reqwest4j is now async first

build.gradle

@@ -40,7 +40,7 @@ dependencies {
     implementation 'com.squareup.okhttp3:okhttp'
     implementation 'com.squareup.okhttp3:okhttp-brotli'
     implementation 'io.sentry:sentry:6.24.0'
-    implementation 'rocks.kavin:reqwest4j:1.0.4'
+    implementation 'rocks.kavin:reqwest4j:1.0.5'
     implementation 'io.minio:minio:8.5.4'
 }
 

src/main/java/me/kavin/piped/consts/Constants.java

@@ -199,7 +199,7 @@ public class Constants {
             h2_no_redir_client = builder_noredir.build();
             String temp = null;
             try {
-                var html = RequestUtils.sendGet("https://www.youtube.com/");
+                var html = RequestUtils.sendGet("https://www.youtube.com/").get();
                 var regex = Pattern.compile("GL\":\"([A-Z]{2})\"", Pattern.MULTILINE);
                 var matcher = regex.matcher(html);
                 if (matcher.find()) {

src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java

@@ -150,7 +150,7 @@ public class StreamHandlers {
                 List<String> allowedCountries = new ObjectArrayList<>();
 
                 {
-                    var restrictedTree = RequestUtils.sendGetJson(Constants.GEO_RESTRICTION_CHECKER_URL + "/api/region/check?video_id=" + videoId);
+                    var restrictedTree = RequestUtils.sendGetJson(Constants.GEO_RESTRICTION_CHECKER_URL + "/api/region/check?video_id=" + videoId).get();
                     if (!restrictedTree.get("restricted").asBoolean()) {
                         assert exception != null;
                         throw (Exception) exception;

src/main/java/me/kavin/piped/server/handlers/auth/UserHandlers.java

@@ -28,7 +28,7 @@ public class UserHandlers {
     private static final Argon2PasswordEncoder argon2PasswordEncoder = Argon2PasswordEncoder.defaultsForSpringSecurity_v5_8();
     private static final BCryptPasswordEncoder bcryptPasswordEncoder = new BCryptPasswordEncoder();
 
-    public static byte[] registerResponse(String user, String pass) throws IOException {
+    public static byte[] registerResponse(String user, String pass) throws Exception {
 
         if (Constants.DISABLE_REGISTRATION)
             ExceptionHandler.throwErrorResponse(new DisabledRegistrationResponse());
@@ -57,7 +57,8 @@ public class UserHandlers {
                 String suffix = sha1Hash.substring(5);
                 String[] entries = RequestUtils
                         .sendGet("https://api.pwnedpasswords.com/range/" + prefix, "github.com/TeamPiped/Piped-Backend")
-                        .split("\n");
+                        .thenApplyAsync(str -> str.split("\n"))
+                        .get();
                 for (String entry : entries)
                     if (StringUtils.substringBefore(entry, ":").equals(suffix))
                         ExceptionHandler.throwErrorResponse(new CompromisedPasswordResponse());

src/main/java/me/kavin/piped/utils/DownloaderImpl.java

@@ -1,13 +1,6 @@
 package me.kavin.piped.utils;
 
-import com.grack.nanojson.JsonParserException;
 import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
-import me.kavin.piped.consts.Constants;
-import me.kavin.piped.utils.obj.SolvedCaptcha;
-import okhttp3.FormBody;
-import org.apache.commons.lang3.StringUtils;
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Request;
 import org.schabi.newpipe.extractor.downloader.Response;
@@ -18,7 +11,7 @@ import java.io.IOException;
 import java.net.HttpCookie;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutionException;
 
 public class DownloaderImpl extends Downloader {
 
@@ -41,70 +34,85 @@ public class DownloaderImpl extends Downloader {
 
         request.headers().forEach((name, values) -> values.forEach(value -> headers.put(name, value)));
 
-        var resp = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers);
+        var future = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers);
 
-        if (resp.status() == 429) {
+        // Recaptcha solver code
+        // Commented out, as it hasn't been ported to reqwest4j yet
+        // Also, this was last seen a long time back
 
-            synchronized (cookie_lock) {
+//        future.thenAcceptAsync(resp -> {
+//            if (resp.status() == 429) {
+//                synchronized (cookie_lock) {
+//
+//                    if (saved_cookie != null && saved_cookie.hasExpired()
+//                            || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
+//                        saved_cookie = null;
+//
+//                    String redir_url = String.valueOf(resp.finalUrl());
+//
+//                    if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
+//
+//                        var formBuilder = new FormBody.Builder();
+//                        String sitekey = null, data_s = null;
+//
+//                        for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) {
+//                            String name;
+//                            if (!(name = el.tagName()).equals("script")) {
+//                                if (name.equals("input"))
+//                                    formBuilder.add(el.attr("name"), el.attr("value"));
+//                                else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
+//                                    sitekey = el.attr("data-sitekey");
+//                                    data_s = el.attr("data-s");
+//                                }
+//                            }
+//                        }
+//
+//                        if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s))
+//                            ExceptionHandler.handle(new ReCaptchaException("Could not get recaptcha", redir_url));
+//
+//                        SolvedCaptcha solved = null;
+//
+//                        try {
+//                            solved = CaptchaSolver.solve(redir_url, sitekey, data_s);
+//                        } catch (JsonParserException | InterruptedException | IOException e) {
+//                            e.printStackTrace();
+//                        }
+//
+//                        formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse());
+//
+//                        var formReqBuilder = new okhttp3.Request.Builder()
+//                                .url("https://www.google.com/sorry/index")
+//                                .header("User-Agent", Constants.USER_AGENT)
+//                                .post(formBuilder.build());
+//
+//                        okhttp3.Response formResponse;
+//                        try {
+//                            formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute();
+//                        } catch (IOException e) {
+//                            throw new RuntimeException(e);
+//                        }
+//
+//                        saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
+//                                        .substringAfter(formResponse.headers().get("Location"), "google_abuse=")))
+//                                .get(0);
+//                        cookie_received = System.currentTimeMillis();
+//                    }
+//                }
+//            }
+//        }, Multithreading.getCachedExecutor());
 
-                if (saved_cookie != null && saved_cookie.hasExpired()
+        var responseFuture = future.thenApplyAsync(resp -> {
-                        || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
+            Map<String, List<String>> headerMap = resp.headers().entrySet().stream()
-                    saved_cookie = null;
+                    .collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll);
 
-                String redir_url = String.valueOf(resp.finalUrl());
+            return new Response(resp.status(), null, headerMap, new String(resp.body()),
-
+                    resp.finalUrl());
-                if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
+        }, Multithreading.getCachedExecutor());
-
-                    var formBuilder = new FormBody.Builder();
-                    String sitekey = null, data_s = null;
-
-                    for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) {
-                        String name;
-                        if (!(name = el.tagName()).equals("script")) {
-                            if (name.equals("input"))
-                                formBuilder.add(el.attr("name"), el.attr("value"));
-                            else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
-                                sitekey = el.attr("data-sitekey");
-                                data_s = el.attr("data-s");
-                            }
-                        }
-                    }
-                    if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s))
-                        throw new ReCaptchaException("Could not get recaptcha", redir_url);
-
-                    SolvedCaptcha solved = null;
-
-                    try {
-                        solved = CaptchaSolver.solve(redir_url, sitekey, data_s);
-                    } catch (JsonParserException | InterruptedException e) {
-                        e.printStackTrace();
-                    }
-
-                    formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse());
-
-                    var formReqBuilder = new okhttp3.Request.Builder()
-                            .url("https://www.google.com/sorry/index")
-                            .header("User-Agent", Constants.USER_AGENT)
-                            .post(formBuilder.build());
-
-                    var formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute();
-
-                    saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
-                                    .substringAfter(formResponse.headers().get("Location"), "google_abuse=")))
-                            .get(0);
-                    cookie_received = System.currentTimeMillis();
-                }
-
-                if (saved_cookie != null) // call again as captcha has been solved or cookie has not expired.
-                    execute(request);
-            }
 
+        try {
+            return responseFuture.get();
+        } catch (InterruptedException | ExecutionException e) {
+            throw new IOException(e);
         }
-
-        Map<String, List<String>> headerMap = resp.headers().entrySet().stream()
-                .collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll);
-
-        return new Response(resp.status(), null, headerMap, new String(resp.body()),
-                resp.finalUrl());
     }
 }

src/main/java/me/kavin/piped/utils/LbryHelper.java

@@ -3,8 +3,8 @@ package me.kavin.piped.utils;
 import me.kavin.piped.consts.Constants;
 import org.apache.commons.lang3.StringUtils;
 import rocks.kavin.reqwest4j.ReqwestUtils;
+import rocks.kavin.reqwest4j.Response;
 
-import java.io.IOException;
 import java.net.URI;
 import java.util.Map;
 
@@ -13,19 +13,20 @@ import static me.kavin.piped.utils.URLUtils.silentEncode;
 
 public class LbryHelper {
 
-    public static String getLBRYId(String videoId) throws IOException {
+    public static String getLBRYId(String videoId) throws Exception {
 
         if (Constants.DISABLE_LBRY)
             return null;
 
         return RequestUtils.sendGetJson("https://api.lbry.com/yt/resolve?video_ids=" + silentEncode(videoId))
-                .at("/data/videos")
+                .thenApplyAsync(json -> json.at("/data/videos")
-                .path(videoId)
+                        .path(videoId)
-                .asText(null);
+                        .asText(null)
+                ).get();
     }
 
     public static String getLBRYStreamURL(String lbryId)
-            throws IOException {
+            throws Exception {
 
         if (StringUtils.isEmpty(lbryId))
             return null;
@@ -41,7 +42,7 @@ public class LbryHelper {
                                                 .put("uri", "lbry://" + lbryId)
                                                 .put("save_file", true)
                                 )
-                ), Map.of("Content-Type", "application/json"));
+                ), Map.of("Content-Type", "application/json")).get();
         if (resp.status() / 100 == 2) {
             return mapper.readTree(resp.body())
                     .at("/result/streaming_url")
@@ -60,12 +61,12 @@ public class LbryHelper {
         // LBRY provides non UTF-8 characters in the URL, which causes issues
         streamUrl = new URI(streamUrl).toASCIIString();
 
-        var resp = ReqwestUtils.fetch(streamUrl, "HEAD", null, Map.of(
+        final String lastLocation = ReqwestUtils.fetch(streamUrl, "HEAD", null, Map.of(
-                "Origin", "https://odysee.com",
+                        "Origin", "https://odysee.com",
-                "Referer", "https://odysee.com/"
+                        "Referer", "https://odysee.com/"
-        ));
+                ))
-
+                .thenApply(Response::finalUrl)
-        final String lastLocation = resp.finalUrl();
+                .get();
 
         return streamUrl.equals(lastLocation) ? null : lastLocation;
     }

src/main/java/me/kavin/piped/utils/PubSubHelper.java

@@ -44,11 +44,11 @@ public class PubSubHelper {
             var buffer = new Buffer();
             formBuilder.build().writeTo(buffer);
 
-            var resp = ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of());
+            ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of())
-
+                    .thenAccept(resp -> {
-            if (resp.status() != 202)
+                        if (resp.status() != 202)
-                System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body()));
+                            System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body()));
-
+                    });
         }
     }
 

src/main/java/me/kavin/piped/utils/RequestUtils.java

@@ -1,7 +1,6 @@
 package me.kavin.piped.utils;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import me.kavin.piped.consts.Constants;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
 import rocks.kavin.reqwest4j.ReqwestUtils;
@@ -9,27 +8,26 @@ import rocks.kavin.reqwest4j.Response;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 
 import static me.kavin.piped.consts.Constants.mapper;
 
 public class RequestUtils {
 
-    public static Response sendGetRaw(String url) throws IOException {
+    public static CompletableFuture<Response> sendGetRaw(String url) throws Exception {
         return ReqwestUtils.fetch(url, "GET", null, Map.of());
     }
 
-    public static String sendGet(String url) throws IOException {
+    public static CompletableFuture<String> sendGet(String url) throws Exception {
-        return new String(
+        return ReqwestUtils.fetch(url, "GET", null, Map.of())
-                ReqwestUtils.fetch(url, "GET", null, Map.of())
+                .thenApply(Response::body)
-                        .body()
+                .thenApplyAsync(String::new);
-        );
     }
 
-    public static String sendGet(String url, String ua) throws IOException {
+    public static CompletableFuture<String> sendGet(String url, String ua) throws Exception {
-        return new String(
+        return ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua))
-                ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua))
+                .thenApply(Response::body)
-                        .body()
+                .thenApplyAsync(String::new);
-        );
     }
 
     public static JsonNode getJsonNode(OkHttpClient client, Request request) throws IOException {
@@ -44,13 +42,13 @@ public class RequestUtils {
         }
     }
 
-    public static JsonNode sendGetJson(String url, String ua) throws IOException {
+    public static CompletableFuture<JsonNode> sendGetJson(String url) throws Exception {
-        return getJsonNode(Constants.h2client, new Request.Builder().header("User-Agent", ua).url(url).build());
+        return ReqwestUtils.fetch(url, "GET", null, Map.of()).thenApply(Response::body).thenApplyAsync(resp -> {
-    }
+            try {
-
+                return mapper.readTree(resp);
-    public static JsonNode sendGetJson(String url) throws IOException {
+            } catch (Exception e) {
-
+                throw new RuntimeException("Failed to parse JSON", e);
-        return mapper.readTree(ReqwestUtils.fetch(url, "GET", null, Map.of()).body());
+            }
-
+        }, Multithreading.getCachedExecutor());
     }
 }

src/main/java/me/kavin/piped/utils/RydHelper.java

@@ -3,25 +3,17 @@ package me.kavin.piped.utils;
 
 import me.kavin.piped.consts.Constants;
 
-import java.io.IOException;
+import static me.kavin.piped.utils.RequestUtils.sendGetJson;
-
-import static me.kavin.piped.consts.Constants.mapper;
-import static me.kavin.piped.utils.RequestUtils.sendGetRaw;
 
 public class RydHelper {
-    public static double getDislikeRating(String videoId) throws IOException {
+    public static double getDislikeRating(String videoId) throws Exception {
 
         if (Constants.DISABLE_RYD)
             return -1;
 
-        var resp = sendGetRaw(Constants.RYD_PROXY_URL + "/votes/" + videoId);
+        return sendGetJson(Constants.RYD_PROXY_URL + "/votes/" + videoId)
-
+                .thenApply(tree -> tree.path("rating").asDouble(-1))
-        if (resp.status() / 100 != 2)
+                .get();
-            return -1;
-
-        return mapper.readTree(resp.body())
-                .path("rating")
-                .asDouble(-1);
 
     }
 }

src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java

@@ -24,7 +24,7 @@ public class SponsorBlockUtils {
             try {
 
                 var resp = RequestUtils.sendGetRaw(url + "/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4))
-                        + "?categories=" + URLUtils.silentEncode(categories));
+                        + "?categories=" + URLUtils.silentEncode(categories)).get();
 
                 if (resp.status() == 200) {
                     var any = mapper.readTree(resp.body());

src/main/java/me/kavin/piped/utils/obj/MatrixHelper.java

@@ -66,7 +66,7 @@ public class MatrixHelper {
                         .asText();
             }
 
-            AUTHORIZED_USERS = (ArrayNode) mapper.readTree(RequestUtils.sendGet("https://raw.githubusercontent.com/TeamPiped/piped-federation/main/authorized-users.json"));
+            AUTHORIZED_USERS = (ArrayNode) RequestUtils.sendGetJson("https://raw.githubusercontent.com/TeamPiped/piped-federation/main/authorized-users.json").get();
 
         } catch (Exception e) {
             ExceptionHandler.handle(e);