From 36af3af096ba7aac0853b49507e12089166080d1 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Sun, 29 Jan 2023 12:16:57 +0000 Subject: [PATCH] Use reqwest4j instead of okhttp. --- build.gradle | 121 +++++++++--------- settings.gradle | 4 +- .../java/me/kavin/piped/consts/Constants.java | 2 +- .../me/kavin/piped/utils/DownloaderImpl.java | 35 +++-- .../java/me/kavin/piped/utils/LbryHelper.java | 29 ++--- .../me/kavin/piped/utils/PubSubHelper.java | 22 ++-- .../me/kavin/piped/utils/RequestUtils.java | 34 ++--- .../java/me/kavin/piped/utils/RydHelper.java | 13 +- .../kavin/piped/utils/SponsorBlockUtils.java | 10 +- 9 files changed, 129 insertions(+), 141 deletions(-) diff --git a/build.gradle b/build.gradle index 70be964..6fe0163 100644 --- a/build.gradle +++ b/build.gradle @@ -1,60 +1,61 @@ -plugins { - id "com.github.johnrengelman.shadow" version "7.1.2" - id "java" - id "io.freefair.lombok" version "6.6.1" - id "eclipse" -} - -repositories { - mavenCentral() - maven { url 'https://jitpack.io' } -} - -dependencies { - implementation 'org.apache.commons:commons-lang3:3.12.0' - implementation 'org.apache.commons:commons-text:1.10.0' - implementation 'commons-io:commons-io:2.11.0' - implementation 'it.unimi.dsi:fastutil-core:8.5.11' - implementation 'commons-codec:commons-codec:1.15' - implementation 'org.bouncycastle:bcprov-jdk15on:1.70' - implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:85bff7662e56f5570dd0e5f8b666b4ffaf57a3b0' - implementation 'com.github.FireMasterK:nanojson:885effc25ad9ff920d43a4b09849719926ed89b2' - implementation 'com.fasterxml.jackson.core:jackson-core:2.14.1' - implementation 'com.fasterxml.jackson.core:jackson-annotations:2.14.1' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.1' - implementation 'com.rometools:rome:1.18.0' - implementation 'org.jsoup:jsoup:1.15.3' - implementation 'io.activej:activej-common:5.4.3' - implementation 'io.activej:activej-http:5.4.3' - implementation 'io.activej:activej-boot:5.4.3' - implementation 'io.activej:activej-specializer:5.4.3' - implementation 'io.activej:activej-launchers-http:5.4.3' - implementation 'org.hsqldb:hsqldb:2.7.1' - implementation 'org.postgresql:postgresql:42.5.1' - implementation 'org.hibernate:hibernate-core:6.1.6.Final' - implementation 'org.hibernate:hibernate-hikaricp:6.1.6.Final' - implementation 'com.zaxxer:HikariCP:5.0.1' - implementation 'org.springframework.security:spring-security-crypto:6.0.1' - implementation 'commons-logging:commons-logging:1.2' - implementation(platform("com.squareup.okhttp3:okhttp-bom:4.10.0")) - implementation 'com.squareup.okhttp3:okhttp' - implementation 'com.squareup.okhttp3:okhttp-brotli' - implementation 'io.sentry:sentry:6.11.0' -} - -shadowJar { - // minimize() -} - -jar { - manifest { - attributes( - 'Main-Class': 'me.kavin.piped.Main' - ) - } -} - -group = 'me.kavin.piped' -version = '1.0' -sourceCompatibility = JavaVersion.VERSION_17 -targetCompatibility = JavaVersion.VERSION_17 +plugins { + id "com.github.johnrengelman.shadow" version "7.1.2" + id "java" + id "io.freefair.lombok" version "6.6.1" + id "eclipse" +} + +repositories { + mavenCentral() + maven { url 'https://jitpack.io' } +} + +dependencies { + implementation 'org.apache.commons:commons-lang3:3.12.0' + implementation 'org.apache.commons:commons-text:1.10.0' + implementation 'commons-io:commons-io:2.11.0' + implementation 'it.unimi.dsi:fastutil-core:8.5.11' + implementation 'commons-codec:commons-codec:1.15' + implementation 'org.bouncycastle:bcprov-jdk15on:1.70' + implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:85bff7662e56f5570dd0e5f8b666b4ffaf57a3b0' + implementation 'com.github.FireMasterK:nanojson:885effc25ad9ff920d43a4b09849719926ed89b2' + implementation 'com.fasterxml.jackson.core:jackson-core:2.14.1' + implementation 'com.fasterxml.jackson.core:jackson-annotations:2.14.1' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.1' + implementation 'com.rometools:rome:1.18.0' + implementation 'org.jsoup:jsoup:1.15.3' + implementation 'io.activej:activej-common:5.4.3' + implementation 'io.activej:activej-http:5.4.3' + implementation 'io.activej:activej-boot:5.4.3' + implementation 'io.activej:activej-specializer:5.4.3' + implementation 'io.activej:activej-launchers-http:5.4.3' + implementation 'org.hsqldb:hsqldb:2.7.1' + implementation 'org.postgresql:postgresql:42.5.1' + implementation 'org.hibernate:hibernate-core:6.1.6.Final' + implementation 'org.hibernate:hibernate-hikaricp:6.1.6.Final' + implementation 'com.zaxxer:HikariCP:5.0.1' + implementation 'org.springframework.security:spring-security-crypto:6.0.1' + implementation 'commons-logging:commons-logging:1.2' + implementation(platform("com.squareup.okhttp3:okhttp-bom:4.10.0")) + implementation 'com.squareup.okhttp3:okhttp' + implementation 'com.squareup.okhttp3:okhttp-brotli' + implementation 'io.sentry:sentry:6.11.0' + implementation 'rocks.kavin:reqwest4j:1.0' +} + +shadowJar { + // minimize() +} + +jar { + manifest { + attributes( + 'Main-Class': 'me.kavin.piped.Main' + ) + } +} + +group = 'me.kavin.piped' +version = '1.0' +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 diff --git a/settings.gradle b/settings.gradle index 664e539..4e3f036 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,2 +1,2 @@ - -rootProject.name = 'piped' + +rootProject.name = 'piped' diff --git a/src/main/java/me/kavin/piped/consts/Constants.java b/src/main/java/me/kavin/piped/consts/Constants.java index 2aea2c5..760adde 100644 --- a/src/main/java/me/kavin/piped/consts/Constants.java +++ b/src/main/java/me/kavin/piped/consts/Constants.java @@ -29,7 +29,7 @@ import java.util.regex.Pattern; public class Constants { - public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0"; + public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0"; public static final int PORT; public static final String HTTP_WORKERS; diff --git a/src/main/java/me/kavin/piped/utils/DownloaderImpl.java b/src/main/java/me/kavin/piped/utils/DownloaderImpl.java index fd65c51..ecf1250 100644 --- a/src/main/java/me/kavin/piped/utils/DownloaderImpl.java +++ b/src/main/java/me/kavin/piped/utils/DownloaderImpl.java @@ -1,10 +1,10 @@ package me.kavin.piped.utils; import com.grack.nanojson.JsonParserException; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; import me.kavin.piped.consts.Constants; import me.kavin.piped.utils.obj.SolvedCaptcha; import okhttp3.FormBody; -import okhttp3.RequestBody; import org.apache.commons.lang3.StringUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Element; @@ -12,9 +12,12 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Request; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import rocks.kavin.reqwest4j.ReqwestUtils; import java.io.IOException; import java.net.HttpCookie; +import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; public class DownloaderImpl extends Downloader { @@ -31,23 +34,16 @@ public class DownloaderImpl extends Downloader { // TODO: HTTP/3 aka QUIC var bytes = request.dataToSend(); - RequestBody body = null; - if (bytes != null) - body = RequestBody.create(bytes); - - var builder = new okhttp3.Request.Builder() - .url(request.url()) - .method(request.httpMethod(), body) - .header("User-Agent", Constants.USER_AGENT); + Map headers = new Object2ObjectOpenHashMap<>(); if (saved_cookie != null && !saved_cookie.hasExpired()) - builder.header("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue()); + headers.put("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue()); - request.headers().forEach((name, values) -> values.forEach(value -> builder.header(name, value))); + request.headers().forEach((name, values) -> values.forEach(value -> headers.put(name, value))); - var resp = Constants.h2client.newCall(builder.build()).execute(); + var resp = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers); - if (resp.code() == 429) { + if (resp.status() == 429) { synchronized (cookie_lock) { @@ -55,14 +51,14 @@ public class DownloaderImpl extends Downloader { || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30))) saved_cookie = null; - String redir_url = String.valueOf(resp.request().url()); + String redir_url = String.valueOf(resp.finalUrl()); if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) { var formBuilder = new FormBody.Builder(); String sitekey = null, data_s = null; - for (Element el : Jsoup.parse(resp.body().string()).selectFirst("form").children()) { + for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) { String name; if (!(name = el.tagName()).equals("script")) { if (name.equals("input")) @@ -105,11 +101,10 @@ public class DownloaderImpl extends Downloader { } - var response = new Response(resp.code(), resp.message(), resp.headers().toMultimap(), resp.body().string(), - String.valueOf(resp.request().url())); + Map> headerMap = resp.headers().entrySet().stream() + .collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll); - resp.close(); - - return response; + return new Response(resp.status(), null, headerMap, new String(resp.body()), + resp.finalUrl()); } } diff --git a/src/main/java/me/kavin/piped/utils/LbryHelper.java b/src/main/java/me/kavin/piped/utils/LbryHelper.java index ed87949..4936059 100644 --- a/src/main/java/me/kavin/piped/utils/LbryHelper.java +++ b/src/main/java/me/kavin/piped/utils/LbryHelper.java @@ -1,16 +1,13 @@ package me.kavin.piped.utils; import me.kavin.piped.consts.Constants; -import okhttp3.MediaType; -import okhttp3.Request; -import okhttp3.RequestBody; import org.apache.commons.lang3.StringUtils; +import rocks.kavin.reqwest4j.ReqwestUtils; import java.io.IOException; +import java.util.Map; -import static me.kavin.piped.consts.Constants.h2client; import static me.kavin.piped.consts.Constants.mapper; -import static me.kavin.piped.utils.RequestUtils.sendGet; import static me.kavin.piped.utils.URLUtils.silentEncode; public class LbryHelper { @@ -20,7 +17,7 @@ public class LbryHelper { if (Constants.DISABLE_LBRY) return null; - return mapper.readTree(sendGet("https://api.lbry.com/yt/resolve?video_ids=" + silentEncode(videoId))) + return RequestUtils.sendGetJson("https://api.lbry.com/yt/resolve?video_ids=" + silentEncode(videoId)) .at("/data/videos") .path(videoId) .asText(null); @@ -32,11 +29,9 @@ public class LbryHelper { if (StringUtils.isEmpty(lbryId)) return null; - var request = new Request.Builder() - .url("https://api.na-backend.odysee.com/api/v1/proxy?m=get") - .post(RequestBody.create(mapper.writeValueAsBytes( + var resp = ReqwestUtils.fetch("https://api.na-backend.odysee.com/api/v1/proxy?m=get", "POST", + mapper.writeValueAsBytes( mapper.createObjectNode() - .put("id", System.currentTimeMillis()) .put("id", System.currentTimeMillis()) .put("jsonrpc", "2.0") .put("method", "get") @@ -45,15 +40,11 @@ public class LbryHelper { .put("uri", "lbry://" + lbryId) .put("save_file", true) ) - ), MediaType.get("application/json"))) - .build(); - - try (var resp = h2client.newCall(request).execute()) { - if (resp.isSuccessful()) { - return mapper.readTree(resp.body().byteStream()) - .at("/result/streaming_url") - .asText(null); - } + ), Map.of("Content-Type", "application/json")); + if (resp.status() / 100 == 2) { + return mapper.readTree(resp.body()) + .at("/result/streaming_url") + .asText(null); } return null; diff --git a/src/main/java/me/kavin/piped/utils/PubSubHelper.java b/src/main/java/me/kavin/piped/utils/PubSubHelper.java index b1eb282..fadaeb1 100644 --- a/src/main/java/me/kavin/piped/utils/PubSubHelper.java +++ b/src/main/java/me/kavin/piped/utils/PubSubHelper.java @@ -3,11 +3,12 @@ package me.kavin.piped.utils; import me.kavin.piped.consts.Constants; import me.kavin.piped.utils.obj.db.PubSub; import okhttp3.FormBody; -import okhttp3.Request; +import okio.Buffer; import org.hibernate.StatelessSession; +import rocks.kavin.reqwest4j.ReqwestUtils; import java.io.IOException; -import java.util.Objects; +import java.util.Map; import java.util.concurrent.TimeUnit; public class PubSubHelper { @@ -23,9 +24,6 @@ public class PubSubHelper { String callback = Constants.PUBSUB_URL + "/webhooks/pubsub"; String topic = "https://www.youtube.com/xml/feeds/videos.xml?channel_id=" + channelId; - var builder = new Request.Builder() - .url(Constants.PUBSUB_HUB_URL); - var formBuilder = new FormBody.Builder(); formBuilder.add("hub.callback", callback); @@ -42,16 +40,16 @@ public class PubSubHelper { tr.commit(); } - try (var resp = Constants.h2client - .newCall(builder.post(formBuilder.build()) - .build()).execute()) { + // write form to read later + var buffer = new Buffer(); + formBuilder.build().writeTo(buffer); - if (resp.code() != 202) - System.out.println("Failed to subscribe: " + resp.code() + "\n" + Objects.requireNonNull(resp.body()).string()); + var resp = ReqwestUtils.fetch(callback, "POST", buffer.readByteArray(), Map.of()); + + if (resp.status() != 202) + System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body())); - } } - } public static void updatePubSub(String channelId) { diff --git a/src/main/java/me/kavin/piped/utils/RequestUtils.java b/src/main/java/me/kavin/piped/utils/RequestUtils.java index a5968f5..6285c8c 100644 --- a/src/main/java/me/kavin/piped/utils/RequestUtils.java +++ b/src/main/java/me/kavin/piped/utils/RequestUtils.java @@ -4,38 +4,38 @@ import com.fasterxml.jackson.databind.JsonNode; import me.kavin.piped.consts.Constants; import okhttp3.OkHttpClient; import okhttp3.Request; -import okhttp3.Response; +import rocks.kavin.reqwest4j.ReqwestUtils; +import rocks.kavin.reqwest4j.Response; import java.io.IOException; +import java.util.Map; + +import static me.kavin.piped.consts.Constants.mapper; public class RequestUtils { public static Response sendGetRaw(String url) throws IOException { - return sendGetRaw(url, Constants.USER_AGENT); - } - - public static Response sendGetRaw(String url, String ua) throws IOException { - var request = new Request.Builder().header("User-Agent", ua).url(url).build(); - return Constants.h2client.newCall(request).execute(); + return ReqwestUtils.fetch(url, "GET", null, Map.of()); } public static String sendGet(String url) throws IOException { - return sendGet(url, Constants.USER_AGENT); + return new String( + ReqwestUtils.fetch(url, "GET", null, Map.of()) + .body() + ); } public static String sendGet(String url, String ua) throws IOException { - - var response = sendGetRaw(url, ua); - var responseString = response.body().string(); - response.close(); - - return responseString; + return new String( + ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua)) + .body() + ); } public static JsonNode getJsonNode(OkHttpClient client, Request request) throws IOException { try (var resp = client.newCall(request).execute()) { try { - return Constants.mapper.readTree(resp.body().byteStream()); + return mapper.readTree(resp.body().byteStream()); } catch (Exception e) { if (!resp.isSuccessful()) ExceptionHandler.handle(e); @@ -49,6 +49,8 @@ public class RequestUtils { } public static JsonNode sendGetJson(String url) throws IOException { - return sendGetJson(url, Constants.USER_AGENT); + + return mapper.readTree(ReqwestUtils.fetch(url, "GET", null, Map.of()).body()); + } } diff --git a/src/main/java/me/kavin/piped/utils/RydHelper.java b/src/main/java/me/kavin/piped/utils/RydHelper.java index 0579f0a..44c34ce 100644 --- a/src/main/java/me/kavin/piped/utils/RydHelper.java +++ b/src/main/java/me/kavin/piped/utils/RydHelper.java @@ -14,15 +14,14 @@ public class RydHelper { if (Constants.DISABLE_RYD) return -1; - try (var resp = sendGetRaw(Constants.RYD_PROXY_URL + "/votes/" + videoId)) { + var resp = sendGetRaw(Constants.RYD_PROXY_URL + "/votes/" + videoId); - if (!resp.isSuccessful()) - return -1; + if (resp.status() / 100 != 2) + return -1; - return mapper.readTree(resp.body().byteStream()) - .path("rating") - .asDouble(-1); + return mapper.readTree(resp.body()) + .path("rating") + .asDouble(-1); - } } } diff --git a/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java b/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java index 6af6840..bb86d73 100644 --- a/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java +++ b/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java @@ -25,11 +25,13 @@ public class SponsorBlockUtils { String hash = toSha256(id); for (String url : Constants.SPONSORBLOCK_SERVERS) { - try (var resp = RequestUtils.sendGetRaw(url + "/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4)) - + "?categories=" + URLUtils.silentEncode(categories))) { + try { - if (resp.code() == 200) { - JsonArray jArray = JsonParser.array().from(resp.body().string()); + var resp = RequestUtils.sendGetRaw(url + "/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4)) + + "?categories=" + URLUtils.silentEncode(categories)); + + if (resp.status() == 200) { + JsonArray jArray = JsonParser.array().from(new String(resp.body())); jArray.removeIf(jObject -> !((JsonObject) jObject).getString("videoID").equalsIgnoreCase(id));