From 71bd6d539ee186ddb5ed9d28a16fbcbc1b99988a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 15 Nov 2023 01:05:25 +0000 Subject: [PATCH 1/7] fix(deps): update dependency org.liquibase:liquibase-core to v4.25.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5a58433..016427d 100644 --- a/build.gradle +++ b/build.gradle @@ -32,7 +32,7 @@ dependencies { implementation 'org.postgresql:postgresql:42.6.0' implementation 'org.hibernate:hibernate-core:6.3.1.Final' implementation 'org.hibernate:hibernate-hikaricp:6.3.1.Final' - implementation 'org.liquibase:liquibase-core:4.23.2' + implementation 'org.liquibase:liquibase-core:4.25.0' implementation('org.liquibase.ext:liquibase-yugabytedb:4.23.2') { exclude group: 'org.liquibase' } implementation 'com.zaxxer:HikariCP:5.0.1' implementation 'org.springframework.security:spring-security-crypto:6.1.4' From 455a4c0bdae9fe8fc1831ea87a4d07c752bc8f85 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 15 Nov 2023 01:52:42 +0000 Subject: [PATCH 2/7] fix(deps): update dependency org.liquibase.ext:liquibase-yugabytedb to v4.24.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 016427d..8ceed09 100644 --- a/build.gradle +++ b/build.gradle @@ -33,7 +33,7 @@ dependencies { implementation 'org.hibernate:hibernate-core:6.3.1.Final' implementation 'org.hibernate:hibernate-hikaricp:6.3.1.Final' implementation 'org.liquibase:liquibase-core:4.25.0' - implementation('org.liquibase.ext:liquibase-yugabytedb:4.23.2') { exclude group: 'org.liquibase' } + implementation('org.liquibase.ext:liquibase-yugabytedb:4.24.0') { exclude group: 'org.liquibase' } implementation 'com.zaxxer:HikariCP:5.0.1' implementation 'org.springframework.security:spring-security-crypto:6.1.4' implementation 'commons-logging:commons-logging:1.2' From 2c5f6ed6af91acbff39e02b98f9c0957d1a49a2e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 16 Nov 2023 01:51:45 +0000 Subject: [PATCH 3/7] fix(deps): update jackson to v2.16.0 --- build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 8ceed09..68ca8db 100644 --- a/build.gradle +++ b/build.gradle @@ -18,9 +18,9 @@ dependencies { implementation 'org.bouncycastle:bcprov-jdk15on:1.70' implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:8cf9a4aef0919df2ef1baafd30ab5bfefefc0844' implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7' - implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2' - implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.2' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2' + implementation 'com.fasterxml.jackson.core:jackson-core:2.16.0' + implementation 'com.fasterxml.jackson.core:jackson-annotations:2.16.0' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.16.0' implementation 'com.rometools:rome:2.1.0' implementation 'com.rometools:rome-modules:2.1.0' implementation 'org.jsoup:jsoup:1.16.1' From 268e599421aade6368141f64618a60a12d40f614 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 20 Nov 2023 05:17:29 +0000 Subject: [PATCH 4/7] Implement blake3 proxy query hash --- build.gradle | 2 +- config.properties | 3 + src/main/java/me/kavin/piped/Main.java | 5 ++ .../java/me/kavin/piped/consts/Constants.java | 5 ++ .../java/me/kavin/piped/utils/URLUtils.java | 84 ++++++++++++++++++- 5 files changed, 97 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 68ca8db..e5b1fac 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation 'commons-io:commons-io:2.14.0' implementation 'it.unimi.dsi:fastutil-core:8.5.12' implementation 'commons-codec:commons-codec:1.16.0' - implementation 'org.bouncycastle:bcprov-jdk15on:1.70' + implementation 'org.bouncycastle:bcprov-jdk18on:1.77' implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:8cf9a4aef0919df2ef1baafd30ab5bfefefc0844' implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7' implementation 'com.fasterxml.jackson.core:jackson-core:2.16.0' diff --git a/config.properties b/config.properties index 9b5fed0..4b99288 100644 --- a/config.properties +++ b/config.properties @@ -6,6 +6,9 @@ HTTP_WORKERS:2 # Proxy PROXY_PART:https://pipedproxy-cdg.kavin.rocks +# Proxy Hash Secret +#PROXY_HASH_SECRET:INSERT_HERE + # Outgoing proxy to be used by reqwest4j - eg: socks5://127.0.0.1:1080 #REQWEST_PROXY: socks5://127.0.0.1:1080 # Optional proxy username and password diff --git a/src/main/java/me/kavin/piped/Main.java b/src/main/java/me/kavin/piped/Main.java index b80d31f..934dc8c 100644 --- a/src/main/java/me/kavin/piped/Main.java +++ b/src/main/java/me/kavin/piped/Main.java @@ -13,6 +13,7 @@ import me.kavin.piped.utils.obj.db.PlaylistVideo; import me.kavin.piped.utils.obj.db.PubSub; import me.kavin.piped.utils.obj.db.Video; import okhttp3.OkHttpClient; +import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.hibernate.Session; import org.hibernate.StatelessSession; import org.schabi.newpipe.extractor.NewPipe; @@ -21,6 +22,7 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import java.security.Security; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; @@ -33,6 +35,9 @@ public class Main { public static void main(String[] args) throws Exception { + Security.setProperty("crypto.policy", "unlimited"); + Security.addProvider(new BouncyCastleProvider()); + NewPipe.init(new DownloaderImpl(), new Localization("en", "US"), ContentCountry.DEFAULT, Multithreading.getCachedExecutor()); YoutubeStreamExtractor.forceFetchAndroidClient(true); YoutubeStreamExtractor.forceFetchIosClient(true); diff --git a/src/main/java/me/kavin/piped/consts/Constants.java b/src/main/java/me/kavin/piped/consts/Constants.java index 516ece6..840069a 100644 --- a/src/main/java/me/kavin/piped/consts/Constants.java +++ b/src/main/java/me/kavin/piped/consts/Constants.java @@ -25,7 +25,9 @@ import java.io.File; import java.io.FileReader; import java.net.InetSocketAddress; import java.net.ProxySelector; +import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Optional; import java.util.Properties; import java.util.regex.Pattern; @@ -40,6 +42,8 @@ public class Constants { public static final String IMAGE_PROXY_PART; + public static final byte[] PROXY_HASH_SECRET; + public static final String CAPTCHA_BASE_URL, CAPTCHA_API_KEY; public static final StreamingService YOUTUBE_SERVICE; @@ -127,6 +131,7 @@ public class Constants { String.valueOf(Runtime.getRuntime().availableProcessors())); PROXY_PART = getProperty(prop, "PROXY_PART"); IMAGE_PROXY_PART = getProperty(prop, "IMAGE_PROXY_PART", PROXY_PART); + PROXY_HASH_SECRET = Optional.ofNullable(getProperty(prop, "PROXY_HASH_SECRET")).map(s -> s.getBytes(StandardCharsets.UTF_8)).orElse(null); CAPTCHA_BASE_URL = getProperty(prop, "CAPTCHA_BASE_URL"); CAPTCHA_API_KEY = getProperty(prop, "CAPTCHA_API_KEY"); PUBLIC_URL = getProperty(prop, "API_URL"); diff --git a/src/main/java/me/kavin/piped/utils/URLUtils.java b/src/main/java/me/kavin/piped/utils/URLUtils.java index a36f3e9..20b5122 100644 --- a/src/main/java/me/kavin/piped/utils/URLUtils.java +++ b/src/main/java/me/kavin/piped/utils/URLUtils.java @@ -1,6 +1,7 @@ package me.kavin.piped.utils; import me.kavin.piped.consts.Constants; +import org.apache.commons.codec.binary.Hex; import org.apache.commons.lang3.StringUtils; import org.schabi.newpipe.extractor.Image; @@ -9,7 +10,14 @@ import java.net.URL; import java.net.URLDecoder; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Comparator; import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import static me.kavin.piped.consts.Constants.PROXY_HASH_SECRET; public class URLUtils { @@ -65,13 +73,87 @@ public class URLUtils { boolean hasQuery = query != null; + Comparator> listComparator = (o1, o2) -> { + for (int i = 0; i < Math.min(o1.size(), o2.size()); i++) { + int result = o1.get(i).compareTo(o2.get(i)); + if (result != 0) { + return result; + } + } + return Integer.compare(o1.size(), o2.size()); // compare list sizes if all elements are equal + }; + + Set> queryPairs = new TreeSet<>(listComparator); + + if (hasQuery) { + String[] pairs = query.split("&"); + + for (String pair : pairs) { + int idx = pair.indexOf("="); + queryPairs.add(List.of( + silentDecode(pair.substring(0, idx)), + silentDecode(pair.substring(idx + 1)) + )); + } + } + + // look for host param, and add it if it doesn't exist + boolean hasHost = false; + for (List pair : queryPairs) { + if (pair.get(0).equals("host")) { + hasHost = true; + break; + } + } + if (!hasHost) { + queryPairs.add(List.of("host", host)); + } + + if (PROXY_HASH_SECRET != null) + try { + MessageDigest md = MessageDigest.getInstance("BLAKE3-256"); + for (List pair : queryPairs) { + md.update(pair.get(0).getBytes(StandardCharsets.UTF_8)); + md.update(pair.get(1).getBytes(StandardCharsets.UTF_8)); + } + + md.update(PROXY_HASH_SECRET); + + queryPairs.add(List.of("qhash", Hex.encodeHexString(md.digest()).substring(0, 8))); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } + + String path = url.getPath(); if (path.contains("=")) { path = StringUtils.substringBefore(path, "=") + "=" + StringUtils.substringAfter(path, "=").replace("-rj", "-rw"); } - return proxy + path + (hasQuery ? "?" + query + "&host=" : "?host=") + silentEncode(host); + String newUrl = proxy + path; + + StringBuilder qstring = null; + + if (hasQuery) { + for (List pair : queryPairs) { + if (qstring == null) { + qstring = new StringBuilder(); + } else { + qstring.append("&"); + } + + qstring.append(pair.get(0)); + qstring.append("="); + qstring.append(pair.get(1)); + } + } + + if (qstring != null) { + newUrl += "?" + qstring; + } + + return newUrl; } } From 8d04aca336c1b649c03cd03de3e006570d4eada4 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 20 Nov 2023 05:46:57 +0000 Subject: [PATCH 5/7] Always add new query string. --- src/main/java/me/kavin/piped/utils/URLUtils.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/me/kavin/piped/utils/URLUtils.java b/src/main/java/me/kavin/piped/utils/URLUtils.java index 20b5122..6ec5b14 100644 --- a/src/main/java/me/kavin/piped/utils/URLUtils.java +++ b/src/main/java/me/kavin/piped/utils/URLUtils.java @@ -149,9 +149,7 @@ public class URLUtils { } } - if (qstring != null) { - newUrl += "?" + qstring; - } + newUrl += "?" + qstring; return newUrl; From fdbeffaf37bfc2cabec9b4443b6b6f86cbd46185 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 20 Nov 2023 06:09:26 +0000 Subject: [PATCH 6/7] Fix null qstring. --- .../java/me/kavin/piped/utils/URLUtils.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/main/java/me/kavin/piped/utils/URLUtils.java b/src/main/java/me/kavin/piped/utils/URLUtils.java index 6ec5b14..72e69d4 100644 --- a/src/main/java/me/kavin/piped/utils/URLUtils.java +++ b/src/main/java/me/kavin/piped/utils/URLUtils.java @@ -135,18 +135,16 @@ public class URLUtils { StringBuilder qstring = null; - if (hasQuery) { - for (List pair : queryPairs) { - if (qstring == null) { - qstring = new StringBuilder(); - } else { - qstring.append("&"); - } - - qstring.append(pair.get(0)); - qstring.append("="); - qstring.append(pair.get(1)); + for (List pair : queryPairs) { + if (qstring == null) { + qstring = new StringBuilder(); + } else { + qstring.append("&"); } + + qstring.append(pair.get(0)); + qstring.append("="); + qstring.append(pair.get(1)); } newUrl += "?" + qstring; From 39d31682274dd60d9fda7e55db06b7bece312101 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 20 Nov 2023 08:31:19 +0000 Subject: [PATCH 7/7] Include path in qhash --- .../java/me/kavin/piped/utils/URLUtils.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/java/me/kavin/piped/utils/URLUtils.java b/src/main/java/me/kavin/piped/utils/URLUtils.java index 72e69d4..7a8184a 100644 --- a/src/main/java/me/kavin/piped/utils/URLUtils.java +++ b/src/main/java/me/kavin/piped/utils/URLUtils.java @@ -109,6 +109,12 @@ public class URLUtils { queryPairs.add(List.of("host", host)); } + String path = url.getPath(); + + if (path.contains("=")) { + path = StringUtils.substringBefore(path, "=") + "=" + StringUtils.substringAfter(path, "=").replace("-rj", "-rw"); + } + if (PROXY_HASH_SECRET != null) try { MessageDigest md = MessageDigest.getInstance("BLAKE3-256"); @@ -117,6 +123,8 @@ public class URLUtils { md.update(pair.get(1).getBytes(StandardCharsets.UTF_8)); } + md.update(path.getBytes(StandardCharsets.UTF_8)); + md.update(PROXY_HASH_SECRET); queryPairs.add(List.of("qhash", Hex.encodeHexString(md.digest()).substring(0, 8))); @@ -124,13 +132,6 @@ public class URLUtils { e.printStackTrace(); } - - String path = url.getPath(); - - if (path.contains("=")) { - path = StringUtils.substringBefore(path, "=") + "=" + StringUtils.substringAfter(path, "=").replace("-rj", "-rw"); - } - String newUrl = proxy + path; StringBuilder qstring = null; @@ -142,9 +143,9 @@ public class URLUtils { qstring.append("&"); } - qstring.append(pair.get(0)); + qstring.append(silentEncode(pair.get(0))); qstring.append("="); - qstring.append(pair.get(1)); + qstring.append(silentEncode(pair.get(1))); } newUrl += "?" + qstring;