diff --git a/build.gradle b/build.gradle index 7b7f322..34a1dde 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation 'it.unimi.dsi:fastutil-core:8.5.13' implementation 'commons-codec:commons-codec:1.17.0' implementation 'org.bouncycastle:bcprov-jdk18on:1.78.1' - implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:a64e202bb498032e817a702145263590829f3c1d' + implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:e45fa4d37f809a07dd5bdf6f920adabe0077704f' implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7' implementation 'com.fasterxml.jackson.core:jackson-core:2.17.2' implementation 'com.fasterxml.jackson.core:jackson-annotations:2.17.2' diff --git a/config.properties b/config.properties index ff8592f..f78325b 100644 --- a/config.properties +++ b/config.properties @@ -73,6 +73,9 @@ MATRIX_SERVER:https://matrix-client.matrix.org # Geo Restriction Checker for federated bypassing of Geo Restrictions #GEO_RESTRICTION_CHECKER_URL:INSERT_HERE +# BG Helper URL for supplying PoTokens +#BG_HELPER_URL:INSERT_HERE + # S3 Configuration Data (compatible with any provider that offers an S3 compatible API) #S3_ENDPOINT:INSERT_HERE #S3_ACCESS_KEY:INSERT_HERE diff --git a/src/main/java/me/kavin/piped/Main.java b/src/main/java/me/kavin/piped/Main.java index ab710fa..624d5f8 100644 --- a/src/main/java/me/kavin/piped/Main.java +++ b/src/main/java/me/kavin/piped/Main.java @@ -13,6 +13,7 @@ import me.kavin.piped.utils.obj.db.PlaylistVideo; import me.kavin.piped.utils.obj.db.PubSub; import me.kavin.piped.utils.obj.db.Video; import okhttp3.OkHttpClient; +import org.apache.commons.lang3.StringUtils; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.hibernate.Session; import org.hibernate.StatelessSession; @@ -21,6 +22,7 @@ import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfo; import rocks.kavin.reqwest4j.ReqwestUtils; @@ -44,6 +46,8 @@ public class Main { ReqwestUtils.init(REQWEST_PROXY, REQWEST_PROXY_USER, REQWEST_PROXY_PASS); NewPipe.init(new DownloaderImpl(), new Localization("en", "US"), ContentCountry.DEFAULT); + if (!StringUtils.isEmpty(Constants.BG_HELPER_URL)) + YoutubeStreamExtractor.setPoTokenProvider(new BgPoTokenProvider(Constants.BG_HELPER_URL)); YoutubeParsingHelper.setConsentAccepted(CONSENT_COOKIE); // Warm up the extractor @@ -82,7 +86,7 @@ public class Main { System.exit(1); } - Multithreading.runAsync(() -> Thread.ofVirtual().start(new SyncRunner( + Multithreading.runAsync(() -> Thread.ofVirtual().start(new SyncRunner( new OkHttpClient.Builder().readTimeout(60, TimeUnit.SECONDS).build(), MATRIX_SERVER, MatrixHelper.MATRIX_TOKEN) diff --git a/src/main/java/me/kavin/piped/consts/Constants.java b/src/main/java/me/kavin/piped/consts/Constants.java index 2a2624d..984088f 100644 --- a/src/main/java/me/kavin/piped/consts/Constants.java +++ b/src/main/java/me/kavin/piped/consts/Constants.java @@ -28,7 +28,7 @@ import java.util.Properties; public class Constants { - public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0"; + public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:128.0) Gecko/20100101 Firefox/128.0"; public static final int PORT; public static final String HTTP_WORKERS; @@ -100,6 +100,8 @@ public class Constants { public static final String GEO_RESTRICTION_CHECKER_URL; + public static final String BG_HELPER_URL; + public static String YOUTUBE_COUNTRY; public static final String VERSION; @@ -170,6 +172,7 @@ public class Constants { MATRIX_SERVER = getProperty(prop, "MATRIX_SERVER", "https://matrix-client.matrix.org"); MATRIX_TOKEN = getProperty(prop, "MATRIX_TOKEN"); GEO_RESTRICTION_CHECKER_URL = getProperty(prop, "GEO_RESTRICTION_CHECKER_URL"); + BG_HELPER_URL = getProperty(prop, "BG_HELPER_URL"); prop.forEach((_key, _value) -> { String key = String.valueOf(_key), value = String.valueOf(_value); if (key.startsWith("hibernate")) diff --git a/src/main/java/me/kavin/piped/utils/BgPoTokenProvider.java b/src/main/java/me/kavin/piped/utils/BgPoTokenProvider.java new file mode 100644 index 0000000..722f070 --- /dev/null +++ b/src/main/java/me/kavin/piped/utils/BgPoTokenProvider.java @@ -0,0 +1,93 @@ +package me.kavin.piped.utils; + +import lombok.RequiredArgsConstructor; +import org.jetbrains.annotations.Nullable; +import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider; +import org.schabi.newpipe.extractor.services.youtube.PoTokenResult; +import rocks.kavin.reqwest4j.ReqwestUtils; + +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.*; +import java.util.regex.Pattern; + +import static me.kavin.piped.consts.Constants.mapper; + +@RequiredArgsConstructor +public class BgPoTokenProvider implements PoTokenProvider { + + private final String bgHelperUrl; + + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + + private String getWebVisitorData() throws Exception { + var html = RequestUtils.sendGet("https://www.youtube.com").get(); + var matcher = Pattern.compile("visitorData\":\"([\\w%-]+)\"").matcher(html); + + if (matcher.find()) { + return matcher.group(1); + } + + throw new RuntimeException("Failed to get visitor data"); + } + + private final Queue validPoTokens = new ConcurrentLinkedQueue<>(); + + private PoTokenResult getPoTokenPooled() throws Exception { + PoTokenResult poToken = validPoTokens.poll(); + + if (poToken == null) { + poToken = createWebClientPoToken(); + } + + // if still null, return null + if (poToken == null) { + return null; + } + + // timer to insert back into queue after 10 + random seconds + int delay = 10_000 + ThreadLocalRandom.current().nextInt(5000); + PoTokenResult finalPoToken = poToken; + scheduler.schedule(() -> validPoTokens.offer(finalPoToken), delay, TimeUnit.MILLISECONDS); + + return poToken; + } + + private PoTokenResult createWebClientPoToken() throws Exception { + String visitorDate = getWebVisitorData(); + + String poToken = ReqwestUtils.fetch(bgHelperUrl + "/generate", "POST", mapper.writeValueAsBytes(mapper.createObjectNode().put( + "visitorData", visitorDate + )), Map.of( + "Content-Type", "application/json" + )).thenApply(response -> { + try { + return mapper.readTree(response.body()).get("poToken").asText(); + } catch (Exception e) { + return null; + } + }).join(); + + if (poToken != null) { + return new PoTokenResult(visitorDate, poToken); + } + + return null; + } + + @Override + public @Nullable PoTokenResult getWebClientPoToken() { + try { + return getPoTokenPooled(); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + @Override + public @Nullable PoTokenResult getAndroidClientPoToken() { + // TODO: allow setting from config maybe? + return null; + } +} diff --git a/src/main/java/me/kavin/piped/utils/RequestUtils.java b/src/main/java/me/kavin/piped/utils/RequestUtils.java index f5c25b0..eeeeac4 100644 --- a/src/main/java/me/kavin/piped/utils/RequestUtils.java +++ b/src/main/java/me/kavin/piped/utils/RequestUtils.java @@ -1,6 +1,7 @@ package me.kavin.piped.utils; import com.fasterxml.jackson.databind.JsonNode; +import me.kavin.piped.consts.Constants; import okhttp3.OkHttpClient; import okhttp3.Request; import rocks.kavin.reqwest4j.ReqwestUtils; @@ -15,11 +16,15 @@ import static me.kavin.piped.consts.Constants.mapper; public class RequestUtils { public static CompletableFuture sendGetRaw(String url) throws Exception { - return ReqwestUtils.fetch(url, "GET", null, Map.of()); + return ReqwestUtils.fetch(url, "GET", null, Map.of( + "User-Agent", Constants.USER_AGENT + )); } public static CompletableFuture sendGet(String url) throws Exception { - return ReqwestUtils.fetch(url, "GET", null, Map.of()) + return ReqwestUtils.fetch(url, "GET", null, Map.of( + "User-Agent", Constants.USER_AGENT + )) .thenApply(Response::body) .thenApplyAsync(String::new); }