From 545522b80fc68bc12366aa0c48f91bd095ff5089 Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Tue, 1 Feb 2022 19:52:41 +0100 Subject: [PATCH] [Youtube] Make throttling decryption more resilient to api change --- .../youtube/YoutubeThrottlingDecrypter.java | 97 ++++++++++++------- .../extractors/YoutubeStreamExtractor.java | 25 +++-- .../YoutubeThrottlingDecrypterTest.java | 6 +- 3 files changed, 83 insertions(+), 45 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java index 21925392a..02566c4b3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java @@ -38,7 +38,9 @@ public class YoutubeThrottlingDecrypter { private static final Pattern FUNCTION_NAME_PATTERN = Pattern.compile( "b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\S+)\\(b\\),a\\.set\\(\"n\",b\\)"); - private static final Map nParams = new HashMap<>(); + private static final Map N_PARAMS_CACHE = new HashMap<>(); + private static String FUNCTION; + private static String FUNCTION_NAME; private final String functionName; private final String function; @@ -49,6 +51,8 @@ public class YoutubeThrottlingDecrypter { * is requested. *

* Otherwise use the no-arg constructor which uses a constant value. + * + * @deprecated Use static function instead */ public YoutubeThrottlingDecrypter(final String videoId) throws ParsingException { final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId); @@ -57,6 +61,9 @@ public class YoutubeThrottlingDecrypter { function = parseDecodeFunction(playerJsCode, functionName); } + /** + * @deprecated Use static function instead + */ public YoutubeThrottlingDecrypter() throws ParsingException { final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(); @@ -64,27 +71,50 @@ public class YoutubeThrottlingDecrypter { function = parseDecodeFunction(playerJsCode, functionName); } - private String parseDecodeFunctionName(final String playerJsCode) - throws Parser.RegexException { - String functionName = Parser.matchGroup1(FUNCTION_NAME_PATTERN, playerJsCode); - final int arrayStartBrace = functionName.indexOf("["); - - if (arrayStartBrace > 0) { - final String arrayVarName = functionName.substring(0, arrayStartBrace); - final String order = functionName.substring( - arrayStartBrace + 1, functionName.indexOf("]")); - final int arrayNum = Integer.parseInt(order); - final Pattern arrayPattern = Pattern.compile( - String.format("var %s=\\[(.+?)\\];", arrayVarName)); - final String arrayStr = Parser.matchGroup1(arrayPattern, playerJsCode); - final String[] names = arrayStr.split(","); - functionName = names[arrayNum]; + /** + *

+ * The videoId is only used to fetch the decryption function. + * It can be a constant value of any existing video. + * A constant value is discouraged, because it could allow tracking. + */ + public static String apply(final String url, final String videoId) throws ParsingException { + if (containsNParam(url)) { + if (FUNCTION == null) { + final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId); + + FUNCTION_NAME = parseDecodeFunctionName(playerJsCode); + FUNCTION = parseDecodeFunction(playerJsCode, FUNCTION_NAME); } - return functionName; + + final String oldNParam = parseNParam(url); + final String newNParam = decryptNParam(FUNCTION, FUNCTION_NAME, oldNParam); + return replaceNParam(url, oldNParam, newNParam); + } else { + return url; + } + } + + private static String parseDecodeFunctionName(final String playerJsCode) + throws Parser.RegexException { + String functionName = Parser.matchGroup1(FUNCTION_NAME_PATTERN, playerJsCode); + final int arrayStartBrace = functionName.indexOf("["); + + if (arrayStartBrace > 0) { + final String arrayVarName = functionName.substring(0, arrayStartBrace); + final String order = functionName.substring( + arrayStartBrace + 1, functionName.indexOf("]")); + final int arrayNum = Integer.parseInt(order); + final Pattern arrayPattern = Pattern.compile( + String.format("var %s=\\[(.+?)\\];", arrayVarName)); + final String arrayStr = Parser.matchGroup1(arrayPattern, playerJsCode); + final String[] names = arrayStr.split(","); + functionName = names[arrayNum]; + } + return functionName; } @Nonnull - private String parseDecodeFunction(final String playerJsCode, final String functionName) + private static String parseDecodeFunction(final String playerJsCode, final String functionName) throws Parser.RegexException { try { return parseWithParenthesisMatching(playerJsCode, functionName); @@ -94,49 +124,50 @@ public class YoutubeThrottlingDecrypter { } @Nonnull - private String parseWithParenthesisMatching(final String playerJsCode, final String functionName) { + private static String parseWithParenthesisMatching(final String playerJsCode, final String functionName) { final String functionBase = functionName + "=function"; return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase) + ";"; } @Nonnull - private String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException { + private static String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException { final Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n", Pattern.DOTALL); return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode); } - public String apply(final String url) throws Parser.RegexException { + @Deprecated + public String apply(final String url) throws ParsingException { if (containsNParam(url)) { final String oldNParam = parseNParam(url); - final String newNParam = decryptNParam(oldNParam); + final String newNParam = decryptNParam(function, functionName, oldNParam); return replaceNParam(url, oldNParam, newNParam); } else { return url; } } - private boolean containsNParam(final String url) { + private static boolean containsNParam(final String url) { return Parser.isMatch(N_PARAM_PATTERN, url); } - private String parseNParam(final String url) throws Parser.RegexException { + private static String parseNParam(final String url) throws Parser.RegexException { return Parser.matchGroup1(N_PARAM_PATTERN, url); } - private String decryptNParam(final String nParam) { - if (nParams.containsKey(nParam)) { - return nParams.get(nParam); + private static String decryptNParam(final String function, final String functionName, final String nParam) { + if (N_PARAMS_CACHE.containsKey(nParam)) { + return N_PARAMS_CACHE.get(nParam); } final String decryptedNParam = JavaScript.run(function, functionName, nParam); - nParams.put(nParam, decryptedNParam); + N_PARAMS_CACHE.put(nParam, decryptedNParam); return decryptedNParam; } @Nonnull - private String replaceNParam(@Nonnull final String url, - final String oldValue, - final String newValue) { + private static String replaceNParam(@Nonnull final String url, + final String oldValue, + final String newValue) { return url.replace(oldValue, newValue); } @@ -144,13 +175,13 @@ public class YoutubeThrottlingDecrypter { * @return the number of the cached "n" query parameters. */ public static int getCacheSize() { - return nParams.size(); + return N_PARAMS_CACHE.size(); } /** * Clears all stored "n" query parameters. */ public static void clearCache() { - nParams.clear(); + N_PARAMS_CACHE.clear(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 5a04a772b..e83eeb257 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -469,14 +469,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { public List getAudioStreams() throws ExtractionException { assertPageFetched(); final List audioStreams = new ArrayList<>(); - final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); try { for (final Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) { final ItagItem itag = entry.getValue(); - String url = entry.getKey(); - url = throttlingDecrypter.apply(url); + final String url = tryDecryption(entry.getKey(), getId()); final AudioStream audioStream = new AudioStream(url, itag); if (!Stream.containSimilarStream(audioStream, audioStreams)) { @@ -494,14 +492,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { public List getVideoStreams() throws ExtractionException { assertPageFetched(); final List videoStreams = new ArrayList<>(); - final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); try { for (final Map.Entry entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) { final ItagItem itag = entry.getValue(); - String url = entry.getKey(); - url = throttlingDecrypter.apply(url); + final String url = tryDecryption(entry.getKey(), getId()); final VideoStream videoStream = new VideoStream(url, false, itag); if (!Stream.containSimilarStream(videoStream, videoStreams)) { @@ -519,14 +515,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { public List getVideoOnlyStreams() throws ExtractionException { assertPageFetched(); final List videoOnlyStreams = new ArrayList<>(); - final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); try { for (final Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { final ItagItem itag = entry.getValue(); - String url = entry.getKey(); - url = throttlingDecrypter.apply(url); + final String url = tryDecryption(entry.getKey(), getId()); final VideoStream videoStream = new VideoStream(url, true, itag); if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) { @@ -540,6 +534,19 @@ public class YoutubeStreamExtractor extends StreamExtractor { return videoOnlyStreams; } + /** + * Try to decrypt url and fallback to given url, because decryption is not + * always needed. + * This way a breaking change from YouTube does not result in a broken extractor. + */ + private String tryDecryption(final String url, final String videoId) { + try { + return YoutubeThrottlingDecrypter.apply(url, videoId); + } catch (ParsingException e) { + return url; + } + } + @Override @Nonnull public List getSubtitlesDefault() throws ParsingException { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java index 92690964a..11312fcf1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java @@ -28,7 +28,7 @@ public class YoutubeThrottlingDecrypterTest { for (final String videoId : videoIds) { try { - final String decryptedUrl = new YoutubeThrottlingDecrypter(videoId).apply(encryptedUrl); + final String decryptedUrl = YoutubeThrottlingDecrypter.apply(encryptedUrl, videoId); assertNotEquals(encryptedUrl, decryptedUrl); } catch (EvaluatorException e) { fail("Failed to extract n param decrypt function for video " + videoId + "\n" + e); @@ -40,7 +40,7 @@ public class YoutubeThrottlingDecrypterTest { public void testDecode__success() throws ParsingException { // URL extracted from browser with the dev tools final String encryptedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=N9BWSTFT7vvBJrvQ&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190"; - final String decryptedUrl = new YoutubeThrottlingDecrypter().apply(encryptedUrl); + final String decryptedUrl = YoutubeThrottlingDecrypter.apply(encryptedUrl, "jE1USQrs1rw"); // The cipher function changes over time, so we just check if the n param changed. assertNotEquals(encryptedUrl, decryptedUrl); } @@ -48,7 +48,7 @@ public class YoutubeThrottlingDecrypterTest { @Test public void testDecode__noNParam__success() throws ParsingException { final String noNParamUrl = "https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?expire=1626553257&ei=SefyYPmIFoKT1wLtqbjgCQ&ip=127.0.0.1&id=o-AIT5xGifsaEAdEOAb5vd06J9VNtm-KHHolnaZRGPjHZi&itag=140&source=youtube&requiressl=yes&mh=xO&mm=31%2C29&mn=sn-4g5ednsz%2Csn-4g5e6nsr&ms=au%2Crdu&mv=m&mvi=5&pl=24&initcwndbps=1322500&vprv=1&mime=audio%2Fmp4&ns=cA2SS5atEe0mH8tMwGTO4RIG&gir=yes&clen=3009275&dur=185.898&lmt=1626356984653961&mt=1626531173&fvip=5&keepalive=yes&fexp=24001373%2C24007246&beids=23886212&c=WEB&txp=6411222&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&lsig=AG3C_xAwRgIhAPueRlTutSlzPafxrqBmgZz5m7-Zfbw3QweDp3j4XO9SAiEA5tF7_ZCJFKmS-D6I1jlUURjpjoiTbsYyKuarV4u6E8Y%3D&sig=AOq0QJ8wRQIgRD_4WwkPeTEKGVSQqPsznMJGqq4nVJ8o1ChGBCgi4Y0CIQCZT3tI40YLKBWJCh2Q7AlvuUIpN0ficzdSElLeQpJdrw=="; - final String decrypted = new YoutubeThrottlingDecrypter().apply(noNParamUrl); + final String decrypted = YoutubeThrottlingDecrypter.apply(noNParamUrl, "jE1USQrs1rw"); assertEquals(noNParamUrl, decrypted); }