From 56595bd9d5cb3a326d499269f9a6e346dda3877b Mon Sep 17 00:00:00 2001 From: gechoto <124326167+gechoto@users.noreply.github.com> Date: Sun, 29 Dec 2024 01:33:58 +0100 Subject: [PATCH 01/11] [YouTube] Fix extraction of n param deobfuscation function name and fixup function to prevent early return --- .../YoutubeThrottlingParameterUtils.java | 49 ++++++++++++++++--- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java index 2f8f4484b..3bb998373 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java @@ -30,17 +30,31 @@ final class YoutubeThrottlingParameterUtils { private static final Pattern[] DEOBFUSCATION_FUNCTION_NAME_REGEXES = { /* - * The first regex matches the following text, where we want Wma and the array index + * The first regex matches the following text, where we want SDa and the array index * accessed: * - * a.D&&(b="nn"[+a.D],WL(a),c=a.j[b]||null)&&(c=SDa[0](c),a.set(b,c),SDa.length||Wma("") + * WL(a),c=a.j[b]||null)&&(c=SDa[0]( */ - Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "=\"nn\"\\[\\+" + MULTIPLE_CHARS_REGEX - + "\\." + MULTIPLE_CHARS_REGEX + "]," + MULTIPLE_CHARS_REGEX + "\\(" + Pattern.compile(MULTIPLE_CHARS_REGEX + "\\(" + MULTIPLE_CHARS_REGEX + "\\)," + MULTIPLE_CHARS_REGEX + "=" + MULTIPLE_CHARS_REGEX + "\\." + MULTIPLE_CHARS_REGEX + "\\[" - + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\).+\\|\\|(" + MULTIPLE_CHARS_REGEX - + ")\\(\"\"\\)"), + + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\)&&\\(" + MULTIPLE_CHARS_REGEX + "=(" + + MULTIPLE_CHARS_REGEX + ")" + ARRAY_ACCESS_REGEX + "\\("), + + /* + * This regex matches the following text, where we want Wma: + * + * a.D&&(b="nn"[+a.D],WL(a),c=a.j[b]||null)&&(c=SDa[0](c),a.set(b,c),SDa.length||Wma("") + * + * UPDATE: This is broken and Wma is not the function needed anymore. + * We need the function which is in SDa[0] instead. + */ +// Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "=\"nn\"\\[\\+" + MULTIPLE_CHARS_REGEX +// + "\\." + MULTIPLE_CHARS_REGEX + "]," + MULTIPLE_CHARS_REGEX + "\\(" +// + MULTIPLE_CHARS_REGEX + "\\)," + MULTIPLE_CHARS_REGEX + "=" +// + MULTIPLE_CHARS_REGEX + "\\." + MULTIPLE_CHARS_REGEX + "\\[" +// + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\).+\\|\\|(" + MULTIPLE_CHARS_REGEX +// + ")\\(\"\"\\)"), /* * The second regex matches the following text, where we want SDa and the array index @@ -112,6 +126,12 @@ final class YoutubeThrottlingParameterUtils { private static final String FUNCTION_NAMES_IN_DEOBFUSCATION_ARRAY_REGEX = "\\s*=\\s*\\[(.+?)][;,]"; + private static final String FUNCTION_ARGUMENTS_REGEX = + "=\\s*function\\s*\\(\\s*([^)]*)\\s*\\)"; + + private static final String EARLY_RETURN_REGEX = + ";\\s*if\\s*\\(\\s*typeof\\s+" + MULTIPLE_CHARS_REGEX + "+\\s*===?\\s*([\"'])undefined\\1\\s*\\)\\s*return\\s+"; + private YoutubeThrottlingParameterUtils() { } @@ -162,11 +182,13 @@ final class YoutubeThrottlingParameterUtils { static String getDeobfuscationFunction(@Nonnull final String javaScriptPlayerCode, @Nonnull final String functionName) throws ParsingException { + String function; try { - return parseFunctionWithLexer(javaScriptPlayerCode, functionName); + function = parseFunctionWithLexer(javaScriptPlayerCode, functionName); } catch (final Exception e) { - return parseFunctionWithRegex(javaScriptPlayerCode, functionName); + function = parseFunctionWithRegex(javaScriptPlayerCode, functionName); } + return fixupFunction(function); } /** @@ -214,4 +236,15 @@ final class YoutubeThrottlingParameterUtils { JavaScript.compileOrThrow(function); return function; } + + @Nonnull + private static String fixupFunction(@Nonnull final String function) + throws Parser.RegexException { + final String firstArgName = Parser.matchGroup1(FUNCTION_ARGUMENTS_REGEX, function).split(",")[0].trim(); + final Pattern earlyReturnPattern = Pattern.compile( + EARLY_RETURN_REGEX + firstArgName + ";", + Pattern.DOTALL); + final Matcher earlyReturnCodeMatcher = earlyReturnPattern.matcher(function); + return earlyReturnCodeMatcher.replaceFirst(";"); + } } From 68465b27fe9a8b00ed0ccb370ec701056747fffc Mon Sep 17 00:00:00 2001 From: gechoto <124326167+gechoto@users.noreply.github.com> Date: Sun, 29 Dec 2024 13:49:51 +0100 Subject: [PATCH 02/11] fix formatting --- .../youtube/YoutubeThrottlingParameterUtils.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java index 3bb998373..7be94c8f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java @@ -130,7 +130,8 @@ final class YoutubeThrottlingParameterUtils { "=\\s*function\\s*\\(\\s*([^)]*)\\s*\\)"; private static final String EARLY_RETURN_REGEX = - ";\\s*if\\s*\\(\\s*typeof\\s+" + MULTIPLE_CHARS_REGEX + "+\\s*===?\\s*([\"'])undefined\\1\\s*\\)\\s*return\\s+"; + ";\\s*if\\s*\\(\\s*typeof\\s+" + MULTIPLE_CHARS_REGEX + + "+\\s*===?\\s*([\"'])undefined\\1\\s*\\)\\s*return\\s+"; private YoutubeThrottlingParameterUtils() { } @@ -206,7 +207,7 @@ final class YoutubeThrottlingParameterUtils { // If the throttling parameter could not be parsed from the URL, it means that there is // no throttling parameter // Return null in this case - return null; + return null; } } @@ -240,7 +241,9 @@ final class YoutubeThrottlingParameterUtils { @Nonnull private static String fixupFunction(@Nonnull final String function) throws Parser.RegexException { - final String firstArgName = Parser.matchGroup1(FUNCTION_ARGUMENTS_REGEX, function).split(",")[0].trim(); + final String firstArgName = Parser + .matchGroup1(FUNCTION_ARGUMENTS_REGEX, function) + .split(",")[0].trim(); final Pattern earlyReturnPattern = Pattern.compile( EARLY_RETURN_REGEX + firstArgName + ";", Pattern.DOTALL); From 1393644f444b850fe629bbfdf0ea21d6121311cc Mon Sep 17 00:00:00 2001 From: gechoto <124326167+gechoto@users.noreply.github.com> Date: Sun, 29 Dec 2024 17:33:35 +0100 Subject: [PATCH 03/11] formatting again --- .../services/youtube/YoutubeThrottlingParameterUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java index 7be94c8f3..d9756e388 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java @@ -207,7 +207,7 @@ final class YoutubeThrottlingParameterUtils { // If the throttling parameter could not be parsed from the URL, it means that there is // no throttling parameter // Return null in this case - return null; + return null; } } From e0d39b606c1d489208eca329610dc967076f80b8 Mon Sep 17 00:00:00 2001 From: gechoto <124326167+gechoto@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:31:23 +0100 Subject: [PATCH 04/11] correct comment --- .../youtube/YoutubeThrottlingParameterUtils.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java index d9756e388..1fa74a162 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java @@ -175,7 +175,7 @@ final class YoutubeThrottlingParameterUtils { * Get the throttling parameter deobfuscation code of YouTube's base JavaScript file. * * @param javaScriptPlayerCode the complete JavaScript base player code - * @return the throttling parameter deobfuscation function name + * @return the throttling parameter deobfuscation function code * @throws ParsingException if the throttling parameter deobfuscation code couldn't be * extracted */ @@ -238,14 +238,20 @@ final class YoutubeThrottlingParameterUtils { return function; } + /** + * + * @param function + * @return + * @throws Parser.RegexException + */ @Nonnull private static String fixupFunction(@Nonnull final String function) throws Parser.RegexException { final String firstArgName = Parser - .matchGroup1(FUNCTION_ARGUMENTS_REGEX, function) + .matchGroup1(FUNCTION_ARGUMENTS_REGEX + "aba([a-z]\\d)", function) .split(",")[0].trim(); final Pattern earlyReturnPattern = Pattern.compile( - EARLY_RETURN_REGEX + firstArgName + ";", + EARLY_RETURN_REGEX + firstArgName + "aba;", Pattern.DOTALL); final Matcher earlyReturnCodeMatcher = earlyReturnPattern.matcher(function); return earlyReturnCodeMatcher.replaceFirst(";"); From 4181625c2cfd41be6e3260a2f476df0f0930888f Mon Sep 17 00:00:00 2001 From: gechoto <124326167+gechoto@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:50:14 +0100 Subject: [PATCH 05/11] add comment to throttling parameter deobfuscation function fixup --- .../youtube/YoutubeThrottlingParameterUtils.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java index 1fa74a162..ef7a1d0d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java @@ -239,10 +239,20 @@ final class YoutubeThrottlingParameterUtils { } /** + * Removes an early return statement from the code of the throttling parameter deobfuscation function. * - * @param function - * @return - * @throws Parser.RegexException + *

In newer version of the player code the function contains a check for something defined outside of the function. + * If that was not found it will return early. + * + *

The check can look like this (JS):
+ * if(typeof RUQ==="undefined")return p; + * + *

In this example RUQ will always be undefined when running the function as standalone. + * If the check is kept it would just return p which is the input parameter and would be wrong. + * For that reason this check and return statement needs to be removed. + * + * @param function the original throttling parameter deobfuscation function code + * @return the throttling parameter deobfuscation function code with the early return statement removed */ @Nonnull private static String fixupFunction(@Nonnull final String function) From d007e368f3030118ce5c57518a5e5038c6828bfe Mon Sep 17 00:00:00 2001 From: gechoto <124326167+gechoto@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:54:57 +0100 Subject: [PATCH 06/11] don't commit testing code kids --- .../services/youtube/YoutubeThrottlingParameterUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java index ef7a1d0d8..2947689fd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java @@ -258,10 +258,10 @@ final class YoutubeThrottlingParameterUtils { private static String fixupFunction(@Nonnull final String function) throws Parser.RegexException { final String firstArgName = Parser - .matchGroup1(FUNCTION_ARGUMENTS_REGEX + "aba([a-z]\\d)", function) + .matchGroup1(FUNCTION_ARGUMENTS_REGEX, function) .split(",")[0].trim(); final Pattern earlyReturnPattern = Pattern.compile( - EARLY_RETURN_REGEX + firstArgName + "aba;", + EARLY_RETURN_REGEX + firstArgName + ";", Pattern.DOTALL); final Matcher earlyReturnCodeMatcher = earlyReturnPattern.matcher(function); return earlyReturnCodeMatcher.replaceFirst(";"); From 14f7ef1093ece92d81f8f7aed75de9ce90e9086d Mon Sep 17 00:00:00 2001 From: gechoto <124326167+gechoto@users.noreply.github.com> Date: Sun, 5 Jan 2025 20:39:53 +0000 Subject: [PATCH 07/11] YoutubeThrottlingParameterUtils: add back second regex and swap it with the third one --- .../YoutubeThrottlingParameterUtils.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java index 2947689fd..5f6997cf0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java @@ -41,21 +41,6 @@ final class YoutubeThrottlingParameterUtils { + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\)&&\\(" + MULTIPLE_CHARS_REGEX + "=(" + MULTIPLE_CHARS_REGEX + ")" + ARRAY_ACCESS_REGEX + "\\("), - /* - * This regex matches the following text, where we want Wma: - * - * a.D&&(b="nn"[+a.D],WL(a),c=a.j[b]||null)&&(c=SDa[0](c),a.set(b,c),SDa.length||Wma("") - * - * UPDATE: This is broken and Wma is not the function needed anymore. - * We need the function which is in SDa[0] instead. - */ -// Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "=\"nn\"\\[\\+" + MULTIPLE_CHARS_REGEX -// + "\\." + MULTIPLE_CHARS_REGEX + "]," + MULTIPLE_CHARS_REGEX + "\\(" -// + MULTIPLE_CHARS_REGEX + "\\)," + MULTIPLE_CHARS_REGEX + "=" -// + MULTIPLE_CHARS_REGEX + "\\." + MULTIPLE_CHARS_REGEX + "\\[" -// + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\).+\\|\\|(" + MULTIPLE_CHARS_REGEX -// + ")\\(\"\"\\)"), - /* * The second regex matches the following text, where we want SDa and the array index * accessed: @@ -70,7 +55,22 @@ final class YoutubeThrottlingParameterUtils { + MULTIPLE_CHARS_REGEX + ")" + ARRAY_ACCESS_REGEX), /* - * The third regex matches the following text, where we want rma: + * The third regex matches the following text, where we want Wma: + * + * a.D&&(b="nn"[+a.D],WL(a),c=a.j[b]||null)&&(c=SDa[0](c),a.set(b,c),SDa.length||Wma("") + * + * UPDATE: This is prone to find the wrong function on recent version of the player. + * For this reason a regex which finds the Array + index should be preferred / above. + */ + Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "=\"nn\"\\[\\+" + MULTIPLE_CHARS_REGEX + + "\\." + MULTIPLE_CHARS_REGEX + "]," + MULTIPLE_CHARS_REGEX + "\\(" + + MULTIPLE_CHARS_REGEX + "\\)," + MULTIPLE_CHARS_REGEX + "=" + + MULTIPLE_CHARS_REGEX + "\\." + MULTIPLE_CHARS_REGEX + "\\[" + + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\).+\\|\\|(" + MULTIPLE_CHARS_REGEX + + ")\\(\"\"\\)"), + + /* + * The fourth regex matches the following text, where we want rma: * * a.D&&(b="nn"[+a.D],c=a.get(b))&&(c=rDa[0](c),a.set(b,c),rDa.length||rma("") */ @@ -80,7 +80,7 @@ final class YoutubeThrottlingParameterUtils { + MULTIPLE_CHARS_REGEX + ")\\(\"\"\\)"), /* - * The fourth regex matches the following text, where we want rDa and the array index + * The fifth regex matches the following text, where we want rDa and the array index * accessed: * * a.D&&(b="nn"[+a.D],c=a.get(b))&&(c=rDa[0](c),a.set(b,c),rDa.length||rma("") @@ -91,7 +91,7 @@ final class YoutubeThrottlingParameterUtils { + MULTIPLE_CHARS_REGEX + "=(" + MULTIPLE_CHARS_REGEX + ")\\[(\\d+)]"), /* - * The fifth regex matches the following text, where we want BDa and the array index + * The sixth regex matches the following text, where we want BDa and the array index * accessed: * * (b=String.fromCharCode(110),c=a.get(b))&&(c=BDa[0](c) @@ -103,7 +103,7 @@ final class YoutubeThrottlingParameterUtils { + SINGLE_CHAR_VARIABLE_REGEX + "\\)"), /* - * The sixth regex matches the following text, where we want Yva and the array index + * The seventh regex matches the following text, where we want Yva and the array index * accessed: * * .get("n"))&&(b=Yva[0](b) From c7d598869cdcd124acddff5b9b2e0abf204c514c Mon Sep 17 00:00:00 2001 From: gechoto <124326167+gechoto@users.noreply.github.com> Date: Mon, 6 Jan 2025 21:32:05 +0000 Subject: [PATCH 08/11] YoutubeThrottlingParameterUtils: swap regex positions again --- .../YoutubeThrottlingParameterUtils.java | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java index 5f6997cf0..832f9129d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java @@ -33,18 +33,6 @@ final class YoutubeThrottlingParameterUtils { * The first regex matches the following text, where we want SDa and the array index * accessed: * - * WL(a),c=a.j[b]||null)&&(c=SDa[0]( - */ - Pattern.compile(MULTIPLE_CHARS_REGEX + "\\(" - + MULTIPLE_CHARS_REGEX + "\\)," + MULTIPLE_CHARS_REGEX + "=" - + MULTIPLE_CHARS_REGEX + "\\." + MULTIPLE_CHARS_REGEX + "\\[" - + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\)&&\\(" + MULTIPLE_CHARS_REGEX + "=(" - + MULTIPLE_CHARS_REGEX + ")" + ARRAY_ACCESS_REGEX + "\\("), - - /* - * The second regex matches the following text, where we want SDa and the array index - * accessed: - * * a.D&&(b="nn"[+a.D],WL(a),c=a.j[b]||null)&&(c=SDa[0](c),a.set(b,c),SDa.length||Wma("") */ Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "=\"nn\"\\[\\+" + MULTIPLE_CHARS_REGEX @@ -55,12 +43,9 @@ final class YoutubeThrottlingParameterUtils { + MULTIPLE_CHARS_REGEX + ")" + ARRAY_ACCESS_REGEX), /* - * The third regex matches the following text, where we want Wma: + * The second regex matches the following text, where we want Wma: * * a.D&&(b="nn"[+a.D],WL(a),c=a.j[b]||null)&&(c=SDa[0](c),a.set(b,c),SDa.length||Wma("") - * - * UPDATE: This is prone to find the wrong function on recent version of the player. - * For this reason a regex which finds the Array + index should be preferred / above. */ Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "=\"nn\"\\[\\+" + MULTIPLE_CHARS_REGEX + "\\." + MULTIPLE_CHARS_REGEX + "]," + MULTIPLE_CHARS_REGEX + "\\(" @@ -69,6 +54,18 @@ final class YoutubeThrottlingParameterUtils { + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\).+\\|\\|(" + MULTIPLE_CHARS_REGEX + ")\\(\"\"\\)"), + /* + * The third regex matches the following text, where we want SDa and the array index + * accessed: + * + * WL(a),c=a.j[b]||null)&&(c=SDa[0]( + */ + Pattern.compile(MULTIPLE_CHARS_REGEX + "\\(" + + MULTIPLE_CHARS_REGEX + "\\)," + MULTIPLE_CHARS_REGEX + "=" + + MULTIPLE_CHARS_REGEX + "\\." + MULTIPLE_CHARS_REGEX + "\\[" + + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\)&&\\(" + MULTIPLE_CHARS_REGEX + "=(" + + MULTIPLE_CHARS_REGEX + ")" + ARRAY_ACCESS_REGEX + "\\("), + /* * The fourth regex matches the following text, where we want rma: * From ca9f1935d3ab985d866006f55bbdf0c09d08a3d0 Mon Sep 17 00:00:00 2001 From: gechoto <124326167+gechoto@users.noreply.github.com> Date: Mon, 6 Jan 2025 21:44:16 +0000 Subject: [PATCH 09/11] YoutubeThrottlingParameterUtils: make third regex longer to ensure it is a function which takes one parameter --- .../services/youtube/YoutubeThrottlingParameterUtils.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java index 832f9129d..fea444cdb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java @@ -58,13 +58,14 @@ final class YoutubeThrottlingParameterUtils { * The third regex matches the following text, where we want SDa and the array index * accessed: * - * WL(a),c=a.j[b]||null)&&(c=SDa[0]( + * WL(a),c=a.j[b]||null)&&(c=SDa[0](c) */ Pattern.compile(MULTIPLE_CHARS_REGEX + "\\(" + MULTIPLE_CHARS_REGEX + "\\)," + MULTIPLE_CHARS_REGEX + "=" + MULTIPLE_CHARS_REGEX + "\\." + MULTIPLE_CHARS_REGEX + "\\[" + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\)&&\\(" + MULTIPLE_CHARS_REGEX + "=(" - + MULTIPLE_CHARS_REGEX + ")" + ARRAY_ACCESS_REGEX + "\\("), + + MULTIPLE_CHARS_REGEX + ")" + ARRAY_ACCESS_REGEX + + "\\(" + MULTIPLE_CHARS_REGEX + "\\)"), /* * The fourth regex matches the following text, where we want rma: From 2b70b5f737536ecac27488e233ade4d8d851e0f8 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sun, 26 Jan 2025 12:55:07 +0100 Subject: [PATCH 10/11] Improve new regex to account for more context --- .../youtube/YoutubeThrottlingParameterUtils.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java index fea444cdb..621447469 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java @@ -58,14 +58,16 @@ final class YoutubeThrottlingParameterUtils { * The third regex matches the following text, where we want SDa and the array index * accessed: * - * WL(a),c=a.j[b]||null)&&(c=SDa[0](c) + * ,Vb(m),W=m.j[c]||null)&&(W=cvb[0](W),m.set(c,W) */ - Pattern.compile(MULTIPLE_CHARS_REGEX + "\\(" + Pattern.compile("," + MULTIPLE_CHARS_REGEX + "\\(" + MULTIPLE_CHARS_REGEX + "\\)," + MULTIPLE_CHARS_REGEX + "=" + MULTIPLE_CHARS_REGEX + "\\." + MULTIPLE_CHARS_REGEX + "\\[" - + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\)&&\\(" + MULTIPLE_CHARS_REGEX + "=(" - + MULTIPLE_CHARS_REGEX + ")" + ARRAY_ACCESS_REGEX - + "\\(" + MULTIPLE_CHARS_REGEX + "\\)"), + + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\)&&\\(\\b" + MULTIPLE_CHARS_REGEX + "=(" + + MULTIPLE_CHARS_REGEX + ")" + ARRAY_ACCESS_REGEX + "\\(" + + SINGLE_CHAR_VARIABLE_REGEX + "\\)," + MULTIPLE_CHARS_REGEX + + "\\.set\\((?:\"n+\"|" + MULTIPLE_CHARS_REGEX + ")," + MULTIPLE_CHARS_REGEX + + "\\)"), /* * The fourth regex matches the following text, where we want rma: From b74a8d0df811d12699a475035783f172dca9e3af Mon Sep 17 00:00:00 2001 From: Stypox Date: Sun, 26 Jan 2025 12:57:31 +0100 Subject: [PATCH 11/11] Remove manually writing cardinals of regexes --- .../YoutubeThrottlingParameterUtils.java | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java index 621447469..b25d89026 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java @@ -30,8 +30,7 @@ final class YoutubeThrottlingParameterUtils { private static final Pattern[] DEOBFUSCATION_FUNCTION_NAME_REGEXES = { /* - * The first regex matches the following text, where we want SDa and the array index - * accessed: + * Matches the following text, where we want SDa and the array index accessed: * * a.D&&(b="nn"[+a.D],WL(a),c=a.j[b]||null)&&(c=SDa[0](c),a.set(b,c),SDa.length||Wma("") */ @@ -43,7 +42,7 @@ final class YoutubeThrottlingParameterUtils { + MULTIPLE_CHARS_REGEX + ")" + ARRAY_ACCESS_REGEX), /* - * The second regex matches the following text, where we want Wma: + * Matches the following text, where we want Wma: * * a.D&&(b="nn"[+a.D],WL(a),c=a.j[b]||null)&&(c=SDa[0](c),a.set(b,c),SDa.length||Wma("") */ @@ -55,8 +54,7 @@ final class YoutubeThrottlingParameterUtils { + ")\\(\"\"\\)"), /* - * The third regex matches the following text, where we want SDa and the array index - * accessed: + * Matches the following text, where we want cvb and the array index accessed: * * ,Vb(m),W=m.j[c]||null)&&(W=cvb[0](W),m.set(c,W) */ @@ -70,7 +68,7 @@ final class YoutubeThrottlingParameterUtils { + "\\)"), /* - * The fourth regex matches the following text, where we want rma: + * Matches the following text, where we want rma: * * a.D&&(b="nn"[+a.D],c=a.get(b))&&(c=rDa[0](c),a.set(b,c),rDa.length||rma("") */ @@ -80,8 +78,7 @@ final class YoutubeThrottlingParameterUtils { + MULTIPLE_CHARS_REGEX + ")\\(\"\"\\)"), /* - * The fifth regex matches the following text, where we want rDa and the array index - * accessed: + * Matches the following text, where we want rDa and the array index accessed: * * a.D&&(b="nn"[+a.D],c=a.get(b))&&(c=rDa[0](c),a.set(b,c),rDa.length||rma("") */ @@ -91,8 +88,7 @@ final class YoutubeThrottlingParameterUtils { + MULTIPLE_CHARS_REGEX + "=(" + MULTIPLE_CHARS_REGEX + ")\\[(\\d+)]"), /* - * The sixth regex matches the following text, where we want BDa and the array index - * accessed: + * Matches the following text, where we want BDa and the array index accessed: * * (b=String.fromCharCode(110),c=a.get(b))&&(c=BDa[0](c) */ @@ -103,8 +99,7 @@ final class YoutubeThrottlingParameterUtils { + SINGLE_CHAR_VARIABLE_REGEX + "\\)"), /* - * The seventh regex matches the following text, where we want Yva and the array index - * accessed: + * Matches the following text, where we want Yva and the array index accessed: * * .get("n"))&&(b=Yva[0](b) */