From 7b5d41a7413548056967204e06e2588c8301a85e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 21:11:54 +0000 Subject: [PATCH 1/9] Bump org.mozilla:rhino from 1.7.15 to 1.8.0 Bumps [org.mozilla:rhino](https://github.com/mozilla/rhino) from 1.7.15 to 1.8.0. - [Release notes](https://github.com/mozilla/rhino/releases) - [Changelog](https://github.com/mozilla/rhino/blob/master/RELEASE-NOTES.md) - [Commits](https://github.com/mozilla/rhino/commits) --- updated-dependencies: - dependency-name: org.mozilla:rhino dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- extractor/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/build.gradle b/extractor/build.gradle index 2b46253ea..be14bc953 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -29,7 +29,7 @@ dependencies { implementation 'org.jsoup:jsoup:1.17.2' implementation "com.google.code.findbugs:jsr305:$jsr305Version" - implementation 'org.mozilla:rhino:1.7.15' + implementation 'org.mozilla:rhino:1.8.0' checkstyle "com.puppycrawl.tools:checkstyle:$checkstyleVersion" From 2d3342bf56cae2905a118133d52c8e3076304c99 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Wed, 5 Mar 2025 15:44:25 +0100 Subject: [PATCH 2/9] Improve tests --- .../extractor/utils/JavaScriptExtractorTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JavaScriptExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JavaScriptExtractorTest.java index 807054fce..ad47e3e74 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JavaScriptExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JavaScriptExtractorTest.java @@ -1,5 +1,9 @@ package org.schabi.newpipe.extractor.utils; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.FileUtils.resolveTestResource; + import org.junit.jupiter.api.Test; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.jsextractor.JavaScriptExtractor; @@ -10,11 +14,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.FileUtils.resolveTestResource; - -public class JavaScriptExtractorTest +class JavaScriptExtractorTest { @Test void testJsExtractor() throws ParsingException { From b376539062b76bf04d3e2ad1b4c1a1fa4fb74f68 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Wed, 5 Mar 2025 15:44:33 +0100 Subject: [PATCH 3/9] Also use rhino engine --- extractor/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/extractor/build.gradle b/extractor/build.gradle index be14bc953..951a9e029 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -30,6 +30,7 @@ dependencies { implementation "com.google.code.findbugs:jsr305:$jsr305Version" implementation 'org.mozilla:rhino:1.8.0' + implementation 'org.mozilla:rhino-engine:1.8.0' checkstyle "com.puppycrawl.tools:checkstyle:$checkstyleVersion" From 470a7198619e12409a9b0c542227dc523c7d587b Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Wed, 5 Mar 2025 15:44:46 +0100 Subject: [PATCH 4/9] Fix compile problems in JavaScript class --- .../schabi/newpipe/extractor/utils/JavaScript.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java index ab30ed806..4522b51a4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java @@ -10,31 +10,25 @@ public final class JavaScript { } public static void compileOrThrow(final String function) { - try { - final Context context = Context.enter(); - context.setOptimizationLevel(-1); + try (Context context = Context.enter()) { + context.setInterpretedMode(true); // If it doesn't compile it throws an exception here context.compileString(function, null, 1, null); - } finally { - Context.exit(); } } public static String run(final String function, final String functionName, final String... parameters) { - try { - final Context context = Context.enter(); - context.setOptimizationLevel(-1); + try (Context context = Context.enter()) { + context.setInterpretedMode(true); final ScriptableObject scope = context.initSafeStandardObjects(); context.evaluateString(scope, function, functionName, 1, null); final Function jsFunction = (Function) scope.get(functionName, scope); final Object result = jsFunction.call(context, scope, scope, parameters); return result.toString(); - } finally { - Context.exit(); } } From bcf24def8b36d38527d152751083d807771a3201 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Wed, 5 Mar 2025 15:44:57 +0100 Subject: [PATCH 5/9] Cleanup test --- .../youtube/YoutubeThrottlingParameterDeobfuscationTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterDeobfuscationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterDeobfuscationTest.java index f90284713..dcb569c99 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterDeobfuscationTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterDeobfuscationTest.java @@ -10,14 +10,12 @@ import org.schabi.newpipe.downloader.DownloaderFactory; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import java.io.IOException; - class YoutubeThrottlingParameterDeobfuscationTest { private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/parameterDeobf"; @BeforeEach - void setup() throws IOException { + void setup() { YoutubeTestsUtils.ensureStateless(); NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH)); } From 907fc2ac52a735852e5c48ae2cb784c081bb7844 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Wed, 5 Mar 2025 15:45:08 +0100 Subject: [PATCH 6/9] Fix compile problems and optimize TokenStream --- .../utils/jsextractor/TokenStream.java | 100 ++++-------------- 1 file changed, 18 insertions(+), 82 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/jsextractor/TokenStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/jsextractor/TokenStream.java index 81651d227..6533860a0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/jsextractor/TokenStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/jsextractor/TokenStream.java @@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.utils.jsextractor; import org.mozilla.javascript.Context; import org.mozilla.javascript.Kit; -import org.mozilla.javascript.ObjToIntMap; import org.mozilla.javascript.ScriptRuntime; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -38,12 +37,10 @@ class TokenStream { this.languageVersion = languageVersion; } - static boolean isKeyword(final String s, final int version, final boolean isStrict) { - return Token.EOF != stringToKeyword(s, version, isStrict); - } - - private static Token stringToKeyword(final String name, final int version, - final boolean isStrict) { + private static Token stringToKeyword( + final String name, + final int version, + final boolean isStrict) { if (version < Context.VERSION_ES6) { return stringToKeywordForJS(name); } @@ -343,7 +340,7 @@ class TokenStream { } ungetChar(c); - String str = getStringFromBuffer(); + final String str = getStringFromBuffer(); if (!containsEscape) { // OPT we shouldn't have to make a string (object!) to // check if it's a keyword. @@ -353,30 +350,17 @@ class TokenStream { if (result != Token.EOF) { if ((result == Token.LET || result == Token.YIELD) && languageVersion < Context.VERSION_1_7) { - // LET and YIELD are tokens only in 1.7 and later - string = result == Token.LET ? "let" : "yield"; result = Token.NAME; } // Save the string in case we need to use in // object literal definitions. - this.string = (String) allStrings.intern(str); - if (result != Token.RESERVED) { - return result; - } else if (languageVersion >= Context.VERSION_ES6) { - return result; - } else if (!IS_RESERVED_KEYWORD_AS_IDENTIFIER) { + if (result != Token.RESERVED + || languageVersion >= Context.VERSION_ES6 + || !IS_RESERVED_KEYWORD_AS_IDENTIFIER) { return result; } } - } else if (isKeyword( - str, - languageVersion, - STRICT_MODE)) { - // If a string contains unicodes, and converted to a keyword, - // we convert the last character back to unicode - str = convertLastCharToHex(str); } - this.string = (String) allStrings.intern(str); return Token.NAME; } @@ -466,7 +450,7 @@ class TokenStream { } } ungetChar(c); - this.string = getStringFromBuffer(); + tokenEnd = cursor; return Token.NUMBER; } @@ -562,7 +546,7 @@ class TokenStream { escapeVal = Kit.xDigitToInt(c, 0); if (escapeVal < 0) { addToString('x'); - continue strLoop; + continue; } final int c1 = c; c = getChar(); @@ -570,7 +554,7 @@ class TokenStream { if (escapeVal < 0) { addToString('x'); addToString(c1); - continue strLoop; + continue; } // got 2 hex digits c = escapeVal; @@ -580,7 +564,7 @@ class TokenStream { // Remove line terminator after escape to follow // SpiderMonkey and C/C++ c = getChar(); - continue strLoop; + continue; default: if ('0' <= c && c < '8') { @@ -605,8 +589,7 @@ class TokenStream { c = getChar(false); } - final String str = getStringFromBuffer(); - this.string = (String) allStrings.intern(str); + tokenEnd = cursor; return quoteChar == '`' ? Token.TEMPLATE_LITERAL : Token.STRING; } @@ -722,14 +705,13 @@ class TokenStream { return Token.GT; case '*': - if (languageVersion >= Context.VERSION_ES6) { - if (matchChar('*')) { - if (matchChar('=')) { - return Token.ASSIGN_EXP; - } - return Token.EXP; + if (languageVersion >= Context.VERSION_ES6 && matchChar('*')) { + if (matchChar('=')) { + return Token.ASSIGN_EXP; } + return Token.EXP; } + if (matchChar('=')) { return Token.ASSIGN_MUL; } @@ -920,7 +902,6 @@ class TokenStream { } if (peekChar() == '*') { tokenEnd = cursor - 1; - this.string = new String(stringBuffer, 0, stringBufferTop); throw new ParsingException("msg.unterminated.re.lit"); } } @@ -944,7 +925,6 @@ class TokenStream { } addToString(c); } - final int reEnd = stringBufferTop; while (true) { c = getCharIgnoreLineEnd(); @@ -959,7 +939,6 @@ class TokenStream { } tokenEnd = start + stringBufferTop + 2; // include slashes - this.string = new String(stringBuffer, 0, reEnd); } private String getStringFromBuffer() { @@ -1019,7 +998,6 @@ class TokenStream { for (;;) { if (sourceCursor == sourceString.length()) { - hitEOF = true; return EOF_CHAR; } cursor++; @@ -1031,7 +1009,6 @@ class TokenStream { continue; } lineEndChar = -1; - lineStart = sourceCursor - 1; lineno++; } @@ -1078,42 +1055,6 @@ class TokenStream { tokenEnd = cursor; } - /** Return the current position of the scanner cursor. */ - public int getCursor() { - return cursor; - } - - /** Return the absolute source offset of the last scanned token. */ - public int getTokenBeg() { - return tokenBeg; - } - - /** Return the absolute source end-offset of the last scanned token. */ - public int getTokenEnd() { - return tokenEnd; - } - - /** Return tokenEnd - tokenBeg */ - public int getTokenLength() { - return tokenEnd - tokenBeg; - } - - public String getTokenRaw() { - return sourceString.substring(tokenBeg, tokenEnd); - } - - private static String convertLastCharToHex(final String str) { - final int lastIndex = str.length() - 1; - final StringBuilder buf = new StringBuilder(str.substring(0, lastIndex)); - buf.append("\\u"); - final String hexCode = Integer.toHexString(str.charAt(lastIndex)); - for (int i = 0; i < 4 - hexCode.length(); ++i) { - buf.append('0'); - } - buf.append(hexCode); - return buf.toString(); - } - public Token nextToken() throws ParsingException { Token tt = getToken(); while (tt == Token.EOL || tt == Token.COMMENT) { @@ -1124,19 +1065,14 @@ class TokenStream { // stuff other than whitespace since start of line private boolean dirtyLine; - private String string = ""; private char[] stringBuffer = new char[128]; private int stringBufferTop; - private final ObjToIntMap allStrings = new ObjToIntMap(50); // Room to backtrace from to < on failed match of the last - in