Merge pull request #1256 from TeamNewPipe/dependabot/gradle/org.mozilla-rhino-1.8.0

Bump org.mozilla:rhino from 1.7.15 to 1.8.0

extractor/build.gradle

@@ -22,6 +22,10 @@ checkstyleTest {
     enabled false // do not checkstyle test files
 }
 
+ext {
+    rhinoVersion = '1.8.0'
+}
+
 dependencies {
     implementation project(':timeago-parser')
 
@@ -29,7 +33,8 @@ dependencies {
     implementation 'org.jsoup:jsoup:1.18.3'
     implementation "com.google.code.findbugs:jsr305:$jsr305Version"
 
-    implementation 'org.mozilla:rhino:1.7.15'
+    implementation "org.mozilla:rhino:$rhinoVersion"
+    implementation "org.mozilla:rhino-engine:$rhinoVersion"
 
     checkstyle "com.puppycrawl.tools:checkstyle:$checkstyleVersion"
 

extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java

@@ -10,31 +10,25 @@ public final class JavaScript {
     }
 
     public static void compileOrThrow(final String function) {
-        try {
+        try (Context context = Context.enter()) {
-            final Context context = Context.enter();
+            context.setInterpretedMode(true);
-            context.setOptimizationLevel(-1);
 
             // If it doesn't compile it throws an exception here
             context.compileString(function, null, 1, null);
-        } finally {
-            Context.exit();
         }
     }
 
     public static String run(final String function,
                              final String functionName,
                              final String... parameters) {
-        try {
+        try (Context context = Context.enter()) {
-            final Context context = Context.enter();
+            context.setInterpretedMode(true);
-            context.setOptimizationLevel(-1);
             final ScriptableObject scope = context.initSafeStandardObjects();
 
             context.evaluateString(scope, function, functionName, 1, null);
             final Function jsFunction = (Function) scope.get(functionName, scope);
             final Object result = jsFunction.call(context, scope, scope, parameters);
             return result.toString();
-        } finally {
-            Context.exit();
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/utils/jsextractor/EcmaScriptTokenStream.java

@@ -1,18 +1,28 @@
-package org.schabi.newpipe.extractor.utils.jsextractor;
+/*
-
+ * Source: Mozilla Rhino, org.mozilla.javascript.TokenStream
-import org.mozilla.javascript.Context;
-import org.mozilla.javascript.Kit;
-import org.mozilla.javascript.ObjToIntMap;
-import org.mozilla.javascript.ScriptRuntime;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-
-/* Source: Mozilla Rhino, org.mozilla.javascript.Token
  *
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- * */
+ *
-class TokenStream {
+ */
+package org.schabi.newpipe.extractor.utils.jsextractor;
+
+import org.mozilla.javascript.Kit;
+import org.mozilla.javascript.ScriptRuntime;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+/**
+ * Based on Mozilla Rhino's (v1.7.14) org.mozilla.javascript.TokenStream
+ * <p/>
+ * Changes:
+ * <ul>
+ *     <li>Tailored for {@link Lexer}</li>
+ *     <li>Removed all not needed code to improve performance</li>
+ *     <li>Optimized for ECMAScript6/2015</li>
+ * </ul>
+ */
+class EcmaScriptTokenStream {
     /*
      * For chars - because we need something out-of-range
      * to check.  (And checking EOF by exception is annoying.)
@@ -29,127 +39,17 @@ class TokenStream {
     private static final char BYTE_ORDER_MARK = '\uFEFF';
     private static final char NUMERIC_SEPARATOR = '_';
 
-    TokenStream(final String sourceString, final int lineno, final int languageVersion) {
+    EcmaScriptTokenStream(final String sourceString, final int lineno, final boolean strictMode) {
         this.sourceString = sourceString;
         this.sourceCursor = 0;
         this.cursor = 0;
 
         this.lineno = lineno;
-        this.languageVersion = languageVersion;
+        this.strictMode = strictMode;
     }
 
-    static boolean isKeyword(final String s, final int version, final boolean isStrict) {
+    private Token stringToKeyword(final String name) {
-        return Token.EOF != stringToKeyword(s, version, isStrict);
+        return stringToKeywordForES(name, strictMode);
-    }
-
-    private static Token stringToKeyword(final String name, final int version,
-                                       final boolean isStrict) {
-        if (version < Context.VERSION_ES6) {
-            return stringToKeywordForJS(name);
-        }
-        return stringToKeywordForES(name, isStrict);
-    }
-
-    /** JavaScript 1.8 and earlier */
-    private static Token stringToKeywordForJS(final String name) {
-        switch (name) {
-            case "break":
-                return Token.BREAK;
-            case "case":
-                return Token.CASE;
-            case "continue":
-                return Token.CONTINUE;
-            case "default":
-                return Token.DEFAULT;
-            case "delete":
-                return Token.DELPROP;
-            case "do":
-                return Token.DO;
-            case "else":
-                return Token.ELSE;
-            case "export":
-                return Token.EXPORT;
-            case "false":
-                return Token.FALSE;
-            case "for":
-                return Token.FOR;
-            case "function":
-                return Token.FUNCTION;
-            case "if":
-                return Token.IF;
-            case "in":
-                return Token.IN;
-            case "let":
-                return Token.LET;
-            case "new":
-                return Token.NEW;
-            case "null":
-                return Token.NULL;
-            case "return":
-                return Token.RETURN;
-            case "switch":
-                return Token.SWITCH;
-            case "this":
-                return Token.THIS;
-            case "true":
-                return Token.TRUE;
-            case "typeof":
-                return Token.TYPEOF;
-            case "var":
-                return Token.VAR;
-            case "void":
-                return Token.VOID;
-            case "while":
-                return Token.WHILE;
-            case "with":
-                return Token.WITH;
-            case "yield":
-                return Token.YIELD;
-            case "throw":
-                return Token.THROW;
-            case "catch":
-                return Token.CATCH;
-            case "const":
-                return Token.CONST;
-            case "debugger":
-                return Token.DEBUGGER;
-            case "finally":
-                return Token.FINALLY;
-            case "instanceof":
-                return Token.INSTANCEOF;
-            case "try":
-                return Token.TRY;
-            case "abstract":
-            case "boolean":
-            case "byte":
-            case "char":
-            case "class":
-            case "double":
-            case "enum":
-            case "extends":
-            case "final":
-            case "float":
-            case "goto":
-            case "implements":
-            case "import":
-            case "int":
-            case "interface":
-            case "long":
-            case "native":
-            case "package":
-            case "private":
-            case "protected":
-            case "public":
-            case "short":
-            case "static":
-            case "super":
-            case "synchronized":
-            case "throws":
-            case "transient":
-            case "volatile":
-                return Token.RESERVED;
-        }
-        return Token.EOF;
     }
 
     /** ECMAScript 6. */
@@ -343,40 +243,17 @@ class TokenStream {
                 }
                 ungetChar(c);
 
-                String str = getStringFromBuffer();
+                final String str = getStringFromBuffer();
                 if (!containsEscape) {
                     // OPT we shouldn't have to make a string (object!) to
                     // check if it's a keyword.
 
                     // Return the corresponding token if it's a keyword
-                    Token result = stringToKeyword(str, languageVersion, STRICT_MODE);
+                    final Token result = stringToKeyword(str);
                     if (result != Token.EOF) {
-                        if ((result == Token.LET || result == Token.YIELD)
+                        return result; // Always needed due to ECMAScript
-                                && languageVersion < Context.VERSION_1_7) {
-                            // LET and YIELD are tokens only in 1.7 and later
-                            string = result == Token.LET ? "let" : "yield";
-                            result = Token.NAME;
-                        }
-                        // Save the string in case we need to use in
-                        // object literal definitions.
-                        this.string = (String) allStrings.intern(str);
-                        if (result != Token.RESERVED) {
-                            return result;
-                        } else if (languageVersion >= Context.VERSION_ES6) {
-                            return result;
-                        } else if (!IS_RESERVED_KEYWORD_AS_IDENTIFIER) {
-                            return result;
-                        }
                     }
-                } else if (isKeyword(
-                        str,
-                        languageVersion,
-                        STRICT_MODE)) {
-                    // If a string contains unicodes, and converted to a keyword,
-                    // we convert the last character back to unicode
-                    str = convertLastCharToHex(str);
                 }
-                this.string = (String) allStrings.intern(str);
                 return Token.NAME;
             }
 
@@ -384,7 +261,6 @@ class TokenStream {
             if (isDigit(c) || (c == '.' && isDigit(peekChar()))) {
                 stringBufferTop = 0;
                 int base = 10;
-                final boolean es6 = languageVersion >= Context.VERSION_ES6;
                 boolean isOldOctal = false;
 
                 if (c == '0') {
@@ -392,10 +268,10 @@ class TokenStream {
                     if (c == 'x' || c == 'X') {
                         base = 16;
                         c = getChar();
-                    } else if (es6 && (c == 'o' || c == 'O')) {
+                    } else if (c == 'o' || c == 'O') {
                         base = 8;
                         c = getChar();
-                    } else if (es6 && (c == 'b' || c == 'B')) {
+                    } else if (c == 'b' || c == 'B') {
                         base = 2;
                         c = getChar();
                     } else if (isDigit(c)) {
@@ -438,7 +314,7 @@ class TokenStream {
                     throw new ParsingException("number format error");
                 }
 
-                if (es6 && c == 'n') {
+                if (c == 'n') {
                     c = getChar();
                 } else if (base == 10 && (c == '.' || c == 'e' || c == 'E')) {
                     if (c == '.') {
@@ -466,7 +342,7 @@ class TokenStream {
                     }
                 }
                 ungetChar(c);
-                this.string = getStringFromBuffer();
+                tokenEnd = cursor;
                 return Token.NUMBER;
             }
 
@@ -562,7 +438,7 @@ class TokenStream {
                                 escapeVal = Kit.xDigitToInt(c, 0);
                                 if (escapeVal < 0) {
                                     addToString('x');
-                                    continue strLoop;
+                                    continue;
                                 }
                                 final int c1 = c;
                                 c = getChar();
@@ -570,7 +446,7 @@ class TokenStream {
                                 if (escapeVal < 0) {
                                     addToString('x');
                                     addToString(c1);
-                                    continue strLoop;
+                                    continue;
                                 }
                                 // got 2 hex digits
                                 c = escapeVal;
@@ -580,7 +456,7 @@ class TokenStream {
                                 // Remove line terminator after escape to follow
                                 // SpiderMonkey and C/C++
                                 c = getChar();
-                                continue strLoop;
+                                continue;
 
                             default:
                                 if ('0' <= c && c < '8') {
@@ -605,8 +481,7 @@ class TokenStream {
                     c = getChar(false);
                 }
 
-                final String str = getStringFromBuffer();
+                tokenEnd = cursor;
-                this.string = (String) allStrings.intern(str);
                 return quoteChar == '`' ? Token.TEMPLATE_LITERAL : Token.STRING;
             }
 
@@ -722,14 +597,13 @@ class TokenStream {
                     return Token.GT;
 
                 case '*':
-                    if (languageVersion >= Context.VERSION_ES6) {
+                    if (matchChar('*')) {
-                        if (matchChar('*')) {
+                        if (matchChar('=')) {
-                            if (matchChar('=')) {
+                            return Token.ASSIGN_EXP;
-                                return Token.ASSIGN_EXP;
-                            }
-                            return Token.EXP;
                         }
+                        return Token.EXP;
                     }
+
                     if (matchChar('=')) {
                         return Token.ASSIGN_MUL;
                     }
@@ -920,7 +794,6 @@ class TokenStream {
             }
             if (peekChar() == '*') {
                 tokenEnd = cursor - 1;
-                this.string = new String(stringBuffer, 0, stringBufferTop);
                 throw new ParsingException("msg.unterminated.re.lit");
             }
         }
@@ -944,7 +817,6 @@ class TokenStream {
             }
             addToString(c);
         }
-        final int reEnd = stringBufferTop;
 
         while (true) {
             c = getCharIgnoreLineEnd();
@@ -959,7 +831,6 @@ class TokenStream {
         }
 
         tokenEnd = start + stringBufferTop + 2; // include slashes
-        this.string = new String(stringBuffer, 0, reEnd);
     }
 
     private String getStringFromBuffer() {
@@ -1019,7 +890,6 @@ class TokenStream {
 
         for (;;) {
             if (sourceCursor == sourceString.length()) {
-                hitEOF = true;
                 return EOF_CHAR;
             }
             cursor++;
@@ -1031,7 +901,6 @@ class TokenStream {
                     continue;
                 }
                 lineEndChar = -1;
-                lineStart = sourceCursor - 1;
                 lineno++;
             }
 
@@ -1078,42 +947,6 @@ class TokenStream {
         tokenEnd = cursor;
     }
 
-    /** Return the current position of the scanner cursor. */
-    public int getCursor() {
-        return cursor;
-    }
-
-    /** Return the absolute source offset of the last scanned token. */
-    public int getTokenBeg() {
-        return tokenBeg;
-    }
-
-    /** Return the absolute source end-offset of the last scanned token. */
-    public int getTokenEnd() {
-        return tokenEnd;
-    }
-
-    /** Return tokenEnd - tokenBeg */
-    public int getTokenLength() {
-        return tokenEnd - tokenBeg;
-    }
-
-    public String getTokenRaw() {
-        return sourceString.substring(tokenBeg, tokenEnd);
-    }
-
-    private static String convertLastCharToHex(final String str) {
-        final int lastIndex = str.length() - 1;
-        final StringBuilder buf = new StringBuilder(str.substring(0, lastIndex));
-        buf.append("\\u");
-        final String hexCode = Integer.toHexString(str.charAt(lastIndex));
-        for (int i = 0; i < 4 - hexCode.length(); ++i) {
-            buf.append('0');
-        }
-        buf.append(hexCode);
-        return buf.toString();
-    }
-
     public Token nextToken() throws ParsingException {
         Token tt = getToken();
         while (tt == Token.EOL || tt == Token.COMMENT) {
@@ -1124,19 +957,14 @@ class TokenStream {
 
     // stuff other than whitespace since start of line
     private boolean dirtyLine;
-    private String string = "";
 
     private char[] stringBuffer = new char[128];
     private int stringBufferTop;
-    private final ObjToIntMap allStrings = new ObjToIntMap(50);
 
     // Room to backtrace from to < on failed match of the last - in <!--
     private final int[] ungetBuffer = new int[3];
     private int ungetCursor;
 
-    private boolean hitEOF = false;
-
-    private int lineStart = 0;
     private int lineEndChar = -1;
     int lineno;
 
@@ -1144,18 +972,16 @@ class TokenStream {
 
     // sourceCursor is an index into a small buffer that keeps a
     // sliding window of the source stream.
-    int sourceCursor;
+    private int sourceCursor;
 
     // cursor is a monotonically increasing index into the original
     // source stream, tracking exactly how far scanning has progressed.
     // Its value is the index of the next character to be scanned.
-    int cursor;
+    private int cursor;
 
     // Record start and end positions of last scanned token.
     int tokenBeg;
     int tokenEnd;
 
-    private final int languageVersion;
+    private final boolean strictMode;
-    private static final boolean IS_RESERVED_KEYWORD_AS_IDENTIFIER = true;
-    private static final boolean STRICT_MODE = false;
 }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/jsextractor/Lexer.java

@@ -1,6 +1,5 @@
 package org.schabi.newpipe.extractor.utils.jsextractor;
 
-import org.mozilla.javascript.Context;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
 import java.util.Stack;
@@ -119,7 +118,7 @@ public class Lexer {
         }
     }
 
-    private final TokenStream stream;
+    private final EcmaScriptTokenStream stream;
     private final LookBehind lastThree;
     private final Stack<Brace> braceStack;
     private final Stack<Paren> parenStack;
@@ -128,24 +127,14 @@ public class Lexer {
      * Create a new JavaScript lexer with the given source code
      *
      * @param js JavaScript code
-     * @param languageVersion JavaScript version (from Rhino)
      */
-    public Lexer(final String js, final int languageVersion) {
+    public Lexer(final String js) {
-        stream = new TokenStream(js, 0, languageVersion);
+        stream = new EcmaScriptTokenStream(js, 0, false);
         lastThree = new LookBehind();
         braceStack = new Stack<>();
         parenStack = new Stack<>();
     }
 
-    /**
-     * Create a new JavaScript lexer with the given source code
-     *
-     * @param js JavaScript code
-     */
-    public Lexer(final String js) {
-        this(js, Context.VERSION_DEFAULT);
-    }
-
     /**
      * Continue parsing and return the next token
      * @return next token
@@ -256,7 +245,7 @@ public class Lexer {
      */
     void handleCloseParenBooks(final int start) throws ParsingException {
         if (parenStack.isEmpty()) {
-            throw new ParsingException("unmached closing paren at " + start);
+            throw new ParsingException("unmatched closing paren at " + start);
         }
         lastThree.push(new ParenMetaToken(Token.RP, stream.lineno, parenStack.pop()));
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterDeobfuscationTest.java

@@ -10,14 +10,12 @@ import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
-import java.io.IOException;
-
 class YoutubeThrottlingParameterDeobfuscationTest {
     private static final String RESOURCE_PATH =
             DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/parameterDeobf";
 
     @BeforeEach
-    void setup() throws IOException {
+    void setup() {
         YoutubeTestsUtils.ensureStateless();
         NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH));
     }

extractor/src/test/java/org/schabi/newpipe/extractor/utils/JavaScriptExtractorTest.java

@@ -1,5 +1,9 @@
 package org.schabi.newpipe.extractor.utils;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.FileUtils.resolveTestResource;
+
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.jsextractor.JavaScriptExtractor;
@@ -10,11 +14,7 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
+class JavaScriptExtractorTest
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.schabi.newpipe.FileUtils.resolveTestResource;
-
-public class JavaScriptExtractorTest
 {
     @Test
     void testJsExtractor() throws ParsingException {