Merge pull request #780 from TiA4f8R/yt-more-params-innertube-requests

[YouTube] Add more parameters to InnerTube requests, use the iOS client for livestreams and fix extraction of embeddable age-restricted videos and contents with a warning before playback

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -1,3 +1,23 @@
+/*
+ * Created by Christian Schabesberger on 02.03.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * YoutubeParsingHelper.java is part of NewPipe Extractor.
+ *
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe Extractor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package org.schabi.newpipe.extractor.services.youtube;
 
 import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
@@ -5,6 +25,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
 import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
+import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 import com.grack.nanojson.JsonArray;
@@ -15,7 +36,6 @@ import com.grack.nanojson.JsonParserException;
 import com.grack.nanojson.JsonWriter;
 
 import org.schabi.newpipe.extractor.MetaInfo;
-import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@@ -28,6 +48,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
+import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
@@ -35,11 +56,14 @@ import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.security.SecureRandom;
 import java.time.LocalDate;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeParseException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -51,58 +75,171 @@ import java.util.Random;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
-/*
- * Created by Christian Schabesberger on 02.03.16.
- *
- * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
- * YoutubeParsingHelper.java is part of NewPipe.
- *
- * NewPipe is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * NewPipe is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 public final class YoutubeParsingHelper {
 
     private YoutubeParsingHelper() {
     }
 
+    /**
+     * The base URL of requests of the {@code WEB} clients to the InnerTube internal API.
+     */
     public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
 
-    private static final String HARDCODED_CLIENT_VERSION = "2.20210728.00.00";
+    /**
+     * The base URL of requests of non-web clients to the InnerTube internal API.
+     */
+    public static final String YOUTUBEI_V1_GAPIS_URL =
+            "https://youtubei.googleapis.com/youtubei/v1/";
+
+    /**
+     * A parameter to disable pretty-printed response of InnerTube requests, to reduce response
+     * sizes.
+     *
+     * <p>
+     * Sent in query parameters of the requests, <b>after</b> the API key.
+     * </p>
+     **/
+    public static final String DISABLE_PRETTY_PRINT_PARAMETER = "&prettyPrint=false";
+
+    /**
+     * A parameter sent by official clients named {@code contentPlaybackNonce}.
+     *
+     * <p>
+     * It is sent by official clients on videoplayback requests, and by all clients (except the
+     * {@code WEB} one to the player requests.
+     * </p>
+     *
+     * <p>
+     * It is composed of 16 characters which are generated from
+     * {@link #CONTENT_PLAYBACK_NONCE_ALPHABET this alphabet}, with the use of strong random
+     * values.
+     * </p>
+     *
+     * @see #generateContentPlaybackNonce()
+     */
+    public static final String CPN = "cpn";
+    public static final String VIDEO_ID = "videoId";
+
+    /**
+     * A parameter sent by official clients named {@code contentCheckOk}.
+     *
+     * <p>
+     * Setting it to {@code true} allows us to get streaming data on videos with a warning about
+     * what the sensible content they contain.
+     * </p>
+     */
+    public static final String CONTENT_CHECK_OK = "contentCheckOk";
+
+    /**
+     * A parameter which may be send by official clients named {@code racyCheckOk}.
+     *
+     * <p>
+     * What this parameter does is not really known, but it seems to be linked to sensitive
+     * contents such as age-restricted content.
+     * </p>
+     */
+    public static final String RACY_CHECK_OK = "racyCheckOk";
+
+    /**
+     * The client version for InnerTube requests with the {@code WEB} client, used as the last
+     * fallback if the extraction of the real one failed.
+     *
+     * You can get it directly either into YouTube pages or the service worker JavaScript file
+     * ({@code https://www.youtube.com/sw.js}) (also applies for YouTube Music).
+     */
+    private static final String HARDCODED_CLIENT_VERSION = "2.20220315.01.00";
     private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
-    private static final String MOBILE_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
+
-    private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.29.38";
+    /**
+     * The InnerTube API key used by the {@code ANDROID} client. Found with the help of
+     * reverse-engineering app network requests.
+     */
+    private static final String ANDROID_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
+
+    /**
+     * The InnerTube API key used by the {@code iOS} client. Found with the help of
+     * reverse-engineering app network requests.
+     */
+    private static final String IOS_YOUTUBE_KEY = "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc";
+
+    /**
+     * The hardcoded client version of the Android app used for InnerTube requests with this
+     * client.
+     *
+     * <p>
+     * It can be extracted by getting the latest release version of the app in an APK repository
+     * such as APKMirror.
+     * </p>
+     *
+     * @implNote This version is also used for the {@code iOS} client, as getting the app version
+     * without an iPhone device is not so easily.
+     */
+    private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "17.10.35";
+
+    /**
+     * The hardcoded client version of the Android app used for InnerTube requests with this
+     * client.
+     */
+    private static final String TVHTML5_SIMPLY_EMBED_CLIENT_VERSION = "2.0";
+
     private static String clientVersion;
     private static String key;
 
     private static final String[] HARDCODED_YOUTUBE_MUSIC_KEY =
-            {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20210726.00.01"};
+            {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20220309.01.00"};
     private static String[] youtubeMusicKey;
 
     private static boolean keyAndVersionExtracted = false;
     @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
     private static Optional<Boolean> hardcodedClientVersionAndKeyValid = Optional.empty();
 
-    private static Random numberGenerator = new Random();
+    private static final String[] INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES =
+            {"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
+                    "innertube_context_client_version\":\"([0-9\\.]+?)\"",
+                    "client.version=([0-9\\.]+)"};
+    private static final String[] INNERTUBE_API_KEY_REGEXES =
+            {"INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"",
+                    "innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\""};
+    private static final String[] INITIAL_DATA_REGEXES =
+            {"window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});",
+                    "var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});"};
+    private static final String INNERTUBE_CLIENT_NAME_REGEX =
+            "INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),";
+
+    private static final String CONTENT_PLAYBACK_NONCE_ALPHABET =
+            "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
 
     /**
-     * <code>PENDING+</code> means that the user did not yet submit their choices.
+     * The device machine id for the iPhone 13, used to get 60fps with the {@code iOS} client.
+     *
+     * <p>
+     * See <a href="https://gist.github.com/adamawolf/3048717">this GitHub Gist</a> for more
+     * information.
+     * </p>
+     */
+    private static final String IOS_DEVICE_MODEL = "iPhone14,5";
+
+    private static Random numberGenerator = new SecureRandom();
+
+    /**
+     * {@code PENDING+} means that the user did not yet submit their choices.
+     *
+     * <p>
      * Therefore, YouTube & Google should not track the user, because they did not give consent.
+     * </p>
+     *
+     * <p>
      * The three digits at the end can be random, but are required.
+     * </p>
      */
     private static final String CONSENT_COOKIE_VALUE = "PENDING+";
+
     /**
-     * Youtube <code>CONSENT</code> cookie. Should prevent redirect to consent.youtube.com
+     * YouTube {@code CONSENT} cookie.
+     *
+     * <p>
+     * Should prevent redirect to {@code consent.youtube.com}.
+     * </p>
      */
     private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
 
@@ -419,17 +556,10 @@ public final class YoutubeParsingHelper {
         }
     }
 
-    public static JsonObject getInitialData(final String html) throws ParsingException {
+    private static JsonObject getInitialData(final String html) throws ParsingException {
         try {
-            try {
+            return JsonParser.object().from(getStringResultFromRegexArray(html,
-                final String initialData = Parser.matchGroup1(
+                    INITIAL_DATA_REGEXES, 1));
-                        "window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
-                return JsonParser.object().from(initialData);
-            } catch (final Parser.RegexException e) {
-                final String initialData = Parser.matchGroup1(
-                        "var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html);
-                return JsonParser.object().from(initialData);
-            }
         } catch (final JsonParserException | Parser.RegexException e) {
             throw new ParsingException("Could not get ytInitialData", e);
         }
@@ -466,7 +596,7 @@ public final class YoutubeParsingHelper {
         // This endpoint is fetched by the YouTube website to get the items of its main menu and is
         // pretty lightweight (around 30kB)
         final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
-                        + HARDCODED_KEY, headers, body);
+                        + HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, headers, body);
         final String responseBody = response.responseBody();
         final int responseCode = response.responseCode();
 
@@ -475,12 +605,34 @@ public final class YoutubeParsingHelper {
         return hardcodedClientVersionAndKeyValid.get();
     }
 
-    private static void extractClientVersionAndKey() throws IOException, ExtractionException {
+
+    private static void extractClientVersionAndKeyFromSwJs()
+            throws IOException, ExtractionException {
+        if (keyAndVersionExtracted) {
+            return;
+        }
+        final String url = "https://www.youtube.com/sw.js";
+        final Map<String, List<String>> headers = new HashMap<>();
+        headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
+        headers.put("Referer", Collections.singletonList("https://www.youtube.com"));
+        final String response = getDownloader().get(url, headers).responseBody();
+        try {
+            clientVersion = getStringResultFromRegexArray(response,
+                    INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
+            key = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
+        } catch (final Parser.RegexException e) {
+            throw new ParsingException("Could not extract YouTube WEB InnerTube client version "
+                    + "and API key from sw.js", e);
+        }
+        keyAndVersionExtracted = true;
+    }
+
+    private static void extractClientVersionAndKeyFromHtmlSearchResultsPage()
+            throws IOException, ExtractionException {
         // Don't extract the client version and the InnerTube key if it has been already extracted
         if (keyAndVersionExtracted) {
             return;
         }
-
         // Don't provide a search term in order to have a smaller response
         final String url = "https://www.youtube.com/results?search_query=&ucbcb=1";
         final Map<String, List<String>> headers = new HashMap<>();
@@ -517,74 +669,90 @@ public final class YoutubeParsingHelper {
             }
         }
 
-        String contextClientVersion;
-        final String[] patterns = {
-                "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
-                "innertube_context_client_version\":\"([0-9\\.]+?)\"",
-                "client.version=([0-9\\.]+)"
-        };
-        for (final String pattern : patterns) {
         try {
-                contextClientVersion = Parser.matchGroup1(pattern, html);
+            clientVersion = getStringResultFromRegexArray(html,
-                if (!isNullOrEmpty(contextClientVersion)) {
+                    INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
-                    clientVersion = contextClientVersion;
-                    break;
-                }
         } catch (final Parser.RegexException ignored) {
         }
-        }
 
         if (!isNullOrEmpty(clientVersion) && !isNullOrEmpty(shortClientVersion)) {
             clientVersion = shortClientVersion;
         }
 
         try {
-            key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
+            key = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1);
-        } catch (final Parser.RegexException e1) {
+        } catch (final Parser.RegexException e) {
-            try {
+            throw new ParsingException("Could not extract YouTube WEB InnerTube client version "
-                key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
+                    + "and API key from HTML search results page", e);
-            } catch (final Parser.RegexException e2) {
-                throw new ParsingException("Could not extract client version and key");
-            }
         }
         keyAndVersionExtracted = true;
     }
 
     /**
-     * Get the client version
+     * Get the client version used by YouTube website on InnerTube requests.
      */
     public static String getClientVersion() throws IOException, ExtractionException {
         if (!isNullOrEmpty(clientVersion)) {
             return clientVersion;
         }
+
+        // Always extract latest client version, by trying first to extract it from the JavaScript
+        // service worker, then from HTML search results page as a fallback, to prevent
+        // fingerprinting based on the client version used
+        try {
+            extractClientVersionAndKeyFromSwJs();
+        } catch (final Exception e) {
+            extractClientVersionAndKeyFromHtmlSearchResultsPage();
+        }
+
+        if (keyAndVersionExtracted) {
+            return clientVersion;
+        }
+
+        // Fallback to the hardcoded one if it's valid
         if (areHardcodedClientVersionAndKeyValid()) {
             clientVersion = HARDCODED_CLIENT_VERSION;
             return clientVersion;
         }
 
-        extractClientVersionAndKey();
+        throw new ExtractionException("Could not get YouTube WEB client version");
-        return clientVersion;
     }
 
     /**
-     * Get the key
+     * Get the internal API key used by YouTube website on InnerTube requests.
      */
     public static String getKey() throws IOException, ExtractionException {
         if (!isNullOrEmpty(key)) {
             return key;
         }
+
+        // Always extract the key used by the webiste, by trying first to extract it from the
+        // JavaScript service worker, then from HTML search results page as a fallback, to prevent
+        // fingerprinting based on the key and/or invalid key issues
+        try {
+            extractClientVersionAndKeyFromSwJs();
+        } catch (final Exception e) {
+            extractClientVersionAndKeyFromHtmlSearchResultsPage();
+        }
+
+        if (keyAndVersionExtracted) {
+            return key;
+        }
+
+        // Fallback to the hardcoded one if it's valid
         if (areHardcodedClientVersionAndKeyValid()) {
             key = HARDCODED_KEY;
             return key;
         }
 
-        extractClientVersionAndKey();
+        // The ANDROID API key is also valid with the WEB client so return it if we couldn't
-        return key;
+        // extract the WEB API key.
+        return ANDROID_YOUTUBE_KEY;
     }
 
     /**
      * <p>
-     * <b>Only use in tests.</b>
+     * <b>Only used in tests.</b>
      * </p>
      *
      * <p>
@@ -600,11 +768,12 @@ public final class YoutubeParsingHelper {
     public static void resetClientVersionAndKey() {
         clientVersion = null;
         key = null;
+        keyAndVersionExtracted = false;
     }
 
     /**
      * <p>
-     * <b>Only use in tests.</b>
+     * <b>Only used in tests.</b>
      * </p>
      */
     public static void setNumberGenerator(final Random random) {
@@ -615,7 +784,7 @@ public final class YoutubeParsingHelper {
             ReCaptchaException {
         final String url =
                 "https://music.youtube.com/youtubei/v1/music/get_search_suggestions?alt=json&key="
-                + HARDCODED_YOUTUBE_MUSIC_KEY[0];
+                        + HARDCODED_YOUTUBE_MUSIC_KEY[0] + DISABLE_PRETTY_PRINT_PARAMETER;
 
         // @formatter:off
         final byte[] json = JsonWriter.string()
@@ -659,8 +828,8 @@ public final class YoutubeParsingHelper {
         return response.responseBody().length() > 500 && response.responseCode() == 200;
     }
 
-    public static String[] getYoutubeMusicKey() throws IOException, ReCaptchaException,
+    public static String[] getYoutubeMusicKey()
-            Parser.RegexException {
+            throws IOException, ReCaptchaException, Parser.RegexException {
         if (youtubeMusicKey != null && youtubeMusicKey.length == 3) {
             return youtubeMusicKey;
         }
@@ -669,40 +838,33 @@ public final class YoutubeParsingHelper {
             return youtubeMusicKey;
         }
 
+        String musicClientVersion;
+        String musicKey;
+        String musicClientName;
+
+        try {
+            final String url = "https://music.youtube.com/sw.js";
+            final Map<String, List<String>> headers = new HashMap<>();
+            headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
+            headers.put("Referer", Collections.singletonList("https://music.youtube.com"));
+            final String response = getDownloader().get(url, headers).responseBody();
+                musicClientVersion = getStringResultFromRegexArray(response,
+                        INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
+                musicKey = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
+                musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, response);
+        } catch (final Exception e) {
             final String url = "https://music.youtube.com/";
             final Map<String, List<String>> headers = new HashMap<>();
             addCookieHeader(headers);
             final String html = getDownloader().get(url, headers).responseBody();
 
-        String innertubeApiKey;
+            musicKey = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1);
-        try {
+            musicClientVersion = getStringResultFromRegexArray(html,
-            innertubeApiKey = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
+                    INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES);
-        } catch (final Parser.RegexException e) {
+            musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, html);
-            innertubeApiKey = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html);
         }
 
-        final String innertubeClientName
+        youtubeMusicKey = new String[] {musicKey, musicClientName, musicClientVersion};
-                = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", html);
-
-        String innertubeClientVersion;
-        try {
-            innertubeClientVersion = Parser.matchGroup1(
-                    "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
-        } catch (final Parser.RegexException e) {
-            try {
-                innertubeClientVersion = Parser.matchGroup1(
-                        "INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
-            } catch (final Parser.RegexException ee) {
-                innertubeClientVersion = Parser.matchGroup1(
-                        "innertube_context_client_version\":\"([0-9\\.]+?)\"", html);
-            }
-        }
-
-        youtubeMusicKey = new String[]{
-                innertubeApiKey,
-                innertubeClientName,
-                innertubeClientVersion
-        };
         return youtubeMusicKey;
     }
 
@@ -754,7 +916,7 @@ public final class YoutubeParsingHelper {
         } else if (navigationEndpoint.has("watchEndpoint")) {
             final StringBuilder url = new StringBuilder();
             url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint
-                    .getObject("watchEndpoint").getString("videoId"));
+                    .getObject("watchEndpoint").getString(VIDEO_ID));
             if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
                 url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint")
                         .getString("playlistId"));
@@ -772,10 +934,11 @@ public final class YoutubeParsingHelper {
     }
 
     /**
-     * Get the text from a JSON object that has either a simpleText or a runs array.
+     * Get the text from a JSON object that has either a {@code simpleText} or a {@code runs}
+     * array.
      *
      * @param textObject JSON object to get the text from
-     * @param html       whether to return HTML, by parsing the navigationEndpoint
+     * @param html       whether to return HTML, by parsing the {@code navigationEndpoint}
      * @return text in the JSON object or {@code null}
      */
     @Nullable
@@ -891,17 +1054,6 @@ public final class YoutubeParsingHelper {
         return responseBody;
     }
 
-    public static Response getResponse(final String url, final Localization localization)
-            throws IOException, ExtractionException {
-        final Map<String, List<String>> headers = new HashMap<>();
-        addYouTubeHeaders(headers);
-
-        final Response response = getDownloader().get(url, headers, localization);
-        getValidJsonResponseBody(response);
-
-        return response;
-    }
-
     public static JsonObject getJsonPostResponse(final String endpoint,
                                                  final byte[] body,
                                                  final Localization localization)
@@ -911,53 +1063,50 @@ public final class YoutubeParsingHelper {
         headers.put("Content-Type", Collections.singletonList("application/json"));
 
         final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
-                + getKey(), headers, body, localization);
+                + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
 
         return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
     }
 
-    public static JsonObject getJsonMobilePostResponse(final String endpoint,
+    public static JsonObject getJsonAndroidPostResponse(
+            final String endpoint,
             final byte[] body,
-                                                       @Nonnull final ContentCountry
+            @Nonnull final Localization localization,
-                                                               contentCountry,
+            @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
-                                                       final Localization localization)
+        return getMobilePostResponse(endpoint, body, localization,
-            throws IOException, ExtractionException {
+                getAndroidUserAgent(localization), ANDROID_YOUTUBE_KEY, endPartOfUrlRequest);
+    }
+
+    public static JsonObject getJsonIosPostResponse(
+            final String endpoint,
+            final byte[] body,
+            @Nonnull final Localization localization,
+            @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
+        return getMobilePostResponse(endpoint, body, localization, getIosUserAgent(localization),
+                IOS_YOUTUBE_KEY, endPartOfUrlRequest);
+    }
+
+    private static JsonObject getMobilePostResponse(
+            final String endpoint,
+            final byte[] body,
+            @Nonnull final Localization localization,
+            @Nonnull final String userAgent,
+            @Nonnull final String innerTubeApiKey,
+            @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
         final Map<String, List<String>> headers = new HashMap<>();
         headers.put("Content-Type", Collections.singletonList("application/json"));
-        // Spoofing an Android 11 device with the hardcoded version of the Android app
+        headers.put("User-Agent", Collections.singletonList(userAgent));
-        headers.put("User-Agent", Collections.singletonList("com.google.android.youtube/"
+        headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
-                + MOBILE_YOUTUBE_CLIENT_VERSION + "Linux; U; Android 11; "
-                + contentCountry.getCountryCode() + ") gzip"));
-        headers.put("x-goog-api-format-version", Collections.singletonList("2"));
 
-        final Response response = getDownloader().post(
+        final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey
-                "https://youtubei.googleapis.com/youtubei/v1/" + endpoint + "?key="
+                + DISABLE_PRETTY_PRINT_PARAMETER;
-                        + MOBILE_YOUTUBE_KEY, headers, body, localization);
 
+        final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
+                        ? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
+                headers, body, localization);
         return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
     }
 
-    public static JsonArray getJsonResponse(final String url, final Localization localization)
-            throws IOException, ExtractionException {
-        final Map<String, List<String>> headers = new HashMap<>();
-        addYouTubeHeaders(headers);
-
-        final Response response = getDownloader().get(url, headers, localization);
-
-        return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
-    }
-
-    public static JsonArray getJsonResponse(@Nonnull final Page page,
-                                            final Localization localization)
-            throws IOException, ExtractionException {
-        final Map<String, List<String>> headers = new HashMap<>();
-        addYouTubeHeaders(headers);
-
-        final Response response = getDownloader().get(page.getUrl(), headers, localization);
-
-        return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
-    }
-
     @Nonnull
     public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
             @Nonnull final Localization localization,
@@ -971,6 +1120,13 @@ public final class YoutubeParsingHelper {
                         .value("gl", contentCountry.getCountryCode())
                         .value("clientName", "WEB")
                         .value("clientVersion", getClientVersion())
+                        .value("originalUrl", "https://www.youtube.com")
+                        .value("platform", "DESKTOP")
+                    .end()
+                    .object("request")
+                        .array("internalExperimentFlags")
+                        .end()
+                        .value("useSsl", true)
                     .end()
                     .object("user")
                         // TO DO: provide a way to enable restricted mode with:
@@ -991,6 +1147,7 @@ public final class YoutubeParsingHelper {
                     .object("client")
                         .value("clientName", "ANDROID")
                         .value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION)
+                        .value("platform", "MOBILE")
                         .value("hl", localization.getLocalizationCode())
                         .value("gl", contentCountry.getCountryCode())
                     .end()
@@ -1004,35 +1161,32 @@ public final class YoutubeParsingHelper {
     }
 
     @Nonnull
-    public static JsonBuilder<JsonObject> prepareDesktopEmbedVideoJsonBuilder(
+    public static JsonBuilder<JsonObject> prepareIosMobileJsonBuilder(
             @Nonnull final Localization localization,
-            @Nonnull final ContentCountry contentCountry,
+            @Nonnull final ContentCountry contentCountry) {
-            @Nonnull final String videoId) throws IOException, ExtractionException {
         // @formatter:off
         return JsonObject.builder()
                 .object("context")
                     .object("client")
+                        .value("clientName", "IOS")
+                        .value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION)
+                        // Device model is required to get 60fps streams
+                        .value("deviceModel", IOS_DEVICE_MODEL)
+                        .value("platform", "MOBILE")
                         .value("hl", localization.getLocalizationCode())
                         .value("gl", contentCountry.getCountryCode())
-                        .value("clientName", "WEB")
-                        .value("clientVersion", getClientVersion())
-                        .value("clientScreen", "EMBED")
-                    .end()
-                    .object("thirdParty")
-                        .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
                     .end()
                     .object("user")
                         // TO DO: provide a way to enable restricted mode with:
                         // .value("enableSafetyMode", boolean)
                         .value("lockedSafetyMode", false)
                     .end()
-                .end()
+                .end();
-                .value("videoId", videoId);
         // @formatter:on
     }
 
     @Nonnull
-    public static JsonBuilder<JsonObject> prepareAndroidMobileEmbedVideoJsonBuilder(
+    public static JsonBuilder<JsonObject> prepareTvHtml5EmbedJsonBuilder(
             @Nonnull final Localization localization,
             @Nonnull final ContentCountry contentCountry,
             @Nonnull final String videoId) {
@@ -1040,9 +1194,10 @@ public final class YoutubeParsingHelper {
         return JsonObject.builder()
                 .object("context")
                     .object("client")
-                        .value("clientName", "ANDROID")
+                        .value("clientName", "TVHTML5_SIMPLY_EMBEDDED_PLAYER")
-                        .value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION)
+                        .value("clientVersion", TVHTML5_SIMPLY_EMBED_CLIENT_VERSION)
                         .value("clientScreen", "EMBED")
+                        .value("platform", "TV")
                         .value("hl", localization.getLocalizationCode())
                         .value("gl", contentCountry.getCountryCode())
                     .end()
@@ -1054,43 +1209,80 @@ public final class YoutubeParsingHelper {
                         // .value("enableSafetyMode", boolean)
                         .value("lockedSafetyMode", false)
                     .end()
-                .end()
+                .end();
-                .value("videoId", videoId);
         // @formatter:on
     }
 
     @Nonnull
-    public static byte[] createPlayerBodyWithSts(final Localization localization,
+    public static byte[] createDesktopPlayerBody(
-                                                 final ContentCountry contentCountry,
+            @Nonnull final Localization localization,
-                                                 final String videoId,
+            @Nonnull final ContentCountry contentCountry,
-                                                 final boolean withThirdParty,
+            @Nonnull final String videoId,
-                                                 @Nullable final String sts)
+            @Nonnull final String sts,
-            throws IOException, ExtractionException {
+            final boolean isTvHtml5DesktopJsonBuilder,
-        if (withThirdParty) {
+            @Nonnull final String contentPlaybackNonce) throws IOException, ExtractionException {
         // @formatter:off
-            return JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(
+        return JsonWriter.string((isTvHtml5DesktopJsonBuilder
-                    localization, contentCountry, videoId)
+                        ? prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId)
+                        : prepareDesktopJsonBuilder(localization, contentCountry))
                 .object("playbackContext")
                     .object("contentPlaybackContext")
+                        // Some parameters which are sent by the official WEB client in player
+                        // requests, which seems to avoid throttling on streams from it
                         .value("signatureTimestamp", sts)
+                        .value("referer", "https://www.youtube.com/watch?v=" + videoId)
                     .end()
                 .end()
+                .value(CPN, contentPlaybackNonce)
+                .value(VIDEO_ID, videoId)
+                .value(CONTENT_CHECK_OK, true)
+                .value(RACY_CHECK_OK, true)
                 .done())
-                    .getBytes(UTF_8);
+                .getBytes(StandardCharsets.UTF_8);
-            // @formatter:on
-        } else {
-            // @formatter:off
-            return JsonWriter.string(prepareDesktopJsonBuilder(localization, contentCountry)
-                    .value("videoId", videoId)
-                    .object("playbackContext")
-                        .object("contentPlaybackContext")
-                            .value("signatureTimestamp", sts)
-                        .end()
-                    .end()
-                    .done())
-                    .getBytes(UTF_8);
         // @formatter:on
     }
+
+    /**
+     * Get the user-agent string used as the user-agent for InnerTube requests with the Android
+     * client.
+     *
+     * If the {@link Localization} provided is {@code null}, fallbacks to
+     * {@link Localization#DEFAULT the default one}.
+     *
+     * @param localization the {@link Localization} to set in the user-agent
+     * @return the Android user-agent used for InnerTube requests with the Android client,
+     * depending on the {@link Localization} provided
+     */
+    @Nonnull
+    public static String getAndroidUserAgent(@Nullable final Localization localization) {
+        // Spoofing an Android 12 device with the hardcoded version of the Android app
+        return "com.google.android.youtube/" + MOBILE_YOUTUBE_CLIENT_VERSION
+                + " (Linux; U; Android 12; "
+                + (localization != null ? localization.getCountryCode()
+                        : Localization.DEFAULT.getCountryCode())
+                + ") gzip";
+    }
+
+    /**
+     * Get the user-agent string used as the user-agent for InnerTube requests with the iOS
+     * client.
+     *
+     * If the {@link Localization} provided is {@code null}, fallbacks to
+     * {@link Localization#DEFAULT the default one}.
+     *
+     * @param localization the {@link Localization} to set in the user-agent
+     * @return the iOS user-agent used for InnerTube requests with the iOS client, depending on the
+     * {@link Localization} provided
+     */
+    @Nonnull
+    public static String getIosUserAgent(@Nullable final Localization localization) {
+        // Spoofing an iPhone running iOS 15.4 with the hardcoded mobile client version
+        return "com.google.ios.youtube/" + MOBILE_YOUTUBE_CLIENT_VERSION
+                + "(" + IOS_DEVICE_MODEL
+                + "; U; CPU iOS 15_4 like Mac OS X; "
+                + (localization != null ? localization.getCountryCode()
+                        : Localization.DEFAULT.getCountryCode())
+                + ")";
     }
 
     /**
@@ -1126,9 +1318,10 @@ public final class YoutubeParsingHelper {
      * @see #CONSENT_COOKIE
      * @param headers the headers which should be completed
      */
+    @SuppressWarnings("ArraysAsListWithZeroOrOneArgument")
     public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
         if (headers.get("Cookie") == null) {
-            headers.put("Cookie", Collections.singletonList(generateConsentCookie()));
+            headers.put("Cookie", Arrays.asList(generateConsentCookie()));
         } else {
             headers.get("Cookie").add(generateConsentCookie());
         }
@@ -1366,4 +1559,33 @@ public final class YoutubeParsingHelper {
                 .replaceAll("\\\\x5b", "[")
                 .replaceAll("\\\\x5d", "]");
     }
+
+    /**
+     * Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
+     * playback requests (and also for some clients, in the player request body).
+     *
+     * @return a content playback nonce string
+     */
+    @Nonnull
+    public static String generateContentPlaybackNonce() {
+        return RandomStringFromAlphabetGenerator.generate(
+                CONTENT_PLAYBACK_NONCE_ALPHABET, 16, numberGenerator);
+    }
+
+    /**
+     * Try to generate a {@code t} parameter, sent by mobile clients as a query of the player
+     * request.
+     *
+     * <p>
+     * Some researches needs to be done to know how this parameter, unique at each request, is
+     * generated.
+     * </p>
+     *
+     * @return a 12 characters string to try to reproduce the {@code} parameter
+     */
+    @Nonnull
+    public static String generateTParameter() {
+        return RandomStringFromAlphabetGenerator.generate(
+                CONTENT_PLAYBACK_NONCE_ALPHABET, 12, numberGenerator);
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
@@ -395,7 +396,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
                         .done())
                 .getBytes(UTF_8);
 
-        return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelIds, null, body);
+        return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+                + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
@@ -90,8 +91,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
         final Map<String, List<String>> headers = new HashMap<>();
         addClientInfoHeaders(headers);
 
-        final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey(),
+        final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
-                headers, body, localization);
+                + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
 
         initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
         playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
@@ -60,7 +61,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
         final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
 
         final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key="
-                + youtubeMusicKeys[0];
+                + youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
 
         final String params;
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl;
@@ -317,7 +318,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
                             .done())
                     .getBytes(StandardCharsets.UTF_8);
 
-            return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
+            return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+                    + DISABLE_PRETTY_PRINT_PARAMETER, body);
         } else {
             return null;
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
@@ -239,7 +240,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
         final String token = continuationItemRenderer.getObject("continuationEndpoint")
                 .getObject("continuationCommand").getString("token");
 
-        final String url = YOUTUBEI_V1_URL + "search?key=" + getKey();
+        final String url = YOUTUBEI_V1_URL + "search?key=" + getKey()
+                + DISABLE_PRETTY_PRINT_PARAMETER;
 
         return new Page(url, token);
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -1,14 +1,20 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createPlayerBodyWithSts;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createDesktopPlayerBody;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonMobilePostResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileEmbedVideoJsonBuilder;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopEmbedVideoJsonBuilder;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder;
 import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -63,13 +69,14 @@ import java.time.LocalDate;
 import java.time.OffsetDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
@@ -113,19 +120,35 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nullable
     private static String playerCode = null;
 
+    private static boolean isAndroidClientFetchForced = false;
+    private static boolean isIosClientFetchForced = false;
+
     private JsonObject playerResponse;
     private JsonObject nextResponse;
 
     @Nullable
-    private JsonObject desktopStreamingData;
+    private JsonObject html5StreamingData;
     @Nullable
-    private JsonObject mobileStreamingData;
+    private JsonObject androidStreamingData;
+    @Nullable
+    private JsonObject iosStreamingData;
+
     private JsonObject videoPrimaryInfoRenderer;
     private JsonObject videoSecondaryInfoRenderer;
+    private JsonObject playerMicroFormatRenderer;
     private int ageLimit = -1;
+    private StreamType streamType;
     @Nullable
     private List<SubtitlesStream> subtitles = null;
 
+    // We need to store the contentPlaybackNonces because we need to append them to videoplayback
+    // URLs (with the cpn parameter).
+    // Also because a nonce should be unique, it should be different between clients used, so
+    // three different strings are used.
+    private String html5Cpn;
+    private String androidCpn;
+    private String iosCpn;
+
     public YoutubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) {
         super(service, linkHandler);
     }
@@ -160,14 +183,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nullable
     @Override
     public String getTextualUploadDate() throws ParsingException {
-        final JsonObject micro = playerResponse.getObject("microformat")
+        if (!playerMicroFormatRenderer.getString("uploadDate", EMPTY_STRING).isEmpty()) {
-                .getObject("playerMicroformatRenderer");
+            return playerMicroFormatRenderer.getString("uploadDate");
-        if (!micro.getString("uploadDate", EMPTY_STRING).isEmpty()) {
+        } else if (!playerMicroFormatRenderer.getString("publishDate", EMPTY_STRING).isEmpty()) {
-            return micro.getString("uploadDate");
+            return playerMicroFormatRenderer.getString("publishDate");
-        } else if (!micro.getString("publishDate", EMPTY_STRING).isEmpty()) {
-            return micro.getString("publishDate");
         } else {
-            final JsonObject liveDetails = micro.getObject("liveBroadcastDetails");
+            final JsonObject liveDetails = playerMicroFormatRenderer.getObject(
+                    "liveBroadcastDetails");
             if (!liveDetails.getString("endTimestamp", EMPTY_STRING).isEmpty()) {
                 // an ended live stream
                 return liveDetails.getString("endTimestamp");
@@ -183,7 +205,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))
                 .startsWith("Premiered")) {
             final String time = getTextFromObject(
-                    getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10);
+                    getVideoPrimaryInfoRenderer().getObject("dateText")).substring(13);
 
             try { // Premiered 20 hours ago
                 final TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(
@@ -199,6 +221,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
             } catch (final Exception ignored) {
             }
+
+            try { // Premiered on 21 Feb 2020
+                final LocalDate localDate = LocalDate.parse(time,
+                        DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
+                return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
+            } catch (final Exception ignored) {
+            }
         }
 
         try {
@@ -208,10 +237,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                     getVideoPrimaryInfoRenderer().getObject("dateText")),
                     DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
             return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
-        } catch (final Exception ignored) {
+        } catch (final Exception e) {
+            throw new ParsingException("Could not get upload date", e);
         }
 
-        throw new ParsingException("Could not get upload date");
     }
 
     @Override
@@ -260,8 +289,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         String description = playerResponse.getObject("videoDetails")
                 .getString("shortDescription");
         if (description == null) {
-            final JsonObject descriptionObject = playerResponse.getObject("microformat")
+            final JsonObject descriptionObject = playerMicroFormatRenderer.getObject("description");
-                    .getObject("playerMicroformatRenderer").getObject("description");
             description = getTextFromObject(descriptionObject);
         }
 
@@ -305,20 +333,28 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                     .getString("lengthSeconds");
             return Long.parseLong(duration);
         } catch (final Exception e) {
-            if (desktopStreamingData != null) {
+            return getDurationFromFirstAdaptiveFormat(Arrays.asList(
-                final JsonArray adaptiveFormats = desktopStreamingData.getArray(ADAPTIVE_FORMATS);
+                    html5StreamingData, androidStreamingData, iosStreamingData));
-                final String durationMs = adaptiveFormats.getObject(0)
-                        .getString("approxDurationMs");
-                return Math.round(Long.parseLong(durationMs) / 1000f);
-            } else if (mobileStreamingData != null) {
-                final JsonArray adaptiveFormats = mobileStreamingData.getArray(ADAPTIVE_FORMATS);
-                final String durationMs = adaptiveFormats.getObject(0)
-                        .getString("approxDurationMs");
-                return Math.round(Long.parseLong(durationMs) / 1000f);
-            } else {
-                throw new ParsingException("Could not get duration", e);
         }
     }
+
+    private int getDurationFromFirstAdaptiveFormat(@Nonnull final List<JsonObject> streamingDatas)
+            throws ParsingException {
+        for (final JsonObject streamingData : streamingDatas) {
+            final JsonArray adaptiveFormats = streamingData.getArray(ADAPTIVE_FORMATS);
+            if (adaptiveFormats.isEmpty()) {
+                continue;
+            }
+
+            final String durationMs = adaptiveFormats.getObject(0)
+                    .getString("approxDurationMs");
+            try {
+                return Math.round(Long.parseLong(durationMs) / 1000f);
+            } catch (final NumberFormatException ignored) {
+            }
+        }
+
+        throw new ParsingException("Could not get duration");
     }
 
     /**
@@ -465,7 +501,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             if (ageLimit == NO_AGE_LIMIT) {
                 throw new ParsingException("Could not get uploader avatar URL");
             }
-            return "";
+
+            return EMPTY_STRING;
         }
 
         return fixThumbnailUrl(url);
@@ -491,13 +528,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public String getDashMpdUrl() throws ParsingException {
         assertPageFetched();
 
-        if (desktopStreamingData != null) {
+        // There is no DASH manifest available in the iOS clients and the DASH manifest of the
-            return desktopStreamingData.getString("dashManifestUrl");
+        // Android client doesn't contain all available streams (mainly the WEBM ones)
-        } else if (mobileStreamingData != null) {
+        return getManifestUrl("dash", Arrays.asList(html5StreamingData,
-            return mobileStreamingData.getString("dashManifestUrl");
+                androidStreamingData));
-        } else {
-            return EMPTY_STRING;
-        }
     }
 
     @Nonnull
@@ -505,13 +539,27 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public String getHlsUrl() throws ParsingException {
         assertPageFetched();
 
-        if (desktopStreamingData != null) {
+        // Return HLS manifest of the iOS client first because on livestreams, the HLS manifest
-            return desktopStreamingData.getString("hlsManifestUrl");
+        // returned has separated audio and video streams
-        } else if (mobileStreamingData != null) {
+        // Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response
-            return mobileStreamingData.getString("hlsManifestUrl");
+        return getManifestUrl("hls", Arrays.asList(iosStreamingData, html5StreamingData,
-        } else {
+                androidStreamingData));
-            return EMPTY_STRING;
     }
+
+    @Nonnull
+    private static String getManifestUrl(@Nonnull final String manifestType,
+                                         @Nonnull final List<JsonObject> streamingDataObjects) {
+        final String manifestKey = manifestType + "ManifestUrl";
+        for (final JsonObject streamingDataObject : streamingDataObjects) {
+            if (streamingDataObject != null) {
+                final String manifestKeyValue = streamingDataObject.getString(manifestKey);
+                if (manifestKeyValue != null) {
+                    return manifestKeyValue;
+                }
+            }
+        }
+
+        return EMPTY_STRING;
     }
 
     @Override
@@ -645,11 +693,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public StreamType getStreamType() {
         assertPageFetched();
 
+        return streamType;
+    }
+
+    private void setStreamType() {
         if (playerResponse.getObject("playabilityStatus").has("liveStreamability")
                 || playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false)) {
-            return StreamType.LIVE_STREAM;
+            streamType = StreamType.LIVE_STREAM;
+        } else {
+            streamType = StreamType.VIDEO_STREAM;
         }
-        return StreamType.VIDEO_STREAM;
     }
 
     @Nullable
@@ -710,6 +763,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     private static final String FORMATS = "formats";
     private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
     private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate";
+    private static final String STREAMING_DATA = "streamingData";
+    private static final String PLAYER = "player";
+    private static final String NEXT = "next";
 
     private static final String[] REGEXES = {
             "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)"
@@ -725,24 +781,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
+        if (sts == null) {
+            getStsFromPlayerJs();
+        }
+
         final String videoId = getId();
         final Localization localization = getExtractorLocalization();
         final ContentCountry contentCountry = getExtractorContentCountry();
-        final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
+        html5Cpn = generateContentPlaybackNonce();
-                localization, contentCountry)
-                .value("videoId", videoId)
-                .done())
-                .getBytes(UTF_8);
 
-        // Put the sts string if we already know it so we don't have to fetch again the player
+        playerResponse = getJsonPostResponse(PLAYER,
-        // endpoint of the desktop internal API if something went wrong when parsing the Android
+                createDesktopPlayerBody(localization, contentCountry, videoId, sts, false,
-        // API.
+                        html5Cpn),
-        if (sts != null) {
+                localization);
-            playerResponse = getJsonPostResponse("player", createPlayerBodyWithSts(localization,
-                    contentCountry, videoId, false, sts), localization);
-        } else {
-            playerResponse = getJsonPostResponse("player", body, localization);
-        }
 
         // Save the playerResponse from the player endpoint of the desktop internal API because
         // there can be restrictions on the embedded player.
@@ -759,44 +810,56 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         final boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING)
                 .contains("age");
 
-        if (!playerResponse.has("streamingData")) {
+        setStreamType();
+
+        if (!playerResponse.has(STREAMING_DATA)) {
             try {
-                fetchDesktopEmbedJsonPlayer(contentCountry, localization, videoId);
+                fetchTvHtml5EmbedJsonPlayer(contentCountry, localization, videoId);
-            } catch (final Exception ignored) {
-            }
-            try {
-                fetchAndroidEmbedJsonPlayer(contentCountry, localization, videoId);
             } catch (final Exception ignored) {
             }
+
+            // Refresh the stream type because the stream type may be not properly known for
+            // age-restricted videos
+            setStreamType();
         }
 
-        if (desktopStreamingData == null && playerResponse.has("streamingData")) {
+        if (html5StreamingData == null && playerResponse.has(STREAMING_DATA)) {
-            desktopStreamingData = playerResponse.getObject("streamingData");
+            html5StreamingData = playerResponse.getObject(STREAMING_DATA);
         }
 
-        if (desktopStreamingData == null) {
+        if (html5StreamingData == null) {
             checkPlayabilityStatus(youtubePlayerResponse, playabilityStatus);
         }
 
-        if (ageRestricted) {
+        // The microformat JSON object of the content is not returned on the client we use to
-            final byte[] ageRestrictedBody = JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(
+        // try to get streams of unavailable contents but is still returned on the WEB client,
-                    localization, contentCountry, videoId)
+        // so we need to store it instead of getting it directly from the playerResponse
+        playerMicroFormatRenderer = youtubePlayerResponse.getObject("microformat")
+                .getObject("playerMicroformatRenderer");
+
+        final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
+                contentCountry)
+                .value(VIDEO_ID, videoId)
+                .value(CONTENT_CHECK_OK, true)
+                .value(RACY_CHECK_OK, true)
                 .done())
                 .getBytes(UTF_8);
-            nextResponse = getJsonPostResponse("next", ageRestrictedBody, localization);
+        nextResponse = getJsonPostResponse(NEXT, body, localization);
-        } else {
-            nextResponse = getJsonPostResponse("next", body, localization);
-        }
 
-        if (!ageRestricted) {
+        if ((!ageRestricted && streamType == StreamType.VIDEO_STREAM)
+                || isAndroidClientFetchForced) {
             try {
                 fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId);
             } catch (final Exception ignored) {
             }
         }
 
-        if (isCipherProtectedContent()) {
+        if ((!ageRestricted && streamType == StreamType.LIVE_STREAM)
-            fetchDesktopJsonPlayerWithSts(contentCountry, localization, videoId);
+                || isIosClientFetchForced) {
+            try {
+                fetchIosMobileJsonPlayer(contentCountry, localization, videoId);
+            } catch (final Exception ignored) {
+            }
         }
     }
 
@@ -825,8 +888,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                             "This age-restricted video cannot be watched.");
                 }
             }
-            if (status.equalsIgnoreCase("unplayable")) {
+
-                if (reason != null) {
+            if (status.equalsIgnoreCase("unplayable") && reason != null) {
                 if (reason.contains("Music Premium")) {
                     throw new YoutubeMusicPremiumContentException();
                 }
@@ -837,16 +900,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                     throw new PaidContentException("This video is only available"
                             + " for members of the channel of this video");
                 }
+
                 if (reason.contains("unavailable")) {
                     final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus
                             .getObject("errorScreen").getObject("playerErrorMessageRenderer")
                             .getObject("subreason"));
-                        if (detailedErrorMessage != null) {
+                    if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) {
-                            if (detailedErrorMessage.contains("country")) {
                         throw new GeographicRestrictionException(
-                                        "This video is not available in user's country.");
+                                "This video is not available in client's country.");
-                            }
-                        }
                     }
                 }
             }
@@ -856,103 +917,99 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     /**
-     * Fetch the Android Mobile API and assign the streaming data to the mobileStreamingData JSON
+     * Fetch the Android Mobile API and assign the streaming data to the androidStreamingData JSON
      * object.
      */
-    private void fetchAndroidMobileJsonPlayer(final ContentCountry contentCountry,
+    private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentCountry,
-                                              final Localization localization,
+                                              @Nonnull final Localization localization,
-                                              final String videoId)
+                                              @Nonnull final String videoId)
             throws IOException, ExtractionException {
+        androidCpn = generateContentPlaybackNonce();
         final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder(
                 localization, contentCountry)
-                .value("videoId", videoId)
+                .value(VIDEO_ID, videoId)
+                .value(CPN, androidCpn)
+                .value(CONTENT_CHECK_OK, true)
+                .value(RACY_CHECK_OK, true)
                 .done())
                 .getBytes(UTF_8);
-        final JsonObject mobilePlayerResponse = getJsonMobilePostResponse("player",
-                mobileBody, contentCountry, localization);
 
-        final JsonObject streamingData = mobilePlayerResponse.getObject("streamingData");
+        final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(PLAYER,
+                mobileBody, localization, "&t=" + generateTParameter()
+                        + "&id=" + videoId);
+
+        final JsonObject streamingData = androidPlayerResponse.getObject(STREAMING_DATA);
         if (!isNullOrEmpty(streamingData)) {
-            mobileStreamingData = streamingData;
+            androidStreamingData = streamingData;
-            if (desktopStreamingData == null) {
+            if (html5StreamingData == null) {
-                playerResponse = mobilePlayerResponse;
+                playerResponse = androidPlayerResponse;
             }
         }
     }
 
     /**
-     * Fetch the desktop API with the {@code signatureTimestamp} and assign the streaming data to
+     * Fetch the iOS Mobile API and assign the streaming data to the iosStreamingData JSON
-     * the {@code desktopStreamingData} JSON object.
+     * object.
-     * The cipher signatures from the player endpoint without a signatureTimestamp are invalid so
-     * if the content is protected by signatureCiphers and if signatureTimestamp is not known, we
-     * need to fetch again the desktop InnerTube API.
      */
-    private void fetchDesktopJsonPlayerWithSts(final ContentCountry contentCountry,
+    private void fetchIosMobileJsonPlayer(@Nonnull final ContentCountry contentCountry,
-                                               final Localization localization,
+                                          @Nonnull final Localization localization,
-                                               final String videoId)
+                                          @Nonnull final String videoId)
+            throws IOException, ExtractionException {
+        iosCpn = generateContentPlaybackNonce();
+        final byte[] mobileBody = JsonWriter.string(prepareIosMobileJsonBuilder(
+                        localization, contentCountry)
+                        .value(VIDEO_ID, videoId)
+                        .value(CPN, iosCpn)
+                        .value(CONTENT_CHECK_OK, true)
+                        .value(RACY_CHECK_OK, true)
+                        .done())
+                .getBytes(UTF_8);
+
+        final JsonObject iosPlayerResponse = getJsonIosPostResponse(PLAYER,
+                mobileBody, localization, "&t=" + generateTParameter()
+                        + "&id=" + videoId);
+
+        final JsonObject streamingData = iosPlayerResponse.getObject(STREAMING_DATA);
+        if (!isNullOrEmpty(streamingData)) {
+            iosStreamingData = streamingData;
+            if (html5StreamingData == null) {
+                playerResponse = iosPlayerResponse;
+            }
+        }
+    }
+
+    /**
+     * Download the {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} JSON player as an embed client to bypass
+     * some age-restrictions and assign the streaming data to the {@code html5StreamingData} JSON
+     * object.
+     *
+     * @param contentCountry the content country to use
+     * @param localization   the localization to use
+     * @param videoId        the video id
+     */
+    private void fetchTvHtml5EmbedJsonPlayer(@Nonnull final ContentCountry contentCountry,
+                                             @Nonnull final Localization localization,
+                                             @Nonnull final String videoId)
             throws IOException, ExtractionException {
         if (sts == null) {
             getStsFromPlayerJs();
         }
-        final JsonObject playerResponseWithSignatureTimestamp = getJsonPostResponse(
-                "player", createPlayerBodyWithSts(
-                        localization, contentCountry, videoId, false, sts),
-                localization);
-        if (playerResponseWithSignatureTimestamp.has("streamingData")) {
-            desktopStreamingData = playerResponseWithSignatureTimestamp.getObject("streamingData");
-        }
-    }
 
-    /**
+        // Because a cpn is unique to each request, we need to generate it again
-     * Download again the desktop JSON player as an embed client to bypass some age-restrictions.
+        html5Cpn = generateContentPlaybackNonce();
-     * <p>
+
-     * We need also to get the {@code signatureTimestamp}, if it isn't known because we don't know
+        final JsonObject tvHtml5EmbedPlayerResponse = getJsonPostResponse(PLAYER,
-     * if the video will have signature ciphers or not.
+                createDesktopPlayerBody(localization, contentCountry, videoId, sts, true,
-     * </p>
+                        html5Cpn), localization);
-     */
+        final JsonObject streamingData = tvHtml5EmbedPlayerResponse.getObject(
-    private void fetchDesktopEmbedJsonPlayer(final ContentCountry contentCountry,
+                STREAMING_DATA);
-                                             final Localization localization,
-                                             final String videoId)
-            throws IOException, ExtractionException {
-        if (sts == null) {
-            getStsFromPlayerJs();
-        }
-        final JsonObject desktopWebEmbedPlayerResponse = getJsonPostResponse(
-                "player", createPlayerBodyWithSts(
-                        localization, contentCountry, videoId, true, sts),
-                localization);
-        final JsonObject streamingData = desktopWebEmbedPlayerResponse.getObject(
-                "streamingData");
         if (!isNullOrEmpty(streamingData)) {
-            playerResponse = desktopWebEmbedPlayerResponse;
+            playerResponse = tvHtml5EmbedPlayerResponse;
-            desktopStreamingData = streamingData;
+            html5StreamingData = streamingData;
         }
     }
 
-    /**
+    private static void storePlayerJs() throws ParsingException {
-     * Download the Android mobile JSON player as an embed client to bypass some age-restrictions.
-     */
-    private void fetchAndroidEmbedJsonPlayer(final ContentCountry contentCountry,
-                                             final Localization localization,
-                                             final String videoId)
-            throws IOException, ExtractionException {
-        final byte[] androidMobileEmbedBody = JsonWriter.string(
-                prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
-                        .done())
-                .getBytes(UTF_8);
-        final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player",
-                androidMobileEmbedBody, contentCountry, localization);
-        final JsonObject streamingData = androidMobileEmbedPlayerResponse.getObject(
-                "streamingData");
-        if (!isNullOrEmpty(streamingData)) {
-            if (desktopStreamingData == null) {
-                playerResponse = androidMobileEmbedPlayerResponse;
-            }
-            mobileStreamingData = androidMobileEmbedPlayerResponse.getObject("streamingData");
-        }
-    }
-
-    private void storePlayerJs() throws ParsingException {
         try {
             playerCode = YoutubeJavaScriptExtractor.extractJavaScriptCode();
         } catch (final Exception e) {
@@ -960,37 +1017,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
     }
 
-    private boolean isCipherProtectedContent() {
+    private static String getDeobfuscationFuncName(final String thePlayerCode)
-        if (desktopStreamingData != null) {
-            if (desktopStreamingData.has(ADAPTIVE_FORMATS)) {
-                final JsonArray adaptiveFormats = desktopStreamingData.getArray(ADAPTIVE_FORMATS);
-                if (!isNullOrEmpty(adaptiveFormats)) {
-                    for (final Object adaptiveFormat : adaptiveFormats) {
-                        final JsonObject adaptiveFormatJsonObject = ((JsonObject) adaptiveFormat);
-                        if (adaptiveFormatJsonObject.has("signatureCipher")
-                                || adaptiveFormatJsonObject.has("cipher")) {
-                            return true;
-                        }
-                    }
-                }
-            }
-            if (desktopStreamingData.has(FORMATS)) {
-                final JsonArray formats = desktopStreamingData.getArray(FORMATS);
-                if (!isNullOrEmpty(formats)) {
-                    for (final Object format : formats) {
-                        final JsonObject formatJsonObject = ((JsonObject) format);
-                        if (formatJsonObject.has("signatureCipher")
-                                || formatJsonObject.has("cipher")) {
-                            return true;
-                        }
-                    }
-                }
-            }
-        }
-        return false;
-    }
-
-    private String getDeobfuscationFuncName(final String thePlayerCode)
             throws DeobfuscateException {
         Parser.RegexException exception = null;
         for (final String regex : REGEXES) {
@@ -1007,7 +1034,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Nonnull
-    private String loadDeobfuscationCode() throws DeobfuscateException {
+    private static String loadDeobfuscationCode() throws DeobfuscateException {
         try {
             final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode);
 
@@ -1024,7 +1051,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                     "(var " + helperObjectName.replace("$", "\\$")
                             + "=\\{.+?\\}\\};)";
             final String helperObject =
-                    Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));
+                    Parser.matchGroup1(helperPattern, Objects.requireNonNull(playerCode).replace(
+                            "\n", ""));
 
             final String callerFunction =
                     "function " + DEOBFUSCATION_FUNC_NAME + "(a){return "
@@ -1037,7 +1065,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Nonnull
-    private String getDeobfuscationCode() throws ParsingException {
+    private static String getDeobfuscationCode() throws ParsingException {
         if (cachedDeobfuscationCode == null) {
             if (isNullOrEmpty(playerCode)) {
                 throw new ParsingException("playerCode is null");
@@ -1048,7 +1076,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         return cachedDeobfuscationCode;
     }
 
-    private void getStsFromPlayerJs() throws ParsingException {
+    private static void getStsFromPlayerJs() throws ParsingException {
         if (!isNullOrEmpty(sts)) {
             return;
         }
@@ -1085,8 +1113,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     //////////////////////////////////////////////////////////////////////////*/
 
     private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException {
-        if (this.videoPrimaryInfoRenderer != null) {
+        if (videoPrimaryInfoRenderer != null) {
-            return this.videoPrimaryInfoRenderer;
+            return videoPrimaryInfoRenderer;
         }
 
         final JsonArray contents = nextResponse.getObject("contents")
@@ -1106,18 +1134,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             throw new ParsingException("Could not find videoPrimaryInfoRenderer");
         }
 
-        this.videoPrimaryInfoRenderer = theVideoPrimaryInfoRenderer;
+        videoPrimaryInfoRenderer = theVideoPrimaryInfoRenderer;
         return theVideoPrimaryInfoRenderer;
     }
 
     private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
-        if (this.videoSecondaryInfoRenderer != null) {
+        if (videoSecondaryInfoRenderer != null) {
-            return this.videoSecondaryInfoRenderer;
+            return videoSecondaryInfoRenderer;
         }
 
         final JsonArray contents = nextResponse.getObject("contents")
                 .getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
                 .getArray("contents");
+
         JsonObject theVideoSecondaryInfoRenderer = null;
 
         for (final Object content : contents) {
@@ -1132,24 +1161,33 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             throw new ParsingException("Could not find videoSecondaryInfoRenderer");
         }
 
-        this.videoSecondaryInfoRenderer = theVideoSecondaryInfoRenderer;
+        videoSecondaryInfoRenderer = theVideoSecondaryInfoRenderer;
         return theVideoSecondaryInfoRenderer;
     }
 
     @Nonnull
-    private Map<String, ItagItem> getItags(final String streamingDataKey,
+    private Map<String, ItagItem> getItags(@Nonnull final String streamingDataKey,
-                                           final ItagItem.ItagType itagTypeWanted) {
+                                           @Nonnull final ItagItem.ItagType itagTypeWanted) {
         final Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
-        if (desktopStreamingData == null && mobileStreamingData == null) {
+        if (html5StreamingData == null && androidStreamingData == null
+                && iosStreamingData == null) {
             return urlAndItags;
         }
 
-        // Use the mobileStreamingData object first because there is no n param and no
+        final Map<String, JsonObject> streamingDataAndCpnLoopMap = new HashMap<>();
+        // Use the androidStreamingData object first because there is no n param and no
         // signatureCiphers in streaming URLs of the Android client
-        urlAndItags.putAll(getStreamsFromStreamingDataKey(
+        streamingDataAndCpnLoopMap.put(androidCpn, androidStreamingData);
-                mobileStreamingData, streamingDataKey, itagTypeWanted));
+        streamingDataAndCpnLoopMap.put(html5Cpn, html5StreamingData);
-        urlAndItags.putAll(getStreamsFromStreamingDataKey(
+        // Use the iosStreamingData object in the last position because most of the available
-                desktopStreamingData, streamingDataKey, itagTypeWanted));
+        // streams can be extracted with the Android and web clients and also because the iOS
+        // client is only enabled by default on livestreams
+        streamingDataAndCpnLoopMap.put(iosCpn, iosStreamingData);
+
+        for (final Map.Entry<String, JsonObject> entry : streamingDataAndCpnLoopMap.entrySet()) {
+            urlAndItags.putAll(getStreamsFromStreamingDataKey(entry.getValue(), streamingDataKey,
+                    itagTypeWanted, entry.getKey()));
+        }
 
         return urlAndItags;
     }
@@ -1157,8 +1195,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nonnull
     private Map<String, ItagItem> getStreamsFromStreamingDataKey(
             final JsonObject streamingData,
-            final String streamingDataKey,
+            @Nonnull final String streamingDataKey,
-            final ItagItem.ItagType itagTypeWanted) {
+            @Nonnull final ItagItem.ItagType itagTypeWanted,
+            @Nonnull final String contentPlaybackNonce) {
 
         final Map<String, ItagItem> urlAndItagsFromStreamingDataObject = new LinkedHashMap<>();
         if (streamingData != null && streamingData.has(streamingDataKey)) {
@@ -1180,7 +1219,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
                             final String streamUrl;
                             if (formatData.has("url")) {
-                                streamUrl = formatData.getString("url");
+                                streamUrl = formatData.getString("url") + "&cpn="
+                                        + contentPlaybackNonce;
                             } else {
                                 // This url has an obfuscated signature
                                 final String cipherString = formatData.has("cipher")
@@ -1292,16 +1332,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public Privacy getPrivacy() {
-        final boolean isUnlisted = playerResponse.getObject("microformat")
+        final boolean isUnlisted = playerMicroFormatRenderer.getBoolean("isUnlisted");
-                .getObject("playerMicroformatRenderer").getBoolean("isUnlisted");
         return isUnlisted ? Privacy.UNLISTED : Privacy.PUBLIC;
     }
 
     @Nonnull
     @Override
     public String getCategory() {
-        return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer")
+        return playerMicroFormatRenderer.getString("category", EMPTY_STRING);
-                .getString("category", EMPTY_STRING);
     }
 
     @Nonnull
@@ -1404,6 +1442,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     /**
      * Reset YouTube's deobfuscation code.
+     *
      * <p>
      * This is needed for mocks in YouTube stream tests, because when they are ran, the
      * {@code signatureTimestamp} is known (the {@code sts} string) so a different body than the
@@ -1418,4 +1457,42 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         sts = null;
         YoutubeJavaScriptExtractor.resetJavaScriptCode();
     }
+
+    /**
+     * Enable or disable the fetch of the Android client for all stream types.
+     *
+     * <p>
+     * By default, the fetch of the Android client will be made only on videos, in order to reduce
+     * data usage, because available streams of the Android client will be almost equal to the ones
+     * available on the {@code WEB} client: you can get exclusively a 48kbps audio stream and a
+     * 3GPP very low stream (which is, most of times, a 144p8 stream).
+     * </p>
+     *
+     * @param forceFetchAndroidClientValue whether to always fetch the Android client and not only
+     *                                     for videos
+     */
+    public static void forceFetchAndroidClient(final boolean forceFetchAndroidClientValue) {
+        isAndroidClientFetchForced = forceFetchAndroidClientValue;
+    }
+
+    /**
+     * Enable or disable the fetch of the iOS client for all stream types.
+     *
+     * <p>
+     * By default, the fetch of the iOS client will be made only on livestreams, in order to get an
+     * HLS manifest with separated audio and video which has also an higher replay time (up to one
+     * hour, depending of the content instead of 30 seconds with non-iOS clients).
+     * </p>
+     *
+     * <p>
+     * Enabling this option will allow you to get an HLS manifest also for regular videos, which
+     * contains resolutions up to 1080p60.
+     * </p>
+     *
+     * @param forceFetchIosClientValue whether to always fetch the iOS client and not only for
+     *                                 livestreams
+     */
+    public static void forceFetchIosClient(final boolean forceFetchIosClientValue) {
+        isIosClientFetchForced = forceFetchIosClientValue;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java

@@ -1,3 +1,23 @@
+/*
+ * Created by Christian Schabesberger on 02.02.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * Parser.java is part of NewPipe Extractor.
+ *
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe Extractor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package org.schabi.newpipe.extractor.utils;
 
 import org.nibor.autolink.LinkExtractor;
@@ -5,39 +25,21 @@ import org.nibor.autolink.LinkSpan;
 import org.nibor.autolink.LinkType;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
+import javax.annotation.Nonnull;
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 
-/*
- * Created by Christian Schabesberger on 02.02.16.
- *
- * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
- * Parser.java is part of NewPipe.
- *
- * NewPipe is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * NewPipe is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 /**
- * avoid using regex !!!
+ * Avoid using regex !!!
  */
 public final class Parser {
 
@@ -66,8 +68,9 @@ public final class Parser {
         return matchGroup(Pattern.compile(pattern), input, group);
     }
 
-    public static String matchGroup(final Pattern pat, final String input, final int group)
+    public static String matchGroup(@Nonnull final Pattern pat,
-            throws RegexException {
+                                    final String input,
+                                    final int group) throws RegexException {
         final Matcher matcher = pat.matcher(input);
         final boolean foundMatch = matcher.find();
         if (foundMatch) {
@@ -75,9 +78,9 @@ public final class Parser {
         } else {
             // only pass input to exception message when it is not too long
             if (input.length() > 1024) {
-                throw new RegexException("failed to find pattern \"" + pat.pattern() + "\"");
+                throw new RegexException("Failed to find pattern \"" + pat.pattern() + "\"");
             } else {
-                throw new RegexException("failed to find pattern \"" + pat.pattern()
+                throw new RegexException("Failed to find pattern \"" + pat.pattern()
                         + "\" inside of \"" + input + "\"");
             }
         }
@@ -89,12 +92,13 @@ public final class Parser {
         return mat.find();
     }
 
-    public static boolean isMatch(final Pattern pattern, final String input) {
+    public static boolean isMatch(@Nonnull final Pattern pattern, final String input) {
         final Matcher mat = pattern.matcher(input);
         return mat.find();
     }
 
-    public static Map<String, String> compatParseMap(final String input)
+    @Nonnull
+    public static Map<String, String> compatParseMap(@Nonnull final String input)
             throws UnsupportedEncodingException {
         final Map<String, String> map = new HashMap<>();
         for (final String arg : input.split("&")) {
@@ -108,9 +112,10 @@ public final class Parser {
         return map;
     }
 
+    @Nonnull
     public static String[] getLinksFromString(final String txt) throws ParsingException {
         try {
-            final ArrayList<String> links = new ArrayList<>();
+            final List<String> links = new ArrayList<>();
             final LinkExtractor linkExtractor = LinkExtractor.builder()
                     .linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW))
                     .build();

extractor/src/main/java/org/schabi/newpipe/extractor/utils/RandomStringFromAlphabetGenerator.java

@@ -0,0 +1,36 @@
+package org.schabi.newpipe.extractor.utils;
+
+import java.util.Random;
+
+import javax.annotation.Nonnull;
+
+/**
+ * Generates a random string from a predefined alphabet.
+ */
+public final class RandomStringFromAlphabetGenerator {
+    private RandomStringFromAlphabetGenerator() {
+        // No impl
+    }
+
+    /**
+     * Generate a random string from an alphabet.
+     *
+     * @param alphabet the characters' alphabet to use
+     * @param length   the length of the returned string (> 0)
+     * @param random   {@link Random} (or better {@link java.security.SecureRandom}) used for
+     *                 generating the random string
+     * @return a random string of the requested length made of only characters from the provided
+     * alphabet
+     */
+    @Nonnull
+    public static String generate(
+            final String alphabet,
+            final int length,
+            final Random random) {
+        final StringBuilder stringBuilder = new StringBuilder(length);
+        for (int i = 0; i < length; i++) {
+            stringBuilder.append(alphabet.charAt(random.nextInt(alphabet.length())));
+        }
+        return stringBuilder.toString();
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -2,16 +2,20 @@ package org.schabi.newpipe.extractor.utils;
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.regex.Pattern;
 
 public final class Utils {
@@ -28,21 +32,32 @@ public final class Utils {
     }
 
     /**
-     * Remove all non-digit characters from a string.<p>
+     * Remove all non-digit characters from a string.
-     * Examples:<p>
+     *
-     * <ul><li>1 234 567 views -&gt; 1234567</li>
+     * <p>
-     * <li>$31,133.124 -&gt; 31133124</li></ul>
+     * Examples:
+     * </p>
+     *
+     * <ul>
+     *     <li>1 234 567 views -&gt; 1234567</li>
+     *     <li>$31,133.124 -&gt; 31133124</li>
+     * </ul>
      *
      * @param toRemove string to remove non-digit chars
      * @return a string that contains only digits
      */
-    public static String removeNonDigitCharacters(final String toRemove) {
+    @Nonnull
+    public static String removeNonDigitCharacters(@Nonnull final String toRemove) {
         return toRemove.replaceAll("\\D+", "");
     }
 
     /**
-     * <p>Convert a mixed number word to a long.</p>
+     * Convert a mixed number word to a long.
-     * <p>Examples:</p>
+     *
+     * <p>
+     * Examples:
+     * </p>
+     *
      * <ul>
      *     <li>123 -&gt; 123</li>
      *     <li>1.23K -&gt; 1230</li>
@@ -52,8 +67,8 @@ public final class Utils {
      * @param numberWord string to be converted to a long
      * @return a long
      */
-    public static long mixedNumberWordToLong(final String numberWord) throws NumberFormatException,
+    public static long mixedNumberWordToLong(final String numberWord)
-            ParsingException {
+            throws NumberFormatException, ParsingException {
         String multiplier = "";
         try {
             multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2);
@@ -94,7 +109,7 @@ public final class Utils {
             return null;
         }
 
-        if (!url.isEmpty() && url.startsWith(HTTP)) {
+        if (url.startsWith(HTTP)) {
             return HTTPS + url.substring(HTTP.length());
         }
         return url;
@@ -102,13 +117,19 @@ public final class Utils {
 
     /**
      * Get the value of a URL-query by name.
-     * If a url-query is give multiple times, only the value of the first query is returned
+     *
+     * <p>
+     * If an url-query is give multiple times, only the value of the first query is returned.
+     * </p>
      *
      * @param url           the url to be used
      * @param parameterName the pattern that will be used to check the url
-     * @return a string that contains the value of the query parameter or null if nothing was found
+     * @return a string that contains the value of the query parameter or {@code null} if nothing
+     * was found
      */
-    public static String getQueryValue(final URL url, final String parameterName) {
+    @Nullable
+    public static String getQueryValue(@Nonnull final URL url,
+                                       final String parameterName) {
         final String urlQuery = url.getQuery();
 
         if (urlQuery != null) {
@@ -138,17 +159,21 @@ public final class Utils {
     }
 
     /**
-     * converts a string to a URL-Object.
+     * Convert a string to a {@link URL URL object}.
-     * defaults to HTTP if no protocol is given
+     *
+     * <p>
+     * Defaults to HTTP if no protocol is given.
+     * </p>
      *
      * @param url the string to be converted to a URL-Object
-     * @return a URL-Object containing the url
+     * @return a {@link URL URL object} containing the url
      */
+    @Nonnull
     public static URL stringToURL(final String url) throws MalformedURLException {
         try {
             return new URL(url);
         } catch (final MalformedURLException e) {
-            // if no protocol is given try prepending "https://"
+            // If no protocol is given try prepending "https://"
             if (e.getMessage().equals("no protocol: " + url)) {
                 return new URL(HTTPS + url);
             }
@@ -157,8 +182,8 @@ public final class Utils {
         }
     }
 
-    public static boolean isHTTP(final URL url) {
+    public static boolean isHTTP(@Nonnull final URL url) {
-        // make sure its http or https
+        // Make sure it's HTTP or HTTPS
         final String protocol = url.getProtocol();
         if (!protocol.equals("http") && !protocol.equals("https")) {
             return false;
@@ -180,7 +205,8 @@ public final class Utils {
         return url;
     }
 
-    public static String removeUTF8BOM(final String s) {
+    @Nonnull
+    public static String removeUTF8BOM(@Nonnull final String s) {
         String result = s;
         if (result.startsWith("\uFEFF")) {
             result = result.substring(1);
@@ -191,6 +217,7 @@ public final class Utils {
         return result;
     }
 
+    @Nonnull
     public static String getBaseUrl(final String url) throws ParsingException {
         try {
             final URL uri = stringToURL(url);
@@ -198,7 +225,7 @@ public final class Utils {
         } catch (final MalformedURLException e) {
             final String message = e.getMessage();
             if (message.startsWith("unknown protocol: ")) {
-                // return just the protocol (e.g. vnd.youtube)
+                // Return just the protocol (e.g. vnd.youtube)
                 return message.substring("unknown protocol: ".length());
             }
 
@@ -214,17 +241,16 @@ public final class Utils {
      * @return an url with no Google search redirects
      */
     public static String followGoogleRedirectIfNeeded(final String url) {
-        // if the url is a redirect from a Google search, extract the actual url
+        // If the url is a redirect from a Google search, extract the actual URL
         try {
             final URL decoded = Utils.stringToURL(url);
             if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
-                return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url),
+                return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8);
-                        UTF_8);
             }
         } catch (final Exception ignored) {
         }
 
-        // url is not a google search redirect
+        // URL is not a Google search redirect
         return url;
     }
 
@@ -232,13 +258,31 @@ public final class Utils {
         return str == null || str.isEmpty();
     }
 
-    // can be used for JsonArrays
+    /**
+     * Checks if a collection is null or empty.
+     *
+     * <p>
+     * This method can be also used for {@link com.grack.nanojson.JsonArray JsonArray}s.
+     * </p>
+     *
+     * @param collection the collection on which check if it's null or empty
+     * @return whether the collection is null or empty
+     */
     public static boolean isNullOrEmpty(final Collection<?> collection) {
         return collection == null || collection.isEmpty();
     }
 
-    // can be used for JsonObjects
+    /**
-    public static boolean isNullOrEmpty(final Map<?, ?> map) {
+     * Checks if a {@link Map map} is null or empty.
+     *
+     * <p>
+     * This method can be also used for {@link com.grack.nanojson.JsonObject JsonObject}s.
+     * </p>
+     *
+     * @param map the {@link Map map} on which check if it's null or empty
+     * @return whether the {@link Map map} is null or empty
+     */
+    public static <K, V> boolean isNullOrEmpty(final Map<K, V> map) {
         return map == null || map.isEmpty();
     }
 
@@ -261,8 +305,9 @@ public final class Utils {
         return true;
     }
 
+    @Nonnull
     public static String join(final CharSequence delimiter,
-                              final Iterable<? extends CharSequence> elements) {
+                              @Nonnull final Iterable<? extends CharSequence> elements) {
         final StringBuilder stringBuilder = new StringBuilder();
         final Iterator<? extends CharSequence> iterator = elements.iterator();
         while (iterator.hasNext()) {
@@ -274,11 +319,14 @@ public final class Utils {
         return stringBuilder.toString();
     }
 
-    public static String join(final String delimiter, final String mapJoin,
+    @Nonnull
-                              final Map<? extends CharSequence, ? extends CharSequence> elements) {
+    public static String join(
+            final String delimiter,
+            final String mapJoin,
+            @Nonnull final Map<? extends CharSequence, ? extends CharSequence> elements) {
         final List<String> list = new LinkedList<>();
-        for (final Map.Entry<? extends CharSequence, ? extends CharSequence> entry : elements
+        for (final Map.Entry<? extends CharSequence, ? extends CharSequence> entry
-                .entrySet()) {
+                : elements.entrySet()) {
             list.add(entry.getKey() + mapJoin + entry.getValue());
         }
         return join(delimiter, list);
@@ -287,10 +335,98 @@ public final class Utils {
     /**
      * Concatenate all non-null, non-empty and strings which are not equal to <code>"null"</code>.
      */
+    @Nonnull
     public static String nonEmptyAndNullJoin(final CharSequence delimiter,
                                              final String[] elements) {
-        final List<String> list = new java.util.ArrayList<>(Arrays.asList(elements));
+        final List<String> list = new ArrayList<>(Arrays.asList(elements));
         list.removeIf(s -> isNullOrEmpty(s) || s.equals("null"));
         return join(delimiter, list);
     }
+
+    /**
+     * Find the result of an array of string regular expressions inside an input on the first
+     * group ({@code 0}).
+     *
+     * @param input   the input on which using the regular expressions
+     * @param regexes the string array of regular expressions
+     * @return the result
+     * @throws Parser.RegexException if none of the patterns match the input
+     */
+    @Nonnull
+    public static String getStringResultFromRegexArray(@Nonnull final String input,
+                                                       @Nonnull final String[] regexes)
+            throws Parser.RegexException {
+        return getStringResultFromRegexArray(input, regexes, 0);
+    }
+
+    /**
+     * Find the result of an array of {@link Pattern}s inside an input on the first group
+     * ({@code 0}).
+     *
+     * @param input   the input on which using the regular expressions
+     * @param regexes the {@link Pattern} array
+     * @return the result
+     * @throws Parser.RegexException if none of the patterns match the input
+     */
+    @Nonnull
+    public static String getStringResultFromRegexArray(@Nonnull final String input,
+                                                       @Nonnull final Pattern[] regexes)
+            throws Parser.RegexException {
+        return getStringResultFromRegexArray(input, regexes, 0);
+    }
+
+    /**
+     * Find the result of an array of string regular expressions inside an input on a specific
+     * group.
+     *
+     * @param input   the input on which using the regular expressions
+     * @param regexes the string array of regular expressions
+     * @param group   the group to match
+     * @return the result
+     * @throws Parser.RegexException if none of the patterns match the input, or at least in the
+     * specified group
+     */
+    @Nonnull
+    public static String getStringResultFromRegexArray(@Nonnull final String input,
+                                                       @Nonnull final String[] regexes,
+                                                       final int group)
+            throws Parser.RegexException {
+        return getStringResultFromRegexArray(input,
+                Arrays.stream(regexes)
+                        .filter(Objects::nonNull)
+                        .map(Pattern::compile)
+                        .toArray(Pattern[]::new),
+                group);
+    }
+
+    /**
+     * Find the result of an array of {@link Pattern}s inside an input on a specific
+     * group.
+     *
+     * @param input   the input on which using the regular expressions
+     * @param regexes the {@link Pattern} array
+     * @param group   the group to match
+     * @return the result
+     * @throws Parser.RegexException if none of the patterns match the input, or at least in the
+     * specified group
+     */
+    @Nonnull
+    public static String getStringResultFromRegexArray(@Nonnull final String input,
+                                                       @Nonnull final Pattern[] regexes,
+                                                       final int group)
+            throws Parser.RegexException {
+        for (final Pattern regex : regexes) {
+            try {
+                final String result = Parser.matchGroup(regex, input, group);
+                if (result != null) {
+                    return result;
+                }
+
+                // Continue if the result is null
+            } catch (final Parser.RegexException ignored) {
+            }
+        }
+
+        throw new Parser.RegexException("No regex matched the input on group " + group);
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -1,5 +1,17 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -15,13 +27,6 @@ import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
 
 import java.io.IOException;
-import java.util.Random;
-
-import static org.junit.jupiter.api.Assertions.*;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.DefaultTests.*;
 
 /**
  * Test for {@link ChannelExtractor}
@@ -33,8 +38,7 @@ public class YoutubeChannelExtractorTest {
     public static class NotAvailable {
         @BeforeAll
         public static void setUp() throws IOException {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "notAvailable"));
         }
 
@@ -130,8 +134,7 @@ public class YoutubeChannelExtractorTest {
     public static class NotSupported {
         @BeforeAll
         public static void setUp() throws IOException {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "notSupported"));
         }
 
@@ -149,8 +152,7 @@ public class YoutubeChannelExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "gronkh"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("http://www.youtube.com/user/Gronkh");
@@ -246,8 +248,7 @@ public class YoutubeChannelExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "VSauce"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/user/Vsauce");
@@ -342,8 +343,7 @@ public class YoutubeChannelExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "kurzgesagt"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
@@ -460,8 +460,7 @@ public class YoutubeChannelExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "captainDisillusion"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/user/CaptainDisillusion/videos");
@@ -555,8 +554,7 @@ public class YoutubeChannelExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "random"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w");

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java

@@ -1,5 +1,9 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
+
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.ListExtractor;
@@ -14,11 +18,6 @@ import java.time.temporal.ChronoUnit;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
-
-import static org.junit.jupiter.api.Assertions.fail;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
 
 /**
  * A class that tests multiple channels and ranges of "time ago".
@@ -30,8 +29,7 @@ public class YoutubeChannelLocalizationTest {
 
     @Test
     public void testAllSupportedLocalizations() throws Exception {
-        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeTestsUtils.ensureStateless();
-        YoutubeParsingHelper.setNumberGenerator(new Random(1));
         NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "localization"));
 
         testLocalizationsFor("https://www.youtube.com/user/NBCNews");

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java

@@ -1,5 +1,11 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -17,13 +23,6 @@ import org.schabi.newpipe.extractor.utils.Utils;
 import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
-import java.util.Random;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 public class YoutubeCommentsExtractorTest {
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/comments/";
@@ -38,8 +37,7 @@ public class YoutubeCommentsExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "thomas"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);
@@ -127,8 +125,7 @@ public class YoutubeCommentsExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "empty"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);
@@ -167,8 +164,7 @@ public class YoutubeCommentsExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "hearted"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);
@@ -210,8 +206,7 @@ public class YoutubeCommentsExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "pinned"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);
@@ -252,8 +247,7 @@ public class YoutubeCommentsExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "likes"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);
@@ -284,8 +278,7 @@ public class YoutubeCommentsExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "localized_vote_count"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);
@@ -313,8 +306,7 @@ public class YoutubeCommentsExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "replies"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java

@@ -1,5 +1,12 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -10,14 +17,6 @@ import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
 
 import java.io.IOException;
-import java.util.Random;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
 
 public class YoutubeFeedExtractorTest {
 
@@ -28,8 +27,7 @@ public class YoutubeFeedExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH));
             extractor = (YoutubeFeedExtractor) YouTube
                     .getFeedExtractor("https://www.youtube.com/user/Kurzgesagt");

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java

@@ -1,5 +1,10 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -8,13 +13,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
 
-import java.util.Random;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
-
 public class YoutubeKioskExtractorTest {
 
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/kiosk/";
@@ -24,8 +22,7 @@ public class YoutubeKioskExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "trending"));
             extractor = (YoutubeTrendingExtractor) YouTube.getKioskList().getDefaultKioskExtractor();
             extractor.fetchPage();

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java

@@ -6,9 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
 
 import com.grack.nanojson.JsonWriter;
 
@@ -31,7 +29,6 @@ import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 
 public class YoutubeMixPlaylistExtractorTest {
@@ -47,8 +44,7 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mix"));
             dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -95,7 +91,8 @@ public class YoutubeMixPlaylistExtractorTest {
                     .getBytes(StandardCharsets.UTF_8);
 
             final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
-                    YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+                    YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+                    null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
@@ -142,8 +139,7 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mixWithIndex"));
             dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -186,7 +182,8 @@ public class YoutubeMixPlaylistExtractorTest {
                     .getBytes(StandardCharsets.UTF_8);
 
             final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
-                    YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+                    YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+                    null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
@@ -228,8 +225,7 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "myMix"));
             dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -274,7 +270,8 @@ public class YoutubeMixPlaylistExtractorTest {
                     .getBytes(StandardCharsets.UTF_8);
 
             final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
-                    YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+                    YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+                    null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
@@ -318,8 +315,7 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @BeforeAll
         public static void setUp() throws IOException {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "invalid"));
             dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
         }
@@ -353,8 +349,7 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channelMix"));
             dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -395,7 +390,8 @@ public class YoutubeMixPlaylistExtractorTest {
                     .getBytes(StandardCharsets.UTF_8);
 
             final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
-                    YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+                    YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+                    null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
@@ -417,9 +413,8 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "genreMix"));
-            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "genreMix"));
             dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                     .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java

@@ -1,5 +1,8 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -8,10 +11,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
 import java.io.IOException;
-import java.util.Random;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class YoutubeParsingHelperTest {
 
@@ -19,8 +18,7 @@ public class YoutubeParsingHelperTest {
 
     @BeforeAll
     public static void setUp() throws IOException {
-        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeTestsUtils.ensureStateless();
-        YoutubeParsingHelper.setNumberGenerator(new Random(1));
         NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "youtubeParsingHelper"));
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -1,5 +1,18 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+import static org.schabi.newpipe.extractor.ListExtractor.ITEM_COUNT_UNKNOWN;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -16,20 +29,6 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistE
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 import java.io.IOException;
-import java.util.Random;
-
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
-import static org.schabi.newpipe.extractor.ListExtractor.ITEM_COUNT_UNKNOWN;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
 
 /**
  * Test for {@link YoutubePlaylistExtractor}
@@ -41,8 +40,7 @@ public class YoutubePlaylistExtractorTest {
     public static class NotAvailable {
         @BeforeAll
         public static void setUp() throws IOException {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "notAvailable"));
         }
 
@@ -66,8 +64,7 @@ public class YoutubePlaylistExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "TimelessPopHits"));
             extractor = (YoutubePlaylistExtractor) YouTube
                     .getPlaylistExtractor("http://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj");
@@ -174,8 +171,7 @@ public class YoutubePlaylistExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "huge"));
             extractor = (YoutubePlaylistExtractor) YouTube
                     .getPlaylistExtractor("https://www.youtube.com/watch?v=8SbUC-UaAxE&list=PLWwAypAcFRgKAIIFqBr9oy-ZYZnixa_Fj");
@@ -298,8 +294,7 @@ public class YoutubePlaylistExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "learning"));
             extractor = (YoutubePlaylistExtractor) YouTube
                     .getPlaylistExtractor("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8");
@@ -407,8 +402,7 @@ public class YoutubePlaylistExtractorTest {
 
         @BeforeAll
         public static void setUp() throws IOException {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "continuations"));
         }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java

@@ -20,6 +20,9 @@ package org.schabi.newpipe.extractor.services.youtube;
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -29,10 +32,6 @@ import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 
 import java.io.IOException;
-import java.util.Random;
-
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 /**
  * Test for {@link SuggestionExtractor}
@@ -45,8 +44,7 @@ public class YoutubeSuggestionExtractorTest {
 
     @BeforeAll
     public static void setUp() throws Exception {
-        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeTestsUtils.ensureStateless();
-        YoutubeParsingHelper.setNumberGenerator(new Random(1));
         NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + ""), new Localization("de", "DE"));
         suggestionExtractor = YouTube.getSuggestionExtractor();
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java

@@ -0,0 +1,28 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
+
+import java.util.Random;
+
+/**
+ * Utility class for keeping YouTube tests stateless.
+ */
+public final class YoutubeTestsUtils {
+    private YoutubeTestsUtils() {
+        // No impl
+    }
+
+    /**
+     * Clears static YT states.
+     *
+     * <p>
+     * This method needs to be called to generate all mocks of a test when running different tests
+     * at the same time.
+     * </p>
+     */
+    public static void ensureStateless() {
+        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeParsingHelper.setNumberGenerator(new Random(1));
+        YoutubeStreamExtractor.resetDeobfuscationCode();
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java

@@ -20,6 +20,10 @@ package org.schabi.newpipe.extractor.services.youtube;
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -28,12 +32,6 @@ import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.kiosk.KioskInfo;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 
-import java.util.Random;
-
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-
 /**
  * Test for {@link KioskInfo}
  */
@@ -46,8 +44,7 @@ public class YoutubeTrendingKioskInfoTest {
     @BeforeAll
     public static void setUp()
             throws Exception {
-        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeTestsUtils.ensureStateless();
-        YoutubeParsingHelper.setNumberGenerator(new Random(1));
         NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH));
         LinkHandlerFactory LinkHandlerFactory = ((StreamingService) YouTube).getKioskList().getListLinkHandlerFactoryByType("Trending");
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java

@@ -25,7 +25,7 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
@@ -35,7 +35,6 @@ import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Random;
 
 import javax.annotation.Nullable;
 
@@ -49,8 +48,7 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "all"));
             extractor = YouTube.getSearchExtractor(QUERY);
             extractor.fetchPage();
@@ -72,8 +70,7 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channel"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(CHANNELS), "");
             extractor.fetchPage();
@@ -97,8 +94,7 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "playlist"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(PLAYLISTS), "");
             extractor.fetchPage();
@@ -122,8 +118,7 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "videos"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
             extractor.fetchPage();
@@ -149,8 +144,7 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "suggestions"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
             extractor.fetchPage();
@@ -174,8 +168,7 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "corrected"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
             extractor.fetchPage();
@@ -199,8 +192,7 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "random"));
             extractor = YouTube.getSearchExtractor(QUERY);
             extractor.fetchPage();
@@ -235,8 +227,7 @@ public class YoutubeSearchExtractorTest {
     public static class PagingTest {
         @Test
         public void duplicatedItemsCheck() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "paging"));
             final SearchExtractor extractor = YouTube.getSearchExtractor("cirque du soleil", singletonList(VIDEOS), "");
             extractor.fetchPage();
@@ -254,8 +245,7 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "metaInfo"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
             extractor.fetchPage();
@@ -291,8 +281,7 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "verified"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(CHANNELS), "");
             extractor.fetchPage();
@@ -329,8 +318,7 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "video_uploader_avatar"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
             extractor.fetchPage();
@@ -361,8 +349,7 @@ public class YoutubeSearchExtractorTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "video_description"));
             extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
             extractor.fetchPage();

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -1,23 +1,21 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
 import java.util.Collections;
 import java.util.List;
-import java.util.Random;
 
 import javax.annotation.Nullable;
 
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-
 public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtractorTest {
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
     private static final String ID = "rwcfPqbAx-0";
@@ -27,9 +25,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
 
     @BeforeAll
     public static void setUp() throws Exception {
-        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeTestsUtils.ensureStateless();
-        YoutubeParsingHelper.setNumberGenerator(new Random(1));
-        YoutubeStreamExtractor.resetDeobfuscationCode();
         NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "ageRestricted"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -1,24 +1,22 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
 import java.util.Arrays;
 import java.util.List;
-import java.util.Random;
 
 import javax.annotation.Nullable;
 
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-
 /**
  * Test for {@link YoutubeStreamLinkHandlerFactory}
  */
@@ -31,9 +29,7 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
 
     @BeforeAll
     public static void setUp() throws Exception {
-        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeTestsUtils.ensureStateless();
-        YoutubeParsingHelper.setNumberGenerator(new Random(1));
-        YoutubeStreamExtractor.resetDeobfuscationCode();
         NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "controversial"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java

@@ -1,5 +1,31 @@
+/*
+ * Created by Christian Schabesberger on 30.12.15.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * YoutubeVideoExtractorDefault.java is part of NewPipe Extractor.
+ *
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe Extractor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -15,9 +41,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
 import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
-import org.schabi.newpipe.extractor.stream.*;
+import org.schabi.newpipe.extractor.stream.Description;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.stream.StreamSegment;
+import org.schabi.newpipe.extractor.stream.StreamType;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
@@ -25,35 +54,9 @@ import java.net.URL;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.Random;
 
 import javax.annotation.Nullable;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
-
-/*
- * Created by Christian Schabesberger on 30.12.15.
- *
- * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
- * YoutubeVideoExtractorDefault.java is part of NewPipe.
- *
- * NewPipe is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * NewPipe is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
- */
 public class YoutubeStreamExtractorDefaultTest {
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
     static final String BASE_URL = "https://www.youtube.com/watch?v=";
@@ -62,9 +65,7 @@ public class YoutubeStreamExtractorDefaultTest {
     public static class NotAvailable {
         @BeforeAll
         public static void setUp() throws IOException {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
-            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "notAvailable"));
         }
 
@@ -119,9 +120,7 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
-            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "pewdiwpie"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
@@ -164,9 +163,7 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
-            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "unboxing"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
@@ -219,9 +216,7 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
-            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "ratingsDisabled"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
@@ -257,9 +252,7 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
-            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "streamSegmentsTagesschau"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
@@ -294,7 +287,7 @@ public class YoutubeStreamExtractorDefaultTest {
         // @formatter:on
 
         @Test
-        public void testStreamSegment0() throws Exception {
+        void testStreamSegment0() throws Exception {
             final StreamSegment segment = extractor.getStreamSegments().get(0);
             assertEquals(0, segment.getStartTimeSeconds());
             assertEquals("Guten Abend", segment.getTitle());
@@ -303,7 +296,7 @@ public class YoutubeStreamExtractorDefaultTest {
         }
 
         @Test
-        public void testStreamSegment3() throws Exception {
+        void testStreamSegment3() throws Exception {
             final StreamSegment segment = extractor.getStreamSegments().get(3);
             assertEquals(224, segment.getStartTimeSeconds());
             assertEquals("Pandemie dämpft Konjunkturprognose für 2021", segment.getTitle());
@@ -320,9 +313,7 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
-            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "streamSegmentsMaiLab"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
@@ -363,7 +354,7 @@ public class YoutubeStreamExtractorDefaultTest {
         // @formatter:on
 
         @Test
-        public void testStreamSegment() throws Exception {
+        void testStreamSegment() throws Exception {
             final StreamSegment segment = extractor.getStreamSegments().get(1);
             assertEquals(164, segment.getStartTimeSeconds());
             assertEquals("Was ist Vitamin D?", segment.getTitle());
@@ -390,11 +381,9 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeTestsUtils.ensureStateless();
-            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "publicBroadcast"));
             extractor = YouTube.getStreamExtractor(URL);
-            YoutubeStreamExtractor.resetDeobfuscationCode();
             extractor.fetchPage();
         }
 
@@ -445,7 +434,7 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeStreamExtractor.resetDeobfuscationCode();
+            YoutubeTestsUtils.ensureStateless();
             NewPipe.init(DownloaderTestImpl.getInstance());
             extractor = (YoutubeStreamExtractor) YouTube
                     .getStreamExtractor("https://www.youtube.com/watch?v=tjz2u2DiveM");
@@ -453,7 +442,7 @@ public class YoutubeStreamExtractorDefaultTest {
         }
 
         @Test
-        public void testGetUnlisted() {
+        void testGetUnlisted() {
             assertEquals(StreamExtractor.Privacy.UNLISTED, extractor.getPrivacy());
         }
     }
@@ -465,14 +454,14 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @BeforeAll
         public static void setUp() throws Exception {
-            YoutubeStreamExtractor.resetDeobfuscationCode();
+            YoutubeTestsUtils.ensureStateless();
             NewPipe.init(DownloaderTestImpl.getInstance());
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
         }
 
         @Test
-        public void testGetLicence() throws ParsingException {
+        void testGetLicence() throws ParsingException {
             assertEquals("Creative Commons Attribution licence (reuse allowed)", extractor.getLicence());
         }
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java

@@ -1,24 +1,22 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
 import java.util.Arrays;
 import java.util.List;
-import java.util.Random;
 
 import javax.annotation.Nullable;
 
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-
 public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractorTest {
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
     private static final String ID = "5qap5aO4i9A";
@@ -28,9 +26,7 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
 
     @BeforeAll
     public static void setUp() throws Exception {
-        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeTestsUtils.ensureStateless();
-        YoutubeParsingHelper.setNumberGenerator(new Random(1));
-        YoutubeStreamExtractor.resetDeobfuscationCode();
         NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "live"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java

@@ -9,24 +9,22 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 import static org.schabi.newpipe.extractor.services.youtube.stream.YoutubeStreamExtractorDefaultTest.YOUTUBE_LICENCE;
 
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
-import org.schabi.newpipe.downloader.MockOnly;
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfo.PlaylistType;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
-import java.util.Random;
 import java.util.stream.Collectors;
 
 import javax.annotation.Nullable;
@@ -40,9 +38,7 @@ public class YoutubeStreamExtractorRelatedMixTest extends DefaultStreamExtractor
 
     @BeforeAll
     public static void setUp() throws Exception {
-        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeTestsUtils.ensureStateless();
-        YoutubeParsingHelper.setNumberGenerator(new Random(1));
-        YoutubeStreamExtractor.resetDeobfuscationCode();
         NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "relatedMix"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();
@@ -88,7 +84,7 @@ public class YoutubeStreamExtractorRelatedMixTest extends DefaultStreamExtractor
     // @formatter:on
 
     @Test
-    @MockOnly("related items keep changing, and so do the mixes contained within them")
+    @Disabled("Mixes are not available in related items anymore, see https://github.com/TeamNewPipe/NewPipeExtractor/issues/820")
     @Override
     public void testRelatedItems() throws Exception {
         super.testRelatedItems();

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java

@@ -1,24 +1,22 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.stream.StreamExtractor.Privacy.UNLISTED;
+
 import org.junit.jupiter.api.BeforeAll;
 import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
 import java.util.Arrays;
 import java.util.List;
-import java.util.Random;
 
 import javax.annotation.Nullable;
 
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.stream.StreamExtractor.Privacy.UNLISTED;
-
 public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTest {
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
     static final String ID = "udsB8KnIJTg";
@@ -27,9 +25,7 @@ public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTe
 
     @BeforeAll
     public static void setUp() throws Exception {
-        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeTestsUtils.ensureStateless();
-        YoutubeParsingHelper.setNumberGenerator(new Random(1));
-        YoutubeStreamExtractor.resetDeobfuscationCode();
         NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "unlisted"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_1.json

@@ -1,11 +1,8 @@
 {
   "request": {
     "httpMethod": "POST",
-    "url": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
+    "url": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\u0026prettyPrint\u003dfalse",
     "headers": {
-      "Accept-Language": [
-        "en-GB, en;q\u003d0.9"
-      ],
       "Origin": [
         "https://www.youtube.com"
       ],
@@ -16,10 +13,13 @@
         "https://www.youtube.com"
       ],
       "X-YouTube-Client-Version": [
-        "2.20210728.00.00"
+        "2.20220325.00.00"
       ],
       "Content-Type": [
         "application/json"
+      ],
+      "Accept-Language": [
+        "en-GB, en;q\u003d0.9"
       ]
     },
     "dataToSend": [
@@ -36,6 +36,61 @@
       58,
       123,
       34,
+      114,
+      101,
+      113,
+      117,
+      101,
+      115,
+      116,
+      34,
+      58,
+      123,
+      34,
+      105,
+      110,
+      116,
+      101,
+      114,
+      110,
+      97,
+      108,
+      69,
+      120,
+      112,
+      101,
+      114,
+      105,
+      109,
+      101,
+      110,
+      116,
+      70,
+      108,
+      97,
+      103,
+      115,
+      34,
+      58,
+      91,
+      93,
+      44,
+      34,
+      117,
+      115,
+      101,
+      83,
+      115,
+      108,
+      34,
+      58,
+      116,
+      114,
+      117,
+      101,
+      125,
+      44,
+      34,
       99,
       108,
       105,
@@ -88,6 +143,46 @@
       34,
       44,
       34,
+      111,
+      114,
+      105,
+      103,
+      105,
+      110,
+      97,
+      108,
+      85,
+      114,
+      108,
+      34,
+      58,
+      34,
+      104,
+      116,
+      116,
+      112,
+      115,
+      58,
+      47,
+      47,
+      119,
+      119,
+      119,
+      46,
+      121,
+      111,
+      117,
+      116,
+      117,
+      98,
+      101,
+      46,
+      99,
+      111,
+      109,
+      34,
+      44,
+      34,
       99,
       108,
       105,
@@ -109,11 +204,11 @@
       50,
       48,
       50,
-      49,
-      48,
-      55,
       50,
-      56,
+      48,
+      51,
+      50,
+      53,
       46,
       48,
       48,
@@ -121,6 +216,27 @@
       48,
       48,
       34,
+      44,
+      34,
+      112,
+      108,
+      97,
+      116,
+      102,
+      111,
+      114,
+      109,
+      34,
+      58,
+      34,
+      68,
+      69,
+      83,
+      75,
+      84,
+      79,
+      80,
+      34,
       125,
       44,
       34,
@@ -222,10 +338,10 @@
         "application/json; charset\u003dUTF-8"
       ],
       "date": [
-        "Thu, 17 Mar 2022 13:52:23 GMT"
+        "Sun, 27 Mar 2022 19:33:37 GMT"
       ],
       "expires": [
-        "Thu, 17 Mar 2022 13:52:23 GMT"
+        "Sun, 27 Mar 2022 19:33:37 GMT"
       ],
       "p3p": [
         "CP\u003d\"This is not a P3P policy! See g.co/p3phelp for more info.\""
@@ -234,7 +350,7 @@
         "scaffolding on HTTPServer2"
       ],
       "set-cookie": [
-        "CONSENT\u003dPENDING+046; expires\u003dSat, 16-Mar-2024 13:52:23 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "CONSENT\u003dPENDING+165; expires\u003dTue, 26-Mar-2024 19:33:37 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "vary": [
         "Origin",
@@ -251,7 +367,7 @@
         "0"
       ]
     },
-    "responseBody": "{\n  \"responseContext\": {\n    \"visitorData\": \"CgtuS0lURlJ0dTBubyiX-syRBg%3D%3D\",\n    \"serviceTrackingParams\": [\n      {\n        \"service\": \"CSI\",\n        \"params\": [\n          {\n            \"key\": \"c\",\n            \"value\": \"WEB\"\n          },\n          {\n            \"key\": \"cver\",\n            \"value\": \"2.20210728.00.00\"\n          },\n          {\n            \"key\": \"yt_li\",\n            \"value\": \"0\"\n          },\n          {\n            \"key\": \"ResolveUrl_rid\",\n            \"value\": \"0x233c2bae5fb15af0\"\n          }\n        ]\n      },\n      {\n        \"service\": \"GFEEDBACK\",\n        \"params\": [\n          {\n            \"key\": \"logged_in\",\n            \"value\": \"0\"\n          },\n          {\n            \"key\": \"e\",\n            \"value\": \"24123058,24152443,23943577,24145515,23934970,24007790,24167385,23998056,23986025,24036947,24138064,24180220,24180069,24177193,24180014,23966208,24138442,24140247,24158010,24179789,24164187,24176755,24077266,24007246,24165479,24135310,23918597,23946420,23804281,24004644,24077241,24161848,23983296,1714257,24080738,24002022,24141412,23744176,24148484,24165080,24181588,24166867,24045475,24175488,24002025,24082662,24154616,24181308,24166201,24110902,23882685,24027707,24053419,24144329,24062268,24108219,24085811,24120819,24106839,39321475,24045476,24001373,24174214,24175889,24180089,24034168,24169726\"\n          }\n        ]\n      },\n      {\n        \"service\": \"GUIDED_HELP\",\n        \"params\": [\n          {\n            \"key\": \"logged_in\",\n            \"value\": \"0\"\n          }\n        ]\n      },\n      {\n        \"service\": \"ECATCHER\",\n        \"params\": [\n          {\n            \"key\": \"client.version\",\n            \"value\": \"2.20211103\"\n          },\n          {\n            \"key\": \"client.name\",\n            \"value\": \"WEB\"\n          },\n          {\n            \"key\": \"client.fexp\",\n            \"value\": \"24123058,24152443,23943577,24145515,23934970,24007790,24167385,23998056,23986025,24036947,24138064,24180220,24180069,24177193,24180014,23966208,24138442,24140247,24158010,24179789,24164187,24176755,24077266,24007246,24165479,24135310,23918597,23946420,23804281,24004644,24077241,24161848,23983296,1714257,24080738,24002022,24141412,23744176,24148484,24165080,24181588,24166867,24045475,24175488,24002025,24082662,24154616,24181308,24166201,24110902,23882685,24027707,24053419,24144329,24062268,24108219,24085811,24120819,24106839,39321475,24045476,24001373,24174214,24175889,24180089,24034168,24169726\"\n          }\n        ]\n      }\n    ],\n    \"mainAppWebResponseContext\": {\n      \"loggedOut\": true\n    },\n    \"webResponseContextExtensionData\": {\n      \"hasDecorated\": true\n    }\n  },\n  \"endpoint\": {\n    \"clickTrackingParams\": \"IhMIs_K1t6XN9gIVNOYRCB1JBgjjMghleHRlcm5hbA\u003d\u003d\",\n    \"commandMetadata\": {\n      \"webCommandMetadata\": {\n        \"url\": \"/youtubei/v1/navigation/resolve_url\",\n        \"webPageType\": \"WEB_PAGE_TYPE_CHANNEL\",\n        \"rootVe\": 3611,\n        \"apiUrl\": \"/youtubei/v1/browse\"\n      },\n      \"resolveUrlCommandMetadata\": {\n        \"isVanityUrl\": true\n      }\n    },\n    \"browseEndpoint\": {\n      \"browseId\": \"UC6nSFpj9HTCZ5t-N3Rm3-HA\",\n      \"params\": \"EgC4AQA%3D\"\n    }\n  }\n}\n",
+    "responseBody": "{\"responseContext\":{\"visitorData\":\"CgtELW0wN1pqQlZHcyiR-IKSBg%3D%3D\",\"serviceTrackingParams\":[{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20220325.00.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"ResolveUrl_rid\",\"value\":\"0xeb6dd9911efcb72c\"}]},{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"24148483,24004644,23946420,24007246,24161848,24036948,24077241,24177978,23744176,24186125,24145515,23998056,24175488,23918597,24062267,24135310,24180015,23804281,24120820,23966208,24138442,24140247,24185349,24147969,1714240,24154616,24034168,24077266,24185065,23983296,24180089,24174214,24001373,24045469,24152442,24141412,24181361,9405994,23934970,24177193,23940248,24165080,23882503,24085811,24082661,24002022,23986023,24169726,24187043,24166867,24080738,24045470,24183272,24110902,24106839,24179789,24176755,24002025,24180070,39321475,24138064,24189899\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20220325\"},{\"key\":\"client.name\",\"value\":\"WEB\"},{\"key\":\"client.fexp\",\"value\":\"24148483,24004644,23946420,24007246,24161848,24036948,24077241,24177978,23744176,24186125,24145515,23998056,24175488,23918597,24062267,24135310,24180015,23804281,24120820,23966208,24138442,24140247,24185349,24147969,1714240,24154616,24034168,24077266,24185065,23983296,24180089,24174214,24001373,24045469,24152442,24141412,24181361,9405994,23934970,24177193,23940248,24165080,23882503,24085811,24082661,24002022,23986023,24169726,24187043,24166867,24080738,24045470,24183272,24110902,24106839,24179789,24176755,24002025,24180070,39321475,24138064,24189899\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"endpoint\":{\"clickTrackingParams\":\"IhMIo-zbrYTn9gIVNjvxBR1VVgfnMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/youtubei/v1/navigation/resolve_url\",\"webPageType\":\"WEB_PAGE_TYPE_CHANNEL\",\"rootVe\":3611,\"apiUrl\":\"/youtubei/v1/browse\"},\"resolveUrlCommandMetadata\":{\"isVanityUrl\":true}},\"browseEndpoint\":{\"browseId\":\"UC6nSFpj9HTCZ5t-N3Rm3-HA\",\"params\":\"EgC4AQA%3D\"}}}",
-    "latestUrl": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
+    "latestUrl": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\u0026prettyPrint\u003dfalse"
   }
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/captainDisillusion/generated_mock_1.json

@@ -1,7 +1,7 @@
 {
   "request": {
     "httpMethod": "POST",
-    "url": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
+    "url": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\u0026prettyPrint\u003dfalse",
     "headers": {
       "Accept-Language": [
         "en-GB, en;q\u003d0.9"
@@ -16,7 +16,7 @@
         "https://www.youtube.com"
       ],
       "X-YouTube-Client-Version": [
-        "2.20210728.00.00"
+        "2.20220315.01.00"
       ],
       "Content-Type": [
         "application/json"
@@ -36,6 +36,61 @@
       58,
       123,
       34,
+      114,
+      101,
+      113,
+      117,
+      101,
+      115,
+      116,
+      34,
+      58,
+      123,
+      34,
+      105,
+      110,
+      116,
+      101,
+      114,
+      110,
+      97,
+      108,
+      69,
+      120,
+      112,
+      101,
+      114,
+      105,
+      109,
+      101,
+      110,
+      116,
+      70,
+      108,
+      97,
+      103,
+      115,
+      34,
+      58,
+      91,
+      93,
+      44,
+      34,
+      117,
+      115,
+      101,
+      83,
+      115,
+      108,
+      34,
+      58,
+      116,
+      114,
+      117,
+      101,
+      125,
+      44,
+      34,
       99,
       108,
       105,
@@ -88,6 +143,46 @@
       34,
       44,
       34,
+      111,
+      114,
+      105,
+      103,
+      105,
+      110,
+      97,
+      108,
+      85,
+      114,
+      108,
+      34,
+      58,
+      34,
+      104,
+      116,
+      116,
+      112,
+      115,
+      58,
+      47,
+      47,
+      119,
+      119,
+      119,
+      46,
+      121,
+      111,
+      117,
+      116,
+      117,
+      98,
+      101,
+      46,
+      99,
+      111,
+      109,
+      34,
+      44,
+      34,
       99,
       108,
       105,
@@ -109,18 +204,39 @@
       50,
       48,
       50,
-      49,
-      48,
-      55,
       50,
-      56,
+      48,
-      46,
+      51,
-      48,
+      49,
-      48,
+      53,
+      46,
+      48,
+      49,
       46,
       48,
       48,
       34,
+      44,
+      34,
+      112,
+      108,
+      97,
+      116,
+      102,
+      111,
+      114,
+      109,
+      34,
+      58,
+      34,
+      68,
+      69,
+      83,
+      75,
+      84,
+      79,
+      80,
+      34,
       125,
       44,
       34,
@@ -225,7 +341,7 @@
     "responseMessage": "",
     "responseHeaders": {
       "alt-svc": [
-        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
       ],
       "cache-control": [
         "private"
@@ -234,19 +350,19 @@
         "application/json; charset\u003dUTF-8"
       ],
       "date": [
-        "Fri, 30 Jul 2021 17:15:09 GMT"
+        "Tue, 15 Mar 2022 19:12:38 GMT"
       ],
       "expires": [
-        "Fri, 30 Jul 2021 17:15:09 GMT"
+        "Tue, 15 Mar 2022 19:12:38 GMT"
       ],
       "p3p": [
         "CP\u003d\"This is not a P3P policy! See g.co/p3phelp for more info.\""
       ],
       "server": [
-        "ESF"
+        "scaffolding on HTTPServer2"
       ],
       "set-cookie": [
-        "CONSENT\u003dPENDING+106; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "CONSENT\u003dPENDING+708; expires\u003dThu, 14-Mar-2024 19:12:38 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "vary": [
         "Origin",
@@ -263,7 +379,7 @@
         "0"
       ]
     },
-    "responseBody": "{\n  \"responseContext\": {\n    \"visitorData\": \"Cgs5bFRGcWFiLXJsYyid55CIBg%3D%3D\",\n    \"serviceTrackingParams\": [\n      {\n        \"service\": \"CSI\",\n        \"params\": [\n          {\n            \"key\": \"c\",\n            \"value\": \"WEB\"\n          },\n          {\n            \"key\": \"cver\",\n            \"value\": \"2.20210728.00.00\"\n          },\n          {\n            \"key\": \"yt_li\",\n            \"value\": \"0\"\n          },\n          {\n            \"key\": \"ResolveUrl_rid\",\n            \"value\": \"0x2c1228855582a8e5\"\n          }\n        ]\n      },\n      {\n        \"service\": \"GFEEDBACK\",\n        \"params\": [\n          {\n            \"key\": \"logged_in\",\n            \"value\": \"0\"\n          },\n          {\n            \"key\": \"e\",\n            \"value\": \"23918597,24070036,24068842,24056264,24045411,24002025,24059521,24050503,24049820,23996512,24044723,24028143,23968386,23987907,24016284,24049573,23944779,24053866,23857949,24058812,23983813,24080913,23882685,23974595,24049571,24058380,23986027,23843507,23744176,24002922,1714254,24036236,24007246,24004644,24043240,24036947,23998056,24060921,24030040,23804281,23891344,23966208,24070943,23946420,24076876,24001373,23983296,24057238,23891346,23884386,23996830,24056274,24002022,24079272,23934970,24058128,24077274,24077957,24042870,24037794\"\n          }\n        ]\n      },\n      {\n        \"service\": \"GUIDED_HELP\",\n        \"params\": [\n          {\n            \"key\": \"logged_in\",\n            \"value\": \"0\"\n          }\n        ]\n      },\n      {\n        \"service\": \"ECATCHER\",\n        \"params\": [\n          {\n            \"key\": \"client.version\",\n            \"value\": \"2.20210728\"\n          },\n          {\n            \"key\": \"client.name\",\n            \"value\": \"WEB\"\n          }\n        ]\n      }\n    ],\n    \"mainAppWebResponseContext\": {\n      \"loggedOut\": true\n    },\n    \"webResponseContextExtensionData\": {\n      \"hasDecorated\": true\n    }\n  },\n  \"endpoint\": {\n    \"clickTrackingParams\": \"IhMI7uO75KWL8gIVQhXxBR1KjgvfMghleHRlcm5hbA\u003d\u003d\",\n    \"commandMetadata\": {\n      \"webCommandMetadata\": {\n        \"url\": \"/youtubei/v1/navigation/resolve_url\",\n        \"webPageType\": \"WEB_PAGE_TYPE_CHANNEL\",\n        \"rootVe\": 3611,\n        \"apiUrl\": \"/youtubei/v1/browse\"\n      },\n      \"resolveUrlCommandMetadata\": {\n        \"isVanityUrl\": true\n      }\n    },\n    \"browseEndpoint\": {\n      \"browseId\": \"UCEOXxzW2vU0P-0THehuIIeg\",\n      \"params\": \"EgC4AQA%3D\"\n    }\n  }\n}\n",
+    "responseBody": "{\"responseContext\":{\"visitorData\":\"CgtsRmxxcVhCdUJMMCimysORBg%3D%3D\",\"serviceTrackingParams\":[{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20220315.01.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"ResolveUrl_rid\",\"value\":\"0x9dff30845b199ee1\"}]},{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"23966208,24001373,24152442,24161848,24120829,23986026,24062268,24154616,24106839,24077266,24138064,24182586,23940247,24176755,24179789,1714259,23744176,24174213,24082662,24002025,23998056,24036947,23918597,24175488,24007790,24002022,24135310,24080738,24148482,24139173,24145515,23858057,23882502,24177193,24034168,24110902,24165080,23946420,23804281,24180089,23934970,24077241,24166867,24140247,24004644,24138442,24169726,24120819,23983296,24085811,39321475,24182870,24007246\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20220315\"},{\"key\":\"client.name\",\"value\":\"WEB\"},{\"key\":\"client.fexp\",\"value\":\"23966208,24001373,24152442,24161848,24120829,23986026,24062268,24154616,24106839,24077266,24138064,24182586,23940247,24176755,24179789,1714259,23744176,24174213,24082662,24002025,23998056,24036947,23918597,24175488,24007790,24002022,24135310,24080738,24148482,24139173,24145515,23858057,23882502,24177193,24034168,24110902,24165080,23946420,23804281,24180089,23934970,24077241,24166867,24140247,24004644,24138442,24169726,24120819,23983296,24085811,39321475,24182870,24007246\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"endpoint\":{\"clickTrackingParams\":\"IhMIwp_CpOnI9gIVWCvxBR3T2gynMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/youtubei/v1/navigation/resolve_url\",\"webPageType\":\"WEB_PAGE_TYPE_CHANNEL\",\"rootVe\":3611,\"apiUrl\":\"/youtubei/v1/browse\"},\"resolveUrlCommandMetadata\":{\"isVanityUrl\":true}},\"browseEndpoint\":{\"browseId\":\"UCEOXxzW2vU0P-0THehuIIeg\",\"params\":\"EgC4AQA%3D\"}}}",
-    "latestUrl": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
+    "latestUrl": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\u0026prettyPrint\u003dfalse"
   }
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/gronkh/generated_mock_1.json

@@ -1,7 +1,7 @@
 {
   "request": {
     "httpMethod": "POST",
-    "url": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
+    "url": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\u0026prettyPrint\u003dfalse",
     "headers": {
       "Accept-Language": [
         "en-GB, en;q\u003d0.9"
@@ -16,7 +16,7 @@
         "https://www.youtube.com"
       ],
       "X-YouTube-Client-Version": [
-        "2.20210728.00.00"
+        "2.20220315.01.00"
       ],
       "Content-Type": [
         "application/json"
@@ -36,6 +36,61 @@
       58,
       123,
       34,
+      114,
+      101,
+      113,
+      117,
+      101,
+      115,
+      116,
+      34,
+      58,
+      123,
+      34,
+      105,
+      110,
+      116,
+      101,
+      114,
+      110,
+      97,
+      108,
+      69,
+      120,
+      112,
+      101,
+      114,
+      105,
+      109,
+      101,
+      110,
+      116,
+      70,
+      108,
+      97,
+      103,
+      115,
+      34,
+      58,
+      91,
+      93,
+      44,
+      34,
+      117,
+      115,
+      101,
+      83,
+      115,
+      108,
+      34,
+      58,
+      116,
+      114,
+      117,
+      101,
+      125,
+      44,
+      34,
       99,
       108,
       105,
@@ -88,6 +143,46 @@
       34,
       44,
       34,
+      111,
+      114,
+      105,
+      103,
+      105,
+      110,
+      97,
+      108,
+      85,
+      114,
+      108,
+      34,
+      58,
+      34,
+      104,
+      116,
+      116,
+      112,
+      115,
+      58,
+      47,
+      47,
+      119,
+      119,
+      119,
+      46,
+      121,
+      111,
+      117,
+      116,
+      117,
+      98,
+      101,
+      46,
+      99,
+      111,
+      109,
+      34,
+      44,
+      34,
       99,
       108,
       105,
@@ -109,18 +204,39 @@
       50,
       48,
       50,
-      49,
-      48,
-      55,
       50,
-      56,
+      48,
-      46,
+      51,
-      48,
+      49,
-      48,
+      53,
+      46,
+      48,
+      49,
       46,
       48,
       48,
       34,
+      44,
+      34,
+      112,
+      108,
+      97,
+      116,
+      102,
+      111,
+      114,
+      109,
+      34,
+      58,
+      34,
+      68,
+      69,
+      83,
+      75,
+      84,
+      79,
+      80,
+      34,
       125,
       44,
       34,
@@ -213,7 +329,7 @@
     "responseMessage": "",
     "responseHeaders": {
       "alt-svc": [
-        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
       ],
       "cache-control": [
         "private"
@@ -222,10 +338,10 @@
         "application/json; charset\u003dUTF-8"
       ],
       "date": [
-        "Fri, 30 Jul 2021 17:14:06 GMT"
+        "Tue, 15 Mar 2022 19:12:39 GMT"
       ],
       "expires": [
-        "Fri, 30 Jul 2021 17:14:06 GMT"
+        "Tue, 15 Mar 2022 19:12:39 GMT"
       ],
       "p3p": [
         "CP\u003d\"This is not a P3P policy! See g.co/p3phelp for more info.\""
@@ -234,7 +350,7 @@
         "ESF"
       ],
       "set-cookie": [
-        "CONSENT\u003dPENDING+790; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "CONSENT\u003dPENDING+303; expires\u003dThu, 14-Mar-2024 19:12:39 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "vary": [
         "Origin",
@@ -251,7 +367,7 @@
         "0"
       ]
     },
-    "responseBody": "{\n  \"responseContext\": {\n    \"visitorData\": \"CgtpZENTV1kxN2NBTSje5pCIBg%3D%3D\",\n    \"serviceTrackingParams\": [\n      {\n        \"service\": \"CSI\",\n        \"params\": [\n          {\n            \"key\": \"c\",\n            \"value\": \"WEB\"\n          },\n          {\n            \"key\": \"cver\",\n            \"value\": \"2.20210728.00.00\"\n          },\n          {\n            \"key\": \"yt_li\",\n            \"value\": \"0\"\n          },\n          {\n            \"key\": \"ResolveUrl_rid\",\n            \"value\": \"0xbdf69059f69c2534\"\n          }\n        ]\n      },\n      {\n        \"service\": \"GFEEDBACK\",\n        \"params\": [\n          {\n            \"key\": \"logged_in\",\n            \"value\": \"0\"\n          },\n          {\n            \"key\": \"e\",\n            \"value\": \"24076879,24028143,23882503,24049573,23934970,24066753,23744176,24047617,24053866,23996830,23974595,24065803,24002923,24064474,24002022,24049571,24056264,23877025,23857950,24064095,23886243,24059521,23918597,24001373,24071949,23968386,23998056,24058380,24049820,24070035,24050503,24070258,24075143,24016724,24007246,24058812,23983813,24004644,24045411,24036237,23804281,24042870,24037794,23944779,23946420,24060921,24016284,1714259,23884386,23990877,23986031,24030040,24037583,23891346,24022464,24075684,23891344,39321255,24068842,24070942,24036947,24067274,23966208,24058128,24056274,24002025,23983296,23996512,24057238,24043240\"\n          }\n        ]\n      },\n      {\n        \"service\": \"GUIDED_HELP\",\n        \"params\": [\n          {\n            \"key\": \"logged_in\",\n            \"value\": \"0\"\n          }\n        ]\n      },\n      {\n        \"service\": \"ECATCHER\",\n        \"params\": [\n          {\n            \"key\": \"client.version\",\n            \"value\": \"2.20210728\"\n          },\n          {\n            \"key\": \"client.name\",\n            \"value\": \"WEB\"\n          }\n        ]\n      }\n    ],\n    \"mainAppWebResponseContext\": {\n      \"loggedOut\": true\n    },\n    \"webResponseContextExtensionData\": {\n      \"hasDecorated\": true\n    }\n  },\n  \"endpoint\": {\n    \"clickTrackingParams\": \"IhMIz5njxqWL8gIVjCBMCh3Iog9qMghleHRlcm5hbA\u003d\u003d\",\n    \"commandMetadata\": {\n      \"webCommandMetadata\": {\n        \"url\": \"/youtubei/v1/navigation/resolve_url\",\n        \"webPageType\": \"WEB_PAGE_TYPE_CHANNEL\",\n        \"rootVe\": 3611,\n        \"apiUrl\": \"/youtubei/v1/browse\"\n      },\n      \"resolveUrlCommandMetadata\": {\n        \"isVanityUrl\": true\n      }\n    },\n    \"browseEndpoint\": {\n      \"browseId\": \"UCYJ61XIK64sp6ZFFS8sctxw\",\n      \"params\": \"EgC4AQA%3D\"\n    }\n  }\n}\n",
+    "responseBody": "{\"responseContext\":{\"visitorData\":\"CgtXS3hrQllaQWpFYyinysORBg%3D%3D\",\"serviceTrackingParams\":[{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20220315.01.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"ResolveUrl_rid\",\"value\":\"0xf45eb6ef455aaff6\"}]},{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"24165080,24034168,1714252,24082661,24180089,24179787,24186023,23744176,23934970,24085811,24169726,24180222,23735348,24120829,24138064,24175488,24062268,24002022,24166867,24110902,39321475,24174213,24002025,24154586,24181422,24169458,24161848,23918597,24106839,24151647,24176755,24179290,23748146,23804281,23946420,24166202,24004644,24077241,24007246,24120820,24036947,23966208,24027708,24007790,23882503,9405988,24177193,24148482,24080738,24141413,24181182,23983296,24138442,24140247,23986021,23998056,24001373,24152442,24077266,24154616,24145515,24135310,24037231,24139173\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20220315\"},{\"key\":\"client.name\",\"value\":\"WEB\"},{\"key\":\"client.fexp\",\"value\":\"24165080,24034168,1714252,24082661,24180089,24179787,24186023,23744176,23934970,24085811,24169726,24180222,23735348,24120829,24138064,24175488,24062268,24002022,24166867,24110902,39321475,24174213,24002025,24154586,24181422,24169458,24161848,23918597,24106839,24151647,24176755,24179290,23748146,23804281,23946420,24166202,24004644,24077241,24007246,24120820,24036947,23966208,24027708,24007790,23882503,9405988,24177193,24148482,24080738,24141413,24181182,23983296,24138442,24140247,23986021,23998056,24001373,24152442,24077266,24154616,24145515,24135310,24037231,24139173\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"endpoint\":{\"clickTrackingParams\":\"IhMIjrjdpOnI9gIVQjzxBR1btwbaMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/youtubei/v1/navigation/resolve_url\",\"webPageType\":\"WEB_PAGE_TYPE_CHANNEL\",\"rootVe\":3611,\"apiUrl\":\"/youtubei/v1/browse\"},\"resolveUrlCommandMetadata\":{\"isVanityUrl\":true}},\"browseEndpoint\":{\"browseId\":\"UCYJ61XIK64sp6ZFFS8sctxw\",\"params\":\"EgC4AQA%3D\"}}}",
-    "latestUrl": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
+    "latestUrl": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\u0026prettyPrint\u003dfalse"
   }
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/localization/generated_mock_1.json

@@ -1,7 +1,7 @@
 {
   "request": {
     "httpMethod": "POST",
-    "url": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
+    "url": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\u0026prettyPrint\u003dfalse",
     "headers": {
       "Accept-Language": [
         "en-GB, en;q\u003d0.9"
@@ -16,7 +16,7 @@
         "https://www.youtube.com"
       ],
       "X-YouTube-Client-Version": [
-        "2.20210728.00.00"
+        "2.20220314.01.00"
       ],
       "Content-Type": [
         "application/json"
@@ -36,6 +36,61 @@
       58,
       123,
       34,
+      114,
+      101,
+      113,
+      117,
+      101,
+      115,
+      116,
+      34,
+      58,
+      123,
+      34,
+      105,
+      110,
+      116,
+      101,
+      114,
+      110,
+      97,
+      108,
+      69,
+      120,
+      112,
+      101,
+      114,
+      105,
+      109,
+      101,
+      110,
+      116,
+      70,
+      108,
+      97,
+      103,
+      115,
+      34,
+      58,
+      91,
+      93,
+      44,
+      34,
+      117,
+      115,
+      101,
+      83,
+      115,
+      108,
+      34,
+      58,
+      116,
+      114,
+      117,
+      101,
+      125,
+      44,
+      34,
       99,
       108,
       105,
@@ -88,6 +143,46 @@
       34,
       44,
       34,
+      111,
+      114,
+      105,
+      103,
+      105,
+      110,
+      97,
+      108,
+      85,
+      114,
+      108,
+      34,
+      58,
+      34,
+      104,
+      116,
+      116,
+      112,
+      115,
+      58,
+      47,
+      47,
+      119,
+      119,
+      119,
+      46,
+      121,
+      111,
+      117,
+      116,
+      117,
+      98,
+      101,
+      46,
+      99,
+      111,
+      109,
+      34,
+      44,
+      34,
       99,
       108,
       105,
@@ -109,18 +204,39 @@
       50,
       48,
       50,
-      49,
-      48,
-      55,
       50,
-      56,
+      48,
-      46,
+      51,
-      48,
+      49,
-      48,
+      52,
+      46,
+      48,
+      49,
       46,
       48,
       48,
       34,
+      44,
+      34,
+      112,
+      108,
+      97,
+      116,
+      102,
+      111,
+      114,
+      109,
+      34,
+      58,
+      34,
+      68,
+      69,
+      83,
+      75,
+      84,
+      79,
+      80,
+      34,
       125,
       44,
       34,
@@ -214,7 +330,7 @@
     "responseMessage": "",
     "responseHeaders": {
       "alt-svc": [
-        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
       ],
       "cache-control": [
         "private"
@@ -223,19 +339,19 @@
         "application/json; charset\u003dUTF-8"
       ],
       "date": [
-        "Fri, 30 Jul 2021 17:16:13 GMT"
+        "Tue, 15 Mar 2022 17:06:19 GMT"
       ],
       "expires": [
-        "Fri, 30 Jul 2021 17:16:13 GMT"
+        "Tue, 15 Mar 2022 17:06:19 GMT"
       ],
       "p3p": [
         "CP\u003d\"This is not a P3P policy! See g.co/p3phelp for more info.\""
       ],
       "server": [
-        "ESF"
+        "scaffolding on HTTPServer2"
       ],
       "set-cookie": [
-        "CONSENT\u003dPENDING+608; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "CONSENT\u003dPENDING+943; expires\u003dThu, 14-Mar-2024 17:06:19 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "vary": [
         "Origin",
@@ -252,7 +368,7 @@
         "0"
       ]
     },
-    "responseBody": "{\n  \"responseContext\": {\n    \"visitorData\": \"CgtTWjdEZHNiazVjYyjd55CIBg%3D%3D\",\n    \"serviceTrackingParams\": [\n      {\n        \"service\": \"CSI\",\n        \"params\": [\n          {\n            \"key\": \"c\",\n            \"value\": \"WEB\"\n          },\n          {\n            \"key\": \"cver\",\n            \"value\": \"2.20210728.00.00\"\n          },\n          {\n            \"key\": \"yt_li\",\n            \"value\": \"0\"\n          },\n          {\n            \"key\": \"ResolveUrl_rid\",\n            \"value\": \"0x7e38e2014f16d006\"\n          }\n        ]\n      },\n      {\n        \"service\": \"GFEEDBACK\",\n        \"params\": [\n          {\n            \"key\": \"logged_in\",\n            \"value\": \"0\"\n          },\n          {\n            \"key\": \"e\",\n            \"value\": \"24039151,39321255,23891346,23804281,23986015,23884386,24030040,24060921,23946420,24037794,24043240,24037585,24004644,24045475,24027396,24078419,24007246,24068842,24002025,24080914,23934970,24042870,24047616,23983814,24070036,24057238,23983296,24066054,23891344,23996512,9405960,1714258,23966208,24058128,23996830,24002022,24050503,23968386,24028143,24076877,24056264,24059521,24049820,24045411,24045476,24076598,23744176,24077267,23748147,23918597,23940237,24002923,23997322,24049573,23998056,24058812,24056275,24078367,23776346,24036948,23974595,24053866,24065282,23882685,24016284,23944779,24001373,24036237,24049571,24070943,24058380,23857949,24014916\"\n          }\n        ]\n      },\n      {\n        \"service\": \"GUIDED_HELP\",\n        \"params\": [\n          {\n            \"key\": \"logged_in\",\n            \"value\": \"0\"\n          }\n        ]\n      },\n      {\n        \"service\": \"ECATCHER\",\n        \"params\": [\n          {\n            \"key\": \"client.version\",\n            \"value\": \"2.20210728\"\n          },\n          {\n            \"key\": \"client.name\",\n            \"value\": \"WEB\"\n          }\n        ]\n      }\n    ],\n    \"mainAppWebResponseContext\": {\n      \"loggedOut\": true\n    },\n    \"webResponseContextExtensionData\": {\n      \"hasDecorated\": true\n    }\n  },\n  \"endpoint\": {\n    \"clickTrackingParams\": \"IhMIorj8gqaL8gIVIhDxBR0XzAL7MghleHRlcm5hbA\u003d\u003d\",\n    \"commandMetadata\": {\n      \"webCommandMetadata\": {\n        \"url\": \"/youtubei/v1/navigation/resolve_url\",\n        \"webPageType\": \"WEB_PAGE_TYPE_CHANNEL\",\n        \"rootVe\": 3611,\n        \"apiUrl\": \"/youtubei/v1/browse\"\n      },\n      \"resolveUrlCommandMetadata\": {\n        \"isVanityUrl\": true\n      }\n    },\n    \"browseEndpoint\": {\n      \"browseId\": \"UCeY0bbntWzzVIaj2z3QigXg\",\n      \"params\": \"EgC4AQA%3D\"\n    }\n  }\n}\n",
+    "responseBody": "{\"responseContext\":{\"visitorData\":\"CgtVNFVYa3p6R21EZyiLj8ORBg%3D%3D\",\"serviceTrackingParams\":[{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20220314.01.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"ResolveUrl_rid\",\"value\":\"0x2a498ca871cd2cc7\"}]},{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"23744176,24138064,24175488,24166867,24002025,24120829,24156613,24177978,24169726,24045475,24085811,24165080,24034168,24179787,24180089,24175889,23934970,24174792,24176755,24106839,24062267,24110902,23918597,24118607,24154586,24161848,24036948,24174213,24082662,24120819,39321475,24002022,24179579,24180220,24177193,24007790,24080738,23986031,24175560,24152443,23946420,24001373,24004644,24077241,23966208,24148481,24007246,23748146,23804281,24077266,24154616,24139173,24135310,24145515,23998056,23882685,24173491,24045476,1714251,24140247,24138442,24168663,24053419,23983296,24141413\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20220314\"},{\"key\":\"client.name\",\"value\":\"WEB\"},{\"key\":\"client.fexp\",\"value\":\"23744176,24138064,24175488,24166867,24002025,24120829,24156613,24177978,24169726,24045475,24085811,24165080,24034168,24179787,24180089,24175889,23934970,24174792,24176755,24106839,24062267,24110902,23918597,24118607,24154586,24161848,24036948,24174213,24082662,24120819,39321475,24002022,24179579,24180220,24177193,24007790,24080738,23986031,24175560,24152443,23946420,24001373,24004644,24077241,23966208,24148481,24007246,23748146,23804281,24077266,24154616,24139173,24135310,24145515,23998056,23882685,24173491,24045476,1714251,24140247,24138442,24168663,24053419,23983296,24141413\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"endpoint\":{\"clickTrackingParams\":\"IhMIrqirhs3I9gIVZUVPBB2SJQkwMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/youtubei/v1/navigation/resolve_url\",\"webPageType\":\"WEB_PAGE_TYPE_CHANNEL\",\"rootVe\":3611,\"apiUrl\":\"/youtubei/v1/browse\"},\"resolveUrlCommandMetadata\":{\"isVanityUrl\":true}},\"browseEndpoint\":{\"browseId\":\"UCeY0bbntWzzVIaj2z3QigXg\",\"params\":\"EgC4AQA%3D\"}}}",
-    "latestUrl": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
+    "latestUrl": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\u0026prettyPrint\u003dfalse"
   }
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_8.json

@@ -1,7 +1,7 @@
 {
   "request": {
     "httpMethod": "POST",
-    "url": "https://www.youtube.com/youtubei/v1/browse?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
+    "url": "https://www.youtube.com/youtubei/v1/browse?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\u0026prettyPrint\u003dfalse",
     "headers": {
       "Accept-Language": [
         "en-GB, en;q\u003d0.9"
@@ -16,7 +16,7 @@
         "https://www.youtube.com"
       ],
       "X-YouTube-Client-Version": [
-        "2.20210728.00.00"
+        "2.20220315.01.00"
       ],
       "Content-Type": [
         "application/json"
@@ -62,6 +62,61 @@
       58,
       123,
       34,
+      114,
+      101,
+      113,
+      117,
+      101,
+      115,
+      116,
+      34,
+      58,
+      123,
+      34,
+      105,
+      110,
+      116,
+      101,
+      114,
+      110,
+      97,
+      108,
+      69,
+      120,
+      112,
+      101,
+      114,
+      105,
+      109,
+      101,
+      110,
+      116,
+      70,
+      108,
+      97,
+      103,
+      115,
+      34,
+      58,
+      91,
+      93,
+      44,
+      34,
+      117,
+      115,
+      101,
+      83,
+      115,
+      108,
+      34,
+      58,
+      116,
+      114,
+      117,
+      101,
+      125,
+      44,
+      34,
       99,
       108,
       105,
@@ -114,6 +169,46 @@
       34,
       44,
       34,
+      111,
+      114,
+      105,
+      103,
+      105,
+      110,
+      97,
+      108,
+      85,
+      114,
+      108,
+      34,
+      58,
+      34,
+      104,
+      116,
+      116,
+      112,
+      115,
+      58,
+      47,
+      47,
+      119,
+      119,
+      119,
+      46,
+      121,
+      111,
+      117,
+      116,
+      117,
+      98,
+      101,
+      46,
+      99,
+      111,
+      109,
+      34,
+      44,
+      34,
       99,
       108,
       105,
@@ -135,18 +230,39 @@
       50,
       48,
       50,
-      49,
-      48,
-      55,
       50,
-      56,
+      48,
-      46,
+      51,
-      48,
+      49,
-      48,
+      53,
+      46,
+      48,
+      49,
       46,
       48,
       48,
       34,
+      44,
+      34,
+      112,
+      108,
+      97,
+      116,
+      102,
+      111,
+      114,
+      109,
+      34,
+      58,
+      34,
+      68,
+      69,
+      83,
+      75,
+      84,
+      79,
+      80,
+      34,
       125,
       44,
       34,
@@ -230,7 +346,7 @@
         "application/json; charset\u003dUTF-8"
       ],
       "date": [
-        "Wed, 08 Dec 2021 20:50:38 GMT"
+        "Tue, 15 Mar 2022 17:06:17 GMT"
       ],
       "server": [
         "ESF"
@@ -251,6 +367,6 @@
       ]
     },
     "responseBody": "{\n  \"error\": {\n    \"code\": 400,\n    \"message\": \"Request contains an invalid argument.\",\n    \"errors\": [\n      {\n        \"message\": \"Request contains an invalid argument.\",\n        \"domain\": \"global\",\n        \"reason\": \"badRequest\"\n      }\n    ],\n    \"status\": \"INVALID_ARGUMENT\"\n  }\n}\n",
-    "latestUrl": "https://www.youtube.com/youtubei/v1/browse?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
+    "latestUrl": "https://www.youtube.com/youtubei/v1/browse?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\u0026prettyPrint\u003dfalse"
   }
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/feed/notAvailable/generated_mock_0.json

@@ -17,7 +17,7 @@
     "responseMessage": "",
     "responseHeaders": {
       "alt-svc": [
-        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
       ],
       "content-length": [
         "1613"
@@ -26,7 +26,7 @@
         "text/html; charset\u003dUTF-8"
       ],
       "date": [
-        "Sun, 04 Jul 2021 16:47:38 GMT"
+        "Tue, 15 Mar 2022 17:06:27 GMT"
       ],
       "server": [
         "YouTube RSS Feeds server"