From fd82ec585b79142681b06cb93754dc979aa1bbb8 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Tue, 28 Jan 2025 20:16:13 +0100 Subject: [PATCH 01/17] [YouTube] Move to their own file and update clients' constants Also update client version and device info and add TVHTML5 client and WEB_EMBEDDED_PLAYER constants, these will be used in the future. TVHTML5_SIMPLY_EMBED_CLIENT_VERSION has been kept in its place as the corresponding client does not work anonymously anymore, so it will be removed soon. --- .../services/youtube/ClientsConstants.java | 116 ++++++++++++++++ .../youtube/YoutubeParsingHelper.java | 130 ++++-------------- 2 files changed, 144 insertions(+), 102 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java new file mode 100644 index 000000000..22b002e8f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java @@ -0,0 +1,116 @@ +package org.schabi.newpipe.extractor.services.youtube; + +final class ClientsConstants { + private ClientsConstants() { + } + + // Common client fields + + static final String DESKTOP_CLIENT_PLATFORM = "DESKTOP"; + static final String MOBILE_CLIENT_PLATFORM = "MOBILE"; + static final String WATCH_CLIENT_SCREEN = "WATCH"; + static final String EMBED_CLIENT_SCREEN = "EMBED"; + + // WEB (YouTube desktop) client fields + + static final String WEB_CLIENT_ID = "1"; + static final String WEB_CLIENT_NAME = "WEB"; + /** + * The client version for InnerTube requests with the {@code WEB} client, used as the last + * fallback if the extraction of the real one failed. + */ + static final String WEB_HARDCODED_CLIENT_VERSION = "2.20250122.04.00"; + + // WEB_REMIX (YouTube Music) client fields + + static final String WEB_REMIX_CLIENT_ID = "67"; + static final String WEB_REMIX_CLIENT_NAME = "WEB_REMIX"; + static final String WEB_REMIX_HARDCODED_CLIENT_VERSION = "1.20250122.01.00"; + + // TVHTML5 (YouTube on TVs and consoles using HTML5) client fields + static final String TVHTML5_CLIENT_ID = "7"; + static final String TVHTML5_CLIENT_NAME = "TVHTML5"; + static final String TVHTML5_CLIENT_VERSION = "7.20250122.15.00"; + static final String TVHTML5_CLIENT_PLATFORM = "GAME_CONSOLE"; + static final String TVHTML5_DEVICE_MAKE = "Sony"; + static final String TVHTML5_DEVICE_MODEL_AND_OS_NAME = "PlayStation 4"; + // CHECKSTYLE:OFF + static final String TVHTML5_USER_AGENT = + "Mozilla/5.0 (PlayStation; PlayStation 4/12.00) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15"; + // CHECKSTYLE:ON + + // WEB_EMBEDDED_PLAYER (YouTube embeds) + + static final String WEB_EMBEDDED_CLIENT_ID = "56"; + static final String WEB_EMBEDDED_CLIENT_NAME = "WEB_EMBEDDED_PLAYER"; + static final String WEB_EMBEDDED_CLIENT_VERSION = "1.20250121.00.00"; + + // IOS (iOS YouTube app) client fields + + static final String IOS_CLIENT_NAME = "IOS"; + + /** + * The hardcoded client version of the iOS app used for InnerTube requests with this client. + * + *

+ * It can be extracted by getting the latest release version of the app on + * the App + * Store page of the YouTube app, in the {@code What’s New} section. + *

+ */ + static final String IOS_CLIENT_VERSION = "20.03.02"; + + /** + * The device machine id for the iPhone 15 Pro Max, used to get 60fps with the {@code iOS} + * client. + * + *

+ * See this GitHub Gist for more + * information. + *

+ */ + static final String IOS_DEVICE_MODEL = "iPhone16,2"; + + /** + * The iOS version to be used in JSON POST requests, the one of an iPhone 15 Pro Max running + * iOS 18.2.1 with the hardcoded version of the iOS app (for the {@code "osVersion"} field). + * + *

+ * The value of this field seems to use the following structure: + * "iOS major version.minor version.patch version.build version", where + * "patch version" is equal to 0 if it isn't set + * The build version corresponding to the iOS version used can be found on + * + * https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max + *

+ * + * @see #IOS_USER_AGENT_VERSION + */ + static final String IOS_OS_VERSION = "18.2.1.22C161"; + + /** + * The iOS version to be used in the HTTP user agent for requests. + * + *

+ * This should be the same of as {@link #IOS_OS_VERSION}. + *

+ * + * @see #IOS_OS_VERSION + */ + static final String IOS_USER_AGENT_VERSION = "18_2_1"; + + // ANDROID (Android YouTube app) client fields + + static final String ANDROID_CLIENT_NAME = "ANDROID"; + + /** + * The hardcoded client version of the Android app used for InnerTube requests with this + * client. + * + *

+ * It can be extracted by getting the latest release version of the app in an APK repository + * such as APKMirror. + *

+ */ + static final String ANDROID_CLIENT_VERSION = "19.28.35"; +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 54b4b90fb..d3c88deab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -21,6 +21,17 @@ package org.schabi.newpipe.extractor.services.youtube; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray; @@ -144,55 +155,11 @@ public final class YoutubeParsingHelper { */ public static final String RACY_CHECK_OK = "racyCheckOk"; - /** - * The hardcoded client ID used for InnerTube requests with the {@code WEB} client. - */ - private static final String WEB_CLIENT_ID = "1"; - - /** - * The client version for InnerTube requests with the {@code WEB} client, used as the last - * fallback if the extraction of the real one failed. - */ - private static final String HARDCODED_CLIENT_VERSION = "2.20240718.01.00"; - - /** - * The hardcoded client version of the Android app used for InnerTube requests with this - * client. - * - *

- * It can be extracted by getting the latest release version of the app in an APK repository - * such as APKMirror. - *

- */ - private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "19.28.35"; - - /** - * The hardcoded client version of the iOS app used for InnerTube requests with this client. - * - *

- * It can be extracted by getting the latest release version of the app on - * the App - * Store page of the YouTube app, in the {@code What’s New} section. - *

- */ - private static final String IOS_YOUTUBE_CLIENT_VERSION = "20.03.02"; - /** * The hardcoded client version used for InnerTube requests with the TV HTML5 embed client. */ private static final String TVHTML5_SIMPLY_EMBED_CLIENT_VERSION = "2.0"; - /** - * The hardcoded client ID used for InnerTube requests with the YouTube Music desktop client. - */ - private static final String YOUTUBE_MUSIC_CLIENT_ID = "67"; - - /** - * The hardcoded client version used for InnerTube requests with the YouTube Music desktop - * client. - */ - private static final String HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION = "1.20240715.01.00"; - private static String clientVersion; private static String youtubeMusicClientVersion; @@ -212,41 +179,6 @@ public final class YoutubeParsingHelper { private static final String CONTENT_PLAYBACK_NONCE_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; - /** - * The device machine id for the iPhone 15 Pro Max, - * used to get 60fps with the {@code iOS} client. - * - *

- * See this GitHub Gist for more - * information. - *

- */ - private static final String IOS_DEVICE_MODEL = "iPhone16,2"; - - /** - * Spoofing an iPhone 15 Pro Max running iOS 18.2.1 with the hardcoded version of the iOS app. - * To be used for the {@code "osVersion"} field in JSON POST requests. - *

- * The value of this field seems to use the following structure: - * "iOS major version.minor version.patch version.build version", where - * "patch version" is equal to 0 if it isn't set - * The build version corresponding to the iOS version used can be found on - * - * https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max - *

- * - * @see #IOS_USER_AGENT_VERSION - */ - private static final String IOS_OS_VERSION = "18.2.1.22C161"; - - /** - * Spoofing an iPhone 15 Pro Max running iOS 18.2.1 with the hardcoded version of the iOS app. - * To be used in the user agent for requests. - * - * @see #IOS_OS_VERSION - */ - private static final String IOS_USER_AGENT_VERSION = "18_2_1"; - private static Random numberGenerator = new Random(); private static final String FEED_BASE_CHANNEL_ID = @@ -561,9 +493,9 @@ public final class YoutubeParsingHelper { .object("client") .value("hl", "en-GB") .value("gl", "GB") - .value("clientName", "WEB") - .value("clientVersion", HARDCODED_CLIENT_VERSION) - .value("platform", "DESKTOP") + .value("clientName", WEB_CLIENT_NAME) + .value("clientVersion", WEB_HARDCODED_CLIENT_VERSION) + .value("platform", DESKTOP_CLIENT_PLATFORM) .value("utcOffsetMinutes", 0) .end() .object("request") @@ -581,7 +513,7 @@ public final class YoutubeParsingHelper { .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on - final var headers = getClientHeaders(WEB_CLIENT_ID, HARDCODED_CLIENT_VERSION); + final var headers = getClientHeaders(WEB_CLIENT_ID, WEB_HARDCODED_CLIENT_VERSION); // This endpoint is fetched by the YouTube website to get the items of its main menu and is // pretty lightweight (around 30kB) @@ -705,7 +637,7 @@ public final class YoutubeParsingHelper { // Fallback to the hardcoded one if it is valid if (isHardcodedClientVersionValid()) { - clientVersion = HARDCODED_CLIENT_VERSION; + clientVersion = WEB_HARDCODED_CLIENT_VERSION; return clientVersion; } @@ -752,11 +684,11 @@ public final class YoutubeParsingHelper { .object() .object("context") .object("client") - .value("clientName", "WEB_REMIX") - .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION) + .value("clientName", WEB_REMIX_CLIENT_NAME) + .value("clientVersion", WEB_REMIX_HARDCODED_CLIENT_VERSION) .value("hl", "en-GB") .value("gl", "GB") - .value("platform", "DESKTOP") + .value("platform", DESKTOP_CLIENT_PLATFORM) .value("utcOffsetMinutes", 0) .end() .object("request") @@ -775,8 +707,7 @@ public final class YoutubeParsingHelper { // @formatter:on final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL)); - headers.putAll(getClientHeaders(YOUTUBE_MUSIC_CLIENT_ID, - HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION)); + headers.putAll(getClientHeaders(WEB_REMIX_CLIENT_ID, WEB_HARDCODED_CLIENT_VERSION)); final Response response = getDownloader().postWithContentTypeJson(url, headers, json); // Ensure to have a valid response @@ -789,7 +720,7 @@ public final class YoutubeParsingHelper { return youtubeMusicClientVersion; } if (isHardcodedYoutubeMusicClientVersionValid()) { - youtubeMusicClientVersion = HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION; + youtubeMusicClientVersion = WEB_REMIX_HARDCODED_CLIENT_VERSION; return youtubeMusicClientVersion; } @@ -1196,10 +1127,10 @@ public final class YoutubeParsingHelper { .object("client") .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) - .value("clientName", "WEB") + .value("clientName", WEB_CLIENT_NAME) .value("clientVersion", getClientVersion()) .value("originalUrl", "https://www.youtube.com") - .value("platform", "DESKTOP") + .value("platform", DESKTOP_CLIENT_PLATFORM) .value("utcOffsetMinutes", 0) .value("visitorData", vData) .end() @@ -1391,9 +1322,8 @@ public final class YoutubeParsingHelper { */ @Nonnull public static String getAndroidUserAgent(@Nullable final Localization localization) { - // Spoofing an Android 14 device with the hardcoded version of the Android app - return "com.google.android.youtube/" + ANDROID_YOUTUBE_CLIENT_VERSION - + " (Linux; U; Android 14; " + return "com.google.android.youtube/" + ANDROID_CLIENT_VERSION + + " (Linux; U; Android 15; " + (localization != null ? localization : Localization.DEFAULT).getCountryCode() + ") gzip"; } @@ -1413,11 +1343,8 @@ public final class YoutubeParsingHelper { */ @Nonnull public static String getIosUserAgent(@Nullable final Localization localization) { - // Spoofing an iPhone 15 Pro Max running iOS 18.1.0 - // with the hardcoded version of the iOS app - return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION - + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS " - + IOS_USER_AGENT_VERSION + " like Mac OS X; " + return "com.google.ios.youtube/" + IOS_CLIENT_VERSION + "(" + IOS_DEVICE_MODEL + + "; U; CPU iOS " + IOS_USER_AGENT_VERSION + " like Mac OS X; " + (localization != null ? localization : Localization.DEFAULT).getCountryCode() + ")"; } @@ -1428,8 +1355,7 @@ public final class YoutubeParsingHelper { @Nonnull public static Map> getYoutubeMusicHeaders() { final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL)); - headers.putAll(getClientHeaders(YOUTUBE_MUSIC_CLIENT_ID, - youtubeMusicClientVersion)); + headers.putAll(getClientHeaders(WEB_REMIX_CLIENT_ID, youtubeMusicClientVersion)); return headers; } From 3691fc22c62667d007c0eb48ea06f44aa7754b42 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Tue, 28 Jan 2025 20:45:54 +0100 Subject: [PATCH 02/17] [YouTube] Add an interface and a class to fetch and provide poTokens --- .../services/youtube/PoTokenProvider.java | 120 ++++++++++++++++++ .../services/youtube/PoTokenResult.java | 52 ++++++++ 2 files changed, 172 insertions(+) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java new file mode 100644 index 000000000..a11a25bae --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java @@ -0,0 +1,120 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import javax.annotation.Nullable; + +/** + * Interface to provide {@code poToken}s to YouTube player requests. + * + *

+ * On some major clients, YouTube requires that the integrity of the device passes some checks to + * allow playback. + *

+ * + *

+ * These checks involve running codes to verify the integrity and using their result to generate + * one or multiple {@code poToken}(s) (which stands for proof of origin token(s)). + *

+ * + *

+ * These tokens may have a role in triggering the sign in requirement. + *

+ * + *

+ * If an implementation does not want to return a {@code poToken} for a specific client, it must + * return {@code null}. + *

+ * + *

+ * Implementations of this interface are expected to be thread-safe, as they may be accessed by + * multiple threads. + *

+ */ +public interface PoTokenProvider { + + /** + * Get a {@link PoTokenResult} specific to the desktop website, a.k.a. the WEB InnerTube client. + * + *

+ * To be generated and valid, {@code poToken}s from this client must be generated using Google's + * BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They + * must be added to adaptive/DASH streaming URLs with the {@code pot} parameter. + *

+ * + *

+ * Note that YouTube desktop website generates two {@code poToken}s: + * - one for the player requests {@code poToken}s, using the videoId as the minter value; + * - one for the streaming URLs, using a visitor data for logged-out users as the minter value. + *

+ * + * @return a {@link PoTokenResult} specific to the WEB InnerTube client + */ + @Nullable + PoTokenResult getWebClientPoToken(String videoId); + + /** + * Get a {@link PoTokenResult} specific to the web embeds, a.k.a. the WEB_EMBEDDED_PLAYER + * InnerTube client. + * + *

+ * To be generated and valid, {@code poToken}s from this client must be generated using Google's + * BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They + * should be added to adaptive/DASH streaming URLs with the {@code pot} parameter. + *

+ * + *

+ * As of writing, like the YouTube desktop website previously did, it generates only one + * {@code poToken}, sent in player requests and streaming URLs, using a visitor data for + * logged-out users. {@code poToken}s do not seem to be mandatory for now on this client. + *

+ * + * @return a {@link PoTokenResult} specific to the WEB_EMBEDDED_PLAYER InnerTube client + */ + @Nullable + PoTokenResult getWebEmbedClientPoToken(String videoId); + + /** + * Get a {@link PoTokenResult} specific to the Android app, a.k.a. the ANDROID InnerTube client. + * + *

+ * Implementation details are not known, the app uses DroidGuard, a downloaded native virtual + * machine ran by Google Play Services for which its code is updated pretty frequently. + *

+ * + *

+ * As of writing, DroidGuard seem to check for the Android app signature and package ID, as + * non-rooted YouTube patched with reVanced doesn't work without spoofing another InnerTube + * client while the rooted version works without any client spoofing. + *

+ * + *

+ * There should be only one {@code poToken} needed for the player requests, it shouldn't be + * required for regular adaptive URLs (i.e. not server adaptive bitrate (SABR) URLs). HLS + * formats returned (only for premieres and running and post-live livestreams) in the client's + * HLS manifest URL should work without {@code poToken}s. + *

+ * + * @return a {@link PoTokenResult} specific to the ANDROID InnerTube client + */ + @Nullable + PoTokenResult getAndroidClientPoToken(String videoId); + + /** + * Get a {@link PoTokenResult} specific to the iOS app, a.k.a. the IOS InnerTube client. + * + *

+ * Implementation details are not known, the app seem to use something called iosGuard which + * should be similar to Android's DroidGuard. It may rely on Apple's attestation APIs. + *

+ * + *

+ * As of writing, there should be only one {@code poToken} needed for the player requests, it + * shouldn't be required for regular adaptive URLs (i.e. not server adaptive bitrate (SABR) + * URLs). HLS formats returned in the client's HLS manifest URL should also work without a + * {@code poToken}. + *

+ * + * @return a {@link PoTokenResult} specific to the IOS InnerTube client + */ + @Nullable + PoTokenResult getIosClientPoToken(String videoId); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java new file mode 100644 index 000000000..4ccbaf1ba --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java @@ -0,0 +1,52 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Objects; + +/** + * The result of a supported/successful {@code poToken} extraction request by a + * {@link PoTokenProvider}. + */ +public final class PoTokenResult { + + /** + * The visitor data associated with a {@code poToken}. + */ + @Nonnull + public final String visitorData; + + /** + * The {@code poToken} of a player request, a Protobuf object encoded as a base 64 string. + */ + @Nonnull + public final String playerRequestPoToken; + + /** + * The {@code poToken} to be appended to streaming URLs, a Protobuf object encoded as a base + * 64 string. + * + *

+ * It may be required on some clients such as HTML5 ones and may also differ from the player + * request {@code poToken}. + *

+ */ + @Nullable + public final String streamingDataPoToken; + + /** + * Construct a {@link PoTokenResult} instance. + * + * @param visitorData see {@link #visitorData} + * @param playerRequestPoToken see {@link #playerRequestPoToken} + * @param streamingDataPoToken see {@link #streamingDataPoToken} + * @throws NullPointerException if a non-null parameter is null + */ + public PoTokenResult(@Nonnull final String visitorData, + @Nonnull final String playerRequestPoToken, + @Nullable final String streamingDataPoToken) { + this.visitorData = Objects.requireNonNull(visitorData); + this.playerRequestPoToken = Objects.requireNonNull(playerRequestPoToken); + this.streamingDataPoToken = streamingDataPoToken; + } +} From 1df0267440f9b307c46c0fff1a11baccdf25603c Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Tue, 28 Jan 2025 21:07:37 +0100 Subject: [PATCH 03/17] [YouTube] Make client and origin HTTP headers methods package-private This allows their internal usage in an upcoming new class to be used on other InnerTube clients than the WEB one. --- .../extractor/services/youtube/YoutubeParsingHelper.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index d3c88deab..ece76ecef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -1387,7 +1387,7 @@ public final class YoutubeParsingHelper { * * @param url The URL to be set as the origin and referrer. */ - private static Map> getOriginReferrerHeaders(@Nonnull final String url) { + static Map> getOriginReferrerHeaders(@Nonnull final String url) { final var urlList = List.of(url); return Map.of("Origin", urlList, "Referer", urlList); } @@ -1399,8 +1399,8 @@ public final class YoutubeParsingHelper { * @param name The X-YouTube-Client-Name value. * @param version X-YouTube-Client-Version value. */ - private static Map> getClientHeaders(@Nonnull final String name, - @Nonnull final String version) { + static Map> getClientHeaders(@Nonnull final String name, + @Nonnull final String version) { return Map.of("X-YouTube-Client-Name", List.of(name), "X-YouTube-Client-Version", List.of(version)); } From 9d2b840cfb1ed4b21a41676d271d5a3a0cbed31e Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Tue, 28 Jan 2025 21:18:58 +0100 Subject: [PATCH 04/17] [YouTube] Add utility class to handle player requests fetching This internal class, YoutubeStreamHelper, has to be public in order to be used by subpackages of the service's root one. It supports poTokens, HTML5 signatureTimestamp property, embed context and multiple InnerTube clients. It is meant to replace the corresponding methods in YoutubeParsingHelper, in order to reduce the class' size, code duplicates and improve its readability. It adds a new way to get age-restricted videos' streams, the only ones which are playable in YouTube embeds, which is not very common. --- .../services/youtube/YoutubeStreamHelper.java | 476 ++++++++++++++++++ 1 file changed, 476 insertions(+) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java new file mode 100644 index 000000000..7d2da95f8 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java @@ -0,0 +1,476 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import com.grack.nanojson.JsonBuilder; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonWriter; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.localization.ContentCountry; +import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.EMBED_CLIENT_SCREEN; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_OS_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.MOBILE_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MAKE; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MODEL_AND_OS_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WATCH_CLIENT_SCREEN; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_GAPIS_URL; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientHeaders; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getOriginReferrerHeaders; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +public final class YoutubeStreamHelper { + + private static final String PLAYER = "player"; + private static final String SERVICE_INTEGRITY_DIMENSIONS = "serviceIntegrityDimensions"; + private static final String PO_TOKEN = "poToken"; + private static final String BASE_YT_DESKTOP_WATCH_URL = "https://www.youtube.com/watch?v="; + + private YoutubeStreamHelper() { + } + + @Nonnull + public static JsonObject getWebMetadataPlayerResponse( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId) throws IOException, ExtractionException { + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + WEB_CLIENT_NAME, + getClientVersion(), + WATCH_CLIENT_SCREEN, + DESKTOP_CLIENT_PLATFORM, + YoutubeParsingHelper.randomVisitorData(contentCountry), + null, + null, + null, + null, + null, + -1); + + addVideoIdCpnAndOkChecks(builder, videoId, null); + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER + + "&$fields=microformat,playabilityStatus,storyboards,videoDetails"; + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson( + url, getYouTubeHeaders(), body, localization))); + } + + @Nonnull + public static JsonObject getTvHtml5PlayerResponse( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nonnull final String cpn) throws IOException, ExtractionException { + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + TVHTML5_CLIENT_NAME, + TVHTML5_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, + TVHTML5_CLIENT_PLATFORM, + YoutubeParsingHelper.randomVisitorData(contentCountry), + TVHTML5_DEVICE_MAKE, + TVHTML5_DEVICE_MODEL_AND_OS_NAME, + TVHTML5_DEVICE_MODEL_AND_OS_NAME, + "", + null, + -1); + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; + + final Map> headers = new HashMap<>( + getClientHeaders(TVHTML5_CLIENT_ID, TVHTML5_CLIENT_VERSION)); + headers.putAll(getOriginReferrerHeaders("https://www.youtube.com")); + headers.put("User-Agent", List.of(TVHTML5_USER_AGENT)); + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson( + url, headers, body, localization))); + } + + @Nonnull + public static JsonObject getWebFullPlayerResponse( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nonnull final String cpn, + @Nonnull final PoTokenResult webPoTokenResult, + final int signatureTimestamp) throws IOException, ExtractionException { + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + WEB_CLIENT_NAME, + getClientVersion(), + WATCH_CLIENT_SCREEN, + DESKTOP_CLIENT_PLATFORM, + webPoTokenResult.visitorData, + null, + null, + null, + null, + null, + -1); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + addPlaybackContext( + builder, + BASE_YT_DESKTOP_WATCH_URL + videoId, + signatureTimestamp); + + addPoToken(builder, webPoTokenResult.playerRequestPoToken); + + final byte[] body = JsonWriter.string(builder.end().done()) + .getBytes(StandardCharsets.UTF_8); + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson( + url, getYouTubeHeaders(), body, localization))); + } + + @Nonnull + public static JsonObject getWebEmbeddedPlayerResponse( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nonnull final String cpn, + @Nullable final PoTokenResult webEmbeddedPoTokenResult, + final int signatureTimestamp) throws IOException, ExtractionException { + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + WEB_EMBEDDED_CLIENT_NAME, + WEB_REMIX_HARDCODED_CLIENT_VERSION, + EMBED_CLIENT_SCREEN, + DESKTOP_CLIENT_PLATFORM, + webEmbeddedPoTokenResult == null + ? YoutubeParsingHelper.randomVisitorData(contentCountry) + : webEmbeddedPoTokenResult.visitorData, + null, + null, + null, + null, + BASE_YT_DESKTOP_WATCH_URL + videoId, + -1); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + addPlaybackContext( + builder, + BASE_YT_DESKTOP_WATCH_URL + videoId, + signatureTimestamp); + + if (webEmbeddedPoTokenResult != null) { + addPoToken(builder, webEmbeddedPoTokenResult.playerRequestPoToken); + } + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; + + final Map> headers = new HashMap<>( + getClientHeaders(WEB_EMBEDDED_CLIENT_ID, WEB_EMBEDDED_CLIENT_VERSION)); + headers.putAll(getOriginReferrerHeaders("https://www.youtube.com")); + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson( + url, headers, body, localization))); + } + + public static JsonObject getAndroidPlayerResponse( + @Nonnull final ContentCountry contentCountry, + @Nonnull final Localization localization, + @Nonnull final String videoId, + @Nonnull final String cpn, + @Nonnull final PoTokenResult androidPoTokenResult) + throws IOException, ExtractionException { + + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + ANDROID_CLIENT_NAME, + ANDROID_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, + MOBILE_CLIENT_PLATFORM, + androidPoTokenResult.visitorData, + null, + null, + "Android", + "15", + null, + 35); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + addPoToken(builder, androidPoTokenResult.playerRequestPoToken); + + final byte[] body = JsonWriter.string(builder.end().done()) + .getBytes(StandardCharsets.UTF_8); + + return getJsonAndroidPostResponse( + PLAYER, + body, + localization, + "&t=" + generateTParameter() + "&id=" + videoId); + } + + public static JsonObject getAndroidReelPlayerResponse( + @Nonnull final ContentCountry contentCountry, + @Nonnull final Localization localization, + @Nonnull final String videoId, + @Nonnull final String cpn) throws IOException, ExtractionException { + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + ANDROID_CLIENT_NAME, + ANDROID_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, + MOBILE_CLIENT_PLATFORM, + YoutubeParsingHelper.randomVisitorData(contentCountry), + null, + null, + "Android", + "15", + null, + 35); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + builder.object("playerRequest") + .value(VIDEO_ID, videoId) + .end() + .value("disablePlayerResponse", false); + + final byte[] mobileBody = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + final JsonObject androidPlayerResponse = getJsonAndroidPostResponse( + "reel/reel_item_watch", + mobileBody, + localization, + "&t=" + generateTParameter() + "&id=" + videoId + "&$fields=playerResponse"); + + return androidPlayerResponse.getObject("playerResponse"); + } + + public static JsonObject getIosPlayerResponse(@Nonnull final ContentCountry contentCountry, + @Nonnull final Localization localization, + @Nonnull final String videoId, + @Nonnull final String cpn, + @Nullable final PoTokenResult iosPoTokenResult) + throws IOException, ExtractionException { + final boolean noPoTokenResult = iosPoTokenResult == null; + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + IOS_CLIENT_NAME, + IOS_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, + MOBILE_CLIENT_PLATFORM, + noPoTokenResult + ? YoutubeParsingHelper.randomVisitorData(contentCountry) + : iosPoTokenResult.visitorData, + "Apple", + IOS_DEVICE_MODEL, + "iOS", + IOS_OS_VERSION, + null, + -1); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + if (!noPoTokenResult) { + addPoToken(builder, iosPoTokenResult.playerRequestPoToken); + } + + final byte[] mobileBody = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + return getJsonIosPostResponse( + mobileBody, localization, "&t=" + generateTParameter() + + "&id=" + videoId + "&fields=streamingData.hlsManifestUrl"); + } + + public static JsonObject getJsonAndroidPostResponse(final String endpoint, + final byte[] body, + @Nonnull final Localization localization, + @Nullable final String endPartOfUrlRequest) + throws IOException, ExtractionException { + return getMobilePostResponse(endpoint, body, localization, + getAndroidUserAgent(localization), endPartOfUrlRequest); + } + + private static JsonObject getJsonIosPostResponse(final byte[] body, + @Nonnull final Localization localization, + @Nullable final String endPartOfUrlRequest) + throws IOException, ExtractionException { + return getMobilePostResponse(YoutubeStreamHelper.PLAYER, body, localization, + getIosUserAgent(localization), + endPartOfUrlRequest); + } + + @SuppressWarnings("checkstyle:ParameterNumber") + @Nonnull + private static JsonBuilder prepareJsonBuilder( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String clientName, + @Nonnull final String clientVersion, + @Nonnull final String clientScreen, + @Nonnull final String platform, + @Nonnull final String visitorData, + @Nullable final String deviceMake, + @Nullable final String deviceModel, + @Nullable final String osName, + @Nullable final String osVersion, + @Nullable final String embedUrl, + final int androidSdkVersion) { + final JsonBuilder builder = JsonObject.builder() + .object("context") + .object("client") + .value("clientName", clientName) + .value("clientVersion", clientVersion) + .value("clientScreen", clientScreen) + .value("platform", platform) + .value("visitorData", visitorData); + + if (deviceMake != null) { + builder.value("deviceMake", deviceMake); + } + if (deviceModel != null) { + builder.value("deviceModel", deviceModel); + } + if (osName != null) { + builder.value("osName", osName); + } + if (osVersion != null) { + builder.value("osVersion", osVersion); + } + if (androidSdkVersion > 0) { + builder.value("androidSdkVersion", androidSdkVersion); + } + + builder.value("hl", localization.getLocalizationCode()) + .value("gl", contentCountry.getCountryCode()) + .value("utcOffsetMinutes", 0) + .end(); + + if (embedUrl != null) { + builder.object("thirdParty") + .value("embedUrl", embedUrl) + .end(); + } + + builder.object("request") + .array("internalExperimentFlags") + .end() + .value("useSsl", true) + .end() + .object("user") + // TODO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) + .end() + .end(); + + return builder; + } + + private static JsonObject getMobilePostResponse(@Nonnull final String endpoint, + final byte[] body, + @Nonnull final Localization localization, + @Nonnull final String userAgent, + @Nullable final String endPartOfUrlRequest) + throws IOException, ExtractionException { + final Map> headers = Map.of("User-Agent", List.of(userAgent), + "X-Goog-Api-Format-Version", List.of("2")); + + final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?" + + DISABLE_PRETTY_PRINT_PARAMETER; + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest) + ? baseEndpointUrl + : baseEndpointUrl + endPartOfUrlRequest, + headers, body, localization))); + } + + private static void addVideoIdCpnAndOkChecks(@Nonnull final JsonBuilder builder, + @Nonnull final String videoId, + @Nullable final String cpn) { + builder.value(VIDEO_ID, videoId); + + if (cpn != null) { + builder.value(CPN, cpn); + } + + builder.value(CONTENT_CHECK_OK, true) + .value(RACY_CHECK_OK, true); + } + + private static void addPlaybackContext(@Nonnull final JsonBuilder builder, + @Nonnull final String referer, + final int signatureTimestamp) { + builder.object("playbackContext") + .object("contentPlaybackContext") + .value("signatureTimestamp", signatureTimestamp) + .value("referer", referer) + .end() + .end(); + } + + private static void addPoToken(@Nonnull final JsonBuilder builder, + @Nonnull final String poToken) { + builder.object(SERVICE_INTEGRITY_DIMENSIONS) + .value(PO_TOKEN, poToken) + .end(); + } +} From 3878696b2c4e419a3809e23e28b250d487ab71db Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Tue, 28 Jan 2025 21:44:14 +0100 Subject: [PATCH 05/17] [YouTube] Add support for poTokens, refactor player clients' fetching Also improve detection of age-restricted statuses in playability error messages returned by the service and provide version 7 of DASH manifests. --- .../extractors/YoutubeStreamExtractor.java | 566 +++++++++++------- 1 file changed, 355 insertions(+), 211 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 988a1bcc2..e2a49f7da 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -27,18 +27,12 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createTvHtml5EmbedPlayerBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -66,9 +60,12 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider; +import org.schabi.newpipe.extractor.services.youtube.PoTokenResult; import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager; import org.schabi.newpipe.extractor.services.youtube.YoutubeMetaInfoHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.YoutubeStreamHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.DeliveryMethod; @@ -104,6 +101,10 @@ import javax.annotation.Nullable; public class YoutubeStreamExtractor extends StreamExtractor { + @Nullable + private static PoTokenProvider poTokenProvider; + private static boolean fetchIosClient; + private JsonObject playerResponse; private JsonObject nextResponse; @@ -112,7 +113,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nullable private JsonObject androidStreamingData; @Nullable - private JsonObject tvHtml5SimplyEmbedStreamingData; + private JsonObject html5StreamingData; private JsonObject videoPrimaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer; @@ -127,7 +128,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { // three different strings are used. private String iosCpn; private String androidCpn; - private String tvHtml5SimplyEmbedCpn; + private String html5Cpn; + + @Nullable + private String html5StreamingUrlsPoToken; + @Nullable + private String androidStreamingUrlsPoToken; + @Nullable + private String iosStreamingUrlsPoToken; public YoutubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) { super(service, linkHandler); @@ -321,7 +329,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { return Long.parseLong(duration); } catch (final Exception e) { return getDurationFromFirstAdaptiveFormat(Arrays.asList( - iosStreamingData, androidStreamingData, tvHtml5SimplyEmbedStreamingData)); + html5StreamingData, androidStreamingData, iosStreamingData)); } } @@ -579,11 +587,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getDashMpdUrl() throws ParsingException { assertPageFetched(); - // There is no DASH manifest available in the iOS clients and the DASH manifest of the - // Android client doesn't contain all available streams (mainly the WEBM ones) + // There is no DASH manifest available with the iOS client return getManifestUrl( "dash", - Arrays.asList(androidStreamingData, tvHtml5SimplyEmbedStreamingData)); + Arrays.asList( + new Pair<>(androidStreamingData, androidStreamingUrlsPoToken), + new Pair<>(html5StreamingData, html5StreamingUrlsPoToken)), + // Return version 7 of the DASH manifest, which is the latest one, reducing + // manifest size and allowing playback with some DASH players + "mpd_version=7"); } @Nonnull @@ -592,25 +604,44 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); // Return HLS manifest of the iOS client first because on livestreams, the HLS manifest - // returned has separated audio and video streams + // returned has separated audio and video streams and poTokens requirement do not seem to + // impact HLS formats (if a poToken is provided, it is added) // Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response + // unless a Safari macOS user agent is used return getManifestUrl( "hls", Arrays.asList( - iosStreamingData, androidStreamingData, tvHtml5SimplyEmbedStreamingData)); + new Pair<>(iosStreamingData, iosStreamingUrlsPoToken), + new Pair<>(androidStreamingData, androidStreamingUrlsPoToken), + new Pair<>(html5StreamingData, html5StreamingUrlsPoToken)), + ""); } @Nonnull - private static String getManifestUrl(@Nonnull final String manifestType, - @Nonnull final List streamingDataObjects) { + private static String getManifestUrl( + @Nonnull final String manifestType, + @Nonnull final List> streamingDataObjects, + @Nonnull final String partToAppendToManifestUrlEnd) { final String manifestKey = manifestType + "ManifestUrl"; - return streamingDataObjects.stream() - .filter(Objects::nonNull) - .map(streamingDataObject -> streamingDataObject.getString(manifestKey)) - .filter(Objects::nonNull) - .findFirst() - .orElse(""); + for (final Pair streamingDataObj : streamingDataObjects) { + if (streamingDataObj.getFirst() != null) { + final String manifestUrl = streamingDataObj.getFirst().getString(manifestKey); + if (isNullOrEmpty(manifestUrl)) { + continue; + } + + // If poToken is not null, add it to manifest URL + if (streamingDataObj.getSecond() == null) { + return manifestUrl + "?" + partToAppendToManifestUrlEnd; + } else { + return manifestUrl + "?pot=" + streamingDataObj.getSecond() + "&" + + partToAppendToManifestUrlEnd; + } + } + } + + return ""; } @Override @@ -684,7 +715,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } private void setStreamType() { - if (playerResponse.getObject("playabilityStatus").has("liveStreamability")) { + if (playerResponse.getObject(PLAYABILITY_STATUS).has("liveStreamability")) { streamType = StreamType.LIVE_STREAM; } else if (playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false)) { streamType = StreamType.POST_LIVE_STREAM; @@ -751,7 +782,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getErrorMessage() { try { - return getTextFromObject(playerResponse.getObject("playabilityStatus") + return getTextFromObject(playerResponse.getObject(PLAYABILITY_STATUS) .getObject("errorScreen").getObject("playerErrorMessageRenderer") .getObject("reason")); } catch (final NullPointerException e) { @@ -766,10 +797,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String FORMATS = "formats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String STREAMING_DATA = "streamingData"; - private static final String PLAYER = "player"; private static final String NEXT = "next"; private static final String SIGNATURE_CIPHER = "signatureCipher"; private static final String CIPHER = "cipher"; + private static final String PLAYER_CAPTIONS_TRACKLIST_RENDERER + = "playerCaptionsTracklistRenderer"; + private static final String CAPTIONS = "captions"; + private static final String PLAYABILITY_STATUS = "playabilityStatus"; @Override public void onFetchPage(@Nonnull final Downloader downloader) @@ -779,98 +813,53 @@ public class YoutubeStreamExtractor extends StreamExtractor { final Localization localization = getExtractorLocalization(); final ContentCountry contentCountry = getExtractorContentCountry(); - final JsonObject webPlayerResponse = YoutubeParsingHelper.getWebPlayerResponse( - localization, contentCountry, videoId); + final PoTokenProvider poTokenproviderInstance = poTokenProvider; + final boolean noPoTokenProviderSet = poTokenproviderInstance == null; - if (isPlayerResponseNotValid(webPlayerResponse, videoId)) { - // Check the playability status, as private and deleted videos and invalid video IDs do - // not return the ID provided in the player response - // When the requested video is playable and a different video ID is returned, it has - // the OK playability status, meaning the ExtractionException after this check will be - // thrown - checkPlayabilityStatus( - webPlayerResponse, webPlayerResponse.getObject("playabilityStatus")); - throw new ExtractionException("Initial WEB player response is not valid"); - } - - // Save the webPlayerResponse into playerResponse in the case the video cannot be played, - // so some metadata can be retrieved - playerResponse = webPlayerResponse; - - // Use the player response from the player endpoint of the desktop internal API because - // there can be restrictions on videos in the embedded player. - // E.g. if a video is age-restricted, the embedded player's playabilityStatus says that - // the video cannot be played outside of YouTube, but does not show the original message. - final JsonObject playabilityStatus = webPlayerResponse.getObject("playabilityStatus"); - - final boolean isAgeRestricted = "login_required".equalsIgnoreCase( - playabilityStatus.getString("status")) - && playabilityStatus.getString("reason", "") - .contains("age"); + fetchHtml5Client(localization, contentCountry, videoId, poTokenproviderInstance, + noPoTokenProviderSet); setStreamType(); - if (isAgeRestricted) { - fetchTvHtml5EmbedJsonPlayer(contentCountry, localization, videoId); + final PoTokenResult androidPoTokenResult = noPoTokenProviderSet ? null + : poTokenproviderInstance.getAndroidClientPoToken(videoId); - // If no streams can be fetched in the TVHTML5 simply embed client, the video should be - // age-restricted, therefore throw an AgeRestrictedContentException explicitly. - if (tvHtml5SimplyEmbedStreamingData == null) { - throw new AgeRestrictedContentException( - "This age-restricted video cannot be watched."); - } + fetchAndroidClient(localization, contentCountry, videoId, androidPoTokenResult); - // Refresh the stream type because the stream type may be not properly known for - // age-restricted videos - setStreamType(); - } else { - checkPlayabilityStatus(webPlayerResponse, playabilityStatus); - - // Fetching successfully the iOS player is mandatory to get streams - fetchIosMobileJsonPlayer(contentCountry, localization, videoId); - - try { - fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId); - } catch (final Exception ignored) { - // Ignore exceptions related to ANDROID client fetch or parsing, as it is not - // compulsory to play contents - } + if (fetchIosClient) { + final PoTokenResult iosPoTokenResult = noPoTokenProviderSet ? null + : poTokenproviderInstance.getIosClientPoToken(videoId); + fetchIosClient(localization, contentCountry, videoId, iosPoTokenResult); } - // The microformat JSON object of the content is only returned on the WEB client, - // so we need to store it instead of getting it directly from the playerResponse - playerMicroFormatRenderer = webPlayerResponse.getObject("microformat") - .getObject("playerMicroformatRenderer"); - - final byte[] body = JsonWriter.string( + final byte[] nextBody = JsonWriter.string( prepareDesktopJsonBuilder(localization, contentCountry) .value(VIDEO_ID, videoId) .value(CONTENT_CHECK_OK, true) .value(RACY_CHECK_OK, true) .done()) .getBytes(StandardCharsets.UTF_8); - nextResponse = getJsonPostResponse(NEXT, body, localization); + nextResponse = getJsonPostResponse(NEXT, nextBody, localization); } - private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse, - @Nonnull final JsonObject playabilityStatus) + private static void checkPlayabilityStatus(@Nonnull final JsonObject playabilityStatus) throws ParsingException { - String status = playabilityStatus.getString("status"); + final String status = playabilityStatus.getString("status"); if (status == null || status.equalsIgnoreCase("ok")) { return; } - // If status exist, and is not "OK", throw the specific exception based on error message - // or a ContentNotAvailableException with the reason text if it's an unknown reason. - final JsonObject newPlayabilityStatus = - youtubePlayerResponse.getObject("playabilityStatus"); - status = newPlayabilityStatus.getString("status"); - final String reason = newPlayabilityStatus.getString("reason"); + final String reason = playabilityStatus.getString("reason"); - if (status.equalsIgnoreCase("login_required") && reason == null) { - final String message = newPlayabilityStatus.getArray("messages").getString(0); - if (message != null && message.contains("private")) { - throw new PrivateContentException("This video is private."); + if (status.equalsIgnoreCase("login_required")) { + if (reason == null) { + final String message = playabilityStatus.getArray("messages").getString(0); + if (message != null && message.contains("private")) { + throw new PrivateContentException("This video is private"); + } + } else if (reason.contains("age")) { + throw new AgeRestrictedContentException( + "This age-restricted video cannot be watched anonymously"); } } @@ -879,16 +868,18 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (reason.contains("Music Premium")) { throw new YoutubeMusicPremiumContentException(); } + if (reason.contains("payment")) { throw new PaidContentException("This video is a paid video"); } + if (reason.contains("members-only")) { throw new PaidContentException("This video is only available" + " for members of the channel of this video"); } if (reason.contains("unavailable")) { - final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus + final String detailedErrorMessage = getTextFromObject(playabilityStatus .getObject("errorScreen") .getObject("playerErrorMessageRenderer") .getObject("subreason")); @@ -900,120 +891,204 @@ public class YoutubeStreamExtractor extends StreamExtractor { Objects.requireNonNullElse(detailedErrorMessage, reason)); } } + + if (reason.contains("age-restricted")) { + throw new AgeRestrictedContentException( + "This age-restricted video cannot be watched anonymously"); + } } throw new ContentNotAvailableException("Got error: \"" + reason + "\""); } - /** - * Fetch the Android Mobile API and assign the streaming data to the androidStreamingData JSON - * object. - */ - private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentCountry, - @Nonnull final Localization localization, - @Nonnull final String videoId) + private void fetchHtml5Client(@Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nullable final PoTokenProvider poTokenProviderInstance, + final boolean noPoTokenProviderSet) throws IOException, ExtractionException { - androidCpn = generateContentPlaybackNonce(); - final byte[] mobileBody = JsonWriter.string( - prepareAndroidMobileJsonBuilder(localization, contentCountry) - .object("playerRequest") - .value(VIDEO_ID, videoId) - .end() - .value("disablePlayerResponse", false) - .value(VIDEO_ID, videoId) - .value(CPN, androidCpn) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) - .getBytes(StandardCharsets.UTF_8); + html5Cpn = generateContentPlaybackNonce(); - final JsonObject androidPlayerResponse = getJsonAndroidPostResponse( - "reel/reel_item_watch", - mobileBody, - localization, - "&t=" + generateTParameter() + "&id=" + videoId + "&$fields=playerResponse"); + // Suppress NPE warning as nullability is already checked before and passed with + // noPoTokenProviderSet + //noinspection DataFlowIssue + final PoTokenResult webPoTokenResult = noPoTokenProviderSet ? null + : poTokenProviderInstance.getWebClientPoToken(videoId); + final JsonObject webPlayerResponse; + if (noPoTokenProviderSet || webPoTokenResult == null) { + webPlayerResponse = YoutubeStreamHelper.getWebMetadataPlayerResponse( + localization, contentCountry, videoId); - final JsonObject playerResponseObject = androidPlayerResponse.getObject("playerResponse"); - if (isPlayerResponseNotValid(playerResponseObject, videoId)) { - return; - } + throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId); - final JsonObject streamingData = playerResponseObject.getObject(STREAMING_DATA); - if (!isNullOrEmpty(streamingData)) { - androidStreamingData = streamingData; - if (isNullOrEmpty(playerCaptionsTracklistRenderer)) { - playerCaptionsTracklistRenderer = playerResponseObject.getObject("captions") - .getObject("playerCaptionsTracklistRenderer"); + // Save the webPlayerResponse into playerResponse in the case the video cannot be + // played, so some metadata can be retrieved + playerResponse = webPlayerResponse; + + // The microformat JSON object of the content is only returned on the WEB client, + // so we need to store it instead of getting it directly from the playerResponse + playerMicroFormatRenderer = playerResponse.getObject("microformat") + .getObject("playerMicroformatRenderer"); + + final JsonObject playabilityStatus = webPlayerResponse.getObject(PLAYABILITY_STATUS); + + if (isVideoAgeRestricted(playabilityStatus)) { + fetchHtml5EmbedClient(localization, contentCountry, videoId, + noPoTokenProviderSet ? null + : poTokenProviderInstance.getWebEmbedClientPoToken(videoId)); + } else { + checkPlayabilityStatus(playabilityStatus); + + final JsonObject tvHtml5PlayerResponse = + YoutubeStreamHelper.getTvHtml5PlayerResponse( + localization, contentCountry, videoId, html5Cpn); + + if (isPlayerResponseNotValid(tvHtml5PlayerResponse, videoId)) { + throw new ExtractionException("TVHTML5 player response is not valid"); + } + + html5StreamingData = tvHtml5PlayerResponse.getObject(STREAMING_DATA); + playerCaptionsTracklistRenderer = tvHtml5PlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + } + } else { + webPlayerResponse = YoutubeStreamHelper.getWebFullPlayerResponse( + localization, contentCountry, videoId, html5Cpn, webPoTokenResult, + YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId)); + + throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId); + + // Save the webPlayerResponse into playerResponse in the case the video cannot be + // played, so some metadata can be retrieved + playerResponse = webPlayerResponse; + + // The microformat JSON object of the content is only returned on the WEB client, + // so we need to store it instead of getting it directly from the playerResponse + playerMicroFormatRenderer = playerResponse.getObject("microformat") + .getObject("playerMicroformatRenderer"); + + final JsonObject playabilityStatus = webPlayerResponse.getObject(PLAYABILITY_STATUS); + + if (isVideoAgeRestricted(playabilityStatus)) { + fetchHtml5EmbedClient(localization, contentCountry, videoId, + poTokenProviderInstance.getWebEmbedClientPoToken(videoId)); + } else { + checkPlayabilityStatus(playabilityStatus); + html5StreamingData = webPlayerResponse.getObject(STREAMING_DATA); + playerCaptionsTracklistRenderer = webPlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + html5StreamingUrlsPoToken = webPoTokenResult.streamingDataPoToken; } } } - /** - * Fetch the iOS Mobile API and assign the streaming data to the iosStreamingData JSON - * object. - */ - private void fetchIosMobileJsonPlayer(@Nonnull final ContentCountry contentCountry, - @Nonnull final Localization localization, - @Nonnull final String videoId) - throws IOException, ExtractionException { - iosCpn = generateContentPlaybackNonce(); - final byte[] mobileBody = JsonWriter.string( - prepareIosMobileJsonBuilder(localization, contentCountry) - .value(VIDEO_ID, videoId) - .value(CPN, iosCpn) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) - .getBytes(StandardCharsets.UTF_8); - - final JsonObject iosPlayerResponse = getJsonIosPostResponse(PLAYER, - mobileBody, localization, "&t=" + generateTParameter() - + "&id=" + videoId); - - if (isPlayerResponseNotValid(iosPlayerResponse, videoId)) { - throw new ExtractionException("IOS player response is not valid"); - } - - final JsonObject streamingData = iosPlayerResponse.getObject(STREAMING_DATA); - if (!isNullOrEmpty(streamingData)) { - iosStreamingData = streamingData; - playerCaptionsTracklistRenderer = iosPlayerResponse.getObject("captions") - .getObject("playerCaptionsTracklistRenderer"); + private static void throwExceptionIfPlayerResponseNotValid( + @Nonnull final JsonObject webPlayerResponse, + @Nonnull final String videoId) throws ExtractionException { + if (isPlayerResponseNotValid(webPlayerResponse, videoId)) { + // Check the playability status, as private and deleted videos and invalid video + // IDs do not return the ID provided in the player response + // When the requested video is playable and a different video ID is returned, it + // has the OK playability status, meaning the ExtractionException after this check + // will be thrown + checkPlayabilityStatus(webPlayerResponse.getObject(PLAYABILITY_STATUS)); + throw new ExtractionException("WEB player response is not valid"); } } - /** - * Download the {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} JSON player as an embed client to bypass - * some age-restrictions and assign the streaming data to the {@code html5StreamingData} JSON - * object. - * - * @param contentCountry the content country to use - * @param localization the localization to use - * @param videoId the video id - */ - private void fetchTvHtml5EmbedJsonPlayer(@Nonnull final ContentCountry contentCountry, - @Nonnull final Localization localization, - @Nonnull final String videoId) + private void fetchHtml5EmbedClient(@Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nullable final PoTokenResult webEmbedPoTokenResult) throws IOException, ExtractionException { - tvHtml5SimplyEmbedCpn = generateContentPlaybackNonce(); + html5Cpn = generateContentPlaybackNonce(); - final JsonObject tvHtml5EmbedPlayerResponse = getJsonPostResponse(PLAYER, - createTvHtml5EmbedPlayerBody(localization, - contentCountry, - videoId, - YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId), - tvHtml5SimplyEmbedCpn), localization); + final JsonObject webEmbeddedPlayerResponse = + YoutubeStreamHelper.getWebEmbeddedPlayerResponse(localization, contentCountry, + videoId, html5Cpn, webEmbedPoTokenResult, + YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId)); - if (isPlayerResponseNotValid(tvHtml5EmbedPlayerResponse, videoId)) { - throw new ExtractionException("TVHTML5 embed player response is not valid"); + // Save the webEmbeddedPlayerResponse into playerResponse in the case the video cannot be + // played, so some metadata can be retrieved + playerResponse = webEmbeddedPlayerResponse; + + // Check if the playability status in the player response, if the age-restriction could not + // be bypassed, an exception will be thrown + checkPlayabilityStatus(webEmbeddedPlayerResponse.getObject(PLAYABILITY_STATUS)); + + if (isPlayerResponseNotValid(webEmbeddedPlayerResponse, videoId)) { + throw new ExtractionException("WEB_EMBEDDED_PLAYER player response is not valid"); } - final JsonObject streamingData = tvHtml5EmbedPlayerResponse.getObject(STREAMING_DATA); - if (!isNullOrEmpty(streamingData)) { - playerResponse = tvHtml5EmbedPlayerResponse; - tvHtml5SimplyEmbedStreamingData = streamingData; - playerCaptionsTracklistRenderer = playerResponse.getObject("captions") - .getObject("playerCaptionsTracklistRenderer"); + html5StreamingData = webEmbeddedPlayerResponse.getObject(STREAMING_DATA); + playerCaptionsTracklistRenderer = webEmbeddedPlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + if (webEmbedPoTokenResult != null) { + html5StreamingUrlsPoToken = webEmbedPoTokenResult.streamingDataPoToken; + } + } + + private void fetchAndroidClient(@Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nullable final PoTokenResult androidPoTokenResult) { + try { + androidCpn = generateContentPlaybackNonce(); + + final JsonObject androidPlayerResponse; + if (androidPoTokenResult == null) { + androidPlayerResponse = YoutubeStreamHelper.getAndroidReelPlayerResponse( + contentCountry, localization, videoId, androidCpn); + } else { + androidPlayerResponse = YoutubeStreamHelper.getAndroidPlayerResponse( + contentCountry, localization, videoId, androidCpn, + androidPoTokenResult); + } + + if (!isPlayerResponseNotValid(androidPlayerResponse, videoId)) { + androidStreamingData = androidPlayerResponse.getObject(STREAMING_DATA); + + if (isNullOrEmpty(playerCaptionsTracklistRenderer)) { + playerCaptionsTracklistRenderer = + androidPlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + } + + if (androidPoTokenResult != null) { + androidStreamingUrlsPoToken = androidPoTokenResult.streamingDataPoToken; + } + } + } catch (final Exception ignored) { + // Ignore exceptions related to ANDROID client fetch or parsing, as it is not + // compulsory to play contents + } + } + + private void fetchIosClient(@Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nullable final PoTokenResult iosPoTokenResult) { + try { + iosCpn = generateContentPlaybackNonce(); + + final JsonObject iosPlayerResponse = YoutubeStreamHelper.getIosPlayerResponse( + contentCountry, localization, videoId, iosCpn, iosPoTokenResult); + + if (!isPlayerResponseNotValid(iosPlayerResponse, videoId)) { + iosStreamingData = iosPlayerResponse.getObject(STREAMING_DATA); + + if (isNullOrEmpty(playerCaptionsTracklistRenderer)) { + playerCaptionsTracklistRenderer = iosPlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + } + + if (iosPoTokenResult != null) { + iosStreamingUrlsPoToken = iosPoTokenResult.streamingDataPoToken; + } + } + } catch (final Exception ignored) { + // Ignore exceptions related to IOS client fetch or parsing, as it is not + // compulsory to play contents } } @@ -1021,7 +1096,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { * Checks whether a player response is invalid. * *

- * If YouTube detect that requests come from a third party client, they may replace the real + * If YouTube detects that requests come from a third party client, they may replace the real * player response by another one of a video saying that this content is not available on this * app and to watch it on the latest version of YouTube. This behavior has been observed on the * {@code ANDROID} client, see @@ -1054,6 +1129,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getString("videoId")); } + private static boolean isVideoAgeRestricted(@Nonnull final JsonObject playabilityStatus) { + // This is language dependent + return "login_required".equalsIgnoreCase(playabilityStatus.getString("status")) + && playabilityStatus.getString("reason", "") + .contains("age"); + } + /*////////////////////////////////////////////////////////////////////////// // Utils //////////////////////////////////////////////////////////////////////////*/ @@ -1106,22 +1188,28 @@ public class YoutubeStreamExtractor extends StreamExtractor { java.util.stream.Stream.of( /* - Use the iosStreamingData object first because there is no n param and no - signatureCiphers in streaming URLs of the iOS client + Use the html5StreamingData object first because YouTube should have less + control on HTML5 clients, especially for poTokens - The androidStreamingData is used as second way as it isn't used on livestreams, - it doesn't return all available streams, and the Android client extraction is - more likely to break + The androidStreamingData is used as second way as the Android client extraction + is more likely to break - As age-restricted videos are not common, use tvHtml5SimplyEmbedStreamingData - last, which will be the only one not empty for age-restricted content + As iOS streaming data is affected by poTokens and not passing them should lead + to 403 responses, it should be used in the last resort */ - new Pair<>(iosStreamingData, iosCpn), - new Pair<>(androidStreamingData, androidCpn), - new Pair<>(tvHtml5SimplyEmbedStreamingData, tvHtml5SimplyEmbedCpn) - ) - .flatMap(pair -> getStreamsFromStreamingDataKey(videoId, pair.getFirst(), - streamingDataKey, itagTypeWanted, pair.getSecond())) + new Pair<>(html5StreamingData, + new Pair<>(html5Cpn, html5StreamingUrlsPoToken)), + new Pair<>(androidStreamingData, + new Pair<>(androidCpn, androidStreamingUrlsPoToken)), + new Pair<>(iosStreamingData, + new Pair<>(iosCpn, iosStreamingUrlsPoToken))) + .flatMap(pair -> getStreamsFromStreamingDataKey( + videoId, + pair.getFirst(), + streamingDataKey, + itagTypeWanted, + pair.getSecond().getFirst(), + pair.getSecond().getSecond())) .map(streamBuilderHelper) .forEachOrdered(stream -> { if (!Stream.containSimilarStream(stream, streamList)) { @@ -1146,7 +1234,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { *

    *
  • the {@link ItagItem}'s id of the stream as its id;
  • *
  • {@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and - * and as the value of {@code isUrl};
  • + * as the value of {@code isUrl}; *
  • the media format returned by the {@link ItagItem} as its media format;
  • *
  • its average bitrate with the value returned by {@link * ItagItem#getAverageBitrate()};
  • @@ -1199,7 +1287,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { *
      *
    • the {@link ItagItem}'s id of the stream as its id;
    • *
    • {@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and - * and as the value of {@code isUrl};
    • + * as the value of {@code isUrl}; *
    • the media format returned by the {@link ItagItem} as its media format;
    • *
    • whether it is video-only with the {@code areStreamsVideoOnly} parameter
    • *
    • the {@link ItagItem};
    • @@ -1255,7 +1343,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final JsonObject streamingData, final String streamingDataKey, @Nonnull final ItagItem.ItagType itagTypeWanted, - @Nonnull final String contentPlaybackNonce) { + @Nonnull final String contentPlaybackNonce, + @Nullable final String poToken) { if (streamingData == null || !streamingData.has(streamingDataKey)) { return java.util.stream.Stream.empty(); } @@ -1268,7 +1357,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final ItagItem itagItem = ItagItem.getItag(formatData.getInt("itag")); if (itagItem.itagType == itagTypeWanted) { return buildAndAddItagInfoToList(videoId, formatData, itagItem, - itagItem.itagType, contentPlaybackNonce); + itagItem.itagType, contentPlaybackNonce, poToken); } } catch (final ExtractionException ignored) { // If the itag is not supported, the n parameter of HTML5 clients cannot be @@ -1284,7 +1373,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull final JsonObject formatData, @Nonnull final ItagItem itagItem, @Nonnull final ItagItem.ItagType itagType, - @Nonnull final String contentPlaybackNonce) throws ExtractionException { + @Nonnull final String contentPlaybackNonce, + @Nullable final String poToken) throws ExtractionException { String streamUrl; if (formatData.has("url")) { streamUrl = formatData.getString("url"); @@ -1298,9 +1388,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + signature; } - // Add the content playback nonce to the stream URL - streamUrl += "&" + CPN + "=" + contentPlaybackNonce; - // Decode the n parameter if it is present // If it cannot be decoded, the stream cannot be used as streaming URLs return HTTP 403 // responses if it has not the right value @@ -1310,6 +1397,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { streamUrl = YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated( videoId, streamUrl); + // Add the content playback nonce to the stream URL + streamUrl += "&" + CPN + "=" + contentPlaybackNonce; + + // Add the poToken, if there is one + if (poToken != null) { + streamUrl += "&pot=" + poToken; + } + final JsonObject initRange = formatData.getObject("initRange"); final JsonObject indexRange = formatData.getObject("indexRange"); final String mimeType = formatData.getString("mimeType", ""); @@ -1587,4 +1682,53 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getObject("results") .getArray("contents")); } + + /** + * Set the {@link PoTokenProvider} instance to be used for fetching {@code poToken}s. + * + *

      + * This method allows setting an implementation of {@link PoTokenProvider} which will be used + * to obtain poTokens required for YouTube player requests and streaming URLs. These tokens + * are used by YouTube to verify the integrity of the user's device or browser and are required + * for playback with several clients. + *

      + * + *

      + * Without a {@link PoTokenProvider}, the extractor makes its best effort to fetch as many + * streams as possible, but without {@code poToken}s, some formats may be not available or + * fetching may be slower due to additional requests done to get streams. + *

      + * + *

      + * Note that any provider change will be only applied on the next {@link #fetchPage()} request. + *

      + * + * @param poTokenProvider the {@link PoTokenProvider} instance to set, which can be null to + * remove a provider already passed + * @see PoTokenProvider + */ + @SuppressWarnings("unused") + public static void setPoTokenProvider(@Nullable final PoTokenProvider poTokenProvider) { + YoutubeStreamExtractor.poTokenProvider = poTokenProvider; + } + + /** + * Set whether to fetch the iOS player responses. + * + *

      + * This method allows fetching the iOS player response, which can be useful in scenarios where + * streams from the iOS player response are needed, especially HLS manifests. + *

      + * + *

      + * Note that at the time of writing, YouTube is rolling out a {@code poToken} requirement on + * this client, formats from HLS manifests do not seem to be affected. + *

      + * + * @param fetchIosClient whether to fetch the iOS client + */ + @SuppressWarnings("unused") + public static void setFetchIosClient(final boolean fetchIosClient) { + YoutubeStreamExtractor.fetchIosClient = fetchIosClient; + } } From 38e2b67cb7c08fb1b517d5811e164735e10680b6 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Tue, 28 Jan 2025 21:44:50 +0100 Subject: [PATCH 06/17] [YouTube] Remove unused methods and constants in YoutubeParsingHelper --- .../youtube/YoutubeParsingHelper.java | 201 ------------------ 1 file changed, 201 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index ece76ecef..6b8742531 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -155,11 +155,6 @@ public final class YoutubeParsingHelper { */ public static final String RACY_CHECK_OK = "racyCheckOk"; - /** - * The hardcoded client version used for InnerTube requests with the TV HTML5 embed client. - */ - private static final String TVHTML5_SIMPLY_EMBED_CLIENT_VERSION = "2.0"; - private static String clientVersion; private static String youtubeMusicClientVersion; @@ -1065,43 +1060,6 @@ public final class YoutubeParsingHelper { + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization))); } - public static JsonObject getJsonAndroidPostResponse( - final String endpoint, - final byte[] body, - @Nonnull final Localization localization, - @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { - return getMobilePostResponse(endpoint, body, localization, - getAndroidUserAgent(localization), endPartOfUrlRequest); - } - - public static JsonObject getJsonIosPostResponse( - final String endpoint, - final byte[] body, - @Nonnull final Localization localization, - @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { - return getMobilePostResponse(endpoint, body, localization, getIosUserAgent(localization), - endPartOfUrlRequest); - } - - private static JsonObject getMobilePostResponse( - final String endpoint, - final byte[] body, - @Nonnull final Localization localization, - @Nonnull final String userAgent, - @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { - final var headers = Map.of("User-Agent", List.of(userAgent), - "X-Goog-Api-Format-Version", List.of("2")); - - final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?" - + DISABLE_PRETTY_PRINT_PARAMETER; - - return JsonUtils.toJsonObject(getValidJsonResponseBody( - getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest) - ? baseEndpointUrl - : baseEndpointUrl + endPartOfUrlRequest, - headers, body, localization))); - } - @Nonnull public static JsonBuilder prepareDesktopJsonBuilder( @Nonnull final Localization localization, @@ -1148,165 +1106,6 @@ public final class YoutubeParsingHelper { // @formatter:on } - @Nonnull - public static JsonBuilder prepareAndroidMobileJsonBuilder( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry) { - // @formatter:off - return JsonObject.builder() - .object("context") - .object("client") - .value("clientName", "ANDROID") - .value("clientVersion", ANDROID_YOUTUBE_CLIENT_VERSION) - .value("platform", "MOBILE") - .value("osName", "Android") - .value("osVersion", "14") - /* - A valid Android SDK version is required to be sure to get a valid player - response - If this parameter is not provided, the player response is replaced by an - error saying the message "The following content is not available on this - app. Watch this content on the latest version on YouTube" (it was - previously a 5-minute video with this message) - See https://github.com/TeamNewPipe/NewPipe/issues/8713 - The Android SDK version corresponding to the Android version used in - requests is sent - */ - .value("androidSdkVersion", 34) - .value("hl", localization.getLocalizationCode()) - .value("gl", contentCountry.getCountryCode()) - .value("utcOffsetMinutes", 0) - .end() - .object("request") - .array("internalExperimentFlags") - .end() - .value("useSsl", true) - .end() - .object("user") - // TODO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) - .value("lockedSafetyMode", false) - .end() - .end(); - // @formatter:on - } - - @Nonnull - public static JsonBuilder prepareIosMobileJsonBuilder( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry) { - // @formatter:off - return JsonObject.builder() - .object("context") - .object("client") - .value("clientName", "IOS") - .value("clientVersion", IOS_YOUTUBE_CLIENT_VERSION) - .value("deviceMake", "Apple") - // Device model is required to get 60fps streams - .value("deviceModel", IOS_DEVICE_MODEL) - .value("platform", "MOBILE") - .value("osName", "iOS") - .value("osVersion", IOS_OS_VERSION) - .value("visitorData", randomVisitorData(contentCountry)) - .value("hl", localization.getLocalizationCode()) - .value("gl", contentCountry.getCountryCode()) - .value("utcOffsetMinutes", 0) - .end() - .object("request") - .array("internalExperimentFlags") - .end() - .value("useSsl", true) - .end() - .object("user") - // TODO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) - .value("lockedSafetyMode", false) - .end() - .end(); - // @formatter:on - } - - @Nonnull - public static JsonBuilder prepareTvHtml5EmbedJsonBuilder( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nonnull final String videoId) { - // @formatter:off - return JsonObject.builder() - .object("context") - .object("client") - .value("clientName", "TVHTML5_SIMPLY_EMBEDDED_PLAYER") - .value("clientVersion", TVHTML5_SIMPLY_EMBED_CLIENT_VERSION) - .value("clientScreen", "EMBED") - .value("platform", "TV") - .value("hl", localization.getLocalizationCode()) - .value("gl", contentCountry.getCountryCode()) - .value("utcOffsetMinutes", 0) - .end() - .object("thirdParty") - .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId) - .end() - .object("request") - .array("internalExperimentFlags") - .end() - .value("useSsl", true) - .end() - .object("user") - // TODO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) - .value("lockedSafetyMode", false) - .end() - .end(); - // @formatter:on - } - - @Nonnull - public static JsonObject getWebPlayerResponse( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nonnull final String videoId) throws IOException, ExtractionException { - final byte[] body = JsonWriter.string( - prepareDesktopJsonBuilder(localization, contentCountry) - .value(VIDEO_ID, videoId) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) - .getBytes(StandardCharsets.UTF_8); - final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER - + "&$fields=microformat,playabilityStatus,storyboards,videoDetails"; - - return JsonUtils.toJsonObject(getValidJsonResponseBody( - getDownloader().postWithContentTypeJson( - url, getYouTubeHeaders(), body, localization))); - } - - @Nonnull - public static byte[] createTvHtml5EmbedPlayerBody( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nonnull final String videoId, - @Nonnull final Integer sts, - @Nonnull final String contentPlaybackNonce) { - // @formatter:off - return JsonWriter.string( - prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId) - .object("playbackContext") - .object("contentPlaybackContext") - // Signature timestamp from the JavaScript base player is needed to get - // working obfuscated URLs - .value("signatureTimestamp", sts) - .value("referer", "https://www.youtube.com/watch?v=" + videoId) - .end() - .end() - .value(CPN, contentPlaybackNonce) - .value(VIDEO_ID, videoId) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) - .getBytes(StandardCharsets.UTF_8); - // @formatter:on - } - /** * Get the user-agent string used as the user-agent for InnerTube requests with the Android * client. From 9333d7fcdc85e318f4368a6b932b8052d2a8a1ed Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Thu, 30 Jan 2025 20:17:31 +0100 Subject: [PATCH 07/17] [YouTube] Update DASH manifest creation clients' handling - Use POST requests with the same body as official clients do; - Update methods checking the client streaming URLs come from: - Replace TVHTML5_SIMPLY_EMBEDDED_PLAYER by TVHTML5; - Add WEB_EMBEDDED_PLAYER. --- .../youtube/YoutubeParsingHelper.java | 29 +++++++++++++------ .../YoutubeDashManifestCreatorsUtils.java | 13 +++++---- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 6b8742531..782ec6cb4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -180,8 +180,10 @@ public final class YoutubeParsingHelper { "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; private static final Pattern C_WEB_PATTERN = Pattern.compile("&c=WEB"); - private static final Pattern C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN = - Pattern.compile("&c=TVHTML5_SIMPLY_EMBEDDED_PLAYER"); + private static final Pattern C_WEB_EMBEDDED_PLAYER_PATTERN = + Pattern.compile("&c=WEB_EMBEDDED_PLAYER"); + private static final Pattern C_TVHTML5_PLAYER_PATTERN = + Pattern.compile("&c=TVHTML5"); private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID"); private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS"); @@ -1394,15 +1396,24 @@ public final class YoutubeParsingHelper { } /** - * Check if the streaming URL is a URL from the YouTube {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} - * client. + * Check if the streaming URL is from the YouTube {@code WEB_EMBEDDED_PLAYER} client. * - * @param url the streaming URL on which check if it's a {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} - * streaming URL. - * @return true if it's a {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} streaming URL, false otherwise + * @param url the streaming URL to be checked. + * @return true if it's a {@code WEB_EMBEDDED_PLAYER} streaming URL, false otherwise */ - public static boolean isTvHtml5SimplyEmbeddedPlayerStreamingUrl(@Nonnull final String url) { - return Parser.isMatch(C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN, url); + public static boolean isWebEmbeddedPlayerStreamingUrl(@Nonnull final String url) { + return Parser.isMatch(C_WEB_EMBEDDED_PLAYER_PATTERN, url); + } + + /** + * Check if the streaming URL is a URL from the YouTube {@code TVHTML5} client. + * + * @param url the streaming URL on which check if it's a {@code TVHTML5} + * streaming URL. + * @return true if it's a {@code TVHTML5} streaming URL, false otherwise + */ + public static boolean isTvHtml5StreamingUrl(@Nonnull final String url) { + return Parser.isMatch(C_TVHTML5_PLAYER_PATTERN, url); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java index 0b4c41157..49dce80e4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java @@ -5,8 +5,9 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5StreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebEmbeddedPlayerStreamingUrl; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import org.schabi.newpipe.extractor.MediaFormat; @@ -582,8 +583,8 @@ public final class YoutubeDashManifestCreatorsUtils { * This method fetches, for OTF streams and for post-live-DVR streams: *
        *
      • the base URL of the stream, to which are appended {@link #SQ_0} and - * {@link #RN_0} parameters, with a {@code GET} request for streaming URLs from HTML5 - * clients and a {@code POST} request for the ones from the {@code ANDROID} and the + * {@link #RN_0} parameters, with a {@code POS} request for streaming URLs from + * {@code WEB}, {@code TVHTML5}, {@code WEB_EMBEDDED_PLAYER}, {@code ANDROID} and * {@code IOS} clients;
      • *
      • for streaming URLs from HTML5 clients, the {@link #ALR_YES} param is also added. *
      • @@ -602,7 +603,8 @@ public final class YoutubeDashManifestCreatorsUtils { final DeliveryType deliveryType) throws CreationException { final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl) - || isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl); + || isTvHtml5StreamingUrl(baseStreamingUrl) + || isWebEmbeddedPlayerStreamingUrl(baseStreamingUrl); final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl); final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl); if (isHtml5StreamingUrl) { @@ -748,7 +750,8 @@ public final class YoutubeDashManifestCreatorsUtils { int redirectsCount = 0; while (!responseMimeType.equals(responseMimeTypeExpected) && redirectsCount < MAXIMUM_REDIRECT_COUNT) { - final Response response = downloader.get(streamingUrl, headers); + final byte[] html5Body = new byte[] {0x78, 0}; + final Response response = downloader.post(streamingUrl, headers, html5Body); final int responseCode = response.responseCode(); if (responseCode != 200) { From 94541d2d94f17dbc540e172f89749eb9eefd7112 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 31 Jan 2025 10:30:56 +0100 Subject: [PATCH 08/17] [YouTube] Add utility data class to store client and device info This new class, InnertubeClientRequestInfo, composed of two mutable subclasses, ClientInfo and DeviceInfo, allows to store client and device info in a better way, without requiring to pass more than 10 method parameters like in YoutubeStreamHelper currently. Mutability has been added in order to allow changing some fields easily, especially visitorData. --- .../youtube/InnertubeClientRequestInfo.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java new file mode 100644 index 000000000..c1a91b0f9 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java @@ -0,0 +1,73 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public final class InnertubeClientRequestInfo { + + @Nonnull + public ClientInfo clientInfo; + @Nonnull + public DeviceInfo deviceInfo; + + public static final class ClientInfo { + + @Nonnull + public String clientName; + @Nonnull + public String clientVersion; + @Nonnull + public String clientScreen; + @Nullable + public String clientId; + @Nullable + public String visitorData; + + public ClientInfo(@Nonnull final String clientName, + @Nonnull final String clientVersion, + @Nonnull final String clientScreen, + @Nullable final String clientId, + @Nullable final String visitorData) { + this.clientName = clientName; + this.clientVersion = clientVersion; + this.clientScreen = clientScreen; + this.clientId = clientId; + this.visitorData = visitorData; + } + } + + public static final class DeviceInfo { + + @Nonnull + public String platform; + @Nullable + public String deviceMake; + @Nullable + public String deviceModel; + @Nullable + public String osName; + @Nullable + public String osVersion; + public int androidSdkVersion; + + public DeviceInfo(@Nonnull final String platform, + @Nullable final String deviceMake, + @Nullable final String deviceModel, + @Nullable final String osName, + @Nullable final String osVersion, + final int androidSdkVersion) { + this.platform = platform; + this.deviceMake = deviceMake; + this.deviceModel = deviceModel; + this.osName = osName; + this.osVersion = osVersion; + this.androidSdkVersion = androidSdkVersion; + } + } + + public InnertubeClientRequestInfo(@Nonnull final ClientInfo clientInfo, + @Nonnull final DeviceInfo deviceInfo) { + this.clientInfo = clientInfo; + this.deviceInfo = deviceInfo; + } +} From 9e45c8001abe549dd59271b07101e6110c620249 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 31 Jan 2025 10:48:13 +0100 Subject: [PATCH 09/17] [YouTube] Do not send a visitorData for every InnerTube request As YouTube is disabling ability to use a random visitor ID in a visitorData on player requests and BotGuard challenges, it shouldn't matter if we use a random one or not for other request types. --- .../services/youtube/YoutubeParsingHelper.java | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 782ec6cb4..d6efbfb94 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -1065,22 +1065,7 @@ public final class YoutubeParsingHelper { @Nonnull public static JsonBuilder prepareDesktopJsonBuilder( @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry) - throws IOException, ExtractionException { - return prepareDesktopJsonBuilder(localization, contentCountry, null); - } - - @Nonnull - public static JsonBuilder prepareDesktopJsonBuilder( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nullable final String visitorData) - throws IOException, ExtractionException { - String vData = visitorData; - if (vData == null) { - vData = randomVisitorData(contentCountry); - } - + @Nonnull final ContentCountry contentCountry) throws IOException, ExtractionException { // @formatter:off return JsonObject.builder() .object("context") @@ -1092,7 +1077,6 @@ public final class YoutubeParsingHelper { .value("originalUrl", "https://www.youtube.com") .value("platform", DESKTOP_CLIENT_PLATFORM) .value("utcOffsetMinutes", 0) - .value("visitorData", vData) .end() .object("request") .array("internalExperimentFlags") From 6533a333fa840d25f56d3ed4670cbe0f322ef033 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 31 Jan 2025 10:51:07 +0100 Subject: [PATCH 10/17] [YouTube] Add ability to get a visitorData from InnerTube --- .../youtube/YoutubeParsingHelper.java | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index d6efbfb94..8e9643efc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -1495,4 +1495,90 @@ public final class YoutubeParsingHelper { return null; } } + + public static String getVisitorDataFromInnertube( + @Nonnull final InnertubeClientRequestInfo innertubeClientRequestInfo, + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final Map> httpHeaders, + @Nonnull final String innerTubeDomain, + @Nullable final String embedUrl) throws IOException, ExtractionException { + final JsonBuilder builder = prepareJsonBuilder( + localization, contentCountry, innertubeClientRequestInfo, embedUrl); + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + final String visitorData = JsonUtils.toJsonObject(getValidJsonResponseBody(getDownloader() + .postWithContentTypeJson(innerTubeDomain + "visitor_id", httpHeaders, body))) + .getObject("responseContext") + .getString("visitorData"); + + if (isNullOrEmpty(visitorData)) { + throw new ParsingException("Could not get visitorData"); + } + + return visitorData; + } + + @Nonnull + static JsonBuilder prepareJsonBuilder( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final InnertubeClientRequestInfo innertubeClientRequestInfo, + @Nullable final String embedUrl) { + final JsonBuilder builder = JsonObject.builder() + .object("context") + .object("client") + .value("clientName", innertubeClientRequestInfo.clientInfo.clientName) + .value("clientVersion", innertubeClientRequestInfo.clientInfo.clientVersion) + .value("clientScreen", innertubeClientRequestInfo.clientInfo.clientScreen) + .value("platform", innertubeClientRequestInfo.deviceInfo.platform); + + if (innertubeClientRequestInfo.clientInfo.visitorData != null) { + builder.value("visitorData", innertubeClientRequestInfo.clientInfo.visitorData); + } + + if (innertubeClientRequestInfo.deviceInfo.deviceMake != null) { + builder.value("deviceMake", innertubeClientRequestInfo.deviceInfo.deviceMake); + } + if (innertubeClientRequestInfo.deviceInfo.deviceModel != null) { + builder.value("deviceModel", innertubeClientRequestInfo.deviceInfo.deviceModel); + } + if (innertubeClientRequestInfo.deviceInfo.osName != null) { + builder.value("osName", innertubeClientRequestInfo.deviceInfo.osName); + } + if (innertubeClientRequestInfo.deviceInfo.osVersion != null) { + builder.value("osVersion", innertubeClientRequestInfo.deviceInfo.osVersion); + } + if (innertubeClientRequestInfo.deviceInfo.androidSdkVersion > 0) { + builder.value("androidSdkVersion", + innertubeClientRequestInfo.deviceInfo.androidSdkVersion); + } + + builder.value("hl", localization.getLocalizationCode()) + .value("gl", contentCountry.getCountryCode()) + .value("utcOffsetMinutes", 0) + .end(); + + if (embedUrl != null) { + builder.object("thirdParty") + .value("embedUrl", embedUrl) + .end(); + } + + builder.object("request") + .array("internalExperimentFlags") + .end() + .value("useSsl", true) + .end() + .object("user") + // TODO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) + .end() + .end(); + + return builder; + } } From 862a607fc6da1611be86474c41faef29d15c973a Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 31 Jan 2025 10:51:51 +0100 Subject: [PATCH 11/17] [YouTube] Add IOS and ANDROID client IDs in ClientsConstants --- .../newpipe/extractor/services/youtube/ClientsConstants.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java index 22b002e8f..4b5e291ff 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java @@ -47,6 +47,7 @@ final class ClientsConstants { // IOS (iOS YouTube app) client fields + static final String IOS_CLIENT_ID = "5"; static final String IOS_CLIENT_NAME = "IOS"; /** @@ -101,6 +102,7 @@ final class ClientsConstants { // ANDROID (Android YouTube app) client fields + static final String ANDROID_CLIENT_ID = "3"; static final String ANDROID_CLIENT_NAME = "ANDROID"; /** From d08331dbcfc70039a67d8765898d2fc5f5107252 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 31 Jan 2025 11:23:58 +0100 Subject: [PATCH 12/17] [YouTube] Add ability to use the guide endpoint to get a visitorData Some clients like TVHTML5 are not allowed on the visitor_id endpoint (with this client, a 400 HTTP response is returned with a precondition check failed error). Also disable pretty printing for these requests, like we do for others. --- .../extractor/services/youtube/YoutubeParsingHelper.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 8e9643efc..a41773726 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -1502,7 +1502,8 @@ public final class YoutubeParsingHelper { @Nonnull final ContentCountry contentCountry, @Nonnull final Map> httpHeaders, @Nonnull final String innerTubeDomain, - @Nullable final String embedUrl) throws IOException, ExtractionException { + @Nullable final String embedUrl, + final boolean useGuideEndpoint) throws IOException, ExtractionException { final JsonBuilder builder = prepareJsonBuilder( localization, contentCountry, innertubeClientRequestInfo, embedUrl); @@ -1510,7 +1511,10 @@ public final class YoutubeParsingHelper { .getBytes(StandardCharsets.UTF_8); final String visitorData = JsonUtils.toJsonObject(getValidJsonResponseBody(getDownloader() - .postWithContentTypeJson(innerTubeDomain + "visitor_id", httpHeaders, body))) + .postWithContentTypeJson( + innerTubeDomain + (useGuideEndpoint ? "guide" : "visitor_id") + + "?" + DISABLE_PRETTY_PRINT_PARAMETER, + httpHeaders, body))) .getObject("responseContext") .getString("visitorData"); From 61f67854ed3801de967b8cbe5790e688e685da42 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 31 Jan 2025 11:25:02 +0100 Subject: [PATCH 13/17] [YouTube] Get visitorData for player requests if not provided and do some fixes visitorData are get using InnertubeClientRequestInfo and YoutubeParsingHelper.prepareJsonBuilder, which is replacing the corresponding method in YoutubeStreamHelper, removed in this commit. Also fix some bugs like JsonBuilder usages in some places and remove HLS manifest filtering for the iOS client, as we still use for now the full player response, in the case having streams requiring poTokens after some time as a last resort is useful. --- .../services/youtube/YoutubeStreamHelper.java | 464 ++++++++---------- 1 file changed, 218 insertions(+), 246 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java index 7d2da95f8..1b17058a1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java @@ -17,10 +17,12 @@ import java.util.List; import java.util.Map; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_ID; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_NAME; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.EMBED_CLIENT_SCREEN; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_ID; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_NAME; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; @@ -34,6 +36,7 @@ import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVH import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MODEL_AND_OS_NAME; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WATCH_CLIENT_SCREEN; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_NAME; @@ -54,7 +57,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getOriginReferrerHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder; public final class YoutubeStreamHelper { @@ -71,20 +74,32 @@ public final class YoutubeStreamHelper { @Nonnull final Localization localization, @Nonnull final ContentCountry contentCountry, @Nonnull final String videoId) throws IOException, ExtractionException { - final JsonBuilder builder = prepareJsonBuilder( - localization, - contentCountry, - WEB_CLIENT_NAME, - getClientVersion(), - WATCH_CLIENT_SCREEN, - DESKTOP_CLIENT_PLATFORM, - YoutubeParsingHelper.randomVisitorData(contentCountry), - null, - null, - null, - null, - null, - -1); + final InnertubeClientRequestInfo innertubeClientRequestInfo = + new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo( + WEB_CLIENT_NAME, + getClientVersion(), + WATCH_CLIENT_SCREEN, + WEB_CLIENT_ID, + null), + new InnertubeClientRequestInfo.DeviceInfo( + DESKTOP_CLIENT_PLATFORM, + null, + null, + null, + null, + -1)); + + final Map> headers = getYouTubeHeaders(); + + // We must always pass a valid visitorData to get valid player responses, which needs to be + // got from YouTube + innertubeClientRequestInfo.clientInfo.visitorData = + YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo, + localization, contentCountry, headers, YOUTUBEI_V1_URL, null, false); + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, null); addVideoIdCpnAndOkChecks(builder, videoId, null); @@ -96,7 +111,7 @@ public final class YoutubeStreamHelper { return JsonUtils.toJsonObject(getValidJsonResponseBody( getDownloader().postWithContentTypeJson( - url, getYouTubeHeaders(), body, localization))); + url, headers, body, localization))); } @Nonnull @@ -105,20 +120,38 @@ public final class YoutubeStreamHelper { @Nonnull final ContentCountry contentCountry, @Nonnull final String videoId, @Nonnull final String cpn) throws IOException, ExtractionException { - final JsonBuilder builder = prepareJsonBuilder( - localization, - contentCountry, - TVHTML5_CLIENT_NAME, - TVHTML5_CLIENT_VERSION, - WATCH_CLIENT_SCREEN, - TVHTML5_CLIENT_PLATFORM, - YoutubeParsingHelper.randomVisitorData(contentCountry), - TVHTML5_DEVICE_MAKE, - TVHTML5_DEVICE_MODEL_AND_OS_NAME, - TVHTML5_DEVICE_MODEL_AND_OS_NAME, - "", - null, - -1); + final InnertubeClientRequestInfo innertubeClientRequestInfo = + new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo( + TVHTML5_CLIENT_NAME, + TVHTML5_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, + TVHTML5_CLIENT_ID, + null), + new InnertubeClientRequestInfo.DeviceInfo( + TVHTML5_CLIENT_PLATFORM, + TVHTML5_DEVICE_MAKE, + TVHTML5_DEVICE_MODEL_AND_OS_NAME, + TVHTML5_DEVICE_MODEL_AND_OS_NAME, + "", + -1)); + + final Map> headers = new HashMap<>( + getClientHeaders(TVHTML5_CLIENT_ID, TVHTML5_CLIENT_VERSION)); + headers.putAll(getOriginReferrerHeaders("https://www.youtube.com")); + headers.put("User-Agent", List.of(TVHTML5_USER_AGENT)); + + // We must always pass a valid visitorData to get valid player responses, which needs to be + // got from YouTube + // For some reason, the TVHTML5 client doesn't support the visitor_id endpoint, use the + // guide one instead, which is quite lightweight + innertubeClientRequestInfo.clientInfo.visitorData = + YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo, + localization, contentCountry, headers, YOUTUBEI_V1_URL, null, true); + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, null); + addVideoIdCpnAndOkChecks(builder, videoId, cpn); final byte[] body = JsonWriter.string(builder.done()) @@ -126,14 +159,8 @@ public final class YoutubeStreamHelper { final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; - final Map> headers = new HashMap<>( - getClientHeaders(TVHTML5_CLIENT_ID, TVHTML5_CLIENT_VERSION)); - headers.putAll(getOriginReferrerHeaders("https://www.youtube.com")); - headers.put("User-Agent", List.of(TVHTML5_USER_AGENT)); - return JsonUtils.toJsonObject(getValidJsonResponseBody( - getDownloader().postWithContentTypeJson( - url, headers, body, localization))); + getDownloader().postWithContentTypeJson(url, headers, body, localization))); } @Nonnull @@ -144,32 +171,34 @@ public final class YoutubeStreamHelper { @Nonnull final String cpn, @Nonnull final PoTokenResult webPoTokenResult, final int signatureTimestamp) throws IOException, ExtractionException { - final JsonBuilder builder = prepareJsonBuilder( - localization, - contentCountry, - WEB_CLIENT_NAME, - getClientVersion(), - WATCH_CLIENT_SCREEN, - DESKTOP_CLIENT_PLATFORM, - webPoTokenResult.visitorData, - null, - null, - null, - null, - null, - -1); + final InnertubeClientRequestInfo innertubeClientRequestInfo = + new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo( + WEB_CLIENT_NAME, + getClientVersion(), + WATCH_CLIENT_SCREEN, + WEB_CLIENT_ID, + webPoTokenResult.visitorData), + new InnertubeClientRequestInfo.DeviceInfo( + DESKTOP_CLIENT_PLATFORM, + null, + null, + null, + null, + -1)); + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, null); addVideoIdCpnAndOkChecks(builder, videoId, cpn); - addPlaybackContext( - builder, - BASE_YT_DESKTOP_WATCH_URL + videoId, - signatureTimestamp); + addPlaybackContext(builder, BASE_YT_DESKTOP_WATCH_URL + videoId, signatureTimestamp); addPoToken(builder, webPoTokenResult.playerRequestPoToken); final byte[] body = JsonWriter.string(builder.end().done()) .getBytes(StandardCharsets.UTF_8); + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; return JsonUtils.toJsonObject(getValidJsonResponseBody( @@ -185,29 +214,41 @@ public final class YoutubeStreamHelper { @Nonnull final String cpn, @Nullable final PoTokenResult webEmbeddedPoTokenResult, final int signatureTimestamp) throws IOException, ExtractionException { - final JsonBuilder builder = prepareJsonBuilder( - localization, - contentCountry, - WEB_EMBEDDED_CLIENT_NAME, - WEB_REMIX_HARDCODED_CLIENT_VERSION, - EMBED_CLIENT_SCREEN, - DESKTOP_CLIENT_PLATFORM, - webEmbeddedPoTokenResult == null - ? YoutubeParsingHelper.randomVisitorData(contentCountry) - : webEmbeddedPoTokenResult.visitorData, - null, - null, - null, - null, - BASE_YT_DESKTOP_WATCH_URL + videoId, - -1); + final InnertubeClientRequestInfo innertubeClientRequestInfo = + new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo( + WEB_EMBEDDED_CLIENT_NAME, + WEB_REMIX_HARDCODED_CLIENT_VERSION, + EMBED_CLIENT_SCREEN, + WEB_EMBEDDED_CLIENT_ID, + null), + new InnertubeClientRequestInfo.DeviceInfo( + DESKTOP_CLIENT_PLATFORM, + null, + null, + null, + null, + -1)); + + final Map> headers = new HashMap<>( + getClientHeaders(WEB_EMBEDDED_CLIENT_ID, WEB_EMBEDDED_CLIENT_VERSION)); + headers.putAll(getOriginReferrerHeaders("https://www.youtube.com")); + + final String embedUrl = BASE_YT_DESKTOP_WATCH_URL + videoId; + + // We must always pass a valid visitorData to get valid player responses, which needs to be + // got from YouTube + innertubeClientRequestInfo.clientInfo.visitorData = webEmbeddedPoTokenResult == null + ? YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo, + localization, contentCountry, headers, YOUTUBEI_V1_URL, embedUrl, false) + : webEmbeddedPoTokenResult.visitorData; + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, embedUrl); addVideoIdCpnAndOkChecks(builder, videoId, cpn); - addPlaybackContext( - builder, - BASE_YT_DESKTOP_WATCH_URL + videoId, - signatureTimestamp); + addPlaybackContext(builder, embedUrl, signatureTimestamp); if (webEmbeddedPoTokenResult != null) { addPoToken(builder, webEmbeddedPoTokenResult.playerRequestPoToken); @@ -217,13 +258,8 @@ public final class YoutubeStreamHelper { .getBytes(StandardCharsets.UTF_8); final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; - final Map> headers = new HashMap<>( - getClientHeaders(WEB_EMBEDDED_CLIENT_ID, WEB_EMBEDDED_CLIENT_VERSION)); - headers.putAll(getOriginReferrerHeaders("https://www.youtube.com")); - return JsonUtils.toJsonObject(getValidJsonResponseBody( - getDownloader().postWithContentTypeJson( - url, headers, body, localization))); + getDownloader().postWithContentTypeJson(url, headers, body, localization))); } public static JsonObject getAndroidPlayerResponse( @@ -233,34 +269,40 @@ public final class YoutubeStreamHelper { @Nonnull final String cpn, @Nonnull final PoTokenResult androidPoTokenResult) throws IOException, ExtractionException { + final InnertubeClientRequestInfo innertubeClientRequestInfo = + new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo( + ANDROID_CLIENT_NAME, + ANDROID_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, + ANDROID_CLIENT_ID, + androidPoTokenResult.visitorData), + new InnertubeClientRequestInfo.DeviceInfo( + MOBILE_CLIENT_PLATFORM, + null, + null, + "Android", + "15", + 35)); - final JsonBuilder builder = prepareJsonBuilder( - localization, - contentCountry, - ANDROID_CLIENT_NAME, - ANDROID_CLIENT_VERSION, - WATCH_CLIENT_SCREEN, - MOBILE_CLIENT_PLATFORM, - androidPoTokenResult.visitorData, - null, - null, - "Android", - "15", - null, - 35); + final Map> headers = + getMobileClientHeaders(getAndroidUserAgent(localization)); + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, null); addVideoIdCpnAndOkChecks(builder, videoId, cpn); addPoToken(builder, androidPoTokenResult.playerRequestPoToken); - final byte[] body = JsonWriter.string(builder.end().done()) + final byte[] body = JsonWriter.string(builder.done()) .getBytes(StandardCharsets.UTF_8); - return getJsonAndroidPostResponse( - PLAYER, - body, - localization, - "&t=" + generateTParameter() + "&id=" + videoId); + final String url = YOUTUBEI_V1_GAPIS_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER + + "&t=" + generateTParameter() + "&id=" + videoId; + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson(url, headers, body, localization))); } public static JsonObject getAndroidReelPlayerResponse( @@ -268,20 +310,33 @@ public final class YoutubeStreamHelper { @Nonnull final Localization localization, @Nonnull final String videoId, @Nonnull final String cpn) throws IOException, ExtractionException { - final JsonBuilder builder = prepareJsonBuilder( - localization, - contentCountry, - ANDROID_CLIENT_NAME, - ANDROID_CLIENT_VERSION, - WATCH_CLIENT_SCREEN, - MOBILE_CLIENT_PLATFORM, - YoutubeParsingHelper.randomVisitorData(contentCountry), - null, - null, - "Android", - "15", - null, - 35); + final InnertubeClientRequestInfo innertubeClientRequestInfo = + new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo( + ANDROID_CLIENT_NAME, + ANDROID_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, + ANDROID_CLIENT_ID, + null), + new InnertubeClientRequestInfo.DeviceInfo( + MOBILE_CLIENT_PLATFORM, + null, + null, + "Android", + "15", + 35)); + + final Map> headers = + getMobileClientHeaders(getAndroidUserAgent(localization)); + + // We must always pass a valid visitorData to get valid player responses, which needs to be + // got from YouTube + innertubeClientRequestInfo.clientInfo.visitorData = + YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo, + localization, contentCountry, headers, YOUTUBEI_V1_GAPIS_URL, null, false); + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, null); addVideoIdCpnAndOkChecks(builder, videoId, cpn); @@ -290,16 +345,16 @@ public final class YoutubeStreamHelper { .end() .value("disablePlayerResponse", false); - final byte[] mobileBody = JsonWriter.string(builder.done()) + final byte[] body = JsonWriter.string(builder.done()) .getBytes(StandardCharsets.UTF_8); - final JsonObject androidPlayerResponse = getJsonAndroidPostResponse( - "reel/reel_item_watch", - mobileBody, - localization, - "&t=" + generateTParameter() + "&id=" + videoId + "&$fields=playerResponse"); + final String url = YOUTUBEI_V1_GAPIS_URL + "reel/reel_item_watch" + "?" + + DISABLE_PRETTY_PRINT_PARAMETER + "&t=" + generateTParameter() + "&id=" + videoId + + "&$fields=playerResponse"; - return androidPlayerResponse.getObject("playerResponse"); + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson(url, headers, body, localization))) + .getObject("playerResponse"); } public static JsonObject getIosPlayerResponse(@Nonnull final ContentCountry contentCountry, @@ -308,139 +363,49 @@ public final class YoutubeStreamHelper { @Nonnull final String cpn, @Nullable final PoTokenResult iosPoTokenResult) throws IOException, ExtractionException { - final boolean noPoTokenResult = iosPoTokenResult == null; - final JsonBuilder builder = prepareJsonBuilder( - localization, - contentCountry, - IOS_CLIENT_NAME, - IOS_CLIENT_VERSION, - WATCH_CLIENT_SCREEN, - MOBILE_CLIENT_PLATFORM, - noPoTokenResult - ? YoutubeParsingHelper.randomVisitorData(contentCountry) - : iosPoTokenResult.visitorData, - "Apple", - IOS_DEVICE_MODEL, - "iOS", - IOS_OS_VERSION, - null, - -1); + final InnertubeClientRequestInfo innertubeClientRequestInfo = + new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo( + IOS_CLIENT_NAME, + IOS_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, + IOS_CLIENT_ID, + null), + new InnertubeClientRequestInfo.DeviceInfo( + MOBILE_CLIENT_PLATFORM, + "Apple", + IOS_DEVICE_MODEL, + "iOS", + IOS_OS_VERSION, + -1)); + + final Map> headers = + getMobileClientHeaders(getIosUserAgent(localization)); + + // We must always pass a valid visitorData to get valid player responses, which needs to be + // got from YouTube + innertubeClientRequestInfo.clientInfo.visitorData = iosPoTokenResult == null + ? YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo, + localization, contentCountry, headers, YOUTUBEI_V1_URL, null, false) + : iosPoTokenResult.visitorData; + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, null); addVideoIdCpnAndOkChecks(builder, videoId, cpn); - if (!noPoTokenResult) { + + if (iosPoTokenResult != null) { addPoToken(builder, iosPoTokenResult.playerRequestPoToken); } - final byte[] mobileBody = JsonWriter.string(builder.done()) + final byte[] body = JsonWriter.string(builder.done()) .getBytes(StandardCharsets.UTF_8); - return getJsonIosPostResponse( - mobileBody, localization, "&t=" + generateTParameter() - + "&id=" + videoId + "&fields=streamingData.hlsManifestUrl"); - } - - public static JsonObject getJsonAndroidPostResponse(final String endpoint, - final byte[] body, - @Nonnull final Localization localization, - @Nullable final String endPartOfUrlRequest) - throws IOException, ExtractionException { - return getMobilePostResponse(endpoint, body, localization, - getAndroidUserAgent(localization), endPartOfUrlRequest); - } - - private static JsonObject getJsonIosPostResponse(final byte[] body, - @Nonnull final Localization localization, - @Nullable final String endPartOfUrlRequest) - throws IOException, ExtractionException { - return getMobilePostResponse(YoutubeStreamHelper.PLAYER, body, localization, - getIosUserAgent(localization), - endPartOfUrlRequest); - } - - @SuppressWarnings("checkstyle:ParameterNumber") - @Nonnull - private static JsonBuilder prepareJsonBuilder( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nonnull final String clientName, - @Nonnull final String clientVersion, - @Nonnull final String clientScreen, - @Nonnull final String platform, - @Nonnull final String visitorData, - @Nullable final String deviceMake, - @Nullable final String deviceModel, - @Nullable final String osName, - @Nullable final String osVersion, - @Nullable final String embedUrl, - final int androidSdkVersion) { - final JsonBuilder builder = JsonObject.builder() - .object("context") - .object("client") - .value("clientName", clientName) - .value("clientVersion", clientVersion) - .value("clientScreen", clientScreen) - .value("platform", platform) - .value("visitorData", visitorData); - - if (deviceMake != null) { - builder.value("deviceMake", deviceMake); - } - if (deviceModel != null) { - builder.value("deviceModel", deviceModel); - } - if (osName != null) { - builder.value("osName", osName); - } - if (osVersion != null) { - builder.value("osVersion", osVersion); - } - if (androidSdkVersion > 0) { - builder.value("androidSdkVersion", androidSdkVersion); - } - - builder.value("hl", localization.getLocalizationCode()) - .value("gl", contentCountry.getCountryCode()) - .value("utcOffsetMinutes", 0) - .end(); - - if (embedUrl != null) { - builder.object("thirdParty") - .value("embedUrl", embedUrl) - .end(); - } - - builder.object("request") - .array("internalExperimentFlags") - .end() - .value("useSsl", true) - .end() - .object("user") - // TODO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) - .value("lockedSafetyMode", false) - .end() - .end(); - - return builder; - } - - private static JsonObject getMobilePostResponse(@Nonnull final String endpoint, - final byte[] body, - @Nonnull final Localization localization, - @Nonnull final String userAgent, - @Nullable final String endPartOfUrlRequest) - throws IOException, ExtractionException { - final Map> headers = Map.of("User-Agent", List.of(userAgent), - "X-Goog-Api-Format-Version", List.of("2")); - - final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?" - + DISABLE_PRETTY_PRINT_PARAMETER; + final String url = YOUTUBEI_V1_GAPIS_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER + + "&t=" + generateTParameter() + "&id=" + videoId; return JsonUtils.toJsonObject(getValidJsonResponseBody( - getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest) - ? baseEndpointUrl - : baseEndpointUrl + endPartOfUrlRequest, - headers, body, localization))); + getDownloader().postWithContentTypeJson(url, headers, body, localization))); } private static void addVideoIdCpnAndOkChecks(@Nonnull final JsonBuilder builder, @@ -473,4 +438,11 @@ public final class YoutubeStreamHelper { .value(PO_TOKEN, poToken) .end(); } + + @Nonnull + private static Map> getMobileClientHeaders( + @Nonnull final String userAgent) { + return Map.of("User-Agent", List.of(userAgent), + "X-Goog-Api-Format-Version", List.of("2")); + } } From 4644e1744b336f5bc8c3400351d2344c94346fa8 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 31 Jan 2025 21:26:41 +0100 Subject: [PATCH 14/17] [YouTube] Add signatureTimestamp argument to TVHTML5 client requests This argument, which has been forgot, is required to get valid streaming URLs with this client. --- .../extractor/services/youtube/YoutubeStreamHelper.java | 5 ++++- .../services/youtube/extractors/YoutubeStreamExtractor.java | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java index 1b17058a1..9f005299b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java @@ -119,7 +119,8 @@ public final class YoutubeStreamHelper { @Nonnull final Localization localization, @Nonnull final ContentCountry contentCountry, @Nonnull final String videoId, - @Nonnull final String cpn) throws IOException, ExtractionException { + @Nonnull final String cpn, + final int signatureTimestamp) throws IOException, ExtractionException { final InnertubeClientRequestInfo innertubeClientRequestInfo = new InnertubeClientRequestInfo( new InnertubeClientRequestInfo.ClientInfo( @@ -154,6 +155,8 @@ public final class YoutubeStreamHelper { addVideoIdCpnAndOkChecks(builder, videoId, cpn); + addPlaybackContext(builder, BASE_YT_DESKTOP_WATCH_URL + videoId, signatureTimestamp); + final byte[] body = JsonWriter.string(builder.done()) .getBytes(StandardCharsets.UTF_8); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index e2a49f7da..76ef2771b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -941,7 +941,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final JsonObject tvHtml5PlayerResponse = YoutubeStreamHelper.getTvHtml5PlayerResponse( - localization, contentCountry, videoId, html5Cpn); + localization, contentCountry, videoId, html5Cpn, + YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId)); if (isPlayerResponseNotValid(tvHtml5PlayerResponse, videoId)) { throw new ExtractionException("TVHTML5 player response is not valid"); From 0952431121f49a97c3ad289971e12ccf44410501 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 31 Jan 2025 21:38:31 +0100 Subject: [PATCH 15/17] [YouTube] Move InnertubeClientRequestInfo creations in class' methods This commits provides methods to get InnertubeClientRequestInfo instances, which can be used by extractor clients to get visitor data to pass to PoTokenProvider implementations using YoutubeParsingHelper. Ability to create custom instances has been removed, but returned objects can be modified. This is what YoutubeStreamHelper now uses to set the visitorData property. --- .../youtube/InnertubeClientRequestInfo.java | 101 ++++++++++++-- .../services/youtube/YoutubeStreamHelper.java | 131 ++---------------- 2 files changed, 100 insertions(+), 132 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java index c1a91b0f9..c8848b086 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java @@ -1,8 +1,35 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.EMBED_CLIENT_SCREEN; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_OS_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.MOBILE_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MAKE; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MODEL_AND_OS_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WATCH_CLIENT_SCREEN; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION; + import javax.annotation.Nonnull; import javax.annotation.Nullable; +// TODO: add docs + public final class InnertubeClientRequestInfo { @Nonnull @@ -23,11 +50,11 @@ public final class InnertubeClientRequestInfo { @Nullable public String visitorData; - public ClientInfo(@Nonnull final String clientName, - @Nonnull final String clientVersion, - @Nonnull final String clientScreen, - @Nullable final String clientId, - @Nullable final String visitorData) { + private ClientInfo(@Nonnull final String clientName, + @Nonnull final String clientVersion, + @Nonnull final String clientScreen, + @Nullable final String clientId, + @Nullable final String visitorData) { this.clientName = clientName; this.clientVersion = clientVersion; this.clientScreen = clientScreen; @@ -50,12 +77,12 @@ public final class InnertubeClientRequestInfo { public String osVersion; public int androidSdkVersion; - public DeviceInfo(@Nonnull final String platform, - @Nullable final String deviceMake, - @Nullable final String deviceModel, - @Nullable final String osName, - @Nullable final String osVersion, - final int androidSdkVersion) { + private DeviceInfo(@Nonnull final String platform, + @Nullable final String deviceMake, + @Nullable final String deviceModel, + @Nullable final String osName, + @Nullable final String osVersion, + final int androidSdkVersion) { this.platform = platform; this.deviceMake = deviceMake; this.deviceModel = deviceModel; @@ -65,9 +92,57 @@ public final class InnertubeClientRequestInfo { } } - public InnertubeClientRequestInfo(@Nonnull final ClientInfo clientInfo, - @Nonnull final DeviceInfo deviceInfo) { + private InnertubeClientRequestInfo(@Nonnull final ClientInfo clientInfo, + @Nonnull final DeviceInfo deviceInfo) { this.clientInfo = clientInfo; this.deviceInfo = deviceInfo; } + + @Nonnull + public static InnertubeClientRequestInfo ofWebClient() { + return new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo( + WEB_CLIENT_NAME, WEB_HARDCODED_CLIENT_VERSION, WATCH_CLIENT_SCREEN, + WEB_CLIENT_ID, null), + new InnertubeClientRequestInfo.DeviceInfo(DESKTOP_CLIENT_PLATFORM, null, null, + null, null, -1)); + } + + @Nonnull + public static InnertubeClientRequestInfo ofWebEmbeddedPlayerClient() { + return new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo(WEB_EMBEDDED_CLIENT_NAME, + WEB_REMIX_HARDCODED_CLIENT_VERSION, EMBED_CLIENT_SCREEN, + WEB_EMBEDDED_CLIENT_ID, null), + new InnertubeClientRequestInfo.DeviceInfo(DESKTOP_CLIENT_PLATFORM, null, null, + null, null, -1)); + } + + @Nonnull + public static InnertubeClientRequestInfo ofTvHtml5Client() { + return new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo(TVHTML5_CLIENT_NAME, + TVHTML5_CLIENT_VERSION, WATCH_CLIENT_SCREEN, TVHTML5_CLIENT_ID, null), + new InnertubeClientRequestInfo.DeviceInfo(TVHTML5_CLIENT_PLATFORM, + TVHTML5_DEVICE_MAKE, TVHTML5_DEVICE_MODEL_AND_OS_NAME, + TVHTML5_DEVICE_MODEL_AND_OS_NAME, "", -1)); + } + + @Nonnull + public static InnertubeClientRequestInfo ofAndroidClient() { + return new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo(ANDROID_CLIENT_NAME, + ANDROID_CLIENT_VERSION, WATCH_CLIENT_SCREEN, ANDROID_CLIENT_ID, null), + new InnertubeClientRequestInfo.DeviceInfo(MOBILE_CLIENT_PLATFORM, null, null, + "Android", "15", 35)); + } + + @Nonnull + public static InnertubeClientRequestInfo ofIosClient() { + return new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo(IOS_CLIENT_NAME, IOS_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, IOS_CLIENT_ID, null), + new InnertubeClientRequestInfo.DeviceInfo(MOBILE_CLIENT_PLATFORM, "Apple", + IOS_DEVICE_MODEL, "iOS", IOS_OS_VERSION, -1)); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java index 9f005299b..1eb56f250 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java @@ -17,31 +17,11 @@ import java.util.List; import java.util.Map; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_ID; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_NAME; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.EMBED_CLIENT_SCREEN; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_ID; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_NAME; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_OS_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.MOBILE_CLIENT_PLATFORM; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_ID; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_NAME; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_PLATFORM; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MAKE; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MODEL_AND_OS_NAME; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WATCH_CLIENT_SCREEN; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_NAME; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; @@ -75,20 +55,8 @@ public final class YoutubeStreamHelper { @Nonnull final ContentCountry contentCountry, @Nonnull final String videoId) throws IOException, ExtractionException { final InnertubeClientRequestInfo innertubeClientRequestInfo = - new InnertubeClientRequestInfo( - new InnertubeClientRequestInfo.ClientInfo( - WEB_CLIENT_NAME, - getClientVersion(), - WATCH_CLIENT_SCREEN, - WEB_CLIENT_ID, - null), - new InnertubeClientRequestInfo.DeviceInfo( - DESKTOP_CLIENT_PLATFORM, - null, - null, - null, - null, - -1)); + InnertubeClientRequestInfo.ofWebClient(); + innertubeClientRequestInfo.clientInfo.clientVersion = getClientVersion(); final Map> headers = getYouTubeHeaders(); @@ -122,20 +90,7 @@ public final class YoutubeStreamHelper { @Nonnull final String cpn, final int signatureTimestamp) throws IOException, ExtractionException { final InnertubeClientRequestInfo innertubeClientRequestInfo = - new InnertubeClientRequestInfo( - new InnertubeClientRequestInfo.ClientInfo( - TVHTML5_CLIENT_NAME, - TVHTML5_CLIENT_VERSION, - WATCH_CLIENT_SCREEN, - TVHTML5_CLIENT_ID, - null), - new InnertubeClientRequestInfo.DeviceInfo( - TVHTML5_CLIENT_PLATFORM, - TVHTML5_DEVICE_MAKE, - TVHTML5_DEVICE_MODEL_AND_OS_NAME, - TVHTML5_DEVICE_MODEL_AND_OS_NAME, - "", - -1)); + InnertubeClientRequestInfo.ofTvHtml5Client(); final Map> headers = new HashMap<>( getClientHeaders(TVHTML5_CLIENT_ID, TVHTML5_CLIENT_VERSION)); @@ -175,20 +130,9 @@ public final class YoutubeStreamHelper { @Nonnull final PoTokenResult webPoTokenResult, final int signatureTimestamp) throws IOException, ExtractionException { final InnertubeClientRequestInfo innertubeClientRequestInfo = - new InnertubeClientRequestInfo( - new InnertubeClientRequestInfo.ClientInfo( - WEB_CLIENT_NAME, - getClientVersion(), - WATCH_CLIENT_SCREEN, - WEB_CLIENT_ID, - webPoTokenResult.visitorData), - new InnertubeClientRequestInfo.DeviceInfo( - DESKTOP_CLIENT_PLATFORM, - null, - null, - null, - null, - -1)); + InnertubeClientRequestInfo.ofWebClient(); + innertubeClientRequestInfo.clientInfo.clientVersion = getClientVersion(); + innertubeClientRequestInfo.clientInfo.visitorData = webPoTokenResult.visitorData; final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, innertubeClientRequestInfo, null); @@ -199,7 +143,7 @@ public final class YoutubeStreamHelper { addPoToken(builder, webPoTokenResult.playerRequestPoToken); - final byte[] body = JsonWriter.string(builder.end().done()) + final byte[] body = JsonWriter.string(builder.done()) .getBytes(StandardCharsets.UTF_8); final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; @@ -218,20 +162,7 @@ public final class YoutubeStreamHelper { @Nullable final PoTokenResult webEmbeddedPoTokenResult, final int signatureTimestamp) throws IOException, ExtractionException { final InnertubeClientRequestInfo innertubeClientRequestInfo = - new InnertubeClientRequestInfo( - new InnertubeClientRequestInfo.ClientInfo( - WEB_EMBEDDED_CLIENT_NAME, - WEB_REMIX_HARDCODED_CLIENT_VERSION, - EMBED_CLIENT_SCREEN, - WEB_EMBEDDED_CLIENT_ID, - null), - new InnertubeClientRequestInfo.DeviceInfo( - DESKTOP_CLIENT_PLATFORM, - null, - null, - null, - null, - -1)); + InnertubeClientRequestInfo.ofWebEmbeddedPlayerClient(); final Map> headers = new HashMap<>( getClientHeaders(WEB_EMBEDDED_CLIENT_ID, WEB_EMBEDDED_CLIENT_VERSION)); @@ -273,20 +204,8 @@ public final class YoutubeStreamHelper { @Nonnull final PoTokenResult androidPoTokenResult) throws IOException, ExtractionException { final InnertubeClientRequestInfo innertubeClientRequestInfo = - new InnertubeClientRequestInfo( - new InnertubeClientRequestInfo.ClientInfo( - ANDROID_CLIENT_NAME, - ANDROID_CLIENT_VERSION, - WATCH_CLIENT_SCREEN, - ANDROID_CLIENT_ID, - androidPoTokenResult.visitorData), - new InnertubeClientRequestInfo.DeviceInfo( - MOBILE_CLIENT_PLATFORM, - null, - null, - "Android", - "15", - 35)); + InnertubeClientRequestInfo.ofAndroidClient(); + innertubeClientRequestInfo.clientInfo.visitorData = androidPoTokenResult.visitorData; final Map> headers = getMobileClientHeaders(getAndroidUserAgent(localization)); @@ -314,20 +233,7 @@ public final class YoutubeStreamHelper { @Nonnull final String videoId, @Nonnull final String cpn) throws IOException, ExtractionException { final InnertubeClientRequestInfo innertubeClientRequestInfo = - new InnertubeClientRequestInfo( - new InnertubeClientRequestInfo.ClientInfo( - ANDROID_CLIENT_NAME, - ANDROID_CLIENT_VERSION, - WATCH_CLIENT_SCREEN, - ANDROID_CLIENT_ID, - null), - new InnertubeClientRequestInfo.DeviceInfo( - MOBILE_CLIENT_PLATFORM, - null, - null, - "Android", - "15", - 35)); + InnertubeClientRequestInfo.ofAndroidClient(); final Map> headers = getMobileClientHeaders(getAndroidUserAgent(localization)); @@ -367,20 +273,7 @@ public final class YoutubeStreamHelper { @Nullable final PoTokenResult iosPoTokenResult) throws IOException, ExtractionException { final InnertubeClientRequestInfo innertubeClientRequestInfo = - new InnertubeClientRequestInfo( - new InnertubeClientRequestInfo.ClientInfo( - IOS_CLIENT_NAME, - IOS_CLIENT_VERSION, - WATCH_CLIENT_SCREEN, - IOS_CLIENT_ID, - null), - new InnertubeClientRequestInfo.DeviceInfo( - MOBILE_CLIENT_PLATFORM, - "Apple", - IOS_DEVICE_MODEL, - "iOS", - IOS_OS_VERSION, - -1)); + InnertubeClientRequestInfo.ofIosClient(); final Map> headers = getMobileClientHeaders(getIosUserAgent(localization)); From c48d44985324c4669a11ce898acb71b66643cc0c Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 31 Jan 2025 21:43:51 +0100 Subject: [PATCH 16/17] [YouTube] Add ability to get TVHTML5 user agent used Also make YoutubeParsingHelper.getOriginReferrerHeaders public, in order to be used by other extractor classes and improve the name of a parameter of YoutubeParsingHelper.getVisitorDataFromInnertube. --- .../youtube/YoutubeParsingHelper.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index a41773726..da5870833 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -26,6 +26,7 @@ import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DES import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION; @@ -1134,6 +1135,17 @@ public final class YoutubeParsingHelper { + ")"; } + /** + * Get the user-agent string used as the user-agent for InnerTube requests with the HTML5 TV + * client. + * + * @return the user-agent used for InnerTube requests with the TVHTML5 client + */ + @Nonnull + public static String getTvHtml5UserAgent() { + return TVHTML5_USER_AGENT; + } + /** * Returns a {@link Map} containing the required YouTube Music headers. */ @@ -1172,7 +1184,7 @@ public final class YoutubeParsingHelper { * * @param url The URL to be set as the origin and referrer. */ - static Map> getOriginReferrerHeaders(@Nonnull final String url) { + public static Map> getOriginReferrerHeaders(@Nonnull final String url) { final var urlList = List.of(url); return Map.of("Origin", urlList, "Referer", urlList); } @@ -1496,12 +1508,13 @@ public final class YoutubeParsingHelper { } } + @Nonnull public static String getVisitorDataFromInnertube( @Nonnull final InnertubeClientRequestInfo innertubeClientRequestInfo, @Nonnull final Localization localization, @Nonnull final ContentCountry contentCountry, @Nonnull final Map> httpHeaders, - @Nonnull final String innerTubeDomain, + @Nonnull final String innertubeDomainAndVersionEndpoint, @Nullable final String embedUrl, final boolean useGuideEndpoint) throws IOException, ExtractionException { final JsonBuilder builder = prepareJsonBuilder( @@ -1512,8 +1525,9 @@ public final class YoutubeParsingHelper { final String visitorData = JsonUtils.toJsonObject(getValidJsonResponseBody(getDownloader() .postWithContentTypeJson( - innerTubeDomain + (useGuideEndpoint ? "guide" : "visitor_id") - + "?" + DISABLE_PRETTY_PRINT_PARAMETER, + innertubeDomainAndVersionEndpoint + + (useGuideEndpoint ? "guide" : "visitor_id") + "?" + + DISABLE_PRETTY_PRINT_PARAMETER, httpHeaders, body))) .getObject("responseContext") .getString("visitorData"); From 96911ae2a48b619704303353fbf6f407f8ba2831 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 31 Jan 2025 21:46:40 +0100 Subject: [PATCH 17/17] [YouTube] Fix usage of WEB client headers for all HTML5 URLs in DASH creators Also use TVHTML5 user agent for requests from this client in these DASH manifests creators. --- .../YoutubeDashManifestCreatorsUtils.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java index 49dce80e4..f1083b0ab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java @@ -1,8 +1,9 @@ package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientInfoHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getOriginReferrerHeaders; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTvHtml5UserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5StreamingUrl; @@ -27,6 +28,7 @@ import org.w3c.dom.Element; import java.io.IOException; import java.io.StringWriter; import java.nio.charset.StandardCharsets; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -602,8 +604,9 @@ public final class YoutubeDashManifestCreatorsUtils { @Nonnull final ItagItem itagItem, final DeliveryType deliveryType) throws CreationException { + final boolean isTvHtml5StreamingUrl = isTvHtml5StreamingUrl(baseStreamingUrl); final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl) - || isTvHtml5StreamingUrl(baseStreamingUrl) + || isTvHtml5StreamingUrl || isWebEmbeddedPlayerStreamingUrl(baseStreamingUrl); final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl); final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl); @@ -617,7 +620,7 @@ public final class YoutubeDashManifestCreatorsUtils { final String mimeTypeExpected = itagItem.getMediaFormat().getMimeType(); if (!isNullOrEmpty(mimeTypeExpected)) { return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl, - mimeTypeExpected); + mimeTypeExpected, isTvHtml5StreamingUrl); } } else if (isAndroidStreamingUrl || isIosStreamingUrl) { try { @@ -732,6 +735,8 @@ public final class YoutubeDashManifestCreatorsUtils { * @param downloader the {@link Downloader} instance to be used * @param streamingUrl the streaming URL which we are trying to get a streaming URL * without any redirection on the network and/or IP used + * @param isTvHtml5StreamingUrl whether the streaming URL comes from TVHTML5 client, in + * order to use an appropriate HTTP User-Agent header * @param responseMimeTypeExpected the response mime type expected from Google video servers * @return the {@link Response} of the stream, which should have no redirections */ @@ -740,10 +745,15 @@ public final class YoutubeDashManifestCreatorsUtils { private static Response getStreamingWebUrlWithoutRedirects( @Nonnull final Downloader downloader, @Nonnull String streamingUrl, - @Nonnull final String responseMimeTypeExpected) + @Nonnull final String responseMimeTypeExpected, + final boolean isTvHtml5StreamingUrl) throws CreationException { try { - final var headers = getClientInfoHeaders(); + final var headers = new HashMap<>( + getOriginReferrerHeaders("https://www.youtube.com")); + if (isTvHtml5StreamingUrl) { + headers.put("User-Agent", List.of(getTvHtml5UserAgent())); + } String responseMimeType = "";