diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java new file mode 100644 index 000000000..4b5e291ff --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java @@ -0,0 +1,118 @@ +package org.schabi.newpipe.extractor.services.youtube; + +final class ClientsConstants { + private ClientsConstants() { + } + + // Common client fields + + static final String DESKTOP_CLIENT_PLATFORM = "DESKTOP"; + static final String MOBILE_CLIENT_PLATFORM = "MOBILE"; + static final String WATCH_CLIENT_SCREEN = "WATCH"; + static final String EMBED_CLIENT_SCREEN = "EMBED"; + + // WEB (YouTube desktop) client fields + + static final String WEB_CLIENT_ID = "1"; + static final String WEB_CLIENT_NAME = "WEB"; + /** + * The client version for InnerTube requests with the {@code WEB} client, used as the last + * fallback if the extraction of the real one failed. + */ + static final String WEB_HARDCODED_CLIENT_VERSION = "2.20250122.04.00"; + + // WEB_REMIX (YouTube Music) client fields + + static final String WEB_REMIX_CLIENT_ID = "67"; + static final String WEB_REMIX_CLIENT_NAME = "WEB_REMIX"; + static final String WEB_REMIX_HARDCODED_CLIENT_VERSION = "1.20250122.01.00"; + + // TVHTML5 (YouTube on TVs and consoles using HTML5) client fields + static final String TVHTML5_CLIENT_ID = "7"; + static final String TVHTML5_CLIENT_NAME = "TVHTML5"; + static final String TVHTML5_CLIENT_VERSION = "7.20250122.15.00"; + static final String TVHTML5_CLIENT_PLATFORM = "GAME_CONSOLE"; + static final String TVHTML5_DEVICE_MAKE = "Sony"; + static final String TVHTML5_DEVICE_MODEL_AND_OS_NAME = "PlayStation 4"; + // CHECKSTYLE:OFF + static final String TVHTML5_USER_AGENT = + "Mozilla/5.0 (PlayStation; PlayStation 4/12.00) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15"; + // CHECKSTYLE:ON + + // WEB_EMBEDDED_PLAYER (YouTube embeds) + + static final String WEB_EMBEDDED_CLIENT_ID = "56"; + static final String WEB_EMBEDDED_CLIENT_NAME = "WEB_EMBEDDED_PLAYER"; + static final String WEB_EMBEDDED_CLIENT_VERSION = "1.20250121.00.00"; + + // IOS (iOS YouTube app) client fields + + static final String IOS_CLIENT_ID = "5"; + static final String IOS_CLIENT_NAME = "IOS"; + + /** + * The hardcoded client version of the iOS app used for InnerTube requests with this client. + * + *

+ * It can be extracted by getting the latest release version of the app on + * the App + * Store page of the YouTube app, in the {@code What’s New} section. + *

+ */ + static final String IOS_CLIENT_VERSION = "20.03.02"; + + /** + * The device machine id for the iPhone 15 Pro Max, used to get 60fps with the {@code iOS} + * client. + * + *

+ * See this GitHub Gist for more + * information. + *

+ */ + static final String IOS_DEVICE_MODEL = "iPhone16,2"; + + /** + * The iOS version to be used in JSON POST requests, the one of an iPhone 15 Pro Max running + * iOS 18.2.1 with the hardcoded version of the iOS app (for the {@code "osVersion"} field). + * + *

+ * The value of this field seems to use the following structure: + * "iOS major version.minor version.patch version.build version", where + * "patch version" is equal to 0 if it isn't set + * The build version corresponding to the iOS version used can be found on + * + * https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max + *

+ * + * @see #IOS_USER_AGENT_VERSION + */ + static final String IOS_OS_VERSION = "18.2.1.22C161"; + + /** + * The iOS version to be used in the HTTP user agent for requests. + * + *

+ * This should be the same of as {@link #IOS_OS_VERSION}. + *

+ * + * @see #IOS_OS_VERSION + */ + static final String IOS_USER_AGENT_VERSION = "18_2_1"; + + // ANDROID (Android YouTube app) client fields + + static final String ANDROID_CLIENT_ID = "3"; + static final String ANDROID_CLIENT_NAME = "ANDROID"; + + /** + * The hardcoded client version of the Android app used for InnerTube requests with this + * client. + * + *

+ * It can be extracted by getting the latest release version of the app in an APK repository + * such as APKMirror. + *

+ */ + static final String ANDROID_CLIENT_VERSION = "19.28.35"; +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java new file mode 100644 index 000000000..c8848b086 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java @@ -0,0 +1,148 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.EMBED_CLIENT_SCREEN; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_OS_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.MOBILE_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MAKE; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MODEL_AND_OS_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WATCH_CLIENT_SCREEN; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +// TODO: add docs + +public final class InnertubeClientRequestInfo { + + @Nonnull + public ClientInfo clientInfo; + @Nonnull + public DeviceInfo deviceInfo; + + public static final class ClientInfo { + + @Nonnull + public String clientName; + @Nonnull + public String clientVersion; + @Nonnull + public String clientScreen; + @Nullable + public String clientId; + @Nullable + public String visitorData; + + private ClientInfo(@Nonnull final String clientName, + @Nonnull final String clientVersion, + @Nonnull final String clientScreen, + @Nullable final String clientId, + @Nullable final String visitorData) { + this.clientName = clientName; + this.clientVersion = clientVersion; + this.clientScreen = clientScreen; + this.clientId = clientId; + this.visitorData = visitorData; + } + } + + public static final class DeviceInfo { + + @Nonnull + public String platform; + @Nullable + public String deviceMake; + @Nullable + public String deviceModel; + @Nullable + public String osName; + @Nullable + public String osVersion; + public int androidSdkVersion; + + private DeviceInfo(@Nonnull final String platform, + @Nullable final String deviceMake, + @Nullable final String deviceModel, + @Nullable final String osName, + @Nullable final String osVersion, + final int androidSdkVersion) { + this.platform = platform; + this.deviceMake = deviceMake; + this.deviceModel = deviceModel; + this.osName = osName; + this.osVersion = osVersion; + this.androidSdkVersion = androidSdkVersion; + } + } + + private InnertubeClientRequestInfo(@Nonnull final ClientInfo clientInfo, + @Nonnull final DeviceInfo deviceInfo) { + this.clientInfo = clientInfo; + this.deviceInfo = deviceInfo; + } + + @Nonnull + public static InnertubeClientRequestInfo ofWebClient() { + return new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo( + WEB_CLIENT_NAME, WEB_HARDCODED_CLIENT_VERSION, WATCH_CLIENT_SCREEN, + WEB_CLIENT_ID, null), + new InnertubeClientRequestInfo.DeviceInfo(DESKTOP_CLIENT_PLATFORM, null, null, + null, null, -1)); + } + + @Nonnull + public static InnertubeClientRequestInfo ofWebEmbeddedPlayerClient() { + return new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo(WEB_EMBEDDED_CLIENT_NAME, + WEB_REMIX_HARDCODED_CLIENT_VERSION, EMBED_CLIENT_SCREEN, + WEB_EMBEDDED_CLIENT_ID, null), + new InnertubeClientRequestInfo.DeviceInfo(DESKTOP_CLIENT_PLATFORM, null, null, + null, null, -1)); + } + + @Nonnull + public static InnertubeClientRequestInfo ofTvHtml5Client() { + return new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo(TVHTML5_CLIENT_NAME, + TVHTML5_CLIENT_VERSION, WATCH_CLIENT_SCREEN, TVHTML5_CLIENT_ID, null), + new InnertubeClientRequestInfo.DeviceInfo(TVHTML5_CLIENT_PLATFORM, + TVHTML5_DEVICE_MAKE, TVHTML5_DEVICE_MODEL_AND_OS_NAME, + TVHTML5_DEVICE_MODEL_AND_OS_NAME, "", -1)); + } + + @Nonnull + public static InnertubeClientRequestInfo ofAndroidClient() { + return new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo(ANDROID_CLIENT_NAME, + ANDROID_CLIENT_VERSION, WATCH_CLIENT_SCREEN, ANDROID_CLIENT_ID, null), + new InnertubeClientRequestInfo.DeviceInfo(MOBILE_CLIENT_PLATFORM, null, null, + "Android", "15", 35)); + } + + @Nonnull + public static InnertubeClientRequestInfo ofIosClient() { + return new InnertubeClientRequestInfo( + new InnertubeClientRequestInfo.ClientInfo(IOS_CLIENT_NAME, IOS_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, IOS_CLIENT_ID, null), + new InnertubeClientRequestInfo.DeviceInfo(MOBILE_CLIENT_PLATFORM, "Apple", + IOS_DEVICE_MODEL, "iOS", IOS_OS_VERSION, -1)); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java new file mode 100644 index 000000000..a11a25bae --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java @@ -0,0 +1,120 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import javax.annotation.Nullable; + +/** + * Interface to provide {@code poToken}s to YouTube player requests. + * + *

+ * On some major clients, YouTube requires that the integrity of the device passes some checks to + * allow playback. + *

+ * + *

+ * These checks involve running codes to verify the integrity and using their result to generate + * one or multiple {@code poToken}(s) (which stands for proof of origin token(s)). + *

+ * + *

+ * These tokens may have a role in triggering the sign in requirement. + *

+ * + *

+ * If an implementation does not want to return a {@code poToken} for a specific client, it must + * return {@code null}. + *

+ * + *

+ * Implementations of this interface are expected to be thread-safe, as they may be accessed by + * multiple threads. + *

+ */ +public interface PoTokenProvider { + + /** + * Get a {@link PoTokenResult} specific to the desktop website, a.k.a. the WEB InnerTube client. + * + *

+ * To be generated and valid, {@code poToken}s from this client must be generated using Google's + * BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They + * must be added to adaptive/DASH streaming URLs with the {@code pot} parameter. + *

+ * + *

+ * Note that YouTube desktop website generates two {@code poToken}s: + * - one for the player requests {@code poToken}s, using the videoId as the minter value; + * - one for the streaming URLs, using a visitor data for logged-out users as the minter value. + *

+ * + * @return a {@link PoTokenResult} specific to the WEB InnerTube client + */ + @Nullable + PoTokenResult getWebClientPoToken(String videoId); + + /** + * Get a {@link PoTokenResult} specific to the web embeds, a.k.a. the WEB_EMBEDDED_PLAYER + * InnerTube client. + * + *

+ * To be generated and valid, {@code poToken}s from this client must be generated using Google's + * BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They + * should be added to adaptive/DASH streaming URLs with the {@code pot} parameter. + *

+ * + *

+ * As of writing, like the YouTube desktop website previously did, it generates only one + * {@code poToken}, sent in player requests and streaming URLs, using a visitor data for + * logged-out users. {@code poToken}s do not seem to be mandatory for now on this client. + *

+ * + * @return a {@link PoTokenResult} specific to the WEB_EMBEDDED_PLAYER InnerTube client + */ + @Nullable + PoTokenResult getWebEmbedClientPoToken(String videoId); + + /** + * Get a {@link PoTokenResult} specific to the Android app, a.k.a. the ANDROID InnerTube client. + * + *

+ * Implementation details are not known, the app uses DroidGuard, a downloaded native virtual + * machine ran by Google Play Services for which its code is updated pretty frequently. + *

+ * + *

+ * As of writing, DroidGuard seem to check for the Android app signature and package ID, as + * non-rooted YouTube patched with reVanced doesn't work without spoofing another InnerTube + * client while the rooted version works without any client spoofing. + *

+ * + *

+ * There should be only one {@code poToken} needed for the player requests, it shouldn't be + * required for regular adaptive URLs (i.e. not server adaptive bitrate (SABR) URLs). HLS + * formats returned (only for premieres and running and post-live livestreams) in the client's + * HLS manifest URL should work without {@code poToken}s. + *

+ * + * @return a {@link PoTokenResult} specific to the ANDROID InnerTube client + */ + @Nullable + PoTokenResult getAndroidClientPoToken(String videoId); + + /** + * Get a {@link PoTokenResult} specific to the iOS app, a.k.a. the IOS InnerTube client. + * + *

+ * Implementation details are not known, the app seem to use something called iosGuard which + * should be similar to Android's DroidGuard. It may rely on Apple's attestation APIs. + *

+ * + *

+ * As of writing, there should be only one {@code poToken} needed for the player requests, it + * shouldn't be required for regular adaptive URLs (i.e. not server adaptive bitrate (SABR) + * URLs). HLS formats returned in the client's HLS manifest URL should also work without a + * {@code poToken}. + *

+ * + * @return a {@link PoTokenResult} specific to the IOS InnerTube client + */ + @Nullable + PoTokenResult getIosClientPoToken(String videoId); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java new file mode 100644 index 000000000..4ccbaf1ba --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java @@ -0,0 +1,52 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Objects; + +/** + * The result of a supported/successful {@code poToken} extraction request by a + * {@link PoTokenProvider}. + */ +public final class PoTokenResult { + + /** + * The visitor data associated with a {@code poToken}. + */ + @Nonnull + public final String visitorData; + + /** + * The {@code poToken} of a player request, a Protobuf object encoded as a base 64 string. + */ + @Nonnull + public final String playerRequestPoToken; + + /** + * The {@code poToken} to be appended to streaming URLs, a Protobuf object encoded as a base + * 64 string. + * + *

+ * It may be required on some clients such as HTML5 ones and may also differ from the player + * request {@code poToken}. + *

+ */ + @Nullable + public final String streamingDataPoToken; + + /** + * Construct a {@link PoTokenResult} instance. + * + * @param visitorData see {@link #visitorData} + * @param playerRequestPoToken see {@link #playerRequestPoToken} + * @param streamingDataPoToken see {@link #streamingDataPoToken} + * @throws NullPointerException if a non-null parameter is null + */ + public PoTokenResult(@Nonnull final String visitorData, + @Nonnull final String playerRequestPoToken, + @Nullable final String streamingDataPoToken) { + this.visitorData = Objects.requireNonNull(visitorData); + this.playerRequestPoToken = Objects.requireNonNull(playerRequestPoToken); + this.streamingDataPoToken = streamingDataPoToken; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 54b4b90fb..da5870833 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -21,6 +21,18 @@ package org.schabi.newpipe.extractor.services.youtube; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray; @@ -144,55 +156,6 @@ public final class YoutubeParsingHelper { */ public static final String RACY_CHECK_OK = "racyCheckOk"; - /** - * The hardcoded client ID used for InnerTube requests with the {@code WEB} client. - */ - private static final String WEB_CLIENT_ID = "1"; - - /** - * The client version for InnerTube requests with the {@code WEB} client, used as the last - * fallback if the extraction of the real one failed. - */ - private static final String HARDCODED_CLIENT_VERSION = "2.20240718.01.00"; - - /** - * The hardcoded client version of the Android app used for InnerTube requests with this - * client. - * - *

- * It can be extracted by getting the latest release version of the app in an APK repository - * such as APKMirror. - *

- */ - private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "19.28.35"; - - /** - * The hardcoded client version of the iOS app used for InnerTube requests with this client. - * - *

- * It can be extracted by getting the latest release version of the app on - * the App - * Store page of the YouTube app, in the {@code What’s New} section. - *

- */ - private static final String IOS_YOUTUBE_CLIENT_VERSION = "20.03.02"; - - /** - * The hardcoded client version used for InnerTube requests with the TV HTML5 embed client. - */ - private static final String TVHTML5_SIMPLY_EMBED_CLIENT_VERSION = "2.0"; - - /** - * The hardcoded client ID used for InnerTube requests with the YouTube Music desktop client. - */ - private static final String YOUTUBE_MUSIC_CLIENT_ID = "67"; - - /** - * The hardcoded client version used for InnerTube requests with the YouTube Music desktop - * client. - */ - private static final String HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION = "1.20240715.01.00"; - private static String clientVersion; private static String youtubeMusicClientVersion; @@ -212,49 +175,16 @@ public final class YoutubeParsingHelper { private static final String CONTENT_PLAYBACK_NONCE_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; - /** - * The device machine id for the iPhone 15 Pro Max, - * used to get 60fps with the {@code iOS} client. - * - *

- * See this GitHub Gist for more - * information. - *

- */ - private static final String IOS_DEVICE_MODEL = "iPhone16,2"; - - /** - * Spoofing an iPhone 15 Pro Max running iOS 18.2.1 with the hardcoded version of the iOS app. - * To be used for the {@code "osVersion"} field in JSON POST requests. - *

- * The value of this field seems to use the following structure: - * "iOS major version.minor version.patch version.build version", where - * "patch version" is equal to 0 if it isn't set - * The build version corresponding to the iOS version used can be found on - * - * https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max - *

- * - * @see #IOS_USER_AGENT_VERSION - */ - private static final String IOS_OS_VERSION = "18.2.1.22C161"; - - /** - * Spoofing an iPhone 15 Pro Max running iOS 18.2.1 with the hardcoded version of the iOS app. - * To be used in the user agent for requests. - * - * @see #IOS_OS_VERSION - */ - private static final String IOS_USER_AGENT_VERSION = "18_2_1"; - private static Random numberGenerator = new Random(); private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; private static final Pattern C_WEB_PATTERN = Pattern.compile("&c=WEB"); - private static final Pattern C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN = - Pattern.compile("&c=TVHTML5_SIMPLY_EMBEDDED_PLAYER"); + private static final Pattern C_WEB_EMBEDDED_PLAYER_PATTERN = + Pattern.compile("&c=WEB_EMBEDDED_PLAYER"); + private static final Pattern C_TVHTML5_PLAYER_PATTERN = + Pattern.compile("&c=TVHTML5"); private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID"); private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS"); @@ -561,9 +491,9 @@ public final class YoutubeParsingHelper { .object("client") .value("hl", "en-GB") .value("gl", "GB") - .value("clientName", "WEB") - .value("clientVersion", HARDCODED_CLIENT_VERSION) - .value("platform", "DESKTOP") + .value("clientName", WEB_CLIENT_NAME) + .value("clientVersion", WEB_HARDCODED_CLIENT_VERSION) + .value("platform", DESKTOP_CLIENT_PLATFORM) .value("utcOffsetMinutes", 0) .end() .object("request") @@ -581,7 +511,7 @@ public final class YoutubeParsingHelper { .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on - final var headers = getClientHeaders(WEB_CLIENT_ID, HARDCODED_CLIENT_VERSION); + final var headers = getClientHeaders(WEB_CLIENT_ID, WEB_HARDCODED_CLIENT_VERSION); // This endpoint is fetched by the YouTube website to get the items of its main menu and is // pretty lightweight (around 30kB) @@ -705,7 +635,7 @@ public final class YoutubeParsingHelper { // Fallback to the hardcoded one if it is valid if (isHardcodedClientVersionValid()) { - clientVersion = HARDCODED_CLIENT_VERSION; + clientVersion = WEB_HARDCODED_CLIENT_VERSION; return clientVersion; } @@ -752,11 +682,11 @@ public final class YoutubeParsingHelper { .object() .object("context") .object("client") - .value("clientName", "WEB_REMIX") - .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION) + .value("clientName", WEB_REMIX_CLIENT_NAME) + .value("clientVersion", WEB_REMIX_HARDCODED_CLIENT_VERSION) .value("hl", "en-GB") .value("gl", "GB") - .value("platform", "DESKTOP") + .value("platform", DESKTOP_CLIENT_PLATFORM) .value("utcOffsetMinutes", 0) .end() .object("request") @@ -775,8 +705,7 @@ public final class YoutubeParsingHelper { // @formatter:on final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL)); - headers.putAll(getClientHeaders(YOUTUBE_MUSIC_CLIENT_ID, - HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION)); + headers.putAll(getClientHeaders(WEB_REMIX_CLIENT_ID, WEB_HARDCODED_CLIENT_VERSION)); final Response response = getDownloader().postWithContentTypeJson(url, headers, json); // Ensure to have a valid response @@ -789,7 +718,7 @@ public final class YoutubeParsingHelper { return youtubeMusicClientVersion; } if (isHardcodedYoutubeMusicClientVersionValid()) { - youtubeMusicClientVersion = HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION; + youtubeMusicClientVersion = WEB_REMIX_HARDCODED_CLIENT_VERSION; return youtubeMusicClientVersion; } @@ -1134,116 +1063,20 @@ public final class YoutubeParsingHelper { + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization))); } - public static JsonObject getJsonAndroidPostResponse( - final String endpoint, - final byte[] body, - @Nonnull final Localization localization, - @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { - return getMobilePostResponse(endpoint, body, localization, - getAndroidUserAgent(localization), endPartOfUrlRequest); - } - - public static JsonObject getJsonIosPostResponse( - final String endpoint, - final byte[] body, - @Nonnull final Localization localization, - @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { - return getMobilePostResponse(endpoint, body, localization, getIosUserAgent(localization), - endPartOfUrlRequest); - } - - private static JsonObject getMobilePostResponse( - final String endpoint, - final byte[] body, - @Nonnull final Localization localization, - @Nonnull final String userAgent, - @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { - final var headers = Map.of("User-Agent", List.of(userAgent), - "X-Goog-Api-Format-Version", List.of("2")); - - final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?" - + DISABLE_PRETTY_PRINT_PARAMETER; - - return JsonUtils.toJsonObject(getValidJsonResponseBody( - getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest) - ? baseEndpointUrl - : baseEndpointUrl + endPartOfUrlRequest, - headers, body, localization))); - } - @Nonnull public static JsonBuilder prepareDesktopJsonBuilder( @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry) - throws IOException, ExtractionException { - return prepareDesktopJsonBuilder(localization, contentCountry, null); - } - - @Nonnull - public static JsonBuilder prepareDesktopJsonBuilder( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nullable final String visitorData) - throws IOException, ExtractionException { - String vData = visitorData; - if (vData == null) { - vData = randomVisitorData(contentCountry); - } - + @Nonnull final ContentCountry contentCountry) throws IOException, ExtractionException { // @formatter:off return JsonObject.builder() .object("context") .object("client") .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) - .value("clientName", "WEB") + .value("clientName", WEB_CLIENT_NAME) .value("clientVersion", getClientVersion()) .value("originalUrl", "https://www.youtube.com") - .value("platform", "DESKTOP") - .value("utcOffsetMinutes", 0) - .value("visitorData", vData) - .end() - .object("request") - .array("internalExperimentFlags") - .end() - .value("useSsl", true) - .end() - .object("user") - // TODO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) - .value("lockedSafetyMode", false) - .end() - .end(); - // @formatter:on - } - - @Nonnull - public static JsonBuilder prepareAndroidMobileJsonBuilder( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry) { - // @formatter:off - return JsonObject.builder() - .object("context") - .object("client") - .value("clientName", "ANDROID") - .value("clientVersion", ANDROID_YOUTUBE_CLIENT_VERSION) - .value("platform", "MOBILE") - .value("osName", "Android") - .value("osVersion", "14") - /* - A valid Android SDK version is required to be sure to get a valid player - response - If this parameter is not provided, the player response is replaced by an - error saying the message "The following content is not available on this - app. Watch this content on the latest version on YouTube" (it was - previously a 5-minute video with this message) - See https://github.com/TeamNewPipe/NewPipe/issues/8713 - The Android SDK version corresponding to the Android version used in - requests is sent - */ - .value("androidSdkVersion", 34) - .value("hl", localization.getLocalizationCode()) - .value("gl", contentCountry.getCountryCode()) + .value("platform", DESKTOP_CLIENT_PLATFORM) .value("utcOffsetMinutes", 0) .end() .object("request") @@ -1260,122 +1093,6 @@ public final class YoutubeParsingHelper { // @formatter:on } - @Nonnull - public static JsonBuilder prepareIosMobileJsonBuilder( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry) { - // @formatter:off - return JsonObject.builder() - .object("context") - .object("client") - .value("clientName", "IOS") - .value("clientVersion", IOS_YOUTUBE_CLIENT_VERSION) - .value("deviceMake", "Apple") - // Device model is required to get 60fps streams - .value("deviceModel", IOS_DEVICE_MODEL) - .value("platform", "MOBILE") - .value("osName", "iOS") - .value("osVersion", IOS_OS_VERSION) - .value("visitorData", randomVisitorData(contentCountry)) - .value("hl", localization.getLocalizationCode()) - .value("gl", contentCountry.getCountryCode()) - .value("utcOffsetMinutes", 0) - .end() - .object("request") - .array("internalExperimentFlags") - .end() - .value("useSsl", true) - .end() - .object("user") - // TODO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) - .value("lockedSafetyMode", false) - .end() - .end(); - // @formatter:on - } - - @Nonnull - public static JsonBuilder prepareTvHtml5EmbedJsonBuilder( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nonnull final String videoId) { - // @formatter:off - return JsonObject.builder() - .object("context") - .object("client") - .value("clientName", "TVHTML5_SIMPLY_EMBEDDED_PLAYER") - .value("clientVersion", TVHTML5_SIMPLY_EMBED_CLIENT_VERSION) - .value("clientScreen", "EMBED") - .value("platform", "TV") - .value("hl", localization.getLocalizationCode()) - .value("gl", contentCountry.getCountryCode()) - .value("utcOffsetMinutes", 0) - .end() - .object("thirdParty") - .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId) - .end() - .object("request") - .array("internalExperimentFlags") - .end() - .value("useSsl", true) - .end() - .object("user") - // TODO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) - .value("lockedSafetyMode", false) - .end() - .end(); - // @formatter:on - } - - @Nonnull - public static JsonObject getWebPlayerResponse( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nonnull final String videoId) throws IOException, ExtractionException { - final byte[] body = JsonWriter.string( - prepareDesktopJsonBuilder(localization, contentCountry) - .value(VIDEO_ID, videoId) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) - .getBytes(StandardCharsets.UTF_8); - final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER - + "&$fields=microformat,playabilityStatus,storyboards,videoDetails"; - - return JsonUtils.toJsonObject(getValidJsonResponseBody( - getDownloader().postWithContentTypeJson( - url, getYouTubeHeaders(), body, localization))); - } - - @Nonnull - public static byte[] createTvHtml5EmbedPlayerBody( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nonnull final String videoId, - @Nonnull final Integer sts, - @Nonnull final String contentPlaybackNonce) { - // @formatter:off - return JsonWriter.string( - prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId) - .object("playbackContext") - .object("contentPlaybackContext") - // Signature timestamp from the JavaScript base player is needed to get - // working obfuscated URLs - .value("signatureTimestamp", sts) - .value("referer", "https://www.youtube.com/watch?v=" + videoId) - .end() - .end() - .value(CPN, contentPlaybackNonce) - .value(VIDEO_ID, videoId) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) - .getBytes(StandardCharsets.UTF_8); - // @formatter:on - } - /** * Get the user-agent string used as the user-agent for InnerTube requests with the Android * client. @@ -1391,9 +1108,8 @@ public final class YoutubeParsingHelper { */ @Nonnull public static String getAndroidUserAgent(@Nullable final Localization localization) { - // Spoofing an Android 14 device with the hardcoded version of the Android app - return "com.google.android.youtube/" + ANDROID_YOUTUBE_CLIENT_VERSION - + " (Linux; U; Android 14; " + return "com.google.android.youtube/" + ANDROID_CLIENT_VERSION + + " (Linux; U; Android 15; " + (localization != null ? localization : Localization.DEFAULT).getCountryCode() + ") gzip"; } @@ -1413,23 +1129,30 @@ public final class YoutubeParsingHelper { */ @Nonnull public static String getIosUserAgent(@Nullable final Localization localization) { - // Spoofing an iPhone 15 Pro Max running iOS 18.1.0 - // with the hardcoded version of the iOS app - return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION - + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS " - + IOS_USER_AGENT_VERSION + " like Mac OS X; " + return "com.google.ios.youtube/" + IOS_CLIENT_VERSION + "(" + IOS_DEVICE_MODEL + + "; U; CPU iOS " + IOS_USER_AGENT_VERSION + " like Mac OS X; " + (localization != null ? localization : Localization.DEFAULT).getCountryCode() + ")"; } + /** + * Get the user-agent string used as the user-agent for InnerTube requests with the HTML5 TV + * client. + * + * @return the user-agent used for InnerTube requests with the TVHTML5 client + */ + @Nonnull + public static String getTvHtml5UserAgent() { + return TVHTML5_USER_AGENT; + } + /** * Returns a {@link Map} containing the required YouTube Music headers. */ @Nonnull public static Map> getYoutubeMusicHeaders() { final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL)); - headers.putAll(getClientHeaders(YOUTUBE_MUSIC_CLIENT_ID, - youtubeMusicClientVersion)); + headers.putAll(getClientHeaders(WEB_REMIX_CLIENT_ID, youtubeMusicClientVersion)); return headers; } @@ -1461,7 +1184,7 @@ public final class YoutubeParsingHelper { * * @param url The URL to be set as the origin and referrer. */ - private static Map> getOriginReferrerHeaders(@Nonnull final String url) { + public static Map> getOriginReferrerHeaders(@Nonnull final String url) { final var urlList = List.of(url); return Map.of("Origin", urlList, "Referer", urlList); } @@ -1473,8 +1196,8 @@ public final class YoutubeParsingHelper { * @param name The X-YouTube-Client-Name value. * @param version X-YouTube-Client-Version value. */ - private static Map> getClientHeaders(@Nonnull final String name, - @Nonnull final String version) { + static Map> getClientHeaders(@Nonnull final String name, + @Nonnull final String version) { return Map.of("X-YouTube-Client-Name", List.of(name), "X-YouTube-Client-Version", List.of(version)); } @@ -1669,15 +1392,24 @@ public final class YoutubeParsingHelper { } /** - * Check if the streaming URL is a URL from the YouTube {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} - * client. + * Check if the streaming URL is from the YouTube {@code WEB_EMBEDDED_PLAYER} client. * - * @param url the streaming URL on which check if it's a {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} - * streaming URL. - * @return true if it's a {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} streaming URL, false otherwise + * @param url the streaming URL to be checked. + * @return true if it's a {@code WEB_EMBEDDED_PLAYER} streaming URL, false otherwise */ - public static boolean isTvHtml5SimplyEmbeddedPlayerStreamingUrl(@Nonnull final String url) { - return Parser.isMatch(C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN, url); + public static boolean isWebEmbeddedPlayerStreamingUrl(@Nonnull final String url) { + return Parser.isMatch(C_WEB_EMBEDDED_PLAYER_PATTERN, url); + } + + /** + * Check if the streaming URL is a URL from the YouTube {@code TVHTML5} client. + * + * @param url the streaming URL on which check if it's a {@code TVHTML5} + * streaming URL. + * @return true if it's a {@code TVHTML5} streaming URL, false otherwise + */ + public static boolean isTvHtml5StreamingUrl(@Nonnull final String url) { + return Parser.isMatch(C_TVHTML5_PLAYER_PATTERN, url); } /** @@ -1775,4 +1507,96 @@ public final class YoutubeParsingHelper { return null; } } + + @Nonnull + public static String getVisitorDataFromInnertube( + @Nonnull final InnertubeClientRequestInfo innertubeClientRequestInfo, + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final Map> httpHeaders, + @Nonnull final String innertubeDomainAndVersionEndpoint, + @Nullable final String embedUrl, + final boolean useGuideEndpoint) throws IOException, ExtractionException { + final JsonBuilder builder = prepareJsonBuilder( + localization, contentCountry, innertubeClientRequestInfo, embedUrl); + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + final String visitorData = JsonUtils.toJsonObject(getValidJsonResponseBody(getDownloader() + .postWithContentTypeJson( + innertubeDomainAndVersionEndpoint + + (useGuideEndpoint ? "guide" : "visitor_id") + "?" + + DISABLE_PRETTY_PRINT_PARAMETER, + httpHeaders, body))) + .getObject("responseContext") + .getString("visitorData"); + + if (isNullOrEmpty(visitorData)) { + throw new ParsingException("Could not get visitorData"); + } + + return visitorData; + } + + @Nonnull + static JsonBuilder prepareJsonBuilder( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final InnertubeClientRequestInfo innertubeClientRequestInfo, + @Nullable final String embedUrl) { + final JsonBuilder builder = JsonObject.builder() + .object("context") + .object("client") + .value("clientName", innertubeClientRequestInfo.clientInfo.clientName) + .value("clientVersion", innertubeClientRequestInfo.clientInfo.clientVersion) + .value("clientScreen", innertubeClientRequestInfo.clientInfo.clientScreen) + .value("platform", innertubeClientRequestInfo.deviceInfo.platform); + + if (innertubeClientRequestInfo.clientInfo.visitorData != null) { + builder.value("visitorData", innertubeClientRequestInfo.clientInfo.visitorData); + } + + if (innertubeClientRequestInfo.deviceInfo.deviceMake != null) { + builder.value("deviceMake", innertubeClientRequestInfo.deviceInfo.deviceMake); + } + if (innertubeClientRequestInfo.deviceInfo.deviceModel != null) { + builder.value("deviceModel", innertubeClientRequestInfo.deviceInfo.deviceModel); + } + if (innertubeClientRequestInfo.deviceInfo.osName != null) { + builder.value("osName", innertubeClientRequestInfo.deviceInfo.osName); + } + if (innertubeClientRequestInfo.deviceInfo.osVersion != null) { + builder.value("osVersion", innertubeClientRequestInfo.deviceInfo.osVersion); + } + if (innertubeClientRequestInfo.deviceInfo.androidSdkVersion > 0) { + builder.value("androidSdkVersion", + innertubeClientRequestInfo.deviceInfo.androidSdkVersion); + } + + builder.value("hl", localization.getLocalizationCode()) + .value("gl", contentCountry.getCountryCode()) + .value("utcOffsetMinutes", 0) + .end(); + + if (embedUrl != null) { + builder.object("thirdParty") + .value("embedUrl", embedUrl) + .end(); + } + + builder.object("request") + .array("internalExperimentFlags") + .end() + .value("useSsl", true) + .end() + .object("user") + // TODO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) + .end() + .end(); + + return builder; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java new file mode 100644 index 000000000..1eb56f250 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java @@ -0,0 +1,344 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import com.grack.nanojson.JsonBuilder; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonWriter; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.localization.ContentCountry; +import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_GAPIS_URL; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientHeaders; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getOriginReferrerHeaders; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder; + +public final class YoutubeStreamHelper { + + private static final String PLAYER = "player"; + private static final String SERVICE_INTEGRITY_DIMENSIONS = "serviceIntegrityDimensions"; + private static final String PO_TOKEN = "poToken"; + private static final String BASE_YT_DESKTOP_WATCH_URL = "https://www.youtube.com/watch?v="; + + private YoutubeStreamHelper() { + } + + @Nonnull + public static JsonObject getWebMetadataPlayerResponse( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId) throws IOException, ExtractionException { + final InnertubeClientRequestInfo innertubeClientRequestInfo = + InnertubeClientRequestInfo.ofWebClient(); + innertubeClientRequestInfo.clientInfo.clientVersion = getClientVersion(); + + final Map> headers = getYouTubeHeaders(); + + // We must always pass a valid visitorData to get valid player responses, which needs to be + // got from YouTube + innertubeClientRequestInfo.clientInfo.visitorData = + YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo, + localization, contentCountry, headers, YOUTUBEI_V1_URL, null, false); + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, null); + + addVideoIdCpnAndOkChecks(builder, videoId, null); + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER + + "&$fields=microformat,playabilityStatus,storyboards,videoDetails"; + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson( + url, headers, body, localization))); + } + + @Nonnull + public static JsonObject getTvHtml5PlayerResponse( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nonnull final String cpn, + final int signatureTimestamp) throws IOException, ExtractionException { + final InnertubeClientRequestInfo innertubeClientRequestInfo = + InnertubeClientRequestInfo.ofTvHtml5Client(); + + final Map> headers = new HashMap<>( + getClientHeaders(TVHTML5_CLIENT_ID, TVHTML5_CLIENT_VERSION)); + headers.putAll(getOriginReferrerHeaders("https://www.youtube.com")); + headers.put("User-Agent", List.of(TVHTML5_USER_AGENT)); + + // We must always pass a valid visitorData to get valid player responses, which needs to be + // got from YouTube + // For some reason, the TVHTML5 client doesn't support the visitor_id endpoint, use the + // guide one instead, which is quite lightweight + innertubeClientRequestInfo.clientInfo.visitorData = + YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo, + localization, contentCountry, headers, YOUTUBEI_V1_URL, null, true); + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, null); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + addPlaybackContext(builder, BASE_YT_DESKTOP_WATCH_URL + videoId, signatureTimestamp); + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson(url, headers, body, localization))); + } + + @Nonnull + public static JsonObject getWebFullPlayerResponse( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nonnull final String cpn, + @Nonnull final PoTokenResult webPoTokenResult, + final int signatureTimestamp) throws IOException, ExtractionException { + final InnertubeClientRequestInfo innertubeClientRequestInfo = + InnertubeClientRequestInfo.ofWebClient(); + innertubeClientRequestInfo.clientInfo.clientVersion = getClientVersion(); + innertubeClientRequestInfo.clientInfo.visitorData = webPoTokenResult.visitorData; + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, null); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + addPlaybackContext(builder, BASE_YT_DESKTOP_WATCH_URL + videoId, signatureTimestamp); + + addPoToken(builder, webPoTokenResult.playerRequestPoToken); + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson( + url, getYouTubeHeaders(), body, localization))); + } + + @Nonnull + public static JsonObject getWebEmbeddedPlayerResponse( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nonnull final String cpn, + @Nullable final PoTokenResult webEmbeddedPoTokenResult, + final int signatureTimestamp) throws IOException, ExtractionException { + final InnertubeClientRequestInfo innertubeClientRequestInfo = + InnertubeClientRequestInfo.ofWebEmbeddedPlayerClient(); + + final Map> headers = new HashMap<>( + getClientHeaders(WEB_EMBEDDED_CLIENT_ID, WEB_EMBEDDED_CLIENT_VERSION)); + headers.putAll(getOriginReferrerHeaders("https://www.youtube.com")); + + final String embedUrl = BASE_YT_DESKTOP_WATCH_URL + videoId; + + // We must always pass a valid visitorData to get valid player responses, which needs to be + // got from YouTube + innertubeClientRequestInfo.clientInfo.visitorData = webEmbeddedPoTokenResult == null + ? YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo, + localization, contentCountry, headers, YOUTUBEI_V1_URL, embedUrl, false) + : webEmbeddedPoTokenResult.visitorData; + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, embedUrl); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + addPlaybackContext(builder, embedUrl, signatureTimestamp); + + if (webEmbeddedPoTokenResult != null) { + addPoToken(builder, webEmbeddedPoTokenResult.playerRequestPoToken); + } + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson(url, headers, body, localization))); + } + + public static JsonObject getAndroidPlayerResponse( + @Nonnull final ContentCountry contentCountry, + @Nonnull final Localization localization, + @Nonnull final String videoId, + @Nonnull final String cpn, + @Nonnull final PoTokenResult androidPoTokenResult) + throws IOException, ExtractionException { + final InnertubeClientRequestInfo innertubeClientRequestInfo = + InnertubeClientRequestInfo.ofAndroidClient(); + innertubeClientRequestInfo.clientInfo.visitorData = androidPoTokenResult.visitorData; + + final Map> headers = + getMobileClientHeaders(getAndroidUserAgent(localization)); + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, null); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + addPoToken(builder, androidPoTokenResult.playerRequestPoToken); + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + final String url = YOUTUBEI_V1_GAPIS_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER + + "&t=" + generateTParameter() + "&id=" + videoId; + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson(url, headers, body, localization))); + } + + public static JsonObject getAndroidReelPlayerResponse( + @Nonnull final ContentCountry contentCountry, + @Nonnull final Localization localization, + @Nonnull final String videoId, + @Nonnull final String cpn) throws IOException, ExtractionException { + final InnertubeClientRequestInfo innertubeClientRequestInfo = + InnertubeClientRequestInfo.ofAndroidClient(); + + final Map> headers = + getMobileClientHeaders(getAndroidUserAgent(localization)); + + // We must always pass a valid visitorData to get valid player responses, which needs to be + // got from YouTube + innertubeClientRequestInfo.clientInfo.visitorData = + YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo, + localization, contentCountry, headers, YOUTUBEI_V1_GAPIS_URL, null, false); + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, null); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + builder.object("playerRequest") + .value(VIDEO_ID, videoId) + .end() + .value("disablePlayerResponse", false); + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + final String url = YOUTUBEI_V1_GAPIS_URL + "reel/reel_item_watch" + "?" + + DISABLE_PRETTY_PRINT_PARAMETER + "&t=" + generateTParameter() + "&id=" + videoId + + "&$fields=playerResponse"; + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson(url, headers, body, localization))) + .getObject("playerResponse"); + } + + public static JsonObject getIosPlayerResponse(@Nonnull final ContentCountry contentCountry, + @Nonnull final Localization localization, + @Nonnull final String videoId, + @Nonnull final String cpn, + @Nullable final PoTokenResult iosPoTokenResult) + throws IOException, ExtractionException { + final InnertubeClientRequestInfo innertubeClientRequestInfo = + InnertubeClientRequestInfo.ofIosClient(); + + final Map> headers = + getMobileClientHeaders(getIosUserAgent(localization)); + + // We must always pass a valid visitorData to get valid player responses, which needs to be + // got from YouTube + innertubeClientRequestInfo.clientInfo.visitorData = iosPoTokenResult == null + ? YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo, + localization, contentCountry, headers, YOUTUBEI_V1_URL, null, false) + : iosPoTokenResult.visitorData; + + final JsonBuilder builder = prepareJsonBuilder(localization, contentCountry, + innertubeClientRequestInfo, null); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + if (iosPoTokenResult != null) { + addPoToken(builder, iosPoTokenResult.playerRequestPoToken); + } + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + final String url = YOUTUBEI_V1_GAPIS_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER + + "&t=" + generateTParameter() + "&id=" + videoId; + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson(url, headers, body, localization))); + } + + private static void addVideoIdCpnAndOkChecks(@Nonnull final JsonBuilder builder, + @Nonnull final String videoId, + @Nullable final String cpn) { + builder.value(VIDEO_ID, videoId); + + if (cpn != null) { + builder.value(CPN, cpn); + } + + builder.value(CONTENT_CHECK_OK, true) + .value(RACY_CHECK_OK, true); + } + + private static void addPlaybackContext(@Nonnull final JsonBuilder builder, + @Nonnull final String referer, + final int signatureTimestamp) { + builder.object("playbackContext") + .object("contentPlaybackContext") + .value("signatureTimestamp", signatureTimestamp) + .value("referer", referer) + .end() + .end(); + } + + private static void addPoToken(@Nonnull final JsonBuilder builder, + @Nonnull final String poToken) { + builder.object(SERVICE_INTEGRITY_DIMENSIONS) + .value(PO_TOKEN, poToken) + .end(); + } + + @Nonnull + private static Map> getMobileClientHeaders( + @Nonnull final String userAgent) { + return Map.of("User-Agent", List.of(userAgent), + "X-Goog-Api-Format-Version", List.of("2")); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java index 0b4c41157..f1083b0ab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java @@ -1,12 +1,14 @@ package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientInfoHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getOriginReferrerHeaders; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTvHtml5UserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5StreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebEmbeddedPlayerStreamingUrl; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import org.schabi.newpipe.extractor.MediaFormat; @@ -26,6 +28,7 @@ import org.w3c.dom.Element; import java.io.IOException; import java.io.StringWriter; import java.nio.charset.StandardCharsets; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -582,8 +585,8 @@ public final class YoutubeDashManifestCreatorsUtils { * This method fetches, for OTF streams and for post-live-DVR streams: *
    *
  • the base URL of the stream, to which are appended {@link #SQ_0} and - * {@link #RN_0} parameters, with a {@code GET} request for streaming URLs from HTML5 - * clients and a {@code POST} request for the ones from the {@code ANDROID} and the + * {@link #RN_0} parameters, with a {@code POS} request for streaming URLs from + * {@code WEB}, {@code TVHTML5}, {@code WEB_EMBEDDED_PLAYER}, {@code ANDROID} and * {@code IOS} clients;
    • *
  • for streaming URLs from HTML5 clients, the {@link #ALR_YES} param is also added. *
    • @@ -601,8 +604,10 @@ public final class YoutubeDashManifestCreatorsUtils { @Nonnull final ItagItem itagItem, final DeliveryType deliveryType) throws CreationException { + final boolean isTvHtml5StreamingUrl = isTvHtml5StreamingUrl(baseStreamingUrl); final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl) - || isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl); + || isTvHtml5StreamingUrl + || isWebEmbeddedPlayerStreamingUrl(baseStreamingUrl); final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl); final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl); if (isHtml5StreamingUrl) { @@ -615,7 +620,7 @@ public final class YoutubeDashManifestCreatorsUtils { final String mimeTypeExpected = itagItem.getMediaFormat().getMimeType(); if (!isNullOrEmpty(mimeTypeExpected)) { return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl, - mimeTypeExpected); + mimeTypeExpected, isTvHtml5StreamingUrl); } } else if (isAndroidStreamingUrl || isIosStreamingUrl) { try { @@ -730,6 +735,8 @@ public final class YoutubeDashManifestCreatorsUtils { * @param downloader the {@link Downloader} instance to be used * @param streamingUrl the streaming URL which we are trying to get a streaming URL * without any redirection on the network and/or IP used + * @param isTvHtml5StreamingUrl whether the streaming URL comes from TVHTML5 client, in + * order to use an appropriate HTTP User-Agent header * @param responseMimeTypeExpected the response mime type expected from Google video servers * @return the {@link Response} of the stream, which should have no redirections */ @@ -738,17 +745,23 @@ public final class YoutubeDashManifestCreatorsUtils { private static Response getStreamingWebUrlWithoutRedirects( @Nonnull final Downloader downloader, @Nonnull String streamingUrl, - @Nonnull final String responseMimeTypeExpected) + @Nonnull final String responseMimeTypeExpected, + final boolean isTvHtml5StreamingUrl) throws CreationException { try { - final var headers = getClientInfoHeaders(); + final var headers = new HashMap<>( + getOriginReferrerHeaders("https://www.youtube.com")); + if (isTvHtml5StreamingUrl) { + headers.put("User-Agent", List.of(getTvHtml5UserAgent())); + } String responseMimeType = ""; int redirectsCount = 0; while (!responseMimeType.equals(responseMimeTypeExpected) && redirectsCount < MAXIMUM_REDIRECT_COUNT) { - final Response response = downloader.get(streamingUrl, headers); + final byte[] html5Body = new byte[] {0x78, 0}; + final Response response = downloader.post(streamingUrl, headers, html5Body); final int responseCode = response.responseCode(); if (responseCode != 200) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 988a1bcc2..76ef2771b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -27,18 +27,12 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createTvHtml5EmbedPlayerBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -66,9 +60,12 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider; +import org.schabi.newpipe.extractor.services.youtube.PoTokenResult; import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager; import org.schabi.newpipe.extractor.services.youtube.YoutubeMetaInfoHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.YoutubeStreamHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.DeliveryMethod; @@ -104,6 +101,10 @@ import javax.annotation.Nullable; public class YoutubeStreamExtractor extends StreamExtractor { + @Nullable + private static PoTokenProvider poTokenProvider; + private static boolean fetchIosClient; + private JsonObject playerResponse; private JsonObject nextResponse; @@ -112,7 +113,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nullable private JsonObject androidStreamingData; @Nullable - private JsonObject tvHtml5SimplyEmbedStreamingData; + private JsonObject html5StreamingData; private JsonObject videoPrimaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer; @@ -127,7 +128,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { // three different strings are used. private String iosCpn; private String androidCpn; - private String tvHtml5SimplyEmbedCpn; + private String html5Cpn; + + @Nullable + private String html5StreamingUrlsPoToken; + @Nullable + private String androidStreamingUrlsPoToken; + @Nullable + private String iosStreamingUrlsPoToken; public YoutubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) { super(service, linkHandler); @@ -321,7 +329,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { return Long.parseLong(duration); } catch (final Exception e) { return getDurationFromFirstAdaptiveFormat(Arrays.asList( - iosStreamingData, androidStreamingData, tvHtml5SimplyEmbedStreamingData)); + html5StreamingData, androidStreamingData, iosStreamingData)); } } @@ -579,11 +587,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getDashMpdUrl() throws ParsingException { assertPageFetched(); - // There is no DASH manifest available in the iOS clients and the DASH manifest of the - // Android client doesn't contain all available streams (mainly the WEBM ones) + // There is no DASH manifest available with the iOS client return getManifestUrl( "dash", - Arrays.asList(androidStreamingData, tvHtml5SimplyEmbedStreamingData)); + Arrays.asList( + new Pair<>(androidStreamingData, androidStreamingUrlsPoToken), + new Pair<>(html5StreamingData, html5StreamingUrlsPoToken)), + // Return version 7 of the DASH manifest, which is the latest one, reducing + // manifest size and allowing playback with some DASH players + "mpd_version=7"); } @Nonnull @@ -592,25 +604,44 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); // Return HLS manifest of the iOS client first because on livestreams, the HLS manifest - // returned has separated audio and video streams + // returned has separated audio and video streams and poTokens requirement do not seem to + // impact HLS formats (if a poToken is provided, it is added) // Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response + // unless a Safari macOS user agent is used return getManifestUrl( "hls", Arrays.asList( - iosStreamingData, androidStreamingData, tvHtml5SimplyEmbedStreamingData)); + new Pair<>(iosStreamingData, iosStreamingUrlsPoToken), + new Pair<>(androidStreamingData, androidStreamingUrlsPoToken), + new Pair<>(html5StreamingData, html5StreamingUrlsPoToken)), + ""); } @Nonnull - private static String getManifestUrl(@Nonnull final String manifestType, - @Nonnull final List streamingDataObjects) { + private static String getManifestUrl( + @Nonnull final String manifestType, + @Nonnull final List> streamingDataObjects, + @Nonnull final String partToAppendToManifestUrlEnd) { final String manifestKey = manifestType + "ManifestUrl"; - return streamingDataObjects.stream() - .filter(Objects::nonNull) - .map(streamingDataObject -> streamingDataObject.getString(manifestKey)) - .filter(Objects::nonNull) - .findFirst() - .orElse(""); + for (final Pair streamingDataObj : streamingDataObjects) { + if (streamingDataObj.getFirst() != null) { + final String manifestUrl = streamingDataObj.getFirst().getString(manifestKey); + if (isNullOrEmpty(manifestUrl)) { + continue; + } + + // If poToken is not null, add it to manifest URL + if (streamingDataObj.getSecond() == null) { + return manifestUrl + "?" + partToAppendToManifestUrlEnd; + } else { + return manifestUrl + "?pot=" + streamingDataObj.getSecond() + "&" + + partToAppendToManifestUrlEnd; + } + } + } + + return ""; } @Override @@ -684,7 +715,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } private void setStreamType() { - if (playerResponse.getObject("playabilityStatus").has("liveStreamability")) { + if (playerResponse.getObject(PLAYABILITY_STATUS).has("liveStreamability")) { streamType = StreamType.LIVE_STREAM; } else if (playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false)) { streamType = StreamType.POST_LIVE_STREAM; @@ -751,7 +782,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getErrorMessage() { try { - return getTextFromObject(playerResponse.getObject("playabilityStatus") + return getTextFromObject(playerResponse.getObject(PLAYABILITY_STATUS) .getObject("errorScreen").getObject("playerErrorMessageRenderer") .getObject("reason")); } catch (final NullPointerException e) { @@ -766,10 +797,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String FORMATS = "formats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String STREAMING_DATA = "streamingData"; - private static final String PLAYER = "player"; private static final String NEXT = "next"; private static final String SIGNATURE_CIPHER = "signatureCipher"; private static final String CIPHER = "cipher"; + private static final String PLAYER_CAPTIONS_TRACKLIST_RENDERER + = "playerCaptionsTracklistRenderer"; + private static final String CAPTIONS = "captions"; + private static final String PLAYABILITY_STATUS = "playabilityStatus"; @Override public void onFetchPage(@Nonnull final Downloader downloader) @@ -779,98 +813,53 @@ public class YoutubeStreamExtractor extends StreamExtractor { final Localization localization = getExtractorLocalization(); final ContentCountry contentCountry = getExtractorContentCountry(); - final JsonObject webPlayerResponse = YoutubeParsingHelper.getWebPlayerResponse( - localization, contentCountry, videoId); + final PoTokenProvider poTokenproviderInstance = poTokenProvider; + final boolean noPoTokenProviderSet = poTokenproviderInstance == null; - if (isPlayerResponseNotValid(webPlayerResponse, videoId)) { - // Check the playability status, as private and deleted videos and invalid video IDs do - // not return the ID provided in the player response - // When the requested video is playable and a different video ID is returned, it has - // the OK playability status, meaning the ExtractionException after this check will be - // thrown - checkPlayabilityStatus( - webPlayerResponse, webPlayerResponse.getObject("playabilityStatus")); - throw new ExtractionException("Initial WEB player response is not valid"); - } - - // Save the webPlayerResponse into playerResponse in the case the video cannot be played, - // so some metadata can be retrieved - playerResponse = webPlayerResponse; - - // Use the player response from the player endpoint of the desktop internal API because - // there can be restrictions on videos in the embedded player. - // E.g. if a video is age-restricted, the embedded player's playabilityStatus says that - // the video cannot be played outside of YouTube, but does not show the original message. - final JsonObject playabilityStatus = webPlayerResponse.getObject("playabilityStatus"); - - final boolean isAgeRestricted = "login_required".equalsIgnoreCase( - playabilityStatus.getString("status")) - && playabilityStatus.getString("reason", "") - .contains("age"); + fetchHtml5Client(localization, contentCountry, videoId, poTokenproviderInstance, + noPoTokenProviderSet); setStreamType(); - if (isAgeRestricted) { - fetchTvHtml5EmbedJsonPlayer(contentCountry, localization, videoId); + final PoTokenResult androidPoTokenResult = noPoTokenProviderSet ? null + : poTokenproviderInstance.getAndroidClientPoToken(videoId); - // If no streams can be fetched in the TVHTML5 simply embed client, the video should be - // age-restricted, therefore throw an AgeRestrictedContentException explicitly. - if (tvHtml5SimplyEmbedStreamingData == null) { - throw new AgeRestrictedContentException( - "This age-restricted video cannot be watched."); - } + fetchAndroidClient(localization, contentCountry, videoId, androidPoTokenResult); - // Refresh the stream type because the stream type may be not properly known for - // age-restricted videos - setStreamType(); - } else { - checkPlayabilityStatus(webPlayerResponse, playabilityStatus); - - // Fetching successfully the iOS player is mandatory to get streams - fetchIosMobileJsonPlayer(contentCountry, localization, videoId); - - try { - fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId); - } catch (final Exception ignored) { - // Ignore exceptions related to ANDROID client fetch or parsing, as it is not - // compulsory to play contents - } + if (fetchIosClient) { + final PoTokenResult iosPoTokenResult = noPoTokenProviderSet ? null + : poTokenproviderInstance.getIosClientPoToken(videoId); + fetchIosClient(localization, contentCountry, videoId, iosPoTokenResult); } - // The microformat JSON object of the content is only returned on the WEB client, - // so we need to store it instead of getting it directly from the playerResponse - playerMicroFormatRenderer = webPlayerResponse.getObject("microformat") - .getObject("playerMicroformatRenderer"); - - final byte[] body = JsonWriter.string( + final byte[] nextBody = JsonWriter.string( prepareDesktopJsonBuilder(localization, contentCountry) .value(VIDEO_ID, videoId) .value(CONTENT_CHECK_OK, true) .value(RACY_CHECK_OK, true) .done()) .getBytes(StandardCharsets.UTF_8); - nextResponse = getJsonPostResponse(NEXT, body, localization); + nextResponse = getJsonPostResponse(NEXT, nextBody, localization); } - private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse, - @Nonnull final JsonObject playabilityStatus) + private static void checkPlayabilityStatus(@Nonnull final JsonObject playabilityStatus) throws ParsingException { - String status = playabilityStatus.getString("status"); + final String status = playabilityStatus.getString("status"); if (status == null || status.equalsIgnoreCase("ok")) { return; } - // If status exist, and is not "OK", throw the specific exception based on error message - // or a ContentNotAvailableException with the reason text if it's an unknown reason. - final JsonObject newPlayabilityStatus = - youtubePlayerResponse.getObject("playabilityStatus"); - status = newPlayabilityStatus.getString("status"); - final String reason = newPlayabilityStatus.getString("reason"); + final String reason = playabilityStatus.getString("reason"); - if (status.equalsIgnoreCase("login_required") && reason == null) { - final String message = newPlayabilityStatus.getArray("messages").getString(0); - if (message != null && message.contains("private")) { - throw new PrivateContentException("This video is private."); + if (status.equalsIgnoreCase("login_required")) { + if (reason == null) { + final String message = playabilityStatus.getArray("messages").getString(0); + if (message != null && message.contains("private")) { + throw new PrivateContentException("This video is private"); + } + } else if (reason.contains("age")) { + throw new AgeRestrictedContentException( + "This age-restricted video cannot be watched anonymously"); } } @@ -879,16 +868,18 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (reason.contains("Music Premium")) { throw new YoutubeMusicPremiumContentException(); } + if (reason.contains("payment")) { throw new PaidContentException("This video is a paid video"); } + if (reason.contains("members-only")) { throw new PaidContentException("This video is only available" + " for members of the channel of this video"); } if (reason.contains("unavailable")) { - final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus + final String detailedErrorMessage = getTextFromObject(playabilityStatus .getObject("errorScreen") .getObject("playerErrorMessageRenderer") .getObject("subreason")); @@ -900,120 +891,205 @@ public class YoutubeStreamExtractor extends StreamExtractor { Objects.requireNonNullElse(detailedErrorMessage, reason)); } } + + if (reason.contains("age-restricted")) { + throw new AgeRestrictedContentException( + "This age-restricted video cannot be watched anonymously"); + } } throw new ContentNotAvailableException("Got error: \"" + reason + "\""); } - /** - * Fetch the Android Mobile API and assign the streaming data to the androidStreamingData JSON - * object. - */ - private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentCountry, - @Nonnull final Localization localization, - @Nonnull final String videoId) + private void fetchHtml5Client(@Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nullable final PoTokenProvider poTokenProviderInstance, + final boolean noPoTokenProviderSet) throws IOException, ExtractionException { - androidCpn = generateContentPlaybackNonce(); - final byte[] mobileBody = JsonWriter.string( - prepareAndroidMobileJsonBuilder(localization, contentCountry) - .object("playerRequest") - .value(VIDEO_ID, videoId) - .end() - .value("disablePlayerResponse", false) - .value(VIDEO_ID, videoId) - .value(CPN, androidCpn) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) - .getBytes(StandardCharsets.UTF_8); + html5Cpn = generateContentPlaybackNonce(); - final JsonObject androidPlayerResponse = getJsonAndroidPostResponse( - "reel/reel_item_watch", - mobileBody, - localization, - "&t=" + generateTParameter() + "&id=" + videoId + "&$fields=playerResponse"); + // Suppress NPE warning as nullability is already checked before and passed with + // noPoTokenProviderSet + //noinspection DataFlowIssue + final PoTokenResult webPoTokenResult = noPoTokenProviderSet ? null + : poTokenProviderInstance.getWebClientPoToken(videoId); + final JsonObject webPlayerResponse; + if (noPoTokenProviderSet || webPoTokenResult == null) { + webPlayerResponse = YoutubeStreamHelper.getWebMetadataPlayerResponse( + localization, contentCountry, videoId); - final JsonObject playerResponseObject = androidPlayerResponse.getObject("playerResponse"); - if (isPlayerResponseNotValid(playerResponseObject, videoId)) { - return; - } + throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId); - final JsonObject streamingData = playerResponseObject.getObject(STREAMING_DATA); - if (!isNullOrEmpty(streamingData)) { - androidStreamingData = streamingData; - if (isNullOrEmpty(playerCaptionsTracklistRenderer)) { - playerCaptionsTracklistRenderer = playerResponseObject.getObject("captions") - .getObject("playerCaptionsTracklistRenderer"); + // Save the webPlayerResponse into playerResponse in the case the video cannot be + // played, so some metadata can be retrieved + playerResponse = webPlayerResponse; + + // The microformat JSON object of the content is only returned on the WEB client, + // so we need to store it instead of getting it directly from the playerResponse + playerMicroFormatRenderer = playerResponse.getObject("microformat") + .getObject("playerMicroformatRenderer"); + + final JsonObject playabilityStatus = webPlayerResponse.getObject(PLAYABILITY_STATUS); + + if (isVideoAgeRestricted(playabilityStatus)) { + fetchHtml5EmbedClient(localization, contentCountry, videoId, + noPoTokenProviderSet ? null + : poTokenProviderInstance.getWebEmbedClientPoToken(videoId)); + } else { + checkPlayabilityStatus(playabilityStatus); + + final JsonObject tvHtml5PlayerResponse = + YoutubeStreamHelper.getTvHtml5PlayerResponse( + localization, contentCountry, videoId, html5Cpn, + YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId)); + + if (isPlayerResponseNotValid(tvHtml5PlayerResponse, videoId)) { + throw new ExtractionException("TVHTML5 player response is not valid"); + } + + html5StreamingData = tvHtml5PlayerResponse.getObject(STREAMING_DATA); + playerCaptionsTracklistRenderer = tvHtml5PlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + } + } else { + webPlayerResponse = YoutubeStreamHelper.getWebFullPlayerResponse( + localization, contentCountry, videoId, html5Cpn, webPoTokenResult, + YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId)); + + throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId); + + // Save the webPlayerResponse into playerResponse in the case the video cannot be + // played, so some metadata can be retrieved + playerResponse = webPlayerResponse; + + // The microformat JSON object of the content is only returned on the WEB client, + // so we need to store it instead of getting it directly from the playerResponse + playerMicroFormatRenderer = playerResponse.getObject("microformat") + .getObject("playerMicroformatRenderer"); + + final JsonObject playabilityStatus = webPlayerResponse.getObject(PLAYABILITY_STATUS); + + if (isVideoAgeRestricted(playabilityStatus)) { + fetchHtml5EmbedClient(localization, contentCountry, videoId, + poTokenProviderInstance.getWebEmbedClientPoToken(videoId)); + } else { + checkPlayabilityStatus(playabilityStatus); + html5StreamingData = webPlayerResponse.getObject(STREAMING_DATA); + playerCaptionsTracklistRenderer = webPlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + html5StreamingUrlsPoToken = webPoTokenResult.streamingDataPoToken; } } } - /** - * Fetch the iOS Mobile API and assign the streaming data to the iosStreamingData JSON - * object. - */ - private void fetchIosMobileJsonPlayer(@Nonnull final ContentCountry contentCountry, - @Nonnull final Localization localization, - @Nonnull final String videoId) - throws IOException, ExtractionException { - iosCpn = generateContentPlaybackNonce(); - final byte[] mobileBody = JsonWriter.string( - prepareIosMobileJsonBuilder(localization, contentCountry) - .value(VIDEO_ID, videoId) - .value(CPN, iosCpn) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) - .getBytes(StandardCharsets.UTF_8); - - final JsonObject iosPlayerResponse = getJsonIosPostResponse(PLAYER, - mobileBody, localization, "&t=" + generateTParameter() - + "&id=" + videoId); - - if (isPlayerResponseNotValid(iosPlayerResponse, videoId)) { - throw new ExtractionException("IOS player response is not valid"); - } - - final JsonObject streamingData = iosPlayerResponse.getObject(STREAMING_DATA); - if (!isNullOrEmpty(streamingData)) { - iosStreamingData = streamingData; - playerCaptionsTracklistRenderer = iosPlayerResponse.getObject("captions") - .getObject("playerCaptionsTracklistRenderer"); + private static void throwExceptionIfPlayerResponseNotValid( + @Nonnull final JsonObject webPlayerResponse, + @Nonnull final String videoId) throws ExtractionException { + if (isPlayerResponseNotValid(webPlayerResponse, videoId)) { + // Check the playability status, as private and deleted videos and invalid video + // IDs do not return the ID provided in the player response + // When the requested video is playable and a different video ID is returned, it + // has the OK playability status, meaning the ExtractionException after this check + // will be thrown + checkPlayabilityStatus(webPlayerResponse.getObject(PLAYABILITY_STATUS)); + throw new ExtractionException("WEB player response is not valid"); } } - /** - * Download the {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} JSON player as an embed client to bypass - * some age-restrictions and assign the streaming data to the {@code html5StreamingData} JSON - * object. - * - * @param contentCountry the content country to use - * @param localization the localization to use - * @param videoId the video id - */ - private void fetchTvHtml5EmbedJsonPlayer(@Nonnull final ContentCountry contentCountry, - @Nonnull final Localization localization, - @Nonnull final String videoId) + private void fetchHtml5EmbedClient(@Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nullable final PoTokenResult webEmbedPoTokenResult) throws IOException, ExtractionException { - tvHtml5SimplyEmbedCpn = generateContentPlaybackNonce(); + html5Cpn = generateContentPlaybackNonce(); - final JsonObject tvHtml5EmbedPlayerResponse = getJsonPostResponse(PLAYER, - createTvHtml5EmbedPlayerBody(localization, - contentCountry, - videoId, - YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId), - tvHtml5SimplyEmbedCpn), localization); + final JsonObject webEmbeddedPlayerResponse = + YoutubeStreamHelper.getWebEmbeddedPlayerResponse(localization, contentCountry, + videoId, html5Cpn, webEmbedPoTokenResult, + YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId)); - if (isPlayerResponseNotValid(tvHtml5EmbedPlayerResponse, videoId)) { - throw new ExtractionException("TVHTML5 embed player response is not valid"); + // Save the webEmbeddedPlayerResponse into playerResponse in the case the video cannot be + // played, so some metadata can be retrieved + playerResponse = webEmbeddedPlayerResponse; + + // Check if the playability status in the player response, if the age-restriction could not + // be bypassed, an exception will be thrown + checkPlayabilityStatus(webEmbeddedPlayerResponse.getObject(PLAYABILITY_STATUS)); + + if (isPlayerResponseNotValid(webEmbeddedPlayerResponse, videoId)) { + throw new ExtractionException("WEB_EMBEDDED_PLAYER player response is not valid"); } - final JsonObject streamingData = tvHtml5EmbedPlayerResponse.getObject(STREAMING_DATA); - if (!isNullOrEmpty(streamingData)) { - playerResponse = tvHtml5EmbedPlayerResponse; - tvHtml5SimplyEmbedStreamingData = streamingData; - playerCaptionsTracklistRenderer = playerResponse.getObject("captions") - .getObject("playerCaptionsTracklistRenderer"); + html5StreamingData = webEmbeddedPlayerResponse.getObject(STREAMING_DATA); + playerCaptionsTracklistRenderer = webEmbeddedPlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + if (webEmbedPoTokenResult != null) { + html5StreamingUrlsPoToken = webEmbedPoTokenResult.streamingDataPoToken; + } + } + + private void fetchAndroidClient(@Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nullable final PoTokenResult androidPoTokenResult) { + try { + androidCpn = generateContentPlaybackNonce(); + + final JsonObject androidPlayerResponse; + if (androidPoTokenResult == null) { + androidPlayerResponse = YoutubeStreamHelper.getAndroidReelPlayerResponse( + contentCountry, localization, videoId, androidCpn); + } else { + androidPlayerResponse = YoutubeStreamHelper.getAndroidPlayerResponse( + contentCountry, localization, videoId, androidCpn, + androidPoTokenResult); + } + + if (!isPlayerResponseNotValid(androidPlayerResponse, videoId)) { + androidStreamingData = androidPlayerResponse.getObject(STREAMING_DATA); + + if (isNullOrEmpty(playerCaptionsTracklistRenderer)) { + playerCaptionsTracklistRenderer = + androidPlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + } + + if (androidPoTokenResult != null) { + androidStreamingUrlsPoToken = androidPoTokenResult.streamingDataPoToken; + } + } + } catch (final Exception ignored) { + // Ignore exceptions related to ANDROID client fetch or parsing, as it is not + // compulsory to play contents + } + } + + private void fetchIosClient(@Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nullable final PoTokenResult iosPoTokenResult) { + try { + iosCpn = generateContentPlaybackNonce(); + + final JsonObject iosPlayerResponse = YoutubeStreamHelper.getIosPlayerResponse( + contentCountry, localization, videoId, iosCpn, iosPoTokenResult); + + if (!isPlayerResponseNotValid(iosPlayerResponse, videoId)) { + iosStreamingData = iosPlayerResponse.getObject(STREAMING_DATA); + + if (isNullOrEmpty(playerCaptionsTracklistRenderer)) { + playerCaptionsTracklistRenderer = iosPlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + } + + if (iosPoTokenResult != null) { + iosStreamingUrlsPoToken = iosPoTokenResult.streamingDataPoToken; + } + } + } catch (final Exception ignored) { + // Ignore exceptions related to IOS client fetch or parsing, as it is not + // compulsory to play contents } } @@ -1021,7 +1097,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { * Checks whether a player response is invalid. * *

    - * If YouTube detect that requests come from a third party client, they may replace the real + * If YouTube detects that requests come from a third party client, they may replace the real * player response by another one of a video saying that this content is not available on this * app and to watch it on the latest version of YouTube. This behavior has been observed on the * {@code ANDROID} client, see @@ -1054,6 +1130,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getString("videoId")); } + private static boolean isVideoAgeRestricted(@Nonnull final JsonObject playabilityStatus) { + // This is language dependent + return "login_required".equalsIgnoreCase(playabilityStatus.getString("status")) + && playabilityStatus.getString("reason", "") + .contains("age"); + } + /*////////////////////////////////////////////////////////////////////////// // Utils //////////////////////////////////////////////////////////////////////////*/ @@ -1106,22 +1189,28 @@ public class YoutubeStreamExtractor extends StreamExtractor { java.util.stream.Stream.of( /* - Use the iosStreamingData object first because there is no n param and no - signatureCiphers in streaming URLs of the iOS client + Use the html5StreamingData object first because YouTube should have less + control on HTML5 clients, especially for poTokens - The androidStreamingData is used as second way as it isn't used on livestreams, - it doesn't return all available streams, and the Android client extraction is - more likely to break + The androidStreamingData is used as second way as the Android client extraction + is more likely to break - As age-restricted videos are not common, use tvHtml5SimplyEmbedStreamingData - last, which will be the only one not empty for age-restricted content + As iOS streaming data is affected by poTokens and not passing them should lead + to 403 responses, it should be used in the last resort */ - new Pair<>(iosStreamingData, iosCpn), - new Pair<>(androidStreamingData, androidCpn), - new Pair<>(tvHtml5SimplyEmbedStreamingData, tvHtml5SimplyEmbedCpn) - ) - .flatMap(pair -> getStreamsFromStreamingDataKey(videoId, pair.getFirst(), - streamingDataKey, itagTypeWanted, pair.getSecond())) + new Pair<>(html5StreamingData, + new Pair<>(html5Cpn, html5StreamingUrlsPoToken)), + new Pair<>(androidStreamingData, + new Pair<>(androidCpn, androidStreamingUrlsPoToken)), + new Pair<>(iosStreamingData, + new Pair<>(iosCpn, iosStreamingUrlsPoToken))) + .flatMap(pair -> getStreamsFromStreamingDataKey( + videoId, + pair.getFirst(), + streamingDataKey, + itagTypeWanted, + pair.getSecond().getFirst(), + pair.getSecond().getSecond())) .map(streamBuilderHelper) .forEachOrdered(stream -> { if (!Stream.containSimilarStream(stream, streamList)) { @@ -1146,7 +1235,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { *

      *
    • the {@link ItagItem}'s id of the stream as its id;
      • *
    • {@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and - * and as the value of {@code isUrl};
      • + * as the value of {@code isUrl}; *
    • the media format returned by the {@link ItagItem} as its media format;
      • *
    • its average bitrate with the value returned by {@link * ItagItem#getAverageBitrate()};
      • @@ -1199,7 +1288,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { *
        *
      • the {@link ItagItem}'s id of the stream as its id;
        • *
      • {@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and - * and as the value of {@code isUrl};
        • + * as the value of {@code isUrl}; *
      • the media format returned by the {@link ItagItem} as its media format;
        • *
      • whether it is video-only with the {@code areStreamsVideoOnly} parameter
        • *
      • the {@link ItagItem};
        • @@ -1255,7 +1344,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final JsonObject streamingData, final String streamingDataKey, @Nonnull final ItagItem.ItagType itagTypeWanted, - @Nonnull final String contentPlaybackNonce) { + @Nonnull final String contentPlaybackNonce, + @Nullable final String poToken) { if (streamingData == null || !streamingData.has(streamingDataKey)) { return java.util.stream.Stream.empty(); } @@ -1268,7 +1358,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final ItagItem itagItem = ItagItem.getItag(formatData.getInt("itag")); if (itagItem.itagType == itagTypeWanted) { return buildAndAddItagInfoToList(videoId, formatData, itagItem, - itagItem.itagType, contentPlaybackNonce); + itagItem.itagType, contentPlaybackNonce, poToken); } } catch (final ExtractionException ignored) { // If the itag is not supported, the n parameter of HTML5 clients cannot be @@ -1284,7 +1374,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull final JsonObject formatData, @Nonnull final ItagItem itagItem, @Nonnull final ItagItem.ItagType itagType, - @Nonnull final String contentPlaybackNonce) throws ExtractionException { + @Nonnull final String contentPlaybackNonce, + @Nullable final String poToken) throws ExtractionException { String streamUrl; if (formatData.has("url")) { streamUrl = formatData.getString("url"); @@ -1298,9 +1389,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + signature; } - // Add the content playback nonce to the stream URL - streamUrl += "&" + CPN + "=" + contentPlaybackNonce; - // Decode the n parameter if it is present // If it cannot be decoded, the stream cannot be used as streaming URLs return HTTP 403 // responses if it has not the right value @@ -1310,6 +1398,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { streamUrl = YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated( videoId, streamUrl); + // Add the content playback nonce to the stream URL + streamUrl += "&" + CPN + "=" + contentPlaybackNonce; + + // Add the poToken, if there is one + if (poToken != null) { + streamUrl += "&pot=" + poToken; + } + final JsonObject initRange = formatData.getObject("initRange"); final JsonObject indexRange = formatData.getObject("indexRange"); final String mimeType = formatData.getString("mimeType", ""); @@ -1587,4 +1683,53 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getObject("results") .getArray("contents")); } + + /** + * Set the {@link PoTokenProvider} instance to be used for fetching {@code poToken}s. + * + *

        + * This method allows setting an implementation of {@link PoTokenProvider} which will be used + * to obtain poTokens required for YouTube player requests and streaming URLs. These tokens + * are used by YouTube to verify the integrity of the user's device or browser and are required + * for playback with several clients. + *

        + * + *

        + * Without a {@link PoTokenProvider}, the extractor makes its best effort to fetch as many + * streams as possible, but without {@code poToken}s, some formats may be not available or + * fetching may be slower due to additional requests done to get streams. + *

        + * + *

        + * Note that any provider change will be only applied on the next {@link #fetchPage()} request. + *

        + * + * @param poTokenProvider the {@link PoTokenProvider} instance to set, which can be null to + * remove a provider already passed + * @see PoTokenProvider + */ + @SuppressWarnings("unused") + public static void setPoTokenProvider(@Nullable final PoTokenProvider poTokenProvider) { + YoutubeStreamExtractor.poTokenProvider = poTokenProvider; + } + + /** + * Set whether to fetch the iOS player responses. + * + *

        + * This method allows fetching the iOS player response, which can be useful in scenarios where + * streams from the iOS player response are needed, especially HLS manifests. + *

        + * + *

        + * Note that at the time of writing, YouTube is rolling out a {@code poToken} requirement on + * this client, formats from HLS manifests do not seem to be affected. + *

        + * + * @param fetchIosClient whether to fetch the iOS client + */ + @SuppressWarnings("unused") + public static void setFetchIosClient(final boolean fetchIosClient) { + YoutubeStreamExtractor.fetchIosClient = fetchIosClient; + } }