kavin/NewPipeExtractor
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipeExtractor.git synced 2025-04-28 16:00:33 +05:30
Code Issues Projects Releases Wiki Activity

Merge pull request #1272 from AudricV/yt_clients_changes_and_potokens_support

Browse Source 
[YouTube] Refactor player clients, add support for poTokens, extract visitor data from the service and more
This commit is contained in:
Stypox 2025-02-05 10:02:51 +01:00 committed by GitHub
commit fe168aba0b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 1316 additions and 552 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java Normal file
View File

@ -0,0 +1,118 @@
package org.schabi.newpipe.extractor.services.youtube;
final class ClientsConstants {
private ClientsConstants() {
}
// Common client fields
static final String DESKTOP_CLIENT_PLATFORM = "DESKTOP";
static final String MOBILE_CLIENT_PLATFORM = "MOBILE";
static final String WATCH_CLIENT_SCREEN = "WATCH";
static final String EMBED_CLIENT_SCREEN = "EMBED";
// WEB (YouTube desktop) client fields
static final String WEB_CLIENT_ID = "1";
static final String WEB_CLIENT_NAME = "WEB";
/**
* The client version for InnerTube requests with the {@code WEB} client, used as the last
* fallback if the extraction of the real one failed.
*/
static final String WEB_HARDCODED_CLIENT_VERSION = "2.20250122.04.00";
// WEB_REMIX (YouTube Music) client fields
static final String WEB_REMIX_CLIENT_ID = "67";
static final String WEB_REMIX_CLIENT_NAME = "WEB_REMIX";
static final String WEB_REMIX_HARDCODED_CLIENT_VERSION = "1.20250122.01.00";
// TVHTML5 (YouTube on TVs and consoles using HTML5) client fields
static final String TVHTML5_CLIENT_ID = "7";
static final String TVHTML5_CLIENT_NAME = "TVHTML5";
static final String TVHTML5_CLIENT_VERSION = "7.20250122.15.00";
static final String TVHTML5_CLIENT_PLATFORM = "GAME_CONSOLE";
static final String TVHTML5_DEVICE_MAKE = "Sony";
static final String TVHTML5_DEVICE_MODEL_AND_OS_NAME = "PlayStation 4";
// CHECKSTYLE:OFF
static final String TVHTML5_USER_AGENT =
"Mozilla/5.0 (PlayStation; PlayStation 4/12.00) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15";
// CHECKSTYLE:ON
// WEB_EMBEDDED_PLAYER (YouTube embeds)
static final String WEB_EMBEDDED_CLIENT_ID = "56";
static final String WEB_EMBEDDED_CLIENT_NAME = "WEB_EMBEDDED_PLAYER";
static final String WEB_EMBEDDED_CLIENT_VERSION = "1.20250121.00.00";
// IOS (iOS YouTube app) client fields
static final String IOS_CLIENT_ID = "5";
static final String IOS_CLIENT_NAME = "IOS";
/**
* The hardcoded client version of the iOS app used for InnerTube requests with this client.
*
* <p>
* It can be extracted by getting the latest release version of the app on
* <a href="https://apps.apple.com/us/app/youtube-watch-listen-stream/id544007664/">the App
* Store page of the YouTube app</a>, in the {@code What’s New} section.
* </p>
*/
static final String IOS_CLIENT_VERSION = "20.03.02";
/**
* The device machine id for the iPhone 15 Pro Max, used to get 60fps with the {@code iOS}
* client.
*
* <p>
* See <a href="https://gist.github.com/adamawolf/3048717">this GitHub Gist</a> for more
* information.
* </p>
*/
static final String IOS_DEVICE_MODEL = "iPhone16,2";
/**
* The iOS version to be used in JSON POST requests, the one of an iPhone 15 Pro Max running
* iOS 18.2.1 with the hardcoded version of the iOS app (for the {@code "osVersion"} field).
*
* <p>
* The value of this field seems to use the following structure:
* "iOS major version.minor version.patch version.build version", where
* "patch version" is equal to 0 if it isn't set
* The build version corresponding to the iOS version used can be found on
* <a href="https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max">
* https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max</a>
* </p>
*
* @see #IOS_USER_AGENT_VERSION
*/
static final String IOS_OS_VERSION = "18.2.1.22C161";
/**
* The iOS version to be used in the HTTP user agent for requests.
*
* <p>
* This should be the same of as {@link #IOS_OS_VERSION}.
* </p>
*
* @see #IOS_OS_VERSION
*/
static final String IOS_USER_AGENT_VERSION = "18_2_1";
// ANDROID (Android YouTube app) client fields
static final String ANDROID_CLIENT_ID = "3";
static final String ANDROID_CLIENT_NAME = "ANDROID";
/**
* The hardcoded client version of the Android app used for InnerTube requests with this
* client.
*
* <p>
* It can be extracted by getting the latest release version of the app in an APK repository
* such as <a href="https://www.apkmirror.com/apk/google-inc/youtube/">APKMirror</a>.
* </p>
*/
static final String ANDROID_CLIENT_VERSION = "19.28.35";
}

148
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java Normal file
View File

@ -0,0 +1,148 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_NAME;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.EMBED_CLIENT_SCREEN;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_NAME;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_OS_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.MOBILE_CLIENT_PLATFORM;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_NAME;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_PLATFORM;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MAKE;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MODEL_AND_OS_NAME;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WATCH_CLIENT_SCREEN;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_NAME;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
// TODO: add docs
public final class InnertubeClientRequestInfo {
@Nonnull
public ClientInfo clientInfo;
@Nonnull
public DeviceInfo deviceInfo;
public static final class ClientInfo {
@Nonnull
public String clientName;
@Nonnull
public String clientVersion;
@Nonnull
public String clientScreen;
@Nullable
public String clientId;
@Nullable
public String visitorData;
private ClientInfo(@Nonnull final String clientName,
@Nonnull final String clientVersion,
@Nonnull final String clientScreen,
@Nullable final String clientId,
@Nullable final String visitorData) {
this.clientName = clientName;
this.clientVersion = clientVersion;
this.clientScreen = clientScreen;
this.clientId = clientId;
this.visitorData = visitorData;
}
}
public static final class DeviceInfo {
@Nonnull
public String platform;
@Nullable
public String deviceMake;
@Nullable
public String deviceModel;
@Nullable
public String osName;
@Nullable
public String osVersion;
public int androidSdkVersion;
private DeviceInfo(@Nonnull final String platform,
@Nullable final String deviceMake,
@Nullable final String deviceModel,
@Nullable final String osName,
@Nullable final String osVersion,
final int androidSdkVersion) {
this.platform = platform;
this.deviceMake = deviceMake;
this.deviceModel = deviceModel;
this.osName = osName;
this.osVersion = osVersion;
this.androidSdkVersion = androidSdkVersion;
}
}
private InnertubeClientRequestInfo(@Nonnull final ClientInfo clientInfo,
@Nonnull final DeviceInfo deviceInfo) {
this.clientInfo = clientInfo;
this.deviceInfo = deviceInfo;
}
@Nonnull
public static InnertubeClientRequestInfo ofWebClient() {
return new InnertubeClientRequestInfo(
new InnertubeClientRequestInfo.ClientInfo(
WEB_CLIENT_NAME, WEB_HARDCODED_CLIENT_VERSION, WATCH_CLIENT_SCREEN,
WEB_CLIENT_ID, null),
new InnertubeClientRequestInfo.DeviceInfo(DESKTOP_CLIENT_PLATFORM, null, null,
null, null, -1));
}
@Nonnull
public static InnertubeClientRequestInfo ofWebEmbeddedPlayerClient() {
return new InnertubeClientRequestInfo(
new InnertubeClientRequestInfo.ClientInfo(WEB_EMBEDDED_CLIENT_NAME,
WEB_REMIX_HARDCODED_CLIENT_VERSION, EMBED_CLIENT_SCREEN,
WEB_EMBEDDED_CLIENT_ID, null),
new InnertubeClientRequestInfo.DeviceInfo(DESKTOP_CLIENT_PLATFORM, null, null,
null, null, -1));
}
@Nonnull
public static InnertubeClientRequestInfo ofTvHtml5Client() {
return new InnertubeClientRequestInfo(
new InnertubeClientRequestInfo.ClientInfo(TVHTML5_CLIENT_NAME,
TVHTML5_CLIENT_VERSION, WATCH_CLIENT_SCREEN, TVHTML5_CLIENT_ID, null),
new InnertubeClientRequestInfo.DeviceInfo(TVHTML5_CLIENT_PLATFORM,
TVHTML5_DEVICE_MAKE, TVHTML5_DEVICE_MODEL_AND_OS_NAME,
TVHTML5_DEVICE_MODEL_AND_OS_NAME, "", -1));
}
@Nonnull
public static InnertubeClientRequestInfo ofAndroidClient() {
return new InnertubeClientRequestInfo(
new InnertubeClientRequestInfo.ClientInfo(ANDROID_CLIENT_NAME,
ANDROID_CLIENT_VERSION, WATCH_CLIENT_SCREEN, ANDROID_CLIENT_ID, null),
new InnertubeClientRequestInfo.DeviceInfo(MOBILE_CLIENT_PLATFORM, null, null,
"Android", "15", 35));
}
@Nonnull
public static InnertubeClientRequestInfo ofIosClient() {
return new InnertubeClientRequestInfo(
new InnertubeClientRequestInfo.ClientInfo(IOS_CLIENT_NAME, IOS_CLIENT_VERSION,
WATCH_CLIENT_SCREEN, IOS_CLIENT_ID, null),
new InnertubeClientRequestInfo.DeviceInfo(MOBILE_CLIENT_PLATFORM, "Apple",
IOS_DEVICE_MODEL, "iOS", IOS_OS_VERSION, -1));
}
}

120
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java Normal file
View File

@ -0,0 +1,120 @@
package org.schabi.newpipe.extractor.services.youtube;
import javax.annotation.Nullable;
/**
* Interface to provide {@code poToken}s to YouTube player requests.
*
* <p>
* On some major clients, YouTube requires that the integrity of the device passes some checks to
* allow playback.
* </p>
*
* <p>
* These checks involve running codes to verify the integrity and using their result to generate
* one or multiple {@code poToken}(s) (which stands for proof of origin token(s)).
* </p>
*
* <p>
* These tokens may have a role in triggering the sign in requirement.
* </p>
*
* <p>
* If an implementation does not want to return a {@code poToken} for a specific client, it <b>must
* return {@code null}</b>.
* </p>
*
* <p>
* <b>Implementations of this interface are expected to be thread-safe, as they may be accessed by
* multiple threads.</b>
* </p>
*/
public interface PoTokenProvider {
/**
* Get a {@link PoTokenResult} specific to the desktop website, a.k.a. the WEB InnerTube client.
*
* <p>
* To be generated and valid, {@code poToken}s from this client must be generated using Google's
* BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They
* must be added to adaptive/DASH streaming URLs with the {@code pot} parameter.
* </p>
*
* <p>
* Note that YouTube desktop website generates two {@code poToken}s:
* - one for the player requests {@code poToken}s, using the videoId as the minter value;
* - one for the streaming URLs, using a visitor data for logged-out users as the minter value.
* </p>
*
* @return a {@link PoTokenResult} specific to the WEB InnerTube client
*/
@Nullable
PoTokenResult getWebClientPoToken(String videoId);
/**
* Get a {@link PoTokenResult} specific to the web embeds, a.k.a. the WEB_EMBEDDED_PLAYER
* InnerTube client.
*
* <p>
* To be generated and valid, {@code poToken}s from this client must be generated using Google's
* BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They
* should be added to adaptive/DASH streaming URLs with the {@code pot} parameter.
* </p>
*
* <p>
* As of writing, like the YouTube desktop website previously did, it generates only one
* {@code poToken}, sent in player requests and streaming URLs, using a visitor data for
* logged-out users. {@code poToken}s do not seem to be mandatory for now on this client.
* </p>
*
* @return a {@link PoTokenResult} specific to the WEB_EMBEDDED_PLAYER InnerTube client
*/
@Nullable
PoTokenResult getWebEmbedClientPoToken(String videoId);
/**
* Get a {@link PoTokenResult} specific to the Android app, a.k.a. the ANDROID InnerTube client.
*
* <p>
* Implementation details are not known, the app uses DroidGuard, a downloaded native virtual
* machine ran by Google Play Services for which its code is updated pretty frequently.
* </p>
*
* <p>
* As of writing, DroidGuard seem to check for the Android app signature and package ID, as
* non-rooted YouTube patched with reVanced doesn't work without spoofing another InnerTube
* client while the rooted version works without any client spoofing.
* </p>
*
* <p>
* There should be only one {@code poToken} needed for the player requests, it shouldn't be
* required for regular adaptive URLs (i.e. not server adaptive bitrate (SABR) URLs). HLS
* formats returned (only for premieres and running and post-live livestreams) in the client's
* HLS manifest URL should work without {@code poToken}s.
* </p>
*
* @return a {@link PoTokenResult} specific to the ANDROID InnerTube client
*/
@Nullable
PoTokenResult getAndroidClientPoToken(String videoId);
/**
* Get a {@link PoTokenResult} specific to the iOS app, a.k.a. the IOS InnerTube client.
*
* <p>
* Implementation details are not known, the app seem to use something called iosGuard which
* should be similar to Android's DroidGuard. It may rely on Apple's attestation APIs.
* </p>
*
* <p>
* As of writing, there should be only one {@code poToken} needed for the player requests, it
* shouldn't be required for regular adaptive URLs (i.e. not server adaptive bitrate (SABR)
* URLs). HLS formats returned in the client's HLS manifest URL should also work without a
* {@code poToken}.
* </p>
*
* @return a {@link PoTokenResult} specific to the IOS InnerTube client
*/
@Nullable
PoTokenResult getIosClientPoToken(String videoId);
}

52
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java Normal file
View File

@ -0,0 +1,52 @@
package org.schabi.newpipe.extractor.services.youtube;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.Objects;
/**
* The result of a supported/successful {@code poToken} extraction request by a
* {@link PoTokenProvider}.
*/
public final class PoTokenResult {
/**
* The visitor data associated with a {@code poToken}.
*/
@Nonnull
public final String visitorData;
/**
* The {@code poToken} of a player request, a Protobuf object encoded as a base 64 string.
*/
@Nonnull
public final String playerRequestPoToken;
/**
* The {@code poToken} to be appended to streaming URLs, a Protobuf object encoded as a base
* 64 string.
*
* <p>
* It may be required on some clients such as HTML5 ones and may also differ from the player
* request {@code poToken}.
* </p>
*/
@Nullable
public final String streamingDataPoToken;
/**
* Construct a {@link PoTokenResult} instance.
*
* @param visitorData see {@link #visitorData}
* @param playerRequestPoToken see {@link #playerRequestPoToken}
* @param streamingDataPoToken see {@link #streamingDataPoToken}
* @throws NullPointerException if a non-null parameter is null
*/
public PoTokenResult(@Nonnull final String visitorData,
@Nonnull final String playerRequestPoToken,
@Nullable final String streamingDataPoToken) {
this.visitorData = Objects.requireNonNull(visitorData);
this.playerRequestPoToken = Objects.requireNonNull(playerRequestPoToken);
this.streamingDataPoToken = streamingDataPoToken;
}
}

486
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -21,6 +21,18 @@
package org.schabi.newpipe.extractor.services.youtube; package org.schabi.newpipe.extractor.services.youtube;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader; import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_NAME;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray; import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
@ -144,55 +156,6 @@ public final class YoutubeParsingHelper {
*/ */
public static final String RACY_CHECK_OK = "racyCheckOk"; public static final String RACY_CHECK_OK = "racyCheckOk";
/**
* The hardcoded client ID used for InnerTube requests with the {@code WEB} client.
*/
private static final String WEB_CLIENT_ID = "1";
/**
* The client version for InnerTube requests with the {@code WEB} client, used as the last
* fallback if the extraction of the real one failed.
*/
private static final String HARDCODED_CLIENT_VERSION = "2.20240718.01.00";
/**
* The hardcoded client version of the Android app used for InnerTube requests with this
* client.
*
* <p>
* It can be extracted by getting the latest release version of the app in an APK repository
* such as <a href="https://www.apkmirror.com/apk/google-inc/youtube/">APKMirror</a>.
* </p>
*/
private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "19.28.35";
/**
* The hardcoded client version of the iOS app used for InnerTube requests with this client.
*
* <p>
* It can be extracted by getting the latest release version of the app on
* <a href="https://apps.apple.com/us/app/youtube-watch-listen-stream/id544007664/">the App
* Store page of the YouTube app</a>, in the {@code What’s New} section.
* </p>
*/
private static final String IOS_YOUTUBE_CLIENT_VERSION = "20.03.02";
/**
* The hardcoded client version used for InnerTube requests with the TV HTML5 embed client.
*/
private static final String TVHTML5_SIMPLY_EMBED_CLIENT_VERSION = "2.0";
/**
* The hardcoded client ID used for InnerTube requests with the YouTube Music desktop client.
*/
private static final String YOUTUBE_MUSIC_CLIENT_ID = "67";
/**
* The hardcoded client version used for InnerTube requests with the YouTube Music desktop
* client.
*/
private static final String HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION = "1.20240715.01.00";
private static String clientVersion; private static String clientVersion;
private static String youtubeMusicClientVersion; private static String youtubeMusicClientVersion;
@ -212,49 +175,16 @@ public final class YoutubeParsingHelper {
private static final String CONTENT_PLAYBACK_NONCE_ALPHABET = private static final String CONTENT_PLAYBACK_NONCE_ALPHABET =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
/**
* The device machine id for the iPhone 15 Pro Max,
* used to get 60fps with the {@code iOS} client.
*
* <p>
* See <a href="https://gist.github.com/adamawolf/3048717">this GitHub Gist</a> for more
* information.
* </p>
*/
private static final String IOS_DEVICE_MODEL = "iPhone16,2";
/**
* Spoofing an iPhone 15 Pro Max running iOS 18.2.1 with the hardcoded version of the iOS app.
* To be used for the {@code "osVersion"} field in JSON POST requests.
* <p>
* The value of this field seems to use the following structure:
* "iOS major version.minor version.patch version.build version", where
* "patch version" is equal to 0 if it isn't set
* The build version corresponding to the iOS version used can be found on
* <a href="https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max">
* https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max</a>
* </p>
*
* @see #IOS_USER_AGENT_VERSION
*/
private static final String IOS_OS_VERSION = "18.2.1.22C161";
/**
* Spoofing an iPhone 15 Pro Max running iOS 18.2.1 with the hardcoded version of the iOS app.
* To be used in the user agent for requests.
*
* @see #IOS_OS_VERSION
*/
private static final String IOS_USER_AGENT_VERSION = "18_2_1";
private static Random numberGenerator = new Random(); private static Random numberGenerator = new Random();
private static final String FEED_BASE_CHANNEL_ID = private static final String FEED_BASE_CHANNEL_ID =
"https://www.youtube.com/feeds/videos.xml?channel_id="; "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
private static final Pattern C_WEB_PATTERN = Pattern.compile("&c=WEB"); private static final Pattern C_WEB_PATTERN = Pattern.compile("&c=WEB");
private static final Pattern C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN = private static final Pattern C_WEB_EMBEDDED_PLAYER_PATTERN =
Pattern.compile("&c=TVHTML5_SIMPLY_EMBEDDED_PLAYER"); Pattern.compile("&c=WEB_EMBEDDED_PLAYER");
private static final Pattern C_TVHTML5_PLAYER_PATTERN =
Pattern.compile("&c=TVHTML5");
private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID"); private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS"); private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
@ -561,9 +491,9 @@ public final class YoutubeParsingHelper {
.object("client") .object("client")
.value("hl", "en-GB") .value("hl", "en-GB")
.value("gl", "GB") .value("gl", "GB")
.value("clientName", "WEB") .value("clientName", WEB_CLIENT_NAME)
.value("clientVersion", HARDCODED_CLIENT_VERSION) .value("clientVersion", WEB_HARDCODED_CLIENT_VERSION)
.value("platform", "DESKTOP") .value("platform", DESKTOP_CLIENT_PLATFORM)
.value("utcOffsetMinutes", 0) .value("utcOffsetMinutes", 0)
.end() .end()
.object("request") .object("request")
@ -581,7 +511,7 @@ public final class YoutubeParsingHelper {
.end().done().getBytes(StandardCharsets.UTF_8); .end().done().getBytes(StandardCharsets.UTF_8);
// @formatter:on // @formatter:on
final var headers = getClientHeaders(WEB_CLIENT_ID, HARDCODED_CLIENT_VERSION); final var headers = getClientHeaders(WEB_CLIENT_ID, WEB_HARDCODED_CLIENT_VERSION);
// This endpoint is fetched by the YouTube website to get the items of its main menu and is // This endpoint is fetched by the YouTube website to get the items of its main menu and is
// pretty lightweight (around 30kB) // pretty lightweight (around 30kB)
@ -705,7 +635,7 @@ public final class YoutubeParsingHelper {
// Fallback to the hardcoded one if it is valid // Fallback to the hardcoded one if it is valid
if (isHardcodedClientVersionValid()) { if (isHardcodedClientVersionValid()) {
clientVersion = HARDCODED_CLIENT_VERSION; clientVersion = WEB_HARDCODED_CLIENT_VERSION;
return clientVersion; return clientVersion;
} }
@ -752,11 +682,11 @@ public final class YoutubeParsingHelper {
.object() .object()
.object("context") .object("context")
.object("client") .object("client")
.value("clientName", "WEB_REMIX") .value("clientName", WEB_REMIX_CLIENT_NAME)
.value("clientVersion", HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION) .value("clientVersion", WEB_REMIX_HARDCODED_CLIENT_VERSION)
.value("hl", "en-GB") .value("hl", "en-GB")
.value("gl", "GB") .value("gl", "GB")
.value("platform", "DESKTOP") .value("platform", DESKTOP_CLIENT_PLATFORM)
.value("utcOffsetMinutes", 0) .value("utcOffsetMinutes", 0)
.end() .end()
.object("request") .object("request")
@ -775,8 +705,7 @@ public final class YoutubeParsingHelper {
// @formatter:on // @formatter:on
final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL)); final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL));
headers.putAll(getClientHeaders(YOUTUBE_MUSIC_CLIENT_ID, headers.putAll(getClientHeaders(WEB_REMIX_CLIENT_ID, WEB_HARDCODED_CLIENT_VERSION));
HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION));
final Response response = getDownloader().postWithContentTypeJson(url, headers, json); final Response response = getDownloader().postWithContentTypeJson(url, headers, json);
// Ensure to have a valid response // Ensure to have a valid response
@ -789,7 +718,7 @@ public final class YoutubeParsingHelper {
return youtubeMusicClientVersion; return youtubeMusicClientVersion;
} }
if (isHardcodedYoutubeMusicClientVersionValid()) { if (isHardcodedYoutubeMusicClientVersionValid()) {
youtubeMusicClientVersion = HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION; youtubeMusicClientVersion = WEB_REMIX_HARDCODED_CLIENT_VERSION;
return youtubeMusicClientVersion; return youtubeMusicClientVersion;
} }
@ -1134,116 +1063,20 @@ public final class YoutubeParsingHelper {
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization))); + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
} }
public static JsonObject getJsonAndroidPostResponse(
final String endpoint,
final byte[] body,
@Nonnull final Localization localization,
@Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
return getMobilePostResponse(endpoint, body, localization,
getAndroidUserAgent(localization), endPartOfUrlRequest);
}
public static JsonObject getJsonIosPostResponse(
final String endpoint,
final byte[] body,
@Nonnull final Localization localization,
@Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
return getMobilePostResponse(endpoint, body, localization, getIosUserAgent(localization),
endPartOfUrlRequest);
}
private static JsonObject getMobilePostResponse(
final String endpoint,
final byte[] body,
@Nonnull final Localization localization,
@Nonnull final String userAgent,
@Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
final var headers = Map.of("User-Agent", List.of(userAgent),
"X-Goog-Api-Format-Version", List.of("2"));
final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?"
+ DISABLE_PRETTY_PRINT_PARAMETER;
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl
: baseEndpointUrl + endPartOfUrlRequest,
headers, body, localization)));
}
@Nonnull @Nonnull
public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder( public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
@Nonnull final Localization localization, @Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry) @Nonnull final ContentCountry contentCountry) throws IOException, ExtractionException {
throws IOException, ExtractionException {
return prepareDesktopJsonBuilder(localization, contentCountry, null);
}
@Nonnull
public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nullable final String visitorData)
throws IOException, ExtractionException {
String vData = visitorData;
if (vData == null) {
vData = randomVisitorData(contentCountry);
}
// @formatter:off // @formatter:off
return JsonObject.builder() return JsonObject.builder()
.object("context") .object("context")
.object("client") .object("client")
.value("hl", localization.getLocalizationCode()) .value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode()) .value("gl", contentCountry.getCountryCode())
.value("clientName", "WEB") .value("clientName", WEB_CLIENT_NAME)
.value("clientVersion", getClientVersion()) .value("clientVersion", getClientVersion())
.value("originalUrl", "https://www.youtube.com") .value("originalUrl", "https://www.youtube.com")
.value("platform", "DESKTOP") .value("platform", DESKTOP_CLIENT_PLATFORM)
.value("utcOffsetMinutes", 0)
.value("visitorData", vData)
.end()
.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user")
// TODO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
// @formatter:on
}
@Nonnull
public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry) {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("clientName", "ANDROID")
.value("clientVersion", ANDROID_YOUTUBE_CLIENT_VERSION)
.value("platform", "MOBILE")
.value("osName", "Android")
.value("osVersion", "14")
/*
A valid Android SDK version is required to be sure to get a valid player
response
If this parameter is not provided, the player response is replaced by an
error saying the message "The following content is not available on this
app. Watch this content on the latest version on YouTube" (it was
previously a 5-minute video with this message)
See https://github.com/TeamNewPipe/NewPipe/issues/8713
The Android SDK version corresponding to the Android version used in
requests is sent
*/
.value("androidSdkVersion", 34)
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("utcOffsetMinutes", 0) .value("utcOffsetMinutes", 0)
.end() .end()
.object("request") .object("request")
@ -1260,122 +1093,6 @@ public final class YoutubeParsingHelper {
// @formatter:on // @formatter:on
} }
@Nonnull
public static JsonBuilder<JsonObject> prepareIosMobileJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry) {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("clientName", "IOS")
.value("clientVersion", IOS_YOUTUBE_CLIENT_VERSION)
.value("deviceMake", "Apple")
// Device model is required to get 60fps streams
.value("deviceModel", IOS_DEVICE_MODEL)
.value("platform", "MOBILE")
.value("osName", "iOS")
.value("osVersion", IOS_OS_VERSION)
.value("visitorData", randomVisitorData(contentCountry))
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("utcOffsetMinutes", 0)
.end()
.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user")
// TODO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
// @formatter:on
}
@Nonnull
public static JsonBuilder<JsonObject> prepareTvHtml5EmbedJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId) {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("clientName", "TVHTML5_SIMPLY_EMBEDDED_PLAYER")
.value("clientVersion", TVHTML5_SIMPLY_EMBED_CLIENT_VERSION)
.value("clientScreen", "EMBED")
.value("platform", "TV")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("utcOffsetMinutes", 0)
.end()
.object("thirdParty")
.value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
.end()
.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user")
// TODO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
// @formatter:on
}
@Nonnull
public static JsonObject getWebPlayerResponse(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId) throws IOException, ExtractionException {
final byte[] body = JsonWriter.string(
prepareDesktopJsonBuilder(localization, contentCountry)
.value(VIDEO_ID, videoId)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(StandardCharsets.UTF_8);
final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER
+ "&$fields=microformat,playabilityStatus,storyboards,videoDetails";
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(
url, getYouTubeHeaders(), body, localization)));
}
@Nonnull
public static byte[] createTvHtml5EmbedPlayerBody(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId,
@Nonnull final Integer sts,
@Nonnull final String contentPlaybackNonce) {
// @formatter:off
return JsonWriter.string(
prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId)
.object("playbackContext")
.object("contentPlaybackContext")
// Signature timestamp from the JavaScript base player is needed to get
// working obfuscated URLs
.value("signatureTimestamp", sts)
.value("referer", "https://www.youtube.com/watch?v=" + videoId)
.end()
.end()
.value(CPN, contentPlaybackNonce)
.value(VIDEO_ID, videoId)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(StandardCharsets.UTF_8);
// @formatter:on
}
/** /**
* Get the user-agent string used as the user-agent for InnerTube requests with the Android * Get the user-agent string used as the user-agent for InnerTube requests with the Android
* client. * client.
@ -1391,9 +1108,8 @@ public final class YoutubeParsingHelper {
*/ */
@Nonnull @Nonnull
public static String getAndroidUserAgent(@Nullable final Localization localization) { public static String getAndroidUserAgent(@Nullable final Localization localization) {
// Spoofing an Android 14 device with the hardcoded version of the Android app return "com.google.android.youtube/" + ANDROID_CLIENT_VERSION
return "com.google.android.youtube/" + ANDROID_YOUTUBE_CLIENT_VERSION + " (Linux; U; Android 15; "
+ " (Linux; U; Android 14; "
+ (localization != null ? localization : Localization.DEFAULT).getCountryCode() + (localization != null ? localization : Localization.DEFAULT).getCountryCode()
+ ") gzip"; + ") gzip";
} }
@ -1413,23 +1129,30 @@ public final class YoutubeParsingHelper {
*/ */
@Nonnull @Nonnull
public static String getIosUserAgent(@Nullable final Localization localization) { public static String getIosUserAgent(@Nullable final Localization localization) {
// Spoofing an iPhone 15 Pro Max running iOS 18.1.0 return "com.google.ios.youtube/" + IOS_CLIENT_VERSION + "(" + IOS_DEVICE_MODEL
// with the hardcoded version of the iOS app + "; U; CPU iOS " + IOS_USER_AGENT_VERSION + " like Mac OS X; "
return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION
+ "(" + IOS_DEVICE_MODEL + "; U; CPU iOS "
+ IOS_USER_AGENT_VERSION + " like Mac OS X; "
+ (localization != null ? localization : Localization.DEFAULT).getCountryCode() + (localization != null ? localization : Localization.DEFAULT).getCountryCode()
+ ")"; + ")";
} }
/**
* Get the user-agent string used as the user-agent for InnerTube requests with the HTML5 TV
* client.
*
* @return the user-agent used for InnerTube requests with the TVHTML5 client
*/
@Nonnull
public static String getTvHtml5UserAgent() {
return TVHTML5_USER_AGENT;
}
/** /**
* Returns a {@link Map} containing the required YouTube Music headers. * Returns a {@link Map} containing the required YouTube Music headers.
*/ */
@Nonnull @Nonnull
public static Map<String, List<String>> getYoutubeMusicHeaders() { public static Map<String, List<String>> getYoutubeMusicHeaders() {
final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL)); final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL));
headers.putAll(getClientHeaders(YOUTUBE_MUSIC_CLIENT_ID, headers.putAll(getClientHeaders(WEB_REMIX_CLIENT_ID, youtubeMusicClientVersion));
youtubeMusicClientVersion));
return headers; return headers;
} }
@ -1461,7 +1184,7 @@ public final class YoutubeParsingHelper {
* *
* @param url The URL to be set as the origin and referrer. * @param url The URL to be set as the origin and referrer.
*/ */
private static Map<String, List<String>> getOriginReferrerHeaders(@Nonnull final String url) { public static Map<String, List<String>> getOriginReferrerHeaders(@Nonnull final String url) {
final var urlList = List.of(url); final var urlList = List.of(url);
return Map.of("Origin", urlList, "Referer", urlList); return Map.of("Origin", urlList, "Referer", urlList);
} }
@ -1473,7 +1196,7 @@ public final class YoutubeParsingHelper {
* @param name The X-YouTube-Client-Name value. * @param name The X-YouTube-Client-Name value.
* @param version X-YouTube-Client-Version value. * @param version X-YouTube-Client-Version value.
*/ */
private static Map<String, List<String>> getClientHeaders(@Nonnull final String name, static Map<String, List<String>> getClientHeaders(@Nonnull final String name,
@Nonnull final String version) { @Nonnull final String version) {
return Map.of("X-YouTube-Client-Name", List.of(name), return Map.of("X-YouTube-Client-Name", List.of(name),
"X-YouTube-Client-Version", List.of(version)); "X-YouTube-Client-Version", List.of(version));
@ -1669,15 +1392,24 @@ public final class YoutubeParsingHelper {
} }
/** /**
* Check if the streaming URL is a URL from the YouTube {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} * Check if the streaming URL is from the YouTube {@code WEB_EMBEDDED_PLAYER} client.
* client.
* *
* @param url the streaming URL on which check if it's a {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} * @param url the streaming URL to be checked.
* streaming URL. * @return true if it's a {@code WEB_EMBEDDED_PLAYER} streaming URL, false otherwise
* @return true if it's a {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} streaming URL, false otherwise
*/ */
public static boolean isTvHtml5SimplyEmbeddedPlayerStreamingUrl(@Nonnull final String url) { public static boolean isWebEmbeddedPlayerStreamingUrl(@Nonnull final String url) {
return Parser.isMatch(C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN, url); return Parser.isMatch(C_WEB_EMBEDDED_PLAYER_PATTERN, url);
}
/**
* Check if the streaming URL is a URL from the YouTube {@code TVHTML5} client.
*
* @param url the streaming URL on which check if it's a {@code TVHTML5}
* streaming URL.
* @return true if it's a {@code TVHTML5} streaming URL, false otherwise
*/
public static boolean isTvHtml5StreamingUrl(@Nonnull final String url) {
return Parser.isMatch(C_TVHTML5_PLAYER_PATTERN, url);
} }
/** /**
@ -1775,4 +1507,96 @@ public final class YoutubeParsingHelper {
return null; return null;
} }
} }
@Nonnull
public static String getVisitorDataFromInnertube(
@Nonnull final InnertubeClientRequestInfo innertubeClientRequestInfo,
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final Map<String, List<String>> httpHeaders,
@Nonnull final String innertubeDomainAndVersionEndpoint,
@Nullable final String embedUrl,
final boolean useGuideEndpoint) throws IOException, ExtractionException {
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(
localization, contentCountry, innertubeClientRequestInfo, embedUrl);
final byte[] body = JsonWriter.string(builder.done())
.getBytes(StandardCharsets.UTF_8);
final String visitorData = JsonUtils.toJsonObject(getValidJsonResponseBody(getDownloader()
.postWithContentTypeJson(
innertubeDomainAndVersionEndpoint
+ (useGuideEndpoint ? "guide" : "visitor_id") + "?"
+ DISABLE_PRETTY_PRINT_PARAMETER,
httpHeaders, body)))
.getObject("responseContext")
.getString("visitorData");
if (isNullOrEmpty(visitorData)) {
throw new ParsingException("Could not get visitorData");
}
return visitorData;
}
@Nonnull
static JsonBuilder<JsonObject> prepareJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final InnertubeClientRequestInfo innertubeClientRequestInfo,
@Nullable final String embedUrl) {
final JsonBuilder<JsonObject> builder = JsonObject.builder()
.object("context")
.object("client")
.value("clientName", innertubeClientRequestInfo.clientInfo.clientName)
.value("clientVersion", innertubeClientRequestInfo.clientInfo.clientVersion)
.value("clientScreen", innertubeClientRequestInfo.clientInfo.clientScreen)
.value("platform", innertubeClientRequestInfo.deviceInfo.platform);
if (innertubeClientRequestInfo.clientInfo.visitorData != null) {
builder.value("visitorData", innertubeClientRequestInfo.clientInfo.visitorData);
}
if (innertubeClientRequestInfo.deviceInfo.deviceMake != null) {
builder.value("deviceMake", innertubeClientRequestInfo.deviceInfo.deviceMake);
}
if (innertubeClientRequestInfo.deviceInfo.deviceModel != null) {
builder.value("deviceModel", innertubeClientRequestInfo.deviceInfo.deviceModel);
}
if (innertubeClientRequestInfo.deviceInfo.osName != null) {
builder.value("osName", innertubeClientRequestInfo.deviceInfo.osName);
}
if (innertubeClientRequestInfo.deviceInfo.osVersion != null) {
builder.value("osVersion", innertubeClientRequestInfo.deviceInfo.osVersion);
}
if (innertubeClientRequestInfo.deviceInfo.androidSdkVersion > 0) {
builder.value("androidSdkVersion",
innertubeClientRequestInfo.deviceInfo.androidSdkVersion);
}
builder.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("utcOffsetMinutes", 0)
.end();
if (embedUrl != null) {
builder.object("thirdParty")
.value("embedUrl", embedUrl)
.end();
}
builder.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user")
// TODO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
return builder;
}
} }

344
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java Normal file
View File

@ -0,0 +1,344 @@
package org.schabi.newpipe.extractor.services.youtube;
import com.grack.nanojson.JsonBuilder;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_GAPIS_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getOriginReferrerHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
public final class YoutubeStreamHelper {
private static final String PLAYER = "player";
private static final String SERVICE_INTEGRITY_DIMENSIONS = "serviceIntegrityDimensions";
private static final String PO_TOKEN = "poToken";
private static final String BASE_YT_DESKTOP_WATCH_URL = "https://www.youtube.com/watch?v=";
private YoutubeStreamHelper() {
}
@Nonnull
public static JsonObject getWebMetadataPlayerResponse(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId) throws IOException, ExtractionException {
final InnertubeClientRequestInfo innertubeClientRequestInfo =
InnertubeClientRequestInfo.ofWebClient();
innertubeClientRequestInfo.clientInfo.clientVersion = getClientVersion();
final Map<String, List<String>> headers = getYouTubeHeaders();
// We must always pass a valid visitorData to get valid player responses, which needs to be
// got from YouTube
innertubeClientRequestInfo.clientInfo.visitorData =
YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo,
localization, contentCountry, headers, YOUTUBEI_V1_URL, null, false);
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, null);
addVideoIdCpnAndOkChecks(builder, videoId, null);
final byte[] body = JsonWriter.string(builder.done())
.getBytes(StandardCharsets.UTF_8);
final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER
+ "&$fields=microformat,playabilityStatus,storyboards,videoDetails";
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(
url, headers, body, localization)));
}
@Nonnull
public static JsonObject getTvHtml5PlayerResponse(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId,
@Nonnull final String cpn,
final int signatureTimestamp) throws IOException, ExtractionException {
final InnertubeClientRequestInfo innertubeClientRequestInfo =
InnertubeClientRequestInfo.ofTvHtml5Client();
final Map<String, List<String>> headers = new HashMap<>(
getClientHeaders(TVHTML5_CLIENT_ID, TVHTML5_CLIENT_VERSION));
headers.putAll(getOriginReferrerHeaders("https://www.youtube.com"));
headers.put("User-Agent", List.of(TVHTML5_USER_AGENT));
// We must always pass a valid visitorData to get valid player responses, which needs to be
// got from YouTube
// For some reason, the TVHTML5 client doesn't support the visitor_id endpoint, use the
// guide one instead, which is quite lightweight
innertubeClientRequestInfo.clientInfo.visitorData =
YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo,
localization, contentCountry, headers, YOUTUBEI_V1_URL, null, true);
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, null);
addVideoIdCpnAndOkChecks(builder, videoId, cpn);
addPlaybackContext(builder, BASE_YT_DESKTOP_WATCH_URL + videoId, signatureTimestamp);
final byte[] body = JsonWriter.string(builder.done())
.getBytes(StandardCharsets.UTF_8);
final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(url, headers, body, localization)));
}
@Nonnull
public static JsonObject getWebFullPlayerResponse(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId,
@Nonnull final String cpn,
@Nonnull final PoTokenResult webPoTokenResult,
final int signatureTimestamp) throws IOException, ExtractionException {
final InnertubeClientRequestInfo innertubeClientRequestInfo =
InnertubeClientRequestInfo.ofWebClient();
innertubeClientRequestInfo.clientInfo.clientVersion = getClientVersion();
innertubeClientRequestInfo.clientInfo.visitorData = webPoTokenResult.visitorData;
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, null);
addVideoIdCpnAndOkChecks(builder, videoId, cpn);
addPlaybackContext(builder, BASE_YT_DESKTOP_WATCH_URL + videoId, signatureTimestamp);
addPoToken(builder, webPoTokenResult.playerRequestPoToken);
final byte[] body = JsonWriter.string(builder.done())
.getBytes(StandardCharsets.UTF_8);
final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(
url, getYouTubeHeaders(), body, localization)));
}
@Nonnull
public static JsonObject getWebEmbeddedPlayerResponse(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId,
@Nonnull final String cpn,
@Nullable final PoTokenResult webEmbeddedPoTokenResult,
final int signatureTimestamp) throws IOException, ExtractionException {
final InnertubeClientRequestInfo innertubeClientRequestInfo =
InnertubeClientRequestInfo.ofWebEmbeddedPlayerClient();
final Map<String, List<String>> headers = new HashMap<>(
getClientHeaders(WEB_EMBEDDED_CLIENT_ID, WEB_EMBEDDED_CLIENT_VERSION));
headers.putAll(getOriginReferrerHeaders("https://www.youtube.com"));
final String embedUrl = BASE_YT_DESKTOP_WATCH_URL + videoId;
// We must always pass a valid visitorData to get valid player responses, which needs to be
// got from YouTube
innertubeClientRequestInfo.clientInfo.visitorData = webEmbeddedPoTokenResult == null
? YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo,
localization, contentCountry, headers, YOUTUBEI_V1_URL, embedUrl, false)
: webEmbeddedPoTokenResult.visitorData;
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, embedUrl);
addVideoIdCpnAndOkChecks(builder, videoId, cpn);
addPlaybackContext(builder, embedUrl, signatureTimestamp);
if (webEmbeddedPoTokenResult != null) {
addPoToken(builder, webEmbeddedPoTokenResult.playerRequestPoToken);
}
final byte[] body = JsonWriter.string(builder.done())
.getBytes(StandardCharsets.UTF_8);
final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(url, headers, body, localization)));
}
public static JsonObject getAndroidPlayerResponse(
@Nonnull final ContentCountry contentCountry,
@Nonnull final Localization localization,
@Nonnull final String videoId,
@Nonnull final String cpn,
@Nonnull final PoTokenResult androidPoTokenResult)
throws IOException, ExtractionException {
final InnertubeClientRequestInfo innertubeClientRequestInfo =
InnertubeClientRequestInfo.ofAndroidClient();
innertubeClientRequestInfo.clientInfo.visitorData = androidPoTokenResult.visitorData;
final Map<String, List<String>> headers =
getMobileClientHeaders(getAndroidUserAgent(localization));
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, null);
addVideoIdCpnAndOkChecks(builder, videoId, cpn);
addPoToken(builder, androidPoTokenResult.playerRequestPoToken);
final byte[] body = JsonWriter.string(builder.done())
.getBytes(StandardCharsets.UTF_8);
final String url = YOUTUBEI_V1_GAPIS_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER
+ "&t=" + generateTParameter() + "&id=" + videoId;
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(url, headers, body, localization)));
}
public static JsonObject getAndroidReelPlayerResponse(
@Nonnull final ContentCountry contentCountry,
@Nonnull final Localization localization,
@Nonnull final String videoId,
@Nonnull final String cpn) throws IOException, ExtractionException {
final InnertubeClientRequestInfo innertubeClientRequestInfo =
InnertubeClientRequestInfo.ofAndroidClient();
final Map<String, List<String>> headers =
getMobileClientHeaders(getAndroidUserAgent(localization));
// We must always pass a valid visitorData to get valid player responses, which needs to be
// got from YouTube
innertubeClientRequestInfo.clientInfo.visitorData =
YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo,
localization, contentCountry, headers, YOUTUBEI_V1_GAPIS_URL, null, false);
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, null);
addVideoIdCpnAndOkChecks(builder, videoId, cpn);
builder.object("playerRequest")
.value(VIDEO_ID, videoId)
.end()
.value("disablePlayerResponse", false);
final byte[] body = JsonWriter.string(builder.done())
.getBytes(StandardCharsets.UTF_8);
final String url = YOUTUBEI_V1_GAPIS_URL + "reel/reel_item_watch" + "?"
+ DISABLE_PRETTY_PRINT_PARAMETER + "&t=" + generateTParameter() + "&id=" + videoId
+ "&$fields=playerResponse";
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(url, headers, body, localization)))
.getObject("playerResponse");
}
public static JsonObject getIosPlayerResponse(@Nonnull final ContentCountry contentCountry,
@Nonnull final Localization localization,
@Nonnull final String videoId,
@Nonnull final String cpn,
@Nullable final PoTokenResult iosPoTokenResult)
throws IOException, ExtractionException {
final InnertubeClientRequestInfo innertubeClientRequestInfo =
InnertubeClientRequestInfo.ofIosClient();
final Map<String, List<String>> headers =
getMobileClientHeaders(getIosUserAgent(localization));
// We must always pass a valid visitorData to get valid player responses, which needs to be
// got from YouTube
innertubeClientRequestInfo.clientInfo.visitorData = iosPoTokenResult == null
? YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo,
localization, contentCountry, headers, YOUTUBEI_V1_URL, null, false)
: iosPoTokenResult.visitorData;
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, null);
addVideoIdCpnAndOkChecks(builder, videoId, cpn);
if (iosPoTokenResult != null) {
addPoToken(builder, iosPoTokenResult.playerRequestPoToken);
}
final byte[] body = JsonWriter.string(builder.done())
.getBytes(StandardCharsets.UTF_8);
final String url = YOUTUBEI_V1_GAPIS_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER
+ "&t=" + generateTParameter() + "&id=" + videoId;
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(url, headers, body, localization)));
}
private static void addVideoIdCpnAndOkChecks(@Nonnull final JsonBuilder<JsonObject> builder,
@Nonnull final String videoId,
@Nullable final String cpn) {
builder.value(VIDEO_ID, videoId);
if (cpn != null) {
builder.value(CPN, cpn);
}
builder.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true);
}
private static void addPlaybackContext(@Nonnull final JsonBuilder<JsonObject> builder,
@Nonnull final String referer,
final int signatureTimestamp) {
builder.object("playbackContext")
.object("contentPlaybackContext")
.value("signatureTimestamp", signatureTimestamp)
.value("referer", referer)
.end()
.end();
}
private static void addPoToken(@Nonnull final JsonBuilder<JsonObject> builder,
@Nonnull final String poToken) {
builder.object(SERVICE_INTEGRITY_DIMENSIONS)
.value(PO_TOKEN, poToken)
.end();
}
@Nonnull
private static Map<String, List<String>> getMobileClientHeaders(
@Nonnull final String userAgent) {
return Map.of("User-Agent", List.of(userAgent),
"X-Goog-Api-Format-Version", List.of("2"));
}
}

31
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java
View File

@ -1,12 +1,14 @@
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators; package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getOriginReferrerHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTvHtml5UserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5StreamingUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebEmbeddedPlayerStreamingUrl;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MediaFormat;
@ -26,6 +28,7 @@ import org.w3c.dom.Element;
import java.io.IOException; import java.io.IOException;
import java.io.StringWriter; import java.io.StringWriter;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
@ -582,8 +585,8 @@ public final class YoutubeDashManifestCreatorsUtils {
* This method fetches, for OTF streams and for post-live-DVR streams: * This method fetches, for OTF streams and for post-live-DVR streams:
* <ul> * <ul>
* <li>the base URL of the stream, to which are appended {@link #SQ_0} and * <li>the base URL of the stream, to which are appended {@link #SQ_0} and
* {@link #RN_0} parameters, with a {@code GET} request for streaming URLs from HTML5 * {@link #RN_0} parameters, with a {@code POS} request for streaming URLs from
* clients and a {@code POST} request for the ones from the {@code ANDROID} and the * {@code WEB}, {@code TVHTML5}, {@code WEB_EMBEDDED_PLAYER}, {@code ANDROID} and
* {@code IOS} clients;</li> * {@code IOS} clients;</li>
* <li>for streaming URLs from HTML5 clients, the {@link #ALR_YES} param is also added. * <li>for streaming URLs from HTML5 clients, the {@link #ALR_YES} param is also added.
* </li> * </li>
@ -601,8 +604,10 @@ public final class YoutubeDashManifestCreatorsUtils {
@Nonnull final ItagItem itagItem, @Nonnull final ItagItem itagItem,
final DeliveryType deliveryType) final DeliveryType deliveryType)
throws CreationException { throws CreationException {
final boolean isTvHtml5StreamingUrl = isTvHtml5StreamingUrl(baseStreamingUrl);
final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl) final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl)
|| isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl); || isTvHtml5StreamingUrl
|| isWebEmbeddedPlayerStreamingUrl(baseStreamingUrl);
final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl); final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl);
final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl); final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl);
if (isHtml5StreamingUrl) { if (isHtml5StreamingUrl) {
@ -615,7 +620,7 @@ public final class YoutubeDashManifestCreatorsUtils {
final String mimeTypeExpected = itagItem.getMediaFormat().getMimeType(); final String mimeTypeExpected = itagItem.getMediaFormat().getMimeType();
if (!isNullOrEmpty(mimeTypeExpected)) { if (!isNullOrEmpty(mimeTypeExpected)) {
return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl, return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl,
mimeTypeExpected); mimeTypeExpected, isTvHtml5StreamingUrl);
} }
} else if (isAndroidStreamingUrl || isIosStreamingUrl) { } else if (isAndroidStreamingUrl || isIosStreamingUrl) {
try { try {
@ -730,6 +735,8 @@ public final class YoutubeDashManifestCreatorsUtils {
* @param downloader the {@link Downloader} instance to be used * @param downloader the {@link Downloader} instance to be used
* @param streamingUrl the streaming URL which we are trying to get a streaming URL * @param streamingUrl the streaming URL which we are trying to get a streaming URL
* without any redirection on the network and/or IP used * without any redirection on the network and/or IP used
* @param isTvHtml5StreamingUrl whether the streaming URL comes from TVHTML5 client, in
* order to use an appropriate HTTP User-Agent header
* @param responseMimeTypeExpected the response mime type expected from Google video servers * @param responseMimeTypeExpected the response mime type expected from Google video servers
* @return the {@link Response} of the stream, which should have no redirections * @return the {@link Response} of the stream, which should have no redirections
*/ */
@ -738,17 +745,23 @@ public final class YoutubeDashManifestCreatorsUtils {
private static Response getStreamingWebUrlWithoutRedirects( private static Response getStreamingWebUrlWithoutRedirects(
@Nonnull final Downloader downloader, @Nonnull final Downloader downloader,
@Nonnull String streamingUrl, @Nonnull String streamingUrl,
@Nonnull final String responseMimeTypeExpected) @Nonnull final String responseMimeTypeExpected,
final boolean isTvHtml5StreamingUrl)
throws CreationException { throws CreationException {
try { try {
final var headers = getClientInfoHeaders(); final var headers = new HashMap<>(
getOriginReferrerHeaders("https://www.youtube.com"));
if (isTvHtml5StreamingUrl) {
headers.put("User-Agent", List.of(getTvHtml5UserAgent()));
}
String responseMimeType = ""; String responseMimeType = "";
int redirectsCount = 0; int redirectsCount = 0;
while (!responseMimeType.equals(responseMimeTypeExpected) while (!responseMimeType.equals(responseMimeTypeExpected)
&& redirectsCount < MAXIMUM_REDIRECT_COUNT) { && redirectsCount < MAXIMUM_REDIRECT_COUNT) {
final Response response = downloader.get(streamingUrl, headers); final byte[] html5Body = new byte[] {0x78, 0};
final Response response = downloader.post(streamingUrl, headers, html5Body);
final int responseCode = response.responseCode(); final int responseCode = response.responseCode();
if (responseCode != 200) { if (responseCode != 200) {

565
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -27,18 +27,12 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createTvHtml5EmbedPlayerBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
@ -66,9 +60,12 @@ import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider;
import org.schabi.newpipe.extractor.services.youtube.PoTokenResult;
import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager; import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager;
import org.schabi.newpipe.extractor.services.youtube.YoutubeMetaInfoHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeMetaInfoHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeStreamHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.DeliveryMethod;
@ -104,6 +101,10 @@ import javax.annotation.Nullable;
public class YoutubeStreamExtractor extends StreamExtractor { public class YoutubeStreamExtractor extends StreamExtractor {
@Nullable
private static PoTokenProvider poTokenProvider;
private static boolean fetchIosClient;
private JsonObject playerResponse; private JsonObject playerResponse;
private JsonObject nextResponse; private JsonObject nextResponse;
@ -112,7 +113,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nullable @Nullable
private JsonObject androidStreamingData; private JsonObject androidStreamingData;
@Nullable @Nullable
private JsonObject tvHtml5SimplyEmbedStreamingData; private JsonObject html5StreamingData;
private JsonObject videoPrimaryInfoRenderer; private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer;
@ -127,7 +128,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// three different strings are used. // three different strings are used.
private String iosCpn; private String iosCpn;
private String androidCpn; private String androidCpn;
private String tvHtml5SimplyEmbedCpn; private String html5Cpn;
@Nullable
private String html5StreamingUrlsPoToken;
@Nullable
private String androidStreamingUrlsPoToken;
@Nullable
private String iosStreamingUrlsPoToken;
public YoutubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) { public YoutubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
@ -321,7 +329,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return Long.parseLong(duration); return Long.parseLong(duration);
} catch (final Exception e) { } catch (final Exception e) {
return getDurationFromFirstAdaptiveFormat(Arrays.asList( return getDurationFromFirstAdaptiveFormat(Arrays.asList(
iosStreamingData, androidStreamingData, tvHtml5SimplyEmbedStreamingData)); html5StreamingData, androidStreamingData, iosStreamingData));
} }
} }
@ -579,11 +587,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getDashMpdUrl() throws ParsingException { public String getDashMpdUrl() throws ParsingException {
assertPageFetched(); assertPageFetched();
// There is no DASH manifest available in the iOS clients and the DASH manifest of the // There is no DASH manifest available with the iOS client
// Android client doesn't contain all available streams (mainly the WEBM ones)
return getManifestUrl( return getManifestUrl(
"dash", "dash",
Arrays.asList(androidStreamingData, tvHtml5SimplyEmbedStreamingData)); Arrays.asList(
new Pair<>(androidStreamingData, androidStreamingUrlsPoToken),
new Pair<>(html5StreamingData, html5StreamingUrlsPoToken)),
// Return version 7 of the DASH manifest, which is the latest one, reducing
// manifest size and allowing playback with some DASH players
"mpd_version=7");
} }
@Nonnull @Nonnull
@ -592,25 +604,44 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched(); assertPageFetched();
// Return HLS manifest of the iOS client first because on livestreams, the HLS manifest // Return HLS manifest of the iOS client first because on livestreams, the HLS manifest
// returned has separated audio and video streams // returned has separated audio and video streams and poTokens requirement do not seem to
// impact HLS formats (if a poToken is provided, it is added)
// Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response // Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response
// unless a Safari macOS user agent is used
return getManifestUrl( return getManifestUrl(
"hls", "hls",
Arrays.asList( Arrays.asList(
iosStreamingData, androidStreamingData, tvHtml5SimplyEmbedStreamingData)); new Pair<>(iosStreamingData, iosStreamingUrlsPoToken),
new Pair<>(androidStreamingData, androidStreamingUrlsPoToken),
new Pair<>(html5StreamingData, html5StreamingUrlsPoToken)),
"");
} }
@Nonnull @Nonnull
private static String getManifestUrl(@Nonnull final String manifestType, private static String getManifestUrl(
@Nonnull final List<JsonObject> streamingDataObjects) { @Nonnull final String manifestType,
@Nonnull final List<Pair<JsonObject, String>> streamingDataObjects,
@Nonnull final String partToAppendToManifestUrlEnd) {
final String manifestKey = manifestType + "ManifestUrl"; final String manifestKey = manifestType + "ManifestUrl";
return streamingDataObjects.stream() for (final Pair<JsonObject, String> streamingDataObj : streamingDataObjects) {
.filter(Objects::nonNull) if (streamingDataObj.getFirst() != null) {
.map(streamingDataObject -> streamingDataObject.getString(manifestKey)) final String manifestUrl = streamingDataObj.getFirst().getString(manifestKey);
.filter(Objects::nonNull) if (isNullOrEmpty(manifestUrl)) {
.findFirst() continue;
.orElse(""); }
// If poToken is not null, add it to manifest URL
if (streamingDataObj.getSecond() == null) {
return manifestUrl + "?" + partToAppendToManifestUrlEnd;
} else {
return manifestUrl + "?pot=" + streamingDataObj.getSecond() + "&"
+ partToAppendToManifestUrlEnd;
}
}
}
return "";
} }
@Override @Override
@ -684,7 +715,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
private void setStreamType() { private void setStreamType() {
if (playerResponse.getObject("playabilityStatus").has("liveStreamability")) { if (playerResponse.getObject(PLAYABILITY_STATUS).has("liveStreamability")) {
streamType = StreamType.LIVE_STREAM; streamType = StreamType.LIVE_STREAM;
} else if (playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false)) { } else if (playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false)) {
streamType = StreamType.POST_LIVE_STREAM; streamType = StreamType.POST_LIVE_STREAM;
@ -751,7 +782,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public String getErrorMessage() { public String getErrorMessage() {
try { try {
return getTextFromObject(playerResponse.getObject("playabilityStatus") return getTextFromObject(playerResponse.getObject(PLAYABILITY_STATUS)
.getObject("errorScreen").getObject("playerErrorMessageRenderer") .getObject("errorScreen").getObject("playerErrorMessageRenderer")
.getObject("reason")); .getObject("reason"));
} catch (final NullPointerException e) { } catch (final NullPointerException e) {
@ -766,10 +797,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static final String FORMATS = "formats"; private static final String FORMATS = "formats";
private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
private static final String STREAMING_DATA = "streamingData"; private static final String STREAMING_DATA = "streamingData";
private static final String PLAYER = "player";
private static final String NEXT = "next"; private static final String NEXT = "next";
private static final String SIGNATURE_CIPHER = "signatureCipher"; private static final String SIGNATURE_CIPHER = "signatureCipher";
private static final String CIPHER = "cipher"; private static final String CIPHER = "cipher";
private static final String PLAYER_CAPTIONS_TRACKLIST_RENDERER
= "playerCaptionsTracklistRenderer";
private static final String CAPTIONS = "captions";
private static final String PLAYABILITY_STATUS = "playabilityStatus";
@Override @Override
public void onFetchPage(@Nonnull final Downloader downloader) public void onFetchPage(@Nonnull final Downloader downloader)
@ -779,98 +813,53 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final Localization localization = getExtractorLocalization(); final Localization localization = getExtractorLocalization();
final ContentCountry contentCountry = getExtractorContentCountry(); final ContentCountry contentCountry = getExtractorContentCountry();
final JsonObject webPlayerResponse = YoutubeParsingHelper.getWebPlayerResponse( final PoTokenProvider poTokenproviderInstance = poTokenProvider;
localization, contentCountry, videoId); final boolean noPoTokenProviderSet = poTokenproviderInstance == null;
if (isPlayerResponseNotValid(webPlayerResponse, videoId)) { fetchHtml5Client(localization, contentCountry, videoId, poTokenproviderInstance,
// Check the playability status, as private and deleted videos and invalid video IDs do noPoTokenProviderSet);
// not return the ID provided in the player response
// When the requested video is playable and a different video ID is returned, it has
// the OK playability status, meaning the ExtractionException after this check will be
// thrown
checkPlayabilityStatus(
webPlayerResponse, webPlayerResponse.getObject("playabilityStatus"));
throw new ExtractionException("Initial WEB player response is not valid");
}
// Save the webPlayerResponse into playerResponse in the case the video cannot be played,
// so some metadata can be retrieved
playerResponse = webPlayerResponse;
// Use the player response from the player endpoint of the desktop internal API because
// there can be restrictions on videos in the embedded player.
// E.g. if a video is age-restricted, the embedded player's playabilityStatus says that
// the video cannot be played outside of YouTube, but does not show the original message.
final JsonObject playabilityStatus = webPlayerResponse.getObject("playabilityStatus");
final boolean isAgeRestricted = "login_required".equalsIgnoreCase(
playabilityStatus.getString("status"))
&& playabilityStatus.getString("reason", "")
.contains("age");
setStreamType(); setStreamType();
if (isAgeRestricted) { final PoTokenResult androidPoTokenResult = noPoTokenProviderSet ? null
fetchTvHtml5EmbedJsonPlayer(contentCountry, localization, videoId); : poTokenproviderInstance.getAndroidClientPoToken(videoId);
// If no streams can be fetched in the TVHTML5 simply embed client, the video should be fetchAndroidClient(localization, contentCountry, videoId, androidPoTokenResult);
// age-restricted, therefore throw an AgeRestrictedContentException explicitly.
if (tvHtml5SimplyEmbedStreamingData == null) { if (fetchIosClient) {
throw new AgeRestrictedContentException( final PoTokenResult iosPoTokenResult = noPoTokenProviderSet ? null
"This age-restricted video cannot be watched."); : poTokenproviderInstance.getIosClientPoToken(videoId);
fetchIosClient(localization, contentCountry, videoId, iosPoTokenResult);
} }
// Refresh the stream type because the stream type may be not properly known for final byte[] nextBody = JsonWriter.string(
// age-restricted videos
setStreamType();
} else {
checkPlayabilityStatus(webPlayerResponse, playabilityStatus);
// Fetching successfully the iOS player is mandatory to get streams
fetchIosMobileJsonPlayer(contentCountry, localization, videoId);
try {
fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId);
} catch (final Exception ignored) {
// Ignore exceptions related to ANDROID client fetch or parsing, as it is not
// compulsory to play contents
}
}
// The microformat JSON object of the content is only returned on the WEB client,
// so we need to store it instead of getting it directly from the playerResponse
playerMicroFormatRenderer = webPlayerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer");
final byte[] body = JsonWriter.string(
prepareDesktopJsonBuilder(localization, contentCountry) prepareDesktopJsonBuilder(localization, contentCountry)
.value(VIDEO_ID, videoId) .value(VIDEO_ID, videoId)
.value(CONTENT_CHECK_OK, true) .value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true) .value(RACY_CHECK_OK, true)
.done()) .done())
.getBytes(StandardCharsets.UTF_8); .getBytes(StandardCharsets.UTF_8);
nextResponse = getJsonPostResponse(NEXT, body, localization); nextResponse = getJsonPostResponse(NEXT, nextBody, localization);
} }
private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse, private static void checkPlayabilityStatus(@Nonnull final JsonObject playabilityStatus)
@Nonnull final JsonObject playabilityStatus)
throws ParsingException { throws ParsingException {
String status = playabilityStatus.getString("status"); final String status = playabilityStatus.getString("status");
if (status == null || status.equalsIgnoreCase("ok")) { if (status == null || status.equalsIgnoreCase("ok")) {
return; return;
} }
// If status exist, and is not "OK", throw the specific exception based on error message final String reason = playabilityStatus.getString("reason");
// or a ContentNotAvailableException with the reason text if it's an unknown reason.
final JsonObject newPlayabilityStatus =
youtubePlayerResponse.getObject("playabilityStatus");
status = newPlayabilityStatus.getString("status");
final String reason = newPlayabilityStatus.getString("reason");
if (status.equalsIgnoreCase("login_required") && reason == null) { if (status.equalsIgnoreCase("login_required")) {
final String message = newPlayabilityStatus.getArray("messages").getString(0); if (reason == null) {
final String message = playabilityStatus.getArray("messages").getString(0);
if (message != null && message.contains("private")) { if (message != null && message.contains("private")) {
throw new PrivateContentException("This video is private."); throw new PrivateContentException("This video is private");
}
} else if (reason.contains("age")) {
throw new AgeRestrictedContentException(
"This age-restricted video cannot be watched anonymously");
} }
} }
@ -879,16 +868,18 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (reason.contains("Music Premium")) { if (reason.contains("Music Premium")) {
throw new YoutubeMusicPremiumContentException(); throw new YoutubeMusicPremiumContentException();
} }
if (reason.contains("payment")) { if (reason.contains("payment")) {
throw new PaidContentException("This video is a paid video"); throw new PaidContentException("This video is a paid video");
} }
if (reason.contains("members-only")) { if (reason.contains("members-only")) {
throw new PaidContentException("This video is only available" throw new PaidContentException("This video is only available"
+ " for members of the channel of this video"); + " for members of the channel of this video");
} }
if (reason.contains("unavailable")) { if (reason.contains("unavailable")) {
final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus final String detailedErrorMessage = getTextFromObject(playabilityStatus
.getObject("errorScreen") .getObject("errorScreen")
.getObject("playerErrorMessageRenderer") .getObject("playerErrorMessageRenderer")
.getObject("subreason")); .getObject("subreason"));
@ -900,120 +891,205 @@ public class YoutubeStreamExtractor extends StreamExtractor {
Objects.requireNonNullElse(detailedErrorMessage, reason)); Objects.requireNonNullElse(detailedErrorMessage, reason));
} }
} }
if (reason.contains("age-restricted")) {
throw new AgeRestrictedContentException(
"This age-restricted video cannot be watched anonymously");
}
} }
throw new ContentNotAvailableException("Got error: \"" + reason + "\""); throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
} }
/** private void fetchHtml5Client(@Nonnull final Localization localization,
* Fetch the Android Mobile API and assign the streaming data to the androidStreamingData JSON @Nonnull final ContentCountry contentCountry,
* object. @Nonnull final String videoId,
*/ @Nullable final PoTokenProvider poTokenProviderInstance,
private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentCountry, final boolean noPoTokenProviderSet)
@Nonnull final Localization localization,
@Nonnull final String videoId)
throws IOException, ExtractionException { throws IOException, ExtractionException {
html5Cpn = generateContentPlaybackNonce();
// Suppress NPE warning as nullability is already checked before and passed with
// noPoTokenProviderSet
//noinspection DataFlowIssue
final PoTokenResult webPoTokenResult = noPoTokenProviderSet ? null
: poTokenProviderInstance.getWebClientPoToken(videoId);
final JsonObject webPlayerResponse;
if (noPoTokenProviderSet || webPoTokenResult == null) {
webPlayerResponse = YoutubeStreamHelper.getWebMetadataPlayerResponse(
localization, contentCountry, videoId);
throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId);
// Save the webPlayerResponse into playerResponse in the case the video cannot be
// played, so some metadata can be retrieved
playerResponse = webPlayerResponse;
// The microformat JSON object of the content is only returned on the WEB client,
// so we need to store it instead of getting it directly from the playerResponse
playerMicroFormatRenderer = playerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer");
final JsonObject playabilityStatus = webPlayerResponse.getObject(PLAYABILITY_STATUS);
if (isVideoAgeRestricted(playabilityStatus)) {
fetchHtml5EmbedClient(localization, contentCountry, videoId,
noPoTokenProviderSet ? null
: poTokenProviderInstance.getWebEmbedClientPoToken(videoId));
} else {
checkPlayabilityStatus(playabilityStatus);
final JsonObject tvHtml5PlayerResponse =
YoutubeStreamHelper.getTvHtml5PlayerResponse(
localization, contentCountry, videoId, html5Cpn,
YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId));
if (isPlayerResponseNotValid(tvHtml5PlayerResponse, videoId)) {
throw new ExtractionException("TVHTML5 player response is not valid");
}
html5StreamingData = tvHtml5PlayerResponse.getObject(STREAMING_DATA);
playerCaptionsTracklistRenderer = tvHtml5PlayerResponse.getObject(CAPTIONS)
.getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER);
}
} else {
webPlayerResponse = YoutubeStreamHelper.getWebFullPlayerResponse(
localization, contentCountry, videoId, html5Cpn, webPoTokenResult,
YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId));
throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId);
// Save the webPlayerResponse into playerResponse in the case the video cannot be
// played, so some metadata can be retrieved
playerResponse = webPlayerResponse;
// The microformat JSON object of the content is only returned on the WEB client,
// so we need to store it instead of getting it directly from the playerResponse
playerMicroFormatRenderer = playerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer");
final JsonObject playabilityStatus = webPlayerResponse.getObject(PLAYABILITY_STATUS);
if (isVideoAgeRestricted(playabilityStatus)) {
fetchHtml5EmbedClient(localization, contentCountry, videoId,
poTokenProviderInstance.getWebEmbedClientPoToken(videoId));
} else {
checkPlayabilityStatus(playabilityStatus);
html5StreamingData = webPlayerResponse.getObject(STREAMING_DATA);
playerCaptionsTracklistRenderer = webPlayerResponse.getObject(CAPTIONS)
.getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER);
html5StreamingUrlsPoToken = webPoTokenResult.streamingDataPoToken;
}
}
}
private static void throwExceptionIfPlayerResponseNotValid(
@Nonnull final JsonObject webPlayerResponse,
@Nonnull final String videoId) throws ExtractionException {
if (isPlayerResponseNotValid(webPlayerResponse, videoId)) {
// Check the playability status, as private and deleted videos and invalid video
// IDs do not return the ID provided in the player response
// When the requested video is playable and a different video ID is returned, it
// has the OK playability status, meaning the ExtractionException after this check
// will be thrown
checkPlayabilityStatus(webPlayerResponse.getObject(PLAYABILITY_STATUS));
throw new ExtractionException("WEB player response is not valid");
}
}
private void fetchHtml5EmbedClient(@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId,
@Nullable final PoTokenResult webEmbedPoTokenResult)
throws IOException, ExtractionException {
html5Cpn = generateContentPlaybackNonce();
final JsonObject webEmbeddedPlayerResponse =
YoutubeStreamHelper.getWebEmbeddedPlayerResponse(localization, contentCountry,
videoId, html5Cpn, webEmbedPoTokenResult,
YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId));
// Save the webEmbeddedPlayerResponse into playerResponse in the case the video cannot be
// played, so some metadata can be retrieved
playerResponse = webEmbeddedPlayerResponse;
// Check if the playability status in the player response, if the age-restriction could not
// be bypassed, an exception will be thrown
checkPlayabilityStatus(webEmbeddedPlayerResponse.getObject(PLAYABILITY_STATUS));
if (isPlayerResponseNotValid(webEmbeddedPlayerResponse, videoId)) {
throw new ExtractionException("WEB_EMBEDDED_PLAYER player response is not valid");
}
html5StreamingData = webEmbeddedPlayerResponse.getObject(STREAMING_DATA);
playerCaptionsTracklistRenderer = webEmbeddedPlayerResponse.getObject(CAPTIONS)
.getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER);
if (webEmbedPoTokenResult != null) {
html5StreamingUrlsPoToken = webEmbedPoTokenResult.streamingDataPoToken;
}
}
private void fetchAndroidClient(@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId,
@Nullable final PoTokenResult androidPoTokenResult) {
try {
androidCpn = generateContentPlaybackNonce(); androidCpn = generateContentPlaybackNonce();
final byte[] mobileBody = JsonWriter.string(
prepareAndroidMobileJsonBuilder(localization, contentCountry)
.object("playerRequest")
.value(VIDEO_ID, videoId)
.end()
.value("disablePlayerResponse", false)
.value(VIDEO_ID, videoId)
.value(CPN, androidCpn)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject androidPlayerResponse = getJsonAndroidPostResponse( final JsonObject androidPlayerResponse;
"reel/reel_item_watch", if (androidPoTokenResult == null) {
mobileBody, androidPlayerResponse = YoutubeStreamHelper.getAndroidReelPlayerResponse(
localization, contentCountry, localization, videoId, androidCpn);
"&t=" + generateTParameter() + "&id=" + videoId + "&$fields=playerResponse"); } else {
androidPlayerResponse = YoutubeStreamHelper.getAndroidPlayerResponse(
final JsonObject playerResponseObject = androidPlayerResponse.getObject("playerResponse"); contentCountry, localization, videoId, androidCpn,
if (isPlayerResponseNotValid(playerResponseObject, videoId)) { androidPoTokenResult);
return;
} }
final JsonObject streamingData = playerResponseObject.getObject(STREAMING_DATA); if (!isPlayerResponseNotValid(androidPlayerResponse, videoId)) {
if (!isNullOrEmpty(streamingData)) { androidStreamingData = androidPlayerResponse.getObject(STREAMING_DATA);
androidStreamingData = streamingData;
if (isNullOrEmpty(playerCaptionsTracklistRenderer)) { if (isNullOrEmpty(playerCaptionsTracklistRenderer)) {
playerCaptionsTracklistRenderer = playerResponseObject.getObject("captions") playerCaptionsTracklistRenderer =
.getObject("playerCaptionsTracklistRenderer"); androidPlayerResponse.getObject(CAPTIONS)
.getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER);
} }
if (androidPoTokenResult != null) {
androidStreamingUrlsPoToken = androidPoTokenResult.streamingDataPoToken;
}
}
} catch (final Exception ignored) {
// Ignore exceptions related to ANDROID client fetch or parsing, as it is not
// compulsory to play contents
} }
} }
/** private void fetchIosClient(@Nonnull final Localization localization,
* Fetch the iOS Mobile API and assign the streaming data to the iosStreamingData JSON @Nonnull final ContentCountry contentCountry,
* object. @Nonnull final String videoId,
*/ @Nullable final PoTokenResult iosPoTokenResult) {
private void fetchIosMobileJsonPlayer(@Nonnull final ContentCountry contentCountry, try {
@Nonnull final Localization localization,
@Nonnull final String videoId)
throws IOException, ExtractionException {
iosCpn = generateContentPlaybackNonce(); iosCpn = generateContentPlaybackNonce();
final byte[] mobileBody = JsonWriter.string(
prepareIosMobileJsonBuilder(localization, contentCountry)
.value(VIDEO_ID, videoId)
.value(CPN, iosCpn)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject iosPlayerResponse = getJsonIosPostResponse(PLAYER, final JsonObject iosPlayerResponse = YoutubeStreamHelper.getIosPlayerResponse(
mobileBody, localization, "&t=" + generateTParameter() contentCountry, localization, videoId, iosCpn, iosPoTokenResult);
+ "&id=" + videoId);
if (isPlayerResponseNotValid(iosPlayerResponse, videoId)) { if (!isPlayerResponseNotValid(iosPlayerResponse, videoId)) {
throw new ExtractionException("IOS player response is not valid"); iosStreamingData = iosPlayerResponse.getObject(STREAMING_DATA);
if (isNullOrEmpty(playerCaptionsTracklistRenderer)) {
playerCaptionsTracklistRenderer = iosPlayerResponse.getObject(CAPTIONS)
.getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER);
} }
final JsonObject streamingData = iosPlayerResponse.getObject(STREAMING_DATA); if (iosPoTokenResult != null) {
if (!isNullOrEmpty(streamingData)) { iosStreamingUrlsPoToken = iosPoTokenResult.streamingDataPoToken;
iosStreamingData = streamingData;
playerCaptionsTracklistRenderer = iosPlayerResponse.getObject("captions")
.getObject("playerCaptionsTracklistRenderer");
} }
} }
} catch (final Exception ignored) {
/** // Ignore exceptions related to IOS client fetch or parsing, as it is not
* Download the {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} JSON player as an embed client to bypass // compulsory to play contents
* some age-restrictions and assign the streaming data to the {@code html5StreamingData} JSON
* object.
*
* @param contentCountry the content country to use
* @param localization the localization to use
* @param videoId the video id
*/
private void fetchTvHtml5EmbedJsonPlayer(@Nonnull final ContentCountry contentCountry,
@Nonnull final Localization localization,
@Nonnull final String videoId)
throws IOException, ExtractionException {
tvHtml5SimplyEmbedCpn = generateContentPlaybackNonce();
final JsonObject tvHtml5EmbedPlayerResponse = getJsonPostResponse(PLAYER,
createTvHtml5EmbedPlayerBody(localization,
contentCountry,
videoId,
YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId),
tvHtml5SimplyEmbedCpn), localization);
if (isPlayerResponseNotValid(tvHtml5EmbedPlayerResponse, videoId)) {
throw new ExtractionException("TVHTML5 embed player response is not valid");
}
final JsonObject streamingData = tvHtml5EmbedPlayerResponse.getObject(STREAMING_DATA);
if (!isNullOrEmpty(streamingData)) {
playerResponse = tvHtml5EmbedPlayerResponse;
tvHtml5SimplyEmbedStreamingData = streamingData;
playerCaptionsTracklistRenderer = playerResponse.getObject("captions")
.getObject("playerCaptionsTracklistRenderer");
} }
} }
@ -1021,7 +1097,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
* Checks whether a player response is invalid. * Checks whether a player response is invalid.
* *
* <p> * <p>
* If YouTube detect that requests come from a third party client, they may replace the real * If YouTube detects that requests come from a third party client, they may replace the real
* player response by another one of a video saying that this content is not available on this * player response by another one of a video saying that this content is not available on this
* app and to watch it on the latest version of YouTube. This behavior has been observed on the * app and to watch it on the latest version of YouTube. This behavior has been observed on the
* {@code ANDROID} client, see * {@code ANDROID} client, see
@ -1054,6 +1130,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
.getString("videoId")); .getString("videoId"));
} }
private static boolean isVideoAgeRestricted(@Nonnull final JsonObject playabilityStatus) {
// This is language dependent
return "login_required".equalsIgnoreCase(playabilityStatus.getString("status"))
&& playabilityStatus.getString("reason", "")
.contains("age");
}
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
// Utils // Utils
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
@ -1106,22 +1189,28 @@ public class YoutubeStreamExtractor extends StreamExtractor {
java.util.stream.Stream.of( java.util.stream.Stream.of(
/* /*
Use the iosStreamingData object first because there is no n param and no Use the html5StreamingData object first because YouTube should have less
signatureCiphers in streaming URLs of the iOS client control on HTML5 clients, especially for poTokens
The androidStreamingData is used as second way as it isn't used on livestreams, The androidStreamingData is used as second way as the Android client extraction
it doesn't return all available streams, and the Android client extraction is is more likely to break
more likely to break
As age-restricted videos are not common, use tvHtml5SimplyEmbedStreamingData As iOS streaming data is affected by poTokens and not passing them should lead
last, which will be the only one not empty for age-restricted content to 403 responses, it should be used in the last resort
*/ */
new Pair<>(iosStreamingData, iosCpn), new Pair<>(html5StreamingData,
new Pair<>(androidStreamingData, androidCpn), new Pair<>(html5Cpn, html5StreamingUrlsPoToken)),
new Pair<>(tvHtml5SimplyEmbedStreamingData, tvHtml5SimplyEmbedCpn) new Pair<>(androidStreamingData,
) new Pair<>(androidCpn, androidStreamingUrlsPoToken)),
.flatMap(pair -> getStreamsFromStreamingDataKey(videoId, pair.getFirst(), new Pair<>(iosStreamingData,
streamingDataKey, itagTypeWanted, pair.getSecond())) new Pair<>(iosCpn, iosStreamingUrlsPoToken)))
.flatMap(pair -> getStreamsFromStreamingDataKey(
videoId,
pair.getFirst(),
streamingDataKey,
itagTypeWanted,
pair.getSecond().getFirst(),
pair.getSecond().getSecond()))
.map(streamBuilderHelper) .map(streamBuilderHelper)
.forEachOrdered(stream -> { .forEachOrdered(stream -> {
if (!Stream.containSimilarStream(stream, streamList)) { if (!Stream.containSimilarStream(stream, streamList)) {
@ -1146,7 +1235,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
* <ul> * <ul>
* <li>the {@link ItagItem}'s id of the stream as its id;</li> * <li>the {@link ItagItem}'s id of the stream as its id;</li>
* <li>{@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and * <li>{@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and
* and as the value of {@code isUrl};</li> * as the value of {@code isUrl};</li>
* <li>the media format returned by the {@link ItagItem} as its media format;</li> * <li>the media format returned by the {@link ItagItem} as its media format;</li>
* <li>its average bitrate with the value returned by {@link * <li>its average bitrate with the value returned by {@link
* ItagItem#getAverageBitrate()};</li> * ItagItem#getAverageBitrate()};</li>
@ -1199,7 +1288,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
* <ul> * <ul>
* <li>the {@link ItagItem}'s id of the stream as its id;</li> * <li>the {@link ItagItem}'s id of the stream as its id;</li>
* <li>{@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and * <li>{@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and
* and as the value of {@code isUrl};</li> * as the value of {@code isUrl};</li>
* <li>the media format returned by the {@link ItagItem} as its media format;</li> * <li>the media format returned by the {@link ItagItem} as its media format;</li>
* <li>whether it is video-only with the {@code areStreamsVideoOnly} parameter</li> * <li>whether it is video-only with the {@code areStreamsVideoOnly} parameter</li>
* <li>the {@link ItagItem};</li> * <li>the {@link ItagItem};</li>
@ -1255,7 +1344,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final JsonObject streamingData, final JsonObject streamingData,
final String streamingDataKey, final String streamingDataKey,
@Nonnull final ItagItem.ItagType itagTypeWanted, @Nonnull final ItagItem.ItagType itagTypeWanted,
@Nonnull final String contentPlaybackNonce) { @Nonnull final String contentPlaybackNonce,
@Nullable final String poToken) {
if (streamingData == null || !streamingData.has(streamingDataKey)) { if (streamingData == null || !streamingData.has(streamingDataKey)) {
return java.util.stream.Stream.empty(); return java.util.stream.Stream.empty();
} }
@ -1268,7 +1358,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final ItagItem itagItem = ItagItem.getItag(formatData.getInt("itag")); final ItagItem itagItem = ItagItem.getItag(formatData.getInt("itag"));
if (itagItem.itagType == itagTypeWanted) { if (itagItem.itagType == itagTypeWanted) {
return buildAndAddItagInfoToList(videoId, formatData, itagItem, return buildAndAddItagInfoToList(videoId, formatData, itagItem,
itagItem.itagType, contentPlaybackNonce); itagItem.itagType, contentPlaybackNonce, poToken);
} }
} catch (final ExtractionException ignored) { } catch (final ExtractionException ignored) {
// If the itag is not supported, the n parameter of HTML5 clients cannot be // If the itag is not supported, the n parameter of HTML5 clients cannot be
@ -1284,7 +1374,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull final JsonObject formatData, @Nonnull final JsonObject formatData,
@Nonnull final ItagItem itagItem, @Nonnull final ItagItem itagItem,
@Nonnull final ItagItem.ItagType itagType, @Nonnull final ItagItem.ItagType itagType,
@Nonnull final String contentPlaybackNonce) throws ExtractionException { @Nonnull final String contentPlaybackNonce,
@Nullable final String poToken) throws ExtractionException {
String streamUrl; String streamUrl;
if (formatData.has("url")) { if (formatData.has("url")) {
streamUrl = formatData.getString("url"); streamUrl = formatData.getString("url");
@ -1298,9 +1389,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + signature; streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + signature;
} }
// Add the content playback nonce to the stream URL
streamUrl += "&" + CPN + "=" + contentPlaybackNonce;
// Decode the n parameter if it is present // Decode the n parameter if it is present
// If it cannot be decoded, the stream cannot be used as streaming URLs return HTTP 403 // If it cannot be decoded, the stream cannot be used as streaming URLs return HTTP 403
// responses if it has not the right value // responses if it has not the right value
@ -1310,6 +1398,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
streamUrl = YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated( streamUrl = YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated(
videoId, streamUrl); videoId, streamUrl);
// Add the content playback nonce to the stream URL
streamUrl += "&" + CPN + "=" + contentPlaybackNonce;
// Add the poToken, if there is one
if (poToken != null) {
streamUrl += "&pot=" + poToken;
}
final JsonObject initRange = formatData.getObject("initRange"); final JsonObject initRange = formatData.getObject("initRange");
final JsonObject indexRange = formatData.getObject("indexRange"); final JsonObject indexRange = formatData.getObject("indexRange");
final String mimeType = formatData.getString("mimeType", ""); final String mimeType = formatData.getString("mimeType", "");
@ -1587,4 +1683,53 @@ public class YoutubeStreamExtractor extends StreamExtractor {
.getObject("results") .getObject("results")
.getArray("contents")); .getArray("contents"));
} }
/**
* Set the {@link PoTokenProvider} instance to be used for fetching {@code poToken}s.
*
* <p>
* This method allows setting an implementation of {@link PoTokenProvider} which will be used
* to obtain poTokens required for YouTube player requests and streaming URLs. These tokens
* are used by YouTube to verify the integrity of the user's device or browser and are required
* for playback with several clients.
* </p>
*
* <p>
* Without a {@link PoTokenProvider}, the extractor makes its best effort to fetch as many
* streams as possible, but without {@code poToken}s, some formats may be not available or
* fetching may be slower due to additional requests done to get streams.
* </p>
*
* <p>
* Note that any provider change will be only applied on the next {@link #fetchPage()} request.
* </p>
*
* @param poTokenProvider the {@link PoTokenProvider} instance to set, which can be null to
* remove a provider already passed
* @see PoTokenProvider
*/
@SuppressWarnings("unused")
public static void setPoTokenProvider(@Nullable final PoTokenProvider poTokenProvider) {
YoutubeStreamExtractor.poTokenProvider = poTokenProvider;
}
/**
* Set whether to fetch the iOS player responses.
*
* <p>
* This method allows fetching the iOS player response, which can be useful in scenarios where
* streams from the iOS player response are needed, especially HLS manifests.
* </p>
*
* <p>
* Note that at the time of writing, YouTube is rolling out a {@code poToken} requirement on
* this client, formats from HLS manifests do not seem to be affected.
* </p>
*
* @param fetchIosClient whether to fetch the iOS client
*/
@SuppressWarnings("unused")
public static void setFetchIosClient(final boolean fetchIosClient) {
YoutubeStreamExtractor.fetchIosClient = fetchIosClient;
}
} }