[YouTube] Get visitorData for player requests if not provided and do some fixes

visitorData are get using InnertubeClientRequestInfo and
YoutubeParsingHelper.prepareJsonBuilder, which is replacing the
corresponding method in YoutubeStreamHelper, removed in this commit.

Also fix some bugs like JsonBuilder usages in some places and remove
HLS manifest filtering for the iOS client, as we still use for now
the full player response, in the case having streams requiring poTokens
after some time as a last resort is useful.
This commit is contained in:
AudricV 2025-01-31 11:25:02 +01:00 committed by Stypox
parent d08331dbcf
commit 61f67854ed
No known key found for this signature in database
GPG Key ID: 4BDF1B40A49FDD23

View File

@ -17,10 +17,12 @@ import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_NAME;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.EMBED_CLIENT_SCREEN;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_NAME;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL;
@ -34,6 +36,7 @@ import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVH
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MODEL_AND_OS_NAME;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WATCH_CLIENT_SCREEN;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID;
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_NAME;
@ -54,7 +57,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getOriginReferrerHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
public final class YoutubeStreamHelper {
@ -71,20 +74,32 @@ public final class YoutubeStreamHelper {
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId) throws IOException, ExtractionException {
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(
localization,
contentCountry,
final InnertubeClientRequestInfo innertubeClientRequestInfo =
new InnertubeClientRequestInfo(
new InnertubeClientRequestInfo.ClientInfo(
WEB_CLIENT_NAME,
getClientVersion(),
WATCH_CLIENT_SCREEN,
WEB_CLIENT_ID,
null),
new InnertubeClientRequestInfo.DeviceInfo(
DESKTOP_CLIENT_PLATFORM,
YoutubeParsingHelper.randomVisitorData(contentCountry),
null,
null,
null,
null,
null,
-1);
-1));
final Map<String, List<String>> headers = getYouTubeHeaders();
// We must always pass a valid visitorData to get valid player responses, which needs to be
// got from YouTube
innertubeClientRequestInfo.clientInfo.visitorData =
YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo,
localization, contentCountry, headers, YOUTUBEI_V1_URL, null, false);
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, null);
addVideoIdCpnAndOkChecks(builder, videoId, null);
@ -96,7 +111,7 @@ public final class YoutubeStreamHelper {
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(
url, getYouTubeHeaders(), body, localization)));
url, headers, body, localization)));
}
@Nonnull
@ -105,20 +120,38 @@ public final class YoutubeStreamHelper {
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId,
@Nonnull final String cpn) throws IOException, ExtractionException {
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(
localization,
contentCountry,
final InnertubeClientRequestInfo innertubeClientRequestInfo =
new InnertubeClientRequestInfo(
new InnertubeClientRequestInfo.ClientInfo(
TVHTML5_CLIENT_NAME,
TVHTML5_CLIENT_VERSION,
WATCH_CLIENT_SCREEN,
TVHTML5_CLIENT_ID,
null),
new InnertubeClientRequestInfo.DeviceInfo(
TVHTML5_CLIENT_PLATFORM,
YoutubeParsingHelper.randomVisitorData(contentCountry),
TVHTML5_DEVICE_MAKE,
TVHTML5_DEVICE_MODEL_AND_OS_NAME,
TVHTML5_DEVICE_MODEL_AND_OS_NAME,
"",
null,
-1);
-1));
final Map<String, List<String>> headers = new HashMap<>(
getClientHeaders(TVHTML5_CLIENT_ID, TVHTML5_CLIENT_VERSION));
headers.putAll(getOriginReferrerHeaders("https://www.youtube.com"));
headers.put("User-Agent", List.of(TVHTML5_USER_AGENT));
// We must always pass a valid visitorData to get valid player responses, which needs to be
// got from YouTube
// For some reason, the TVHTML5 client doesn't support the visitor_id endpoint, use the
// guide one instead, which is quite lightweight
innertubeClientRequestInfo.clientInfo.visitorData =
YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo,
localization, contentCountry, headers, YOUTUBEI_V1_URL, null, true);
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, null);
addVideoIdCpnAndOkChecks(builder, videoId, cpn);
final byte[] body = JsonWriter.string(builder.done())
@ -126,14 +159,8 @@ public final class YoutubeStreamHelper {
final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
final Map<String, List<String>> headers = new HashMap<>(
getClientHeaders(TVHTML5_CLIENT_ID, TVHTML5_CLIENT_VERSION));
headers.putAll(getOriginReferrerHeaders("https://www.youtube.com"));
headers.put("User-Agent", List.of(TVHTML5_USER_AGENT));
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(
url, headers, body, localization)));
getDownloader().postWithContentTypeJson(url, headers, body, localization)));
}
@Nonnull
@ -144,32 +171,34 @@ public final class YoutubeStreamHelper {
@Nonnull final String cpn,
@Nonnull final PoTokenResult webPoTokenResult,
final int signatureTimestamp) throws IOException, ExtractionException {
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(
localization,
contentCountry,
final InnertubeClientRequestInfo innertubeClientRequestInfo =
new InnertubeClientRequestInfo(
new InnertubeClientRequestInfo.ClientInfo(
WEB_CLIENT_NAME,
getClientVersion(),
WATCH_CLIENT_SCREEN,
WEB_CLIENT_ID,
webPoTokenResult.visitorData),
new InnertubeClientRequestInfo.DeviceInfo(
DESKTOP_CLIENT_PLATFORM,
webPoTokenResult.visitorData,
null,
null,
null,
null,
null,
-1);
-1));
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, null);
addVideoIdCpnAndOkChecks(builder, videoId, cpn);
addPlaybackContext(
builder,
BASE_YT_DESKTOP_WATCH_URL + videoId,
signatureTimestamp);
addPlaybackContext(builder, BASE_YT_DESKTOP_WATCH_URL + videoId, signatureTimestamp);
addPoToken(builder, webPoTokenResult.playerRequestPoToken);
final byte[] body = JsonWriter.string(builder.end().done())
.getBytes(StandardCharsets.UTF_8);
final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
return JsonUtils.toJsonObject(getValidJsonResponseBody(
@ -185,29 +214,41 @@ public final class YoutubeStreamHelper {
@Nonnull final String cpn,
@Nullable final PoTokenResult webEmbeddedPoTokenResult,
final int signatureTimestamp) throws IOException, ExtractionException {
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(
localization,
contentCountry,
final InnertubeClientRequestInfo innertubeClientRequestInfo =
new InnertubeClientRequestInfo(
new InnertubeClientRequestInfo.ClientInfo(
WEB_EMBEDDED_CLIENT_NAME,
WEB_REMIX_HARDCODED_CLIENT_VERSION,
EMBED_CLIENT_SCREEN,
WEB_EMBEDDED_CLIENT_ID,
null),
new InnertubeClientRequestInfo.DeviceInfo(
DESKTOP_CLIENT_PLATFORM,
webEmbeddedPoTokenResult == null
? YoutubeParsingHelper.randomVisitorData(contentCountry)
: webEmbeddedPoTokenResult.visitorData,
null,
null,
null,
null,
BASE_YT_DESKTOP_WATCH_URL + videoId,
-1);
-1));
final Map<String, List<String>> headers = new HashMap<>(
getClientHeaders(WEB_EMBEDDED_CLIENT_ID, WEB_EMBEDDED_CLIENT_VERSION));
headers.putAll(getOriginReferrerHeaders("https://www.youtube.com"));
final String embedUrl = BASE_YT_DESKTOP_WATCH_URL + videoId;
// We must always pass a valid visitorData to get valid player responses, which needs to be
// got from YouTube
innertubeClientRequestInfo.clientInfo.visitorData = webEmbeddedPoTokenResult == null
? YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo,
localization, contentCountry, headers, YOUTUBEI_V1_URL, embedUrl, false)
: webEmbeddedPoTokenResult.visitorData;
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, embedUrl);
addVideoIdCpnAndOkChecks(builder, videoId, cpn);
addPlaybackContext(
builder,
BASE_YT_DESKTOP_WATCH_URL + videoId,
signatureTimestamp);
addPlaybackContext(builder, embedUrl, signatureTimestamp);
if (webEmbeddedPoTokenResult != null) {
addPoToken(builder, webEmbeddedPoTokenResult.playerRequestPoToken);
@ -217,13 +258,8 @@ public final class YoutubeStreamHelper {
.getBytes(StandardCharsets.UTF_8);
final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
final Map<String, List<String>> headers = new HashMap<>(
getClientHeaders(WEB_EMBEDDED_CLIENT_ID, WEB_EMBEDDED_CLIENT_VERSION));
headers.putAll(getOriginReferrerHeaders("https://www.youtube.com"));
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(
url, headers, body, localization)));
getDownloader().postWithContentTypeJson(url, headers, body, localization)));
}
public static JsonObject getAndroidPlayerResponse(
@ -233,34 +269,40 @@ public final class YoutubeStreamHelper {
@Nonnull final String cpn,
@Nonnull final PoTokenResult androidPoTokenResult)
throws IOException, ExtractionException {
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(
localization,
contentCountry,
final InnertubeClientRequestInfo innertubeClientRequestInfo =
new InnertubeClientRequestInfo(
new InnertubeClientRequestInfo.ClientInfo(
ANDROID_CLIENT_NAME,
ANDROID_CLIENT_VERSION,
WATCH_CLIENT_SCREEN,
ANDROID_CLIENT_ID,
androidPoTokenResult.visitorData),
new InnertubeClientRequestInfo.DeviceInfo(
MOBILE_CLIENT_PLATFORM,
androidPoTokenResult.visitorData,
null,
null,
"Android",
"15",
null,
35);
35));
final Map<String, List<String>> headers =
getMobileClientHeaders(getAndroidUserAgent(localization));
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, null);
addVideoIdCpnAndOkChecks(builder, videoId, cpn);
addPoToken(builder, androidPoTokenResult.playerRequestPoToken);
final byte[] body = JsonWriter.string(builder.end().done())
final byte[] body = JsonWriter.string(builder.done())
.getBytes(StandardCharsets.UTF_8);
return getJsonAndroidPostResponse(
PLAYER,
body,
localization,
"&t=" + generateTParameter() + "&id=" + videoId);
final String url = YOUTUBEI_V1_GAPIS_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER
+ "&t=" + generateTParameter() + "&id=" + videoId;
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(url, headers, body, localization)));
}
public static JsonObject getAndroidReelPlayerResponse(
@ -268,20 +310,33 @@ public final class YoutubeStreamHelper {
@Nonnull final Localization localization,
@Nonnull final String videoId,
@Nonnull final String cpn) throws IOException, ExtractionException {
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(
localization,
contentCountry,
final InnertubeClientRequestInfo innertubeClientRequestInfo =
new InnertubeClientRequestInfo(
new InnertubeClientRequestInfo.ClientInfo(
ANDROID_CLIENT_NAME,
ANDROID_CLIENT_VERSION,
WATCH_CLIENT_SCREEN,
ANDROID_CLIENT_ID,
null),
new InnertubeClientRequestInfo.DeviceInfo(
MOBILE_CLIENT_PLATFORM,
YoutubeParsingHelper.randomVisitorData(contentCountry),
null,
null,
"Android",
"15",
null,
35);
35));
final Map<String, List<String>> headers =
getMobileClientHeaders(getAndroidUserAgent(localization));
// We must always pass a valid visitorData to get valid player responses, which needs to be
// got from YouTube
innertubeClientRequestInfo.clientInfo.visitorData =
YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo,
localization, contentCountry, headers, YOUTUBEI_V1_GAPIS_URL, null, false);
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, null);
addVideoIdCpnAndOkChecks(builder, videoId, cpn);
@ -290,16 +345,16 @@ public final class YoutubeStreamHelper {
.end()
.value("disablePlayerResponse", false);
final byte[] mobileBody = JsonWriter.string(builder.done())
final byte[] body = JsonWriter.string(builder.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(
"reel/reel_item_watch",
mobileBody,
localization,
"&t=" + generateTParameter() + "&id=" + videoId + "&$fields=playerResponse");
final String url = YOUTUBEI_V1_GAPIS_URL + "reel/reel_item_watch" + "?"
+ DISABLE_PRETTY_PRINT_PARAMETER + "&t=" + generateTParameter() + "&id=" + videoId
+ "&$fields=playerResponse";
return androidPlayerResponse.getObject("playerResponse");
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(url, headers, body, localization)))
.getObject("playerResponse");
}
public static JsonObject getIosPlayerResponse(@Nonnull final ContentCountry contentCountry,
@ -308,139 +363,49 @@ public final class YoutubeStreamHelper {
@Nonnull final String cpn,
@Nullable final PoTokenResult iosPoTokenResult)
throws IOException, ExtractionException {
final boolean noPoTokenResult = iosPoTokenResult == null;
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(
localization,
contentCountry,
final InnertubeClientRequestInfo innertubeClientRequestInfo =
new InnertubeClientRequestInfo(
new InnertubeClientRequestInfo.ClientInfo(
IOS_CLIENT_NAME,
IOS_CLIENT_VERSION,
WATCH_CLIENT_SCREEN,
IOS_CLIENT_ID,
null),
new InnertubeClientRequestInfo.DeviceInfo(
MOBILE_CLIENT_PLATFORM,
noPoTokenResult
? YoutubeParsingHelper.randomVisitorData(contentCountry)
: iosPoTokenResult.visitorData,
"Apple",
IOS_DEVICE_MODEL,
"iOS",
IOS_OS_VERSION,
null,
-1);
-1));
final Map<String, List<String>> headers =
getMobileClientHeaders(getIosUserAgent(localization));
// We must always pass a valid visitorData to get valid player responses, which needs to be
// got from YouTube
innertubeClientRequestInfo.clientInfo.visitorData = iosPoTokenResult == null
? YoutubeParsingHelper.getVisitorDataFromInnertube(innertubeClientRequestInfo,
localization, contentCountry, headers, YOUTUBEI_V1_URL, null, false)
: iosPoTokenResult.visitorData;
final JsonBuilder<JsonObject> builder = prepareJsonBuilder(localization, contentCountry,
innertubeClientRequestInfo, null);
addVideoIdCpnAndOkChecks(builder, videoId, cpn);
if (!noPoTokenResult) {
if (iosPoTokenResult != null) {
addPoToken(builder, iosPoTokenResult.playerRequestPoToken);
}
final byte[] mobileBody = JsonWriter.string(builder.done())
final byte[] body = JsonWriter.string(builder.done())
.getBytes(StandardCharsets.UTF_8);
return getJsonIosPostResponse(
mobileBody, localization, "&t=" + generateTParameter()
+ "&id=" + videoId + "&fields=streamingData.hlsManifestUrl");
}
public static JsonObject getJsonAndroidPostResponse(final String endpoint,
final byte[] body,
@Nonnull final Localization localization,
@Nullable final String endPartOfUrlRequest)
throws IOException, ExtractionException {
return getMobilePostResponse(endpoint, body, localization,
getAndroidUserAgent(localization), endPartOfUrlRequest);
}
private static JsonObject getJsonIosPostResponse(final byte[] body,
@Nonnull final Localization localization,
@Nullable final String endPartOfUrlRequest)
throws IOException, ExtractionException {
return getMobilePostResponse(YoutubeStreamHelper.PLAYER, body, localization,
getIosUserAgent(localization),
endPartOfUrlRequest);
}
@SuppressWarnings("checkstyle:ParameterNumber")
@Nonnull
private static JsonBuilder<JsonObject> prepareJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String clientName,
@Nonnull final String clientVersion,
@Nonnull final String clientScreen,
@Nonnull final String platform,
@Nonnull final String visitorData,
@Nullable final String deviceMake,
@Nullable final String deviceModel,
@Nullable final String osName,
@Nullable final String osVersion,
@Nullable final String embedUrl,
final int androidSdkVersion) {
final JsonBuilder<JsonObject> builder = JsonObject.builder()
.object("context")
.object("client")
.value("clientName", clientName)
.value("clientVersion", clientVersion)
.value("clientScreen", clientScreen)
.value("platform", platform)
.value("visitorData", visitorData);
if (deviceMake != null) {
builder.value("deviceMake", deviceMake);
}
if (deviceModel != null) {
builder.value("deviceModel", deviceModel);
}
if (osName != null) {
builder.value("osName", osName);
}
if (osVersion != null) {
builder.value("osVersion", osVersion);
}
if (androidSdkVersion > 0) {
builder.value("androidSdkVersion", androidSdkVersion);
}
builder.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("utcOffsetMinutes", 0)
.end();
if (embedUrl != null) {
builder.object("thirdParty")
.value("embedUrl", embedUrl)
.end();
}
builder.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user")
// TODO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
return builder;
}
private static JsonObject getMobilePostResponse(@Nonnull final String endpoint,
final byte[] body,
@Nonnull final Localization localization,
@Nonnull final String userAgent,
@Nullable final String endPartOfUrlRequest)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = Map.of("User-Agent", List.of(userAgent),
"X-Goog-Api-Format-Version", List.of("2"));
final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?"
+ DISABLE_PRETTY_PRINT_PARAMETER;
final String url = YOUTUBEI_V1_GAPIS_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER
+ "&t=" + generateTParameter() + "&id=" + videoId;
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl
: baseEndpointUrl + endPartOfUrlRequest,
headers, body, localization)));
getDownloader().postWithContentTypeJson(url, headers, body, localization)));
}
private static void addVideoIdCpnAndOkChecks(@Nonnull final JsonBuilder<JsonObject> builder,
@ -473,4 +438,11 @@ public final class YoutubeStreamHelper {
.value(PO_TOKEN, poToken)
.end();
}
@Nonnull
private static Map<String, List<String>> getMobileClientHeaders(
@Nonnull final String userAgent) {
return Map.of("User-Agent", List.of(userAgent),
"X-Goog-Api-Format-Version", List.of("2"));
}
}