From 40059ed6879b8abd556ea4bb5ae4997af2e3c331 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Sun, 19 Jan 2025 19:59:10 +0100 Subject: [PATCH] fix: update iOS client, add visitor data to YouTube requests --- .../services/youtube/ProtoBuilder.java | 71 +++++++++++++++++++ .../youtube/YoutubeParsingHelper.java | 42 +++++++---- 2 files changed, 101 insertions(+), 12 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ProtoBuilder.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ProtoBuilder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ProtoBuilder.java new file mode 100644 index 000000000..01368ca78 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ProtoBuilder.java @@ -0,0 +1,71 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class ProtoBuilder { + ByteArrayOutputStream byteBuffer; + + public ProtoBuilder() { + this.byteBuffer = new ByteArrayOutputStream(); + } + + public byte[] toBytes() { + return byteBuffer.toByteArray(); + } + + public String toUrlencodedBase64() { + final String b64 = Base64.getUrlEncoder().encodeToString(toBytes()); + return URLEncoder.encode(b64, StandardCharsets.UTF_8); + } + + private void writeVarint(long val) { + try { + if (val == 0) { + byteBuffer.write(new byte[]{(byte) 0}); + } else { + while (val != 0) { + byte b = (byte) (val & 0x7f); + val >>= 7; + + if (val != 0) { + b |= (byte) 0x80; + } + byteBuffer.write(new byte[]{b}); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void field(final int field, final byte wire) { + final long fbits = ((long) field) << 3; + final long wbits = ((long) wire) & 0x07; + final long val = fbits | wbits; + writeVarint(val); + } + + public void varint(final int field, final long val) { + field(field, (byte) 0); + writeVarint(val); + } + + public void string(final int field, final String string) { + final byte[] strBts = string.getBytes(StandardCharsets.UTF_8); + bytes(field, strBts); + } + + public void bytes(final int field, final byte[] bytes) { + field(field, (byte) 2); + writeVarint(bytes.length); + try { + byteBuffer.write(bytes); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index f514b61d6..9a811737a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -174,7 +174,7 @@ public final class YoutubeParsingHelper { * Store page of the YouTube app, in the {@code What’s New} section. *

*/ - private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.28.1"; + private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.45.4"; /** * The hardcoded client version used for InnerTube requests with the TV HTML5 embed client. @@ -235,7 +235,7 @@ public final class YoutubeParsingHelper { * * @see #IOS_USER_AGENT_VERSION */ - private static final String IOS_OS_VERSION = "17.5.1.21F90"; + private static final String IOS_OS_VERSION = "18.1.0.22B83"; /** * Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app. To be @@ -243,7 +243,7 @@ public final class YoutubeParsingHelper { * * @see #IOS_OS_VERSION */ - private static final String IOS_USER_AGENT_VERSION = "17_5_1"; + private static final String IOS_USER_AGENT_VERSION = "18_1_0"; private static Random numberGenerator = new Random(); @@ -303,6 +303,23 @@ public final class YoutubeParsingHelper { return url.getHost().equalsIgnoreCase("y2u.be"); } + public static String randomVisitorData(final ContentCountry country) { + final ProtoBuilder pbE2 = new ProtoBuilder(); + pbE2.string(2, ""); + pbE2.varint(4, numberGenerator.nextInt(1, 256)); + + final ProtoBuilder pbE = new ProtoBuilder(); + pbE.string(1, country.getCountryCode()); + pbE.bytes(2, pbE2.toBytes()); + + final ProtoBuilder pb = new ProtoBuilder(); + pb.string(1, RandomStringFromAlphabetGenerator.generate( + CONTENT_PLAYBACK_NONCE_ALPHABET, 11, numberGenerator)); + pb.varint(5, System.currentTimeMillis() / 1000 - numberGenerator.nextInt(600000)); + pb.bytes(6, pbE.toBytes()); + return pb.toUrlencodedBase64(); + } + /** * Parses the duration string of the video expecting ":" or "." as separators * @@ -1164,10 +1181,14 @@ public final class YoutubeParsingHelper { public static JsonBuilder prepareDesktopJsonBuilder( @Nonnull final Localization localization, @Nonnull final ContentCountry contentCountry, - @Nullable final String visitorData) + @Nullable String visitorData) throws IOException, ExtractionException { + if (visitorData == null) { + visitorData = randomVisitorData(contentCountry); + } + // @formatter:off - final JsonBuilder builder = JsonObject.builder() + return JsonObject.builder() .object("context") .object("client") .value("hl", localization.getLocalizationCode()) @@ -1176,13 +1197,9 @@ public final class YoutubeParsingHelper { .value("clientVersion", getClientVersion()) .value("originalUrl", "https://www.youtube.com") .value("platform", "DESKTOP") - .value("utcOffsetMinutes", 0); - - if (visitorData != null) { - builder.value("visitorData", visitorData); - } - - return builder.end() + .value("utcOffsetMinutes", 0) + .value("visitorData", visitorData) + .end() .object("request") .array("internalExperimentFlags") .end() @@ -1256,6 +1273,7 @@ public final class YoutubeParsingHelper { .value("platform", "MOBILE") .value("osName", "iOS") .value("osVersion", IOS_OS_VERSION) + .value("visitorData", randomVisitorData(contentCountry)) .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) .value("utcOffsetMinutes", 0)