From 40059ed6879b8abd556ea4bb5ae4997af2e3c331 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Sun, 19 Jan 2025 19:59:10 +0100 Subject: [PATCH 1/5] fix: update iOS client, add visitor data to YouTube requests --- .../services/youtube/ProtoBuilder.java | 71 +++++++++++++++++++ .../youtube/YoutubeParsingHelper.java | 42 +++++++---- 2 files changed, 101 insertions(+), 12 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ProtoBuilder.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ProtoBuilder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ProtoBuilder.java new file mode 100644 index 000000000..01368ca78 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ProtoBuilder.java @@ -0,0 +1,71 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class ProtoBuilder { + ByteArrayOutputStream byteBuffer; + + public ProtoBuilder() { + this.byteBuffer = new ByteArrayOutputStream(); + } + + public byte[] toBytes() { + return byteBuffer.toByteArray(); + } + + public String toUrlencodedBase64() { + final String b64 = Base64.getUrlEncoder().encodeToString(toBytes()); + return URLEncoder.encode(b64, StandardCharsets.UTF_8); + } + + private void writeVarint(long val) { + try { + if (val == 0) { + byteBuffer.write(new byte[]{(byte) 0}); + } else { + while (val != 0) { + byte b = (byte) (val & 0x7f); + val >>= 7; + + if (val != 0) { + b |= (byte) 0x80; + } + byteBuffer.write(new byte[]{b}); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void field(final int field, final byte wire) { + final long fbits = ((long) field) << 3; + final long wbits = ((long) wire) & 0x07; + final long val = fbits | wbits; + writeVarint(val); + } + + public void varint(final int field, final long val) { + field(field, (byte) 0); + writeVarint(val); + } + + public void string(final int field, final String string) { + final byte[] strBts = string.getBytes(StandardCharsets.UTF_8); + bytes(field, strBts); + } + + public void bytes(final int field, final byte[] bytes) { + field(field, (byte) 2); + writeVarint(bytes.length); + try { + byteBuffer.write(bytes); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index f514b61d6..9a811737a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -174,7 +174,7 @@ public final class YoutubeParsingHelper { * Store page of the YouTube app, in the {@code What’s New} section. *

*/ - private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.28.1"; + private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.45.4"; /** * The hardcoded client version used for InnerTube requests with the TV HTML5 embed client. @@ -235,7 +235,7 @@ public final class YoutubeParsingHelper { * * @see #IOS_USER_AGENT_VERSION */ - private static final String IOS_OS_VERSION = "17.5.1.21F90"; + private static final String IOS_OS_VERSION = "18.1.0.22B83"; /** * Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app. To be @@ -243,7 +243,7 @@ public final class YoutubeParsingHelper { * * @see #IOS_OS_VERSION */ - private static final String IOS_USER_AGENT_VERSION = "17_5_1"; + private static final String IOS_USER_AGENT_VERSION = "18_1_0"; private static Random numberGenerator = new Random(); @@ -303,6 +303,23 @@ public final class YoutubeParsingHelper { return url.getHost().equalsIgnoreCase("y2u.be"); } + public static String randomVisitorData(final ContentCountry country) { + final ProtoBuilder pbE2 = new ProtoBuilder(); + pbE2.string(2, ""); + pbE2.varint(4, numberGenerator.nextInt(1, 256)); + + final ProtoBuilder pbE = new ProtoBuilder(); + pbE.string(1, country.getCountryCode()); + pbE.bytes(2, pbE2.toBytes()); + + final ProtoBuilder pb = new ProtoBuilder(); + pb.string(1, RandomStringFromAlphabetGenerator.generate( + CONTENT_PLAYBACK_NONCE_ALPHABET, 11, numberGenerator)); + pb.varint(5, System.currentTimeMillis() / 1000 - numberGenerator.nextInt(600000)); + pb.bytes(6, pbE.toBytes()); + return pb.toUrlencodedBase64(); + } + /** * Parses the duration string of the video expecting ":" or "." as separators * @@ -1164,10 +1181,14 @@ public final class YoutubeParsingHelper { public static JsonBuilder prepareDesktopJsonBuilder( @Nonnull final Localization localization, @Nonnull final ContentCountry contentCountry, - @Nullable final String visitorData) + @Nullable String visitorData) throws IOException, ExtractionException { + if (visitorData == null) { + visitorData = randomVisitorData(contentCountry); + } + // @formatter:off - final JsonBuilder builder = JsonObject.builder() + return JsonObject.builder() .object("context") .object("client") .value("hl", localization.getLocalizationCode()) @@ -1176,13 +1197,9 @@ public final class YoutubeParsingHelper { .value("clientVersion", getClientVersion()) .value("originalUrl", "https://www.youtube.com") .value("platform", "DESKTOP") - .value("utcOffsetMinutes", 0); - - if (visitorData != null) { - builder.value("visitorData", visitorData); - } - - return builder.end() + .value("utcOffsetMinutes", 0) + .value("visitorData", visitorData) + .end() .object("request") .array("internalExperimentFlags") .end() @@ -1256,6 +1273,7 @@ public final class YoutubeParsingHelper { .value("platform", "MOBILE") .value("osName", "iOS") .value("osVersion", IOS_OS_VERSION) + .value("visitorData", randomVisitorData(contentCountry)) .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) .value("utcOffsetMinutes", 0) From 65d888f4cff7b1070851bed172fbc142a587c204 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Tue, 21 Jan 2025 03:05:54 +0100 Subject: [PATCH 2/5] move ProtoBuilder, add tests, fix incompatible rand call --- .../youtube/YoutubeParsingHelper.java | 24 ++++++++++--------- .../youtube => utils}/ProtoBuilder.java | 17 ++++++------- .../extractor/utils/ProtoBuilderTest.java | 18 ++++++++++++++ 3 files changed, 40 insertions(+), 19 deletions(-) rename extractor/src/main/java/org/schabi/newpipe/extractor/{services/youtube => utils}/ProtoBuilder.java (82%) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/utils/ProtoBuilderTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 9a811737a..0f97bd12a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -48,6 +48,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfo; import org.schabi.newpipe.extractor.stream.AudioTrackType; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.ProtoBuilder; import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator; import org.schabi.newpipe.extractor.utils.Utils; @@ -212,7 +213,7 @@ public final class YoutubeParsingHelper { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; /** - * The device machine id for the iPhone 15, used to get 60fps with the {@code iOS} client. + * The device machine id for the iPhone 16, used to get 60fps with the {@code iOS} client. * *

* See this GitHub Gist for more @@ -222,15 +223,15 @@ public final class YoutubeParsingHelper { private static final String IOS_DEVICE_MODEL = "iPhone16,2"; /** - * Spoofing an iPhone 15 Pro Max running iOS 17.5.1 with the hardcoded version of the iOS app. + * Spoofing an iPhone 16 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app. * To be used for the {@code "osVersion"} field in JSON POST requests. *

* The value of this field seems to use the following structure: * "iOS major version.minor version.patch version.build version", where * "patch version" is equal to 0 if it isn't set * The build version corresponding to the iOS version used can be found on - * - * https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15_Pro_Max + * + * https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_16_Pro_Max *

* * @see #IOS_USER_AGENT_VERSION @@ -238,7 +239,7 @@ public final class YoutubeParsingHelper { private static final String IOS_OS_VERSION = "18.1.0.22B83"; /** - * Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app. To be + * Spoofing an iPhone 16 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app. To be * used in the user agent for requests. * * @see #IOS_OS_VERSION @@ -306,7 +307,7 @@ public final class YoutubeParsingHelper { public static String randomVisitorData(final ContentCountry country) { final ProtoBuilder pbE2 = new ProtoBuilder(); pbE2.string(2, ""); - pbE2.varint(4, numberGenerator.nextInt(1, 256)); + pbE2.varint(4, numberGenerator.nextInt(255) + 1); final ProtoBuilder pbE = new ProtoBuilder(); pbE.string(1, country.getCountryCode()); @@ -1181,10 +1182,11 @@ public final class YoutubeParsingHelper { public static JsonBuilder prepareDesktopJsonBuilder( @Nonnull final Localization localization, @Nonnull final ContentCountry contentCountry, - @Nullable String visitorData) + @Nullable final String visitorData) throws IOException, ExtractionException { - if (visitorData == null) { - visitorData = randomVisitorData(contentCountry); + String vData = visitorData; + if (vData == null) { + vData = randomVisitorData(contentCountry); } // @formatter:off @@ -1198,7 +1200,7 @@ public final class YoutubeParsingHelper { .value("originalUrl", "https://www.youtube.com") .value("platform", "DESKTOP") .value("utcOffsetMinutes", 0) - .value("visitorData", visitorData) + .value("visitorData", vData) .end() .object("request") .array("internalExperimentFlags") @@ -1410,7 +1412,7 @@ public final class YoutubeParsingHelper { */ @Nonnull public static String getIosUserAgent(@Nullable final Localization localization) { - // Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app + // Spoofing an iPhone 16 Pro Max running iOS 17.5.1 with the hardcoded version of the iOS app return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS " + IOS_USER_AGENT_VERSION + " like Mac OS X; " diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ProtoBuilder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ProtoBuilder.java similarity index 82% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ProtoBuilder.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/utils/ProtoBuilder.java index 01368ca78..f0e223a9c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ProtoBuilder.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ProtoBuilder.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.utils; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -22,22 +22,23 @@ public class ProtoBuilder { return URLEncoder.encode(b64, StandardCharsets.UTF_8); } - private void writeVarint(long val) { + private void writeVarint(final long val) { try { if (val == 0) { byteBuffer.write(new byte[]{(byte) 0}); } else { - while (val != 0) { - byte b = (byte) (val & 0x7f); - val >>= 7; + long v = val; + while (v != 0) { + byte b = (byte) (v & 0x7f); + v >>= 7; - if (val != 0) { + if (v != 0) { b |= (byte) 0x80; } byteBuffer.write(new byte[]{b}); } } - } catch (IOException e) { + } catch (final IOException e) { throw new RuntimeException(e); } } @@ -64,7 +65,7 @@ public class ProtoBuilder { writeVarint(bytes.length); try { byteBuffer.write(bytes); - } catch (IOException e) { + } catch (final IOException e) { throw new RuntimeException(e); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ProtoBuilderTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ProtoBuilderTest.java new file mode 100644 index 000000000..b39879451 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ProtoBuilderTest.java @@ -0,0 +1,18 @@ +package org.schabi.newpipe.extractor.utils; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ProtoBuilderTest { + @Test + public void testProtoBuilder() { + final ProtoBuilder pb = new ProtoBuilder(); + pb.varint(1, 128); + pb.varint(2, 1234567890); + pb.varint(3, 1234567890123456789L); + pb.string(4, "Hello"); + pb.bytes(5, new byte[]{1, 2, 3}); + assertEquals("CIABENKF2MwEGJWCpu_HnoSRESIFSGVsbG8qAwECAw%3D%3D", pb.toUrlencodedBase64()); + } +} From 45645b043ba071a3de8da03d1f039b35a2bd79f7 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Tue, 21 Jan 2025 13:12:51 +0100 Subject: [PATCH 3/5] update comment --- .../extractor/services/youtube/YoutubeParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 0f97bd12a..14118037e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -1412,7 +1412,7 @@ public final class YoutubeParsingHelper { */ @Nonnull public static String getIosUserAgent(@Nullable final Localization localization) { - // Spoofing an iPhone 16 Pro Max running iOS 17.5.1 with the hardcoded version of the iOS app + // Spoofing an iPhone 16 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS " + IOS_USER_AGENT_VERSION + " like Mac OS X; " From 5b31ff20e524ae8f1fced13b6a1bb361f99adc02 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 21 Jan 2025 23:09:00 +0100 Subject: [PATCH 4/5] iPhone 15 Pro Max instead of 16 in comments --- .../services/youtube/YoutubeParsingHelper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 14118037e..198fb1097 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -213,7 +213,7 @@ public final class YoutubeParsingHelper { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; /** - * The device machine id for the iPhone 16, used to get 60fps with the {@code iOS} client. + * The device machine id for the iPhone 15, used to get 60fps with the {@code iOS} client. * *

* See this GitHub Gist for more @@ -223,15 +223,15 @@ public final class YoutubeParsingHelper { private static final String IOS_DEVICE_MODEL = "iPhone16,2"; /** - * Spoofing an iPhone 16 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app. + * Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app. * To be used for the {@code "osVersion"} field in JSON POST requests. *

* The value of this field seems to use the following structure: * "iOS major version.minor version.patch version.build version", where * "patch version" is equal to 0 if it isn't set * The build version corresponding to the iOS version used can be found on - * - * https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_16_Pro_Max + * + * https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max *

* * @see #IOS_USER_AGENT_VERSION @@ -1412,7 +1412,7 @@ public final class YoutubeParsingHelper { */ @Nonnull public static String getIosUserAgent(@Nullable final Localization localization) { - // Spoofing an iPhone 16 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app + // Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS " + IOS_USER_AGENT_VERSION + " like Mac OS X; " From 936bf2d71beaeee30afcab1210bc27c475328328 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Tue, 21 Jan 2025 23:09:40 +0100 Subject: [PATCH 5/5] Update extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java Co-authored-by: Stypox --- .../extractor/services/youtube/YoutubeParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 198fb1097..c2502139a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -239,7 +239,7 @@ public final class YoutubeParsingHelper { private static final String IOS_OS_VERSION = "18.1.0.22B83"; /** - * Spoofing an iPhone 16 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app. To be + * Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app. To be * used in the user agent for requests. * * @see #IOS_OS_VERSION