diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index f514b61d6..c2502139a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -48,6 +48,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfo; import org.schabi.newpipe.extractor.stream.AudioTrackType; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.ProtoBuilder; import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator; import org.schabi.newpipe.extractor.utils.Utils; @@ -174,7 +175,7 @@ public final class YoutubeParsingHelper { * Store page of the YouTube app, in the {@code What’s New} section. *

*/ - private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.28.1"; + private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.45.4"; /** * The hardcoded client version used for InnerTube requests with the TV HTML5 embed client. @@ -222,28 +223,28 @@ public final class YoutubeParsingHelper { private static final String IOS_DEVICE_MODEL = "iPhone16,2"; /** - * Spoofing an iPhone 15 Pro Max running iOS 17.5.1 with the hardcoded version of the iOS app. + * Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app. * To be used for the {@code "osVersion"} field in JSON POST requests. *

* The value of this field seems to use the following structure: * "iOS major version.minor version.patch version.build version", where * "patch version" is equal to 0 if it isn't set * The build version corresponding to the iOS version used can be found on - * - * https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15_Pro_Max + * + * https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max *

* * @see #IOS_USER_AGENT_VERSION */ - private static final String IOS_OS_VERSION = "17.5.1.21F90"; + private static final String IOS_OS_VERSION = "18.1.0.22B83"; /** - * Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app. To be + * Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app. To be * used in the user agent for requests. * * @see #IOS_OS_VERSION */ - private static final String IOS_USER_AGENT_VERSION = "17_5_1"; + private static final String IOS_USER_AGENT_VERSION = "18_1_0"; private static Random numberGenerator = new Random(); @@ -303,6 +304,23 @@ public final class YoutubeParsingHelper { return url.getHost().equalsIgnoreCase("y2u.be"); } + public static String randomVisitorData(final ContentCountry country) { + final ProtoBuilder pbE2 = new ProtoBuilder(); + pbE2.string(2, ""); + pbE2.varint(4, numberGenerator.nextInt(255) + 1); + + final ProtoBuilder pbE = new ProtoBuilder(); + pbE.string(1, country.getCountryCode()); + pbE.bytes(2, pbE2.toBytes()); + + final ProtoBuilder pb = new ProtoBuilder(); + pb.string(1, RandomStringFromAlphabetGenerator.generate( + CONTENT_PLAYBACK_NONCE_ALPHABET, 11, numberGenerator)); + pb.varint(5, System.currentTimeMillis() / 1000 - numberGenerator.nextInt(600000)); + pb.bytes(6, pbE.toBytes()); + return pb.toUrlencodedBase64(); + } + /** * Parses the duration string of the video expecting ":" or "." as separators * @@ -1166,8 +1184,13 @@ public final class YoutubeParsingHelper { @Nonnull final ContentCountry contentCountry, @Nullable final String visitorData) throws IOException, ExtractionException { + String vData = visitorData; + if (vData == null) { + vData = randomVisitorData(contentCountry); + } + // @formatter:off - final JsonBuilder builder = JsonObject.builder() + return JsonObject.builder() .object("context") .object("client") .value("hl", localization.getLocalizationCode()) @@ -1176,13 +1199,9 @@ public final class YoutubeParsingHelper { .value("clientVersion", getClientVersion()) .value("originalUrl", "https://www.youtube.com") .value("platform", "DESKTOP") - .value("utcOffsetMinutes", 0); - - if (visitorData != null) { - builder.value("visitorData", visitorData); - } - - return builder.end() + .value("utcOffsetMinutes", 0) + .value("visitorData", vData) + .end() .object("request") .array("internalExperimentFlags") .end() @@ -1256,6 +1275,7 @@ public final class YoutubeParsingHelper { .value("platform", "MOBILE") .value("osName", "iOS") .value("osVersion", IOS_OS_VERSION) + .value("visitorData", randomVisitorData(contentCountry)) .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) .value("utcOffsetMinutes", 0) @@ -1392,7 +1412,7 @@ public final class YoutubeParsingHelper { */ @Nonnull public static String getIosUserAgent(@Nullable final Localization localization) { - // Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app + // Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS " + IOS_USER_AGENT_VERSION + " like Mac OS X; " diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ProtoBuilder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ProtoBuilder.java new file mode 100644 index 000000000..f0e223a9c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ProtoBuilder.java @@ -0,0 +1,72 @@ +package org.schabi.newpipe.extractor.utils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class ProtoBuilder { + ByteArrayOutputStream byteBuffer; + + public ProtoBuilder() { + this.byteBuffer = new ByteArrayOutputStream(); + } + + public byte[] toBytes() { + return byteBuffer.toByteArray(); + } + + public String toUrlencodedBase64() { + final String b64 = Base64.getUrlEncoder().encodeToString(toBytes()); + return URLEncoder.encode(b64, StandardCharsets.UTF_8); + } + + private void writeVarint(final long val) { + try { + if (val == 0) { + byteBuffer.write(new byte[]{(byte) 0}); + } else { + long v = val; + while (v != 0) { + byte b = (byte) (v & 0x7f); + v >>= 7; + + if (v != 0) { + b |= (byte) 0x80; + } + byteBuffer.write(new byte[]{b}); + } + } + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + + private void field(final int field, final byte wire) { + final long fbits = ((long) field) << 3; + final long wbits = ((long) wire) & 0x07; + final long val = fbits | wbits; + writeVarint(val); + } + + public void varint(final int field, final long val) { + field(field, (byte) 0); + writeVarint(val); + } + + public void string(final int field, final String string) { + final byte[] strBts = string.getBytes(StandardCharsets.UTF_8); + bytes(field, strBts); + } + + public void bytes(final int field, final byte[] bytes) { + field(field, (byte) 2); + writeVarint(bytes.length); + try { + byteBuffer.write(bytes); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ProtoBuilderTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ProtoBuilderTest.java new file mode 100644 index 000000000..b39879451 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ProtoBuilderTest.java @@ -0,0 +1,18 @@ +package org.schabi.newpipe.extractor.utils; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ProtoBuilderTest { + @Test + public void testProtoBuilder() { + final ProtoBuilder pb = new ProtoBuilder(); + pb.varint(1, 128); + pb.varint(2, 1234567890); + pb.varint(3, 1234567890123456789L); + pb.string(4, "Hello"); + pb.bytes(5, new byte[]{1, 2, 3}); + assertEquals("CIABENKF2MwEGJWCpu_HnoSRESIFSGVsbG8qAwECAw%3D%3D", pb.toUrlencodedBase64()); + } +}