mirror of
https://github.com/TeamNewPipe/NewPipeExtractor.git
synced 2024-12-13 13:50:33 +05:30
Fixed all YTMixPlaylists
Added option to choose if you want to consent or not - currently this is done by a static variable in ``YoutubeParsingHelper`` - may not be the best long-term solution but for now the tests work again (in EU countries) 🥳
This commit is contained in:
litetex
parent
2b6fe294b2
commit
ecfc370685
@ -0,0 +1,12 @@
|
||||
package org.schabi.newpipe.extractor.exceptions;
|
||||
|
||||
public class ConsentRequiredException extends ParsingException {
|
||||
|
||||
public ConsentRequiredException(final String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public ConsentRequiredException(final String message, final Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@ -27,7 +27,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
@ -245,16 +244,19 @@ public final class YoutubeParsingHelper {
|
||||
* The three digits at the end can be random, but are required.
|
||||
* </p>
|
||||
*/
|
||||
private static final String CONSENT_COOKIE_VALUE = "PENDING+";
|
||||
|
||||
private static final String CONSENT_COOKIE_PENDING_VALUE = "PENDING+";
|
||||
/**
|
||||
* YouTube {@code CONSENT} cookie.
|
||||
* {@code YES+} means that the user did submit their choices and accepted all cookies.
|
||||
*
|
||||
* <p>
|
||||
* Should prevent redirect to {@code consent.youtube.com}.
|
||||
* Therefore, YouTube & Google can track the user, because they did give consent.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The three digits at the end can be random, but are required.
|
||||
* </p>
|
||||
*/
|
||||
private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
|
||||
private static final String CONSENT_COOKIE_YES_VALUE = "YES+";
|
||||
|
||||
private static final String FEED_BASE_CHANNEL_ID =
|
||||
"https://www.youtube.com/feeds/videos.xml?channel_id=";
|
||||
@ -265,6 +267,13 @@ public final class YoutubeParsingHelper {
|
||||
private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
|
||||
private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
|
||||
|
||||
/**
|
||||
* {@code false} (default) will use {@link #CONSENT_COOKIE_PENDING_VALUE}.
|
||||
* <br/>
|
||||
* {@code true} will use {@link #CONSENT_COOKIE_YES_VALUE}.
|
||||
*/
|
||||
private static boolean consentAccepted = false;
|
||||
|
||||
private static boolean isGoogleURL(final String url) {
|
||||
final String cachedUrl = extractCachedUrlIfNeeded(url);
|
||||
try {
|
||||
@ -1378,7 +1387,6 @@ public final class YoutubeParsingHelper {
|
||||
|
||||
/**
|
||||
* Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
|
||||
* @see #CONSENT_COOKIE
|
||||
* @param headers the headers which should be completed
|
||||
*/
|
||||
public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
|
||||
@ -1391,8 +1399,9 @@ public final class YoutubeParsingHelper {
|
||||
|
||||
@Nonnull
|
||||
public static String generateConsentCookie() {
|
||||
final int statusCode = 100 + numberGenerator.nextInt(900);
|
||||
return CONSENT_COOKIE + statusCode;
|
||||
return "CONSENT="
|
||||
+ (isConsentAccepted() ? CONSENT_COOKIE_YES_VALUE : CONSENT_COOKIE_PENDING_VALUE)
|
||||
+ (100 + numberGenerator.nextInt(900));
|
||||
}
|
||||
|
||||
public static String extractCookieValue(final String cookieName,
|
||||
@ -1612,16 +1621,6 @@ public final class YoutubeParsingHelper {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
public static String unescapeDocument(@Nonnull final String doc) {
|
||||
return doc
|
||||
.replaceAll("\\\\x22", "\"")
|
||||
.replaceAll("\\\\x7b", "{")
|
||||
.replaceAll("\\\\x7d", "}")
|
||||
.replaceAll("\\\\x5b", "[")
|
||||
.replaceAll("\\\\x5d", "]");
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
|
||||
* playback requests (and also for some clients, in the player request body).
|
||||
@ -1692,4 +1691,12 @@ public final class YoutubeParsingHelper {
|
||||
public static boolean isIosStreamingUrl(@Nonnull final String url) {
|
||||
return Parser.isMatch(C_IOS_PATTERN, url);
|
||||
}
|
||||
|
||||
public static void setConsentAccepted(final boolean accepted) {
|
||||
consentAccepted = accepted;
|
||||
}
|
||||
|
||||
public static boolean isConsentAccepted() {
|
||||
return consentAccepted;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,28 +1,15 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonBuilder;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.exceptions.ConsentRequiredException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
@ -35,6 +22,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@ -43,8 +32,18 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addYouTubeHeaders;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
|
||||
|
||||
/**
|
||||
* A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
|
||||
@ -89,16 +88,26 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
||||
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
addClientInfoHeaders(headers);
|
||||
// Cookie is required due to consent
|
||||
addYouTubeHeaders(headers);
|
||||
|
||||
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
|
||||
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
|
||||
|
||||
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
|
||||
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
|
||||
.getObject("playlist").getObject("playlist");
|
||||
playlistData = initialData
|
||||
.getObject("contents")
|
||||
.getObject("twoColumnWatchNextResults")
|
||||
.getObject("playlist")
|
||||
.getObject("playlist");
|
||||
if (isNullOrEmpty(playlistData)) {
|
||||
throw new ExtractionException("Could not get playlistData");
|
||||
final ExtractionException ex = new ExtractionException("Could not get playlistData");
|
||||
if (!YoutubeParsingHelper.isConsentAccepted()) {
|
||||
throw new ConsentRequiredException(
|
||||
"Consent is required in some countries to view Mix playlists",
|
||||
ex);
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
cookieValue = extractCookieValue(COOKIE_NAME, response);
|
||||
}
|
||||
@ -212,7 +221,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
||||
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
addClientInfoHeaders(headers);
|
||||
// Cookie is required due to consent
|
||||
addYouTubeHeaders(headers);
|
||||
|
||||
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
|
||||
getExtractorLocalization());
|
||||
|
||||
@ -1,15 +1,6 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
|
||||
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.schabi.newpipe.downloader.DownloaderFactory;
|
||||
@ -31,6 +22,17 @@ import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
|
||||
|
||||
public class YoutubeMixPlaylistExtractorTest {
|
||||
|
||||
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
|
||||
@ -45,6 +47,7 @@ public class YoutubeMixPlaylistExtractorTest {
|
||||
@BeforeAll
|
||||
public static void setUp() throws Exception {
|
||||
YoutubeTestsUtils.ensureStateless();
|
||||
YoutubeParsingHelper.setConsentAccepted(true);
|
||||
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mix"));
|
||||
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
|
||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||
@ -140,6 +143,7 @@ public class YoutubeMixPlaylistExtractorTest {
|
||||
@BeforeAll
|
||||
public static void setUp() throws Exception {
|
||||
YoutubeTestsUtils.ensureStateless();
|
||||
YoutubeParsingHelper.setConsentAccepted(true);
|
||||
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mixWithIndex"));
|
||||
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
|
||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||
@ -221,11 +225,12 @@ public class YoutubeMixPlaylistExtractorTest {
|
||||
}
|
||||
|
||||
public static class MyMix {
|
||||
private static final String VIDEO_ID = "_AzeUSL9lZc";
|
||||
private static final String VIDEO_ID = "YVkUvmDQ3HY";
|
||||
|
||||
@BeforeAll
|
||||
public static void setUp() throws Exception {
|
||||
YoutubeTestsUtils.ensureStateless();
|
||||
YoutubeParsingHelper.setConsentAccepted(true);
|
||||
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "myMix"));
|
||||
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
|
||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||
@ -249,7 +254,7 @@ public class YoutubeMixPlaylistExtractorTest {
|
||||
void getThumbnailUrl() throws Exception {
|
||||
final String thumbnailUrl = extractor.getThumbnailUrl();
|
||||
assertIsSecureUrl(thumbnailUrl);
|
||||
assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc"));
|
||||
assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/" + VIDEO_ID));
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -316,6 +321,7 @@ public class YoutubeMixPlaylistExtractorTest {
|
||||
@BeforeAll
|
||||
public static void setUp() throws IOException {
|
||||
YoutubeTestsUtils.ensureStateless();
|
||||
YoutubeParsingHelper.setConsentAccepted(true);
|
||||
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "invalid"));
|
||||
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
|
||||
}
|
||||
@ -350,6 +356,7 @@ public class YoutubeMixPlaylistExtractorTest {
|
||||
@BeforeAll
|
||||
public static void setUp() throws Exception {
|
||||
YoutubeTestsUtils.ensureStateless();
|
||||
YoutubeParsingHelper.setConsentAccepted(true);
|
||||
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channelMix"));
|
||||
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
|
||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||
@ -414,6 +421,7 @@ public class YoutubeMixPlaylistExtractorTest {
|
||||
@BeforeAll
|
||||
public static void setUp() throws Exception {
|
||||
YoutubeTestsUtils.ensureStateless();
|
||||
YoutubeParsingHelper.setConsentAccepted(true);
|
||||
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "genreMix"));
|
||||
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
|
||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||
|
||||
@ -21,6 +21,7 @@ public final class YoutubeTestsUtils {
|
||||
* </p>
|
||||
*/
|
||||
public static void ensureStateless() {
|
||||
YoutubeParsingHelper.setConsentAccepted(false);
|
||||
YoutubeParsingHelper.resetClientVersionAndKey();
|
||||
YoutubeParsingHelper.setNumberGenerator(new Random(1));
|
||||
YoutubeStreamExtractor.resetDeobfuscationCode();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user