diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java
new file mode 100644
index 000000000..9adee6191
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java
@@ -0,0 +1,271 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonWriter;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.localization.ContentCountry;
+import org.schabi.newpipe.extractor.localization.Localization;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+/**
+ * Shared functions for extracting YouTube channel pages and tabs.
+ */
+public final class YoutubeChannelHelper {
+ private YoutubeChannelHelper() {
+ }
+
+ /**
+ * Take a YouTube channel ID or URL path, resolve it if necessary and return a channel ID.
+ *
+ * @param idOrPath a YouTube channel ID or URL path
+ * @return a YouTube channel ID
+ * @throws IOException if a channel resolve request failed
+ * @throws ExtractionException if a channel resolve request response could not be parsed or is
+ * invalid
+ */
+ @Nonnull
+ public static String resolveChannelId(@Nonnull final String idOrPath)
+ throws ExtractionException, IOException {
+ final String[] channelId = idOrPath.split("/");
+
+ if (channelId[0].startsWith("UC")) {
+ return channelId[0];
+ }
+
+ // If the URL is not a /channel URL, we need to use the navigation/resolve_url endpoint of
+ // the InnerTube API to get the channel id.
+ // Otherwise, we couldn't get information about the channel associated with this URL, if
+ // there is one.
+ if (!channelId[0].equals("channel")) {
+ final byte[] body = JsonWriter.string(
+ prepareDesktopJsonBuilder(Localization.DEFAULT, ContentCountry.DEFAULT)
+ .value("url", "https://www.youtube.com/" + idOrPath)
+ .done())
+ .getBytes(StandardCharsets.UTF_8);
+
+ final JsonObject jsonResponse = getJsonPostResponse(
+ "navigation/resolve_url", body, Localization.DEFAULT);
+
+ checkIfChannelResponseIsValid(jsonResponse);
+
+ final JsonObject endpoint = jsonResponse.getObject("endpoint");
+
+ final String webPageType = endpoint.getObject("commandMetadata")
+ .getObject("webCommandMetadata")
+ .getString("webPageType", "");
+
+ final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
+ final String browseId = browseEndpoint.getString("browseId", "");
+
+ if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
+ || webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
+ && !browseId.isEmpty()) {
+ if (!browseId.startsWith("UC")) {
+ throw new ExtractionException("Redirected id is not pointing to a channel");
+ }
+
+ return browseId;
+ }
+ }
+
+ return channelId[1];
+ }
+
+ /**
+ * Response data object for {@link #getChannelResponse(String, String, Localization,
+ * ContentCountry)}, after any redirection in the allowed redirects count ({@code 3}).
+ */
+ public static final class ChannelResponseData {
+
+ /**
+ * The channel response as a JSON object, after all redirects.
+ */
+ @Nonnull
+ public final JsonObject jsonResponse;
+
+ /**
+ * The channel ID after all redirects.
+ */
+ @Nonnull
+ public final String channelId;
+
+ private ChannelResponseData(@Nonnull final JsonObject jsonResponse,
+ @Nonnull final String channelId) {
+ this.jsonResponse = jsonResponse;
+ this.channelId = channelId;
+ }
+ }
+
+ /**
+ * Fetch a YouTube channel tab response, using the given channel ID and tab parameters.
+ *
+ *
+ * Redirections to other channels such as are supported to up to 3 redirects, which could
+ * happen for instance for localized channels or auto-generated ones such as the {@code Movies
+ * and Shows} (channel IDs {@code UCuJcl0Ju-gPDoksRjK1ya-w}, {@code UChBfWrfBXL9wS6tQtgjt_OQ}
+ * and {@code UCok7UTQQEP1Rsctxiv3gwSQ} of this channel redirect to the
+ * {@code UClgRkhTL3_hImCAmdLfDE4g} one).
+ *
+ *
+ * @param channelId a valid YouTube channel ID
+ * @param parameters the parameters to specify the YouTube channel tab; if invalid ones are
+ * specified, YouTube should return the {@code Home} tab
+ * @param localization the {@link Localization} to use
+ * @param country the {@link ContentCountry} to use
+ * @return a {@link ChannelResponseData channel response data}
+ * @throws IOException if a channel request failed
+ * @throws ExtractionException if a channel request response could not be parsed or is invalid
+ */
+ @Nonnull
+ public static ChannelResponseData getChannelResponse(@Nonnull final String channelId,
+ @Nonnull final String parameters,
+ @Nonnull final Localization localization,
+ @Nonnull final ContentCountry country)
+ throws ExtractionException, IOException {
+ String id = channelId;
+ JsonObject ajaxJson = null;
+
+ int level = 0;
+ while (level < 3) {
+ final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
+ localization, country)
+ .value("browseId", id)
+ .value("params", parameters)
+ .done())
+ .getBytes(StandardCharsets.UTF_8);
+
+ final JsonObject jsonResponse = getJsonPostResponse(
+ "browse", body, localization);
+
+ checkIfChannelResponseIsValid(jsonResponse);
+
+ final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
+ .getObject(0)
+ .getObject("navigateAction")
+ .getObject("endpoint");
+
+ final String webPageType = endpoint.getObject("commandMetadata")
+ .getObject("webCommandMetadata")
+ .getString("webPageType", "");
+
+ final String browseId = endpoint.getObject("browseEndpoint")
+ .getString("browseId", "");
+
+ if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
+ || webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
+ && !browseId.isEmpty()) {
+ if (!browseId.startsWith("UC")) {
+ throw new ExtractionException("Redirected id is not pointing to a channel");
+ }
+
+ id = browseId;
+ level++;
+ } else {
+ ajaxJson = jsonResponse;
+ break;
+ }
+ }
+
+ if (ajaxJson == null) {
+ throw new ExtractionException("Got no channel response");
+ }
+
+ defaultAlertsCheck(ajaxJson);
+
+ return new ChannelResponseData(ajaxJson, id);
+ }
+
+ /**
+ * Assert that a channel JSON response does not contain an {@code error} JSON object.
+ *
+ * @param jsonResponse a channel JSON response
+ * @throws ContentNotAvailableException if the channel was not found
+ */
+ private static void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse)
+ throws ContentNotAvailableException {
+ if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
+ final JsonObject errorJsonObject = jsonResponse.getObject("error");
+ final int errorCode = errorJsonObject.getInt("code");
+ if (errorCode == 404) {
+ throw new ContentNotAvailableException("This channel doesn't exist.");
+ } else {
+ throw new ContentNotAvailableException("Got error:\""
+ + errorJsonObject.getString("status") + "\": "
+ + errorJsonObject.getString("message"));
+ }
+ }
+ }
+
+ /**
+ * A channel header response.
+ *
+ *
+ * This class allows the distinction between a classic header and a carousel one, used for
+ * auto-generated ones like the gaming or music topic channels and for big events such as the
+ * Coachella music festival, which have a different data structure and do not return the same
+ * properties.
+ *
+ */
+ public static final class ChannelHeader {
+
+ /**
+ * The channel header JSON response.
+ */
+ @Nonnull
+ public final JsonObject json;
+
+ /**
+ * Whether the header is a {@code carouselHeaderRenderer}.
+ *
+ *
+ * See the class documentation for more details.
+ *
+ */
+ public final boolean isCarouselHeader;
+
+ private ChannelHeader(@Nonnull final JsonObject json, final boolean isCarouselHeader) {
+ this.json = json;
+ this.isCarouselHeader = isCarouselHeader;
+ }
+ }
+
+ /**
+ * Get a channel header as an {@link Optional} it if exists.
+ *
+ * @param channelResponse a full channel JSON response
+ * @return an {@link Optional} containing a {@link ChannelHeader} or an empty {@link Optional}
+ * if no supported header has been found
+ */
+ @Nonnull
+ public static Optional getChannelHeader(
+ @Nonnull final JsonObject channelResponse) {
+ final JsonObject header = channelResponse.getObject("header");
+
+ if (header.has("c4TabbedHeaderRenderer")) {
+ return Optional.of(header.getObject("c4TabbedHeaderRenderer"))
+ .map(json -> new ChannelHeader(json, false));
+ } else if (header.has("carouselHeaderRenderer")) {
+ return header.getObject("carouselHeaderRenderer")
+ .getArray("contents")
+ .stream()
+ .filter(JsonObject.class::isInstance)
+ .map(JsonObject.class::cast)
+ .filter(item -> item.has("topicChannelDetailsRenderer"))
+ .findFirst()
+ .map(item -> item.getObject("topicChannelDetailsRenderer"))
+ .map(json -> new ChannelHeader(json, true));
+ } else {
+ return Optional.empty();
+ }
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
index 74fee3336..aec550897 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
@@ -1230,8 +1230,17 @@ public final class YoutubeParsingHelper {
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry)
throws IOException, ExtractionException {
+ return prepareDesktopJsonBuilder(localization, contentCountry, null);
+ }
+
+ @Nonnull
+ public static JsonBuilder prepareDesktopJsonBuilder(
+ @Nonnull final Localization localization,
+ @Nonnull final ContentCountry contentCountry,
+ @Nullable final String visitorData)
+ throws IOException, ExtractionException {
// @formatter:off
- return JsonObject.builder()
+ final JsonBuilder builder = JsonObject.builder()
.object("context")
.object("client")
.value("hl", localization.getLocalizationCode())
@@ -1239,8 +1248,13 @@ public final class YoutubeParsingHelper {
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("originalUrl", "https://www.youtube.com")
- .value("platform", "DESKTOP")
- .end()
+ .value("platform", "DESKTOP");
+
+ if (visitorData != null) {
+ builder.value("visitorData", visitorData);
+ }
+
+ return builder.end()
.object("request")
.array("internalExperimentFlags")
.end()
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
index 7470971d5..599b9a5c1 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
@@ -8,6 +8,7 @@ import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.feed.FeedExtractor;
@@ -16,6 +17,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
+import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.localization.ContentCountry;
@@ -23,6 +25,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
@@ -34,6 +37,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscript
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
@@ -88,6 +92,11 @@ public class YoutubeService extends StreamingService {
return YoutubeChannelLinkHandlerFactory.getInstance();
}
+ @Override
+ public ListLinkHandlerFactory getChannelTabLHFactory() {
+ return YoutubeChannelTabLinkHandlerFactory.getInstance();
+ }
+
@Override
public ListLinkHandlerFactory getPlaylistLHFactory() {
return YoutubePlaylistLinkHandlerFactory.getInstance();
@@ -108,6 +117,15 @@ public class YoutubeService extends StreamingService {
return new YoutubeChannelExtractor(this, linkHandler);
}
+ @Override
+ public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
+ if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
+ return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
+ } else {
+ return new YoutubeChannelTabExtractor(this, linkHandler);
+ }
+ }
+
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
@@ -136,16 +154,17 @@ public class YoutubeService extends StreamingService {
@Override
public KioskList getKioskList() throws ExtractionException {
final KioskList list = new KioskList(this);
+ final ListLinkHandlerFactory h = YoutubeTrendingLinkHandlerFactory.getInstance();
// add kiosks here e.g.:
try {
list.addKioskEntry(
(streamingService, url, id) -> new YoutubeTrendingExtractor(
YoutubeService.this,
- new YoutubeTrendingLinkHandlerFactory().fromUrl(url),
+ h.fromUrl(url),
id
),
- new YoutubeTrendingLinkHandlerFactory(),
+ h,
YoutubeTrendingExtractor.KIOSK_ID
);
list.setDefaultKiosk(YoutubeTrendingExtractor.KIOSK_ID);
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
index 0b599f3c5..02709975a 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
@@ -1,39 +1,35 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonWriter;
-import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
-import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
-import org.schabi.newpipe.extractor.localization.TimeAgoParser;
+import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor.VideosTabExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
-import org.schabi.newpipe.extractor.stream.StreamInfoItem;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
-import java.util.Objects;
import java.util.Optional;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -59,22 +55,24 @@ import javax.annotation.Nullable;
*/
public class YoutubeChannelExtractor extends ChannelExtractor {
- private JsonObject initialData;
- private Optional channelHeader;
- private boolean isCarouselHeader = false;
- private JsonObject videoTab;
+
+ private JsonObject jsonResponse;
+ @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
+ private Optional channelHeader;
+
+ private String channelId;
/**
- * Some channels have response redirects and the only way to reliably get the id is by saving it
+ * If a channel is age-restricted, its pages are only accessible to logged-in and
+ * age-verified users, we get an {@code channelAgeGateRenderer} in this case, containing only
+ * the following metadata: channel name and channel avatar.
+ *
*
- * "Movies & Shows":
- *
- * UCuJcl0Ju-gPDoksRjK1ya-w ┐
- * UChBfWrfBXL9wS6tQtgjt_OQ ├ UClgRkhTL3_hImCAmdLfDE4g
- * UCok7UTQQEP1Rsctxiv3gwSQ ┘
- *
+ * This restriction doesn't seem to apply to all countries.
+ *
*/
- private String redirectedChannelId;
+ @Nullable
+ private JsonObject channelAgeGateRenderer;
public YoutubeChannelExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
@@ -85,132 +83,42 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String channelPath = super.getId();
- final String[] channelId = channelPath.split("/");
- String id = "";
- // If the url is an URL which is not a /channel URL, we need to use the
- // navigation/resolve_url endpoint of the InnerTube API to get the channel id. Otherwise,
- // we couldn't get information about the channel associated with this URL, if there is one.
- if (!channelId[0].equals("channel")) {
- final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
- getExtractorLocalization(), getExtractorContentCountry())
- .value("url", "https://www.youtube.com/" + channelPath)
- .done())
- .getBytes(StandardCharsets.UTF_8);
+ final String id = resolveChannelId(channelPath);
+ // Fetch Videos tab
+ final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(id,
+ "EgZ2aWRlb3PyBgQKAjoA", getExtractorLocalization(), getExtractorContentCountry());
- final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
- body, getExtractorLocalization());
-
- checkIfChannelResponseIsValid(jsonResponse);
-
- final JsonObject endpoint = jsonResponse.getObject("endpoint");
-
- final String webPageType = endpoint.getObject("commandMetadata")
- .getObject("webCommandMetadata")
- .getString("webPageType", "");
-
- final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
- final String browseId = browseEndpoint.getString("browseId", "");
-
- if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
- || webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
- && !browseId.isEmpty()) {
- if (!browseId.startsWith("UC")) {
- throw new ExtractionException("Redirected id is not pointing to a channel");
- }
-
- id = browseId;
- redirectedChannelId = browseId;
- }
- } else {
- id = channelId[1];
- }
- JsonObject ajaxJson = null;
-
- int level = 0;
- while (level < 3) {
- final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
- getExtractorLocalization(), getExtractorContentCountry())
- .value("browseId", id)
- .value("params", "EgZ2aWRlb3M%3D") // Equal to videos
- .done())
- .getBytes(StandardCharsets.UTF_8);
-
- final JsonObject jsonResponse = getJsonPostResponse("browse", body,
- getExtractorLocalization());
-
- checkIfChannelResponseIsValid(jsonResponse);
-
- final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
- .getObject(0)
- .getObject("navigateAction")
- .getObject("endpoint");
-
- final String webPageType = endpoint.getObject("commandMetadata")
- .getObject("webCommandMetadata")
- .getString("webPageType", "");
-
- final String browseId = endpoint.getObject("browseEndpoint").getString("browseId",
- "");
-
- if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
- || webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
- && !browseId.isEmpty()) {
- if (!browseId.startsWith("UC")) {
- throw new ExtractionException("Redirected id is not pointing to a channel");
- }
-
- id = browseId;
- redirectedChannelId = browseId;
- level++;
- } else {
- ajaxJson = jsonResponse;
- break;
- }
- }
-
- if (ajaxJson == null) {
- throw new ExtractionException("Could not fetch initial JSON data");
- }
-
- initialData = ajaxJson;
- YoutubeParsingHelper.defaultAlertsCheck(initialData);
+ jsonResponse = data.jsonResponse;
+ channelId = data.channelId;
+ channelAgeGateRenderer = getChannelAgeGateRenderer();
}
- private void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse)
- throws ContentNotAvailableException {
- if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
- final JsonObject errorJsonObject = jsonResponse.getObject("error");
- final int errorCode = errorJsonObject.getInt("code");
- if (errorCode == 404) {
- throw new ContentNotAvailableException("This channel doesn't exist.");
- } else {
- throw new ContentNotAvailableException("Got error:\""
- + errorJsonObject.getString("status") + "\": "
- + errorJsonObject.getString("message"));
- }
- }
- }
-
- @Nonnull
- private Optional getChannelHeader() {
- if (channelHeader == null) {
- final JsonObject h = initialData.getObject("header");
-
- if (h.has("c4TabbedHeaderRenderer")) {
- channelHeader = Optional.of(h.getObject("c4TabbedHeaderRenderer"));
- } else if (h.has("carouselHeaderRenderer")) {
- isCarouselHeader = true;
- channelHeader = h.getObject("carouselHeaderRenderer")
+ @Nullable
+ private JsonObject getChannelAgeGateRenderer() {
+ return jsonResponse.getObject("contents")
+ .getObject("twoColumnBrowseResultsRenderer")
+ .getArray("tabs")
+ .stream()
+ .filter(JsonObject.class::isInstance)
+ .map(JsonObject.class::cast)
+ .flatMap(tab -> tab.getObject("tabRenderer")
+ .getObject("content")
+ .getObject("sectionListRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
- .map(JsonObject.class::cast)
- .filter(itm -> itm.has("topicChannelDetailsRenderer"))
- .findFirst()
- .map(itm -> itm.getObject("topicChannelDetailsRenderer"));
- } else {
- channelHeader = Optional.empty();
- }
+ .map(JsonObject.class::cast))
+ .filter(content -> content.has("channelAgeGateRenderer"))
+ .map(content -> content.getObject("channelAgeGateRenderer"))
+ .findFirst()
+ .orElse(null);
+ }
+
+ @Nonnull
+ private Optional getChannelHeader() {
+ //noinspection OptionalAssignedToNull
+ if (channelHeader == null) {
+ channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
}
return channelHeader;
}
@@ -229,57 +137,70 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getId() throws ParsingException {
return getChannelHeader()
- .flatMap(header -> Optional.ofNullable(header.getString("channelId")).or(
- () -> Optional.ofNullable(header.getObject("navigationEndpoint")
+ .flatMap(header -> Optional.ofNullable(header.json.getString("channelId"))
+ .or(() -> Optional.ofNullable(header.json.getObject("navigationEndpoint")
.getObject("browseEndpoint")
.getString("browseId"))
))
- .or(() -> Optional.ofNullable(redirectedChannelId))
- .orElseThrow(() -> new ParsingException("Could not get channel id"));
+ .or(() -> Optional.ofNullable(channelId))
+ .orElseThrow(() -> new ParsingException("Could not get channel ID"));
}
@Nonnull
@Override
public String getName() throws ParsingException {
- final String mdName = initialData.getObject("metadata")
- .getObject("channelMetadataRenderer")
- .getString("title");
- if (!isNullOrEmpty(mdName)) {
- return mdName;
+ if (channelAgeGateRenderer != null) {
+ return channelAgeGateRenderer.getString("channelTitle");
}
- final Optional header = getChannelHeader();
- if (header.isPresent()) {
- final Object title = header.get().get("title");
+ final String metadataRendererTitle = jsonResponse.getObject("metadata")
+ .getObject("channelMetadataRenderer")
+ .getString("title");
+ if (!isNullOrEmpty(metadataRendererTitle)) {
+ return metadataRendererTitle;
+ }
+
+ return getChannelHeader().flatMap(header -> {
+ final Object title = header.json.get("title");
if (title instanceof String) {
- return (String) title;
+ return Optional.of((String) title);
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
- return headerName;
+ return Optional.of(headerName);
}
}
- }
-
- throw new ParsingException("Could not get channel name");
+ return Optional.empty();
+ }).orElseThrow(() -> new ParsingException("Could not get channel name"));
}
@Override
public String getAvatarUrl() throws ParsingException {
- return getChannelHeader().flatMap(header -> Optional.ofNullable(
- header.getObject("avatar").getArray("thumbnails")
- .getObject(0).getString("url")
- ))
- .map(YoutubeParsingHelper::fixThumbnailUrl)
- .orElseThrow(() -> new ParsingException("Could not get avatar"));
+ final JsonObject avatarJsonObjectContainer;
+ if (channelAgeGateRenderer != null) {
+ avatarJsonObjectContainer = channelAgeGateRenderer;
+ } else {
+ avatarJsonObjectContainer = getChannelHeader().map(header -> header.json)
+ .orElseThrow(() -> new ParsingException("Could not get avatar URL"));
+ }
+
+ return YoutubeParsingHelper.fixThumbnailUrl(avatarJsonObjectContainer.getObject("avatar")
+ .getArray("thumbnails")
+ .getObject(0)
+ .getString("url"));
}
@Override
public String getBannerUrl() throws ParsingException {
+ if (channelAgeGateRenderer != null) {
+ return "";
+ }
+
return getChannelHeader().flatMap(header -> Optional.ofNullable(
- header.getObject("banner").getArray("thumbnails")
- .getObject(0).getString("url")
- ))
+ header.json.getObject("banner")
+ .getArray("thumbnails")
+ .getObject(0)
+ .getString("url")))
.filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
.map(YoutubeParsingHelper::fixThumbnailUrl)
// Channels may not have a banner, so no exception should be thrown if no banner is
@@ -290,6 +211,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getFeedUrl() throws ParsingException {
+ // RSS feeds are accessible for age-restricted channels, no need to check whether a channel
+ // has a channelAgeGateRenderer
try {
return YoutubeParsingHelper.getFeedUrlFrom(getId());
} catch (final Exception e) {
@@ -299,14 +222,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public long getSubscriberCount() throws ParsingException {
- final Optional header = getChannelHeader();
- if (header.isPresent()) {
+ if (channelAgeGateRenderer != null) {
+ return UNKNOWN_SUBSCRIBER_COUNT;
+ }
+
+ final Optional headerOpt = getChannelHeader();
+ if (headerOpt.isPresent()) {
+ final JsonObject header = headerOpt.get().json;
JsonObject textObject = null;
- if (header.get().has("subscriberCountText")) {
- textObject = header.get().getObject("subscriberCountText");
- } else if (header.get().has("subtitle")) {
- textObject = header.get().getObject("subtitle");
+ if (header.has("subscriberCountText")) {
+ textObject = header.getObject("subscriberCountText");
+ } else if (header.has("subtitle")) {
+ textObject = header.getObject("subtitle");
}
if (textObject != null) {
@@ -317,13 +245,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
}
}
+
return UNKNOWN_SUBSCRIBER_COUNT;
}
@Override
public String getDescription() throws ParsingException {
+ if (channelAgeGateRenderer != null) {
+ return null;
+ }
+
try {
- return initialData.getObject("metadata").getObject("channelMetadataRenderer")
+ return jsonResponse.getObject("metadata")
+ .getObject("channelMetadataRenderer")
.getString("description");
} catch (final Exception e) {
throw new ParsingException("Could not get channel description", e);
@@ -347,190 +281,139 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public boolean isVerified() throws ParsingException {
- // The CarouselHeaderRenderer does not contain any verification badges.
- // Since it is only shown on YT-internal channels or on channels of large organizations
- // broadcasting live events, we can assume the channel to be verified.
- if (isCarouselHeader) {
- return true;
+ if (channelAgeGateRenderer != null) {
+ return false;
}
- return getChannelHeader()
- .map(header -> header.getArray("badges"))
- .map(YoutubeParsingHelper::isVerified)
- .orElse(false);
+ final Optional headerOpt = getChannelHeader();
+ if (headerOpt.isPresent()) {
+ final YoutubeChannelHelper.ChannelHeader header = headerOpt.get();
+
+ // The CarouselHeaderRenderer does not contain any verification badges.
+ // Since it is only shown on YT-internal channels or on channels of large organizations
+ // broadcasting live events, we can assume the channel to be verified.
+ if (header.isCarouselHeader) {
+ return true;
+ }
+ return YoutubeParsingHelper.isVerified(header.json.getArray("badges"));
+ }
+ return false;
}
@Nonnull
@Override
- public InfoItemsPage getInitialPage() throws IOException, ExtractionException {
- final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-
- Page nextPage = null;
-
- if (getVideoTab() != null) {
- final JsonObject tabContent = getVideoTab().getObject("content");
- JsonArray items = tabContent
- .getObject("sectionListRenderer")
- .getArray("contents").getObject(0).getObject("itemSectionRenderer")
- .getArray("contents").getObject(0).getObject("gridRenderer").getArray("items");
-
- if (items.isEmpty()) {
- items = tabContent.getObject("richGridRenderer").getArray("contents");
- }
-
- final List channelIds = new ArrayList<>();
- channelIds.add(getName());
- channelIds.add(getUrl());
- final JsonObject continuation = collectStreamsFrom(collector, items, channelIds);
-
- nextPage = getNextPageFrom(continuation, channelIds);
+ public List getTabs() throws ParsingException {
+ if (channelAgeGateRenderer == null) {
+ return getTabsForNonAgeRestrictedChannels();
}
- return new InfoItemsPage<>(collector, nextPage);
+ return getTabsForAgeRestrictedChannels();
}
- @Override
- public InfoItemsPage getPage(final Page page)
- throws IOException, ExtractionException {
- if (page == null || isNullOrEmpty(page.getUrl())) {
- throw new IllegalArgumentException("Page doesn't contain an URL");
- }
-
- final List channelIds = page.getIds();
-
- final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-
- final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
- getExtractorLocalization());
-
- final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
- .getObject(0)
- .getObject("appendContinuationItemsAction");
-
- final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation
- .getArray("continuationItems"), channelIds);
-
- return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
- }
-
- @Nullable
- private Page getNextPageFrom(final JsonObject continuations,
- final List channelIds)
- throws IOException, ExtractionException {
- if (isNullOrEmpty(continuations)) {
- return null;
- }
-
- final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
- final String continuation = continuationEndpoint.getObject("continuationCommand")
- .getString("token");
-
- final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
- getExtractorContentCountry())
- .value("continuation", continuation)
- .done())
- .getBytes(StandardCharsets.UTF_8);
-
- return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
- + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
- }
-
- /**
- * Collect streams from an array of items
- *
- * @param collector the collector where videos will be committed
- * @param videos the array to get videos from
- * @param channelIds the ids of the channel, which are its name and its URL
- * @return the continuation object
- */
- private JsonObject collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
- @Nonnull final JsonArray videos,
- @Nonnull final List channelIds) {
- collector.reset();
-
- final String uploaderName = channelIds.get(0);
- final String uploaderUrl = channelIds.get(1);
- final TimeAgoParser timeAgoParser = getTimeAgoParser();
-
- JsonObject continuation = null;
-
- for (final Object object : videos) {
- final JsonObject video = (JsonObject) object;
- if (video.has("gridVideoRenderer")) {
- collector.commit(new YoutubeStreamInfoItemExtractor(
- video.getObject("gridVideoRenderer"), timeAgoParser) {
- @Override
- public String getUploaderName() {
- return uploaderName;
- }
-
- @Override
- public String getUploaderUrl() {
- return uploaderUrl;
- }
- });
- } else if (video.has("richItemRenderer")) {
- collector.commit(new YoutubeStreamInfoItemExtractor(
- video.getObject("richItemRenderer")
- .getObject("content").getObject("videoRenderer"), timeAgoParser) {
- @Override
- public String getUploaderName() {
- return uploaderName;
- }
-
- @Override
- public String getUploaderUrl() {
- return uploaderUrl;
- }
- });
-
- } else if (video.has("continuationItemRenderer")) {
- continuation = video.getObject("continuationItemRenderer");
- }
- }
-
- return continuation;
- }
-
- @Nullable
- private JsonObject getVideoTab() throws ParsingException {
- if (videoTab != null) {
- return videoTab;
- }
-
- final JsonArray tabs = initialData.getObject("contents")
+ @Nonnull
+ private List getTabsForNonAgeRestrictedChannels() throws ParsingException {
+ final JsonArray responseTabs = jsonResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs");
- final JsonObject foundVideoTab = tabs.stream()
- .filter(Objects::nonNull)
+ final List tabs = new ArrayList<>();
+ final Consumer addNonVideosTab = tabName -> {
+ try {
+ tabs.add(YoutubeChannelTabLinkHandlerFactory.getInstance().fromQuery(
+ channelId, List.of(tabName), ""));
+ } catch (final ParsingException ignored) {
+ // Do not add the tab if we couldn't create the LinkHandler
+ }
+ };
+
+ final String name = getName();
+ final String url = getUrl();
+ final String id = getId();
+
+ responseTabs.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
- .filter(tab -> tab.has("tabRenderer")
- && tab.getObject("tabRenderer")
- .getString("title", "")
- .equals("Videos"))
- .findFirst()
+ .filter(tab -> tab.has("tabRenderer"))
.map(tab -> tab.getObject("tabRenderer"))
- .orElseThrow(
- () -> new ContentNotSupportedException("This channel has no Videos tab"));
+ .forEach(tabRenderer -> {
+ final String tabUrl = tabRenderer.getObject("endpoint")
+ .getObject("commandMetadata")
+ .getObject("webCommandMetadata")
+ .getString("url");
+ if (tabUrl != null) {
+ final String[] urlParts = tabUrl.split("/");
+ if (urlParts.length == 0) {
+ return;
+ }
- final String messageRendererText = getTextFromObject(
- foundVideoTab.getObject("content")
- .getObject("sectionListRenderer")
- .getArray("contents")
- .getObject(0)
- .getObject("itemSectionRenderer")
- .getArray("contents")
- .getObject(0)
- .getObject("messageRenderer")
- .getObject("text"));
- if (messageRendererText != null
- && messageRendererText.equals("This channel has no videos.")) {
- return null;
+ final String urlSuffix = urlParts[urlParts.length - 1];
+
+ switch (urlSuffix) {
+ case "videos":
+ // Since the Videos tab has already its contents fetched, make
+ // sure it is in the first position
+ // YoutubeChannelTabExtractor still supports fetching this tab
+ tabs.add(0, new ReadyChannelTabListLinkHandler(
+ tabUrl,
+ channelId,
+ ChannelTabs.VIDEOS,
+ (service, linkHandler) -> new VideosTabExtractor(
+ service, linkHandler, tabRenderer, name, id, url)));
+
+ break;
+ case "shorts":
+ addNonVideosTab.accept(ChannelTabs.SHORTS);
+ break;
+ case "streams":
+ addNonVideosTab.accept(ChannelTabs.LIVESTREAMS);
+ break;
+ case "playlists":
+ addNonVideosTab.accept(ChannelTabs.PLAYLISTS);
+ break;
+ case "channels":
+ addNonVideosTab.accept(ChannelTabs.CHANNELS);
+ break;
+ }
+ }
+ });
+
+ return Collections.unmodifiableList(tabs);
+ }
+
+ @Nonnull
+ private List getTabsForAgeRestrictedChannels() throws ParsingException {
+ // As we don't have access to the channel tabs list, consider that the channel has videos,
+ // shorts and livestreams, the data only accessible without login on YouTube's desktop
+ // client using uploads system playlists
+ // The playlists channel tab is still available on YouTube Music, but this is not
+ // implemented in the extractor
+
+ final List tabs = new ArrayList<>();
+ final String channelUrl = getUrl();
+
+ final Consumer addTab = tabName ->
+ tabs.add(new ReadyChannelTabListLinkHandler(channelUrl + "/" + tabName,
+ channelId, tabName, YoutubeChannelTabPlaylistExtractor::new));
+
+ addTab.accept(ChannelTabs.VIDEOS);
+ addTab.accept(ChannelTabs.SHORTS);
+ addTab.accept(ChannelTabs.LIVESTREAMS);
+ return Collections.unmodifiableList(tabs);
+ }
+
+ @Nonnull
+ @Override
+ public List getTags() throws ParsingException {
+ if (channelAgeGateRenderer != null) {
+ return List.of();
}
- videoTab = foundVideoTab;
- return foundVideoTab;
+ return jsonResponse.getObject("microformat")
+ .getObject("microformatDataRenderer")
+ .getArray("tags")
+ .stream()
+ .filter(String.class::isInstance)
+ .map(String.class::cast)
+ .collect(Collectors.toUnmodifiableList());
}
}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
new file mode 100644
index 000000000..d0f116af0
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
@@ -0,0 +1,486 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonWriter;
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
+import org.schabi.newpipe.extractor.Page;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
+import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.localization.TimeAgoParser;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Consumer;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+/**
+ * A {@link ChannelTabExtractor} implementation for the YouTube service.
+ *
+ *
+ * It currently supports {@code Videos}, {@code Shorts}, {@code Live}, {@code Playlists} and
+ * {@code Channels} tabs.
+ *
+ */
+public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
+
+ /**
+ * Whether the visitor data extracted from the initial channel response is required to be used
+ * for continuations.
+ *
+ *
+ * A valid {@code visitorData} is required to get continuations of shorts in channels.
+ *
+ *
+ *
+ * It should be not used when it is not needed, in order to reduce YouTube's tracking.
+ *
+ */
+ private final boolean useVisitorData;
+ private JsonObject jsonResponse;
+ private String channelId;
+ @Nullable
+ private String visitorData;
+
+ public YoutubeChannelTabExtractor(final StreamingService service,
+ final ListLinkHandler linkHandler) {
+ super(service, linkHandler);
+ useVisitorData = getName().equals(ChannelTabs.SHORTS);
+ }
+
+ @Nonnull
+ private String getChannelTabsParameters() throws ParsingException {
+ final String name = getName();
+ switch (name) {
+ case ChannelTabs.VIDEOS:
+ return "EgZ2aWRlb3PyBgQKAjoA";
+ case ChannelTabs.SHORTS:
+ return "EgZzaG9ydHPyBgUKA5oBAA%3D%3D";
+ case ChannelTabs.LIVESTREAMS:
+ return "EgdzdHJlYW1z8gYECgJ6AA%3D%3D";
+ case ChannelTabs.PLAYLISTS:
+ return "EglwbGF5bGlzdHPyBgQKAkIA";
+ case ChannelTabs.CHANNELS:
+ return "EghjaGFubmVsc_IGBAoCUgA%3D";
+ }
+ throw new ParsingException("Unsupported channel tab: " + name);
+ }
+
+ @Override
+ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
+ ExtractionException {
+ channelId = resolveChannelId(super.getId());
+
+ final String params = getChannelTabsParameters();
+
+ final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(channelId,
+ params, getExtractorLocalization(), getExtractorContentCountry());
+
+ jsonResponse = data.jsonResponse;
+ channelId = data.channelId;
+ if (useVisitorData) {
+ visitorData = jsonResponse.getObject("responseContext").getString("visitorData");
+ }
+ }
+
+ @Nonnull
+ @Override
+ public String getUrl() throws ParsingException {
+ try {
+ return YoutubeChannelTabLinkHandlerFactory.getInstance()
+ .getUrl("channel/" + getId(), List.of(getName()), "");
+ } catch (final ParsingException e) {
+ return super.getUrl();
+ }
+ }
+
+ @Nonnull
+ @Override
+ public String getId() throws ParsingException {
+ final String id = jsonResponse.getObject("header")
+ .getObject("c4TabbedHeaderRenderer")
+ .getString("channelId", "");
+
+ if (!id.isEmpty()) {
+ return id;
+ }
+
+ final Optional carouselHeaderId = jsonResponse.getObject("header")
+ .getObject("carouselHeaderRenderer")
+ .getArray("contents")
+ .stream()
+ .filter(JsonObject.class::isInstance)
+ .map(JsonObject.class::cast)
+ .filter(item -> item.has("topicChannelDetailsRenderer"))
+ .findFirst()
+ .flatMap(item ->
+ Optional.ofNullable(item.getObject("topicChannelDetailsRenderer")
+ .getObject("navigationEndpoint")
+ .getObject("browseEndpoint")
+ .getString("browseId")));
+ if (carouselHeaderId.isPresent()) {
+ return carouselHeaderId.get();
+ }
+
+ if (!isNullOrEmpty(channelId)) {
+ return channelId;
+ } else {
+ throw new ParsingException("Could not get channel ID");
+ }
+ }
+
+ protected String getChannelName() {
+ final String metadataName = jsonResponse.getObject("metadata")
+ .getObject("channelMetadataRenderer")
+ .getString("title");
+ if (!isNullOrEmpty(metadataName)) {
+ return metadataName;
+ }
+
+ return YoutubeChannelHelper.getChannelHeader(jsonResponse)
+ .map(header -> {
+ final Object title = header.json.get("title");
+ if (title instanceof String) {
+ return (String) title;
+ } else if (title instanceof JsonObject) {
+ final String headerName = getTextFromObject((JsonObject) title);
+ if (!isNullOrEmpty(headerName)) {
+ return headerName;
+ }
+ }
+ return "";
+ })
+ .orElse("");
+ }
+
+ @Nonnull
+ @Override
+ public InfoItemsPage getInitialPage() throws IOException, ExtractionException {
+ final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
+
+ JsonArray items = new JsonArray();
+ final Optional tab = getTabData();
+
+ if (tab.isPresent()) {
+ final JsonObject tabContent = tab.get().getObject("content");
+
+ items = tabContent.getObject("sectionListRenderer")
+ .getArray("contents")
+ .getObject(0)
+ .getObject("itemSectionRenderer")
+ .getArray("contents")
+ .getObject(0)
+ .getObject("gridRenderer")
+ .getArray("items");
+
+ if (items.isEmpty()) {
+ items = tabContent.getObject("richGridRenderer")
+ .getArray("contents");
+
+ if (items.isEmpty()) {
+ items = tabContent.getObject("sectionListRenderer")
+ .getArray("contents");
+ }
+ }
+ }
+
+ // If a channel tab is fetched, the next page requires channel ID and name, as channel
+ // streams don't have their channel specified.
+ // We also need to set the visitor data here when it should be enabled, as it is required
+ // to get continuations on some channel tabs, and we need a way to pass it between pages
+ final List channelIds = useVisitorData && !isNullOrEmpty(visitorData)
+ ? List.of(getChannelName(), getUrl(), visitorData)
+ : List.of(getChannelName(), getUrl());
+
+ final JsonObject continuation = collectItemsFrom(collector, items, channelIds)
+ .orElse(null);
+
+ final Page nextPage = getNextPageFrom(continuation, channelIds);
+
+ return new InfoItemsPage<>(collector, nextPage);
+ }
+
+ @Override
+ public InfoItemsPage getPage(final Page page)
+ throws IOException, ExtractionException {
+ if (page == null || isNullOrEmpty(page.getUrl())) {
+ throw new IllegalArgumentException("Page doesn't contain an URL");
+ }
+
+ final List channelIds = page.getIds();
+
+ final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
+
+ final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
+ getExtractorLocalization());
+
+ final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
+ .stream()
+ .filter(JsonObject.class::isInstance)
+ .map(JsonObject.class::cast)
+ .filter(jsonObject -> jsonObject.has("appendContinuationItemsAction"))
+ .map(jsonObject -> jsonObject.getObject("appendContinuationItemsAction"))
+ .findFirst()
+ .orElse(new JsonObject());
+
+ final JsonObject continuation = collectItemsFrom(collector,
+ sectionListContinuation.getArray("continuationItems"), channelIds)
+ .orElse(null);
+
+ return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
+ }
+
+ Optional getTabData() {
+ final String urlSuffix = YoutubeChannelTabLinkHandlerFactory.getUrlSuffix(getName());
+
+ return jsonResponse.getObject("contents")
+ .getObject("twoColumnBrowseResultsRenderer")
+ .getArray("tabs")
+ .stream()
+ .filter(JsonObject.class::isInstance)
+ .map(JsonObject.class::cast)
+ .filter(tab -> tab.has("tabRenderer"))
+ .map(tab -> tab.getObject("tabRenderer"))
+ .filter(tabRenderer -> tabRenderer.getObject("endpoint")
+ .getObject("commandMetadata").getObject("webCommandMetadata")
+ .getString("url", "").endsWith(urlSuffix))
+ .findFirst()
+ // Check if tab has no content
+ .filter(tabRenderer -> {
+ final JsonArray tabContents = tabRenderer.getObject("content")
+ .getObject("sectionListRenderer")
+ .getArray("contents")
+ .getObject(0)
+ .getObject("itemSectionRenderer")
+ .getArray("contents");
+ return tabContents.size() != 1
+ || !tabContents.getObject(0).has("messageRenderer");
+ });
+ }
+
+ private Optional collectItemsFrom(@Nonnull final MultiInfoItemsCollector collector,
+ @Nonnull final JsonArray items,
+ @Nonnull final List channelIds) {
+ return items.stream()
+ .filter(JsonObject.class::isInstance)
+ .map(JsonObject.class::cast)
+ .map(item -> collectItem(collector, item, channelIds))
+ .reduce(Optional.empty(), (c1, c2) -> c1.or(() -> c2));
+ }
+
+ private Optional collectItem(@Nonnull final MultiInfoItemsCollector collector,
+ @Nonnull final JsonObject item,
+ @Nonnull final List channelIds) {
+ final TimeAgoParser timeAgoParser = getTimeAgoParser();
+
+ if (item.has("richItemRenderer")) {
+ final JsonObject richItem = item.getObject("richItemRenderer")
+ .getObject("content");
+
+ if (richItem.has("videoRenderer")) {
+ getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept(
+ richItem.getObject("videoRenderer"));
+ } else if (richItem.has("reelItemRenderer")) {
+ getCommitReelItemConsumer(collector, timeAgoParser, channelIds).accept(
+ richItem.getObject("reelItemRenderer"));
+ } else if (richItem.has("playlistRenderer")) {
+ getCommitPlaylistConsumer(collector, channelIds).accept(
+ item.getObject("playlistRenderer"));
+ }
+ } else if (item.has("gridVideoRenderer")) {
+ getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept(
+ item.getObject("gridVideoRenderer"));
+ } else if (item.has("gridPlaylistRenderer")) {
+ getCommitPlaylistConsumer(collector, channelIds).accept(
+ item.getObject("gridPlaylistRenderer"));
+ } else if (item.has("gridChannelRenderer")) {
+ collector.commit(new YoutubeChannelInfoItemExtractor(
+ item.getObject("gridChannelRenderer")));
+ } else if (item.has("shelfRenderer")) {
+ return collectItem(collector, item.getObject("shelfRenderer")
+ .getObject("content"), channelIds);
+ } else if (item.has("itemSectionRenderer")) {
+ return collectItemsFrom(collector, item.getObject("itemSectionRenderer")
+ .getArray("contents"), channelIds);
+ } else if (item.has("horizontalListRenderer")) {
+ return collectItemsFrom(collector, item.getObject("horizontalListRenderer")
+ .getArray("items"), channelIds);
+ } else if (item.has("expandedShelfContentsRenderer")) {
+ return collectItemsFrom(collector, item.getObject("expandedShelfContentsRenderer")
+ .getArray("items"), channelIds);
+ } else if (item.has("continuationItemRenderer")) {
+ return Optional.ofNullable(item.getObject("continuationItemRenderer"));
+ }
+
+ return Optional.empty();
+ }
+
+ @Nonnull
+ private Consumer getCommitVideoConsumer(
+ @Nonnull final MultiInfoItemsCollector collector,
+ @Nonnull final TimeAgoParser timeAgoParser,
+ @Nonnull final List channelIds) {
+ return videoRenderer -> collector.commit(
+ new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser) {
+ @Override
+ public String getUploaderName() throws ParsingException {
+ if (channelIds.size() >= 2) {
+ return channelIds.get(0);
+ }
+ return super.getUploaderName();
+ }
+
+ @Override
+ public String getUploaderUrl() throws ParsingException {
+ if (channelIds.size() >= 2) {
+ return channelIds.get(1);
+ }
+ return super.getUploaderUrl();
+ }
+ });
+ }
+
+ @Nonnull
+ private Consumer getCommitReelItemConsumer(
+ @Nonnull final MultiInfoItemsCollector collector,
+ @Nonnull final TimeAgoParser timeAgoParser,
+ @Nonnull final List channelIds) {
+ return reelItemRenderer -> collector.commit(
+ new YoutubeReelInfoItemExtractor(reelItemRenderer, timeAgoParser) {
+ @Override
+ public String getUploaderName() throws ParsingException {
+ if (channelIds.size() >= 2) {
+ return channelIds.get(0);
+ }
+ return super.getUploaderName();
+ }
+
+ @Override
+ public String getUploaderUrl() throws ParsingException {
+ if (channelIds.size() >= 2) {
+ return channelIds.get(1);
+ }
+ return super.getUploaderUrl();
+ }
+ });
+ }
+
+ @Nonnull
+ private Consumer getCommitPlaylistConsumer(
+ @Nonnull final MultiInfoItemsCollector collector,
+ @Nonnull final List channelIds) {
+ return playlistRenderer -> collector.commit(
+ new YoutubePlaylistInfoItemExtractor(playlistRenderer) {
+ @Override
+ public String getUploaderName() throws ParsingException {
+ if (channelIds.size() >= 2) {
+ return channelIds.get(0);
+ }
+ return super.getUploaderName();
+ }
+
+ @Override
+ public String getUploaderUrl() throws ParsingException {
+ if (channelIds.size() >= 2) {
+ return channelIds.get(1);
+ }
+ return super.getUploaderUrl();
+ }
+ });
+ }
+
+ @Nullable
+ private Page getNextPageFrom(final JsonObject continuations,
+ final List channelIds) throws IOException,
+ ExtractionException {
+ if (isNullOrEmpty(continuations)) {
+ return null;
+ }
+
+ final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
+ final String continuation = continuationEndpoint.getObject("continuationCommand")
+ .getString("token");
+
+ final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
+ getExtractorContentCountry(),
+ useVisitorData && channelIds.size() >= 3 ? channelIds.get(2) : null)
+ .value("continuation", continuation)
+ .done())
+ .getBytes(StandardCharsets.UTF_8);
+
+ return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
+ }
+
+ /**
+ * A {@link YoutubeChannelTabExtractor} for the {@code Videos} tab, if it has been already
+ * fetched.
+ */
+ public static final class VideosTabExtractor extends YoutubeChannelTabExtractor {
+ private final JsonObject tabRenderer;
+ private final String channelName;
+ private final String channelId;
+ private final String channelUrl;
+
+ VideosTabExtractor(final StreamingService service,
+ final ListLinkHandler linkHandler,
+ final JsonObject tabRenderer,
+ final String channelName,
+ final String channelId,
+ final String channelUrl) {
+ super(service, linkHandler);
+ this.tabRenderer = tabRenderer;
+ this.channelName = channelName;
+ this.channelId = channelId;
+ this.channelUrl = channelUrl;
+ }
+
+ @Override
+ public void onFetchPage(@Nonnull final Downloader downloader) {
+ // Nothing to do, the initial data was already fetched and is stored in the link handler
+ }
+
+ @Nonnull
+ @Override
+ public String getId() throws ParsingException {
+ return channelId;
+ }
+
+ @Nonnull
+ @Override
+ public String getUrl() throws ParsingException {
+ return channelUrl;
+ }
+
+ @Override
+ protected String getChannelName() {
+ return channelName;
+ }
+
+ @Override
+ Optional getTabData() {
+ return Optional.of(tabRenderer);
+ }
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabPlaylistExtractor.java
new file mode 100644
index 000000000..da50db631
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabPlaylistExtractor.java
@@ -0,0 +1,191 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.Page;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
+import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+/**
+ * A {@link ChannelTabExtractor} for YouTube system playlists using a
+ * {@link YoutubePlaylistExtractor} instance.
+ *
+ *
+ * It is currently used to bypass age-restrictions on channels marked as age-restricted by their
+ * owner(s).
+ *
+ */
+public class YoutubeChannelTabPlaylistExtractor extends ChannelTabExtractor {
+
+ private final PlaylistExtractor playlistExtractorInstance;
+ private boolean playlistExisting;
+
+ /**
+ * Construct a {@link YoutubeChannelTabPlaylistExtractor} instance.
+ *
+ * @param service a {@link StreamingService} implementation, which must be the YouTube
+ * one
+ * @param linkHandler a {@link ListLinkHandler} which must have a valid channel ID (starting
+ * with `UC`) and one of the given and supported content filters:
+ * {@link ChannelTabs#VIDEOS}, {@link ChannelTabs#SHORTS},
+ * {@link ChannelTabs#LIVESTREAMS}
+ * @throws IllegalArgumentException if the given {@link ListLinkHandler} doesn't have the
+ * required arguments
+ * @throws SystemPlaylistUrlCreationException if the system playlist URL could not be created,
+ * which should never happen
+ */
+ public YoutubeChannelTabPlaylistExtractor(@Nonnull final StreamingService service,
+ @Nonnull final ListLinkHandler linkHandler)
+ throws IllegalArgumentException, SystemPlaylistUrlCreationException {
+ super(service, linkHandler);
+ final ListLinkHandler playlistLinkHandler = getPlaylistLinkHandler(linkHandler);
+ this.playlistExtractorInstance = new YoutubePlaylistExtractor(service, playlistLinkHandler);
+ }
+
+ @Override
+ public void onFetchPage(@Nonnull final Downloader downloader)
+ throws IOException, ExtractionException {
+ try {
+ playlistExtractorInstance.onFetchPage(downloader);
+ if (!playlistExisting) {
+ playlistExisting = true;
+ }
+ } catch (final ContentNotAvailableException e) {
+ // If a channel has no content of the type requested, the corresponding system playlist
+ // won't exist, so a ContentNotAvailableException would be thrown
+ // Ignore such issues in this case
+ }
+ }
+
+ @Nonnull
+ @Override
+ public InfoItemsPage getInitialPage() throws IOException, ExtractionException {
+ if (!playlistExisting) {
+ return InfoItemsPage.emptyPage();
+ }
+
+ final InfoItemsPage playlistInitialPage =
+ playlistExtractorInstance.getInitialPage();
+
+ // We can't provide the playlist page as it is due to a type conflict, we need to wrap the
+ // page items and provide a new InfoItemsPage
+ final List infoItems = new ArrayList<>(playlistInitialPage.getItems());
+ return new InfoItemsPage<>(infoItems, playlistInitialPage.getNextPage(),
+ playlistInitialPage.getErrors());
+ }
+
+ @Override
+ public InfoItemsPage getPage(final Page page)
+ throws IOException, ExtractionException {
+ if (!playlistExisting) {
+ return InfoItemsPage.emptyPage();
+ }
+
+ final InfoItemsPage playlistPage = playlistExtractorInstance.getPage(page);
+
+ // We can't provide the playlist page as it is due to a type conflict, we need to wrap the
+ // page items and provide a new InfoItemsPage
+ final List infoItems = new ArrayList<>(playlistPage.getItems());
+ return new InfoItemsPage<>(infoItems, playlistPage.getNextPage(),
+ playlistPage.getErrors());
+ }
+
+ /**
+ * Get a playlist {@link ListLinkHandler} from a channel tab one.
+ *
+ *
+ * This method converts a channel ID without its {@code UC} prefix into a YouTube system
+ * playlist, depending on the first content filter provided in the given
+ * {@link ListLinkHandler}.
+ *
+ *
+ *
+ * The first content filter must be a channel tabs one among the
+ * {@link ChannelTabs#VIDEOS videos}, {@link ChannelTabs#SHORTS shorts} and
+ * {@link ChannelTabs#LIVESTREAMS} ones, which would be converted respectively into playlists
+ * with the ID {@code UULF}, {@code UUSH} and {@code UULV} on which the channel ID without the
+ * {@code UC} part is appended.
+ *
+ *
+ * @param originalLinkHandler the original {@link ListLinkHandler} with which a
+ * {@link YoutubeChannelTabPlaylistExtractor} instance is being constructed
+ *
+ * @return a {@link ListLinkHandler} to use for the {@link YoutubePlaylistExtractor} instance
+ * needed to extract channel tabs data from a system playlist
+ * @throws IllegalArgumentException if the original {@link ListLinkHandler} does not meet the
+ * required criteria above
+ * @throws SystemPlaylistUrlCreationException if the system playlist URL could not be created,
+ * which should never happen
+ */
+ @Nonnull
+ private ListLinkHandler getPlaylistLinkHandler(
+ @Nonnull final ListLinkHandler originalLinkHandler)
+ throws IllegalArgumentException, SystemPlaylistUrlCreationException {
+ final List contentFilters = originalLinkHandler.getContentFilters();
+ if (contentFilters.isEmpty()) {
+ throw new IllegalArgumentException("A content filter is required");
+ }
+
+ final String channelId = originalLinkHandler.getId();
+ if (isNullOrEmpty(channelId) || !channelId.startsWith("UC")) {
+ throw new IllegalArgumentException("Invalid channel ID");
+ }
+
+ final String channelIdWithoutUc = channelId.substring(2);
+
+ final String playlistId;
+ switch (contentFilters.get(0)) {
+ case ChannelTabs.VIDEOS:
+ playlistId = "UULF" + channelIdWithoutUc;
+ break;
+ case ChannelTabs.SHORTS:
+ playlistId = "UUSH" + channelIdWithoutUc;
+ break;
+ case ChannelTabs.LIVESTREAMS:
+ playlistId = "UULV" + channelIdWithoutUc;
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Only Videos, Shorts and Livestreams tabs can extracted as playlists");
+ }
+
+ try {
+ final String newUrl = YoutubePlaylistLinkHandlerFactory.getInstance()
+ .getUrl(playlistId);
+ return new ListLinkHandler(newUrl, newUrl, playlistId, List.of(), "");
+ } catch (final ParsingException e) {
+ // This should be not reachable, as the given playlist ID should be valid and
+ // YoutubePlaylistLinkHandlerFactory doesn't throw any exception
+ throw new SystemPlaylistUrlCreationException(
+ "Could not create a YouTube playlist from a valid playlist ID", e);
+ }
+ }
+
+ /**
+ * Exception thrown when a YouTube system playlist URL could not be created.
+ *
+ *
+ * This exception should be never thrown, as given playlist IDs should be always valid.
+ *
+ */
+ public static final class SystemPlaylistUrlCreationException extends RuntimeException {
+ SystemPlaylistUrlCreationException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java
index ef09a505a..c3a7707dc 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java
@@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
+import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
@@ -22,10 +23,15 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public String getThumbnailUrl() throws ParsingException {
try {
- final String url = playlistInfoItem.getArray("thumbnails").getObject(0)
- .getArray("thumbnails").getObject(0).getString("url");
+ JsonArray thumbnails = playlistInfoItem.getArray("thumbnails")
+ .getObject(0)
+ .getArray("thumbnails");
+ if (thumbnails.isEmpty()) {
+ thumbnails = playlistInfoItem.getObject("thumbnail")
+ .getArray("thumbnails");
+ }
- return fixThumbnailUrl(url);
+ return fixThumbnailUrl(thumbnails.getObject(0).getString("url"));
} catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
@@ -79,9 +85,21 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public long getStreamCount() throws ParsingException {
+ String videoCountText = playlistInfoItem.getString("videoCount");
+ if (videoCountText == null) {
+ videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountText"));
+ }
+
+ if (videoCountText == null) {
+ videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountShortText"));
+ }
+
+ if (videoCountText == null) {
+ throw new ParsingException("Could not get stream count");
+ }
+
try {
- return Long.parseLong(Utils.removeNonDigitCharacters(
- playlistInfoItem.getString("videoCount")));
+ return Long.parseLong(Utils.removeNonDigitCharacters(videoCountText));
} catch (final Exception e) {
throw new ParsingException("Could not get stream count", e);
}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java
new file mode 100644
index 000000000..1b68c2ddf
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java
@@ -0,0 +1,147 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.localization.TimeAgoParser;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
+import org.schabi.newpipe.extractor.stream.StreamType;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+/**
+ * A {@link StreamInfoItemExtractor} for YouTube's {@code reelItemRenderers}.
+ *
+ *
+ * {@code reelItemRenderers} are returned on YouTube for their short-form contents on almost every
+ * place and every major client. They provide a limited amount of information and do not provide
+ * the exact view count, any uploader info (name, URL, avatar, verified status) and the upload date.
+ *
+ */
+public class YoutubeReelInfoItemExtractor implements StreamInfoItemExtractor {
+
+ @Nonnull
+ private final JsonObject reelInfo;
+ @Nullable
+ private final TimeAgoParser timeAgoParser;
+
+ public YoutubeReelInfoItemExtractor(@Nonnull final JsonObject reelInfo,
+ @Nullable final TimeAgoParser timeAgoParser) {
+ this.reelInfo = reelInfo;
+ this.timeAgoParser = timeAgoParser;
+ }
+
+ @Override
+ public String getName() throws ParsingException {
+ return getTextFromObject(reelInfo.getObject("headline"));
+ }
+
+ @Override
+ public String getUrl() throws ParsingException {
+ try {
+ final String videoId = reelInfo.getString("videoId");
+ return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId);
+ } catch (final Exception e) {
+ throw new ParsingException("Could not get URL", e);
+ }
+ }
+
+ @Override
+ public String getThumbnailUrl() throws ParsingException {
+ return getThumbnailUrlFromInfoItem(reelInfo);
+ }
+
+ @Override
+ public StreamType getStreamType() throws ParsingException {
+ return StreamType.VIDEO_STREAM;
+ }
+
+ @Override
+ public long getDuration() throws ParsingException {
+ // Duration of reelItems is only provided in the accessibility data
+ // example: "VIDEO TITLE - 49 seconds - play video"
+ // "VIDEO TITLE - 1 minute, 1 second - play video"
+ final String accessibilityLabel = reelInfo.getObject("accessibility")
+ .getObject("accessibilityData").getString("label");
+ if (accessibilityLabel == null || timeAgoParser == null) {
+ return 0;
+ }
+
+ // This approach may be language dependent
+ final String[] labelParts = accessibilityLabel.split(" [\u2013-] ");
+
+ if (labelParts.length > 2) {
+ final String textualDuration = labelParts[labelParts.length - 2];
+ return timeAgoParser.parseDuration(textualDuration);
+ }
+
+ return -1;
+ }
+
+ @Override
+ public long getViewCount() throws ParsingException {
+ final String viewCountText = getTextFromObject(reelInfo.getObject("viewCountText"));
+ if (!isNullOrEmpty(viewCountText)) {
+ // This approach is language dependent
+ if (viewCountText.toLowerCase().contains("no views")) {
+ return 0;
+ }
+
+ return Utils.mixedNumberWordToLong(viewCountText);
+ }
+
+ throw new ParsingException("Could not get short view count");
+ }
+
+ @Override
+ public boolean isShortFormContent() throws ParsingException {
+ return true;
+ }
+
+ // All the following properties cannot be obtained from reelItemRenderers
+
+ @Override
+ public boolean isAd() throws ParsingException {
+ return false;
+ }
+
+ @Override
+ public String getUploaderName() throws ParsingException {
+ return null;
+ }
+
+ @Override
+ public String getUploaderUrl() throws ParsingException {
+ return null;
+ }
+
+ @Nullable
+ @Override
+ public String getUploaderAvatarUrl() throws ParsingException {
+ return null;
+ }
+
+ @Override
+ public boolean isUploaderVerified() throws ParsingException {
+ return false;
+ }
+
+ @Nullable
+ @Override
+ public String getTextualUploadDate() throws ParsingException {
+ return null;
+ }
+
+ @Nullable
+ @Override
+ public DateWrapper getUploadDate() throws ParsingException {
+ return null;
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java
index ba370e49b..1273c7a3e 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java
@@ -58,7 +58,8 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto
@Override
public String getUrl(final String id,
final List contentFilters,
- final String searchFilter) {
+ final String searchFilter)
+ throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/" + id;
}
@@ -84,7 +85,7 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto
}
@Override
- public String getId(final String url) throws ParsingException {
+ public String getId(final String url) throws ParsingException, UnsupportedOperationException {
try {
final URL urlObj = Utils.stringToURL(url);
String path = urlObj.getPath();
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java
new file mode 100644
index 000000000..e83808a63
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java
@@ -0,0 +1,73 @@
+package org.schabi.newpipe.extractor.services.youtube.linkHandler;
+
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.UnsupportedTabException;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFactory {
+ private static final YoutubeChannelTabLinkHandlerFactory INSTANCE =
+ new YoutubeChannelTabLinkHandlerFactory();
+
+ private YoutubeChannelTabLinkHandlerFactory() {
+ }
+
+ public static YoutubeChannelTabLinkHandlerFactory getInstance() {
+ return INSTANCE;
+ }
+
+ @Nonnull
+ public static String getUrlSuffix(@Nonnull final String tab)
+ throws UnsupportedTabException {
+ switch (tab) {
+ case ChannelTabs.VIDEOS:
+ return "/videos";
+ case ChannelTabs.SHORTS:
+ return "/shorts";
+ case ChannelTabs.LIVESTREAMS:
+ return "/streams";
+ case ChannelTabs.PLAYLISTS:
+ return "/playlists";
+ case ChannelTabs.CHANNELS:
+ return "/channels";
+ }
+ throw new UnsupportedTabException(tab);
+ }
+
+ @Override
+ public String getUrl(final String id,
+ final List contentFilter,
+ final String sortFilter)
+ throws ParsingException, UnsupportedOperationException {
+ return "https://www.youtube.com/" + id + getUrlSuffix(contentFilter.get(0));
+ }
+
+ @Override
+ public String getId(final String url) throws ParsingException, UnsupportedOperationException {
+ return YoutubeChannelLinkHandlerFactory.getInstance().getId(url);
+ }
+
+ @Override
+ public boolean onAcceptUrl(final String url) throws ParsingException {
+ try {
+ getId(url);
+ } catch (final ParsingException e) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String[] getAvailableContentFilter() {
+ return new String[] {
+ ChannelTabs.VIDEOS,
+ ChannelTabs.SHORTS,
+ ChannelTabs.LIVESTREAMS,
+ ChannelTabs.PLAYLISTS,
+ ChannelTabs.CHANNELS
+ };
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java
index a15801b22..0af682f87 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java
@@ -19,13 +19,14 @@ public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFact
}
@Override
- public String getUrl(final String id) {
+ public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/watch?v=" + id;
}
@Override
- public String getId(final String urlString) throws ParsingException, IllegalArgumentException {
- // we need the same id, avoids duplicate code
+ public String getId(final String urlString)
+ throws ParsingException, UnsupportedOperationException {
+ // We need the same id, avoids duplicate code
return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString);
}
@@ -44,7 +45,8 @@ public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFact
@Override
public String getUrl(final String id,
final List contentFilter,
- final String sortFilter) throws ParsingException {
+ final String sortFilter)
+ throws ParsingException, UnsupportedOperationException {
return getUrl(id);
}
}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java
index 6b8c40be9..f6d2e10e9 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java
@@ -26,12 +26,13 @@ public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFact
@Override
public String getUrl(final String id, final List contentFilters,
- final String sortFilter) {
+ final String sortFilter)
+ throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/playlist?list=" + id;
}
@Override
- public String getId(final String url) throws ParsingException {
+ public String getId(final String url) throws ParsingException, UnsupportedOperationException {
try {
final URL urlObj = Utils.stringToURL(url);
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java
index 3c2bc7073..44b036be6 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java
@@ -13,6 +13,9 @@ import javax.annotation.Nonnull;
public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
+ private static final YoutubeSearchQueryHandlerFactory INSTANCE =
+ new YoutubeSearchQueryHandlerFactory();
+
public static final String ALL = "all";
public static final String VIDEOS = "videos";
public static final String CHANNELS = "channels";
@@ -29,20 +32,18 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa
@Nonnull
public static YoutubeSearchQueryHandlerFactory getInstance() {
- return new YoutubeSearchQueryHandlerFactory();
+ return INSTANCE;
}
@Override
public String getUrl(final String searchString,
@Nonnull final List contentFilters,
- final String sortFilter) throws ParsingException {
+ final String sortFilter)
+ throws ParsingException, UnsupportedOperationException {
try {
if (!contentFilters.isEmpty()) {
final String contentFilter = contentFilters.get(0);
switch (contentFilter) {
- case ALL:
- default:
- break;
case VIDEOS:
return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAQ%253D%253D";
case CHANNELS:
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java
index 0872bbe2d..e5e39aa35 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java
@@ -79,7 +79,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
@Nonnull
@Override
- public String getUrl(final String id) {
+ public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/watch?v=" + id;
}
@@ -87,7 +87,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
@Nonnull
@Override
public String getId(final String theUrlString)
- throws ParsingException, IllegalArgumentException {
+ throws ParsingException, UnsupportedOperationException {
String urlString = theUrlString;
try {
final URI uri = new URI(urlString);
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java
index 82f6bb26c..305eeb6a5 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java
@@ -1,28 +1,29 @@
-package org.schabi.newpipe.extractor.services.youtube.linkHandler;
-
/*
* Created by Christian Schabesberger on 12.08.17.
*
* Copyright (C) Christian Schabesberger 2018
- * YoutubeTrendingLinkHandlerFactory.java is part of NewPipe.
+ * YoutubeTrendingLinkHandlerFactory.java is part of NewPipe Extractor.
*
- * NewPipe is free software: you can redistribute it and/or modify
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
- * NewPipe is distributed in the hope that it will be useful,
+ * NewPipe Extractor is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
- * along with NewPipe. If not, see .
+ * along with NewPipe Extractor. If not, see .
*/
+package org.schabi.newpipe.extractor.services.youtube.linkHandler;
+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidiousURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
@@ -30,16 +31,27 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
-public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
+public final class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
+
+ private static final YoutubeTrendingLinkHandlerFactory INSTANCE =
+ new YoutubeTrendingLinkHandlerFactory();
+
+ private YoutubeTrendingLinkHandlerFactory() {
+ }
+
+ public static YoutubeTrendingLinkHandlerFactory getInstance() {
+ return INSTANCE;
+ }
public String getUrl(final String id,
final List contentFilters,
- final String sortFilter) {
+ final String sortFilter)
+ throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/feed/trending";
}
@Override
- public String getId(final String url) {
+ public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return "Trending";
}