From 7366eab156cc36dd53837611eed5667454944f20 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 14 Jul 2023 23:46:48 +0200 Subject: [PATCH] [YouTube] Add support for channel tabs and tags and age-restricted channels Support of tags and videos, shorts, live, playlists and channels tabs has been added for non-age restricted channels. Age-restricted channels are now also supported and always returned the videos, shorts and live tabs, accessible using system playlists. These tabs are the only ones which can be accessed using YouTube's desktop website without being logged-in. The videos channel tab parameter has been updated to the one used by the desktop website and when a channel extraction is fetched, this tab is returned in the list of tabs as a cached one in the corresponding link handler. Visitor data support per request has been added, as a valid visitor data is required to fetch continuations with contents on the shorts tab. It is only used in this case to enhance privacy. A dedicated shorts UI elements (reelItemRenderers) extractor has been added, YoutubeReelInfoItemExtractor. These elements do not provide the exact view count, any uploader info (name, URL, avatar, verified status) and the upload date. All service's LinkHandlers are now using the singleton pattern and some code has been also improved on the files changed. Co-authored-by: ThetaDev Co-authored-by: Stypox --- .../youtube/YoutubeChannelHelper.java | 271 +++++++++ .../youtube/YoutubeParsingHelper.java | 20 +- .../services/youtube/YoutubeService.java | 23 +- .../extractors/YoutubeChannelExtractor.java | 573 +++++++----------- .../YoutubeChannelTabExtractor.java | 486 +++++++++++++++ .../YoutubeChannelTabPlaylistExtractor.java | 191 ++++++ .../YoutubePlaylistInfoItemExtractor.java | 28 +- .../YoutubeReelInfoItemExtractor.java | 147 +++++ .../YoutubeChannelLinkHandlerFactory.java | 5 +- .../YoutubeChannelTabLinkHandlerFactory.java | 73 +++ .../YoutubeCommentsLinkHandlerFactory.java | 10 +- .../YoutubePlaylistLinkHandlerFactory.java | 5 +- .../YoutubeSearchQueryHandlerFactory.java | 11 +- .../YoutubeStreamLinkHandlerFactory.java | 4 +- .../YoutubeTrendingLinkHandlerFactory.java | 30 +- 15 files changed, 1498 insertions(+), 379 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabPlaylistExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java new file mode 100644 index 000000000..9adee6191 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java @@ -0,0 +1,271 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonWriter; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.localization.ContentCountry; +import org.schabi.newpipe.extractor.localization.Localization; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +/** + * Shared functions for extracting YouTube channel pages and tabs. + */ +public final class YoutubeChannelHelper { + private YoutubeChannelHelper() { + } + + /** + * Take a YouTube channel ID or URL path, resolve it if necessary and return a channel ID. + * + * @param idOrPath a YouTube channel ID or URL path + * @return a YouTube channel ID + * @throws IOException if a channel resolve request failed + * @throws ExtractionException if a channel resolve request response could not be parsed or is + * invalid + */ + @Nonnull + public static String resolveChannelId(@Nonnull final String idOrPath) + throws ExtractionException, IOException { + final String[] channelId = idOrPath.split("/"); + + if (channelId[0].startsWith("UC")) { + return channelId[0]; + } + + // If the URL is not a /channel URL, we need to use the navigation/resolve_url endpoint of + // the InnerTube API to get the channel id. + // Otherwise, we couldn't get information about the channel associated with this URL, if + // there is one. + if (!channelId[0].equals("channel")) { + final byte[] body = JsonWriter.string( + prepareDesktopJsonBuilder(Localization.DEFAULT, ContentCountry.DEFAULT) + .value("url", "https://www.youtube.com/" + idOrPath) + .done()) + .getBytes(StandardCharsets.UTF_8); + + final JsonObject jsonResponse = getJsonPostResponse( + "navigation/resolve_url", body, Localization.DEFAULT); + + checkIfChannelResponseIsValid(jsonResponse); + + final JsonObject endpoint = jsonResponse.getObject("endpoint"); + + final String webPageType = endpoint.getObject("commandMetadata") + .getObject("webCommandMetadata") + .getString("webPageType", ""); + + final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint"); + final String browseId = browseEndpoint.getString("browseId", ""); + + if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") + || webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL") + && !browseId.isEmpty()) { + if (!browseId.startsWith("UC")) { + throw new ExtractionException("Redirected id is not pointing to a channel"); + } + + return browseId; + } + } + + return channelId[1]; + } + + /** + * Response data object for {@link #getChannelResponse(String, String, Localization, + * ContentCountry)}, after any redirection in the allowed redirects count ({@code 3}). + */ + public static final class ChannelResponseData { + + /** + * The channel response as a JSON object, after all redirects. + */ + @Nonnull + public final JsonObject jsonResponse; + + /** + * The channel ID after all redirects. + */ + @Nonnull + public final String channelId; + + private ChannelResponseData(@Nonnull final JsonObject jsonResponse, + @Nonnull final String channelId) { + this.jsonResponse = jsonResponse; + this.channelId = channelId; + } + } + + /** + * Fetch a YouTube channel tab response, using the given channel ID and tab parameters. + * + *

+ * Redirections to other channels such as are supported to up to 3 redirects, which could + * happen for instance for localized channels or auto-generated ones such as the {@code Movies + * and Shows} (channel IDs {@code UCuJcl0Ju-gPDoksRjK1ya-w}, {@code UChBfWrfBXL9wS6tQtgjt_OQ} + * and {@code UCok7UTQQEP1Rsctxiv3gwSQ} of this channel redirect to the + * {@code UClgRkhTL3_hImCAmdLfDE4g} one). + *

+ * + * @param channelId a valid YouTube channel ID + * @param parameters the parameters to specify the YouTube channel tab; if invalid ones are + * specified, YouTube should return the {@code Home} tab + * @param localization the {@link Localization} to use + * @param country the {@link ContentCountry} to use + * @return a {@link ChannelResponseData channel response data} + * @throws IOException if a channel request failed + * @throws ExtractionException if a channel request response could not be parsed or is invalid + */ + @Nonnull + public static ChannelResponseData getChannelResponse(@Nonnull final String channelId, + @Nonnull final String parameters, + @Nonnull final Localization localization, + @Nonnull final ContentCountry country) + throws ExtractionException, IOException { + String id = channelId; + JsonObject ajaxJson = null; + + int level = 0; + while (level < 3) { + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( + localization, country) + .value("browseId", id) + .value("params", parameters) + .done()) + .getBytes(StandardCharsets.UTF_8); + + final JsonObject jsonResponse = getJsonPostResponse( + "browse", body, localization); + + checkIfChannelResponseIsValid(jsonResponse); + + final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions") + .getObject(0) + .getObject("navigateAction") + .getObject("endpoint"); + + final String webPageType = endpoint.getObject("commandMetadata") + .getObject("webCommandMetadata") + .getString("webPageType", ""); + + final String browseId = endpoint.getObject("browseEndpoint") + .getString("browseId", ""); + + if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") + || webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL") + && !browseId.isEmpty()) { + if (!browseId.startsWith("UC")) { + throw new ExtractionException("Redirected id is not pointing to a channel"); + } + + id = browseId; + level++; + } else { + ajaxJson = jsonResponse; + break; + } + } + + if (ajaxJson == null) { + throw new ExtractionException("Got no channel response"); + } + + defaultAlertsCheck(ajaxJson); + + return new ChannelResponseData(ajaxJson, id); + } + + /** + * Assert that a channel JSON response does not contain an {@code error} JSON object. + * + * @param jsonResponse a channel JSON response + * @throws ContentNotAvailableException if the channel was not found + */ + private static void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse) + throws ContentNotAvailableException { + if (!isNullOrEmpty(jsonResponse.getObject("error"))) { + final JsonObject errorJsonObject = jsonResponse.getObject("error"); + final int errorCode = errorJsonObject.getInt("code"); + if (errorCode == 404) { + throw new ContentNotAvailableException("This channel doesn't exist."); + } else { + throw new ContentNotAvailableException("Got error:\"" + + errorJsonObject.getString("status") + "\": " + + errorJsonObject.getString("message")); + } + } + } + + /** + * A channel header response. + * + *

+ * This class allows the distinction between a classic header and a carousel one, used for + * auto-generated ones like the gaming or music topic channels and for big events such as the + * Coachella music festival, which have a different data structure and do not return the same + * properties. + *

+ */ + public static final class ChannelHeader { + + /** + * The channel header JSON response. + */ + @Nonnull + public final JsonObject json; + + /** + * Whether the header is a {@code carouselHeaderRenderer}. + * + *

+ * See the class documentation for more details. + *

+ */ + public final boolean isCarouselHeader; + + private ChannelHeader(@Nonnull final JsonObject json, final boolean isCarouselHeader) { + this.json = json; + this.isCarouselHeader = isCarouselHeader; + } + } + + /** + * Get a channel header as an {@link Optional} it if exists. + * + * @param channelResponse a full channel JSON response + * @return an {@link Optional} containing a {@link ChannelHeader} or an empty {@link Optional} + * if no supported header has been found + */ + @Nonnull + public static Optional getChannelHeader( + @Nonnull final JsonObject channelResponse) { + final JsonObject header = channelResponse.getObject("header"); + + if (header.has("c4TabbedHeaderRenderer")) { + return Optional.of(header.getObject("c4TabbedHeaderRenderer")) + .map(json -> new ChannelHeader(json, false)); + } else if (header.has("carouselHeaderRenderer")) { + return header.getObject("carouselHeaderRenderer") + .getArray("contents") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(item -> item.has("topicChannelDetailsRenderer")) + .findFirst() + .map(item -> item.getObject("topicChannelDetailsRenderer")) + .map(json -> new ChannelHeader(json, true)); + } else { + return Optional.empty(); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 74fee3336..aec550897 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -1230,8 +1230,17 @@ public final class YoutubeParsingHelper { @Nonnull final Localization localization, @Nonnull final ContentCountry contentCountry) throws IOException, ExtractionException { + return prepareDesktopJsonBuilder(localization, contentCountry, null); + } + + @Nonnull + public static JsonBuilder prepareDesktopJsonBuilder( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nullable final String visitorData) + throws IOException, ExtractionException { // @formatter:off - return JsonObject.builder() + final JsonBuilder builder = JsonObject.builder() .object("context") .object("client") .value("hl", localization.getLocalizationCode()) @@ -1239,8 +1248,13 @@ public final class YoutubeParsingHelper { .value("clientName", "WEB") .value("clientVersion", getClientVersion()) .value("originalUrl", "https://www.youtube.com") - .value("platform", "DESKTOP") - .end() + .value("platform", "DESKTOP"); + + if (visitorData != null) { + builder.value("visitorData", visitorData); + } + + return builder.end() .object("request") .array("internalExperimentFlags") .end() diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 7470971d5..599b9a5c1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -8,6 +8,7 @@ import static java.util.Arrays.asList; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.feed.FeedExtractor; @@ -16,6 +17,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.localization.ContentCountry; @@ -23,6 +25,7 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; @@ -34,6 +37,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscript import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; @@ -88,6 +92,11 @@ public class YoutubeService extends StreamingService { return YoutubeChannelLinkHandlerFactory.getInstance(); } + @Override + public ListLinkHandlerFactory getChannelTabLHFactory() { + return YoutubeChannelTabLinkHandlerFactory.getInstance(); + } + @Override public ListLinkHandlerFactory getPlaylistLHFactory() { return YoutubePlaylistLinkHandlerFactory.getInstance(); @@ -108,6 +117,15 @@ public class YoutubeService extends StreamingService { return new YoutubeChannelExtractor(this, linkHandler); } + @Override + public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) { + if (linkHandler instanceof ReadyChannelTabListLinkHandler) { + return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this); + } else { + return new YoutubeChannelTabExtractor(this, linkHandler); + } + } + @Override public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) { @@ -136,16 +154,17 @@ public class YoutubeService extends StreamingService { @Override public KioskList getKioskList() throws ExtractionException { final KioskList list = new KioskList(this); + final ListLinkHandlerFactory h = YoutubeTrendingLinkHandlerFactory.getInstance(); // add kiosks here e.g.: try { list.addKioskEntry( (streamingService, url, id) -> new YoutubeTrendingExtractor( YoutubeService.this, - new YoutubeTrendingLinkHandlerFactory().fromUrl(url), + h.fromUrl(url), id ), - new YoutubeTrendingLinkHandlerFactory(), + h, YoutubeTrendingExtractor.KIOSK_ID ); list.setDefaultKiosk(YoutubeTrendingExtractor.KIOSK_ID); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 0b599f3c5..02709975a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -1,39 +1,35 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonWriter; -import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; -import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler; +import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor.VideosTabExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.Optional; +import java.util.function.Consumer; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -59,22 +55,24 @@ import javax.annotation.Nullable; */ public class YoutubeChannelExtractor extends ChannelExtractor { - private JsonObject initialData; - private Optional channelHeader; - private boolean isCarouselHeader = false; - private JsonObject videoTab; + + private JsonObject jsonResponse; + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + private Optional channelHeader; + + private String channelId; /** - * Some channels have response redirects and the only way to reliably get the id is by saving it + * If a channel is age-restricted, its pages are only accessible to logged-in and + * age-verified users, we get an {@code channelAgeGateRenderer} in this case, containing only + * the following metadata: channel name and channel avatar. + * *

- * "Movies & Shows": - * 

-     * UCuJcl0Ju-gPDoksRjK1ya-w ┐
-     * UChBfWrfBXL9wS6tQtgjt_OQ ├ UClgRkhTL3_hImCAmdLfDE4g
-     * UCok7UTQQEP1Rsctxiv3gwSQ ┘
-     *
+ * This restriction doesn't seem to apply to all countries. + *

*/ - private String redirectedChannelId; + @Nullable + private JsonObject channelAgeGateRenderer; public YoutubeChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) { @@ -85,132 +83,42 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { final String channelPath = super.getId(); - final String[] channelId = channelPath.split("/"); - String id = ""; - // If the url is an URL which is not a /channel URL, we need to use the - // navigation/resolve_url endpoint of the InnerTube API to get the channel id. Otherwise, - // we couldn't get information about the channel associated with this URL, if there is one. - if (!channelId[0].equals("channel")) { - final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( - getExtractorLocalization(), getExtractorContentCountry()) - .value("url", "https://www.youtube.com/" + channelPath) - .done()) - .getBytes(StandardCharsets.UTF_8); + final String id = resolveChannelId(channelPath); + // Fetch Videos tab + final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(id, + "EgZ2aWRlb3PyBgQKAjoA", getExtractorLocalization(), getExtractorContentCountry()); - final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url", - body, getExtractorLocalization()); - - checkIfChannelResponseIsValid(jsonResponse); - - final JsonObject endpoint = jsonResponse.getObject("endpoint"); - - final String webPageType = endpoint.getObject("commandMetadata") - .getObject("webCommandMetadata") - .getString("webPageType", ""); - - final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint"); - final String browseId = browseEndpoint.getString("browseId", ""); - - if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") - || webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL") - && !browseId.isEmpty()) { - if (!browseId.startsWith("UC")) { - throw new ExtractionException("Redirected id is not pointing to a channel"); - } - - id = browseId; - redirectedChannelId = browseId; - } - } else { - id = channelId[1]; - } - JsonObject ajaxJson = null; - - int level = 0; - while (level < 3) { - final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( - getExtractorLocalization(), getExtractorContentCountry()) - .value("browseId", id) - .value("params", "EgZ2aWRlb3M%3D") // Equal to videos - .done()) - .getBytes(StandardCharsets.UTF_8); - - final JsonObject jsonResponse = getJsonPostResponse("browse", body, - getExtractorLocalization()); - - checkIfChannelResponseIsValid(jsonResponse); - - final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions") - .getObject(0) - .getObject("navigateAction") - .getObject("endpoint"); - - final String webPageType = endpoint.getObject("commandMetadata") - .getObject("webCommandMetadata") - .getString("webPageType", ""); - - final String browseId = endpoint.getObject("browseEndpoint").getString("browseId", - ""); - - if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") - || webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL") - && !browseId.isEmpty()) { - if (!browseId.startsWith("UC")) { - throw new ExtractionException("Redirected id is not pointing to a channel"); - } - - id = browseId; - redirectedChannelId = browseId; - level++; - } else { - ajaxJson = jsonResponse; - break; - } - } - - if (ajaxJson == null) { - throw new ExtractionException("Could not fetch initial JSON data"); - } - - initialData = ajaxJson; - YoutubeParsingHelper.defaultAlertsCheck(initialData); + jsonResponse = data.jsonResponse; + channelId = data.channelId; + channelAgeGateRenderer = getChannelAgeGateRenderer(); } - private void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse) - throws ContentNotAvailableException { - if (!isNullOrEmpty(jsonResponse.getObject("error"))) { - final JsonObject errorJsonObject = jsonResponse.getObject("error"); - final int errorCode = errorJsonObject.getInt("code"); - if (errorCode == 404) { - throw new ContentNotAvailableException("This channel doesn't exist."); - } else { - throw new ContentNotAvailableException("Got error:\"" - + errorJsonObject.getString("status") + "\": " - + errorJsonObject.getString("message")); - } - } - } - - @Nonnull - private Optional getChannelHeader() { - if (channelHeader == null) { - final JsonObject h = initialData.getObject("header"); - - if (h.has("c4TabbedHeaderRenderer")) { - channelHeader = Optional.of(h.getObject("c4TabbedHeaderRenderer")); - } else if (h.has("carouselHeaderRenderer")) { - isCarouselHeader = true; - channelHeader = h.getObject("carouselHeaderRenderer") + @Nullable + private JsonObject getChannelAgeGateRenderer() { + return jsonResponse.getObject("contents") + .getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .flatMap(tab -> tab.getObject("tabRenderer") + .getObject("content") + .getObject("sectionListRenderer") .getArray("contents") .stream() .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .filter(itm -> itm.has("topicChannelDetailsRenderer")) - .findFirst() - .map(itm -> itm.getObject("topicChannelDetailsRenderer")); - } else { - channelHeader = Optional.empty(); - } + .map(JsonObject.class::cast)) + .filter(content -> content.has("channelAgeGateRenderer")) + .map(content -> content.getObject("channelAgeGateRenderer")) + .findFirst() + .orElse(null); + } + + @Nonnull + private Optional getChannelHeader() { + //noinspection OptionalAssignedToNull + if (channelHeader == null) { + channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse); } return channelHeader; } @@ -229,57 +137,70 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getId() throws ParsingException { return getChannelHeader() - .flatMap(header -> Optional.ofNullable(header.getString("channelId")).or( - () -> Optional.ofNullable(header.getObject("navigationEndpoint") + .flatMap(header -> Optional.ofNullable(header.json.getString("channelId")) + .or(() -> Optional.ofNullable(header.json.getObject("navigationEndpoint") .getObject("browseEndpoint") .getString("browseId")) )) - .or(() -> Optional.ofNullable(redirectedChannelId)) - .orElseThrow(() -> new ParsingException("Could not get channel id")); + .or(() -> Optional.ofNullable(channelId)) + .orElseThrow(() -> new ParsingException("Could not get channel ID")); } @Nonnull @Override public String getName() throws ParsingException { - final String mdName = initialData.getObject("metadata") - .getObject("channelMetadataRenderer") - .getString("title"); - if (!isNullOrEmpty(mdName)) { - return mdName; + if (channelAgeGateRenderer != null) { + return channelAgeGateRenderer.getString("channelTitle"); } - final Optional header = getChannelHeader(); - if (header.isPresent()) { - final Object title = header.get().get("title"); + final String metadataRendererTitle = jsonResponse.getObject("metadata") + .getObject("channelMetadataRenderer") + .getString("title"); + if (!isNullOrEmpty(metadataRendererTitle)) { + return metadataRendererTitle; + } + + return getChannelHeader().flatMap(header -> { + final Object title = header.json.get("title"); if (title instanceof String) { - return (String) title; + return Optional.of((String) title); } else if (title instanceof JsonObject) { final String headerName = getTextFromObject((JsonObject) title); if (!isNullOrEmpty(headerName)) { - return headerName; + return Optional.of(headerName); } } - } - - throw new ParsingException("Could not get channel name"); + return Optional.empty(); + }).orElseThrow(() -> new ParsingException("Could not get channel name")); } @Override public String getAvatarUrl() throws ParsingException { - return getChannelHeader().flatMap(header -> Optional.ofNullable( - header.getObject("avatar").getArray("thumbnails") - .getObject(0).getString("url") - )) - .map(YoutubeParsingHelper::fixThumbnailUrl) - .orElseThrow(() -> new ParsingException("Could not get avatar")); + final JsonObject avatarJsonObjectContainer; + if (channelAgeGateRenderer != null) { + avatarJsonObjectContainer = channelAgeGateRenderer; + } else { + avatarJsonObjectContainer = getChannelHeader().map(header -> header.json) + .orElseThrow(() -> new ParsingException("Could not get avatar URL")); + } + + return YoutubeParsingHelper.fixThumbnailUrl(avatarJsonObjectContainer.getObject("avatar") + .getArray("thumbnails") + .getObject(0) + .getString("url")); } @Override public String getBannerUrl() throws ParsingException { + if (channelAgeGateRenderer != null) { + return ""; + } + return getChannelHeader().flatMap(header -> Optional.ofNullable( - header.getObject("banner").getArray("thumbnails") - .getObject(0).getString("url") - )) + header.json.getObject("banner") + .getArray("thumbnails") + .getObject(0) + .getString("url"))) .filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner")) .map(YoutubeParsingHelper::fixThumbnailUrl) // Channels may not have a banner, so no exception should be thrown if no banner is @@ -290,6 +211,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getFeedUrl() throws ParsingException { + // RSS feeds are accessible for age-restricted channels, no need to check whether a channel + // has a channelAgeGateRenderer try { return YoutubeParsingHelper.getFeedUrlFrom(getId()); } catch (final Exception e) { @@ -299,14 +222,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - final Optional header = getChannelHeader(); - if (header.isPresent()) { + if (channelAgeGateRenderer != null) { + return UNKNOWN_SUBSCRIBER_COUNT; + } + + final Optional headerOpt = getChannelHeader(); + if (headerOpt.isPresent()) { + final JsonObject header = headerOpt.get().json; JsonObject textObject = null; - if (header.get().has("subscriberCountText")) { - textObject = header.get().getObject("subscriberCountText"); - } else if (header.get().has("subtitle")) { - textObject = header.get().getObject("subtitle"); + if (header.has("subscriberCountText")) { + textObject = header.getObject("subscriberCountText"); + } else if (header.has("subtitle")) { + textObject = header.getObject("subtitle"); } if (textObject != null) { @@ -317,13 +245,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } } } + return UNKNOWN_SUBSCRIBER_COUNT; } @Override public String getDescription() throws ParsingException { + if (channelAgeGateRenderer != null) { + return null; + } + try { - return initialData.getObject("metadata").getObject("channelMetadataRenderer") + return jsonResponse.getObject("metadata") + .getObject("channelMetadataRenderer") .getString("description"); } catch (final Exception e) { throw new ParsingException("Could not get channel description", e); @@ -347,190 +281,139 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public boolean isVerified() throws ParsingException { - // The CarouselHeaderRenderer does not contain any verification badges. - // Since it is only shown on YT-internal channels or on channels of large organizations - // broadcasting live events, we can assume the channel to be verified. - if (isCarouselHeader) { - return true; + if (channelAgeGateRenderer != null) { + return false; } - return getChannelHeader() - .map(header -> header.getArray("badges")) - .map(YoutubeParsingHelper::isVerified) - .orElse(false); + final Optional headerOpt = getChannelHeader(); + if (headerOpt.isPresent()) { + final YoutubeChannelHelper.ChannelHeader header = headerOpt.get(); + + // The CarouselHeaderRenderer does not contain any verification badges. + // Since it is only shown on YT-internal channels or on channels of large organizations + // broadcasting live events, we can assume the channel to be verified. + if (header.isCarouselHeader) { + return true; + } + return YoutubeParsingHelper.isVerified(header.json.getArray("badges")); + } + return false; } @Nonnull @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - - Page nextPage = null; - - if (getVideoTab() != null) { - final JsonObject tabContent = getVideoTab().getObject("content"); - JsonArray items = tabContent - .getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer") - .getArray("contents").getObject(0).getObject("gridRenderer").getArray("items"); - - if (items.isEmpty()) { - items = tabContent.getObject("richGridRenderer").getArray("contents"); - } - - final List channelIds = new ArrayList<>(); - channelIds.add(getName()); - channelIds.add(getUrl()); - final JsonObject continuation = collectStreamsFrom(collector, items, channelIds); - - nextPage = getNextPageFrom(continuation, channelIds); + public List getTabs() throws ParsingException { + if (channelAgeGateRenderer == null) { + return getTabsForNonAgeRestrictedChannels(); } - return new InfoItemsPage<>(collector, nextPage); + return getTabsForAgeRestrictedChannels(); } - @Override - public InfoItemsPage getPage(final Page page) - throws IOException, ExtractionException { - if (page == null || isNullOrEmpty(page.getUrl())) { - throw new IllegalArgumentException("Page doesn't contain an URL"); - } - - final List channelIds = page.getIds(); - - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - - final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(), - getExtractorLocalization()); - - final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions") - .getObject(0) - .getObject("appendContinuationItemsAction"); - - final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation - .getArray("continuationItems"), channelIds); - - return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds)); - } - - @Nullable - private Page getNextPageFrom(final JsonObject continuations, - final List channelIds) - throws IOException, ExtractionException { - if (isNullOrEmpty(continuations)) { - return null; - } - - final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint"); - final String continuation = continuationEndpoint.getObject("continuationCommand") - .getString("token"); - - final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), - getExtractorContentCountry()) - .value("continuation", continuation) - .done()) - .getBytes(StandardCharsets.UTF_8); - - return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey() - + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body); - } - - /** - * Collect streams from an array of items - * - * @param collector the collector where videos will be committed - * @param videos the array to get videos from - * @param channelIds the ids of the channel, which are its name and its URL - * @return the continuation object - */ - private JsonObject collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector, - @Nonnull final JsonArray videos, - @Nonnull final List channelIds) { - collector.reset(); - - final String uploaderName = channelIds.get(0); - final String uploaderUrl = channelIds.get(1); - final TimeAgoParser timeAgoParser = getTimeAgoParser(); - - JsonObject continuation = null; - - for (final Object object : videos) { - final JsonObject video = (JsonObject) object; - if (video.has("gridVideoRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor( - video.getObject("gridVideoRenderer"), timeAgoParser) { - @Override - public String getUploaderName() { - return uploaderName; - } - - @Override - public String getUploaderUrl() { - return uploaderUrl; - } - }); - } else if (video.has("richItemRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor( - video.getObject("richItemRenderer") - .getObject("content").getObject("videoRenderer"), timeAgoParser) { - @Override - public String getUploaderName() { - return uploaderName; - } - - @Override - public String getUploaderUrl() { - return uploaderUrl; - } - }); - - } else if (video.has("continuationItemRenderer")) { - continuation = video.getObject("continuationItemRenderer"); - } - } - - return continuation; - } - - @Nullable - private JsonObject getVideoTab() throws ParsingException { - if (videoTab != null) { - return videoTab; - } - - final JsonArray tabs = initialData.getObject("contents") + @Nonnull + private List getTabsForNonAgeRestrictedChannels() throws ParsingException { + final JsonArray responseTabs = jsonResponse.getObject("contents") .getObject("twoColumnBrowseResultsRenderer") .getArray("tabs"); - final JsonObject foundVideoTab = tabs.stream() - .filter(Objects::nonNull) + final List tabs = new ArrayList<>(); + final Consumer addNonVideosTab = tabName -> { + try { + tabs.add(YoutubeChannelTabLinkHandlerFactory.getInstance().fromQuery( + channelId, List.of(tabName), "")); + } catch (final ParsingException ignored) { + // Do not add the tab if we couldn't create the LinkHandler + } + }; + + final String name = getName(); + final String url = getUrl(); + final String id = getId(); + + responseTabs.stream() .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) - .filter(tab -> tab.has("tabRenderer") - && tab.getObject("tabRenderer") - .getString("title", "") - .equals("Videos")) - .findFirst() + .filter(tab -> tab.has("tabRenderer")) .map(tab -> tab.getObject("tabRenderer")) - .orElseThrow( - () -> new ContentNotSupportedException("This channel has no Videos tab")); + .forEach(tabRenderer -> { + final String tabUrl = tabRenderer.getObject("endpoint") + .getObject("commandMetadata") + .getObject("webCommandMetadata") + .getString("url"); + if (tabUrl != null) { + final String[] urlParts = tabUrl.split("/"); + if (urlParts.length == 0) { + return; + } - final String messageRendererText = getTextFromObject( - foundVideoTab.getObject("content") - .getObject("sectionListRenderer") - .getArray("contents") - .getObject(0) - .getObject("itemSectionRenderer") - .getArray("contents") - .getObject(0) - .getObject("messageRenderer") - .getObject("text")); - if (messageRendererText != null - && messageRendererText.equals("This channel has no videos.")) { - return null; + final String urlSuffix = urlParts[urlParts.length - 1]; + + switch (urlSuffix) { + case "videos": + // Since the Videos tab has already its contents fetched, make + // sure it is in the first position + // YoutubeChannelTabExtractor still supports fetching this tab + tabs.add(0, new ReadyChannelTabListLinkHandler( + tabUrl, + channelId, + ChannelTabs.VIDEOS, + (service, linkHandler) -> new VideosTabExtractor( + service, linkHandler, tabRenderer, name, id, url))); + + break; + case "shorts": + addNonVideosTab.accept(ChannelTabs.SHORTS); + break; + case "streams": + addNonVideosTab.accept(ChannelTabs.LIVESTREAMS); + break; + case "playlists": + addNonVideosTab.accept(ChannelTabs.PLAYLISTS); + break; + case "channels": + addNonVideosTab.accept(ChannelTabs.CHANNELS); + break; + } + } + }); + + return Collections.unmodifiableList(tabs); + } + + @Nonnull + private List getTabsForAgeRestrictedChannels() throws ParsingException { + // As we don't have access to the channel tabs list, consider that the channel has videos, + // shorts and livestreams, the data only accessible without login on YouTube's desktop + // client using uploads system playlists + // The playlists channel tab is still available on YouTube Music, but this is not + // implemented in the extractor + + final List tabs = new ArrayList<>(); + final String channelUrl = getUrl(); + + final Consumer addTab = tabName -> + tabs.add(new ReadyChannelTabListLinkHandler(channelUrl + "/" + tabName, + channelId, tabName, YoutubeChannelTabPlaylistExtractor::new)); + + addTab.accept(ChannelTabs.VIDEOS); + addTab.accept(ChannelTabs.SHORTS); + addTab.accept(ChannelTabs.LIVESTREAMS); + return Collections.unmodifiableList(tabs); + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + if (channelAgeGateRenderer != null) { + return List.of(); } - videoTab = foundVideoTab; - return foundVideoTab; + return jsonResponse.getObject("microformat") + .getObject("microformatDataRenderer") + .getArray("tags") + .stream() + .filter(String.class::isInstance) + .map(String.class::cast) + .collect(Collectors.toUnmodifiableList()); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java new file mode 100644 index 000000000..d0f116af0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java @@ -0,0 +1,486 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonWriter; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor; +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +/** + * A {@link ChannelTabExtractor} implementation for the YouTube service. + * + *

+ * It currently supports {@code Videos}, {@code Shorts}, {@code Live}, {@code Playlists} and + * {@code Channels} tabs. + *

+ */ +public class YoutubeChannelTabExtractor extends ChannelTabExtractor { + + /** + * Whether the visitor data extracted from the initial channel response is required to be used + * for continuations. + * + *

+ * A valid {@code visitorData} is required to get continuations of shorts in channels. + *

+ * + *

+ * It should be not used when it is not needed, in order to reduce YouTube's tracking. + *

+ */ + private final boolean useVisitorData; + private JsonObject jsonResponse; + private String channelId; + @Nullable + private String visitorData; + + public YoutubeChannelTabExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { + super(service, linkHandler); + useVisitorData = getName().equals(ChannelTabs.SHORTS); + } + + @Nonnull + private String getChannelTabsParameters() throws ParsingException { + final String name = getName(); + switch (name) { + case ChannelTabs.VIDEOS: + return "EgZ2aWRlb3PyBgQKAjoA"; + case ChannelTabs.SHORTS: + return "EgZzaG9ydHPyBgUKA5oBAA%3D%3D"; + case ChannelTabs.LIVESTREAMS: + return "EgdzdHJlYW1z8gYECgJ6AA%3D%3D"; + case ChannelTabs.PLAYLISTS: + return "EglwbGF5bGlzdHPyBgQKAkIA"; + case ChannelTabs.CHANNELS: + return "EghjaGFubmVsc_IGBAoCUgA%3D"; + } + throw new ParsingException("Unsupported channel tab: " + name); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, + ExtractionException { + channelId = resolveChannelId(super.getId()); + + final String params = getChannelTabsParameters(); + + final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(channelId, + params, getExtractorLocalization(), getExtractorContentCountry()); + + jsonResponse = data.jsonResponse; + channelId = data.channelId; + if (useVisitorData) { + visitorData = jsonResponse.getObject("responseContext").getString("visitorData"); + } + } + + @Nonnull + @Override + public String getUrl() throws ParsingException { + try { + return YoutubeChannelTabLinkHandlerFactory.getInstance() + .getUrl("channel/" + getId(), List.of(getName()), ""); + } catch (final ParsingException e) { + return super.getUrl(); + } + } + + @Nonnull + @Override + public String getId() throws ParsingException { + final String id = jsonResponse.getObject("header") + .getObject("c4TabbedHeaderRenderer") + .getString("channelId", ""); + + if (!id.isEmpty()) { + return id; + } + + final Optional carouselHeaderId = jsonResponse.getObject("header") + .getObject("carouselHeaderRenderer") + .getArray("contents") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(item -> item.has("topicChannelDetailsRenderer")) + .findFirst() + .flatMap(item -> + Optional.ofNullable(item.getObject("topicChannelDetailsRenderer") + .getObject("navigationEndpoint") + .getObject("browseEndpoint") + .getString("browseId"))); + if (carouselHeaderId.isPresent()) { + return carouselHeaderId.get(); + } + + if (!isNullOrEmpty(channelId)) { + return channelId; + } else { + throw new ParsingException("Could not get channel ID"); + } + } + + protected String getChannelName() { + final String metadataName = jsonResponse.getObject("metadata") + .getObject("channelMetadataRenderer") + .getString("title"); + if (!isNullOrEmpty(metadataName)) { + return metadataName; + } + + return YoutubeChannelHelper.getChannelHeader(jsonResponse) + .map(header -> { + final Object title = header.json.get("title"); + if (title instanceof String) { + return (String) title; + } else if (title instanceof JsonObject) { + final String headerName = getTextFromObject((JsonObject) title); + if (!isNullOrEmpty(headerName)) { + return headerName; + } + } + return ""; + }) + .orElse(""); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); + + JsonArray items = new JsonArray(); + final Optional tab = getTabData(); + + if (tab.isPresent()) { + final JsonObject tabContent = tab.get().getObject("content"); + + items = tabContent.getObject("sectionListRenderer") + .getArray("contents") + .getObject(0) + .getObject("itemSectionRenderer") + .getArray("contents") + .getObject(0) + .getObject("gridRenderer") + .getArray("items"); + + if (items.isEmpty()) { + items = tabContent.getObject("richGridRenderer") + .getArray("contents"); + + if (items.isEmpty()) { + items = tabContent.getObject("sectionListRenderer") + .getArray("contents"); + } + } + } + + // If a channel tab is fetched, the next page requires channel ID and name, as channel + // streams don't have their channel specified. + // We also need to set the visitor data here when it should be enabled, as it is required + // to get continuations on some channel tabs, and we need a way to pass it between pages + final List channelIds = useVisitorData && !isNullOrEmpty(visitorData) + ? List.of(getChannelName(), getUrl(), visitorData) + : List.of(getChannelName(), getUrl()); + + final JsonObject continuation = collectItemsFrom(collector, items, channelIds) + .orElse(null); + + final Page nextPage = getNextPageFrom(continuation, channelIds); + + return new InfoItemsPage<>(collector, nextPage); + } + + @Override + public InfoItemsPage getPage(final Page page) + throws IOException, ExtractionException { + if (page == null || isNullOrEmpty(page.getUrl())) { + throw new IllegalArgumentException("Page doesn't contain an URL"); + } + + final List channelIds = page.getIds(); + + final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); + + final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(), + getExtractorLocalization()); + + final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(jsonObject -> jsonObject.has("appendContinuationItemsAction")) + .map(jsonObject -> jsonObject.getObject("appendContinuationItemsAction")) + .findFirst() + .orElse(new JsonObject()); + + final JsonObject continuation = collectItemsFrom(collector, + sectionListContinuation.getArray("continuationItems"), channelIds) + .orElse(null); + + return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds)); + } + + Optional getTabData() { + final String urlSuffix = YoutubeChannelTabLinkHandlerFactory.getUrlSuffix(getName()); + + return jsonResponse.getObject("contents") + .getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(tab -> tab.has("tabRenderer")) + .map(tab -> tab.getObject("tabRenderer")) + .filter(tabRenderer -> tabRenderer.getObject("endpoint") + .getObject("commandMetadata").getObject("webCommandMetadata") + .getString("url", "").endsWith(urlSuffix)) + .findFirst() + // Check if tab has no content + .filter(tabRenderer -> { + final JsonArray tabContents = tabRenderer.getObject("content") + .getObject("sectionListRenderer") + .getArray("contents") + .getObject(0) + .getObject("itemSectionRenderer") + .getArray("contents"); + return tabContents.size() != 1 + || !tabContents.getObject(0).has("messageRenderer"); + }); + } + + private Optional collectItemsFrom(@Nonnull final MultiInfoItemsCollector collector, + @Nonnull final JsonArray items, + @Nonnull final List channelIds) { + return items.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(item -> collectItem(collector, item, channelIds)) + .reduce(Optional.empty(), (c1, c2) -> c1.or(() -> c2)); + } + + private Optional collectItem(@Nonnull final MultiInfoItemsCollector collector, + @Nonnull final JsonObject item, + @Nonnull final List channelIds) { + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + if (item.has("richItemRenderer")) { + final JsonObject richItem = item.getObject("richItemRenderer") + .getObject("content"); + + if (richItem.has("videoRenderer")) { + getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept( + richItem.getObject("videoRenderer")); + } else if (richItem.has("reelItemRenderer")) { + getCommitReelItemConsumer(collector, timeAgoParser, channelIds).accept( + richItem.getObject("reelItemRenderer")); + } else if (richItem.has("playlistRenderer")) { + getCommitPlaylistConsumer(collector, channelIds).accept( + item.getObject("playlistRenderer")); + } + } else if (item.has("gridVideoRenderer")) { + getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept( + item.getObject("gridVideoRenderer")); + } else if (item.has("gridPlaylistRenderer")) { + getCommitPlaylistConsumer(collector, channelIds).accept( + item.getObject("gridPlaylistRenderer")); + } else if (item.has("gridChannelRenderer")) { + collector.commit(new YoutubeChannelInfoItemExtractor( + item.getObject("gridChannelRenderer"))); + } else if (item.has("shelfRenderer")) { + return collectItem(collector, item.getObject("shelfRenderer") + .getObject("content"), channelIds); + } else if (item.has("itemSectionRenderer")) { + return collectItemsFrom(collector, item.getObject("itemSectionRenderer") + .getArray("contents"), channelIds); + } else if (item.has("horizontalListRenderer")) { + return collectItemsFrom(collector, item.getObject("horizontalListRenderer") + .getArray("items"), channelIds); + } else if (item.has("expandedShelfContentsRenderer")) { + return collectItemsFrom(collector, item.getObject("expandedShelfContentsRenderer") + .getArray("items"), channelIds); + } else if (item.has("continuationItemRenderer")) { + return Optional.ofNullable(item.getObject("continuationItemRenderer")); + } + + return Optional.empty(); + } + + @Nonnull + private Consumer getCommitVideoConsumer( + @Nonnull final MultiInfoItemsCollector collector, + @Nonnull final TimeAgoParser timeAgoParser, + @Nonnull final List channelIds) { + return videoRenderer -> collector.commit( + new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser) { + @Override + public String getUploaderName() throws ParsingException { + if (channelIds.size() >= 2) { + return channelIds.get(0); + } + return super.getUploaderName(); + } + + @Override + public String getUploaderUrl() throws ParsingException { + if (channelIds.size() >= 2) { + return channelIds.get(1); + } + return super.getUploaderUrl(); + } + }); + } + + @Nonnull + private Consumer getCommitReelItemConsumer( + @Nonnull final MultiInfoItemsCollector collector, + @Nonnull final TimeAgoParser timeAgoParser, + @Nonnull final List channelIds) { + return reelItemRenderer -> collector.commit( + new YoutubeReelInfoItemExtractor(reelItemRenderer, timeAgoParser) { + @Override + public String getUploaderName() throws ParsingException { + if (channelIds.size() >= 2) { + return channelIds.get(0); + } + return super.getUploaderName(); + } + + @Override + public String getUploaderUrl() throws ParsingException { + if (channelIds.size() >= 2) { + return channelIds.get(1); + } + return super.getUploaderUrl(); + } + }); + } + + @Nonnull + private Consumer getCommitPlaylistConsumer( + @Nonnull final MultiInfoItemsCollector collector, + @Nonnull final List channelIds) { + return playlistRenderer -> collector.commit( + new YoutubePlaylistInfoItemExtractor(playlistRenderer) { + @Override + public String getUploaderName() throws ParsingException { + if (channelIds.size() >= 2) { + return channelIds.get(0); + } + return super.getUploaderName(); + } + + @Override + public String getUploaderUrl() throws ParsingException { + if (channelIds.size() >= 2) { + return channelIds.get(1); + } + return super.getUploaderUrl(); + } + }); + } + + @Nullable + private Page getNextPageFrom(final JsonObject continuations, + final List channelIds) throws IOException, + ExtractionException { + if (isNullOrEmpty(continuations)) { + return null; + } + + final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint"); + final String continuation = continuationEndpoint.getObject("continuationCommand") + .getString("token"); + + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), + getExtractorContentCountry(), + useVisitorData && channelIds.size() >= 3 ? channelIds.get(2) : null) + .value("continuation", continuation) + .done()) + .getBytes(StandardCharsets.UTF_8); + + return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey() + + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body); + } + + /** + * A {@link YoutubeChannelTabExtractor} for the {@code Videos} tab, if it has been already + * fetched. + */ + public static final class VideosTabExtractor extends YoutubeChannelTabExtractor { + private final JsonObject tabRenderer; + private final String channelName; + private final String channelId; + private final String channelUrl; + + VideosTabExtractor(final StreamingService service, + final ListLinkHandler linkHandler, + final JsonObject tabRenderer, + final String channelName, + final String channelId, + final String channelUrl) { + super(service, linkHandler); + this.tabRenderer = tabRenderer; + this.channelName = channelName; + this.channelId = channelId; + this.channelUrl = channelUrl; + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) { + // Nothing to do, the initial data was already fetched and is stored in the link handler + } + + @Nonnull + @Override + public String getId() throws ParsingException { + return channelId; + } + + @Nonnull + @Override + public String getUrl() throws ParsingException { + return channelUrl; + } + + @Override + protected String getChannelName() { + return channelName; + } + + @Override + Optional getTabData() { + return Optional.of(tabRenderer); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabPlaylistExtractor.java new file mode 100644 index 000000000..da50db631 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabPlaylistExtractor.java @@ -0,0 +1,191 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor; +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +/** + * A {@link ChannelTabExtractor} for YouTube system playlists using a + * {@link YoutubePlaylistExtractor} instance. + * + *

+ * It is currently used to bypass age-restrictions on channels marked as age-restricted by their + * owner(s). + *

+ */ +public class YoutubeChannelTabPlaylistExtractor extends ChannelTabExtractor { + + private final PlaylistExtractor playlistExtractorInstance; + private boolean playlistExisting; + + /** + * Construct a {@link YoutubeChannelTabPlaylistExtractor} instance. + * + * @param service a {@link StreamingService} implementation, which must be the YouTube + * one + * @param linkHandler a {@link ListLinkHandler} which must have a valid channel ID (starting + * with `UC`) and one of the given and supported content filters: + * {@link ChannelTabs#VIDEOS}, {@link ChannelTabs#SHORTS}, + * {@link ChannelTabs#LIVESTREAMS} + * @throws IllegalArgumentException if the given {@link ListLinkHandler} doesn't have the + * required arguments + * @throws SystemPlaylistUrlCreationException if the system playlist URL could not be created, + * which should never happen + */ + public YoutubeChannelTabPlaylistExtractor(@Nonnull final StreamingService service, + @Nonnull final ListLinkHandler linkHandler) + throws IllegalArgumentException, SystemPlaylistUrlCreationException { + super(service, linkHandler); + final ListLinkHandler playlistLinkHandler = getPlaylistLinkHandler(linkHandler); + this.playlistExtractorInstance = new YoutubePlaylistExtractor(service, playlistLinkHandler); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { + try { + playlistExtractorInstance.onFetchPage(downloader); + if (!playlistExisting) { + playlistExisting = true; + } + } catch (final ContentNotAvailableException e) { + // If a channel has no content of the type requested, the corresponding system playlist + // won't exist, so a ContentNotAvailableException would be thrown + // Ignore such issues in this case + } + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + if (!playlistExisting) { + return InfoItemsPage.emptyPage(); + } + + final InfoItemsPage playlistInitialPage = + playlistExtractorInstance.getInitialPage(); + + // We can't provide the playlist page as it is due to a type conflict, we need to wrap the + // page items and provide a new InfoItemsPage + final List infoItems = new ArrayList<>(playlistInitialPage.getItems()); + return new InfoItemsPage<>(infoItems, playlistInitialPage.getNextPage(), + playlistInitialPage.getErrors()); + } + + @Override + public InfoItemsPage getPage(final Page page) + throws IOException, ExtractionException { + if (!playlistExisting) { + return InfoItemsPage.emptyPage(); + } + + final InfoItemsPage playlistPage = playlistExtractorInstance.getPage(page); + + // We can't provide the playlist page as it is due to a type conflict, we need to wrap the + // page items and provide a new InfoItemsPage + final List infoItems = new ArrayList<>(playlistPage.getItems()); + return new InfoItemsPage<>(infoItems, playlistPage.getNextPage(), + playlistPage.getErrors()); + } + + /** + * Get a playlist {@link ListLinkHandler} from a channel tab one. + * + *

+ * This method converts a channel ID without its {@code UC} prefix into a YouTube system + * playlist, depending on the first content filter provided in the given + * {@link ListLinkHandler}. + *

+ * + *

+ * The first content filter must be a channel tabs one among the + * {@link ChannelTabs#VIDEOS videos}, {@link ChannelTabs#SHORTS shorts} and + * {@link ChannelTabs#LIVESTREAMS} ones, which would be converted respectively into playlists + * with the ID {@code UULF}, {@code UUSH} and {@code UULV} on which the channel ID without the + * {@code UC} part is appended. + *

+ * + * @param originalLinkHandler the original {@link ListLinkHandler} with which a + * {@link YoutubeChannelTabPlaylistExtractor} instance is being constructed + * + * @return a {@link ListLinkHandler} to use for the {@link YoutubePlaylistExtractor} instance + * needed to extract channel tabs data from a system playlist + * @throws IllegalArgumentException if the original {@link ListLinkHandler} does not meet the + * required criteria above + * @throws SystemPlaylistUrlCreationException if the system playlist URL could not be created, + * which should never happen + */ + @Nonnull + private ListLinkHandler getPlaylistLinkHandler( + @Nonnull final ListLinkHandler originalLinkHandler) + throws IllegalArgumentException, SystemPlaylistUrlCreationException { + final List contentFilters = originalLinkHandler.getContentFilters(); + if (contentFilters.isEmpty()) { + throw new IllegalArgumentException("A content filter is required"); + } + + final String channelId = originalLinkHandler.getId(); + if (isNullOrEmpty(channelId) || !channelId.startsWith("UC")) { + throw new IllegalArgumentException("Invalid channel ID"); + } + + final String channelIdWithoutUc = channelId.substring(2); + + final String playlistId; + switch (contentFilters.get(0)) { + case ChannelTabs.VIDEOS: + playlistId = "UULF" + channelIdWithoutUc; + break; + case ChannelTabs.SHORTS: + playlistId = "UUSH" + channelIdWithoutUc; + break; + case ChannelTabs.LIVESTREAMS: + playlistId = "UULV" + channelIdWithoutUc; + break; + default: + throw new IllegalArgumentException( + "Only Videos, Shorts and Livestreams tabs can extracted as playlists"); + } + + try { + final String newUrl = YoutubePlaylistLinkHandlerFactory.getInstance() + .getUrl(playlistId); + return new ListLinkHandler(newUrl, newUrl, playlistId, List.of(), ""); + } catch (final ParsingException e) { + // This should be not reachable, as the given playlist ID should be valid and + // YoutubePlaylistLinkHandlerFactory doesn't throw any exception + throw new SystemPlaylistUrlCreationException( + "Could not create a YouTube playlist from a valid playlist ID", e); + } + } + + /** + * Exception thrown when a YouTube system playlist URL could not be created. + * + *

+ * This exception should be never thrown, as given playlist IDs should be always valid. + *

+ */ + public static final class SystemPlaylistUrlCreationException extends RuntimeException { + SystemPlaylistUrlCreationException(final String message, final Throwable cause) { + super(message, cause); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index ef09a505a..c3a7707dc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; @@ -22,10 +23,15 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract @Override public String getThumbnailUrl() throws ParsingException { try { - final String url = playlistInfoItem.getArray("thumbnails").getObject(0) - .getArray("thumbnails").getObject(0).getString("url"); + JsonArray thumbnails = playlistInfoItem.getArray("thumbnails") + .getObject(0) + .getArray("thumbnails"); + if (thumbnails.isEmpty()) { + thumbnails = playlistInfoItem.getObject("thumbnail") + .getArray("thumbnails"); + } - return fixThumbnailUrl(url); + return fixThumbnailUrl(thumbnails.getObject(0).getString("url")); } catch (final Exception e) { throw new ParsingException("Could not get thumbnail url", e); } @@ -79,9 +85,21 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract @Override public long getStreamCount() throws ParsingException { + String videoCountText = playlistInfoItem.getString("videoCount"); + if (videoCountText == null) { + videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountText")); + } + + if (videoCountText == null) { + videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountShortText")); + } + + if (videoCountText == null) { + throw new ParsingException("Could not get stream count"); + } + try { - return Long.parseLong(Utils.removeNonDigitCharacters( - playlistInfoItem.getString("videoCount"))); + return Long.parseLong(Utils.removeNonDigitCharacters(videoCountText)); } catch (final Exception e) { throw new ParsingException("Could not get stream count", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java new file mode 100644 index 000000000..1b68c2ddf --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java @@ -0,0 +1,147 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.Utils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +/** + * A {@link StreamInfoItemExtractor} for YouTube's {@code reelItemRenderers}. + * + *

+ * {@code reelItemRenderers} are returned on YouTube for their short-form contents on almost every + * place and every major client. They provide a limited amount of information and do not provide + * the exact view count, any uploader info (name, URL, avatar, verified status) and the upload date. + *

+ */ +public class YoutubeReelInfoItemExtractor implements StreamInfoItemExtractor { + + @Nonnull + private final JsonObject reelInfo; + @Nullable + private final TimeAgoParser timeAgoParser; + + public YoutubeReelInfoItemExtractor(@Nonnull final JsonObject reelInfo, + @Nullable final TimeAgoParser timeAgoParser) { + this.reelInfo = reelInfo; + this.timeAgoParser = timeAgoParser; + } + + @Override + public String getName() throws ParsingException { + return getTextFromObject(reelInfo.getObject("headline")); + } + + @Override + public String getUrl() throws ParsingException { + try { + final String videoId = reelInfo.getString("videoId"); + return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId); + } catch (final Exception e) { + throw new ParsingException("Could not get URL", e); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return getThumbnailUrlFromInfoItem(reelInfo); + } + + @Override + public StreamType getStreamType() throws ParsingException { + return StreamType.VIDEO_STREAM; + } + + @Override + public long getDuration() throws ParsingException { + // Duration of reelItems is only provided in the accessibility data + // example: "VIDEO TITLE - 49 seconds - play video" + // "VIDEO TITLE - 1 minute, 1 second - play video" + final String accessibilityLabel = reelInfo.getObject("accessibility") + .getObject("accessibilityData").getString("label"); + if (accessibilityLabel == null || timeAgoParser == null) { + return 0; + } + + // This approach may be language dependent + final String[] labelParts = accessibilityLabel.split(" [\u2013-] "); + + if (labelParts.length > 2) { + final String textualDuration = labelParts[labelParts.length - 2]; + return timeAgoParser.parseDuration(textualDuration); + } + + return -1; + } + + @Override + public long getViewCount() throws ParsingException { + final String viewCountText = getTextFromObject(reelInfo.getObject("viewCountText")); + if (!isNullOrEmpty(viewCountText)) { + // This approach is language dependent + if (viewCountText.toLowerCase().contains("no views")) { + return 0; + } + + return Utils.mixedNumberWordToLong(viewCountText); + } + + throw new ParsingException("Could not get short view count"); + } + + @Override + public boolean isShortFormContent() throws ParsingException { + return true; + } + + // All the following properties cannot be obtained from reelItemRenderers + + @Override + public boolean isAd() throws ParsingException { + return false; + } + + @Override + public String getUploaderName() throws ParsingException { + return null; + } + + @Override + public String getUploaderUrl() throws ParsingException { + return null; + } + + @Nullable + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return null; + } + + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + + @Nullable + @Override + public String getTextualUploadDate() throws ParsingException { + return null; + } + + @Nullable + @Override + public DateWrapper getUploadDate() throws ParsingException { + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index ba370e49b..1273c7a3e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -58,7 +58,8 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto @Override public String getUrl(final String id, final List contentFilters, - final String searchFilter) { + final String searchFilter) + throws ParsingException, UnsupportedOperationException { return "https://www.youtube.com/" + id; } @@ -84,7 +85,7 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto } @Override - public String getId(final String url) throws ParsingException { + public String getId(final String url) throws ParsingException, UnsupportedOperationException { try { final URL urlObj = Utils.stringToURL(url); String path = urlObj.getPath(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java new file mode 100644 index 000000000..e83808a63 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java @@ -0,0 +1,73 @@ +package org.schabi.newpipe.extractor.services.youtube.linkHandler; + +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.UnsupportedTabException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + +import javax.annotation.Nonnull; +import java.util.List; + +public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFactory { + private static final YoutubeChannelTabLinkHandlerFactory INSTANCE = + new YoutubeChannelTabLinkHandlerFactory(); + + private YoutubeChannelTabLinkHandlerFactory() { + } + + public static YoutubeChannelTabLinkHandlerFactory getInstance() { + return INSTANCE; + } + + @Nonnull + public static String getUrlSuffix(@Nonnull final String tab) + throws UnsupportedTabException { + switch (tab) { + case ChannelTabs.VIDEOS: + return "/videos"; + case ChannelTabs.SHORTS: + return "/shorts"; + case ChannelTabs.LIVESTREAMS: + return "/streams"; + case ChannelTabs.PLAYLISTS: + return "/playlists"; + case ChannelTabs.CHANNELS: + return "/channels"; + } + throw new UnsupportedTabException(tab); + } + + @Override + public String getUrl(final String id, + final List contentFilter, + final String sortFilter) + throws ParsingException, UnsupportedOperationException { + return "https://www.youtube.com/" + id + getUrlSuffix(contentFilter.get(0)); + } + + @Override + public String getId(final String url) throws ParsingException, UnsupportedOperationException { + return YoutubeChannelLinkHandlerFactory.getInstance().getId(url); + } + + @Override + public boolean onAcceptUrl(final String url) throws ParsingException { + try { + getId(url); + } catch (final ParsingException e) { + return false; + } + return true; + } + + @Override + public String[] getAvailableContentFilter() { + return new String[] { + ChannelTabs.VIDEOS, + ChannelTabs.SHORTS, + ChannelTabs.LIVESTREAMS, + ChannelTabs.PLAYLISTS, + ChannelTabs.CHANNELS + }; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java index a15801b22..0af682f87 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java @@ -19,13 +19,14 @@ public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFact } @Override - public String getUrl(final String id) { + public String getUrl(final String id) throws ParsingException, UnsupportedOperationException { return "https://www.youtube.com/watch?v=" + id; } @Override - public String getId(final String urlString) throws ParsingException, IllegalArgumentException { - // we need the same id, avoids duplicate code + public String getId(final String urlString) + throws ParsingException, UnsupportedOperationException { + // We need the same id, avoids duplicate code return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString); } @@ -44,7 +45,8 @@ public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFact @Override public String getUrl(final String id, final List contentFilter, - final String sortFilter) throws ParsingException { + final String sortFilter) + throws ParsingException, UnsupportedOperationException { return getUrl(id); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 6b8c40be9..f6d2e10e9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -26,12 +26,13 @@ public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFact @Override public String getUrl(final String id, final List contentFilters, - final String sortFilter) { + final String sortFilter) + throws ParsingException, UnsupportedOperationException { return "https://www.youtube.com/playlist?list=" + id; } @Override - public String getId(final String url) throws ParsingException { + public String getId(final String url) throws ParsingException, UnsupportedOperationException { try { final URL urlObj = Utils.stringToURL(url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 3c2bc7073..44b036be6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -13,6 +13,9 @@ import javax.annotation.Nonnull; public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { + private static final YoutubeSearchQueryHandlerFactory INSTANCE = + new YoutubeSearchQueryHandlerFactory(); + public static final String ALL = "all"; public static final String VIDEOS = "videos"; public static final String CHANNELS = "channels"; @@ -29,20 +32,18 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa @Nonnull public static YoutubeSearchQueryHandlerFactory getInstance() { - return new YoutubeSearchQueryHandlerFactory(); + return INSTANCE; } @Override public String getUrl(final String searchString, @Nonnull final List contentFilters, - final String sortFilter) throws ParsingException { + final String sortFilter) + throws ParsingException, UnsupportedOperationException { try { if (!contentFilters.isEmpty()) { final String contentFilter = contentFilters.get(0); switch (contentFilter) { - case ALL: - default: - break; case VIDEOS: return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAQ%253D%253D"; case CHANNELS: diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index 0872bbe2d..e5e39aa35 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -79,7 +79,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { @Nonnull @Override - public String getUrl(final String id) { + public String getUrl(final String id) throws ParsingException, UnsupportedOperationException { return "https://www.youtube.com/watch?v=" + id; } @@ -87,7 +87,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { @Nonnull @Override public String getId(final String theUrlString) - throws ParsingException, IllegalArgumentException { + throws ParsingException, UnsupportedOperationException { String urlString = theUrlString; try { final URI uri = new URI(urlString); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java index 82f6bb26c..305eeb6a5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java @@ -1,28 +1,29 @@ -package org.schabi.newpipe.extractor.services.youtube.linkHandler; - /* * Created by Christian Schabesberger on 12.08.17. * * Copyright (C) Christian Schabesberger 2018 - * YoutubeTrendingLinkHandlerFactory.java is part of NewPipe. + * YoutubeTrendingLinkHandlerFactory.java is part of NewPipe Extractor. * - * NewPipe is free software: you can redistribute it and/or modify + * NewPipe Extractor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * NewPipe is distributed in the hope that it will be useful, + * NewPipe Extractor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . + * along with NewPipe Extractor. If not, see . */ +package org.schabi.newpipe.extractor.services.youtube.linkHandler; + import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidiousURL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; @@ -30,16 +31,27 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.List; -public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { +public final class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { + + private static final YoutubeTrendingLinkHandlerFactory INSTANCE = + new YoutubeTrendingLinkHandlerFactory(); + + private YoutubeTrendingLinkHandlerFactory() { + } + + public static YoutubeTrendingLinkHandlerFactory getInstance() { + return INSTANCE; + } public String getUrl(final String id, final List contentFilters, - final String sortFilter) { + final String sortFilter) + throws ParsingException, UnsupportedOperationException { return "https://www.youtube.com/feed/trending"; } @Override - public String getId(final String url) { + public String getId(final String url) throws ParsingException, UnsupportedOperationException { return "Trending"; }