Merge pull request #1082 from AudricV/channel-tabs-and-tags-support

Add support for channel tabs and channel tags
This commit is contained in:
Stypox 2023-08-06 12:21:42 +02:00 committed by GitHub
commit 35f3a4ad01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
182 changed files with 16319 additions and 4292 deletions

View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -140,6 +141,14 @@ public abstract class StreamingService {
*/
public abstract ListLinkHandlerFactory getChannelLHFactory();
/**
* Must return a new instance of an implementation of ListLinkHandlerFactory for channel tabs.
* If support for channel tabs is not given null must be returned.
*
* @return an instance of a ListLinkHandlerFactory for channels or null
*/
public abstract ListLinkHandlerFactory getChannelTabLHFactory();
/**
* Must return a new instance of an implementation of ListLinkHandlerFactory for playlists.
* If support for playlists is not given null must be returned.
@ -204,6 +213,15 @@ public abstract class StreamingService {
public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler)
throws ExtractionException;
/**
* Must create a new instance of a ChannelTabExtractor implementation.
*
* @param linkHandler is pointing to the channel which should be handled by this new instance.
* @return a new ChannelTabExtractor
*/
public abstract ChannelTabExtractor getChannelTabExtractor(ListLinkHandler linkHandler)
throws ExtractionException;
/**
* Must crete a new instance of a PlaylistExtractor implementation.
* @param linkHandler is pointing to the playlist which should be handled by this new instance.
@ -262,6 +280,20 @@ public abstract class StreamingService {
return getChannelExtractor(getChannelLHFactory().fromUrl(url));
}
public ChannelTabExtractor getChannelTabExtractorFromId(final String id, final String tab)
throws ExtractionException {
return getChannelTabExtractor(getChannelTabLHFactory().fromQuery(
id, Collections.singletonList(tab), ""));
}
public ChannelTabExtractor getChannelTabExtractorFromIdAndBaseUrl(final String id,
final String tab,
final String baseUrl)
throws ExtractionException {
return getChannelTabExtractor(getChannelTabLHFactory().fromQuery(
id, Collections.singletonList(tab), "", baseUrl));
}
public PlaylistExtractor getPlaylistExtractor(final String url) throws ExtractionException {
return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url));
}

View File

@ -1,10 +1,12 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import javax.annotation.Nonnull;
import java.util.List;
/*
* Created by Christian Schabesberger on 25.07.16.
@ -26,11 +28,11 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
public abstract class ChannelExtractor extends Extractor {
public static final long UNKNOWN_SUBSCRIBER_COUNT = -1;
public ChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
protected ChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@ -43,5 +45,10 @@ public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
public abstract String getParentChannelUrl() throws ParsingException;
public abstract String getParentChannelAvatarUrl() throws ParsingException;
public abstract boolean isVerified() throws ParsingException;
@Nonnull
public abstract List<ListLinkHandler> getTabs() throws ParsingException;
@Nonnull
public List<String> getTags() throws ParsingException {
return List.of();
}
}

View File

@ -1,16 +1,15 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
import java.util.List;
import javax.annotation.Nonnull;
/*
* Created by Christian Schabesberger on 31.07.16.
@ -32,16 +31,14 @@ import java.io.IOException;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ChannelInfo extends ListInfo<StreamInfoItem> {
public class ChannelInfo extends Info {
public ChannelInfo(final int serviceId,
final String id,
final String url,
final String originalUrl,
final String name,
final ListLinkHandler listLinkHandler) {
super(serviceId, id, url, originalUrl, name, listLinkHandler.getContentFilters(),
listLinkHandler.getSortFilter());
final String name) {
super(serviceId, id, url, originalUrl, name);
}
public static ChannelInfo getInfo(final String url) throws IOException, ExtractionException {
@ -55,13 +52,6 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
return getInfo(extractor);
}
public static InfoItemsPage<StreamInfoItem> getMoreItems(final StreamingService service,
final String url,
final Page page)
throws IOException, ExtractionException {
return service.getChannelExtractor(url).getPage(page);
}
public static ChannelInfo getInfo(final ChannelExtractor extractor)
throws IOException, ExtractionException {
@ -71,35 +61,32 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
final String originalUrl = extractor.getOriginalUrl();
final String name = extractor.getName();
final ChannelInfo info =
new ChannelInfo(serviceId, id, url, originalUrl, name, extractor.getLinkHandler());
final ChannelInfo info = new ChannelInfo(serviceId, id, url, originalUrl, name);
try {
info.setAvatarUrl(extractor.getAvatarUrl());
} catch (final Exception e) {
info.addError(e);
}
try {
info.setBannerUrl(extractor.getBannerUrl());
} catch (final Exception e) {
info.addError(e);
}
try {
info.setFeedUrl(extractor.getFeedUrl());
} catch (final Exception e) {
info.addError(e);
}
final InfoItemsPage<StreamInfoItem> itemsPage =
ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(itemsPage.getItems());
info.setNextPage(itemsPage.getNextPage());
try {
info.setSubscriberCount(extractor.getSubscriberCount());
} catch (final Exception e) {
info.addError(e);
}
try {
info.setDescription(extractor.getDescription());
} catch (final Exception e) {
@ -130,6 +117,18 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
info.addError(e);
}
try {
info.setTabs(extractor.getTabs());
} catch (final Exception e) {
info.addError(e);
}
try {
info.setTags(extractor.getTags());
} catch (final Exception e) {
info.addError(e);
}
return info;
}
@ -143,6 +142,8 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
private String description;
private String[] donationLinks;
private boolean verified;
private List<ListLinkHandler> tabs = List.of();
private List<String> tags = List.of();
public String getParentChannelName() {
return parentChannelName;
@ -223,4 +224,22 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
public void setVerified(final boolean verified) {
this.verified = verified;
}
@Nonnull
public List<ListLinkHandler> getTabs() {
return tabs;
}
public void setTabs(@Nonnull final List<ListLinkHandler> tabs) {
this.tabs = tabs;
}
@Nonnull
public List<String> getTags() {
return tags;
}
public void setTags(@Nonnull final List<String> tags) {
this.tags = tags;
}
}

View File

@ -0,0 +1,25 @@
package org.schabi.newpipe.extractor.channel.tabs;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import javax.annotation.Nonnull;
/**
* A {@link ListExtractor} of {@link InfoItem}s for tabs of channels.
*/
public abstract class ChannelTabExtractor extends ListExtractor<InfoItem> {
protected ChannelTabExtractor(@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Nonnull
@Override
public String getName() {
return getLinkHandler().getContentFilters().get(0);
}
}

View File

@ -0,0 +1,70 @@
package org.schabi.newpipe.extractor.channel.tabs;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfo;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import javax.annotation.Nonnull;
import java.io.IOException;
public class ChannelTabInfo extends ListInfo<InfoItem> {
public ChannelTabInfo(final int serviceId,
@Nonnull final ListLinkHandler linkHandler) {
super(serviceId, linkHandler, linkHandler.getContentFilters().get(0));
}
/**
* Get a {@link ChannelTabInfo} instance from the given service and tab handler.
*
* @param service streaming service
* @param linkHandler Channel tab handler (from {@link ChannelInfo})
* @return the extracted {@link ChannelTabInfo}
*/
@Nonnull
public static ChannelTabInfo getInfo(@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler)
throws ExtractionException, IOException {
final ChannelTabExtractor extractor = service.getChannelTabExtractor(linkHandler);
extractor.fetchPage();
return getInfo(extractor);
}
/**
* Get a {@link ChannelTabInfo} instance from a {@link ChannelTabExtractor}.
*
* @param extractor an extractor where {@code fetchPage()} was already got called on
* @return the extracted {@link ChannelTabInfo}
*/
@Nonnull
public static ChannelTabInfo getInfo(@Nonnull final ChannelTabExtractor extractor) {
final ChannelTabInfo info =
new ChannelTabInfo(extractor.getServiceId(), extractor.getLinkHandler());
try {
info.setOriginalUrl(extractor.getOriginalUrl());
} catch (final Exception e) {
info.addError(e);
}
final ListExtractor.InfoItemsPage<InfoItem> page
= ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(page.getItems());
info.setNextPage(page.getNextPage());
return info;
}
public static ListExtractor.InfoItemsPage<InfoItem> getMoreItems(
@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler,
@Nonnull final Page page) throws ExtractionException, IOException {
return service.getChannelTabExtractor(linkHandler).getPage(page);
}
}

View File

@ -0,0 +1,17 @@
package org.schabi.newpipe.extractor.channel.tabs;
/**
* Constants of channel tabs supported by the extractor.
*/
public final class ChannelTabs {
public static final String VIDEOS = "videos";
public static final String TRACKS = "tracks";
public static final String SHORTS = "shorts";
public static final String LIVESTREAMS = "livestreams";
public static final String CHANNELS = "channels";
public static final String PLAYLISTS = "playlists";
public static final String ALBUMS = "albums";
private ChannelTabs() {
}
}

View File

@ -0,0 +1,7 @@
package org.schabi.newpipe.extractor.exceptions;
public final class UnsupportedTabException extends UnsupportedOperationException {
public UnsupportedTabException(final String unsupportedTab) {
super("Unsupported tab " + unsupportedTab);
}
}

View File

@ -31,13 +31,14 @@ public abstract class LinkHandlerFactory {
// To Override
///////////////////////////////////
public abstract String getId(String url) throws ParsingException;
public abstract String getId(String url) throws ParsingException, UnsupportedOperationException;
public abstract String getUrl(String id) throws ParsingException;
public abstract String getUrl(String id) throws ParsingException, UnsupportedOperationException;
public abstract boolean onAcceptUrl(String url) throws ParsingException;
public String getUrl(final String id, final String baseUrl) throws ParsingException {
public String getUrl(final String id, final String baseUrl)
throws ParsingException, UnsupportedOperationException {
return getUrl(id);
}

View File

@ -14,12 +14,13 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
///////////////////////////////////
public abstract String getUrl(String id, List<String> contentFilter, String sortFilter)
throws ParsingException;
throws ParsingException, UnsupportedOperationException;
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter,
final String baseUrl) throws ParsingException {
final String baseUrl)
throws ParsingException, UnsupportedOperationException {
return getUrl(id, contentFilter, sortFilter);
}
@ -72,7 +73,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
*
* @return the url corresponding to id without any filters applied
*/
public String getUrl(final String id) throws ParsingException {
public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
return getUrl(id, new ArrayList<>(0), "");
}

View File

@ -0,0 +1,55 @@
package org.schabi.newpipe.extractor.linkhandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import javax.annotation.Nonnull;
import java.io.Serializable;
import java.util.List;
/**
* A {@link ListLinkHandler} which can be used to be returned from {@link
* org.schabi.newpipe.extractor.channel.ChannelInfo#getTabs() ChannelInfo#getTabs()} when a
* specific tab's data has already been fetched.
*
* <p>
* This class allows passing a builder for a {@link ChannelTabExtractor} that can hold references
* to variables.
* </p>
*
* <p>
* Note: a service that wishes to use this class in one of its {@link
* org.schabi.newpipe.extractor.channel.ChannelExtractor ChannelExtractor}s must also add the
* following snippet of code in the service's
* {@link StreamingService#getChannelTabExtractor(ListLinkHandler)}:
* <pre>
* if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
* return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
* }
* </pre>
* </p>
*/
public class ReadyChannelTabListLinkHandler extends ListLinkHandler {
public interface ChannelTabExtractorBuilder extends Serializable {
@Nonnull
ChannelTabExtractor build(@Nonnull StreamingService service,
@Nonnull ListLinkHandler linkHandler);
}
private final ChannelTabExtractorBuilder extractorBuilder;
public ReadyChannelTabListLinkHandler(
final String url,
final String channelId,
@Nonnull final String channelTab,
@Nonnull final ChannelTabExtractorBuilder extractorBuilder) {
super(url, url, channelId, List.of(channelTab), "");
this.extractorBuilder = extractorBuilder;
}
@Nonnull
public ChannelTabExtractor getChannelTabExtractor(@Nonnull final StreamingService service) {
return extractorBuilder.build(service, new ListLinkHandler(this));
}
}

View File

@ -13,7 +13,7 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
@Override
public abstract String getUrl(String query, List<String> contentFilter, String sortFilter)
throws ParsingException;
throws ParsingException, UnsupportedOperationException;
@SuppressWarnings("unused")
public String getSearchString(final String url) {
@ -25,7 +25,7 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
///////////////////////////////////
@Override
public String getId(final String url) {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return getSearchString(url);
}

View File

@ -7,14 +7,21 @@ import org.schabi.newpipe.extractor.utils.Parser;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.util.regex.MatchResult;
/**
* A helper class that is meant to be used by services that need to parse upload dates in the
* format '2 days ago' or similar.
* A helper class that is meant to be used by services that need to parse durations such as
* {@code 23 seconds} and/or upload dates in the format {@code 2 days ago} or similar.
*/
public class TimeAgoParser {
private static final Pattern DURATION_PATTERN = Pattern.compile("(?:(\\d+) )?([A-z]+)");
private final PatternsHolder patternsHolder;
private final OffsetDateTime now;
@ -60,6 +67,48 @@ public class TimeAgoParser {
return getResultFor(parseTimeAgoAmount(textualDate), parseChronoUnit(textualDate));
}
/**
* Parses a textual duration into a duration computer number.
*
* @param textualDuration the textual duration to parse
* @return the textual duration parsed, as a primitive {@code long}
* @throws ParsingException if the textual duration could not be parsed
*/
public long parseDuration(final String textualDuration) throws ParsingException {
// We can't use Matcher.results, as it is only available on Android 14 and above
final Matcher matcher = DURATION_PATTERN.matcher(textualDuration);
final List<MatchResult> results = new ArrayList<>();
while (matcher.find()) {
results.add(matcher.toMatchResult());
}
return results.stream()
.map(match -> {
final String digits = match.group(1);
final String word = match.group(2);
int amount;
try {
amount = Integer.parseInt(digits);
} catch (final NumberFormatException ignored) {
amount = 1;
}
final ChronoUnit unit;
try {
unit = parseChronoUnit(word);
} catch (final ParsingException ignored) {
return 0L;
}
return amount * unit.getDuration().getSeconds();
})
.filter(n -> n > 0)
.reduce(Long::sum)
.orElseThrow(() -> new ParsingException(
"Could not parse duration \"" + textualDuration + "\""));
}
private int parseTimeAgoAmount(final String textualDate) {
try {
return Integer.parseInt(textualDate.replaceAll("\\D+", ""));

View File

@ -12,6 +12,7 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
@ -19,11 +20,13 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelTabExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampCommentsExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor;
@ -34,6 +37,7 @@ import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchE
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSuggestionExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampPlaylistLinkHandlerFactory;
@ -58,27 +62,32 @@ public class BandcampService extends StreamingService {
@Override
public LinkHandlerFactory getStreamLHFactory() {
return new BandcampStreamLinkHandlerFactory();
return BandcampStreamLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getChannelLHFactory() {
return new BandcampChannelLinkHandlerFactory();
return BandcampChannelLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getChannelTabLHFactory() {
return BandcampChannelTabLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getPlaylistLHFactory() {
return new BandcampPlaylistLinkHandlerFactory();
return BandcampPlaylistLinkHandlerFactory.getInstance();
}
@Override
public SearchQueryHandlerFactory getSearchQHFactory() {
return new BandcampSearchQueryHandlerFactory();
return BandcampSearchQueryHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getCommentsLHFactory() {
return new BandcampCommentsLinkHandlerFactory();
return BandcampCommentsLinkHandlerFactory.getInstance();
}
@Override
@ -98,27 +107,27 @@ public class BandcampService extends StreamingService {
@Override
public KioskList getKioskList() throws ExtractionException {
final KioskList kioskList = new KioskList(this);
final ListLinkHandlerFactory h = BandcampFeaturedLinkHandlerFactory.getInstance();
try {
kioskList.addKioskEntry(
(streamingService, url, kioskId) -> new BandcampFeaturedExtractor(
BandcampService.this,
new BandcampFeaturedLinkHandlerFactory().fromUrl(FEATURED_API_URL),
h.fromUrl(FEATURED_API_URL),
kioskId
),
new BandcampFeaturedLinkHandlerFactory(),
h,
KIOSK_FEATURED
);
kioskList.addKioskEntry(
(streamingService, url, kioskId) -> new BandcampRadioExtractor(
BandcampService.this,
new BandcampFeaturedLinkHandlerFactory().fromUrl(RADIO_API_URL),
h.fromUrl(RADIO_API_URL),
kioskId
),
new BandcampFeaturedLinkHandlerFactory(),
h,
KIOSK_RADIO
);
@ -136,6 +145,15 @@ public class BandcampService extends StreamingService {
return new BandcampChannelExtractor(this, linkHandler);
}
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
} else {
return new BandcampChannelTabExtractor(this, linkHandler);
}
}
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return new BandcampPlaylistExtractor(this, linkHandler);

View File

@ -0,0 +1,55 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
public class BandcampAlbumInfoItemExtractor implements PlaylistInfoItemExtractor {
private final JsonObject albumInfoItem;
private final String uploaderUrl;
public BandcampAlbumInfoItemExtractor(final JsonObject albumInfoItem,
final String uploaderUrl) {
this.albumInfoItem = albumInfoItem;
this.uploaderUrl = uploaderUrl;
}
@Override
public String getName() throws ParsingException {
return albumInfoItem.getString("title");
}
@Override
public String getUrl() throws ParsingException {
return BandcampExtractorHelper.getStreamUrlFromIds(
albumInfoItem.getLong("band_id"),
albumInfoItem.getLong("item_id"),
albumInfoItem.getString("item_type"));
}
@Override
public String getThumbnailUrl() throws ParsingException {
return BandcampExtractorHelper.getImageUrl(albumInfoItem.getLong("art_id"), true);
}
@Override
public String getUploaderName() throws ParsingException {
return albumInfoItem.getString("band_name");
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
@Override
public boolean isUploaderVerified() {
return false;
}
@Override
public long getStreamCount() {
return ListExtractor.ITEM_COUNT_UNKNOWN;
}
}

View File

@ -8,19 +8,22 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.jsoup.Jsoup;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampDiscographStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelTabLinkHandlerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;
@ -52,8 +55,8 @@ public class BandcampChannelExtractor extends ChannelExtractor {
*/
try {
final String html = getDownloader()
.get(replaceHttpWithHttps(channelInfo.getString("bandcamp_url")))
.responseBody();
.get(replaceHttpWithHttps(channelInfo.getString("bandcamp_url")))
.responseBody();
return Stream.of(Jsoup.parse(html).getElementById("customHeader"))
.filter(Objects::nonNull)
@ -107,29 +110,47 @@ public class BandcampChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
public List<ListLinkHandler> getTabs() throws ParsingException {
final JsonArray discography = channelInfo.getArray("discography");
final TabExtractorBuilder builder = new TabExtractorBuilder(discography);
for (int i = 0; i < discography.size(); i++) {
// A discograph is as an item appears in a discography
final JsonObject discograph = discography.getObject(i);
final List<ListLinkHandler> tabs = new ArrayList<>();
if (!discograph.getString("item_type").equals("track")) {
boolean foundTrackItem = false;
boolean foundAlbumItem = false;
for (final Object discographyItem : discography) {
if (foundTrackItem && foundAlbumItem) {
break;
}
if (!(discographyItem instanceof JsonObject)) {
continue;
}
collector.commit(new BandcampDiscographStreamInfoItemExtractor(discograph, getUrl()));
final JsonObject discographyJsonItem = (JsonObject) discographyItem;
final String itemType = discographyJsonItem.getString("item_type");
if (!foundTrackItem && "track".equals(itemType)) {
foundTrackItem = true;
tabs.add(new ReadyChannelTabListLinkHandler(getUrl()
+ BandcampChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.TRACKS),
getId(),
ChannelTabs.TRACKS,
builder));
}
if (!foundAlbumItem && "album".equals(itemType)) {
foundAlbumItem = true;
tabs.add(new ReadyChannelTabListLinkHandler(getUrl()
+ BandcampChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.ALBUMS),
getId(),
ChannelTabs.ALBUMS,
builder));
}
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
return null;
return Collections.unmodifiableList(tabs);
}
@Override
@ -143,4 +164,20 @@ public class BandcampChannelExtractor extends ChannelExtractor {
public String getName() {
return channelInfo.getString("name");
}
private static final class TabExtractorBuilder
implements ReadyChannelTabListLinkHandler.ChannelTabExtractorBuilder {
private final JsonArray discography;
TabExtractorBuilder(final JsonArray discography) {
this.discography = discography;
}
@Nonnull
@Override
public ChannelTabExtractor build(@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler) {
return BandcampChannelTabExtractor.fromDiscography(service, linkHandler, discography);
}
}
}

View File

@ -0,0 +1,95 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampDiscographStreamInfoItemExtractor;
import javax.annotation.Nonnull;
import java.io.IOException;
public class BandcampChannelTabExtractor extends ChannelTabExtractor {
private JsonArray discography;
private final String filter;
public BandcampChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
final String tab = linkHandler.getContentFilters().get(0);
switch (tab) {
case ChannelTabs.TRACKS:
filter = "track";
break;
case ChannelTabs.ALBUMS:
filter = "album";
break;
default:
throw new IllegalArgumentException("Unsupported channel tab: " + tab);
}
}
public static BandcampChannelTabExtractor fromDiscography(final StreamingService service,
final ListLinkHandler linkHandler,
final JsonArray discography) {
final BandcampChannelTabExtractor tabExtractor =
new BandcampChannelTabExtractor(service, linkHandler);
tabExtractor.discography = discography;
return tabExtractor;
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws ParsingException {
if (discography == null) {
discography = BandcampExtractorHelper.getArtistDetails(getId())
.getArray("discography");
}
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
for (final Object discograph : discography) {
// A discograph is as an item appears in a discography
if (!(discograph instanceof JsonObject)) {
continue;
}
final JsonObject discographJsonObject = (JsonObject) discograph;
final String itemType = discographJsonObject.getString("item_type", "");
if (!itemType.equals(filter)) {
continue;
}
switch (itemType) {
case "track":
collector.commit(new BandcampDiscographStreamInfoItemExtractor(
discographJsonObject, getUrl()));
break;
case "album":
collector.commit(new BandcampAlbumInfoItemExtractor(
discographJsonObject, getUrl()));
break;
}
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page) {
return null;
}
}

View File

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.List;
@ -17,11 +18,20 @@ import java.util.List;
/**
* Artist do have IDs that are useful
*/
public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory {
public final class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final BandcampChannelLinkHandlerFactory INSTANCE
= new BandcampChannelLinkHandlerFactory();
private BandcampChannelLinkHandlerFactory() {
}
public static BandcampChannelLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
try {
final String response = NewPipe.getDownloader().get(url).responseBody();
@ -41,16 +51,13 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory {
*/
@Override
public String getUrl(final String id, final List<String> contentFilter, final String sortFilter)
throws ParsingException {
try {
return BandcampExtractorHelper.getArtistDetails(id)
.getString("bandcamp_url")
.replace("http://", "https://");
} catch (final NullPointerException e) {
throws ParsingException, UnsupportedOperationException {
final JsonObject artistDetails = BandcampExtractorHelper.getArtistDetails(id);
if (artistDetails.getBoolean("error")) {
throw new ParsingException(
"JSON does not contain URL (invalid id?) or is otherwise invalid", e);
"JSON does not contain a channel URL (invalid id?) or is otherwise invalid");
}
return Utils.replaceHttpWithHttps(artistDetails.getString("bandcamp_url"));
}
/**
@ -61,22 +68,21 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory {
final String lowercaseUrl = url.toLowerCase();
// https: | | artist.bandcamp.com | releases
// 0 1 2 3
// https: | | artist.bandcamp.com | releases - music - album - track ( | name)
// 0 1 2 3 (4)
final String[] splitUrl = lowercaseUrl.split("/");
// URL is too short
if (splitUrl.length < 3) {
if (splitUrl.length != 3 && splitUrl.length != 4) {
return false;
}
// Must have "releases" or "music" as segment after url or none at all
if (splitUrl.length > 3 && !(
splitUrl[3].equals("releases") || splitUrl[3].equals("music")
)) {
// Must have "releases", "music", "album" or "track" as segment after URL or none at all
if (splitUrl.length == 4 && !(splitUrl[3].equals("releases")
|| splitUrl[3].equals("music")
|| splitUrl[3].equals("album")
|| splitUrl[3].equals("track"))) {
return false;
} else {
if (splitUrl[2].equals("daily.bandcamp.com")) {
// Refuse links to daily.bandcamp.com as that is not an artist

View File

@ -0,0 +1,72 @@
package org.schabi.newpipe.extractor.services.bandcamp.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.UnsupportedTabException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import javax.annotation.Nonnull;
import java.util.List;
public final class BandcampChannelTabLinkHandlerFactory extends ListLinkHandlerFactory {
private static final BandcampChannelTabLinkHandlerFactory INSTANCE
= new BandcampChannelTabLinkHandlerFactory();
private BandcampChannelTabLinkHandlerFactory() {
}
public static BandcampChannelTabLinkHandlerFactory getInstance() {
return INSTANCE;
}
/**
* Get a tab's URL suffix.
*
* <p>
* These URLs don't actually exist on the Bandcamp website, as both albums and tracks are
* listed on the main page, but redirect to the main page, which is perfect for us as we need a
* unique URL for each tab.
* </p>
*
* @param tab the tab value, which must not be null
* @return a URL suffix
* @throws UnsupportedTabException if the tab is not supported
*/
@Nonnull
public static String getUrlSuffix(@Nonnull final String tab) throws UnsupportedTabException {
switch (tab) {
case ChannelTabs.TRACKS:
return "/track";
case ChannelTabs.ALBUMS:
return "/album";
}
throw new UnsupportedTabException(tab);
}
@Override
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return BandcampChannelLinkHandlerFactory.getInstance().getId(url);
}
@Override
public String getUrl(final String id, final List<String> contentFilter, final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return BandcampChannelLinkHandlerFactory.getInstance().getUrl(id)
+ getUrlSuffix(contentFilter.get(0));
}
@Override
public boolean onAcceptUrl(final String url) throws ParsingException {
return BandcampChannelLinkHandlerFactory.getInstance().onAcceptUrl(url);
}
@Override
public String[] getAvailableContentFilter() {
return new String[]{
ChannelTabs.TRACKS,
ChannelTabs.ALBUMS,
};
}
}

View File

@ -10,10 +10,20 @@ import java.util.List;
* Like in {@link BandcampStreamLinkHandlerFactory}, tracks have no meaningful IDs except for
* their URLs
*/
public class BandcampCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
public final class BandcampCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final BandcampCommentsLinkHandlerFactory INSTANCE
= new BandcampCommentsLinkHandlerFactory();
private BandcampCommentsLinkHandlerFactory() {
}
public static BandcampCommentsLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return url;
}
@ -35,7 +45,8 @@ public class BandcampCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return id;
}
}

View File

@ -2,6 +2,7 @@
package org.schabi.newpipe.extractor.services.bandcamp.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
import org.schabi.newpipe.extractor.utils.Utils;
@ -13,12 +14,23 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.KIOSK_RADIO;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.RADIO_API_URL;
public class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory {
public final class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory {
private static final BandcampFeaturedLinkHandlerFactory INSTANCE =
new BandcampFeaturedLinkHandlerFactory();
private BandcampFeaturedLinkHandlerFactory() {
}
public static BandcampFeaturedLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
if (id.equals(KIOSK_FEATURED)) {
return FEATURED_API_URL; // doesn't have a website
} else if (id.equals(KIOSK_RADIO)) {
@ -29,7 +41,7 @@ public class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory {
}
@Override
public String getId(final String url) {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
final String fixedUrl = Utils.replaceHttpWithHttps(url);
if (BandcampExtractorHelper.isRadioUrl(fixedUrl) || fixedUrl.equals(RADIO_API_URL)) {
return KIOSK_RADIO;

View File

@ -11,16 +11,28 @@ import java.util.List;
/**
* Just as with streams, the album ids are essentially useless for us.
*/
public class BandcampPlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
public final class BandcampPlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
private static final BandcampPlaylistLinkHandlerFactory INSTANCE
= new BandcampPlaylistLinkHandlerFactory();
private BandcampPlaylistLinkHandlerFactory() {
}
public static BandcampPlaylistLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return getUrl(url);
}
@Override
public String getUrl(final String url,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return url;
}

View File

@ -11,11 +11,23 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.UnsupportedEncodingException;
import java.util.List;
public class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public final class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
private static final BandcampSearchQueryHandlerFactory INSTANCE
= new BandcampSearchQueryHandlerFactory();
private BandcampSearchQueryHandlerFactory() {
}
public static BandcampSearchQueryHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String getUrl(final String query,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try {
return BASE_URL + "/search?q=" + Utils.encodeUrlUtf8(query) + "&page=1";
} catch (final UnsupportedEncodingException e) {

View File

@ -14,14 +14,24 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp
*
* <p>Radio (bandcamp weekly) shows do have ids.</p>
*/
public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory {
public final class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final BandcampStreamLinkHandlerFactory INSTANCE
= new BandcampStreamLinkHandlerFactory();
private BandcampStreamLinkHandlerFactory() {
}
public static BandcampStreamLinkHandlerFactory getInstance() {
return INSTANCE;
}
/**
* @see BandcampStreamLinkHandlerFactory
*/
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
if (BandcampExtractorHelper.isRadioUrl(url)) {
return url.split("bandcamp.com/\\?show=")[1];
} else {
@ -34,7 +44,8 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory {
* @see BandcampStreamLinkHandlerFactory
*/
@Override
public String getUrl(final String input) {
public String getUrl(final String input)
throws ParsingException, UnsupportedOperationException {
if (input.matches("\\d+")) {
return BASE_URL + "/?show=" + input;
} else {

View File

@ -6,6 +6,7 @@ import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
@ -13,6 +14,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
@ -27,8 +29,6 @@ import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearch
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCLiveListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCRecentListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
@ -47,12 +47,17 @@ public class MediaCCCService extends StreamingService {
@Override
public LinkHandlerFactory getStreamLHFactory() {
return new MediaCCCStreamLinkHandlerFactory();
return MediaCCCStreamLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getChannelLHFactory() {
return new MediaCCCConferenceLinkHandlerFactory();
return MediaCCCConferenceLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getChannelTabLHFactory() {
return null;
}
@Override
@ -62,7 +67,7 @@ public class MediaCCCService extends StreamingService {
@Override
public SearchQueryHandlerFactory getSearchQHFactory() {
return new MediaCCCSearchQueryHandlerFactory();
return MediaCCCSearchQueryHandlerFactory.getInstance();
}
@Override
@ -78,6 +83,22 @@ public class MediaCCCService extends StreamingService {
return new MediaCCCConferenceExtractor(this, linkHandler);
}
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
}
/*
Channel tab extractors are only supported in conferences and should only come from a
ReadyChannelTabListLinkHandler instance with a ChannelTabExtractorBuilder instance of the
conferences extractor
If that's not the case, return null in this case, so no channel tabs support
*/
return null;
}
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return null;
@ -91,36 +112,37 @@ public class MediaCCCService extends StreamingService {
@Override
public KioskList getKioskList() throws ExtractionException {
final KioskList list = new KioskList(this);
final ListLinkHandlerFactory h = MediaCCCConferencesListLinkHandlerFactory.getInstance();
// add kiosks here e.g.:
try {
list.addKioskEntry(
(streamingService, url, kioskId) -> new MediaCCCConferenceKiosk(
MediaCCCService.this,
new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url),
h.fromUrl(url),
kioskId
),
new MediaCCCConferencesListLinkHandlerFactory(),
h,
MediaCCCConferenceKiosk.KIOSK_ID
);
list.addKioskEntry(
(streamingService, url, kioskId) -> new MediaCCCRecentKiosk(
MediaCCCService.this,
new MediaCCCRecentListLinkHandlerFactory().fromUrl(url),
h.fromUrl(url),
kioskId
),
new MediaCCCRecentListLinkHandlerFactory(),
h,
MediaCCCRecentKiosk.KIOSK_ID
);
list.addKioskEntry(
(streamingService, url, kioskId) -> new MediaCCCLiveStreamKiosk(
MediaCCCService.this,
new MediaCCCLiveListLinkHandlerFactory().fromUrl(url),
h.fromUrl(url),
kioskId
),
new MediaCCCLiveListLinkHandlerFactory(),
h,
MediaCCCLiveStreamKiosk.KIOSK_ID
);

View File

@ -1,23 +1,29 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException;
import java.util.List;
import javax.annotation.Nonnull;
import java.io.IOException;
public class MediaCCCConferenceExtractor extends ChannelExtractor {
private JsonObject conferenceData;
@ -74,18 +80,9 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray events = conferenceData.getArray("events");
for (int i = 0; i < events.size(); i++) {
collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i)));
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
return InfoItemsPage.emptyPage();
public List<ListLinkHandler> getTabs() throws ParsingException {
return List.of(new ReadyChannelTabListLinkHandler(getUrl(), getId(),
ChannelTabs.VIDEOS, new VideosTabExtractorBuilder(conferenceData)));
}
@Override
@ -105,4 +102,55 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
public String getName() throws ParsingException {
return conferenceData.getString("title");
}
private static final class VideosTabExtractorBuilder
implements ReadyChannelTabListLinkHandler.ChannelTabExtractorBuilder {
private final JsonObject conferenceData;
VideosTabExtractorBuilder(final JsonObject conferenceData) {
this.conferenceData = conferenceData;
}
@Nonnull
@Override
public ChannelTabExtractor build(@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler) {
return new VideosChannelTabExtractor(service, linkHandler, conferenceData);
}
}
private static final class VideosChannelTabExtractor extends ChannelTabExtractor {
private final JsonObject conferenceData;
VideosChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final JsonObject conferenceData) {
super(service, linkHandler);
this.conferenceData = conferenceData;
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) {
// Nothing to do here, as data was already fetched
}
@Nonnull
@Override
public ListExtractor.InfoItemsPage<InfoItem> getInitialPage() {
final MultiInfoItemsCollector collector =
new MultiInfoItemsCollector(getServiceId());
conferenceData.getArray("events")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.forEach(event -> collector.commit(new MediaCCCStreamInfoItemExtractor(event)));
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page) {
return InfoItemsPage.emptyPage();
}
}
}

View File

@ -65,7 +65,7 @@ public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
return new MediaCCCConferenceLinkHandlerFactory()
return MediaCCCConferenceLinkHandlerFactory.getInstance()
.fromUrl(event.getString("conference_url")) // API URL
.getUrl(); // web URL
}

View File

@ -38,7 +38,8 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
super(service, linkHandler);
try {
conferenceKiosk = new MediaCCCConferenceKiosk(service,
new MediaCCCConferencesListLinkHandlerFactory().fromId("conferences"),
MediaCCCConferencesListLinkHandlerFactory.getInstance()
.fromId("conferences"),
"conferences");
} catch (final Exception e) {
e.printStackTrace();

View File

@ -6,7 +6,11 @@ import org.schabi.newpipe.extractor.utils.Parser;
import java.util.List;
public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory {
public final class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory {
private static final MediaCCCConferenceLinkHandlerFactory INSTANCE
= new MediaCCCConferenceLinkHandlerFactory();
public static final String CONFERENCE_API_ENDPOINT
= "https://api.media.ccc.de/public/conferences/";
public static final String CONFERENCE_PATH = "https://media.ccc.de/c/";
@ -14,15 +18,23 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory
= "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/conferences/)"
+ "|(?:media\\.ccc\\.de/[bc]/))([^/?&#]*)";
private MediaCCCConferenceLinkHandlerFactory() {
}
public static MediaCCCConferenceLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return CONFERENCE_PATH + id;
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return Parser.matchGroup1(ID_PATTERN, url);
}

View File

@ -5,15 +5,28 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List;
public class MediaCCCConferencesListLinkHandlerFactory extends ListLinkHandlerFactory {
public final class MediaCCCConferencesListLinkHandlerFactory extends ListLinkHandlerFactory {
private static final MediaCCCConferencesListLinkHandlerFactory INSTANCE =
new MediaCCCConferencesListLinkHandlerFactory();
private MediaCCCConferencesListLinkHandlerFactory() {
}
public static MediaCCCConferencesListLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return "conferences";
}
@Override
public String getUrl(final String id, final List<String> contentFilter,
final String sortFilter) throws ParsingException {
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://media.ccc.de/public/conferences";
}

View File

@ -6,11 +6,22 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List;
import java.util.regex.Pattern;
public class MediaCCCLiveListLinkHandlerFactory extends ListLinkHandlerFactory {
public final class MediaCCCLiveListLinkHandlerFactory extends ListLinkHandlerFactory {
private static final MediaCCCLiveListLinkHandlerFactory INSTANCE =
new MediaCCCLiveListLinkHandlerFactory();
private static final String STREAM_PATTERN = "^(?:https?://)?media\\.ccc\\.de/live$";
private MediaCCCLiveListLinkHandlerFactory() {
}
public static MediaCCCLiveListLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return "live";
}
@ -22,7 +33,8 @@ public class MediaCCCLiveListLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
// FIXME: wrong URL; should be https://streaming.media.ccc.de/{conference_slug}/{room_slug}
return "https://media.ccc.de/live";
}

View File

@ -1,32 +0,0 @@
package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
public class MediaCCCLiveStreamLinkHandlerFactory extends LinkHandlerFactory {
public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/";
private static final String VIDEO_PATH = "https://streaming.media.ccc.de/v/";
private static final String ID_PATTERN
= "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)"
+ "|(?:media\\.ccc\\.de/v/))([^/?&#]*)";
@Override
public String getId(final String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);
}
@Override
public String getUrl(final String id) throws ParsingException {
return VIDEO_PATH + id;
}
@Override
public boolean onAcceptUrl(final String url) {
try {
return getId(url) != null;
} catch (final ParsingException e) {
return false;
}
}
}

View File

@ -1,15 +1,27 @@
package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List;
import java.util.regex.Pattern;
public class MediaCCCRecentListLinkHandlerFactory extends ListLinkHandlerFactory {
public final class MediaCCCRecentListLinkHandlerFactory extends ListLinkHandlerFactory {
private static final MediaCCCRecentListLinkHandlerFactory INSTANCE =
new MediaCCCRecentListLinkHandlerFactory();
private static final String PATTERN = "^(https?://)?media\\.ccc\\.de/recent/?$";
private MediaCCCRecentListLinkHandlerFactory() {
}
public static MediaCCCRecentListLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String getId(final String url) {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return "recent";
}
@ -21,7 +33,8 @@ public class MediaCCCRecentListLinkHandlerFactory extends ListLinkHandlerFactory
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://media.ccc.de/recent";
}
}

View File

@ -7,11 +7,22 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.UnsupportedEncodingException;
import java.util.List;
public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public final class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
private static final MediaCCCSearchQueryHandlerFactory INSTANCE =
new MediaCCCSearchQueryHandlerFactory();
public static final String ALL = "all";
public static final String CONFERENCES = "conferences";
public static final String EVENTS = "events";
private MediaCCCSearchQueryHandlerFactory() {
}
public static MediaCCCSearchQueryHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String[] getAvailableContentFilter() {
return new String[]{
@ -27,8 +38,10 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory
}
@Override
public String getUrl(final String query, final List<String> contentFilter,
final String sortFilter) throws ParsingException {
public String getUrl(final String query,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try {
return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query);
} catch (final UnsupportedEncodingException e) {

View File

@ -5,7 +5,11 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper;
import org.schabi.newpipe.extractor.utils.Parser;
public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
public final class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final MediaCCCStreamLinkHandlerFactory INSTANCE =
new MediaCCCStreamLinkHandlerFactory();
public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/";
private static final String VIDEO_PATH = "https://media.ccc.de/v/";
private static final String RECORDING_ID_PATTERN
@ -15,8 +19,15 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final String LIVE_STREAM_ID_PATTERN
= "streaming\\.media\\.ccc\\.de\\/(\\w+\\/\\w+)";
private MediaCCCStreamLinkHandlerFactory() {
}
public static MediaCCCStreamLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
String streamId = null;
try {
streamId = Parser.matchGroup1(LIVE_STREAM_ID_PATTERN, url);
@ -30,7 +41,7 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
}
@Override
public String getUrl(final String id) throws ParsingException {
public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
if (MediaCCCParsingHelper.isLiveStreamId(id)) {
return LIVE_STREAM_PATH + id;
}

View File

@ -72,24 +72,29 @@ public final class PeertubeParsingHelper {
}
}
public static void collectStreamsFrom(final InfoItemsCollector collector,
final JsonObject json,
final String baseUrl) throws ParsingException {
collectStreamsFrom(collector, json, baseUrl, false);
public static void collectItemsFrom(final InfoItemsCollector collector,
final JsonObject json,
final String baseUrl) throws ParsingException {
collectItemsFrom(collector, json, baseUrl, false);
}
/**
* Collect stream from json with collector
* Collect items from the given JSON object with the given collector.
*
* <p>
* Supported info item types are streams with their Sepia variant, channels and playlists.
* </p>
*
* @param collector the collector used to collect information
* @param json the file to retrieve data from
* @param baseUrl the base Url of the instance
* @param sepia if we should use PeertubeSepiaStreamInfoItemExtractor
* @param json the JSOn response to retrieve data from
* @param baseUrl the base URL of the instance
* @param sepia if we should use {@code PeertubeSepiaStreamInfoItemExtractor} to extract
* streams or {@code PeertubeStreamInfoItemExtractor} otherwise
*/
public static void collectStreamsFrom(final InfoItemsCollector collector,
final JsonObject json,
final String baseUrl,
final boolean sepia) throws ParsingException {
public static void collectItemsFrom(final InfoItemsCollector collector,
final JsonObject json,
final String baseUrl,
final boolean sepia) throws ParsingException {
final JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");

View File

@ -6,6 +6,7 @@ import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
@ -19,6 +20,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelTabExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor;
@ -26,6 +28,7 @@ import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamE
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSuggestionExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
@ -60,6 +63,11 @@ public class PeertubeService extends StreamingService {
return PeertubeChannelLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getChannelTabLHFactory() {
return PeertubeChannelTabLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getPlaylistLHFactory() {
return PeertubePlaylistLinkHandlerFactory.getInstance();
@ -103,6 +111,12 @@ public class PeertubeService extends StreamingService {
}
}
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler)
throws ExtractionException {
return new PeertubeChannelTabExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler)
throws ExtractionException {
@ -136,17 +150,20 @@ public class PeertubeService extends StreamingService {
@Override
public KioskList getKioskList() throws ExtractionException {
final PeertubeTrendingLinkHandlerFactory h =
PeertubeTrendingLinkHandlerFactory.getInstance();
final KioskList.KioskExtractorFactory kioskFactory = (streamingService, url, id) ->
new PeertubeTrendingExtractor(
PeertubeService.this,
new PeertubeTrendingLinkHandlerFactory().fromId(id),
h.fromId(id),
id
);
final KioskList list = new KioskList(this);
// add kiosks here e.g.:
final PeertubeTrendingLinkHandlerFactory h = new PeertubeTrendingLinkHandlerFactory();
try {
list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING);
list.addKioskEntry(kioskFactory, h,
@ -160,6 +177,4 @@ public class PeertubeService extends StreamingService {
return list;
}
}

View File

@ -4,31 +4,23 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import java.util.List;
public class PeertubeAccountExtractor extends ChannelExtractor {
private JsonObject json;
@ -119,54 +111,19 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
return getPage(new Page(baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&"
+ COUNT_KEY + "=" + ITEMS_PER_PAGE));
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
JsonObject pageJson = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
pageJson = JsonParser.object().from(response.responseBody());
} catch (final Exception e) {
throw new ParsingException("Could not parse json data for account info", e);
}
}
if (pageJson != null) {
PeertubeParsingHelper.validate(pageJson);
final long total = pageJson.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, pageJson, getBaseUrl());
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {
throw new ExtractionException("Unable to get PeerTube account info");
}
public List<ListLinkHandler> getTabs() throws ParsingException {
return List.of(
PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(),
List.of(ChannelTabs.VIDEOS), "", getBaseUrl()),
PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(),
List.of(ChannelTabs.CHANNELS), "", getBaseUrl()));
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
String accountUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT;
if (getId().contains(ACCOUNTS)) {
accountUrl += getId();
} else {
accountUrl += ACCOUNTS + getId();
}
final Response response = downloader.get(accountUrl);
final Response response = downloader.get(baseUrl
+ PeertubeChannelLinkHandlerFactory.API_ENDPOINT + getId());
if (response != null) {
setInitialData(response.responseBody());
} else {

View File

@ -3,30 +3,22 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import java.util.List;
public class PeertubeChannelExtractor extends ChannelExtractor {
private JsonObject json;
@ -98,41 +90,12 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
return getPage(new Page(baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&"
+ COUNT_KEY + "=" + ITEMS_PER_PAGE));
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
JsonObject pageJson = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
pageJson = JsonParser.object().from(response.responseBody());
} catch (final Exception e) {
throw new ParsingException("Could not parse json data for channel info", e);
}
}
if (pageJson != null) {
PeertubeParsingHelper.validate(pageJson);
final long total = pageJson.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, pageJson, getBaseUrl());
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {
throw new ExtractionException("Unable to get PeerTube channel info");
}
public List<ListLinkHandler> getTabs() throws ParsingException {
return List.of(
PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(),
List.of(ChannelTabs.VIDEOS), "", getBaseUrl()),
PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(),
List.of(ChannelTabs.PLAYLISTS), "", getBaseUrl()));
}
@Override

View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -52,7 +52,7 @@ public class PeertubeChannelInfoItemExtractor implements ChannelInfoItemExtracto
@Override
public long getStreamCount() throws ParsingException {
return ChannelExtractor.ITEM_COUNT_UNKNOWN;
return ListExtractor.ITEM_COUNT_UNKNOWN;
}
@Override

View File

@ -0,0 +1,80 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom;
import static org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory.getUrlSuffix;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeChannelTabExtractor extends ChannelTabExtractor {
private final String baseUrl;
public PeertubeChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler)
throws ParsingException {
super(service, linkHandler);
baseUrl = getBaseUrl();
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) {
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return getPage(new Page(baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT
+ getId() + getUrlSuffix(getName()) + "?" + START_KEY + "=0&" + COUNT_KEY + "="
+ ITEMS_PER_PAGE));
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
JsonObject pageJson = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
pageJson = JsonParser.object().from(response.responseBody());
} catch (final Exception e) {
throw new ParsingException("Could not parse json data for account info", e);
}
}
if (pageJson == null) {
throw new ExtractionException("Unable to get account channel list");
}
PeertubeParsingHelper.validate(pageJson);
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
collectItemsFrom(collector, pageJson, getBaseUrl());
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), pageJson.getLong("total")));
}
}

View File

@ -23,7 +23,7 @@ import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubePlaylistExtractor extends PlaylistExtractor {
@ -125,7 +125,7 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor {
final long total = json.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl());
collectItemsFrom(collector, json, getBaseUrl());
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total));

View File

@ -26,7 +26,7 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeSearchExtractor extends SearchExtractor {
@ -93,7 +93,7 @@ public class PeertubeSearchExtractor extends SearchExtractor {
final long total = json.getLong("total");
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl(), sepia);
collectItemsFrom(collector, json, getBaseUrl(), sepia);
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total));

View File

@ -23,7 +23,7 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@ -69,7 +69,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
final long total = json.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl());
collectItemsFrom(collector, json, getBaseUrl());
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total));

View File

@ -22,14 +22,15 @@ public final class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFact
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return fixId(Parser.matchGroup(ID_PATTERN, url, 0));
}
@Override
public String getUrl(final String id,
final List<String> contentFilters,
final String searchFilter) throws ParsingException {
final String searchFilter)
throws ParsingException, UnsupportedOperationException {
return getUrl(id, contentFilters, searchFilter, ServiceList.PeerTube.getBaseUrl());
}
@ -38,7 +39,7 @@ public final class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFact
final List<String> contentFilter,
final String sortFilter,
final String baseUrl)
throws ParsingException {
throws ParsingException, UnsupportedOperationException {
if (id.matches(ID_PATTERN)) {
return baseUrl + "/" + fixId(id);
} else {

View File

@ -0,0 +1,71 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.UnsupportedTabException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import javax.annotation.Nonnull;
import java.util.List;
public final class PeertubeChannelTabLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubeChannelTabLinkHandlerFactory INSTANCE
= new PeertubeChannelTabLinkHandlerFactory();
private PeertubeChannelTabLinkHandlerFactory() {
}
public static PeertubeChannelTabLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Nonnull
public static String getUrlSuffix(@Nonnull final String tab)
throws UnsupportedTabException {
switch (tab) {
case ChannelTabs.VIDEOS:
return "/videos";
case ChannelTabs.CHANNELS: // only available on accounts
return "/video-channels";
case ChannelTabs.PLAYLISTS: // only available on channels
return "/video-playlists";
}
throw new UnsupportedTabException(tab);
}
@Override
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return PeertubeChannelLinkHandlerFactory.getInstance().getId(url);
}
@Override
public String getUrl(final String id, final List<String> contentFilter, final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return PeertubeChannelLinkHandlerFactory.getInstance().getUrl(id)
+ getUrlSuffix(contentFilter.get(0));
}
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter,
final String baseUrl)
throws ParsingException, UnsupportedOperationException {
return PeertubeChannelLinkHandlerFactory.getInstance().getUrl(id, null, null, baseUrl)
+ getUrlSuffix(contentFilter.get(0));
}
@Override
public boolean onAcceptUrl(final String url) throws ParsingException {
return PeertubeChannelLinkHandlerFactory.getInstance().onAcceptUrl(url);
}
@Override
public String[] getAvailableContentFilter() {
return new String[] {
ChannelTabs.VIDEOS,
ChannelTabs.CHANNELS,
ChannelTabs.PLAYLISTS,
};
}
}

View File

@ -21,7 +21,7 @@ public final class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFac
}
@Override
public String getId(final String url) throws ParsingException, IllegalArgumentException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return PeertubeStreamLinkHandlerFactory.getInstance().getId(url); // the same id is needed
}
@ -33,7 +33,8 @@ public final class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFac
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return getUrl(id, contentFilter, sortFilter, ServiceList.PeerTube.getBaseUrl());
}
@ -41,7 +42,8 @@ public final class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFac
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter,
final String baseUrl) throws ParsingException {
final String baseUrl)
throws ParsingException, UnsupportedOperationException {
return baseUrl + String.format(COMMENTS_ENDPOINT, id);
}

View File

@ -25,7 +25,8 @@ public final class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFac
@Override
public String getUrl(final String id,
final List<String> contentFilters,
final String sortFilter) {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return getUrl(id, contentFilters, sortFilter, ServiceList.PeerTube.getBaseUrl());
}
@ -33,12 +34,13 @@ public final class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFac
public String getUrl(final String id,
final List<String> contentFilters,
final String sortFilter,
final String baseUrl) {
final String baseUrl)
throws ParsingException, UnsupportedOperationException {
return baseUrl + "/api/v1/video-playlists/" + id;
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
try {
return Parser.matchGroup(ID_PATTERN, url, 2);
} catch (final ParsingException ignored) {

View File

@ -10,6 +10,9 @@ import java.util.List;
public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
private static final PeertubeSearchQueryHandlerFactory INSTANCE =
new PeertubeSearchQueryHandlerFactory();
public static final String VIDEOS = "videos";
public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index
public static final String PLAYLISTS = "playlists";
@ -23,13 +26,14 @@ public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerF
}
public static PeertubeSearchQueryHandlerFactory getInstance() {
return new PeertubeSearchQueryHandlerFactory();
return INSTANCE;
}
@Override
public String getUrl(final String searchString,
final List<String> contentFilters,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
final String baseUrl;
if (!contentFilters.isEmpty() && contentFilters.get(0).startsWith("sepia_")) {
baseUrl = SEPIA_BASE_URL;
@ -43,7 +47,8 @@ public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerF
public String getUrl(final String searchString,
final List<String> contentFilters,
final String sortFilter,
final String baseUrl) throws ParsingException {
final String baseUrl)
throws ParsingException, UnsupportedOperationException {
try {
final String endpoint;
if (contentFilters.isEmpty()

View File

@ -27,7 +27,7 @@ public final class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
}
@Override
public String getUrl(final String id) {
public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
return getUrl(id, ServiceList.PeerTube.getBaseUrl());
}
@ -37,7 +37,7 @@ public final class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
}
@Override
public String getId(final String url) throws ParsingException, IllegalArgumentException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return Parser.matchGroup(ID_PATTERN, url, 4);
}

View File

@ -23,6 +23,9 @@ public final class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFac
KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt",
KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local");
private PeertubeTrendingLinkHandlerFactory() {
}
public static PeertubeTrendingLinkHandlerFactory getInstance() {
return INSTANCE;
}
@ -30,7 +33,8 @@ public final class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFac
@Override
public String getUrl(final String id,
final List<String> contentFilters,
final String sortFilter) {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return getUrl(id, contentFilters, sortFilter, ServiceList.PeerTube.getBaseUrl());
}
@ -38,12 +42,13 @@ public final class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFac
public String getUrl(final String id,
final List<String> contentFilters,
final String sortFilter,
final String baseUrl) {
final String baseUrl)
throws ParsingException, UnsupportedOperationException {
return String.format(KIOSK_MAP.get(id), baseUrl);
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
final String cleanUrl = url.replace(ServiceList.PeerTube.getBaseUrl(), "%s");
if (cleanUrl.contains("/videos/trending")) {
return KIOSK_TRENDING;

View File

@ -8,6 +8,7 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -16,6 +17,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
@ -300,6 +302,54 @@ public final class SoundcloudParsingHelper {
return getStreamsFromApi(collector, apiUrl, false);
}
public static String getInfoItemsFromApi(final MultiInfoItemsCollector collector,
final String apiUrl) throws ReCaptchaException,
ParsingException, IOException {
final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization());
if (response.responseCode() >= 400) {
throw new IOException("Could not get streams from API, HTTP "
+ response.responseCode());
}
final JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response.responseBody());
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
responseObject.getArray("collection")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.forEach(searchResult -> {
final String kind = searchResult.getString("kind", "");
switch (kind) {
case "user":
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
break;
case "track":
collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult));
break;
case "playlist":
collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult));
break;
}
});
String nextPageUrl;
try {
nextPageUrl = responseObject.getString("next_href");
if (!nextPageUrl.contains("client_id=")) {
nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
}
} catch (final Exception ignored) {
nextPageUrl = "";
}
return nextPageUrl;
}
@Nonnull
public static String getUploaderUrl(final JsonObject object) {
final String url = object.getObject("user").getString("permalink_url", "");
@ -312,6 +362,7 @@ public final class SoundcloudParsingHelper {
return replaceHttpWithHttps(url);
}
@Nonnull
public static String getUploaderName(final JsonObject object) {
return object.getObject("user").getString("username", "");
}

View File

@ -6,6 +6,7 @@ import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
@ -19,6 +20,7 @@ import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelTabExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChartsExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCommentsExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistExtractor;
@ -27,6 +29,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStr
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudSubscriptionExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudSuggestionExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChartsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudPlaylistLinkHandlerFactory;
@ -50,7 +53,7 @@ public class SoundcloudService extends StreamingService {
@Override
public SearchQueryHandlerFactory getSearchQHFactory() {
return new SoundcloudSearchQueryHandlerFactory();
return SoundcloudSearchQueryHandlerFactory.getInstance();
}
@Override
@ -63,6 +66,11 @@ public class SoundcloudService extends StreamingService {
return SoundcloudChannelLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getChannelTabLHFactory() {
return SoundcloudChannelTabLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getPlaylistLHFactory() {
return SoundcloudPlaylistLinkHandlerFactory.getInstance();
@ -86,6 +94,11 @@ public class SoundcloudService extends StreamingService {
return new SoundcloudChannelExtractor(this, linkHandler);
}
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
return new SoundcloudChannelTabExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return new SoundcloudPlaylistExtractor(this, linkHandler);
@ -103,14 +116,15 @@ public class SoundcloudService extends StreamingService {
@Override
public KioskList getKioskList() throws ExtractionException {
final KioskList.KioskExtractorFactory chartsFactory = (streamingService, url, id) ->
new SoundcloudChartsExtractor(SoundcloudService.this,
new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id);
final KioskList list = new KioskList(this);
final SoundcloudChartsLinkHandlerFactory h =
SoundcloudChartsLinkHandlerFactory.getInstance();
final KioskList.KioskExtractorFactory chartsFactory = (streamingService, url, id) ->
new SoundcloudChartsExtractor(SoundcloudService.this,
h.fromUrl(url), id);
// add kiosks here e.g.:
final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory();
try {
list.addKioskEntry(chartsFactory, h, "Top 50");
list.addKioskEntry(chartsFactory, h, "New & hot");

View File

@ -1,24 +1,23 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChannelTabLinkHandlerFactory;
import java.io.IOException;
import java.util.List;
import javax.annotation.Nonnull;
@ -108,34 +107,22 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
try {
final StreamInfoItemsCollector streamInfoItemsCollector =
new StreamInfoItemsCollector(getServiceId());
public List<ListLinkHandler> getTabs() throws ParsingException {
final String url = getUrl();
final String urlTracks = url
+ SoundcloudChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.TRACKS);
final String urlPlaylists = url
+ SoundcloudChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.PLAYLISTS);
final String urlAlbums = url
+ SoundcloudChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.ALBUMS);
final String id = getId();
final String apiUrl = USERS_ENDPOINT + getId() + "/tracks" + "?client_id="
+ SoundcloudParsingHelper.clientId() + "&limit=20" + "&linked_partitioning=1";
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15,
streamInfoItemsCollector, apiUrl);
return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl));
} catch (final Exception e) {
throw new ExtractionException("Could not get next page", e);
}
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector,
page.getUrl());
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
return List.of(
new ListLinkHandler(urlTracks, urlTracks, id,
List.of(ChannelTabs.TRACKS), ""),
new ListLinkHandler(urlPlaylists, urlPlaylists, id,
List.of(ChannelTabs.PLAYLISTS), ""),
new ListLinkHandler(urlAlbums, urlAlbums, id,
List.of(ChannelTabs.ALBUMS), ""));
}
}

View File

@ -0,0 +1,76 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudChannelTabExtractor extends ChannelTabExtractor {
private static final String USERS_ENDPOINT = SOUNDCLOUD_API_V2_URL + "users/";
private final String userId;
public SoundcloudChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
userId = getLinkHandler().getId();
}
@Nonnull
private String getEndpoint() throws ParsingException {
switch (getName()) {
case ChannelTabs.TRACKS:
return "/tracks";
case ChannelTabs.PLAYLISTS:
return "/playlists_without_albums";
case ChannelTabs.ALBUMS:
return "/albums";
}
throw new ParsingException("Unsupported tab: " + getName());
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) {
}
@Nonnull
@Override
public String getId() {
return userId;
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return getPage(new Page(USERS_ENDPOINT + userId + getEndpoint() + "?client_id="
+ SoundcloudParsingHelper.clientId() + "&limit=20" + "&linked_partitioning=1"));
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getInfoItemsFromApi(
collector, page.getUrl());
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
}
}

View File

@ -23,7 +23,7 @@ public final class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFa
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
Utils.checkUrl(URL_PATTERN, url);
try {
@ -36,7 +36,8 @@ public final class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFa
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
"https://api.soundcloud.com/users/" + id);

View File

@ -0,0 +1,61 @@
package org.schabi.newpipe.extractor.services.soundcloud.linkHandler;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.UnsupportedTabException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import javax.annotation.Nonnull;
import java.util.List;
public final class SoundcloudChannelTabLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudChannelTabLinkHandlerFactory INSTANCE
= new SoundcloudChannelTabLinkHandlerFactory();
private SoundcloudChannelTabLinkHandlerFactory() {
}
public static SoundcloudChannelTabLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Nonnull
public static String getUrlSuffix(final String tab) throws UnsupportedOperationException {
switch (tab) {
case ChannelTabs.TRACKS:
return "/tracks";
case ChannelTabs.PLAYLISTS:
return "/sets";
case ChannelTabs.ALBUMS:
return "/albums";
}
throw new UnsupportedTabException(tab);
}
@Override
public String getId(final String url) throws ParsingException {
return SoundcloudChannelLinkHandlerFactory.getInstance().getId(url);
}
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
return SoundcloudChannelLinkHandlerFactory.getInstance().getUrl(id)
+ getUrlSuffix(contentFilter.get(0));
}
@Override
public boolean onAcceptUrl(final String url) throws ParsingException {
return SoundcloudChannelLinkHandlerFactory.getInstance().onAcceptUrl(url);
}
@Override
public String[] getAvailableContentFilter() {
return new String[] {
ChannelTabs.TRACKS,
ChannelTabs.PLAYLISTS,
ChannelTabs.ALBUMS,
};
}
}

View File

@ -1,18 +1,30 @@
package org.schabi.newpipe.extractor.services.soundcloud.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
import java.util.List;
public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory {
public final class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudChartsLinkHandlerFactory INSTANCE =
new SoundcloudChartsLinkHandlerFactory();
private static final String TOP_URL_PATTERN =
"^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
private static final String URL_PATTERN =
"^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
private SoundcloudChartsLinkHandlerFactory() {
}
public static SoundcloudChartsLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String getId(final String url) {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
return "Top 50";
} else {
@ -23,7 +35,8 @@ public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
if (id.equals("Top 50")) {
return "https://soundcloud.com/charts/top";
} else {

View File

@ -24,7 +24,8 @@ public final class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerF
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try {
return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id="
+ clientId() + "&threaded=0" + "&filter_replies=1";
@ -37,7 +38,7 @@ public final class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerF
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
// Delegation to avoid duplicate code, as we need the same id
return SoundcloudStreamLinkHandlerFactory.getInstance().getId(url);
}

View File

@ -22,7 +22,7 @@ public final class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerF
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
Utils.checkUrl(URL_PATTERN, url);
try {
@ -37,7 +37,7 @@ public final class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerF
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException {
throws ParsingException, UnsupportedOperationException {
try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
"https://api.soundcloud.com/playlists/" + id);

View File

@ -13,7 +13,10 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.List;
public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public final class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
private static final SoundcloudSearchQueryHandlerFactory INSTANCE =
new SoundcloudSearchQueryHandlerFactory();
public static final String TRACKS = "tracks";
public static final String USERS = "users";
@ -22,11 +25,18 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
public static final int ITEMS_PER_PAGE = 10;
private SoundcloudSearchQueryHandlerFactory() {
}
public static SoundcloudSearchQueryHandlerFactory getInstance() {
return INSTANCE;
}
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException {
throws ParsingException, UnsupportedOperationException {
try {
String url = SOUNDCLOUD_API_V2_URL + "search";

View File

@ -21,7 +21,7 @@ public final class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory
}
@Override
public String getUrl(final String id) throws ParsingException {
public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
"https://api.soundcloud.com/tracks/" + id);
@ -31,7 +31,7 @@ public final class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
if (Parser.isMatch(API_URL_PATTERN, url)) {
return Parser.matchGroup1(API_URL_PATTERN, url);
}

View File

@ -0,0 +1,271 @@
package org.schabi.newpipe.extractor.services.youtube;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* Shared functions for extracting YouTube channel pages and tabs.
*/
public final class YoutubeChannelHelper {
private YoutubeChannelHelper() {
}
/**
* Take a YouTube channel ID or URL path, resolve it if necessary and return a channel ID.
*
* @param idOrPath a YouTube channel ID or URL path
* @return a YouTube channel ID
* @throws IOException if a channel resolve request failed
* @throws ExtractionException if a channel resolve request response could not be parsed or is
* invalid
*/
@Nonnull
public static String resolveChannelId(@Nonnull final String idOrPath)
throws ExtractionException, IOException {
final String[] channelId = idOrPath.split("/");
if (channelId[0].startsWith("UC")) {
return channelId[0];
}
// If the URL is not a /channel URL, we need to use the navigation/resolve_url endpoint of
// the InnerTube API to get the channel id.
// Otherwise, we couldn't get information about the channel associated with this URL, if
// there is one.
if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(
prepareDesktopJsonBuilder(Localization.DEFAULT, ContentCountry.DEFAULT)
.value("url", "https://www.youtube.com/" + idOrPath)
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject jsonResponse = getJsonPostResponse(
"navigation/resolve_url", body, Localization.DEFAULT);
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
final String browseId = browseEndpoint.getString("browseId", "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
return browseId;
}
}
return channelId[1];
}
/**
* Response data object for {@link #getChannelResponse(String, String, Localization,
* ContentCountry)}, after any redirection in the allowed redirects count ({@code 3}).
*/
public static final class ChannelResponseData {
/**
* The channel response as a JSON object, after all redirects.
*/
@Nonnull
public final JsonObject jsonResponse;
/**
* The channel ID after all redirects.
*/
@Nonnull
public final String channelId;
private ChannelResponseData(@Nonnull final JsonObject jsonResponse,
@Nonnull final String channelId) {
this.jsonResponse = jsonResponse;
this.channelId = channelId;
}
}
/**
* Fetch a YouTube channel tab response, using the given channel ID and tab parameters.
*
* <p>
* Redirections to other channels such as are supported to up to 3 redirects, which could
* happen for instance for localized channels or auto-generated ones such as the {@code Movies
* and Shows} (channel IDs {@code UCuJcl0Ju-gPDoksRjK1ya-w}, {@code UChBfWrfBXL9wS6tQtgjt_OQ}
* and {@code UCok7UTQQEP1Rsctxiv3gwSQ} of this channel redirect to the
* {@code UClgRkhTL3_hImCAmdLfDE4g} one).
* </p>
*
* @param channelId a valid YouTube channel ID
* @param parameters the parameters to specify the YouTube channel tab; if invalid ones are
* specified, YouTube should return the {@code Home} tab
* @param localization the {@link Localization} to use
* @param country the {@link ContentCountry} to use
* @return a {@link ChannelResponseData channel response data}
* @throws IOException if a channel request failed
* @throws ExtractionException if a channel request response could not be parsed or is invalid
*/
@Nonnull
public static ChannelResponseData getChannelResponse(@Nonnull final String channelId,
@Nonnull final String parameters,
@Nonnull final Localization localization,
@Nonnull final ContentCountry country)
throws ExtractionException, IOException {
String id = channelId;
JsonObject ajaxJson = null;
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
localization, country)
.value("browseId", id)
.value("params", parameters)
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject jsonResponse = getJsonPostResponse(
"browse", body, localization);
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("navigateAction")
.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final String browseId = endpoint.getObject("browseEndpoint")
.getString("browseId", "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
level++;
} else {
ajaxJson = jsonResponse;
break;
}
}
if (ajaxJson == null) {
throw new ExtractionException("Got no channel response");
}
defaultAlertsCheck(ajaxJson);
return new ChannelResponseData(ajaxJson, id);
}
/**
* Assert that a channel JSON response does not contain an {@code error} JSON object.
*
* @param jsonResponse a channel JSON response
* @throws ContentNotAvailableException if the channel was not found
*/
private static void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse)
throws ContentNotAvailableException {
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
}
/**
* A channel header response.
*
* <p>
* This class allows the distinction between a classic header and a carousel one, used for
* auto-generated ones like the gaming or music topic channels and for big events such as the
* Coachella music festival, which have a different data structure and do not return the same
* properties.
* </p>
*/
public static final class ChannelHeader {
/**
* The channel header JSON response.
*/
@Nonnull
public final JsonObject json;
/**
* Whether the header is a {@code carouselHeaderRenderer}.
*
* <p>
* See the class documentation for more details.
* </p>
*/
public final boolean isCarouselHeader;
private ChannelHeader(@Nonnull final JsonObject json, final boolean isCarouselHeader) {
this.json = json;
this.isCarouselHeader = isCarouselHeader;
}
}
/**
* Get a channel header as an {@link Optional} it if exists.
*
* @param channelResponse a full channel JSON response
* @return an {@link Optional} containing a {@link ChannelHeader} or an empty {@link Optional}
* if no supported header has been found
*/
@Nonnull
public static Optional<ChannelHeader> getChannelHeader(
@Nonnull final JsonObject channelResponse) {
final JsonObject header = channelResponse.getObject("header");
if (header.has("c4TabbedHeaderRenderer")) {
return Optional.of(header.getObject("c4TabbedHeaderRenderer"))
.map(json -> new ChannelHeader(json, false));
} else if (header.has("carouselHeaderRenderer")) {
return header.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(item -> item.has("topicChannelDetailsRenderer"))
.findFirst()
.map(item -> item.getObject("topicChannelDetailsRenderer"))
.map(json -> new ChannelHeader(json, true));
} else {
return Optional.empty();
}
}
}

View File

@ -1230,8 +1230,17 @@ public final class YoutubeParsingHelper {
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry)
throws IOException, ExtractionException {
return prepareDesktopJsonBuilder(localization, contentCountry, null);
}
@Nonnull
public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nullable final String visitorData)
throws IOException, ExtractionException {
// @formatter:off
return JsonObject.builder()
final JsonBuilder<JsonObject> builder = JsonObject.builder()
.object("context")
.object("client")
.value("hl", localization.getLocalizationCode())
@ -1239,8 +1248,13 @@ public final class YoutubeParsingHelper {
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("originalUrl", "https://www.youtube.com")
.value("platform", "DESKTOP")
.end()
.value("platform", "DESKTOP");
if (visitorData != null) {
builder.value("visitorData", visitorData);
}
return builder.end()
.object("request")
.array("internalExperimentFlags")
.end()

View File

@ -8,6 +8,7 @@ import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.feed.FeedExtractor;
@ -16,6 +17,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.localization.ContentCountry;
@ -23,6 +25,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
@ -34,6 +37,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscript
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
@ -88,6 +92,11 @@ public class YoutubeService extends StreamingService {
return YoutubeChannelLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getChannelTabLHFactory() {
return YoutubeChannelTabLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getPlaylistLHFactory() {
return YoutubePlaylistLinkHandlerFactory.getInstance();
@ -108,6 +117,15 @@ public class YoutubeService extends StreamingService {
return new YoutubeChannelExtractor(this, linkHandler);
}
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
} else {
return new YoutubeChannelTabExtractor(this, linkHandler);
}
}
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
@ -136,16 +154,17 @@ public class YoutubeService extends StreamingService {
@Override
public KioskList getKioskList() throws ExtractionException {
final KioskList list = new KioskList(this);
final ListLinkHandlerFactory h = YoutubeTrendingLinkHandlerFactory.getInstance();
// add kiosks here e.g.:
try {
list.addKioskEntry(
(streamingService, url, id) -> new YoutubeTrendingExtractor(
YoutubeService.this,
new YoutubeTrendingLinkHandlerFactory().fromUrl(url),
h.fromUrl(url),
id
),
new YoutubeTrendingLinkHandlerFactory(),
h,
YoutubeTrendingExtractor.KIOSK_ID
);
list.setDefaultKiosk(YoutubeTrendingExtractor.KIOSK_ID);

View File

@ -1,39 +1,35 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor.VideosTabExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -59,22 +55,24 @@ import javax.annotation.Nullable;
*/
public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject initialData;
private Optional<JsonObject> channelHeader;
private boolean isCarouselHeader = false;
private JsonObject videoTab;
private JsonObject jsonResponse;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private Optional<YoutubeChannelHelper.ChannelHeader> channelHeader;
private String channelId;
/**
* Some channels have response redirects and the only way to reliably get the id is by saving it
* If a channel is age-restricted, its pages are only accessible to logged-in and
* age-verified users, we get an {@code channelAgeGateRenderer} in this case, containing only
* the following metadata: channel name and channel avatar.
*
* <p>
* "Movies & Shows":
* <pre>
* UCuJcl0Ju-gPDoksRjK1ya-w
* UChBfWrfBXL9wS6tQtgjt_OQ UClgRkhTL3_hImCAmdLfDE4g
* UCok7UTQQEP1Rsctxiv3gwSQ
* </pre>
* This restriction doesn't seem to apply to all countries.
* </p>
*/
private String redirectedChannelId;
@Nullable
private JsonObject channelAgeGateRenderer;
public YoutubeChannelExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
@ -85,132 +83,42 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String channelPath = super.getId();
final String[] channelId = channelPath.split("/");
String id = "";
// If the url is an URL which is not a /channel URL, we need to use the
// navigation/resolve_url endpoint of the InnerTube API to get the channel id. Otherwise,
// we couldn't get information about the channel associated with this URL, if there is one.
if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("url", "https://www.youtube.com/" + channelPath)
.done())
.getBytes(StandardCharsets.UTF_8);
final String id = resolveChannelId(channelPath);
// Fetch Videos tab
final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(id,
"EgZ2aWRlb3PyBgQKAjoA", getExtractorLocalization(), getExtractorContentCountry());
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
body, getExtractorLocalization());
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
final String browseId = browseEndpoint.getString("browseId", "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
redirectedChannelId = browseId;
}
} else {
id = channelId[1];
}
JsonObject ajaxJson = null;
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("browseId", id)
.value("params", "EgZ2aWRlb3M%3D") // Equal to videos
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject jsonResponse = getJsonPostResponse("browse", body,
getExtractorLocalization());
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("navigateAction")
.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId",
"");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
redirectedChannelId = browseId;
level++;
} else {
ajaxJson = jsonResponse;
break;
}
}
if (ajaxJson == null) {
throw new ExtractionException("Could not fetch initial JSON data");
}
initialData = ajaxJson;
YoutubeParsingHelper.defaultAlertsCheck(initialData);
jsonResponse = data.jsonResponse;
channelId = data.channelId;
channelAgeGateRenderer = getChannelAgeGateRenderer();
}
private void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse)
throws ContentNotAvailableException {
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
}
@Nonnull
private Optional<JsonObject> getChannelHeader() {
if (channelHeader == null) {
final JsonObject h = initialData.getObject("header");
if (h.has("c4TabbedHeaderRenderer")) {
channelHeader = Optional.of(h.getObject("c4TabbedHeaderRenderer"));
} else if (h.has("carouselHeaderRenderer")) {
isCarouselHeader = true;
channelHeader = h.getObject("carouselHeaderRenderer")
@Nullable
private JsonObject getChannelAgeGateRenderer() {
return jsonResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.flatMap(tab -> tab.getObject("tabRenderer")
.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(itm -> itm.has("topicChannelDetailsRenderer"))
.findFirst()
.map(itm -> itm.getObject("topicChannelDetailsRenderer"));
} else {
channelHeader = Optional.empty();
}
.map(JsonObject.class::cast))
.filter(content -> content.has("channelAgeGateRenderer"))
.map(content -> content.getObject("channelAgeGateRenderer"))
.findFirst()
.orElse(null);
}
@Nonnull
private Optional<YoutubeChannelHelper.ChannelHeader> getChannelHeader() {
//noinspection OptionalAssignedToNull
if (channelHeader == null) {
channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
}
return channelHeader;
}
@ -229,57 +137,70 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getId() throws ParsingException {
return getChannelHeader()
.flatMap(header -> Optional.ofNullable(header.getString("channelId")).or(
() -> Optional.ofNullable(header.getObject("navigationEndpoint")
.flatMap(header -> Optional.ofNullable(header.json.getString("channelId"))
.or(() -> Optional.ofNullable(header.json.getObject("navigationEndpoint")
.getObject("browseEndpoint")
.getString("browseId"))
))
.or(() -> Optional.ofNullable(redirectedChannelId))
.orElseThrow(() -> new ParsingException("Could not get channel id"));
.or(() -> Optional.ofNullable(channelId))
.orElseThrow(() -> new ParsingException("Could not get channel ID"));
}
@Nonnull
@Override
public String getName() throws ParsingException {
final String mdName = initialData.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("title");
if (!isNullOrEmpty(mdName)) {
return mdName;
if (channelAgeGateRenderer != null) {
return channelAgeGateRenderer.getString("channelTitle");
}
final Optional<JsonObject> header = getChannelHeader();
if (header.isPresent()) {
final Object title = header.get().get("title");
final String metadataRendererTitle = jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("title");
if (!isNullOrEmpty(metadataRendererTitle)) {
return metadataRendererTitle;
}
return getChannelHeader().flatMap(header -> {
final Object title = header.json.get("title");
if (title instanceof String) {
return (String) title;
return Optional.of((String) title);
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
return headerName;
return Optional.of(headerName);
}
}
}
throw new ParsingException("Could not get channel name");
return Optional.empty();
}).orElseThrow(() -> new ParsingException("Could not get channel name"));
}
@Override
public String getAvatarUrl() throws ParsingException {
return getChannelHeader().flatMap(header -> Optional.ofNullable(
header.getObject("avatar").getArray("thumbnails")
.getObject(0).getString("url")
))
.map(YoutubeParsingHelper::fixThumbnailUrl)
.orElseThrow(() -> new ParsingException("Could not get avatar"));
final JsonObject avatarJsonObjectContainer;
if (channelAgeGateRenderer != null) {
avatarJsonObjectContainer = channelAgeGateRenderer;
} else {
avatarJsonObjectContainer = getChannelHeader().map(header -> header.json)
.orElseThrow(() -> new ParsingException("Could not get avatar URL"));
}
return YoutubeParsingHelper.fixThumbnailUrl(avatarJsonObjectContainer.getObject("avatar")
.getArray("thumbnails")
.getObject(0)
.getString("url"));
}
@Override
public String getBannerUrl() throws ParsingException {
if (channelAgeGateRenderer != null) {
return "";
}
return getChannelHeader().flatMap(header -> Optional.ofNullable(
header.getObject("banner").getArray("thumbnails")
.getObject(0).getString("url")
))
header.json.getObject("banner")
.getArray("thumbnails")
.getObject(0)
.getString("url")))
.filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
.map(YoutubeParsingHelper::fixThumbnailUrl)
// Channels may not have a banner, so no exception should be thrown if no banner is
@ -290,6 +211,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getFeedUrl() throws ParsingException {
// RSS feeds are accessible for age-restricted channels, no need to check whether a channel
// has a channelAgeGateRenderer
try {
return YoutubeParsingHelper.getFeedUrlFrom(getId());
} catch (final Exception e) {
@ -299,14 +222,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public long getSubscriberCount() throws ParsingException {
final Optional<JsonObject> header = getChannelHeader();
if (header.isPresent()) {
if (channelAgeGateRenderer != null) {
return UNKNOWN_SUBSCRIBER_COUNT;
}
final Optional<YoutubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
if (headerOpt.isPresent()) {
final JsonObject header = headerOpt.get().json;
JsonObject textObject = null;
if (header.get().has("subscriberCountText")) {
textObject = header.get().getObject("subscriberCountText");
} else if (header.get().has("subtitle")) {
textObject = header.get().getObject("subtitle");
if (header.has("subscriberCountText")) {
textObject = header.getObject("subscriberCountText");
} else if (header.has("subtitle")) {
textObject = header.getObject("subtitle");
}
if (textObject != null) {
@ -317,13 +245,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
}
}
return UNKNOWN_SUBSCRIBER_COUNT;
}
@Override
public String getDescription() throws ParsingException {
if (channelAgeGateRenderer != null) {
return null;
}
try {
return initialData.getObject("metadata").getObject("channelMetadataRenderer")
return jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("description");
} catch (final Exception e) {
throw new ParsingException("Could not get channel description", e);
@ -347,190 +281,139 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public boolean isVerified() throws ParsingException {
// The CarouselHeaderRenderer does not contain any verification badges.
// Since it is only shown on YT-internal channels or on channels of large organizations
// broadcasting live events, we can assume the channel to be verified.
if (isCarouselHeader) {
return true;
if (channelAgeGateRenderer != null) {
return false;
}
return getChannelHeader()
.map(header -> header.getArray("badges"))
.map(YoutubeParsingHelper::isVerified)
.orElse(false);
final Optional<YoutubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
if (headerOpt.isPresent()) {
final YoutubeChannelHelper.ChannelHeader header = headerOpt.get();
// The CarouselHeaderRenderer does not contain any verification badges.
// Since it is only shown on YT-internal channels or on channels of large organizations
// broadcasting live events, we can assume the channel to be verified.
if (header.isCarouselHeader) {
return true;
}
return YoutubeParsingHelper.isVerified(header.json.getArray("badges"));
}
return false;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
if (getVideoTab() != null) {
final JsonObject tabContent = getVideoTab().getObject("content");
JsonArray items = tabContent
.getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("items");
if (items.isEmpty()) {
items = tabContent.getObject("richGridRenderer").getArray("contents");
}
final List<String> channelIds = new ArrayList<>();
channelIds.add(getName());
channelIds.add(getUrl());
final JsonObject continuation = collectStreamsFrom(collector, items, channelIds);
nextPage = getNextPageFrom(continuation, channelIds);
public List<ListLinkHandler> getTabs() throws ParsingException {
if (channelAgeGateRenderer == null) {
return getTabsForNonAgeRestrictedChannels();
}
return new InfoItemsPage<>(collector, nextPage);
return getTabsForAgeRestrictedChannels();
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final List<String> channelIds = page.getIds();
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
getExtractorLocalization());
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction");
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation
.getArray("continuationItems"), channelIds);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
}
@Nullable
private Page getNextPageFrom(final JsonObject continuations,
final List<String> channelIds)
throws IOException, ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("continuation", continuation)
.done())
.getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
}
/**
* Collect streams from an array of items
*
* @param collector the collector where videos will be committed
* @param videos the array to get videos from
* @param channelIds the ids of the channel, which are its name and its URL
* @return the continuation object
*/
private JsonObject collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
@Nonnull final JsonArray videos,
@Nonnull final List<String> channelIds) {
collector.reset();
final String uploaderName = channelIds.get(0);
final String uploaderUrl = channelIds.get(1);
final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonObject continuation = null;
for (final Object object : videos) {
final JsonObject video = (JsonObject) object;
if (video.has("gridVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
video.getObject("gridVideoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
});
} else if (video.has("richItemRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
video.getObject("richItemRenderer")
.getObject("content").getObject("videoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
});
} else if (video.has("continuationItemRenderer")) {
continuation = video.getObject("continuationItemRenderer");
}
}
return continuation;
}
@Nullable
private JsonObject getVideoTab() throws ParsingException {
if (videoTab != null) {
return videoTab;
}
final JsonArray tabs = initialData.getObject("contents")
@Nonnull
private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws ParsingException {
final JsonArray responseTabs = jsonResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs");
final JsonObject foundVideoTab = tabs.stream()
.filter(Objects::nonNull)
final List<ListLinkHandler> tabs = new ArrayList<>();
final Consumer<String> addNonVideosTab = tabName -> {
try {
tabs.add(YoutubeChannelTabLinkHandlerFactory.getInstance().fromQuery(
channelId, List.of(tabName), ""));
} catch (final ParsingException ignored) {
// Do not add the tab if we couldn't create the LinkHandler
}
};
final String name = getName();
final String url = getUrl();
final String id = getId();
responseTabs.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(tab -> tab.has("tabRenderer")
&& tab.getObject("tabRenderer")
.getString("title", "")
.equals("Videos"))
.findFirst()
.filter(tab -> tab.has("tabRenderer"))
.map(tab -> tab.getObject("tabRenderer"))
.orElseThrow(
() -> new ContentNotSupportedException("This channel has no Videos tab"));
.forEach(tabRenderer -> {
final String tabUrl = tabRenderer.getObject("endpoint")
.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("url");
if (tabUrl != null) {
final String[] urlParts = tabUrl.split("/");
if (urlParts.length == 0) {
return;
}
final String messageRendererText = getTextFromObject(
foundVideoTab.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents")
.getObject(0)
.getObject("itemSectionRenderer")
.getArray("contents")
.getObject(0)
.getObject("messageRenderer")
.getObject("text"));
if (messageRendererText != null
&& messageRendererText.equals("This channel has no videos.")) {
return null;
final String urlSuffix = urlParts[urlParts.length - 1];
switch (urlSuffix) {
case "videos":
// Since the Videos tab has already its contents fetched, make
// sure it is in the first position
// YoutubeChannelTabExtractor still supports fetching this tab
tabs.add(0, new ReadyChannelTabListLinkHandler(
tabUrl,
channelId,
ChannelTabs.VIDEOS,
(service, linkHandler) -> new VideosTabExtractor(
service, linkHandler, tabRenderer, name, id, url)));
break;
case "shorts":
addNonVideosTab.accept(ChannelTabs.SHORTS);
break;
case "streams":
addNonVideosTab.accept(ChannelTabs.LIVESTREAMS);
break;
case "playlists":
addNonVideosTab.accept(ChannelTabs.PLAYLISTS);
break;
case "channels":
addNonVideosTab.accept(ChannelTabs.CHANNELS);
break;
}
}
});
return Collections.unmodifiableList(tabs);
}
@Nonnull
private List<ListLinkHandler> getTabsForAgeRestrictedChannels() throws ParsingException {
// As we don't have access to the channel tabs list, consider that the channel has videos,
// shorts and livestreams, the data only accessible without login on YouTube's desktop
// client using uploads system playlists
// The playlists channel tab is still available on YouTube Music, but this is not
// implemented in the extractor
final List<ListLinkHandler> tabs = new ArrayList<>();
final String channelUrl = getUrl();
final Consumer<String> addTab = tabName ->
tabs.add(new ReadyChannelTabListLinkHandler(channelUrl + "/" + tabName,
channelId, tabName, YoutubeChannelTabPlaylistExtractor::new));
addTab.accept(ChannelTabs.VIDEOS);
addTab.accept(ChannelTabs.SHORTS);
addTab.accept(ChannelTabs.LIVESTREAMS);
return Collections.unmodifiableList(tabs);
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
if (channelAgeGateRenderer != null) {
return List.of();
}
videoTab = foundVideoTab;
return foundVideoTab;
return jsonResponse.getObject("microformat")
.getObject("microformatDataRenderer")
.getArray("tags")
.stream()
.filter(String.class::isInstance)
.map(String.class::cast)
.collect(Collectors.toUnmodifiableList());
}
}

View File

@ -0,0 +1,486 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
import java.util.function.Consumer;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* A {@link ChannelTabExtractor} implementation for the YouTube service.
*
* <p>
* It currently supports {@code Videos}, {@code Shorts}, {@code Live}, {@code Playlists} and
* {@code Channels} tabs.
* </p>
*/
public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
/**
* Whether the visitor data extracted from the initial channel response is required to be used
* for continuations.
*
* <p>
* A valid {@code visitorData} is required to get continuations of shorts in channels.
* </p>
*
* <p>
* It should be not used when it is not needed, in order to reduce YouTube's tracking.
* </p>
*/
private final boolean useVisitorData;
private JsonObject jsonResponse;
private String channelId;
@Nullable
private String visitorData;
public YoutubeChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
useVisitorData = getName().equals(ChannelTabs.SHORTS);
}
@Nonnull
private String getChannelTabsParameters() throws ParsingException {
final String name = getName();
switch (name) {
case ChannelTabs.VIDEOS:
return "EgZ2aWRlb3PyBgQKAjoA";
case ChannelTabs.SHORTS:
return "EgZzaG9ydHPyBgUKA5oBAA%3D%3D";
case ChannelTabs.LIVESTREAMS:
return "EgdzdHJlYW1z8gYECgJ6AA%3D%3D";
case ChannelTabs.PLAYLISTS:
return "EglwbGF5bGlzdHPyBgQKAkIA";
case ChannelTabs.CHANNELS:
return "EghjaGFubmVsc_IGBAoCUgA%3D";
}
throw new ParsingException("Unsupported channel tab: " + name);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
channelId = resolveChannelId(super.getId());
final String params = getChannelTabsParameters();
final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(channelId,
params, getExtractorLocalization(), getExtractorContentCountry());
jsonResponse = data.jsonResponse;
channelId = data.channelId;
if (useVisitorData) {
visitorData = jsonResponse.getObject("responseContext").getString("visitorData");
}
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
try {
return YoutubeChannelTabLinkHandlerFactory.getInstance()
.getUrl("channel/" + getId(), List.of(getName()), "");
} catch (final ParsingException e) {
return super.getUrl();
}
}
@Nonnull
@Override
public String getId() throws ParsingException {
final String id = jsonResponse.getObject("header")
.getObject("c4TabbedHeaderRenderer")
.getString("channelId", "");
if (!id.isEmpty()) {
return id;
}
final Optional<String> carouselHeaderId = jsonResponse.getObject("header")
.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(item -> item.has("topicChannelDetailsRenderer"))
.findFirst()
.flatMap(item ->
Optional.ofNullable(item.getObject("topicChannelDetailsRenderer")
.getObject("navigationEndpoint")
.getObject("browseEndpoint")
.getString("browseId")));
if (carouselHeaderId.isPresent()) {
return carouselHeaderId.get();
}
if (!isNullOrEmpty(channelId)) {
return channelId;
} else {
throw new ParsingException("Could not get channel ID");
}
}
protected String getChannelName() {
final String metadataName = jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("title");
if (!isNullOrEmpty(metadataName)) {
return metadataName;
}
return YoutubeChannelHelper.getChannelHeader(jsonResponse)
.map(header -> {
final Object title = header.json.get("title");
if (title instanceof String) {
return (String) title;
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
return headerName;
}
}
return "";
})
.orElse("");
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
JsonArray items = new JsonArray();
final Optional<JsonObject> tab = getTabData();
if (tab.isPresent()) {
final JsonObject tabContent = tab.get().getObject("content");
items = tabContent.getObject("sectionListRenderer")
.getArray("contents")
.getObject(0)
.getObject("itemSectionRenderer")
.getArray("contents")
.getObject(0)
.getObject("gridRenderer")
.getArray("items");
if (items.isEmpty()) {
items = tabContent.getObject("richGridRenderer")
.getArray("contents");
if (items.isEmpty()) {
items = tabContent.getObject("sectionListRenderer")
.getArray("contents");
}
}
}
// If a channel tab is fetched, the next page requires channel ID and name, as channel
// streams don't have their channel specified.
// We also need to set the visitor data here when it should be enabled, as it is required
// to get continuations on some channel tabs, and we need a way to pass it between pages
final List<String> channelIds = useVisitorData && !isNullOrEmpty(visitorData)
? List.of(getChannelName(), getUrl(), visitorData)
: List.of(getChannelName(), getUrl());
final JsonObject continuation = collectItemsFrom(collector, items, channelIds)
.orElse(null);
final Page nextPage = getNextPageFrom(continuation, channelIds);
return new InfoItemsPage<>(collector, nextPage);
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final List<String> channelIds = page.getIds();
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
getExtractorLocalization());
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(jsonObject -> jsonObject.has("appendContinuationItemsAction"))
.map(jsonObject -> jsonObject.getObject("appendContinuationItemsAction"))
.findFirst()
.orElse(new JsonObject());
final JsonObject continuation = collectItemsFrom(collector,
sectionListContinuation.getArray("continuationItems"), channelIds)
.orElse(null);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
}
Optional<JsonObject> getTabData() {
final String urlSuffix = YoutubeChannelTabLinkHandlerFactory.getUrlSuffix(getName());
return jsonResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(tab -> tab.has("tabRenderer"))
.map(tab -> tab.getObject("tabRenderer"))
.filter(tabRenderer -> tabRenderer.getObject("endpoint")
.getObject("commandMetadata").getObject("webCommandMetadata")
.getString("url", "").endsWith(urlSuffix))
.findFirst()
// Check if tab has no content
.filter(tabRenderer -> {
final JsonArray tabContents = tabRenderer.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents")
.getObject(0)
.getObject("itemSectionRenderer")
.getArray("contents");
return tabContents.size() != 1
|| !tabContents.getObject(0).has("messageRenderer");
});
}
private Optional<JsonObject> collectItemsFrom(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonArray items,
@Nonnull final List<String> channelIds) {
return items.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(item -> collectItem(collector, item, channelIds))
.reduce(Optional.empty(), (c1, c2) -> c1.or(() -> c2));
}
private Optional<JsonObject> collectItem(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonObject item,
@Nonnull final List<String> channelIds) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
if (item.has("richItemRenderer")) {
final JsonObject richItem = item.getObject("richItemRenderer")
.getObject("content");
if (richItem.has("videoRenderer")) {
getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept(
richItem.getObject("videoRenderer"));
} else if (richItem.has("reelItemRenderer")) {
getCommitReelItemConsumer(collector, timeAgoParser, channelIds).accept(
richItem.getObject("reelItemRenderer"));
} else if (richItem.has("playlistRenderer")) {
getCommitPlaylistConsumer(collector, channelIds).accept(
item.getObject("playlistRenderer"));
}
} else if (item.has("gridVideoRenderer")) {
getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept(
item.getObject("gridVideoRenderer"));
} else if (item.has("gridPlaylistRenderer")) {
getCommitPlaylistConsumer(collector, channelIds).accept(
item.getObject("gridPlaylistRenderer"));
} else if (item.has("gridChannelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(
item.getObject("gridChannelRenderer")));
} else if (item.has("shelfRenderer")) {
return collectItem(collector, item.getObject("shelfRenderer")
.getObject("content"), channelIds);
} else if (item.has("itemSectionRenderer")) {
return collectItemsFrom(collector, item.getObject("itemSectionRenderer")
.getArray("contents"), channelIds);
} else if (item.has("horizontalListRenderer")) {
return collectItemsFrom(collector, item.getObject("horizontalListRenderer")
.getArray("items"), channelIds);
} else if (item.has("expandedShelfContentsRenderer")) {
return collectItemsFrom(collector, item.getObject("expandedShelfContentsRenderer")
.getArray("items"), channelIds);
} else if (item.has("continuationItemRenderer")) {
return Optional.ofNullable(item.getObject("continuationItemRenderer"));
}
return Optional.empty();
}
@Nonnull
private Consumer<JsonObject> getCommitVideoConsumer(
@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final TimeAgoParser timeAgoParser,
@Nonnull final List<String> channelIds) {
return videoRenderer -> collector.commit(
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser) {
@Override
public String getUploaderName() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(0);
}
return super.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(1);
}
return super.getUploaderUrl();
}
});
}
@Nonnull
private Consumer<JsonObject> getCommitReelItemConsumer(
@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final TimeAgoParser timeAgoParser,
@Nonnull final List<String> channelIds) {
return reelItemRenderer -> collector.commit(
new YoutubeReelInfoItemExtractor(reelItemRenderer, timeAgoParser) {
@Override
public String getUploaderName() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(0);
}
return super.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(1);
}
return super.getUploaderUrl();
}
});
}
@Nonnull
private Consumer<JsonObject> getCommitPlaylistConsumer(
@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final List<String> channelIds) {
return playlistRenderer -> collector.commit(
new YoutubePlaylistInfoItemExtractor(playlistRenderer) {
@Override
public String getUploaderName() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(0);
}
return super.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(1);
}
return super.getUploaderUrl();
}
});
}
@Nullable
private Page getNextPageFrom(final JsonObject continuations,
final List<String> channelIds) throws IOException,
ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry(),
useVisitorData && channelIds.size() >= 3 ? channelIds.get(2) : null)
.value("continuation", continuation)
.done())
.getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
}
/**
* A {@link YoutubeChannelTabExtractor} for the {@code Videos} tab, if it has been already
* fetched.
*/
public static final class VideosTabExtractor extends YoutubeChannelTabExtractor {
private final JsonObject tabRenderer;
private final String channelName;
private final String channelId;
private final String channelUrl;
VideosTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final JsonObject tabRenderer,
final String channelName,
final String channelId,
final String channelUrl) {
super(service, linkHandler);
this.tabRenderer = tabRenderer;
this.channelName = channelName;
this.channelId = channelId;
this.channelUrl = channelUrl;
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) {
// Nothing to do, the initial data was already fetched and is stored in the link handler
}
@Nonnull
@Override
public String getId() throws ParsingException {
return channelId;
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
return channelUrl;
}
@Override
protected String getChannelName() {
return channelName;
}
@Override
Optional<JsonObject> getTabData() {
return Optional.of(tabRenderer);
}
}
}

View File

@ -0,0 +1,191 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* A {@link ChannelTabExtractor} for YouTube system playlists using a
* {@link YoutubePlaylistExtractor} instance.
*
* <p>
* It is currently used to bypass age-restrictions on channels marked as age-restricted by their
* owner(s).
* </p>
*/
public class YoutubeChannelTabPlaylistExtractor extends ChannelTabExtractor {
private final PlaylistExtractor playlistExtractorInstance;
private boolean playlistExisting;
/**
* Construct a {@link YoutubeChannelTabPlaylistExtractor} instance.
*
* @param service a {@link StreamingService} implementation, which must be the YouTube
* one
* @param linkHandler a {@link ListLinkHandler} which must have a valid channel ID (starting
* with `UC`) and one of the given and supported content filters:
* {@link ChannelTabs#VIDEOS}, {@link ChannelTabs#SHORTS},
* {@link ChannelTabs#LIVESTREAMS}
* @throws IllegalArgumentException if the given {@link ListLinkHandler} doesn't have the
* required arguments
* @throws SystemPlaylistUrlCreationException if the system playlist URL could not be created,
* which should never happen
*/
public YoutubeChannelTabPlaylistExtractor(@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler)
throws IllegalArgumentException, SystemPlaylistUrlCreationException {
super(service, linkHandler);
final ListLinkHandler playlistLinkHandler = getPlaylistLinkHandler(linkHandler);
this.playlistExtractorInstance = new YoutubePlaylistExtractor(service, playlistLinkHandler);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
try {
playlistExtractorInstance.onFetchPage(downloader);
if (!playlistExisting) {
playlistExisting = true;
}
} catch (final ContentNotAvailableException e) {
// If a channel has no content of the type requested, the corresponding system playlist
// won't exist, so a ContentNotAvailableException would be thrown
// Ignore such issues in this case
}
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
if (!playlistExisting) {
return InfoItemsPage.emptyPage();
}
final InfoItemsPage<StreamInfoItem> playlistInitialPage =
playlistExtractorInstance.getInitialPage();
// We can't provide the playlist page as it is due to a type conflict, we need to wrap the
// page items and provide a new InfoItemsPage
final List<InfoItem> infoItems = new ArrayList<>(playlistInitialPage.getItems());
return new InfoItemsPage<>(infoItems, playlistInitialPage.getNextPage(),
playlistInitialPage.getErrors());
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (!playlistExisting) {
return InfoItemsPage.emptyPage();
}
final InfoItemsPage<StreamInfoItem> playlistPage = playlistExtractorInstance.getPage(page);
// We can't provide the playlist page as it is due to a type conflict, we need to wrap the
// page items and provide a new InfoItemsPage
final List<InfoItem> infoItems = new ArrayList<>(playlistPage.getItems());
return new InfoItemsPage<>(infoItems, playlistPage.getNextPage(),
playlistPage.getErrors());
}
/**
* Get a playlist {@link ListLinkHandler} from a channel tab one.
*
* <p>
* This method converts a channel ID without its {@code UC} prefix into a YouTube system
* playlist, depending on the first content filter provided in the given
* {@link ListLinkHandler}.
* </p>
*
* <p>
* The first content filter must be a channel tabs one among the
* {@link ChannelTabs#VIDEOS videos}, {@link ChannelTabs#SHORTS shorts} and
* {@link ChannelTabs#LIVESTREAMS} ones, which would be converted respectively into playlists
* with the ID {@code UULF}, {@code UUSH} and {@code UULV} on which the channel ID without the
* {@code UC} part is appended.
* </p>
*
* @param originalLinkHandler the original {@link ListLinkHandler} with which a
* {@link YoutubeChannelTabPlaylistExtractor} instance is being constructed
*
* @return a {@link ListLinkHandler} to use for the {@link YoutubePlaylistExtractor} instance
* needed to extract channel tabs data from a system playlist
* @throws IllegalArgumentException if the original {@link ListLinkHandler} does not meet the
* required criteria above
* @throws SystemPlaylistUrlCreationException if the system playlist URL could not be created,
* which should never happen
*/
@Nonnull
private ListLinkHandler getPlaylistLinkHandler(
@Nonnull final ListLinkHandler originalLinkHandler)
throws IllegalArgumentException, SystemPlaylistUrlCreationException {
final List<String> contentFilters = originalLinkHandler.getContentFilters();
if (contentFilters.isEmpty()) {
throw new IllegalArgumentException("A content filter is required");
}
final String channelId = originalLinkHandler.getId();
if (isNullOrEmpty(channelId) || !channelId.startsWith("UC")) {
throw new IllegalArgumentException("Invalid channel ID");
}
final String channelIdWithoutUc = channelId.substring(2);
final String playlistId;
switch (contentFilters.get(0)) {
case ChannelTabs.VIDEOS:
playlistId = "UULF" + channelIdWithoutUc;
break;
case ChannelTabs.SHORTS:
playlistId = "UUSH" + channelIdWithoutUc;
break;
case ChannelTabs.LIVESTREAMS:
playlistId = "UULV" + channelIdWithoutUc;
break;
default:
throw new IllegalArgumentException(
"Only Videos, Shorts and Livestreams tabs can extracted as playlists");
}
try {
final String newUrl = YoutubePlaylistLinkHandlerFactory.getInstance()
.getUrl(playlistId);
return new ListLinkHandler(newUrl, newUrl, playlistId, List.of(), "");
} catch (final ParsingException e) {
// This should be not reachable, as the given playlist ID should be valid and
// YoutubePlaylistLinkHandlerFactory doesn't throw any exception
throw new SystemPlaylistUrlCreationException(
"Could not create a YouTube playlist from a valid playlist ID", e);
}
}
/**
* Exception thrown when a YouTube system playlist URL could not be created.
*
* <p>
* This exception should be never thrown, as given playlist IDs should be always valid.
* </p>
*/
public static final class SystemPlaylistUrlCreationException extends RuntimeException {
SystemPlaylistUrlCreationException(final String message, final Throwable cause) {
super(message, cause);
}
}
}

View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
@ -22,10 +23,15 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public String getThumbnailUrl() throws ParsingException {
try {
final String url = playlistInfoItem.getArray("thumbnails").getObject(0)
.getArray("thumbnails").getObject(0).getString("url");
JsonArray thumbnails = playlistInfoItem.getArray("thumbnails")
.getObject(0)
.getArray("thumbnails");
if (thumbnails.isEmpty()) {
thumbnails = playlistInfoItem.getObject("thumbnail")
.getArray("thumbnails");
}
return fixThumbnailUrl(url);
return fixThumbnailUrl(thumbnails.getObject(0).getString("url"));
} catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
@ -79,9 +85,21 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public long getStreamCount() throws ParsingException {
String videoCountText = playlistInfoItem.getString("videoCount");
if (videoCountText == null) {
videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountText"));
}
if (videoCountText == null) {
videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountShortText"));
}
if (videoCountText == null) {
throw new ParsingException("Could not get stream count");
}
try {
return Long.parseLong(Utils.removeNonDigitCharacters(
playlistInfoItem.getString("videoCount")));
return Long.parseLong(Utils.removeNonDigitCharacters(videoCountText));
} catch (final Exception e) {
throw new ParsingException("Could not get stream count", e);
}

View File

@ -0,0 +1,147 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* A {@link StreamInfoItemExtractor} for YouTube's {@code reelItemRenderers}.
*
* <p>
* {@code reelItemRenderers} are returned on YouTube for their short-form contents on almost every
* place and every major client. They provide a limited amount of information and do not provide
* the exact view count, any uploader info (name, URL, avatar, verified status) and the upload date.
* </p>
*/
public class YoutubeReelInfoItemExtractor implements StreamInfoItemExtractor {
@Nonnull
private final JsonObject reelInfo;
@Nullable
private final TimeAgoParser timeAgoParser;
public YoutubeReelInfoItemExtractor(@Nonnull final JsonObject reelInfo,
@Nullable final TimeAgoParser timeAgoParser) {
this.reelInfo = reelInfo;
this.timeAgoParser = timeAgoParser;
}
@Override
public String getName() throws ParsingException {
return getTextFromObject(reelInfo.getObject("headline"));
}
@Override
public String getUrl() throws ParsingException {
try {
final String videoId = reelInfo.getString("videoId");
return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId);
} catch (final Exception e) {
throw new ParsingException("Could not get URL", e);
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
return getThumbnailUrlFromInfoItem(reelInfo);
}
@Override
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
}
@Override
public long getDuration() throws ParsingException {
// Duration of reelItems is only provided in the accessibility data
// example: "VIDEO TITLE - 49 seconds - play video"
// "VIDEO TITLE - 1 minute, 1 second - play video"
final String accessibilityLabel = reelInfo.getObject("accessibility")
.getObject("accessibilityData").getString("label");
if (accessibilityLabel == null || timeAgoParser == null) {
return 0;
}
// This approach may be language dependent
final String[] labelParts = accessibilityLabel.split(" [\u2013-] ");
if (labelParts.length > 2) {
final String textualDuration = labelParts[labelParts.length - 2];
return timeAgoParser.parseDuration(textualDuration);
}
return -1;
}
@Override
public long getViewCount() throws ParsingException {
final String viewCountText = getTextFromObject(reelInfo.getObject("viewCountText"));
if (!isNullOrEmpty(viewCountText)) {
// This approach is language dependent
if (viewCountText.toLowerCase().contains("no views")) {
return 0;
}
return Utils.mixedNumberWordToLong(viewCountText);
}
throw new ParsingException("Could not get short view count");
}
@Override
public boolean isShortFormContent() throws ParsingException {
return true;
}
// All the following properties cannot be obtained from reelItemRenderers
@Override
public boolean isAd() throws ParsingException {
return false;
}
@Override
public String getUploaderName() throws ParsingException {
return null;
}
@Override
public String getUploaderUrl() throws ParsingException {
return null;
}
@Nullable
@Override
public String getUploaderAvatarUrl() throws ParsingException {
return null;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Nullable
@Override
public String getTextualUploadDate() throws ParsingException {
return null;
}
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
return null;
}
}

View File

@ -58,7 +58,8 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto
@Override
public String getUrl(final String id,
final List<String> contentFilters,
final String searchFilter) {
final String searchFilter)
throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/" + id;
}
@ -84,7 +85,7 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
try {
final URL urlObj = Utils.stringToURL(url);
String path = urlObj.getPath();

View File

@ -0,0 +1,73 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.UnsupportedTabException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import javax.annotation.Nonnull;
import java.util.List;
public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubeChannelTabLinkHandlerFactory INSTANCE =
new YoutubeChannelTabLinkHandlerFactory();
private YoutubeChannelTabLinkHandlerFactory() {
}
public static YoutubeChannelTabLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Nonnull
public static String getUrlSuffix(@Nonnull final String tab)
throws UnsupportedTabException {
switch (tab) {
case ChannelTabs.VIDEOS:
return "/videos";
case ChannelTabs.SHORTS:
return "/shorts";
case ChannelTabs.LIVESTREAMS:
return "/streams";
case ChannelTabs.PLAYLISTS:
return "/playlists";
case ChannelTabs.CHANNELS:
return "/channels";
}
throw new UnsupportedTabException(tab);
}
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/" + id + getUrlSuffix(contentFilter.get(0));
}
@Override
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return YoutubeChannelLinkHandlerFactory.getInstance().getId(url);
}
@Override
public boolean onAcceptUrl(final String url) throws ParsingException {
try {
getId(url);
} catch (final ParsingException e) {
return false;
}
return true;
}
@Override
public String[] getAvailableContentFilter() {
return new String[] {
ChannelTabs.VIDEOS,
ChannelTabs.SHORTS,
ChannelTabs.LIVESTREAMS,
ChannelTabs.PLAYLISTS,
ChannelTabs.CHANNELS
};
}
}

View File

@ -19,13 +19,14 @@ public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFact
}
@Override
public String getUrl(final String id) {
public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/watch?v=" + id;
}
@Override
public String getId(final String urlString) throws ParsingException, IllegalArgumentException {
// we need the same id, avoids duplicate code
public String getId(final String urlString)
throws ParsingException, UnsupportedOperationException {
// We need the same id, avoids duplicate code
return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString);
}
@ -44,7 +45,8 @@ public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFact
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return getUrl(id);
}
}

View File

@ -26,12 +26,13 @@ public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFact
@Override
public String getUrl(final String id, final List<String> contentFilters,
final String sortFilter) {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/playlist?list=" + id;
}
@Override
public String getId(final String url) throws ParsingException {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
try {
final URL urlObj = Utils.stringToURL(url);

View File

@ -13,6 +13,9 @@ import javax.annotation.Nonnull;
public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
private static final YoutubeSearchQueryHandlerFactory INSTANCE =
new YoutubeSearchQueryHandlerFactory();
public static final String ALL = "all";
public static final String VIDEOS = "videos";
public static final String CHANNELS = "channels";
@ -29,20 +32,18 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa
@Nonnull
public static YoutubeSearchQueryHandlerFactory getInstance() {
return new YoutubeSearchQueryHandlerFactory();
return INSTANCE;
}
@Override
public String getUrl(final String searchString,
@Nonnull final List<String> contentFilters,
final String sortFilter) throws ParsingException {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try {
if (!contentFilters.isEmpty()) {
final String contentFilter = contentFilters.get(0);
switch (contentFilter) {
case ALL:
default:
break;
case VIDEOS:
return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAQ%253D%253D";
case CHANNELS:

View File

@ -79,7 +79,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
@Nonnull
@Override
public String getUrl(final String id) {
public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/watch?v=" + id;
}
@ -87,7 +87,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
@Nonnull
@Override
public String getId(final String theUrlString)
throws ParsingException, IllegalArgumentException {
throws ParsingException, UnsupportedOperationException {
String urlString = theUrlString;
try {
final URI uri = new URI(urlString);

View File

@ -1,28 +1,29 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
/*
* Created by Christian Schabesberger on 12.08.17.
*
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
* YoutubeTrendingLinkHandlerFactory.java is part of NewPipe.
* YoutubeTrendingLinkHandlerFactory.java is part of NewPipe Extractor.
*
* NewPipe is free software: you can redistribute it and/or modify
* NewPipe Extractor is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* NewPipe Extractor is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
* along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
*/
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidiousURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
@ -30,16 +31,27 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
public final class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubeTrendingLinkHandlerFactory INSTANCE =
new YoutubeTrendingLinkHandlerFactory();
private YoutubeTrendingLinkHandlerFactory() {
}
public static YoutubeTrendingLinkHandlerFactory getInstance() {
return INSTANCE;
}
public String getUrl(final String id,
final List<String> contentFilters,
final String sortFilter) {
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/feed/trending";
}
@Override
public String getId(final String url) {
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return "Trending";
}

View File

@ -3,8 +3,11 @@ package org.schabi.newpipe.extractor;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -15,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.Utils;
public class ExtractorAsserts {
@ -144,6 +148,16 @@ public class ExtractorAsserts {
assertNotNull(shouldBeContained, "shouldBeContained is null");
assertNotNull(container, "container is null");
assertTrue(container.contains(shouldBeContained),
"'" + shouldBeContained + "' should be contained inside '" + container +"'");
"'" + shouldBeContained + "' should be contained inside '" + container + "'");
}
public static void assertTabsContained(@Nonnull final List<ListLinkHandler> tabs,
@Nonnull final String... expectedTabs) {
final Set<String> tabSet = tabs.stream()
.map(linkHandler -> linkHandler.getContentFilters().get(0))
.collect(Collectors.toUnmodifiableSet());
Arrays.stream(expectedTabs)
.forEach(expectedTab -> assertTrue(tabSet.contains(expectedTab),
"Missing " + expectedTab + " tab (got " + tabSet + ")"));
}
}

View File

@ -0,0 +1,31 @@
package org.schabi.newpipe.extractor.localization;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
class TimeAgoParserTest {
private static TimeAgoParser timeAgoParser;
@BeforeAll
static void setUp() {
timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
}
@Test
void testGetDuration() throws ParsingException {
assertEquals(1, timeAgoParser.parseDuration("one second"));
assertEquals(1, timeAgoParser.parseDuration("second"));
assertEquals(49, timeAgoParser.parseDuration("49 seconds"));
assertEquals(61, timeAgoParser.parseDuration("1 minute, 1 second"));
}
@Test
void testGetDurationError() {
assertThrows(ParsingException.class, () -> timeAgoParser.parseDuration("abcd"));
assertThrows(ParsingException.class, () -> timeAgoParser.parseDuration("12 abcd"));
}
}

View File

@ -1,11 +1,22 @@
package org.schabi.newpipe.extractor.services;
@SuppressWarnings("unused")
public interface BaseChannelExtractorTest extends BaseListExtractorTest {
import org.junit.jupiter.api.Test;
public interface BaseChannelExtractorTest extends BaseExtractorTest {
@Test
void testDescription() throws Exception;
@Test
void testAvatarUrl() throws Exception;
@Test
void testBannerUrl() throws Exception;
@Test
void testFeedUrl() throws Exception;
@Test
void testSubscriberCount() throws Exception;
@Test
void testVerified() throws Exception;
@Test
void testTabs() throws Exception;
@Test
void testTags() throws Exception;
}

View File

@ -1,10 +1,16 @@
package org.schabi.newpipe.extractor.services;
@SuppressWarnings("unused")
import org.junit.jupiter.api.Test;
public interface BaseExtractorTest {
@Test
void testServiceId() throws Exception;
@Test
void testName() throws Exception;
@Test
void testId() throws Exception;
@Test
void testUrl() throws Exception;
@Test
void testOriginalUrl() throws Exception;
}

View File

@ -1,7 +1,10 @@
package org.schabi.newpipe.extractor.services;
@SuppressWarnings("unused")
import org.junit.jupiter.api.Test;
public interface BaseListExtractorTest extends BaseExtractorTest {
@Test
void testRelatedItems() throws Exception;
@Test
void testMoreRelatedItems() throws Exception;
}

View File

@ -1,11 +1,18 @@
package org.schabi.newpipe.extractor.services;
@SuppressWarnings("unused")
import org.junit.jupiter.api.Test;
public interface BasePlaylistExtractorTest extends BaseListExtractorTest {
@Test
void testThumbnailUrl() throws Exception;
@Test
void testBannerUrl() throws Exception;
@Test
void testUploaderName() throws Exception;
@Test
void testUploaderAvatarUrl() throws Exception;
@Test
void testStreamCount() throws Exception;
@Test
void testUploaderVerified() throws Exception;
}

View File

@ -1,8 +1,12 @@
package org.schabi.newpipe.extractor.services;
@SuppressWarnings("unused")
import org.junit.jupiter.api.Test;
public interface BaseSearchExtractorTest extends BaseListExtractorTest {
@Test
void testSearchString() throws Exception;
@Test
void testSearchSuggestion() throws Exception;
@Test
void testSearchCorrected() throws Exception;
}

View File

@ -1,36 +1,70 @@
package org.schabi.newpipe.extractor.services;
import org.junit.jupiter.api.Test;
public interface BaseStreamExtractorTest extends BaseExtractorTest {
@Test
void testStreamType() throws Exception;
@Test
void testUploaderName() throws Exception;
@Test
void testUploaderUrl() throws Exception;
@Test
void testUploaderAvatarUrl() throws Exception;
@Test
void testSubscriberCount() throws Exception;
@Test
void testSubChannelName() throws Exception;
@Test
void testSubChannelUrl() throws Exception;
@Test
void testSubChannelAvatarUrl() throws Exception;
@Test
void testThumbnailUrl() throws Exception;
@Test
void testDescription() throws Exception;
@Test
void testLength() throws Exception;
@Test
void testTimestamp() throws Exception;
@Test
void testViewCount() throws Exception;
@Test
void testUploadDate() throws Exception;
@Test
void testTextualUploadDate() throws Exception;
@Test
void testLikeCount() throws Exception;
@Test
void testDislikeCount() throws Exception;
@Test
void testRelatedItems() throws Exception;
@Test
void testAgeLimit() throws Exception;
@Test
void testErrorMessage() throws Exception;
@Test
void testAudioStreams() throws Exception;
@Test
void testVideoStreams() throws Exception;
@Test
void testSubtitles() throws Exception;
@Test
void testGetDashMpdUrl() throws Exception;
@Test
void testFrames() throws Exception;
@Test
void testHost() throws Exception;
@Test
void testPrivacy() throws Exception;
@Test
void testCategory() throws Exception;
@Test
void testLicence() throws Exception;
@Test
void testLanguageInfo() throws Exception;
@Test
void testTags() throws Exception;
@Test
void testSupportInfo() throws Exception;
}

View File

@ -7,12 +7,11 @@ import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContained;
import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
@ -27,12 +26,7 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
}
@Test
public void testLength() throws ExtractionException, IOException {
assertTrue(extractor.getInitialPage().getItems().size() >= 0);
}
@Override
@Test
public void testDescription() throws Exception {
assertEquals("making music:)", extractor.getDescription());
}
@ -62,16 +56,6 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
assertFalse(extractor.isVerified());
}
@Override
public void testRelatedItems() throws Exception {
// not implemented
}
@Override
public void testMoreRelatedItems() throws Exception {
// not implemented
}
@Override
public void testServiceId() {
assertEquals(Bandcamp.getServiceId(), extractor.getServiceId());
@ -84,7 +68,7 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
@Override
public void testId() throws Exception {
assertEquals("https://toupie.bandcamp.com/", extractor.getId());
assertEquals("2450875064", extractor.getId());
}
@Override
@ -96,4 +80,16 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
public void testOriginalUrl() throws Exception {
assertEquals("https://toupie.bandcamp.com", extractor.getUrl());
}
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.ALBUMS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
}

View File

@ -19,7 +19,7 @@ public class BandcampChannelLinkHandlerFactoryTest {
@BeforeAll
public static void setUp() {
linkHandler = new BandcampChannelLinkHandlerFactory();
linkHandler = BandcampChannelLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}
@ -55,7 +55,7 @@ public class BandcampChannelLinkHandlerFactoryTest {
public void testGetId() throws ParsingException {
assertEquals("1196681540", linkHandler.getId("https://macbenson.bandcamp.com/"));
assertEquals("1196681540", linkHandler.getId("http://macbenson.bandcamp.com/"));
assertEquals("1581461772", linkHandler.getId("https://interovgm.bandcamp.com/releases"));
assertEquals("1581461772", linkHandler.getId("https://shirakumon.bandcamp.com/releases"));
assertEquals("3321800855", linkHandler.getId("https://infiniteammo.bandcamp.com/"));
assertEquals("3775652329", linkHandler.getId("https://npet.bandcamp.com/"));
@ -67,7 +67,7 @@ public class BandcampChannelLinkHandlerFactoryTest {
@Test
public void testGetUrl() throws ParsingException {
assertEquals("https://macbenson.bandcamp.com", linkHandler.getUrl("1196681540"));
assertEquals("https://interovgm.bandcamp.com", linkHandler.getUrl("1581461772"));
assertEquals("https://shirakumon.bandcamp.com", linkHandler.getUrl("1581461772"));
assertEquals("https://infiniteammo.bandcamp.com", linkHandler.getUrl("3321800855"));
assertEquals("https://lobstertheremin.com", linkHandler.getUrl("2735462545"));
@ -82,5 +82,4 @@ public class BandcampChannelLinkHandlerFactoryTest {
public void testGetIdWithInvalidUrl() {
assertThrows(ParsingException.class, () -> linkHandler.getUrl("https://bandcamp.com"));
}
}

View File

@ -0,0 +1,128 @@
package org.schabi.newpipe.extractor.services.bandcamp;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelTabExtractor;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
class BandcampChannelTabExtractorTest {
static class Tracks implements BaseListExtractorTest {
private static BandcampChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws IOException, ExtractionException {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (BandcampChannelTabExtractor) Bandcamp
.getChannelTabExtractorFromId("2464198920", ChannelTabs.TRACKS);
extractor.fetchPage();
}
@Test
@Override
public void testServiceId() throws Exception {
assertEquals(Bandcamp.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws Exception {
assertEquals(ChannelTabs.TRACKS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("2464198920", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://wintergatan.bandcamp.com/track", extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws Exception {
assertEquals("https://wintergatan.bandcamp.com/track", extractor.getOriginalUrl());
}
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
// Bandcamp only return a single page
}
}
static class Albums implements BaseListExtractorTest {
private static BandcampChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws IOException, ExtractionException {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (BandcampChannelTabExtractor) Bandcamp
.getChannelTabExtractorFromId("2450875064", ChannelTabs.ALBUMS);
extractor.fetchPage();
}
@Test
@Override
public void testServiceId() {
assertEquals(Bandcamp.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws Exception {
assertEquals(ChannelTabs.ALBUMS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("2450875064", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://toupie.bandcamp.com/album", extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws Exception {
assertEquals("https://toupie.bandcamp.com/album", extractor.getOriginalUrl());
}
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
// Bandcamp only return a single page
}
}
}

View File

@ -20,7 +20,7 @@ public class BandcampCommentsLinkHandlerFactoryTest {
@BeforeAll
public static void setUp() {
linkHandler = new BandcampCommentsLinkHandlerFactory();
linkHandler = BandcampCommentsLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}

View File

@ -18,7 +18,7 @@ public class BandcampFeaturedLinkHandlerFactoryTest {
@BeforeAll
public static void setUp() {
linkHandler = new BandcampFeaturedLinkHandlerFactory();
linkHandler = BandcampFeaturedLinkHandlerFactory.getInstance();
}
@ -42,7 +42,7 @@ public class BandcampFeaturedLinkHandlerFactoryTest {
}
@Test
public void testGetId() {
public void testGetId() throws ParsingException {
assertEquals("Featured", linkHandler.getId("http://bandcamp.com/api/mobile/24/bootstrap_data"));
assertEquals("Featured", linkHandler.getId("https://bandcamp.com/api/mobile/24/bootstrap_data"));
assertEquals("Radio", linkHandler.getId("http://bandcamp.com/?show=1"));

View File

@ -21,7 +21,7 @@ public class BandcampPlaylistLinkHandlerFactoryTest {
@BeforeAll
public static void setUp() {
linkHandler = new BandcampPlaylistLinkHandlerFactory();
linkHandler = BandcampPlaylistLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}

View File

@ -20,12 +20,12 @@ public class BandcampStreamLinkHandlerFactoryTest {
@BeforeAll
public static void setUp() {
linkHandler = new BandcampStreamLinkHandlerFactory();
linkHandler = BandcampStreamLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void testGetRadioUrl() {
public void testGetRadioUrl() throws ParsingException {
assertEquals("https://bandcamp.com/?show=1", linkHandler.getUrl("1"));
}

View File

@ -4,6 +4,7 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -16,12 +17,16 @@ import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
public class MediaCCCConferenceExtractorTest {
public static class FrOSCon2017 {
private static MediaCCCConferenceExtractor extractor;
private static ChannelTabExtractor tabExtractor;
@BeforeAll
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/froscon2017");
extractor.fetchPage();
tabExtractor = MediaCCC.getChannelTabExtractor(extractor.getTabs().get(0));
tabExtractor.fetchPage();
}
@Test
@ -46,18 +51,22 @@ public class MediaCCCConferenceExtractorTest {
@Test
public void testGetInitalPage() throws Exception {
assertEquals(97, extractor.getInitialPage().getItems().size());
assertEquals(97, tabExtractor.getInitialPage().getItems().size());
}
}
public static class Oscal2019 {
private static MediaCCCConferenceExtractor extractor;
private static ChannelTabExtractor tabExtractor;
@BeforeAll
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/oscal19");
extractor.fetchPage();
tabExtractor = MediaCCC.getChannelTabExtractor(extractor.getTabs().get(0));
tabExtractor.fetchPage();
}
@Test
@ -82,7 +91,7 @@ public class MediaCCCConferenceExtractorTest {
@Test
public void testGetInitalPage() throws Exception {
assertTrue(extractor.getInitialPage().getItems().size() >= 21);
assertTrue(tabExtractor.getInitialPage().getItems().size() >= 21);
}
}
}

View File

@ -14,7 +14,7 @@ public class MediaCCCConferenceLinkHandlerFactoryTest {
@BeforeAll
public static void setUp() {
linkHandler = new MediaCCCConferenceLinkHandlerFactory();
linkHandler = MediaCCCConferenceLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}

View File

@ -14,7 +14,7 @@ public class MediaCCCStreamLinkHandlerFactoryTest {
@BeforeAll
public static void setUp() {
linkHandler = new MediaCCCStreamLinkHandlerFactory();
linkHandler = MediaCCCStreamLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}

View File

@ -5,7 +5,7 @@ import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ExtractorAsserts;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor;
@ -14,11 +14,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContained;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
/**
* Test for {@link PeertubeAccountExtractor}
@ -67,20 +66,6 @@ public class PeertubeAccountExtractorTest {
assertEquals("https://framatube.org/accounts/framasoft", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@ -114,6 +99,18 @@ public class PeertubeAccountExtractorTest {
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.VIDEOS, ChannelTabs.CHANNELS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
}
public static class FreeSoftwareFoundation implements BaseChannelExtractorTest {
@ -129,16 +126,6 @@ public class PeertubeAccountExtractorTest {
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@ -168,20 +155,6 @@ public class PeertubeAccountExtractorTest {
assertEquals("https://framatube.org/api/v1/accounts/fsf", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@ -215,5 +188,17 @@ public class PeertubeAccountExtractorTest {
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.VIDEOS, ChannelTabs.CHANNELS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
}
}

View File

@ -0,0 +1,150 @@
package org.schabi.newpipe.extractor.services.peertube;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelTabExtractor;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
class PeertubeAccountTabExtractorTest {
static class Videos implements BaseListExtractorTest {
private static PeertubeChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = (PeertubeChannelTabExtractor) PeerTube
.getChannelTabExtractorFromId("accounts/framasoft", ChannelTabs.VIDEOS);
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws ParsingException {
assertEquals(ChannelTabs.VIDEOS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("accounts/framasoft", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://framatube.org/accounts/framasoft/videos",
extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws ParsingException {
assertEquals("https://framatube.org/accounts/framasoft/videos",
extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
}
static class Channels implements BaseListExtractorTest {
private static PeertubeChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = (PeertubeChannelTabExtractor) PeerTube
.getChannelTabExtractorFromId("accounts/framasoft", ChannelTabs.CHANNELS);
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws ParsingException {
assertEquals(ChannelTabs.CHANNELS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("accounts/framasoft", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://framatube.org/accounts/framasoft/video-channels",
extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws ParsingException {
assertEquals("https://framatube.org/accounts/framasoft/video-channels",
extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
}
}

View File

@ -5,15 +5,19 @@ import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ExtractorAsserts;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import static org.junit.jupiter.api.Assertions.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContained;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
/**
* Test for {@link PeertubeChannelExtractor}
@ -62,20 +66,6 @@ public class PeertubeChannelExtractorTest {
assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/videos", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@ -124,6 +114,18 @@ public class PeertubeChannelExtractorTest {
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.VIDEOS, ChannelTabs.PLAYLISTS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
}
public static class ChatSceptique implements BaseChannelExtractorTest {
@ -140,16 +142,6 @@ public class PeertubeChannelExtractorTest {
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@ -179,20 +171,6 @@ public class PeertubeChannelExtractorTest {
assertEquals("https://framatube.org/api/v1/video-channels/chatsceptique@skeptikon.fr", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@ -241,5 +219,17 @@ public class PeertubeChannelExtractorTest {
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.VIDEOS, ChannelTabs.PLAYLISTS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
}
}

View File

@ -0,0 +1,212 @@
package org.schabi.newpipe.extractor.services.peertube;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelTabExtractor;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
class PeertubeChannelTabExtractorTest {
static class Videos implements BaseListExtractorTest {
private static PeertubeChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = (PeertubeChannelTabExtractor) PeerTube.getChannelTabExtractorFromId(
"video-channels/lqdn_channel@video.lqdn.fr", ChannelTabs.VIDEOS);
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws ParsingException {
assertEquals(ChannelTabs.VIDEOS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("video-channels/lqdn_channel@video.lqdn.fr", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/videos",
extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws ParsingException {
assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/videos",
extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
void testGetPageInNewExtractor() throws Exception {
final ChannelTabExtractor newTabExtractor = PeerTube.getChannelTabExtractorFromId(
"video-channels/lqdn_channel@video.lqdn.fr", ChannelTabs.VIDEOS);
defaultTestGetPageInNewExtractor(extractor, newTabExtractor);
}
}
static class Playlists implements BaseListExtractorTest {
private static PeertubeChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws IOException, ExtractionException {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeChannelTabExtractor)
PeerTube.getChannelTabExtractorFromIdAndBaseUrl(
"video-channels/lqdn_channel@video.lqdn.fr", ChannelTabs.PLAYLISTS,
"https://framatube.org");
extractor.fetchPage();
}
@Test
@Override
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws Exception {
assertEquals(ChannelTabs.PLAYLISTS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("video-channels/lqdn_channel@video.lqdn.fr", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/video-playlists",
extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws Exception {
assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/video-playlists",
extractor.getOriginalUrl());
}
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Override
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
}
static class Channels implements BaseListExtractorTest {
private static PeertubeChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws IOException, ExtractionException {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeChannelTabExtractor)
PeerTube.getChannelTabExtractorFromIdAndBaseUrl("accounts/framasoft",
ChannelTabs.CHANNELS, "https://framatube.org");
extractor.fetchPage();
}
@Test
@Override
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws Exception {
assertEquals(ChannelTabs.CHANNELS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("accounts/framasoft", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://framatube.org/accounts/framasoft/video-channels",
extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws Exception {
assertEquals("https://framatube.org/accounts/framasoft/video-channels",
extractor.getOriginalUrl());
}
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
}
}

View File

@ -22,7 +22,7 @@ public class PeertubeTrendingLinkHandlerFactoryTest {
public static void setUp() throws Exception {
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
LinkHandlerFactory = new PeertubeTrendingLinkHandlerFactory();
LinkHandlerFactory = PeertubeTrendingLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}

View File

@ -4,7 +4,7 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelExtractor;
@ -12,8 +12,8 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCha
import static org.junit.jupiter.api.Assertions.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContained;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
/**
* Test for {@link SoundcloudChannelExtractor}
@ -59,20 +59,6 @@ public class SoundcloudChannelExtractorTest {
assertEquals("http://soundcloud.com/liluzivert/sets", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@ -106,6 +92,19 @@ public class SoundcloudChannelExtractorTest {
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.TRACKS, ChannelTabs.PLAYLISTS,
ChannelTabs.ALBUMS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
}
public static class DubMatix implements BaseChannelExtractorTest {
@ -119,16 +118,6 @@ public class SoundcloudChannelExtractorTest {
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@ -158,20 +147,6 @@ public class SoundcloudChannelExtractorTest {
assertEquals("https://soundcloud.com/dubmatix", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@ -205,5 +180,18 @@ public class SoundcloudChannelExtractorTest {
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.TRACKS, ChannelTabs.PLAYLISTS,
ChannelTabs.ALBUMS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
}
}

Some files were not shown because too many files have changed in this diff Show More