Merge pull request #1082 from AudricV/channel-tabs-and-tags-support

Add support for channel tabs and channel tags
This commit is contained in:
Stypox 2023-08-06 12:21:42 +02:00 committed by GitHub
commit 35f3a4ad01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
182 changed files with 16319 additions and 4292 deletions

View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor; package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -140,6 +141,14 @@ public abstract class StreamingService {
*/ */
public abstract ListLinkHandlerFactory getChannelLHFactory(); public abstract ListLinkHandlerFactory getChannelLHFactory();
/**
* Must return a new instance of an implementation of ListLinkHandlerFactory for channel tabs.
* If support for channel tabs is not given null must be returned.
*
* @return an instance of a ListLinkHandlerFactory for channels or null
*/
public abstract ListLinkHandlerFactory getChannelTabLHFactory();
/** /**
* Must return a new instance of an implementation of ListLinkHandlerFactory for playlists. * Must return a new instance of an implementation of ListLinkHandlerFactory for playlists.
* If support for playlists is not given null must be returned. * If support for playlists is not given null must be returned.
@ -204,6 +213,15 @@ public abstract class StreamingService {
public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler)
throws ExtractionException; throws ExtractionException;
/**
* Must create a new instance of a ChannelTabExtractor implementation.
*
* @param linkHandler is pointing to the channel which should be handled by this new instance.
* @return a new ChannelTabExtractor
*/
public abstract ChannelTabExtractor getChannelTabExtractor(ListLinkHandler linkHandler)
throws ExtractionException;
/** /**
* Must crete a new instance of a PlaylistExtractor implementation. * Must crete a new instance of a PlaylistExtractor implementation.
* @param linkHandler is pointing to the playlist which should be handled by this new instance. * @param linkHandler is pointing to the playlist which should be handled by this new instance.
@ -262,6 +280,20 @@ public abstract class StreamingService {
return getChannelExtractor(getChannelLHFactory().fromUrl(url)); return getChannelExtractor(getChannelLHFactory().fromUrl(url));
} }
public ChannelTabExtractor getChannelTabExtractorFromId(final String id, final String tab)
throws ExtractionException {
return getChannelTabExtractor(getChannelTabLHFactory().fromQuery(
id, Collections.singletonList(tab), ""));
}
public ChannelTabExtractor getChannelTabExtractorFromIdAndBaseUrl(final String id,
final String tab,
final String baseUrl)
throws ExtractionException {
return getChannelTabExtractor(getChannelTabLHFactory().fromQuery(
id, Collections.singletonList(tab), "", baseUrl));
}
public PlaylistExtractor getPlaylistExtractor(final String url) throws ExtractionException { public PlaylistExtractor getPlaylistExtractor(final String url) throws ExtractionException {
return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url)); return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url));
} }

View File

@ -1,10 +1,12 @@
package org.schabi.newpipe.extractor.channel; package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import javax.annotation.Nonnull;
import java.util.List;
/* /*
* Created by Christian Schabesberger on 25.07.16. * Created by Christian Schabesberger on 25.07.16.
@ -26,11 +28,11 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> { public abstract class ChannelExtractor extends Extractor {
public static final long UNKNOWN_SUBSCRIBER_COUNT = -1; public static final long UNKNOWN_SUBSCRIBER_COUNT = -1;
public ChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) { protected ChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
} }
@ -43,5 +45,10 @@ public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
public abstract String getParentChannelUrl() throws ParsingException; public abstract String getParentChannelUrl() throws ParsingException;
public abstract String getParentChannelAvatarUrl() throws ParsingException; public abstract String getParentChannelAvatarUrl() throws ParsingException;
public abstract boolean isVerified() throws ParsingException; public abstract boolean isVerified() throws ParsingException;
@Nonnull
public abstract List<ListLinkHandler> getTabs() throws ParsingException;
@Nonnull
public List<String> getTags() throws ParsingException {
return List.of();
}
} }

View File

@ -1,16 +1,15 @@
package org.schabi.newpipe.extractor.channel; package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import javax.annotation.Nonnull;
/* /*
* Created by Christian Schabesberger on 31.07.16. * Created by Christian Schabesberger on 31.07.16.
@ -32,16 +31,14 @@ import java.io.IOException;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public class ChannelInfo extends ListInfo<StreamInfoItem> { public class ChannelInfo extends Info {
public ChannelInfo(final int serviceId, public ChannelInfo(final int serviceId,
final String id, final String id,
final String url, final String url,
final String originalUrl, final String originalUrl,
final String name, final String name) {
final ListLinkHandler listLinkHandler) { super(serviceId, id, url, originalUrl, name);
super(serviceId, id, url, originalUrl, name, listLinkHandler.getContentFilters(),
listLinkHandler.getSortFilter());
} }
public static ChannelInfo getInfo(final String url) throws IOException, ExtractionException { public static ChannelInfo getInfo(final String url) throws IOException, ExtractionException {
@ -55,13 +52,6 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
return getInfo(extractor); return getInfo(extractor);
} }
public static InfoItemsPage<StreamInfoItem> getMoreItems(final StreamingService service,
final String url,
final Page page)
throws IOException, ExtractionException {
return service.getChannelExtractor(url).getPage(page);
}
public static ChannelInfo getInfo(final ChannelExtractor extractor) public static ChannelInfo getInfo(final ChannelExtractor extractor)
throws IOException, ExtractionException { throws IOException, ExtractionException {
@ -71,35 +61,32 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
final String originalUrl = extractor.getOriginalUrl(); final String originalUrl = extractor.getOriginalUrl();
final String name = extractor.getName(); final String name = extractor.getName();
final ChannelInfo info = final ChannelInfo info = new ChannelInfo(serviceId, id, url, originalUrl, name);
new ChannelInfo(serviceId, id, url, originalUrl, name, extractor.getLinkHandler());
try { try {
info.setAvatarUrl(extractor.getAvatarUrl()); info.setAvatarUrl(extractor.getAvatarUrl());
} catch (final Exception e) { } catch (final Exception e) {
info.addError(e); info.addError(e);
} }
try { try {
info.setBannerUrl(extractor.getBannerUrl()); info.setBannerUrl(extractor.getBannerUrl());
} catch (final Exception e) { } catch (final Exception e) {
info.addError(e); info.addError(e);
} }
try { try {
info.setFeedUrl(extractor.getFeedUrl()); info.setFeedUrl(extractor.getFeedUrl());
} catch (final Exception e) { } catch (final Exception e) {
info.addError(e); info.addError(e);
} }
final InfoItemsPage<StreamInfoItem> itemsPage =
ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(itemsPage.getItems());
info.setNextPage(itemsPage.getNextPage());
try { try {
info.setSubscriberCount(extractor.getSubscriberCount()); info.setSubscriberCount(extractor.getSubscriberCount());
} catch (final Exception e) { } catch (final Exception e) {
info.addError(e); info.addError(e);
} }
try { try {
info.setDescription(extractor.getDescription()); info.setDescription(extractor.getDescription());
} catch (final Exception e) { } catch (final Exception e) {
@ -130,6 +117,18 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
info.addError(e); info.addError(e);
} }
try {
info.setTabs(extractor.getTabs());
} catch (final Exception e) {
info.addError(e);
}
try {
info.setTags(extractor.getTags());
} catch (final Exception e) {
info.addError(e);
}
return info; return info;
} }
@ -143,6 +142,8 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
private String description; private String description;
private String[] donationLinks; private String[] donationLinks;
private boolean verified; private boolean verified;
private List<ListLinkHandler> tabs = List.of();
private List<String> tags = List.of();
public String getParentChannelName() { public String getParentChannelName() {
return parentChannelName; return parentChannelName;
@ -223,4 +224,22 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
public void setVerified(final boolean verified) { public void setVerified(final boolean verified) {
this.verified = verified; this.verified = verified;
} }
@Nonnull
public List<ListLinkHandler> getTabs() {
return tabs;
}
public void setTabs(@Nonnull final List<ListLinkHandler> tabs) {
this.tabs = tabs;
}
@Nonnull
public List<String> getTags() {
return tags;
}
public void setTags(@Nonnull final List<String> tags) {
this.tags = tags;
}
} }

View File

@ -0,0 +1,25 @@
package org.schabi.newpipe.extractor.channel.tabs;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import javax.annotation.Nonnull;
/**
* A {@link ListExtractor} of {@link InfoItem}s for tabs of channels.
*/
public abstract class ChannelTabExtractor extends ListExtractor<InfoItem> {
protected ChannelTabExtractor(@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Nonnull
@Override
public String getName() {
return getLinkHandler().getContentFilters().get(0);
}
}

View File

@ -0,0 +1,70 @@
package org.schabi.newpipe.extractor.channel.tabs;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfo;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import javax.annotation.Nonnull;
import java.io.IOException;
public class ChannelTabInfo extends ListInfo<InfoItem> {
public ChannelTabInfo(final int serviceId,
@Nonnull final ListLinkHandler linkHandler) {
super(serviceId, linkHandler, linkHandler.getContentFilters().get(0));
}
/**
* Get a {@link ChannelTabInfo} instance from the given service and tab handler.
*
* @param service streaming service
* @param linkHandler Channel tab handler (from {@link ChannelInfo})
* @return the extracted {@link ChannelTabInfo}
*/
@Nonnull
public static ChannelTabInfo getInfo(@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler)
throws ExtractionException, IOException {
final ChannelTabExtractor extractor = service.getChannelTabExtractor(linkHandler);
extractor.fetchPage();
return getInfo(extractor);
}
/**
* Get a {@link ChannelTabInfo} instance from a {@link ChannelTabExtractor}.
*
* @param extractor an extractor where {@code fetchPage()} was already got called on
* @return the extracted {@link ChannelTabInfo}
*/
@Nonnull
public static ChannelTabInfo getInfo(@Nonnull final ChannelTabExtractor extractor) {
final ChannelTabInfo info =
new ChannelTabInfo(extractor.getServiceId(), extractor.getLinkHandler());
try {
info.setOriginalUrl(extractor.getOriginalUrl());
} catch (final Exception e) {
info.addError(e);
}
final ListExtractor.InfoItemsPage<InfoItem> page
= ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(page.getItems());
info.setNextPage(page.getNextPage());
return info;
}
public static ListExtractor.InfoItemsPage<InfoItem> getMoreItems(
@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler,
@Nonnull final Page page) throws ExtractionException, IOException {
return service.getChannelTabExtractor(linkHandler).getPage(page);
}
}

View File

@ -0,0 +1,17 @@
package org.schabi.newpipe.extractor.channel.tabs;
/**
* Constants of channel tabs supported by the extractor.
*/
public final class ChannelTabs {
public static final String VIDEOS = "videos";
public static final String TRACKS = "tracks";
public static final String SHORTS = "shorts";
public static final String LIVESTREAMS = "livestreams";
public static final String CHANNELS = "channels";
public static final String PLAYLISTS = "playlists";
public static final String ALBUMS = "albums";
private ChannelTabs() {
}
}

View File

@ -0,0 +1,7 @@
package org.schabi.newpipe.extractor.exceptions;
public final class UnsupportedTabException extends UnsupportedOperationException {
public UnsupportedTabException(final String unsupportedTab) {
super("Unsupported tab " + unsupportedTab);
}
}

View File

@ -31,13 +31,14 @@ public abstract class LinkHandlerFactory {
// To Override // To Override
/////////////////////////////////// ///////////////////////////////////
public abstract String getId(String url) throws ParsingException; public abstract String getId(String url) throws ParsingException, UnsupportedOperationException;
public abstract String getUrl(String id) throws ParsingException; public abstract String getUrl(String id) throws ParsingException, UnsupportedOperationException;
public abstract boolean onAcceptUrl(String url) throws ParsingException; public abstract boolean onAcceptUrl(String url) throws ParsingException;
public String getUrl(final String id, final String baseUrl) throws ParsingException { public String getUrl(final String id, final String baseUrl)
throws ParsingException, UnsupportedOperationException {
return getUrl(id); return getUrl(id);
} }

View File

@ -14,12 +14,13 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
/////////////////////////////////// ///////////////////////////////////
public abstract String getUrl(String id, List<String> contentFilter, String sortFilter) public abstract String getUrl(String id, List<String> contentFilter, String sortFilter)
throws ParsingException; throws ParsingException, UnsupportedOperationException;
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter, final String sortFilter,
final String baseUrl) throws ParsingException { final String baseUrl)
throws ParsingException, UnsupportedOperationException {
return getUrl(id, contentFilter, sortFilter); return getUrl(id, contentFilter, sortFilter);
} }
@ -72,7 +73,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
* *
* @return the url corresponding to id without any filters applied * @return the url corresponding to id without any filters applied
*/ */
public String getUrl(final String id) throws ParsingException { public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
return getUrl(id, new ArrayList<>(0), ""); return getUrl(id, new ArrayList<>(0), "");
} }

View File

@ -0,0 +1,55 @@
package org.schabi.newpipe.extractor.linkhandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import javax.annotation.Nonnull;
import java.io.Serializable;
import java.util.List;
/**
* A {@link ListLinkHandler} which can be used to be returned from {@link
* org.schabi.newpipe.extractor.channel.ChannelInfo#getTabs() ChannelInfo#getTabs()} when a
* specific tab's data has already been fetched.
*
* <p>
* This class allows passing a builder for a {@link ChannelTabExtractor} that can hold references
* to variables.
* </p>
*
* <p>
* Note: a service that wishes to use this class in one of its {@link
* org.schabi.newpipe.extractor.channel.ChannelExtractor ChannelExtractor}s must also add the
* following snippet of code in the service's
* {@link StreamingService#getChannelTabExtractor(ListLinkHandler)}:
* <pre>
* if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
* return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
* }
* </pre>
* </p>
*/
public class ReadyChannelTabListLinkHandler extends ListLinkHandler {
public interface ChannelTabExtractorBuilder extends Serializable {
@Nonnull
ChannelTabExtractor build(@Nonnull StreamingService service,
@Nonnull ListLinkHandler linkHandler);
}
private final ChannelTabExtractorBuilder extractorBuilder;
public ReadyChannelTabListLinkHandler(
final String url,
final String channelId,
@Nonnull final String channelTab,
@Nonnull final ChannelTabExtractorBuilder extractorBuilder) {
super(url, url, channelId, List.of(channelTab), "");
this.extractorBuilder = extractorBuilder;
}
@Nonnull
public ChannelTabExtractor getChannelTabExtractor(@Nonnull final StreamingService service) {
return extractorBuilder.build(service, new ListLinkHandler(this));
}
}

View File

@ -13,7 +13,7 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
@Override @Override
public abstract String getUrl(String query, List<String> contentFilter, String sortFilter) public abstract String getUrl(String query, List<String> contentFilter, String sortFilter)
throws ParsingException; throws ParsingException, UnsupportedOperationException;
@SuppressWarnings("unused") @SuppressWarnings("unused")
public String getSearchString(final String url) { public String getSearchString(final String url) {
@ -25,7 +25,7 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
/////////////////////////////////// ///////////////////////////////////
@Override @Override
public String getId(final String url) { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return getSearchString(url); return getSearchString(url);
} }

View File

@ -7,14 +7,21 @@ import org.schabi.newpipe.extractor.utils.Parser;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.time.ZoneOffset; import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit; import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.util.regex.MatchResult;
/** /**
* A helper class that is meant to be used by services that need to parse upload dates in the * A helper class that is meant to be used by services that need to parse durations such as
* format '2 days ago' or similar. * {@code 23 seconds} and/or upload dates in the format {@code 2 days ago} or similar.
*/ */
public class TimeAgoParser { public class TimeAgoParser {
private static final Pattern DURATION_PATTERN = Pattern.compile("(?:(\\d+) )?([A-z]+)");
private final PatternsHolder patternsHolder; private final PatternsHolder patternsHolder;
private final OffsetDateTime now; private final OffsetDateTime now;
@ -60,6 +67,48 @@ public class TimeAgoParser {
return getResultFor(parseTimeAgoAmount(textualDate), parseChronoUnit(textualDate)); return getResultFor(parseTimeAgoAmount(textualDate), parseChronoUnit(textualDate));
} }
/**
* Parses a textual duration into a duration computer number.
*
* @param textualDuration the textual duration to parse
* @return the textual duration parsed, as a primitive {@code long}
* @throws ParsingException if the textual duration could not be parsed
*/
public long parseDuration(final String textualDuration) throws ParsingException {
// We can't use Matcher.results, as it is only available on Android 14 and above
final Matcher matcher = DURATION_PATTERN.matcher(textualDuration);
final List<MatchResult> results = new ArrayList<>();
while (matcher.find()) {
results.add(matcher.toMatchResult());
}
return results.stream()
.map(match -> {
final String digits = match.group(1);
final String word = match.group(2);
int amount;
try {
amount = Integer.parseInt(digits);
} catch (final NumberFormatException ignored) {
amount = 1;
}
final ChronoUnit unit;
try {
unit = parseChronoUnit(word);
} catch (final ParsingException ignored) {
return 0L;
}
return amount * unit.getDuration().getSeconds();
})
.filter(n -> n > 0)
.reduce(Long::sum)
.orElseThrow(() -> new ParsingException(
"Could not parse duration \"" + textualDuration + "\""));
}
private int parseTimeAgoAmount(final String textualDate) { private int parseTimeAgoAmount(final String textualDate) {
try { try {
return Integer.parseInt(textualDate.replaceAll("\\D+", "")); return Integer.parseInt(textualDate.replaceAll("\\D+", ""));

View File

@ -12,6 +12,7 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.kiosk.KioskList;
@ -19,11 +20,13 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelTabExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampCommentsExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampCommentsExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor;
@ -34,6 +37,7 @@ import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchE
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSuggestionExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSuggestionExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampCommentsLinkHandlerFactory; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampPlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampPlaylistLinkHandlerFactory;
@ -58,27 +62,32 @@ public class BandcampService extends StreamingService {
@Override @Override
public LinkHandlerFactory getStreamLHFactory() { public LinkHandlerFactory getStreamLHFactory() {
return new BandcampStreamLinkHandlerFactory(); return BandcampStreamLinkHandlerFactory.getInstance();
} }
@Override @Override
public ListLinkHandlerFactory getChannelLHFactory() { public ListLinkHandlerFactory getChannelLHFactory() {
return new BandcampChannelLinkHandlerFactory(); return BandcampChannelLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getChannelTabLHFactory() {
return BandcampChannelTabLinkHandlerFactory.getInstance();
} }
@Override @Override
public ListLinkHandlerFactory getPlaylistLHFactory() { public ListLinkHandlerFactory getPlaylistLHFactory() {
return new BandcampPlaylistLinkHandlerFactory(); return BandcampPlaylistLinkHandlerFactory.getInstance();
} }
@Override @Override
public SearchQueryHandlerFactory getSearchQHFactory() { public SearchQueryHandlerFactory getSearchQHFactory() {
return new BandcampSearchQueryHandlerFactory(); return BandcampSearchQueryHandlerFactory.getInstance();
} }
@Override @Override
public ListLinkHandlerFactory getCommentsLHFactory() { public ListLinkHandlerFactory getCommentsLHFactory() {
return new BandcampCommentsLinkHandlerFactory(); return BandcampCommentsLinkHandlerFactory.getInstance();
} }
@Override @Override
@ -98,27 +107,27 @@ public class BandcampService extends StreamingService {
@Override @Override
public KioskList getKioskList() throws ExtractionException { public KioskList getKioskList() throws ExtractionException {
final KioskList kioskList = new KioskList(this); final KioskList kioskList = new KioskList(this);
final ListLinkHandlerFactory h = BandcampFeaturedLinkHandlerFactory.getInstance();
try { try {
kioskList.addKioskEntry( kioskList.addKioskEntry(
(streamingService, url, kioskId) -> new BandcampFeaturedExtractor( (streamingService, url, kioskId) -> new BandcampFeaturedExtractor(
BandcampService.this, BandcampService.this,
new BandcampFeaturedLinkHandlerFactory().fromUrl(FEATURED_API_URL), h.fromUrl(FEATURED_API_URL),
kioskId kioskId
), ),
new BandcampFeaturedLinkHandlerFactory(), h,
KIOSK_FEATURED KIOSK_FEATURED
); );
kioskList.addKioskEntry( kioskList.addKioskEntry(
(streamingService, url, kioskId) -> new BandcampRadioExtractor( (streamingService, url, kioskId) -> new BandcampRadioExtractor(
BandcampService.this, BandcampService.this,
new BandcampFeaturedLinkHandlerFactory().fromUrl(RADIO_API_URL), h.fromUrl(RADIO_API_URL),
kioskId kioskId
), ),
new BandcampFeaturedLinkHandlerFactory(), h,
KIOSK_RADIO KIOSK_RADIO
); );
@ -136,6 +145,15 @@ public class BandcampService extends StreamingService {
return new BandcampChannelExtractor(this, linkHandler); return new BandcampChannelExtractor(this, linkHandler);
} }
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
} else {
return new BandcampChannelTabExtractor(this, linkHandler);
}
}
@Override @Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return new BandcampPlaylistExtractor(this, linkHandler); return new BandcampPlaylistExtractor(this, linkHandler);

View File

@ -0,0 +1,55 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
public class BandcampAlbumInfoItemExtractor implements PlaylistInfoItemExtractor {
private final JsonObject albumInfoItem;
private final String uploaderUrl;
public BandcampAlbumInfoItemExtractor(final JsonObject albumInfoItem,
final String uploaderUrl) {
this.albumInfoItem = albumInfoItem;
this.uploaderUrl = uploaderUrl;
}
@Override
public String getName() throws ParsingException {
return albumInfoItem.getString("title");
}
@Override
public String getUrl() throws ParsingException {
return BandcampExtractorHelper.getStreamUrlFromIds(
albumInfoItem.getLong("band_id"),
albumInfoItem.getLong("item_id"),
albumInfoItem.getString("item_type"));
}
@Override
public String getThumbnailUrl() throws ParsingException {
return BandcampExtractorHelper.getImageUrl(albumInfoItem.getLong("art_id"), true);
}
@Override
public String getUploaderName() throws ParsingException {
return albumInfoItem.getString("band_name");
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
@Override
public boolean isUploaderVerified() {
return false;
}
@Override
public long getStreamCount() {
return ListExtractor.ITEM_COUNT_UNKNOWN;
}
}

View File

@ -8,19 +8,22 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampDiscographStreamInfoItemExtractor; import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.stream.Stream; import java.util.stream.Stream;
@ -52,8 +55,8 @@ public class BandcampChannelExtractor extends ChannelExtractor {
*/ */
try { try {
final String html = getDownloader() final String html = getDownloader()
.get(replaceHttpWithHttps(channelInfo.getString("bandcamp_url"))) .get(replaceHttpWithHttps(channelInfo.getString("bandcamp_url")))
.responseBody(); .responseBody();
return Stream.of(Jsoup.parse(html).getElementById("customHeader")) return Stream.of(Jsoup.parse(html).getElementById("customHeader"))
.filter(Objects::nonNull) .filter(Objects::nonNull)
@ -107,29 +110,47 @@ public class BandcampChannelExtractor extends ChannelExtractor {
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException { public List<ListLinkHandler> getTabs() throws ParsingException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray discography = channelInfo.getArray("discography"); final JsonArray discography = channelInfo.getArray("discography");
final TabExtractorBuilder builder = new TabExtractorBuilder(discography);
for (int i = 0; i < discography.size(); i++) { final List<ListLinkHandler> tabs = new ArrayList<>();
// A discograph is as an item appears in a discography
final JsonObject discograph = discography.getObject(i);
if (!discograph.getString("item_type").equals("track")) { boolean foundTrackItem = false;
boolean foundAlbumItem = false;
for (final Object discographyItem : discography) {
if (foundTrackItem && foundAlbumItem) {
break;
}
if (!(discographyItem instanceof JsonObject)) {
continue; continue;
} }
collector.commit(new BandcampDiscographStreamInfoItemExtractor(discograph, getUrl())); final JsonObject discographyJsonItem = (JsonObject) discographyItem;
final String itemType = discographyJsonItem.getString("item_type");
if (!foundTrackItem && "track".equals(itemType)) {
foundTrackItem = true;
tabs.add(new ReadyChannelTabListLinkHandler(getUrl()
+ BandcampChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.TRACKS),
getId(),
ChannelTabs.TRACKS,
builder));
}
if (!foundAlbumItem && "album".equals(itemType)) {
foundAlbumItem = true;
tabs.add(new ReadyChannelTabListLinkHandler(getUrl()
+ BandcampChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.ALBUMS),
getId(),
ChannelTabs.ALBUMS,
builder));
}
} }
return new InfoItemsPage<>(collector, null); return Collections.unmodifiableList(tabs);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
return null;
} }
@Override @Override
@ -143,4 +164,20 @@ public class BandcampChannelExtractor extends ChannelExtractor {
public String getName() { public String getName() {
return channelInfo.getString("name"); return channelInfo.getString("name");
} }
private static final class TabExtractorBuilder
implements ReadyChannelTabListLinkHandler.ChannelTabExtractorBuilder {
private final JsonArray discography;
TabExtractorBuilder(final JsonArray discography) {
this.discography = discography;
}
@Nonnull
@Override
public ChannelTabExtractor build(@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler) {
return BandcampChannelTabExtractor.fromDiscography(service, linkHandler, discography);
}
}
} }

View File

@ -0,0 +1,95 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampDiscographStreamInfoItemExtractor;
import javax.annotation.Nonnull;
import java.io.IOException;
public class BandcampChannelTabExtractor extends ChannelTabExtractor {
private JsonArray discography;
private final String filter;
public BandcampChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
final String tab = linkHandler.getContentFilters().get(0);
switch (tab) {
case ChannelTabs.TRACKS:
filter = "track";
break;
case ChannelTabs.ALBUMS:
filter = "album";
break;
default:
throw new IllegalArgumentException("Unsupported channel tab: " + tab);
}
}
public static BandcampChannelTabExtractor fromDiscography(final StreamingService service,
final ListLinkHandler linkHandler,
final JsonArray discography) {
final BandcampChannelTabExtractor tabExtractor =
new BandcampChannelTabExtractor(service, linkHandler);
tabExtractor.discography = discography;
return tabExtractor;
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws ParsingException {
if (discography == null) {
discography = BandcampExtractorHelper.getArtistDetails(getId())
.getArray("discography");
}
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
for (final Object discograph : discography) {
// A discograph is as an item appears in a discography
if (!(discograph instanceof JsonObject)) {
continue;
}
final JsonObject discographJsonObject = (JsonObject) discograph;
final String itemType = discographJsonObject.getString("item_type", "");
if (!itemType.equals(filter)) {
continue;
}
switch (itemType) {
case "track":
collector.commit(new BandcampDiscographStreamInfoItemExtractor(
discographJsonObject, getUrl()));
break;
case "album":
collector.commit(new BandcampAlbumInfoItemExtractor(
discographJsonObject, getUrl()));
break;
}
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page) {
return null;
}
}

View File

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
@ -17,11 +18,20 @@ import java.util.List;
/** /**
* Artist do have IDs that are useful * Artist do have IDs that are useful
*/ */
public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { public final class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final BandcampChannelLinkHandlerFactory INSTANCE
= new BandcampChannelLinkHandlerFactory();
private BandcampChannelLinkHandlerFactory() {
}
public static BandcampChannelLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
try { try {
final String response = NewPipe.getDownloader().get(url).responseBody(); final String response = NewPipe.getDownloader().get(url).responseBody();
@ -41,16 +51,13 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory {
*/ */
@Override @Override
public String getUrl(final String id, final List<String> contentFilter, final String sortFilter) public String getUrl(final String id, final List<String> contentFilter, final String sortFilter)
throws ParsingException { throws ParsingException, UnsupportedOperationException {
try { final JsonObject artistDetails = BandcampExtractorHelper.getArtistDetails(id);
return BandcampExtractorHelper.getArtistDetails(id) if (artistDetails.getBoolean("error")) {
.getString("bandcamp_url")
.replace("http://", "https://");
} catch (final NullPointerException e) {
throw new ParsingException( throw new ParsingException(
"JSON does not contain URL (invalid id?) or is otherwise invalid", e); "JSON does not contain a channel URL (invalid id?) or is otherwise invalid");
} }
return Utils.replaceHttpWithHttps(artistDetails.getString("bandcamp_url"));
} }
/** /**
@ -61,22 +68,21 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory {
final String lowercaseUrl = url.toLowerCase(); final String lowercaseUrl = url.toLowerCase();
// https: | | artist.bandcamp.com | releases // https: | | artist.bandcamp.com | releases - music - album - track ( | name)
// 0 1 2 3 // 0 1 2 3 (4)
final String[] splitUrl = lowercaseUrl.split("/"); final String[] splitUrl = lowercaseUrl.split("/");
// URL is too short // URL is too short
if (splitUrl.length < 3) { if (splitUrl.length != 3 && splitUrl.length != 4) {
return false; return false;
} }
// Must have "releases" or "music" as segment after url or none at all // Must have "releases", "music", "album" or "track" as segment after URL or none at all
if (splitUrl.length > 3 && !( if (splitUrl.length == 4 && !(splitUrl[3].equals("releases")
splitUrl[3].equals("releases") || splitUrl[3].equals("music") || splitUrl[3].equals("music")
)) { || splitUrl[3].equals("album")
|| splitUrl[3].equals("track"))) {
return false; return false;
} else { } else {
if (splitUrl[2].equals("daily.bandcamp.com")) { if (splitUrl[2].equals("daily.bandcamp.com")) {
// Refuse links to daily.bandcamp.com as that is not an artist // Refuse links to daily.bandcamp.com as that is not an artist

View File

@ -0,0 +1,72 @@
package org.schabi.newpipe.extractor.services.bandcamp.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.UnsupportedTabException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import javax.annotation.Nonnull;
import java.util.List;
public final class BandcampChannelTabLinkHandlerFactory extends ListLinkHandlerFactory {
private static final BandcampChannelTabLinkHandlerFactory INSTANCE
= new BandcampChannelTabLinkHandlerFactory();
private BandcampChannelTabLinkHandlerFactory() {
}
public static BandcampChannelTabLinkHandlerFactory getInstance() {
return INSTANCE;
}
/**
* Get a tab's URL suffix.
*
* <p>
* These URLs don't actually exist on the Bandcamp website, as both albums and tracks are
* listed on the main page, but redirect to the main page, which is perfect for us as we need a
* unique URL for each tab.
* </p>
*
* @param tab the tab value, which must not be null
* @return a URL suffix
* @throws UnsupportedTabException if the tab is not supported
*/
@Nonnull
public static String getUrlSuffix(@Nonnull final String tab) throws UnsupportedTabException {
switch (tab) {
case ChannelTabs.TRACKS:
return "/track";
case ChannelTabs.ALBUMS:
return "/album";
}
throw new UnsupportedTabException(tab);
}
@Override
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return BandcampChannelLinkHandlerFactory.getInstance().getId(url);
}
@Override
public String getUrl(final String id, final List<String> contentFilter, final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return BandcampChannelLinkHandlerFactory.getInstance().getUrl(id)
+ getUrlSuffix(contentFilter.get(0));
}
@Override
public boolean onAcceptUrl(final String url) throws ParsingException {
return BandcampChannelLinkHandlerFactory.getInstance().onAcceptUrl(url);
}
@Override
public String[] getAvailableContentFilter() {
return new String[]{
ChannelTabs.TRACKS,
ChannelTabs.ALBUMS,
};
}
}

View File

@ -10,10 +10,20 @@ import java.util.List;
* Like in {@link BandcampStreamLinkHandlerFactory}, tracks have no meaningful IDs except for * Like in {@link BandcampStreamLinkHandlerFactory}, tracks have no meaningful IDs except for
* their URLs * their URLs
*/ */
public class BandcampCommentsLinkHandlerFactory extends ListLinkHandlerFactory { public final class BandcampCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final BandcampCommentsLinkHandlerFactory INSTANCE
= new BandcampCommentsLinkHandlerFactory();
private BandcampCommentsLinkHandlerFactory() {
}
public static BandcampCommentsLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return url; return url;
} }
@ -35,7 +45,8 @@ public class BandcampCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) throws ParsingException { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return id; return id;
} }
} }

View File

@ -2,6 +2,7 @@
package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; package org.schabi.newpipe.extractor.services.bandcamp.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
@ -13,12 +14,23 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.KIOSK_RADIO; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.KIOSK_RADIO;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.RADIO_API_URL; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.RADIO_API_URL;
public class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory { public final class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory {
private static final BandcampFeaturedLinkHandlerFactory INSTANCE =
new BandcampFeaturedLinkHandlerFactory();
private BandcampFeaturedLinkHandlerFactory() {
}
public static BandcampFeaturedLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
if (id.equals(KIOSK_FEATURED)) { if (id.equals(KIOSK_FEATURED)) {
return FEATURED_API_URL; // doesn't have a website return FEATURED_API_URL; // doesn't have a website
} else if (id.equals(KIOSK_RADIO)) { } else if (id.equals(KIOSK_RADIO)) {
@ -29,7 +41,7 @@ public class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory {
} }
@Override @Override
public String getId(final String url) { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
final String fixedUrl = Utils.replaceHttpWithHttps(url); final String fixedUrl = Utils.replaceHttpWithHttps(url);
if (BandcampExtractorHelper.isRadioUrl(fixedUrl) || fixedUrl.equals(RADIO_API_URL)) { if (BandcampExtractorHelper.isRadioUrl(fixedUrl) || fixedUrl.equals(RADIO_API_URL)) {
return KIOSK_RADIO; return KIOSK_RADIO;

View File

@ -11,16 +11,28 @@ import java.util.List;
/** /**
* Just as with streams, the album ids are essentially useless for us. * Just as with streams, the album ids are essentially useless for us.
*/ */
public class BandcampPlaylistLinkHandlerFactory extends ListLinkHandlerFactory { public final class BandcampPlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
private static final BandcampPlaylistLinkHandlerFactory INSTANCE
= new BandcampPlaylistLinkHandlerFactory();
private BandcampPlaylistLinkHandlerFactory() {
}
public static BandcampPlaylistLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return getUrl(url); return getUrl(url);
} }
@Override @Override
public String getUrl(final String url, public String getUrl(final String url,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) throws ParsingException { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return url; return url;
} }

View File

@ -11,11 +11,23 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.List; import java.util.List;
public class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public final class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
private static final BandcampSearchQueryHandlerFactory INSTANCE
= new BandcampSearchQueryHandlerFactory();
private BandcampSearchQueryHandlerFactory() {
}
public static BandcampSearchQueryHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String getUrl(final String query, public String getUrl(final String query,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) throws ParsingException { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try { try {
return BASE_URL + "/search?q=" + Utils.encodeUrlUtf8(query) + "&page=1"; return BASE_URL + "/search?q=" + Utils.encodeUrlUtf8(query) + "&page=1";
} catch (final UnsupportedEncodingException e) { } catch (final UnsupportedEncodingException e) {

View File

@ -14,14 +14,24 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp
* *
* <p>Radio (bandcamp weekly) shows do have ids.</p> * <p>Radio (bandcamp weekly) shows do have ids.</p>
*/ */
public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { public final class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final BandcampStreamLinkHandlerFactory INSTANCE
= new BandcampStreamLinkHandlerFactory();
private BandcampStreamLinkHandlerFactory() {
}
public static BandcampStreamLinkHandlerFactory getInstance() {
return INSTANCE;
}
/** /**
* @see BandcampStreamLinkHandlerFactory * @see BandcampStreamLinkHandlerFactory
*/ */
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
if (BandcampExtractorHelper.isRadioUrl(url)) { if (BandcampExtractorHelper.isRadioUrl(url)) {
return url.split("bandcamp.com/\\?show=")[1]; return url.split("bandcamp.com/\\?show=")[1];
} else { } else {
@ -34,7 +44,8 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory {
* @see BandcampStreamLinkHandlerFactory * @see BandcampStreamLinkHandlerFactory
*/ */
@Override @Override
public String getUrl(final String input) { public String getUrl(final String input)
throws ParsingException, UnsupportedOperationException {
if (input.matches("\\d+")) { if (input.matches("\\d+")) {
return BASE_URL + "/?show=" + input; return BASE_URL + "/?show=" + input;
} else { } else {

View File

@ -6,6 +6,7 @@ import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.kiosk.KioskList;
@ -13,6 +14,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
@ -27,8 +29,6 @@ import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearch
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCLiveListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCRecentListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor;
@ -47,12 +47,17 @@ public class MediaCCCService extends StreamingService {
@Override @Override
public LinkHandlerFactory getStreamLHFactory() { public LinkHandlerFactory getStreamLHFactory() {
return new MediaCCCStreamLinkHandlerFactory(); return MediaCCCStreamLinkHandlerFactory.getInstance();
} }
@Override @Override
public ListLinkHandlerFactory getChannelLHFactory() { public ListLinkHandlerFactory getChannelLHFactory() {
return new MediaCCCConferenceLinkHandlerFactory(); return MediaCCCConferenceLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getChannelTabLHFactory() {
return null;
} }
@Override @Override
@ -62,7 +67,7 @@ public class MediaCCCService extends StreamingService {
@Override @Override
public SearchQueryHandlerFactory getSearchQHFactory() { public SearchQueryHandlerFactory getSearchQHFactory() {
return new MediaCCCSearchQueryHandlerFactory(); return MediaCCCSearchQueryHandlerFactory.getInstance();
} }
@Override @Override
@ -78,6 +83,22 @@ public class MediaCCCService extends StreamingService {
return new MediaCCCConferenceExtractor(this, linkHandler); return new MediaCCCConferenceExtractor(this, linkHandler);
} }
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
}
/*
Channel tab extractors are only supported in conferences and should only come from a
ReadyChannelTabListLinkHandler instance with a ChannelTabExtractorBuilder instance of the
conferences extractor
If that's not the case, return null in this case, so no channel tabs support
*/
return null;
}
@Override @Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return null; return null;
@ -91,36 +112,37 @@ public class MediaCCCService extends StreamingService {
@Override @Override
public KioskList getKioskList() throws ExtractionException { public KioskList getKioskList() throws ExtractionException {
final KioskList list = new KioskList(this); final KioskList list = new KioskList(this);
final ListLinkHandlerFactory h = MediaCCCConferencesListLinkHandlerFactory.getInstance();
// add kiosks here e.g.: // add kiosks here e.g.:
try { try {
list.addKioskEntry( list.addKioskEntry(
(streamingService, url, kioskId) -> new MediaCCCConferenceKiosk( (streamingService, url, kioskId) -> new MediaCCCConferenceKiosk(
MediaCCCService.this, MediaCCCService.this,
new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), h.fromUrl(url),
kioskId kioskId
), ),
new MediaCCCConferencesListLinkHandlerFactory(), h,
MediaCCCConferenceKiosk.KIOSK_ID MediaCCCConferenceKiosk.KIOSK_ID
); );
list.addKioskEntry( list.addKioskEntry(
(streamingService, url, kioskId) -> new MediaCCCRecentKiosk( (streamingService, url, kioskId) -> new MediaCCCRecentKiosk(
MediaCCCService.this, MediaCCCService.this,
new MediaCCCRecentListLinkHandlerFactory().fromUrl(url), h.fromUrl(url),
kioskId kioskId
), ),
new MediaCCCRecentListLinkHandlerFactory(), h,
MediaCCCRecentKiosk.KIOSK_ID MediaCCCRecentKiosk.KIOSK_ID
); );
list.addKioskEntry( list.addKioskEntry(
(streamingService, url, kioskId) -> new MediaCCCLiveStreamKiosk( (streamingService, url, kioskId) -> new MediaCCCLiveStreamKiosk(
MediaCCCService.this, MediaCCCService.this,
new MediaCCCLiveListLinkHandlerFactory().fromUrl(url), h.fromUrl(url),
kioskId kioskId
), ),
new MediaCCCLiveListLinkHandlerFactory(), h,
MediaCCCLiveStreamKiosk.KIOSK_ID MediaCCCLiveStreamKiosk.KIOSK_ID
); );

View File

@ -1,23 +1,29 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors; package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.io.IOException;
import java.util.List;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException;
public class MediaCCCConferenceExtractor extends ChannelExtractor { public class MediaCCCConferenceExtractor extends ChannelExtractor {
private JsonObject conferenceData; private JsonObject conferenceData;
@ -74,18 +80,9 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() { public List<ListLinkHandler> getTabs() throws ParsingException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); return List.of(new ReadyChannelTabListLinkHandler(getUrl(), getId(),
final JsonArray events = conferenceData.getArray("events"); ChannelTabs.VIDEOS, new VideosTabExtractorBuilder(conferenceData)));
for (int i = 0; i < events.size(); i++) {
collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i)));
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
return InfoItemsPage.emptyPage();
} }
@Override @Override
@ -105,4 +102,55 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
public String getName() throws ParsingException { public String getName() throws ParsingException {
return conferenceData.getString("title"); return conferenceData.getString("title");
} }
private static final class VideosTabExtractorBuilder
implements ReadyChannelTabListLinkHandler.ChannelTabExtractorBuilder {
private final JsonObject conferenceData;
VideosTabExtractorBuilder(final JsonObject conferenceData) {
this.conferenceData = conferenceData;
}
@Nonnull
@Override
public ChannelTabExtractor build(@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler) {
return new VideosChannelTabExtractor(service, linkHandler, conferenceData);
}
}
private static final class VideosChannelTabExtractor extends ChannelTabExtractor {
private final JsonObject conferenceData;
VideosChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final JsonObject conferenceData) {
super(service, linkHandler);
this.conferenceData = conferenceData;
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) {
// Nothing to do here, as data was already fetched
}
@Nonnull
@Override
public ListExtractor.InfoItemsPage<InfoItem> getInitialPage() {
final MultiInfoItemsCollector collector =
new MultiInfoItemsCollector(getServiceId());
conferenceData.getArray("events")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.forEach(event -> collector.commit(new MediaCCCStreamInfoItemExtractor(event)));
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page) {
return InfoItemsPage.emptyPage();
}
}
} }

View File

@ -65,7 +65,7 @@ public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor {
@Override @Override
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
return new MediaCCCConferenceLinkHandlerFactory() return MediaCCCConferenceLinkHandlerFactory.getInstance()
.fromUrl(event.getString("conference_url")) // API URL .fromUrl(event.getString("conference_url")) // API URL
.getUrl(); // web URL .getUrl(); // web URL
} }

View File

@ -38,7 +38,8 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
super(service, linkHandler); super(service, linkHandler);
try { try {
conferenceKiosk = new MediaCCCConferenceKiosk(service, conferenceKiosk = new MediaCCCConferenceKiosk(service,
new MediaCCCConferencesListLinkHandlerFactory().fromId("conferences"), MediaCCCConferencesListLinkHandlerFactory.getInstance()
.fromId("conferences"),
"conferences"); "conferences");
} catch (final Exception e) { } catch (final Exception e) {
e.printStackTrace(); e.printStackTrace();

View File

@ -6,7 +6,11 @@ import org.schabi.newpipe.extractor.utils.Parser;
import java.util.List; import java.util.List;
public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory { public final class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory {
private static final MediaCCCConferenceLinkHandlerFactory INSTANCE
= new MediaCCCConferenceLinkHandlerFactory();
public static final String CONFERENCE_API_ENDPOINT public static final String CONFERENCE_API_ENDPOINT
= "https://api.media.ccc.de/public/conferences/"; = "https://api.media.ccc.de/public/conferences/";
public static final String CONFERENCE_PATH = "https://media.ccc.de/c/"; public static final String CONFERENCE_PATH = "https://media.ccc.de/c/";
@ -14,15 +18,23 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory
= "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/conferences/)" = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/conferences/)"
+ "|(?:media\\.ccc\\.de/[bc]/))([^/?&#]*)"; + "|(?:media\\.ccc\\.de/[bc]/))([^/?&#]*)";
private MediaCCCConferenceLinkHandlerFactory() {
}
public static MediaCCCConferenceLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) throws ParsingException { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return CONFERENCE_PATH + id; return CONFERENCE_PATH + id;
} }
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return Parser.matchGroup1(ID_PATTERN, url); return Parser.matchGroup1(ID_PATTERN, url);
} }

View File

@ -5,15 +5,28 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List; import java.util.List;
public class MediaCCCConferencesListLinkHandlerFactory extends ListLinkHandlerFactory { public final class MediaCCCConferencesListLinkHandlerFactory extends ListLinkHandlerFactory {
private static final MediaCCCConferencesListLinkHandlerFactory INSTANCE =
new MediaCCCConferencesListLinkHandlerFactory();
private MediaCCCConferencesListLinkHandlerFactory() {
}
public static MediaCCCConferencesListLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return "conferences"; return "conferences";
} }
@Override @Override
public String getUrl(final String id, final List<String> contentFilter, public String getUrl(final String id,
final String sortFilter) throws ParsingException { final List<String> contentFilter,
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://media.ccc.de/public/conferences"; return "https://media.ccc.de/public/conferences";
} }

View File

@ -6,11 +6,22 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List; import java.util.List;
import java.util.regex.Pattern; import java.util.regex.Pattern;
public class MediaCCCLiveListLinkHandlerFactory extends ListLinkHandlerFactory { public final class MediaCCCLiveListLinkHandlerFactory extends ListLinkHandlerFactory {
private static final MediaCCCLiveListLinkHandlerFactory INSTANCE =
new MediaCCCLiveListLinkHandlerFactory();
private static final String STREAM_PATTERN = "^(?:https?://)?media\\.ccc\\.de/live$"; private static final String STREAM_PATTERN = "^(?:https?://)?media\\.ccc\\.de/live$";
private MediaCCCLiveListLinkHandlerFactory() {
}
public static MediaCCCLiveListLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return "live"; return "live";
} }
@ -22,7 +33,8 @@ public class MediaCCCLiveListLinkHandlerFactory extends ListLinkHandlerFactory {
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) throws ParsingException { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
// FIXME: wrong URL; should be https://streaming.media.ccc.de/{conference_slug}/{room_slug} // FIXME: wrong URL; should be https://streaming.media.ccc.de/{conference_slug}/{room_slug}
return "https://media.ccc.de/live"; return "https://media.ccc.de/live";
} }

View File

@ -1,32 +0,0 @@
package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
public class MediaCCCLiveStreamLinkHandlerFactory extends LinkHandlerFactory {
public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/";
private static final String VIDEO_PATH = "https://streaming.media.ccc.de/v/";
private static final String ID_PATTERN
= "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)"
+ "|(?:media\\.ccc\\.de/v/))([^/?&#]*)";
@Override
public String getId(final String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);
}
@Override
public String getUrl(final String id) throws ParsingException {
return VIDEO_PATH + id;
}
@Override
public boolean onAcceptUrl(final String url) {
try {
return getId(url) != null;
} catch (final ParsingException e) {
return false;
}
}
}

View File

@ -1,15 +1,27 @@
package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List; import java.util.List;
import java.util.regex.Pattern; import java.util.regex.Pattern;
public class MediaCCCRecentListLinkHandlerFactory extends ListLinkHandlerFactory { public final class MediaCCCRecentListLinkHandlerFactory extends ListLinkHandlerFactory {
private static final MediaCCCRecentListLinkHandlerFactory INSTANCE =
new MediaCCCRecentListLinkHandlerFactory();
private static final String PATTERN = "^(https?://)?media\\.ccc\\.de/recent/?$"; private static final String PATTERN = "^(https?://)?media\\.ccc\\.de/recent/?$";
private MediaCCCRecentListLinkHandlerFactory() {
}
public static MediaCCCRecentListLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String getId(final String url) { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return "recent"; return "recent";
} }
@ -21,7 +33,8 @@ public class MediaCCCRecentListLinkHandlerFactory extends ListLinkHandlerFactory
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://media.ccc.de/recent"; return "https://media.ccc.de/recent";
} }
} }

View File

@ -7,11 +7,22 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.List; import java.util.List;
public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public final class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
private static final MediaCCCSearchQueryHandlerFactory INSTANCE =
new MediaCCCSearchQueryHandlerFactory();
public static final String ALL = "all"; public static final String ALL = "all";
public static final String CONFERENCES = "conferences"; public static final String CONFERENCES = "conferences";
public static final String EVENTS = "events"; public static final String EVENTS = "events";
private MediaCCCSearchQueryHandlerFactory() {
}
public static MediaCCCSearchQueryHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String[] getAvailableContentFilter() { public String[] getAvailableContentFilter() {
return new String[]{ return new String[]{
@ -27,8 +38,10 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory
} }
@Override @Override
public String getUrl(final String query, final List<String> contentFilter, public String getUrl(final String query,
final String sortFilter) throws ParsingException { final List<String> contentFilter,
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try { try {
return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query); return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query);
} catch (final UnsupportedEncodingException e) { } catch (final UnsupportedEncodingException e) {

View File

@ -5,7 +5,11 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { public final class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final MediaCCCStreamLinkHandlerFactory INSTANCE =
new MediaCCCStreamLinkHandlerFactory();
public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/"; public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/";
private static final String VIDEO_PATH = "https://media.ccc.de/v/"; private static final String VIDEO_PATH = "https://media.ccc.de/v/";
private static final String RECORDING_ID_PATTERN private static final String RECORDING_ID_PATTERN
@ -15,8 +19,15 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final String LIVE_STREAM_ID_PATTERN private static final String LIVE_STREAM_ID_PATTERN
= "streaming\\.media\\.ccc\\.de\\/(\\w+\\/\\w+)"; = "streaming\\.media\\.ccc\\.de\\/(\\w+\\/\\w+)";
private MediaCCCStreamLinkHandlerFactory() {
}
public static MediaCCCStreamLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
String streamId = null; String streamId = null;
try { try {
streamId = Parser.matchGroup1(LIVE_STREAM_ID_PATTERN, url); streamId = Parser.matchGroup1(LIVE_STREAM_ID_PATTERN, url);
@ -30,7 +41,7 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
} }
@Override @Override
public String getUrl(final String id) throws ParsingException { public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
if (MediaCCCParsingHelper.isLiveStreamId(id)) { if (MediaCCCParsingHelper.isLiveStreamId(id)) {
return LIVE_STREAM_PATH + id; return LIVE_STREAM_PATH + id;
} }

View File

@ -72,24 +72,29 @@ public final class PeertubeParsingHelper {
} }
} }
public static void collectStreamsFrom(final InfoItemsCollector collector, public static void collectItemsFrom(final InfoItemsCollector collector,
final JsonObject json, final JsonObject json,
final String baseUrl) throws ParsingException { final String baseUrl) throws ParsingException {
collectStreamsFrom(collector, json, baseUrl, false); collectItemsFrom(collector, json, baseUrl, false);
} }
/** /**
* Collect stream from json with collector * Collect items from the given JSON object with the given collector.
*
* <p>
* Supported info item types are streams with their Sepia variant, channels and playlists.
* </p>
* *
* @param collector the collector used to collect information * @param collector the collector used to collect information
* @param json the file to retrieve data from * @param json the JSOn response to retrieve data from
* @param baseUrl the base Url of the instance * @param baseUrl the base URL of the instance
* @param sepia if we should use PeertubeSepiaStreamInfoItemExtractor * @param sepia if we should use {@code PeertubeSepiaStreamInfoItemExtractor} to extract
* streams or {@code PeertubeStreamInfoItemExtractor} otherwise
*/ */
public static void collectStreamsFrom(final InfoItemsCollector collector, public static void collectItemsFrom(final InfoItemsCollector collector,
final JsonObject json, final JsonObject json,
final String baseUrl, final String baseUrl,
final boolean sepia) throws ParsingException { final boolean sepia) throws ParsingException {
final JsonArray contents; final JsonArray contents;
try { try {
contents = (JsonArray) JsonUtils.getValue(json, "data"); contents = (JsonArray) JsonUtils.getValue(json, "data");

View File

@ -6,6 +6,7 @@ import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.kiosk.KioskList;
@ -19,6 +20,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelTabExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor;
@ -26,6 +28,7 @@ import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamE
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSuggestionExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSuggestionExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
@ -60,6 +63,11 @@ public class PeertubeService extends StreamingService {
return PeertubeChannelLinkHandlerFactory.getInstance(); return PeertubeChannelLinkHandlerFactory.getInstance();
} }
@Override
public ListLinkHandlerFactory getChannelTabLHFactory() {
return PeertubeChannelTabLinkHandlerFactory.getInstance();
}
@Override @Override
public ListLinkHandlerFactory getPlaylistLHFactory() { public ListLinkHandlerFactory getPlaylistLHFactory() {
return PeertubePlaylistLinkHandlerFactory.getInstance(); return PeertubePlaylistLinkHandlerFactory.getInstance();
@ -103,6 +111,12 @@ public class PeertubeService extends StreamingService {
} }
} }
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler)
throws ExtractionException {
return new PeertubeChannelTabExtractor(this, linkHandler);
}
@Override @Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler)
throws ExtractionException { throws ExtractionException {
@ -136,17 +150,20 @@ public class PeertubeService extends StreamingService {
@Override @Override
public KioskList getKioskList() throws ExtractionException { public KioskList getKioskList() throws ExtractionException {
final PeertubeTrendingLinkHandlerFactory h =
PeertubeTrendingLinkHandlerFactory.getInstance();
final KioskList.KioskExtractorFactory kioskFactory = (streamingService, url, id) -> final KioskList.KioskExtractorFactory kioskFactory = (streamingService, url, id) ->
new PeertubeTrendingExtractor( new PeertubeTrendingExtractor(
PeertubeService.this, PeertubeService.this,
new PeertubeTrendingLinkHandlerFactory().fromId(id), h.fromId(id),
id id
); );
final KioskList list = new KioskList(this); final KioskList list = new KioskList(this);
// add kiosks here e.g.: // add kiosks here e.g.:
final PeertubeTrendingLinkHandlerFactory h = new PeertubeTrendingLinkHandlerFactory();
try { try {
list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING);
list.addKioskEntry(kioskFactory, h, list.addKioskEntry(kioskFactory, h,
@ -160,6 +177,4 @@ public class PeertubeService extends StreamingService {
return list; return list;
} }
} }

View File

@ -4,31 +4,23 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeAccountExtractor extends ChannelExtractor { public class PeertubeAccountExtractor extends ChannelExtractor {
private JsonObject json; private JsonObject json;
@ -119,54 +111,19 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException { public List<ListLinkHandler> getTabs() throws ParsingException {
return getPage(new Page(baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&" return List.of(
+ COUNT_KEY + "=" + ITEMS_PER_PAGE)); PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(),
} List.of(ChannelTabs.VIDEOS), "", getBaseUrl()),
PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(),
@Override List.of(ChannelTabs.CHANNELS), "", getBaseUrl()));
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
JsonObject pageJson = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
pageJson = JsonParser.object().from(response.responseBody());
} catch (final Exception e) {
throw new ParsingException("Could not parse json data for account info", e);
}
}
if (pageJson != null) {
PeertubeParsingHelper.validate(pageJson);
final long total = pageJson.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, pageJson, getBaseUrl());
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {
throw new ExtractionException("Unable to get PeerTube account info");
}
} }
@Override @Override
public void onFetchPage(@Nonnull final Downloader downloader) public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException { throws IOException, ExtractionException {
String accountUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT; final Response response = downloader.get(baseUrl
if (getId().contains(ACCOUNTS)) { + PeertubeChannelLinkHandlerFactory.API_ENDPOINT + getId());
accountUrl += getId();
} else {
accountUrl += ACCOUNTS + getId();
}
final Response response = downloader.get(accountUrl);
if (response != null) { if (response != null) {
setInitialData(response.responseBody()); setInitialData(response.responseBody());
} else { } else {

View File

@ -3,30 +3,22 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeChannelExtractor extends ChannelExtractor { public class PeertubeChannelExtractor extends ChannelExtractor {
private JsonObject json; private JsonObject json;
@ -98,41 +90,12 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException { public List<ListLinkHandler> getTabs() throws ParsingException {
return getPage(new Page(baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&" return List.of(
+ COUNT_KEY + "=" + ITEMS_PER_PAGE)); PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(),
} List.of(ChannelTabs.VIDEOS), "", getBaseUrl()),
PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(),
@Override List.of(ChannelTabs.PLAYLISTS), "", getBaseUrl()));
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
JsonObject pageJson = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
pageJson = JsonParser.object().from(response.responseBody());
} catch (final Exception e) {
throw new ParsingException("Could not parse json data for channel info", e);
}
}
if (pageJson != null) {
PeertubeParsingHelper.validate(pageJson);
final long total = pageJson.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, pageJson, getBaseUrl());
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {
throw new ExtractionException("Unable to get PeerTube channel info");
}
} }
@Override @Override

View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.services.peertube.extractors; package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -52,7 +52,7 @@ public class PeertubeChannelInfoItemExtractor implements ChannelInfoItemExtracto
@Override @Override
public long getStreamCount() throws ParsingException { public long getStreamCount() throws ParsingException {
return ChannelExtractor.ITEM_COUNT_UNKNOWN; return ListExtractor.ITEM_COUNT_UNKNOWN;
} }
@Override @Override

View File

@ -0,0 +1,80 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom;
import static org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory.getUrlSuffix;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeChannelTabExtractor extends ChannelTabExtractor {
private final String baseUrl;
public PeertubeChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler)
throws ParsingException {
super(service, linkHandler);
baseUrl = getBaseUrl();
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) {
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return getPage(new Page(baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT
+ getId() + getUrlSuffix(getName()) + "?" + START_KEY + "=0&" + COUNT_KEY + "="
+ ITEMS_PER_PAGE));
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
JsonObject pageJson = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
pageJson = JsonParser.object().from(response.responseBody());
} catch (final Exception e) {
throw new ParsingException("Could not parse json data for account info", e);
}
}
if (pageJson == null) {
throw new ExtractionException("Unable to get account channel list");
}
PeertubeParsingHelper.validate(pageJson);
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
collectItemsFrom(collector, pageJson, getBaseUrl());
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), pageJson.getLong("total")));
}
}

View File

@ -23,7 +23,7 @@ import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubePlaylistExtractor extends PlaylistExtractor { public class PeertubePlaylistExtractor extends PlaylistExtractor {
@ -125,7 +125,7 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor {
final long total = json.getLong("total"); final long total = json.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl()); collectItemsFrom(collector, json, getBaseUrl());
return new InfoItemsPage<>(collector, return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total)); PeertubeParsingHelper.getNextPage(page.getUrl(), total));

View File

@ -26,7 +26,7 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeSearchExtractor extends SearchExtractor { public class PeertubeSearchExtractor extends SearchExtractor {
@ -93,7 +93,7 @@ public class PeertubeSearchExtractor extends SearchExtractor {
final long total = json.getLong("total"); final long total = json.getLong("total");
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl(), sepia); collectItemsFrom(collector, json, getBaseUrl(), sepia);
return new InfoItemsPage<>(collector, return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total)); PeertubeParsingHelper.getNextPage(page.getUrl(), total));

View File

@ -23,7 +23,7 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> { public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@ -69,7 +69,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
final long total = json.getLong("total"); final long total = json.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl()); collectItemsFrom(collector, json, getBaseUrl());
return new InfoItemsPage<>(collector, return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total)); PeertubeParsingHelper.getNextPage(page.getUrl(), total));

View File

@ -22,14 +22,15 @@ public final class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFact
} }
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return fixId(Parser.matchGroup(ID_PATTERN, url, 0)); return fixId(Parser.matchGroup(ID_PATTERN, url, 0));
} }
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilters, final List<String> contentFilters,
final String searchFilter) throws ParsingException { final String searchFilter)
throws ParsingException, UnsupportedOperationException {
return getUrl(id, contentFilters, searchFilter, ServiceList.PeerTube.getBaseUrl()); return getUrl(id, contentFilters, searchFilter, ServiceList.PeerTube.getBaseUrl());
} }
@ -38,7 +39,7 @@ public final class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFact
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter, final String sortFilter,
final String baseUrl) final String baseUrl)
throws ParsingException { throws ParsingException, UnsupportedOperationException {
if (id.matches(ID_PATTERN)) { if (id.matches(ID_PATTERN)) {
return baseUrl + "/" + fixId(id); return baseUrl + "/" + fixId(id);
} else { } else {

View File

@ -0,0 +1,71 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.UnsupportedTabException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import javax.annotation.Nonnull;
import java.util.List;
public final class PeertubeChannelTabLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubeChannelTabLinkHandlerFactory INSTANCE
= new PeertubeChannelTabLinkHandlerFactory();
private PeertubeChannelTabLinkHandlerFactory() {
}
public static PeertubeChannelTabLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Nonnull
public static String getUrlSuffix(@Nonnull final String tab)
throws UnsupportedTabException {
switch (tab) {
case ChannelTabs.VIDEOS:
return "/videos";
case ChannelTabs.CHANNELS: // only available on accounts
return "/video-channels";
case ChannelTabs.PLAYLISTS: // only available on channels
return "/video-playlists";
}
throw new UnsupportedTabException(tab);
}
@Override
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return PeertubeChannelLinkHandlerFactory.getInstance().getId(url);
}
@Override
public String getUrl(final String id, final List<String> contentFilter, final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return PeertubeChannelLinkHandlerFactory.getInstance().getUrl(id)
+ getUrlSuffix(contentFilter.get(0));
}
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter,
final String baseUrl)
throws ParsingException, UnsupportedOperationException {
return PeertubeChannelLinkHandlerFactory.getInstance().getUrl(id, null, null, baseUrl)
+ getUrlSuffix(contentFilter.get(0));
}
@Override
public boolean onAcceptUrl(final String url) throws ParsingException {
return PeertubeChannelLinkHandlerFactory.getInstance().onAcceptUrl(url);
}
@Override
public String[] getAvailableContentFilter() {
return new String[] {
ChannelTabs.VIDEOS,
ChannelTabs.CHANNELS,
ChannelTabs.PLAYLISTS,
};
}
}

View File

@ -21,7 +21,7 @@ public final class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFac
} }
@Override @Override
public String getId(final String url) throws ParsingException, IllegalArgumentException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return PeertubeStreamLinkHandlerFactory.getInstance().getId(url); // the same id is needed return PeertubeStreamLinkHandlerFactory.getInstance().getId(url); // the same id is needed
} }
@ -33,7 +33,8 @@ public final class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFac
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) throws ParsingException { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return getUrl(id, contentFilter, sortFilter, ServiceList.PeerTube.getBaseUrl()); return getUrl(id, contentFilter, sortFilter, ServiceList.PeerTube.getBaseUrl());
} }
@ -41,7 +42,8 @@ public final class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFac
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter, final String sortFilter,
final String baseUrl) throws ParsingException { final String baseUrl)
throws ParsingException, UnsupportedOperationException {
return baseUrl + String.format(COMMENTS_ENDPOINT, id); return baseUrl + String.format(COMMENTS_ENDPOINT, id);
} }

View File

@ -25,7 +25,8 @@ public final class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFac
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilters, final List<String> contentFilters,
final String sortFilter) { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return getUrl(id, contentFilters, sortFilter, ServiceList.PeerTube.getBaseUrl()); return getUrl(id, contentFilters, sortFilter, ServiceList.PeerTube.getBaseUrl());
} }
@ -33,12 +34,13 @@ public final class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFac
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilters, final List<String> contentFilters,
final String sortFilter, final String sortFilter,
final String baseUrl) { final String baseUrl)
throws ParsingException, UnsupportedOperationException {
return baseUrl + "/api/v1/video-playlists/" + id; return baseUrl + "/api/v1/video-playlists/" + id;
} }
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
try { try {
return Parser.matchGroup(ID_PATTERN, url, 2); return Parser.matchGroup(ID_PATTERN, url, 2);
} catch (final ParsingException ignored) { } catch (final ParsingException ignored) {

View File

@ -10,6 +10,9 @@ import java.util.List;
public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
private static final PeertubeSearchQueryHandlerFactory INSTANCE =
new PeertubeSearchQueryHandlerFactory();
public static final String VIDEOS = "videos"; public static final String VIDEOS = "videos";
public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index
public static final String PLAYLISTS = "playlists"; public static final String PLAYLISTS = "playlists";
@ -23,13 +26,14 @@ public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerF
} }
public static PeertubeSearchQueryHandlerFactory getInstance() { public static PeertubeSearchQueryHandlerFactory getInstance() {
return new PeertubeSearchQueryHandlerFactory(); return INSTANCE;
} }
@Override @Override
public String getUrl(final String searchString, public String getUrl(final String searchString,
final List<String> contentFilters, final List<String> contentFilters,
final String sortFilter) throws ParsingException { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
final String baseUrl; final String baseUrl;
if (!contentFilters.isEmpty() && contentFilters.get(0).startsWith("sepia_")) { if (!contentFilters.isEmpty() && contentFilters.get(0).startsWith("sepia_")) {
baseUrl = SEPIA_BASE_URL; baseUrl = SEPIA_BASE_URL;
@ -43,7 +47,8 @@ public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerF
public String getUrl(final String searchString, public String getUrl(final String searchString,
final List<String> contentFilters, final List<String> contentFilters,
final String sortFilter, final String sortFilter,
final String baseUrl) throws ParsingException { final String baseUrl)
throws ParsingException, UnsupportedOperationException {
try { try {
final String endpoint; final String endpoint;
if (contentFilters.isEmpty() if (contentFilters.isEmpty()

View File

@ -27,7 +27,7 @@ public final class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
} }
@Override @Override
public String getUrl(final String id) { public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
return getUrl(id, ServiceList.PeerTube.getBaseUrl()); return getUrl(id, ServiceList.PeerTube.getBaseUrl());
} }
@ -37,7 +37,7 @@ public final class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
} }
@Override @Override
public String getId(final String url) throws ParsingException, IllegalArgumentException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return Parser.matchGroup(ID_PATTERN, url, 4); return Parser.matchGroup(ID_PATTERN, url, 4);
} }

View File

@ -23,6 +23,9 @@ public final class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFac
KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt", KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt",
KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local"); KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local");
private PeertubeTrendingLinkHandlerFactory() {
}
public static PeertubeTrendingLinkHandlerFactory getInstance() { public static PeertubeTrendingLinkHandlerFactory getInstance() {
return INSTANCE; return INSTANCE;
} }
@ -30,7 +33,8 @@ public final class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFac
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilters, final List<String> contentFilters,
final String sortFilter) { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return getUrl(id, contentFilters, sortFilter, ServiceList.PeerTube.getBaseUrl()); return getUrl(id, contentFilters, sortFilter, ServiceList.PeerTube.getBaseUrl());
} }
@ -38,12 +42,13 @@ public final class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFac
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilters, final List<String> contentFilters,
final String sortFilter, final String sortFilter,
final String baseUrl) { final String baseUrl)
throws ParsingException, UnsupportedOperationException {
return String.format(KIOSK_MAP.get(id), baseUrl); return String.format(KIOSK_MAP.get(id), baseUrl);
} }
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
final String cleanUrl = url.replace(ServiceList.PeerTube.getBaseUrl(), "%s"); final String cleanUrl = url.replace(ServiceList.PeerTube.getBaseUrl(), "%s");
if (cleanUrl.contains("/videos/trending")) { if (cleanUrl.contains("/videos/trending")) {
return KIOSK_TRENDING; return KIOSK_TRENDING;

View File

@ -8,6 +8,7 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
@ -16,6 +17,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelInfoItemExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
@ -300,6 +302,54 @@ public final class SoundcloudParsingHelper {
return getStreamsFromApi(collector, apiUrl, false); return getStreamsFromApi(collector, apiUrl, false);
} }
public static String getInfoItemsFromApi(final MultiInfoItemsCollector collector,
final String apiUrl) throws ReCaptchaException,
ParsingException, IOException {
final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization());
if (response.responseCode() >= 400) {
throw new IOException("Could not get streams from API, HTTP "
+ response.responseCode());
}
final JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response.responseBody());
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
responseObject.getArray("collection")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.forEach(searchResult -> {
final String kind = searchResult.getString("kind", "");
switch (kind) {
case "user":
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
break;
case "track":
collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult));
break;
case "playlist":
collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult));
break;
}
});
String nextPageUrl;
try {
nextPageUrl = responseObject.getString("next_href");
if (!nextPageUrl.contains("client_id=")) {
nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
}
} catch (final Exception ignored) {
nextPageUrl = "";
}
return nextPageUrl;
}
@Nonnull @Nonnull
public static String getUploaderUrl(final JsonObject object) { public static String getUploaderUrl(final JsonObject object) {
final String url = object.getObject("user").getString("permalink_url", ""); final String url = object.getObject("user").getString("permalink_url", "");
@ -312,6 +362,7 @@ public final class SoundcloudParsingHelper {
return replaceHttpWithHttps(url); return replaceHttpWithHttps(url);
} }
@Nonnull
public static String getUploaderName(final JsonObject object) { public static String getUploaderName(final JsonObject object) {
return object.getObject("user").getString("username", ""); return object.getObject("user").getString("username", "");
} }

View File

@ -6,6 +6,7 @@ import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.kiosk.KioskList;
@ -19,6 +20,7 @@ import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelTabExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChartsExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChartsExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCommentsExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCommentsExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistExtractor;
@ -27,6 +29,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStr
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudSubscriptionExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudSubscriptionExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudSuggestionExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudSuggestionExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChartsLinkHandlerFactory; import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChartsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudCommentsLinkHandlerFactory; import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudPlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudPlaylistLinkHandlerFactory;
@ -50,7 +53,7 @@ public class SoundcloudService extends StreamingService {
@Override @Override
public SearchQueryHandlerFactory getSearchQHFactory() { public SearchQueryHandlerFactory getSearchQHFactory() {
return new SoundcloudSearchQueryHandlerFactory(); return SoundcloudSearchQueryHandlerFactory.getInstance();
} }
@Override @Override
@ -63,6 +66,11 @@ public class SoundcloudService extends StreamingService {
return SoundcloudChannelLinkHandlerFactory.getInstance(); return SoundcloudChannelLinkHandlerFactory.getInstance();
} }
@Override
public ListLinkHandlerFactory getChannelTabLHFactory() {
return SoundcloudChannelTabLinkHandlerFactory.getInstance();
}
@Override @Override
public ListLinkHandlerFactory getPlaylistLHFactory() { public ListLinkHandlerFactory getPlaylistLHFactory() {
return SoundcloudPlaylistLinkHandlerFactory.getInstance(); return SoundcloudPlaylistLinkHandlerFactory.getInstance();
@ -86,6 +94,11 @@ public class SoundcloudService extends StreamingService {
return new SoundcloudChannelExtractor(this, linkHandler); return new SoundcloudChannelExtractor(this, linkHandler);
} }
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
return new SoundcloudChannelTabExtractor(this, linkHandler);
}
@Override @Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return new SoundcloudPlaylistExtractor(this, linkHandler); return new SoundcloudPlaylistExtractor(this, linkHandler);
@ -103,14 +116,15 @@ public class SoundcloudService extends StreamingService {
@Override @Override
public KioskList getKioskList() throws ExtractionException { public KioskList getKioskList() throws ExtractionException {
final KioskList.KioskExtractorFactory chartsFactory = (streamingService, url, id) ->
new SoundcloudChartsExtractor(SoundcloudService.this,
new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id);
final KioskList list = new KioskList(this); final KioskList list = new KioskList(this);
final SoundcloudChartsLinkHandlerFactory h =
SoundcloudChartsLinkHandlerFactory.getInstance();
final KioskList.KioskExtractorFactory chartsFactory = (streamingService, url, id) ->
new SoundcloudChartsExtractor(SoundcloudService.this,
h.fromUrl(url), id);
// add kiosks here e.g.: // add kiosks here e.g.:
final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory();
try { try {
list.addKioskEntry(chartsFactory, h, "Top 50"); list.addKioskEntry(chartsFactory, h, "Top 50");
list.addKioskEntry(chartsFactory, h, "New & hot"); list.addKioskEntry(chartsFactory, h, "New & hot");

View File

@ -1,24 +1,23 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors; package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
@ -108,34 +107,22 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException { public List<ListLinkHandler> getTabs() throws ParsingException {
try { final String url = getUrl();
final StreamInfoItemsCollector streamInfoItemsCollector = final String urlTracks = url
new StreamInfoItemsCollector(getServiceId()); + SoundcloudChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.TRACKS);
final String urlPlaylists = url
+ SoundcloudChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.PLAYLISTS);
final String urlAlbums = url
+ SoundcloudChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.ALBUMS);
final String id = getId();
final String apiUrl = USERS_ENDPOINT + getId() + "/tracks" + "?client_id=" return List.of(
+ SoundcloudParsingHelper.clientId() + "&limit=20" + "&linked_partitioning=1"; new ListLinkHandler(urlTracks, urlTracks, id,
List.of(ChannelTabs.TRACKS), ""),
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, new ListLinkHandler(urlPlaylists, urlPlaylists, id,
streamInfoItemsCollector, apiUrl); List.of(ChannelTabs.PLAYLISTS), ""),
new ListLinkHandler(urlAlbums, urlAlbums, id,
return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl)); List.of(ChannelTabs.ALBUMS), ""));
} catch (final Exception e) {
throw new ExtractionException("Could not get next page", e);
}
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector,
page.getUrl());
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
} }
} }

View File

@ -0,0 +1,76 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudChannelTabExtractor extends ChannelTabExtractor {
private static final String USERS_ENDPOINT = SOUNDCLOUD_API_V2_URL + "users/";
private final String userId;
public SoundcloudChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
userId = getLinkHandler().getId();
}
@Nonnull
private String getEndpoint() throws ParsingException {
switch (getName()) {
case ChannelTabs.TRACKS:
return "/tracks";
case ChannelTabs.PLAYLISTS:
return "/playlists_without_albums";
case ChannelTabs.ALBUMS:
return "/albums";
}
throw new ParsingException("Unsupported tab: " + getName());
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) {
}
@Nonnull
@Override
public String getId() {
return userId;
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return getPage(new Page(USERS_ENDPOINT + userId + getEndpoint() + "?client_id="
+ SoundcloudParsingHelper.clientId() + "&limit=20" + "&linked_partitioning=1"));
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getInfoItemsFromApi(
collector, page.getUrl());
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
}
}

View File

@ -23,7 +23,7 @@ public final class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFa
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
Utils.checkUrl(URL_PATTERN, url); Utils.checkUrl(URL_PATTERN, url);
try { try {
@ -36,7 +36,8 @@ public final class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFa
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) throws ParsingException { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try { try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer( return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
"https://api.soundcloud.com/users/" + id); "https://api.soundcloud.com/users/" + id);

View File

@ -0,0 +1,61 @@
package org.schabi.newpipe.extractor.services.soundcloud.linkHandler;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.UnsupportedTabException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import javax.annotation.Nonnull;
import java.util.List;
public final class SoundcloudChannelTabLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudChannelTabLinkHandlerFactory INSTANCE
= new SoundcloudChannelTabLinkHandlerFactory();
private SoundcloudChannelTabLinkHandlerFactory() {
}
public static SoundcloudChannelTabLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Nonnull
public static String getUrlSuffix(final String tab) throws UnsupportedOperationException {
switch (tab) {
case ChannelTabs.TRACKS:
return "/tracks";
case ChannelTabs.PLAYLISTS:
return "/sets";
case ChannelTabs.ALBUMS:
return "/albums";
}
throw new UnsupportedTabException(tab);
}
@Override
public String getId(final String url) throws ParsingException {
return SoundcloudChannelLinkHandlerFactory.getInstance().getId(url);
}
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
return SoundcloudChannelLinkHandlerFactory.getInstance().getUrl(id)
+ getUrlSuffix(contentFilter.get(0));
}
@Override
public boolean onAcceptUrl(final String url) throws ParsingException {
return SoundcloudChannelLinkHandlerFactory.getInstance().onAcceptUrl(url);
}
@Override
public String[] getAvailableContentFilter() {
return new String[] {
ChannelTabs.TRACKS,
ChannelTabs.PLAYLISTS,
ChannelTabs.ALBUMS,
};
}
}

View File

@ -1,18 +1,30 @@
package org.schabi.newpipe.extractor.services.soundcloud.linkHandler; package org.schabi.newpipe.extractor.services.soundcloud.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import java.util.List; import java.util.List;
public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory { public final class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudChartsLinkHandlerFactory INSTANCE =
new SoundcloudChartsLinkHandlerFactory();
private static final String TOP_URL_PATTERN = private static final String TOP_URL_PATTERN =
"^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$"; "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
private static final String URL_PATTERN = private static final String URL_PATTERN =
"^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$"; "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
private SoundcloudChartsLinkHandlerFactory() {
}
public static SoundcloudChartsLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String getId(final String url) { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) { if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
return "Top 50"; return "Top 50";
} else { } else {
@ -23,7 +35,8 @@ public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory {
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
if (id.equals("Top 50")) { if (id.equals("Top 50")) {
return "https://soundcloud.com/charts/top"; return "https://soundcloud.com/charts/top";
} else { } else {

View File

@ -24,7 +24,8 @@ public final class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerF
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) throws ParsingException { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try { try {
return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id=" return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id="
+ clientId() + "&threaded=0" + "&filter_replies=1"; + clientId() + "&threaded=0" + "&filter_replies=1";
@ -37,7 +38,7 @@ public final class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerF
} }
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
// Delegation to avoid duplicate code, as we need the same id // Delegation to avoid duplicate code, as we need the same id
return SoundcloudStreamLinkHandlerFactory.getInstance().getId(url); return SoundcloudStreamLinkHandlerFactory.getInstance().getId(url);
} }

View File

@ -22,7 +22,7 @@ public final class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerF
} }
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
Utils.checkUrl(URL_PATTERN, url); Utils.checkUrl(URL_PATTERN, url);
try { try {
@ -37,7 +37,7 @@ public final class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerF
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) final String sortFilter)
throws ParsingException { throws ParsingException, UnsupportedOperationException {
try { try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer( return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
"https://api.soundcloud.com/playlists/" + id); "https://api.soundcloud.com/playlists/" + id);

View File

@ -13,7 +13,10 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.List; import java.util.List;
public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public final class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
private static final SoundcloudSearchQueryHandlerFactory INSTANCE =
new SoundcloudSearchQueryHandlerFactory();
public static final String TRACKS = "tracks"; public static final String TRACKS = "tracks";
public static final String USERS = "users"; public static final String USERS = "users";
@ -22,11 +25,18 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
public static final int ITEMS_PER_PAGE = 10; public static final int ITEMS_PER_PAGE = 10;
private SoundcloudSearchQueryHandlerFactory() {
}
public static SoundcloudSearchQueryHandlerFactory getInstance() {
return INSTANCE;
}
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) final String sortFilter)
throws ParsingException { throws ParsingException, UnsupportedOperationException {
try { try {
String url = SOUNDCLOUD_API_V2_URL + "search"; String url = SOUNDCLOUD_API_V2_URL + "search";

View File

@ -21,7 +21,7 @@ public final class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory
} }
@Override @Override
public String getUrl(final String id) throws ParsingException { public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
try { try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer( return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
"https://api.soundcloud.com/tracks/" + id); "https://api.soundcloud.com/tracks/" + id);
@ -31,7 +31,7 @@ public final class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory
} }
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
if (Parser.isMatch(API_URL_PATTERN, url)) { if (Parser.isMatch(API_URL_PATTERN, url)) {
return Parser.matchGroup1(API_URL_PATTERN, url); return Parser.matchGroup1(API_URL_PATTERN, url);
} }

View File

@ -0,0 +1,271 @@
package org.schabi.newpipe.extractor.services.youtube;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* Shared functions for extracting YouTube channel pages and tabs.
*/
public final class YoutubeChannelHelper {
private YoutubeChannelHelper() {
}
/**
* Take a YouTube channel ID or URL path, resolve it if necessary and return a channel ID.
*
* @param idOrPath a YouTube channel ID or URL path
* @return a YouTube channel ID
* @throws IOException if a channel resolve request failed
* @throws ExtractionException if a channel resolve request response could not be parsed or is
* invalid
*/
@Nonnull
public static String resolveChannelId(@Nonnull final String idOrPath)
throws ExtractionException, IOException {
final String[] channelId = idOrPath.split("/");
if (channelId[0].startsWith("UC")) {
return channelId[0];
}
// If the URL is not a /channel URL, we need to use the navigation/resolve_url endpoint of
// the InnerTube API to get the channel id.
// Otherwise, we couldn't get information about the channel associated with this URL, if
// there is one.
if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(
prepareDesktopJsonBuilder(Localization.DEFAULT, ContentCountry.DEFAULT)
.value("url", "https://www.youtube.com/" + idOrPath)
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject jsonResponse = getJsonPostResponse(
"navigation/resolve_url", body, Localization.DEFAULT);
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
final String browseId = browseEndpoint.getString("browseId", "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
return browseId;
}
}
return channelId[1];
}
/**
* Response data object for {@link #getChannelResponse(String, String, Localization,
* ContentCountry)}, after any redirection in the allowed redirects count ({@code 3}).
*/
public static final class ChannelResponseData {
/**
* The channel response as a JSON object, after all redirects.
*/
@Nonnull
public final JsonObject jsonResponse;
/**
* The channel ID after all redirects.
*/
@Nonnull
public final String channelId;
private ChannelResponseData(@Nonnull final JsonObject jsonResponse,
@Nonnull final String channelId) {
this.jsonResponse = jsonResponse;
this.channelId = channelId;
}
}
/**
* Fetch a YouTube channel tab response, using the given channel ID and tab parameters.
*
* <p>
* Redirections to other channels such as are supported to up to 3 redirects, which could
* happen for instance for localized channels or auto-generated ones such as the {@code Movies
* and Shows} (channel IDs {@code UCuJcl0Ju-gPDoksRjK1ya-w}, {@code UChBfWrfBXL9wS6tQtgjt_OQ}
* and {@code UCok7UTQQEP1Rsctxiv3gwSQ} of this channel redirect to the
* {@code UClgRkhTL3_hImCAmdLfDE4g} one).
* </p>
*
* @param channelId a valid YouTube channel ID
* @param parameters the parameters to specify the YouTube channel tab; if invalid ones are
* specified, YouTube should return the {@code Home} tab
* @param localization the {@link Localization} to use
* @param country the {@link ContentCountry} to use
* @return a {@link ChannelResponseData channel response data}
* @throws IOException if a channel request failed
* @throws ExtractionException if a channel request response could not be parsed or is invalid
*/
@Nonnull
public static ChannelResponseData getChannelResponse(@Nonnull final String channelId,
@Nonnull final String parameters,
@Nonnull final Localization localization,
@Nonnull final ContentCountry country)
throws ExtractionException, IOException {
String id = channelId;
JsonObject ajaxJson = null;
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
localization, country)
.value("browseId", id)
.value("params", parameters)
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject jsonResponse = getJsonPostResponse(
"browse", body, localization);
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("navigateAction")
.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final String browseId = endpoint.getObject("browseEndpoint")
.getString("browseId", "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
level++;
} else {
ajaxJson = jsonResponse;
break;
}
}
if (ajaxJson == null) {
throw new ExtractionException("Got no channel response");
}
defaultAlertsCheck(ajaxJson);
return new ChannelResponseData(ajaxJson, id);
}
/**
* Assert that a channel JSON response does not contain an {@code error} JSON object.
*
* @param jsonResponse a channel JSON response
* @throws ContentNotAvailableException if the channel was not found
*/
private static void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse)
throws ContentNotAvailableException {
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
}
/**
* A channel header response.
*
* <p>
* This class allows the distinction between a classic header and a carousel one, used for
* auto-generated ones like the gaming or music topic channels and for big events such as the
* Coachella music festival, which have a different data structure and do not return the same
* properties.
* </p>
*/
public static final class ChannelHeader {
/**
* The channel header JSON response.
*/
@Nonnull
public final JsonObject json;
/**
* Whether the header is a {@code carouselHeaderRenderer}.
*
* <p>
* See the class documentation for more details.
* </p>
*/
public final boolean isCarouselHeader;
private ChannelHeader(@Nonnull final JsonObject json, final boolean isCarouselHeader) {
this.json = json;
this.isCarouselHeader = isCarouselHeader;
}
}
/**
* Get a channel header as an {@link Optional} it if exists.
*
* @param channelResponse a full channel JSON response
* @return an {@link Optional} containing a {@link ChannelHeader} or an empty {@link Optional}
* if no supported header has been found
*/
@Nonnull
public static Optional<ChannelHeader> getChannelHeader(
@Nonnull final JsonObject channelResponse) {
final JsonObject header = channelResponse.getObject("header");
if (header.has("c4TabbedHeaderRenderer")) {
return Optional.of(header.getObject("c4TabbedHeaderRenderer"))
.map(json -> new ChannelHeader(json, false));
} else if (header.has("carouselHeaderRenderer")) {
return header.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(item -> item.has("topicChannelDetailsRenderer"))
.findFirst()
.map(item -> item.getObject("topicChannelDetailsRenderer"))
.map(json -> new ChannelHeader(json, true));
} else {
return Optional.empty();
}
}
}

View File

@ -1230,8 +1230,17 @@ public final class YoutubeParsingHelper {
@Nonnull final Localization localization, @Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry) @Nonnull final ContentCountry contentCountry)
throws IOException, ExtractionException { throws IOException, ExtractionException {
return prepareDesktopJsonBuilder(localization, contentCountry, null);
}
@Nonnull
public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nullable final String visitorData)
throws IOException, ExtractionException {
// @formatter:off // @formatter:off
return JsonObject.builder() final JsonBuilder<JsonObject> builder = JsonObject.builder()
.object("context") .object("context")
.object("client") .object("client")
.value("hl", localization.getLocalizationCode()) .value("hl", localization.getLocalizationCode())
@ -1239,8 +1248,13 @@ public final class YoutubeParsingHelper {
.value("clientName", "WEB") .value("clientName", "WEB")
.value("clientVersion", getClientVersion()) .value("clientVersion", getClientVersion())
.value("originalUrl", "https://www.youtube.com") .value("originalUrl", "https://www.youtube.com")
.value("platform", "DESKTOP") .value("platform", "DESKTOP");
.end()
if (visitorData != null) {
builder.value("visitorData", visitorData);
}
return builder.end()
.object("request") .object("request")
.array("internalExperimentFlags") .array("internalExperimentFlags")
.end() .end()

View File

@ -8,6 +8,7 @@ import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.feed.FeedExtractor;
@ -16,6 +17,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.ContentCountry;
@ -23,6 +25,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
@ -34,6 +37,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscript
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
@ -88,6 +92,11 @@ public class YoutubeService extends StreamingService {
return YoutubeChannelLinkHandlerFactory.getInstance(); return YoutubeChannelLinkHandlerFactory.getInstance();
} }
@Override
public ListLinkHandlerFactory getChannelTabLHFactory() {
return YoutubeChannelTabLinkHandlerFactory.getInstance();
}
@Override @Override
public ListLinkHandlerFactory getPlaylistLHFactory() { public ListLinkHandlerFactory getPlaylistLHFactory() {
return YoutubePlaylistLinkHandlerFactory.getInstance(); return YoutubePlaylistLinkHandlerFactory.getInstance();
@ -108,6 +117,15 @@ public class YoutubeService extends StreamingService {
return new YoutubeChannelExtractor(this, linkHandler); return new YoutubeChannelExtractor(this, linkHandler);
} }
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
} else {
return new YoutubeChannelTabExtractor(this, linkHandler);
}
}
@Override @Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) { if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
@ -136,16 +154,17 @@ public class YoutubeService extends StreamingService {
@Override @Override
public KioskList getKioskList() throws ExtractionException { public KioskList getKioskList() throws ExtractionException {
final KioskList list = new KioskList(this); final KioskList list = new KioskList(this);
final ListLinkHandlerFactory h = YoutubeTrendingLinkHandlerFactory.getInstance();
// add kiosks here e.g.: // add kiosks here e.g.:
try { try {
list.addKioskEntry( list.addKioskEntry(
(streamingService, url, id) -> new YoutubeTrendingExtractor( (streamingService, url, id) -> new YoutubeTrendingExtractor(
YoutubeService.this, YoutubeService.this,
new YoutubeTrendingLinkHandlerFactory().fromUrl(url), h.fromUrl(url),
id id
), ),
new YoutubeTrendingLinkHandlerFactory(), h,
YoutubeTrendingExtractor.KIOSK_ID YoutubeTrendingExtractor.KIOSK_ID
); );
list.setDefaultKiosk(YoutubeTrendingExtractor.KIOSK_ID); list.setDefaultKiosk(YoutubeTrendingExtractor.KIOSK_ID);

View File

@ -1,39 +1,35 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor.VideosTabExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
@ -59,22 +55,24 @@ import javax.annotation.Nullable;
*/ */
public class YoutubeChannelExtractor extends ChannelExtractor { public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject initialData;
private Optional<JsonObject> channelHeader; private JsonObject jsonResponse;
private boolean isCarouselHeader = false; @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private JsonObject videoTab; private Optional<YoutubeChannelHelper.ChannelHeader> channelHeader;
private String channelId;
/** /**
* Some channels have response redirects and the only way to reliably get the id is by saving it * If a channel is age-restricted, its pages are only accessible to logged-in and
* age-verified users, we get an {@code channelAgeGateRenderer} in this case, containing only
* the following metadata: channel name and channel avatar.
*
* <p> * <p>
* "Movies & Shows": * This restriction doesn't seem to apply to all countries.
* <pre> * </p>
* UCuJcl0Ju-gPDoksRjK1ya-w
* UChBfWrfBXL9wS6tQtgjt_OQ UClgRkhTL3_hImCAmdLfDE4g
* UCok7UTQQEP1Rsctxiv3gwSQ
* </pre>
*/ */
private String redirectedChannelId; @Nullable
private JsonObject channelAgeGateRenderer;
public YoutubeChannelExtractor(final StreamingService service, public YoutubeChannelExtractor(final StreamingService service,
final ListLinkHandler linkHandler) { final ListLinkHandler linkHandler) {
@ -85,132 +83,42 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public void onFetchPage(@Nonnull final Downloader downloader) public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException { throws IOException, ExtractionException {
final String channelPath = super.getId(); final String channelPath = super.getId();
final String[] channelId = channelPath.split("/"); final String id = resolveChannelId(channelPath);
String id = ""; // Fetch Videos tab
// If the url is an URL which is not a /channel URL, we need to use the final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(id,
// navigation/resolve_url endpoint of the InnerTube API to get the channel id. Otherwise, "EgZ2aWRlb3PyBgQKAjoA", getExtractorLocalization(), getExtractorContentCountry());
// we couldn't get information about the channel associated with this URL, if there is one.
if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("url", "https://www.youtube.com/" + channelPath)
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url", jsonResponse = data.jsonResponse;
body, getExtractorLocalization()); channelId = data.channelId;
channelAgeGateRenderer = getChannelAgeGateRenderer();
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
final String browseId = browseEndpoint.getString("browseId", "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
redirectedChannelId = browseId;
}
} else {
id = channelId[1];
}
JsonObject ajaxJson = null;
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("browseId", id)
.value("params", "EgZ2aWRlb3M%3D") // Equal to videos
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject jsonResponse = getJsonPostResponse("browse", body,
getExtractorLocalization());
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("navigateAction")
.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId",
"");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
redirectedChannelId = browseId;
level++;
} else {
ajaxJson = jsonResponse;
break;
}
}
if (ajaxJson == null) {
throw new ExtractionException("Could not fetch initial JSON data");
}
initialData = ajaxJson;
YoutubeParsingHelper.defaultAlertsCheck(initialData);
} }
private void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse) @Nullable
throws ContentNotAvailableException { private JsonObject getChannelAgeGateRenderer() {
if (!isNullOrEmpty(jsonResponse.getObject("error"))) { return jsonResponse.getObject("contents")
final JsonObject errorJsonObject = jsonResponse.getObject("error"); .getObject("twoColumnBrowseResultsRenderer")
final int errorCode = errorJsonObject.getInt("code"); .getArray("tabs")
if (errorCode == 404) { .stream()
throw new ContentNotAvailableException("This channel doesn't exist."); .filter(JsonObject.class::isInstance)
} else { .map(JsonObject.class::cast)
throw new ContentNotAvailableException("Got error:\"" .flatMap(tab -> tab.getObject("tabRenderer")
+ errorJsonObject.getString("status") + "\": " .getObject("content")
+ errorJsonObject.getString("message")); .getObject("sectionListRenderer")
}
}
}
@Nonnull
private Optional<JsonObject> getChannelHeader() {
if (channelHeader == null) {
final JsonObject h = initialData.getObject("header");
if (h.has("c4TabbedHeaderRenderer")) {
channelHeader = Optional.of(h.getObject("c4TabbedHeaderRenderer"));
} else if (h.has("carouselHeaderRenderer")) {
isCarouselHeader = true;
channelHeader = h.getObject("carouselHeaderRenderer")
.getArray("contents") .getArray("contents")
.stream() .stream()
.filter(JsonObject.class::isInstance) .filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast) .map(JsonObject.class::cast))
.filter(itm -> itm.has("topicChannelDetailsRenderer")) .filter(content -> content.has("channelAgeGateRenderer"))
.findFirst() .map(content -> content.getObject("channelAgeGateRenderer"))
.map(itm -> itm.getObject("topicChannelDetailsRenderer")); .findFirst()
} else { .orElse(null);
channelHeader = Optional.empty(); }
}
@Nonnull
private Optional<YoutubeChannelHelper.ChannelHeader> getChannelHeader() {
//noinspection OptionalAssignedToNull
if (channelHeader == null) {
channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
} }
return channelHeader; return channelHeader;
} }
@ -229,57 +137,70 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getId() throws ParsingException { public String getId() throws ParsingException {
return getChannelHeader() return getChannelHeader()
.flatMap(header -> Optional.ofNullable(header.getString("channelId")).or( .flatMap(header -> Optional.ofNullable(header.json.getString("channelId"))
() -> Optional.ofNullable(header.getObject("navigationEndpoint") .or(() -> Optional.ofNullable(header.json.getObject("navigationEndpoint")
.getObject("browseEndpoint") .getObject("browseEndpoint")
.getString("browseId")) .getString("browseId"))
)) ))
.or(() -> Optional.ofNullable(redirectedChannelId)) .or(() -> Optional.ofNullable(channelId))
.orElseThrow(() -> new ParsingException("Could not get channel id")); .orElseThrow(() -> new ParsingException("Could not get channel ID"));
} }
@Nonnull @Nonnull
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
final String mdName = initialData.getObject("metadata") if (channelAgeGateRenderer != null) {
.getObject("channelMetadataRenderer") return channelAgeGateRenderer.getString("channelTitle");
.getString("title");
if (!isNullOrEmpty(mdName)) {
return mdName;
} }
final Optional<JsonObject> header = getChannelHeader(); final String metadataRendererTitle = jsonResponse.getObject("metadata")
if (header.isPresent()) { .getObject("channelMetadataRenderer")
final Object title = header.get().get("title"); .getString("title");
if (!isNullOrEmpty(metadataRendererTitle)) {
return metadataRendererTitle;
}
return getChannelHeader().flatMap(header -> {
final Object title = header.json.get("title");
if (title instanceof String) { if (title instanceof String) {
return (String) title; return Optional.of((String) title);
} else if (title instanceof JsonObject) { } else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title); final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) { if (!isNullOrEmpty(headerName)) {
return headerName; return Optional.of(headerName);
} }
} }
} return Optional.empty();
}).orElseThrow(() -> new ParsingException("Could not get channel name"));
throw new ParsingException("Could not get channel name");
} }
@Override @Override
public String getAvatarUrl() throws ParsingException { public String getAvatarUrl() throws ParsingException {
return getChannelHeader().flatMap(header -> Optional.ofNullable( final JsonObject avatarJsonObjectContainer;
header.getObject("avatar").getArray("thumbnails") if (channelAgeGateRenderer != null) {
.getObject(0).getString("url") avatarJsonObjectContainer = channelAgeGateRenderer;
)) } else {
.map(YoutubeParsingHelper::fixThumbnailUrl) avatarJsonObjectContainer = getChannelHeader().map(header -> header.json)
.orElseThrow(() -> new ParsingException("Could not get avatar")); .orElseThrow(() -> new ParsingException("Could not get avatar URL"));
}
return YoutubeParsingHelper.fixThumbnailUrl(avatarJsonObjectContainer.getObject("avatar")
.getArray("thumbnails")
.getObject(0)
.getString("url"));
} }
@Override @Override
public String getBannerUrl() throws ParsingException { public String getBannerUrl() throws ParsingException {
if (channelAgeGateRenderer != null) {
return "";
}
return getChannelHeader().flatMap(header -> Optional.ofNullable( return getChannelHeader().flatMap(header -> Optional.ofNullable(
header.getObject("banner").getArray("thumbnails") header.json.getObject("banner")
.getObject(0).getString("url") .getArray("thumbnails")
)) .getObject(0)
.getString("url")))
.filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner")) .filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
.map(YoutubeParsingHelper::fixThumbnailUrl) .map(YoutubeParsingHelper::fixThumbnailUrl)
// Channels may not have a banner, so no exception should be thrown if no banner is // Channels may not have a banner, so no exception should be thrown if no banner is
@ -290,6 +211,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getFeedUrl() throws ParsingException { public String getFeedUrl() throws ParsingException {
// RSS feeds are accessible for age-restricted channels, no need to check whether a channel
// has a channelAgeGateRenderer
try { try {
return YoutubeParsingHelper.getFeedUrlFrom(getId()); return YoutubeParsingHelper.getFeedUrlFrom(getId());
} catch (final Exception e) { } catch (final Exception e) {
@ -299,14 +222,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public long getSubscriberCount() throws ParsingException { public long getSubscriberCount() throws ParsingException {
final Optional<JsonObject> header = getChannelHeader(); if (channelAgeGateRenderer != null) {
if (header.isPresent()) { return UNKNOWN_SUBSCRIBER_COUNT;
}
final Optional<YoutubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
if (headerOpt.isPresent()) {
final JsonObject header = headerOpt.get().json;
JsonObject textObject = null; JsonObject textObject = null;
if (header.get().has("subscriberCountText")) { if (header.has("subscriberCountText")) {
textObject = header.get().getObject("subscriberCountText"); textObject = header.getObject("subscriberCountText");
} else if (header.get().has("subtitle")) { } else if (header.has("subtitle")) {
textObject = header.get().getObject("subtitle"); textObject = header.getObject("subtitle");
} }
if (textObject != null) { if (textObject != null) {
@ -317,13 +245,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} }
} }
} }
return UNKNOWN_SUBSCRIBER_COUNT; return UNKNOWN_SUBSCRIBER_COUNT;
} }
@Override @Override
public String getDescription() throws ParsingException { public String getDescription() throws ParsingException {
if (channelAgeGateRenderer != null) {
return null;
}
try { try {
return initialData.getObject("metadata").getObject("channelMetadataRenderer") return jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("description"); .getString("description");
} catch (final Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get channel description", e); throw new ParsingException("Could not get channel description", e);
@ -347,190 +281,139 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public boolean isVerified() throws ParsingException { public boolean isVerified() throws ParsingException {
// The CarouselHeaderRenderer does not contain any verification badges. if (channelAgeGateRenderer != null) {
// Since it is only shown on YT-internal channels or on channels of large organizations return false;
// broadcasting live events, we can assume the channel to be verified.
if (isCarouselHeader) {
return true;
} }
return getChannelHeader() final Optional<YoutubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
.map(header -> header.getArray("badges")) if (headerOpt.isPresent()) {
.map(YoutubeParsingHelper::isVerified) final YoutubeChannelHelper.ChannelHeader header = headerOpt.get();
.orElse(false);
// The CarouselHeaderRenderer does not contain any verification badges.
// Since it is only shown on YT-internal channels or on channels of large organizations
// broadcasting live events, we can assume the channel to be verified.
if (header.isCarouselHeader) {
return true;
}
return YoutubeParsingHelper.isVerified(header.json.getArray("badges"));
}
return false;
} }
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException { public List<ListLinkHandler> getTabs() throws ParsingException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); if (channelAgeGateRenderer == null) {
return getTabsForNonAgeRestrictedChannels();
Page nextPage = null;
if (getVideoTab() != null) {
final JsonObject tabContent = getVideoTab().getObject("content");
JsonArray items = tabContent
.getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("items");
if (items.isEmpty()) {
items = tabContent.getObject("richGridRenderer").getArray("contents");
}
final List<String> channelIds = new ArrayList<>();
channelIds.add(getName());
channelIds.add(getUrl());
final JsonObject continuation = collectStreamsFrom(collector, items, channelIds);
nextPage = getNextPageFrom(continuation, channelIds);
} }
return new InfoItemsPage<>(collector, nextPage); return getTabsForAgeRestrictedChannels();
} }
@Override @Nonnull
public InfoItemsPage<StreamInfoItem> getPage(final Page page) private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws ParsingException {
throws IOException, ExtractionException { final JsonArray responseTabs = jsonResponse.getObject("contents")
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final List<String> channelIds = page.getIds();
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
getExtractorLocalization());
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction");
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation
.getArray("continuationItems"), channelIds);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
}
@Nullable
private Page getNextPageFrom(final JsonObject continuations,
final List<String> channelIds)
throws IOException, ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("continuation", continuation)
.done())
.getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
}
/**
* Collect streams from an array of items
*
* @param collector the collector where videos will be committed
* @param videos the array to get videos from
* @param channelIds the ids of the channel, which are its name and its URL
* @return the continuation object
*/
private JsonObject collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
@Nonnull final JsonArray videos,
@Nonnull final List<String> channelIds) {
collector.reset();
final String uploaderName = channelIds.get(0);
final String uploaderUrl = channelIds.get(1);
final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonObject continuation = null;
for (final Object object : videos) {
final JsonObject video = (JsonObject) object;
if (video.has("gridVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
video.getObject("gridVideoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
});
} else if (video.has("richItemRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
video.getObject("richItemRenderer")
.getObject("content").getObject("videoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
});
} else if (video.has("continuationItemRenderer")) {
continuation = video.getObject("continuationItemRenderer");
}
}
return continuation;
}
@Nullable
private JsonObject getVideoTab() throws ParsingException {
if (videoTab != null) {
return videoTab;
}
final JsonArray tabs = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer") .getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs"); .getArray("tabs");
final JsonObject foundVideoTab = tabs.stream() final List<ListLinkHandler> tabs = new ArrayList<>();
.filter(Objects::nonNull) final Consumer<String> addNonVideosTab = tabName -> {
try {
tabs.add(YoutubeChannelTabLinkHandlerFactory.getInstance().fromQuery(
channelId, List.of(tabName), ""));
} catch (final ParsingException ignored) {
// Do not add the tab if we couldn't create the LinkHandler
}
};
final String name = getName();
final String url = getUrl();
final String id = getId();
responseTabs.stream()
.filter(JsonObject.class::isInstance) .filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast) .map(JsonObject.class::cast)
.filter(tab -> tab.has("tabRenderer") .filter(tab -> tab.has("tabRenderer"))
&& tab.getObject("tabRenderer")
.getString("title", "")
.equals("Videos"))
.findFirst()
.map(tab -> tab.getObject("tabRenderer")) .map(tab -> tab.getObject("tabRenderer"))
.orElseThrow( .forEach(tabRenderer -> {
() -> new ContentNotSupportedException("This channel has no Videos tab")); final String tabUrl = tabRenderer.getObject("endpoint")
.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("url");
if (tabUrl != null) {
final String[] urlParts = tabUrl.split("/");
if (urlParts.length == 0) {
return;
}
final String messageRendererText = getTextFromObject( final String urlSuffix = urlParts[urlParts.length - 1];
foundVideoTab.getObject("content")
.getObject("sectionListRenderer") switch (urlSuffix) {
.getArray("contents") case "videos":
.getObject(0) // Since the Videos tab has already its contents fetched, make
.getObject("itemSectionRenderer") // sure it is in the first position
.getArray("contents") // YoutubeChannelTabExtractor still supports fetching this tab
.getObject(0) tabs.add(0, new ReadyChannelTabListLinkHandler(
.getObject("messageRenderer") tabUrl,
.getObject("text")); channelId,
if (messageRendererText != null ChannelTabs.VIDEOS,
&& messageRendererText.equals("This channel has no videos.")) { (service, linkHandler) -> new VideosTabExtractor(
return null; service, linkHandler, tabRenderer, name, id, url)));
break;
case "shorts":
addNonVideosTab.accept(ChannelTabs.SHORTS);
break;
case "streams":
addNonVideosTab.accept(ChannelTabs.LIVESTREAMS);
break;
case "playlists":
addNonVideosTab.accept(ChannelTabs.PLAYLISTS);
break;
case "channels":
addNonVideosTab.accept(ChannelTabs.CHANNELS);
break;
}
}
});
return Collections.unmodifiableList(tabs);
}
@Nonnull
private List<ListLinkHandler> getTabsForAgeRestrictedChannels() throws ParsingException {
// As we don't have access to the channel tabs list, consider that the channel has videos,
// shorts and livestreams, the data only accessible without login on YouTube's desktop
// client using uploads system playlists
// The playlists channel tab is still available on YouTube Music, but this is not
// implemented in the extractor
final List<ListLinkHandler> tabs = new ArrayList<>();
final String channelUrl = getUrl();
final Consumer<String> addTab = tabName ->
tabs.add(new ReadyChannelTabListLinkHandler(channelUrl + "/" + tabName,
channelId, tabName, YoutubeChannelTabPlaylistExtractor::new));
addTab.accept(ChannelTabs.VIDEOS);
addTab.accept(ChannelTabs.SHORTS);
addTab.accept(ChannelTabs.LIVESTREAMS);
return Collections.unmodifiableList(tabs);
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
if (channelAgeGateRenderer != null) {
return List.of();
} }
videoTab = foundVideoTab; return jsonResponse.getObject("microformat")
return foundVideoTab; .getObject("microformatDataRenderer")
.getArray("tags")
.stream()
.filter(String.class::isInstance)
.map(String.class::cast)
.collect(Collectors.toUnmodifiableList());
} }
} }

View File

@ -0,0 +1,486 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
import java.util.function.Consumer;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* A {@link ChannelTabExtractor} implementation for the YouTube service.
*
* <p>
* It currently supports {@code Videos}, {@code Shorts}, {@code Live}, {@code Playlists} and
* {@code Channels} tabs.
* </p>
*/
public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
/**
* Whether the visitor data extracted from the initial channel response is required to be used
* for continuations.
*
* <p>
* A valid {@code visitorData} is required to get continuations of shorts in channels.
* </p>
*
* <p>
* It should be not used when it is not needed, in order to reduce YouTube's tracking.
* </p>
*/
private final boolean useVisitorData;
private JsonObject jsonResponse;
private String channelId;
@Nullable
private String visitorData;
public YoutubeChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
useVisitorData = getName().equals(ChannelTabs.SHORTS);
}
@Nonnull
private String getChannelTabsParameters() throws ParsingException {
final String name = getName();
switch (name) {
case ChannelTabs.VIDEOS:
return "EgZ2aWRlb3PyBgQKAjoA";
case ChannelTabs.SHORTS:
return "EgZzaG9ydHPyBgUKA5oBAA%3D%3D";
case ChannelTabs.LIVESTREAMS:
return "EgdzdHJlYW1z8gYECgJ6AA%3D%3D";
case ChannelTabs.PLAYLISTS:
return "EglwbGF5bGlzdHPyBgQKAkIA";
case ChannelTabs.CHANNELS:
return "EghjaGFubmVsc_IGBAoCUgA%3D";
}
throw new ParsingException("Unsupported channel tab: " + name);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
channelId = resolveChannelId(super.getId());
final String params = getChannelTabsParameters();
final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(channelId,
params, getExtractorLocalization(), getExtractorContentCountry());
jsonResponse = data.jsonResponse;
channelId = data.channelId;
if (useVisitorData) {
visitorData = jsonResponse.getObject("responseContext").getString("visitorData");
}
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
try {
return YoutubeChannelTabLinkHandlerFactory.getInstance()
.getUrl("channel/" + getId(), List.of(getName()), "");
} catch (final ParsingException e) {
return super.getUrl();
}
}
@Nonnull
@Override
public String getId() throws ParsingException {
final String id = jsonResponse.getObject("header")
.getObject("c4TabbedHeaderRenderer")
.getString("channelId", "");
if (!id.isEmpty()) {
return id;
}
final Optional<String> carouselHeaderId = jsonResponse.getObject("header")
.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(item -> item.has("topicChannelDetailsRenderer"))
.findFirst()
.flatMap(item ->
Optional.ofNullable(item.getObject("topicChannelDetailsRenderer")
.getObject("navigationEndpoint")
.getObject("browseEndpoint")
.getString("browseId")));
if (carouselHeaderId.isPresent()) {
return carouselHeaderId.get();
}
if (!isNullOrEmpty(channelId)) {
return channelId;
} else {
throw new ParsingException("Could not get channel ID");
}
}
protected String getChannelName() {
final String metadataName = jsonResponse.getObject("metadata")
.getObject("channelMetadataRenderer")
.getString("title");
if (!isNullOrEmpty(metadataName)) {
return metadataName;
}
return YoutubeChannelHelper.getChannelHeader(jsonResponse)
.map(header -> {
final Object title = header.json.get("title");
if (title instanceof String) {
return (String) title;
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
return headerName;
}
}
return "";
})
.orElse("");
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
JsonArray items = new JsonArray();
final Optional<JsonObject> tab = getTabData();
if (tab.isPresent()) {
final JsonObject tabContent = tab.get().getObject("content");
items = tabContent.getObject("sectionListRenderer")
.getArray("contents")
.getObject(0)
.getObject("itemSectionRenderer")
.getArray("contents")
.getObject(0)
.getObject("gridRenderer")
.getArray("items");
if (items.isEmpty()) {
items = tabContent.getObject("richGridRenderer")
.getArray("contents");
if (items.isEmpty()) {
items = tabContent.getObject("sectionListRenderer")
.getArray("contents");
}
}
}
// If a channel tab is fetched, the next page requires channel ID and name, as channel
// streams don't have their channel specified.
// We also need to set the visitor data here when it should be enabled, as it is required
// to get continuations on some channel tabs, and we need a way to pass it between pages
final List<String> channelIds = useVisitorData && !isNullOrEmpty(visitorData)
? List.of(getChannelName(), getUrl(), visitorData)
: List.of(getChannelName(), getUrl());
final JsonObject continuation = collectItemsFrom(collector, items, channelIds)
.orElse(null);
final Page nextPage = getNextPageFrom(continuation, channelIds);
return new InfoItemsPage<>(collector, nextPage);
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final List<String> channelIds = page.getIds();
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
getExtractorLocalization());
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(jsonObject -> jsonObject.has("appendContinuationItemsAction"))
.map(jsonObject -> jsonObject.getObject("appendContinuationItemsAction"))
.findFirst()
.orElse(new JsonObject());
final JsonObject continuation = collectItemsFrom(collector,
sectionListContinuation.getArray("continuationItems"), channelIds)
.orElse(null);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
}
Optional<JsonObject> getTabData() {
final String urlSuffix = YoutubeChannelTabLinkHandlerFactory.getUrlSuffix(getName());
return jsonResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(tab -> tab.has("tabRenderer"))
.map(tab -> tab.getObject("tabRenderer"))
.filter(tabRenderer -> tabRenderer.getObject("endpoint")
.getObject("commandMetadata").getObject("webCommandMetadata")
.getString("url", "").endsWith(urlSuffix))
.findFirst()
// Check if tab has no content
.filter(tabRenderer -> {
final JsonArray tabContents = tabRenderer.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents")
.getObject(0)
.getObject("itemSectionRenderer")
.getArray("contents");
return tabContents.size() != 1
|| !tabContents.getObject(0).has("messageRenderer");
});
}
private Optional<JsonObject> collectItemsFrom(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonArray items,
@Nonnull final List<String> channelIds) {
return items.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(item -> collectItem(collector, item, channelIds))
.reduce(Optional.empty(), (c1, c2) -> c1.or(() -> c2));
}
private Optional<JsonObject> collectItem(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonObject item,
@Nonnull final List<String> channelIds) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
if (item.has("richItemRenderer")) {
final JsonObject richItem = item.getObject("richItemRenderer")
.getObject("content");
if (richItem.has("videoRenderer")) {
getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept(
richItem.getObject("videoRenderer"));
} else if (richItem.has("reelItemRenderer")) {
getCommitReelItemConsumer(collector, timeAgoParser, channelIds).accept(
richItem.getObject("reelItemRenderer"));
} else if (richItem.has("playlistRenderer")) {
getCommitPlaylistConsumer(collector, channelIds).accept(
item.getObject("playlistRenderer"));
}
} else if (item.has("gridVideoRenderer")) {
getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept(
item.getObject("gridVideoRenderer"));
} else if (item.has("gridPlaylistRenderer")) {
getCommitPlaylistConsumer(collector, channelIds).accept(
item.getObject("gridPlaylistRenderer"));
} else if (item.has("gridChannelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(
item.getObject("gridChannelRenderer")));
} else if (item.has("shelfRenderer")) {
return collectItem(collector, item.getObject("shelfRenderer")
.getObject("content"), channelIds);
} else if (item.has("itemSectionRenderer")) {
return collectItemsFrom(collector, item.getObject("itemSectionRenderer")
.getArray("contents"), channelIds);
} else if (item.has("horizontalListRenderer")) {
return collectItemsFrom(collector, item.getObject("horizontalListRenderer")
.getArray("items"), channelIds);
} else if (item.has("expandedShelfContentsRenderer")) {
return collectItemsFrom(collector, item.getObject("expandedShelfContentsRenderer")
.getArray("items"), channelIds);
} else if (item.has("continuationItemRenderer")) {
return Optional.ofNullable(item.getObject("continuationItemRenderer"));
}
return Optional.empty();
}
@Nonnull
private Consumer<JsonObject> getCommitVideoConsumer(
@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final TimeAgoParser timeAgoParser,
@Nonnull final List<String> channelIds) {
return videoRenderer -> collector.commit(
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser) {
@Override
public String getUploaderName() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(0);
}
return super.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(1);
}
return super.getUploaderUrl();
}
});
}
@Nonnull
private Consumer<JsonObject> getCommitReelItemConsumer(
@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final TimeAgoParser timeAgoParser,
@Nonnull final List<String> channelIds) {
return reelItemRenderer -> collector.commit(
new YoutubeReelInfoItemExtractor(reelItemRenderer, timeAgoParser) {
@Override
public String getUploaderName() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(0);
}
return super.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(1);
}
return super.getUploaderUrl();
}
});
}
@Nonnull
private Consumer<JsonObject> getCommitPlaylistConsumer(
@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final List<String> channelIds) {
return playlistRenderer -> collector.commit(
new YoutubePlaylistInfoItemExtractor(playlistRenderer) {
@Override
public String getUploaderName() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(0);
}
return super.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
if (channelIds.size() >= 2) {
return channelIds.get(1);
}
return super.getUploaderUrl();
}
});
}
@Nullable
private Page getNextPageFrom(final JsonObject continuations,
final List<String> channelIds) throws IOException,
ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry(),
useVisitorData && channelIds.size() >= 3 ? channelIds.get(2) : null)
.value("continuation", continuation)
.done())
.getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
}
/**
* A {@link YoutubeChannelTabExtractor} for the {@code Videos} tab, if it has been already
* fetched.
*/
public static final class VideosTabExtractor extends YoutubeChannelTabExtractor {
private final JsonObject tabRenderer;
private final String channelName;
private final String channelId;
private final String channelUrl;
VideosTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final JsonObject tabRenderer,
final String channelName,
final String channelId,
final String channelUrl) {
super(service, linkHandler);
this.tabRenderer = tabRenderer;
this.channelName = channelName;
this.channelId = channelId;
this.channelUrl = channelUrl;
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) {
// Nothing to do, the initial data was already fetched and is stored in the link handler
}
@Nonnull
@Override
public String getId() throws ParsingException {
return channelId;
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
return channelUrl;
}
@Override
protected String getChannelName() {
return channelName;
}
@Override
Optional<JsonObject> getTabData() {
return Optional.of(tabRenderer);
}
}
}

View File

@ -0,0 +1,191 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* A {@link ChannelTabExtractor} for YouTube system playlists using a
* {@link YoutubePlaylistExtractor} instance.
*
* <p>
* It is currently used to bypass age-restrictions on channels marked as age-restricted by their
* owner(s).
* </p>
*/
public class YoutubeChannelTabPlaylistExtractor extends ChannelTabExtractor {
private final PlaylistExtractor playlistExtractorInstance;
private boolean playlistExisting;
/**
* Construct a {@link YoutubeChannelTabPlaylistExtractor} instance.
*
* @param service a {@link StreamingService} implementation, which must be the YouTube
* one
* @param linkHandler a {@link ListLinkHandler} which must have a valid channel ID (starting
* with `UC`) and one of the given and supported content filters:
* {@link ChannelTabs#VIDEOS}, {@link ChannelTabs#SHORTS},
* {@link ChannelTabs#LIVESTREAMS}
* @throws IllegalArgumentException if the given {@link ListLinkHandler} doesn't have the
* required arguments
* @throws SystemPlaylistUrlCreationException if the system playlist URL could not be created,
* which should never happen
*/
public YoutubeChannelTabPlaylistExtractor(@Nonnull final StreamingService service,
@Nonnull final ListLinkHandler linkHandler)
throws IllegalArgumentException, SystemPlaylistUrlCreationException {
super(service, linkHandler);
final ListLinkHandler playlistLinkHandler = getPlaylistLinkHandler(linkHandler);
this.playlistExtractorInstance = new YoutubePlaylistExtractor(service, playlistLinkHandler);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
try {
playlistExtractorInstance.onFetchPage(downloader);
if (!playlistExisting) {
playlistExisting = true;
}
} catch (final ContentNotAvailableException e) {
// If a channel has no content of the type requested, the corresponding system playlist
// won't exist, so a ContentNotAvailableException would be thrown
// Ignore such issues in this case
}
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
if (!playlistExisting) {
return InfoItemsPage.emptyPage();
}
final InfoItemsPage<StreamInfoItem> playlistInitialPage =
playlistExtractorInstance.getInitialPage();
// We can't provide the playlist page as it is due to a type conflict, we need to wrap the
// page items and provide a new InfoItemsPage
final List<InfoItem> infoItems = new ArrayList<>(playlistInitialPage.getItems());
return new InfoItemsPage<>(infoItems, playlistInitialPage.getNextPage(),
playlistInitialPage.getErrors());
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (!playlistExisting) {
return InfoItemsPage.emptyPage();
}
final InfoItemsPage<StreamInfoItem> playlistPage = playlistExtractorInstance.getPage(page);
// We can't provide the playlist page as it is due to a type conflict, we need to wrap the
// page items and provide a new InfoItemsPage
final List<InfoItem> infoItems = new ArrayList<>(playlistPage.getItems());
return new InfoItemsPage<>(infoItems, playlistPage.getNextPage(),
playlistPage.getErrors());
}
/**
* Get a playlist {@link ListLinkHandler} from a channel tab one.
*
* <p>
* This method converts a channel ID without its {@code UC} prefix into a YouTube system
* playlist, depending on the first content filter provided in the given
* {@link ListLinkHandler}.
* </p>
*
* <p>
* The first content filter must be a channel tabs one among the
* {@link ChannelTabs#VIDEOS videos}, {@link ChannelTabs#SHORTS shorts} and
* {@link ChannelTabs#LIVESTREAMS} ones, which would be converted respectively into playlists
* with the ID {@code UULF}, {@code UUSH} and {@code UULV} on which the channel ID without the
* {@code UC} part is appended.
* </p>
*
* @param originalLinkHandler the original {@link ListLinkHandler} with which a
* {@link YoutubeChannelTabPlaylistExtractor} instance is being constructed
*
* @return a {@link ListLinkHandler} to use for the {@link YoutubePlaylistExtractor} instance
* needed to extract channel tabs data from a system playlist
* @throws IllegalArgumentException if the original {@link ListLinkHandler} does not meet the
* required criteria above
* @throws SystemPlaylistUrlCreationException if the system playlist URL could not be created,
* which should never happen
*/
@Nonnull
private ListLinkHandler getPlaylistLinkHandler(
@Nonnull final ListLinkHandler originalLinkHandler)
throws IllegalArgumentException, SystemPlaylistUrlCreationException {
final List<String> contentFilters = originalLinkHandler.getContentFilters();
if (contentFilters.isEmpty()) {
throw new IllegalArgumentException("A content filter is required");
}
final String channelId = originalLinkHandler.getId();
if (isNullOrEmpty(channelId) || !channelId.startsWith("UC")) {
throw new IllegalArgumentException("Invalid channel ID");
}
final String channelIdWithoutUc = channelId.substring(2);
final String playlistId;
switch (contentFilters.get(0)) {
case ChannelTabs.VIDEOS:
playlistId = "UULF" + channelIdWithoutUc;
break;
case ChannelTabs.SHORTS:
playlistId = "UUSH" + channelIdWithoutUc;
break;
case ChannelTabs.LIVESTREAMS:
playlistId = "UULV" + channelIdWithoutUc;
break;
default:
throw new IllegalArgumentException(
"Only Videos, Shorts and Livestreams tabs can extracted as playlists");
}
try {
final String newUrl = YoutubePlaylistLinkHandlerFactory.getInstance()
.getUrl(playlistId);
return new ListLinkHandler(newUrl, newUrl, playlistId, List.of(), "");
} catch (final ParsingException e) {
// This should be not reachable, as the given playlist ID should be valid and
// YoutubePlaylistLinkHandlerFactory doesn't throw any exception
throw new SystemPlaylistUrlCreationException(
"Could not create a YouTube playlist from a valid playlist ID", e);
}
}
/**
* Exception thrown when a YouTube system playlist URL could not be created.
*
* <p>
* This exception should be never thrown, as given playlist IDs should be always valid.
* </p>
*/
public static final class SystemPlaylistUrlCreationException extends RuntimeException {
SystemPlaylistUrlCreationException(final String message, final Throwable cause) {
super(message, cause);
}
}
}

View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
@ -22,10 +23,15 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
final String url = playlistInfoItem.getArray("thumbnails").getObject(0) JsonArray thumbnails = playlistInfoItem.getArray("thumbnails")
.getArray("thumbnails").getObject(0).getString("url"); .getObject(0)
.getArray("thumbnails");
if (thumbnails.isEmpty()) {
thumbnails = playlistInfoItem.getObject("thumbnail")
.getArray("thumbnails");
}
return fixThumbnailUrl(url); return fixThumbnailUrl(thumbnails.getObject(0).getString("url"));
} catch (final Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e); throw new ParsingException("Could not get thumbnail url", e);
} }
@ -79,9 +85,21 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override @Override
public long getStreamCount() throws ParsingException { public long getStreamCount() throws ParsingException {
String videoCountText = playlistInfoItem.getString("videoCount");
if (videoCountText == null) {
videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountText"));
}
if (videoCountText == null) {
videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountShortText"));
}
if (videoCountText == null) {
throw new ParsingException("Could not get stream count");
}
try { try {
return Long.parseLong(Utils.removeNonDigitCharacters( return Long.parseLong(Utils.removeNonDigitCharacters(videoCountText));
playlistInfoItem.getString("videoCount")));
} catch (final Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get stream count", e); throw new ParsingException("Could not get stream count", e);
} }

View File

@ -0,0 +1,147 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* A {@link StreamInfoItemExtractor} for YouTube's {@code reelItemRenderers}.
*
* <p>
* {@code reelItemRenderers} are returned on YouTube for their short-form contents on almost every
* place and every major client. They provide a limited amount of information and do not provide
* the exact view count, any uploader info (name, URL, avatar, verified status) and the upload date.
* </p>
*/
public class YoutubeReelInfoItemExtractor implements StreamInfoItemExtractor {
@Nonnull
private final JsonObject reelInfo;
@Nullable
private final TimeAgoParser timeAgoParser;
public YoutubeReelInfoItemExtractor(@Nonnull final JsonObject reelInfo,
@Nullable final TimeAgoParser timeAgoParser) {
this.reelInfo = reelInfo;
this.timeAgoParser = timeAgoParser;
}
@Override
public String getName() throws ParsingException {
return getTextFromObject(reelInfo.getObject("headline"));
}
@Override
public String getUrl() throws ParsingException {
try {
final String videoId = reelInfo.getString("videoId");
return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId);
} catch (final Exception e) {
throw new ParsingException("Could not get URL", e);
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
return getThumbnailUrlFromInfoItem(reelInfo);
}
@Override
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
}
@Override
public long getDuration() throws ParsingException {
// Duration of reelItems is only provided in the accessibility data
// example: "VIDEO TITLE - 49 seconds - play video"
// "VIDEO TITLE - 1 minute, 1 second - play video"
final String accessibilityLabel = reelInfo.getObject("accessibility")
.getObject("accessibilityData").getString("label");
if (accessibilityLabel == null || timeAgoParser == null) {
return 0;
}
// This approach may be language dependent
final String[] labelParts = accessibilityLabel.split(" [\u2013-] ");
if (labelParts.length > 2) {
final String textualDuration = labelParts[labelParts.length - 2];
return timeAgoParser.parseDuration(textualDuration);
}
return -1;
}
@Override
public long getViewCount() throws ParsingException {
final String viewCountText = getTextFromObject(reelInfo.getObject("viewCountText"));
if (!isNullOrEmpty(viewCountText)) {
// This approach is language dependent
if (viewCountText.toLowerCase().contains("no views")) {
return 0;
}
return Utils.mixedNumberWordToLong(viewCountText);
}
throw new ParsingException("Could not get short view count");
}
@Override
public boolean isShortFormContent() throws ParsingException {
return true;
}
// All the following properties cannot be obtained from reelItemRenderers
@Override
public boolean isAd() throws ParsingException {
return false;
}
@Override
public String getUploaderName() throws ParsingException {
return null;
}
@Override
public String getUploaderUrl() throws ParsingException {
return null;
}
@Nullable
@Override
public String getUploaderAvatarUrl() throws ParsingException {
return null;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Nullable
@Override
public String getTextualUploadDate() throws ParsingException {
return null;
}
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
return null;
}
}

View File

@ -58,7 +58,8 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilters, final List<String> contentFilters,
final String searchFilter) { final String searchFilter)
throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/" + id; return "https://www.youtube.com/" + id;
} }
@ -84,7 +85,7 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto
} }
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
try { try {
final URL urlObj = Utils.stringToURL(url); final URL urlObj = Utils.stringToURL(url);
String path = urlObj.getPath(); String path = urlObj.getPath();

View File

@ -0,0 +1,73 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.UnsupportedTabException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import javax.annotation.Nonnull;
import java.util.List;
public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubeChannelTabLinkHandlerFactory INSTANCE =
new YoutubeChannelTabLinkHandlerFactory();
private YoutubeChannelTabLinkHandlerFactory() {
}
public static YoutubeChannelTabLinkHandlerFactory getInstance() {
return INSTANCE;
}
@Nonnull
public static String getUrlSuffix(@Nonnull final String tab)
throws UnsupportedTabException {
switch (tab) {
case ChannelTabs.VIDEOS:
return "/videos";
case ChannelTabs.SHORTS:
return "/shorts";
case ChannelTabs.LIVESTREAMS:
return "/streams";
case ChannelTabs.PLAYLISTS:
return "/playlists";
case ChannelTabs.CHANNELS:
return "/channels";
}
throw new UnsupportedTabException(tab);
}
@Override
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/" + id + getUrlSuffix(contentFilter.get(0));
}
@Override
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return YoutubeChannelLinkHandlerFactory.getInstance().getId(url);
}
@Override
public boolean onAcceptUrl(final String url) throws ParsingException {
try {
getId(url);
} catch (final ParsingException e) {
return false;
}
return true;
}
@Override
public String[] getAvailableContentFilter() {
return new String[] {
ChannelTabs.VIDEOS,
ChannelTabs.SHORTS,
ChannelTabs.LIVESTREAMS,
ChannelTabs.PLAYLISTS,
ChannelTabs.CHANNELS
};
}
}

View File

@ -19,13 +19,14 @@ public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFact
} }
@Override @Override
public String getUrl(final String id) { public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/watch?v=" + id; return "https://www.youtube.com/watch?v=" + id;
} }
@Override @Override
public String getId(final String urlString) throws ParsingException, IllegalArgumentException { public String getId(final String urlString)
// we need the same id, avoids duplicate code throws ParsingException, UnsupportedOperationException {
// We need the same id, avoids duplicate code
return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString); return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString);
} }
@ -44,7 +45,8 @@ public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFact
@Override @Override
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilter, final List<String> contentFilter,
final String sortFilter) throws ParsingException { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return getUrl(id); return getUrl(id);
} }
} }

View File

@ -26,12 +26,13 @@ public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFact
@Override @Override
public String getUrl(final String id, final List<String> contentFilters, public String getUrl(final String id, final List<String> contentFilters,
final String sortFilter) { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/playlist?list=" + id; return "https://www.youtube.com/playlist?list=" + id;
} }
@Override @Override
public String getId(final String url) throws ParsingException { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
try { try {
final URL urlObj = Utils.stringToURL(url); final URL urlObj = Utils.stringToURL(url);

View File

@ -13,6 +13,9 @@ import javax.annotation.Nonnull;
public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
private static final YoutubeSearchQueryHandlerFactory INSTANCE =
new YoutubeSearchQueryHandlerFactory();
public static final String ALL = "all"; public static final String ALL = "all";
public static final String VIDEOS = "videos"; public static final String VIDEOS = "videos";
public static final String CHANNELS = "channels"; public static final String CHANNELS = "channels";
@ -29,20 +32,18 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa
@Nonnull @Nonnull
public static YoutubeSearchQueryHandlerFactory getInstance() { public static YoutubeSearchQueryHandlerFactory getInstance() {
return new YoutubeSearchQueryHandlerFactory(); return INSTANCE;
} }
@Override @Override
public String getUrl(final String searchString, public String getUrl(final String searchString,
@Nonnull final List<String> contentFilters, @Nonnull final List<String> contentFilters,
final String sortFilter) throws ParsingException { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try { try {
if (!contentFilters.isEmpty()) { if (!contentFilters.isEmpty()) {
final String contentFilter = contentFilters.get(0); final String contentFilter = contentFilters.get(0);
switch (contentFilter) { switch (contentFilter) {
case ALL:
default:
break;
case VIDEOS: case VIDEOS:
return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAQ%253D%253D"; return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAQ%253D%253D";
case CHANNELS: case CHANNELS:

View File

@ -79,7 +79,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
@Nonnull @Nonnull
@Override @Override
public String getUrl(final String id) { public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/watch?v=" + id; return "https://www.youtube.com/watch?v=" + id;
} }
@ -87,7 +87,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
@Nonnull @Nonnull
@Override @Override
public String getId(final String theUrlString) public String getId(final String theUrlString)
throws ParsingException, IllegalArgumentException { throws ParsingException, UnsupportedOperationException {
String urlString = theUrlString; String urlString = theUrlString;
try { try {
final URI uri = new URI(urlString); final URI uri = new URI(urlString);

View File

@ -1,28 +1,29 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
/* /*
* Created by Christian Schabesberger on 12.08.17. * Created by Christian Schabesberger on 12.08.17.
* *
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
* YoutubeTrendingLinkHandlerFactory.java is part of NewPipe. * YoutubeTrendingLinkHandlerFactory.java is part of NewPipe Extractor.
* *
* NewPipe is free software: you can redistribute it and/or modify * NewPipe Extractor is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or * the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version. * (at your option) any later version.
* *
* NewPipe is distributed in the hope that it will be useful, * NewPipe Extractor is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of * but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details. * GNU General Public License for more details.
* *
* You should have received a copy of the GNU General Public License * You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
*/ */
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidiousURL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidiousURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
@ -30,16 +31,27 @@ import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.List; import java.util.List;
public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { public final class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubeTrendingLinkHandlerFactory INSTANCE =
new YoutubeTrendingLinkHandlerFactory();
private YoutubeTrendingLinkHandlerFactory() {
}
public static YoutubeTrendingLinkHandlerFactory getInstance() {
return INSTANCE;
}
public String getUrl(final String id, public String getUrl(final String id,
final List<String> contentFilters, final List<String> contentFilters,
final String sortFilter) { final String sortFilter)
throws ParsingException, UnsupportedOperationException {
return "https://www.youtube.com/feed/trending"; return "https://www.youtube.com/feed/trending";
} }
@Override @Override
public String getId(final String url) { public String getId(final String url) throws ParsingException, UnsupportedOperationException {
return "Trending"; return "Trending";
} }

View File

@ -3,8 +3,11 @@ package org.schabi.newpipe.extractor;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
@ -15,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
public class ExtractorAsserts { public class ExtractorAsserts {
@ -144,6 +148,16 @@ public class ExtractorAsserts {
assertNotNull(shouldBeContained, "shouldBeContained is null"); assertNotNull(shouldBeContained, "shouldBeContained is null");
assertNotNull(container, "container is null"); assertNotNull(container, "container is null");
assertTrue(container.contains(shouldBeContained), assertTrue(container.contains(shouldBeContained),
"'" + shouldBeContained + "' should be contained inside '" + container +"'"); "'" + shouldBeContained + "' should be contained inside '" + container + "'");
}
public static void assertTabsContained(@Nonnull final List<ListLinkHandler> tabs,
@Nonnull final String... expectedTabs) {
final Set<String> tabSet = tabs.stream()
.map(linkHandler -> linkHandler.getContentFilters().get(0))
.collect(Collectors.toUnmodifiableSet());
Arrays.stream(expectedTabs)
.forEach(expectedTab -> assertTrue(tabSet.contains(expectedTab),
"Missing " + expectedTab + " tab (got " + tabSet + ")"));
} }
} }

View File

@ -0,0 +1,31 @@
package org.schabi.newpipe.extractor.localization;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
class TimeAgoParserTest {
private static TimeAgoParser timeAgoParser;
@BeforeAll
static void setUp() {
timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
}
@Test
void testGetDuration() throws ParsingException {
assertEquals(1, timeAgoParser.parseDuration("one second"));
assertEquals(1, timeAgoParser.parseDuration("second"));
assertEquals(49, timeAgoParser.parseDuration("49 seconds"));
assertEquals(61, timeAgoParser.parseDuration("1 minute, 1 second"));
}
@Test
void testGetDurationError() {
assertThrows(ParsingException.class, () -> timeAgoParser.parseDuration("abcd"));
assertThrows(ParsingException.class, () -> timeAgoParser.parseDuration("12 abcd"));
}
}

View File

@ -1,11 +1,22 @@
package org.schabi.newpipe.extractor.services; package org.schabi.newpipe.extractor.services;
@SuppressWarnings("unused") import org.junit.jupiter.api.Test;
public interface BaseChannelExtractorTest extends BaseListExtractorTest {
public interface BaseChannelExtractorTest extends BaseExtractorTest {
@Test
void testDescription() throws Exception; void testDescription() throws Exception;
@Test
void testAvatarUrl() throws Exception; void testAvatarUrl() throws Exception;
@Test
void testBannerUrl() throws Exception; void testBannerUrl() throws Exception;
@Test
void testFeedUrl() throws Exception; void testFeedUrl() throws Exception;
@Test
void testSubscriberCount() throws Exception; void testSubscriberCount() throws Exception;
@Test
void testVerified() throws Exception; void testVerified() throws Exception;
@Test
void testTabs() throws Exception;
@Test
void testTags() throws Exception;
} }

View File

@ -1,10 +1,16 @@
package org.schabi.newpipe.extractor.services; package org.schabi.newpipe.extractor.services;
@SuppressWarnings("unused") import org.junit.jupiter.api.Test;
public interface BaseExtractorTest { public interface BaseExtractorTest {
@Test
void testServiceId() throws Exception; void testServiceId() throws Exception;
@Test
void testName() throws Exception; void testName() throws Exception;
@Test
void testId() throws Exception; void testId() throws Exception;
@Test
void testUrl() throws Exception; void testUrl() throws Exception;
@Test
void testOriginalUrl() throws Exception; void testOriginalUrl() throws Exception;
} }

View File

@ -1,7 +1,10 @@
package org.schabi.newpipe.extractor.services; package org.schabi.newpipe.extractor.services;
@SuppressWarnings("unused") import org.junit.jupiter.api.Test;
public interface BaseListExtractorTest extends BaseExtractorTest { public interface BaseListExtractorTest extends BaseExtractorTest {
@Test
void testRelatedItems() throws Exception; void testRelatedItems() throws Exception;
@Test
void testMoreRelatedItems() throws Exception; void testMoreRelatedItems() throws Exception;
} }

View File

@ -1,11 +1,18 @@
package org.schabi.newpipe.extractor.services; package org.schabi.newpipe.extractor.services;
@SuppressWarnings("unused") import org.junit.jupiter.api.Test;
public interface BasePlaylistExtractorTest extends BaseListExtractorTest { public interface BasePlaylistExtractorTest extends BaseListExtractorTest {
@Test
void testThumbnailUrl() throws Exception; void testThumbnailUrl() throws Exception;
@Test
void testBannerUrl() throws Exception; void testBannerUrl() throws Exception;
@Test
void testUploaderName() throws Exception; void testUploaderName() throws Exception;
@Test
void testUploaderAvatarUrl() throws Exception; void testUploaderAvatarUrl() throws Exception;
@Test
void testStreamCount() throws Exception; void testStreamCount() throws Exception;
@Test
void testUploaderVerified() throws Exception; void testUploaderVerified() throws Exception;
} }

View File

@ -1,8 +1,12 @@
package org.schabi.newpipe.extractor.services; package org.schabi.newpipe.extractor.services;
@SuppressWarnings("unused") import org.junit.jupiter.api.Test;
public interface BaseSearchExtractorTest extends BaseListExtractorTest { public interface BaseSearchExtractorTest extends BaseListExtractorTest {
@Test
void testSearchString() throws Exception; void testSearchString() throws Exception;
@Test
void testSearchSuggestion() throws Exception; void testSearchSuggestion() throws Exception;
@Test
void testSearchCorrected() throws Exception; void testSearchCorrected() throws Exception;
} }

View File

@ -1,36 +1,70 @@
package org.schabi.newpipe.extractor.services; package org.schabi.newpipe.extractor.services;
import org.junit.jupiter.api.Test;
public interface BaseStreamExtractorTest extends BaseExtractorTest { public interface BaseStreamExtractorTest extends BaseExtractorTest {
@Test
void testStreamType() throws Exception; void testStreamType() throws Exception;
@Test
void testUploaderName() throws Exception; void testUploaderName() throws Exception;
@Test
void testUploaderUrl() throws Exception; void testUploaderUrl() throws Exception;
@Test
void testUploaderAvatarUrl() throws Exception; void testUploaderAvatarUrl() throws Exception;
@Test
void testSubscriberCount() throws Exception; void testSubscriberCount() throws Exception;
@Test
void testSubChannelName() throws Exception; void testSubChannelName() throws Exception;
@Test
void testSubChannelUrl() throws Exception; void testSubChannelUrl() throws Exception;
@Test
void testSubChannelAvatarUrl() throws Exception; void testSubChannelAvatarUrl() throws Exception;
@Test
void testThumbnailUrl() throws Exception; void testThumbnailUrl() throws Exception;
@Test
void testDescription() throws Exception; void testDescription() throws Exception;
@Test
void testLength() throws Exception; void testLength() throws Exception;
@Test
void testTimestamp() throws Exception; void testTimestamp() throws Exception;
@Test
void testViewCount() throws Exception; void testViewCount() throws Exception;
@Test
void testUploadDate() throws Exception; void testUploadDate() throws Exception;
@Test
void testTextualUploadDate() throws Exception; void testTextualUploadDate() throws Exception;
@Test
void testLikeCount() throws Exception; void testLikeCount() throws Exception;
@Test
void testDislikeCount() throws Exception; void testDislikeCount() throws Exception;
@Test
void testRelatedItems() throws Exception; void testRelatedItems() throws Exception;
@Test
void testAgeLimit() throws Exception; void testAgeLimit() throws Exception;
@Test
void testErrorMessage() throws Exception; void testErrorMessage() throws Exception;
@Test
void testAudioStreams() throws Exception; void testAudioStreams() throws Exception;
@Test
void testVideoStreams() throws Exception; void testVideoStreams() throws Exception;
@Test
void testSubtitles() throws Exception; void testSubtitles() throws Exception;
@Test
void testGetDashMpdUrl() throws Exception; void testGetDashMpdUrl() throws Exception;
@Test
void testFrames() throws Exception; void testFrames() throws Exception;
@Test
void testHost() throws Exception; void testHost() throws Exception;
@Test
void testPrivacy() throws Exception; void testPrivacy() throws Exception;
@Test
void testCategory() throws Exception; void testCategory() throws Exception;
@Test
void testLicence() throws Exception; void testLicence() throws Exception;
@Test
void testLanguageInfo() throws Exception; void testLanguageInfo() throws Exception;
@Test
void testTags() throws Exception; void testTags() throws Exception;
@Test
void testSupportInfo() throws Exception; void testSupportInfo() throws Exception;
} }

View File

@ -7,12 +7,11 @@ import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContained;
import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
public class BandcampChannelExtractorTest implements BaseChannelExtractorTest { public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
@ -27,12 +26,7 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
} }
@Test @Test
public void testLength() throws ExtractionException, IOException {
assertTrue(extractor.getInitialPage().getItems().size() >= 0);
}
@Override @Override
@Test
public void testDescription() throws Exception { public void testDescription() throws Exception {
assertEquals("making music:)", extractor.getDescription()); assertEquals("making music:)", extractor.getDescription());
} }
@ -62,16 +56,6 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
assertFalse(extractor.isVerified()); assertFalse(extractor.isVerified());
} }
@Override
public void testRelatedItems() throws Exception {
// not implemented
}
@Override
public void testMoreRelatedItems() throws Exception {
// not implemented
}
@Override @Override
public void testServiceId() { public void testServiceId() {
assertEquals(Bandcamp.getServiceId(), extractor.getServiceId()); assertEquals(Bandcamp.getServiceId(), extractor.getServiceId());
@ -84,7 +68,7 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
@Override @Override
public void testId() throws Exception { public void testId() throws Exception {
assertEquals("https://toupie.bandcamp.com/", extractor.getId()); assertEquals("2450875064", extractor.getId());
} }
@Override @Override
@ -96,4 +80,16 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
public void testOriginalUrl() throws Exception { public void testOriginalUrl() throws Exception {
assertEquals("https://toupie.bandcamp.com", extractor.getUrl()); assertEquals("https://toupie.bandcamp.com", extractor.getUrl());
} }
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.ALBUMS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
} }

View File

@ -19,7 +19,7 @@ public class BandcampChannelLinkHandlerFactoryTest {
@BeforeAll @BeforeAll
public static void setUp() { public static void setUp() {
linkHandler = new BandcampChannelLinkHandlerFactory(); linkHandler = BandcampChannelLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
} }
@ -55,7 +55,7 @@ public class BandcampChannelLinkHandlerFactoryTest {
public void testGetId() throws ParsingException { public void testGetId() throws ParsingException {
assertEquals("1196681540", linkHandler.getId("https://macbenson.bandcamp.com/")); assertEquals("1196681540", linkHandler.getId("https://macbenson.bandcamp.com/"));
assertEquals("1196681540", linkHandler.getId("http://macbenson.bandcamp.com/")); assertEquals("1196681540", linkHandler.getId("http://macbenson.bandcamp.com/"));
assertEquals("1581461772", linkHandler.getId("https://interovgm.bandcamp.com/releases")); assertEquals("1581461772", linkHandler.getId("https://shirakumon.bandcamp.com/releases"));
assertEquals("3321800855", linkHandler.getId("https://infiniteammo.bandcamp.com/")); assertEquals("3321800855", linkHandler.getId("https://infiniteammo.bandcamp.com/"));
assertEquals("3775652329", linkHandler.getId("https://npet.bandcamp.com/")); assertEquals("3775652329", linkHandler.getId("https://npet.bandcamp.com/"));
@ -67,7 +67,7 @@ public class BandcampChannelLinkHandlerFactoryTest {
@Test @Test
public void testGetUrl() throws ParsingException { public void testGetUrl() throws ParsingException {
assertEquals("https://macbenson.bandcamp.com", linkHandler.getUrl("1196681540")); assertEquals("https://macbenson.bandcamp.com", linkHandler.getUrl("1196681540"));
assertEquals("https://interovgm.bandcamp.com", linkHandler.getUrl("1581461772")); assertEquals("https://shirakumon.bandcamp.com", linkHandler.getUrl("1581461772"));
assertEquals("https://infiniteammo.bandcamp.com", linkHandler.getUrl("3321800855")); assertEquals("https://infiniteammo.bandcamp.com", linkHandler.getUrl("3321800855"));
assertEquals("https://lobstertheremin.com", linkHandler.getUrl("2735462545")); assertEquals("https://lobstertheremin.com", linkHandler.getUrl("2735462545"));
@ -82,5 +82,4 @@ public class BandcampChannelLinkHandlerFactoryTest {
public void testGetIdWithInvalidUrl() { public void testGetIdWithInvalidUrl() {
assertThrows(ParsingException.class, () -> linkHandler.getUrl("https://bandcamp.com")); assertThrows(ParsingException.class, () -> linkHandler.getUrl("https://bandcamp.com"));
} }
} }

View File

@ -0,0 +1,128 @@
package org.schabi.newpipe.extractor.services.bandcamp;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelTabExtractor;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
class BandcampChannelTabExtractorTest {
static class Tracks implements BaseListExtractorTest {
private static BandcampChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws IOException, ExtractionException {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (BandcampChannelTabExtractor) Bandcamp
.getChannelTabExtractorFromId("2464198920", ChannelTabs.TRACKS);
extractor.fetchPage();
}
@Test
@Override
public void testServiceId() throws Exception {
assertEquals(Bandcamp.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws Exception {
assertEquals(ChannelTabs.TRACKS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("2464198920", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://wintergatan.bandcamp.com/track", extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws Exception {
assertEquals("https://wintergatan.bandcamp.com/track", extractor.getOriginalUrl());
}
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
// Bandcamp only return a single page
}
}
static class Albums implements BaseListExtractorTest {
private static BandcampChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws IOException, ExtractionException {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (BandcampChannelTabExtractor) Bandcamp
.getChannelTabExtractorFromId("2450875064", ChannelTabs.ALBUMS);
extractor.fetchPage();
}
@Test
@Override
public void testServiceId() {
assertEquals(Bandcamp.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws Exception {
assertEquals(ChannelTabs.ALBUMS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("2450875064", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://toupie.bandcamp.com/album", extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws Exception {
assertEquals("https://toupie.bandcamp.com/album", extractor.getOriginalUrl());
}
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
// Bandcamp only return a single page
}
}
}

View File

@ -20,7 +20,7 @@ public class BandcampCommentsLinkHandlerFactoryTest {
@BeforeAll @BeforeAll
public static void setUp() { public static void setUp() {
linkHandler = new BandcampCommentsLinkHandlerFactory(); linkHandler = BandcampCommentsLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
} }

View File

@ -18,7 +18,7 @@ public class BandcampFeaturedLinkHandlerFactoryTest {
@BeforeAll @BeforeAll
public static void setUp() { public static void setUp() {
linkHandler = new BandcampFeaturedLinkHandlerFactory(); linkHandler = BandcampFeaturedLinkHandlerFactory.getInstance();
} }
@ -42,7 +42,7 @@ public class BandcampFeaturedLinkHandlerFactoryTest {
} }
@Test @Test
public void testGetId() { public void testGetId() throws ParsingException {
assertEquals("Featured", linkHandler.getId("http://bandcamp.com/api/mobile/24/bootstrap_data")); assertEquals("Featured", linkHandler.getId("http://bandcamp.com/api/mobile/24/bootstrap_data"));
assertEquals("Featured", linkHandler.getId("https://bandcamp.com/api/mobile/24/bootstrap_data")); assertEquals("Featured", linkHandler.getId("https://bandcamp.com/api/mobile/24/bootstrap_data"));
assertEquals("Radio", linkHandler.getId("http://bandcamp.com/?show=1")); assertEquals("Radio", linkHandler.getId("http://bandcamp.com/?show=1"));

View File

@ -21,7 +21,7 @@ public class BandcampPlaylistLinkHandlerFactoryTest {
@BeforeAll @BeforeAll
public static void setUp() { public static void setUp() {
linkHandler = new BandcampPlaylistLinkHandlerFactory(); linkHandler = BandcampPlaylistLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
} }

View File

@ -20,12 +20,12 @@ public class BandcampStreamLinkHandlerFactoryTest {
@BeforeAll @BeforeAll
public static void setUp() { public static void setUp() {
linkHandler = new BandcampStreamLinkHandlerFactory(); linkHandler = BandcampStreamLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
} }
@Test @Test
public void testGetRadioUrl() { public void testGetRadioUrl() throws ParsingException {
assertEquals("https://bandcamp.com/?show=1", linkHandler.getUrl("1")); assertEquals("https://bandcamp.com/?show=1", linkHandler.getUrl("1"));
} }

View File

@ -4,6 +4,7 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
@ -16,12 +17,16 @@ import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
public class MediaCCCConferenceExtractorTest { public class MediaCCCConferenceExtractorTest {
public static class FrOSCon2017 { public static class FrOSCon2017 {
private static MediaCCCConferenceExtractor extractor; private static MediaCCCConferenceExtractor extractor;
private static ChannelTabExtractor tabExtractor;
@BeforeAll @BeforeAll
public static void setUpClass() throws Exception { public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/froscon2017"); extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/froscon2017");
extractor.fetchPage(); extractor.fetchPage();
tabExtractor = MediaCCC.getChannelTabExtractor(extractor.getTabs().get(0));
tabExtractor.fetchPage();
} }
@Test @Test
@ -46,18 +51,22 @@ public class MediaCCCConferenceExtractorTest {
@Test @Test
public void testGetInitalPage() throws Exception { public void testGetInitalPage() throws Exception {
assertEquals(97, extractor.getInitialPage().getItems().size()); assertEquals(97, tabExtractor.getInitialPage().getItems().size());
} }
} }
public static class Oscal2019 { public static class Oscal2019 {
private static MediaCCCConferenceExtractor extractor; private static MediaCCCConferenceExtractor extractor;
private static ChannelTabExtractor tabExtractor;
@BeforeAll @BeforeAll
public static void setUpClass() throws Exception { public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/oscal19"); extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/oscal19");
extractor.fetchPage(); extractor.fetchPage();
tabExtractor = MediaCCC.getChannelTabExtractor(extractor.getTabs().get(0));
tabExtractor.fetchPage();
} }
@Test @Test
@ -82,7 +91,7 @@ public class MediaCCCConferenceExtractorTest {
@Test @Test
public void testGetInitalPage() throws Exception { public void testGetInitalPage() throws Exception {
assertTrue(extractor.getInitialPage().getItems().size() >= 21); assertTrue(tabExtractor.getInitialPage().getItems().size() >= 21);
} }
} }
} }

View File

@ -14,7 +14,7 @@ public class MediaCCCConferenceLinkHandlerFactoryTest {
@BeforeAll @BeforeAll
public static void setUp() { public static void setUp() {
linkHandler = new MediaCCCConferenceLinkHandlerFactory(); linkHandler = MediaCCCConferenceLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
} }

View File

@ -14,7 +14,7 @@ public class MediaCCCStreamLinkHandlerFactoryTest {
@BeforeAll @BeforeAll
public static void setUp() { public static void setUp() {
linkHandler = new MediaCCCStreamLinkHandlerFactory(); linkHandler = MediaCCCStreamLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
} }

View File

@ -5,7 +5,7 @@ import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.ExtractorAsserts;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor;
@ -14,11 +14,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContained;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
/** /**
* Test for {@link PeertubeAccountExtractor} * Test for {@link PeertubeAccountExtractor}
@ -67,20 +66,6 @@ public class PeertubeAccountExtractorTest {
assertEquals("https://framatube.org/accounts/framasoft", extractor.getOriginalUrl()); assertEquals("https://framatube.org/accounts/framasoft", extractor.getOriginalUrl());
} }
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor // ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
@ -114,6 +99,18 @@ public class PeertubeAccountExtractorTest {
public void testVerified() throws Exception { public void testVerified() throws Exception {
assertFalse(extractor.isVerified()); assertFalse(extractor.isVerified());
} }
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.VIDEOS, ChannelTabs.CHANNELS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
} }
public static class FreeSoftwareFoundation implements BaseChannelExtractorTest { public static class FreeSoftwareFoundation implements BaseChannelExtractorTest {
@ -129,16 +126,6 @@ public class PeertubeAccountExtractorTest {
extractor.fetchPage(); extractor.fetchPage();
} }
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
// Extractor // Extractor
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
@ -168,20 +155,6 @@ public class PeertubeAccountExtractorTest {
assertEquals("https://framatube.org/api/v1/accounts/fsf", extractor.getOriginalUrl()); assertEquals("https://framatube.org/api/v1/accounts/fsf", extractor.getOriginalUrl());
} }
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor // ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
@ -215,5 +188,17 @@ public class PeertubeAccountExtractorTest {
public void testVerified() throws Exception { public void testVerified() throws Exception {
assertFalse(extractor.isVerified()); assertFalse(extractor.isVerified());
} }
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.VIDEOS, ChannelTabs.CHANNELS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
} }
} }

View File

@ -0,0 +1,150 @@
package org.schabi.newpipe.extractor.services.peertube;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelTabExtractor;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
class PeertubeAccountTabExtractorTest {
static class Videos implements BaseListExtractorTest {
private static PeertubeChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = (PeertubeChannelTabExtractor) PeerTube
.getChannelTabExtractorFromId("accounts/framasoft", ChannelTabs.VIDEOS);
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws ParsingException {
assertEquals(ChannelTabs.VIDEOS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("accounts/framasoft", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://framatube.org/accounts/framasoft/videos",
extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws ParsingException {
assertEquals("https://framatube.org/accounts/framasoft/videos",
extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
}
static class Channels implements BaseListExtractorTest {
private static PeertubeChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = (PeertubeChannelTabExtractor) PeerTube
.getChannelTabExtractorFromId("accounts/framasoft", ChannelTabs.CHANNELS);
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws ParsingException {
assertEquals(ChannelTabs.CHANNELS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("accounts/framasoft", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://framatube.org/accounts/framasoft/video-channels",
extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws ParsingException {
assertEquals("https://framatube.org/accounts/framasoft/video-channels",
extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
}
}

View File

@ -5,15 +5,19 @@ import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.ExtractorAsserts;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContained;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
/** /**
* Test for {@link PeertubeChannelExtractor} * Test for {@link PeertubeChannelExtractor}
@ -62,20 +66,6 @@ public class PeertubeChannelExtractorTest {
assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/videos", extractor.getOriginalUrl()); assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/videos", extractor.getOriginalUrl());
} }
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor // ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
@ -124,6 +114,18 @@ public class PeertubeChannelExtractorTest {
public void testVerified() throws Exception { public void testVerified() throws Exception {
assertFalse(extractor.isVerified()); assertFalse(extractor.isVerified());
} }
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.VIDEOS, ChannelTabs.PLAYLISTS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
} }
public static class ChatSceptique implements BaseChannelExtractorTest { public static class ChatSceptique implements BaseChannelExtractorTest {
@ -140,16 +142,6 @@ public class PeertubeChannelExtractorTest {
extractor.fetchPage(); extractor.fetchPage();
} }
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
// Extractor // Extractor
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
@ -179,20 +171,6 @@ public class PeertubeChannelExtractorTest {
assertEquals("https://framatube.org/api/v1/video-channels/chatsceptique@skeptikon.fr", extractor.getOriginalUrl()); assertEquals("https://framatube.org/api/v1/video-channels/chatsceptique@skeptikon.fr", extractor.getOriginalUrl());
} }
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor // ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
@ -241,5 +219,17 @@ public class PeertubeChannelExtractorTest {
public void testVerified() throws Exception { public void testVerified() throws Exception {
assertFalse(extractor.isVerified()); assertFalse(extractor.isVerified());
} }
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.VIDEOS, ChannelTabs.PLAYLISTS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
} }
} }

View File

@ -0,0 +1,212 @@
package org.schabi.newpipe.extractor.services.peertube;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelTabExtractor;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
class PeertubeChannelTabExtractorTest {
static class Videos implements BaseListExtractorTest {
private static PeertubeChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = (PeertubeChannelTabExtractor) PeerTube.getChannelTabExtractorFromId(
"video-channels/lqdn_channel@video.lqdn.fr", ChannelTabs.VIDEOS);
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws ParsingException {
assertEquals(ChannelTabs.VIDEOS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("video-channels/lqdn_channel@video.lqdn.fr", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/videos",
extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws ParsingException {
assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/videos",
extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
void testGetPageInNewExtractor() throws Exception {
final ChannelTabExtractor newTabExtractor = PeerTube.getChannelTabExtractorFromId(
"video-channels/lqdn_channel@video.lqdn.fr", ChannelTabs.VIDEOS);
defaultTestGetPageInNewExtractor(extractor, newTabExtractor);
}
}
static class Playlists implements BaseListExtractorTest {
private static PeertubeChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws IOException, ExtractionException {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeChannelTabExtractor)
PeerTube.getChannelTabExtractorFromIdAndBaseUrl(
"video-channels/lqdn_channel@video.lqdn.fr", ChannelTabs.PLAYLISTS,
"https://framatube.org");
extractor.fetchPage();
}
@Test
@Override
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws Exception {
assertEquals(ChannelTabs.PLAYLISTS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("video-channels/lqdn_channel@video.lqdn.fr", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/video-playlists",
extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws Exception {
assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/video-playlists",
extractor.getOriginalUrl());
}
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Override
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
}
static class Channels implements BaseListExtractorTest {
private static PeertubeChannelTabExtractor extractor;
@BeforeAll
static void setUp() throws IOException, ExtractionException {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeChannelTabExtractor)
PeerTube.getChannelTabExtractorFromIdAndBaseUrl("accounts/framasoft",
ChannelTabs.CHANNELS, "https://framatube.org");
extractor.fetchPage();
}
@Test
@Override
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
@Override
public void testName() throws Exception {
assertEquals(ChannelTabs.CHANNELS, extractor.getName());
}
@Test
@Override
public void testId() throws ParsingException {
assertEquals("accounts/framasoft", extractor.getId());
}
@Test
@Override
public void testUrl() throws ParsingException {
assertEquals("https://framatube.org/accounts/framasoft/video-channels",
extractor.getUrl());
}
@Test
@Override
public void testOriginalUrl() throws Exception {
assertEquals("https://framatube.org/accounts/framasoft/video-channels",
extractor.getOriginalUrl());
}
@Test
@Override
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
@Override
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
}
}

View File

@ -22,7 +22,7 @@ public class PeertubeTrendingLinkHandlerFactoryTest {
public static void setUp() throws Exception { public static void setUp() throws Exception {
// setting instance might break test when running in parallel // setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
LinkHandlerFactory = new PeertubeTrendingLinkHandlerFactory(); LinkHandlerFactory = PeertubeTrendingLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
} }

View File

@ -4,7 +4,7 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelExtractor;
@ -12,8 +12,8 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCha
import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertTabsContained;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
/** /**
* Test for {@link SoundcloudChannelExtractor} * Test for {@link SoundcloudChannelExtractor}
@ -59,20 +59,6 @@ public class SoundcloudChannelExtractorTest {
assertEquals("http://soundcloud.com/liluzivert/sets", extractor.getOriginalUrl()); assertEquals("http://soundcloud.com/liluzivert/sets", extractor.getOriginalUrl());
} }
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor // ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
@ -106,6 +92,19 @@ public class SoundcloudChannelExtractorTest {
public void testVerified() throws Exception { public void testVerified() throws Exception {
assertTrue(extractor.isVerified()); assertTrue(extractor.isVerified());
} }
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.TRACKS, ChannelTabs.PLAYLISTS,
ChannelTabs.ALBUMS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
} }
public static class DubMatix implements BaseChannelExtractorTest { public static class DubMatix implements BaseChannelExtractorTest {
@ -119,16 +118,6 @@ public class SoundcloudChannelExtractorTest {
extractor.fetchPage(); extractor.fetchPage();
} }
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
// Extractor // Extractor
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
@ -158,20 +147,6 @@ public class SoundcloudChannelExtractorTest {
assertEquals("https://soundcloud.com/dubmatix", extractor.getOriginalUrl()); assertEquals("https://soundcloud.com/dubmatix", extractor.getOriginalUrl());
} }
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor // ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
@ -205,5 +180,18 @@ public class SoundcloudChannelExtractorTest {
public void testVerified() throws Exception { public void testVerified() throws Exception {
assertTrue(extractor.isVerified()); assertTrue(extractor.isVerified());
} }
@Test
@Override
public void testTabs() throws Exception {
assertTabsContained(extractor.getTabs(), ChannelTabs.TRACKS, ChannelTabs.PLAYLISTS,
ChannelTabs.ALBUMS);
}
@Test
@Override
public void testTags() throws Exception {
assertTrue(extractor.getTags().isEmpty());
}
} }
} }

Some files were not shown because too many files have changed in this diff Show More