mirror of
https://github.com/TeamNewPipe/NewPipeExtractor.git
synced 2024-12-12 21:30:33 +05:30
Merge pull request #788 from Stypox/mix
Add MixInfoItem and extract YouTube mixes in related items
This commit is contained in:
XiangRongLin
GitHub
commit
7f2ea133f0
@ -1,8 +1,5 @@
|
||||
package org.schabi.newpipe.extractor.search;
|
||||
package org.schabi.newpipe.extractor;
|
||||
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.InfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.InfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
@ -36,7 +33,8 @@ import java.util.List;
|
||||
*/
|
||||
|
||||
/**
|
||||
* Collector for search results
|
||||
* A collector that can handle many extractor types, to be used when a list contains items of
|
||||
* different types (e.g. search)
|
||||
* <p>
|
||||
* This collector can handle the following extractor types:
|
||||
* <ul>
|
||||
@ -44,15 +42,15 @@ import java.util.List;
|
||||
* <li>{@link ChannelInfoItemExtractor}</li>
|
||||
* <li>{@link PlaylistInfoItemExtractor}</li>
|
||||
* </ul>
|
||||
* Calling {@link #extract(InfoItemExtractor)} or {@link #commit(Object)} with any
|
||||
* Calling {@link #extract(InfoItemExtractor)} or {@link #commit(InfoItemExtractor)} with any
|
||||
* other extractor type will raise an exception.
|
||||
*/
|
||||
public class InfoItemsSearchCollector extends InfoItemsCollector<InfoItem, InfoItemExtractor> {
|
||||
public class MultiInfoItemsCollector extends InfoItemsCollector<InfoItem, InfoItemExtractor> {
|
||||
private final StreamInfoItemsCollector streamCollector;
|
||||
private final ChannelInfoItemsCollector userCollector;
|
||||
private final PlaylistInfoItemsCollector playlistCollector;
|
||||
|
||||
public InfoItemsSearchCollector(int serviceId) {
|
||||
public MultiInfoItemsCollector(int serviceId) {
|
||||
super(serviceId);
|
||||
streamCollector = new StreamInfoItemsCollector(serviceId);
|
||||
userCollector = new ChannelInfoItemsCollector(serviceId);
|
||||
@ -49,4 +49,8 @@ public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> {
|
||||
public String getSubChannelAvatarUrl() throws ParsingException {
|
||||
return EMPTY_STRING;
|
||||
}
|
||||
|
||||
public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException {
|
||||
return PlaylistInfo.PlaylistType.NORMAL;
|
||||
}
|
||||
}
|
||||
|
||||
@ -17,6 +17,41 @@ import java.util.List;
|
||||
|
||||
public class PlaylistInfo extends ListInfo<StreamInfoItem> {
|
||||
|
||||
/**
|
||||
* Mixes are handled as particular playlists in NewPipeExtractor. {@link PlaylistType#NORMAL} is
|
||||
* for non-mixes, while other values are for the different types of mixes. The type of a mix
|
||||
* depends on how its contents are autogenerated.
|
||||
*/
|
||||
public enum PlaylistType {
|
||||
/**
|
||||
* A normal playlist (not a mix)
|
||||
*/
|
||||
NORMAL,
|
||||
|
||||
/**
|
||||
* A mix made only of streams related to a particular stream, for example YouTube mixes
|
||||
*/
|
||||
MIX_STREAM,
|
||||
|
||||
/**
|
||||
* A mix made only of music streams related to a particular stream, for example YouTube
|
||||
* music mixes
|
||||
*/
|
||||
MIX_MUSIC,
|
||||
|
||||
/**
|
||||
* A mix made only of streams from (or related to) the same channel, for example YouTube
|
||||
* channel mixes
|
||||
*/
|
||||
MIX_CHANNEL,
|
||||
|
||||
/**
|
||||
* A mix made only of streams related to a particular (musical) genre, for example YouTube
|
||||
* genre mixes
|
||||
*/
|
||||
MIX_GENRE,
|
||||
}
|
||||
|
||||
private PlaylistInfo(int serviceId, ListLinkHandler linkHandler, String name) throws ParsingException {
|
||||
super(serviceId, linkHandler, name);
|
||||
}
|
||||
@ -105,6 +140,11 @@ public class PlaylistInfo extends ListInfo<StreamInfoItem> {
|
||||
} catch (Exception e) {
|
||||
info.addError(e);
|
||||
}
|
||||
try {
|
||||
info.setPlaylistType(extractor.getPlaylistType());
|
||||
} catch (Exception e) {
|
||||
info.addError(e);
|
||||
}
|
||||
// do not fail if everything but the uploader infos could be collected
|
||||
if (!uploaderParsingErrors.isEmpty() &&
|
||||
(!info.getErrors().isEmpty() || uploaderParsingErrors.size() < 3)) {
|
||||
@ -127,6 +167,7 @@ public class PlaylistInfo extends ListInfo<StreamInfoItem> {
|
||||
private String subChannelName;
|
||||
private String subChannelAvatarUrl;
|
||||
private long streamCount = 0;
|
||||
private PlaylistType playlistType;
|
||||
|
||||
public String getThumbnailUrl() {
|
||||
return thumbnailUrl;
|
||||
@ -199,4 +240,12 @@ public class PlaylistInfo extends ListInfo<StreamInfoItem> {
|
||||
public void setStreamCount(long streamCount) {
|
||||
this.streamCount = streamCount;
|
||||
}
|
||||
|
||||
public PlaylistType getPlaylistType() {
|
||||
return playlistType;
|
||||
}
|
||||
|
||||
public void setPlaylistType(final PlaylistType playlistType) {
|
||||
this.playlistType = playlistType;
|
||||
}
|
||||
}
|
||||
|
||||
@ -9,6 +9,7 @@ public class PlaylistInfoItem extends InfoItem {
|
||||
* How many streams this playlist have
|
||||
*/
|
||||
private long streamCount = 0;
|
||||
private PlaylistInfo.PlaylistType playlistType;
|
||||
|
||||
public PlaylistInfoItem(int serviceId, String url, String name) {
|
||||
super(InfoType.PLAYLIST, serviceId, url, name);
|
||||
@ -29,4 +30,12 @@ public class PlaylistInfoItem extends InfoItem {
|
||||
public void setStreamCount(long stream_count) {
|
||||
this.streamCount = stream_count;
|
||||
}
|
||||
|
||||
public PlaylistInfo.PlaylistType getPlaylistType() {
|
||||
return playlistType;
|
||||
}
|
||||
|
||||
public void setPlaylistType(final PlaylistInfo.PlaylistType playlistType) {
|
||||
this.playlistType = playlistType;
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.playlist;
|
||||
import org.schabi.newpipe.extractor.InfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
public interface PlaylistInfoItemExtractor extends InfoItemExtractor {
|
||||
|
||||
/**
|
||||
@ -18,4 +20,13 @@ public interface PlaylistInfoItemExtractor extends InfoItemExtractor {
|
||||
* @throws ParsingException
|
||||
*/
|
||||
long getStreamCount() throws ParsingException;
|
||||
|
||||
/**
|
||||
* @return the type of this playlist, see {@link PlaylistInfo.PlaylistType} for a description
|
||||
* of types. If not overridden always returns {@link PlaylistInfo.PlaylistType#NORMAL}.
|
||||
*/
|
||||
@Nonnull
|
||||
default PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException {
|
||||
return PlaylistInfo.PlaylistType.NORMAL;
|
||||
}
|
||||
}
|
||||
|
||||
@ -33,6 +33,11 @@ public class PlaylistInfoItemsCollector extends InfoItemsCollector<PlaylistInfoI
|
||||
} catch (Exception e) {
|
||||
addError(e);
|
||||
}
|
||||
try {
|
||||
resultItem.setPlaylistType(extractor.getPlaylistType());
|
||||
} catch (Exception e) {
|
||||
addError(e);
|
||||
}
|
||||
return resultItem;
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,7 +15,7 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
|
||||
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
|
||||
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampSearchStreamInfoItemExtractor;
|
||||
|
||||
@ -50,7 +50,7 @@ public class BandcampSearchExtractor extends SearchExtractor {
|
||||
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, ExtractionException {
|
||||
final String html = getDownloader().get(page.getUrl()).responseBody();
|
||||
|
||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
|
||||
|
||||
|
||||
final Document d = Jsoup.parse(html);
|
||||
|
||||
@ -15,7 +15,7 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
|
||||
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
|
||||
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory;
|
||||
@ -66,7 +66,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
|
||||
@Nonnull
|
||||
@Override
|
||||
public InfoItemsPage<InfoItem> getInitialPage() {
|
||||
final InfoItemsSearchCollector searchItems = new InfoItemsSearchCollector(getServiceId());
|
||||
final MultiInfoItemsCollector searchItems = new MultiInfoItemsCollector(getServiceId());
|
||||
|
||||
if (getLinkHandler().getContentFilters().contains(CONFERENCES)
|
||||
|| getLinkHandler().getContentFilters().contains(ALL)
|
||||
@ -122,7 +122,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
|
||||
|
||||
private void searchConferences(final String searchString,
|
||||
final List<ChannelInfoItem> channelItems,
|
||||
final InfoItemsSearchCollector collector) {
|
||||
final MultiInfoItemsCollector collector) {
|
||||
for (final ChannelInfoItem item : channelItems) {
|
||||
if (item.getName().toUpperCase().contains(
|
||||
searchString.toUpperCase())) {
|
||||
|
||||
@ -12,7 +12,7 @@ import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
|
||||
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
|
||||
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
@ -87,7 +87,7 @@ public class PeertubeSearchExtractor extends SearchExtractor {
|
||||
PeertubeParsingHelper.validate(json);
|
||||
final long total = json.getLong("total");
|
||||
|
||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
|
||||
collectStreamsFrom(collector, json, getBaseUrl(), sepia);
|
||||
|
||||
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total));
|
||||
|
||||
@ -9,7 +9,7 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
|
||||
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
|
||||
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
|
||||
@ -100,7 +100,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
|
||||
|
||||
private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(
|
||||
final JsonArray searchCollection) {
|
||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
|
||||
|
||||
for (final Object result : searchCollection) {
|
||||
if (!(result instanceof JsonObject)) continue;
|
||||
|
||||
@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.exceptions.*;
|
||||
import org.schabi.newpipe.extractor.localization.ContentCountry;
|
||||
import org.schabi.newpipe.extractor.localization.Localization;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
|
||||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
@ -246,7 +247,19 @@ public class YoutubeParsingHelper {
|
||||
* @return Whether given id belongs to a YouTube Mix
|
||||
*/
|
||||
public static boolean isYoutubeMixId(@Nonnull final String playlistId) {
|
||||
return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId);
|
||||
return playlistId.startsWith("RD")
|
||||
&& !isYoutubeMusicMixId(playlistId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the given playlist id is a YouTube My Mix (auto-generated playlist)
|
||||
* Ids from a YouTube My Mix start with "RDMM"
|
||||
*
|
||||
* @param playlistId the playlist id
|
||||
* @return Whether given id belongs to a YouTube My Mix
|
||||
*/
|
||||
public static boolean isYoutubeMyMixId(@Nonnull final String playlistId) {
|
||||
return playlistId.startsWith("RDMM");
|
||||
}
|
||||
|
||||
/**
|
||||
@ -271,33 +284,106 @@ public class YoutubeParsingHelper {
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the video id from the playlist id for Mixes.
|
||||
* Checks if the given playlist id is a YouTube Genre Mix (auto-generated playlist)
|
||||
* Ids from a YouTube Genre Mix start with "RDGMEM"
|
||||
*
|
||||
* @throws ParsingException If the playlistId is a Channel Mix or not a mix.
|
||||
* @return Whether given id belongs to a YouTube Genre Mix
|
||||
*/
|
||||
public static boolean isYoutubeGenreMixId(@Nonnull final String playlistId) {
|
||||
return playlistId.startsWith("RDGMEM");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param playlistId the playlist id to parse
|
||||
* @return the {@link PlaylistInfo.PlaylistType} extracted from the playlistId (mix playlist
|
||||
* types included)
|
||||
* @throws ParsingException if the playlistId is null or empty, if the playlistId is not a mix,
|
||||
* if it is a mix but it's not based on a specific stream (this is the
|
||||
* case for channel or genre mixes)
|
||||
*/
|
||||
@Nonnull
|
||||
public static String extractVideoIdFromMixId(@Nonnull final String playlistId)
|
||||
public static String extractVideoIdFromMixId(final String playlistId)
|
||||
throws ParsingException {
|
||||
if (playlistId.startsWith("RDMM")) { // My Mix
|
||||
if (isNullOrEmpty(playlistId)) {
|
||||
throw new ParsingException("Video id could not be determined from empty playlist id");
|
||||
|
||||
} else if (isYoutubeMyMixId(playlistId)) {
|
||||
return playlistId.substring(4);
|
||||
|
||||
} else if (isYoutubeMusicMixId(playlistId)) { // starts with "RDAMVM" or "RDCLAK"
|
||||
} else if (isYoutubeMusicMixId(playlistId)) {
|
||||
return playlistId.substring(6);
|
||||
|
||||
} else if (isYoutubeChannelMixId(playlistId)) { // starts with "RMCM"
|
||||
// Channel mix are build with RMCM{channelId}, so videoId can't be determined
|
||||
throw new ParsingException("Video id could not be determined from mix id: "
|
||||
} else if (isYoutubeChannelMixId(playlistId)) {
|
||||
// Channel mixes are of the form RMCM{channelId}, so videoId can't be determined
|
||||
throw new ParsingException("Video id could not be determined from channel mix id: "
|
||||
+ playlistId);
|
||||
|
||||
} else if (isYoutubeMixId(playlistId)) { // normal mix, starts with "RD"
|
||||
} else if (isYoutubeGenreMixId(playlistId)) {
|
||||
// Genre mixes are of the form RDGMEM{garbage}, so videoId can't be determined
|
||||
throw new ParsingException("Video id could not be determined from genre mix id: "
|
||||
+ playlistId);
|
||||
|
||||
} else if (isYoutubeMixId(playlistId)) { // normal mix
|
||||
if (playlistId.length() != 13) {
|
||||
// Stream YouTube mixes are of the form RD{videoId}, but if videoId is not exactly
|
||||
// 11 characters then it can't be a video id, hence we are dealing with a different
|
||||
// type of mix (e.g. genre mixes handled above, of the form RDGMEM{garbage})
|
||||
throw new ParsingException("Video id could not be determined from mix id: "
|
||||
+ playlistId);
|
||||
}
|
||||
return playlistId.substring(2);
|
||||
|
||||
} else { // not a mix
|
||||
throw new ParsingException("Video id could not be determined from mix id: "
|
||||
throw new ParsingException("Video id could not be determined from playlist id: "
|
||||
+ playlistId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param playlistId the playlist id to parse
|
||||
* @return the {@link PlaylistInfo.PlaylistType} extracted from the playlistId (mix playlist
|
||||
* types included)
|
||||
* @throws ParsingException if the playlistId is null or empty
|
||||
*/
|
||||
@Nonnull
|
||||
public static PlaylistInfo.PlaylistType extractPlaylistTypeFromPlaylistId(
|
||||
final String playlistId) throws ParsingException {
|
||||
if (isNullOrEmpty(playlistId)) {
|
||||
throw new ParsingException("Could not extract playlist type from empty playlist id");
|
||||
} else if (isYoutubeMusicMixId(playlistId)) {
|
||||
return PlaylistInfo.PlaylistType.MIX_MUSIC;
|
||||
} else if (isYoutubeChannelMixId(playlistId)) {
|
||||
return PlaylistInfo.PlaylistType.MIX_CHANNEL;
|
||||
} else if (isYoutubeGenreMixId(playlistId)) {
|
||||
return PlaylistInfo.PlaylistType.MIX_GENRE;
|
||||
} else if (isYoutubeMixId(playlistId)) { // normal mix
|
||||
// Either a normal mix based on a stream, or a "my mix" (still based on a stream).
|
||||
// NOTE: if YouTube introduces even more types of mixes that still start with RD,
|
||||
// they will default to this, even though they might not be based on a stream.
|
||||
return PlaylistInfo.PlaylistType.MIX_STREAM;
|
||||
} else {
|
||||
// not a known type of mix: just consider it a normal playlist
|
||||
return PlaylistInfo.PlaylistType.NORMAL;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param playlistUrl the playlist url to parse
|
||||
* @return the {@link PlaylistInfo.PlaylistType} extracted from the playlistUrl's list param
|
||||
* (mix playlist types included)
|
||||
* @throws ParsingException if the playlistUrl is malformed, if has no list param or if the list
|
||||
* param is empty
|
||||
*/
|
||||
public static PlaylistInfo.PlaylistType extractPlaylistTypeFromPlaylistUrl(
|
||||
final String playlistUrl) throws ParsingException {
|
||||
try {
|
||||
return extractPlaylistTypeFromPlaylistId(
|
||||
Utils.getQueryValue(Utils.stringToURL(playlistUrl), "list"));
|
||||
} catch (final MalformedURLException e) {
|
||||
throw new ParsingException("Could not extract playlist type from malformed url", e);
|
||||
}
|
||||
}
|
||||
|
||||
public static JsonObject getInitialData(final String html) throws ParsingException {
|
||||
try {
|
||||
try {
|
||||
@ -705,6 +791,17 @@ public class YoutubeParsingHelper {
|
||||
return thumbnailUrl;
|
||||
}
|
||||
|
||||
public static String getThumbnailUrlFromInfoItem(final JsonObject infoItem)
|
||||
throws ParsingException {
|
||||
// TODO: Don't simply get the first item, but look at all thumbnails and their resolution
|
||||
try {
|
||||
return fixThumbnailUrl(infoItem.getObject("thumbnail").getArray("thumbnails")
|
||||
.getObject(0).getString("url"));
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get thumbnail url", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
public static String getValidJsonResponseBody(@Nonnull final Response response)
|
||||
throws ParsingException, MalformedURLException {
|
||||
|
||||
@ -0,0 +1,76 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
import com.grack.nanojson.JsonObject;
|
||||
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
public class YoutubeMixOrPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
|
||||
private final JsonObject mixInfoItem;
|
||||
|
||||
public YoutubeMixOrPlaylistInfoItemExtractor(final JsonObject mixInfoItem) {
|
||||
this.mixInfoItem = mixInfoItem;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
final String name = getTextFromObject(mixInfoItem.getObject("title"));
|
||||
if (isNullOrEmpty(name)) {
|
||||
throw new ParsingException("Could not get name");
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
final String url = mixInfoItem.getString("shareUrl");
|
||||
if (isNullOrEmpty(url)) {
|
||||
throw new ParsingException("Could not get url");
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
return getThumbnailUrlFromInfoItem(mixInfoItem);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderName() throws ParsingException {
|
||||
// this will be "YouTube" for mixes
|
||||
return YoutubeParsingHelper.getTextFromObject(mixInfoItem.getObject("longBylineText"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getStreamCount() throws ParsingException {
|
||||
final String countString = YoutubeParsingHelper.getTextFromObject(
|
||||
mixInfoItem.getObject("videoCountShortText"));
|
||||
if (countString == null) {
|
||||
throw new ParsingException("Could not extract item count for playlist/mix info item");
|
||||
}
|
||||
|
||||
try {
|
||||
return Integer.parseInt(countString);
|
||||
} catch (final NumberFormatException ignored) {
|
||||
// un-parsable integer: this is a mix with infinite items and "50+" as count string
|
||||
// (though youtube music mixes do not necessarily have an infinite count of songs)
|
||||
return ListExtractor.ITEM_COUNT_INFINITE;
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException {
|
||||
return extractPlaylistTypeFromPlaylistUrl(getUrl());
|
||||
}
|
||||
}
|
||||
@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.localization.Localization;
|
||||
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
@ -232,23 +233,19 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId) throws ParsingException {
|
||||
final String videoId;
|
||||
if (playlistId.startsWith("RDMM")) {
|
||||
videoId = playlistId.substring(4);
|
||||
} else if (playlistId.startsWith("RDCMUC")) {
|
||||
throw new ParsingException("This playlist is a channel mix");
|
||||
} else {
|
||||
videoId = playlistId.substring(2);
|
||||
}
|
||||
if (videoId.isEmpty()) {
|
||||
throw new ParsingException("videoId is empty");
|
||||
}
|
||||
return getThumbnailUrlFromVideoId(videoId);
|
||||
private String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId)
|
||||
throws ParsingException {
|
||||
return getThumbnailUrlFromVideoId(YoutubeParsingHelper.extractVideoIdFromMixId(playlistId));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String getThumbnailUrlFromVideoId(final String videoId) {
|
||||
return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException {
|
||||
return extractPlaylistTypeFromPlaylistId(playlistData.getString("playlistId"));
|
||||
}
|
||||
}
|
||||
|
||||
@ -12,7 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
|
||||
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
||||
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
||||
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
|
||||
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
@ -177,7 +177,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
|
||||
@Nonnull
|
||||
@Override
|
||||
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
|
||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
|
||||
|
||||
final JsonArray contents = JsonUtils.getArray(JsonUtils.getArray(initialData,
|
||||
"contents.tabbedSearchResultsRenderer.tabs").getObject(0),
|
||||
@ -206,7 +206,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
|
||||
throw new IllegalArgumentException("Page doesn't contain an URL");
|
||||
}
|
||||
|
||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
|
||||
|
||||
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
|
||||
|
||||
@ -264,7 +264,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
|
||||
return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
|
||||
}
|
||||
|
||||
private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector,
|
||||
private void collectMusicStreamsFrom(final MultiInfoItemsCollector collector,
|
||||
@Nonnull final JsonArray videos) {
|
||||
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
||||
|
||||
|
||||
@ -14,6 +14,7 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.localization.Localization;
|
||||
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
@ -329,4 +330,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
})
|
||||
.forEachOrdered(collector::commit);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException {
|
||||
return extractPlaylistTypeFromPlaylistUrl(getUrl());
|
||||
}
|
||||
}
|
||||
|
||||
@ -11,7 +11,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
|
||||
import org.schabi.newpipe.extractor.localization.Localization;
|
||||
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
||||
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
|
||||
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
@ -132,7 +132,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
||||
@Nonnull
|
||||
@Override
|
||||
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
|
||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
|
||||
|
||||
final JsonArray sections = initialData.getObject("contents")
|
||||
.getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
|
||||
@ -163,7 +163,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
||||
}
|
||||
|
||||
final Localization localization = getExtractorLocalization();
|
||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
|
||||
|
||||
// @formatter:off
|
||||
final byte[] json = JsonWriter.string(prepareDesktopJsonBuilder(localization,
|
||||
@ -195,7 +195,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
||||
.getObject("continuationItemRenderer")));
|
||||
}
|
||||
|
||||
private void collectStreamsFrom(final InfoItemsSearchCollector collector,
|
||||
private void collectStreamsFrom(final MultiInfoItemsCollector collector,
|
||||
final JsonArray contents) throws NothingFoundException,
|
||||
ParsingException {
|
||||
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
||||
|
||||
@ -10,6 +10,7 @@ import org.mozilla.javascript.ScriptableObject;
|
||||
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.MetaInfo;
|
||||
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
|
||||
@ -618,7 +619,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public StreamInfoItemsCollector getRelatedItems() throws ExtractionException {
|
||||
public MultiInfoItemsCollector getRelatedItems() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
|
||||
if (getAgeLimit() != NO_AGE_LIMIT) {
|
||||
@ -626,8 +627,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
|
||||
try {
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(
|
||||
getServiceId());
|
||||
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
|
||||
|
||||
final JsonArray results = nextResponse.getObject("contents")
|
||||
.getObject("twoColumnWatchNextResults").getObject("secondaryResults")
|
||||
@ -635,10 +635,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
|
||||
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
||||
|
||||
for (final Object ul : results) {
|
||||
if (((JsonObject) ul).has("compactVideoRenderer")) {
|
||||
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) ul)
|
||||
.getObject("compactVideoRenderer"), timeAgoParser));
|
||||
for (final Object resultObject : results) {
|
||||
final JsonObject result = (JsonObject) resultObject;
|
||||
if (result.has("compactVideoRenderer")) {
|
||||
collector.commit(new YoutubeStreamInfoItemExtractor(
|
||||
result.getObject("compactVideoRenderer"), timeAgoParser));
|
||||
} else if (result.has("compactRadioRenderer")) {
|
||||
collector.commit(new YoutubeMixOrPlaylistInfoItemExtractor(
|
||||
result.getObject("compactRadioRenderer")));
|
||||
} else if (result.has("compactPlaylistRenderer")) {
|
||||
collector.commit(new YoutubeMixOrPlaylistInfoItemExtractor(
|
||||
result.getObject("compactPlaylistRenderer")));
|
||||
}
|
||||
}
|
||||
return collector;
|
||||
|
||||
@ -252,15 +252,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
||||
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
try {
|
||||
// TODO: Don't simply get the first item, but look at all thumbnails and their resolution
|
||||
String url = videoInfo.getObject("thumbnail").getArray("thumbnails")
|
||||
.getObject(0).getString("url");
|
||||
|
||||
return fixThumbnailUrl(url);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get thumbnail url", e);
|
||||
}
|
||||
return getThumbnailUrlFromInfoItem(videoInfo);
|
||||
}
|
||||
|
||||
private boolean isPremium() {
|
||||
|
||||
@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.schabi.newpipe.downloader.DownloaderFactory;
|
||||
import org.schabi.newpipe.extractor.ExtractorAsserts;
|
||||
@ -11,6 +10,8 @@ import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
|
||||
@ -34,8 +35,8 @@ public class YoutubeMixPlaylistExtractorTest {
|
||||
private static YoutubeMixPlaylistExtractor extractor;
|
||||
|
||||
public static class Mix {
|
||||
private static final String VIDEO_ID = "QMVCAPd5cwBcg";
|
||||
private static final String VIDEO_TITLE = "Mix – ";
|
||||
private static final String VIDEO_ID = "UtF6Jej8yb4";
|
||||
private static final String VIDEO_TITLE = "Avicii - The Nights";
|
||||
|
||||
@BeforeAll
|
||||
public static void setUp() throws Exception {
|
||||
@ -118,14 +119,19 @@ public class YoutubeMixPlaylistExtractorTest {
|
||||
void getStreamCount() {
|
||||
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getPlaylistType() throws ParsingException {
|
||||
assertEquals(PlaylistInfo.PlaylistType.MIX_STREAM, extractor.getPlaylistType());
|
||||
}
|
||||
}
|
||||
|
||||
public static class MixWithIndex {
|
||||
|
||||
private static final String VIDEO_ID = "QMVCAPd5cwBcg";
|
||||
private static final String VIDEO_TITLE = "Mix – ";
|
||||
private static final String VIDEO_ID = "UtF6Jej8yb4";
|
||||
private static final String VIDEO_TITLE = "Avicii - The Nights";
|
||||
private static final int INDEX = 4;
|
||||
private static final String VIDEO_ID_NUMBER_4 = "lWA2pjMjpBs";
|
||||
private static final String VIDEO_ID_NUMBER_4 = "ebXbLfLACGM";
|
||||
|
||||
@BeforeAll
|
||||
public static void setUp() throws Exception {
|
||||
@ -203,6 +209,11 @@ public class YoutubeMixPlaylistExtractorTest {
|
||||
void getStreamCount() {
|
||||
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getPlaylistType() throws ParsingException {
|
||||
assertEquals(PlaylistInfo.PlaylistType.MIX_STREAM, extractor.getPlaylistType());
|
||||
}
|
||||
}
|
||||
|
||||
public static class MyMix {
|
||||
@ -287,6 +298,11 @@ public class YoutubeMixPlaylistExtractorTest {
|
||||
void getStreamCount() {
|
||||
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getPlaylistType() throws ParsingException {
|
||||
assertEquals(PlaylistInfo.PlaylistType.MIX_STREAM, extractor.getPlaylistType());
|
||||
}
|
||||
}
|
||||
|
||||
public static class Invalid {
|
||||
@ -381,5 +397,100 @@ public class YoutubeMixPlaylistExtractorTest {
|
||||
void getStreamCount() {
|
||||
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getPlaylistType() throws ParsingException {
|
||||
assertEquals(PlaylistInfo.PlaylistType.MIX_CHANNEL, extractor.getPlaylistType());
|
||||
}
|
||||
}
|
||||
|
||||
public static class GenreMix {
|
||||
private static final String VIDEO_ID = "kINJeTNFbpg";
|
||||
private static final String MIX_TITLE = "Mix – Electronic music";
|
||||
|
||||
@BeforeAll
|
||||
public static void setUp() throws Exception {
|
||||
YoutubeParsingHelper.resetClientVersionAndKey();
|
||||
YoutubeParsingHelper.setNumberGenerator(new Random(1));
|
||||
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "genreMix"));
|
||||
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
|
||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
|
||||
+ "&list=RDGMEMYH9CUrFO7CfLJpaD7UR85w");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Test
|
||||
void getServiceId() {
|
||||
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getName() throws Exception {
|
||||
assertEquals(MIX_TITLE, extractor.getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getThumbnailUrl() throws Exception {
|
||||
final String thumbnailUrl = extractor.getThumbnailUrl();
|
||||
assertIsSecureUrl(thumbnailUrl);
|
||||
ExtractorAsserts.assertContains("yt", thumbnailUrl);
|
||||
ExtractorAsserts.assertContains(VIDEO_ID, thumbnailUrl);
|
||||
}
|
||||
|
||||
@Test
|
||||
void getInitialPage() throws Exception {
|
||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
|
||||
assertFalse(streams.getItems().isEmpty());
|
||||
assertTrue(streams.hasNextPage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getPage() throws Exception {
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
|
||||
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
|
||||
.value("videoId", VIDEO_ID)
|
||||
.value("playlistId", "RD" + VIDEO_ID)
|
||||
.value("params", "OAE%3D")
|
||||
.done())
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
||||
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
|
||||
assertFalse(streams.getItems().isEmpty());
|
||||
assertTrue(streams.hasNextPage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getContinuations() throws Exception {
|
||||
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
|
||||
final Set<String> urls = new HashSet<>();
|
||||
|
||||
// Should work infinitely, but for testing purposes only 3 times
|
||||
for (int i = 0; i < 3; i++) {
|
||||
assertTrue(streams.hasNextPage());
|
||||
assertFalse(streams.getItems().isEmpty());
|
||||
|
||||
for (final StreamInfoItem item : streams.getItems()) {
|
||||
// TODO Duplicates are appearing
|
||||
// assertFalse(urls.contains(item.getUrl()));
|
||||
urls.add(item.getUrl());
|
||||
}
|
||||
|
||||
streams = extractor.getPage(streams.getNextPage());
|
||||
}
|
||||
assertTrue(streams.hasNextPage());
|
||||
assertFalse(streams.getItems().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getStreamCount() {
|
||||
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getPlaylistType() throws ParsingException {
|
||||
assertEquals(PlaylistInfo.PlaylistType.MIX_GENRE, extractor.getPlaylistType());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,9 +8,9 @@ import org.schabi.newpipe.extractor.ExtractorAsserts;
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
|
||||
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
@ -162,6 +162,11 @@ public class YoutubePlaylistExtractorTest {
|
||||
public void testUploaderVerified() throws Exception {
|
||||
assertFalse(extractor.isUploaderVerified());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getPlaylistType() throws ParsingException {
|
||||
assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
|
||||
}
|
||||
}
|
||||
|
||||
public static class HugePlaylist implements BasePlaylistExtractorTest {
|
||||
@ -281,6 +286,11 @@ public class YoutubePlaylistExtractorTest {
|
||||
public void testUploaderVerified() throws Exception {
|
||||
assertTrue(extractor.isUploaderVerified());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getPlaylistType() throws ParsingException {
|
||||
assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
|
||||
}
|
||||
}
|
||||
|
||||
public static class LearningPlaylist implements BasePlaylistExtractorTest {
|
||||
@ -386,6 +396,11 @@ public class YoutubePlaylistExtractorTest {
|
||||
public void testUploaderVerified() throws Exception {
|
||||
assertTrue(extractor.isUploaderVerified());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getPlaylistType() throws ParsingException {
|
||||
assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
|
||||
}
|
||||
}
|
||||
|
||||
public static class ContinuationsTests {
|
||||
|
||||
@ -0,0 +1,130 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.stream;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.stream.YoutubeStreamExtractorDefaultTest.YOUTUBE_LICENCE;
|
||||
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.schabi.newpipe.downloader.DownloaderFactory;
|
||||
import org.schabi.newpipe.downloader.MockOnly;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfo.PlaylistType;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
|
||||
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Random;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
public class YoutubeStreamExtractorRelatedMixTest extends DefaultStreamExtractorTest {
|
||||
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
|
||||
static final String ID = "K4DyBUG242c";
|
||||
static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID;
|
||||
static final String TITLE = "Cartoon - On & On (feat. Daniel Levi) [NCS Release]";
|
||||
private static StreamExtractor extractor;
|
||||
|
||||
@BeforeAll
|
||||
public static void setUp() throws Exception {
|
||||
YoutubeParsingHelper.resetClientVersionAndKey();
|
||||
YoutubeParsingHelper.setNumberGenerator(new Random(1));
|
||||
YoutubeStreamExtractor.resetDeobfuscationCode();
|
||||
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "relatedMix"));
|
||||
extractor = YouTube.getStreamExtractor(URL);
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
// @formatter:off
|
||||
@Override public StreamExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return YouTube; }
|
||||
@Override public String expectedName() { return TITLE; }
|
||||
@Override public String expectedId() { return ID; }
|
||||
@Override public String expectedUrlContains() { return URL; }
|
||||
@Override public String expectedOriginalUrlContains() { return URL; }
|
||||
|
||||
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
|
||||
@Override public String expectedUploaderName() { return "NoCopyrightSounds"; }
|
||||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg"; }
|
||||
@Override public List<String> expectedDescriptionContains() {
|
||||
return Arrays.asList("https://www.youtube.com/user/danielleviband/", "©");
|
||||
}
|
||||
@Override public boolean expectedUploaderVerified() { return true; }
|
||||
@Override public long expectedUploaderSubscriberCountAtLeast() { return 32_000_000; }
|
||||
@Override public long expectedLength() { return 208; }
|
||||
@Override public long expectedTimestamp() { return 0; }
|
||||
@Override public long expectedViewCountAtLeast() { return 449_000_000; }
|
||||
@Nullable @Override public String expectedUploadDate() { return "2015-07-09 00:00:00.000"; }
|
||||
@Nullable @Override public String expectedTextualUploadDate() { return "2015-07-09"; }
|
||||
@Override public long expectedLikeCountAtLeast() { return 6_400_000; }
|
||||
@Override public long expectedDislikeCountAtLeast() { return -1; }
|
||||
@Override public boolean expectedHasSubtitles() { return true; }
|
||||
@Override public int expectedStreamSegmentsCount() { return 0; }
|
||||
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
|
||||
@Override public String expectedCategory() { return "Music"; }
|
||||
@Override public List<String> expectedTags() {
|
||||
return Arrays.asList("Cartoon", "Cartoon - On & On", "Cartoon Baboon",
|
||||
"Cartoon NCS Release", "Cartoon On & On (feat. Daniel Levi)", "Cartoon best songs",
|
||||
"Copyright Free Music", "Daniel Levi", "NCS", "NCS Best Songs",
|
||||
"NCS Cartoon Daniel Levi", "NCS Cartoon On & On", "NCS On & On", "NCS On and On",
|
||||
"NCS Release", "NCS Release Daniel Levi", "NCS release Cartoon", "Official",
|
||||
"On & On", "On & On NCS", "On and on", "Royalty Free Cartoon", "Royalty Free Music",
|
||||
"electronic", "no copyright sounds", "nocopyrightsounds", "on & on lyrics",
|
||||
"on and on lyrics");
|
||||
}
|
||||
// @formatter:on
|
||||
|
||||
@Test
|
||||
@MockOnly // related items keep changing, and so do the mixes contained within them
|
||||
@Override
|
||||
public void testRelatedItems() throws Exception {
|
||||
super.testRelatedItems();
|
||||
|
||||
final List<PlaylistInfoItem> playlists = Objects.requireNonNull(extractor.getRelatedItems())
|
||||
.getItems()
|
||||
.stream()
|
||||
.filter(PlaylistInfoItem.class::isInstance)
|
||||
.map(PlaylistInfoItem.class::cast)
|
||||
.collect(Collectors.toList());
|
||||
playlists.forEach(item -> assertNotEquals(PlaylistType.NORMAL, item.getPlaylistType(),
|
||||
"Unexpected normal playlist in related items"));
|
||||
|
||||
final List<PlaylistInfoItem> streamMixes = playlists.stream()
|
||||
.filter(item -> item.getPlaylistType().equals(PlaylistType.MIX_STREAM))
|
||||
.collect(Collectors.toList());
|
||||
assertEquals(1, streamMixes.size(), "Not found exactly one stream mix in related items");
|
||||
|
||||
final PlaylistInfoItem streamMix = streamMixes.get(0);
|
||||
assertSame(InfoItem.InfoType.PLAYLIST, streamMix.getInfoType());
|
||||
assertEquals(YouTube.getServiceId(), streamMix.getServiceId());
|
||||
assertContains(URL, streamMix.getUrl());
|
||||
assertContains("list=RD" + ID, streamMix.getUrl());
|
||||
assertEquals("Mix – " + TITLE, streamMix.getName());
|
||||
assertIsSecureUrl(streamMix.getThumbnailUrl());
|
||||
|
||||
final List<PlaylistInfoItem> musicMixes = playlists.stream()
|
||||
.filter(item -> item.getPlaylistType().equals(PlaylistType.MIX_MUSIC))
|
||||
.collect(Collectors.toList());
|
||||
assertEquals(1, musicMixes.size(), "Not found exactly one music mix in related items");
|
||||
|
||||
final PlaylistInfoItem musicMix = musicMixes.get(0);
|
||||
assertSame(InfoItem.InfoType.PLAYLIST, musicMix.getInfoType());
|
||||
assertEquals(YouTube.getServiceId(), musicMix.getServiceId());
|
||||
assertContains("list=RDCLAK", musicMix.getUrl());
|
||||
assertEquals("Hip Hop Essentials", musicMix.getName());
|
||||
assertIsSecureUrl(musicMix.getThumbnailUrl());
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -0,0 +1,73 @@
|
||||
{
|
||||
"request": {
|
||||
"httpMethod": "GET",
|
||||
"url": "https://www.youtube.com/iframe_api",
|
||||
"headers": {
|
||||
"Accept-Language": [
|
||||
"en-GB, en;q\u003d0.9"
|
||||
]
|
||||
},
|
||||
"localization": {
|
||||
"languageCode": "en",
|
||||
"countryCode": "GB"
|
||||
}
|
||||
},
|
||||
"response": {
|
||||
"responseCode": 200,
|
||||
"responseMessage": "",
|
||||
"responseHeaders": {
|
||||
"alt-svc": [
|
||||
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
|
||||
],
|
||||
"cache-control": [
|
||||
"private, max-age\u003d0"
|
||||
],
|
||||
"content-type": [
|
||||
"text/javascript; charset\u003dutf-8"
|
||||
],
|
||||
"cross-origin-opener-policy-report-only": [
|
||||
"same-origin; report-to\u003d\"ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\""
|
||||
],
|
||||
"cross-origin-resource-policy": [
|
||||
"cross-origin"
|
||||
],
|
||||
"date": [
|
||||
"Mon, 28 Feb 2022 18:41:20 GMT"
|
||||
],
|
||||
"expires": [
|
||||
"Mon, 28 Feb 2022 18:41:20 GMT"
|
||||
],
|
||||
"p3p": [
|
||||
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
|
||||
],
|
||||
"permissions-policy": [
|
||||
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
|
||||
],
|
||||
"report-to": [
|
||||
"{\"group\":\"ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/encsid_ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\"}]}"
|
||||
],
|
||||
"server": [
|
||||
"ESF"
|
||||
],
|
||||
"set-cookie": [
|
||||
"YSC\u003dRRetMyo289w; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
|
||||
"VISITOR_INFO1_LIVE\u003d65TC-XopBj0; Domain\u003d.youtube.com; Expires\u003dSat, 27-Aug-2022 18:41:20 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
|
||||
"CONSENT\u003dPENDING+472; expires\u003dWed, 28-Feb-2024 18:41:20 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
|
||||
],
|
||||
"strict-transport-security": [
|
||||
"max-age\u003d31536000"
|
||||
],
|
||||
"x-content-type-options": [
|
||||
"nosniff"
|
||||
],
|
||||
"x-frame-options": [
|
||||
"SAMEORIGIN"
|
||||
],
|
||||
"x-xss-protection": [
|
||||
"0"
|
||||
]
|
||||
},
|
||||
"responseBody": "var scriptUrl \u003d \u0027https:\\/\\/www.youtube.com\\/s\\/player\\/450209b9\\/www-widgetapi.vflset\\/www-widgetapi.js\u0027;try{var ttPolicy\u003dwindow.trustedTypes.createPolicy(\"youtube-widget-api\",{createScriptURL:function(x){return x}});scriptUrl\u003dttPolicy.createScriptURL(scriptUrl)}catch(e){}if(!window[\"YT\"])var YT\u003d{loading:0,loaded:0};if(!window[\"YTConfig\"])var YTConfig\u003d{\"host\":\"https://www.youtube.com\"};\nif(!YT.loading){YT.loading\u003d1;(function(){var l\u003d[];YT.ready\u003dfunction(f){if(YT.loaded)f();else l.push(f)};window.onYTReady\u003dfunction(){YT.loaded\u003d1;for(var i\u003d0;i\u003cl.length;i++)try{l[i]()}catch(e$0){}};YT.setConfig\u003dfunction(c){for(var k in c)if(c.hasOwnProperty(k))YTConfig[k]\u003dc[k]};var a\u003ddocument.createElement(\"script\");a.type\u003d\"text/javascript\";a.id\u003d\"www-widgetapi-script\";a.src\u003dscriptUrl;a.async\u003dtrue;var c\u003ddocument.currentScript;if(c){var n\u003dc.nonce||c.getAttribute(\"nonce\");if(n)a.setAttribute(\"nonce\",n)}var b\u003d\ndocument.getElementsByTagName(\"script\")[0];b.parentNode.insertBefore(a,b)})()};\n",
|
||||
"latestUrl": "https://www.youtube.com/iframe_api"
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue
Block a user