mirror of
https://github.com/TeamNewPipe/NewPipeExtractor.git
synced 2024-12-14 22:30:33 +05:30
[Youtube] apply wb9688 suggestion (mix)
Channel mix adjusments and test Don't accept youtube music mix urls as playlist Don't override playlistData to keep getInitialPage() Remove json constants Indentation
This commit is contained in:
parent
822cf307f7
commit
3ff8619bcc
@ -193,13 +193,23 @@ public class YoutubeParsingHelper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Checks if the given playlist id is a mix (auto-generated playlist)
|
* Checks if the given playlist id is a youtube mix (auto-generated playlist)
|
||||||
* Ids from a mix start with "RD"
|
* Ids from a youtube mix start with "RD"
|
||||||
* @param playlistId
|
* @param playlistId
|
||||||
* @return Whether given id belongs to a mix
|
* @return Whether given id belongs to a youtube mix
|
||||||
*/
|
*/
|
||||||
public static boolean isYoutubeMixId(String playlistId) {
|
public static boolean isYoutubeMixId(String playlistId) {
|
||||||
return playlistId.startsWith("RD");
|
return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks if the given playlist id is a youtube music mix (auto-generated playlist)
|
||||||
|
* Ids from a youtube music mix start with "RD"
|
||||||
|
* @param playlistId
|
||||||
|
* @return Whether given id belongs to a youtube music mix
|
||||||
|
*/
|
||||||
|
public static boolean isYoutubeMusicMixId(String playlistId) {
|
||||||
|
return playlistId.startsWith("RDAMVM");
|
||||||
}
|
}
|
||||||
|
|
||||||
public static JsonObject getInitialData(String html) throws ParsingException {
|
public static JsonObject getInitialData(String html) throws ParsingException {
|
||||||
@ -427,9 +437,9 @@ public class YoutubeParsingHelper {
|
|||||||
StringBuilder url = new StringBuilder();
|
StringBuilder url = new StringBuilder();
|
||||||
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
|
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
|
||||||
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
|
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
|
||||||
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
|
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
|
||||||
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
|
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
|
||||||
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
|
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
|
||||||
return url.toString();
|
return url.toString();
|
||||||
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
|
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
|
||||||
return "https://www.youtube.com/playlist?list=" +
|
return "https://www.youtube.com/playlist?list=" +
|
||||||
@ -457,6 +467,7 @@ public class YoutubeParsingHelper {
|
|||||||
if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
|
if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
|
||||||
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
|
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
|
||||||
if (!isNullOrEmpty(url)) {
|
if (!isNullOrEmpty(url)) {
|
||||||
|
url = url.replaceAll("&", "&");
|
||||||
textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
|
textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||||
|
|
||||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
|
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
|
||||||
|
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
|
||||||
|
|
||||||
import com.grack.nanojson.JsonArray;
|
import com.grack.nanojson.JsonArray;
|
||||||
import com.grack.nanojson.JsonObject;
|
import com.grack.nanojson.JsonObject;
|
||||||
@ -25,13 +26,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
|||||||
*/
|
*/
|
||||||
public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
||||||
|
|
||||||
|
private JsonObject initialData;
|
||||||
private final static String CONTENTS = "contents";
|
|
||||||
private final static String RESPONSE = "response";
|
|
||||||
private final static String PLAYLIST = "playlist";
|
|
||||||
private final static String TWO_COLUMN_WATCH_NEXT_RESULTS = "twoColumnWatchNextResults";
|
|
||||||
private final static String PLAYLIST_PANEL_VIDEO_RENDERER = "playlistPanelVideoRenderer";
|
|
||||||
|
|
||||||
private JsonObject playlistData;
|
private JsonObject playlistData;
|
||||||
|
|
||||||
public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
|
public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
|
||||||
@ -43,9 +38,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||||||
throws IOException, ExtractionException {
|
throws IOException, ExtractionException {
|
||||||
final String url = getUrl() + "&pbj=1";
|
final String url = getUrl() + "&pbj=1";
|
||||||
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
|
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
|
||||||
JsonObject initialData = ajaxJson.getObject(3).getObject(RESPONSE);
|
initialData = ajaxJson.getObject(3).getObject("response");
|
||||||
playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMN_WATCH_NEXT_RESULTS)
|
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
|
||||||
.getObject(PLAYLIST).getObject(PLAYLIST);
|
.getObject("playlist").getObject("playlist");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Nonnull
|
@Nonnull
|
||||||
@ -62,7 +57,14 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||||||
public String getThumbnailUrl() throws ParsingException {
|
public String getThumbnailUrl() throws ParsingException {
|
||||||
try {
|
try {
|
||||||
final String playlistId = playlistData.getString("playlistId");
|
final String playlistId = playlistData.getString("playlistId");
|
||||||
return getThumbnailUrlFromId(playlistId);
|
try {
|
||||||
|
return getThumbnailUrlFromPlaylistId(playlistId);
|
||||||
|
} catch (ParsingException e) {
|
||||||
|
//fallback to thumbnail of current video. Always the case for channel mix
|
||||||
|
return getThumbnailUrlFromVideoId(
|
||||||
|
initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint")
|
||||||
|
.getString("videoId"));
|
||||||
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new ParsingException("Could not get playlist thumbnail", e);
|
throw new ParsingException("Could not get playlist thumbnail", e);
|
||||||
}
|
}
|
||||||
@ -101,20 +103,26 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||||||
@Override
|
@Override
|
||||||
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
|
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
|
||||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||||
collectStreamsFrom(collector, playlistData.getArray(CONTENTS));
|
collectStreamsFrom(collector, playlistData.getArray("contents"));
|
||||||
return new InfoItemsPage<>(collector, getNextPageUrl());
|
return new InfoItemsPage<>(collector, getNextPageUrl());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getNextPageUrl() throws ExtractionException {
|
public String getNextPageUrl() throws ExtractionException {
|
||||||
final JsonObject lastStream = ((JsonObject) playlistData.getArray(CONTENTS)
|
return getNextPageUrlFrom(playlistData);
|
||||||
.get(playlistData.getArray(CONTENTS).size() - 1));
|
}
|
||||||
if (lastStream == null || lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) == null) {
|
|
||||||
|
private String getNextPageUrlFrom(JsonObject playlistData) throws ExtractionException {
|
||||||
|
final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents")
|
||||||
|
.get(playlistData.getArray("contents").size() - 1));
|
||||||
|
if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) {
|
||||||
throw new ExtractionException("Could not extract next page url");
|
throw new ExtractionException("Could not extract next page url");
|
||||||
}
|
}
|
||||||
return "https://youtube.com" + lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER)
|
//Index of video in mix is missing, but adding it doesn't appear to have any effect.
|
||||||
.getObject("navigationEndpoint").getObject("commandMetadata")
|
//And since the index needs to be tracked by us, it is left out
|
||||||
.getObject("webCommandMetadata").getString("url") + "&pbj=1";
|
return getUrlFromNavigationEndpoint(
|
||||||
|
lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint"))
|
||||||
|
+ "&pbj=1";
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -127,21 +135,20 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||||||
|
|
||||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||||
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
|
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
|
||||||
playlistData =
|
JsonObject playlistData =
|
||||||
ajaxJson.getObject(3).getObject(RESPONSE).getObject(CONTENTS)
|
ajaxJson.getObject(3).getObject("response").getObject("contents")
|
||||||
.getObject(TWO_COLUMN_WATCH_NEXT_RESULTS).getObject(PLAYLIST)
|
.getObject("twoColumnWatchNextResults").getObject("playlist")
|
||||||
.getObject(PLAYLIST);
|
.getObject("playlist");
|
||||||
final JsonArray streams = playlistData.getArray(CONTENTS);
|
final JsonArray streams = playlistData.getArray("contents");
|
||||||
//Because continuation requests are created with the last video of previous request as start
|
//Because continuation requests are created with the last video of previous request as start
|
||||||
streams.remove(0);
|
streams.remove(0);
|
||||||
collectStreamsFrom(collector, streams);
|
collectStreamsFrom(collector, streams);
|
||||||
return new InfoItemsPage<>(collector, getNextPageUrl());
|
return new InfoItemsPage<>(collector, getNextPageUrlFrom(playlistData));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void collectStreamsFrom(
|
private void collectStreamsFrom(
|
||||||
@Nonnull StreamInfoItemsCollector collector,
|
@Nonnull StreamInfoItemsCollector collector,
|
||||||
@Nullable JsonArray streams) {
|
@Nullable JsonArray streams) {
|
||||||
collector.reset();
|
|
||||||
|
|
||||||
if (streams == null) {
|
if (streams == null) {
|
||||||
return;
|
return;
|
||||||
@ -152,7 +159,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||||||
for (Object stream : streams) {
|
for (Object stream : streams) {
|
||||||
if (stream instanceof JsonObject) {
|
if (stream instanceof JsonObject) {
|
||||||
JsonObject streamInfo = ((JsonObject) stream)
|
JsonObject streamInfo = ((JsonObject) stream)
|
||||||
.getObject(PLAYLIST_PANEL_VIDEO_RENDERER);
|
.getObject("playlistPanelVideoRenderer");
|
||||||
if (streamInfo != null) {
|
if (streamInfo != null) {
|
||||||
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
|
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
|
||||||
}
|
}
|
||||||
@ -160,16 +167,22 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getThumbnailUrlFromId(String playlistId) throws ParsingException {
|
private String getThumbnailUrlFromPlaylistId(String playlistId) throws ParsingException {
|
||||||
final String videoId;
|
final String videoId;
|
||||||
if (playlistId.startsWith("RDMM")) {
|
if (playlistId.startsWith("RDMM")) {
|
||||||
videoId = playlistId.substring(4);
|
videoId = playlistId.substring(4);
|
||||||
|
} else if (playlistId.startsWith("RDCMUC")) {
|
||||||
|
throw new ParsingException("is channel mix");
|
||||||
} else {
|
} else {
|
||||||
videoId = playlistId.substring(2);
|
videoId = playlistId.substring(2);
|
||||||
}
|
}
|
||||||
if (videoId.isEmpty()) {
|
if (videoId.isEmpty()) {
|
||||||
throw new ParsingException("videoId is empty");
|
throw new ParsingException("videoId is empty");
|
||||||
}
|
}
|
||||||
|
return getThumbnailUrlFromVideoId(videoId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getThumbnailUrlFromVideoId(String videoId) {
|
||||||
return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
|
return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -64,11 +64,12 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
|
|||||||
@Override
|
@Override
|
||||||
public boolean onAcceptUrl(final String url) {
|
public boolean onAcceptUrl(final String url) {
|
||||||
try {
|
try {
|
||||||
getId(url);
|
String playlistId = getId(url);
|
||||||
|
//Because youtube music mix are not supported yet.
|
||||||
|
return !YoutubeParsingHelper.isYoutubeMusicMixId(playlistId);
|
||||||
} catch (ParsingException e) {
|
} catch (ParsingException e) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.ListExtractor;
|
|||||||
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
|
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
|
||||||
import org.schabi.newpipe.extractor.NewPipe;
|
import org.schabi.newpipe.extractor.NewPipe;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
|
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||||
|
|
||||||
@ -93,11 +94,6 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||||||
public void getStreamCount() throws Exception {
|
public void getStreamCount() throws Exception {
|
||||||
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
public void getStreamCount() throws Exception {
|
|
||||||
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class MixWithIndex {
|
public static class MixWithIndex {
|
||||||
@ -166,11 +162,6 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||||||
assertFalse(streams.getItems().isEmpty());
|
assertFalse(streams.getItems().isEmpty());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
public void getStreamCount() {
|
|
||||||
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void getStreamCount() throws Exception {
|
public void getStreamCount() throws Exception {
|
||||||
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
||||||
@ -264,12 +255,13 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||||||
extractor.getPage("");
|
extractor.getPage("");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = NullPointerException.class)
|
@Test(expected = ExtractionException.class)
|
||||||
public void invalidVideoId() throws Exception {
|
public void invalidVideoId() throws Exception {
|
||||||
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
extractor = (YoutubeMixPlaylistExtractor) YouTube
|
||||||
.getPlaylistExtractor(
|
.getPlaylistExtractor(
|
||||||
"https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
|
"https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
|
||||||
extractor.fetchPage();
|
extractor.fetchPage();
|
||||||
|
extractor.getName();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -329,10 +321,5 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||||||
public void getStreamCount() throws Exception {
|
public void getStreamCount() throws Exception {
|
||||||
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
public void getStreamCount() throws Exception {
|
|
||||||
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
Loading…
Reference in New Issue
Block a user