mirror of
https://github.com/TeamNewPipe/NewPipeExtractor.git
synced 2025-04-28 16:00:33 +05:30
[YouTube] Apply changes in Extractors except YoutubeMusicSearchExtractor
Also improve a bit some code related to the changes.
This commit is contained in:
parent
4cc99f9ce1
commit
c1981ed54f
@ -1,3 +1,23 @@
|
||||
/*
|
||||
* Created by Christian Schabesberger on 25.07.16.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
|
||||
* YoutubeChannelExtractor.java is part of NewPipe Extractor.
|
||||
*
|
||||
* NewPipe Extractor is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe Extractor is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
|
||||
@ -8,6 +28,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
|
||||
import org.schabi.newpipe.extractor.Image;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
|
||||
@ -36,26 +57,6 @@ import java.util.stream.Collectors;
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 25.07.16.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
|
||||
* YoutubeChannelExtractor.java is part of NewPipe.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
|
||||
private JsonObject jsonResponse;
|
||||
@ -190,16 +191,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
.orElseThrow(() -> new ParsingException("Could not get channel name"));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getAvatarUrl() throws ParsingException {
|
||||
public List<Image> getAvatars() throws ParsingException {
|
||||
assertPageFetched();
|
||||
if (channelAgeGateRenderer != null) {
|
||||
return Optional.ofNullable(channelAgeGateRenderer.getObject("avatar")
|
||||
.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getString("url"))
|
||||
.map(YoutubeParsingHelper::fixThumbnailUrl)
|
||||
.orElseThrow(() -> new ParsingException("Could not get avatar URL"));
|
||||
.getArray("thumbnails"))
|
||||
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
|
||||
.orElseThrow(() -> new ParsingException("Could not get avatars"));
|
||||
}
|
||||
|
||||
return channelHeader.map(header -> {
|
||||
@ -210,56 +210,37 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
.getObject("image")
|
||||
.getObject("contentPreviewImageViewModel")
|
||||
.getObject("image")
|
||||
.getArray("sources")
|
||||
.getObject(0)
|
||||
.getString("url");
|
||||
.getArray("sources");
|
||||
|
||||
case INTERACTIVE_TABBED:
|
||||
return header.json.getObject("boxArt")
|
||||
.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getString("url");
|
||||
.getArray("thumbnails");
|
||||
|
||||
case C4_TABBED:
|
||||
case CAROUSEL:
|
||||
default:
|
||||
return header.json.getObject("avatar")
|
||||
.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getString("url");
|
||||
.getArray("thumbnails");
|
||||
}
|
||||
})
|
||||
.map(YoutubeParsingHelper::fixThumbnailUrl)
|
||||
.orElseThrow(() -> new ParsingException("Could not get avatar URL"));
|
||||
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
|
||||
.orElseThrow(() -> new ParsingException("Could not get avatars"));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getBannerUrl() throws ParsingException {
|
||||
public List<Image> getBanners() {
|
||||
assertPageFetched();
|
||||
if (channelAgeGateRenderer != null) {
|
||||
return null;
|
||||
return List.of();
|
||||
}
|
||||
|
||||
if (channelHeader.isPresent()) {
|
||||
final ChannelHeader header = channelHeader.get();
|
||||
if (header.headerType == HeaderType.PAGE) {
|
||||
// No banner is available on pageHeaderRenderer headers
|
||||
return null;
|
||||
}
|
||||
|
||||
return Optional.ofNullable(header.json.getObject("banner")
|
||||
.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getString("url"))
|
||||
.filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
|
||||
.map(YoutubeParsingHelper::fixThumbnailUrl)
|
||||
// Channels may not have a banner, so no exception should be thrown if no
|
||||
// banner is found
|
||||
// Return null in this case
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
return null;
|
||||
return channelHeader.filter(header -> header.headerType != HeaderType.PAGE)
|
||||
.map(header -> header.json.getObject("banner")
|
||||
.getArray("thumbnails"))
|
||||
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
|
||||
.orElse(List.of());
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -359,9 +340,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getParentChannelAvatarUrl() {
|
||||
return "";
|
||||
public List<Image> getParentChannelAvatars() {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -17,6 +17,8 @@ import com.grack.nanojson.JsonBuilder;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import org.schabi.newpipe.extractor.Image;
|
||||
import org.schabi.newpipe.extractor.Image.ResolutionLevel;
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
@ -34,6 +36,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.utils.ImageSuffix;
|
||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
@ -43,6 +46,7 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
@ -53,6 +57,12 @@ import javax.annotation.Nullable;
|
||||
* {@code youtube.com/watch?v=videoId&list=playlistId}
|
||||
*/
|
||||
public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
||||
private static final List<ImageSuffix> IMAGE_URL_SUFFIXES_AND_RESOLUTIONS = List.of(
|
||||
// sqdefault and maxresdefault image resolutions are not available on all
|
||||
// videos, so don't add them in the list of available resolutions
|
||||
new ImageSuffix("default.jpg", 90, 120, ResolutionLevel.LOW),
|
||||
new ImageSuffix("mqdefault.jpg", 180, 320, ResolutionLevel.MEDIUM),
|
||||
new ImageSuffix("hqdefault.jpg", 360, 480, ResolutionLevel.MEDIUM));
|
||||
|
||||
/**
|
||||
* YouTube identifies mixes based on this cookie. With this information it can generate
|
||||
@ -126,18 +136,18 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
public List<Image> getThumbnails() throws ParsingException {
|
||||
try {
|
||||
return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId"));
|
||||
return getThumbnailsFromPlaylistId(playlistData.getString("playlistId"));
|
||||
} catch (final Exception e) {
|
||||
try {
|
||||
// Fallback to thumbnail of current video. Always the case for channel mix
|
||||
return getThumbnailUrlFromVideoId(initialData.getObject("currentVideoEndpoint")
|
||||
// Fallback to thumbnail of current video. Always the case for channel mixes
|
||||
return getThumbnailsFromVideoId(initialData.getObject("currentVideoEndpoint")
|
||||
.getObject("watchEndpoint").getString("videoId"));
|
||||
} catch (final Exception ignored) {
|
||||
}
|
||||
|
||||
throw new ParsingException("Could not get playlist thumbnail", e);
|
||||
throw new ParsingException("Could not get playlist thumbnails", e);
|
||||
}
|
||||
}
|
||||
|
||||
@ -153,10 +163,11 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
||||
return "YouTube";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getUploaderAvatarUrl() {
|
||||
public List<Image> getUploaderAvatars() {
|
||||
// YouTube mixes are auto-generated by YouTube
|
||||
return "";
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -264,14 +275,19 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId)
|
||||
private List<Image> getThumbnailsFromPlaylistId(@Nonnull final String playlistId)
|
||||
throws ParsingException {
|
||||
return getThumbnailUrlFromVideoId(YoutubeParsingHelper.extractVideoIdFromMixId(playlistId));
|
||||
return getThumbnailsFromVideoId(YoutubeParsingHelper.extractVideoIdFromMixId(playlistId));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String getThumbnailUrlFromVideoId(final String videoId) {
|
||||
return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
|
||||
private List<Image> getThumbnailsFromVideoId(@Nonnull final String videoId) {
|
||||
final String baseUrl = "https://i.ytimg.com/vi/" + videoId + "/";
|
||||
return IMAGE_URL_SUFFIXES_AND_RESOLUTIONS.stream()
|
||||
.map(imageSuffix -> new Image(baseUrl + imageSuffix.getSuffix(),
|
||||
imageSuffix.getHeight(), imageSuffix.getWidth(),
|
||||
imageSuffix.getResolutionLevel()))
|
||||
.collect(Collectors.toUnmodifiableList());
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
@ -3,10 +3,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
@ -15,6 +15,7 @@ import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import org.schabi.newpipe.extractor.Image;
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
@ -33,6 +34,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
@ -160,39 +162,35 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
String url;
|
||||
public List<Image> getThumbnails() throws ParsingException {
|
||||
final JsonArray playlistMetadataThumbnailsArray;
|
||||
if (isNewPlaylistInterface) {
|
||||
url = getPlaylistHeader().getObject("playlistHeaderBanner")
|
||||
playlistMetadataThumbnailsArray = getPlaylistHeader().getObject("playlistHeaderBanner")
|
||||
.getObject("heroPlaylistThumbnailRenderer")
|
||||
.getObject("thumbnail")
|
||||
.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getString("url");
|
||||
.getArray("thumbnails");
|
||||
} else {
|
||||
url = getPlaylistInfo().getObject("thumbnailRenderer")
|
||||
playlistMetadataThumbnailsArray = playlistInfo.getObject("thumbnailRenderer")
|
||||
.getObject("playlistVideoThumbnailRenderer")
|
||||
.getObject("thumbnail")
|
||||
.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getString("url");
|
||||
.getArray("thumbnails");
|
||||
}
|
||||
|
||||
if (!isNullOrEmpty(playlistMetadataThumbnailsArray)) {
|
||||
return getImagesFromThumbnailsArray(playlistMetadataThumbnailsArray);
|
||||
}
|
||||
|
||||
// This data structure is returned in both layouts
|
||||
if (isNullOrEmpty(url)) {
|
||||
url = browseResponse.getObject("microformat")
|
||||
final JsonArray microFormatThumbnailsArray = browseResponse.getObject("microformat")
|
||||
.getObject("microformatDataRenderer")
|
||||
.getObject("thumbnail")
|
||||
.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getString("url");
|
||||
.getArray("thumbnails");
|
||||
|
||||
if (isNullOrEmpty(url)) {
|
||||
throw new ParsingException("Could not get playlist thumbnail");
|
||||
}
|
||||
if (!isNullOrEmpty(microFormatThumbnailsArray)) {
|
||||
return getImagesFromThumbnailsArray(microFormatThumbnailsArray);
|
||||
}
|
||||
|
||||
return fixThumbnailUrl(url);
|
||||
throw new ParsingException("Could not get playlist thumbnails");
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -220,23 +218,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getUploaderAvatarUrl() throws ParsingException {
|
||||
public List<Image> getUploaderAvatars() throws ParsingException {
|
||||
if (isNewPlaylistInterface) {
|
||||
// The new playlist interface doesn't provide an uploader avatar
|
||||
return "";
|
||||
return List.of();
|
||||
}
|
||||
|
||||
try {
|
||||
final String url = getUploaderInfo()
|
||||
.getObject("thumbnail")
|
||||
.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getString("url");
|
||||
|
||||
return fixThumbnailUrl(url);
|
||||
return getImagesFromThumbnailsArray(getUploaderInfo().getObject("thumbnail")
|
||||
.getArray("thumbnails"));
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get playlist uploader avatar", e);
|
||||
throw new ParsingException("Could not get playlist uploader avatars", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -30,11 +30,12 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAttributedDescription;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAttributedDescription;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder;
|
||||
@ -47,6 +48,7 @@ import com.grack.nanojson.JsonWriter;
|
||||
import org.mozilla.javascript.Context;
|
||||
import org.mozilla.javascript.Function;
|
||||
import org.mozilla.javascript.ScriptableObject;
|
||||
import org.schabi.newpipe.extractor.Image;
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.MetaInfo;
|
||||
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
|
||||
@ -258,23 +260,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
public List<Image> getThumbnails() throws ParsingException {
|
||||
assertPageFetched();
|
||||
try {
|
||||
final JsonArray thumbnails = playerResponse
|
||||
.getObject("videoDetails")
|
||||
return getImagesFromThumbnailsArray(playerResponse.getObject("videoDetails")
|
||||
.getObject("thumbnail")
|
||||
.getArray("thumbnails");
|
||||
// the last thumbnail is the one with the highest resolution
|
||||
final String url = thumbnails
|
||||
.getObject(thumbnails.size() - 1)
|
||||
.getString("url");
|
||||
|
||||
return fixThumbnailUrl(url);
|
||||
.getArray("thumbnails"));
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get thumbnail url");
|
||||
throw new ParsingException("Could not get thumbnails");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@ -552,26 +546,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getUploaderAvatarUrl() throws ParsingException {
|
||||
public List<Image> getUploaderAvatars() throws ParsingException {
|
||||
assertPageFetched();
|
||||
|
||||
final String url = getVideoSecondaryInfoRenderer()
|
||||
.getObject("owner")
|
||||
final List<Image> imageList = getImagesFromThumbnailsArray(
|
||||
getVideoSecondaryInfoRenderer().getObject("owner")
|
||||
.getObject("videoOwnerRenderer")
|
||||
.getObject("thumbnail")
|
||||
.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getString("url");
|
||||
.getArray("thumbnails"));
|
||||
|
||||
if (isNullOrEmpty(url)) {
|
||||
if (ageLimit == NO_AGE_LIMIT) {
|
||||
throw new ParsingException("Could not get uploader avatar URL");
|
||||
if (imageList.isEmpty() && ageLimit == NO_AGE_LIMIT) {
|
||||
throw new ParsingException("Could not get uploader avatars");
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
return fixThumbnailUrl(url);
|
||||
return imageList;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
Loading…
x
Reference in New Issue
Block a user