mirror of
https://github.com/TeamNewPipe/NewPipeExtractor.git
synced 2024-12-14 06:10:33 +05:30
Merge pull request #810 from TiA4f8R/delivery-methods-v2
Support delivery methods other than progressive HTTP
This commit is contained in:
commit
c8a77da2ab
@ -30,9 +30,12 @@ import java.util.List;
|
||||
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL;
|
||||
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
|
||||
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
|
||||
public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
|
||||
|
||||
private static final String OPUS_LO = "opus-lo";
|
||||
private static final String MP3_128 = "mp3-128";
|
||||
private JsonObject showInfo;
|
||||
|
||||
public BandcampRadioStreamExtractor(final StreamingService service,
|
||||
@ -116,23 +119,27 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() {
|
||||
final ArrayList<AudioStream> list = new ArrayList<>();
|
||||
final List<AudioStream> audioStreams = new ArrayList<>();
|
||||
final JsonObject streams = showInfo.getObject("audio_stream");
|
||||
|
||||
if (streams.has("opus-lo")) {
|
||||
list.add(new AudioStream(
|
||||
streams.getString("opus-lo"),
|
||||
MediaFormat.OPUS, 100
|
||||
));
|
||||
}
|
||||
if (streams.has("mp3-128")) {
|
||||
list.add(new AudioStream(
|
||||
streams.getString("mp3-128"),
|
||||
MediaFormat.MP3, 128
|
||||
));
|
||||
if (streams.has(MP3_128)) {
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId(MP3_128)
|
||||
.setContent(streams.getString(MP3_128), true)
|
||||
.setMediaFormat(MediaFormat.MP3)
|
||||
.setAverageBitrate(128)
|
||||
.build());
|
||||
}
|
||||
|
||||
return list;
|
||||
if (streams.has(OPUS_LO)) {
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId(OPUS_LO)
|
||||
.setContent(streams.getString(OPUS_LO), true)
|
||||
.setMediaFormat(MediaFormat.OPUS)
|
||||
.setAverageBitrate(100).build());
|
||||
}
|
||||
|
||||
return audioStreams;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@ -156,14 +163,14 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
|
||||
@Override
|
||||
public String getLicence() {
|
||||
// Contrary to other Bandcamp streams, radio streams don't have a license
|
||||
return "";
|
||||
return EMPTY_STRING;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getCategory() {
|
||||
// Contrary to other Bandcamp streams, radio streams don't have categories
|
||||
return "";
|
||||
return EMPTY_STRING;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
@ -3,6 +3,8 @@
|
||||
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
|
||||
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
@ -10,7 +12,6 @@ import com.grack.nanojson.JsonParserException;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
@ -27,16 +28,15 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class BandcampStreamExtractor extends StreamExtractor {
|
||||
|
||||
private JsonObject albumJson;
|
||||
private JsonObject current;
|
||||
private Document document;
|
||||
@ -88,7 +88,7 @@ public class BandcampStreamExtractor extends StreamExtractor {
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
final String[] parts = getUrl().split("/");
|
||||
// https: (/) (/) * .bandcamp.com (/) and leave out the rest
|
||||
return "https://" + parts[2] + "/";
|
||||
return HTTPS + parts[2] + "/";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@ -119,10 +119,10 @@ public class BandcampStreamExtractor extends StreamExtractor {
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
if (albumJson.isNull("art_id")) {
|
||||
return Utils.EMPTY_STRING;
|
||||
} else {
|
||||
return getImageUrl(albumJson.getLong("art_id"), true);
|
||||
return EMPTY_STRING;
|
||||
}
|
||||
|
||||
return getImageUrl(albumJson.getLong("art_id"), true);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@ -139,24 +139,26 @@ public class BandcampStreamExtractor extends StreamExtractor {
|
||||
public Description getDescription() {
|
||||
final String s = Utils.nonEmptyAndNullJoin(
|
||||
"\n\n",
|
||||
new String[]{
|
||||
new String[] {
|
||||
current.getString("about"),
|
||||
current.getString("lyrics"),
|
||||
current.getString("credits")
|
||||
}
|
||||
);
|
||||
});
|
||||
return new Description(s, Description.PLAIN_TEXT);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() {
|
||||
final List<AudioStream> audioStreams = new ArrayList<>();
|
||||
|
||||
audioStreams.add(new AudioStream(
|
||||
albumJson.getArray("trackinfo").getObject(0)
|
||||
.getObject("file").getString("mp3-128"),
|
||||
MediaFormat.MP3, 128
|
||||
));
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId("mp3-128")
|
||||
.setContent(albumJson.getArray("trackinfo")
|
||||
.getObject(0)
|
||||
.getObject("file")
|
||||
.getString("mp3-128"), true)
|
||||
.setMediaFormat(MediaFormat.MP3)
|
||||
.setAverageBitrate(128)
|
||||
.build());
|
||||
return audioStreams;
|
||||
}
|
||||
|
||||
@ -184,11 +186,11 @@ public class BandcampStreamExtractor extends StreamExtractor {
|
||||
@Override
|
||||
public PlaylistInfoItemsCollector getRelatedItems() {
|
||||
final PlaylistInfoItemsCollector collector = new PlaylistInfoItemsCollector(getServiceId());
|
||||
final Elements recommendedAlbums = document.getElementsByClass("recommended-album");
|
||||
document.getElementsByClass("recommended-album")
|
||||
.stream()
|
||||
.map(BandcampRelatedPlaylistInfoItemExtractor::new)
|
||||
.forEach(collector::commit);
|
||||
|
||||
for (final Element album : recommendedAlbums) {
|
||||
collector.commit(new BandcampRelatedPlaylistInfoItemExtractor(album));
|
||||
}
|
||||
return collector;
|
||||
}
|
||||
|
||||
@ -200,15 +202,17 @@ public class BandcampStreamExtractor extends StreamExtractor {
|
||||
.flatMap(element -> element.getElementsByClass("tag").stream())
|
||||
.map(Element::text)
|
||||
.findFirst()
|
||||
.orElse("");
|
||||
.orElse(EMPTY_STRING);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getLicence() {
|
||||
/* Tests resulted in this mapping of ints to licence:
|
||||
/*
|
||||
Tests resulted in this mapping of ints to licence:
|
||||
https://cloud.disroot.org/s/ZTWBxbQ9fKRmRWJ/preview (screenshot from a Bandcamp artist's
|
||||
account) */
|
||||
account)
|
||||
*/
|
||||
|
||||
switch (current.getInt("license_type")) {
|
||||
case 1:
|
||||
@ -233,14 +237,9 @@ public class BandcampStreamExtractor extends StreamExtractor {
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<String> getTags() {
|
||||
final Elements tagElements = document.getElementsByAttributeValue("itemprop", "keywords");
|
||||
|
||||
final List<String> tags = new ArrayList<>();
|
||||
|
||||
for (final Element e : tagElements) {
|
||||
tags.add(e.text());
|
||||
}
|
||||
|
||||
return tags;
|
||||
return document.getElementsByAttributeValue("itemprop", "keywords")
|
||||
.stream()
|
||||
.map(Element::text)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
|
@ -10,18 +10,30 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
|
||||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
|
||||
import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
|
||||
public class MediaCCCLiveStreamExtractor extends StreamExtractor {
|
||||
private static final String STREAMS = "streams";
|
||||
private static final String URLS = "urls";
|
||||
private static final String URL = "url";
|
||||
|
||||
private JsonObject conference = null;
|
||||
private String group = "";
|
||||
private JsonObject room = null;
|
||||
@ -34,19 +46,22 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
|
||||
@Override
|
||||
public void onFetchPage(@Nonnull final Downloader downloader)
|
||||
throws IOException, ExtractionException {
|
||||
final JsonArray doc =
|
||||
MediaCCCParsingHelper.getLiveStreams(downloader, getExtractorLocalization());
|
||||
// find correct room
|
||||
final JsonArray doc = MediaCCCParsingHelper.getLiveStreams(downloader,
|
||||
getExtractorLocalization());
|
||||
// Find the correct room
|
||||
for (int c = 0; c < doc.size(); c++) {
|
||||
conference = doc.getObject(c);
|
||||
final JsonArray groups = conference.getArray("groups");
|
||||
final JsonObject conferenceObject = doc.getObject(c);
|
||||
final JsonArray groups = conferenceObject.getArray("groups");
|
||||
for (int g = 0; g < groups.size(); g++) {
|
||||
group = groups.getObject(g).getString("group");
|
||||
final String groupObject = groups.getObject(g).getString("group");
|
||||
final JsonArray rooms = groups.getObject(g).getArray("rooms");
|
||||
for (int r = 0; r < rooms.size(); r++) {
|
||||
room = rooms.getObject(r);
|
||||
if (getId().equals(
|
||||
conference.getString("slug") + "/" + room.getString("slug"))) {
|
||||
final JsonObject roomObject = rooms.getObject(r);
|
||||
if (getId().equals(conferenceObject.getString("slug") + "/"
|
||||
+ roomObject.getString("slug"))) {
|
||||
conference = conferenceObject;
|
||||
group = groupObject;
|
||||
room = roomObject;
|
||||
return;
|
||||
}
|
||||
}
|
||||
@ -91,69 +106,155 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
|
||||
return conference.getString("conference");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the URL of the first DASH stream found.
|
||||
*
|
||||
* <p>
|
||||
* There can be several DASH streams, so the URL of the first one found is returned by this
|
||||
* method.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* You can find the other DASH video streams by using {@link #getVideoStreams()}
|
||||
* </p>
|
||||
*/
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getDashMpdUrl() throws ParsingException {
|
||||
return getManifestOfDeliveryMethodWanted("dash");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the URL of the first HLS stream found.
|
||||
*
|
||||
* <p>
|
||||
* There can be several HLS streams, so the URL of the first one found is returned by this
|
||||
* method.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* You can find the other HLS video streams by using {@link #getVideoStreams()}
|
||||
* </p>
|
||||
*/
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getHlsUrl() {
|
||||
// TODO: There are multiple HLS streams.
|
||||
// Make getHlsUrl() and getDashMpdUrl() return lists of VideoStreams,
|
||||
// so the user can choose a resolution.
|
||||
for (int s = 0; s < room.getArray("streams").size(); s++) {
|
||||
final JsonObject stream = room.getArray("streams").getObject(s);
|
||||
if (stream.getString("type").equals("video")) {
|
||||
if (stream.has("hls")) {
|
||||
return stream.getObject("urls").getObject("hls").getString("url");
|
||||
}
|
||||
}
|
||||
}
|
||||
return "";
|
||||
return getManifestOfDeliveryMethodWanted("hls");
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String getManifestOfDeliveryMethodWanted(@Nonnull final String deliveryMethod) {
|
||||
return room.getArray(STREAMS).stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.map(streamObject -> streamObject.getObject(URLS))
|
||||
.filter(urls -> urls.has(deliveryMethod))
|
||||
.map(urls -> urls.getObject(deliveryMethod).getString(URL, EMPTY_STRING))
|
||||
.findFirst()
|
||||
.orElse(EMPTY_STRING);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
|
||||
final List<AudioStream> audioStreams = new ArrayList<>();
|
||||
for (int s = 0; s < room.getArray("streams").size(); s++) {
|
||||
final JsonObject stream = room.getArray("streams").getObject(s);
|
||||
if (stream.getString("type").equals("audio")) {
|
||||
for (final String type : stream.getObject("urls").keySet()) {
|
||||
final JsonObject url = stream.getObject("urls").getObject(type);
|
||||
audioStreams.add(new AudioStream(url.getString("url"),
|
||||
MediaFormat.getFromSuffix(type), -1));
|
||||
}
|
||||
}
|
||||
}
|
||||
return audioStreams;
|
||||
return getStreams("audio",
|
||||
dto -> {
|
||||
final AudioStream.Builder builder = new AudioStream.Builder()
|
||||
.setId(dto.urlValue.getString("tech", ID_UNKNOWN))
|
||||
.setContent(dto.urlValue.getString(URL), true)
|
||||
.setAverageBitrate(UNKNOWN_BITRATE);
|
||||
|
||||
if ("hls".equals(dto.urlKey)) {
|
||||
// We don't know with the type string what media format will
|
||||
// have HLS streams.
|
||||
// However, the tech string may contain some information
|
||||
// about the media format used.
|
||||
return builder.setDeliveryMethod(DeliveryMethod.HLS)
|
||||
.build();
|
||||
}
|
||||
|
||||
return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey))
|
||||
.build();
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
|
||||
final List<VideoStream> videoStreams = new ArrayList<>();
|
||||
for (int s = 0; s < room.getArray("streams").size(); s++) {
|
||||
final JsonObject stream = room.getArray("streams").getObject(s);
|
||||
if (stream.getString("type").equals("video")) {
|
||||
final String resolution = stream.getArray("videoSize").getInt(0) + "x"
|
||||
+ stream.getArray("videoSize").getInt(1);
|
||||
for (final String type : stream.getObject("urls").keySet()) {
|
||||
if (!type.equals("hls")) {
|
||||
final JsonObject url = stream.getObject("urls").getObject(type);
|
||||
videoStreams.add(new VideoStream(
|
||||
url.getString("url"),
|
||||
MediaFormat.getFromSuffix(type),
|
||||
resolution));
|
||||
return getStreams("video",
|
||||
dto -> {
|
||||
final JsonArray videoSize = dto.streamJsonObj.getArray("videoSize");
|
||||
|
||||
final VideoStream.Builder builder = new VideoStream.Builder()
|
||||
.setId(dto.urlValue.getString("tech", ID_UNKNOWN))
|
||||
.setContent(dto.urlValue.getString(URL), true)
|
||||
.setIsVideoOnly(false)
|
||||
.setResolution(videoSize.getInt(0) + "x" + videoSize.getInt(1));
|
||||
|
||||
if ("hls".equals(dto.urlKey)) {
|
||||
// We don't know with the type string what media format will
|
||||
// have HLS streams.
|
||||
// However, the tech string may contain some information
|
||||
// about the media format used.
|
||||
return builder.setDeliveryMethod(DeliveryMethod.HLS)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey))
|
||||
.build();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This is just an internal class used in {@link #getStreams(String, Function)} to tie together
|
||||
* the stream json object, its URL key and its URL value. An object of this class would be
|
||||
* temporary and the three values it holds would be <b>convert</b>ed to a proper {@link Stream}
|
||||
* object based on the wanted stream type.
|
||||
*/
|
||||
private static final class MediaCCCLiveStreamMapperDTO {
|
||||
final JsonObject streamJsonObj;
|
||||
final String urlKey;
|
||||
final JsonObject urlValue;
|
||||
|
||||
MediaCCCLiveStreamMapperDTO(final JsonObject streamJsonObj,
|
||||
final String urlKey,
|
||||
final JsonObject urlValue) {
|
||||
this.streamJsonObj = streamJsonObj;
|
||||
this.urlKey = urlKey;
|
||||
this.urlValue = urlValue;
|
||||
}
|
||||
return videoStreams;
|
||||
}
|
||||
|
||||
private <T extends Stream> List<T> getStreams(
|
||||
@Nonnull final String streamType,
|
||||
@Nonnull final Function<MediaCCCLiveStreamMapperDTO, T> converter) {
|
||||
return room.getArray(STREAMS).stream()
|
||||
// Ensure that we use only process JsonObjects
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
// Only process streams of requested type
|
||||
.filter(streamJsonObj -> streamType.equals(streamJsonObj.getString("type")))
|
||||
// Flatmap Urls and ensure that we use only process JsonObjects
|
||||
.flatMap(streamJsonObj -> streamJsonObj.getObject(URLS).entrySet().stream()
|
||||
.filter(e -> e.getValue() instanceof JsonObject)
|
||||
.map(e -> new MediaCCCLiveStreamMapperDTO(
|
||||
streamJsonObj,
|
||||
e.getKey(),
|
||||
(JsonObject) e.getValue())))
|
||||
// The DASH manifest will be extracted with getDashMpdUrl
|
||||
.filter(dto -> !"dash".equals(dto.urlKey))
|
||||
// Convert
|
||||
.map(converter)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoOnlyStreams() {
|
||||
return null;
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamType getStreamType() throws ParsingException {
|
||||
return StreamType.LIVE_STREAM; // TODO: video and audio only streams are both available
|
||||
return StreamType.LIVE_STREAM;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
@ -1,5 +1,8 @@
|
||||
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
|
||||
|
||||
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
|
||||
import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
@ -99,7 +102,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
|
||||
final JsonObject recording = recordings.getObject(i);
|
||||
final String mimeType = recording.getString("mime_type");
|
||||
if (mimeType.startsWith("audio")) {
|
||||
//first we need to resolve the actual video data from CDN
|
||||
// First we need to resolve the actual video data from the CDN
|
||||
final MediaFormat mediaFormat;
|
||||
if (mimeType.endsWith("opus")) {
|
||||
mediaFormat = MediaFormat.OPUS;
|
||||
@ -108,11 +111,18 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
|
||||
} else if (mimeType.endsWith("ogg")) {
|
||||
mediaFormat = MediaFormat.OGG;
|
||||
} else {
|
||||
throw new ExtractionException("Unknown media format: " + mimeType);
|
||||
mediaFormat = null;
|
||||
}
|
||||
|
||||
audioStreams.add(new AudioStream(recording.getString("recording_url"),
|
||||
mediaFormat, -1));
|
||||
// Not checking containsSimilarStream here, since MediaCCC does not provide enough
|
||||
// information to decide whether two streams are similar. Hence that method would
|
||||
// always return false, e.g. even for different language variations.
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId(recording.getString("filename", ID_UNKNOWN))
|
||||
.setContent(recording.getString("recording_url"), true)
|
||||
.setMediaFormat(mediaFormat)
|
||||
.setAverageBitrate(UNKNOWN_BITRATE)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
return audioStreams;
|
||||
@ -126,21 +136,29 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
|
||||
final JsonObject recording = recordings.getObject(i);
|
||||
final String mimeType = recording.getString("mime_type");
|
||||
if (mimeType.startsWith("video")) {
|
||||
//first we need to resolve the actual video data from CDN
|
||||
|
||||
// First we need to resolve the actual video data from the CDN
|
||||
final MediaFormat mediaFormat;
|
||||
if (mimeType.endsWith("webm")) {
|
||||
mediaFormat = MediaFormat.WEBM;
|
||||
} else if (mimeType.endsWith("mp4")) {
|
||||
mediaFormat = MediaFormat.MPEG_4;
|
||||
} else {
|
||||
throw new ExtractionException("Unknown media format: " + mimeType);
|
||||
mediaFormat = null;
|
||||
}
|
||||
|
||||
videoStreams.add(new VideoStream(recording.getString("recording_url"),
|
||||
mediaFormat, recording.getInt("height") + "p"));
|
||||
// Not checking containsSimilarStream here, since MediaCCC does not provide enough
|
||||
// information to decide whether two streams are similar. Hence that method would
|
||||
// always return false, e.g. even for different language variations.
|
||||
videoStreams.add(new VideoStream.Builder()
|
||||
.setId(recording.getString("filename", ID_UNKNOWN))
|
||||
.setContent(recording.getString("recording_url"), true)
|
||||
.setIsVideoOnly(false)
|
||||
.setMediaFormat(mediaFormat)
|
||||
.setResolution(recording.getInt("height") + "p")
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
||||
return videoStreams;
|
||||
}
|
||||
|
||||
@ -163,7 +181,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
|
||||
conferenceData = JsonParser.object()
|
||||
.from(downloader.get(data.getString("conference_url")).responseBody());
|
||||
} catch (final JsonParserException jpe) {
|
||||
throw new ExtractionException("Could not parse json returned by url: " + videoUrl, jpe);
|
||||
throw new ExtractionException("Could not parse json returned by URL: " + videoUrl,
|
||||
jpe);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -22,6 +22,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
|
||||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
@ -39,14 +40,30 @@ import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
|
||||
public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
private static final String ACCOUNT_HOST = "account.host";
|
||||
private static final String ACCOUNT_NAME = "account.name";
|
||||
private static final String FILES = "files";
|
||||
private static final String FILE_DOWNLOAD_URL = "fileDownloadUrl";
|
||||
private static final String FILE_URL = "fileUrl";
|
||||
private static final String PLAYLIST_URL = "playlistUrl";
|
||||
private static final String RESOLUTION_ID = "resolution.id";
|
||||
private static final String STREAMING_PLAYLISTS = "streamingPlaylists";
|
||||
|
||||
private final String baseUrl;
|
||||
private JsonObject json;
|
||||
|
||||
private final List<SubtitlesStream> subtitles = new ArrayList<>();
|
||||
private final List<AudioStream> audioStreams = new ArrayList<>();
|
||||
private final List<VideoStream> videoStreams = new ArrayList<>();
|
||||
|
||||
public PeertubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler)
|
||||
throws ParsingException {
|
||||
@ -85,9 +102,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
} catch (final ParsingException e) {
|
||||
return Description.EMPTY_DESCRIPTION;
|
||||
}
|
||||
|
||||
if (text.length() == 250 && text.substring(247).equals("...")) {
|
||||
//if description is shortened, get full description
|
||||
// If description is shortened, get full description
|
||||
final Downloader dl = NewPipe.getDownloader();
|
||||
try {
|
||||
final Response response = dl.get(baseUrl
|
||||
@ -95,8 +111,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
+ getId() + "/description");
|
||||
final JsonObject jsonObject = JsonParser.object().from(response.responseBody());
|
||||
text = JsonUtils.getString(jsonObject, "description");
|
||||
} catch (ReCaptchaException | IOException | JsonParserException e) {
|
||||
e.printStackTrace();
|
||||
} catch (final IOException | ReCaptchaException | JsonParserException ignored) {
|
||||
// Something went wrong when getting the full description, use the shortened one
|
||||
}
|
||||
}
|
||||
return new Description(text, Description.MARKDOWN);
|
||||
@ -119,8 +135,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
|
||||
@Override
|
||||
public long getTimeStamp() throws ParsingException {
|
||||
final long timestamp =
|
||||
getTimestampSeconds("((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
|
||||
final long timestamp = getTimestampSeconds(
|
||||
"((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
|
||||
|
||||
if (timestamp == -2) {
|
||||
// regex for timestamp was not found
|
||||
@ -148,10 +164,10 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
final String name = JsonUtils.getString(json, "account.name");
|
||||
final String host = JsonUtils.getString(json, "account.host");
|
||||
return getService().getChannelLHFactory()
|
||||
.fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
|
||||
final String name = JsonUtils.getString(json, ACCOUNT_NAME);
|
||||
final String host = JsonUtils.getString(json, ACCOUNT_HOST);
|
||||
return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl)
|
||||
.getUrl();
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@ -199,77 +215,51 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getHlsUrl() {
|
||||
return json.getArray("streamingPlaylists").getObject(0).getString("playlistUrl");
|
||||
assertPageFetched();
|
||||
|
||||
if (getStreamType() == StreamType.VIDEO_STREAM
|
||||
&& !isNullOrEmpty(json.getObject(FILES))) {
|
||||
return json.getObject(FILES).getString(PLAYLIST_URL, EMPTY_STRING);
|
||||
}
|
||||
|
||||
return json.getArray(STREAMING_PLAYLISTS).getObject(0).getString(PLAYLIST_URL,
|
||||
EMPTY_STRING);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() {
|
||||
return Collections.emptyList();
|
||||
public List<AudioStream> getAudioStreams() throws ParsingException {
|
||||
assertPageFetched();
|
||||
|
||||
/*
|
||||
Some videos have audio streams; others don't.
|
||||
So an audio stream may be available if a video stream is available.
|
||||
Audio streams are also not returned as separated streams for livestreams.
|
||||
That's why the extraction of audio streams is only run when there are video streams
|
||||
extracted and when the content is not a livestream.
|
||||
*/
|
||||
if (audioStreams.isEmpty() && videoStreams.isEmpty()
|
||||
&& getStreamType() == StreamType.VIDEO_STREAM) {
|
||||
getStreams();
|
||||
}
|
||||
|
||||
return audioStreams;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
final List<VideoStream> videoStreams = new ArrayList<>();
|
||||
|
||||
// mp4
|
||||
try {
|
||||
videoStreams.addAll(getVideoStreamsFromArray(json.getArray("files")));
|
||||
} catch (final Exception ignored) { }
|
||||
|
||||
// HLS
|
||||
try {
|
||||
final JsonArray streamingPlaylists = json.getArray("streamingPlaylists");
|
||||
for (final Object p : streamingPlaylists) {
|
||||
if (!(p instanceof JsonObject)) {
|
||||
continue;
|
||||
}
|
||||
final JsonObject playlist = (JsonObject) p;
|
||||
videoStreams.addAll(getVideoStreamsFromArray(playlist.getArray("files")));
|
||||
if (videoStreams.isEmpty()) {
|
||||
if (getStreamType() == StreamType.VIDEO_STREAM) {
|
||||
getStreams();
|
||||
} else {
|
||||
extractLiveVideoStreams();
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get video streams", e);
|
||||
}
|
||||
|
||||
if (getStreamType() == StreamType.LIVE_STREAM) {
|
||||
videoStreams.add(new VideoStream(getHlsUrl(), MediaFormat.MPEG_4, "720p"));
|
||||
}
|
||||
|
||||
return videoStreams;
|
||||
}
|
||||
|
||||
private List<VideoStream> getVideoStreamsFromArray(final JsonArray streams)
|
||||
throws ParsingException {
|
||||
try {
|
||||
final List<VideoStream> videoStreams = new ArrayList<>();
|
||||
for (final Object s : streams) {
|
||||
if (!(s instanceof JsonObject)) {
|
||||
continue;
|
||||
}
|
||||
final JsonObject stream = (JsonObject) s;
|
||||
final String url;
|
||||
if (stream.has("fileDownloadUrl")) {
|
||||
url = JsonUtils.getString(stream, "fileDownloadUrl");
|
||||
} else {
|
||||
url = JsonUtils.getString(stream, "fileUrl");
|
||||
}
|
||||
final String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
|
||||
final String resolution = JsonUtils.getString(stream, "resolution.label");
|
||||
final String extension = url.substring(url.lastIndexOf(".") + 1);
|
||||
final MediaFormat format = MediaFormat.getFromSuffix(extension);
|
||||
final VideoStream videoStream
|
||||
= new VideoStream(url, torrentUrl, format, resolution);
|
||||
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
|
||||
videoStreams.add(videoStream);
|
||||
}
|
||||
}
|
||||
return videoStreams;
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get video streams from array");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoOnlyStreams() {
|
||||
return Collections.emptyList();
|
||||
@ -284,13 +274,9 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<SubtitlesStream> getSubtitles(final MediaFormat format) {
|
||||
final List<SubtitlesStream> filteredSubs = new ArrayList<>();
|
||||
for (final SubtitlesStream sub : subtitles) {
|
||||
if (sub.getFormat() == format) {
|
||||
filteredSubs.add(sub);
|
||||
}
|
||||
}
|
||||
return filteredSubs;
|
||||
return subtitles.stream()
|
||||
.filter(sub -> sub.getFormat() == format)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -304,8 +290,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
final List<String> tags = getTags();
|
||||
final String apiUrl;
|
||||
if (tags.isEmpty()) {
|
||||
apiUrl = baseUrl + "/api/v1/accounts/" + JsonUtils.getString(json, "account.name")
|
||||
+ "@" + JsonUtils.getString(json, "account.host")
|
||||
apiUrl = baseUrl + "/api/v1/accounts/" + JsonUtils.getString(json, ACCOUNT_NAME)
|
||||
+ "@" + JsonUtils.getString(json, ACCOUNT_HOST)
|
||||
+ "/videos?start=0&count=8";
|
||||
} else {
|
||||
apiUrl = getRelatedItemsUrl(tags);
|
||||
@ -314,7 +300,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
if (Utils.isBlank(apiUrl)) {
|
||||
return null;
|
||||
} else {
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(
|
||||
getServiceId());
|
||||
getStreamsFromApi(collector, apiUrl);
|
||||
return collector;
|
||||
}
|
||||
@ -332,11 +319,13 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
try {
|
||||
return JsonUtils.getString(json, "support");
|
||||
} catch (final ParsingException e) {
|
||||
return "";
|
||||
return EMPTY_STRING;
|
||||
}
|
||||
}
|
||||
|
||||
private String getRelatedItemsUrl(final List<String> tags) throws UnsupportedEncodingException {
|
||||
@Nonnull
|
||||
private String getRelatedItemsUrl(@Nonnull final List<String> tags)
|
||||
throws UnsupportedEncodingException {
|
||||
final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
|
||||
final StringBuilder params = new StringBuilder();
|
||||
params.append("start=0&count=8&sort=-createdAt");
|
||||
@ -348,7 +337,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
|
||||
private void getStreamsFromApi(final StreamInfoItemsCollector collector, final String apiUrl)
|
||||
throws ReCaptchaException, IOException, ParsingException {
|
||||
throws IOException, ReCaptchaException, ParsingException {
|
||||
final Response response = getDownloader().get(apiUrl);
|
||||
JsonObject relatedVideosJson = null;
|
||||
if (response != null && !Utils.isBlank(response.responseBody())) {
|
||||
@ -365,21 +354,20 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
|
||||
private void collectStreamsFrom(final StreamInfoItemsCollector collector,
|
||||
final JsonObject jsonObject)
|
||||
throws ParsingException {
|
||||
final JsonObject jsonObject) throws ParsingException {
|
||||
final JsonArray contents;
|
||||
try {
|
||||
contents = (JsonArray) JsonUtils.getValue(jsonObject, "data");
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("unable to extract related videos", e);
|
||||
throw new ParsingException("Could not extract related videos", e);
|
||||
}
|
||||
|
||||
for (final Object c : contents) {
|
||||
if (c instanceof JsonObject) {
|
||||
final JsonObject item = (JsonObject) c;
|
||||
final PeertubeStreamInfoItemExtractor extractor
|
||||
= new PeertubeStreamInfoItemExtractor(item, baseUrl);
|
||||
//do not add the same stream in related streams
|
||||
final PeertubeStreamInfoItemExtractor extractor =
|
||||
new PeertubeStreamInfoItemExtractor(item, baseUrl);
|
||||
// Do not add the same stream in related streams
|
||||
if (!extractor.getUrl().equals(getUrl())) {
|
||||
collector.commit(extractor);
|
||||
}
|
||||
@ -395,7 +383,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
if (response != null) {
|
||||
setInitialData(response.responseBody());
|
||||
} else {
|
||||
throw new ExtractionException("Unable to extract PeerTube channel data");
|
||||
throw new ExtractionException("Could not extract PeerTube channel data");
|
||||
}
|
||||
|
||||
loadSubtitles();
|
||||
@ -405,10 +393,10 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
try {
|
||||
json = JsonParser.object().from(responseBody);
|
||||
} catch (final JsonParserException e) {
|
||||
throw new ExtractionException("Unable to extract PeerTube stream data", e);
|
||||
throw new ExtractionException("Could not extract PeerTube stream data", e);
|
||||
}
|
||||
if (json == null) {
|
||||
throw new ExtractionException("Unable to extract PeerTube stream data");
|
||||
throw new ExtractionException("Could not extract PeerTube stream data");
|
||||
}
|
||||
PeertubeParsingHelper.validate(json);
|
||||
}
|
||||
@ -429,16 +417,233 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
final String ext = url.substring(url.lastIndexOf(".") + 1);
|
||||
final MediaFormat fmt = MediaFormat.getFromSuffix(ext);
|
||||
if (fmt != null && !isNullOrEmpty(languageCode)) {
|
||||
subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
|
||||
subtitles.add(new SubtitlesStream.Builder()
|
||||
.setContent(url, true)
|
||||
.setMediaFormat(fmt)
|
||||
.setLanguageCode(languageCode)
|
||||
.setAutoGenerated(false)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
// ignore all exceptions
|
||||
} catch (final Exception ignored) {
|
||||
// Ignore all exceptions
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void extractLiveVideoStreams() throws ParsingException {
|
||||
try {
|
||||
final JsonArray streamingPlaylists = json.getArray(STREAMING_PLAYLISTS);
|
||||
streamingPlaylists.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.map(stream -> new VideoStream.Builder()
|
||||
.setId(String.valueOf(stream.getInt("id", -1)))
|
||||
.setContent(stream.getString(PLAYLIST_URL, EMPTY_STRING), true)
|
||||
.setIsVideoOnly(false)
|
||||
.setResolution(EMPTY_STRING)
|
||||
.setMediaFormat(MediaFormat.MPEG_4)
|
||||
.setDeliveryMethod(DeliveryMethod.HLS)
|
||||
.build())
|
||||
// Don't use the containsSimilarStream method because it will always return
|
||||
// false so if there are multiples HLS URLs returned, only the first will be
|
||||
// extracted in this case.
|
||||
.forEachOrdered(videoStreams::add);
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get video streams", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void getStreams() throws ParsingException {
|
||||
// Progressive streams
|
||||
getStreamsFromArray(json.getArray(FILES), EMPTY_STRING);
|
||||
|
||||
// HLS streams
|
||||
try {
|
||||
for (final JsonObject playlist : json.getArray(STREAMING_PLAYLISTS).stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.collect(Collectors.toList())) {
|
||||
getStreamsFromArray(playlist.getArray(FILES), playlist.getString(PLAYLIST_URL));
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get streams", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void getStreamsFromArray(@Nonnull final JsonArray streams,
|
||||
final String playlistUrl) throws ParsingException {
|
||||
try {
|
||||
/*
|
||||
Starting with version 3.4.0 of PeerTube, the HLS playlist of stream resolutions
|
||||
contains the UUID of the streams, so we can't use the same method to get the URL of
|
||||
the HLS playlist without fetching the master playlist.
|
||||
These UUIDs are the same as the ones returned into the fileUrl and fileDownloadUrl
|
||||
strings.
|
||||
*/
|
||||
final boolean isInstanceUsingRandomUuidsForHlsStreams = !isNullOrEmpty(playlistUrl)
|
||||
&& playlistUrl.endsWith("-master.m3u8");
|
||||
|
||||
for (final JsonObject stream : streams.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.collect(Collectors.toList())) {
|
||||
|
||||
// Extract stream version of streams first
|
||||
final String url = JsonUtils.getString(stream,
|
||||
stream.has(FILE_URL) ? FILE_URL : FILE_DOWNLOAD_URL);
|
||||
if (isNullOrEmpty(url)) {
|
||||
// Not a valid stream URL
|
||||
return;
|
||||
}
|
||||
|
||||
final String resolution = JsonUtils.getString(stream, "resolution.label");
|
||||
final String idSuffix = stream.has(FILE_URL) ? FILE_URL : FILE_DOWNLOAD_URL;
|
||||
|
||||
if (resolution.toLowerCase().contains("audio")) {
|
||||
// An audio stream
|
||||
addNewAudioStream(stream, isInstanceUsingRandomUuidsForHlsStreams, resolution,
|
||||
idSuffix, url, playlistUrl);
|
||||
} else {
|
||||
// A video stream
|
||||
addNewVideoStream(stream, isInstanceUsingRandomUuidsForHlsStreams, resolution,
|
||||
idSuffix, url, playlistUrl);
|
||||
}
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get streams from array", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String getHlsPlaylistUrlFromFragmentedFileUrl(
|
||||
@Nonnull final JsonObject streamJsonObject,
|
||||
@Nonnull final String idSuffix,
|
||||
@Nonnull final String format,
|
||||
@Nonnull final String url) throws ParsingException {
|
||||
final String streamUrl = FILE_DOWNLOAD_URL.equals(idSuffix)
|
||||
? JsonUtils.getString(streamJsonObject, FILE_URL)
|
||||
: url;
|
||||
return streamUrl.replace("-fragmented." + format, ".m3u8");
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String getHlsPlaylistUrlFromMasterPlaylist(@Nonnull final JsonObject streamJsonObject,
|
||||
@Nonnull final String playlistUrl)
|
||||
throws ParsingException {
|
||||
return playlistUrl.replace("master", JsonUtils.getNumber(streamJsonObject,
|
||||
RESOLUTION_ID).toString());
|
||||
}
|
||||
|
||||
private void addNewAudioStream(@Nonnull final JsonObject streamJsonObject,
|
||||
final boolean isInstanceUsingRandomUuidsForHlsStreams,
|
||||
@Nonnull final String resolution,
|
||||
@Nonnull final String idSuffix,
|
||||
@Nonnull final String url,
|
||||
@Nullable final String playlistUrl) throws ParsingException {
|
||||
final String extension = url.substring(url.lastIndexOf(".") + 1);
|
||||
final MediaFormat format = MediaFormat.getFromSuffix(extension);
|
||||
final String id = resolution + "-" + extension;
|
||||
|
||||
// Add progressive HTTP streams first
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId(id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP)
|
||||
.setContent(url, true)
|
||||
.setMediaFormat(format)
|
||||
.setAverageBitrate(UNKNOWN_BITRATE)
|
||||
.build());
|
||||
|
||||
// Then add HLS streams
|
||||
if (!isNullOrEmpty(playlistUrl)) {
|
||||
final String hlsStreamUrl;
|
||||
if (isInstanceUsingRandomUuidsForHlsStreams) {
|
||||
hlsStreamUrl = getHlsPlaylistUrlFromFragmentedFileUrl(streamJsonObject, idSuffix,
|
||||
extension, url);
|
||||
|
||||
} else {
|
||||
hlsStreamUrl = getHlsPlaylistUrlFromMasterPlaylist(streamJsonObject, playlistUrl);
|
||||
}
|
||||
final AudioStream audioStream = new AudioStream.Builder()
|
||||
.setId(id + "-" + DeliveryMethod.HLS)
|
||||
.setContent(hlsStreamUrl, true)
|
||||
.setDeliveryMethod(DeliveryMethod.HLS)
|
||||
.setMediaFormat(format)
|
||||
.setAverageBitrate(UNKNOWN_BITRATE)
|
||||
.setManifestUrl(playlistUrl)
|
||||
.build();
|
||||
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
|
||||
audioStreams.add(audioStream);
|
||||
}
|
||||
}
|
||||
|
||||
// Finally, add torrent URLs
|
||||
final String torrentUrl = JsonUtils.getString(streamJsonObject, "torrentUrl");
|
||||
if (!isNullOrEmpty(torrentUrl)) {
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId(id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT)
|
||||
.setContent(torrentUrl, true)
|
||||
.setDeliveryMethod(DeliveryMethod.TORRENT)
|
||||
.setMediaFormat(format)
|
||||
.setAverageBitrate(UNKNOWN_BITRATE)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
||||
private void addNewVideoStream(@Nonnull final JsonObject streamJsonObject,
|
||||
final boolean isInstanceUsingRandomUuidsForHlsStreams,
|
||||
@Nonnull final String resolution,
|
||||
@Nonnull final String idSuffix,
|
||||
@Nonnull final String url,
|
||||
@Nullable final String playlistUrl) throws ParsingException {
|
||||
final String extension = url.substring(url.lastIndexOf(".") + 1);
|
||||
final MediaFormat format = MediaFormat.getFromSuffix(extension);
|
||||
final String id = resolution + "-" + extension;
|
||||
|
||||
// Add progressive HTTP streams first
|
||||
videoStreams.add(new VideoStream.Builder()
|
||||
.setId(id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP)
|
||||
.setContent(url, true)
|
||||
.setIsVideoOnly(false)
|
||||
.setResolution(resolution)
|
||||
.setMediaFormat(format)
|
||||
.build());
|
||||
|
||||
// Then add HLS streams
|
||||
if (!isNullOrEmpty(playlistUrl)) {
|
||||
final String hlsStreamUrl = isInstanceUsingRandomUuidsForHlsStreams
|
||||
? getHlsPlaylistUrlFromFragmentedFileUrl(streamJsonObject, idSuffix, extension,
|
||||
url)
|
||||
: getHlsPlaylistUrlFromMasterPlaylist(streamJsonObject, playlistUrl);
|
||||
|
||||
final VideoStream videoStream = new VideoStream.Builder()
|
||||
.setId(id + "-" + DeliveryMethod.HLS)
|
||||
.setContent(hlsStreamUrl, true)
|
||||
.setIsVideoOnly(false)
|
||||
.setDeliveryMethod(DeliveryMethod.HLS)
|
||||
.setResolution(resolution)
|
||||
.setMediaFormat(format)
|
||||
.setManifestUrl(playlistUrl)
|
||||
.build();
|
||||
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
|
||||
videoStreams.add(videoStream);
|
||||
}
|
||||
}
|
||||
|
||||
// Add finally torrent URLs
|
||||
final String torrentUrl = JsonUtils.getString(streamJsonObject, "torrentUrl");
|
||||
if (!isNullOrEmpty(torrentUrl)) {
|
||||
videoStreams.add(new VideoStream.Builder()
|
||||
.setId(id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT)
|
||||
.setContent(torrentUrl, true)
|
||||
.setIsVideoOnly(false)
|
||||
.setDeliveryMethod(DeliveryMethod.TORRENT)
|
||||
.setResolution(resolution)
|
||||
.setMediaFormat(format)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
@ -448,7 +653,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getHost() throws ParsingException {
|
||||
return JsonUtils.getString(json, "account.host");
|
||||
return JsonUtils.getString(json, ACCOUNT_HOST);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
@ -1,6 +1,9 @@
|
||||
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
|
||||
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.clientId;
|
||||
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
|
||||
import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
@ -25,7 +28,9 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
||||
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
||||
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
|
||||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
@ -58,13 +63,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
||||
final String policy = track.getString("policy", EMPTY_STRING);
|
||||
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
|
||||
isAvailable = false;
|
||||
|
||||
if (policy.equals("SNIP")) {
|
||||
throw new SoundCloudGoPlusContentException();
|
||||
}
|
||||
|
||||
if (policy.equals("BLOCK")) {
|
||||
throw new GeographicRestrictionException(
|
||||
"This track is not available in user's country");
|
||||
}
|
||||
|
||||
throw new ContentNotAvailableException("Content not available: policy " + policy);
|
||||
}
|
||||
}
|
||||
@ -72,7 +80,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getId() {
|
||||
return track.getInt("id") + EMPTY_STRING;
|
||||
return String.valueOf(track.getInt("id"));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@ -168,118 +176,205 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
||||
|
||||
try {
|
||||
final JsonArray transcodings = track.getObject("media").getArray("transcodings");
|
||||
if (transcodings != null) {
|
||||
if (!isNullOrEmpty(transcodings)) {
|
||||
// Get information about what stream formats are available
|
||||
extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings),
|
||||
audioStreams);
|
||||
}
|
||||
|
||||
extractDownloadableFileIfAvailable(audioStreams);
|
||||
} catch (final NullPointerException e) {
|
||||
throw new ExtractionException("Could not get SoundCloud's tracks audio URL", e);
|
||||
throw new ExtractionException("Could not get audio streams", e);
|
||||
}
|
||||
|
||||
return audioStreams;
|
||||
}
|
||||
|
||||
private static boolean checkMp3ProgressivePresence(final JsonArray transcodings) {
|
||||
boolean presence = false;
|
||||
for (final Object transcoding : transcodings) {
|
||||
final JsonObject transcodingJsonObject = (JsonObject) transcoding;
|
||||
if (transcodingJsonObject.getString("preset").contains("mp3")
|
||||
&& transcodingJsonObject.getObject("format").getString("protocol")
|
||||
.equals("progressive")) {
|
||||
presence = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return presence;
|
||||
private static boolean checkMp3ProgressivePresence(@Nonnull final JsonArray transcodings) {
|
||||
return transcodings.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.anyMatch(transcodingJsonObject -> transcodingJsonObject.getString("preset")
|
||||
.contains("mp3") && transcodingJsonObject.getObject("format")
|
||||
.getString("protocol").equals("progressive"));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private static String getTranscodingUrl(final String endpointUrl,
|
||||
final String protocol)
|
||||
private String getTranscodingUrl(final String endpointUrl)
|
||||
throws IOException, ExtractionException {
|
||||
final Downloader downloader = NewPipe.getDownloader();
|
||||
final String apiStreamUrl = endpointUrl + "?client_id="
|
||||
+ SoundcloudParsingHelper.clientId();
|
||||
final String response = downloader.get(apiStreamUrl).responseBody();
|
||||
final String apiStreamUrl = endpointUrl + "?client_id=" + clientId();
|
||||
final String response = NewPipe.getDownloader().get(apiStreamUrl).responseBody();
|
||||
final JsonObject urlObject;
|
||||
try {
|
||||
urlObject = JsonParser.object().from(response);
|
||||
} catch (final JsonParserException e) {
|
||||
throw new ParsingException("Could not parse streamable url", e);
|
||||
throw new ParsingException("Could not parse streamable URL", e);
|
||||
}
|
||||
final String urlString = urlObject.getString("url");
|
||||
|
||||
if (protocol.equals("progressive")) {
|
||||
return urlString;
|
||||
} else if (protocol.equals("hls")) {
|
||||
try {
|
||||
return getSingleUrlFromHlsManifest(urlString);
|
||||
} catch (final ParsingException ignored) {
|
||||
}
|
||||
}
|
||||
// else, unknown protocol
|
||||
return "";
|
||||
return urlObject.getString("url");
|
||||
}
|
||||
|
||||
private static void extractAudioStreams(final JsonArray transcodings,
|
||||
final boolean mp3ProgressiveInStreams,
|
||||
final List<AudioStream> audioStreams) {
|
||||
for (final Object transcoding : transcodings) {
|
||||
final JsonObject transcodingJsonObject = (JsonObject) transcoding;
|
||||
final String url = transcodingJsonObject.getString("url");
|
||||
if (isNullOrEmpty(url)) {
|
||||
continue;
|
||||
}
|
||||
final String mediaUrl;
|
||||
final String preset = transcodingJsonObject.getString("preset");
|
||||
final String protocol = transcodingJsonObject.getObject("format")
|
||||
.getString("protocol");
|
||||
MediaFormat mediaFormat = null;
|
||||
int bitrate = 0;
|
||||
if (preset.contains("mp3")) {
|
||||
// Don't add the MP3 HLS stream if there is a progressive stream present
|
||||
// because the two have the same bitrate
|
||||
if (mp3ProgressiveInStreams && protocol.equals("hls")) {
|
||||
continue;
|
||||
}
|
||||
mediaFormat = MediaFormat.MP3;
|
||||
bitrate = 128;
|
||||
} else if (preset.contains("opus")) {
|
||||
mediaFormat = MediaFormat.OPUS;
|
||||
bitrate = 64;
|
||||
}
|
||||
@Nullable
|
||||
private String getDownloadUrl(@Nonnull final String trackId)
|
||||
throws IOException, ExtractionException {
|
||||
final String response = NewPipe.getDownloader().get(SOUNDCLOUD_API_V2_URL + "tracks/"
|
||||
+ trackId + "/download" + "?client_id=" + clientId()).responseBody();
|
||||
|
||||
if (mediaFormat != null) {
|
||||
try {
|
||||
mediaUrl = getTranscodingUrl(url, protocol);
|
||||
if (!mediaUrl.isEmpty()) {
|
||||
audioStreams.add(new AudioStream(mediaUrl, mediaFormat, bitrate));
|
||||
final JsonObject downloadJsonObject;
|
||||
try {
|
||||
downloadJsonObject = JsonParser.object().from(response);
|
||||
} catch (final JsonParserException e) {
|
||||
throw new ParsingException("Could not parse download URL", e);
|
||||
}
|
||||
final String redirectUri = downloadJsonObject.getString("redirectUri");
|
||||
if (!isNullOrEmpty(redirectUri)) {
|
||||
return redirectUri;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void extractAudioStreams(@Nonnull final JsonArray transcodings,
|
||||
final boolean mp3ProgressiveInStreams,
|
||||
final List<AudioStream> audioStreams) {
|
||||
transcodings.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.forEachOrdered(transcoding -> {
|
||||
final String url = transcoding.getString("url");
|
||||
if (isNullOrEmpty(url)) {
|
||||
return;
|
||||
}
|
||||
} catch (final Exception ignored) {
|
||||
// something went wrong when parsing this transcoding, don't add it to
|
||||
// audioStreams
|
||||
|
||||
final String preset = transcoding.getString("preset", ID_UNKNOWN);
|
||||
final String protocol = transcoding.getObject("format").getString("protocol");
|
||||
final AudioStream.Builder builder = new AudioStream.Builder()
|
||||
.setId(preset);
|
||||
|
||||
try {
|
||||
// streamUrl can be either the MP3 progressive stream URL or the
|
||||
// manifest URL of the HLS MP3 stream (if there is no MP3 progressive
|
||||
// stream, see above)
|
||||
final String streamUrl = getTranscodingUrl(url);
|
||||
|
||||
if (preset.contains("mp3")) {
|
||||
// Don't add the MP3 HLS stream if there is a progressive stream
|
||||
// present because the two have the same bitrate
|
||||
final boolean isHls = protocol.equals("hls");
|
||||
if (mp3ProgressiveInStreams && isHls) {
|
||||
return;
|
||||
}
|
||||
|
||||
builder.setMediaFormat(MediaFormat.MP3);
|
||||
builder.setAverageBitrate(128);
|
||||
|
||||
if (isHls) {
|
||||
builder.setDeliveryMethod(DeliveryMethod.HLS);
|
||||
builder.setContent(streamUrl, true);
|
||||
|
||||
final AudioStream hlsStream = builder.build();
|
||||
if (!Stream.containSimilarStream(hlsStream, audioStreams)) {
|
||||
audioStreams.add(hlsStream);
|
||||
}
|
||||
|
||||
final String progressiveHlsUrl =
|
||||
getSingleUrlFromHlsManifest(streamUrl);
|
||||
builder.setDeliveryMethod(DeliveryMethod.PROGRESSIVE_HTTP);
|
||||
builder.setContent(progressiveHlsUrl, true);
|
||||
|
||||
final AudioStream progressiveHlsStream = builder.build();
|
||||
if (!Stream.containSimilarStream(
|
||||
progressiveHlsStream, audioStreams)) {
|
||||
audioStreams.add(progressiveHlsStream);
|
||||
}
|
||||
|
||||
// The MP3 HLS stream has been added in both versions (HLS and
|
||||
// progressive with the manifest parsing trick), so we need to
|
||||
// continue (otherwise the code would try to add again the stream,
|
||||
// which would be not added because the containsSimilarStream
|
||||
// method would return false and an audio stream object would be
|
||||
// created for nothing)
|
||||
return;
|
||||
} else {
|
||||
builder.setContent(streamUrl, true);
|
||||
}
|
||||
} else if (preset.contains("opus")) {
|
||||
// The HLS manifest trick doesn't work for opus streams
|
||||
builder.setContent(streamUrl, true);
|
||||
builder.setMediaFormat(MediaFormat.OPUS);
|
||||
builder.setAverageBitrate(64);
|
||||
builder.setDeliveryMethod(DeliveryMethod.HLS);
|
||||
} else {
|
||||
// Unknown format, skip to the next audio stream
|
||||
return;
|
||||
}
|
||||
|
||||
final AudioStream audioStream = builder.build();
|
||||
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
|
||||
audioStreams.add(audioStream);
|
||||
}
|
||||
} catch (final ExtractionException | IOException ignored) {
|
||||
// Something went wrong when trying to get and add this audio stream,
|
||||
// skip to the next one
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the downloadable format if it is available.
|
||||
*
|
||||
* <p>
|
||||
* A track can have the {@code downloadable} boolean set to {@code true}, but it doesn't mean
|
||||
* we can download it.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* If the value of the {@code has_download_left} boolean is {@code true}, the track can be
|
||||
* downloaded, and not otherwise.
|
||||
* </p>
|
||||
*
|
||||
* @param audioStreams the audio streams to which the downloadable file is added
|
||||
*/
|
||||
public void extractDownloadableFileIfAvailable(final List<AudioStream> audioStreams) {
|
||||
if (track.getBoolean("downloadable") && track.getBoolean("has_downloads_left")) {
|
||||
try {
|
||||
final String downloadUrl = getDownloadUrl(getId());
|
||||
if (!isNullOrEmpty(downloadUrl)) {
|
||||
audioStreams.add(new AudioStream.Builder()
|
||||
.setId("original-format")
|
||||
.setContent(downloadUrl, true)
|
||||
.setAverageBitrate(UNKNOWN_BITRATE)
|
||||
.build());
|
||||
}
|
||||
} catch (final Exception ignored) {
|
||||
// If something went wrong when trying to get the download URL, ignore the
|
||||
// exception throw because this "stream" is not necessary to play the track
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
|
||||
* Parses a SoundCloud HLS MP3 manifest to get a single URL of HLS streams.
|
||||
*
|
||||
* <p>
|
||||
* This method downloads the provided manifest URL, find all web occurrences in the manifest,
|
||||
* get the last segment URL, changes its segment range to {@code 0/track-length} and return
|
||||
* this string.
|
||||
* This method downloads the provided manifest URL, finds all web occurrences in the manifest,
|
||||
* gets the last segment URL, changes its segment range to {@code 0/track-length}, and return
|
||||
* this as a string.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* This was working before for Opus streams, but has been broken by SoundCloud.
|
||||
* </p>
|
||||
*
|
||||
* @param hlsManifestUrl the URL of the manifest to be parsed
|
||||
* @return a single URL that contains a range equal to the length of the track
|
||||
*/
|
||||
private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl)
|
||||
@Nonnull
|
||||
private static String getSingleUrlFromHlsManifest(@Nonnull final String hlsManifestUrl)
|
||||
throws ParsingException {
|
||||
final Downloader dl = NewPipe.getDownloader();
|
||||
final String hlsManifestResponse;
|
||||
|
||||
try {
|
||||
hlsManifestResponse = dl.get(hlsManifestUrl).responseBody();
|
||||
hlsManifestResponse = NewPipe.getDownloader().get(hlsManifestUrl).responseBody();
|
||||
} catch (final IOException | ReCaptchaException e) {
|
||||
throw new ParsingException("Could not get SoundCloud HLS manifest");
|
||||
}
|
||||
@ -288,12 +383,13 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
||||
for (int l = lines.length - 1; l >= 0; l--) {
|
||||
final String line = lines[l];
|
||||
// Get the last URL from manifest, because it contains the range of the stream
|
||||
if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith("https")) {
|
||||
if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith(HTTPS)) {
|
||||
final String[] hlsLastRangeUrlArray = line.split("/");
|
||||
return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5]
|
||||
+ "/" + hlsLastRangeUrlArray[6];
|
||||
}
|
||||
}
|
||||
|
||||
throw new ParsingException("Could not get any URL from HLS manifest");
|
||||
}
|
||||
|
||||
@ -326,7 +422,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
|
||||
final String apiUrl = SOUNDCLOUD_API_V2_URL + "tracks/" + urlEncode(getId())
|
||||
+ "/related?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
|
||||
+ "/related?client_id=" + urlEncode(clientId());
|
||||
|
||||
SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
|
||||
return collector;
|
||||
@ -355,19 +451,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
||||
// Tags are separated by spaces, but they can be multiple words escaped by quotes "
|
||||
final String[] tagList = track.getString("tag_list").split(" ");
|
||||
final List<String> tags = new ArrayList<>();
|
||||
String escapedTag = "";
|
||||
final StringBuilder escapedTag = new StringBuilder();
|
||||
boolean isEscaped = false;
|
||||
for (final String tag : tagList) {
|
||||
if (tag.startsWith("\"")) {
|
||||
escapedTag += tag.replace("\"", "");
|
||||
escapedTag.append(tag.replace("\"", ""));
|
||||
isEscaped = true;
|
||||
} else if (isEscaped) {
|
||||
if (tag.endsWith("\"")) {
|
||||
escapedTag += " " + tag.replace("\"", "");
|
||||
escapedTag.append(" ").append(tag.replace("\"", ""));
|
||||
isEscaped = false;
|
||||
tags.add(escapedTag);
|
||||
tags.add(escapedTag.toString());
|
||||
} else {
|
||||
escapedTag += " " + tag;
|
||||
escapedTag.append(" ").append(tag);
|
||||
}
|
||||
} else if (!tag.isEmpty()) {
|
||||
tags.add(tag);
|
||||
|
@ -0,0 +1,55 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
/**
|
||||
* Streaming format types used by YouTube in their streams.
|
||||
*
|
||||
* <p>
|
||||
* It is different from {@link org.schabi.newpipe.extractor.stream.DeliveryMethod delivery methods}!
|
||||
* </p>
|
||||
*/
|
||||
public enum DeliveryType {
|
||||
|
||||
/**
|
||||
* YouTube's progressive delivery method, which works with HTTP range headers.
|
||||
* (Note that official clients use the corresponding parameter instead.)
|
||||
*
|
||||
* <p>
|
||||
* Initialization and index ranges are available to get metadata (the corresponding values
|
||||
* are returned in the player response).
|
||||
* </p>
|
||||
*/
|
||||
PROGRESSIVE,
|
||||
|
||||
/**
|
||||
* YouTube's OTF delivery method which uses a sequence parameter to get segments of
|
||||
* streams.
|
||||
*
|
||||
* <p>
|
||||
* The first sequence (which can be fetched with the {@code &sq=0} parameter) contains all the
|
||||
* metadata needed to build the stream source (sidx boxes, segment length, segment count,
|
||||
* duration, ...).
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Only used for videos; mostly those with a small amount of views, or ended livestreams
|
||||
* which have just been re-encoded as normal videos.
|
||||
* </p>
|
||||
*/
|
||||
OTF,
|
||||
|
||||
/**
|
||||
* YouTube's delivery method for livestreams which uses a sequence parameter to get
|
||||
* segments of streams.
|
||||
*
|
||||
* <p>
|
||||
* Each sequence (which can be fetched with the {@code &sq=0} parameter) contains its own
|
||||
* metadata (sidx boxes, segment length, ...), which make no need of an initialization
|
||||
* segment.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Only used for livestreams (ended or running).
|
||||
* </p>
|
||||
*/
|
||||
LIVE
|
||||
}
|
@ -14,16 +14,20 @@ import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
public class ItagItem implements Serializable {
|
||||
|
||||
public class ItagItem {
|
||||
/**
|
||||
* List can be found here
|
||||
* https://github.com/ytdl-org/youtube-dl/blob/9fc5eaf/youtube_dl/extractor/youtube.py#L1071
|
||||
* List can be found here:
|
||||
* https://github.com/ytdl-org/youtube-dl/blob/e988fa4/youtube_dl/extractor/youtube.py#L1195
|
||||
*/
|
||||
private static final ItagItem[] ITAG_LIST = {
|
||||
/////////////////////////////////////////////////////
|
||||
// VIDEO ID Type Format Resolution FPS ///
|
||||
///////////////////////////////////////////////////
|
||||
// VIDEO ID Type Format Resolution FPS ////
|
||||
/////////////////////////////////////////////////////
|
||||
new ItagItem(17, VIDEO, v3GPP, "144p"),
|
||||
new ItagItem(36, VIDEO, v3GPP, "240p"),
|
||||
|
||||
@ -41,8 +45,8 @@ public class ItagItem {
|
||||
new ItagItem(45, VIDEO, WEBM, "720p"),
|
||||
new ItagItem(46, VIDEO, WEBM, "1080p"),
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// AUDIO ID ItagType Format Bitrate ///
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// AUDIO ID ItagType Format Bitrate //
|
||||
//////////////////////////////////////////////////////////////////
|
||||
new ItagItem(171, AUDIO, WEBMA, 128),
|
||||
new ItagItem(172, AUDIO, WEBMA, 256),
|
||||
@ -54,8 +58,8 @@ public class ItagItem {
|
||||
new ItagItem(251, AUDIO, WEBMA_OPUS, 160),
|
||||
|
||||
/// VIDEO ONLY ////////////////////////////////////////////
|
||||
// ID Type Format Resolution FPS ///
|
||||
/////////////////////////////////////////////////////////
|
||||
// ID Type Format Resolution FPS ////
|
||||
///////////////////////////////////////////////////////////
|
||||
new ItagItem(160, VIDEO_ONLY, MPEG_4, "144p"),
|
||||
new ItagItem(133, VIDEO_ONLY, MPEG_4, "240p"),
|
||||
new ItagItem(134, VIDEO_ONLY, MPEG_4, "360p"),
|
||||
@ -102,14 +106,26 @@ public class ItagItem {
|
||||
public static ItagItem getItag(final int itagId) throws ParsingException {
|
||||
for (final ItagItem item : ITAG_LIST) {
|
||||
if (itagId == item.id) {
|
||||
return item;
|
||||
return new ItagItem(item);
|
||||
}
|
||||
}
|
||||
throw new ParsingException("itag=" + itagId + " not supported");
|
||||
throw new ParsingException("itag " + itagId + " is not supported");
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Contructors and misc
|
||||
// Static constants
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
public static final int AVERAGE_BITRATE_UNKNOWN = -1;
|
||||
public static final int SAMPLE_RATE_UNKNOWN = -1;
|
||||
public static final int FPS_NOT_APPLICABLE_OR_UNKNOWN = -1;
|
||||
public static final int TARGET_DURATION_SEC_UNKNOWN = -1;
|
||||
public static final int AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN = -1;
|
||||
public static final long CONTENT_LENGTH_UNKNOWN = -1;
|
||||
public static final long APPROX_DURATION_MS_UNKNOWN = -1;
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Constructors and misc
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
public enum ItagType {
|
||||
@ -134,8 +150,6 @@ public class ItagItem {
|
||||
|
||||
/**
|
||||
* Constructor for videos.
|
||||
*
|
||||
* @param resolution string that will be used in the frontend
|
||||
*/
|
||||
public ItagItem(final int id,
|
||||
final ItagType type,
|
||||
@ -159,22 +173,58 @@ public class ItagItem {
|
||||
this.avgBitrate = avgBitrate;
|
||||
}
|
||||
|
||||
private final MediaFormat mediaFormat;
|
||||
|
||||
/**
|
||||
* Copy constructor of the {@link ItagItem} class.
|
||||
*
|
||||
* @param itagItem the {@link ItagItem} to copy its properties into a new {@link ItagItem}
|
||||
*/
|
||||
public ItagItem(@Nonnull final ItagItem itagItem) {
|
||||
this.mediaFormat = itagItem.mediaFormat;
|
||||
this.id = itagItem.id;
|
||||
this.itagType = itagItem.itagType;
|
||||
this.avgBitrate = itagItem.avgBitrate;
|
||||
this.sampleRate = itagItem.sampleRate;
|
||||
this.audioChannels = itagItem.audioChannels;
|
||||
this.resolutionString = itagItem.resolutionString;
|
||||
this.fps = itagItem.fps;
|
||||
this.bitrate = itagItem.bitrate;
|
||||
this.width = itagItem.width;
|
||||
this.height = itagItem.height;
|
||||
this.initStart = itagItem.initStart;
|
||||
this.initEnd = itagItem.initEnd;
|
||||
this.indexStart = itagItem.indexStart;
|
||||
this.indexEnd = itagItem.indexEnd;
|
||||
this.quality = itagItem.quality;
|
||||
this.codec = itagItem.codec;
|
||||
this.targetDurationSec = itagItem.targetDurationSec;
|
||||
this.approxDurationMs = itagItem.approxDurationMs;
|
||||
this.contentLength = itagItem.contentLength;
|
||||
}
|
||||
|
||||
public MediaFormat getMediaFormat() {
|
||||
return mediaFormat;
|
||||
}
|
||||
|
||||
private final MediaFormat mediaFormat;
|
||||
|
||||
public final int id;
|
||||
public final ItagType itagType;
|
||||
|
||||
// Audio fields
|
||||
public int avgBitrate = -1;
|
||||
/** @deprecated Use {@link #getAverageBitrate()} instead. */
|
||||
@Deprecated
|
||||
public int avgBitrate = AVERAGE_BITRATE_UNKNOWN;
|
||||
private int sampleRate = SAMPLE_RATE_UNKNOWN;
|
||||
private int audioChannels = AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN;
|
||||
|
||||
// Video fields
|
||||
/** @deprecated Use {@link #getResolutionString()} instead. */
|
||||
@Deprecated
|
||||
public String resolutionString;
|
||||
public int fps = -1;
|
||||
|
||||
/** @deprecated Use {@link #getFps()} and {@link #setFps(int)} instead. */
|
||||
@Deprecated
|
||||
public int fps = FPS_NOT_APPLICABLE_OR_UNKNOWN;
|
||||
|
||||
// Fields for Dash
|
||||
private int bitrate;
|
||||
@ -186,6 +236,9 @@ public class ItagItem {
|
||||
private int indexEnd;
|
||||
private String quality;
|
||||
private String codec;
|
||||
private int targetDurationSec = TARGET_DURATION_SEC_UNKNOWN;
|
||||
private long approxDurationMs = APPROX_DURATION_MS_UNKNOWN;
|
||||
private long contentLength = CONTENT_LENGTH_UNKNOWN;
|
||||
|
||||
public int getBitrate() {
|
||||
return bitrate;
|
||||
@ -211,6 +264,43 @@ public class ItagItem {
|
||||
this.height = height;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the frame rate.
|
||||
*
|
||||
* <p>
|
||||
* It is set to the {@code fps} value returned in the corresponding itag in the YouTube player
|
||||
* response.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* It defaults to the standard value associated with this itag.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Note that this value is only known for video itags, so {@link
|
||||
* #FPS_NOT_APPLICABLE_OR_UNKNOWN} is returned for non video itags.
|
||||
* </p>
|
||||
*
|
||||
* @return the frame rate or {@link #FPS_NOT_APPLICABLE_OR_UNKNOWN}
|
||||
*/
|
||||
public int getFps() {
|
||||
return fps;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the frame rate.
|
||||
*
|
||||
* <p>
|
||||
* It is only known for video itags, so {@link #FPS_NOT_APPLICABLE_OR_UNKNOWN} is set/used for
|
||||
* non video itags or if the sample rate value is less than or equal to 0.
|
||||
* </p>
|
||||
*
|
||||
* @param fps the frame rate
|
||||
*/
|
||||
public void setFps(final int fps) {
|
||||
this.fps = fps > 0 ? fps : FPS_NOT_APPLICABLE_OR_UNKNOWN;
|
||||
}
|
||||
|
||||
public int getInitStart() {
|
||||
return initStart;
|
||||
}
|
||||
@ -251,6 +341,21 @@ public class ItagItem {
|
||||
this.quality = quality;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the resolution string associated with this {@code ItagItem}.
|
||||
*
|
||||
* <p>
|
||||
* It is only known for video itags.
|
||||
* </p>
|
||||
*
|
||||
* @return the resolution string associated with this {@code ItagItem} or
|
||||
* {@code null}.
|
||||
*/
|
||||
@Nullable
|
||||
public String getResolutionString() {
|
||||
return resolutionString;
|
||||
}
|
||||
|
||||
public String getCodec() {
|
||||
return codec;
|
||||
}
|
||||
@ -258,4 +363,180 @@ public class ItagItem {
|
||||
public void setCodec(final String codec) {
|
||||
this.codec = codec;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the average bitrate.
|
||||
*
|
||||
* <p>
|
||||
* It is only known for audio itags, so {@link #AVERAGE_BITRATE_UNKNOWN} is always returned for
|
||||
* other itag types.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Bitrate of video itags and precise bitrate of audio itags can be known using
|
||||
* {@link #getBitrate()}.
|
||||
* </p>
|
||||
*
|
||||
* @return the average bitrate or {@link #AVERAGE_BITRATE_UNKNOWN}
|
||||
* @see #getBitrate()
|
||||
*/
|
||||
public int getAverageBitrate() {
|
||||
return avgBitrate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the sample rate.
|
||||
*
|
||||
* <p>
|
||||
* It is only known for audio itags, so {@link #SAMPLE_RATE_UNKNOWN} is returned for non audio
|
||||
* itags, or if the sample rate is unknown.
|
||||
* </p>
|
||||
*
|
||||
* @return the sample rate or {@link #SAMPLE_RATE_UNKNOWN}
|
||||
*/
|
||||
public int getSampleRate() {
|
||||
return sampleRate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the sample rate.
|
||||
*
|
||||
* <p>
|
||||
* It is only known for audio itags, so {@link #SAMPLE_RATE_UNKNOWN} is set/used for non audio
|
||||
* itags, or if the sample rate value is less than or equal to 0.
|
||||
* </p>
|
||||
*
|
||||
* @param sampleRate the sample rate of an audio itag
|
||||
*/
|
||||
public void setSampleRate(final int sampleRate) {
|
||||
this.sampleRate = sampleRate > 0 ? sampleRate : SAMPLE_RATE_UNKNOWN;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of audio channels.
|
||||
*
|
||||
* <p>
|
||||
* It is only known for audio itags, so {@link #AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN} is
|
||||
* returned for non audio itags, or if it is unknown.
|
||||
* </p>
|
||||
*
|
||||
* @return the number of audio channels or {@link #AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN}
|
||||
*/
|
||||
public int getAudioChannels() {
|
||||
return audioChannels;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the number of audio channels.
|
||||
*
|
||||
* <p>
|
||||
* It is only known for audio itags, so {@link #AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN} is
|
||||
* set/used for non audio itags, or if the {@code audioChannels} value is less than or equal to
|
||||
* 0.
|
||||
* </p>
|
||||
*
|
||||
* @param audioChannels the number of audio channels of an audio itag
|
||||
*/
|
||||
public void setAudioChannels(final int audioChannels) {
|
||||
this.audioChannels = audioChannels > 0
|
||||
? audioChannels
|
||||
: AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the {@code targetDurationSec} value.
|
||||
*
|
||||
* <p>
|
||||
* This value is the average time in seconds of the duration of sequences of livestreams and
|
||||
* ended livestreams. It is only returned by YouTube for these stream types, and makes no sense
|
||||
* for videos, so {@link #TARGET_DURATION_SEC_UNKNOWN} is returned for those.
|
||||
* </p>
|
||||
*
|
||||
* @return the {@code targetDurationSec} value or {@link #TARGET_DURATION_SEC_UNKNOWN}
|
||||
*/
|
||||
public int getTargetDurationSec() {
|
||||
return targetDurationSec;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the {@code targetDurationSec} value.
|
||||
*
|
||||
* <p>
|
||||
* This value is the average time in seconds of the duration of sequences of livestreams and
|
||||
* ended livestreams.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* It is only returned for these stream types by YouTube and makes no sense for videos, so
|
||||
* {@link #TARGET_DURATION_SEC_UNKNOWN} will be set/used for video streams or if this value is
|
||||
* less than or equal to 0.
|
||||
* </p>
|
||||
*
|
||||
* @param targetDurationSec the target duration of a segment of streams which are using the
|
||||
* live delivery method type
|
||||
*/
|
||||
public void setTargetDurationSec(final int targetDurationSec) {
|
||||
this.targetDurationSec = targetDurationSec > 0
|
||||
? targetDurationSec
|
||||
: TARGET_DURATION_SEC_UNKNOWN;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the {@code approxDurationMs} value.
|
||||
*
|
||||
* <p>
|
||||
* It is only known for DASH progressive streams, so {@link #APPROX_DURATION_MS_UNKNOWN} is
|
||||
* returned for other stream types or if this value is less than or equal to 0.
|
||||
* </p>
|
||||
*
|
||||
* @return the {@code approxDurationMs} value or {@link #APPROX_DURATION_MS_UNKNOWN}
|
||||
*/
|
||||
public long getApproxDurationMs() {
|
||||
return approxDurationMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the {@code approxDurationMs} value.
|
||||
*
|
||||
* <p>
|
||||
* It is only known for DASH progressive streams, so {@link #APPROX_DURATION_MS_UNKNOWN} is
|
||||
* set/used for other stream types or if this value is less than or equal to 0.
|
||||
* </p>
|
||||
*
|
||||
* @param approxDurationMs the approximate duration of a DASH progressive stream, in
|
||||
* milliseconds
|
||||
*/
|
||||
public void setApproxDurationMs(final long approxDurationMs) {
|
||||
this.approxDurationMs = approxDurationMs > 0
|
||||
? approxDurationMs
|
||||
: APPROX_DURATION_MS_UNKNOWN;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the {@code contentLength} value.
|
||||
*
|
||||
* <p>
|
||||
* It is only known for DASH progressive streams, so {@link #CONTENT_LENGTH_UNKNOWN} is
|
||||
* returned for other stream types or if this value is less than or equal to 0.
|
||||
* </p>
|
||||
*
|
||||
* @return the {@code contentLength} value or {@link #CONTENT_LENGTH_UNKNOWN}
|
||||
*/
|
||||
public long getContentLength() {
|
||||
return contentLength;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the content length of stream.
|
||||
*
|
||||
* <p>
|
||||
* It is only known for DASH progressive streams, so {@link #CONTENT_LENGTH_UNKNOWN} is
|
||||
* set/used for other stream types or if this value is less than or equal to 0.
|
||||
* </p>
|
||||
*
|
||||
* @param contentLength the content length of a DASH progressive stream
|
||||
*/
|
||||
public void setContentLength(final long contentLength) {
|
||||
this.contentLength = contentLength > 0 ? contentLength : CONTENT_LENGTH_UNKNOWN;
|
||||
}
|
||||
}
|
||||
|
@ -71,6 +71,7 @@ import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Random;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
@ -246,6 +247,11 @@ public final class YoutubeParsingHelper {
|
||||
private static final String FEED_BASE_CHANNEL_ID =
|
||||
"https://www.youtube.com/feeds/videos.xml?channel_id=";
|
||||
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
|
||||
private static final Pattern C_WEB_PATTERN = Pattern.compile("&c=WEB");
|
||||
private static final Pattern C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN =
|
||||
Pattern.compile("&c=TVHTML5_SIMPLY_EMBEDDED_PLAYER");
|
||||
private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
|
||||
private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
|
||||
|
||||
private static boolean isGoogleURL(final String url) {
|
||||
final String cachedUrl = extractCachedUrlIfNeeded(url);
|
||||
@ -1190,7 +1196,7 @@ public final class YoutubeParsingHelper {
|
||||
@Nonnull final Localization localization,
|
||||
@Nonnull final ContentCountry contentCountry,
|
||||
@Nonnull final String videoId) {
|
||||
// @formatter:off
|
||||
// @formatter:off
|
||||
return JsonObject.builder()
|
||||
.object("context")
|
||||
.object("client")
|
||||
@ -1258,8 +1264,7 @@ public final class YoutubeParsingHelper {
|
||||
// Spoofing an Android 12 device with the hardcoded version of the Android app
|
||||
return "com.google.android.youtube/" + MOBILE_YOUTUBE_CLIENT_VERSION
|
||||
+ " (Linux; U; Android 12; "
|
||||
+ (localization != null ? localization.getCountryCode()
|
||||
: Localization.DEFAULT.getCountryCode())
|
||||
+ (localization != null ? localization : Localization.DEFAULT).getCountryCode()
|
||||
+ ") gzip";
|
||||
}
|
||||
|
||||
@ -1278,10 +1283,8 @@ public final class YoutubeParsingHelper {
|
||||
public static String getIosUserAgent(@Nullable final Localization localization) {
|
||||
// Spoofing an iPhone running iOS 15.4 with the hardcoded mobile client version
|
||||
return "com.google.ios.youtube/" + MOBILE_YOUTUBE_CLIENT_VERSION
|
||||
+ "(" + IOS_DEVICE_MODEL
|
||||
+ "; U; CPU iOS 15_4 like Mac OS X; "
|
||||
+ (localization != null ? localization.getCountryCode()
|
||||
: Localization.DEFAULT.getCountryCode())
|
||||
+ "(" + IOS_DEVICE_MODEL + "; U; CPU iOS 15_4 like Mac OS X; "
|
||||
+ (localization != null ? localization : Localization.DEFAULT).getCountryCode()
|
||||
+ ")";
|
||||
}
|
||||
|
||||
@ -1588,4 +1591,46 @@ public final class YoutubeParsingHelper {
|
||||
return RandomStringFromAlphabetGenerator.generate(
|
||||
CONTENT_PLAYBACK_NONCE_ALPHABET, 12, numberGenerator);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the streaming URL is from the YouTube {@code WEB} client.
|
||||
*
|
||||
* @param url the streaming URL to be checked.
|
||||
* @return true if it's a {@code WEB} streaming URL, false otherwise
|
||||
*/
|
||||
public static boolean isWebStreamingUrl(@Nonnull final String url) {
|
||||
return Parser.isMatch(C_WEB_PATTERN, url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the streaming URL is a URL from the YouTube {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER}
|
||||
* client.
|
||||
*
|
||||
* @param url the streaming URL on which check if it's a {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER}
|
||||
* streaming URL.
|
||||
* @return true if it's a {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} streaming URL, false otherwise
|
||||
*/
|
||||
public static boolean isTvHtml5SimplyEmbeddedPlayerStreamingUrl(@Nonnull final String url) {
|
||||
return Parser.isMatch(C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN, url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the streaming URL is a URL from the YouTube {@code ANDROID} client.
|
||||
*
|
||||
* @param url the streaming URL to be checked.
|
||||
* @return true if it's a {@code ANDROID} streaming URL, false otherwise
|
||||
*/
|
||||
public static boolean isAndroidStreamingUrl(@Nonnull final String url) {
|
||||
return Parser.isMatch(C_ANDROID_PATTERN, url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the streaming URL is a URL from the YouTube {@code IOS} client.
|
||||
*
|
||||
* @param url the streaming URL on which check if it's a {@code IOS} streaming URL.
|
||||
* @return true if it's a {@code IOS} streaming URL, false otherwise
|
||||
*/
|
||||
public static boolean isIosStreamingUrl(@Nonnull final String url) {
|
||||
return Parser.isMatch(C_IOS_PATTERN, url);
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,63 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
/**
|
||||
* Exception that is thrown when a YouTube DASH manifest creator encounters a problem
|
||||
* while creating a manifest.
|
||||
*/
|
||||
public final class CreationException extends RuntimeException {
|
||||
|
||||
/**
|
||||
* Create a new {@link CreationException} with a detail message.
|
||||
*
|
||||
* @param message the detail message to add in the exception
|
||||
*/
|
||||
public CreationException(final String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new {@link CreationException} with a detail message and a cause.
|
||||
* @param message the detail message to add in the exception
|
||||
* @param cause the exception cause of this {@link CreationException}
|
||||
*/
|
||||
public CreationException(final String message, final Exception cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
// Methods to create exceptions easily without having to use big exception messages and to
|
||||
// reduce duplication
|
||||
|
||||
/**
|
||||
* Create a new {@link CreationException} with a cause and the following detail message format:
|
||||
* <br>
|
||||
* {@code "Could not add " + element + " element", cause}, where {@code element} is an element
|
||||
* of a DASH manifest.
|
||||
*
|
||||
* @param element the element which was not added to the DASH document
|
||||
* @param cause the exception which prevented addition of the element to the DASH document
|
||||
* @return a new {@link CreationException}
|
||||
*/
|
||||
@Nonnull
|
||||
public static CreationException couldNotAddElement(final String element,
|
||||
final Exception cause) {
|
||||
return new CreationException("Could not add " + element + " element", cause);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new {@link CreationException} with a cause and the following detail message format:
|
||||
* <br>
|
||||
* {@code "Could not add " + element + " element: " + reason}, where {@code element} is an
|
||||
* element of a DASH manifest and {@code reason} the reason why this element cannot be added to
|
||||
* the DASH document.
|
||||
*
|
||||
* @param element the element which was not added to the DASH document
|
||||
* @param reason the reason message of why the element has been not added to the DASH document
|
||||
* @return a new {@link CreationException}
|
||||
*/
|
||||
@Nonnull
|
||||
public static CreationException couldNotAddElement(final String element, final String reason) {
|
||||
return new CreationException("Could not add " + element + " element: " + reason);
|
||||
}
|
||||
}
|
@ -0,0 +1,757 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
|
||||
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
|
||||
import org.w3c.dom.Attr;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.xml.XMLConstants;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.transform.OutputKeys;
|
||||
import javax.xml.transform.Transformer;
|
||||
import javax.xml.transform.TransformerException;
|
||||
import javax.xml.transform.TransformerFactory;
|
||||
import javax.xml.transform.dom.DOMSource;
|
||||
import javax.xml.transform.stream.StreamResult;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
/**
|
||||
* Utilities and constants for YouTube DASH manifest creators.
|
||||
*
|
||||
* <p>
|
||||
* This class includes common methods of manifest creators and useful constants.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Generation of DASH documents and their conversion as a string is done using external classes
|
||||
* from {@link org.w3c.dom} and {@link javax.xml} packages.
|
||||
* </p>
|
||||
*/
|
||||
public final class YoutubeDashManifestCreatorsUtils {
|
||||
|
||||
private YoutubeDashManifestCreatorsUtils() {
|
||||
}
|
||||
|
||||
/**
|
||||
* The redirect count limit that this class uses, which is the same limit as OkHttp.
|
||||
*/
|
||||
public static final int MAXIMUM_REDIRECT_COUNT = 20;
|
||||
|
||||
/**
|
||||
* URL parameter of the first sequence for live, post-live-DVR and OTF streams.
|
||||
*/
|
||||
public static final String SQ_0 = "&sq=0";
|
||||
|
||||
/**
|
||||
* URL parameter of the first stream request made by official clients.
|
||||
*/
|
||||
public static final String RN_0 = "&rn=0";
|
||||
|
||||
/**
|
||||
* URL parameter specific to web clients. When this param is added, if a redirection occurs,
|
||||
* the server will not redirect clients to the redirect URL. Instead, it will provide this URL
|
||||
* as the response body.
|
||||
*/
|
||||
public static final String ALR_YES = "&alr=yes";
|
||||
|
||||
// XML elements of DASH MPD manifests
|
||||
// see https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html
|
||||
public static final String MPD = "MPD";
|
||||
public static final String PERIOD = "Period";
|
||||
public static final String ADAPTATION_SET = "AdaptationSet";
|
||||
public static final String ROLE = "Role";
|
||||
public static final String REPRESENTATION = "Representation";
|
||||
public static final String AUDIO_CHANNEL_CONFIGURATION = "AudioChannelConfiguration";
|
||||
public static final String SEGMENT_TEMPLATE = "SegmentTemplate";
|
||||
public static final String SEGMENT_TIMELINE = "SegmentTimeline";
|
||||
public static final String BASE_URL = "BaseURL";
|
||||
public static final String SEGMENT_BASE = "SegmentBase";
|
||||
public static final String INITIALIZATION = "Initialization";
|
||||
|
||||
/**
|
||||
* Create an attribute with {@link Document#createAttribute(String)}, assign to it the provided
|
||||
* name and value, then add it to the provided element using {@link
|
||||
* Element#setAttributeNode(Attr)}.
|
||||
*
|
||||
* @param element element to which to add the created node
|
||||
* @param doc document to use to create the attribute
|
||||
* @param name name of the attribute
|
||||
* @param value value of the attribute, will be set using {@link Attr#setValue(String)}
|
||||
*/
|
||||
public static void setAttribute(final Element element,
|
||||
final Document doc,
|
||||
final String name,
|
||||
final String value) {
|
||||
final Attr attr = doc.createAttribute(name);
|
||||
attr.setValue(value);
|
||||
element.setAttributeNode(attr);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a {@link Document} with common manifest creator elements added to it.
|
||||
*
|
||||
* <p>
|
||||
* Those are:
|
||||
* <ul>
|
||||
* <li>{@code MPD} (using {@link #generateDocumentAndMpdElement(long)});</li>
|
||||
* <li>{@code Period} (using {@link #generatePeriodElement(Document)});</li>
|
||||
* <li>{@code AdaptationSet} (using {@link #generateAdaptationSetElement(Document,
|
||||
* ItagItem)});</li>
|
||||
* <li>{@code Role} (using {@link #generateRoleElement(Document)});</li>
|
||||
* <li>{@code Representation} (using {@link #generateRepresentationElement(Document,
|
||||
* ItagItem)});</li>
|
||||
* <li>and, for audio streams, {@code AudioChannelConfiguration} (using
|
||||
* {@link #generateAudioChannelConfigurationElement(Document, ItagItem)}).</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* @param itagItem the {@link ItagItem} associated to the stream, which must not be null
|
||||
* @param streamDuration the duration of the stream, in milliseconds
|
||||
* @return a {@link Document} with the common elements added in it
|
||||
*/
|
||||
@Nonnull
|
||||
public static Document generateDocumentAndDoCommonElementsGeneration(
|
||||
@Nonnull final ItagItem itagItem,
|
||||
final long streamDuration) throws CreationException {
|
||||
final Document doc = generateDocumentAndMpdElement(streamDuration);
|
||||
|
||||
generatePeriodElement(doc);
|
||||
generateAdaptationSetElement(doc, itagItem);
|
||||
generateRoleElement(doc);
|
||||
generateRepresentationElement(doc, itagItem);
|
||||
if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
|
||||
generateAudioChannelConfigurationElement(doc, itagItem);
|
||||
}
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a {@link Document} instance and generate the {@code <MPD>} element of the manifest.
|
||||
*
|
||||
* <p>
|
||||
* The generated {@code <MPD>} element looks like the manifest returned into the player
|
||||
* response of videos:
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* {@code <MPD xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
* xmlns="urn:mpeg:DASH:schema:MPD:2011"
|
||||
* xsi:schemaLocation="urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd" minBufferTime="PT1.500S"
|
||||
* profiles="urn:mpeg:dash:profile:isoff-main:2011" type="static"
|
||||
* mediaPresentationDuration="PT$duration$S">}
|
||||
* (where {@code $duration$} represents the duration in seconds (a number with 3 digits after
|
||||
* the decimal point)).
|
||||
* </p>
|
||||
*
|
||||
* @param duration the duration of the stream, in milliseconds
|
||||
* @return a {@link Document} instance which contains a {@code <MPD>} element
|
||||
*/
|
||||
@Nonnull
|
||||
public static Document generateDocumentAndMpdElement(final long duration)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Document doc = newDocument();
|
||||
|
||||
final Element mpdElement = doc.createElement(MPD);
|
||||
doc.appendChild(mpdElement);
|
||||
|
||||
setAttribute(mpdElement, doc, "xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
|
||||
setAttribute(mpdElement, doc, "xmlns", "urn:mpeg:DASH:schema:MPD:2011");
|
||||
setAttribute(mpdElement, doc, "xsi:schemaLocation",
|
||||
"urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd");
|
||||
setAttribute(mpdElement, doc, "minBufferTime", "PT1.500S");
|
||||
setAttribute(mpdElement, doc, "profiles", "urn:mpeg:dash:profile:full:2011");
|
||||
setAttribute(mpdElement, doc, "type", "static");
|
||||
setAttribute(mpdElement, doc, "mediaPresentationDuration",
|
||||
String.format(Locale.ENGLISH, "PT%.3fS", duration / 1000.0));
|
||||
|
||||
return doc;
|
||||
} catch (final Exception e) {
|
||||
throw new CreationException(
|
||||
"Could not generate the DASH manifest or append the MPD doc to it", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <Period>} element, appended as a child of the {@code <MPD>} element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <MPD>} element needs to be generated before this element with
|
||||
* {@link #generateDocumentAndMpdElement(long)}.
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the the {@code <Period>} element will be appended
|
||||
*/
|
||||
public static void generatePeriodElement(@Nonnull final Document doc)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element mpdElement = (Element) doc.getElementsByTagName(MPD).item(0);
|
||||
final Element periodElement = doc.createElement(PERIOD);
|
||||
mpdElement.appendChild(periodElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(PERIOD, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <AdaptationSet>} element, appended as a child of the {@code <Period>}
|
||||
* element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Period>} element needs to be generated before this element with
|
||||
* {@link #generatePeriodElement(Document)}.
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <Period>} element will be appended
|
||||
* @param itagItem the {@link ItagItem} corresponding to the stream, which must not be null
|
||||
*/
|
||||
public static void generateAdaptationSetElement(@Nonnull final Document doc,
|
||||
@Nonnull final ItagItem itagItem)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element periodElement = (Element) doc.getElementsByTagName(PERIOD)
|
||||
.item(0);
|
||||
final Element adaptationSetElement = doc.createElement(ADAPTATION_SET);
|
||||
|
||||
setAttribute(adaptationSetElement, doc, "id", "0");
|
||||
|
||||
final MediaFormat mediaFormat = itagItem.getMediaFormat();
|
||||
if (mediaFormat == null || isNullOrEmpty(mediaFormat.getMimeType())) {
|
||||
throw CreationException.couldNotAddElement(ADAPTATION_SET,
|
||||
"the MediaFormat or its mime type is null or empty");
|
||||
}
|
||||
|
||||
setAttribute(adaptationSetElement, doc, "mimeType", mediaFormat.getMimeType());
|
||||
setAttribute(adaptationSetElement, doc, "subsegmentAlignment", "true");
|
||||
|
||||
periodElement.appendChild(adaptationSetElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(ADAPTATION_SET, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <Role>} element, appended as a child of the {@code <AdaptationSet>}
|
||||
* element.
|
||||
*
|
||||
* <p>
|
||||
* This element, with its attributes and values, is:
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* {@code <Role schemeIdUri="urn:mpeg:DASH:role:2011" value="main"/>}
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <AdaptationSet>} element needs to be generated before this element with
|
||||
* {@link #generateAdaptationSetElement(Document, ItagItem)}).
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the the {@code <Role>} element will be appended
|
||||
*/
|
||||
public static void generateRoleElement(@Nonnull final Document doc)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element adaptationSetElement = (Element) doc.getElementsByTagName(
|
||||
ADAPTATION_SET).item(0);
|
||||
final Element roleElement = doc.createElement(ROLE);
|
||||
|
||||
setAttribute(roleElement, doc, "schemeIdUri", "urn:mpeg:DASH:role:2011");
|
||||
setAttribute(roleElement, doc, "value", "main");
|
||||
|
||||
adaptationSetElement.appendChild(roleElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(ROLE, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <Representation>} element, appended as a child of the
|
||||
* {@code <AdaptationSet>} element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <AdaptationSet>} element needs to be generated before this element with
|
||||
* {@link #generateAdaptationSetElement(Document, ItagItem)}).
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the the {@code <SegmentTimeline>} element will be
|
||||
* appended
|
||||
* @param itagItem the {@link ItagItem} to use, which must not be null
|
||||
*/
|
||||
public static void generateRepresentationElement(@Nonnull final Document doc,
|
||||
@Nonnull final ItagItem itagItem)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element adaptationSetElement = (Element) doc.getElementsByTagName(
|
||||
ADAPTATION_SET).item(0);
|
||||
final Element representationElement = doc.createElement(REPRESENTATION);
|
||||
|
||||
final int id = itagItem.id;
|
||||
if (id <= 0) {
|
||||
throw CreationException.couldNotAddElement(REPRESENTATION,
|
||||
"the id of the ItagItem is <= 0");
|
||||
}
|
||||
setAttribute(representationElement, doc, "id", String.valueOf(id));
|
||||
|
||||
final String codec = itagItem.getCodec();
|
||||
if (isNullOrEmpty(codec)) {
|
||||
throw CreationException.couldNotAddElement(ADAPTATION_SET,
|
||||
"the codec value of the ItagItem is null or empty");
|
||||
}
|
||||
setAttribute(representationElement, doc, "codecs", codec);
|
||||
setAttribute(representationElement, doc, "startWithSAP", "1");
|
||||
setAttribute(representationElement, doc, "maxPlayoutRate", "1");
|
||||
|
||||
final int bitrate = itagItem.getBitrate();
|
||||
if (bitrate <= 0) {
|
||||
throw CreationException.couldNotAddElement(REPRESENTATION,
|
||||
"the bitrate of the ItagItem is <= 0");
|
||||
}
|
||||
setAttribute(representationElement, doc, "bandwidth", String.valueOf(bitrate));
|
||||
|
||||
if (itagItem.itagType == ItagItem.ItagType.VIDEO
|
||||
|| itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) {
|
||||
final int height = itagItem.getHeight();
|
||||
final int width = itagItem.getWidth();
|
||||
if (height <= 0 && width <= 0) {
|
||||
throw CreationException.couldNotAddElement(REPRESENTATION,
|
||||
"both width and height of the ItagItem are <= 0");
|
||||
}
|
||||
|
||||
if (width > 0) {
|
||||
setAttribute(representationElement, doc, "width", String.valueOf(width));
|
||||
}
|
||||
setAttribute(representationElement, doc, "height",
|
||||
String.valueOf(itagItem.getHeight()));
|
||||
|
||||
final int fps = itagItem.getFps();
|
||||
if (fps > 0) {
|
||||
setAttribute(representationElement, doc, "frameRate", String.valueOf(fps));
|
||||
}
|
||||
}
|
||||
|
||||
if (itagItem.itagType == ItagItem.ItagType.AUDIO && itagItem.getSampleRate() > 0) {
|
||||
final Attr audioSamplingRateAttribute = doc.createAttribute(
|
||||
"audioSamplingRate");
|
||||
audioSamplingRateAttribute.setValue(String.valueOf(itagItem.getSampleRate()));
|
||||
}
|
||||
|
||||
adaptationSetElement.appendChild(representationElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(REPRESENTATION, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <AudioChannelConfiguration>} element, appended as a child of the
|
||||
* {@code <Representation>} element.
|
||||
*
|
||||
* <p>
|
||||
* This method is only used when generating DASH manifests of audio streams.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* It will produce the following element:
|
||||
* <br>
|
||||
* {@code <AudioChannelConfiguration
|
||||
* schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011"
|
||||
* value="audioChannelsValue"}
|
||||
* <br>
|
||||
* (where {@code audioChannelsValue} is get from the {@link ItagItem} passed as the second
|
||||
* parameter of this method)
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Representation>} element needs to be generated before this element with
|
||||
* {@link #generateRepresentationElement(Document, ItagItem)}).
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <AudioChannelConfiguration>} element will
|
||||
* be appended
|
||||
* @param itagItem the {@link ItagItem} to use, which must not be null
|
||||
*/
|
||||
public static void generateAudioChannelConfigurationElement(
|
||||
@Nonnull final Document doc,
|
||||
@Nonnull final ItagItem itagItem) throws CreationException {
|
||||
try {
|
||||
final Element representationElement = (Element) doc.getElementsByTagName(
|
||||
REPRESENTATION).item(0);
|
||||
final Element audioChannelConfigurationElement = doc.createElement(
|
||||
AUDIO_CHANNEL_CONFIGURATION);
|
||||
|
||||
setAttribute(audioChannelConfigurationElement, doc, "schemeIdUri",
|
||||
"urn:mpeg:dash:23003:3:audio_channel_configuration:2011");
|
||||
|
||||
if (itagItem.getAudioChannels() <= 0) {
|
||||
throw new CreationException("the number of audioChannels in the ItagItem is <= 0: "
|
||||
+ itagItem.getAudioChannels());
|
||||
}
|
||||
setAttribute(audioChannelConfigurationElement, doc, "value",
|
||||
String.valueOf(itagItem.getAudioChannels()));
|
||||
|
||||
representationElement.appendChild(audioChannelConfigurationElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(AUDIO_CHANNEL_CONFIGURATION, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a DASH manifest {@link Document doc} to a string and cache it.
|
||||
*
|
||||
* @param originalBaseStreamingUrl the original base URL of the stream
|
||||
* @param doc the doc to be converted
|
||||
* @param manifestCreatorCache the {@link ManifestCreatorCache} on which store the string
|
||||
* generated
|
||||
* @return the DASH manifest {@link Document doc} converted to a string
|
||||
*/
|
||||
public static String buildAndCacheResult(
|
||||
@Nonnull final String originalBaseStreamingUrl,
|
||||
@Nonnull final Document doc,
|
||||
@Nonnull final ManifestCreatorCache<String, String> manifestCreatorCache)
|
||||
throws CreationException {
|
||||
|
||||
try {
|
||||
final String documentXml = documentToXml(doc);
|
||||
manifestCreatorCache.put(originalBaseStreamingUrl, documentXml);
|
||||
return documentXml;
|
||||
} catch (final Exception e) {
|
||||
throw new CreationException(
|
||||
"Could not convert the DASH manifest generated to a string", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <SegmentTemplate>} element, appended as a child of the
|
||||
* {@code <Representation>} element.
|
||||
*
|
||||
* <p>
|
||||
* This method is only used when generating DASH manifests from OTF and post-live-DVR streams.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* It will produce a {@code <SegmentTemplate>} element with the following attributes:
|
||||
* <ul>
|
||||
* <li>{@code startNumber}, which takes the value {@code 0} for post-live-DVR streams and
|
||||
* {@code 1} for OTF streams;</li>
|
||||
* <li>{@code timescale}, which is always {@code 1000};</li>
|
||||
* <li>{@code media}, which is the base URL of the stream on which is appended
|
||||
* {@code &sq=$Number$};</li>
|
||||
* <li>{@code initialization} (only for OTF streams), which is the base URL of the stream
|
||||
* on which is appended {@link #SQ_0}.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Representation>} element needs to be generated before this element with
|
||||
* {@link #generateRepresentationElement(Document, ItagItem)}).
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <SegmentTemplate>} element will
|
||||
* be appended
|
||||
* @param baseUrl the base URL of the OTF/post-live-DVR stream
|
||||
* @param deliveryType the stream {@link DeliveryType delivery type}, which must be either
|
||||
* {@link DeliveryType#OTF OTF} or {@link DeliveryType#LIVE LIVE}
|
||||
*/
|
||||
public static void generateSegmentTemplateElement(@Nonnull final Document doc,
|
||||
@Nonnull final String baseUrl,
|
||||
final DeliveryType deliveryType)
|
||||
throws CreationException {
|
||||
if (deliveryType != DeliveryType.OTF && deliveryType != DeliveryType.LIVE) {
|
||||
throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, "invalid delivery type: "
|
||||
+ deliveryType);
|
||||
}
|
||||
|
||||
try {
|
||||
final Element representationElement = (Element) doc.getElementsByTagName(
|
||||
REPRESENTATION).item(0);
|
||||
final Element segmentTemplateElement = doc.createElement(SEGMENT_TEMPLATE);
|
||||
|
||||
// The first sequence of post DVR streams is the beginning of the video stream and not
|
||||
// an initialization segment
|
||||
setAttribute(segmentTemplateElement, doc, "startNumber",
|
||||
deliveryType == DeliveryType.LIVE ? "0" : "1");
|
||||
setAttribute(segmentTemplateElement, doc, "timescale", "1000");
|
||||
|
||||
// Post-live-DVR/ended livestreams streams don't require an initialization sequence
|
||||
if (deliveryType != DeliveryType.LIVE) {
|
||||
setAttribute(segmentTemplateElement, doc, "initialization", baseUrl + SQ_0);
|
||||
}
|
||||
|
||||
setAttribute(segmentTemplateElement, doc, "media", baseUrl + "&sq=$Number$");
|
||||
|
||||
representationElement.appendChild(segmentTemplateElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <SegmentTimeline>} element, appended as a child of the
|
||||
* {@code <SegmentTemplate>} element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <SegmentTemplate>} element needs to be generated before this element with
|
||||
* {@link #generateSegmentTemplateElement(Document, String, DeliveryType)}.
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the the {@code <SegmentTimeline>} element will be
|
||||
* appended
|
||||
*/
|
||||
public static void generateSegmentTimelineElement(@Nonnull final Document doc)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element segmentTemplateElement = (Element) doc.getElementsByTagName(
|
||||
SEGMENT_TEMPLATE).item(0);
|
||||
final Element segmentTimelineElement = doc.createElement(SEGMENT_TIMELINE);
|
||||
|
||||
segmentTemplateElement.appendChild(segmentTimelineElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(SEGMENT_TIMELINE, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the "initialization" {@link Response response} of a stream.
|
||||
*
|
||||
* <p>This method fetches, for OTF streams and for post-live-DVR streams:
|
||||
* <ul>
|
||||
* <li>the base URL of the stream, to which are appended {@link #SQ_0} and
|
||||
* {@link #RN_0} parameters, with a {@code GET} request for streaming URLs from HTML5
|
||||
* clients and a {@code POST} request for the ones from the {@code ANDROID} and the
|
||||
* {@code IOS} clients;</li>
|
||||
* <li>for streaming URLs from HTML5 clients, the {@link #ALR_YES} param is also added.
|
||||
* </li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* @param baseStreamingUrl the base URL of the stream, which must not be null
|
||||
* @param itagItem the {@link ItagItem} of stream, which must not be null
|
||||
* @param deliveryType the {@link DeliveryType} of the stream
|
||||
* @return the "initialization" response, without redirections on the network on which the
|
||||
* request(s) is/are made
|
||||
*/
|
||||
@SuppressWarnings("checkstyle:FinalParameters")
|
||||
@Nonnull
|
||||
public static Response getInitializationResponse(@Nonnull String baseStreamingUrl,
|
||||
@Nonnull final ItagItem itagItem,
|
||||
final DeliveryType deliveryType)
|
||||
throws CreationException {
|
||||
final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl)
|
||||
|| isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl);
|
||||
final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl);
|
||||
final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl);
|
||||
if (isHtml5StreamingUrl) {
|
||||
baseStreamingUrl += ALR_YES;
|
||||
}
|
||||
baseStreamingUrl = appendRnSqParamsIfNeeded(baseStreamingUrl, deliveryType);
|
||||
|
||||
final Downloader downloader = NewPipe.getDownloader();
|
||||
if (isHtml5StreamingUrl) {
|
||||
final String mimeTypeExpected = itagItem.getMediaFormat().getMimeType();
|
||||
if (!isNullOrEmpty(mimeTypeExpected)) {
|
||||
return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl,
|
||||
mimeTypeExpected);
|
||||
}
|
||||
} else if (isAndroidStreamingUrl || isIosStreamingUrl) {
|
||||
try {
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
headers.put("User-Agent", Collections.singletonList(
|
||||
isAndroidStreamingUrl ? getAndroidUserAgent(null)
|
||||
: getIosUserAgent(null)));
|
||||
final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8);
|
||||
return downloader.post(baseStreamingUrl, headers, emptyBody);
|
||||
} catch (final IOException | ExtractionException e) {
|
||||
throw new CreationException("Could not get the "
|
||||
+ (isIosStreamingUrl ? "ANDROID" : "IOS") + " streaming URL response", e);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return downloader.get(baseStreamingUrl);
|
||||
} catch (final IOException | ExtractionException e) {
|
||||
throw new CreationException("Could not get the streaming URL response", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a new {@link DocumentBuilder} secured from XXE attacks, on platforms which
|
||||
* support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
|
||||
* {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link DocumentBuilderFactory} instances.
|
||||
*
|
||||
* @return an instance of {@link Document} secured against XXE attacks on supported platforms,
|
||||
* that should then be convertible to an XML string without security problems
|
||||
*/
|
||||
private static Document newDocument() throws ParserConfigurationException {
|
||||
final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
|
||||
try {
|
||||
documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
|
||||
documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
|
||||
} catch (final Exception ignored) {
|
||||
// Ignore exceptions as setting these attributes to secure XML generation is not
|
||||
// supported by all platforms (like the Android implementation)
|
||||
}
|
||||
|
||||
return documentBuilderFactory.newDocumentBuilder().newDocument();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a new {@link TransformerFactory} secured from XXE attacks, on platforms which
|
||||
* support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
|
||||
* {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances.
|
||||
*
|
||||
* @param doc the doc to convert, which must have been created using {@link #newDocument()} to
|
||||
* properly prevent XXE attacks
|
||||
* @return the doc converted to an XML string, making sure there can't be XXE attacks
|
||||
*/
|
||||
// Sonar warning is suppressed because it is still shown even if we apply its solution
|
||||
@SuppressWarnings("squid:S2755")
|
||||
private static String documentToXml(@Nonnull final Document doc)
|
||||
throws TransformerException {
|
||||
|
||||
final TransformerFactory transformerFactory = TransformerFactory.newInstance();
|
||||
try {
|
||||
transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
|
||||
transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
|
||||
} catch (final Exception ignored) {
|
||||
// Ignore exceptions as setting these attributes to secure XML generation is not
|
||||
// supported by all platforms (like the Android implementation)
|
||||
}
|
||||
|
||||
final Transformer transformer = transformerFactory.newTransformer();
|
||||
transformer.setOutputProperty(OutputKeys.VERSION, "1.0");
|
||||
transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
|
||||
transformer.setOutputProperty(OutputKeys.STANDALONE, "no");
|
||||
|
||||
final StringWriter result = new StringWriter();
|
||||
transformer.transform(new DOMSource(doc), new StreamResult(result));
|
||||
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Append {@link #SQ_0} for post-live-DVR and OTF streams and {@link #RN_0} to all streams.
|
||||
*
|
||||
* @param baseStreamingUrl the base streaming URL to which the parameter(s) are being appended
|
||||
* @param deliveryType the {@link DeliveryType} of the stream
|
||||
* @return the base streaming URL to which the param(s) are appended, depending on the
|
||||
* {@link DeliveryType} of the stream
|
||||
*/
|
||||
@Nonnull
|
||||
private static String appendRnSqParamsIfNeeded(@Nonnull final String baseStreamingUrl,
|
||||
@Nonnull final DeliveryType deliveryType) {
|
||||
return baseStreamingUrl + (deliveryType == DeliveryType.PROGRESSIVE ? "" : SQ_0) + RN_0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a URL on which no redirection between playback hosts should be present on the network
|
||||
* and/or IP used to fetch the streaming URL, for HTML5 clients.
|
||||
*
|
||||
* <p>This method will follow redirects which works in the following way:
|
||||
* <ol>
|
||||
* <li>the {@link #ALR_YES} param is appended to all streaming URLs</li>
|
||||
* <li>if no redirection occurs, the video server will return the streaming data;</li>
|
||||
* <li>if a redirection occurs, the server will respond with HTTP status code 200 and a
|
||||
* {@code text/plain} mime type. The redirection URL is the response body;</li>
|
||||
* <li>the redirection URL is requested and the steps above from step 2 are repeated,
|
||||
* until too many redirects are reached of course (the maximum number of redirects is
|
||||
* {@link #MAXIMUM_REDIRECT_COUNT the same as OkHttp}).</li>
|
||||
* </ol>
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* For non-HTML5 clients, redirections are managed in the standard way in
|
||||
* {@link #getInitializationResponse(String, ItagItem, DeliveryType)}.
|
||||
* </p>
|
||||
*
|
||||
* @param downloader the {@link Downloader} instance to be used
|
||||
* @param streamingUrl the streaming URL which we are trying to get a streaming URL
|
||||
* without any redirection on the network and/or IP used
|
||||
* @param responseMimeTypeExpected the response mime type expected from Google video servers
|
||||
* @return the {@link Response} of the stream, which should have no redirections
|
||||
*/
|
||||
@SuppressWarnings("checkstyle:FinalParameters")
|
||||
@Nonnull
|
||||
private static Response getStreamingWebUrlWithoutRedirects(
|
||||
@Nonnull final Downloader downloader,
|
||||
@Nonnull String streamingUrl,
|
||||
@Nonnull final String responseMimeTypeExpected)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
addClientInfoHeaders(headers);
|
||||
|
||||
String responseMimeType = "";
|
||||
|
||||
int redirectsCount = 0;
|
||||
while (!responseMimeType.equals(responseMimeTypeExpected)
|
||||
&& redirectsCount < MAXIMUM_REDIRECT_COUNT) {
|
||||
final Response response = downloader.get(streamingUrl, headers);
|
||||
|
||||
final int responseCode = response.responseCode();
|
||||
if (responseCode != 200) {
|
||||
throw new CreationException(
|
||||
"Could not get the initialization URL: HTTP response code "
|
||||
+ responseCode);
|
||||
}
|
||||
|
||||
// A valid HTTP 1.0+ response should include a Content-Type header, so we can
|
||||
// require that the response from video servers has this header.
|
||||
responseMimeType = Objects.requireNonNull(response.getHeader("Content-Type"),
|
||||
"Could not get the Content-Type header from the response headers");
|
||||
|
||||
// The response body is the redirection URL
|
||||
if (responseMimeType.equals("text/plain")) {
|
||||
streamingUrl = response.responseBody();
|
||||
redirectsCount++;
|
||||
} else {
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
if (redirectsCount >= MAXIMUM_REDIRECT_COUNT) {
|
||||
throw new CreationException(
|
||||
"Too many redirects when trying to get the the streaming URL response of a "
|
||||
+ "HTML5 client");
|
||||
}
|
||||
|
||||
// This should never be reached, but is required because we don't want to return null
|
||||
// here
|
||||
throw new CreationException(
|
||||
"Could not get the streaming URL response of a HTML5 client: unreachable code "
|
||||
+ "reached!");
|
||||
} catch (final IOException | ExtractionException e) {
|
||||
throw new CreationException(
|
||||
"Could not get the streaming URL response of a HTML5 client", e);
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,265 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
|
||||
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ALR_YES;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.RN_0;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SQ_0;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
|
||||
|
||||
/**
|
||||
* Class which generates DASH manifests of YouTube {@link DeliveryType#OTF OTF streams}.
|
||||
*/
|
||||
public final class YoutubeOtfDashManifestCreator {
|
||||
|
||||
/**
|
||||
* Cache of DASH manifests generated for OTF streams.
|
||||
*/
|
||||
private static final ManifestCreatorCache<String, String> OTF_STREAMS_CACHE
|
||||
= new ManifestCreatorCache<>();
|
||||
|
||||
private YoutubeOtfDashManifestCreator() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create DASH manifests from a YouTube OTF stream.
|
||||
*
|
||||
* <p>
|
||||
* OTF streams are YouTube-DASH specific streams which work with sequences and without the need
|
||||
* to get a manifest (even if one is provided, it is not used by official clients).
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* They can be found only on videos; mostly those with a small amount of views, or ended
|
||||
* livestreams which have just been re-encoded as normal videos.
|
||||
* </p>
|
||||
*
|
||||
* <p>This method needs:
|
||||
* <ul>
|
||||
* <li>the base URL of the stream (which, if you try to access to it, returns HTTP
|
||||
* status code 404 after redirects, and if the URL is valid);</li>
|
||||
* <li>an {@link ItagItem}, which needs to contain the following information:
|
||||
* <ul>
|
||||
* <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
|
||||
* an audio or a video stream;</li>
|
||||
* <li>its bitrate;</li>
|
||||
* <li>its mime type;</li>
|
||||
* <li>its codec(s);</li>
|
||||
* <li>for an audio stream: its audio channels;</li>
|
||||
* <li>for a video stream: its width and height.</li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* <li>the duration of the video, which will be used if the duration could not be
|
||||
* parsed from the first sequence of the stream.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* <p>In order to generate the DASH manifest, this method will:
|
||||
* <ul>
|
||||
* <li>request the first sequence of the stream (the base URL on which the first
|
||||
* sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
|
||||
* with a {@code POST} or {@code GET} request (depending of the client on which the
|
||||
* streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));</li>
|
||||
* <li>follow its redirection(s), if any;</li>
|
||||
* <li>save the last URL, remove the first sequence parameter;</li>
|
||||
* <li>use the information provided in the {@link ItagItem} to generate all
|
||||
* elements of the DASH manifest.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
|
||||
* as the stream duration.
|
||||
* </p>
|
||||
*
|
||||
* @param otfBaseStreamingUrl the base URL of the OTF stream, which must not be null
|
||||
* @param itagItem the {@link ItagItem} corresponding to the stream, which
|
||||
* must not be null
|
||||
* @param durationSecondsFallback the duration of the video, which will be used if the duration
|
||||
* could not be extracted from the first sequence
|
||||
* @return the manifest generated into a string
|
||||
*/
|
||||
@Nonnull
|
||||
public static String fromOtfStreamingUrl(
|
||||
@Nonnull final String otfBaseStreamingUrl,
|
||||
@Nonnull final ItagItem itagItem,
|
||||
final long durationSecondsFallback) throws CreationException {
|
||||
if (OTF_STREAMS_CACHE.containsKey(otfBaseStreamingUrl)) {
|
||||
return Objects.requireNonNull(OTF_STREAMS_CACHE.get(otfBaseStreamingUrl)).getSecond();
|
||||
}
|
||||
|
||||
String realOtfBaseStreamingUrl = otfBaseStreamingUrl;
|
||||
// Try to avoid redirects when streaming the content by saving the last URL we get
|
||||
// from video servers.
|
||||
final Response response = getInitializationResponse(realOtfBaseStreamingUrl,
|
||||
itagItem, DeliveryType.OTF);
|
||||
realOtfBaseStreamingUrl = response.latestUrl().replace(SQ_0, EMPTY_STRING)
|
||||
.replace(RN_0, EMPTY_STRING).replace(ALR_YES, EMPTY_STRING);
|
||||
|
||||
final int responseCode = response.responseCode();
|
||||
if (responseCode != 200) {
|
||||
throw new CreationException("Could not get the initialization URL: response code "
|
||||
+ responseCode);
|
||||
}
|
||||
|
||||
final String[] segmentDuration;
|
||||
|
||||
try {
|
||||
final String[] segmentsAndDurationsResponseSplit = response.responseBody()
|
||||
// Get the lines with the durations and the following
|
||||
.split("Segment-Durations-Ms: ")[1]
|
||||
// Remove the other lines
|
||||
.split("\n")[0]
|
||||
// Get all durations and repetitions which are separated by a comma
|
||||
.split(",");
|
||||
final int lastIndex = segmentsAndDurationsResponseSplit.length - 1;
|
||||
if (isBlank(segmentsAndDurationsResponseSplit[lastIndex])) {
|
||||
segmentDuration = Arrays.copyOf(segmentsAndDurationsResponseSplit, lastIndex);
|
||||
} else {
|
||||
segmentDuration = segmentsAndDurationsResponseSplit;
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new CreationException("Could not get segment durations", e);
|
||||
}
|
||||
|
||||
long streamDuration;
|
||||
try {
|
||||
streamDuration = getStreamDuration(segmentDuration);
|
||||
} catch (final CreationException e) {
|
||||
streamDuration = durationSecondsFallback * 1000;
|
||||
}
|
||||
|
||||
final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
|
||||
streamDuration);
|
||||
|
||||
generateSegmentTemplateElement(doc, realOtfBaseStreamingUrl, DeliveryType.OTF);
|
||||
generateSegmentTimelineElement(doc);
|
||||
generateSegmentElementsForOtfStreams(segmentDuration, doc);
|
||||
|
||||
return buildAndCacheResult(otfBaseStreamingUrl, doc, OTF_STREAMS_CACHE);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the cache of DASH manifests generated for OTF streams
|
||||
*/
|
||||
@Nonnull
|
||||
public static ManifestCreatorCache<String, String> getCache() {
|
||||
return OTF_STREAMS_CACHE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate segment elements for OTF streams.
|
||||
*
|
||||
* <p>
|
||||
* By parsing by the first media sequence, we know how many durations and repetitions there are
|
||||
* so we just have to loop into segment durations to generate the following elements for each
|
||||
* duration repeated X times:
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* {@code <S d="segmentDuration" r="durationRepetition" />}
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* If there is no repetition of the duration between two segments, the {@code r} attribute is
|
||||
* not added to the {@code S} element, as it is not needed.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* These elements will be appended as children of the {@code <SegmentTimeline>} element, which
|
||||
* needs to be generated before these elements with
|
||||
* {@link YoutubeDashManifestCreatorsUtils#generateSegmentTimelineElement(Document)}.
|
||||
* </p>
|
||||
*
|
||||
* @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the
|
||||
* regular expressions
|
||||
* @param doc the {@link Document} on which the {@code <S>} elements will be
|
||||
* appended
|
||||
*/
|
||||
private static void generateSegmentElementsForOtfStreams(
|
||||
@Nonnull final String[] segmentDurations,
|
||||
@Nonnull final Document doc) throws CreationException {
|
||||
try {
|
||||
final Element segmentTimelineElement = (Element) doc.getElementsByTagName(
|
||||
SEGMENT_TIMELINE).item(0);
|
||||
|
||||
for (final String segmentDuration : segmentDurations) {
|
||||
final Element sElement = doc.createElement("S");
|
||||
|
||||
final String[] segmentLengthRepeat = segmentDuration.split("\\(r=");
|
||||
// make sure segmentLengthRepeat[0], which is the length, is convertible to int
|
||||
Integer.parseInt(segmentLengthRepeat[0]);
|
||||
|
||||
// There are repetitions of a segment duration in other segments
|
||||
if (segmentLengthRepeat.length > 1) {
|
||||
final int segmentRepeatCount = Integer.parseInt(
|
||||
Utils.removeNonDigitCharacters(segmentLengthRepeat[1]));
|
||||
setAttribute(sElement, doc, "r", String.valueOf(segmentRepeatCount));
|
||||
}
|
||||
setAttribute(sElement, doc, "d", segmentLengthRepeat[0]);
|
||||
|
||||
segmentTimelineElement.appendChild(sElement);
|
||||
}
|
||||
|
||||
} catch (final DOMException | IllegalStateException | IndexOutOfBoundsException
|
||||
| NumberFormatException e) {
|
||||
throw CreationException.couldNotAddElement("segment (S)", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the duration of an OTF stream.
|
||||
*
|
||||
* <p>
|
||||
* The duration of OTF streams is not returned into the player response and needs to be
|
||||
* calculated by adding the duration of each segment.
|
||||
* </p>
|
||||
*
|
||||
* @param segmentDuration the segment duration object extracted from the initialization
|
||||
* sequence of the stream
|
||||
* @return the duration of the OTF stream, in milliseconds
|
||||
*/
|
||||
private static long getStreamDuration(@Nonnull final String[] segmentDuration)
|
||||
throws CreationException {
|
||||
try {
|
||||
long streamLengthMs = 0;
|
||||
|
||||
for (final String segDuration : segmentDuration) {
|
||||
final String[] segmentLengthRepeat = segDuration.split("\\(r=");
|
||||
long segmentRepeatCount = 0;
|
||||
|
||||
// There are repetitions of a segment duration in other segments
|
||||
if (segmentLengthRepeat.length > 1) {
|
||||
segmentRepeatCount = Long.parseLong(Utils.removeNonDigitCharacters(
|
||||
segmentLengthRepeat[1]));
|
||||
}
|
||||
|
||||
final long segmentLength = Integer.parseInt(segmentLengthRepeat[0]);
|
||||
streamLengthMs += segmentLength + segmentRepeatCount * segmentLength;
|
||||
}
|
||||
|
||||
return streamLengthMs;
|
||||
} catch (final NumberFormatException e) {
|
||||
throw new CreationException("Could not get stream length from sequences list", e);
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,217 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
|
||||
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ALR_YES;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.RN_0;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SQ_0;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
/**
|
||||
* Class which generates DASH manifests of YouTube post-live DVR streams (which use the
|
||||
* {@link DeliveryType#LIVE LIVE delivery type}).
|
||||
*/
|
||||
public final class YoutubePostLiveStreamDvrDashManifestCreator {
|
||||
|
||||
/**
|
||||
* Cache of DASH manifests generated for post-live-DVR streams.
|
||||
*/
|
||||
private static final ManifestCreatorCache<String, String> POST_LIVE_DVR_STREAMS_CACHE
|
||||
= new ManifestCreatorCache<>();
|
||||
|
||||
private YoutubePostLiveStreamDvrDashManifestCreator() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create DASH manifests from a YouTube post-live-DVR stream/ended livestream.
|
||||
*
|
||||
* <p>
|
||||
* Post-live-DVR streams/ended livestreams are one of the YouTube DASH specific streams which
|
||||
* works with sequences and without the need to get a manifest (even if one is provided but not
|
||||
* used by main clients (and is not complete for big ended livestreams because it doesn't
|
||||
* return the full stream)).
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* They can be found only on livestreams which have ended very recently (a few hours, most of
|
||||
* the time)
|
||||
* </p>
|
||||
*
|
||||
* <p>This method needs:
|
||||
* <ul>
|
||||
* <li>the base URL of the stream (which, if you try to access to it, returns HTTP
|
||||
* status code 404 after redirects, and if the URL is valid);</li>
|
||||
* <li>an {@link ItagItem}, which needs to contain the following information:
|
||||
* <ul>
|
||||
* <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
|
||||
* an audio or a video stream;</li>
|
||||
* <li>its bitrate;</li>
|
||||
* <li>its mime type;</li>
|
||||
* <li>its codec(s);</li>
|
||||
* <li>for an audio stream: its audio channels;</li>
|
||||
* <li>for a video stream: its width and height.</li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* <li>the duration of the video, which will be used if the duration could not be
|
||||
* parsed from the first sequence of the stream.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* <p>In order to generate the DASH manifest, this method will:
|
||||
* <ul>
|
||||
* <li>request the first sequence of the stream (the base URL on which the first
|
||||
* sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
|
||||
* with a {@code POST} or {@code GET} request (depending of the client on which the
|
||||
* streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));</li>
|
||||
* <li>follow its redirection(s), if any;</li>
|
||||
* <li>save the last URL, remove the first sequence parameters;</li>
|
||||
* <li>use the information provided in the {@link ItagItem} to generate all elements
|
||||
* of the DASH manifest.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
|
||||
* as the stream duration.
|
||||
* </p>
|
||||
*
|
||||
* @param postLiveStreamDvrStreamingUrl the base URL of the post-live-DVR stream/ended
|
||||
* livestream, which must not be null
|
||||
* @param itagItem the {@link ItagItem} corresponding to the stream, which
|
||||
* must not be null
|
||||
* @param targetDurationSec the target duration of each sequence, in seconds (this
|
||||
* value is returned with the {@code targetDurationSec}
|
||||
* field for each stream in YouTube's player response)
|
||||
* @param durationSecondsFallback the duration of the ended livestream, which will be
|
||||
* used if the duration could not be extracted from the
|
||||
* first sequence
|
||||
* @return the manifest generated into a string
|
||||
*/
|
||||
@Nonnull
|
||||
public static String fromPostLiveStreamDvrStreamingUrl(
|
||||
@Nonnull final String postLiveStreamDvrStreamingUrl,
|
||||
@Nonnull final ItagItem itagItem,
|
||||
final int targetDurationSec,
|
||||
final long durationSecondsFallback) throws CreationException {
|
||||
if (POST_LIVE_DVR_STREAMS_CACHE.containsKey(postLiveStreamDvrStreamingUrl)) {
|
||||
return Objects.requireNonNull(
|
||||
POST_LIVE_DVR_STREAMS_CACHE.get(postLiveStreamDvrStreamingUrl)).getSecond();
|
||||
}
|
||||
|
||||
String realPostLiveStreamDvrStreamingUrl = postLiveStreamDvrStreamingUrl;
|
||||
final String streamDurationString;
|
||||
final String segmentCount;
|
||||
|
||||
if (targetDurationSec <= 0) {
|
||||
throw new CreationException("targetDurationSec value is <= 0: " + targetDurationSec);
|
||||
}
|
||||
|
||||
try {
|
||||
// Try to avoid redirects when streaming the content by saving the latest URL we get
|
||||
// from video servers.
|
||||
final Response response = getInitializationResponse(realPostLiveStreamDvrStreamingUrl,
|
||||
itagItem, DeliveryType.LIVE);
|
||||
realPostLiveStreamDvrStreamingUrl = response.latestUrl().replace(SQ_0, EMPTY_STRING)
|
||||
.replace(RN_0, EMPTY_STRING).replace(ALR_YES, EMPTY_STRING);
|
||||
|
||||
final int responseCode = response.responseCode();
|
||||
if (responseCode != 200) {
|
||||
throw new CreationException(
|
||||
"Could not get the initialization sequence: response code " + responseCode);
|
||||
}
|
||||
|
||||
final Map<String, List<String>> responseHeaders = response.responseHeaders();
|
||||
streamDurationString = responseHeaders.get("X-Head-Time-Millis").get(0);
|
||||
segmentCount = responseHeaders.get("X-Head-Seqnum").get(0);
|
||||
} catch (final IndexOutOfBoundsException e) {
|
||||
throw new CreationException(
|
||||
"Could not get the value of the X-Head-Time-Millis or the X-Head-Seqnum header",
|
||||
e);
|
||||
}
|
||||
|
||||
if (isNullOrEmpty(segmentCount)) {
|
||||
throw new CreationException("Could not get the number of segments");
|
||||
}
|
||||
|
||||
long streamDuration;
|
||||
try {
|
||||
streamDuration = Long.parseLong(streamDurationString);
|
||||
} catch (final NumberFormatException e) {
|
||||
streamDuration = durationSecondsFallback;
|
||||
}
|
||||
|
||||
final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
|
||||
streamDuration);
|
||||
|
||||
generateSegmentTemplateElement(doc, realPostLiveStreamDvrStreamingUrl,
|
||||
DeliveryType.LIVE);
|
||||
generateSegmentTimelineElement(doc);
|
||||
generateSegmentElementForPostLiveDvrStreams(doc, targetDurationSec, segmentCount);
|
||||
|
||||
return buildAndCacheResult(postLiveStreamDvrStreamingUrl, doc,
|
||||
POST_LIVE_DVR_STREAMS_CACHE);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the cache of DASH manifests generated for post-live-DVR streams
|
||||
*/
|
||||
@Nonnull
|
||||
public static ManifestCreatorCache<String, String> getCache() {
|
||||
return POST_LIVE_DVR_STREAMS_CACHE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the segment ({@code <S>}) element.
|
||||
*
|
||||
* <p>
|
||||
* We don't know the exact duration of segments for post-live-DVR streams but an
|
||||
* average instead (which is the {@code targetDurationSec} value), so we can use the following
|
||||
* structure to generate the segment timeline for DASH manifests of ended livestreams:
|
||||
* <br>
|
||||
* {@code <S d="targetDurationSecValue" r="segmentCount" />}
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <S>} element will
|
||||
* be appended
|
||||
* @param targetDurationSeconds the {@code targetDurationSec} value from YouTube player
|
||||
* response's stream
|
||||
* @param segmentCount the number of segments, extracted by {@link
|
||||
* #fromPostLiveStreamDvrStreamingUrl(String, ItagItem, int, long)}
|
||||
*/
|
||||
private static void generateSegmentElementForPostLiveDvrStreams(
|
||||
@Nonnull final Document doc,
|
||||
final int targetDurationSeconds,
|
||||
@Nonnull final String segmentCount) throws CreationException {
|
||||
try {
|
||||
final Element segmentTimelineElement = (Element) doc.getElementsByTagName(
|
||||
SEGMENT_TIMELINE).item(0);
|
||||
final Element sElement = doc.createElement("S");
|
||||
|
||||
setAttribute(sElement, doc, "d", String.valueOf(targetDurationSeconds * 1000));
|
||||
setAttribute(sElement, doc, "r", segmentCount);
|
||||
|
||||
segmentTimelineElement.appendChild(sElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement("segment (S)", e);
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,235 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
|
||||
|
||||
import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.util.Objects;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.BASE_URL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.INITIALIZATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.MPD;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.REPRESENTATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
|
||||
|
||||
/**
|
||||
* Class which generates DASH manifests of {@link DeliveryType#PROGRESSIVE YouTube progressive}
|
||||
* streams.
|
||||
*/
|
||||
public final class YoutubeProgressiveDashManifestCreator {
|
||||
|
||||
/**
|
||||
* Cache of DASH manifests generated for progressive streams.
|
||||
*/
|
||||
private static final ManifestCreatorCache<String, String> PROGRESSIVE_STREAMS_CACHE
|
||||
= new ManifestCreatorCache<>();
|
||||
|
||||
private YoutubeProgressiveDashManifestCreator() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create DASH manifests from a YouTube progressive stream.
|
||||
*
|
||||
* <p>
|
||||
* Progressive streams are YouTube DASH streams which work with range requests and without the
|
||||
* need to get a manifest.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* They can be found on all videos, and for all streams for most of videos which come from a
|
||||
* YouTube partner, and on videos with a large number of views.
|
||||
* </p>
|
||||
*
|
||||
* <p>This method needs:
|
||||
* <ul>
|
||||
* <li>the base URL of the stream (which, if you try to access to it, returns the whole
|
||||
* stream, after redirects, and if the URL is valid);</li>
|
||||
* <li>an {@link ItagItem}, which needs to contain the following information:
|
||||
* <ul>
|
||||
* <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
|
||||
* an audio or a video stream;</li>
|
||||
* <li>its bitrate;</li>
|
||||
* <li>its mime type;</li>
|
||||
* <li>its codec(s);</li>
|
||||
* <li>for an audio stream: its audio channels;</li>
|
||||
* <li>for a video stream: its width and height.</li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* <li>the duration of the video (parameter {@code durationSecondsFallback}), which
|
||||
* will be used as the stream duration if the duration could not be parsed from the
|
||||
* {@link ItagItem}.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* @param progressiveStreamingBaseUrl the base URL of the progressive stream, which must not be
|
||||
* null
|
||||
* @param itagItem the {@link ItagItem} corresponding to the stream, which
|
||||
* must not be null
|
||||
* @param durationSecondsFallback the duration of the progressive stream which will be used
|
||||
* if the duration could not be extracted from the
|
||||
* {@link ItagItem}
|
||||
* @return the manifest generated into a string
|
||||
*/
|
||||
@Nonnull
|
||||
public static String fromProgressiveStreamingUrl(
|
||||
@Nonnull final String progressiveStreamingBaseUrl,
|
||||
@Nonnull final ItagItem itagItem,
|
||||
final long durationSecondsFallback) throws CreationException {
|
||||
if (PROGRESSIVE_STREAMS_CACHE.containsKey(progressiveStreamingBaseUrl)) {
|
||||
return Objects.requireNonNull(
|
||||
PROGRESSIVE_STREAMS_CACHE.get(progressiveStreamingBaseUrl)).getSecond();
|
||||
}
|
||||
|
||||
final long itagItemDuration = itagItem.getApproxDurationMs();
|
||||
final long streamDuration;
|
||||
if (itagItemDuration != -1) {
|
||||
streamDuration = itagItemDuration;
|
||||
} else {
|
||||
if (durationSecondsFallback > 0) {
|
||||
streamDuration = durationSecondsFallback * 1000;
|
||||
} else {
|
||||
throw CreationException.couldNotAddElement(MPD, "the duration of the stream "
|
||||
+ "could not be determined and durationSecondsFallback is <= 0");
|
||||
}
|
||||
}
|
||||
|
||||
final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
|
||||
streamDuration);
|
||||
|
||||
generateBaseUrlElement(doc, progressiveStreamingBaseUrl);
|
||||
generateSegmentBaseElement(doc, itagItem);
|
||||
generateInitializationElement(doc, itagItem);
|
||||
|
||||
return buildAndCacheResult(progressiveStreamingBaseUrl, doc,
|
||||
PROGRESSIVE_STREAMS_CACHE);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the cache of DASH manifests generated for progressive streams
|
||||
*/
|
||||
@Nonnull
|
||||
public static ManifestCreatorCache<String, String> getCache() {
|
||||
return PROGRESSIVE_STREAMS_CACHE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <BaseURL>} element, appended as a child of the
|
||||
* {@code <Representation>} element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Representation>} element needs to be generated before this element with
|
||||
* {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}).
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <BaseURL>} element will be appended
|
||||
* @param baseUrl the base URL of the stream, which must not be null and will be set as the
|
||||
* content of the {@code <BaseURL>} element
|
||||
*/
|
||||
private static void generateBaseUrlElement(@Nonnull final Document doc,
|
||||
@Nonnull final String baseUrl)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element representationElement = (Element) doc.getElementsByTagName(
|
||||
REPRESENTATION).item(0);
|
||||
final Element baseURLElement = doc.createElement(BASE_URL);
|
||||
baseURLElement.setTextContent(baseUrl);
|
||||
representationElement.appendChild(baseURLElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(BASE_URL, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <SegmentBase>} element, appended as a child of the
|
||||
* {@code <Representation>} element.
|
||||
*
|
||||
* <p>
|
||||
* It generates the following element:
|
||||
* <br>
|
||||
* {@code <SegmentBase indexRange="indexStart-indexEnd" />}
|
||||
* <br>
|
||||
* (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed
|
||||
* as the second parameter)
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Representation>} element needs to be generated before this element with
|
||||
* {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}),
|
||||
* and the {@code BaseURL} element with {@link #generateBaseUrlElement(Document, String)}
|
||||
* should be generated too.
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <SegmentBase>} element will be appended
|
||||
* @param itagItem the {@link ItagItem} to use, which must not be null
|
||||
*/
|
||||
private static void generateSegmentBaseElement(@Nonnull final Document doc,
|
||||
@Nonnull final ItagItem itagItem)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element representationElement = (Element) doc.getElementsByTagName(
|
||||
REPRESENTATION).item(0);
|
||||
final Element segmentBaseElement = doc.createElement(SEGMENT_BASE);
|
||||
|
||||
final String range = itagItem.getIndexStart() + "-" + itagItem.getIndexEnd();
|
||||
if (itagItem.getIndexStart() < 0 || itagItem.getIndexEnd() < 0) {
|
||||
throw CreationException.couldNotAddElement(SEGMENT_BASE,
|
||||
"ItagItem's indexStart or " + "indexEnd are < 0: " + range);
|
||||
}
|
||||
setAttribute(segmentBaseElement, doc, "indexRange", range);
|
||||
|
||||
representationElement.appendChild(segmentBaseElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(SEGMENT_BASE, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <Initialization>} element, appended as a child of the
|
||||
* {@code <SegmentBase>} element.
|
||||
*
|
||||
* <p>
|
||||
* It generates the following element:
|
||||
* <br>
|
||||
* {@code <Initialization range="initStart-initEnd"/>}
|
||||
* <br>
|
||||
* (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed
|
||||
* as the second parameter)
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <SegmentBase>} element needs to be generated before this element with
|
||||
* {@link #generateSegmentBaseElement(Document, ItagItem)}).
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <Initialization>} element will be
|
||||
* appended
|
||||
* @param itagItem the {@link ItagItem} to use, which must not be null
|
||||
*/
|
||||
private static void generateInitializationElement(@Nonnull final Document doc,
|
||||
@Nonnull final ItagItem itagItem)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element segmentBaseElement = (Element) doc.getElementsByTagName(
|
||||
SEGMENT_BASE).item(0);
|
||||
final Element initializationElement = doc.createElement(INITIALIZATION);
|
||||
|
||||
final String range = itagItem.getInitStart() + "-" + itagItem.getInitEnd();
|
||||
if (itagItem.getInitStart() < 0 || itagItem.getInitEnd() < 0) {
|
||||
throw CreationException.couldNotAddElement(INITIALIZATION,
|
||||
"ItagItem's initStart and/or " + "initEnd are/is < 0: " + range);
|
||||
}
|
||||
setAttribute(initializationElement, doc, "range", range);
|
||||
|
||||
segmentBaseElement.appendChild(initializationElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(INITIALIZATION, e);
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,80 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* Class to build easier {@link org.schabi.newpipe.extractor.stream.Stream}s for
|
||||
* {@link YoutubeStreamExtractor}.
|
||||
*
|
||||
* <p>
|
||||
* It stores, per stream:
|
||||
* <ul>
|
||||
* <li>its content (the URL/the base URL of streams);</li>
|
||||
* <li>whether its content is the URL the content itself or the base URL;</li>
|
||||
* <li>its associated {@link ItagItem}.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*/
|
||||
final class ItagInfo implements Serializable {
|
||||
@Nonnull
|
||||
private final String content;
|
||||
@Nonnull
|
||||
private final ItagItem itagItem;
|
||||
private boolean isUrl;
|
||||
|
||||
/**
|
||||
* Creates a new {@code ItagInfo} instance.
|
||||
*
|
||||
* @param content the content of the stream, which must be not null
|
||||
* @param itagItem the {@link ItagItem} associated with the stream, which must be not null
|
||||
*/
|
||||
ItagInfo(@Nonnull final String content,
|
||||
@Nonnull final ItagItem itagItem) {
|
||||
this.content = content;
|
||||
this.itagItem = itagItem;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets whether the stream is a URL.
|
||||
*
|
||||
* @param isUrl whether the content is a URL
|
||||
*/
|
||||
void setIsUrl(final boolean isUrl) {
|
||||
this.isUrl = isUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the content stored in this {@code ItagInfo} instance, which is either the URL to the
|
||||
* content itself or the base URL.
|
||||
*
|
||||
* @return the content stored in this {@code ItagInfo} instance
|
||||
*/
|
||||
@Nonnull
|
||||
String getContent() {
|
||||
return content;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the {@link ItagItem} associated with this {@code ItagInfo} instance.
|
||||
*
|
||||
* @return the {@link ItagItem} associated with this {@code ItagInfo} instance, which is not
|
||||
* null
|
||||
*/
|
||||
@Nonnull
|
||||
ItagItem getItagItem() {
|
||||
return itagItem;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets whether the content stored is the URL to the content itself or the base URL of it.
|
||||
*
|
||||
* @return whether the content stored is the URL to the content itself or the base URL of it
|
||||
* @see #getContent() for more details
|
||||
*/
|
||||
boolean getIsUrl() {
|
||||
return isUrl;
|
||||
}
|
||||
}
|
@ -1,10 +1,32 @@
|
||||
/*
|
||||
* Created by Christian Schabesberger on 06.08.15.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2019 <chris.schabesberger@mailbox.org>
|
||||
* YoutubeStreamExtractor.java is part of NewPipe Extractor.
|
||||
*
|
||||
* NewPipe Extractor is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe Extractor is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.APPROX_DURATION_MS_UNKNOWN;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createDesktopPlayerBody;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.CONTENT_LENGTH_UNKNOWN;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
|
||||
@ -50,6 +72,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
|
||||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.Frameset;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
@ -64,7 +87,6 @@ import org.schabi.newpipe.extractor.utils.Parser;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.LocalDate;
|
||||
import java.time.OffsetDateTime;
|
||||
@ -72,7 +94,6 @@ import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
@ -82,26 +103,6 @@ import java.util.stream.Collectors;
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 06.08.15.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2019 <chris.schabesberger@mailbox.org>
|
||||
* YoutubeStreamExtractor.java is part of NewPipe.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Exceptions
|
||||
@ -113,7 +114,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////*/
|
||||
/*////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Nullable
|
||||
private static String cachedDeobfuscationCode = null;
|
||||
@ -140,8 +141,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
private JsonObject playerMicroFormatRenderer;
|
||||
private int ageLimit = -1;
|
||||
private StreamType streamType;
|
||||
@Nullable
|
||||
private List<SubtitlesStream> subtitles = null;
|
||||
|
||||
// We need to store the contentPlaybackNonces because we need to append them to videoplayback
|
||||
// URLs (with the cpn parameter).
|
||||
@ -580,73 +579,25 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
.orElse(EMPTY_STRING);
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
interface StreamTypeStreamBuilderHelper<T extends Stream> {
|
||||
T buildStream(String url, ItagItem itagItem);
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract method for
|
||||
* {@link #getAudioStreams()}, {@link #getVideoOnlyStreams()} and {@link #getVideoStreams()}.
|
||||
*
|
||||
* @param itags A map of Urls + ItagItems
|
||||
* @param streamBuilder Builds the stream from the provided data
|
||||
* @param exMsgStreamType Stream type inside the exception message e.g. "video streams"
|
||||
* @param <T> Type of the stream
|
||||
* @return
|
||||
* @throws ExtractionException
|
||||
*/
|
||||
private <T extends Stream> List<T> getStreamsByType(
|
||||
final Map<String, ItagItem> itags,
|
||||
final StreamTypeStreamBuilderHelper<T> streamBuilder,
|
||||
final String exMsgStreamType
|
||||
) throws ExtractionException {
|
||||
final List<T> streams = new ArrayList<>();
|
||||
|
||||
try {
|
||||
for (final Map.Entry<String, ItagItem> entry : itags.entrySet()) {
|
||||
final String url = tryDecryptUrl(entry.getKey(), getId());
|
||||
|
||||
final T stream = streamBuilder.buildStream(url, entry.getValue());
|
||||
if (!Stream.containSimilarStream(stream, streams)) {
|
||||
streams.add(stream);
|
||||
}
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get " + exMsgStreamType, e);
|
||||
}
|
||||
|
||||
return streams;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
return getStreamsByType(
|
||||
getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO),
|
||||
AudioStream::new,
|
||||
"audio streams"
|
||||
);
|
||||
return getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO,
|
||||
getAudioStreamBuilderHelper(), "audio");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
return getStreamsByType(
|
||||
getItags(FORMATS, ItagItem.ItagType.VIDEO),
|
||||
(url, itag) -> new VideoStream(url, false, itag),
|
||||
"video streams"
|
||||
);
|
||||
return getItags(FORMATS, ItagItem.ItagType.VIDEO,
|
||||
getVideoStreamBuilderHelper(false), "video");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoOnlyStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
return getStreamsByType(
|
||||
getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY),
|
||||
(url, itag) -> new VideoStream(url, true, itag),
|
||||
"video only streams"
|
||||
);
|
||||
return getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY,
|
||||
getVideoStreamBuilderHelper(true), "video-only");
|
||||
}
|
||||
|
||||
/**
|
||||
@ -672,18 +623,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
@Nonnull
|
||||
public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws ParsingException {
|
||||
assertPageFetched();
|
||||
if (subtitles != null) {
|
||||
// Already calculated
|
||||
return subtitles;
|
||||
}
|
||||
|
||||
// We cannot store the subtitles list because the media format may change
|
||||
final List<SubtitlesStream> subtitlesToReturn = new ArrayList<>();
|
||||
final JsonObject renderer = playerResponse.getObject("captions")
|
||||
.getObject("playerCaptionsTracklistRenderer");
|
||||
final JsonArray captionsArray = renderer.getArray("captionTracks");
|
||||
// TODO: use this to apply auto translation to different language from a source language
|
||||
// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages");
|
||||
|
||||
subtitles = new ArrayList<>();
|
||||
for (int i = 0; i < captionsArray.size(); i++) {
|
||||
final String languageCode = captionsArray.getObject(i).getString("languageCode");
|
||||
final String baseUrl = captionsArray.getObject(i).getString("baseUrl");
|
||||
@ -692,15 +640,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
if (languageCode != null && baseUrl != null && vssId != null) {
|
||||
final boolean isAutoGenerated = vssId.startsWith("a.");
|
||||
final String cleanUrl = baseUrl
|
||||
.replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists
|
||||
.replaceAll("&tlang=[^&]*", ""); // Remove translation language
|
||||
// Remove preexisting format if exists
|
||||
.replaceAll("&fmt=[^&]*", "")
|
||||
// Remove translation language
|
||||
.replaceAll("&tlang=[^&]*", "");
|
||||
|
||||
subtitles.add(new SubtitlesStream(format, languageCode,
|
||||
cleanUrl + "&fmt=" + format.getSuffix(), isAutoGenerated));
|
||||
subtitlesToReturn.add(new SubtitlesStream.Builder()
|
||||
.setContent(cleanUrl + "&fmt=" + format.getSuffix(), true)
|
||||
.setMediaFormat(format)
|
||||
.setLanguageCode(languageCode)
|
||||
.setAutoGenerated(isAutoGenerated)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
||||
return subtitles;
|
||||
return subtitlesToReturn;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -711,9 +665,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
|
||||
private void setStreamType() {
|
||||
if (playerResponse.getObject("playabilityStatus").has("liveStreamability")
|
||||
|| playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false)) {
|
||||
if (playerResponse.getObject("playabilityStatus").has("liveStreamability")) {
|
||||
streamType = StreamType.LIVE_STREAM;
|
||||
} else if (playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false)) {
|
||||
streamType = StreamType.POST_LIVE_STREAM;
|
||||
} else {
|
||||
streamType = StreamType.VIDEO_STREAM;
|
||||
}
|
||||
@ -788,6 +743,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
private static final String STREAMING_DATA = "streamingData";
|
||||
private static final String PLAYER = "player";
|
||||
private static final String NEXT = "next";
|
||||
private static final String SIGNATURE_CIPHER = "signatureCipher";
|
||||
private static final String CIPHER = "cipher";
|
||||
|
||||
private static final String[] REGEXES = {
|
||||
"(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)"
|
||||
@ -827,7 +784,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
|
||||
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
|
||||
|
||||
final boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING)
|
||||
final boolean isAgeRestricted = playabilityStatus.getString("reason", EMPTY_STRING)
|
||||
.contains("age");
|
||||
|
||||
setStreamType();
|
||||
@ -837,12 +794,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
fetchTvHtml5EmbedJsonPlayer(contentCountry, localization, videoId);
|
||||
} catch (final Exception ignored) {
|
||||
}
|
||||
|
||||
// Refresh the stream type because the stream type may be not properly known for
|
||||
// age-restricted videos
|
||||
setStreamType();
|
||||
}
|
||||
|
||||
// Refresh the stream type because the stream type may be not properly known for
|
||||
// age-restricted videos
|
||||
setStreamType();
|
||||
|
||||
if (html5StreamingData == null && playerResponse.has(STREAMING_DATA)) {
|
||||
html5StreamingData = playerResponse.getObject(STREAMING_DATA);
|
||||
}
|
||||
@ -866,7 +823,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
nextResponse = getJsonPostResponse(NEXT, body, localization);
|
||||
|
||||
if ((!ageRestricted && streamType == StreamType.VIDEO_STREAM)
|
||||
if ((!isAgeRestricted && streamType == StreamType.VIDEO_STREAM)
|
||||
|| isAndroidClientFetchForced) {
|
||||
try {
|
||||
fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId);
|
||||
@ -874,7 +831,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
if ((!ageRestricted && streamType == StreamType.LIVE_STREAM)
|
||||
if ((!isAgeRestricted && streamType == StreamType.LIVE_STREAM)
|
||||
|| isIosClientFetchForced) {
|
||||
try {
|
||||
fetchIosMobileJsonPlayer(contentCountry, localization, videoId);
|
||||
@ -1184,103 +1141,254 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private Map<String, ItagItem> getItags(@Nonnull final String streamingDataKey,
|
||||
@Nonnull final ItagItem.ItagType itagTypeWanted) {
|
||||
final Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
|
||||
if (html5StreamingData == null && androidStreamingData == null
|
||||
&& iosStreamingData == null) {
|
||||
return urlAndItags;
|
||||
private <T extends Stream> List<T> getItags(
|
||||
final String streamingDataKey,
|
||||
final ItagItem.ItagType itagTypeWanted,
|
||||
final java.util.function.Function<ItagInfo, T> streamBuilderHelper,
|
||||
final String streamTypeExceptionMessage) throws ParsingException {
|
||||
try {
|
||||
final String videoId = getId();
|
||||
final List<T> streamList = new ArrayList<>();
|
||||
|
||||
java.util.stream.Stream.of(
|
||||
// Use the androidStreamingData object first because there is no n param and no
|
||||
// signatureCiphers in streaming URLs of the Android client
|
||||
new Pair<>(androidStreamingData, androidCpn),
|
||||
new Pair<>(html5StreamingData, html5Cpn),
|
||||
// Use the iosStreamingData object in the last position because most of the
|
||||
// available streams can be extracted with the Android and web clients and also
|
||||
// because the iOS client is only enabled by default on livestreams
|
||||
new Pair<>(iosStreamingData, iosCpn)
|
||||
)
|
||||
.flatMap(pair -> getStreamsFromStreamingDataKey(videoId, pair.getFirst(),
|
||||
streamingDataKey, itagTypeWanted, pair.getSecond()))
|
||||
.map(streamBuilderHelper)
|
||||
.forEachOrdered(stream -> {
|
||||
if (!Stream.containSimilarStream(stream, streamList)) {
|
||||
streamList.add(stream);
|
||||
}
|
||||
});
|
||||
|
||||
return streamList;
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException(
|
||||
"Could not get " + streamTypeExceptionMessage + " streams", e);
|
||||
}
|
||||
}
|
||||
|
||||
final List<Pair<JsonObject, String>> streamingDataAndCpnLoopList = new ArrayList<>();
|
||||
// Use the androidStreamingData object first because there is no n param and no
|
||||
// signatureCiphers in streaming URLs of the Android client
|
||||
streamingDataAndCpnLoopList.add(new Pair<>(androidStreamingData, androidCpn));
|
||||
streamingDataAndCpnLoopList.add(new Pair<>(html5StreamingData, html5Cpn));
|
||||
// Use the iosStreamingData object in the last position because most of the available
|
||||
// streams can be extracted with the Android and web clients and also because the iOS
|
||||
// client is only enabled by default on livestreams
|
||||
streamingDataAndCpnLoopList.add(new Pair<>(iosStreamingData, iosCpn));
|
||||
/**
|
||||
* Get the stream builder helper which will be used to build {@link AudioStream}s in
|
||||
* {@link #getItags(String, ItagItem.ItagType, java.util.function.Function, String)}
|
||||
*
|
||||
* <p>
|
||||
* The {@code StreamBuilderHelper} will set the following attributes in the
|
||||
* {@link AudioStream}s built:
|
||||
* <ul>
|
||||
* <li>the {@link ItagItem}'s id of the stream as its id;</li>
|
||||
* <li>{@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and
|
||||
* and as the value of {@code isUrl};</li>
|
||||
* <li>the media format returned by the {@link ItagItem} as its media format;</li>
|
||||
* <li>its average bitrate with the value returned by {@link
|
||||
* ItagItem#getAverageBitrate()};</li>
|
||||
* <li>the {@link ItagItem};</li>
|
||||
* <li>the {@link DeliveryMethod#DASH DASH delivery method}, for OTF streams, live streams
|
||||
* and ended streams.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Note that the {@link ItagItem} comes from an {@link ItagInfo} instance.
|
||||
* </p>
|
||||
*
|
||||
* @return a stream builder helper to build {@link AudioStream}s
|
||||
*/
|
||||
@Nonnull
|
||||
private java.util.function.Function<ItagInfo, AudioStream> getAudioStreamBuilderHelper() {
|
||||
return (itagInfo) -> {
|
||||
final ItagItem itagItem = itagInfo.getItagItem();
|
||||
final AudioStream.Builder builder = new AudioStream.Builder()
|
||||
.setId(String.valueOf(itagItem.id))
|
||||
.setContent(itagInfo.getContent(), itagInfo.getIsUrl())
|
||||
.setMediaFormat(itagItem.getMediaFormat())
|
||||
.setAverageBitrate(itagItem.getAverageBitrate())
|
||||
.setItagItem(itagItem);
|
||||
|
||||
for (final Pair<JsonObject, String> pair : streamingDataAndCpnLoopList) {
|
||||
urlAndItags.putAll(getStreamsFromStreamingDataKey(pair.getFirst(), streamingDataKey,
|
||||
itagTypeWanted, pair.getSecond()));
|
||||
}
|
||||
if (streamType == StreamType.LIVE_STREAM
|
||||
|| streamType == StreamType.POST_LIVE_STREAM
|
||||
|| !itagInfo.getIsUrl()) {
|
||||
// For YouTube videos on OTF streams and for all streams of post-live streams
|
||||
// and live streams, only the DASH delivery method can be used.
|
||||
builder.setDeliveryMethod(DeliveryMethod.DASH);
|
||||
}
|
||||
|
||||
return urlAndItags;
|
||||
return builder.build();
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the stream builder helper which will be used to build {@link VideoStream}s in
|
||||
* {@link #getItags(String, ItagItem.ItagType, java.util.function.Function, String)}
|
||||
*
|
||||
* <p>
|
||||
* The {@code StreamBuilderHelper} will set the following attributes in the
|
||||
* {@link VideoStream}s built:
|
||||
* <ul>
|
||||
* <li>the {@link ItagItem}'s id of the stream as its id;</li>
|
||||
* <li>{@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and
|
||||
* and as the value of {@code isUrl};</li>
|
||||
* <li>the media format returned by the {@link ItagItem} as its media format;</li>
|
||||
* <li>whether it is video-only with the {@code areStreamsVideoOnly} parameter</li>
|
||||
* <li>the {@link ItagItem};</li>
|
||||
* <li>the resolution, by trying to use, in this order:
|
||||
* <ol>
|
||||
* <li>the height returned by the {@link ItagItem} + {@code p} + the frame rate if
|
||||
* it is more than 30;</li>
|
||||
* <li>the default resolution string from the {@link ItagItem};</li>
|
||||
* <li>an {@link Utils#EMPTY_STRING empty string}.</li>
|
||||
* </ol>
|
||||
* </li>
|
||||
* <li>the {@link DeliveryMethod#DASH DASH delivery method}, for OTF streams, live streams
|
||||
* and ended streams.</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>
|
||||
* Note that the {@link ItagItem} comes from an {@link ItagInfo} instance.
|
||||
* </p>
|
||||
*
|
||||
* @param areStreamsVideoOnly whether the stream builder helper will set the video
|
||||
* streams as video-only streams
|
||||
* @return a stream builder helper to build {@link VideoStream}s
|
||||
*/
|
||||
@Nonnull
|
||||
private java.util.function.Function<ItagInfo, VideoStream> getVideoStreamBuilderHelper(
|
||||
final boolean areStreamsVideoOnly) {
|
||||
return (itagInfo) -> {
|
||||
final ItagItem itagItem = itagInfo.getItagItem();
|
||||
final VideoStream.Builder builder = new VideoStream.Builder()
|
||||
.setId(String.valueOf(itagItem.id))
|
||||
.setContent(itagInfo.getContent(), itagInfo.getIsUrl())
|
||||
.setMediaFormat(itagItem.getMediaFormat())
|
||||
.setIsVideoOnly(areStreamsVideoOnly)
|
||||
.setItagItem(itagItem);
|
||||
|
||||
final String resolutionString = itagItem.getResolutionString();
|
||||
builder.setResolution(resolutionString != null ? resolutionString
|
||||
: EMPTY_STRING);
|
||||
|
||||
if (streamType != StreamType.VIDEO_STREAM || !itagInfo.getIsUrl()) {
|
||||
// For YouTube videos on OTF streams and for all streams of post-live streams
|
||||
// and live streams, only the DASH delivery method can be used.
|
||||
builder.setDeliveryMethod(DeliveryMethod.DASH);
|
||||
}
|
||||
|
||||
return builder.build();
|
||||
};
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private Map<String, ItagItem> getStreamsFromStreamingDataKey(
|
||||
private java.util.stream.Stream<ItagInfo> getStreamsFromStreamingDataKey(
|
||||
final String videoId,
|
||||
final JsonObject streamingData,
|
||||
@Nonnull final String streamingDataKey,
|
||||
final String streamingDataKey,
|
||||
@Nonnull final ItagItem.ItagType itagTypeWanted,
|
||||
@Nonnull final String contentPlaybackNonce) {
|
||||
if (streamingData == null || !streamingData.has(streamingDataKey)) {
|
||||
return Collections.emptyMap();
|
||||
return java.util.stream.Stream.empty();
|
||||
}
|
||||
|
||||
final Map<String, ItagItem> urlAndItagsFromStreamingDataObject = new LinkedHashMap<>();
|
||||
final JsonArray formats = streamingData.getArray(streamingDataKey);
|
||||
for (int i = 0; i < formats.size(); i++) {
|
||||
final JsonObject formatData = formats.getObject(i);
|
||||
final int itag = formatData.getInt("itag");
|
||||
|
||||
if (!ItagItem.isSupported(itag)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
final ItagItem itagItem = ItagItem.getItag(itag);
|
||||
if (itagItem.itagType != itagTypeWanted) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Ignore streams that are delivered using YouTube's OTF format,
|
||||
// as those only work with DASH and not with progressive HTTP.
|
||||
if ("FORMAT_STREAM_TYPE_OTF".equalsIgnoreCase(formatData.getString("type"))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
final String streamUrl;
|
||||
if (formatData.has("url")) {
|
||||
streamUrl = formatData.getString("url");
|
||||
} else {
|
||||
// This url has an obfuscated signature
|
||||
final String cipherString = formatData.has("cipher")
|
||||
? formatData.getString("cipher")
|
||||
: formatData.getString("signatureCipher");
|
||||
final Map<String, String> cipher = Parser.compatParseMap(
|
||||
cipherString);
|
||||
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
|
||||
+ deobfuscateSignature(cipher.get("s"));
|
||||
}
|
||||
|
||||
final JsonObject initRange = formatData.getObject("initRange");
|
||||
final JsonObject indexRange = formatData.getObject("indexRange");
|
||||
final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
|
||||
final String codec = mimeType.contains("codecs")
|
||||
? mimeType.split("\"")[1]
|
||||
: EMPTY_STRING;
|
||||
|
||||
itagItem.setBitrate(formatData.getInt("bitrate"));
|
||||
itagItem.setWidth(formatData.getInt("width"));
|
||||
itagItem.setHeight(formatData.getInt("height"));
|
||||
itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1")));
|
||||
itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1")));
|
||||
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1")));
|
||||
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1")));
|
||||
itagItem.fps = formatData.getInt("fps");
|
||||
itagItem.setQuality(formatData.getString("quality"));
|
||||
itagItem.setCodec(codec);
|
||||
|
||||
urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem);
|
||||
} catch (final UnsupportedEncodingException | ParsingException ignored) {
|
||||
}
|
||||
}
|
||||
return urlAndItagsFromStreamingDataObject;
|
||||
return streamingData.getArray(streamingDataKey).stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.map(formatData -> {
|
||||
try {
|
||||
final ItagItem itagItem = ItagItem.getItag(formatData.getInt("itag"));
|
||||
if (itagItem.itagType == itagTypeWanted) {
|
||||
return buildAndAddItagInfoToList(videoId, formatData, itagItem,
|
||||
itagItem.itagType, contentPlaybackNonce);
|
||||
}
|
||||
} catch (final IOException | ExtractionException ignored) {
|
||||
// if the itag is not supported and getItag fails, we end up here
|
||||
}
|
||||
return null;
|
||||
})
|
||||
.filter(Objects::nonNull);
|
||||
}
|
||||
|
||||
private ItagInfo buildAndAddItagInfoToList(
|
||||
@Nonnull final String videoId,
|
||||
@Nonnull final JsonObject formatData,
|
||||
@Nonnull final ItagItem itagItem,
|
||||
@Nonnull final ItagItem.ItagType itagType,
|
||||
@Nonnull final String contentPlaybackNonce) throws IOException, ExtractionException {
|
||||
String streamUrl;
|
||||
if (formatData.has("url")) {
|
||||
streamUrl = formatData.getString("url");
|
||||
} else {
|
||||
// This url has an obfuscated signature
|
||||
final String cipherString = formatData.has(CIPHER)
|
||||
? formatData.getString(CIPHER)
|
||||
: formatData.getString(SIGNATURE_CIPHER);
|
||||
final Map<String, String> cipher = Parser.compatParseMap(
|
||||
cipherString);
|
||||
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
|
||||
+ deobfuscateSignature(cipher.get("s"));
|
||||
}
|
||||
|
||||
// Add the content playback nonce to the stream URL
|
||||
streamUrl += "&" + CPN + "=" + contentPlaybackNonce;
|
||||
|
||||
// Decrypt the n parameter if it is present
|
||||
streamUrl = tryDecryptUrl(streamUrl, videoId);
|
||||
|
||||
final JsonObject initRange = formatData.getObject("initRange");
|
||||
final JsonObject indexRange = formatData.getObject("indexRange");
|
||||
final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
|
||||
final String codec = mimeType.contains("codecs")
|
||||
? mimeType.split("\"")[1] : EMPTY_STRING;
|
||||
|
||||
itagItem.setBitrate(formatData.getInt("bitrate"));
|
||||
itagItem.setWidth(formatData.getInt("width"));
|
||||
itagItem.setHeight(formatData.getInt("height"));
|
||||
itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1")));
|
||||
itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1")));
|
||||
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1")));
|
||||
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1")));
|
||||
itagItem.setQuality(formatData.getString("quality"));
|
||||
itagItem.setCodec(codec);
|
||||
|
||||
if (streamType == StreamType.LIVE_STREAM || streamType == StreamType.POST_LIVE_STREAM) {
|
||||
itagItem.setTargetDurationSec(formatData.getInt("targetDurationSec"));
|
||||
} else if (itagType == ItagItem.ItagType.VIDEO
|
||||
|| itagType == ItagItem.ItagType.VIDEO_ONLY) {
|
||||
itagItem.setFps(formatData.getInt("fps"));
|
||||
} else if (itagType == ItagItem.ItagType.AUDIO) {
|
||||
// YouTube return the audio sample rate as a string
|
||||
itagItem.setSampleRate(Integer.parseInt(formatData.getString("audioSampleRate")));
|
||||
itagItem.setAudioChannels(formatData.getInt("audioChannels"));
|
||||
}
|
||||
|
||||
// YouTube return the content length and the approximate duration as strings
|
||||
itagItem.setContentLength(Long.parseLong(formatData.getString("contentLength",
|
||||
String.valueOf(CONTENT_LENGTH_UNKNOWN))));
|
||||
itagItem.setApproxDurationMs(Long.parseLong(formatData.getString("approxDurationMs",
|
||||
String.valueOf(APPROX_DURATION_MS_UNKNOWN))));
|
||||
|
||||
final ItagInfo itagInfo = new ItagInfo(streamUrl, itagItem);
|
||||
|
||||
if (streamType == StreamType.VIDEO_STREAM) {
|
||||
itagInfo.setIsUrl(!formatData.getString("type", EMPTY_STRING)
|
||||
.equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF"));
|
||||
} else {
|
||||
// We are currently not able to generate DASH manifests for running
|
||||
// livestreams, so because of the requirements of StreamInfo
|
||||
// objects, return these streams as DASH URL streams (even if they
|
||||
// are not playable).
|
||||
// Ended livestreams are returned as non URL streams
|
||||
itagInfo.setIsUrl(streamType != StreamType.POST_LIVE_STREAM);
|
||||
}
|
||||
|
||||
return itagInfo;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
|
@ -4,30 +4,35 @@ package org.schabi.newpipe.extractor.stream;
|
||||
* Created by Christian Schabesberger on 04.03.16.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
||||
* AudioStream.java is part of NewPipe.
|
||||
* AudioStream.java is part of NewPipe Extractor.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
* NewPipe Extractor is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe is distributed in the hope that it will be useful,
|
||||
* NewPipe Extractor is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
* along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
|
||||
public class AudioStream extends Stream {
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
public final class AudioStream extends Stream {
|
||||
public static final int UNKNOWN_BITRATE = -1;
|
||||
|
||||
private final int averageBitrate;
|
||||
|
||||
// Fields for Dash
|
||||
private int itag;
|
||||
// Fields for DASH
|
||||
private int itag = ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE;
|
||||
private int bitrate;
|
||||
private int initStart;
|
||||
private int initEnd;
|
||||
@ -35,37 +40,241 @@ public class AudioStream extends Stream {
|
||||
private int indexEnd;
|
||||
private String quality;
|
||||
private String codec;
|
||||
@Nullable
|
||||
private ItagItem itagItem;
|
||||
|
||||
/**
|
||||
* Create a new audio stream
|
||||
* @param url the url
|
||||
* @param format the format
|
||||
* @param averageBitrate the average bitrate
|
||||
* Class to build {@link AudioStream} objects.
|
||||
*/
|
||||
public AudioStream(final String url,
|
||||
final MediaFormat format,
|
||||
final int averageBitrate) {
|
||||
super(url, format);
|
||||
@SuppressWarnings("checkstyle:hiddenField")
|
||||
public static final class Builder {
|
||||
private String id;
|
||||
private String content;
|
||||
private boolean isUrl;
|
||||
private DeliveryMethod deliveryMethod = DeliveryMethod.PROGRESSIVE_HTTP;
|
||||
@Nullable
|
||||
private MediaFormat mediaFormat;
|
||||
@Nullable
|
||||
private String manifestUrl;
|
||||
private int averageBitrate = UNKNOWN_BITRATE;
|
||||
@Nullable
|
||||
private ItagItem itagItem;
|
||||
|
||||
/**
|
||||
* Create a new {@link Builder} instance with its default values.
|
||||
*/
|
||||
public Builder() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the identifier of the {@link AudioStream}.
|
||||
*
|
||||
* <p>
|
||||
* It <b>must not be null</b> and should be non empty.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* If you are not able to get an identifier, use the static constant {@link
|
||||
* Stream#ID_UNKNOWN ID_UNKNOWN} of the {@link Stream} class.
|
||||
* </p>
|
||||
*
|
||||
* @param id the identifier of the {@link AudioStream}, which must not be null
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setId(@Nonnull final String id) {
|
||||
this.id = id;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the content of the {@link AudioStream}.
|
||||
*
|
||||
* <p>
|
||||
* It must not be null, and should be non empty.
|
||||
* </p>
|
||||
*
|
||||
* @param content the content of the {@link AudioStream}
|
||||
* @param isUrl whether the content is a URL
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setContent(@Nonnull final String content,
|
||||
final boolean isUrl) {
|
||||
this.content = content;
|
||||
this.isUrl = isUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the {@link MediaFormat} used by the {@link AudioStream}.
|
||||
*
|
||||
* <p>
|
||||
* It should be one of the audio {@link MediaFormat}s ({@link MediaFormat#M4A M4A},
|
||||
* {@link MediaFormat#WEBMA WEBMA}, {@link MediaFormat#MP3 MP3}, {@link MediaFormat#OPUS
|
||||
* OPUS}, {@link MediaFormat#OGG OGG}, or {@link MediaFormat#WEBMA_OPUS WEBMA_OPUS}) but
|
||||
* can be {@code null} if the media format could not be determined.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The default value is {@code null}.
|
||||
* </p>
|
||||
*
|
||||
* @param mediaFormat the {@link MediaFormat} of the {@link AudioStream}, which can be null
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setMediaFormat(@Nullable final MediaFormat mediaFormat) {
|
||||
this.mediaFormat = mediaFormat;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the {@link DeliveryMethod} of the {@link AudioStream}.
|
||||
*
|
||||
* <p>
|
||||
* It must not be null.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The default delivery method is {@link DeliveryMethod#PROGRESSIVE_HTTP}.
|
||||
* </p>
|
||||
*
|
||||
* @param deliveryMethod the {@link DeliveryMethod} of the {@link AudioStream}, which must
|
||||
* not be null
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setDeliveryMethod(@Nonnull final DeliveryMethod deliveryMethod) {
|
||||
this.deliveryMethod = deliveryMethod;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the URL of the manifest this stream comes from (if applicable, otherwise null).
|
||||
*
|
||||
* @param manifestUrl the URL of the manifest this stream comes from or {@code null}
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setManifestUrl(@Nullable final String manifestUrl) {
|
||||
this.manifestUrl = manifestUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the average bitrate of the {@link AudioStream}.
|
||||
*
|
||||
* <p>
|
||||
* The default value is {@link #UNKNOWN_BITRATE}.
|
||||
* </p>
|
||||
*
|
||||
* @param averageBitrate the average bitrate of the {@link AudioStream}, which should
|
||||
* positive
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setAverageBitrate(final int averageBitrate) {
|
||||
this.averageBitrate = averageBitrate;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the {@link ItagItem} corresponding to the {@link AudioStream}.
|
||||
*
|
||||
* <p>
|
||||
* {@link ItagItem}s are YouTube specific objects, so they are only known for this service
|
||||
* and can be null.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The default value is {@code null}.
|
||||
* </p>
|
||||
*
|
||||
* @param itagItem the {@link ItagItem} of the {@link AudioStream}, which can be null
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setItagItem(@Nullable final ItagItem itagItem) {
|
||||
this.itagItem = itagItem;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an {@link AudioStream} using the builder's current values.
|
||||
*
|
||||
* <p>
|
||||
* The identifier and the content (and so the {@code isUrl} boolean) properties must have
|
||||
* been set.
|
||||
* </p>
|
||||
*
|
||||
* @return a new {@link AudioStream} using the builder's current values
|
||||
* @throws IllegalStateException if {@code id}, {@code content} (and so {@code isUrl}) or
|
||||
* {@code deliveryMethod} have been not set, or have been set as {@code null}
|
||||
*/
|
||||
@Nonnull
|
||||
public AudioStream build() {
|
||||
if (id == null) {
|
||||
throw new IllegalStateException(
|
||||
"The identifier of the audio stream has been not set or is null. If you "
|
||||
+ "are not able to get an identifier, use the static constant "
|
||||
+ "ID_UNKNOWN of the Stream class.");
|
||||
}
|
||||
|
||||
if (content == null) {
|
||||
throw new IllegalStateException("The content of the audio stream has been not set "
|
||||
+ "or is null. Please specify a non-null one with setContent.");
|
||||
}
|
||||
|
||||
if (deliveryMethod == null) {
|
||||
throw new IllegalStateException(
|
||||
"The delivery method of the audio stream has been set as null, which is "
|
||||
+ "not allowed. Pass a valid one instead with setDeliveryMethod.");
|
||||
}
|
||||
|
||||
return new AudioStream(id, content, isUrl, mediaFormat, deliveryMethod, averageBitrate,
|
||||
manifestUrl, itagItem);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create a new audio stream.
|
||||
*
|
||||
* @param id the identifier which uniquely identifies the stream, e.g. for YouTube
|
||||
* this would be the itag
|
||||
* @param content the content or the URL of the stream, depending on whether isUrl is
|
||||
* true
|
||||
* @param isUrl whether content is the URL or the actual content of e.g. a DASH
|
||||
* manifest
|
||||
* @param format the {@link MediaFormat} used by the stream, which can be null
|
||||
* @param deliveryMethod the {@link DeliveryMethod} of the stream
|
||||
* @param averageBitrate the average bitrate of the stream (which can be unknown, see
|
||||
* {@link #UNKNOWN_BITRATE})
|
||||
* @param itagItem the {@link ItagItem} corresponding to the stream, which cannot be null
|
||||
* @param manifestUrl the URL of the manifest this stream comes from (if applicable,
|
||||
* otherwise null)
|
||||
*/
|
||||
@SuppressWarnings("checkstyle:ParameterNumber")
|
||||
private AudioStream(@Nonnull final String id,
|
||||
@Nonnull final String content,
|
||||
final boolean isUrl,
|
||||
@Nullable final MediaFormat format,
|
||||
@Nonnull final DeliveryMethod deliveryMethod,
|
||||
final int averageBitrate,
|
||||
@Nullable final String manifestUrl,
|
||||
@Nullable final ItagItem itagItem) {
|
||||
super(id, content, isUrl, format, deliveryMethod, manifestUrl);
|
||||
if (itagItem != null) {
|
||||
this.itagItem = itagItem;
|
||||
this.itag = itagItem.id;
|
||||
this.quality = itagItem.getQuality();
|
||||
this.bitrate = itagItem.getBitrate();
|
||||
this.initStart = itagItem.getInitStart();
|
||||
this.initEnd = itagItem.getInitEnd();
|
||||
this.indexStart = itagItem.getIndexStart();
|
||||
this.indexEnd = itagItem.getIndexEnd();
|
||||
this.codec = itagItem.getCodec();
|
||||
}
|
||||
this.averageBitrate = averageBitrate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new audio stream
|
||||
* @param url the url
|
||||
* @param itag the ItagItem of the Stream
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
public AudioStream(final String url, final ItagItem itag) {
|
||||
this(url, itag.getMediaFormat(), itag.avgBitrate);
|
||||
this.itag = itag.id;
|
||||
this.quality = itag.getQuality();
|
||||
this.bitrate = itag.getBitrate();
|
||||
this.initStart = itag.getInitStart();
|
||||
this.initEnd = itag.getInitEnd();
|
||||
this.indexStart = itag.getIndexStart();
|
||||
this.indexEnd = itag.getIndexEnd();
|
||||
this.codec = itag.getCodec();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equalStats(final Stream cmp) {
|
||||
return super.equalStats(cmp) && cmp instanceof AudioStream
|
||||
@ -73,42 +282,102 @@ public class AudioStream extends Stream {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the average bitrate
|
||||
* @return the average bitrate or -1
|
||||
* Get the average bitrate of the stream.
|
||||
*
|
||||
* @return the average bitrate or {@link #UNKNOWN_BITRATE} if it is unknown
|
||||
*/
|
||||
public int getAverageBitrate() {
|
||||
return averageBitrate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the itag identifier of the stream.
|
||||
*
|
||||
* <p>
|
||||
* Always equals to {@link #ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE} for other streams than the
|
||||
* ones of the YouTube service.
|
||||
* </p>
|
||||
*
|
||||
* @return the number of the {@link ItagItem} passed in the constructor of the audio stream.
|
||||
*/
|
||||
public int getItag() {
|
||||
return itag;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the bitrate of the stream.
|
||||
*
|
||||
* @return the bitrate set from the {@link ItagItem} passed in the constructor of the stream.
|
||||
*/
|
||||
public int getBitrate() {
|
||||
return bitrate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the initialization start of the stream.
|
||||
*
|
||||
* @return the initialization start value set from the {@link ItagItem} passed in the
|
||||
* constructor of the stream.
|
||||
*/
|
||||
public int getInitStart() {
|
||||
return initStart;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the initialization end of the stream.
|
||||
*
|
||||
* @return the initialization end value set from the {@link ItagItem} passed in the constructor
|
||||
* of the stream.
|
||||
*/
|
||||
public int getInitEnd() {
|
||||
return initEnd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the index start of the stream.
|
||||
*
|
||||
* @return the index start value set from the {@link ItagItem} passed in the constructor of the
|
||||
* stream.
|
||||
*/
|
||||
public int getIndexStart() {
|
||||
return indexStart;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the index end of the stream.
|
||||
*
|
||||
* @return the index end value set from the {@link ItagItem} passed in the constructor of the
|
||||
* stream.
|
||||
*/
|
||||
public int getIndexEnd() {
|
||||
return indexEnd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the quality of the stream.
|
||||
*
|
||||
* @return the quality label set from the {@link ItagItem} passed in the constructor of the
|
||||
* stream.
|
||||
*/
|
||||
public String getQuality() {
|
||||
return quality;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the codec of the stream.
|
||||
*
|
||||
* @return the codec set from the {@link ItagItem} passed in the constructor of the stream.
|
||||
*/
|
||||
public String getCodec() {
|
||||
return codec;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
@Nullable
|
||||
public ItagItem getItagItem() {
|
||||
return itagItem;
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,53 @@
|
||||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
/**
|
||||
* An enum to represent the different delivery methods of {@link Stream streams} which are returned
|
||||
* by the extractor.
|
||||
*/
|
||||
public enum DeliveryMethod {
|
||||
|
||||
/**
|
||||
* Used for {@link Stream}s served using the progressive HTTP streaming method.
|
||||
*/
|
||||
PROGRESSIVE_HTTP,
|
||||
|
||||
/**
|
||||
* Used for {@link Stream}s served using the DASH (Dynamic Adaptive Streaming over HTTP)
|
||||
* adaptive streaming method.
|
||||
*
|
||||
* @see <a href="https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP">the
|
||||
* Dynamic Adaptive Streaming over HTTP Wikipedia page</a> and <a href="https://dashif.org/">
|
||||
* DASH Industry Forum's website</a> for more information about the DASH delivery method
|
||||
*/
|
||||
DASH,
|
||||
|
||||
/**
|
||||
* Used for {@link Stream}s served using the HLS (HTTP Live Streaming) adaptive streaming
|
||||
* method.
|
||||
*
|
||||
* @see <a href="https://en.wikipedia.org/wiki/HTTP_Live_Streaming">the HTTP Live Streaming
|
||||
* page</a> and <a href="https://developer.apple.com/streaming">Apple's developers website page
|
||||
* about HTTP Live Streaming</a> for more information about the HLS delivery method
|
||||
*/
|
||||
HLS,
|
||||
|
||||
/**
|
||||
* Used for {@link Stream}s served using the SmoothStreaming adaptive streaming method.
|
||||
*
|
||||
* @see <a href="https://en.wikipedia.org/wiki/Adaptive_bitrate_streaming
|
||||
* #Microsoft_Smooth_Streaming_(MSS)">Wikipedia's page about adaptive bitrate streaming,
|
||||
* section <i>Microsoft Smooth Streaming (MSS)</i></a> for more information about the
|
||||
* SmoothStreaming delivery method
|
||||
*/
|
||||
SS,
|
||||
|
||||
/**
|
||||
* Used for {@link Stream}s served via a torrent file.
|
||||
*
|
||||
* @see <a href="https://en.wikipedia.org/wiki/BitTorrent">Wikipedia's BitTorrent's page</a>,
|
||||
* <a href="https://en.wikipedia.org/wiki/Torrent_file">Wikipedia's page about torrent files
|
||||
* </a> and <a href="https://www.bittorrent.org">Bitorrent's website</a> for more information
|
||||
* about the BitTorrent protocol
|
||||
*/
|
||||
TORRENT
|
||||
}
|
@ -1,68 +1,72 @@
|
||||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
/**
|
||||
* Creates a stream object from url, format and optional torrent url
|
||||
* Abstract class which represents streams in the extractor.
|
||||
*/
|
||||
public abstract class Stream implements Serializable {
|
||||
private final MediaFormat mediaFormat;
|
||||
private final String url;
|
||||
private final String torrentUrl;
|
||||
public static final int FORMAT_ID_UNKNOWN = -1;
|
||||
public static final String ID_UNKNOWN = " ";
|
||||
|
||||
/**
|
||||
* @deprecated Use {@link #getFormat()} or {@link #getFormatId()}
|
||||
*/
|
||||
@Deprecated
|
||||
public final int format;
|
||||
|
||||
/**
|
||||
* Instantiates a new stream object.
|
||||
* An integer to represent that the itag ID returned is not available (only for YouTube; this
|
||||
* should never happen) or not applicable (for other services than YouTube).
|
||||
*
|
||||
* @param url the url
|
||||
* @param format the format
|
||||
* <p>
|
||||
* An itag should not have a negative value, so {@code -1} is used for this constant.
|
||||
* </p>
|
||||
*/
|
||||
public Stream(final String url, final MediaFormat format) {
|
||||
this(url, null, format);
|
||||
}
|
||||
public static final int ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE = -1;
|
||||
|
||||
private final String id;
|
||||
@Nullable private final MediaFormat mediaFormat;
|
||||
private final String content;
|
||||
private final boolean isUrl;
|
||||
private final DeliveryMethod deliveryMethod;
|
||||
@Nullable private final String manifestUrl;
|
||||
|
||||
/**
|
||||
* Instantiates a new stream object.
|
||||
* Instantiates a new {@code Stream} object.
|
||||
*
|
||||
* @param url the url
|
||||
* @param torrentUrl the url to torrent file, example
|
||||
* https://webtorrent.io/torrents/big-buck-bunny.torrent
|
||||
* @param format the format
|
||||
* @param id the identifier which uniquely identifies the file, e.g. for YouTube
|
||||
* this would be the itag
|
||||
* @param content the content or URL, depending on whether isUrl is true
|
||||
* @param isUrl whether content is the URL or the actual content of e.g. a DASH
|
||||
* manifest
|
||||
* @param format the {@link MediaFormat}, which can be null
|
||||
* @param deliveryMethod the delivery method of the stream
|
||||
* @param manifestUrl the URL of the manifest this stream comes from (if applicable,
|
||||
* otherwise null)
|
||||
*/
|
||||
public Stream(final String url, final String torrentUrl, final MediaFormat format) {
|
||||
this.url = url;
|
||||
this.torrentUrl = torrentUrl;
|
||||
//noinspection deprecation
|
||||
this.format = format.id;
|
||||
public Stream(final String id,
|
||||
final String content,
|
||||
final boolean isUrl,
|
||||
@Nullable final MediaFormat format,
|
||||
final DeliveryMethod deliveryMethod,
|
||||
@Nullable final String manifestUrl) {
|
||||
this.id = id;
|
||||
this.content = content;
|
||||
this.isUrl = isUrl;
|
||||
this.mediaFormat = format;
|
||||
this.deliveryMethod = deliveryMethod;
|
||||
this.manifestUrl = manifestUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reveals whether two streams have the same stats (format and bitrate, for example)
|
||||
*/
|
||||
public boolean equalStats(final Stream cmp) {
|
||||
return cmp != null && getFormatId() == cmp.getFormatId();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reveals whether two Streams are equal
|
||||
*/
|
||||
public boolean equals(final Stream cmp) {
|
||||
return equalStats(cmp) && url.equals(cmp.url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the list already contains one stream with equals stats
|
||||
* Checks if the list already contains a stream with the same statistics.
|
||||
*
|
||||
* @param stream the stream to be compared against the streams in the stream list
|
||||
* @param streamList the list of {@link Stream}s which will be compared
|
||||
* @return whether the list already contains one stream with equals stats
|
||||
*/
|
||||
public static boolean containSimilarStream(final Stream stream,
|
||||
final List<? extends Stream> streamList) {
|
||||
@ -78,38 +82,126 @@ public abstract class Stream implements Serializable {
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the url.
|
||||
* Reveals whether two streams have the same statistics ({@link MediaFormat media format} and
|
||||
* {@link DeliveryMethod delivery method}).
|
||||
*
|
||||
* @return the url
|
||||
* <p>
|
||||
* If the {@link MediaFormat media format} of the stream is unknown, the streams are compared
|
||||
* by using only the {@link DeliveryMethod delivery method} and their ID.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Note: This method always returns false if the stream passed is null.
|
||||
* </p>
|
||||
*
|
||||
* @param other the stream object to be compared to this stream object
|
||||
* @return whether the stream have the same stats or not, based on the criteria above
|
||||
*/
|
||||
public boolean equalStats(@Nullable final Stream other) {
|
||||
if (other == null || mediaFormat == null || other.mediaFormat == null) {
|
||||
return false;
|
||||
}
|
||||
return mediaFormat.id == other.mediaFormat.id && deliveryMethod == other.deliveryMethod
|
||||
&& isUrl == other.isUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the identifier of this stream, e.g. the itag for YouTube.
|
||||
*
|
||||
* <p>
|
||||
* It should normally be unique, but {@link #ID_UNKNOWN} may be returned as the identifier if
|
||||
* the one used by the stream extractor cannot be extracted, which could happen if the
|
||||
* extractor uses a value from a streaming service.
|
||||
* </p>
|
||||
*
|
||||
* @return the identifier (which may be {@link #ID_UNKNOWN})
|
||||
*/
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the URL of this stream if the content is a URL, or {@code null} otherwise.
|
||||
*
|
||||
* @return the URL if the content is a URL, {@code null} otherwise
|
||||
* @deprecated Use {@link #getContent()} instead.
|
||||
*/
|
||||
@Deprecated
|
||||
@Nullable
|
||||
public String getUrl() {
|
||||
return url;
|
||||
return isUrl ? content : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the torrent url.
|
||||
* Gets the content or URL.
|
||||
*
|
||||
* @return the torrent url, example https://webtorrent.io/torrents/big-buck-bunny.torrent
|
||||
* @return the content or URL
|
||||
*/
|
||||
public String getTorrentUrl() {
|
||||
return torrentUrl;
|
||||
public String getContent() {
|
||||
return content;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the format.
|
||||
* Returns whether the content is a URL or not.
|
||||
*
|
||||
* @return {@code true} if the content of this stream is a URL, {@code false} if it's the
|
||||
* actual content
|
||||
*/
|
||||
public boolean isUrl() {
|
||||
return isUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the {@link MediaFormat}, which can be null.
|
||||
*
|
||||
* @return the format
|
||||
*/
|
||||
@Nullable
|
||||
public MediaFormat getFormat() {
|
||||
return mediaFormat;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the format id.
|
||||
* Gets the format ID, which can be unknown.
|
||||
*
|
||||
* @return the format id
|
||||
* @return the format ID or {@link #FORMAT_ID_UNKNOWN}
|
||||
*/
|
||||
public int getFormatId() {
|
||||
return mediaFormat.id;
|
||||
if (mediaFormat != null) {
|
||||
return mediaFormat.id;
|
||||
}
|
||||
return FORMAT_ID_UNKNOWN;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the {@link DeliveryMethod}.
|
||||
*
|
||||
* @return the delivery method
|
||||
*/
|
||||
@Nonnull
|
||||
public DeliveryMethod getDeliveryMethod() {
|
||||
return deliveryMethod;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the URL of the manifest this stream comes from (if applicable, otherwise null).
|
||||
*
|
||||
* @return the URL of the manifest this stream comes from or {@code null}
|
||||
*/
|
||||
@Nullable
|
||||
public String getManifestUrl() {
|
||||
return manifestUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the {@link ItagItem} of a stream.
|
||||
*
|
||||
* <p>
|
||||
* If the stream is not from YouTube, this value will always be null.
|
||||
* </p>
|
||||
*
|
||||
* @return the {@link ItagItem} of the stream or {@code null}
|
||||
*/
|
||||
@Nullable
|
||||
public abstract ItagItem getItagItem();
|
||||
}
|
||||
|
@ -9,7 +9,6 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
||||
import org.schabi.newpipe.extractor.utils.DashMpdParser;
|
||||
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
|
||||
|
||||
import java.io.IOException;
|
||||
@ -26,24 +25,24 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
* Created by Christian Schabesberger on 26.08.15.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
||||
* StreamInfo.java is part of NewPipe.
|
||||
* StreamInfo.java is part of NewPipe Extractor.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
* NewPipe Extractor is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe is distributed in the hope that it will be useful,
|
||||
* NewPipe Extractor is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
* along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Info object for opened videos, ie the video ready to play.
|
||||
* Info object for opened contents, i.e. the content ready to play.
|
||||
*/
|
||||
public class StreamInfo extends Info {
|
||||
|
||||
@ -69,27 +68,26 @@ public class StreamInfo extends Info {
|
||||
return getInfo(NewPipe.getServiceByUrl(url), url);
|
||||
}
|
||||
|
||||
public static StreamInfo getInfo(final StreamingService service,
|
||||
public static StreamInfo getInfo(@Nonnull final StreamingService service,
|
||||
final String url) throws IOException, ExtractionException {
|
||||
return getInfo(service.getStreamExtractor(url));
|
||||
}
|
||||
|
||||
public static StreamInfo getInfo(final StreamExtractor extractor)
|
||||
public static StreamInfo getInfo(@Nonnull final StreamExtractor extractor)
|
||||
throws ExtractionException, IOException {
|
||||
extractor.fetchPage();
|
||||
final StreamInfo streamInfo;
|
||||
try {
|
||||
final StreamInfo streamInfo = extractImportantData(extractor);
|
||||
streamInfo = extractImportantData(extractor);
|
||||
extractStreams(streamInfo, extractor);
|
||||
extractOptionalData(streamInfo, extractor);
|
||||
return streamInfo;
|
||||
|
||||
} catch (final ExtractionException e) {
|
||||
// Currently YouTube does not distinguish between age restricted videos and
|
||||
// videos blocked
|
||||
// by country. This means that during the initialisation of the extractor, the
|
||||
// extractor
|
||||
// will assume that a video is age restricted while in reality it it blocked by
|
||||
// country.
|
||||
// Currently, YouTube does not distinguish between age restricted videos and videos
|
||||
// blocked by country. This means that during the initialisation of the extractor, the
|
||||
// extractor will assume that a video is age restricted while in reality it is blocked
|
||||
// by country.
|
||||
//
|
||||
// We will now detect whether the video is blocked by country or not.
|
||||
|
||||
@ -102,22 +100,27 @@ public class StreamInfo extends Info {
|
||||
}
|
||||
}
|
||||
|
||||
private static StreamInfo extractImportantData(final StreamExtractor extractor)
|
||||
@Nonnull
|
||||
private static StreamInfo extractImportantData(@Nonnull final StreamExtractor extractor)
|
||||
throws ExtractionException {
|
||||
/* ---- important data, without the video can't be displayed goes here: ---- */
|
||||
// if one of these is not available an exception is meant to be thrown directly
|
||||
// into the frontend.
|
||||
// Important data, without it the content can't be displayed.
|
||||
// If one of these is not available, the frontend will receive an exception directly.
|
||||
|
||||
final int serviceId = extractor.getServiceId();
|
||||
final String url = extractor.getUrl();
|
||||
final String originalUrl = extractor.getOriginalUrl();
|
||||
final StreamType streamType = extractor.getStreamType();
|
||||
final String id = extractor.getId();
|
||||
final String name = extractor.getName();
|
||||
final int ageLimit = extractor.getAgeLimit();
|
||||
|
||||
// suppress always-non-null warning as here we double-check it really is not null
|
||||
// Suppress always-non-null warning as here we double-check it really is not null
|
||||
//noinspection ConstantConditions
|
||||
if (streamType == StreamType.NONE || isNullOrEmpty(url) || isNullOrEmpty(id)
|
||||
|| name == null /* but it can be empty of course */ || ageLimit == -1) {
|
||||
if (streamType == StreamType.NONE
|
||||
|| isNullOrEmpty(url)
|
||||
|| isNullOrEmpty(id)
|
||||
|| name == null /* but it can be empty of course */
|
||||
|| ageLimit == -1) {
|
||||
throw new ExtractionException("Some important stream information was not given.");
|
||||
}
|
||||
|
||||
@ -125,16 +128,18 @@ public class StreamInfo extends Info {
|
||||
streamType, id, name, ageLimit);
|
||||
}
|
||||
|
||||
private static void extractStreams(final StreamInfo streamInfo, final StreamExtractor extractor)
|
||||
|
||||
private static void extractStreams(final StreamInfo streamInfo,
|
||||
final StreamExtractor extractor)
|
||||
throws ExtractionException {
|
||||
/* ---- stream extraction goes here ---- */
|
||||
// At least one type of stream has to be available,
|
||||
// otherwise an exception will be thrown directly into the frontend.
|
||||
/* ---- Stream extraction goes here ---- */
|
||||
// At least one type of stream has to be available, otherwise an exception will be thrown
|
||||
// directly into the frontend.
|
||||
|
||||
try {
|
||||
streamInfo.setDashMpdUrl(extractor.getDashMpdUrl());
|
||||
} catch (final Exception e) {
|
||||
streamInfo.addError(new ExtractionException("Couldn't get Dash manifest", e));
|
||||
streamInfo.addError(new ExtractionException("Couldn't get DASH manifest", e));
|
||||
}
|
||||
|
||||
try {
|
||||
@ -151,12 +156,14 @@ public class StreamInfo extends Info {
|
||||
} catch (final Exception e) {
|
||||
streamInfo.addError(new ExtractionException("Couldn't get audio streams", e));
|
||||
}
|
||||
|
||||
/* Extract video stream url */
|
||||
try {
|
||||
streamInfo.setVideoStreams(extractor.getVideoStreams());
|
||||
} catch (final Exception e) {
|
||||
streamInfo.addError(new ExtractionException("Couldn't get video streams", e));
|
||||
}
|
||||
|
||||
/* Extract video only stream url */
|
||||
try {
|
||||
streamInfo.setVideoOnlyStreams(extractor.getVideoOnlyStreams());
|
||||
@ -164,7 +171,7 @@ public class StreamInfo extends Info {
|
||||
streamInfo.addError(new ExtractionException("Couldn't get video only streams", e));
|
||||
}
|
||||
|
||||
// Lists can be null if a exception was thrown during extraction
|
||||
// Lists can be null if an exception was thrown during extraction
|
||||
if (streamInfo.getVideoStreams() == null) {
|
||||
streamInfo.setVideoStreams(Collections.emptyList());
|
||||
}
|
||||
@ -175,37 +182,9 @@ public class StreamInfo extends Info {
|
||||
streamInfo.setAudioStreams(Collections.emptyList());
|
||||
}
|
||||
|
||||
Exception dashMpdError = null;
|
||||
if (!isNullOrEmpty(streamInfo.getDashMpdUrl())) {
|
||||
try {
|
||||
final DashMpdParser.ParserResult result = DashMpdParser.getStreams(streamInfo);
|
||||
streamInfo.getVideoOnlyStreams().addAll(result.getVideoOnlyStreams());
|
||||
streamInfo.getAudioStreams().addAll(result.getAudioStreams());
|
||||
streamInfo.getVideoStreams().addAll(result.getVideoStreams());
|
||||
streamInfo.segmentedVideoOnlyStreams = result.getSegmentedVideoOnlyStreams();
|
||||
streamInfo.segmentedAudioStreams = result.getSegmentedAudioStreams();
|
||||
streamInfo.segmentedVideoStreams = result.getSegmentedVideoStreams();
|
||||
} catch (final Exception e) {
|
||||
// Sometimes we receive 403 (forbidden) error when trying to download the
|
||||
// manifest (similar to what happens with youtube-dl),
|
||||
// just skip the exception (but store it somewhere), as we later check if we
|
||||
// have streams anyway.
|
||||
dashMpdError = e;
|
||||
}
|
||||
}
|
||||
|
||||
// Either audio or video has to be available, otherwise we didn't get a stream
|
||||
// (since videoOnly are optional, they don't count).
|
||||
// Either audio or video has to be available, otherwise we didn't get a stream (since
|
||||
// videoOnly are optional, they don't count).
|
||||
if ((streamInfo.videoStreams.isEmpty()) && (streamInfo.audioStreams.isEmpty())) {
|
||||
|
||||
if (dashMpdError != null) {
|
||||
// If we don't have any video or audio and the dashMpd 'errored', add it to the
|
||||
// error list
|
||||
// (it's optional and it don't get added automatically, but it's good to have
|
||||
// some additional error context)
|
||||
streamInfo.addError(dashMpdError);
|
||||
}
|
||||
|
||||
throw new StreamExtractException(
|
||||
"Could not get any stream. See error variable to get further details.");
|
||||
}
|
||||
@ -214,11 +193,9 @@ public class StreamInfo extends Info {
|
||||
@SuppressWarnings("MethodLength")
|
||||
private static void extractOptionalData(final StreamInfo streamInfo,
|
||||
final StreamExtractor extractor) {
|
||||
/* ---- optional data goes here: ---- */
|
||||
// If one of these fails, the frontend needs to handle that they are not
|
||||
// available.
|
||||
// Exceptions are therefore not thrown into the frontend, but stored into the
|
||||
// error List,
|
||||
/* ---- Optional data goes here: ---- */
|
||||
// If one of these fails, the frontend needs to handle that they are not available.
|
||||
// Exceptions are therefore not thrown into the frontend, but stored into the error list,
|
||||
// so the frontend can afterwards check where errors happened.
|
||||
|
||||
try {
|
||||
@ -314,7 +291,7 @@ public class StreamInfo extends Info {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
|
||||
//additional info
|
||||
// Additional info
|
||||
try {
|
||||
streamInfo.setHost(extractor.getHost());
|
||||
} catch (final Exception e) {
|
||||
@ -360,15 +337,14 @@ public class StreamInfo extends Info {
|
||||
} catch (final Exception e) {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
|
||||
try {
|
||||
streamInfo.setPreviewFrames(extractor.getFrames());
|
||||
} catch (final Exception e) {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
|
||||
streamInfo
|
||||
.setRelatedItems(ExtractorHelper.getRelatedItemsOrLogError(streamInfo, extractor));
|
||||
streamInfo.setRelatedItems(ExtractorHelper.getRelatedItemsOrLogError(streamInfo,
|
||||
extractor));
|
||||
}
|
||||
|
||||
private StreamType streamType;
|
||||
@ -398,11 +374,6 @@ public class StreamInfo extends Info {
|
||||
private List<VideoStream> videoOnlyStreams = new ArrayList<>();
|
||||
|
||||
private String dashMpdUrl = "";
|
||||
private List<VideoStream> segmentedVideoStreams = new ArrayList<>();
|
||||
private List<AudioStream> segmentedAudioStreams = new ArrayList<>();
|
||||
private List<VideoStream> segmentedVideoOnlyStreams = new ArrayList<>();
|
||||
|
||||
|
||||
private String hlsUrl = "";
|
||||
private List<InfoItem> relatedItems = new ArrayList<>();
|
||||
|
||||
@ -625,30 +596,6 @@ public class StreamInfo extends Info {
|
||||
this.dashMpdUrl = dashMpdUrl;
|
||||
}
|
||||
|
||||
public List<VideoStream> getSegmentedVideoStreams() {
|
||||
return segmentedVideoStreams;
|
||||
}
|
||||
|
||||
public void setSegmentedVideoStreams(final List<VideoStream> segmentedVideoStreams) {
|
||||
this.segmentedVideoStreams = segmentedVideoStreams;
|
||||
}
|
||||
|
||||
public List<AudioStream> getSegmentedAudioStreams() {
|
||||
return segmentedAudioStreams;
|
||||
}
|
||||
|
||||
public void setSegmentedAudioStreams(final List<AudioStream> segmentedAudioStreams) {
|
||||
this.segmentedAudioStreams = segmentedAudioStreams;
|
||||
}
|
||||
|
||||
public List<VideoStream> getSegmentedVideoOnlyStreams() {
|
||||
return segmentedVideoOnlyStreams;
|
||||
}
|
||||
|
||||
public void setSegmentedVideoOnlyStreams(final List<VideoStream> segmentedVideoOnlyStreams) {
|
||||
this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams;
|
||||
}
|
||||
|
||||
public String getHlsUrl() {
|
||||
return hlsUrl;
|
||||
}
|
||||
|
@ -1,10 +1,74 @@
|
||||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
/**
|
||||
* An enum representing the stream type of a {@link StreamInfo} extracted by a {@link
|
||||
* StreamExtractor}.
|
||||
*/
|
||||
public enum StreamType {
|
||||
NONE, // placeholder to check if stream type was checked or not
|
||||
|
||||
/**
|
||||
* Placeholder to check if the stream type was checked or not. It doesn't make sense to use this
|
||||
* enum constant outside of the extractor as it will never be returned by an {@link
|
||||
* org.schabi.newpipe.extractor.Extractor} and is only used internally.
|
||||
*/
|
||||
NONE,
|
||||
|
||||
/**
|
||||
* A normal video stream, usually with audio. Note that the {@link StreamInfo} <strong>can also
|
||||
* provide audio-only {@link AudioStream}s</strong> in addition to video or video-only {@link
|
||||
* VideoStream}s.
|
||||
*/
|
||||
VIDEO_STREAM,
|
||||
|
||||
/**
|
||||
* An audio-only stream. There should be no {@link VideoStream}s available! In order to prevent
|
||||
* unexpected behaviors, when {@link StreamExtractor}s return this stream type, they should
|
||||
* ensure that no video stream is returned in {@link StreamExtractor#getVideoStreams()} and
|
||||
* {@link StreamExtractor#getVideoOnlyStreams()}.
|
||||
*/
|
||||
AUDIO_STREAM,
|
||||
|
||||
/**
|
||||
* A video live stream, usually with audio. Note that the {@link StreamInfo} <strong>can also
|
||||
* provide audio-only {@link AudioStream}s</strong> in addition to video or video-only {@link
|
||||
* VideoStream}s.
|
||||
*/
|
||||
LIVE_STREAM,
|
||||
|
||||
/**
|
||||
* An audio-only live stream. There should be no {@link VideoStream}s available! In order to
|
||||
* prevent unexpected behaviors, when {@link StreamExtractor}s return this stream type, they
|
||||
* should ensure that no video stream is returned in {@link StreamExtractor#getVideoStreams()}
|
||||
* and {@link StreamExtractor#getVideoOnlyStreams()}.
|
||||
*/
|
||||
AUDIO_LIVE_STREAM,
|
||||
FILE
|
||||
|
||||
/**
|
||||
* A video live stream that has just ended but has not yet been encoded into a normal video
|
||||
* stream. Note that the {@link StreamInfo} <strong>can also provide audio-only {@link
|
||||
* AudioStream}s</strong> in addition to video or video-only {@link VideoStream}s.
|
||||
*
|
||||
* <p>
|
||||
* Note that most of the content of an ended live video (or audio) may be extracted as {@link
|
||||
* #VIDEO_STREAM regular video contents} (or {@link #AUDIO_STREAM regular audio contents})
|
||||
* later, because the service may encode them again later as normal video/audio streams. That's
|
||||
* the case on YouTube, for example.
|
||||
* </p>
|
||||
*/
|
||||
POST_LIVE_STREAM,
|
||||
|
||||
/**
|
||||
* An audio live stream that has just ended but has not yet been encoded into a normal audio
|
||||
* stream. There should be no {@link VideoStream}s available! In order to prevent unexpected
|
||||
* behaviors, when {@link StreamExtractor}s return this stream type, they should ensure that no
|
||||
* video stream is returned in {@link StreamExtractor#getVideoStreams()} and
|
||||
* {@link StreamExtractor#getVideoOnlyStreams()}.
|
||||
*
|
||||
* <p>
|
||||
* Note that most of ended live audio streams extracted with this value are processed as
|
||||
* {@link #AUDIO_STREAM regular audio streams} later, because the service may encode them
|
||||
* again later.
|
||||
* </p>
|
||||
*/
|
||||
POST_LIVE_AUDIO_STREAM
|
||||
}
|
||||
|
@ -1,53 +1,286 @@
|
||||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Locale;
|
||||
|
||||
public class SubtitlesStream extends Stream implements Serializable {
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
|
||||
public final class SubtitlesStream extends Stream {
|
||||
private final MediaFormat format;
|
||||
private final Locale locale;
|
||||
private final boolean autoGenerated;
|
||||
private final String code;
|
||||
|
||||
public SubtitlesStream(final MediaFormat format,
|
||||
final String languageCode,
|
||||
final String url,
|
||||
final boolean autoGenerated) {
|
||||
super(url, format);
|
||||
/**
|
||||
* Class to build {@link SubtitlesStream} objects.
|
||||
*/
|
||||
@SuppressWarnings("checkstyle:HiddenField")
|
||||
public static final class Builder {
|
||||
private String id;
|
||||
private String content;
|
||||
private boolean isUrl;
|
||||
private DeliveryMethod deliveryMethod = DeliveryMethod.PROGRESSIVE_HTTP;
|
||||
@Nullable
|
||||
private MediaFormat mediaFormat;
|
||||
@Nullable
|
||||
private String manifestUrl;
|
||||
private String languageCode;
|
||||
// Use of the Boolean class instead of the primitive type needed for setter call check
|
||||
private Boolean autoGenerated;
|
||||
|
||||
/**
|
||||
* Create a new {@link Builder} instance with default values.
|
||||
*/
|
||||
public Builder() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the identifier of the {@link SubtitlesStream}.
|
||||
*
|
||||
* @param id the identifier of the {@link SubtitlesStream}, which should not be null
|
||||
* (otherwise the fallback to create the identifier will be used when building
|
||||
* the builder)
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setId(@Nonnull final String id) {
|
||||
this.id = id;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the content of the {@link SubtitlesStream}.
|
||||
*
|
||||
* <p>
|
||||
* It must not be null, and should be non empty.
|
||||
* </p>
|
||||
*
|
||||
* @param content the content of the {@link SubtitlesStream}, which must not be null
|
||||
* @param isUrl whether the content is a URL
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setContent(@Nonnull final String content,
|
||||
final boolean isUrl) {
|
||||
this.content = content;
|
||||
this.isUrl = isUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the {@link MediaFormat} used by the {@link SubtitlesStream}.
|
||||
*
|
||||
* <p>
|
||||
* It should be one of the subtitles {@link MediaFormat}s ({@link MediaFormat#SRT SRT},
|
||||
* {@link MediaFormat#TRANSCRIPT1 TRANSCRIPT1}, {@link MediaFormat#TRANSCRIPT2
|
||||
* TRANSCRIPT2}, {@link MediaFormat#TRANSCRIPT3 TRANSCRIPT3}, {@link MediaFormat#TTML
|
||||
* TTML}, or {@link MediaFormat#VTT VTT}) but can be {@code null} if the media format could
|
||||
* not be determined.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The default value is {@code null}.
|
||||
* </p>
|
||||
*
|
||||
* @param mediaFormat the {@link MediaFormat} of the {@link SubtitlesStream}, which can be
|
||||
* null
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setMediaFormat(@Nullable final MediaFormat mediaFormat) {
|
||||
this.mediaFormat = mediaFormat;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the {@link DeliveryMethod} of the {@link SubtitlesStream}.
|
||||
*
|
||||
* <p>
|
||||
* It must not be null.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The default delivery method is {@link DeliveryMethod#PROGRESSIVE_HTTP}.
|
||||
* </p>
|
||||
*
|
||||
* @param deliveryMethod the {@link DeliveryMethod} of the {@link SubtitlesStream}, which
|
||||
* must not be null
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setDeliveryMethod(@Nonnull final DeliveryMethod deliveryMethod) {
|
||||
this.deliveryMethod = deliveryMethod;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the URL of the manifest this stream comes from (if applicable, otherwise null).
|
||||
*
|
||||
* @param manifestUrl the URL of the manifest this stream comes from or {@code null}
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setManifestUrl(@Nullable final String manifestUrl) {
|
||||
this.manifestUrl = manifestUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the language code of the {@link SubtitlesStream}.
|
||||
*
|
||||
* <p>
|
||||
* It <b>must not be null</b> and should not be an empty string.
|
||||
* </p>
|
||||
*
|
||||
* @param languageCode the language code of the {@link SubtitlesStream}
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setLanguageCode(@Nonnull final String languageCode) {
|
||||
this.languageCode = languageCode;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set whether the subtitles have been auto-generated by the streaming service.
|
||||
*
|
||||
* @param autoGenerated whether the subtitles have been generated by the streaming
|
||||
* service
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setAutoGenerated(final boolean autoGenerated) {
|
||||
this.autoGenerated = autoGenerated;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a {@link SubtitlesStream} using the builder's current values.
|
||||
*
|
||||
* <p>
|
||||
* The content (and so the {@code isUrl} boolean), the language code and the {@code
|
||||
* isAutoGenerated} properties must have been set.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* If no identifier has been set, an identifier will be generated using the language code
|
||||
* and the media format suffix, if the media format is known.
|
||||
* </p>
|
||||
*
|
||||
* @return a new {@link SubtitlesStream} using the builder's current values
|
||||
* @throws IllegalStateException if {@code id}, {@code content} (and so {@code isUrl}),
|
||||
* {@code deliveryMethod}, {@code languageCode} or the {@code isAutogenerated} have been
|
||||
* not set, or have been set as {@code null}
|
||||
*/
|
||||
@Nonnull
|
||||
public SubtitlesStream build() {
|
||||
if (content == null) {
|
||||
throw new IllegalStateException("No valid content was specified. Please specify a "
|
||||
+ "valid one with setContent.");
|
||||
}
|
||||
|
||||
if (deliveryMethod == null) {
|
||||
throw new IllegalStateException(
|
||||
"The delivery method of the subtitles stream has been set as null, which "
|
||||
+ "is not allowed. Pass a valid one instead with"
|
||||
+ "setDeliveryMethod.");
|
||||
}
|
||||
|
||||
if (languageCode == null) {
|
||||
throw new IllegalStateException("The language code of the subtitles stream has "
|
||||
+ "been not set or is null. Make sure you specified an non null language "
|
||||
+ "code with setLanguageCode.");
|
||||
}
|
||||
|
||||
if (autoGenerated == null) {
|
||||
throw new IllegalStateException("The subtitles stream has been not set as an "
|
||||
+ "autogenerated subtitles stream or not. Please specify this information "
|
||||
+ "with setIsAutoGenerated.");
|
||||
}
|
||||
|
||||
if (id == null) {
|
||||
id = languageCode + (mediaFormat != null ? "." + mediaFormat.suffix
|
||||
: EMPTY_STRING);
|
||||
}
|
||||
|
||||
return new SubtitlesStream(id, content, isUrl, mediaFormat, deliveryMethod,
|
||||
languageCode, autoGenerated, manifestUrl);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new subtitles stream.
|
||||
*
|
||||
* @param id the identifier which uniquely identifies the stream, e.g. for YouTube
|
||||
* this would be the itag
|
||||
* @param content the content or the URL of the stream, depending on whether isUrl is
|
||||
* true
|
||||
* @param isUrl whether content is the URL or the actual content of e.g. a DASH
|
||||
* manifest
|
||||
* @param mediaFormat the {@link MediaFormat} used by the stream
|
||||
* @param deliveryMethod the {@link DeliveryMethod} of the stream
|
||||
* @param languageCode the language code of the stream
|
||||
* @param autoGenerated whether the subtitles are auto-generated by the streaming service
|
||||
* @param manifestUrl the URL of the manifest this stream comes from (if applicable,
|
||||
* otherwise null)
|
||||
*/
|
||||
@SuppressWarnings("checkstyle:ParameterNumber")
|
||||
private SubtitlesStream(@Nonnull final String id,
|
||||
@Nonnull final String content,
|
||||
final boolean isUrl,
|
||||
@Nullable final MediaFormat mediaFormat,
|
||||
@Nonnull final DeliveryMethod deliveryMethod,
|
||||
@Nonnull final String languageCode,
|
||||
final boolean autoGenerated,
|
||||
@Nullable final String manifestUrl) {
|
||||
super(id, content, isUrl, mediaFormat, deliveryMethod, manifestUrl);
|
||||
|
||||
/*
|
||||
* Locale.forLanguageTag only for API >= 21
|
||||
* Locale.Builder only for API >= 21
|
||||
* Country codes doesn't work well without
|
||||
*/
|
||||
* Locale.forLanguageTag only for Android API >= 21
|
||||
* Locale.Builder only for Android API >= 21
|
||||
* Country codes doesn't work well without
|
||||
*/
|
||||
final String[] splits = languageCode.split("-");
|
||||
switch (splits.length) {
|
||||
default:
|
||||
this.locale = new Locale(splits[0]);
|
||||
break;
|
||||
case 3:
|
||||
// complex variants doesn't work!
|
||||
this.locale = new Locale(splits[0], splits[1], splits[2]);
|
||||
break;
|
||||
case 2:
|
||||
this.locale = new Locale(splits[0], splits[1]);
|
||||
break;
|
||||
case 3:
|
||||
// Complex variants don't work!
|
||||
this.locale = new Locale(splits[0], splits[1], splits[2]);
|
||||
break;
|
||||
default:
|
||||
this.locale = new Locale(splits[0]);
|
||||
break;
|
||||
}
|
||||
|
||||
this.code = languageCode;
|
||||
this.format = format;
|
||||
this.format = mediaFormat;
|
||||
this.autoGenerated = autoGenerated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the extension of the subtitles.
|
||||
*
|
||||
* @return the extension of the subtitles
|
||||
*/
|
||||
public String getExtension() {
|
||||
return format.suffix;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return whether if the subtitles are auto-generated.
|
||||
* <p>
|
||||
* Some streaming services can generate subtitles for their contents, like YouTube.
|
||||
* </p>
|
||||
*
|
||||
* @return {@code true} if the subtitles are auto-generated, {@code false} otherwise
|
||||
*/
|
||||
public boolean isAutoGenerated() {
|
||||
return autoGenerated;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public boolean equalStats(final Stream cmp) {
|
||||
return super.equalStats(cmp)
|
||||
@ -56,16 +289,42 @@ public class SubtitlesStream extends Stream implements Serializable {
|
||||
&& autoGenerated == ((SubtitlesStream) cmp).autoGenerated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the display language name of the subtitles.
|
||||
*
|
||||
* @return the display language name of the subtitles
|
||||
*/
|
||||
public String getDisplayLanguageName() {
|
||||
return locale.getDisplayName(locale);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the language tag of the subtitles.
|
||||
*
|
||||
* @return the language tag of the subtitles
|
||||
*/
|
||||
public String getLanguageTag() {
|
||||
return code;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the {@link Locale locale} of the subtitles.
|
||||
*
|
||||
* @return the {@link Locale locale} of the subtitles
|
||||
*/
|
||||
public Locale getLocale() {
|
||||
return locale;
|
||||
}
|
||||
|
||||
/**
|
||||
* No subtitles which are currently extracted use an {@link ItagItem}, so {@code null} is
|
||||
* returned by this method.
|
||||
*
|
||||
* @return {@code null}
|
||||
*/
|
||||
@Nullable
|
||||
@Override
|
||||
public ItagItem getItagItem() {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
@ -4,31 +4,41 @@ package org.schabi.newpipe.extractor.stream;
|
||||
* Created by Christian Schabesberger on 04.03.16.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
||||
* VideoStream.java is part of NewPipe.
|
||||
* VideoStream.java is part of NewPipe Extractor.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
* NewPipe Extractor is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe is distributed in the hope that it will be useful,
|
||||
* NewPipe Extractor is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
* along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
|
||||
public class VideoStream extends Stream {
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
public final class VideoStream extends Stream {
|
||||
public static final String RESOLUTION_UNKNOWN = "";
|
||||
|
||||
/** @deprecated Use {@link #getResolution()} instead. */
|
||||
@Deprecated
|
||||
public final String resolution;
|
||||
|
||||
/** @deprecated Use {@link #isVideoOnly()} instead. */
|
||||
@Deprecated
|
||||
public final boolean isVideoOnly;
|
||||
|
||||
// Fields for Dash
|
||||
private int itag;
|
||||
// Fields for DASH
|
||||
private int itag = ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE;
|
||||
private int bitrate;
|
||||
private int initStart;
|
||||
private int initEnd;
|
||||
@ -39,118 +49,437 @@ public class VideoStream extends Stream {
|
||||
private int fps;
|
||||
private String quality;
|
||||
private String codec;
|
||||
@Nullable private ItagItem itagItem;
|
||||
|
||||
public VideoStream(final String url, final MediaFormat format, final String resolution) {
|
||||
this(url, format, resolution, false);
|
||||
/**
|
||||
* Class to build {@link VideoStream} objects.
|
||||
*/
|
||||
@SuppressWarnings("checkstyle:hiddenField")
|
||||
public static final class Builder {
|
||||
private String id;
|
||||
private String content;
|
||||
private boolean isUrl;
|
||||
private DeliveryMethod deliveryMethod = DeliveryMethod.PROGRESSIVE_HTTP;
|
||||
@Nullable
|
||||
private MediaFormat mediaFormat;
|
||||
@Nullable
|
||||
private String manifestUrl;
|
||||
// Use of the Boolean class instead of the primitive type needed for setter call check
|
||||
private Boolean isVideoOnly;
|
||||
private String resolution;
|
||||
@Nullable
|
||||
private ItagItem itagItem;
|
||||
|
||||
/**
|
||||
* Create a new {@link Builder} instance with its default values.
|
||||
*/
|
||||
public Builder() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the identifier of the {@link VideoStream}.
|
||||
*
|
||||
* <p>
|
||||
* It must not be null, and should be non empty.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* If you are not able to get an identifier, use the static constant {@link
|
||||
* Stream#ID_UNKNOWN ID_UNKNOWN} of the {@link Stream} class.
|
||||
* </p>
|
||||
*
|
||||
* @param id the identifier of the {@link VideoStream}, which must not be null
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setId(@Nonnull final String id) {
|
||||
this.id = id;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the content of the {@link VideoStream}.
|
||||
*
|
||||
* <p>
|
||||
* It must not be null, and should be non empty.
|
||||
* </p>
|
||||
*
|
||||
* @param content the content of the {@link VideoStream}
|
||||
* @param isUrl whether the content is a URL
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setContent(@Nonnull final String content,
|
||||
final boolean isUrl) {
|
||||
this.content = content;
|
||||
this.isUrl = isUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the {@link MediaFormat} used by the {@link VideoStream}.
|
||||
*
|
||||
* <p>
|
||||
* It should be one of the video {@link MediaFormat}s ({@link MediaFormat#MPEG_4 MPEG_4},
|
||||
* {@link MediaFormat#v3GPP v3GPP}, or {@link MediaFormat#WEBM WEBM}) but can be {@code
|
||||
* null} if the media format could not be determined.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The default value is {@code null}.
|
||||
* </p>
|
||||
*
|
||||
* @param mediaFormat the {@link MediaFormat} of the {@link VideoStream}, which can be null
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setMediaFormat(@Nullable final MediaFormat mediaFormat) {
|
||||
this.mediaFormat = mediaFormat;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the {@link DeliveryMethod} of the {@link VideoStream}.
|
||||
*
|
||||
* <p>
|
||||
* It must not be null.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The default delivery method is {@link DeliveryMethod#PROGRESSIVE_HTTP}.
|
||||
* </p>
|
||||
*
|
||||
* @param deliveryMethod the {@link DeliveryMethod} of the {@link VideoStream}, which must
|
||||
* not be null
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setDeliveryMethod(@Nonnull final DeliveryMethod deliveryMethod) {
|
||||
this.deliveryMethod = deliveryMethod;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the URL of the manifest this stream comes from (if applicable, otherwise null).
|
||||
*
|
||||
* @param manifestUrl the URL of the manifest this stream comes from or {@code null}
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setManifestUrl(@Nullable final String manifestUrl) {
|
||||
this.manifestUrl = manifestUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set whether the {@link VideoStream} is video-only.
|
||||
*
|
||||
* <p>
|
||||
* This property must be set before building the {@link VideoStream}.
|
||||
* </p>
|
||||
*
|
||||
* @param isVideoOnly whether the {@link VideoStream} is video-only
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setIsVideoOnly(final boolean isVideoOnly) {
|
||||
this.isVideoOnly = isVideoOnly;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the resolution of the {@link VideoStream}.
|
||||
*
|
||||
* <p>
|
||||
* This resolution can be used by clients to know the quality of the video stream.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* If you are not able to know the resolution, you should use {@link #RESOLUTION_UNKNOWN}
|
||||
* as the resolution of the video stream.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* It must be set before building the builder and not null.
|
||||
* </p>
|
||||
*
|
||||
* @param resolution the resolution of the {@link VideoStream}
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setResolution(@Nonnull final String resolution) {
|
||||
this.resolution = resolution;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the {@link ItagItem} corresponding to the {@link VideoStream}.
|
||||
*
|
||||
* <p>
|
||||
* {@link ItagItem}s are YouTube specific objects, so they are only known for this service
|
||||
* and can be null.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The default value is {@code null}.
|
||||
* </p>
|
||||
*
|
||||
* @param itagItem the {@link ItagItem} of the {@link VideoStream}, which can be null
|
||||
* @return this {@link Builder} instance
|
||||
*/
|
||||
public Builder setItagItem(@Nullable final ItagItem itagItem) {
|
||||
this.itagItem = itagItem;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a {@link VideoStream} using the builder's current values.
|
||||
*
|
||||
* <p>
|
||||
* The identifier, the content (and so the {@code isUrl} boolean), the {@code isVideoOnly}
|
||||
* and the {@code resolution} properties must have been set.
|
||||
* </p>
|
||||
*
|
||||
* @return a new {@link VideoStream} using the builder's current values
|
||||
* @throws IllegalStateException if {@code id}, {@code content} (and so {@code isUrl}),
|
||||
* {@code deliveryMethod}, {@code isVideoOnly} or {@code resolution} have been not set, or
|
||||
* have been set as {@code null}
|
||||
*/
|
||||
@Nonnull
|
||||
public VideoStream build() {
|
||||
if (id == null) {
|
||||
throw new IllegalStateException(
|
||||
"The identifier of the video stream has been not set or is null. If you "
|
||||
+ "are not able to get an identifier, use the static constant "
|
||||
+ "ID_UNKNOWN of the Stream class.");
|
||||
}
|
||||
|
||||
if (content == null) {
|
||||
throw new IllegalStateException("The content of the video stream has been not set "
|
||||
+ "or is null. Please specify a non-null one with setContent.");
|
||||
}
|
||||
|
||||
if (deliveryMethod == null) {
|
||||
throw new IllegalStateException(
|
||||
"The delivery method of the video stream has been set as null, which is "
|
||||
+ "not allowed. Pass a valid one instead with setDeliveryMethod.");
|
||||
}
|
||||
|
||||
if (isVideoOnly == null) {
|
||||
throw new IllegalStateException("The video stream has been not set as a "
|
||||
+ "video-only stream or as a video stream with embedded audio. Please "
|
||||
+ "specify this information with setIsVideoOnly.");
|
||||
}
|
||||
|
||||
if (resolution == null) {
|
||||
throw new IllegalStateException(
|
||||
"The resolution of the video stream has been not set. Please specify it "
|
||||
+ "with setResolution (use an empty string if you are not able to "
|
||||
+ "get it).");
|
||||
}
|
||||
|
||||
return new VideoStream(id, content, isUrl, mediaFormat, deliveryMethod, resolution,
|
||||
isVideoOnly, manifestUrl, itagItem);
|
||||
}
|
||||
}
|
||||
|
||||
public VideoStream(final String url,
|
||||
final MediaFormat format,
|
||||
final String resolution,
|
||||
final boolean isVideoOnly) {
|
||||
this(url, null, format, resolution, isVideoOnly);
|
||||
}
|
||||
|
||||
public VideoStream(final String url, final boolean isVideoOnly, final ItagItem itag) {
|
||||
this(url, itag.getMediaFormat(), itag.resolutionString, isVideoOnly);
|
||||
this.itag = itag.id;
|
||||
this.bitrate = itag.getBitrate();
|
||||
this.initStart = itag.getInitStart();
|
||||
this.initEnd = itag.getInitEnd();
|
||||
this.indexStart = itag.getIndexStart();
|
||||
this.indexEnd = itag.getIndexEnd();
|
||||
this.codec = itag.getCodec();
|
||||
this.height = itag.getHeight();
|
||||
this.width = itag.getWidth();
|
||||
this.quality = itag.getQuality();
|
||||
this.fps = itag.fps;
|
||||
}
|
||||
|
||||
public VideoStream(final String url,
|
||||
final String torrentUrl,
|
||||
final MediaFormat format,
|
||||
final String resolution) {
|
||||
this(url, torrentUrl, format, resolution, false);
|
||||
}
|
||||
|
||||
public VideoStream(final String url,
|
||||
final String torrentUrl,
|
||||
final MediaFormat format,
|
||||
final String resolution,
|
||||
final boolean isVideoOnly) {
|
||||
super(url, torrentUrl, format);
|
||||
/**
|
||||
* Create a new video stream.
|
||||
*
|
||||
* @param id the identifier which uniquely identifies the stream, e.g. for YouTube
|
||||
* this would be the itag
|
||||
* @param content the content or the URL of the stream, depending on whether isUrl is
|
||||
* true
|
||||
* @param isUrl whether content is the URL or the actual content of e.g. a DASH
|
||||
* manifest
|
||||
* @param format the {@link MediaFormat} used by the stream, which can be null
|
||||
* @param deliveryMethod the {@link DeliveryMethod} of the stream
|
||||
* @param resolution the resolution of the stream
|
||||
* @param isVideoOnly whether the stream is video-only
|
||||
* @param itagItem the {@link ItagItem} corresponding to the stream, which cannot be null
|
||||
* @param manifestUrl the URL of the manifest this stream comes from (if applicable,
|
||||
* otherwise null)
|
||||
*/
|
||||
@SuppressWarnings("checkstyle:ParameterNumber")
|
||||
private VideoStream(@Nonnull final String id,
|
||||
@Nonnull final String content,
|
||||
final boolean isUrl,
|
||||
@Nullable final MediaFormat format,
|
||||
@Nonnull final DeliveryMethod deliveryMethod,
|
||||
@Nonnull final String resolution,
|
||||
final boolean isVideoOnly,
|
||||
@Nullable final String manifestUrl,
|
||||
@Nullable final ItagItem itagItem) {
|
||||
super(id, content, isUrl, format, deliveryMethod, manifestUrl);
|
||||
if (itagItem != null) {
|
||||
this.itagItem = itagItem;
|
||||
this.itag = itagItem.id;
|
||||
this.bitrate = itagItem.getBitrate();
|
||||
this.initStart = itagItem.getInitStart();
|
||||
this.initEnd = itagItem.getInitEnd();
|
||||
this.indexStart = itagItem.getIndexStart();
|
||||
this.indexEnd = itagItem.getIndexEnd();
|
||||
this.codec = itagItem.getCodec();
|
||||
this.height = itagItem.getHeight();
|
||||
this.width = itagItem.getWidth();
|
||||
this.quality = itagItem.getQuality();
|
||||
this.fps = itagItem.getFps();
|
||||
}
|
||||
this.resolution = resolution;
|
||||
this.isVideoOnly = isVideoOnly;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public boolean equalStats(final Stream cmp) {
|
||||
return super.equalStats(cmp) && cmp instanceof VideoStream
|
||||
return super.equalStats(cmp)
|
||||
&& cmp instanceof VideoStream
|
||||
&& resolution.equals(((VideoStream) cmp).resolution)
|
||||
&& isVideoOnly == ((VideoStream) cmp).isVideoOnly;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the video resolution
|
||||
* Get the video resolution.
|
||||
*
|
||||
* @return the video resolution
|
||||
* <p>
|
||||
* It can be unknown for some streams, like for HLS master playlists. In this case,
|
||||
* {@link #RESOLUTION_UNKNOWN} is returned by this method.
|
||||
* </p>
|
||||
*
|
||||
* @return the video resolution or {@link #RESOLUTION_UNKNOWN}
|
||||
*/
|
||||
@Nonnull
|
||||
public String getResolution() {
|
||||
return resolution;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the video is video only.
|
||||
* <p>
|
||||
* Video only streams have no audio
|
||||
* Return whether the stream is video-only.
|
||||
*
|
||||
* @return {@code true} if this stream is vid
|
||||
* <p>
|
||||
* Video-only streams have no audio.
|
||||
* </p>
|
||||
*
|
||||
* @return {@code true} if this stream is video-only, {@code false} otherwise
|
||||
*/
|
||||
public boolean isVideoOnly() {
|
||||
return isVideoOnly;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the itag identifier of the stream.
|
||||
*
|
||||
* <p>
|
||||
* Always equals to {@link #ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE} for other streams than the
|
||||
* ones of the YouTube service.
|
||||
* </p>
|
||||
*
|
||||
* @return the number of the {@link ItagItem} passed in the constructor of the video stream.
|
||||
*/
|
||||
public int getItag() {
|
||||
return itag;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the bitrate of the stream.
|
||||
*
|
||||
* @return the bitrate set from the {@link ItagItem} passed in the constructor of the stream.
|
||||
*/
|
||||
public int getBitrate() {
|
||||
return bitrate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the initialization start of the stream.
|
||||
*
|
||||
* @return the initialization start value set from the {@link ItagItem} passed in the
|
||||
* constructor of the
|
||||
* stream.
|
||||
*/
|
||||
public int getInitStart() {
|
||||
return initStart;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the initialization end of the stream.
|
||||
*
|
||||
* @return the initialization end value set from the {@link ItagItem} passed in the constructor
|
||||
* of the stream.
|
||||
*/
|
||||
public int getInitEnd() {
|
||||
return initEnd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the index start of the stream.
|
||||
*
|
||||
* @return the index start value set from the {@link ItagItem} passed in the constructor of the
|
||||
* stream.
|
||||
*/
|
||||
public int getIndexStart() {
|
||||
return indexStart;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the index end of the stream.
|
||||
*
|
||||
* @return the index end value set from the {@link ItagItem} passed in the constructor of the
|
||||
* stream.
|
||||
*/
|
||||
public int getIndexEnd() {
|
||||
return indexEnd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the width of the video stream.
|
||||
*
|
||||
* @return the width set from the {@link ItagItem} passed in the constructor of the
|
||||
* stream.
|
||||
*/
|
||||
public int getWidth() {
|
||||
return width;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the height of the video stream.
|
||||
*
|
||||
* @return the height set from the {@link ItagItem} passed in the constructor of the
|
||||
* stream.
|
||||
*/
|
||||
public int getHeight() {
|
||||
return height;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the frames per second of the video stream.
|
||||
*
|
||||
* @return the frames per second set from the {@link ItagItem} passed in the constructor of the
|
||||
* stream.
|
||||
*/
|
||||
public int getFps() {
|
||||
return fps;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the quality of the stream.
|
||||
*
|
||||
* @return the quality label set from the {@link ItagItem} passed in the constructor of the
|
||||
* stream.
|
||||
*/
|
||||
public String getQuality() {
|
||||
return quality;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the codec of the stream.
|
||||
*
|
||||
* @return the codec set from the {@link ItagItem} passed in the constructor of the stream.
|
||||
*/
|
||||
public String getCodec() {
|
||||
return codec;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
@Nullable
|
||||
public ItagItem getItagItem() {
|
||||
return itagItem;
|
||||
}
|
||||
}
|
||||
|
@ -1,225 +0,0 @@
|
||||
package org.schabi.newpipe.extractor.utils;
|
||||
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfo;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 02.02.16.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
||||
* DashMpdParser.java is part of NewPipe.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
public final class DashMpdParser {
|
||||
|
||||
private DashMpdParser() {
|
||||
}
|
||||
|
||||
public static class DashMpdParsingException extends ParsingException {
|
||||
DashMpdParsingException(final String message, final Exception e) {
|
||||
super(message, e);
|
||||
}
|
||||
}
|
||||
|
||||
public static class ParserResult {
|
||||
private final List<VideoStream> videoStreams;
|
||||
private final List<AudioStream> audioStreams;
|
||||
private final List<VideoStream> videoOnlyStreams;
|
||||
|
||||
private final List<VideoStream> segmentedVideoStreams;
|
||||
private final List<AudioStream> segmentedAudioStreams;
|
||||
private final List<VideoStream> segmentedVideoOnlyStreams;
|
||||
|
||||
|
||||
public ParserResult(final List<VideoStream> videoStreams,
|
||||
final List<AudioStream> audioStreams,
|
||||
final List<VideoStream> videoOnlyStreams,
|
||||
final List<VideoStream> segmentedVideoStreams,
|
||||
final List<AudioStream> segmentedAudioStreams,
|
||||
final List<VideoStream> segmentedVideoOnlyStreams) {
|
||||
this.videoStreams = videoStreams;
|
||||
this.audioStreams = audioStreams;
|
||||
this.videoOnlyStreams = videoOnlyStreams;
|
||||
this.segmentedVideoStreams = segmentedVideoStreams;
|
||||
this.segmentedAudioStreams = segmentedAudioStreams;
|
||||
this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams;
|
||||
}
|
||||
|
||||
public List<VideoStream> getVideoStreams() {
|
||||
return videoStreams;
|
||||
}
|
||||
|
||||
public List<AudioStream> getAudioStreams() {
|
||||
return audioStreams;
|
||||
}
|
||||
|
||||
public List<VideoStream> getVideoOnlyStreams() {
|
||||
return videoOnlyStreams;
|
||||
}
|
||||
|
||||
public List<VideoStream> getSegmentedVideoStreams() {
|
||||
return segmentedVideoStreams;
|
||||
}
|
||||
|
||||
public List<AudioStream> getSegmentedAudioStreams() {
|
||||
return segmentedAudioStreams;
|
||||
}
|
||||
|
||||
public List<VideoStream> getSegmentedVideoOnlyStreams() {
|
||||
return segmentedVideoOnlyStreams;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Will try to download (using {@link StreamInfo#getDashMpdUrl()}) and parse the dash manifest,
|
||||
* then it will search for any stream that the ItagItem has (by the id).
|
||||
* <p>
|
||||
* It has video, video only and audio streams and will only add to the list if it don't
|
||||
* find a similar stream in the respective lists (calling {@link Stream#equalStats}).
|
||||
* <p>
|
||||
* Info about dash MPD can be found
|
||||
* <a href="https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html">here</a>.
|
||||
*
|
||||
* @param streamInfo where the parsed streams will be added
|
||||
*/
|
||||
public static ParserResult getStreams(final StreamInfo streamInfo)
|
||||
throws DashMpdParsingException, ReCaptchaException {
|
||||
final String dashDoc;
|
||||
final Downloader downloader = NewPipe.getDownloader();
|
||||
try {
|
||||
dashDoc = downloader.get(streamInfo.getDashMpdUrl()).responseBody();
|
||||
} catch (final IOException ioe) {
|
||||
throw new DashMpdParsingException(
|
||||
"Could not get dash mpd: " + streamInfo.getDashMpdUrl(), ioe);
|
||||
}
|
||||
|
||||
try {
|
||||
final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
||||
final DocumentBuilder builder = factory.newDocumentBuilder();
|
||||
final InputStream stream = new ByteArrayInputStream(dashDoc.getBytes());
|
||||
|
||||
final Document doc = builder.parse(stream);
|
||||
final NodeList representationList = doc.getElementsByTagName("Representation");
|
||||
|
||||
final List<VideoStream> videoStreams = new ArrayList<>();
|
||||
final List<AudioStream> audioStreams = new ArrayList<>();
|
||||
final List<VideoStream> videoOnlyStreams = new ArrayList<>();
|
||||
|
||||
final List<VideoStream> segmentedVideoStreams = new ArrayList<>();
|
||||
final List<AudioStream> segmentedAudioStreams = new ArrayList<>();
|
||||
final List<VideoStream> segmentedVideoOnlyStreams = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < representationList.getLength(); i++) {
|
||||
final Element representation = (Element) representationList.item(i);
|
||||
try {
|
||||
final String mimeType
|
||||
= ((Element) representation.getParentNode()).getAttribute("mimeType");
|
||||
final String id = representation.getAttribute("id");
|
||||
final String url = representation
|
||||
.getElementsByTagName("BaseURL").item(0).getTextContent();
|
||||
final ItagItem itag = ItagItem.getItag(Integer.parseInt(id));
|
||||
final Node segmentationList
|
||||
= representation.getElementsByTagName("SegmentList").item(0);
|
||||
|
||||
// If SegmentList is not null this means that BaseUrl is not representing the
|
||||
// url to the stream. Instead we need to add the "media=" value from the
|
||||
// <SegementURL/> tags inside the <SegmentList/> tag in order to get a full
|
||||
// working url. However each of these is just pointing to a part of the video,
|
||||
// so we can not return a URL with a working stream here. Instead of putting
|
||||
// those streams into the list of regular stream urls we put them in a for
|
||||
// example "segmentedVideoStreams" list.
|
||||
|
||||
final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
|
||||
|
||||
if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
|
||||
if (segmentationList == null) {
|
||||
final AudioStream audioStream
|
||||
= new AudioStream(url, mediaFormat, itag.avgBitrate);
|
||||
if (!Stream.containSimilarStream(audioStream,
|
||||
streamInfo.getAudioStreams())) {
|
||||
audioStreams.add(audioStream);
|
||||
}
|
||||
} else {
|
||||
segmentedAudioStreams.add(
|
||||
new AudioStream(id, mediaFormat, itag.avgBitrate));
|
||||
}
|
||||
} else {
|
||||
final boolean isVideoOnly
|
||||
= itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
|
||||
|
||||
if (segmentationList == null) {
|
||||
final VideoStream videoStream = new VideoStream(url,
|
||||
mediaFormat,
|
||||
itag.resolutionString,
|
||||
isVideoOnly);
|
||||
|
||||
if (isVideoOnly) {
|
||||
if (!Stream.containSimilarStream(videoStream,
|
||||
streamInfo.getVideoOnlyStreams())) {
|
||||
videoOnlyStreams.add(videoStream);
|
||||
}
|
||||
} else if (!Stream.containSimilarStream(videoStream,
|
||||
streamInfo.getVideoStreams())) {
|
||||
videoStreams.add(videoStream);
|
||||
}
|
||||
} else {
|
||||
final VideoStream videoStream = new VideoStream(id,
|
||||
mediaFormat,
|
||||
itag.resolutionString,
|
||||
isVideoOnly);
|
||||
|
||||
if (isVideoOnly) {
|
||||
segmentedVideoOnlyStreams.add(videoStream);
|
||||
} else {
|
||||
segmentedVideoStreams.add(videoStream);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (final Exception ignored) {
|
||||
}
|
||||
}
|
||||
return new ParserResult(
|
||||
videoStreams,
|
||||
audioStreams,
|
||||
videoOnlyStreams,
|
||||
segmentedVideoStreams,
|
||||
segmentedAudioStreams,
|
||||
segmentedVideoOnlyStreams);
|
||||
} catch (final Exception e) {
|
||||
throw new DashMpdParsingException("Could not parse Dash mpd", e);
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,255 @@
|
||||
package org.schabi.newpipe.extractor.utils;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* A {@link Serializable serializable} cache class used by the extractor to cache manifests
|
||||
* generated with extractor's manifests generators.
|
||||
*
|
||||
* <p>
|
||||
* It relies internally on a {@link ConcurrentHashMap} to allow concurrent access to the cache.
|
||||
* </p>
|
||||
*
|
||||
* @param <K> the type of cache keys, which must be {@link Serializable serializable}
|
||||
* @param <V> the type of the second element of {@link Pair pairs} used as values of the cache,
|
||||
* which must be {@link Serializable serializable}
|
||||
*/
|
||||
public final class ManifestCreatorCache<K extends Serializable, V extends Serializable>
|
||||
implements Serializable {
|
||||
|
||||
/**
|
||||
* The default maximum size of a manifest cache.
|
||||
*/
|
||||
public static final int DEFAULT_MAXIMUM_SIZE = Integer.MAX_VALUE;
|
||||
|
||||
/**
|
||||
* The default clear factor of a manifest cache.
|
||||
*/
|
||||
public static final double DEFAULT_CLEAR_FACTOR = 0.75;
|
||||
|
||||
/**
|
||||
* The {@link ConcurrentHashMap} used internally as the cache of manifests.
|
||||
*/
|
||||
private final ConcurrentHashMap<K, Pair<Integer, V>> concurrentHashMap;
|
||||
|
||||
/**
|
||||
* The maximum size of the cache.
|
||||
*
|
||||
* <p>
|
||||
* The default value is {@link #DEFAULT_MAXIMUM_SIZE}.
|
||||
* </p>
|
||||
*/
|
||||
private int maximumSize = DEFAULT_MAXIMUM_SIZE;
|
||||
|
||||
/**
|
||||
* The clear factor of the cache, which is a double between {@code 0} and {@code 1} excluded.
|
||||
*
|
||||
* <p>
|
||||
* The default value is {@link #DEFAULT_CLEAR_FACTOR}.
|
||||
* </p>
|
||||
*/
|
||||
private double clearFactor = DEFAULT_CLEAR_FACTOR;
|
||||
|
||||
/**
|
||||
* Creates a new {@link ManifestCreatorCache}.
|
||||
*/
|
||||
public ManifestCreatorCache() {
|
||||
concurrentHashMap = new ConcurrentHashMap<>();
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests if the specified key is in the cache.
|
||||
*
|
||||
* @param key the key to test its presence in the cache
|
||||
* @return {@code true} if the key is in the cache, {@code false} otherwise.
|
||||
*/
|
||||
public boolean containsKey(final K key) {
|
||||
return concurrentHashMap.containsKey(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the value to which the specified key is mapped, or {@code null} if the cache
|
||||
* contains no mapping for the key.
|
||||
*
|
||||
* @param key the key to which getting its value
|
||||
* @return the value to which the specified key is mapped, or {@code null}
|
||||
*/
|
||||
@Nullable
|
||||
public Pair<Integer, V> get(final K key) {
|
||||
return concurrentHashMap.get(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new element to the cache.
|
||||
*
|
||||
* <p>
|
||||
* If the cache limit is reached, oldest elements will be cleared first using the load factor
|
||||
* and the maximum size.
|
||||
* </p>
|
||||
*
|
||||
* @param key the key to put
|
||||
* @param value the value to associate to the key
|
||||
*
|
||||
* @return the previous value associated with the key, or {@code null} if there was no mapping
|
||||
* for the key (note that a null return can also indicate that the cache previously associated
|
||||
* {@code null} with the key).
|
||||
*/
|
||||
@Nullable
|
||||
public V put(final K key, final V value) {
|
||||
if (!concurrentHashMap.containsKey(key) && concurrentHashMap.size() == maximumSize) {
|
||||
final int newCacheSize = (int) Math.round(maximumSize * clearFactor);
|
||||
keepNewestEntries(newCacheSize != 0 ? newCacheSize : 1);
|
||||
}
|
||||
|
||||
final Pair<Integer, V> returnValue = concurrentHashMap.put(key,
|
||||
new Pair<>(concurrentHashMap.size(), value));
|
||||
return returnValue == null ? null : returnValue.getSecond();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears the cached manifests.
|
||||
*
|
||||
* <p>
|
||||
* The cache will be empty after this method is called.
|
||||
* </p>
|
||||
*/
|
||||
public void clear() {
|
||||
concurrentHashMap.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the cache.
|
||||
*
|
||||
* <p>
|
||||
* The cache will be empty and the clear factor and the maximum size will be reset to their
|
||||
* default values.
|
||||
* </p>
|
||||
*
|
||||
* @see #clear()
|
||||
* @see #resetClearFactor()
|
||||
* @see #resetMaximumSize()
|
||||
*/
|
||||
public void reset() {
|
||||
clear();
|
||||
resetClearFactor();
|
||||
resetMaximumSize();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the number of cached manifests in the cache
|
||||
*/
|
||||
public int size() {
|
||||
return concurrentHashMap.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the maximum size of the cache
|
||||
*/
|
||||
public long getMaximumSize() {
|
||||
return maximumSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the maximum size of the cache.
|
||||
*
|
||||
* If the current cache size is more than the new maximum size, the percentage of one less the
|
||||
* clear factor of the maximum new size of manifests in the cache will be removed.
|
||||
*
|
||||
* @param maximumSize the new maximum size of the cache
|
||||
* @throws IllegalArgumentException if {@code maximumSize} is less than or equal to 0
|
||||
*/
|
||||
public void setMaximumSize(final int maximumSize) {
|
||||
if (maximumSize <= 0) {
|
||||
throw new IllegalArgumentException("Invalid maximum size");
|
||||
}
|
||||
|
||||
if (maximumSize < this.maximumSize && !concurrentHashMap.isEmpty()) {
|
||||
final int newCacheSize = (int) Math.round(maximumSize * clearFactor);
|
||||
keepNewestEntries(newCacheSize != 0 ? newCacheSize : 1);
|
||||
}
|
||||
|
||||
this.maximumSize = maximumSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the maximum size of the cache to its {@link #DEFAULT_MAXIMUM_SIZE default value}.
|
||||
*/
|
||||
public void resetMaximumSize() {
|
||||
this.maximumSize = DEFAULT_MAXIMUM_SIZE;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the current clear factor of the cache, used when the cache limit size is reached
|
||||
*/
|
||||
public double getClearFactor() {
|
||||
return clearFactor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the clear factor of the cache, used when the cache limit size is reached.
|
||||
*
|
||||
* <p>
|
||||
* The clear factor must be a double between {@code 0} excluded and {@code 1} excluded.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Note that it will be only used the next time the cache size limit is reached.
|
||||
* </p>
|
||||
*
|
||||
* @param clearFactor the new clear factor of the cache
|
||||
* @throws IllegalArgumentException if the clear factor passed a parameter is invalid
|
||||
*/
|
||||
public void setClearFactor(final double clearFactor) {
|
||||
if (clearFactor <= 0 || clearFactor >= 1) {
|
||||
throw new IllegalArgumentException("Invalid clear factor");
|
||||
}
|
||||
|
||||
this.clearFactor = clearFactor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the clear factor to its {@link #DEFAULT_CLEAR_FACTOR default value}.
|
||||
*/
|
||||
public void resetClearFactor() {
|
||||
this.clearFactor = DEFAULT_CLEAR_FACTOR;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String toString() {
|
||||
return "ManifestCreatorCache[clearFactor=" + clearFactor + ", maximumSize=" + maximumSize
|
||||
+ ", concurrentHashMap=" + concurrentHashMap + "]";
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps only the newest entries in a cache.
|
||||
*
|
||||
* <p>
|
||||
* This method will first collect the entries to remove by looping through the concurrent hash
|
||||
* map
|
||||
* </p>
|
||||
*
|
||||
* @param newLimit the new limit of the cache
|
||||
*/
|
||||
private void keepNewestEntries(final int newLimit) {
|
||||
final int difference = concurrentHashMap.size() - newLimit;
|
||||
final ArrayList<Map.Entry<K, Pair<Integer, V>>> entriesToRemove = new ArrayList<>();
|
||||
|
||||
concurrentHashMap.entrySet().forEach(entry -> {
|
||||
final Pair<Integer, V> value = entry.getValue();
|
||||
if (value.getFirst() < difference) {
|
||||
entriesToRemove.add(entry);
|
||||
} else {
|
||||
value.setFirst(value.getFirst() - difference);
|
||||
}
|
||||
});
|
||||
|
||||
entriesToRemove.forEach(entry -> concurrentHashMap.remove(entry.getKey(),
|
||||
entry.getValue()));
|
||||
}
|
||||
}
|
@ -15,6 +15,8 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
public class ExtractorAsserts {
|
||||
public static void assertEmptyErrors(String message, List<Throwable> errors) {
|
||||
if (!errors.isEmpty()) {
|
||||
@ -64,6 +66,14 @@ public class ExtractorAsserts {
|
||||
}
|
||||
}
|
||||
|
||||
public static void assertNotBlank(String stringToCheck) {
|
||||
assertNotBlank(stringToCheck, null);
|
||||
}
|
||||
|
||||
public static void assertNotBlank(String stringToCheck, @Nullable String message) {
|
||||
assertFalse(Utils.isBlank(stringToCheck), message);
|
||||
}
|
||||
|
||||
public static void assertGreater(final long expected, final long actual) {
|
||||
assertGreater(expected, actual, actual + " is not > " + expected);
|
||||
}
|
||||
|
@ -271,13 +271,20 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
|
||||
assertFalse(videoStreams.isEmpty());
|
||||
|
||||
for (final VideoStream stream : videoStreams) {
|
||||
assertIsSecureUrl(stream.getUrl());
|
||||
assertFalse(stream.getResolution().isEmpty());
|
||||
|
||||
final int formatId = stream.getFormatId();
|
||||
// see MediaFormat: video stream formats range from 0 to 0x100
|
||||
assertTrue(0 <= formatId && formatId < 0x100,
|
||||
"format id does not fit a video stream: " + formatId);
|
||||
if (stream.isUrl()) {
|
||||
assertIsSecureUrl(stream.getContent());
|
||||
}
|
||||
final StreamType streamType = extractor().getStreamType();
|
||||
// On some video streams, the resolution can be empty and the format be unknown,
|
||||
// especially on livestreams (like streams with HLS master playlists)
|
||||
if (streamType != StreamType.LIVE_STREAM
|
||||
&& streamType != StreamType.AUDIO_LIVE_STREAM) {
|
||||
assertFalse(stream.getResolution().isEmpty());
|
||||
final int formatId = stream.getFormatId();
|
||||
// see MediaFormat: video stream formats range from 0 to 0x100
|
||||
assertTrue(0 <= formatId && formatId < 0x100,
|
||||
"Format id does not fit a video stream: " + formatId);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
assertTrue(videoStreams.isEmpty());
|
||||
@ -294,12 +301,17 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
|
||||
assertFalse(audioStreams.isEmpty());
|
||||
|
||||
for (final AudioStream stream : audioStreams) {
|
||||
assertIsSecureUrl(stream.getUrl());
|
||||
if (stream.isUrl()) {
|
||||
assertIsSecureUrl(stream.getContent());
|
||||
}
|
||||
|
||||
final int formatId = stream.getFormatId();
|
||||
// see MediaFormat: video stream formats range from 0x100 to 0x1000
|
||||
assertTrue(0x100 <= formatId && formatId < 0x1000,
|
||||
"format id does not fit an audio stream: " + formatId);
|
||||
// The media format can be unknown on some audio streams
|
||||
if (stream.getFormat() != null) {
|
||||
final int formatId = stream.getFormat().id;
|
||||
// see MediaFormat: audio stream formats range from 0x100 to 0x1000
|
||||
assertTrue(0x100 <= formatId && formatId < 0x1000,
|
||||
"Format id does not fit an audio stream: " + formatId);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
assertTrue(audioStreams.isEmpty());
|
||||
@ -316,12 +328,14 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
|
||||
assertFalse(subtitles.isEmpty());
|
||||
|
||||
for (final SubtitlesStream stream : subtitles) {
|
||||
assertIsSecureUrl(stream.getUrl());
|
||||
if (stream.isUrl()) {
|
||||
assertIsSecureUrl(stream.getContent());
|
||||
}
|
||||
|
||||
final int formatId = stream.getFormatId();
|
||||
// see MediaFormat: video stream formats range from 0x1000 to 0x10000
|
||||
assertTrue(0x1000 <= formatId && formatId < 0x10000,
|
||||
"format id does not fit a subtitles stream: " + formatId);
|
||||
"Format id does not fit a subtitles stream: " + formatId);
|
||||
}
|
||||
} else {
|
||||
assertTrue(subtitles.isEmpty());
|
||||
@ -344,7 +358,8 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
|
||||
assertTrue(dashMpdUrl.isEmpty());
|
||||
} else {
|
||||
assertIsSecureUrl(dashMpdUrl);
|
||||
ExtractorAsserts.assertContains(expectedDashMpdUrlContains(), extractor().getDashMpdUrl());
|
||||
ExtractorAsserts.assertContains(expectedDashMpdUrlContains(),
|
||||
extractor().getDashMpdUrl());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.SoundCloudGoPlusContentException;
|
||||
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
|
||||
@ -21,7 +22,7 @@ import java.util.List;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
|
||||
|
||||
public class SoundcloudStreamExtractorTest {
|
||||
@ -187,18 +188,27 @@ public class SoundcloudStreamExtractorTest {
|
||||
super.testAudioStreams();
|
||||
final List<AudioStream> audioStreams = extractor.getAudioStreams();
|
||||
assertEquals(2, audioStreams.size());
|
||||
for (final AudioStream audioStream : audioStreams) {
|
||||
final String mediaUrl = audioStream.getUrl();
|
||||
audioStreams.forEach(audioStream -> {
|
||||
final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod();
|
||||
final String mediaUrl = audioStream.getContent();
|
||||
if (audioStream.getFormat() == MediaFormat.OPUS) {
|
||||
// assert that it's an OPUS 64 kbps media URL with a single range which comes from an HLS SoundCloud CDN
|
||||
// Assert that it's an OPUS 64 kbps media URL with a single range which comes
|
||||
// from an HLS SoundCloud CDN
|
||||
ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl);
|
||||
ExtractorAsserts.assertContains(".64.opus", mediaUrl);
|
||||
assertSame(DeliveryMethod.HLS, deliveryMethod,
|
||||
"Wrong delivery method for stream " + audioStream.getId() + ": "
|
||||
+ deliveryMethod);
|
||||
} else if (audioStream.getFormat() == MediaFormat.MP3) {
|
||||
// Assert that it's a MP3 128 kbps media URL which comes from a progressive
|
||||
// SoundCloud CDN
|
||||
ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3",
|
||||
mediaUrl);
|
||||
assertSame(DeliveryMethod.PROGRESSIVE_HTTP, deliveryMethod,
|
||||
"Wrong delivery method for stream " + audioStream.getId() + ": "
|
||||
+ deliveryMethod);
|
||||
}
|
||||
if (audioStream.getFormat() == MediaFormat.MP3) {
|
||||
// assert that it's a MP3 128 kbps media URL which comes from a progressive SoundCloud CDN
|
||||
ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3", mediaUrl);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,363 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.schabi.newpipe.downloader.DownloaderTestImpl;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.CreationException;
|
||||
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeOtfDashManifestCreator;
|
||||
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeProgressiveDashManifestCreator;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.InputSource;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import java.io.StringReader;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertAll;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreater;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotBlank;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ADAPTATION_SET;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.AUDIO_CHANNEL_CONFIGURATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.BASE_URL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.INITIALIZATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.MPD;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.PERIOD;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.REPRESENTATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ROLE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TEMPLATE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
|
||||
|
||||
/**
|
||||
* Test for YouTube DASH manifest creators.
|
||||
*
|
||||
* <p>
|
||||
* Tests the generation of OTF and progressive manifests.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* We cannot test the generation of DASH manifests for ended livestreams because these videos will
|
||||
* be re-encoded as normal videos later, so we can't use a specific video.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The generation of DASH manifests for OTF streams, which can be tested, uses a video licenced
|
||||
* under the Creative Commons Attribution licence (reuse allowed): {@code A New Era of Open?
|
||||
* COVID-19 and the Pursuit for Equitable Solutions} (<a href=
|
||||
* "https://www.youtube.com/watch?v=DJ8GQUNUXGM">https://www.youtube.com/watch?v=DJ8GQUNUXGM</a>)
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* We couldn't use mocks for these tests because the streaming URLs needs to fetched and the IP
|
||||
* address used to get these URLs is required (used as a param in the URLs; without it, video
|
||||
* servers return 403/Forbidden HTTP response code).
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* So the real downloader will be used everytime on this test class.
|
||||
* </p>
|
||||
*/
|
||||
class YoutubeDashManifestCreatorsTest {
|
||||
// Setting a higher number may let Google video servers return 403s
|
||||
private static final int MAX_STREAMS_TO_TEST_PER_METHOD = 3;
|
||||
private static final String url = "https://www.youtube.com/watch?v=DJ8GQUNUXGM";
|
||||
private static YoutubeStreamExtractor extractor;
|
||||
private static long videoLength;
|
||||
|
||||
@BeforeAll
|
||||
public static void setUp() throws Exception {
|
||||
YoutubeParsingHelper.resetClientVersionAndKey();
|
||||
YoutubeParsingHelper.setNumberGenerator(new Random(1));
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
|
||||
extractor = (YoutubeStreamExtractor) YouTube.getStreamExtractor(url);
|
||||
extractor.fetchPage();
|
||||
videoLength = extractor.getLength();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testOtfStreams() throws Exception {
|
||||
assertDashStreams(extractor.getVideoOnlyStreams());
|
||||
assertDashStreams(extractor.getAudioStreams());
|
||||
|
||||
// no video stream with audio uses the DASH delivery method (YouTube OTF stream type)
|
||||
assertEquals(0, assertFilterStreams(extractor.getVideoStreams(),
|
||||
DeliveryMethod.DASH).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testProgressiveStreams() throws Exception {
|
||||
assertProgressiveStreams(extractor.getVideoOnlyStreams());
|
||||
assertProgressiveStreams(extractor.getAudioStreams());
|
||||
|
||||
// we are not able to generate DASH manifests of video formats with audio
|
||||
assertThrows(CreationException.class,
|
||||
() -> assertProgressiveStreams(extractor.getVideoStreams()));
|
||||
}
|
||||
|
||||
private void assertDashStreams(final List<? extends Stream> streams) throws Exception {
|
||||
|
||||
for (final Stream stream : assertFilterStreams(streams, DeliveryMethod.DASH)) {
|
||||
//noinspection ConstantConditions
|
||||
final String manifest = YoutubeOtfDashManifestCreator.fromOtfStreamingUrl(
|
||||
stream.getContent(), stream.getItagItem(), videoLength);
|
||||
assertNotBlank(manifest);
|
||||
|
||||
assertManifestGenerated(
|
||||
manifest,
|
||||
stream.getItagItem(),
|
||||
document -> assertAll(
|
||||
() -> assertSegmentTemplateElement(document),
|
||||
() -> assertSegmentTimelineAndSElements(document)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private void assertProgressiveStreams(final List<? extends Stream> streams) throws Exception {
|
||||
|
||||
for (final Stream stream : assertFilterStreams(streams, DeliveryMethod.PROGRESSIVE_HTTP)) {
|
||||
//noinspection ConstantConditions
|
||||
final String manifest =
|
||||
YoutubeProgressiveDashManifestCreator.fromProgressiveStreamingUrl(
|
||||
stream.getContent(), stream.getItagItem(), videoLength);
|
||||
assertNotBlank(manifest);
|
||||
|
||||
assertManifestGenerated(
|
||||
manifest,
|
||||
stream.getItagItem(),
|
||||
document -> assertAll(
|
||||
() -> assertBaseUrlElement(document),
|
||||
() -> assertSegmentBaseElement(document, stream.getItagItem()),
|
||||
() -> assertInitializationElement(document, stream.getItagItem())
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private List<? extends Stream> assertFilterStreams(
|
||||
@Nonnull final List<? extends Stream> streams,
|
||||
final DeliveryMethod deliveryMethod) {
|
||||
|
||||
final List<? extends Stream> filteredStreams = streams.stream()
|
||||
.filter(stream -> stream.getDeliveryMethod() == deliveryMethod)
|
||||
.limit(MAX_STREAMS_TO_TEST_PER_METHOD)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertAll(filteredStreams.stream()
|
||||
.flatMap(stream -> java.util.stream.Stream.of(
|
||||
() -> assertNotBlank(stream.getContent()),
|
||||
() -> assertNotNull(stream.getItagItem())
|
||||
))
|
||||
);
|
||||
|
||||
return filteredStreams;
|
||||
}
|
||||
|
||||
private void assertManifestGenerated(final String dashManifest,
|
||||
final ItagItem itagItem,
|
||||
final Consumer<Document> additionalAsserts)
|
||||
throws Exception {
|
||||
|
||||
final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory
|
||||
.newInstance();
|
||||
final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
|
||||
final Document document = documentBuilder.parse(new InputSource(
|
||||
new StringReader(dashManifest)));
|
||||
|
||||
assertAll(
|
||||
() -> assertMpdElement(document),
|
||||
() -> assertPeriodElement(document),
|
||||
() -> assertAdaptationSetElement(document, itagItem),
|
||||
() -> assertRoleElement(document),
|
||||
() -> assertRepresentationElement(document, itagItem),
|
||||
() -> {
|
||||
if (itagItem.itagType.equals(ItagItem.ItagType.AUDIO)) {
|
||||
assertAudioChannelConfigurationElement(document, itagItem);
|
||||
}
|
||||
},
|
||||
() -> additionalAsserts.accept(document)
|
||||
);
|
||||
}
|
||||
|
||||
private void assertMpdElement(@Nonnull final Document document) {
|
||||
final Element element = (Element) document.getElementsByTagName(MPD).item(0);
|
||||
assertNotNull(element);
|
||||
assertNull(element.getParentNode().getNodeValue());
|
||||
|
||||
final String mediaPresentationDuration = element.getAttribute("mediaPresentationDuration");
|
||||
assertNotNull(mediaPresentationDuration);
|
||||
assertTrue(mediaPresentationDuration.startsWith("PT"));
|
||||
}
|
||||
|
||||
private void assertPeriodElement(@Nonnull final Document document) {
|
||||
assertGetElement(document, PERIOD, MPD);
|
||||
}
|
||||
|
||||
private void assertAdaptationSetElement(@Nonnull final Document document,
|
||||
@Nonnull final ItagItem itagItem) {
|
||||
final Element element = assertGetElement(document, ADAPTATION_SET, PERIOD);
|
||||
assertAttrEquals(itagItem.getMediaFormat().getMimeType(), element, "mimeType");
|
||||
}
|
||||
|
||||
private void assertRoleElement(@Nonnull final Document document) {
|
||||
assertGetElement(document, ROLE, ADAPTATION_SET);
|
||||
}
|
||||
|
||||
private void assertRepresentationElement(@Nonnull final Document document,
|
||||
@Nonnull final ItagItem itagItem) {
|
||||
final Element element = assertGetElement(document, REPRESENTATION, ADAPTATION_SET);
|
||||
|
||||
assertAttrEquals(itagItem.getBitrate(), element, "bandwidth");
|
||||
assertAttrEquals(itagItem.getCodec(), element, "codecs");
|
||||
|
||||
if (itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY
|
||||
|| itagItem.itagType == ItagItem.ItagType.VIDEO) {
|
||||
assertAttrEquals(itagItem.getFps(), element, "frameRate");
|
||||
assertAttrEquals(itagItem.getHeight(), element, "height");
|
||||
assertAttrEquals(itagItem.getWidth(), element, "width");
|
||||
}
|
||||
|
||||
assertAttrEquals(itagItem.id, element, "id");
|
||||
}
|
||||
|
||||
private void assertAudioChannelConfigurationElement(@Nonnull final Document document,
|
||||
@Nonnull final ItagItem itagItem) {
|
||||
final Element element = assertGetElement(document, AUDIO_CHANNEL_CONFIGURATION,
|
||||
REPRESENTATION);
|
||||
assertAttrEquals(itagItem.getAudioChannels(), element, "value");
|
||||
}
|
||||
|
||||
private void assertSegmentTemplateElement(@Nonnull final Document document) {
|
||||
final Element element = assertGetElement(document, SEGMENT_TEMPLATE, REPRESENTATION);
|
||||
|
||||
final String initializationValue = element.getAttribute("initialization");
|
||||
assertIsValidUrl(initializationValue);
|
||||
assertTrue(initializationValue.endsWith("&sq=0"));
|
||||
|
||||
final String mediaValue = element.getAttribute("media");
|
||||
assertIsValidUrl(mediaValue);
|
||||
assertTrue(mediaValue.endsWith("&sq=$Number$"));
|
||||
|
||||
assertEquals("1", element.getAttribute("startNumber"));
|
||||
}
|
||||
|
||||
private void assertSegmentTimelineAndSElements(@Nonnull final Document document) {
|
||||
final Element element = assertGetElement(document, SEGMENT_TIMELINE, SEGMENT_TEMPLATE);
|
||||
final NodeList childNodes = element.getChildNodes();
|
||||
assertGreater(0, childNodes.getLength());
|
||||
|
||||
assertAll(IntStream.range(0, childNodes.getLength())
|
||||
.mapToObj(childNodes::item)
|
||||
.map(Element.class::cast)
|
||||
.flatMap(sElement -> java.util.stream.Stream.of(
|
||||
() -> assertEquals("S", sElement.getTagName()),
|
||||
() -> assertGreater(0, Integer.parseInt(sElement.getAttribute("d"))),
|
||||
() -> {
|
||||
final String rValue = sElement.getAttribute("r");
|
||||
// A segment duration can or can't be repeated, so test the next segment
|
||||
// if there is no r attribute
|
||||
if (!isBlank(rValue)) {
|
||||
assertGreater(0, Integer.parseInt(rValue));
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private void assertBaseUrlElement(@Nonnull final Document document) {
|
||||
final Element element = assertGetElement(document, BASE_URL, REPRESENTATION);
|
||||
assertIsValidUrl(element.getTextContent());
|
||||
}
|
||||
|
||||
private void assertSegmentBaseElement(@Nonnull final Document document,
|
||||
@Nonnull final ItagItem itagItem) {
|
||||
final Element element = assertGetElement(document, SEGMENT_BASE, REPRESENTATION);
|
||||
assertRangeEquals(itagItem.getIndexStart(), itagItem.getIndexEnd(), element, "indexRange");
|
||||
}
|
||||
|
||||
private void assertInitializationElement(@Nonnull final Document document,
|
||||
@Nonnull final ItagItem itagItem) {
|
||||
final Element element = assertGetElement(document, INITIALIZATION, SEGMENT_BASE);
|
||||
assertRangeEquals(itagItem.getInitStart(), itagItem.getInitEnd(), element, "range");
|
||||
}
|
||||
|
||||
|
||||
private void assertAttrEquals(final int expected,
|
||||
@Nonnull final Element element,
|
||||
final String attribute) {
|
||||
|
||||
final int actual = Integer.parseInt(element.getAttribute(attribute));
|
||||
assertAll(
|
||||
() -> assertGreater(0, actual),
|
||||
() -> assertEquals(expected, actual)
|
||||
);
|
||||
}
|
||||
|
||||
private void assertAttrEquals(final String expected,
|
||||
@Nonnull final Element element,
|
||||
final String attribute) {
|
||||
final String actual = element.getAttribute(attribute);
|
||||
assertAll(
|
||||
() -> assertNotBlank(actual),
|
||||
() -> assertEquals(expected, actual)
|
||||
);
|
||||
}
|
||||
|
||||
private void assertRangeEquals(final int expectedStart,
|
||||
final int expectedEnd,
|
||||
@Nonnull final Element element,
|
||||
final String attribute) {
|
||||
final String range = element.getAttribute(attribute);
|
||||
assertNotBlank(range);
|
||||
final String[] rangeParts = range.split("-");
|
||||
assertEquals(2, rangeParts.length);
|
||||
|
||||
final int actualStart = Integer.parseInt(rangeParts[0]);
|
||||
final int actualEnd = Integer.parseInt(rangeParts[1]);
|
||||
|
||||
assertAll(
|
||||
() -> assertGreaterOrEqual(0, actualStart),
|
||||
() -> assertEquals(expectedStart, actualStart),
|
||||
() -> assertGreater(0, actualEnd),
|
||||
() -> assertEquals(expectedEnd, actualEnd)
|
||||
);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private Element assertGetElement(@Nonnull final Document document,
|
||||
final String tagName,
|
||||
final String expectedParentTagName) {
|
||||
|
||||
final Element element = (Element) document.getElementsByTagName(tagName).item(0);
|
||||
assertNotNull(element);
|
||||
assertTrue(element.getParentNode().isEqualNode(
|
||||
document.getElementsByTagName(expectedParentTagName).item(0)),
|
||||
"Element with tag name \"" + tagName + "\" does not have a parent node"
|
||||
+ " with tag name \"" + expectedParentTagName + "\"");
|
||||
return element;
|
||||
}
|
||||
}
|
@ -0,0 +1,74 @@
|
||||
package org.schabi.newpipe.extractor.utils;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class ManifestCreatorCacheTest {
|
||||
@Test
|
||||
void basicMaximumSizeAndResetTest() {
|
||||
final ManifestCreatorCache<String, String> cache = new ManifestCreatorCache<>();
|
||||
|
||||
// 30 elements set -> cache resized to 23 -> 5 new elements set to the cache -> 28
|
||||
cache.setMaximumSize(30);
|
||||
setCacheContent(cache);
|
||||
assertEquals(28, cache.size(),
|
||||
"Wrong cache size with default clear factor and 30 as the maximum size");
|
||||
cache.reset();
|
||||
|
||||
assertEquals(0, cache.size(),
|
||||
"The cache has been not cleared after a reset call (wrong cache size)");
|
||||
assertEquals(ManifestCreatorCache.DEFAULT_MAXIMUM_SIZE, cache.getMaximumSize(),
|
||||
"Wrong maximum size after cache reset");
|
||||
assertEquals(ManifestCreatorCache.DEFAULT_CLEAR_FACTOR, cache.getClearFactor(),
|
||||
"Wrong clear factor after cache reset");
|
||||
}
|
||||
|
||||
@Test
|
||||
void maximumSizeAndClearFactorSettersAndResettersTest() {
|
||||
final ManifestCreatorCache<String, String> cache = new ManifestCreatorCache<>();
|
||||
cache.setMaximumSize(20);
|
||||
cache.setClearFactor(0.5);
|
||||
|
||||
setCacheContent(cache);
|
||||
// 30 elements set -> cache resized to 10 -> 5 new elements set to the cache -> 15
|
||||
assertEquals(15, cache.size(),
|
||||
"Wrong cache size with 0.5 as the clear factor and 20 as the maximum size");
|
||||
|
||||
// Clear factor and maximum size getters tests
|
||||
assertEquals(0.5, cache.getClearFactor(),
|
||||
"Wrong clear factor gotten from clear factor getter");
|
||||
assertEquals(20, cache.getMaximumSize(),
|
||||
"Wrong maximum cache size gotten from maximum size getter");
|
||||
|
||||
// Resetters tests
|
||||
cache.resetMaximumSize();
|
||||
assertEquals(ManifestCreatorCache.DEFAULT_MAXIMUM_SIZE, cache.getMaximumSize(),
|
||||
"Wrong maximum cache size gotten from maximum size getter after maximum size "
|
||||
+ "resetter call");
|
||||
|
||||
cache.resetClearFactor();
|
||||
assertEquals(ManifestCreatorCache.DEFAULT_CLEAR_FACTOR, cache.getClearFactor(),
|
||||
"Wrong clear factor gotten from clear factor getter after clear factor resetter "
|
||||
+ "call");
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds sample strings to the provided manifest creator cache, in order to test clear factor and
|
||||
* maximum size.
|
||||
* @param cache the cache to fill with some data
|
||||
*/
|
||||
private static void setCacheContent(final ManifestCreatorCache<String, String> cache) {
|
||||
int i = 0;
|
||||
while (i < 26) {
|
||||
cache.put(String.valueOf((char) ('a' + i)), "V");
|
||||
++i;
|
||||
}
|
||||
|
||||
i = 0;
|
||||
while (i < 9) {
|
||||
cache.put("a" + (char) ('a' + i), "V");
|
||||
++i;
|
||||
}
|
||||
}
|
||||
}
|
@ -3,13 +3,14 @@ package org.schabi.newpipe.extractor.utils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.util.Arrays;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
public class UtilsTest {
|
||||
class UtilsTest {
|
||||
@Test
|
||||
public void testMixedNumberWordToLong() throws ParsingException {
|
||||
void testMixedNumberWordToLong() throws ParsingException {
|
||||
assertEquals(10, Utils.mixedNumberWordToLong("10"));
|
||||
assertEquals(10.5e3, Utils.mixedNumberWordToLong("10.5K"), 0.0);
|
||||
assertEquals(10.5e6, Utils.mixedNumberWordToLong("10.5M"), 0.0);
|
||||
@ -18,13 +19,13 @@ public class UtilsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testJoin() {
|
||||
void testJoin() {
|
||||
assertEquals("some,random,stuff", Utils.join(",", Arrays.asList("some", "random", "stuff")));
|
||||
assertEquals("some,random,not-null,stuff", Utils.nonEmptyAndNullJoin(",", new String[]{"some", "null", "random", "", "not-null", null, "stuff"}));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetBaseUrl() throws ParsingException {
|
||||
void testGetBaseUrl() throws ParsingException {
|
||||
assertEquals("https://www.youtube.com", Utils.getBaseUrl("https://www.youtube.com/watch?v=Hu80uDzh8RY"));
|
||||
assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
|
||||
assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube:jZViOEv90dI"));
|
||||
@ -33,7 +34,7 @@ public class UtilsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFollowGoogleRedirect() {
|
||||
void testFollowGoogleRedirect() {
|
||||
assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY",
|
||||
Utils.followGoogleRedirectIfNeeded("https://www.google.it/url?sa=t&rct=j&q=&esrc=s&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DHu80uDzh8RY&source=video"));
|
||||
assertEquals("https://www.youtube.com/watch?v=0b6cFWG45kA",
|
||||
|
Loading…
Reference in New Issue
Block a user