kavin/NewPipeExtractor
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipeExtractor.git synced 2025-04-29 00:10:35 +05:30
Code Issues Projects Releases Wiki Activity

Apply code review and Streams rework

Browse Source
This commit is contained in:
Stypox 2022-05-28 00:26:53 +02:00 committed by TiA4f8R
parent d652e05874
commit b3c620f0d8
No known key found for this signature in database
GPG Key ID: E6D3E7F5949450DD
10 changed files with 248 additions and 351 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java
View File

@ -159,11 +159,11 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
return getStreams("audio", return getStreams("audio",
dto -> { dto -> {
final AudioStream.Builder builder = new AudioStream.Builder() final AudioStream.Builder builder = new AudioStream.Builder()
.setId(dto.getUrlValue().getString("tech", ID_UNKNOWN)) .setId(dto.urlValue.getString("tech", ID_UNKNOWN))
.setContent(dto.getUrlValue().getString(URL), true) .setContent(dto.urlValue.getString(URL), true)
.setAverageBitrate(UNKNOWN_BITRATE); .setAverageBitrate(UNKNOWN_BITRATE);
if ("hls".equals(dto.getUrlKey())) { if ("hls".equals(dto.urlKey)) {
// We don't know with the type string what media format will // We don't know with the type string what media format will
// have HLS streams. // have HLS streams.
// However, the tech string may contain some information // However, the tech string may contain some information
@ -172,7 +172,7 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
.build(); .build();
} }
return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.getUrlKey())) return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey))
.build(); .build();
}); });
} }
@ -181,15 +181,15 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException { public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
return getStreams("video", return getStreams("video",
dto -> { dto -> {
final JsonArray videoSize = dto.getStreamJsonObj().getArray("videoSize"); final JsonArray videoSize = dto.streamJsonObj.getArray("videoSize");
final VideoStream.Builder builder = new VideoStream.Builder() final VideoStream.Builder builder = new VideoStream.Builder()
.setId(dto.getUrlValue().getString("tech", ID_UNKNOWN)) .setId(dto.urlValue.getString("tech", ID_UNKNOWN))
.setContent(dto.getUrlValue().getString(URL), true) .setContent(dto.urlValue.getString(URL), true)
.setIsVideoOnly(false) .setIsVideoOnly(false)
.setResolution(videoSize.getInt(0) + "x" + videoSize.getInt(1)); .setResolution(videoSize.getInt(0) + "x" + videoSize.getInt(1));
if ("hls".equals(dto.getUrlKey())) { if ("hls".equals(dto.urlKey)) {
// We don't know with the type string what media format will // We don't know with the type string what media format will
// have HLS streams. // have HLS streams.
// However, the tech string may contain some information // However, the tech string may contain some information
@ -198,11 +198,32 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
.build(); .build();
} }
return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.getUrlKey())) return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey))
.build(); .build();
}); });
} }
/**
* This is just an internal class used in {@link #getStreams(String, Function)} to tie together
* the stream json object, its URL key and its URL value. An object of this class would be
* temporary and the three values it holds would be <b>convert</b>ed to a proper {@link Stream}
* object based on the wanted stream type.
*/
private static final class MediaCCCLiveStreamMapperDTO {
final JsonObject streamJsonObj;
final String urlKey;
final JsonObject urlValue;
MediaCCCLiveStreamMapperDTO(final JsonObject streamJsonObj,
final String urlKey,
final JsonObject urlValue) {
this.streamJsonObj = streamJsonObj;
this.urlKey = urlKey;
this.urlValue = urlValue;
}
}
private <T extends Stream> List<T> getStreams( private <T extends Stream> List<T> getStreams(
@Nonnull final String streamType, @Nonnull final String streamType,
@Nonnull final Function<MediaCCCLiveStreamMapperDTO, T> converter) { @Nonnull final Function<MediaCCCLiveStreamMapperDTO, T> converter) {
@ -220,7 +241,7 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
e.getKey(), e.getKey(),
(JsonObject) e.getValue()))) (JsonObject) e.getValue())))
// The DASH manifest will be extracted with getDashMpdUrl // The DASH manifest will be extracted with getDashMpdUrl
.filter(dto -> !"dash".equals(dto.getUrlKey())) .filter(dto -> !"dash".equals(dto.urlKey))
// Convert // Convert
.map(converter) .map(converter)
.collect(Collectors.toList()); .collect(Collectors.toList());

29
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamMapperDTO.java
View File

@ -1,29 +0,0 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import com.grack.nanojson.JsonObject;
final class MediaCCCLiveStreamMapperDTO {
private final JsonObject streamJsonObj;
private final String urlKey;
private final JsonObject urlValue;
MediaCCCLiveStreamMapperDTO(final JsonObject streamJsonObj,
final String urlKey,
final JsonObject urlValue) {
this.streamJsonObj = streamJsonObj;
this.urlKey = urlKey;
this.urlValue = urlValue;
}
JsonObject getStreamJsonObj() {
return streamJsonObj;
}
String getUrlKey() {
return urlKey;
}
JsonObject getUrlValue() {
return urlValue;
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java
View File

@ -409,11 +409,7 @@ public class ItagItem implements Serializable {
* @param sampleRate the sample rate of an audio itag * @param sampleRate the sample rate of an audio itag
*/ */
public void setSampleRate(final int sampleRate) { public void setSampleRate(final int sampleRate) {
if (sampleRate > 0) { this.sampleRate = sampleRate > 0 ? sampleRate : SAMPLE_RATE_UNKNOWN;
this.sampleRate = sampleRate;
} else {
this.sampleRate = SAMPLE_RATE_UNKNOWN;
}
} }
/** /**

274
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java
View File

@ -97,6 +97,25 @@ public final class YoutubeDashManifestCreatorsUtils {
public static final String SEGMENT_BASE = "SegmentBase"; public static final String SEGMENT_BASE = "SegmentBase";
public static final String INITIALIZATION = "Initialization"; public static final String INITIALIZATION = "Initialization";
/**
* Create an attribute with {@link Document#createAttribute(String)}, assign to it the provided
* name and value, then add it to the provided element using {@link
* Element#setAttributeNode(Attr)}.
*
* @param element element to which to add the created node
* @param doc document to use to create the attribute
* @param name name of the attribute
* @param value value of the attribute, will be set using {@link Attr#setValue(String)}
*/
public static void setAttribute(final Element element,
final Document doc,
final String name,
final String value) {
final Attr attr = doc.createAttribute(name);
attr.setValue(value);
element.setAttributeNode(attr);
}
/** /**
* Generate a {@link Document} with common manifest creator elements added to it. * Generate a {@link Document} with common manifest creator elements added to it.
* *
@ -123,17 +142,17 @@ public final class YoutubeDashManifestCreatorsUtils {
public static Document generateDocumentAndDoCommonElementsGeneration( public static Document generateDocumentAndDoCommonElementsGeneration(
@Nonnull final ItagItem itagItem, @Nonnull final ItagItem itagItem,
final long streamDuration) throws CreationException { final long streamDuration) throws CreationException {
final Document document = generateDocumentAndMpdElement(streamDuration); final Document doc = generateDocumentAndMpdElement(streamDuration);
generatePeriodElement(document); generatePeriodElement(doc);
generateAdaptationSetElement(document, itagItem); generateAdaptationSetElement(doc, itagItem);
generateRoleElement(document); generateRoleElement(doc);
generateRepresentationElement(document, itagItem); generateRepresentationElement(doc, itagItem);
if (itagItem.itagType == ItagItem.ItagType.AUDIO) { if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
generateAudioChannelConfigurationElement(document, itagItem); generateAudioChannelConfigurationElement(doc, itagItem);
} }
return document; return doc;
} }
/** /**
@ -161,46 +180,25 @@ public final class YoutubeDashManifestCreatorsUtils {
public static Document generateDocumentAndMpdElement(final long duration) public static Document generateDocumentAndMpdElement(final long duration)
throws CreationException { throws CreationException {
try { try {
final Document document = newDocument(); final Document doc = newDocument();
final Element mpdElement = document.createElement(MPD); final Element mpdElement = doc.createElement(MPD);
document.appendChild(mpdElement); doc.appendChild(mpdElement);
final Attr xmlnsXsiAttribute = document.createAttribute("xmlns:xsi"); setAttribute(mpdElement, doc, "xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
xmlnsXsiAttribute.setValue("http://www.w3.org/2001/XMLSchema-instance"); setAttribute(mpdElement, doc, "xmlns", "urn:mpeg:DASH:schema:MPD:2011");
mpdElement.setAttributeNode(xmlnsXsiAttribute); setAttribute(mpdElement, doc, "xsi:schemaLocation",
"urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd");
setAttribute(mpdElement, doc, "minBufferTime", "PT1.500S");
setAttribute(mpdElement, doc, "profiles", "urn:mpeg:dash:profile:full:2011");
setAttribute(mpdElement, doc, "type", "static");
setAttribute(mpdElement, doc, "mediaPresentationDuration",
String.format(Locale.ENGLISH, "PT%.3fS", duration / 1000.0));
final Attr xmlns = document.createAttribute("xmlns"); return doc;
xmlns.setValue("urn:mpeg:DASH:schema:MPD:2011");
mpdElement.setAttributeNode(xmlns);
final Attr xsiSchemaLocationAttribute = document.createAttribute("xsi:schemaLocation");
xsiSchemaLocationAttribute.setValue("urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd");
mpdElement.setAttributeNode(xsiSchemaLocationAttribute);
final Attr minBufferTimeAttribute = document.createAttribute("minBufferTime");
minBufferTimeAttribute.setValue("PT1.500S");
mpdElement.setAttributeNode(minBufferTimeAttribute);
final Attr profilesAttribute = document.createAttribute("profiles");
profilesAttribute.setValue("urn:mpeg:dash:profile:full:2011");
mpdElement.setAttributeNode(profilesAttribute);
final Attr typeAttribute = document.createAttribute("type");
typeAttribute.setValue("static");
mpdElement.setAttributeNode(typeAttribute);
final Attr mediaPresentationDurationAttribute = document.createAttribute(
"mediaPresentationDuration");
final String durationSeconds = String.format(Locale.ENGLISH, "%.3f",
duration / 1000.0);
mediaPresentationDurationAttribute.setValue("PT" + durationSeconds + "S");
mpdElement.setAttributeNode(mediaPresentationDurationAttribute);
return document;
} catch (final Exception e) { } catch (final Exception e) {
throw new CreationException( throw new CreationException(
"Could not generate the DASH manifest or append the MPD document to it", e); "Could not generate the DASH manifest or append the MPD doc to it", e);
} }
} }
@ -212,14 +210,13 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generateDocumentAndMpdElement(long)}. * {@link #generateDocumentAndMpdElement(long)}.
* </p> * </p>
* *
* @param document the {@link Document} on which the the {@code <Period>} element will be * @param doc the {@link Document} on which the the {@code <Period>} element will be appended
* appended
*/ */
public static void generatePeriodElement(@Nonnull final Document document) public static void generatePeriodElement(@Nonnull final Document doc)
throws CreationException { throws CreationException {
try { try {
final Element mpdElement = (Element) document.getElementsByTagName(MPD).item(0); final Element mpdElement = (Element) doc.getElementsByTagName(MPD).item(0);
final Element periodElement = document.createElement(PERIOD); final Element periodElement = doc.createElement(PERIOD);
mpdElement.appendChild(periodElement); mpdElement.appendChild(periodElement);
} catch (final DOMException e) { } catch (final DOMException e) {
throw CreationException.couldNotAddElement(PERIOD, e); throw CreationException.couldNotAddElement(PERIOD, e);
@ -235,21 +232,18 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generatePeriodElement(Document)}. * {@link #generatePeriodElement(Document)}.
* </p> * </p>
* *
* @param document the {@link Document} on which the {@code <Period>} element will be * @param doc the {@link Document} on which the {@code <Period>} element will be appended
* appended
* @param itagItem the {@link ItagItem} corresponding to the stream, which must not be null * @param itagItem the {@link ItagItem} corresponding to the stream, which must not be null
*/ */
public static void generateAdaptationSetElement(@Nonnull final Document document, public static void generateAdaptationSetElement(@Nonnull final Document doc,
@Nonnull final ItagItem itagItem) @Nonnull final ItagItem itagItem)
throws CreationException { throws CreationException {
try { try {
final Element periodElement = (Element) document.getElementsByTagName(PERIOD) final Element periodElement = (Element) doc.getElementsByTagName(PERIOD)
.item(0); .item(0);
final Element adaptationSetElement = document.createElement(ADAPTATION_SET); final Element adaptationSetElement = doc.createElement(ADAPTATION_SET);
final Attr idAttribute = document.createAttribute("id"); setAttribute(adaptationSetElement, doc, "id", "0");
idAttribute.setValue("0");
adaptationSetElement.setAttributeNode(idAttribute);
final MediaFormat mediaFormat = itagItem.getMediaFormat(); final MediaFormat mediaFormat = itagItem.getMediaFormat();
if (mediaFormat == null || isNullOrEmpty(mediaFormat.getMimeType())) { if (mediaFormat == null || isNullOrEmpty(mediaFormat.getMimeType())) {
@ -257,14 +251,8 @@ public final class YoutubeDashManifestCreatorsUtils {
"the MediaFormat or its mime type is null or empty"); "the MediaFormat or its mime type is null or empty");
} }
final Attr mimeTypeAttribute = document.createAttribute("mimeType"); setAttribute(adaptationSetElement, doc, "mimeType", mediaFormat.getMimeType());
mimeTypeAttribute.setValue(mediaFormat.getMimeType()); setAttribute(adaptationSetElement, doc, "subsegmentAlignment", "true");
adaptationSetElement.setAttributeNode(mimeTypeAttribute);
final Attr subsegmentAlignmentAttribute = document.createAttribute(
"subsegmentAlignment");
subsegmentAlignmentAttribute.setValue("true");
adaptationSetElement.setAttributeNode(subsegmentAlignmentAttribute);
periodElement.appendChild(adaptationSetElement); periodElement.appendChild(adaptationSetElement);
} catch (final DOMException e) { } catch (final DOMException e) {
@ -289,23 +277,17 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generateAdaptationSetElement(Document, ItagItem)}). * {@link #generateAdaptationSetElement(Document, ItagItem)}).
* </p> * </p>
* *
* @param document the {@link Document} on which the the {@code <Role>} element will be * @param doc the {@link Document} on which the the {@code <Role>} element will be appended
* appended
*/ */
public static void generateRoleElement(@Nonnull final Document document) public static void generateRoleElement(@Nonnull final Document doc)
throws CreationException { throws CreationException {
try { try {
final Element adaptationSetElement = (Element) document.getElementsByTagName( final Element adaptationSetElement = (Element) doc.getElementsByTagName(
ADAPTATION_SET).item(0); ADAPTATION_SET).item(0);
final Element roleElement = document.createElement(ROLE); final Element roleElement = doc.createElement(ROLE);
final Attr schemeIdUriAttribute = document.createAttribute("schemeIdUri"); setAttribute(roleElement, doc, "schemeIdUri", "urn:mpeg:DASH:role:2011");
schemeIdUriAttribute.setValue("urn:mpeg:DASH:role:2011"); setAttribute(roleElement, doc, "value", "main");
roleElement.setAttributeNode(schemeIdUriAttribute);
final Attr valueAttribute = document.createAttribute("value");
valueAttribute.setValue("main");
roleElement.setAttributeNode(valueAttribute);
adaptationSetElement.appendChild(roleElement); adaptationSetElement.appendChild(roleElement);
} catch (final DOMException e) { } catch (final DOMException e) {
@ -322,56 +304,43 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generateAdaptationSetElement(Document, ItagItem)}). * {@link #generateAdaptationSetElement(Document, ItagItem)}).
* </p> * </p>
* *
* @param document the {@link Document} on which the the {@code <SegmentTimeline>} element will * @param doc the {@link Document} on which the the {@code <SegmentTimeline>} element will be
* be appended * appended
* @param itagItem the {@link ItagItem} to use, which must not be null * @param itagItem the {@link ItagItem} to use, which must not be null
*/ */
public static void generateRepresentationElement(@Nonnull final Document document, public static void generateRepresentationElement(@Nonnull final Document doc,
@Nonnull final ItagItem itagItem) @Nonnull final ItagItem itagItem)
throws CreationException { throws CreationException {
try { try {
final Element adaptationSetElement = (Element) document.getElementsByTagName( final Element adaptationSetElement = (Element) doc.getElementsByTagName(
ADAPTATION_SET).item(0); ADAPTATION_SET).item(0);
final Element representationElement = document.createElement(REPRESENTATION); final Element representationElement = doc.createElement(REPRESENTATION);
final int id = itagItem.id; final int id = itagItem.id;
if (id <= 0) { if (id <= 0) {
throw CreationException.couldNotAddElement(REPRESENTATION, throw CreationException.couldNotAddElement(REPRESENTATION,
"the id of the ItagItem is <= 0"); "the id of the ItagItem is <= 0");
} }
final Attr idAttribute = document.createAttribute("id"); setAttribute(representationElement, doc, "id", String.valueOf(id));
idAttribute.setValue(String.valueOf(id));
representationElement.setAttributeNode(idAttribute);
final String codec = itagItem.getCodec(); final String codec = itagItem.getCodec();
if (isNullOrEmpty(codec)) { if (isNullOrEmpty(codec)) {
throw CreationException.couldNotAddElement(ADAPTATION_SET, throw CreationException.couldNotAddElement(ADAPTATION_SET,
"the codec value of the ItagItem is null or empty"); "the codec value of the ItagItem is null or empty");
} }
final Attr codecsAttribute = document.createAttribute("codecs"); setAttribute(representationElement, doc, "codecs", codec);
codecsAttribute.setValue(codec); setAttribute(representationElement, doc, "startWithSAP", "1");
representationElement.setAttributeNode(codecsAttribute); setAttribute(representationElement, doc, "maxPlayoutRate", "1");
final Attr startWithSAPAttribute = document.createAttribute("startWithSAP");
startWithSAPAttribute.setValue("1");
representationElement.setAttributeNode(startWithSAPAttribute);
final Attr maxPlayoutRateAttribute = document.createAttribute("maxPlayoutRate");
maxPlayoutRateAttribute.setValue("1");
representationElement.setAttributeNode(maxPlayoutRateAttribute);
final int bitrate = itagItem.getBitrate(); final int bitrate = itagItem.getBitrate();
if (bitrate <= 0) { if (bitrate <= 0) {
throw CreationException.couldNotAddElement(REPRESENTATION, throw CreationException.couldNotAddElement(REPRESENTATION,
"the bitrate of the ItagItem is <= 0"); "the bitrate of the ItagItem is <= 0");
} }
final Attr bandwidthAttribute = document.createAttribute("bandwidth"); setAttribute(representationElement, doc, "bandwidth", String.valueOf(bitrate));
bandwidthAttribute.setValue(String.valueOf(bitrate));
representationElement.setAttributeNode(bandwidthAttribute);
final ItagItem.ItagType itagType = itagItem.itagType; if (itagItem.itagType == ItagItem.ItagType.VIDEO
|| itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) {
if (itagType == ItagItem.ItagType.VIDEO || itagType == ItagItem.ItagType.VIDEO_ONLY) {
final int height = itagItem.getHeight(); final int height = itagItem.getHeight();
final int width = itagItem.getWidth(); final int width = itagItem.getWidth();
if (height <= 0 && width <= 0) { if (height <= 0 && width <= 0) {
@ -380,25 +349,19 @@ public final class YoutubeDashManifestCreatorsUtils {
} }
if (width > 0) { if (width > 0) {
final Attr widthAttribute = document.createAttribute("width"); setAttribute(representationElement, doc, "width", String.valueOf(width));
widthAttribute.setValue(String.valueOf(width));
representationElement.setAttributeNode(widthAttribute);
} }
setAttribute(representationElement, doc, "height",
final Attr heightAttribute = document.createAttribute("height"); String.valueOf(itagItem.getHeight()));
heightAttribute.setValue(String.valueOf(itagItem.getHeight()));
representationElement.setAttributeNode(heightAttribute);
final int fps = itagItem.getFps(); final int fps = itagItem.getFps();
if (fps > 0) { if (fps > 0) {
final Attr frameRateAttribute = document.createAttribute("frameRate"); setAttribute(representationElement, doc, "frameRate", String.valueOf(fps));
frameRateAttribute.setValue(String.valueOf(fps));
representationElement.setAttributeNode(frameRateAttribute);
} }
} }
if (itagType == ItagItem.ItagType.AUDIO && itagItem.getSampleRate() > 0) { if (itagItem.itagType == ItagItem.ItagType.AUDIO && itagItem.getSampleRate() > 0) {
final Attr audioSamplingRateAttribute = document.createAttribute( final Attr audioSamplingRateAttribute = doc.createAttribute(
"audioSamplingRate"); "audioSamplingRate");
audioSamplingRateAttribute.setValue(String.valueOf(itagItem.getSampleRate())); audioSamplingRateAttribute.setValue(String.valueOf(itagItem.getSampleRate()));
} }
@ -433,32 +396,28 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generateRepresentationElement(Document, ItagItem)}). * {@link #generateRepresentationElement(Document, ItagItem)}).
* </p> * </p>
* *
* @param document the {@link Document} on which the {@code <AudioChannelConfiguration>} * @param doc the {@link Document} on which the {@code <AudioChannelConfiguration>} element will
* element will be appended * be appended
* @param itagItem the {@link ItagItem} to use, which must not be null * @param itagItem the {@link ItagItem} to use, which must not be null
*/ */
public static void generateAudioChannelConfigurationElement( public static void generateAudioChannelConfigurationElement(
@Nonnull final Document document, @Nonnull final Document doc,
@Nonnull final ItagItem itagItem) throws CreationException { @Nonnull final ItagItem itagItem) throws CreationException {
try { try {
final Element representationElement = (Element) document.getElementsByTagName( final Element representationElement = (Element) doc.getElementsByTagName(
REPRESENTATION).item(0); REPRESENTATION).item(0);
final Element audioChannelConfigurationElement = document.createElement( final Element audioChannelConfigurationElement = doc.createElement(
AUDIO_CHANNEL_CONFIGURATION); AUDIO_CHANNEL_CONFIGURATION);
final Attr schemeIdUriAttribute = document.createAttribute("schemeIdUri"); setAttribute(audioChannelConfigurationElement, doc, "schemeIdUri",
schemeIdUriAttribute.setValue(
"urn:mpeg:dash:23003:3:audio_channel_configuration:2011"); "urn:mpeg:dash:23003:3:audio_channel_configuration:2011");
audioChannelConfigurationElement.setAttributeNode(schemeIdUriAttribute);
final Attr valueAttribute = document.createAttribute("value"); if (itagItem.getAudioChannels() <= 0) {
final int audioChannels = itagItem.getAudioChannels();
if (audioChannels <= 0) {
throw new CreationException("the number of audioChannels in the ItagItem is <= 0: " throw new CreationException("the number of audioChannels in the ItagItem is <= 0: "
+ audioChannels); + itagItem.getAudioChannels());
} }
valueAttribute.setValue(String.valueOf(itagItem.getAudioChannels())); setAttribute(audioChannelConfigurationElement, doc, "value",
audioChannelConfigurationElement.setAttributeNode(valueAttribute); String.valueOf(itagItem.getAudioChannels()));
representationElement.appendChild(audioChannelConfigurationElement); representationElement.appendChild(audioChannelConfigurationElement);
} catch (final DOMException e) { } catch (final DOMException e) {
@ -467,22 +426,22 @@ public final class YoutubeDashManifestCreatorsUtils {
} }
/** /**
* Convert a DASH manifest {@link Document document} to a string and cache it. * Convert a DASH manifest {@link Document doc} to a string and cache it.
* *
* @param originalBaseStreamingUrl the original base URL of the stream * @param originalBaseStreamingUrl the original base URL of the stream
* @param document the document to be converted * @param doc the doc to be converted
* @param manifestCreatorCache the {@link ManifestCreatorCache} on which store the string * @param manifestCreatorCache the {@link ManifestCreatorCache} on which store the string
* generated * generated
* @return the DASH manifest {@link Document document} converted to a string * @return the DASH manifest {@link Document doc} converted to a string
*/ */
public static String buildAndCacheResult( public static String buildAndCacheResult(
@Nonnull final String originalBaseStreamingUrl, @Nonnull final String originalBaseStreamingUrl,
@Nonnull final Document document, @Nonnull final Document doc,
@Nonnull final ManifestCreatorCache<String, String> manifestCreatorCache) @Nonnull final ManifestCreatorCache<String, String> manifestCreatorCache)
throws CreationException { throws CreationException {
try { try {
final String documentXml = documentToXml(document); final String documentXml = documentToXml(doc);
manifestCreatorCache.put(originalBaseStreamingUrl, documentXml); manifestCreatorCache.put(originalBaseStreamingUrl, documentXml);
return documentXml; return documentXml;
} catch (final Exception e) { } catch (final Exception e) {
@ -517,13 +476,13 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generateRepresentationElement(Document, ItagItem)}). * {@link #generateRepresentationElement(Document, ItagItem)}).
* </p> * </p>
* *
* @param document the {@link Document} on which the {@code <SegmentTemplate>} element will * @param doc the {@link Document} on which the {@code <SegmentTemplate>} element will
* be appended * be appended
* @param baseUrl the base URL of the OTF/post-live-DVR stream * @param baseUrl the base URL of the OTF/post-live-DVR stream
* @param deliveryType the stream {@link DeliveryType delivery type}, which must be either * @param deliveryType the stream {@link DeliveryType delivery type}, which must be either
* {@link DeliveryType#OTF OTF} or {@link DeliveryType#LIVE LIVE} * {@link DeliveryType#OTF OTF} or {@link DeliveryType#LIVE LIVE}
*/ */
public static void generateSegmentTemplateElement(@Nonnull final Document document, public static void generateSegmentTemplateElement(@Nonnull final Document doc,
@Nonnull final String baseUrl, @Nonnull final String baseUrl,
final DeliveryType deliveryType) final DeliveryType deliveryType)
throws CreationException { throws CreationException {
@ -533,32 +492,22 @@ public final class YoutubeDashManifestCreatorsUtils {
} }
try { try {
final Element representationElement = (Element) document.getElementsByTagName( final Element representationElement = (Element) doc.getElementsByTagName(
REPRESENTATION).item(0); REPRESENTATION).item(0);
final Element segmentTemplateElement = document.createElement(SEGMENT_TEMPLATE); final Element segmentTemplateElement = doc.createElement(SEGMENT_TEMPLATE);
final Attr startNumberAttribute = document.createAttribute("startNumber");
final boolean isDeliveryTypeLive = deliveryType == DeliveryType.LIVE;
// The first sequence of post DVR streams is the beginning of the video stream and not // The first sequence of post DVR streams is the beginning of the video stream and not
// an initialization segment // an initialization segment
final String startNumberValue = isDeliveryTypeLive ? "0" : "1"; setAttribute(segmentTemplateElement, doc, "startNumber",
startNumberAttribute.setValue(startNumberValue); deliveryType == DeliveryType.LIVE ? "0" : "1");
segmentTemplateElement.setAttributeNode(startNumberAttribute); setAttribute(segmentTemplateElement, doc, "timescale", "1000");
final Attr timescaleAttribute = document.createAttribute("timescale");
timescaleAttribute.setValue("1000");
segmentTemplateElement.setAttributeNode(timescaleAttribute);
// Post-live-DVR/ended livestreams streams don't require an initialization sequence // Post-live-DVR/ended livestreams streams don't require an initialization sequence
if (!isDeliveryTypeLive) { if (deliveryType != DeliveryType.LIVE) {
final Attr initializationAttribute = document.createAttribute("initialization"); setAttribute(segmentTemplateElement, doc, "initialization", baseUrl + SQ_0);
initializationAttribute.setValue(baseUrl + SQ_0);
segmentTemplateElement.setAttributeNode(initializationAttribute);
} }
final Attr mediaAttribute = document.createAttribute("media"); setAttribute(segmentTemplateElement, doc, "media", baseUrl + "&sq=$Number$");
mediaAttribute.setValue(baseUrl + "&sq=$Number$");
segmentTemplateElement.setAttributeNode(mediaAttribute);
representationElement.appendChild(segmentTemplateElement); representationElement.appendChild(segmentTemplateElement);
} catch (final DOMException e) { } catch (final DOMException e) {
@ -575,15 +524,15 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generateSegmentTemplateElement(Document, String, DeliveryType)}. * {@link #generateSegmentTemplateElement(Document, String, DeliveryType)}.
* </p> * </p>
* *
* @param document the {@link Document} on which the the {@code <SegmentTimeline>} element will * @param doc the {@link Document} on which the the {@code <SegmentTimeline>} element will be
* be appended * appended
*/ */
public static void generateSegmentTimelineElement(@Nonnull final Document document) public static void generateSegmentTimelineElement(@Nonnull final Document doc)
throws CreationException { throws CreationException {
try { try {
final Element segmentTemplateElement = (Element) document.getElementsByTagName( final Element segmentTemplateElement = (Element) doc.getElementsByTagName(
SEGMENT_TEMPLATE).item(0); SEGMENT_TEMPLATE).item(0);
final Element segmentTimelineElement = document.createElement(SEGMENT_TIMELINE); final Element segmentTimelineElement = doc.createElement(SEGMENT_TIMELINE);
segmentTemplateElement.appendChild(segmentTimelineElement); segmentTemplateElement.appendChild(segmentTimelineElement);
} catch (final DOMException e) { } catch (final DOMException e) {
@ -672,8 +621,7 @@ public final class YoutubeDashManifestCreatorsUtils {
// supported by all platforms (like the Android implementation) // supported by all platforms (like the Android implementation)
} }
final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder(); return documentBuilderFactory.newDocumentBuilder().newDocument();
return documentBuilder.newDocument();
} }
/** /**
@ -681,13 +629,13 @@ public final class YoutubeDashManifestCreatorsUtils {
* support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
* {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances. * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances.
* *
* @param document the document to convert, which must have been created using * @param doc the doc to convert, which must have been created using {@link #newDocument()} to
* {@link #newDocument()} to properly prevent XXE attacks * properly prevent XXE attacks
* @return the document converted to an XML string, making sure there can't be XXE attacks * @return the doc converted to an XML string, making sure there can't be XXE attacks
*/ */
// Sonar warning is suppressed because it is still shown even if we apply its solution // Sonar warning is suppressed because it is still shown even if we apply its solution
@SuppressWarnings("squid:S2755") @SuppressWarnings("squid:S2755")
private static String documentToXml(@Nonnull final Document document) private static String documentToXml(@Nonnull final Document doc)
throws TransformerException { throws TransformerException {
final TransformerFactory transformerFactory = TransformerFactory.newInstance(); final TransformerFactory transformerFactory = TransformerFactory.newInstance();
@ -705,7 +653,7 @@ public final class YoutubeDashManifestCreatorsUtils {
transformer.setOutputProperty(OutputKeys.STANDALONE, "no"); transformer.setOutputProperty(OutputKeys.STANDALONE, "no");
final StringWriter result = new StringWriter(); final StringWriter result = new StringWriter();
transformer.transform(new DOMSource(document), new StreamResult(result)); transformer.transform(new DOMSource(doc), new StreamResult(result));
return result.toString(); return result.toString();
} }

30
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java
View File

@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import org.w3c.dom.Attr;
import org.w3c.dom.DOMException; import org.w3c.dom.DOMException;
import org.w3c.dom.Document; import org.w3c.dom.Document;
import org.w3c.dom.Element; import org.w3c.dom.Element;
@ -23,6 +22,7 @@ import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement; import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement; import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse; import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isBlank; import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
@ -148,14 +148,14 @@ public final class YoutubeOtfDashManifestCreator {
streamDuration = durationSecondsFallback * 1000; streamDuration = durationSecondsFallback * 1000;
} }
final Document document = generateDocumentAndDoCommonElementsGeneration(itagItem, final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
streamDuration); streamDuration);
generateSegmentTemplateElement(document, realOtfBaseStreamingUrl, DeliveryType.OTF); generateSegmentTemplateElement(doc, realOtfBaseStreamingUrl, DeliveryType.OTF);
generateSegmentTimelineElement(document); generateSegmentTimelineElement(doc);
generateSegmentElementsForOtfStreams(segmentDuration, document); generateSegmentElementsForOtfStreams(segmentDuration, doc);
return buildAndCacheResult(otfBaseStreamingUrl, document, OTF_STREAMS_CACHE); return buildAndCacheResult(otfBaseStreamingUrl, doc, OTF_STREAMS_CACHE);
} }
/** /**
@ -192,17 +192,18 @@ public final class YoutubeOtfDashManifestCreator {
* *
* @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the * @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the
* regular expressions * regular expressions
* @param document the {@link Document} on which the {@code <S>} elements will be appended * @param doc the {@link Document} on which the {@code <S>} elements will be
* appended
*/ */
private static void generateSegmentElementsForOtfStreams( private static void generateSegmentElementsForOtfStreams(
@Nonnull final String[] segmentDurations, @Nonnull final String[] segmentDurations,
@Nonnull final Document document) throws CreationException { @Nonnull final Document doc) throws CreationException {
try { try {
final Element segmentTimelineElement = (Element) document.getElementsByTagName( final Element segmentTimelineElement = (Element) doc.getElementsByTagName(
SEGMENT_TIMELINE).item(0); SEGMENT_TIMELINE).item(0);
for (final String segmentDuration : segmentDurations) { for (final String segmentDuration : segmentDurations) {
final Element sElement = document.createElement("S"); final Element sElement = doc.createElement("S");
final String[] segmentLengthRepeat = segmentDuration.split("\\(r="); final String[] segmentLengthRepeat = segmentDuration.split("\\(r=");
// make sure segmentLengthRepeat[0], which is the length, is convertible to int // make sure segmentLengthRepeat[0], which is the length, is convertible to int
@ -212,14 +213,9 @@ public final class YoutubeOtfDashManifestCreator {
if (segmentLengthRepeat.length > 1) { if (segmentLengthRepeat.length > 1) {
final int segmentRepeatCount = Integer.parseInt( final int segmentRepeatCount = Integer.parseInt(
Utils.removeNonDigitCharacters(segmentLengthRepeat[1])); Utils.removeNonDigitCharacters(segmentLengthRepeat[1]));
final Attr rAttribute = document.createAttribute("r"); setAttribute(sElement, doc, "r", String.valueOf(segmentRepeatCount));
rAttribute.setValue(String.valueOf(segmentRepeatCount));
sElement.setAttributeNode(rAttribute);
} }
setAttribute(sElement, doc, "d", segmentLengthRepeat[0]);
final Attr dAttribute = document.createAttribute("d");
dAttribute.setValue(segmentLengthRepeat[0]);
sElement.setAttributeNode(dAttribute);
segmentTimelineElement.appendChild(sElement); segmentTimelineElement.appendChild(sElement);
} }

29
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java
View File

@ -4,7 +4,6 @@ import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.services.youtube.DeliveryType; import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
import org.w3c.dom.Attr;
import org.w3c.dom.DOMException; import org.w3c.dom.DOMException;
import org.w3c.dom.Document; import org.w3c.dom.Document;
import org.w3c.dom.Element; import org.w3c.dom.Element;
@ -23,6 +22,7 @@ import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement; import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement; import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse; import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -159,15 +159,15 @@ public final class YoutubePostLiveStreamDvrDashManifestCreator {
streamDuration = durationSecondsFallback; streamDuration = durationSecondsFallback;
} }
final Document document = generateDocumentAndDoCommonElementsGeneration(itagItem, final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
streamDuration); streamDuration);
generateSegmentTemplateElement(document, realPostLiveStreamDvrStreamingUrl, generateSegmentTemplateElement(doc, realPostLiveStreamDvrStreamingUrl,
DeliveryType.LIVE); DeliveryType.LIVE);
generateSegmentTimelineElement(document); generateSegmentTimelineElement(doc);
generateSegmentElementForPostLiveDvrStreams(document, targetDurationSec, segmentCount); generateSegmentElementForPostLiveDvrStreams(doc, targetDurationSec, segmentCount);
return buildAndCacheResult(postLiveStreamDvrStreamingUrl, document, return buildAndCacheResult(postLiveStreamDvrStreamingUrl, doc,
POST_LIVE_DVR_STREAMS_CACHE); POST_LIVE_DVR_STREAMS_CACHE);
} }
@ -190,7 +190,7 @@ public final class YoutubePostLiveStreamDvrDashManifestCreator {
* {@code <S d="targetDurationSecValue" r="segmentCount" />} * {@code <S d="targetDurationSecValue" r="segmentCount" />}
* </p> * </p>
* *
* @param document the {@link Document} on which the {@code <S>} element will * @param doc the {@link Document} on which the {@code <S>} element will
* be appended * be appended
* @param targetDurationSeconds the {@code targetDurationSec} value from YouTube player * @param targetDurationSeconds the {@code targetDurationSec} value from YouTube player
* response's stream * response's stream
@ -198,21 +198,16 @@ public final class YoutubePostLiveStreamDvrDashManifestCreator {
* #fromPostLiveStreamDvrStreamingUrl(String, ItagItem, int, long)} * #fromPostLiveStreamDvrStreamingUrl(String, ItagItem, int, long)}
*/ */
private static void generateSegmentElementForPostLiveDvrStreams( private static void generateSegmentElementForPostLiveDvrStreams(
@Nonnull final Document document, @Nonnull final Document doc,
final int targetDurationSeconds, final int targetDurationSeconds,
@Nonnull final String segmentCount) throws CreationException { @Nonnull final String segmentCount) throws CreationException {
try { try {
final Element segmentTimelineElement = (Element) document.getElementsByTagName( final Element segmentTimelineElement = (Element) doc.getElementsByTagName(
SEGMENT_TIMELINE).item(0); SEGMENT_TIMELINE).item(0);
final Element sElement = document.createElement("S"); final Element sElement = doc.createElement("S");
final Attr dAttribute = document.createAttribute("d"); setAttribute(sElement, doc, "d", String.valueOf(targetDurationSeconds * 1000));
dAttribute.setValue(String.valueOf(targetDurationSeconds * 1000)); setAttribute(sElement, doc, "r", segmentCount);
sElement.setAttributeNode(dAttribute);
final Attr rAttribute = document.createAttribute("r");
rAttribute.setValue(segmentCount);
sElement.setAttributeNode(rAttribute);
segmentTimelineElement.appendChild(sElement); segmentTimelineElement.appendChild(sElement);
} catch (final DOMException e) { } catch (final DOMException e) {

60
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java
View File

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
import org.schabi.newpipe.extractor.services.youtube.DeliveryType; import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
import org.w3c.dom.Attr;
import org.w3c.dom.DOMException; import org.w3c.dom.DOMException;
import org.w3c.dom.Document; import org.w3c.dom.Document;
import org.w3c.dom.Element; import org.w3c.dom.Element;
@ -18,6 +17,7 @@ import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE; import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult; import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration; import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
/** /**
* Class which generates DASH manifests of {@link DeliveryType#PROGRESSIVE YouTube progressive} * Class which generates DASH manifests of {@link DeliveryType#PROGRESSIVE YouTube progressive}
@ -100,14 +100,14 @@ public final class YoutubeProgressiveDashManifestCreator {
} }
} }
final Document document = generateDocumentAndDoCommonElementsGeneration(itagItem, final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
streamDuration); streamDuration);
generateBaseUrlElement(document, progressiveStreamingBaseUrl); generateBaseUrlElement(doc, progressiveStreamingBaseUrl);
generateSegmentBaseElement(document, itagItem); generateSegmentBaseElement(doc, itagItem);
generateInitializationElement(document, itagItem); generateInitializationElement(doc, itagItem);
return buildAndCacheResult(progressiveStreamingBaseUrl, document, return buildAndCacheResult(progressiveStreamingBaseUrl, doc,
PROGRESSIVE_STREAMS_CACHE); PROGRESSIVE_STREAMS_CACHE);
} }
@ -128,18 +128,17 @@ public final class YoutubeProgressiveDashManifestCreator {
* {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}). * {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}).
* </p> * </p>
* *
* @param document the {@link Document} on which the {@code <BaseURL>} element will * @param doc the {@link Document} on which the {@code <BaseURL>} element will be appended
* be appended
* @param baseUrl the base URL of the stream, which must not be null and will be set as the * @param baseUrl the base URL of the stream, which must not be null and will be set as the
* content of the {@code <BaseURL>} element * content of the {@code <BaseURL>} element
*/ */
private static void generateBaseUrlElement(@Nonnull final Document document, private static void generateBaseUrlElement(@Nonnull final Document doc,
@Nonnull final String baseUrl) @Nonnull final String baseUrl)
throws CreationException { throws CreationException {
try { try {
final Element representationElement = (Element) document.getElementsByTagName( final Element representationElement = (Element) doc.getElementsByTagName(
REPRESENTATION).item(0); REPRESENTATION).item(0);
final Element baseURLElement = document.createElement(BASE_URL); final Element baseURLElement = doc.createElement(BASE_URL);
baseURLElement.setTextContent(baseUrl); baseURLElement.setTextContent(baseUrl);
representationElement.appendChild(baseURLElement); representationElement.appendChild(baseURLElement);
} catch (final DOMException e) { } catch (final DOMException e) {
@ -167,28 +166,23 @@ public final class YoutubeProgressiveDashManifestCreator {
* should be generated too. * should be generated too.
* </p> * </p>
* *
* @param document the {@link Document} on which the {@code <SegmentBase>} element will be * @param doc the {@link Document} on which the {@code <SegmentBase>} element will be appended
* appended
* @param itagItem the {@link ItagItem} to use, which must not be null * @param itagItem the {@link ItagItem} to use, which must not be null
*/ */
private static void generateSegmentBaseElement(@Nonnull final Document document, private static void generateSegmentBaseElement(@Nonnull final Document doc,
@Nonnull final ItagItem itagItem) @Nonnull final ItagItem itagItem)
throws CreationException { throws CreationException {
try { try {
final Element representationElement = (Element) document.getElementsByTagName( final Element representationElement = (Element) doc.getElementsByTagName(
REPRESENTATION).item(0); REPRESENTATION).item(0);
final Element segmentBaseElement = doc.createElement(SEGMENT_BASE);
final Element segmentBaseElement = document.createElement(SEGMENT_BASE); final String range = itagItem.getIndexStart() + "-" + itagItem.getIndexEnd();
final Attr indexRangeAttribute = document.createAttribute("indexRange");
if (itagItem.getIndexStart() < 0 || itagItem.getIndexEnd() < 0) { if (itagItem.getIndexStart() < 0 || itagItem.getIndexEnd() < 0) {
throw CreationException.couldNotAddElement(SEGMENT_BASE, throw CreationException.couldNotAddElement(SEGMENT_BASE,
"ItagItem's indexStart or " + "indexEnd are < 0: " "ItagItem's indexStart or " + "indexEnd are < 0: " + range);
+ itagItem.getIndexStart() + "-" + itagItem.getIndexEnd());
} }
setAttribute(segmentBaseElement, doc, "indexRange", range);
indexRangeAttribute.setValue(itagItem.getIndexStart() + "-" + itagItem.getIndexEnd());
segmentBaseElement.setAttributeNode(indexRangeAttribute);
representationElement.appendChild(segmentBaseElement); representationElement.appendChild(segmentBaseElement);
} catch (final DOMException e) { } catch (final DOMException e) {
@ -214,28 +208,24 @@ public final class YoutubeProgressiveDashManifestCreator {
* {@link #generateSegmentBaseElement(Document, ItagItem)}). * {@link #generateSegmentBaseElement(Document, ItagItem)}).
* </p> * </p>
* *
* @param document the {@link Document} on which the {@code <Initialization>} element will * @param doc the {@link Document} on which the {@code <Initialization>} element will be
* be appended * appended
* @param itagItem the {@link ItagItem} to use, which must not be null * @param itagItem the {@link ItagItem} to use, which must not be null
*/ */
private static void generateInitializationElement(@Nonnull final Document document, private static void generateInitializationElement(@Nonnull final Document doc,
@Nonnull final ItagItem itagItem) @Nonnull final ItagItem itagItem)
throws CreationException { throws CreationException {
try { try {
final Element segmentBaseElement = (Element) document.getElementsByTagName( final Element segmentBaseElement = (Element) doc.getElementsByTagName(
SEGMENT_BASE).item(0); SEGMENT_BASE).item(0);
final Element initializationElement = doc.createElement(INITIALIZATION);
final Element initializationElement = document.createElement(INITIALIZATION); final String range = itagItem.getInitStart() + "-" + itagItem.getInitEnd();
final Attr rangeAttribute = document.createAttribute("range");
if (itagItem.getInitStart() < 0 || itagItem.getInitEnd() < 0) { if (itagItem.getInitStart() < 0 || itagItem.getInitEnd() < 0) {
throw CreationException.couldNotAddElement(INITIALIZATION, throw CreationException.couldNotAddElement(INITIALIZATION,
"ItagItem's initStart and/or " + "initEnd are/is < 0: " "ItagItem's initStart and/or " + "initEnd are/is < 0: " + range);
+ itagItem.getInitStart() + "-" + itagItem.getInitEnd());
} }
setAttribute(initializationElement, doc, "range", range);
rangeAttribute.setValue(itagItem.getInitStart() + "-" + itagItem.getInitEnd());
initializationElement.setAttributeNode(rangeAttribute);
segmentBaseElement.appendChild(initializationElement); segmentBaseElement.appendChild(initializationElement);
} catch (final DOMException e) { } catch (final DOMException e) {

106
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -1147,34 +1147,27 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final java.util.function.Function<ItagInfo, T> streamBuilderHelper, final java.util.function.Function<ItagInfo, T> streamBuilderHelper,
final String streamTypeExceptionMessage) throws ParsingException { final String streamTypeExceptionMessage) throws ParsingException {
try { try {
final List<ItagInfo> itagInfos = new ArrayList<>(); final String videoId = getId();
if (html5StreamingData == null && androidStreamingData == null
&& iosStreamingData == null) {
return Collections.emptyList();
}
final List<Pair<JsonObject, String>> streamingDataAndCpnLoopList = new ArrayList<>();
// Use the androidStreamingData object first because there is no n param and no
// signatureCiphers in streaming URLs of the Android client
streamingDataAndCpnLoopList.add(new Pair<>(androidStreamingData, androidCpn));
streamingDataAndCpnLoopList.add(new Pair<>(html5StreamingData, html5Cpn));
// Use the iosStreamingData object in the last position because most of the available
// streams can be extracted with the Android and web clients and also because the iOS
// client is only enabled by default on livestreams
streamingDataAndCpnLoopList.add(new Pair<>(iosStreamingData, iosCpn));
for (final Pair<JsonObject, String> pair : streamingDataAndCpnLoopList) {
itagInfos.addAll(getStreamsFromStreamingDataKey(pair.getFirst(), streamingDataKey,
itagTypeWanted, pair.getSecond()));
}
final List<T> streamList = new ArrayList<>(); final List<T> streamList = new ArrayList<>();
for (final ItagInfo itagInfo : itagInfos) {
final T stream = streamBuilderHelper.apply(itagInfo); java.util.stream.Stream.of(
if (!Stream.containSimilarStream(stream, streamList)) { // Use the androidStreamingData object first because there is no n param and no
streamList.add(stream); // signatureCiphers in streaming URLs of the Android client
} new Pair<>(androidStreamingData, androidCpn),
} new Pair<>(html5StreamingData, html5Cpn),
// Use the iosStreamingData object in the last position because most of the
// available streams can be extracted with the Android and web clients and also
// because the iOS client is only enabled by default on livestreams
new Pair<>(iosStreamingData, iosCpn)
)
.flatMap(pair -> getStreamsFromStreamingDataKey(videoId, pair.getFirst(),
streamingDataKey, itagTypeWanted, pair.getSecond()))
.map(streamBuilderHelper)
.forEachOrdered(stream -> {
if (!Stream.containSimilarStream(stream, streamList)) {
streamList.add(stream);
}
});
return streamList; return streamList;
} catch (final Exception e) { } catch (final Exception e) {
@ -1293,43 +1286,36 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Nonnull @Nonnull
private List<ItagInfo> getStreamsFromStreamingDataKey( private java.util.stream.Stream<ItagInfo> getStreamsFromStreamingDataKey(
final String videoId,
final JsonObject streamingData, final JsonObject streamingData,
final String streamingDataKey, final String streamingDataKey,
@Nonnull final ItagItem.ItagType itagTypeWanted, @Nonnull final ItagItem.ItagType itagTypeWanted,
@Nonnull final String contentPlaybackNonce) throws ParsingException { @Nonnull final String contentPlaybackNonce) {
if (streamingData == null || !streamingData.has(streamingDataKey)) { if (streamingData == null || !streamingData.has(streamingDataKey)) {
return Collections.emptyList(); return java.util.stream.Stream.empty();
} }
final String videoId = getId(); return streamingData.getArray(streamingDataKey).stream()
final List<ItagInfo> itagInfos = new ArrayList<>(); .filter(JsonObject.class::isInstance)
final JsonArray formats = streamingData.getArray(streamingDataKey); .map(JsonObject.class::cast)
for (int i = 0; i != formats.size(); ++i) { .map(formatData -> {
final JsonObject formatData = formats.getObject(i); try {
final int itag = formatData.getInt("itag"); final ItagItem itagItem = ItagItem.getItag(formatData.getInt("itag"));
if (itagItem.itagType == itagTypeWanted) {
if (!ItagItem.isSupported(itag)) { return buildAndAddItagInfoToList(videoId, formatData, itagItem,
continue; itagItem.itagType, contentPlaybackNonce);
} }
} catch (final IOException | ExtractionException ignored) {
try { // if the itag is not supported and getItag fails, we end up here
final ItagItem itagItem = ItagItem.getItag(itag); }
final ItagItem.ItagType itagType = itagItem.itagType; return null;
if (itagType == itagTypeWanted) { })
buildAndAddItagInfoToList(videoId, itagInfos, formatData, itagItem, .filter(Objects::nonNull);
itagType, contentPlaybackNonce);
}
} catch (final IOException | ExtractionException ignored) {
}
}
return itagInfos;
} }
private void buildAndAddItagInfoToList( private ItagInfo buildAndAddItagInfoToList(
@Nonnull final String videoId, @Nonnull final String videoId,
@Nonnull final List<ItagInfo> itagInfos,
@Nonnull final JsonObject formatData, @Nonnull final JsonObject formatData,
@Nonnull final ItagItem itagItem, @Nonnull final ItagItem itagItem,
@Nonnull final ItagItem.ItagType itagType, @Nonnull final ItagItem.ItagType itagType,
@ -1372,12 +1358,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (streamType == StreamType.LIVE_STREAM || streamType == StreamType.POST_LIVE_STREAM) { if (streamType == StreamType.LIVE_STREAM || streamType == StreamType.POST_LIVE_STREAM) {
itagItem.setTargetDurationSec(formatData.getInt("targetDurationSec")); itagItem.setTargetDurationSec(formatData.getInt("targetDurationSec"));
} } else if (itagType == ItagItem.ItagType.VIDEO
|| itagType == ItagItem.ItagType.VIDEO_ONLY) {
if (itagType == ItagItem.ItagType.VIDEO || itagType == ItagItem.ItagType.VIDEO_ONLY) {
itagItem.setFps(formatData.getInt("fps")); itagItem.setFps(formatData.getInt("fps"));
} } else if (itagType == ItagItem.ItagType.AUDIO) {
if (itagType == ItagItem.ItagType.AUDIO) {
// YouTube return the audio sample rate as a string // YouTube return the audio sample rate as a string
itagItem.setSampleRate(Integer.parseInt(formatData.getString("audioSampleRate"))); itagItem.setSampleRate(Integer.parseInt(formatData.getString("audioSampleRate")));
itagItem.setAudioChannels(formatData.getInt("audioChannels")); itagItem.setAudioChannels(formatData.getInt("audioChannels"));
@ -1403,7 +1387,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
itagInfo.setIsUrl(streamType != StreamType.POST_LIVE_STREAM); itagInfo.setIsUrl(streamType != StreamType.POST_LIVE_STREAM);
} }
itagInfos.add(itagInfo); return itagInfo;
} }
@Nonnull @Nonnull

19
extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java
View File

@ -94,24 +94,15 @@ public abstract class Stream implements Serializable {
* Note: This method always returns false if the stream passed is null. * Note: This method always returns false if the stream passed is null.
* </p> * </p>
* *
* @param cmp the stream object to be compared to this stream object * @param other the stream object to be compared to this stream object
* @return whether the stream have the same stats or not, based on the criteria above * @return whether the stream have the same stats or not, based on the criteria above
*/ */
public boolean equalStats(@Nullable final Stream cmp) { public boolean equalStats(@Nullable final Stream other) {
if (cmp == null) { if (other == null || mediaFormat == null || other.mediaFormat == null) {
return false; return false;
} }
return mediaFormat.id == other.mediaFormat.id && deliveryMethod == other.deliveryMethod
Boolean haveSameMediaFormatId = null; && isUrl == other.isUrl;
if (mediaFormat != null && cmp.mediaFormat != null) {
haveSameMediaFormatId = mediaFormat.id == cmp.mediaFormat.id;
}
final boolean areUsingSameDeliveryMethodAndAreUrlStreams =
deliveryMethod == cmp.deliveryMethod && isUrl == cmp.isUrl;
return haveSameMediaFormatId != null
? haveSameMediaFormatId && areUsingSameDeliveryMethodAndAreUrlStreams
: areUsingSameDeliveryMethodAndAreUrlStreams;
} }
/** /**

5
extractor/src/test/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCacheTest.java
View File

@ -53,6 +53,11 @@ class ManifestCreatorCacheTest {
+ "call"); + "call");
} }
/**
* Adds sample strings to the provided manifest creator cache, in order to test clear factor and
* maximum size.
* @param cache the cache to fill with some data
*/
private static void setCacheContent(final ManifestCreatorCache<String, String> cache) { private static void setCacheContent(final ManifestCreatorCache<String, String> cache) {
int i = 0; int i = 0;
while (i < 26) { while (i < 26) {