From d3eea4f4be1acd721047ffbe1e3eff16244fc857 Mon Sep 17 00:00:00 2001 From: vkay94 Date: Sat, 12 Dec 2020 10:24:29 +0100 Subject: [PATCH] Add stream segments to StreamInfo --- .../extractors/MediaCCCStreamExtractor.java | 7 ++ .../extractors/PeertubeStreamExtractor.java | 7 ++ .../extractors/SoundcloudStreamExtractor.java | 7 ++ .../extractors/YoutubeStreamExtractor.java | 55 +++++++++++++++ .../extractor/stream/StreamExtractor.java | 10 +++ .../newpipe/extractor/stream/StreamInfo.java | 14 ++++ .../extractor/stream/StreamSegment.java | 69 +++++++++++++++++++ 7 files changed, 169 insertions(+) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamSegment.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 96b1a5cc7..97236ac5c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamSegment; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; @@ -294,4 +295,10 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public String getSupportInfo() { return ""; } + + @Nonnull + @Override + public List getStreamSegments() { + return Collections.emptyList(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index fa02d473a..831bb10f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -23,6 +23,7 @@ import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamSegment; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; @@ -302,6 +303,12 @@ public class PeertubeStreamExtractor extends StreamExtractor { } } + @Nonnull + @Override + public List getStreamSegments() { + return Collections.emptyList(); + } + private String getRelatedStreamsUrl(final List tags) throws UnsupportedEncodingException { final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; final StringBuilder params = new StringBuilder(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index bc1d59026..f24674de6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamSegment; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; @@ -320,4 +321,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public String getSupportInfo() { return ""; } + + @Nonnull + @Override + public List getStreamSegments() { + return Collections.emptyList(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 608ae47cd..6179742dc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -35,6 +35,7 @@ import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamSegment; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; @@ -1061,4 +1062,58 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getSupportInfo() { return ""; } + + @Nonnull + @Override + public List getStreamSegments() throws ParsingException { + final ArrayList segments = new ArrayList<>(); + if (initialData.has("engagementPanels")) { + final JsonArray panels = initialData.getArray("engagementPanels"); + JsonArray segmentsArray = null; + + // Search for correct panel containing the data + for (int i = 0; i < panels.size(); i++) { + if (panels.getObject(i).getObject("engagementPanelSectionListRenderer") + .getString("panelIdentifier").equals("engagement-panel-macro-markers")) { + segmentsArray = panels.getObject(i).getObject("engagementPanelSectionListRenderer") + .getObject("content").getObject("macroMarkersListRenderer").getArray("contents"); + break; + } + } + + if (segmentsArray != null) { + final long duration = getLength(); + for (final Object object : segmentsArray) { + final JsonObject segmentJson = ((JsonObject) object).getObject("macroMarkersListItemRenderer"); + + final int startTimeSeconds = segmentJson.getObject("onTap").getObject("watchEndpoint") + .getInt("startTimeSeconds", -1); + + if (startTimeSeconds == -1) { + throw new ParsingException("Could not get stream segment start time."); + } + if (startTimeSeconds > duration) { + break; + } + + final String title = getTextFromObject(segmentJson.getObject("title")); + if (isNullOrEmpty(title)) { + throw new ParsingException("Could not get stream segment title."); + } + + final StreamSegment segment = new StreamSegment(title, startTimeSeconds); + segment.setUrl(getUrl() + "?t=" + startTimeSeconds); + if (segmentJson.has("thumbnail")) { + final JsonArray previewsArray = segmentJson.getObject("thumbnail").getArray("thumbnails"); + if (!previewsArray.isEmpty()) { + // Assume that the thumbnail with the highest resolution is at the last position + segment.setPreviewUrl(previewsArray.getObject(previewsArray.size() - 1).getString("url")); + } + } + segments.add(segment); + } + } + } + return segments; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 4e109bbab..dca4bbbc3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -476,4 +476,14 @@ public abstract class StreamExtractor extends Extractor { */ @Nonnull public abstract String getSupportInfo() throws ParsingException; + + /** + * The list of stream segments by timestamps for the stream. + * If the segment list is not available you can simply return an empty list. + * + * @return The list of segments of the stream or an empty list. + * @throws ParsingException + */ + @Nonnull + public abstract List getStreamSegments() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 805f26122..18eab21a7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -324,6 +324,11 @@ public class StreamInfo extends Info { } catch (Exception e) { streamInfo.addError(e); } + try { + streamInfo.setStreamSegments(extractor.getStreamSegments()); + } catch (Exception e) { + streamInfo.addError(e); + } streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); @@ -373,6 +378,7 @@ public class StreamInfo extends Info { private String support = ""; private Locale language = null; private List tags = new ArrayList<>(); + private List streamSegments = new ArrayList<>(); /** * Get the stream type @@ -670,4 +676,12 @@ public class StreamInfo extends Info { public String getSupportInfo() { return this.support; } + + public List getStreamSegments() { + return streamSegments; + } + + public void setStreamSegments(List streamSegments) { + this.streamSegments = streamSegments; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamSegment.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamSegment.java new file mode 100644 index 000000000..2bb329aca --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamSegment.java @@ -0,0 +1,69 @@ +package org.schabi.newpipe.extractor.stream; + + +import javax.annotation.Nullable; +import java.io.Serializable; + +public class StreamSegment implements Serializable { + /** + * Title of this segment + */ + private String title; + + /** + * Timestamp of the starting point in seconds + */ + private int startTimeSeconds; + + /** + * Direct url to this segment. This can be null if the service doesn't provide such function. + */ + @Nullable + public String url; + + /** + * Preview url for this segment. This can be null if the service doesn't provide such function + * or there is no resource found. + */ + @Nullable + private String previewUrl = null; + + public StreamSegment(String title, int startTimeSeconds) { + this.title = title; + this.startTimeSeconds = startTimeSeconds; + } + + public String getTitle() { + return title; + } + + public void setTitle(final String title) { + this.title = title; + } + + public int getStartTimeSeconds() { + return startTimeSeconds; + } + + public void setStartTimeSeconds(final int startTimeSeconds) { + this.startTimeSeconds = startTimeSeconds; + } + + @Nullable + public String getUrl() { + return url; + } + + public void setUrl(@Nullable final String url) { + this.url = url; + } + + @Nullable + public String getPreviewUrl() { + return previewUrl; + } + + public void setPreviewUrl(@Nullable final String previewUrl) { + this.previewUrl = previewUrl.replaceAll("\\u0026", "&"); + } +}