From 7129d6db55d92fed6de5365c272b6b7a6610681d Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Wed, 7 Nov 2018 18:28:44 +0100 Subject: [PATCH] add inline documentation for StreamExtractor --- .../soundcloud/SoundcloudStreamExtractor.java | 4 +- .../extractors/YoutubeStreamExtractor.java | 4 +- .../extractor/stream/StreamExtractor.java | 255 +++++++++++++++--- .../newpipe/extractor/stream/StreamInfo.java | 3 +- .../extractor/{ => stream}/Subtitles.java | 2 +- .../extractor/utils/ExtractorHelper.java | 2 +- .../SoundcloudStreamExtractorDefaultTest.java | 2 +- .../YoutubeStreamExtractorDefaultTest.java | 4 +- 8 files changed, 223 insertions(+), 53 deletions(-) rename extractor/src/main/java/org/schabi/newpipe/extractor/{ => stream}/Subtitles.java (94%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index f19f46c33..e7fe83d9f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -188,12 +188,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public StreamInfoItem getNextVideo() throws IOException, ExtractionException { + public StreamInfoItem getNextStream() throws IOException, ExtractionException { return null; } @Override - public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException { + public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + "/related" diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 668985572..6e310b13e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -490,7 +490,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public StreamInfoItem getNextVideo() throws IOException, ExtractionException { + public StreamInfoItem getNextStream() throws IOException, ExtractionException { assertPageFetched(); try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); @@ -504,7 +504,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException { + public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { assertPageFetched(); try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 6071b2dc6..e4a83729a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -22,7 +22,6 @@ package org.schabi.newpipe.extractor.stream; import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.Subtitles; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; @@ -34,7 +33,7 @@ import java.io.IOException; import java.util.List; /** - * Scrapes information from a video streaming service (eg, YouTube). + * Scrapes information from a video/audio streaming service (eg, YouTube). */ public abstract class StreamExtractor extends Extractor { @@ -44,22 +43,234 @@ public abstract class StreamExtractor extends Extractor { super(service, linkHandler, localization); } + /** + * The day on which the stream got uploaded/created. The return information should be in the format + * dd.mm.yyyy, however it NewPipe will not crash if its not. + * @return The day on which the stream got uploaded. + * @throws ParsingException + */ @Nonnull public abstract String getUploadDate() throws ParsingException; + + /** + * This will return the url to the thumbnail of the stream. Try to return the medium resolution here. + * @return The url of the thumbnail. + * @throws ParsingException + */ @Nonnull public abstract String getThumbnailUrl() throws ParsingException; + + /** + * This is the stream description. On YouTube this is the video description. You can return simple HTML here. + * @return The description of the stream/video. + * @throws ParsingException + */ @Nonnull public abstract String getDescription() throws ParsingException; /** - * Get the age limit + * Get the age limit. * @return The age which limits the content or {@value NO_AGE_LIMIT} if there is no limit * @throws ParsingException if an error occurs while parsing */ public abstract int getAgeLimit() throws ParsingException; + /** + * This should return the length of a video in seconds. + * @return The length of the stream in seconds. + * @throws ParsingException + */ public abstract long getLength() throws ParsingException; + + /** + * If the url you are currently handling contains a time stamp/seek, you can return the + * position it represents here. + * If the url has no time stamp simply return zero. + * @return the timestamp in seconds + * @throws ParsingException + */ public abstract long getTimeStamp() throws ParsingException; + + /** + * The count of how many people have watched the video/listened to the audio stream. + * If the current stream has no view count or its not available simply return -1 + * @return amount of views. + * @throws ParsingException + */ + public abstract long getViewCount() throws ParsingException; + + /** + * The Amount of likes a video/audio stream got. + * If the current stream has no likes or its not available simply return -1 + * @return the amount of likes the stream got + * @throws ParsingException + */ + public abstract long getLikeCount() throws ParsingException; + + /** + * The Amount of dislikes a video/audio stream got. + * If the current stream has no dislikes or its not available simply return -1 + * @return the amount of likes the stream got + * @throws ParsingException + */ + public abstract long getDislikeCount() throws ParsingException; + + /** + * The Url to the page of the creator/uploader of the stream. This must not be a homepage, + * but the page offered by the service the extractor handles. This url will be handled by the + * ChannelExtractor, + * so be sure to implement that one before you return a value here, otherwise NewPipe will crash if one selects + * this url. + * @return the url to the page of the creator/uploader of the stream or an empty String + * @throws ParsingException + */ + @Nonnull + public abstract String getUploaderUrl() throws ParsingException; + + /** + * The name of the creator/uploader of the stream. + * If the name is not available you can simply return an empty string. + * @return the name of the creator/uploader of the stream or an empty String + * @throws ParsingException + */ + @Nonnull + public abstract String getUploaderName() throws ParsingException; + + /** + * The url to the image file/profile picture/avatar of the creator/uploader of the stream. + * If the url is not available you can return an empty String. + * @return The url of the image file of the uploader or an empty String + * @throws ParsingException + */ + @Nonnull + public abstract String getUploaderAvatarUrl() throws ParsingException; + + /** + * Get the dash mpd url. If you don't know what a dash MPD is you can read about it + * here. + * @return the url as a string or an empty string + * @throws ParsingException if an error occurs while reading + */ + @Nonnull public abstract String getDashMpdUrl() throws ParsingException; + + /** + * I am not sure if this is in use, and how this is used. However the frontend is missing support + * for HLS streams. Prove me if I am wrong. Please open an + * issue, + * or fix this description if you know whats up with this. + * @return The Url to the hls stream. + * @throws ParsingException + */ + @Nonnull public abstract String getHlsUrl() throws ParsingException; + + /** + * This should return a list of available + * AudioStreams + * You can also return null or an empty list, however be aware that if you don't return anything + * in getVideoStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as + * a failed extraction procedure. + * @return a list of audio only streams in the format of AudioStream + * @throws IOException + * @throws ExtractionException + */ + public abstract List getAudioStreams() throws IOException, ExtractionException; + + /** + * This should return a list of available + * VideoStreams + * Be aware this is the list of video streams which do contain an audio stream. + * You can also return null or an empty list, however be aware that if you don't return anything + * in getAudioStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as + * a failed extraction procedure. + * @return a list of combined video and streams in the format of AudioStream + * @throws IOException + * @throws ExtractionException + */ + public abstract List getVideoStreams() throws IOException, ExtractionException; + + /** + * This should return a list of available + * VideoStreams. + * Be aware this is the list of video streams which do NOT contain an audio stream. + * You can also return null or an empty list, however be aware that if you don't return anything + * in getAudioStreams(), getVideoStreams() and getDashMpdUrl() either the Collector will handle this as + * a failed extraction procedure. + * @return a list of video and streams in the format of AudioStream + * @throws IOException + * @throws ExtractionException + */ + public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; + + /** + * This will return a list of available + * Subtitless. + * If no subtitles are available an empty list can returned. + * @return a list of available subtitles or an empty list + * @throws IOException + * @throws ExtractionException + */ + @Nonnull + public abstract List getSubtitlesDefault() throws IOException, ExtractionException; + + /** + * This will return a list of available + * Subtitless. + * given by a specific type. + * If no subtitles in that specific format are available an empty list can returned. + * @return a list of available subtitles or an empty list + * @throws IOException + * @throws ExtractionException + */ + @Nonnull + public abstract List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException; + + /** + * Get the StreamType. + * @return the type of the stream + * @throws ParsingException + */ + public abstract StreamType getStreamType() throws ParsingException; + + /** + * should return the url of the next stream. NewPipe will automatically play + * the next stream if the user wants that. + * If the next stream is is not available simply return null + * @return the InfoItem of the next stream + * @throws IOException + * @throws ExtractionException + */ + public abstract StreamInfoItem getNextStream() throws IOException, ExtractionException; + + /** + * Should return a list of streams related to the current handled. Many services show suggested + * streams. If you don't like suggested streams you should implement them anyway since they can + * be disabled by the user later in the frontend. + * This list MUST NOT contain the next available video as this should be return through getNextStream() + * If is is not available simply return null + * @return a list of InfoItems showing the related videos/streams + * @throws IOException + * @throws ExtractionException + */ + public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException; + + /** + * Should analyse the webpage's document and extracts any error message there might be. (e.g. GEMA block) + * + * @return Error message; null if there is no error message. + */ + public abstract String getErrorMessage(); + + ////////////////////////////////////////////////////////////////// + /// Helper + ////////////////////////////////////////////////////////////////// + + /** + * Override this function if the format of time stamp in the url is not the same format as that form youtube. + * Honestly I don't even know the time stamp fromat of youtube. + * @param regexPattern + * @return the sime stamp/seek for the video in seconds + * @throws ParsingException + */ protected long getTimestampSeconds(String regexPattern) throws ParsingException { String timeStamp; try { @@ -104,42 +315,4 @@ public abstract class StreamExtractor extends Extractor { } else { return 0; }}; - - public abstract long getViewCount() throws ParsingException; - public abstract long getLikeCount() throws ParsingException; - public abstract long getDislikeCount() throws ParsingException; - - @Nonnull - public abstract String getUploaderUrl() throws ParsingException; - @Nonnull - public abstract String getUploaderName() throws ParsingException; - @Nonnull - public abstract String getUploaderAvatarUrl() throws ParsingException; - - /** - * Get the dash mpd url - * @return the url as a string or an empty string - * @throws ParsingException if an error occurs while reading - */ - @Nonnull public abstract String getDashMpdUrl() throws ParsingException; - @Nonnull public abstract String getHlsUrl() throws ParsingException; - public abstract List getAudioStreams() throws IOException, ExtractionException; - public abstract List getVideoStreams() throws IOException, ExtractionException; - public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; - - @Nonnull - public abstract List getSubtitlesDefault() throws IOException, ExtractionException; - @Nonnull - public abstract List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException; - - public abstract StreamType getStreamType() throws ParsingException; - public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException; - public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException; - - /** - * Analyses the webpage's document and extracts any error message there might be. - * - * @return Error message; null if there is no error message. - */ - public abstract String getErrorMessage(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 65a024983..f9732f4c4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.utils.DashMpdParser; import org.schabi.newpipe.extractor.utils.ExtractorHelper; -import org.schabi.newpipe.extractor.utils.Localization; import java.io.IOException; import java.util.ArrayList; @@ -240,7 +239,7 @@ public class StreamInfo extends Info { streamInfo.addError(e); } try { - streamInfo.setNextVideo(extractor.getNextVideo()); + streamInfo.setNextVideo(extractor.getNextStream()); } catch (Exception e) { streamInfo.addError(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Subtitles.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Subtitles.java similarity index 94% rename from extractor/src/main/java/org/schabi/newpipe/extractor/Subtitles.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/stream/Subtitles.java index affd854e9..814883289 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Subtitles.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Subtitles.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor; +package org.schabi.newpipe.extractor.stream; import org.schabi.newpipe.extractor.stream.SubtitlesFormat; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java index bad595933..445c52c77 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java @@ -29,7 +29,7 @@ public class ExtractorHelper { public static List getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) { try { - InfoItemsCollector collector = extractor.getRelatedVideos(); + InfoItemsCollector collector = extractor.getRelatedStreams(); info.addAllErrors(collector.getErrors()); //noinspection unchecked diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index b6c5b3a2b..3970efde8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -101,7 +101,7 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetRelatedVideos() throws ExtractionException, IOException { - StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos(); + StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); assertFalse(relatedVideos.getItems().isEmpty()); assertTrue(relatedVideos.getErrors().isEmpty()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 2c026f361..75dc5e007 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.*; -import org.schabi.newpipe.extractor.utils.DashMpdParser; import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.extractor.utils.Utils; @@ -17,7 +16,6 @@ import java.io.IOException; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeTrendingExtractorTest.extractor; /* * Created by Christian Schabesberger on 30.12.15. @@ -151,7 +149,7 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetRelatedVideos() throws ExtractionException, IOException { - StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos(); + StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); Utils.printErrors(relatedVideos.getErrors()); assertFalse(relatedVideos.getItems().isEmpty()); assertTrue(relatedVideos.getErrors().isEmpty());