From e4afb2186214509997a8768adcdfb899f0e73c9f Mon Sep 17 00:00:00 2001 From: kapodamy Date: Mon, 24 Sep 2018 16:04:22 -0300 Subject: [PATCH] make Subtitle object Stream compliant * merge SubtitlesFormat into MediaFormat * implement Stream interface on Subtitle class * misc fixes: dont show a snackbar error on age-restricted videos, lint fix. --- .../schabi/newpipe/extractor/MediaFormat.java | 9 ++- .../soundcloud/SoundcloudStreamExtractor.java | 4 +- .../extractors/YoutubeStreamExtractor.java | 26 +++---- .../extractor/stream/StreamExtractor.java | 42 ++++++++++- .../newpipe/extractor/stream/StreamInfo.java | 6 +- .../extractor/stream/SubtitlesStream.java | 73 +++++++++++++++++++ 6 files changed, 140 insertions(+), 20 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java index e39e6977a..21ead42c4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java @@ -36,7 +36,14 @@ public enum MediaFormat { M4A (0x3, "m4a", "m4a", "audio/mp4"), WEBMA (0x4, "WebM", "webm", "audio/webm"), MP3 (0x5, "MP3", "mp3", "audio/mpeg"), - OPUS (0x6, "opus", "opus", "audio/opus"); + OPUS (0x6, "opus", "opus", "audio/opus"), + // subtitles formats + VTT (0x7, "WebVTT", "vtt", "text/vtt"), + TTML (0x8, "Timed Text Markup Language", "ttml", "application/ttml+xml"), + TRANSCRIPT1 (0x9, "TranScript v1", "srv1", "text/xml"), + TRANSCRIPT2 (0xA, "TranScript v2", "srv2", "text/xml"), + TRANSCRIPT3 (0xB, "TranScript v3", "srv3", "text/xml"), + SRT (0xC, "SubRip file format", "srt", "text/srt"); public final int id; public final String name; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index e7fe83d9f..005722e3e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -172,13 +172,13 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override @Nonnull - public List getSubtitlesDefault() throws IOException, ExtractionException { + public List getSubtitlesDefault() throws IOException, ExtractionException { return Collections.emptyList(); } @Override @Nonnull - public List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException { + public List getSubtitles(MediaFormat format) throws IOException, ExtractionException { return Collections.emptyList(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 6e310b13e..e5208d5ad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; @@ -460,15 +461,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override @Nonnull - public List getSubtitlesDefault() throws IOException, ExtractionException { - return getSubtitles(SubtitlesFormat.TTML); + public List getSubtitlesDefault() throws IOException, ExtractionException { + return getSubtitles(MediaFormat.TTML); } @Override @Nonnull - public List getSubtitles(final SubtitlesFormat format) throws IOException, ExtractionException { + public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { assertPageFetched(); - List subtitles = new ArrayList<>(); + List subtitles = new ArrayList<>(); for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) { subtitles.add(subtitlesInfo.getSubtitle(format)); } @@ -494,9 +495,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - collector.commit(extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]") - .first().select("li").first())); + Elements watch = doc.select("div[class=\"watch-sidebar-section\"]"); + if (watch.size() < 1) { + return null;// prevent the snackbar notification "report error" on age-restricted videos + } + + collector.commit(extractVideoPreviewInfo(watch.first().select("li").first())); return collector.getItems().get(0); } catch (Exception e) { throw new ParsingException("Could not get next video", e); @@ -815,21 +820,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String languageCode; final boolean isGenerated; - final Locale locale; - public SubtitlesInfo(final String baseUrl, final String languageCode, final boolean isGenerated) { this.cleanUrl = baseUrl .replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists .replaceAll("&tlang=[^&]*", ""); // Remove translation language this.languageCode = languageCode; this.isGenerated = isGenerated; - - final String[] splits = languageCode.split("-"); - this.locale = splits.length == 2 ? new Locale(splits[0], splits[1]) : new Locale(languageCode); } - public Subtitles getSubtitle(final SubtitlesFormat format) { - return new Subtitles(format, locale, cleanUrl + "&fmt=" + format.getExtension(), isGenerated); + public SubtitlesStream getSubtitle(final MediaFormat format) { + return new SubtitlesStream(format, languageCode, cleanUrl + "&fmt=" + format.getSuffix(), isGenerated); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index e4a83729a..7d1a5e69e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -21,6 +21,7 @@ package org.schabi.newpipe.extractor.stream; */ import org.schabi.newpipe.extractor.Extractor; +import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -314,5 +315,44 @@ public abstract class StreamExtractor extends Extractor { } } else { return 0; - }}; + } + } + + public abstract long getViewCount() throws ParsingException; + public abstract long getLikeCount() throws ParsingException; + public abstract long getDislikeCount() throws ParsingException; + + @Nonnull + public abstract String getUploaderUrl() throws ParsingException; + @Nonnull + public abstract String getUploaderName() throws ParsingException; + @Nonnull + public abstract String getUploaderAvatarUrl() throws ParsingException; + + /** + * Get the dash mpd url + * @return the url as a string or an empty string + * @throws ParsingException if an error occurs while reading + */ + @Nonnull public abstract String getDashMpdUrl() throws ParsingException; + @Nonnull public abstract String getHlsUrl() throws ParsingException; + public abstract List getAudioStreams() throws IOException, ExtractionException; + public abstract List getVideoStreams() throws IOException, ExtractionException; + public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; + + @Nonnull + public abstract List getSubtitlesDefault() throws IOException, ExtractionException; + @Nonnull + public abstract List getSubtitles(MediaFormat format) throws IOException, ExtractionException; + + public abstract StreamType getStreamType() throws ParsingException; + public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException; + public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException; + + /** + * Analyses the webpage's document and extracts any error message there might be. + * + * @return Error message; null if there is no error message. + */ + public abstract String getErrorMessage(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index f9732f4c4..6b3afb970 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -283,7 +283,7 @@ public class StreamInfo extends Info { private List relatedStreams; private long startPosition = 0; - private List subtitles; + private List subtitles; /** * Get the stream type @@ -494,11 +494,11 @@ public class StreamInfo extends Info { this.startPosition = startPosition; } - public List getSubtitles() { + public List getSubtitles() { return subtitles; } - public void setSubtitles(List subtitles) { + public void setSubtitles(List subtitles) { this.subtitles = subtitles; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java new file mode 100644 index 000000000..d0e09ac1b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java @@ -0,0 +1,73 @@ +package org.schabi.newpipe.extractor.stream; + +import org.schabi.newpipe.extractor.MediaFormat; + +import java.io.Serializable; +import java.util.Locale; + +public class SubtitlesStream extends Stream implements Serializable { + private final MediaFormat format; + private final Locale locale; + private final String url; + private final boolean autoGenerated; + private final String code; + + public SubtitlesStream(MediaFormat format, String languageCode, String url, boolean autoGenerated) { + super(url, format); + + /* + * Locale.forLanguageTag only for API >= 21 + * Locale.Builder only for API >= 21 + * Country codes doesn't work well without + */ + final String[] splits = languageCode.split("-"); + switch (splits.length) { + default: + this.locale = new Locale(splits[0]); + break; + case 3: + this.locale = new Locale(splits[0], splits[1], splits[2]);// complex variants doesn't work! + break; + case 2: + this.locale = new Locale(splits[0], splits[1]); + break; + } + this.code = languageCode; + this.format = format; + this.url = url; + this.autoGenerated = autoGenerated; + } + + public String getExtension() { + return format.suffix; + } + + public String getURL() { + return url; + } + + public boolean isAutoGenerated() { + return autoGenerated; + } + + @Override + public boolean equalStats(Stream cmp) { + return super.equalStats(cmp)&& + cmp instanceof SubtitlesStream && + code.equals(((SubtitlesStream) cmp).code) && + autoGenerated == ((SubtitlesStream) cmp).autoGenerated; + } + + public String getDisplayLanguageName() { + return locale.getDisplayName(locale); + } + + public String getLanguageTag() { + return code; + } + + public Locale getLocale() { + return locale; + } + +}