diff --git a/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java b/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java index 2b2618e35..3effd29d4 100644 --- a/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java +++ b/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java @@ -44,23 +44,10 @@ public class SearchResult { this.errors = Collections.unmodifiableList(new ArrayList<>(errors)); } - public static SearchResult getSearchResult(SearchEngine engine, String query, int page, String languageCode, SearchEngine.Filter filter) + public static SearchResult getSearchResult(@Nonnull final SearchEngine engine, final String query, final int page, + final String languageCode, final SearchEngine.Filter filter) throws IOException, ExtractionException { - - SearchResult result = engine - .search(query, page, languageCode, filter) - .getSearchResult(); - if (result.resultList.isEmpty()) { - if (result.suggestion.isEmpty()) { - if (result.errors.isEmpty()) { - throw new ExtractionException("Empty result despite no error"); - } - } else { - // This is used as a fallback. Do not relay on it !!! - throw new SearchEngine.NothingFoundException(result.suggestion); - } - } - return result; + return engine.search(query, page, languageCode, filter).getSearchResult(); } public String getSuggestion() { diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 28353c97a..6a14f3b13 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -116,9 +116,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return SoundcloudParsingHelper.getAvatarUrl(track); } + @Nonnull @Override public String getDashMpdUrl() { - return null; + return ""; + } + + @Nonnull + @Override + public String getHlsUrl() throws ParsingException { + return ""; } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 346ea1184..c5e4c5f95 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -66,12 +66,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - public class LiveStreamException extends ContentNotAvailableException { - LiveStreamException(String message) { - super(message); - } - } - public class SubtitlesException extends ContentNotAvailableException { SubtitlesException(String message, Throwable cause) { super(message, cause); @@ -338,6 +332,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + @Nonnull @Override public String getDashMpdUrl() throws ParsingException { assertPageFetched(); @@ -365,6 +360,24 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + @Nonnull + @Override + public String getHlsUrl() throws ParsingException { + assertPageFetched(); + try { + String hlsvp; + if (playerArgs != null && playerArgs.isString("hlsvp")) { + hlsvp = playerArgs.getString("hlsvp", ""); + } else { + return ""; + } + + return hlsvp; + } catch (Exception e) { + throw new ParsingException("Could not get hls manifest url", e); + } + } + @Override public List getAudioStreams() throws IOException, ExtractionException { assertPageFetched(); @@ -428,7 +441,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override @Nonnull public List getSubtitlesDefault() throws IOException, ExtractionException { - return getSubtitles(SubtitlesFormat.VTT); + return getSubtitles(SubtitlesFormat.TTML); } @Override @@ -444,7 +457,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamType getStreamType() throws ParsingException { - //todo: if implementing livestream support this value should be generated dynamically + assertPageFetched(); + try { + if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") || + playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) { + return StreamType.LIVE_STREAM; + } + } catch (Exception e) { + throw new ParsingException("Could not get hls manifest url", e); + } return StreamType.VIDEO_STREAM; } @@ -517,13 +538,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String CONTENT = "content"; private static final String DECRYPTION_FUNC_NAME = "decrypt"; + private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999"; + private volatile String decryptionCode = ""; private String pageHtml = null; - private String getPageHtml(Downloader downloader) throws IOException, ExtractionException{ + private String getPageHtml(Downloader downloader) throws IOException, ExtractionException { + final String verifiedUrl = getCleanUrl() + VERIFIED_URL_PARAMS; if (pageHtml == null) { - pageHtml = downloader.download(getCleanUrl()); + pageHtml = downloader.download(verifiedUrl); } return pageHtml; } @@ -534,7 +558,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { doc = Jsoup.parse(pageContent, getCleanUrl()); final String playerUrl; - // TODO: use embedded videos to fetch DASH manifest for all videos // Check if the video is age restricted if (pageContent.contains(" related_streams; diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java new file mode 100644 index 000000000..de8fdca6a --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java @@ -0,0 +1,125 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.SubtitlesFormat; +import org.schabi.newpipe.extractor.stream.VideoStream; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +/** + * Test for {@link YoutubeStreamUrlIdHandler} + */ +public class YoutubeStreamExtractorControversialTest { + private static YoutubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = (YoutubeStreamExtractor) YouTube + .getStreamExtractor("https://www.youtube.com/watch?v=T4XJQO3qol8"); + extractor.fetchPage(); + } + + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(extractor.getTimeStamp() + "", extractor.getTimeStamp() <= 0); + } + + @Test + public void testGetValidTimeStamp() throws IOException, ExtractionException { + StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174"); + assertEquals(extractor.getTimeStamp() + "", "174"); + } + + @Test + @Ignore + public void testGetAgeLimit() throws ParsingException { + assertEquals(18, extractor.getAgeLimit()); + } + + @Test + public void testGetName() throws ParsingException { + assertNotNull("name is null", extractor.getName()); + assertFalse("name is empty", extractor.getName().isEmpty()); + } + + @Test + public void testGetDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + assertFalse(extractor.getDescription().isEmpty()); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertNotNull(extractor.getUploaderName()); + assertFalse(extractor.getUploaderName().isEmpty()); + } + + @Ignore // Currently there is no way get the length from restricted videos + @Test + public void testGetLength() throws ParsingException { + assertTrue(extractor.getLength() > 0); + } + + @Test + public void testGetViews() throws ParsingException { + assertTrue(extractor.getViewCount() > 0); + } + + @Test + public void testGetUploadDate() throws ParsingException { + assertTrue(extractor.getUploadDate().length() > 0); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + // FIXME: 25.11.17 Are there no streams or are they not listed? + @Ignore + @Test + public void testGetAudioStreams() throws IOException, ExtractionException { + // audio streams are not always necessary + assertFalse(extractor.getAudioStreams().isEmpty()); + } + + @Test + public void testGetVideoStreams() throws IOException, ExtractionException { + List streams = new ArrayList<>(); + streams.addAll(extractor.getVideoStreams()); + streams.addAll(extractor.getVideoOnlyStreams()); + assertTrue(streams.size() > 0); + } + + + @Test + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(!extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetSubtitlesList() throws IOException, ExtractionException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(!extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty()); + } +}