Merge branch 'dev' into peertube

.travis.yml

@@ -1,7 +1,10 @@
 language: java
+jdk:
+  - openjdk8
 
-script: ./gradlew check
+script:
-after_success: ./gradlew aggregatedJavadocs
+  - ./gradlew check
+  - ./gradlew aggregatedJavadocs
 
 deploy:
   provider: pages

build.gradle

@@ -5,7 +5,7 @@ allprojects {
     sourceCompatibility = 1.7
     targetCompatibility = 1.7
 
-    version 'v0.13.0'
+    version 'v0.18.0'
     group 'com.github.TeamNewPipe'
 
     repositories {
@@ -43,7 +43,8 @@ task aggregatedJavadocs(type: Javadoc, group: 'Documentation') {
     title = "$project.name $version"
     // options.memberLevel = JavadocMemberLevel.PRIVATE
     options.links 'https://docs.oracle.com/javase/7/docs/api/'
-    
+    options.encoding 'UTF-8'
+
     subprojects.each { project ->
         project.tasks.withType(Javadoc).each { javadocTask ->
             source += javadocTask.source

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java

@@ -34,7 +34,7 @@ public abstract class LinkHandlerFactory {
     public abstract String getUrl(String id) throws ParsingException;
     public abstract boolean onAcceptUrl(final String url) throws ParsingException;
 
-    public String getUrl(String id, String baseUrl) throws ParsingException{
+    public String getUrl(String id, String baseUrl) throws ParsingException {
        return getUrl(id);
     }
     
@@ -43,13 +43,14 @@ public abstract class LinkHandlerFactory {
     ///////////////////////////////////
 
     public LinkHandler fromUrl(String url) throws ParsingException {
+        if (url == null) throw new IllegalArgumentException("url can not be null");
         final String baseUrl = Utils.getBaseUrl(url);
         return fromUrl(url, baseUrl);
     }
     
     public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException {
-        if(url == null) throw new IllegalArgumentException("url can not be null");
+        if (url == null) throw new IllegalArgumentException("url can not be null");
-        if(!acceptUrl(url)) {
+        if (!acceptUrl(url)) {
             throw new ParsingException("Malformed unacceptable url: " + url);
         }
 
@@ -58,13 +59,13 @@ public abstract class LinkHandlerFactory {
     }
     
     public LinkHandler fromId(String id) throws ParsingException {
-        if(id == null) throw new IllegalArgumentException("id can not be null");
+        if (id == null) throw new IllegalArgumentException("id can not be null");
         final String url = getUrl(id);
         return new LinkHandler(url, url, id);
     }
 
     public LinkHandler fromId(String id, String baseUrl) throws ParsingException {
-        if(id == null) throw new IllegalArgumentException("id can not be null");
+        if (id == null) throw new IllegalArgumentException("id can not be null");
         final String url = getUrl(id, baseUrl);
         return new LinkHandler(url, url, id);
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -106,19 +106,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public String getName() throws ParsingException {
         assertPageFetched();
-        String name = getStringFromMetaData("title");
+        try {
-        if(name == null) {
+            return playerResponse.getObject("videoDetails").getString("title");
-            // Fallback to HTML method
+
+        } catch (Exception e) {
+            // fallback HTML method
+            String name = null;
             try {
                 name = doc.select("meta[name=title]").attr(CONTENT);
-            } catch (Exception e) {
+            } catch (Exception ignored) {}
-                throw new ParsingException("Could not get the title", e);
+
+            if (name == null) {
+                throw new ParsingException("Could not get name", e);
             }
+            return name;
         }
-        if(name == null || name.isEmpty()) {
-            throw new ParsingException("Could not get the title");
-        }
-        return name;
     }
 
     @Override
@@ -128,9 +130,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
 
         try {
-            return doc.select("meta[itemprop=datePublished]").attr(CONTENT);
+            return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate");
-        } catch (Exception e) {//todo: add fallback method
+        } catch (Exception e) {
-            throw new ParsingException("Could not get upload date", e);
+            String uploadDate = null;
+            try {
+                uploadDate = doc.select("meta[itemprop=datePublished]").attr(CONTENT);
+            } catch (Exception ignored) {}
+
+            if (uploadDate == null) {
+                throw new ParsingException("Could not get upload date", e);
+            }
+            return uploadDate;
         }
     }
 
@@ -149,24 +159,23 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public String getThumbnailUrl() throws ParsingException {
         assertPageFetched();
-        // Try to get high resolution thumbnail first, if it fails, use low res from the player instead
         try {
-            return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
+            JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
-        } catch (Exception ignored) {
+            // the last thumbnail is the one with the highest resolution
-            // Try other method...
+            return thumbnails.getObject(thumbnails.size() - 1).getString("url");
-        }
 
-        try {
-            if (playerArgs != null && playerArgs.isString("thumbnail_url")) return playerArgs.getString("thumbnail_url");
-        } catch (Exception ignored) {
-            // Try other method...
-        }
-
-        try {
-            return videoInfoPage.get("thumbnail_url");
         } catch (Exception e) {
-            throw new ParsingException("Could not get thumbnail url", e);
+            String url = null;
+            try {
+                url = doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
+            } catch (Exception ignored) {}
+
+            if (url == null) {
+                throw new ParsingException("Could not get thumbnail url", e);
+            }
+            return url;
         }
+
     }
 
     @Nonnull
@@ -174,9 +183,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public String getDescription() throws ParsingException {
         assertPageFetched();
         try {
+            // first try to get html-formatted description
             return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html());
         } catch (Exception e) {
-            throw new ParsingException("Could not get the description", e);
+            try {
+                // fallback to raw non-html description
+                return playerResponse.getObject("videoDetails").getString("shortDescription");
+            } catch (Exception ignored) {
+                throw new ParsingException("Could not get the description", e);
+            }
         }
     }
 
@@ -269,25 +284,22 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public long getLength() throws ParsingException {
         assertPageFetched();
 
-        // try getting duration from playerargs
-        try {
-            String durationMs = playerResponse
-                    .getObject("streamingData")
-                    .getArray("formats")
-                    .getObject(0)
-                    .getString("approxDurationMs");
-            return Long.parseLong(durationMs)/1000;
-        } catch (Exception e) {
-        }
-
-        //try getting value from age gated video
         try {
             String duration = playerResponse
                     .getObject("videoDetails")
                     .getString("lengthSeconds");
             return Long.parseLong(duration);
         } catch (Exception e) {
-            throw new ParsingException("Every methode to get the duration has failed: ", e);
+            try {
+                String durationMs = playerResponse
+                        .getObject("streamingData")
+                        .getArray("formats")
+                        .getObject(0)
+                        .getString("approxDurationMs");
+                return Math.round(Long.parseLong(durationMs) / 1000f);
+            } catch (Exception ignored) {
+                throw new ParsingException("Could not get duration", e);
+            }
         }
     }
 
@@ -307,11 +319,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         try {
             if (getStreamType().equals(StreamType.LIVE_STREAM)) {
                 return getLiveStreamWatchingCount();
+            } else {
+                return Long.parseLong(playerResponse.getObject("videoDetails").getString("viewCount"));
+            }
+        } catch (Exception e) {
+            try {
+                return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
+            } catch (Exception ignored) {
+                throw new ParsingException("Could not get view count", e);
             }
-
-            return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
-        } catch (Exception e) {//todo: find fallback method
-            throw new ParsingException("Could not get number of views", e);
         }
     }
 
@@ -373,7 +389,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             try {
                 likesString = button.select("span.yt-uix-button-content").first().text();
             } catch (NullPointerException e) {
-                //if this kicks in our button has no content and therefore likes/dislikes are disabled
+                //if this kicks in our button has no content and therefore ratings must be disabled
+                if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
+                    throw new ParsingException("Ratings are enabled even though the like button is missing", e);
+                }
                 return -1;
             }
             return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
@@ -393,7 +412,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             try {
                 dislikesString = button.select("span.yt-uix-button-content").first().text();
             } catch (NullPointerException e) {
-                //if this kicks in our button has no content and therefore likes/dislikes are disabled
+                //if this kicks in our button has no content and therefore ratings must be disabled
+                if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
+                    throw new ParsingException("Ratings are enabled even though the dislike button is missing", e);
+                }
                 return -1;
             }
             return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
@@ -409,60 +431,59 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public String getUploaderUrl() throws ParsingException {
         assertPageFetched();
         try {
-            return doc.select("div[class=\"yt-user-info\"]").first().children()
+            return "https://www.youtube.com/channel/" +
-                    .select("a").first().attr("abs:href");
+                    playerResponse.getObject("videoDetails").getString("channelId");
         } catch (Exception e) {
-            throw new ParsingException("Could not get channel link", e);
+            String uploaderUrl = null;
-        }
+            try {
-    }
+                uploaderUrl = doc.select("div[class=\"yt-user-info\"]").first().children()
+                        .select("a").first().attr("abs:href");
+            } catch (Exception ignored) {}
 
-
+            if (uploaderUrl == null) {
-    @Nullable
+                throw new ParsingException("Could not get channel link", e);
-    private String getStringFromMetaData(String field) {
+            }
-        assertPageFetched();
+            return uploaderUrl;
-        String value = null;
-        if(playerArgs != null) {
-            // This can not fail
-            value = playerArgs.getString(field);
         }
-        if(value == null) {
-            // This can not fail too
-            value = videoInfoPage.get(field);
-        }
-        return value;
     }
 
     @Nonnull
     @Override
     public String getUploaderName() throws ParsingException {
         assertPageFetched();
-        String name = getStringFromMetaData("author");
+        try {
-
+            return playerResponse.getObject("videoDetails").getString("author");
-        if(name == null) {
+        } catch (Exception e) {
+            String name = null;
             try {
-                // Fallback to HTML method
                 name = doc.select("div.yt-user-info").first().text();
-            } catch (Exception e) {
+            } catch (Exception ignored) {}
-                throw new ParsingException("Could not get uploader name", e);
+
+            if (name == null) {
+                throw new ParsingException("Could not get uploader name");
             }
+            return name;
         }
-        if(name == null || name.isEmpty()) {
-            throw new ParsingException("Could not get uploader name");
-        }
-        return name;
     }
 
     @Nonnull
     @Override
     public String getUploaderAvatarUrl() throws ParsingException {
         assertPageFetched();
+
+        String uploaderAvatarUrl = null;
         try {
-            return doc.select("a[class*=\"yt-user-photo\"]").first()
+            uploaderAvatarUrl = doc.select("a[class*=\"yt-user-photo\"]").first()
                     .select("img").first()
                     .attr("abs:data-thumb");
         } catch (Exception e) {//todo: add fallback method
-            throw new ParsingException("Could not get uploader thumbnail URL.", e);
+            throw new ParsingException("Could not get uploader avatar url", e);
         }
+
+        if (uploaderAvatarUrl == null) {
+            throw new ParsingException("Could not get uploader avatar url");
+        }
+        return uploaderAvatarUrl;
     }
 
     @Nonnull
@@ -590,12 +611,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public StreamType getStreamType() throws ParsingException {
         assertPageFetched();
         try {
-            if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
+            if (!playerResponse.getObject("streamingData").has(FORMATS) ||
-                    (!playerResponse.getObject("streamingData").has(FORMATS)))) {
+                    (playerArgs != null && playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))) {
                 return StreamType.LIVE_STREAM;
             }
         } catch (Exception e) {
-            throw new ParsingException("Could not get hls manifest url", e);
+            throw new ParsingException("Could not get stream type", e);
         }
         return StreamType.VIDEO_STREAM;
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java

@@ -89,7 +89,7 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest {
 
     @Test
     public void testGetTextualUploadDate() throws ParsingException {
-        Assert.assertEquals("2018-05-11", extractor.getTextualUploadDate());
+        Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java

@@ -77,8 +77,8 @@ public class YoutubeStreamLinkHandlerFactoryTest {
         assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.Youtube.com/embed/jZViOEv90dI").getId());
         assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI").getId());
         assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId());
-        assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId());
+        assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI", "youtube.com").getId());
-        assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId());
+        assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI", "youtube.com").getId());
         assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId());
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java

@@ -1,36 +0,0 @@
-package org.schabi.newpipe.extractor.services.youtube.search;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.schabi.newpipe.DownloaderTestImpl;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
-
-import static java.util.Collections.singletonList;
-import static junit.framework.TestCase.assertTrue;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-
-/**
- * Test for {@link YoutubeSearchExtractor}
- */
-public class YoutubeSearchCountTest {
-    public static class YoutubeChannelViewCountTest extends YoutubeSearchExtractorBaseTest {
-        @BeforeClass
-        public static void setUpClass() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
-            extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie",
-                    singletonList(YoutubeSearchQueryHandlerFactory.CHANNELS), null);
-            extractor.fetchPage();
-            itemsPage = extractor.getInitialPage();
-        }
-
-        @Test
-        public void testViewCount() {
-            ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0);
-            assertTrue("Count does not fit: " + Long.toString(ci.getSubscriberCount()),
-                    69043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 103043316);
-        }
-    }
-}

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java

@@ -39,7 +39,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
         boolean equals = true;
         for (int i = 0; i < secondPage.getItems().size()
                 && i < itemsPage.getItems().size(); i++) {
-            if(!secondPage.getItems().get(i).getUrl().equals(
+            if (!secondPage.getItems().get(i).getUrl().equals(
                     itemsPage.getItems().get(i).getUrl())) {
                 equals = false;
             }
@@ -58,7 +58,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
     @Test
     public void testOnlyContainChannels() {
         for(InfoItem item : itemsPage.getItems()) {
-            if(!(item instanceof ChannelInfoItem)) {
+            if (!(item instanceof ChannelInfoItem)) {
                 fail("The following item is no channel item: " + item.toString());
             }
         }
@@ -78,4 +78,11 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
             }
         }
     }
+
+    @Test
+    public void testStreamCount() {
+        ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0);
+        assertTrue("Stream count does not fit: " + ci.getStreamCount(),
+                4000 < ci.getStreamCount() && ci.getStreamCount() < 5500);
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -75,7 +75,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
 
     @Test
     public void testGetLength() throws ParsingException {
-        assertEquals(1789, extractor.getLength());
+        assertEquals(1790, extractor.getLength());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -65,7 +65,7 @@ public class YoutubeStreamExtractorControversialTest {
     @Test
     public void testGetDescription() throws ParsingException {
         assertNotNull(extractor.getDescription());
-//        assertFalse(extractor.getDescription().isEmpty());
+        assertFalse(extractor.getDescription().isEmpty());
     }
 
     @Test
@@ -122,13 +122,13 @@ public class YoutubeStreamExtractorControversialTest {
 
     @Test
     public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
-        // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
+        // Video (/view?v=T4XJQO3qol8) set in the setUp() method has at least auto-generated (English) captions
         assertFalse(extractor.getSubtitlesDefault().isEmpty());
     }
 
     @Test
     public void testGetSubtitlesList() throws IOException, ExtractionException {
-        // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
+        // Video (/view?v=T4XJQO3qol8) set in the setUp() method has at least auto-generated (English) captions
         assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java

@@ -101,7 +101,7 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @Test
         public void testGetLength() throws ParsingException {
-            assertEquals(366, extractor.getLength());
+            assertEquals(367, extractor.getLength());
         }
 
         @Test
@@ -124,7 +124,11 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @Test
         public void testGetUploaderUrl() throws ParsingException {
-            assertEquals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw", extractor.getUploaderUrl());
+            String url = extractor.getUploaderUrl();
+            if (!url.equals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw") &&
+                !url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) {
+                fail("Uploader url is neither the music channel one nor the Vevo one");
+            }
         }
 
         @Test
@@ -183,6 +187,18 @@ public class YoutubeStreamExtractorDefaultTest {
             // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
             assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
         }
+
+        @Test
+        public void testGetLikeCount() throws ParsingException {
+            long likeCount = extractor.getLikeCount();
+            assertTrue("" + likeCount, likeCount >= 15000000);
+        }
+
+        @Test
+        public void testGetDislikeCount() throws ParsingException {
+            long dislikeCount = extractor.getDislikeCount();
+            assertTrue("" + dislikeCount, dislikeCount >= 818000);
+        }
     }
 
     public static class DescriptionTestPewdiepie {
@@ -245,6 +261,29 @@ public class YoutubeStreamExtractorDefaultTest {
         }
     }
 
+    public static class RatingsDisabledTest {
+        private static YoutubeStreamExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = (YoutubeStreamExtractor) YouTube
+                    .getStreamExtractor("https://www.youtube.com/watch?v=HRKu0cvrr_o");
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testGetLikeCount() throws ParsingException {
+            assertEquals(-1, extractor.getLikeCount());
+        }
+
+        @Test
+        public void testGetDislikeCount() throws ParsingException {
+            assertEquals(-1, extractor.getDislikeCount());
+        }
+
+    }
+
     public static class FramesTest {
         private static YoutubeStreamExtractor extractor;