Merge pull request #796 from FireMasterK/streaminfo-subscriber-count

Add support for extracting channel subscriber count in StreamInfo

extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java

@@ -28,6 +28,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
 
+    public static final long UNKNOWN_SUBSCRIBER_COUNT = -1;
+
     public ChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
         super(service, linkHandler);
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -89,9 +89,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         // we couldn't get information about the channel associated with this URL, if there is one.
         if (!channelId[0].equals("channel")) {
             final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
-                    getExtractorLocalization(), getExtractorContentCountry())
+                            getExtractorLocalization(), getExtractorContentCountry())
-                    .value("url", "https://www.youtube.com/" + channelPath)
+                            .value("url", "https://www.youtube.com/" + channelPath)
-                    .done())
+                            .done())
                     .getBytes(UTF_8);
 
             final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
@@ -104,8 +104,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
                     throw new ContentNotAvailableException("This channel doesn't exist.");
                 } else {
                     throw new ContentNotAvailableException("Got error:\""
-                        + errorJsonObject.getString("status") + "\": "
+                            + errorJsonObject.getString("status") + "\": "
-                        + errorJsonObject.getString("message"));
+                            + errorJsonObject.getString("message"));
                 }
             }
 
@@ -136,10 +136,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         int level = 0;
         while (level < 3) {
             final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
-                    getExtractorLocalization(), getExtractorContentCountry())
+                            getExtractorLocalization(), getExtractorContentCountry())
-                    .value("browseId", id)
+                            .value("browseId", id)
-                    .value("params", "EgZ2aWRlb3M%3D") // Equal to videos
+                            .value("params", "EgZ2aWRlb3M%3D") // Equal to videos
-                    .done())
+                            .done())
                     .getBytes(UTF_8);
 
             final JsonObject jsonResponse = getJsonPostResponse("browse", body,
@@ -273,15 +273,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     public long getSubscriberCount() throws ParsingException {
         final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header")
                 .getObject("c4TabbedHeaderRenderer");
-        if (c4TabbedHeaderRenderer.has("subscriberCountText")) {
+        if (!c4TabbedHeaderRenderer.has("subscriberCountText")) {
-            try {
+            return UNKNOWN_SUBSCRIBER_COUNT;
-                return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer
+        }
-                        .getObject("subscriberCountText")));
+        try {
-            } catch (final NumberFormatException e) {
+            return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer
-                throw new ParsingException("Could not get subscriber count", e);
+                    .getObject("subscriberCountText")));
-            }
+        } catch (final NumberFormatException e) {
-        } else {
+            throw new ParsingException("Could not get subscriber count", e);
-            return ITEM_COUNT_UNKNOWN;
         }
     }
 
@@ -385,9 +384,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
                 .getString("token");
 
         final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
-                getExtractorContentCountry())
+                        getExtractorContentCountry())
-                .value("continuation", continuation)
+                        .value("continuation", continuation)
-                .done())
+                        .done())
                 .getBytes(UTF_8);
 
         return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelIds, null, body);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -180,7 +180,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             // TODO: this parses English formatted dates only, we need a better approach to parse
             //  the textual date
             final LocalDate localDate = LocalDate.parse(getTextFromObject(
-                    getVideoPrimaryInfoRenderer().getObject("dateText")),
+                            getVideoPrimaryInfoRenderer().getObject("dateText")),
                     DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
             return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
         } catch (final Exception ignored) {
@@ -437,6 +437,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         return fixThumbnailUrl(url);
     }
 
+    @Override
+    public long getUploaderSubscriberCount() throws ParsingException {
+        final JsonObject videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer, "owner.videoOwnerRenderer");
+        if (!videoOwnerRenderer.has("subscriberCountText")) {
+            return UNKNOWN_SUBSCRIBER_COUNT;
+        }
+        try {
+            return Utils.mixedNumberWordToLong(getTextFromObject(videoOwnerRenderer.getObject("subscriberCountText")));
+        } catch (final NumberFormatException e) {
+            throw new ParsingException("Could not get uploader subscriber count", e);
+        }
+    }
+
     @Nonnull
     @Override
     public String getDashMpdUrl() throws ParsingException {
@@ -666,9 +679,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         final Localization localization = getExtractorLocalization();
         final ContentCountry contentCountry = getExtractorContentCountry();
         final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
-                localization, contentCountry)
+                        localization, contentCountry)
-                .value("videoId", videoId)
+                        .value("videoId", videoId)
-                .done())
+                        .done())
                 .getBytes(UTF_8);
 
         // Put the sts string if we already know it so we don't have to fetch again the player
@@ -717,8 +730,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
         if (ageRestricted) {
             final byte[] ageRestrictedBody = JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(
-                    localization, contentCountry, videoId)
+                            localization, contentCountry, videoId)
-                    .done())
+                            .done())
                     .getBytes(UTF_8);
             nextResponse = getJsonPostResponse("next", ageRestrictedBody, localization);
         } else {
@@ -800,7 +813,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                                               final String videoId)
             throws IOException, ExtractionException {
         final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder(
-                localization, contentCountry)
+                        localization, contentCountry)
                         .value("videoId", videoId)
                         .done())
                 .getBytes(UTF_8);
@@ -873,8 +886,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                                              final String videoId)
             throws IOException, ExtractionException {
         final byte[] androidMobileEmbedBody = JsonWriter.string(
-                prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
+                        prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
-                        .done())
+                                .done())
                 .getBytes(UTF_8);
         final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player",
                 androidMobileEmbedBody, contentCountry, localization);

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -47,6 +47,7 @@ import java.util.Locale;
 public abstract class StreamExtractor extends Extractor {
 
     public static final int NO_AGE_LIMIT = 0;
+    public static final long UNKNOWN_SUBSCRIBER_COUNT = -1;
 
     public StreamExtractor(StreamingService service, LinkHandler linkHandler) {
         super(service, linkHandler);
@@ -202,6 +203,17 @@ public abstract class StreamExtractor extends Extractor {
         return false;
     }
 
+    /**
+     * The subscriber count of the uploader.
+     * If the subscriber count is not implemented, or is unavailable, return <code>-1</code>.
+     *
+     * @return the subscriber count of the uploader or {@value UNKNOWN_SUBSCRIBER_COUNT} if not available
+     * @throws ParsingException
+     */
+    public long getUploaderSubscriberCount() throws ParsingException {
+        return UNKNOWN_SUBSCRIBER_COUNT;
+    }
+
     /**
      * The url to the image file/profile picture/avatar of the creator/uploader of the stream.
      * If the url is not available you can return an empty String.

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -235,6 +235,11 @@ public class StreamInfo extends Info {
         } catch (Exception e) {
             streamInfo.addError(e);
         }
+        try {
+            streamInfo.setUploaderSubscriberCount(extractor.getUploaderSubscriberCount());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
 
         try {
             streamInfo.setSubChannelName(extractor.getSubChannelName());
@@ -367,6 +372,7 @@ public class StreamInfo extends Info {
     private String uploaderUrl = "";
     private String uploaderAvatarUrl = "";
     private boolean uploaderVerified = false;
+    private long uploaderSubscriberCount = -1;
 
     private String subChannelName = "";
     private String subChannelUrl = "";
@@ -540,6 +546,14 @@ public class StreamInfo extends Info {
         this.uploaderVerified = uploaderVerified;
     }
 
+    public long getUploaderSubscriberCount() {
+        return uploaderSubscriberCount;
+    }
+
+    public void setUploaderSubscriberCount(long uploaderSubscriberCount) {
+        this.uploaderSubscriberCount = uploaderSubscriberCount;
+    }
+
     public String getSubChannelName() {
         return subChannelName;
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java

@@ -5,6 +5,7 @@ public interface BaseStreamExtractorTest extends BaseExtractorTest {
     void testUploaderName() throws Exception;
     void testUploaderUrl() throws Exception;
     void testUploaderAvatarUrl() throws Exception;
+    void testSubscriberCount() throws Exception;
     void testSubChannelName() throws Exception;
     void testSubChannelUrl() throws Exception;
     void testSubChannelAvatarUrl() throws Exception;

extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java

@@ -34,6 +34,7 @@ import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEqualsOrderInd
 import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl;
 import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems;
+import static org.schabi.newpipe.extractor.stream.StreamExtractor.UNKNOWN_SUBSCRIBER_COUNT;
 
 /**
  * Test for {@link StreamExtractor}
@@ -45,6 +46,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
     public abstract String expectedUploaderName();
     public abstract String expectedUploaderUrl();
     public boolean expectedUploaderVerified() { return false; }
+    public long expectedUploaderSubscriberCountAtLeast() { return UNKNOWN_SUBSCRIBER_COUNT; }
     public String expectedSubChannelName() { return ""; } // default: there is no subchannel
     public String expectedSubChannelUrl() { return ""; } // default: there is no subchannel
     public boolean expectedDescriptionIsEmpty() { return false; } // default: description is not empty
@@ -105,6 +107,16 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
         assertEquals(expectedUploaderVerified(), extractor().isUploaderVerified());
     }
 
+    @Test
+    @Override
+    public void testSubscriberCount() throws Exception {
+        if (expectedUploaderSubscriberCountAtLeast() == UNKNOWN_SUBSCRIBER_COUNT) {
+            assertEquals(UNKNOWN_SUBSCRIBER_COUNT, extractor().getUploaderSubscriberCount());
+        } else {
+            assertGreaterOrEqual(expectedUploaderSubscriberCountAtLeast(), extractor().getUploaderSubscriberCount());
+        }
+    }
+
     @Test
     @Override
     public void testSubChannelName() throws Exception {

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -45,6 +45,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
     @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
     @Override public String expectedUploaderName() { return "DAN TV"; }
     @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCcQHIVL83g5BEQe2IJFb-6w"; }
+    @Override public long expectedUploaderSubscriberCountAtLeast() { return 50; }
     @Override public boolean expectedUploaderVerified() { return false; }
     @Override public boolean expectedDescriptionIsEmpty() { return true; }
     @Override public List<String> expectedDescriptionContains() { return Collections.emptyList(); }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -49,6 +49,7 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
     @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
     @Override public String expectedUploaderName() { return "Amazing Atheist"; }
     @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCjNxszyFPasDdRoD9J6X-sw"; }
+    @Override public long expectedUploaderSubscriberCountAtLeast() { return 977_000; }
     @Override public List<String> expectedDescriptionContains() {
         return Arrays.asList("http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html",
                 "freedom");

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java

@@ -138,6 +138,7 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
         @Override public String expectedUploaderName() { return "PewDiePie"; }
         @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw"; }
+        @Override public long expectedUploaderSubscriberCountAtLeast() { return 110_000_000; }
         @Override public List<String> expectedDescriptionContains() {
             return Arrays.asList("https://www.youtube.com/channel/UC7l23W7gFi4Uho6WSzckZRA",
                     "https://www.handcraftpictures.com/");
@@ -182,6 +183,7 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
         @Override public String expectedUploaderName() { return "Unbox Therapy"; }
         @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCsTcErHg8oDvUnTzoqsYeNw"; }
+        @Override public long expectedUploaderSubscriberCountAtLeast() { return 18_000_000; }
         @Override public List<String> expectedDescriptionContains() {
             return Arrays.asList("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
                     "https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
@@ -274,6 +276,7 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
         @Override public String expectedUploaderName() { return "tagesschau"; }
         @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC5NOEUbkLheQcaaRldYW5GA"; }
+        @Override public long expectedUploaderSubscriberCountAtLeast() { return 1_000_000; }
         @Override public boolean expectedUploaderVerified() { return true; }
         @Override public List<String> expectedDescriptionContains() {
             return Arrays.asList("Themen der Sendung", "07:15", "Wetter", "Sendung nachträglich bearbeitet");
@@ -336,6 +339,7 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
         @Override public String expectedUploaderName() { return "maiLab"; }
         @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCyHDQ5C6z1NDmJ4g6SerW8g"; }
+        @Override public long expectedUploaderSubscriberCountAtLeast() { return 1_400_000; }
         @Override public List<String> expectedDescriptionContains()  {return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");}
         @Override public boolean expectedUploaderVerified() { return true; }
         @Override public long expectedLength() { return 1010; }
@@ -405,6 +409,7 @@ public class YoutubeStreamExtractorDefaultTest {
         @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
         @Override public String expectedUploaderName() { return "Dinge Erklärt – Kurzgesagt"; }
         @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCwRH985XgMYXQ6NxXDo8npw"; }
+        @Override public long expectedUploaderSubscriberCountAtLeast() { return 1_500_000; }
         @Override public List<String> expectedDescriptionContains() { return Arrays.asList("Lasst uns abtauchen!", "Angebot von funk", "Dinge"); }
         @Override public long expectedLength() { return 631; }
         @Override public long expectedTimestamp() { return TIMESTAMP; }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java

@@ -52,6 +52,7 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
     @Override public StreamType expectedStreamType() { return StreamType.LIVE_STREAM; }
     @Override public String expectedUploaderName() { return "Lofi Girl"; }
     @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow"; }
+    @Override public long expectedUploaderSubscriberCountAtLeast() { return 9_800_000; }
     @Override public List<String> expectedDescriptionContains() {
         return Arrays.asList("Lofi Girl merch",
                 "Thank you for listening, I hope you will have a good time here");

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java

@@ -45,6 +45,7 @@ public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTe
     @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
     @Override public String expectedUploaderName() { return "Hooked"; }
     @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCPysfiuOv4VKBeXFFPhKXyw"; }
+    @Override public long expectedUploaderSubscriberCountAtLeast() { return 24_300; }
     @Override public List<String> expectedDescriptionContains() {
         return Arrays.asList("https://www.youtube.com/user/Roccowschiptune",
                 "https://www.facebook.com/HookedMagazinDE");