add support for segmented streams

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -152,6 +152,9 @@ public class StreamInfo extends Info {
                 streamInfo.getVideoOnlyStreams().addAll(result.getVideoOnlyStreams());
                 streamInfo.getAudioStreams().addAll(result.getAudioStreams());
                 streamInfo.getVideoStreams().addAll(result.getVideoStreams());
+                streamInfo.segmentedVideoOnlyStreams = result.getSegmentedVideoOnlyStreams();
+                streamInfo.segmentedAudioStreams = result.getSegmentedAudioStreams();
+                streamInfo.segmentedVideoStreams = result.getSegmentedVideoStreams();
             } catch (Exception e) {
                 // Sometimes we receive 403 (forbidden) error when trying to download the manifest (similar to what happens with youtube-dl),
                 // just skip the exception (but store it somewhere), as we later check if we have streams anyway.
@@ -271,6 +274,11 @@ public class StreamInfo extends Info {
     private List<VideoStream> videoOnlyStreams;
 
     private String dashMpdUrl;
+    private List<VideoStream> segmentedVideoStreams;
+    private List<AudioStream> segmentedAudioStreams;
+    private List<VideoStream> segmentedVideoOnlyStreams;
+
+
     private String hlsUrl;
     private StreamInfoItem nextVideo;
     private List<InfoItem> relatedStreams;
@@ -431,6 +439,30 @@ public class StreamInfo extends Info {
         this.dashMpdUrl = dashMpdUrl;
     }
 
+    public List<VideoStream> getSegmentedVideoStreams() {
+        return segmentedVideoStreams;
+    }
+
+    public void setSegmentedVideoStreams(List<VideoStream> segmentedVideoStreams) {
+        this.segmentedVideoStreams = segmentedVideoStreams;
+    }
+
+    public List<AudioStream> getSegmentedAudioStreams() {
+        return segmentedAudioStreams;
+    }
+
+    public void setSegmentedAudioStreams(List<AudioStream> segmentedAudioStreams) {
+        this.segmentedAudioStreams = segmentedAudioStreams;
+    }
+
+    public List<VideoStream> getSegmentedVideoOnlyStreams() {
+        return segmentedVideoOnlyStreams;
+    }
+
+    public void setSegmentedVideoOnlyStreams(List<VideoStream> segmentedVideoOnlyStreams) {
+        this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams;
+    }
+
     public String getHlsUrl() {
         return hlsUrl;
     }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java

@@ -59,10 +59,23 @@ public class DashMpdParser {
         private final List<AudioStream> audioStreams;
         private final List<VideoStream> videoOnlyStreams;
 
-        public ParserResult(List<VideoStream> videoStreams, List<AudioStream> audioStreams, List<VideoStream> videoOnlyStreams) {
+        private final List<VideoStream> segmentedVideoStreams;
+        private final List<AudioStream> segmentedAudioStreams;
+        private final List<VideoStream> segmentedVideoOnlyStreams;
+
+
+        public ParserResult(List<VideoStream> videoStreams,
+                            List<AudioStream> audioStreams,
+                            List<VideoStream> videoOnlyStreams,
+                            List<VideoStream> segmentedVideoStreams,
+                            List<AudioStream> segmentedAudioStreams,
+                            List<VideoStream> segmentedVideoOnlyStreams) {
             this.videoStreams = videoStreams;
             this.audioStreams = audioStreams;
             this.videoOnlyStreams = videoOnlyStreams;
+            this.segmentedVideoStreams = segmentedVideoStreams;
+            this.segmentedAudioStreams = segmentedAudioStreams;
+            this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams;
         }
 
         public List<VideoStream> getVideoStreams() {
@@ -76,10 +89,22 @@ public class DashMpdParser {
         public List<VideoStream> getVideoOnlyStreams() {
             return videoOnlyStreams;
         }
+
+        public List<VideoStream> getSegmentedVideoStreams() {
+            return segmentedVideoStreams;
+        }
+
+        public List<AudioStream> getSegmentedAudioStreams() {
+            return segmentedAudioStreams;
+        }
+
+        public List<VideoStream> getSegmentedVideoOnlyStreams() {
+            return segmentedVideoOnlyStreams;
+        }
     }
 
     /**
-     * Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest,
+     * Will try to download (using {@link StreamInfo#getDashMpdUrl()}) and parse the dash manifest,
      * then it will search for any stream that the ItagItem has (by the id).
      * <p>
      * It has video, video only and audio streams and will only add to the list if it don't
@@ -90,7 +115,8 @@ public class DashMpdParser {
      *
      * @param streamInfo where the parsed streams will be added
      */
-    public static ParserResult getStreams(final StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
+    public static ParserResult getStreams(final StreamInfo streamInfo)
+            throws DashMpdParsingException, ReCaptchaException {
         String dashDoc;
         Downloader downloader = NewPipe.getDownloader();
         try {
@@ -113,6 +139,10 @@ public class DashMpdParser {
             final List<AudioStream> audioStreams = new ArrayList<>();
             final List<VideoStream> videoOnlyStreams = new ArrayList<>();
 
+            final List<VideoStream> segmentedVideoStreams = new ArrayList<>();
+            final List<AudioStream> segmentedAudioStreams = new ArrayList<>();
+            final List<VideoStream> segmentedVideoOnlyStreams = new ArrayList<>();
+
             for (int i = 0; i < representationList.getLength(); i++) {
                 final Element representation = (Element) representationList.item(i);
                 try {
@@ -126,34 +156,61 @@ public class DashMpdParser {
                     // instead we need to add the "media=" value from the <SegementURL/> tags inside the <SegmentList/>
                     // tag in order to get a full working url. However each of these is just pointing to a part of the
                     // video, so we can not return a URL with a working stream here.
-                    // We decided not to ignore such streams for the moment.
+                    // Instead of putting those streams into the list of regular stream urls wie put them in a
-                    if (itag != null && segmentationList == null) {
+                    // for example "segmentedVideoStreams" list.
+                    if (itag != null) {
                         final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
 
                         if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
-                            final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate);
+                            if(segmentationList == null) {
-
+                                final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate);
-                            if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) {
+                                if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) {
-                                audioStreams.add(audioStream);
+                                    audioStreams.add(audioStream);
+                                }
+                            } else {
+                                segmentedAudioStreams.add(
+                                        new AudioStream(id, mediaFormat, itag.avgBitrate));
                             }
                         } else {
                             boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
-                            final VideoStream videoStream = new VideoStream(url, mediaFormat, itag.resolutionString, isVideoOnly);
 
-                            if (isVideoOnly) {
+                            if(segmentationList == null) {
-                                if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) {
+                                final VideoStream videoStream = new VideoStream(url,
-                                    streamInfo.getVideoOnlyStreams().add(videoStream);
+                                        mediaFormat,
-                                    videoOnlyStreams.add(videoStream);
+                                        itag.resolutionString,
+                                        isVideoOnly);
+
+                                if (isVideoOnly) {
+                                    if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) {
+                                        videoOnlyStreams.add(videoStream);
+                                    }
+                                } else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) {
+                                    videoStreams.add(videoStream);
+                                }
+                            } else {
+                                final VideoStream videoStream = new VideoStream(id,
+                                        mediaFormat,
+                                        itag.resolutionString,
+                                        isVideoOnly);
+
+                                if(isVideoOnly) {
+                                    segmentedVideoOnlyStreams.add(videoStream);
+                                } else {
+                                    segmentedVideoStreams.add(videoStream);
                                 }
-                            } else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) {
-                                videoStreams.add(videoStream);
                             }
                         }
                     }
                 } catch (Exception ignored) {
                 }
             }
-            return new ParserResult(videoStreams, audioStreams, videoOnlyStreams);
+            return new ParserResult(
+                    videoStreams,
+                    audioStreams,
+                    videoOnlyStreams,
+                    segmentedVideoStreams,
+                    segmentedAudioStreams,
+                    segmentedVideoOnlyStreams);
         } catch (Exception e) {
             throw new DashMpdParsingException("Could not parse Dash mpd", e);
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java

@@ -47,15 +47,21 @@ public class YoutubeStreamExtractorDASHTest {
 
     @Test
     public void testGetDashMpd() {
-        System.out.println(info.getDashMpdUrl());
         assertTrue(info.getDashMpdUrl(),
                 info.getDashMpdUrl() != null && !info.getDashMpdUrl().isEmpty());
     }
 
     @Test
-    public void testDashMpdParser() {
+    public void testRegularStreams() {
         assertEquals(0, info.getAudioStreams().size());
         assertEquals(0, info.getVideoOnlyStreams().size());
         assertEquals(4, info.getVideoStreams().size());
     }
+
+    @Test
+    public void testSegmentedStreams() {
+        assertEquals(2, info.getSegmentedAudioStreams().size());
+        assertEquals(3, info.getSegmentedVideoOnlyStreams().size());
+        assertEquals(0, info.getSegmentedVideoStreams().size());
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java

@@ -28,7 +28,7 @@ public class YoutubeSearchCountTest {
         public void testViewCount() {
             ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0);
             assertTrue("Count does not fit: " + Long.toString(ci.getSubscriberCount()),
-                    65043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 68043316);
+                    69043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 73043316);
         }
     }
 }