Merge pull request #1061 from ChunkyProgrammer/develop/extract-yt-playlist-descrption

Extract playlists description

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java

@@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 import javax.annotation.Nonnull;
@@ -21,6 +22,9 @@ public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> {
 
     public abstract long getStreamCount() throws ParsingException;
 
+    @Nonnull
+    public abstract Description getDescription() throws ParsingException;
+
     @Nonnull
     public String getThumbnailUrl() throws ParsingException {
         return "";

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java

@@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.utils.ExtractorHelper;
 
@@ -102,6 +103,11 @@ public final class PlaylistInfo extends ListInfo<StreamInfoItem> {
         } catch (final Exception e) {
             info.addError(e);
         }
+        try {
+            info.setDescription(extractor.getDescription());
+        } catch (final Exception e) {
+            info.addError(e);
+        }
         try {
             info.setThumbnailUrl(extractor.getThumbnailUrl());
         } catch (final Exception e) {
@@ -174,6 +180,7 @@ public final class PlaylistInfo extends ListInfo<StreamInfoItem> {
     private String subChannelName;
     private String subChannelAvatarUrl;
     private long streamCount = 0;
+    private Description description;
     private PlaylistType playlistType;
 
     public String getThumbnailUrl() {
@@ -248,6 +255,14 @@ public final class PlaylistInfo extends ListInfo<StreamInfoItem> {
         this.streamCount = streamCount;
     }
 
+    public Description getDescription() {
+        return description;
+    }
+
+    public void setDescription(final Description description) {
+        this.description = description;
+    }
+
     public PlaylistType getPlaylistType() {
         return playlistType;
     }

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java

@@ -1,6 +1,7 @@
 package org.schabi.newpipe.extractor.playlist;
 
 import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.stream.Description;
 
 import javax.annotation.Nullable;
 
@@ -13,6 +14,7 @@ public class PlaylistInfoItem extends InfoItem {
      * How many streams this playlist have
      */
     private long streamCount = 0;
+    private Description description;
     private PlaylistInfo.PlaylistType playlistType;
 
     public PlaylistInfoItem(final int serviceId, final String url, final String name) {
@@ -52,6 +54,14 @@ public class PlaylistInfoItem extends InfoItem {
         this.streamCount = streamCount;
     }
 
+    public Description getDescription() {
+        return description;
+    }
+
+    public void setDescription(final Description description) {
+        this.description = description;
+    }
+
     public PlaylistInfo.PlaylistType getPlaylistType() {
         return playlistType;
     }

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java

@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.playlist;
 
 import org.schabi.newpipe.extractor.InfoItemExtractor;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.Description;
 
 import javax.annotation.Nonnull;
 
@@ -31,6 +32,16 @@ public interface PlaylistInfoItemExtractor extends InfoItemExtractor {
      */
     long getStreamCount() throws ParsingException;
 
+    /**
+     * Get the description of the playlist if there is any.
+     * Otherwise, an {@link Description#EMPTY_DESCRIPTION EMPTY_DESCRIPTION} is returned.
+     * @return the playlist's description
+     */
+    @Nonnull
+    default Description getDescription() throws ParsingException {
+        return Description.EMPTY_DESCRIPTION;
+    }
+
     /**
      * @return the type of this playlist, see {@link PlaylistInfo.PlaylistType} for a description
      *         of types. If not overridden always returns {@link PlaylistInfo.PlaylistType#NORMAL}.

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java

@@ -41,6 +41,11 @@ public class PlaylistInfoItemsCollector
         } catch (final Exception e) {
             addError(e);
         }
+        try {
+            resultItem.setDescription(extractor.getDescription());
+        } catch (final Exception e) {
+            addError(e);
+        }
         try {
             resultItem.setPlaylistType(extractor.getPlaylistType());
         } catch (final Exception e) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java

@@ -11,6 +11,8 @@ import com.grack.nanojson.JsonParserException;
 
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -20,10 +22,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampPlaylistStreamInfoItemExtractor;
+import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
 import java.io.IOException;
+import java.util.Objects;
 
 import javax.annotation.Nonnull;
 
@@ -108,6 +112,32 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
         return trackInfo.size();
     }
 
+    @Nonnull
+    @Override
+    public Description getDescription() throws ParsingException {
+        final Element tInfo = document.getElementById("trackInfo");
+        if (tInfo == null) {
+            throw new ParsingException("Could not find trackInfo in document");
+        }
+        final Elements about = tInfo.getElementsByClass("tralbum-about");
+        final Elements credits = tInfo.getElementsByClass("tralbum-credits");
+        final Element license = document.getElementById("license");
+        if (about.isEmpty() && credits.isEmpty() && license == null) {
+            return Description.EMPTY_DESCRIPTION;
+        }
+        final StringBuilder sb = new StringBuilder();
+        if (!about.isEmpty()) {
+            sb.append(Objects.requireNonNull(about.first()).html());
+        }
+        if (!credits.isEmpty()) {
+            sb.append(Objects.requireNonNull(credits.first()).html());
+        }
+        if (license != null) {
+            sb.append(license.html());
+        }
+        return new Description(sb.toString(), Description.HTML);
+    }
+
     @Nonnull
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java

@@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
+import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.Utils;
@@ -65,6 +66,16 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor {
         return playlistInfo.getLong("videosLength");
     }
 
+    @Nonnull
+    @Override
+    public Description getDescription() throws ParsingException {
+        final String description = playlistInfo.getString("description");
+        if (isNullOrEmpty(description)) {
+            return Description.EMPTY_DESCRIPTION;
+        }
+        return new Description(description, Description.PLAIN_TEXT);
+    }
+
     @Nonnull
     @Override
     public String getSubChannelName() {

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java

@@ -4,9 +4,12 @@ import com.grack.nanojson.JsonObject;
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
+import org.schabi.newpipe.extractor.stream.Description;
 
 import javax.annotation.Nonnull;
 
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
 public class PeertubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor  {
 
     final JsonObject item;
@@ -54,4 +57,14 @@ public class PeertubePlaylistInfoItemExtractor implements PlaylistInfoItemExtrac
     public long getStreamCount() throws ParsingException {
         return item.getInt("videosLength");
     }
+
+    @Nonnull
+    @Override
+    public Description getDescription() throws ParsingException {
+        final String description = item.getString("description");
+        if (isNullOrEmpty(description)) {
+            return Description.EMPTY_DESCRIPTION;
+        }
+        return new Description(description, Description.PLAIN_TEXT);
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java

@@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
+import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
@@ -118,6 +119,16 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
         return playlist.getLong("track_count");
     }
 
+    @Nonnull
+    @Override
+    public Description getDescription() throws ParsingException {
+        final String description = playlist.getString("description");
+        if (isNullOrEmpty(description)) {
+            return Description.EMPTY_DESCRIPTION;
+        }
+        return new Description(description, Description.PLAIN_TEXT);
+    }
+
     @Nonnull
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

@@ -31,6 +31,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
@@ -169,6 +170,12 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
         return ListExtractor.ITEM_COUNT_INFINITE;
     }
 
+    @Nonnull
+    @Override
+    public Description getDescription() throws ParsingException {
+        return Description.EMPTY_DESCRIPTION;
+    }
+
     @Nonnull
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage()

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -26,6 +26,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.Utils;
@@ -294,6 +295,17 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         return ITEM_COUNT_UNKNOWN;
     }
 
+    @Nonnull
+    @Override
+    public Description getDescription() throws ParsingException {
+        final String description = getTextFromObject(
+                getPlaylistInfo().getObject("description"),
+                true
+        );
+
+        return new Description(description, Description.HTML);
+    }
+
     @Nonnull
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {

extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java

@@ -13,12 +13,14 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
 import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistExtractor;
+import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 import java.io.IOException;
 import java.util.List;
 
 import static org.junit.jupiter.api.Assertions.*;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
 import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
 
 /**
@@ -135,6 +137,16 @@ public class BandcampPlaylistExtractorTest {
             assertEquals(5, extractor.getStreamCount());
         }
 
+        @Test
+        public void testDescription() throws ParsingException {
+            final Description description = extractor.getDescription();
+            assertNotEquals(Description.EMPTY_DESCRIPTION, description);
+            assertContains("Artwork by Shona Radcliffe", description.getContent()); // about
+            assertContains("All tracks written, produced and recorded by Mac Benson",
+                    description.getContent()); // credits
+            assertContains("all rights reserved", description.getContent()); // license
+        }
+
         @Override
         public void testUploaderVerified() throws Exception {
             assertFalse(extractor.isUploaderVerified());

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java

@@ -60,6 +60,11 @@ public class PeertubePlaylistExtractorTest {
             ExtractorAsserts.assertGreaterOrEqual(39, extractor.getStreamCount());
         }
 
+        @Test
+        void testGetDescription() throws ParsingException {
+            ExtractorAsserts.assertContains("épisodes de Shocking", extractor.getDescription().getContent());
+        }
+
         @Test
         void testGetSubChannelUrl() {
             assertEquals("https://skeptikon.fr/video-channels/metadechoc_channel", extractor.getSubChannelUrl());

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
 import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
@@ -25,6 +26,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
 import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
+import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 import java.io.IOException;
@@ -163,6 +165,12 @@ public class YoutubePlaylistExtractorTest {
         void getPlaylistType() throws ParsingException {
             assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
         }
+
+        @Test
+        public void testDescription() throws ParsingException {
+            final Description description = extractor.getDescription();
+            assertContains("pop songs list", description.getContent());
+        }
     }
 
     public static class HugePlaylist implements BasePlaylistExtractorTest {
@@ -286,6 +294,12 @@ public class YoutubePlaylistExtractorTest {
         void getPlaylistType() throws ParsingException {
             assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
         }
+
+        @Test
+        public void testDescription() throws ParsingException {
+            final Description description = extractor.getDescription();
+            assertContains("I Wanna Rock Super Gigantic Playlist", description.getContent());
+        }
     }
 
     public static class LearningPlaylist implements BasePlaylistExtractorTest {
@@ -394,6 +408,12 @@ public class YoutubePlaylistExtractorTest {
         void getPlaylistType() throws ParsingException {
             assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
         }
+
+        @Test
+        public void testDescription() throws ParsingException {
+            final Description description = extractor.getDescription();
+            assertContains("47 episodes", description.getContent());
+        }
     }
 
     public static class ContinuationsTests {