Merge pull request #25 from karyogamy/playlist_info

Playlist info extraction for search engine

src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java

@@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.FoundAdException;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemCollector;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 
@@ -32,6 +34,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
     private String suggestion = "";
     private StreamInfoItemCollector streamCollector;
     private ChannelInfoItemCollector userCollector;
+    private PlaylistInfoItemCollector playlistCollector;
 
     private SearchResult result = new SearchResult();
 
@@ -39,6 +42,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
         super(serviceId);
         streamCollector = new StreamInfoItemCollector(serviceId);
         userCollector = new ChannelInfoItemCollector(serviceId);
+        playlistCollector = new PlaylistInfoItemCollector(serviceId);
     }
 
     public void setSuggestion(String suggestion) {
@@ -49,6 +53,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
 
         addFromCollector(userCollector);
         addFromCollector(streamCollector);
+        addFromCollector(playlistCollector);
 
         result.suggestion = suggestion;
         result.errors = getErrors();
@@ -74,4 +79,14 @@ public class InfoItemSearchCollector extends InfoItemCollector {
             addError(e);
         }
     }
+
+    public void commit(PlaylistInfoItemExtractor extractor) {
+        try {
+            result.resultList.add(playlistCollector.extract(extractor));
+        } catch (FoundAdException ae) {
+            System.err.println("Found ad");
+        } catch (Exception e) {
+            addError(e);
+        }
+    }
 }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java

@@ -0,0 +1,89 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import com.github.openjson.JSONArray;
+import com.github.openjson.JSONObject;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
+
+public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
+    private static final String USER_KEY = "user";
+    private static final String AVATAR_URL_KEY = "avatar_url";
+    private static final String ARTWORK_URL_KEY = "artwork_url";
+    private static final String NULL_VALUE = "null";
+
+    private JSONObject searchResult;
+
+    public SoundcloudPlaylistInfoItemExtractor(JSONObject searchResult) {
+        this.searchResult = searchResult;
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        try {
+            return searchResult.getString("title");
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist name", e);
+        }
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
+        try {
+            return searchResult.getString("permalink_url");
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist name", e);
+        }
+    }
+
+    @Override
+    public String getThumbnailUrl() throws ParsingException {
+        // Over-engineering at its finest
+        try {
+            final String artworkUrl = searchResult.optString(ARTWORK_URL_KEY);
+            if (!artworkUrl.isEmpty() && !artworkUrl.equals(NULL_VALUE)) return artworkUrl;
+
+            // Look for artwork url inside the track list
+            final JSONArray tracks = searchResult.optJSONArray("tracks");
+            if (tracks == null) return null;
+            for (int i = 0; i < tracks.length(); i++) {
+                if (tracks.isNull(i)) continue;
+                final JSONObject track = tracks.optJSONObject(i);
+                if (track == null) continue;
+
+                // First look for track artwork url
+                final String url = track.optString(ARTWORK_URL_KEY);
+                if (!url.isEmpty() && !url.equals(NULL_VALUE)) return url;
+
+                // Then look for track creator avatar url
+                final JSONObject creator = track.getJSONObject(USER_KEY);
+                final String creatorAvatar = creator.optString(AVATAR_URL_KEY);
+                if (!creatorAvatar.isEmpty() && !creatorAvatar.equals(NULL_VALUE)) return creatorAvatar;
+            }
+
+            // Last resort, use user avatar url. If still not found, then throw exception.
+            final JSONObject user = searchResult.getJSONObject(USER_KEY);
+            return user.getString(AVATAR_URL_KEY);
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist thumbnail url", e);
+        }
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        try {
+            final JSONObject user = searchResult.getJSONObject(USER_KEY);
+            return user.optString("username");
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist uploader", e);
+        }
+    }
+
+    @Override
+    public long getStreamCount() throws ParsingException {
+        try {
+            return Long.parseLong(searchResult.optString("track_count"));
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist stream count", e);
+        }
+    }
+}

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java

@@ -27,10 +27,12 @@ public class SoundcloudSearchEngine extends SearchEngine {
 
         String url = "https://api-v2.soundcloud.com/search";
 
-        if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) {
+        if (filter.contains(Filter.STREAM) && filter.size() == 1) {
             url += "/tracks";
-        } else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) {
+        } else if (filter.contains(Filter.CHANNEL) && filter.size() == 1) {
             url += "/users";
+        } else if (filter.contains(Filter.PLAYLIST) && filter.size() == 1) {
+            url += "/playlists";
         }
 
         url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
@@ -53,6 +55,8 @@ public class SoundcloudSearchEngine extends SearchEngine {
                 collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
             } else if (kind.equals("track")) {
                 collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult));
+            } else if (kind.equals("playlist")) {
+                collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult));
             }
         }
 

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java

@@ -0,0 +1,92 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
+    private Element el;
+
+    public YoutubePlaylistInfoItemExtractor(Element el) {
+        this.el = el;
+    }
+
+    @Override
+    public String getThumbnailUrl() throws ParsingException {
+        String url;
+
+        try {
+            Element te = el.select("div[class=\"yt-thumb video-thumb\"]").first()
+                    .select("img").first();
+            url = te.attr("abs:src");
+
+            if (url.contains(".gif")) {
+                url = te.attr("abs:data-thumb");
+            }
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist thumbnail url", e);
+        }
+
+        return url;
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        String name;
+        try {
+            final Element title = el.select("[class=\"yt-lockup-title\"]").first()
+                    .select("a").first();
+
+            name = title == null ? "" : title.text();
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist name", e);
+        }
+
+        return name;
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
+        String url;
+
+        try {
+            final Element href = el.select("div[class=\"yt-lockup-meta\"]").first()
+                    .select("a").first();
+
+            url = href.attr("abs:href");
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist url", e);
+        }
+
+        return url;
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        String name;
+
+        try {
+            final Element div = el.select("div[class=\"yt-lockup-byline\"]").first()
+                    .select("a").first();
+
+            name = div.text();
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist uploader", e);
+        }
+
+        return name;
+    }
+
+    @Override
+    public long getStreamCount() throws ParsingException {
+        try {
+            final Element count = el.select("span[class=\"formatted-video-count-label\"]").first()
+                    .select("b").first();
+
+            return count == null ? 0 : Long.parseLong(Utils.removeNonDigitCharacters(count.text()));
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist stream count", e);
+        }
+    }
+}

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java

@@ -57,10 +57,13 @@ public class YoutubeSearchEngine extends SearchEngine {
         String url = "https://www.youtube.com/results"
                 + "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
                 + "&page=" + Integer.toString(page + 1);
-        if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) {
+
-            url += "&sp=EgIQAQ%253D%253D";
+        if (filter.contains(Filter.STREAM) && filter.size() == 1) {
-        } else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) {
+            url += "&sp=EgIQAVAU";
-            url += "&sp=EgIQAg%253D%253D";
+        } else if (filter.contains(Filter.CHANNEL) && filter.size() == 1) {
+            url += "&sp=EgIQAlAU"; //EgIQA( lowercase L )AU
+        } else if (filter.contains(Filter.PLAYLIST) && filter.size() == 1) {
+            url += "&sp=EgIQA1AU"; //EgIQA( one )AU
         }
 
         String site;
@@ -105,6 +108,8 @@ public class YoutubeSearchEngine extends SearchEngine {
                 collector.commit(new YoutubeStreamInfoItemExtractor(el));
             } else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
                 collector.commit(new YoutubeChannelInfoItemExtractor(el));
+            } else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null) {
+                collector.commit(new YoutubePlaylistInfoItemExtractor(el));
             } else {
                 // noinspection ConstantConditions
                 // simply ignore not known items

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java

@@ -74,7 +74,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
         try {
             if (getStreamType() == StreamType.LIVE_STREAM) return -1;
 
-            return YoutubeParsingHelper.parseDurationString(item.select("span[class*=\"video-time\"]").first().text());
+            final Element duration = item.select("span[class*=\"video-time\"]").first();
+            // apparently on youtube, video-time element will not show up if the video has a duration of 00:00
+            // see: https://www.youtube.com/results?sp=EgIQAVAU&q=asdfgf
+            return duration == null ? 0 : YoutubeParsingHelper.parseDurationString(duration.text());
         } catch (Exception e) {
             throw new ParsingException("Could not get Duration: " + getUrl(), e);
         }

src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java

@@ -0,0 +1,80 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.search.SearchEngine;
+import org.schabi.newpipe.extractor.search.SearchResult;
+
+import java.util.EnumSet;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
+
+
+/*
+ * Created by Christian Schabesberger on 29.12.15.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * YoutubeSearchEngineStreamTest.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Test for {@link SearchEngine}
+ */
+public class SoundcloudSearchEnginePlaylistTest {
+    private SearchResult result;
+
+    @Before
+    public void setUp() throws Exception {
+        NewPipe.init(Downloader.getInstance());
+        SearchEngine engine = SoundCloud.getService().getSearchEngine();
+
+        // Search by country not yet implemented
+        result = engine.search("parkmemme", 0, "", EnumSet.of(SearchEngine.Filter.PLAYLIST))
+                .getSearchResult();
+    }
+
+    @Test
+    public void testResultList() {
+        assertFalse(result.resultList.isEmpty());
+    }
+
+    @Test
+    public void testUserItemType() {
+        for (InfoItem infoItem : result.resultList) {
+            assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type);
+        }
+    }
+
+    @Test
+    public void testResultErrors() {
+        if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
+        assertTrue(result.errors == null || result.errors.isEmpty());
+    }
+
+    @Ignore
+    @Test
+    public void testSuggestion() {
+        //todo write a real test
+        assertTrue(result.suggestion != null);
+    }
+}

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java

@@ -50,7 +50,8 @@ public class YoutubeSearchEngineAllTest {
         // keep in mind that the suggestions can change by country (the parameter "de")
         result = engine.search("asdgff", 0, "de",
                 EnumSet.of(SearchEngine.Filter.CHANNEL,
-                        SearchEngine.Filter.STREAM)).getSearchResult();
+                        SearchEngine.Filter.STREAM,
+                        SearchEngine.Filter.PLAYLIST)).getSearchResult();
     }
 
     @Test

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java

@@ -0,0 +1,81 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.search.SearchEngine;
+import org.schabi.newpipe.extractor.search.SearchResult;
+
+import java.util.EnumSet;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
+
+/*
+ * Created by Christian Schabesberger on 29.12.15.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * YoutubeSearchEngineStreamTest.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Test for {@link SearchEngine}
+ */
+public class YoutubeSearchEnginePlaylistTest {
+    private SearchResult result;
+
+    @Before
+    public void setUp() throws Exception {
+        NewPipe.init(Downloader.getInstance());
+        SearchEngine engine = YouTube.getService().getSearchEngine();
+
+        // Youtube will suggest "gronkh" instead of "grrunkh"
+        // keep in mind that the suggestions can change by country (the parameter "de")
+        result = engine.search("grrunkh", 0, "de",
+                EnumSet.of(SearchEngine.Filter.PLAYLIST)).getSearchResult();
+    }
+
+    @Test
+    public void testResultList() {
+        assertFalse(result.resultList.isEmpty());
+    }
+
+    @Test
+    public void testUserItemType() {
+        for (InfoItem infoItem : result.resultList) {
+            assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type);
+        }
+    }
+
+    @Test
+    public void testResultErrors() {
+        if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
+        assertTrue(result.errors == null || result.errors.isEmpty());
+    }
+
+    @Ignore
+    @Test
+    public void testSuggestion() {
+        //todo write a real test
+        assertTrue(result.suggestion != null);
+    }
+}