Merge pull request #1081 from TeamNewPipe/fix/sc/search-next-page

[SoundCloud] Detect whether there are any more search results

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -1,14 +1,9 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
-
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -28,6 +23,7 @@ import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -38,7 +34,9 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
-import javax.annotation.Nonnull;
+import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
 
 public final class SoundcloudParsingHelper {
     private static String clientId;
@@ -200,6 +198,7 @@ public final class SoundcloudParsingHelper {
      *
      * @return the next streams url, empty if don't have
      */
+    @Nonnull
     public static String getUsersFromApi(final ChannelInfoItemsCollector collector,
                                          final String apiUrl) throws IOException,
             ReCaptchaException, ParsingException {
@@ -221,17 +220,7 @@ public final class SoundcloudParsingHelper {
             }
         }
 
-        String nextPageUrl;
+        return getNextPageUrl(responseObject);
-        try {
-            nextPageUrl = responseObject.getString("next_href");
-            if (!nextPageUrl.contains("client_id=")) {
-                nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
-            }
-        } catch (final Exception ignored) {
-            nextPageUrl = "";
-        }
-
-        return nextPageUrl;
     }
 
     /**
@@ -261,6 +250,7 @@ public final class SoundcloudParsingHelper {
      *
      * @return the next streams url, empty if don't have
      */
+    @Nonnull
     public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
                                            final String apiUrl,
                                            final boolean charts) throws IOException,
@@ -288,17 +278,20 @@ public final class SoundcloudParsingHelper {
             }
         }
 
-        String nextPageUrl;
+        return getNextPageUrl(responseObject);
+    }
+
+    @Nonnull
+    private static String getNextPageUrl(@Nonnull final JsonObject response) {
         try {
-            nextPageUrl = responseObject.getString("next_href");
+            String nextPageUrl = response.getString("next_href");
             if (!nextPageUrl.contains("client_id=")) {
                 nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
             }
-        } catch (final Exception ignored) {
-            nextPageUrl = "";
-        }
-
             return nextPageUrl;
+        } catch (final Exception ignored) {
+            return "";
+        }
     }
 
     public static String getStreamsFromApi(final StreamInfoItemsCollector collector,

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java

@@ -15,6 +15,7 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 
 import java.io.IOException;
@@ -33,22 +34,7 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
     @Override
     public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException,
             IOException {
-        final Downloader downloader = NewPipe.getDownloader();
+        return getPage(getUrl());
-        final Response response = downloader.get(getUrl());
-
-        final JsonObject json;
-        try {
-            json = JsonParser.object().from(response.responseBody());
-        } catch (final JsonParserException e) {
-            throw new ParsingException("Could not parse json", e);
-        }
-
-        final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
-                getServiceId());
-
-        collectStreamsFrom(collector, json.getArray("collection"));
-
-        return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
     }
 
     @Override
@@ -57,9 +43,14 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
         if (page == null || isNullOrEmpty(page.getUrl())) {
             throw new IllegalArgumentException("Page doesn't contain an URL");
         }
+        return getPage(page.getUrl());
+    }
 
+    @Nonnull
+    private InfoItemsPage<CommentsInfoItem> getPage(@Nonnull final String url)
+            throws ParsingException, IOException, ReCaptchaException {
         final Downloader downloader = NewPipe.getDownloader();
-        final Response response = downloader.get(page.getUrl());
+        final Response response = downloader.get(url);
 
         final JsonObject json;
         try {
@@ -72,8 +63,7 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
                 getServiceId());
 
         collectStreamsFrom(collector, json.getArray("collection"));
-
+        return new InfoItemsPage<>(collector, new Page(json.getString("next_href", null)));
-        return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java

@@ -33,7 +33,9 @@ import java.util.function.IntUnaryOperator;
 import javax.annotation.Nonnull;
 
 public class SoundcloudSearchExtractor extends SearchExtractor {
-    private JsonArray initialSearchCollection;
+    private JsonObject initialSearchObject;
+    private static final String COLLECTION = "collection";
+    private static final String TOTAL_RESULTS = "total_results";
 
     public SoundcloudSearchExtractor(final StreamingService service,
                                      final SearchQueryHandler linkHandler) {
@@ -60,9 +62,15 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
     @Nonnull
     @Override
     public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
+        if (initialSearchObject.getInt(TOTAL_RESULTS) > ITEMS_PER_PAGE) {
             return new InfoItemsPage<>(
-                collectItems(initialSearchCollection),
+                    collectItems(initialSearchObject.getArray(COLLECTION)),
                     getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE));
+        } else {
+            return new InfoItemsPage<>(
+                    collectItems(initialSearchObject.getArray(COLLECTION)), null);
+        }
+
     }
 
     @Override
@@ -74,18 +82,24 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
 
         final Downloader dl = getDownloader();
         final JsonArray searchCollection;
+        final int totalResults;
         try {
             final String response = dl.get(page.getUrl(), getExtractorLocalization())
                     .responseBody();
-            searchCollection = JsonParser.object().from(response).getArray("collection");
+            final JsonObject result = JsonParser.object().from(response);
+            searchCollection = result.getArray(COLLECTION);
+            totalResults = result.getInt(TOTAL_RESULTS);
         } catch (final JsonParserException e) {
             throw new ParsingException("Could not parse json response", e);
         }
 
+        if (getOffsetFromUrl(page.getUrl()) + ITEMS_PER_PAGE < totalResults) {
             return new InfoItemsPage<>(collectItems(searchCollection),
                     getNextPageFromCurrentUrl(page.getUrl(),
                             currentOffset -> currentOffset + ITEMS_PER_PAGE));
         }
+        return new InfoItemsPage<>(collectItems(searchCollection), null);
+    }
 
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
@@ -94,12 +108,12 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
         final String url = getUrl();
         try {
             final String response = dl.get(url, getExtractorLocalization()).responseBody();
-            initialSearchCollection = JsonParser.object().from(response).getArray("collection");
+            initialSearchObject = JsonParser.object().from(response);
         } catch (final JsonParserException e) {
             throw new ParsingException("Could not parse json response", e);
         }
 
-        if (initialSearchCollection.isEmpty()) {
+        if (initialSearchObject.getArray(COLLECTION).isEmpty()) {
             throw new SearchExtractor.NothingFoundException("Nothing found");
         }
     }
@@ -133,13 +147,20 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
 
     private Page getNextPageFromCurrentUrl(final String currentUrl,
                                            final IntUnaryOperator newPageOffsetCalculator)
-            throws MalformedURLException, UnsupportedEncodingException {
+            throws ParsingException {
-        final int currentPageOffset = Integer.parseInt(
+        final int currentPageOffset = getOffsetFromUrl(currentUrl);
-                    Parser.compatParseMap(new URL(currentUrl).getQuery()).get("offset"));
 
         return new Page(
                 currentUrl.replace(
                         "&offset=" + currentPageOffset,
                         "&offset=" + newPageOffsetCalculator.applyAsInt(currentPageOffset)));
     }
+
+    private int getOffsetFromUrl(final String url) throws ParsingException {
+        try {
+            return Integer.parseInt(Parser.compatParseMap(new URL(url).getQuery()).get("offset"));
+        } catch (MalformedURLException | UnsupportedEncodingException e) {
+            throw new ParsingException("Could not get offset from page URL", e);
+        }
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.soundcloud.search;
 
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
@@ -181,4 +182,27 @@ public class SoundcloudSearchExtractorTest {
             assertTrue(verified);
         }
     }
+
+    public static class NoNextPage extends DefaultSearchExtractorTest {
+
+        private static SearchExtractor extractor;
+        private static final String QUERY = "Dan at hor#berlgbd";
+
+        @BeforeAll
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = SoundCloud.getSearchExtractor(QUERY);
+            extractor.fetchPage();
+        }
+
+        @Override public boolean expectedHasMoreItems() { return false; }
+        @Override public SearchExtractor extractor() throws Exception { return extractor; }
+        @Override public StreamingService expectedService() throws Exception { return SoundCloud; }
+        @Override public String expectedName() throws Exception { return QUERY; }
+        @Override public String expectedId() throws Exception { return QUERY; }
+        @Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
+        @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
+        @Override public String expectedSearchString() { return QUERY; }
+        @Nullable @Override public String expectedSearchSuggestion() { return null; }
+    }
 }