Merge pull request #618 from TeamNewPipe/soundcloud_id

Fix Soundcloud extraction

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCha
 import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor;
 import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
 import org.schabi.newpipe.extractor.utils.Utils;
@@ -40,7 +41,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 import static org.schabi.newpipe.extractor.utils.Utils.*;
 
 public class SoundcloudParsingHelper {
-    private static final String HARDCODED_CLIENT_ID = "Kl9G8jQT22DxqatQk09IjWRujGlut5Vd"; // Updated on 04/03/21
+    private static final String HARDCODED_CLIENT_ID = "NcIaRZItQCNQp3Vq9Plvzf7tvjmVJnF6"; // Updated on 26/04/21
     private static String clientId;
 
     private SoundcloudParsingHelper() {
@@ -140,27 +141,37 @@ public class SoundcloudParsingHelper {
     }
 
     /**
-     * Fetch the embed player with the url and return the id (like the id from the json api).
+     * Fetch the widget API with the url and return the id (like the id from the json api).
      *
      * @return the resolved id
      */
-    public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException {
+    public static String resolveIdWithWidgetApi(String urlString) throws IOException, ReCaptchaException, ParsingException {
         // Remove the tailing slash from URLs due to issues with the SoundCloud API
         if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1);
+        // Make URL lower case and remove www. if it exists.
+        // Without doing this, the widget API does not recognize the URL.
+        urlString = Utils.removeWWWFromUrl(urlString.toLowerCase());
 
-        URL url;
+        final URL url;
         try {
             url = Utils.stringToURL(urlString);
         } catch (MalformedURLException e) {
             throw new IllegalArgumentException("The given URL is not valid");
         }
 
-        String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+        try {
-                + URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody();
+            final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url="
-        // handle playlists / sets different and get playlist id via uir field in JSON
+                    + URLEncoder.encode(url.toString(), UTF_8)
-        if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets"))
+                    + "&format=json&client_id=" + SoundcloudParsingHelper.clientId();
-            return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
+            final String response = NewPipe.getDownloader().get(widgetUrl,
-        return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response);
+                    SoundCloud.getLocalization()).responseBody();
+            final JsonObject o = JsonParser.object().from(response);
+            return String.valueOf(JsonUtils.getValue(o, "id"));
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse JSON response", e);
+        } catch (ExtractionException e) {
+            throw new ParsingException("Could not resolve id with embedded player. ClientId not extracted", e);
+        }
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java

@@ -23,7 +23,7 @@ public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory
         Utils.checkUrl(URL_PATTERN, url);
 
         try {
-            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+            return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java

@@ -22,7 +22,7 @@ public class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerFactory
         Utils.checkUrl(URL_PATTERN, url);
 
         try {
-            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+            return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
         } catch (Exception e) {
             throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java

@@ -32,7 +32,7 @@ public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory {
         Utils.checkUrl(URL_PATTERN, url);
 
         try {
-            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+            return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -7,6 +7,7 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
 import java.util.*;
+import java.util.regex.Pattern;
 
 public class Utils {
 
@@ -14,6 +15,7 @@ public class Utils {
     public static final String HTTPS = "https://";
     public static final String UTF_8 = "UTF-8";
     public static final String EMPTY_STRING = "";
+    private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\.");
 
     private Utils() {
         //no instance
@@ -170,6 +172,13 @@ public class Utils {
         return setsNoPort || usesDefaultPort;
     }
 
+    public static String removeWWWFromUrl(String url) {
+        if (WWW_PATTERN.matcher(url).find()) {
+            return url.replace("www.", "");
+        }
+        return url;
+    }
+
     public static String removeUTF8BOM(String s) {
         if (s.startsWith("\uFEFF")) {
             s = s.substring(1);

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java

@@ -29,9 +29,9 @@ public class SoundcloudParsingHelperTest {
     }
 
     @Test
-    public void resolveIdWithEmbedPlayerTest() throws Exception {
+    public void resolveIdWithWidgetApiTest() throws Exception {
-        Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/trapcity"));
+        Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/trapcity"));
-        Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/nocopyrightsounds"));
+        Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/nocopyrightsounds"));
 
     }