From 7607688cb0a90967e6faf05cbeb6e19b16411494 Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sun, 19 Jan 2025 00:08:26 +0100 Subject: [PATCH 1/2] [Soundcloud] Remove DRM-protected and downloadable formats extraction DRM-protected streams have been added to some tracks, mostly from major music companies. We do not support DRM streams in the extractor, so they shouldn't be extracted and so waste time, energy and resources. Extracting downloadable format requires login for a pretty long time, so there is no point again to do requests to extract this stream to get an unauthorized response. Also send the track_authorization property returned in the track info, like the website does and allow duplicate MP3 formats in progressive and HLS delivery methods to be returned. --- .../extractors/SoundcloudStreamExtractor.java | 95 ++++--------------- 1 file changed, 18 insertions(+), 77 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 27cf28e75..6b2abcf59 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -6,7 +6,6 @@ import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsing import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromTrackObject; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAvatarUrl; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDateFrom; -import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -170,14 +169,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } try { - final JsonArray transcodings = track.getObject("media").getArray("transcodings"); + final JsonArray transcodings = track.getObject("media") + .getArray("transcodings"); if (!isNullOrEmpty(transcodings)) { // Get information about what stream formats are available - extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings), - audioStreams); + extractAudioStreams(transcodings, audioStreams); } - - extractDownloadableFileIfAvailable(audioStreams); } catch (final NullPointerException e) { throw new ExtractionException("Could not get audio streams", e); } @@ -185,19 +182,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return audioStreams; } - private static boolean checkMp3ProgressivePresence(@Nonnull final JsonArray transcodings) { - return transcodings.stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .anyMatch(transcodingJsonObject -> transcodingJsonObject.getString("preset") - .contains("mp3") && transcodingJsonObject.getObject("format") - .getString("protocol").equals("progressive")); - } - @Nonnull private String getTranscodingUrl(final String endpointUrl) throws IOException, ExtractionException { - final String apiStreamUrl = endpointUrl + "?client_id=" + clientId(); + String apiStreamUrl = endpointUrl + "?client_id=" + clientId(); + + final String trackAuthorization = track.getString("track_authorization"); + if (!isNullOrEmpty(trackAuthorization)) { + apiStreamUrl += "&track_authorization=" + trackAuthorization; + } + final String response = NewPipe.getDownloader().get(apiStreamUrl).responseBody(); final JsonObject urlObject; try { @@ -209,27 +203,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return urlObject.getString("url"); } - @Nullable - private String getDownloadUrl(@Nonnull final String trackId) - throws IOException, ExtractionException { - final String response = NewPipe.getDownloader().get(SOUNDCLOUD_API_V2_URL + "tracks/" - + trackId + "/download" + "?client_id=" + clientId()).responseBody(); - - final JsonObject downloadJsonObject; - try { - downloadJsonObject = JsonParser.object().from(response); - } catch (final JsonParserException e) { - throw new ParsingException("Could not parse download URL", e); - } - final String redirectUri = downloadJsonObject.getString("redirectUri"); - if (!isNullOrEmpty(redirectUri)) { - return redirectUri; - } - return null; - } - private void extractAudioStreams(@Nonnull final JsonArray transcodings, - final boolean mp3ProgressiveInStreams, final List audioStreams) { transcodings.stream() .filter(JsonObject.class::isInstance) @@ -244,23 +218,23 @@ public class SoundcloudStreamExtractor extends StreamExtractor { final String preset = transcoding.getString("preset", ID_UNKNOWN); final String protocol = transcoding.getObject("format") .getString("protocol"); + + if (protocol.contains("encrypted")) { + // Skip DRM-protected streams, which have encrypted in their protocol + // name + return; + } + final AudioStream.Builder builder = new AudioStream.Builder() .setId(preset); - final boolean isHls = protocol.equals("hls"); - if (isHls) { + if (protocol.equals("hls")) { builder.setDeliveryMethod(DeliveryMethod.HLS); } builder.setContent(getTranscodingUrl(url), true); if (preset.contains("mp3")) { - // Don't add the MP3 HLS stream if there is a progressive stream - // present because both have the same bitrate - if (mp3ProgressiveInStreams && isHls) { - return; - } - builder.setMediaFormat(MediaFormat.MP3); builder.setAverageBitrate(128); } else if (preset.contains("opus")) { @@ -283,39 +257,6 @@ public class SoundcloudStreamExtractor extends StreamExtractor { }); } - /** - * Add the downloadable format if it is available. - * - *

- * A track can have the {@code downloadable} boolean set to {@code true}, but it doesn't mean - * we can download it. - *

- * - *

- * If the value of the {@code has_download_left} boolean is {@code true}, the track can be - * downloaded, and not otherwise. - *

- * - * @param audioStreams the audio streams to which the downloadable file is added - */ - public void extractDownloadableFileIfAvailable(final List audioStreams) { - if (track.getBoolean("downloadable") && track.getBoolean("has_downloads_left")) { - try { - final String downloadUrl = getDownloadUrl(getId()); - if (!isNullOrEmpty(downloadUrl)) { - audioStreams.add(new AudioStream.Builder() - .setId("original-format") - .setContent(downloadUrl, true) - .setAverageBitrate(UNKNOWN_BITRATE) - .build()); - } - } catch (final Exception ignored) { - // If something went wrong when trying to get the download URL, ignore the - // exception throw because this "stream" is not necessary to play the track - } - } - } - @Override public List getVideoStreams() { return Collections.emptyList(); From a336a8ca97ba4b1e3e63faf80fb9f398b404459c Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sun, 19 Jan 2025 00:14:53 +0100 Subject: [PATCH 2/2] [Soundcloud] Update SoundcloudStreamExtractorTest with latest changes --- .../SoundcloudStreamExtractorTest.java | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index 4f7f4e2bc..f47b440cc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -24,6 +24,7 @@ import javax.annotation.Nullable; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.fail; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; public class SoundcloudStreamExtractorTest { @@ -188,26 +189,33 @@ public class SoundcloudStreamExtractorTest { public void testAudioStreams() throws Exception { super.testAudioStreams(); final List audioStreams = extractor.getAudioStreams(); - assertEquals(2, audioStreams.size()); + assertEquals(3, audioStreams.size()); // 2 MP3 streams (1 progressive, 1 HLS) and 1 OPUS audioStreams.forEach(audioStream -> { final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod(); final String mediaUrl = audioStream.getContent(); if (audioStream.getFormat() == MediaFormat.OPUS) { - // Assert that it's an OPUS 64 kbps media URL with a single range which comes - // from an HLS SoundCloud CDN - ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl); - ExtractorAsserts.assertContains(".64.opus", mediaUrl); assertSame(DeliveryMethod.HLS, deliveryMethod, "Wrong delivery method for stream " + audioStream.getId() + ": " + deliveryMethod); - } else if (audioStream.getFormat() == MediaFormat.MP3) { - // Assert that it's a MP3 128 kbps media URL which comes from a progressive + // Assert it's an OPUS 64 kbps media playlist URL which comes from an HLS // SoundCloud CDN - ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3", - mediaUrl); - assertSame(DeliveryMethod.PROGRESSIVE_HTTP, deliveryMethod, - "Wrong delivery method for stream " + audioStream.getId() + ": " - + deliveryMethod); + ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl); + ExtractorAsserts.assertContains(".64.opus", mediaUrl); + } else if (audioStream.getFormat() == MediaFormat.MP3) { + if (deliveryMethod == DeliveryMethod.PROGRESSIVE_HTTP) { + // Assert it's a MP3 128 kbps media URL which comes from a progressive + // SoundCloud CDN + ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3", + mediaUrl); + } else if (deliveryMethod == DeliveryMethod.HLS) { + // Assert it's a MP3 128 kbps media HLS playlist URL which comes from an HLS + // SoundCloud CDN + ExtractorAsserts.assertContains("-hls-media.sndcdn.com", mediaUrl); + ExtractorAsserts.assertContains(".128.mp3", mediaUrl); + } else { + fail("Wrong delivery method for stream " + audioStream.getId() + ": " + + deliveryMethod); + } } }); }