From e2b7cb9c69bf229cac09a88be0d408a15f4856b5 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 20 Aug 2017 10:03:41 +0200 Subject: [PATCH] Add SoundcloudChartsExtractor --- .../soundcloud/SoundcloudChartsExtractor.java | 63 ++++++++++++++ .../SoundcloudChartsUrlIdHandler.java | 37 ++++++++ .../soundcloud/SoundcloudParsingHelper.java | 12 ++- .../soundcloud/SoundcloudService.java | 16 ++-- .../SoundcloudChartsExtractorTest.java | 85 +++++++++++++++++++ .../SoundcloudChartsUrlIdHandlerTest.java | 49 +++++++++++ 6 files changed, 254 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java create mode 100644 src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java new file mode 100644 index 000000000..f4aca4e2d --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -0,0 +1,63 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import java.io.IOException; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; + +public class SoundcloudChartsExtractor extends KioskExtractor { + private String url; + + public SoundcloudChartsExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { + super(service, url, nextStreamsUrl); + this.url = url; + } + + @Override + public void fetchPage() { + } + + @Override + public String getType() throws ParsingException { + return getUrlIdHandler().getId(url); + } + + @Override + public UrlIdHandler getUrlIdHandler() { + return new SoundcloudChartsUrlIdHandler(); + } + + @Override + public NextItemsResult getNextStreams() throws IOException, ExtractionException { + if (!hasMoreStreams()) { + throw new ExtractionException("Chart doesn't have more streams"); + } + + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); + nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl, true); + + return new NextItemsResult(collector.getItemList(), nextStreamsUrl); + } + + @Override + public StreamInfoItemCollector getStreams() throws IOException, ExtractionException { + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); + + String apiUrl = "https://api-v2.soundcloud.com/charts" + + "?genre=soundcloud:genres:all-music" + + "&client_id=" + SoundcloudParsingHelper.clientId(); + + if (getType().equals("Top 50")) { + apiUrl += "&kind=top"; + } else { + apiUrl += "&kind=new"; + } + + nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true); + return collector; + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java new file mode 100644 index 000000000..642e89c84 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.utils.Parser; + +public class SoundcloudChartsUrlIdHandler implements UrlIdHandler { + public String getUrl(String id) { + if (id.equals("Top 50")) { + return "https://soundcloud.com/charts/top"; + } else { + return "https://soundcloud.com/charts/new"; + } + } + + @Override + public String getId(String url) { + if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) { + return "Top 50"; + } else { + return "New & hot"; + } + } + + @Override + public String cleanUrl(String url) { + if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) { + return "https://soundcloud.com/charts/top"; + } else { + return "https://soundcloud.com/charts/new"; + } + } + + @Override + public boolean acceptUrl(String url) { + return Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$", url.toLowerCase()); + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 21b1686d0..fd20ada45 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -121,13 +121,17 @@ public class SoundcloudParsingHelper { * * @return the next streams url, empty if don't have */ - public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { + public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException { String response = NewPipe.getDownloader().download(apiUrl); JSONObject responseObject = new JSONObject(response); JSONArray responseCollection = responseObject.getJSONArray("collection"); for (int i = 0; i < responseCollection.length(); i++) { - collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i))); + if (charts) { + collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i).getJSONObject("track"))); + } else { + collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i))); + } } String nextStreamsUrl; @@ -140,4 +144,8 @@ public class SoundcloudParsingHelper { return nextStreamsUrl; } + + public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException { + return getStreamsFromApi(collector, apiUrl, false); + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 90e794f19..c2bb4f9c1 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -1,18 +1,17 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import java.io.IOException; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import java.io.IOException; - public class SoundcloudService extends StreamingService { public SoundcloudService(int id, String name) { @@ -61,12 +60,17 @@ public class SoundcloudService extends StreamingService { } @Override - public KioskList getKioskList() { + public KioskList getKioskList() throws ExtractionException { KioskList list = new KioskList(getServiceId()); // add kiosks here e.g.: - //list.addKioskEntry("trinding", new TrendingKiosk(), new TrendingUrlIdHandler()); - + SoundcloudChartsUrlIdHandler h = new SoundcloudChartsUrlIdHandler(); + try { + list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("Top 50"), null), h); + list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("New & hot"), null), h); + } catch (Exception e) { + throw new ExtractionException(e); + } return list; } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java new file mode 100644 index 000000000..061c37ff3 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -0,0 +1,85 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItemCollector; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; + +/** + * Test for {@link SoundcloudChartsUrlIdHandler} + */ +public class SoundcloudChartsExtractorTest { + + KioskExtractor extractor; + + @Before + public void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = SoundCloud.getService() + .getKioskList() + .getExtractorByType("Top 50"); + extractor.fetchPage(); + } + + @Test + public void testGetDownloader() throws Exception { + assertNotNull(NewPipe.getDownloader()); + } + + @Test + public void testGetName() throws Exception { + assertEquals(extractor.getName(), "Top 50"); + } + + @Test + public void testId() throws Exception { + assertEquals(extractor.getId(), "Top 50"); + } + + @Test + public void testGetStreams() throws Exception { + InfoItemCollector collector = extractor.getStreams(); + if(!collector.getErrors().isEmpty()) { + System.err.println("----------"); + for(Throwable e : collector.getErrors()) { + e.printStackTrace(); + System.err.println("----------"); + } + } + assertTrue("no streams are received", + !collector.getItemList().isEmpty() + && collector.getErrors().isEmpty()); + } + + @Test + public void testGetStreamsErrors() throws Exception { + assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty()); + } + + @Test + public void testHasMoreStreams() throws Exception { + // Setup the streams + extractor.getStreams(); + assertTrue("has more streams", extractor.hasMoreStreams()); + } + + @Test + public void testGetNextStreams() throws Exception { + extractor.getStreams(); + assertFalse("extractor has next streams", extractor.getNextStreams() == null + || extractor.getNextStreams().nextItemsList.isEmpty()); + } + + @Test + public void testGetCleanUrl() throws Exception { + assertEquals(extractor.getCleanUrl(), "https://soundcloud.com/charts/top"); + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java new file mode 100644 index 000000000..6429f6c3f --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java @@ -0,0 +1,49 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import static junit.framework.TestCase.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; + +/** + * Test for {@link SoundcloudChartsUrlIdHandler} + */ +public class SoundcloudChartsUrlIdHandlerTest { + private SoundcloudChartsUrlIdHandler urlIdHandler; + + @Before + public void setUp() throws Exception { + urlIdHandler = new SoundcloudChartsUrlIdHandler(); + NewPipe.init(Downloader.getInstance()); + } + + @Test + public void getUrl() { + assertEquals(urlIdHandler.getUrl("Top 50"), "https://soundcloud.com/charts/top"); + assertEquals(urlIdHandler.getUrl("New & hot"), "https://soundcloud.com/charts/new"); + } + + @Test + public void getId() { + assertEquals(urlIdHandler.getId("http://soundcloud.com/charts/top?genre=all-music"), "Top 50"); + assertEquals(urlIdHandler.getId("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"), "New & hot"); + } + + @Test + public void acceptUrl() { + assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts")); + assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts/")); + assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/charts/new")); + assertTrue(urlIdHandler.acceptUrl("http://soundcloud.com/charts/top?genre=all-music")); + assertTrue(urlIdHandler.acceptUrl("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries")); + + assertFalse(urlIdHandler.acceptUrl("kdskjfiiejfia")); + assertFalse(urlIdHandler.acceptUrl("soundcloud.com/charts askjkf")); + assertFalse(urlIdHandler.acceptUrl(" soundcloud.com/charts")); + assertFalse(urlIdHandler.acceptUrl("")); + } +}