diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 259e52907..8f67bc5bc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -61,17 +61,17 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { JsonArray arr; try { - arr = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.continuations"); - } catch (ParsingException e) { + arr = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.continuations"); + } catch (Exception e) { return ""; } - if(null == arr || arr.isEmpty()) { + if(arr.isEmpty()) { return ""; } String continuation; try { - continuation = JsonUtils.getValue(arr.getObject(0), "nextContinuationData.continuation"); - } catch (ParsingException e) { + continuation = (String) JsonUtils.getValue(arr.getObject(0), "nextContinuationData.continuation"); + } catch (Exception e) { return ""; } return getNextPageUrl(continuation); @@ -109,14 +109,26 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson, String pageUrl) throws ParsingException { - - JsonArray contents = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.contents"); + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.contents"); + }catch(Exception e) { + throw new ParsingException("unable to get parse youtube comments", e); + } fetchTitle(contents); - List comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer"); + List comments; + try { + comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer"); + }catch(Exception e) { + throw new ParsingException("unable to get parse youtube comments", e); + } - for(JsonObject c: comments) { - CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor(c, pageUrl); - collector.commit(extractor); + + for(Object c: comments) { + if(c instanceof JsonObject) { + CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, pageUrl); + collector.commit(extractor); + } } } @@ -124,7 +136,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { private void fetchTitle(JsonArray contents) { if(null == title) { try { - title = JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle.simpleText"); + title = (String) JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle.simpleText"); } catch (Exception e) { title = "Youtube Comments"; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index f6507f153..79d7a93c8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -7,11 +7,11 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor{ - +public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { + private final JsonObject json; private final String url; - + public YoutubeCommentsInfoItemExtractor(JsonObject json, String url) { this.json = json; this.url = url; @@ -24,55 +24,92 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract @Override public String getThumbnailUrl() throws ParsingException { - JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails"); - return JsonUtils.getValue(arr.getObject(2), "url"); + try { + JsonArray arr = (JsonArray) JsonUtils.getValue(json, "authorThumbnail.thumbnails"); + return (String) JsonUtils.getValue(arr.getObject(2), "url"); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } } @Override public String getName() throws ParsingException { - return JsonUtils.getValue(json, "authorText.simpleText"); + try { + return (String) JsonUtils.getValue(json, "authorText.simpleText"); + } catch (Exception e) { + throw new ParsingException("Could not get author name", e); + } } @Override public String getPublishedTime() throws ParsingException { - JsonArray arr = JsonUtils.getValue(json, "publishedTimeText.runs"); - return JsonUtils.getValue(arr.getObject(0), "text"); + try { + JsonArray arr = (JsonArray) JsonUtils.getValue(json, "publishedTimeText.runs"); + return (String) JsonUtils.getValue(arr.getObject(0), "text"); + } catch (Exception e) { + throw new ParsingException("Could not get publishedTimeText", e); + } } @Override public Integer getLikeCount() throws ParsingException { - return JsonUtils.getValue(json, "likeCount"); + try { + return (Integer) JsonUtils.getValue(json, "likeCount"); + } catch (Exception e) { + throw new ParsingException("Could not get like count", e); + } } @Override public String getCommentText() throws ParsingException { try { - return JsonUtils.getValue(json, "contentText.simpleText"); - } catch (Exception e) { - JsonArray arr = JsonUtils.getValue(json, "contentText.runs"); - return JsonUtils.getValue(arr.getObject(0), "text"); + return (String) JsonUtils.getValue(json, "contentText.simpleText"); + } catch (Exception e1) { + try { + JsonArray arr = (JsonArray) JsonUtils.getValue(json, "contentText.runs"); + return (String) JsonUtils.getValue(arr.getObject(0), "text"); + } catch (Exception e2) { + throw new ParsingException("Could not get comment text", e2); + } } } @Override public String getCommentId() throws ParsingException { - return JsonUtils.getValue(json, "commentId"); + try { + return (String) JsonUtils.getValue(json, "commentId"); + } catch (Exception e) { + throw new ParsingException("Could not get comment id", e); + } } @Override public String getAuthorThumbnail() throws ParsingException { - JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails"); - return JsonUtils.getValue(arr.getObject(2), "url"); + try { + JsonArray arr = (JsonArray) JsonUtils.getValue(json, "authorThumbnail.thumbnails"); + return (String) JsonUtils.getValue(arr.getObject(2), "url"); + } catch (Exception e) { + throw new ParsingException("Could not get author thumbnail", e); + } } @Override public String getAuthorName() throws ParsingException { - return JsonUtils.getValue(json, "authorText.simpleText"); + try { + return (String) JsonUtils.getValue(json, "authorText.simpleText"); + } catch (Exception e) { + throw new ParsingException("Could not get author name", e); + } } @Override public String getAuthorEndpoint() throws ParsingException { - return "https://youtube.com" + JsonUtils.getValue(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl"); + try { + return "https://youtube.com" + + (String) JsonUtils.getValue(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl"); + } catch (Exception e) { + throw new ParsingException("Could not get author endpoint", e); + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index 8aa5cea4b..ddbf47c81 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -18,24 +18,24 @@ public class JsonUtils { } @Nonnull - public static T getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ List keys = Arrays.asList(path.split("\\.")); object = getObject(object, keys.subList(0, keys.size() - 1)); if (null == object) throw new ParsingException("Unable to get " + path); - T result = (T) object.get(keys.get(keys.size() - 1)); + Object result = object.get(keys.get(keys.size() - 1)); if(null == result) throw new ParsingException("Unable to get " + path); return result; } @Nonnull - public static List getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { + public static List getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { - List result = new ArrayList<>(); + List result = new ArrayList<>(); for (int i = 0; i < array.size(); i++) { JsonObject obj = array.getObject(i); - result.add((T)getValue(obj, path)); + result.add(getValue(obj, path)); } return result; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index afef25145..6ed3b1d06 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -58,6 +59,23 @@ public class YoutubeCommentsExtractorTest { assertTrue(result); } + + @Test + public void testGetCommentsAllData() throws IOException, ExtractionException { + InfoItemsPage comments = extractor.getInitialPage(); + for(CommentsInfoItem c: comments.getItems()) { + assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); + assertFalse(StringUtil.isBlank(c.getAuthorName())); + assertFalse(StringUtil.isBlank(c.getAuthorThumbnail())); + assertFalse(StringUtil.isBlank(c.getCommentId())); + assertFalse(StringUtil.isBlank(c.getCommentText())); + assertFalse(StringUtil.isBlank(c.getName())); + assertFalse(StringUtil.isBlank(c.getPublishedTime())); + assertFalse(StringUtil.isBlank(c.getThumbnailUrl())); + assertFalse(StringUtil.isBlank(c.getUrl())); + assertFalse(c.getLikeCount() == null); + } + } private boolean findInComments(InfoItemsPage comments, String comment) { return findInComments(comments.getItems(), comment); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java index b44d3ee9c..dc8f2b04b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java @@ -31,15 +31,15 @@ public class JsonUtilsTest { @Test public void testGetArray() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); - JsonArray arr = JsonUtils.getValue(obj, "batters.batter"); + JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "batters.batter"); assertTrue(!arr.isEmpty()); } @Test public void testGetValues() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); - JsonArray arr = JsonUtils.getValue(obj, "topping"); - List types = JsonUtils.getValues(arr, "type"); + JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "topping"); + List types = JsonUtils.getValues(arr, "type"); assertTrue(types.contains("Chocolate with Sprinkles")); }