diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 3d217ce80..8a1abc68c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -326,14 +326,8 @@ public final class SoundcloudParsingHelper { public static boolean isReplyTo(@Nonnull final JsonObject originalComment, @Nonnull final JsonObject otherComment) { - final String mention = "@" + originalComment.getObject("user").getString("permalink"); - return otherComment.getString("body").startsWith(mention) - && originalComment.getInt("timestamp") == otherComment.getInt("timestamp"); + return originalComment.getInt("timestamp") == otherComment.getInt("timestamp"); } - public static boolean isReply(@Nonnull final JsonObject comment) { - return comment.getString("body").startsWith("@"); - } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java index d51a5fbc9..f253cb695 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; @@ -22,10 +24,9 @@ import java.io.IOException; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - public class SoundcloudCommentsExtractor extends CommentsExtractor { public static final String COLLECTION = "collection"; + public static final String NEXT_HREF = "next_href"; public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) { @@ -49,9 +50,9 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor { final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( getServiceId()); - collectStreamsFrom(collector, json); + collectCommentsFrom(collector, json); - return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); + return new InfoItemsPage<>(collector, new Page(json.getString(NEXT_HREF))); } @Override @@ -83,15 +84,15 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor { try { json = JsonParser.object().from(response.responseBody()); - hasNextPage = json.has("next_href"); + hasNextPage = json.has(NEXT_HREF); } catch (final JsonParserException e) { throw new ParsingException("Could not parse json", e); } - collectStreamsFrom(collector, json); + collectCommentsFrom(collector, json); } if (hasNextPage) { - return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); + return new InfoItemsPage<>(collector, new Page(json.getString(NEXT_HREF))); } else { return new InfoItemsPage<>(collector, null); } @@ -100,17 +101,19 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) { } - private void collectStreamsFrom(final CommentsInfoItemsCollector collector, - final JsonObject json) throws ParsingException { + private void collectCommentsFrom(final CommentsInfoItemsCollector collector, + final JsonObject json) throws ParsingException { final String url = getUrl(); final JsonArray entries = json.getArray(COLLECTION); + JsonObject lastTopComment = null; for (int i = 0; i < entries.size(); i++) { final JsonObject entry = entries.getObject(i); if (i == 0 - || (!SoundcloudParsingHelper.isReply(entry) - && !SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry))) { + || (!SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry) + && !SoundcloudParsingHelper.isReplyTo(lastTopComment, entry))) { + lastTopComment = entry; collector.commit(new SoundcloudCommentsInfoItemExtractor( - json, i, entries.getObject(i), url)); + json, i, entry, url)); } } } @@ -118,7 +121,7 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor { private boolean collectRepliesFrom(final CommentsInfoItemsCollector collector, final JsonObject json, final int id, - final String url) throws ParsingException { + final String url) { JsonObject originalComment = null; final JsonArray entries = json.getArray(COLLECTION); boolean moreReplies = false; @@ -134,7 +137,7 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor { json, i, entries.getObject(i), url, originalComment)); // There might be more replies to the originalComment, // especially if the original comment is at the end of the list. - if (i == entries.size() - 1 && json.has("next_href")) { + if (i == entries.size() - 1 && json.has(NEXT_HREF)) { moreReplies = true; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index a6d2b9a7f..db9ef549d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -1,7 +1,10 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; @@ -12,20 +15,21 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.Description; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; import java.util.Objects; +import javax.annotation.Nullable; + public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { - public static final String USER = "user"; public static final String BODY = "body"; public static final String USER_PERMALINK = "permalink"; + public static final String USER_FULL_NAME = "full_name"; + public static final String USER_USERNAME = "username"; private final JsonObject json; private final int index; private final JsonObject item; private final String url; + private final JsonObject user; private final JsonObject superComment; private int replyCount = CommentsInfoItem.UNKNOWN_REPLY_COUNT; @@ -39,6 +43,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr this.item = item; this.url = url; this.superComment = superComment; + this.user = item.getObject("user"); } public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int index, @@ -50,7 +55,6 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr public String getCommentId() { return Objects.toString(item.getLong("id"), null); } - @Override public Description getCommentText() { String commentContent = item.getString(BODY); @@ -61,32 +65,49 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr // Therefore, the comment starts with the mention of the original comment's author. // The account is automatically linked by the SoundCloud web UI. // We need to do this manually. - final JsonObject user = superComment.getObject("user"); - final String link = "" - + "@" + user.getString("full_name") + ""; - commentContent = commentContent - .replace("@" + user.getString(USER_PERMALINK), link) - .replace("@" + superComment.getInt("user_id"), link); + if (commentContent.startsWith("@")) { + final String authorName = commentContent.split(" ", 2)[0].replace("@", ""); + final JsonArray comments = json.getArray(SoundcloudCommentsExtractor.COLLECTION); + JsonObject author = null; + for (int i = index - 1; i >= 0 && author == null; i--) { + final JsonObject commentsAuthor = comments.getObject(i).getObject("user"); + // use startsWith because sometimes the mention of the user + // is followed by a punctuation character. + if (authorName.startsWith(commentsAuthor.getString(USER_PERMALINK))) { + author = commentsAuthor; + } + } + if (author == null) { + author = superComment.getObject("user"); + } + final String name = isNullOrEmpty(author.getString(USER_FULL_NAME)) + ? author.getString(USER_USERNAME) : author.getString(USER_FULL_NAME); + final String link = "" + + "@" + name + ""; + commentContent = commentContent + .replace("@" + author.getString(USER_PERMALINK), link) + .replace("@" + author.getInt("user_id"), link); + } return new Description(commentContent, Description.HTML); } @Override public String getUploaderName() { - if (isNullOrEmpty(user.getString("full_name"))) { - return user.getString("username"); + if (isNullOrEmpty(user.getString(USER_FULL_NAME))) { + return user.getString(USER_USERNAME); } - return user.getString("full_name"); + return user.getString(USER_FULL_NAME); } @Override public String getUploaderAvatarUrl() { - return item.getObject(USER).getString("avatar_url"); + return user.getString("avatar_url"); } @Override public boolean isUploaderVerified() throws ParsingException { - return item.getObject(USER).getBoolean("verified"); + return user.getBoolean("verified"); } @Override @@ -96,7 +117,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr @Override public String getUploaderUrl() { - return item.getObject(USER).getString("permalink_url"); + return user.getString("permalink_url"); } @Override @@ -112,7 +133,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr @Override public String getName() throws ParsingException { - return item.getObject(USER).getString("permalink"); + return user.getString(USER_PERMALINK); } @Override @@ -122,38 +143,39 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr @Override public String getThumbnailUrl() { - return item.getObject(USER).getString("avatar_url"); + return user.getString("avatar_url"); } @Override public Page getReplies() { if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) { - final List replies = new ArrayList<>(); + final JsonArray replies = new JsonArray(); final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( ServiceList.SoundCloud.getServiceId()); - final JsonArray jsonArray = new JsonArray(); - // Replies start with the mention of the user who created the original comment. - final String mention = "@" + item.getObject(USER).getString(USER_PERMALINK); - // Loop through all comments which come after the original comment to find its replies. - final JsonArray allItems = json.getArray(SoundcloudCommentsExtractor.COLLECTION); - for (int i = index + 1; i < allItems.size(); i++) { - final JsonObject comment = allItems.getObject(i); - final String commentContent = comment.getString("body"); - if (commentContent.startsWith(mention)) { - replies.add(comment); - jsonArray.add(comment); - collector.commit(new SoundcloudCommentsInfoItemExtractor( - json, i, comment, url, item)); - } else if (!commentContent.startsWith("@") || replies.isEmpty()) { - // Only the comments directly after the original comment - // starting with the mention of the comment's creator - // are replies to the original comment. - // The first comment not starting with these letters - // is the next top-level comment. - break; + // SoundCloud has only comments and top level replies, but not nested replies. + // Therefore, replies cannot have further replies. + if (superComment == null) { + // Loop through all comments which come after the original comment + // to find its replies. + final JsonArray allItems = json.getArray(SoundcloudCommentsExtractor.COLLECTION); + boolean foundReply = false; + for (int i = index + 1; i < allItems.size(); i++) { + final JsonObject comment = allItems.getObject(i); + if (SoundcloudParsingHelper.isReplyTo(item, comment)) { + replies.add(comment); + collector.commit(new SoundcloudCommentsInfoItemExtractor( + json, i, comment, url, item)); + foundReply = true; + } else if (foundReply) { + // Only the comments directly after the original comment + // having the same timestamp are replies to the original comment. + // The first comment not having the same timestamp + // is the next top-level comment. + break; + } } } - replyCount = jsonArray.size(); + replyCount = replies.size(); if (collector.getItems().isEmpty()) { return null; } @@ -165,7 +187,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr } @Override - public int getReplyCount() throws ParsingException { + public int getReplyCount() { if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) { getReplies(); }