kavin/Piped-Backend
1
0
Fork 0
mirror of https://github.com/TeamPiped/Piped-Backend.git synced 2025-01-06 01:20:28 +05:30
Code Issues Projects Releases Wiki Activity

Refractor codebase and improve video data update handling. (#185)

Browse Source
This commit is contained in:
Kavin 2022-02-09 03:06:52 +00:00 committed by GitHub
parent ec3b69b7a7
commit 7eeff6eec4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 83 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
src/main/java/me/kavin/piped/utils/ResponseHelper.java
View File

@ -48,7 +48,6 @@ import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery; import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Root; import javax.persistence.criteria.Root;
import java.io.IOException; import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.*; import java.util.*;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
@ -65,7 +64,7 @@ public class ResponseHelper {
.scheduler(Scheduler.systemScheduler()) .scheduler(Scheduler.systemScheduler())
.maximumSize(1000).build(key -> CommentsInfo.getInfo("https://www.youtube.com/watch?v=" + key)); .maximumSize(1000).build(key -> CommentsInfo.getInfo("https://www.youtube.com/watch?v=" + key));
public static final byte[] streamsResponse(String videoId) throws Exception { public static byte[] streamsResponse(String videoId) throws Exception {
CompletableFuture<StreamInfo> futureStream = CompletableFuture.supplyAsync(() -> { CompletableFuture<StreamInfo> futureStream = CompletableFuture.supplyAsync(() -> {
try { try {
@ -87,9 +86,7 @@ public class ResponseHelper {
CompletableFuture<String> futureLBRY = CompletableFuture.supplyAsync(() -> { CompletableFuture<String> futureLBRY = CompletableFuture.supplyAsync(() -> {
try { try {
String lbryId = null; String lbryId = futureLbryId.completeOnTimeout(null, 2, TimeUnit.SECONDS).get();
lbryId = futureLbryId.completeOnTimeout(null, 2, TimeUnit.SECONDS).get();
return LbryHelper.getLBRYStreamURL(lbryId); return LbryHelper.getLBRYStreamURL(lbryId);
} catch (Exception e) { } catch (Exception e) {
@ -103,10 +100,8 @@ public class ResponseHelper {
final StreamInfo info = futureStream.get(); final StreamInfo info = futureStream.get();
info.getStreamSegments().forEach(segment -> { info.getStreamSegments().forEach(segment -> chapters.add(new ChapterSegment(segment.getTitle(), rewriteURL(segment.getPreviewUrl()),
chapters.add(new ChapterSegment(segment.getTitle(), rewriteURL(segment.getPreviewUrl()), segment.getStartTimeSeconds())));
segment.getStartTimeSeconds()));
});
info.getSubtitles() info.getSubtitles()
.forEach(subtitle -> subtitles.add(new Subtitle(rewriteURL(subtitle.getUrl()), .forEach(subtitle -> subtitles.add(new Subtitle(rewriteURL(subtitle.getUrl()),
@ -154,7 +149,7 @@ public class ResponseHelper {
: System.currentTimeMillis(); : System.currentTimeMillis();
if (info.getUploadDate() != null && System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) if (info.getUploadDate() != null && System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
updateViews(info.getId(), info.getViewCount(), time, false); updateVideo(info.getId(), info, time);
final Streams streams = new Streams(info.getName(), info.getDescription().getContent(), final Streams streams = new Streams(info.getName(), info.getDescription().getContent(),
info.getTextualUploadDate(), info.getUploaderName(), substringYouTube(info.getUploaderUrl()), info.getTextualUploadDate(), info.getUploaderName(), substringYouTube(info.getUploaderUrl()),
@ -167,7 +162,7 @@ public class ResponseHelper {
} }
public static final byte[] channelResponse(String channelPath) throws Exception { public static byte[] channelResponse(String channelPath) throws Exception {
final ChannelInfo info = ChannelInfo.getInfo("https://youtube.com/" + channelPath); final ChannelInfo info = ChannelInfo.getInfo("https://youtube.com/" + channelPath);
@ -195,8 +190,12 @@ public class ResponseHelper {
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis(); : System.currentTimeMillis();
if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
updateViews(item.getUrl().substring("https://www.youtube.com/watch?v=".length()), try {
item.getViewCount(), time, true); String id = YOUTUBE_SERVICE.getStreamLHFactory().getId(item.getUrl());
updateVideo(id, item, time, true);
} catch (Exception e) {
ExceptionHandler.handle(e);
}
} }
} }
@ -219,7 +218,7 @@ public class ResponseHelper {
} }
public static final byte[] channelPageResponse(String channelId, String prevpageStr) public static byte[] channelPageResponse(String channelId, String prevpageStr)
throws IOException, ExtractionException { throws IOException, ExtractionException {
Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class); Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class);
@ -243,7 +242,7 @@ public class ResponseHelper {
} }
public static final byte[] trendingResponse(String region) public static byte[] trendingResponse(String region)
throws ExtractionException, IOException { throws ExtractionException, IOException {
if (region == null) if (region == null)
@ -262,7 +261,7 @@ public class ResponseHelper {
return Constants.mapper.writeValueAsBytes(relatedStreams); return Constants.mapper.writeValueAsBytes(relatedStreams);
} }
public static final byte[] playlistResponse(String playlistId) public static byte[] playlistResponse(String playlistId)
throws IOException, ExtractionException { throws IOException, ExtractionException {
final PlaylistInfo info = PlaylistInfo.getInfo("https://www.youtube.com/playlist?list=" + playlistId); final PlaylistInfo info = PlaylistInfo.getInfo("https://www.youtube.com/playlist?list=" + playlistId);
@ -287,7 +286,7 @@ public class ResponseHelper {
} }
public static final byte[] playlistPageResponse(String playlistId, String prevpageStr) public static byte[] playlistPageResponse(String playlistId, String prevpageStr)
throws IOException, ExtractionException { throws IOException, ExtractionException {
Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class); Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class);
@ -311,7 +310,7 @@ public class ResponseHelper {
} }
public static final byte[] playlistRSSResponse(String playlistId) public static byte[] playlistRSSResponse(String playlistId)
throws IOException, ExtractionException, FeedException { throws IOException, ExtractionException, FeedException {
final PlaylistInfo info = PlaylistInfo.getInfo("https://www.youtube.com/playlist?list=" + playlistId); final PlaylistInfo info = PlaylistInfo.getInfo("https://www.youtube.com/playlist?list=" + playlistId);
@ -326,8 +325,7 @@ public class ResponseHelper {
feed.setLink(Constants.FRONTEND_URL + substringYouTube(info.getUrl())); feed.setLink(Constants.FRONTEND_URL + substringYouTube(info.getUrl()));
feed.setPublishedDate(new Date()); feed.setPublishedDate(new Date());
info.getRelatedItems().forEach(o -> { info.getRelatedItems().forEach(item -> {
StreamInfoItem item = o;
SyndEntry entry = new SyndEntryImpl(); SyndEntry entry = new SyndEntryImpl();
entry.setAuthor(item.getUploaderName()); entry.setAuthor(item.getUploaderName());
entry.setLink(item.getUrl()); entry.setLink(item.getUrl());
@ -342,14 +340,14 @@ public class ResponseHelper {
} }
public static final byte[] suggestionsResponse(String query) public static byte[] suggestionsResponse(String query)
throws IOException, ExtractionException { throws IOException, ExtractionException {
return Constants.mapper.writeValueAsBytes(YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query)); return Constants.mapper.writeValueAsBytes(YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query));
} }
public static final byte[] opensearchSuggestionsResponse(String query) public static byte[] opensearchSuggestionsResponse(String query)
throws IOException, ExtractionException { throws IOException, ExtractionException {
return Constants.mapper.writeValueAsBytes(Arrays.asList( return Constants.mapper.writeValueAsBytes(Arrays.asList(
@ -359,7 +357,7 @@ public class ResponseHelper {
} }
public static final byte[] searchResponse(String q, String filter) public static byte[] searchResponse(String q, String filter)
throws IOException, ExtractionException { throws IOException, ExtractionException {
final SearchInfo info = SearchInfo.getInfo(YOUTUBE_SERVICE, final SearchInfo info = SearchInfo.getInfo(YOUTUBE_SERVICE,
@ -395,7 +393,7 @@ public class ResponseHelper {
} }
public static final byte[] searchPageResponse(String q, String filter, String prevpageStr) public static byte[] searchPageResponse(String q, String filter, String prevpageStr)
throws IOException, ExtractionException { throws IOException, ExtractionException {
Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class); Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class);
@ -433,7 +431,7 @@ public class ResponseHelper {
} }
public static final byte[] commentsResponse(String videoId) throws Exception { public static byte[] commentsResponse(String videoId) throws Exception {
CommentsInfo info = commentsCache.get(videoId); CommentsInfo info = commentsCache.get(videoId);
@ -466,7 +464,7 @@ public class ResponseHelper {
} }
public static final byte[] commentsPageResponse(String videoId, String prevpageStr) throws Exception { public static byte[] commentsPageResponse(String videoId, String prevpageStr) throws Exception {
Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class); Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class);
@ -505,8 +503,7 @@ public class ResponseHelper {
private static final Argon2PasswordEncoder argon2PasswordEncoder = new Argon2PasswordEncoder(); private static final Argon2PasswordEncoder argon2PasswordEncoder = new Argon2PasswordEncoder();
public static final byte[] registerResponse(String user, String pass) throws IOException, public static byte[] registerResponse(String user, String pass) throws IOException {
InterruptedException, URISyntaxException {
if (Constants.DISABLE_REGISTRATION) if (Constants.DISABLE_REGISTRATION)
return Constants.mapper.writeValueAsBytes(new DisabledRegistrationResponse()); return Constants.mapper.writeValueAsBytes(new DisabledRegistrationResponse());
@ -554,7 +551,7 @@ public class ResponseHelper {
private static final BCryptPasswordEncoder bcryptPasswordEncoder = new BCryptPasswordEncoder(); private static final BCryptPasswordEncoder bcryptPasswordEncoder = new BCryptPasswordEncoder();
public static final byte[] loginResponse(String user, String pass) public static byte[] loginResponse(String user, String pass)
throws IOException { throws IOException {
if (user == null || pass == null) if (user == null || pass == null)
@ -589,7 +586,7 @@ public class ResponseHelper {
} }
public static final byte[] subscribeResponse(String session, String channelId) public static byte[] subscribeResponse(String session, String channelId)
throws IOException { throws IOException {
Session s = DatabaseSessionFactory.createSession(); Session s = DatabaseSessionFactory.createSession();
@ -655,7 +652,7 @@ public class ResponseHelper {
} }
public static final byte[] unsubscribeResponse(String session, String channelId) public static byte[] unsubscribeResponse(String session, String channelId)
throws IOException { throws IOException {
Session s = DatabaseSessionFactory.createSession(); Session s = DatabaseSessionFactory.createSession();
@ -677,7 +674,7 @@ public class ResponseHelper {
} }
public static final byte[] isSubscribedResponse(String session, String channelId) public static byte[] isSubscribedResponse(String session, String channelId)
throws IOException { throws IOException {
Session s = DatabaseSessionFactory.createSession(); Session s = DatabaseSessionFactory.createSession();
@ -699,7 +696,7 @@ public class ResponseHelper {
} }
public static final byte[] feedResponse(String session) public static byte[] feedResponse(String session)
throws IOException { throws IOException {
Session s = DatabaseSessionFactory.createSession(); Session s = DatabaseSessionFactory.createSession();
@ -740,7 +737,7 @@ public class ResponseHelper {
} }
public static final byte[] feedResponseRSS(String session) public static byte[] feedResponseRSS(String session)
throws IOException, FeedException { throws IOException, FeedException {
Session s = DatabaseSessionFactory.createSession(); Session s = DatabaseSessionFactory.createSession();
@ -801,7 +798,7 @@ public class ResponseHelper {
} }
public static final byte[] importResponse(String session, String[] channelIds, boolean override) public static byte[] importResponse(String session, String[] channelIds, boolean override)
throws IOException { throws IOException {
Session s = DatabaseSessionFactory.createSession(); Session s = DatabaseSessionFactory.createSession();
@ -890,7 +887,7 @@ public class ResponseHelper {
} }
public static final byte[] subscriptionsResponse(String session) public static byte[] subscriptionsResponse(String session)
throws IOException { throws IOException {
Session s = DatabaseSessionFactory.createSession(); Session s = DatabaseSessionFactory.createSession();
@ -923,7 +920,7 @@ public class ResponseHelper {
} }
public static final String registeredBadgeRedirect() { public static String registeredBadgeRedirect() {
Session s = DatabaseSessionFactory.createSession(); Session s = DatabaseSessionFactory.createSession();
@ -965,32 +962,21 @@ public class ResponseHelper {
s.getTransaction().begin(); s.getTransaction().begin();
s.getTransaction().commit(); s.getTransaction().commit();
} else if (video != null) { } else if (video != null) {
video.setViews(info.getViewCount()); updateVideo(info.getId(), info, time);
s.update(video);
if (!s.getTransaction().isActive())
s.getTransaction().begin();
s.getTransaction().commit();
} }
} }
private static void updateViews(String id, long views, long time, boolean addIfNonExistent) { private static void updateVideo(String id, StreamInfoItem item, long time, boolean addIfNotExistent) {
Multithreading.runAsync(() -> { Multithreading.runAsync(() -> {
try { try {
Session s = DatabaseSessionFactory.createSession(); Session s = DatabaseSessionFactory.createSession();
Video video = DatabaseHelper.getVideoFromId(s, id); Video video = DatabaseHelper.getVideoFromId(s, id);
if (video != null) { if (video != null) {
video.setViews(views); updateVideo(s, video, item.getViewCount(), item.getDuration(), item.getName());
s.update(video); } else if (addIfNotExistent) {
s.beginTransaction().commit();
} else if (addIfNonExistent)
handleNewVideo("https://www.youtube.com/watch?v=" + id, time, null, s); handleNewVideo("https://www.youtube.com/watch?v=" + id, time, null, s);
}
s.close();
} catch (Exception e) { } catch (Exception e) {
ExceptionHandler.handle(e); ExceptionHandler.handle(e);
@ -998,7 +984,49 @@ public class ResponseHelper {
}); });
} }
public static void subscribePubSub(String channelId, Session s) throws IOException, InterruptedException { private static void updateVideo(String id, StreamInfo info, long time) {
Multithreading.runAsync(() -> {
try {
Session s = DatabaseSessionFactory.createSession();
Video video = DatabaseHelper.getVideoFromId(s, id);
if (video != null) {
updateVideo(s, video, info.getViewCount(), info.getDuration(), info.getName());
} else {
handleNewVideo(info, time, null, s);
}
} catch (Exception e) {
ExceptionHandler.handle(e);
}
});
}
private static void updateVideo(Session s, Video video, long views, long duration, String title) {
boolean changed = false;
if (duration > 0 && video.getDuration() != duration) {
video.setDuration(duration);
changed = true;
}
if (!video.getTitle().equals(title)) {
video.setTitle(title);
changed = true;
}
if (views > video.getViews()) {
video.setViews(views);
changed = true;
}
if (changed) {
s.update(video);
if (!s.getTransaction().isActive()) s.getTransaction().begin();
s.getTransaction().commit();
}
}
public static void subscribePubSub(String channelId, Session s) throws IOException {
PubSub pubsub = DatabaseHelper.getPubSubFromId(s, channelId); PubSub pubsub = DatabaseHelper.getPubSubFromId(s, channelId);
@ -1034,7 +1062,7 @@ public class ResponseHelper {
s.getTransaction().begin(); s.getTransaction().begin();
s.getTransaction().commit(); s.getTransaction().commit();
} else } else
System.out.println("Failed to subscribe: " + resp.code() + "\n" + resp.body().string()); System.out.println("Failed to subscribe: " + resp.code() + "\n" + Objects.requireNonNull(resp.body()).string());
} }
} }