kavin/Piped-Backend
1
0
Fork 0
mirror of https://github.com/TeamPiped/Piped-Backend.git synced 2024-12-13 13:50:28 +05:30
Code Issues Projects Releases Wiki Activity

Improvements to caffeine caching. (#171)

Browse Source
This commit is contained in:
Kavin 2022-01-26 04:00:56 +00:00 committed by GitHub
parent de9f855fcc
commit 9aa16b3299
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 48 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/me/kavin/piped/utils/DownloaderImpl.java
View File

@ -12,7 +12,9 @@ import java.net.http.HttpResponse.BodyHandlers;
import java.util.Map; import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.github.benmanes.caffeine.cache.Scheduler;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.NotNull;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
@ -34,13 +36,13 @@ public class DownloaderImpl extends Downloader {
private static long cookie_received; private static long cookie_received;
private static final Object cookie_lock = new Object(); private static final Object cookie_lock = new Object();
final LoadingCache<Request, Response> responseCache = Caffeine.newBuilder().expireAfterWrite(1, TimeUnit.MINUTES) final LoadingCache<Request, Response> responseCache = Caffeine.newBuilder()
.maximumSize(1000).build(key -> { .expireAfterWrite(1, TimeUnit.MINUTES)
return executeRequest(key); .scheduler(Scheduler.systemScheduler())
}); .maximumSize(1000).build(this::executeRequest);
@Override @Override
public Response execute(Request request) throws IOException, ReCaptchaException { public Response execute(@NotNull Request request) {
return responseCache.get(request); return responseCache.get(request);
} }
@ -132,7 +134,7 @@ public class DownloaderImpl extends Downloader {
BodyHandlers.ofString()); BodyHandlers.ofString());
saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
.substringAfter(formResponse.headers().firstValue("Location").get(), "google_abuse="))) .substringAfter(formResponse.headers().firstValue("Location").get(), "google_abuse=")))
.get(0); .get(0);
cookie_received = System.currentTimeMillis(); cookie_received = System.currentTimeMillis();

78
src/main/java/me/kavin/piped/utils/ResponseHelper.java
View File

@ -29,6 +29,7 @@ import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery; import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Root; import javax.persistence.criteria.Root;
import com.github.benmanes.caffeine.cache.Scheduler;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -104,8 +105,9 @@ import me.kavin.piped.utils.resp.SubscribeStatusResponse;
public class ResponseHelper { public class ResponseHelper {
public static final LoadingCache<String, CommentsInfo> commentsCache = Caffeine.newBuilder() public static final LoadingCache<String, CommentsInfo> commentsCache = Caffeine.newBuilder()
.expireAfterWrite(1, TimeUnit.HOURS).maximumSize(1000) .expireAfterWrite(1, TimeUnit.HOURS)
.build(key -> CommentsInfo.getInfo("https://www.youtube.com/watch?v=" + key)); .scheduler(Scheduler.systemScheduler())
.maximumSize(1000).build(key -> CommentsInfo.getInfo("https://www.youtube.com/watch?v=" + key));
public static final byte[] streamsResponse(String videoId) throws Exception { public static final byte[] streamsResponse(String videoId) throws Exception {
@ -414,22 +416,22 @@ public class ResponseHelper {
info.getRelatedItems().forEach(item -> { info.getRelatedItems().forEach(item -> {
switch (item.getInfoType()) { switch (item.getInfoType()) {
case STREAM: case STREAM:
items.add(collectRelatedStream(item)); items.add(collectRelatedStream(item));
break; break;
case CHANNEL: case CHANNEL:
ChannelInfoItem channel = (ChannelInfoItem) item; ChannelInfoItem channel = (ChannelInfoItem) item;
items.add(new SearchChannel(item.getName(), rewriteURL(item.getThumbnailUrl()), items.add(new SearchChannel(item.getName(), rewriteURL(item.getThumbnailUrl()),
substringYouTube(item.getUrl()), channel.getDescription(), channel.getSubscriberCount(), substringYouTube(item.getUrl()), channel.getDescription(), channel.getSubscriberCount(),
channel.getStreamCount(), channel.isVerified())); channel.getStreamCount(), channel.isVerified()));
break; break;
case PLAYLIST: case PLAYLIST:
PlaylistInfoItem playlist = (PlaylistInfoItem) item; PlaylistInfoItem playlist = (PlaylistInfoItem) item;
items.add(new SearchPlaylist(item.getName(), rewriteURL(item.getThumbnailUrl()), items.add(new SearchPlaylist(item.getName(), rewriteURL(item.getThumbnailUrl()),
substringYouTube(item.getUrl()), playlist.getUploaderName(), playlist.getStreamCount())); substringYouTube(item.getUrl()), playlist.getUploaderName(), playlist.getStreamCount()));
break; break;
default: default:
break; break;
} }
}); });
@ -452,22 +454,22 @@ public class ResponseHelper {
pages.getItems().forEach(item -> { pages.getItems().forEach(item -> {
switch (item.getInfoType()) { switch (item.getInfoType()) {
case STREAM: case STREAM:
items.add(collectRelatedStream(item)); items.add(collectRelatedStream(item));
break; break;
case CHANNEL: case CHANNEL:
ChannelInfoItem channel = (ChannelInfoItem) item; ChannelInfoItem channel = (ChannelInfoItem) item;
items.add(new SearchChannel(item.getName(), rewriteURL(item.getThumbnailUrl()), items.add(new SearchChannel(item.getName(), rewriteURL(item.getThumbnailUrl()),
substringYouTube(item.getUrl()), channel.getDescription(), channel.getSubscriberCount(), substringYouTube(item.getUrl()), channel.getDescription(), channel.getSubscriberCount(),
channel.getStreamCount(), channel.isVerified())); channel.getStreamCount(), channel.isVerified()));
break; break;
case PLAYLIST: case PLAYLIST:
PlaylistInfoItem playlist = (PlaylistInfoItem) item; PlaylistInfoItem playlist = (PlaylistInfoItem) item;
items.add(new SearchPlaylist(item.getName(), rewriteURL(item.getThumbnailUrl()), items.add(new SearchPlaylist(item.getName(), rewriteURL(item.getThumbnailUrl()),
substringYouTube(item.getUrl()), playlist.getUploaderName(), playlist.getStreamCount())); substringYouTube(item.getUrl()), playlist.getUploaderName(), playlist.getStreamCount()));
break; break;
default: default:
break; break;
} }
}); });
@ -755,7 +757,7 @@ public class ResponseHelper {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
List<Object[]> queryResults = s.createNativeQuery( List<Object[]> queryResults = s.createNativeQuery(
"select Video.*, Channel.* from videos as Video left join channels as Channel on Video.uploader_id = Channel.uploader_id inner join users_subscribed on users_subscribed.channel = Channel.uploader_id where users_subscribed.subscriber = :user") "select Video.*, Channel.* from videos as Video left join channels as Channel on Video.uploader_id = Channel.uploader_id inner join users_subscribed on users_subscribed.channel = Channel.uploader_id where users_subscribed.subscriber = :user")
.setParameter("user", user.getId()).addEntity("Video", Video.class) .setParameter("user", user.getId()).addEntity("Video", Video.class)
.addEntity("Channel", me.kavin.piped.utils.obj.db.Channel.class).getResultList(); .addEntity("Channel", me.kavin.piped.utils.obj.db.Channel.class).getResultList();
@ -805,7 +807,7 @@ public class ResponseHelper {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
List<Object[]> queryResults = s.createNativeQuery( List<Object[]> queryResults = s.createNativeQuery(
"select Video.*, Channel.* from videos as Video left join channels as Channel on Video.uploader_id = Channel.uploader_id inner join users_subscribed on users_subscribed.channel = Channel.uploader_id where users_subscribed.subscriber = :user") "select Video.*, Channel.* from videos as Video left join channels as Channel on Video.uploader_id = Channel.uploader_id inner join users_subscribed on users_subscribed.channel = Channel.uploader_id where users_subscribed.subscriber = :user")
.setParameter("user", user.getId()).addEntity("Video", Video.class) .setParameter("user", user.getId()).addEntity("Video", Video.class)
.addEntity("Channel", me.kavin.piped.utils.obj.db.Channel.class).getResultList(); .addEntity("Channel", me.kavin.piped.utils.obj.db.Channel.class).getResultList();
@ -986,7 +988,7 @@ public class ResponseHelper {
return new JSONObject(Constants.h2client.send(HttpRequest return new JSONObject(Constants.h2client.send(HttpRequest
.newBuilder(URI.create("https://api.lbry.com/yt/resolve?video_ids=" + URLUtils.silentEncode(videoId))) .newBuilder(URI.create("https://api.lbry.com/yt/resolve?video_ids=" + URLUtils.silentEncode(videoId)))
.setHeader("User-Agent", Constants.USER_AGENT).build(), BodyHandlers.ofString()).body()) .setHeader("User-Agent", Constants.USER_AGENT).build(), BodyHandlers.ofString()).body())
.getJSONObject("data").getJSONObject("videos").optString(videoId); .getJSONObject("data").getJSONObject("videos").optString(videoId);
} }
private static final String getLBRYStreamURL(String lbryId) private static final String getLBRYStreamURL(String lbryId)
@ -1017,7 +1019,7 @@ public class ResponseHelper {
} }
private static void handleNewVideo(StreamInfo info, long time, me.kavin.piped.utils.obj.db.Channel channel, private static void handleNewVideo(StreamInfo info, long time, me.kavin.piped.utils.obj.db.Channel channel,
Session s) { Session s) {
if (channel == null) if (channel == null)
channel = DatabaseHelper.getChannelFromId(s, channel = DatabaseHelper.getChannelFromId(s,