kavin/Piped-Backend
1
0
Fork 0
mirror of https://github.com/TeamPiped/Piped-Backend.git synced 2025-01-09 19:10:29 +05:30
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into oidc

Browse Source
This commit is contained in:
Kavin 2023-08-05 16:34:43 +01:00
commit 97889f393e
No known key found for this signature in database
GPG Key ID: 6E4598CA5C92C41F
22 changed files with 318 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile
View File

@ -11,10 +11,12 @@ FROM eclipse-temurin:17-jre
WORKDIR /app/
COPY hotspot-entrypoint.sh /
COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar
COPY VERSION .
EXPOSE 8080
CMD java -server -Xmx1G -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -Xshare:on -jar /app/piped.jar
ENTRYPOINT ["/hotspot-entrypoint.sh"]

4
Dockerfile.azul
View File

@ -11,10 +11,12 @@ FROM azul/zulu-openjdk:17-jre-headless-latest
WORKDIR /app/
COPY hotspot-entrypoint.sh /
COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar
COPY VERSION .
EXPOSE 8080
CMD java -server -Xmx1G -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -jar /app/piped.jar
ENTRYPOINT ["/hotspot-entrypoint.sh"]

4
Dockerfile.azul.ci
View File

@ -2,10 +2,12 @@ FROM azul/zulu-openjdk:17-jre-headless-latest
WORKDIR /app/
COPY hotspot-entrypoint.sh /
COPY ./piped.jar /app/piped.jar
COPY VERSION .
EXPOSE 8080
CMD java -server -Xmx1G -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -jar /app/piped.jar
ENTRYPOINT ["/hotspot-entrypoint.sh"]

4
Dockerfile.ci
View File

@ -2,10 +2,12 @@ FROM eclipse-temurin:17-jre
WORKDIR /app/
COPY hotspot-entrypoint.sh /
COPY ./piped.jar /app/piped.jar
COPY VERSION .
EXPOSE 8080
CMD java -server -Xmx1G -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -Xshare:on -jar /app/piped.jar
ENTRYPOINT ["/hotspot-entrypoint.sh"]

16
build.gradle
View File

@ -11,14 +11,14 @@ repositories {
}
dependencies {
implementation 'org.apache.commons:commons-lang3:3.12.0'
implementation 'org.apache.commons:commons-lang3:3.13.0'
implementation 'org.apache.commons:commons-text:1.10.0'
implementation 'commons-io:commons-io:2.12.0'
implementation 'it.unimi.dsi:fastutil-core:8.5.12'
implementation 'commons-codec:commons-codec:1.16.0'
implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:bdd1366285614f56a245e3baceb26fb8864bda27'
implementation 'com.github.FireMasterK:nanojson:01934924442edda6952f3bedf80ba9e969cba8bc'
implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:88ceba0da4a48b5f4ffecb3b5b2f36f95ec53afe'
implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7'
implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2'
implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.2'
implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
@ -31,17 +31,17 @@ dependencies {
implementation 'io.activej:activej-launchers-http:5.5'
implementation 'org.hsqldb:hsqldb:2.7.2'
implementation 'org.postgresql:postgresql:42.6.0'
implementation 'org.hibernate:hibernate-core:6.2.5.Final'
implementation 'org.hibernate:hibernate-hikaricp:6.2.5.Final'
implementation 'org.hibernate:hibernate-core:6.2.7.Final'
implementation 'org.hibernate:hibernate-hikaricp:6.2.7.Final'
implementation 'com.zaxxer:HikariCP:5.0.1'
implementation 'org.springframework.security:spring-security-crypto:6.1.1'
implementation 'org.springframework.security:spring-security-crypto:6.1.2'
implementation 'commons-logging:commons-logging:1.2'
implementation(platform("com.squareup.okhttp3:okhttp-bom:4.11.0"))
implementation 'com.squareup.okhttp3:okhttp'
implementation 'com.squareup.okhttp3:okhttp-brotli'
implementation 'com.nimbusds:oauth2-oidc-sdk:10.9.1'
implementation 'io.sentry:sentry:6.25.0'
implementation 'rocks.kavin:reqwest4j:1.0.6'
implementation 'io.sentry:sentry:6.28.0'
implementation 'rocks.kavin:reqwest4j:1.0.7'
implementation 'io.minio:minio:8.5.4'
}

2
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.2-bin.zip
distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.2.1-bin.zip
networkTimeout=10000
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME

5
hotspot-entrypoint.sh Executable file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env sh
MAX_MEMORY=${MAX_MEMORY:-1G}
java -server -Xmx"$MAX_MEMORY" -XX:+UnlockExperimentalVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -jar /app/piped.jar

38
src/main/java/me/kavin/piped/Main.java
View File

@ -23,6 +23,8 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExt
import rocks.kavin.reqwest4j.ReqwestUtils;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@ -36,8 +38,6 @@ public class Main {
YoutubeStreamExtractor.forceFetchAndroidClient(true);
YoutubeStreamExtractor.forceFetchIosClient(true);
ReqwestUtils.init(Constants.REQWEST_PROXY);
Sentry.init(options -> {
options.setDsn(Constants.SENTRY_DSN);
options.setRelease(Constants.VERSION);
@ -102,15 +102,39 @@ public class Main {
Collections.shuffle(channelIds);
channelIds.stream()
.parallel()
.forEach(id -> Multithreading.runAsyncLimitedPubSub(() -> {
var queue = new ConcurrentLinkedQueue<>(channelIds);
System.out.println("PubSub: queue size - " + queue.size() + " channels");
for (int i = 0; i < Runtime.getRuntime().availableProcessors(); i++) {
new Thread(() -> {
Object o = new Object();
String channelId;
while ((channelId = queue.poll()) != null) {
try {
PubSubHelper.subscribePubSub(id);
CompletableFuture<?> future = PubSubHelper.subscribePubSub(channelId);
if (future == null)
continue;
future.whenComplete((resp, throwable) -> {
synchronized (o) {
o.notify();
}
});
synchronized (o) {
o.wait();
}
} catch (Exception e) {
ExceptionHandler.handle(e);
}
}));
}
}, "PubSub-" + i).start();
}
} catch (Exception e) {
e.printStackTrace();

3
src/main/java/me/kavin/piped/consts/Constants.java
View File

@ -21,6 +21,7 @@ import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import rocks.kavin.reqwest4j.ReqwestUtils;
import java.io.File;
import java.io.FileReader;
@ -134,6 +135,7 @@ public class Constants {
PUBSUB_URL = getProperty(prop, "PUBSUB_URL", PUBLIC_URL);
PUBSUB_HUB_URL = getProperty(prop, "PUBSUB_HUB_URL", "https://pubsubhubbub.appspot.com/subscribe");
REQWEST_PROXY = getProperty(prop, "REQWEST_PROXY");
ReqwestUtils.init(REQWEST_PROXY);
FRONTEND_URL = getProperty(prop, "FRONTEND_URL", "https://piped.video");
COMPROMISED_PASSWORD_CHECK = Boolean.parseBoolean(getProperty(prop, "COMPROMISED_PASSWORD_CHECK", "true"));
DISABLE_REGISTRATION = Boolean.parseBoolean(getProperty(prop, "DISABLE_REGISTRATION", "false"));
@ -195,6 +197,7 @@ public class Constants {
.map(JsonNodeFactory.instance::textNode).toList()
);
frontendProperties.put("s3Enabled", S3_CLIENT != null);
frontendProperties.put("registrationDisabled", DISABLE_REGISTRATION);
// transform hibernate properties for legacy configurations
hibernateProperties.replace("hibernate.dialect",

29
src/main/java/me/kavin/piped/server/ServerLauncher.java
View File

@ -42,6 +42,7 @@ import com.nimbusds.oauth2.sdk.id.*;
import java.io.ByteArrayInputStream;
import java.net.InetSocketAddress;
import java.net.URI;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
@ -85,18 +86,21 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher {
.map(GET, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> {
var topic = request.getQueryParameter("hub.topic");
if (topic != null)
Multithreading.runAsync(() -> {
Multithreading.runAsyncLimited(() -> {
String channelId = StringUtils.substringAfter(topic, "channel_id=");
PubSubHelper.updatePubSub(channelId);
});
return HttpResponse.ok200().withPlainText(Objects.requireNonNull(request.getQueryParameter("hub.challenge")));
var challenge = request.getQueryParameter("hub.challenge");
return HttpResponse.ok200()
.withPlainText(Objects.requireNonNullElse(challenge, "ok"));
})).map(POST, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> {
try {
SyndFeed feed = new SyndFeedInput().build(
new InputSource(new ByteArrayInputStream(request.loadBody().getResult().asArray())));
Multithreading.runAsync(() -> {
Multithreading.runAsyncLimited(() -> {
for (var entry : feed.getEntries()) {
String url = entry.getLinks().get(0).getHref();
String videoId = StringUtils.substring(url, -11);
@ -104,7 +108,7 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher {
if (DatabaseHelper.doesVideoExist(s, videoId))
continue;
}
Multithreading.runAsync(() -> {
Multithreading.runAsyncLimited(() -> {
try {
Sentry.setExtra("videoId", videoId);
var extractor = YOUTUBE_SERVICE.getStreamExtractor("https://youtube.com/watch?v=" + videoId);
@ -155,6 +159,23 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher {
} catch (Exception e) {
return getErrorResponse(e, request.getPath());
}
})).map(GET, "/dearrow", AsyncServlet.ofBlocking(executor, request -> {
try {
var videoIds = getArray(request.getQueryParameter("videoIds"));
return getJsonResponse(
SponsorBlockUtils.getDeArrowedInfo(videoIds)
.thenApplyAsync(json -> {
try {
return mapper.writeValueAsBytes(json);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}).get(),
"public, max-age=3600");
} catch (Exception e) {
return getErrorResponse(e, request.getPath());
}
})).map(GET, "/streams/:videoId", AsyncServlet.ofBlocking(executor, request -> {
try {
return getJsonResponse(StreamHandlers.streamsResponse(request.getPathParameter("videoId")),

4
src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java
View File

@ -15,10 +15,10 @@ import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.channel.ChannelInfo;
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;

50
src/main/java/me/kavin/piped/utils/Alea.java Normal file
View File

@ -0,0 +1,50 @@
package me.kavin.piped.utils;
public class Alea {
private static final double NORM32 = 2.3283064365386963e-10; // 2^-32
private double s0, s1, s2;
private int c = 1;
public double next() {
double t = 2091639.0 * s0 + c * NORM32; // 2^-32
s0 = s1;
s1 = s2;
return s2 = t - (c = (int) t);
}
public Alea(String seed) {
s0 = mash(" ");
s1 = mash(" ");
s2 = mash(" ");
s0 -= mash(seed);
if (s0 < 0)
s0 += 1;
s1 -= mash(seed);
if (s1 < 0)
s1 += 1;
s2 -= mash(seed);
if (s2 < 0)
s2 += 1;
}
private long n = 0xefc8249dL;
public double mash(String x) {
double h;
for (char c : x.toCharArray()) {
n += c;
h = 0.02519603282416938 * n;
n = (long) h;
h -= n;
h *= n;
n = (long) h;
h -= n;
n += h * 0x100000000L;
}
return n * 2.3283064365386963e-10; // 2^-32
}
}

10
src/main/java/me/kavin/piped/utils/CollectionUtils.java
View File

@ -4,7 +4,6 @@ import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import me.kavin.piped.utils.obj.*;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
@ -16,7 +15,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Optional;
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
import static me.kavin.piped.utils.URLUtils.*;
public class CollectionUtils {
@ -66,12 +64,18 @@ public class CollectionUtils {
final List<ContentItem> relatedStreams = collectRelatedItems(info.getRelatedItems());
final List<MetaInfo> metaInfo = new ObjectArrayList<>();
info.getMetaInfo().forEach(metaInfoItem -> metaInfo.add(new MetaInfo(metaInfoItem.getTitle(), metaInfoItem.getContent().getContent(),
metaInfoItem.getUrls(), metaInfoItem.getUrlTexts()
)));
return new Streams(info.getName(), info.getDescription().getContent(),
info.getTextualUploadDate(), info.getUploaderName(), substringYouTube(info.getUploaderUrl()),
rewriteURL(info.getUploaderAvatarUrl()), rewriteURL(info.getThumbnailUrl()), info.getDuration(),
info.getViewCount(), info.getLikeCount(), info.getDislikeCount(), info.getUploaderSubscriberCount(), info.isUploaderVerified(),
audioStreams, videoStreams, relatedStreams, subtitles, livestream, rewriteVideoURL(info.getHlsUrl()),
rewriteVideoURL(info.getDashMpdUrl()), null, info.getCategory(), chapters, previewFrames);
rewriteVideoURL(info.getDashMpdUrl()), null, info.getCategory(), info.getLicence(),
info.getPrivacy().name().toLowerCase(), info.getTags(), metaInfo, chapters, previewFrames);
}
public static List<ContentItem> collectRelatedItems(List<? extends InfoItem> items) {

3
src/main/java/me/kavin/piped/utils/DatabaseHelper.java
View File

@ -10,9 +10,8 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.hibernate.SharedSessionContract;
import org.hibernate.StatelessSession;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfo;
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;

9
src/main/java/me/kavin/piped/utils/Multithreading.java
View File

@ -1,8 +1,6 @@
package me.kavin.piped.utils;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.*;
import java.util.function.Supplier;
public class Multithreading {
@ -12,11 +10,16 @@ public class Multithreading {
.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 8);
private static final ExecutorService esLimitedPubSub = Executors
.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
private static final ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors());
public static void runAsync(final Runnable runnable) {
es.submit(runnable);
}
public static void runAsyncTask(final ForkJoinTask<?> task) {
forkJoinPool.submit(task);
}
public static void runAsyncLimited(final Runnable runnable) {
esLimited.submit(runnable);
}

45
src/main/java/me/kavin/piped/utils/PubSubHelper.java
View File

@ -6,16 +6,21 @@ import okhttp3.FormBody;
import okio.Buffer;
import org.hibernate.StatelessSession;
import rocks.kavin.reqwest4j.ReqwestUtils;
import rocks.kavin.reqwest4j.Response;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
public class PubSubHelper {
public static void subscribePubSub(String channelId) throws IOException {
@Nullable
public static CompletableFuture<Response> subscribePubSub(String channelId) throws IOException {
if (!ChannelHelpers.isValidId(channelId))
return;
return null;
PubSub pubsub = DatabaseHelper.getPubSubFromId(channelId);
@ -44,30 +49,32 @@ public class PubSubHelper {
var buffer = new Buffer();
formBuilder.build().writeTo(buffer);
ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of())
.thenAccept(resp -> {
if (resp.status() != 202)
var completableFuture = ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of());
completableFuture
.whenComplete((resp, e) -> {
if (e != null) {
ExceptionHandler.handle((Exception) e);
return;
}
if (resp != null && resp.status() != 202)
System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body()));
})
.exceptionally(e -> {
ExceptionHandler.handle((Exception) e);
return null;
});
return completableFuture;
}
return null;
}
public static void updatePubSub(String channelId) {
var pubsub = DatabaseHelper.getPubSubFromId(channelId);
try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) {
s.beginTransaction();
if (pubsub == null) {
pubsub = new PubSub(channelId, System.currentTimeMillis());
s.insert(pubsub);
} else {
pubsub.setSubbedAt(System.currentTimeMillis());
s.update(pubsub);
}
s.getTransaction().commit();
var tr = s.beginTransaction();
s.createNativeMutationQuery("INSERT INTO pubsub (id, subbed_at) VALUES (?, ?) " +
"ON CONFLICT (id) DO UPDATE SET subbed_at = excluded.subbed_at")
.setParameter(1, channelId)
.setParameter(2, System.currentTimeMillis())
.executeUpdate();
tr.commit();
}
}
}

4
src/main/java/me/kavin/piped/utils/RequestUtils.java
View File

@ -42,13 +42,13 @@ public class RequestUtils {
}
}
public static CompletableFuture<JsonNode> sendGetJson(String url) throws Exception {
public static CompletableFuture<JsonNode> sendGetJson(String url) {
return ReqwestUtils.fetch(url, "GET", null, Map.of()).thenApply(Response::body).thenApplyAsync(resp -> {
try {
return mapper.readTree(resp);
} catch (Exception e) {
throw new RuntimeException("Failed to parse JSON", e);
}
}, Multithreading.getCachedExecutor());
});
}
}

82
src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java
View File

@ -1,5 +1,10 @@
package me.kavin.piped.utils;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.NullNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
import me.kavin.piped.consts.Constants;
import me.kavin.piped.utils.resp.InvalidRequestResponse;
import me.kavin.piped.utils.resp.SimpleErrorMessage;
@ -7,7 +12,12 @@ import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import java.io.IOException;
import java.util.Arrays;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ForkJoinTask;
import static me.kavin.piped.consts.Constants.SPONSORBLOCK_SERVERS;
import static me.kavin.piped.consts.Constants.mapper;
public class SponsorBlockUtils {
@ -46,4 +56,76 @@ public class SponsorBlockUtils {
return null;
}
public static CompletableFuture<ObjectNode> getDeArrowedInfo(String[] videoIds) {
ObjectNode objectNode = mapper.createObjectNode();
var futures = Arrays.stream(videoIds)
.map(id -> getDeArrowedInfo(id, SPONSORBLOCK_SERVERS.toArray(new String[0]))
.thenAcceptAsync(jsonNode -> objectNode.set(id, jsonNode.orElse(NullNode.getInstance())))
)
.toArray(CompletableFuture[]::new);
return CompletableFuture.allOf(futures)
.thenApplyAsync(v -> objectNode, Multithreading.getCachedExecutor());
}
private static CompletableFuture<Optional<JsonNode>> getDeArrowedInfo(String videoId, String[] servers) {
String hash = DigestUtils.sha256Hex(videoId);
CompletableFuture<Optional<JsonNode>> future = new CompletableFuture<>();
var task = ForkJoinTask.adapt(() -> {
fetchDeArrowedCf(future, videoId, hash, servers);
});
Multithreading.runAsyncTask(task);
return future;
}
private static final ObjectNode EMPTY_DEARROWED_INFO;
static {
EMPTY_DEARROWED_INFO = mapper.createObjectNode();
EMPTY_DEARROWED_INFO.putArray("titles");
EMPTY_DEARROWED_INFO.putArray("thumbnails");
EMPTY_DEARROWED_INFO.set("videoDuration", NullNode.getInstance());
}
private static void fetchDeArrowedCf(CompletableFuture<Optional<JsonNode>> future, String videoId, String hash, String[] servers) {
var completableFuture = RequestUtils.sendGetJson(servers[0] + "/api/branding/" + URLUtils.silentEncode(hash.substring(0, 4)))
.thenApplyAsync(json -> json.has(videoId) ? Optional.of(json.get(videoId)) : Optional.<JsonNode>empty());
completableFuture.thenAcceptAsync(optional -> optional.ifPresent(jsonNode -> {
ArrayNode nodes = (ArrayNode) jsonNode.get("thumbnails");
for (JsonNode node : nodes) {
if (!node.get("original").booleanValue())
((ObjectNode) node).set("thumbnail", new TextNode(URLUtils.rewriteURL("https://dearrow-thumb.ajay.app/api/v1/getThumbnail?videoID=" + videoId + "&time=" + node.get("timestamp").asText())));
}
}));
completableFuture = completableFuture.thenApplyAsync(optional -> {
if (optional.isEmpty()) {
var clone = EMPTY_DEARROWED_INFO.deepCopy();
clone.put("randomTime", new Alea(videoId).next());
return Optional.of(clone);
} else
return optional;
});
completableFuture.whenComplete((optional, throwable) -> {
if (throwable == null)
future.complete(optional);
else {
if (servers.length == 1)
future.completeExceptionally(new Exception("All SponsorBlock servers are down"));
else
fetchDeArrowedCf(future, videoId, hash, Arrays.copyOfRange(servers, 1, servers.length));
}
});
}
}

45
src/main/java/me/kavin/piped/utils/VideoHelpers.java
View File

@ -6,7 +6,6 @@ import me.kavin.piped.consts.Constants;
import me.kavin.piped.utils.obj.db.Video;
import org.apache.commons.lang3.StringUtils;
import org.hibernate.StatelessSession;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -49,14 +48,7 @@ public class VideoHelpers {
Video video = new Video(info.getId(), info.getName(), info.getViewCount(), info.getDuration(),
Math.max(infoTime, time), info.getThumbnailUrl(), info.isShortFormContent(), channel);
var tr = s.beginTransaction();
try {
s.insert(video);
tr.commit();
} catch (Exception e) {
tr.rollback();
ExceptionHandler.handle(e);
}
insertVideo(video);
return;
}
}
@ -87,14 +79,7 @@ public class VideoHelpers {
Video video = new Video(extractor.getId(), extractor.getName(), extractor.getViewCount(), extractor.getLength(),
Math.max(infoTime, time), extractor.getThumbnailUrl(), isShort, channel);
var tr = s.beginTransaction();
try {
s.insert(video);
tr.commit();
} catch (Exception e) {
tr.rollback();
ExceptionHandler.handle(e);
}
insertVideo(video);
}
}
@ -169,4 +154,30 @@ public class VideoHelpers {
return updated > 0;
}
}
public static void insertVideo(Video video) {
try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) {
var tr = s.beginTransaction();
try {
s.createNativeMutationQuery(
"INSERT INTO videos (uploader_id,duration,is_short,thumbnail,title,uploaded,views,id) values " +
"(:uploader_id,:duration,:is_short,:thumbnail,:title,:uploaded,:views,:id) ON CONFLICT (id) DO UPDATE SET " +
"duration = excluded.duration, title = excluded.title, views = excluded.views"
)
.setParameter("uploader_id", video.getChannel().getUploaderId())
.setParameter("duration", video.getDuration())
.setParameter("is_short", video.isShort())
.setParameter("thumbnail", video.getThumbnail())
.setParameter("title", video.getTitle())
.setParameter("uploaded", video.getUploaded())
.setParameter("views", video.getViews())
.setParameter("id", video.getId())
.executeUpdate();
tr.commit();
} catch (Exception e) {
tr.rollback();
ExceptionHandler.handle(e);
}
}
}
}

15
src/main/java/me/kavin/piped/utils/obj/MetaInfo.java Normal file
View File

@ -0,0 +1,15 @@
package me.kavin.piped.utils.obj;
import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
import java.net.URL;
import java.util.List;
@NoArgsConstructor
@AllArgsConstructor
public class MetaInfo {
public String title, description;
public List<URL> urls;
public List<String> urlTexts;
}

13
src/main/java/me/kavin/piped/utils/obj/Streams.java
View File

@ -9,7 +9,11 @@ import java.util.List;
public class Streams {
public String title, description, uploadDate, uploader, uploaderUrl, uploaderAvatar, thumbnailUrl, hls, dash,
lbryId, category;
lbryId, category, license, visibility;
public List<String> tags;
public List<MetaInfo> metaInfo;
public boolean uploaderVerified;
@ -33,7 +37,8 @@ public class Streams {
String uploaderAvatar, String thumbnailUrl, long duration, long views, long likes, long dislikes, long uploaderSubscriberCount,
boolean uploaderVerified, List<PipedStream> audioStreams, List<PipedStream> videoStreams,
List<ContentItem> relatedStreams, List<Subtitle> subtitles, boolean livestream, String hls, String dash,
String lbryId, String category, List<ChapterSegment> chapters, List<PreviewFrames> previewFrames) {
String lbryId, String category, String license, String visibility, List<String> tags, List<MetaInfo> metaInfo,
List<ChapterSegment> chapters, List<PreviewFrames> previewFrames) {
this.title = title;
this.description = description;
this.uploadDate = uploadDate;
@ -58,5 +63,9 @@ public class Streams {
this.chapters = chapters;
this.previewFrames = previewFrames;
this.category = category;
this.license = license;
this.tags = tags;
this.metaInfo = metaInfo;
this.visibility = visibility;
}
}

3
testing/api-test.sh
View File

@ -39,6 +39,9 @@ curl "${CURLOPTS[@]}" $HOST/clips/Ugkx71jS31nwsms_Cc65oi7yXF1mILflhhrO || exit 1
# Streams
curl "${CURLOPTS[@]}" $HOST/streams/BtN-goy9VOY || exit 1
# Streams with meta info
curl "${CURLOPTS[@]}" $HOST/streams/cJ9to6EmElQ || exit 1
# Comments
curl "${CURLOPTS[@]}" $HOST/comments/BtN-goy9VOY || exit 1