kavin/Piped-Backend
1
0
Fork 0
mirror of https://github.com/TeamPiped/Piped-Backend.git synced 2025-01-09 19:10:29 +05:30
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into oidc

Browse Source 
# Conflicts:
#	build.gradle
This commit is contained in:
Jeidnx 2023-07-04 21:24:27 +02:00
commit 53d9b9d662
No known key found for this signature in database
GPG Key ID: 0E9E697B7E99DF39
17 changed files with 153 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
build.gradle
View File

@ -1,7 +1,7 @@
plugins {
id "com.github.johnrengelman.shadow" version "8.1.1"
id "java"
id "io.freefair.lombok" version "8.0.1"
id "io.freefair.lombok" version "8.1.0"
id "eclipse"
}
@ -15,9 +15,9 @@ dependencies {
implementation 'org.apache.commons:commons-text:1.10.0'
implementation 'commons-io:commons-io:2.12.0'
implementation 'it.unimi.dsi:fastutil-core:8.5.12'
implementation 'commons-codec:commons-codec:1.15'
implementation 'commons-codec:commons-codec:1.16.0'
implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:4ab4bf51ca6e924945a071d359b1ee5cdf9c7605'
implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:bdd1366285614f56a245e3baceb26fb8864bda27'
implementation 'com.github.FireMasterK:nanojson:01934924442edda6952f3bedf80ba9e969cba8bc'
implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2'
implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.2'
@ -34,15 +34,15 @@ dependencies {
implementation 'org.hibernate:hibernate-core:6.2.5.Final'
implementation 'org.hibernate:hibernate-hikaricp:6.2.5.Final'
implementation 'com.zaxxer:HikariCP:5.0.1'
implementation 'org.springframework.security:spring-security-crypto:6.1.0'
implementation 'org.springframework.security:spring-security-crypto:6.1.1'
implementation 'commons-logging:commons-logging:1.2'
implementation(platform("com.squareup.okhttp3:okhttp-bom:4.11.0"))
implementation 'com.squareup.okhttp3:okhttp'
implementation 'com.squareup.okhttp3:okhttp-brotli'
implementation 'io.sentry:sentry:6.23.0'
implementation 'rocks.kavin:reqwest4j:1.0.4'
implementation 'io.minio:minio:8.5.3'
implementation 'com.nimbusds:oauth2-oidc-sdk:10.9.1'
implementation 'io.sentry:sentry:6.25.0'
implementation 'rocks.kavin:reqwest4j:1.0.6'
implementation 'io.minio:minio:8.5.4'
}
shadowJar {

4
config.properties
View File

@ -6,8 +6,8 @@ HTTP_WORKERS:2
# Proxy
PROXY_PART:https://pipedproxy-cdg.kavin.rocks
# Outgoing HTTP Proxy - eg: 127.0.0.1:8118
#HTTP_PROXY: 127.0.0.1:8118
# Outgoing proxy to be used by reqwest4j - eg: socks5://127.0.0.1:1080
#REQWEST_PROXY: socks5://127.0.0.1:1080
# Captcha Parameters
CAPTCHA_BASE_URL:https://api.capmonster.cloud/

BIN
gradle/wrapper/gradle-wrapper.jar vendored
View File

Binary file not shown.

3
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,6 +1,7 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.1.1-bin.zip
distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.2-bin.zip
networkTimeout=10000
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

5
gradlew vendored
View File

@ -130,10 +130,13 @@ location of your Java installation."
fi
else
JAVACMD=java
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
if ! command -v java >/dev/null 2>&1
then
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
fi
# Increase the maximum file descriptors if we can.

3
src/main/java/me/kavin/piped/Main.java
View File

@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import rocks.kavin.reqwest4j.ReqwestUtils;
import java.util.*;
import java.util.concurrent.TimeUnit;
@ -35,6 +36,8 @@ public class Main {
YoutubeStreamExtractor.forceFetchAndroidClient(true);
YoutubeStreamExtractor.forceFetchIosClient(true);
ReqwestUtils.init(Constants.REQWEST_PROXY);
Sentry.init(options -> {
options.setDsn(Constants.SENTRY_DSN);
options.setRelease(Constants.VERSION);

13
src/main/java/me/kavin/piped/consts/Constants.java
View File

@ -52,7 +52,7 @@ public class Constants {
public static final String PUBSUB_HUB_URL;
public static final String HTTP_PROXY;
public static final String REQWEST_PROXY;
public static final String FRONTEND_URL;
@ -133,7 +133,7 @@ public class Constants {
PUBLIC_URL = getProperty(prop, "API_URL");
PUBSUB_URL = getProperty(prop, "PUBSUB_URL", PUBLIC_URL);
PUBSUB_HUB_URL = getProperty(prop, "PUBSUB_HUB_URL", "https://pubsubhubbub.appspot.com/subscribe");
HTTP_PROXY = getProperty(prop, "HTTP_PROXY");
REQWEST_PROXY = getProperty(prop, "REQWEST_PROXY");
FRONTEND_URL = getProperty(prop, "FRONTEND_URL", "https://piped.video");
COMPROMISED_PASSWORD_CHECK = Boolean.parseBoolean(getProperty(prop, "COMPROMISED_PASSWORD_CHECK", "true"));
DISABLE_REGISTRATION = Boolean.parseBoolean(getProperty(prop, "DISABLE_REGISTRATION", "false"));
@ -208,18 +208,11 @@ public class Constants {
OkHttpClient.Builder builder_noredir = new OkHttpClient.Builder()
.followRedirects(false)
.addInterceptor(BrotliInterceptor.INSTANCE);
if (HTTP_PROXY != null && HTTP_PROXY.contains(":")) {
String host = StringUtils.substringBefore(HTTP_PROXY, ":");
String port = StringUtils.substringAfter(HTTP_PROXY, ":");
InetSocketAddress sa = new InetSocketAddress(host, Integer.parseInt(port));
ProxySelector ps = ProxySelector.of(sa);
ProxySelector.setDefault(ps);
}
h2client = builder.build();
h2_no_redir_client = builder_noredir.build();
String temp = null;
try {
var html = RequestUtils.sendGet("https://www.youtube.com/");
var html = RequestUtils.sendGet("https://www.youtube.com/").get();
var regex = Pattern.compile("GL\":\"([A-Z]{2})\"", Pattern.MULTILINE);
var matcher = regex.matcher(html);
if (matcher.find()) {

6
src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java
View File

@ -150,7 +150,7 @@ public class StreamHandlers {
List<String> allowedCountries = new ObjectArrayList<>();
{
var restrictedTree = RequestUtils.sendGetJson(Constants.GEO_RESTRICTION_CHECKER_URL + "/api/region/check?video_id=" + videoId);
var restrictedTree = RequestUtils.sendGetJson(Constants.GEO_RESTRICTION_CHECKER_URL + "/api/region/check?video_id=" + videoId).get();
if (!restrictedTree.get("restricted").asBoolean()) {
assert exception != null;
throw (Exception) exception;
@ -357,7 +357,7 @@ public class StreamHandlers {
nextpage = mapper.writeValueAsString(page);
}
CommentsPage commentsItem = new CommentsPage(comments, nextpage, info.isCommentsDisabled());
CommentsPage commentsItem = new CommentsPage(comments, nextpage, info.isCommentsDisabled(), info.getCommentsCount());
return mapper.writeValueAsBytes(commentsItem);
@ -395,7 +395,7 @@ public class StreamHandlers {
nextpage = mapper.writeValueAsString(page);
}
CommentsPage commentsItem = new CommentsPage(comments, nextpage, false);
CommentsPage commentsItem = new CommentsPage(comments, nextpage, false, -1);
return mapper.writeValueAsBytes(commentsItem);

5
src/main/java/me/kavin/piped/server/handlers/auth/UserHandlers.java
View File

@ -28,7 +28,7 @@ public class UserHandlers {
private static final Argon2PasswordEncoder argon2PasswordEncoder = Argon2PasswordEncoder.defaultsForSpringSecurity_v5_8();
private static final BCryptPasswordEncoder bcryptPasswordEncoder = new BCryptPasswordEncoder();
public static byte[] registerResponse(String user, String pass) throws IOException {
public static byte[] registerResponse(String user, String pass) throws Exception {
if (Constants.DISABLE_REGISTRATION)
ExceptionHandler.throwErrorResponse(new DisabledRegistrationResponse());
@ -57,7 +57,8 @@ public class UserHandlers {
String suffix = sha1Hash.substring(5);
String[] entries = RequestUtils
.sendGet("https://api.pwnedpasswords.com/range/" + prefix, "github.com/TeamPiped/Piped-Backend")
.split("\n");
.thenApplyAsync(str -> str.split("\n"))
.get();
for (String entry : entries)
if (StringUtils.substringBefore(entry, ":").equals(suffix))
ExceptionHandler.throwErrorResponse(new CompromisedPasswordResponse());

138
src/main/java/me/kavin/piped/utils/DownloaderImpl.java
View File

@ -1,13 +1,6 @@
package me.kavin.piped.utils;
import com.grack.nanojson.JsonParserException;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
import me.kavin.piped.consts.Constants;
import me.kavin.piped.utils.obj.SolvedCaptcha;
import okhttp3.FormBody;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Request;
import org.schabi.newpipe.extractor.downloader.Response;
@ -18,7 +11,7 @@ import java.io.IOException;
import java.net.HttpCookie;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.ExecutionException;
public class DownloaderImpl extends Downloader {
@ -41,70 +34,85 @@ public class DownloaderImpl extends Downloader {
request.headers().forEach((name, values) -> values.forEach(value -> headers.put(name, value)));
var resp = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers);
var future = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers);
if (resp.status() == 429) {
// Recaptcha solver code
// Commented out, as it hasn't been ported to reqwest4j yet
// Also, this was last seen a long time back
synchronized (cookie_lock) {
if (saved_cookie != null && saved_cookie.hasExpired()
|| (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
saved_cookie = null;
String redir_url = String.valueOf(resp.finalUrl());
if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
var formBuilder = new FormBody.Builder();
String sitekey = null, data_s = null;
for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) {
String name;
if (!(name = el.tagName()).equals("script")) {
if (name.equals("input"))
formBuilder.add(el.attr("name"), el.attr("value"));
else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
sitekey = el.attr("data-sitekey");
data_s = el.attr("data-s");
}
}
}
if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s))
throw new ReCaptchaException("Could not get recaptcha", redir_url);
SolvedCaptcha solved = null;
try {
solved = CaptchaSolver.solve(redir_url, sitekey, data_s);
} catch (JsonParserException | InterruptedException e) {
e.printStackTrace();
}
formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse());
var formReqBuilder = new okhttp3.Request.Builder()
.url("https://www.google.com/sorry/index")
.header("User-Agent", Constants.USER_AGENT)
.post(formBuilder.build());
var formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute();
saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
.substringAfter(formResponse.headers().get("Location"), "google_abuse=")))
.get(0);
cookie_received = System.currentTimeMillis();
}
if (saved_cookie != null) // call again as captcha has been solved or cookie has not expired.
execute(request);
}
}
// future.thenAcceptAsync(resp -> {
// if (resp.status() == 429) {
// synchronized (cookie_lock) {
//
// if (saved_cookie != null && saved_cookie.hasExpired()
// || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
// saved_cookie = null;
//
// String redir_url = String.valueOf(resp.finalUrl());
//
// if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
//
// var formBuilder = new FormBody.Builder();
// String sitekey = null, data_s = null;
//
// for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) {
// String name;
// if (!(name = el.tagName()).equals("script")) {
// if (name.equals("input"))
// formBuilder.add(el.attr("name"), el.attr("value"));
// else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
// sitekey = el.attr("data-sitekey");
// data_s = el.attr("data-s");
// }
// }
// }
//
// if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s))
// ExceptionHandler.handle(new ReCaptchaException("Could not get recaptcha", redir_url));
//
// SolvedCaptcha solved = null;
//
// try {
// solved = CaptchaSolver.solve(redir_url, sitekey, data_s);
// } catch (JsonParserException | InterruptedException | IOException e) {
// e.printStackTrace();
// }
//
// formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse());
//
// var formReqBuilder = new okhttp3.Request.Builder()
// .url("https://www.google.com/sorry/index")
// .header("User-Agent", Constants.USER_AGENT)
// .post(formBuilder.build());
//
// okhttp3.Response formResponse;
// try {
// formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute();
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
//
// saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
// .substringAfter(formResponse.headers().get("Location"), "google_abuse=")))
// .get(0);
// cookie_received = System.currentTimeMillis();
// }
// }
// }
// }, Multithreading.getCachedExecutor());
var responseFuture = future.thenApplyAsync(resp -> {
Map<String, List<String>> headerMap = resp.headers().entrySet().stream()
.collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll);
return new Response(resp.status(), null, headerMap, new String(resp.body()),
resp.finalUrl());
}, Multithreading.getCachedExecutor());
try {
return responseFuture.get();
} catch (InterruptedException | ExecutionException e) {
throw new IOException(e);
}
}
}

21
src/main/java/me/kavin/piped/utils/LbryHelper.java
View File

@ -3,8 +3,8 @@ package me.kavin.piped.utils;
import me.kavin.piped.consts.Constants;
import org.apache.commons.lang3.StringUtils;
import rocks.kavin.reqwest4j.ReqwestUtils;
import rocks.kavin.reqwest4j.Response;
import java.io.IOException;
import java.net.URI;
import java.util.Map;
@ -13,19 +13,20 @@ import static me.kavin.piped.utils.URLUtils.silentEncode;
public class LbryHelper {
public static String getLBRYId(String videoId) throws IOException {
public static String getLBRYId(String videoId) throws Exception {
if (Constants.DISABLE_LBRY)
return null;
return RequestUtils.sendGetJson("https://api.lbry.com/yt/resolve?video_ids=" + silentEncode(videoId))
.at("/data/videos")
.thenApplyAsync(json -> json.at("/data/videos")
.path(videoId)
.asText(null);
.asText(null)
).get();
}
public static String getLBRYStreamURL(String lbryId)
throws IOException {
throws Exception {
if (StringUtils.isEmpty(lbryId))
return null;
@ -41,7 +42,7 @@ public class LbryHelper {
.put("uri", "lbry://" + lbryId)
.put("save_file", true)
)
), Map.of("Content-Type", "application/json"));
), Map.of("Content-Type", "application/json")).get();
if (resp.status() / 100 == 2) {
return mapper.readTree(resp.body())
.at("/result/streaming_url")
@ -60,12 +61,12 @@ public class LbryHelper {
// LBRY provides non UTF-8 characters in the URL, which causes issues
streamUrl = new URI(streamUrl).toASCIIString();
var resp = ReqwestUtils.fetch(streamUrl, "HEAD", null, Map.of(
final String lastLocation = ReqwestUtils.fetch(streamUrl, "HEAD", null, Map.of(
"Origin", "https://odysee.com",
"Referer", "https://odysee.com/"
));
final String lastLocation = resp.finalUrl();
))
.thenApply(Response::finalUrl)
.get();
return streamUrl.equals(lastLocation) ? null : lastLocation;
}

10
src/main/java/me/kavin/piped/utils/PubSubHelper.java
View File

@ -44,11 +44,15 @@ public class PubSubHelper {
var buffer = new Buffer();
formBuilder.build().writeTo(buffer);
var resp = ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of());
ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of())
.thenAccept(resp -> {
if (resp.status() != 202)
System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body()));
})
.exceptionally(e -> {
ExceptionHandler.handle((Exception) e);
return null;
});
}
}

36
src/main/java/me/kavin/piped/utils/RequestUtils.java
View File

@ -1,7 +1,6 @@
package me.kavin.piped.utils;
import com.fasterxml.jackson.databind.JsonNode;
import me.kavin.piped.consts.Constants;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import rocks.kavin.reqwest4j.ReqwestUtils;
@ -9,27 +8,26 @@ import rocks.kavin.reqwest4j.Response;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import static me.kavin.piped.consts.Constants.mapper;
public class RequestUtils {
public static Response sendGetRaw(String url) throws IOException {
public static CompletableFuture<Response> sendGetRaw(String url) throws Exception {
return ReqwestUtils.fetch(url, "GET", null, Map.of());
}
public static String sendGet(String url) throws IOException {
return new String(
ReqwestUtils.fetch(url, "GET", null, Map.of())
.body()
);
public static CompletableFuture<String> sendGet(String url) throws Exception {
return ReqwestUtils.fetch(url, "GET", null, Map.of())
.thenApply(Response::body)
.thenApplyAsync(String::new);
}
public static String sendGet(String url, String ua) throws IOException {
return new String(
ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua))
.body()
);
public static CompletableFuture<String> sendGet(String url, String ua) throws Exception {
return ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua))
.thenApply(Response::body)
.thenApplyAsync(String::new);
}
public static JsonNode getJsonNode(OkHttpClient client, Request request) throws IOException {
@ -44,13 +42,13 @@ public class RequestUtils {
}
}
public static JsonNode sendGetJson(String url, String ua) throws IOException {
return getJsonNode(Constants.h2client, new Request.Builder().header("User-Agent", ua).url(url).build());
public static CompletableFuture<JsonNode> sendGetJson(String url) throws Exception {
return ReqwestUtils.fetch(url, "GET", null, Map.of()).thenApply(Response::body).thenApplyAsync(resp -> {
try {
return mapper.readTree(resp);
} catch (Exception e) {
throw new RuntimeException("Failed to parse JSON", e);
}
public static JsonNode sendGetJson(String url) throws IOException {
return mapper.readTree(ReqwestUtils.fetch(url, "GET", null, Map.of()).body());
}, Multithreading.getCachedExecutor());
}
}

18
src/main/java/me/kavin/piped/utils/RydHelper.java
View File

@ -3,25 +3,17 @@ package me.kavin.piped.utils;
import me.kavin.piped.consts.Constants;
import java.io.IOException;
import static me.kavin.piped.consts.Constants.mapper;
import static me.kavin.piped.utils.RequestUtils.sendGetRaw;
import static me.kavin.piped.utils.RequestUtils.sendGetJson;
public class RydHelper {
public static double getDislikeRating(String videoId) throws IOException {
public static double getDislikeRating(String videoId) throws Exception {
if (Constants.DISABLE_RYD)
return -1;
var resp = sendGetRaw(Constants.RYD_PROXY_URL + "/votes/" + videoId);
if (resp.status() / 100 != 2)
return -1;
return mapper.readTree(resp.body())
.path("rating")
.asDouble(-1);
return sendGetJson(Constants.RYD_PROXY_URL + "/votes/" + videoId)
.thenApply(tree -> tree.path("rating").asDouble(-1))
.get();
}
}

2
src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java
View File

@ -24,7 +24,7 @@ public class SponsorBlockUtils {
try {
var resp = RequestUtils.sendGetRaw(url + "/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4))
+ "?categories=" + URLUtils.silentEncode(categories));
+ "?categories=" + URLUtils.silentEncode(categories)).get();
if (resp.status() == 200) {
var any = mapper.readTree(resp.body());

4
src/main/java/me/kavin/piped/utils/obj/CommentsPage.java
View File

@ -7,10 +7,12 @@ public class CommentsPage {
public List<Comment> comments;
public String nextpage;
public boolean disabled;
public int commentCount;
public CommentsPage(List<Comment> comments, String nextpage, boolean disabled) {
public CommentsPage(List<Comment> comments, String nextpage, boolean disabled, int commentCount) {
this.comments = comments;
this.nextpage = nextpage;
this.disabled = disabled;
this.commentCount = commentCount;
}
}

2
src/main/java/me/kavin/piped/utils/obj/MatrixHelper.java
View File

@ -66,7 +66,7 @@ public class MatrixHelper {
.asText();
}
AUTHORIZED_USERS = (ArrayNode) mapper.readTree(RequestUtils.sendGet("https://raw.githubusercontent.com/TeamPiped/piped-federation/main/authorized-users.json"));
AUTHORIZED_USERS = (ArrayNode) RequestUtils.sendGetJson("https://raw.githubusercontent.com/TeamPiped/piped-federation/main/authorized-users.json").get();
} catch (Exception e) {
ExceptionHandler.handle(e);