From 2a40f3b9c53f7eaf510028613135d94cbfb7ba68 Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Sun, 13 Oct 2024 19:21:50 +0200 Subject: [PATCH] trying to only use next endpoint for the major data --- locales/en-US.json | 5 +- src/invidious/helpers/errors.cr | 13 +-- src/invidious/routes/watch.cr | 116 +++++++++++----------- src/invidious/videos.cr | 18 ++-- src/invidious/videos/parser.cr | 63 +++++++----- src/invidious/views/components/player.ecr | 2 + src/invidious/views/embed.ecr | 10 +- src/invidious/views/error.ecr | 1 - src/invidious/views/watch.ecr | 19 +++- 9 files changed, 147 insertions(+), 100 deletions(-) diff --git a/locales/en-US.json b/locales/en-US.json index 7827d9c6..caafbc86 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -497,5 +497,8 @@ "toggle_theme": "Toggle Theme", "carousel_slide": "Slide {{current}} of {{total}}", "carousel_skip": "Skip the Carousel", - "carousel_go_to": "Go to slide `x`" + "carousel_go_to": "Go to slide `x`", + "error_from_youtube_unplayable": "Video unplayable due to an error from YouTube:", + "error_processing_data_youtube": "Error while processing the data sent by YouTube", + "refresh_page": "Refresh the page" } diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index b7643194..b3cb5b94 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -73,10 +73,6 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce END_HTML - # Don't show the usual "next steps" widget. The same options are - # proposed above the error message, just worded differently. - next_steps = "" - return templated "error" end @@ -86,8 +82,13 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, mess locale = env.get("preferences").as(Preferences).locale - error_message = translate(locale, message) - next_steps = error_redirect_helper(env) + error_message = <<-END_HTML +
+

#{translate(locale, "error_processing_data_youtube")}

+

#{translate(locale, message)}

+ #{error_redirect_helper(env)} +
+ END_HTML return templated "error" end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index aabe8dfc..e7c451bc 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -117,79 +117,83 @@ module Invidious::Routes::Watch comment_html ||= "" end - fmt_stream = video.fmt_stream - adaptive_fmts = video.adaptive_fmts + if video.reason.nil? + fmt_stream = video.fmt_stream + adaptive_fmts = video.adaptive_fmts - if params.local - fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } - adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } - end - - video_streams = video.video_streams - audio_streams = video.audio_streams - - # Older videos may not have audio sources available. - # We redirect here so they're not unplayable - if audio_streams.empty? && !video.live_now - if params.quality == "dash" - env.params.query.delete_all("quality") - env.params.query["quality"] = "medium" - return env.redirect "/watch?#{env.params.query}" - elsif params.listen - env.params.query.delete_all("listen") - env.params.query["listen"] = "0" - return env.redirect "/watch?#{env.params.query}" + if params.local + fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } + adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } end - end - captions = video.captions + video_streams = video.video_streams + audio_streams = video.audio_streams - preferred_captions = captions.select { |caption| - params.preferred_captions.includes?(caption.name) || - params.preferred_captions.includes?(caption.language_code.split("-")[0]) - } - preferred_captions.sort_by! { |caption| - (params.preferred_captions.index(caption.name) || - params.preferred_captions.index(caption.language_code.split("-")[0])).not_nil! - } - captions = captions - preferred_captions + # Older videos may not have audio sources available. + # We redirect here so they're not unplayable + if audio_streams.empty? && !video.live_now + if params.quality == "dash" + env.params.query.delete_all("quality") + env.params.query["quality"] = "medium" + return env.redirect "/watch?#{env.params.query}" + elsif params.listen + env.params.query.delete_all("listen") + env.params.query["listen"] = "0" + return env.redirect "/watch?#{env.params.query}" + end + end - aspect_ratio = "16:9" + captions = video.captions - thumbnail = "/vi/#{video.id}/maxres.jpg" + preferred_captions = captions.select { |caption| + params.preferred_captions.includes?(caption.name) || + params.preferred_captions.includes?(caption.language_code.split("-")[0]) + } + preferred_captions.sort_by! { |caption| + (params.preferred_captions.index(caption.name) || + params.preferred_captions.index(caption.language_code.split("-")[0])).not_nil! + } + captions = captions - preferred_captions - if params.raw - if params.listen - url = audio_streams[0]["url"].as_s + aspect_ratio = "16:9" - if params.quality.ends_with? "k" - audio_streams.each do |fmt| - if fmt["bitrate"].as_i == params.quality.rchop("k").to_i + thumbnail = "/vi/#{video.id}/maxres.jpg" + + if params.raw + if params.listen + url = audio_streams[0]["url"].as_s + + if params.quality.ends_with? "k" + audio_streams.each do |fmt| + if fmt["bitrate"].as_i == params.quality.rchop("k").to_i + url = fmt["url"].as_s + end + end + end + else + url = fmt_stream[0]["url"].as_s + + fmt_stream.each do |fmt| + if fmt["quality"].as_s == params.quality url = fmt["url"].as_s end end end - else - url = fmt_stream[0]["url"].as_s - fmt_stream.each do |fmt| - if fmt["quality"].as_s == params.quality - url = fmt["url"].as_s - end - end + return env.redirect url end - return env.redirect url + # Structure used for the download widget + video_assets = Invidious::Frontend::WatchPage::VideoAssets.new( + full_videos: fmt_stream, + video_streams: video_streams, + audio_streams: audio_streams, + captions: video.captions + ) + else + env.response.status_code = 500 end - # Structure used for the download widget - video_assets = Invidious::Frontend::WatchPage::VideoAssets.new( - full_videos: fmt_stream, - video_streams: video_streams, - audio_streams: audio_streams, - captions: video.captions - ) - templated "watch" end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index ae09e736..16ccf197 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -313,7 +313,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) end else video = fetch_video(id, region) - Invidious::Database::Videos.insert(video) if !region + Invidious::Database::Videos.insert(video) if !region && !video.info.dig?("reason") end return video @@ -326,13 +326,17 @@ end def fetch_video(id, region) info = extract_video_info(video_id: id) - if reason = info["reason"]? + if info["reason"]? + reason = info["reason"].as_s + if info.dig?("subreason") + subreason = info["subreason"].as_s + else + subreason = "No additional reason" + end if reason == "Video unavailable" - raise NotFoundException.new(reason.as_s || "") - elsif !reason.as_s.starts_with? "Premieres" - # dont error when it's a premiere. - # we already parsed most of the data and display the premiere date - raise InfoException.new(reason.as_s || "") + raise NotFoundException.new(reason + ": " + subreason || "") + elsif {"Private video"}.any?(reason) + raise InfoException.new(reason + ": " + subreason || "") end end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index fb8935d9..84e468b5 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -64,18 +64,19 @@ def extract_video_info(video_id : String) playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s if playability_status != "OK" - subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") - reason = subreason.try &.[]?("simpleText").try &.as_s - reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") - reason ||= player_response.dig("playabilityStatus", "reason").as_s + reason = player_response.dig?("playabilityStatus", "reason").try &.as_s + reason ||= player_response.dig("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "reason", "simpleText").as_s + subreason_main = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") + subreason = subreason_main.try &.[]?("simpleText").try &.as_s + subreason ||= subreason_main.try &.[]("runs").as_a.map(&.[]("text")).join("") # Stop here if video is not a scheduled livestream or # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help - if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) || - playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails") + if {"Private video", "Video unavailable"}.any?(reason) return { - "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), - "reason" => JSON::Any.new(reason), + "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), + "reason" => JSON::Any.new(reason), + "subreason" => JSON::Any.new(subreason), } end elsif video_id != player_response.dig("videoDetails", "videoId") @@ -95,11 +96,8 @@ def extract_video_info(video_id : String) reason = nil end - # Don't fetch the next endpoint if the video is unavailable. - if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) - next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) - player_response = player_response.merge(next_response) - end + next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) + player_response = player_response.merge(next_response) params = parse_video_info(video_id, player_response) params["reason"] = JSON::Any.new(reason) if reason @@ -205,17 +203,22 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer end - video_details = player_response.dig?("videoDetails") + if !(video_details = player_response.dig?("videoDetails")) + video_details = {} of String => JSON::Any + end if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer")) microformat = {} of String => JSON::Any end - raise BrokenTubeException.new("videoDetails") if !video_details - # Basic video infos title = video_details["title"]?.try &.as_s + title ||= extract_text( + video_primary_renderer + .try &.dig?("title") + ) + # We have to try to extract viewCount from videoPrimaryInfoRenderer first, # then from videoDetails, as the latter is "0" for livestreams (we want # to get the amount of viewers watching). @@ -226,17 +229,27 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any views_txt ||= video_details["viewCount"]?.try &.as_s || "" views = views_txt.gsub(/\D/, "").to_i64? - length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) + length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]?) .try &.as_s.to_i64 - published = microformat["publishDate"]? - .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc + published_txt = video_primary_renderer + .try &.dig?("dateText", "simpleText") + + if published_txt.try &.as_s.includes?("ago") && !published_txt.nil? + published = decode_date(published_txt.as_s.lchop("Started streaming ")) + elsif published_txt && published_txt.try &.as_s.matches?(/(\w{3} \d{1,2}, \d{4})$/) + published = Time.parse(published_txt.as_s.match!(/(\w{3} \d{1,2}, \d{4})$/)[0], "%b %-d, %Y", Time::Location::UTC) + else + published = Time.utc + end premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") .try { |t| Time.parse_rfc3339(t.as_s) } live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") - .try &.as_bool || false + .try &.as_bool + live_now ||= video_primary_renderer + .try &.dig?("viewCount", "videoViewCountRenderer", "isLive").try &.as_bool || false post_live_dvr = video_details.dig?("isPostLiveDvr") .try &.as_bool || false @@ -404,9 +417,6 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any # Author infos - author = video_details["author"]?.try &.as_s - ucid = video_details["channelId"]?.try &.as_s - if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url") author_verified = has_verified_badge?(author_info["badges"]?) @@ -414,6 +424,9 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any subs_text = author_info["subscriberCountText"]? .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") } .try &.as_s.split(" ", 2)[0] + + author = author_info.dig?("title", "runs", 0, "text").try &.as_s + ucid = author_info.dig?("title", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId").try &.as_s end # Return data @@ -438,7 +451,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any # Extra video infos "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }), "allowRatings" => JSON::Any.new(allow_ratings || false), - "isFamilyFriendly" => JSON::Any.new(family_friendly || false), + "isFamilyFriendly" => JSON::Any.new(family_friendly || true), "isListed" => JSON::Any.new(is_listed || false), "isUpcoming" => JSON::Any.new(is_upcoming || false), "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }), @@ -448,7 +461,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any # Description "description" => JSON::Any.new(description || ""), "descriptionHtml" => JSON::Any.new(description_html || "

"), - "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), + "shortDescription" => JSON::Any.new(short_description.try &.as_s || ""), # Video metadata "genre" => JSON::Any.new(genre.try &.as_s || ""), "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?), diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 5c28358b..53976461 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -1,3 +1,4 @@ +<% if audio_streams && fmt_stream && preferred_captions && captions %>