kavin/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2024-12-14 14:20:34 +05:30
Code Issues Packages Projects Releases Wiki Activity

Add parser for categories (shelfRenderer)

Browse Source 
This commit adds a new parser for YT's shelfRenderers which are
typically used to denote different categories.The code for featured
channels parsing has also been moved to use the new parser but some
additional refactoring are needed there.

The ContinuationExtractor has also been improved and is now capable of
extraction continuation data that is packaged under
"appendContinuationItemsAction"

In additional this commit adds some useful helper functions to extract
the current selected tab the continuation token. This is to mainly
reduce code size and repetition.
This commit is contained in:
syeopite 2021-05-07 05:13:53 -07:00
parent a027fbf7af
commit 8000d538db
No known key found for this signature in database
GPG Key ID: 6FA616E5A5294A82
9 changed files with 472 additions and 441 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
src/invidious/channels.cr
View File

@ -380,24 +380,73 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
return items, continuation return items, continuation
end end
def fetch_channel_featured_channels(ucid, tab_data, params = nil, continuation = nil, title = nil) def fetch_channel_featured_channels(ucid, tab_data, params = nil, continuation = nil, query_title = nil)
if continuation.is_a?(String) if continuation.is_a?(String)
initial_data = request_youtube_api_browse(continuation) initial_data = request_youtube_api_browse(continuation)
channels_tab_content = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"] items = extract_items(initial_data)
continuation_token = fetch_continuation_token(initial_data)
return process_featured_channels([channels_tab_content], nil, title, continuation_items = true) return [Category.new({
title: query_title.not_nil!, # If continuation contents is requested then the query_title has to be passed along.
contents: items,
browse_endpoint_data: nil,
continuation_token: continuation_token,
badges: nil,
})]
else else
if params.is_a?(String) if params.is_a?(String)
initial_data = request_youtube_api_browse(ucid, params) initial_data = request_youtube_api_browse(ucid, params)
continuation_token = fetch_continuation_token(initial_data)
else else
initial_data = request_youtube_api_browse(ucid, tab_data[1]) initial_data = request_youtube_api_browse(ucid, tab_data[1])
continuation_token = nil
end end
channels_tab = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"][tab_data[0]]["tabRenderer"] channels_tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
channels_tab_content = channels_tab["content"]["sectionListRenderer"]["contents"].as_a submenu = channels_tab["content"]["sectionListRenderer"]["subMenu"]?
submenu_data = channels_tab["content"]["sectionListRenderer"]["subMenu"]?.try &.["channelSubMenuRenderer"]["contentTypeSubMenuItems"] || false
return process_featured_channels(channels_tab_content, submenu_data) # There's no submenu data if the channel doesn't feature any channels.
if !submenu
return [] of Category
end
submenu_data = submenu["channelSubMenuRenderer"]["contentTypeSubMenuItems"]
items = extract_items(initial_data)
fallback_title = submenu_data.as_a.select(&.["selected"].as_bool)[0]["title"].as_s
# Although extract_items parsed everything into the right structs, we still have
# to fill in the title (if missing) attribute since Youtube doesn't return it when requesting
# a full category
category_array = [] of Category
items.each do |category|
# Tell compiler that the result from extract_items has to be an array of Categories
if !category.is_a?(Category)
next
end
category_array << Category.new({
title: category.title.empty? ? fallback_title : category.title,
contents: category.contents,
browse_endpoint_data: category.browse_endpoint_data,
continuation_token: continuation_token,
badges: nil,
})
end
# If we don't have any categories we'll create one.
if category_array.empty?
return [Category.new({
title: fallback_title, # If continuation contents is requested then the query_title has to be passed along.
contents: items,
browse_endpoint_data: nil,
continuation_token: continuation_token,
badges: nil,
})]
end
return category_array
end end
end end

170
src/invidious/featured_channels.cr
View File

@ -1,170 +0,0 @@
struct FeaturedChannel
include DB::Serializable
property author : String
property ucid : String
property author_thumbnail : String
property subscriber_count : Int32
property video_count : Int32
property description_html : String?
def to_json(locale, json : JSON::Builder)
json.object do
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "subCount", self.subscriber_count
json.field "videoCount", self.video_count
json.field "badges", self.badges
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
struct Category
include DB::Serializable
property title : String
property contents : Array(FeaturedChannel) | FeaturedChannel
property browse_endpoint_param : String?
property continuation_token : String?
def to_json(locale, json : JSON::Builder)
json.object do
json.field "title", self.title
json.field "contents", self.contents
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
def _extract_channel_data(channel)
ucid = channel["channelId"].as_s
author = channel["title"]["simpleText"].as_s
author_thumbnail = channel["thumbnail"]["thumbnails"].as_a[0]["url"].as_s
subscriber_count = channel["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s?
.try { |text| short_text_to_number(text.split(" ")[0]) } || 0
video_count = channel["videoCountText"]?.try &.["runs"][0]["text"].as_s.gsub(/\D/, "").to_i || 0
if channel["descriptionSnippet"]?
description = channel["descriptionSnippet"]["runs"][0]["text"].as_s
description_html = HTML.escape(description).gsub("\n", "")
else
description_html = nil
end
FeaturedChannel.new({
author: author,
ucid: ucid,
author_thumbnail: author_thumbnail,
subscriber_count: subscriber_count,
video_count: video_count,
description_html: description_html,
})
end
def process_featured_channels(data, submenu_data, title = nil, continuation_items = false)
all_categories = [] of Category
if submenu_data.is_a?(Bool)
return all_categories
end
# Extraction process differs when there's more than one category
if data.size > 1
data.each do |raw_category|
raw_category = raw_category["itemSectionRenderer"]["contents"].as_a[0]["shelfRenderer"]
category_title = raw_category["title"]["runs"][0]["text"].as_s
browse_endpoint_param = raw_category["endpoint"]["browseEndpoint"]["params"].as_s
# Category has multiple channels
if raw_category["content"].as_h.has_key?("horizontalListRenderer")
contents = [] of FeaturedChannel
raw_category["content"]["horizontalListRenderer"]["items"].as_a.each do |channel|
contents << _extract_channel_data(channel["gridChannelRenderer"])
end
# Single channel
else
channel = raw_category["content"]["expandedShelfContentsRenderer"]["items"][0]["channelRenderer"]
contents = _extract_channel_data(channel)
end
all_categories << Category.new({
title: category_title,
contents: contents,
browse_endpoint_param: browse_endpoint_param,
continuation_token: nil,
})
end
else
if !continuation_items
raw_category_contents = data[0]["itemSectionRenderer"]["contents"].as_a[0]["gridRenderer"]["items"].as_a
else
raw_category_contents = data[0].as_a
end
category_title = submenu_data.try &.[0]["title"].as_s || title || ""
browse_endpoint_param = nil # Not needed
continuation_token = nil
# If a continuation token is needed it'll always be after at least twelve channels
if raw_category_contents.size > 12
continuation_token = raw_category_contents[-1]["continuationItemRenderer"]?.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s || nil
if !continuation_token.nil?
raw_category_contents = raw_category_contents[0..-2]
end
end
contents = [] of FeaturedChannel
raw_category_contents.each do |channel|
contents << _extract_channel_data(channel["gridChannelRenderer"])
end
all_categories << Category.new({
title: category_title,
contents: contents,
browse_endpoint_param: browse_endpoint_param,
continuation_token: continuation_token,
})
end
return all_categories
end

127
src/invidious/helpers/extractors.cr
View File

@ -5,7 +5,7 @@
private ITEM_CONTAINER_EXTRACTOR = { private ITEM_CONTAINER_EXTRACTOR = {
YoutubeTabsExtractor.new, YoutubeTabsExtractor.new,
SearchResultsExtractor.new, SearchResultsExtractor.new,
ContinuationExtractor.new ContinuationExtractor.new,
} }
private ITEM_PARSERS = { private ITEM_PARSERS = {
@ -13,6 +13,7 @@ private ITEM_PARSERS = {
ChannelParser.new, ChannelParser.new,
GridPlaylistParser.new, GridPlaylistParser.new,
PlaylistParser.new, PlaylistParser.new,
CategoryParser.new,
} }
private struct AuthorFallback private struct AuthorFallback
@ -98,7 +99,7 @@ end
private class ChannelParser < ItemParser private class ChannelParser < ItemParser
def process(item, author_fallback) def process(item, author_fallback)
if item_contents = item["channelRenderer"]? if item_contents = (item["channelRenderer"]? || item["gridChannelRenderer"]?)
return self.parse(item_contents, author_fallback) return self.parse(item_contents, author_fallback)
end end
end end
@ -197,6 +198,88 @@ private class PlaylistParser < ItemParser
end end
end end
private class CategoryParser < ItemParser
def process(item, author_fallback)
if item_contents = item["shelfRenderer"]?
return self.parse(item_contents, author_fallback)
end
end
def parse(item_contents, author_fallback)
# Title extraction is a bit complicated. There are two possible routes for it
# as well as times when the title attribute just isn't sent by YT.
title_container = item_contents["title"]? || ""
if !title_container.is_a? String
if title = title_container["simpleText"]?
title = title.as_s
else
title = title_container["runs"][0]["text"].as_s
end
else
title = ""
end
browse_endpoint = item_contents["endpoint"]?.try &.["browseEndpoint"] || nil
browse_endpoint_data = ""
category_type = 0 # 0: Video, 1: Channels, 2: Playlist/feed, 3: trending
# There's no endpoint data for video and trending category
if !item_contents["endpoint"]?
if !item_contents["videoId"]?
category_type = 3
end
end
if !browse_endpoint.nil?
# Playlist/feed categories doesn't need the params value (nor is it even included in yt response)
# instead it uses the browseId parameter. So if there isn't a params value we can assume the
# category is a playlist/feed
if browse_endpoint["params"]?
browse_endpoint_data = browse_endpoint["params"].as_s
category_type = 1
else
browse_endpoint_data = browse_endpoint["browseId"].as_s
category_type = 2
end
end
# Sometimes a category can have badges.
badges = [] of Tuple(String, String) # (Badge style, label)
item_contents["badges"]?.try &.as_a.each do |badge|
badge = badge["metadataBadgeRenderer"]
badges << {badge["style"].as_s, badge["label"].as_s}
end
# Content parsing
contents = [] of SearchItem
# Content could be in three locations.
if content_container = item_contents["content"]["horizontalListRenderer"]?
elsif content_container = item_contents["content"]["expandedShelfContentsRenderer"]
elsif content_container = item_contents["content"]["verticalListRenderer"]
else
content_container = item_contents["contents"]
end
raw_contents = content_container["items"].as_a
raw_contents.each do |item|
result = extract_item(item)
if !result.nil?
contents << result
end
end
Category.new({
title: title,
contents: contents,
browse_endpoint_data: browse_endpoint_data,
continuation_token: nil,
badges: badges,
})
end
end
# The following are the extractors for extracting an array of items from # The following are the extractors for extracting an array of items from
# the internal Youtube API's JSON response. The result is then packaged into # the internal Youtube API's JSON response. The result is then packaged into
# a structure we can more easily use via the parsers above. Their internals are # a structure we can more easily use via the parsers above. Their internals are
@ -220,25 +303,22 @@ private class YoutubeTabsExtractor < ItemsContainerExtractor
private def extract(target) private def extract(target)
raw_items = [] of JSON::Any raw_items = [] of JSON::Any
selected_tab = extract_selected_tab(target["tabs"]) selected_tab = extract_selected_tab(target["tabs"])
content = selected_tab["tabRenderer"]["content"] content = selected_tab["content"]
content["sectionListRenderer"]["contents"].as_a.each do | renderer_container | content["sectionListRenderer"]["contents"].as_a.each do |renderer_container|
renderer_container = renderer_container["itemSectionRenderer"] renderer_container = renderer_container["itemSectionRenderer"]
renderer_container_contents = renderer_container["contents"].as_a[0] renderer_container_contents = renderer_container["contents"].as_a[0]
# Shelf renderer usually refer to a category and would need special handling once # Category extraction
# An extractor for categories are added. But for now it is just used to
# extract items for the trending page
if items_container = renderer_container_contents["shelfRenderer"]? if items_container = renderer_container_contents["shelfRenderer"]?
if items_container["content"]["expandedShelfContentsRenderer"]? raw_items << renderer_container_contents
items_container = items_container["content"]["expandedShelfContentsRenderer"] next
end
elsif items_container = renderer_container_contents["gridRenderer"]? elsif items_container = renderer_container_contents["gridRenderer"]?
else else
items_container = renderer_container_contents items_container = renderer_container_contents
end end
items_container["items"].as_a.each do | item | items_container["items"].as_a.each do |item|
raw_items << item raw_items << item
end end
end end
@ -268,6 +348,8 @@ private class ContinuationExtractor < ItemsContainerExtractor
def process(initial_data) def process(initial_data)
if target = initial_data["continuationContents"]? if target = initial_data["continuationContents"]?
self.extract(target) self.extract(target)
elsif target = initial_data["appendContinuationItemsAction"]?
self.extract(target)
end end
end end
@ -275,13 +357,16 @@ private class ContinuationExtractor < ItemsContainerExtractor
raw_items = [] of JSON::Any raw_items = [] of JSON::Any
if content = target["gridContinuation"]? if content = target["gridContinuation"]?
raw_items = content["items"].as_a raw_items = content["items"].as_a
elsif content = target["continuationItems"]?
raw_items = content.as_a
end end
return raw_items return raw_items
end end
end end
def extract_item(item : JSON::Any, author_fallback : String? = nil, author_id_fallback : String? = nil) def extract_item(item : JSON::Any, author_fallback : String? = nil,
author_id_fallback : String? = nil)
# Parses an item from Youtube's JSON response into a more usable structure. # Parses an item from Youtube's JSON response into a more usable structure.
# The end result can either be a SearchVideo, SearchPlaylist or SearchChannel. # The end result can either be a SearchVideo, SearchPlaylist or SearchChannel.
author_fallback = AuthorFallback.new(author_fallback, author_id_fallback) author_fallback = AuthorFallback.new(author_fallback, author_id_fallback)
@ -298,21 +383,29 @@ def extract_item(item : JSON::Any, author_fallback : String? = nil, author_id_fa
# TODO radioRenderer, showRenderer, shelfRenderer, horizontalCardListRenderer, searchPyvRenderer # TODO radioRenderer, showRenderer, shelfRenderer, horizontalCardListRenderer, searchPyvRenderer
end end
def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
author_id_fallback : String? = nil)
items = [] of SearchItem items = [] of SearchItem
initial_data = initial_data["contents"]?.try &.as_h || initial_data["response"]?.try &.as_h || initial_data
if unpackaged_data = initial_data["contents"]?.try &.as_h
elsif unpackaged_data = initial_data["response"]?.try &.as_h
elsif unpackaged_data = initial_data["onResponseReceivedActions"]?.try &.as_a.[0].as_h
else
unpackaged_data = initial_data
end
# This is identicial to the parser cyling of extract_item(). # This is identicial to the parser cyling of extract_item().
ITEM_CONTAINER_EXTRACTOR.each do | extractor | ITEM_CONTAINER_EXTRACTOR.each do |extractor|
results = extractor.process(initial_data) results = extractor.process(unpackaged_data)
if !results.nil? if !results.nil?
results.each do | item | results.each do |item|
parsed_result = extract_item(item, author_fallback, author_id_fallback) parsed_result = extract_item(item, author_fallback, author_id_fallback)
if !parsed_result.nil? if !parsed_result.nil?
items << parsed_result items << parsed_result
end end
end end
return items
end end
end end

30
src/invidious/helpers/helpers.cr
View File

@ -248,12 +248,38 @@ def html_to_content(description_html : String)
end end
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
extract_items(initial_data, author_fallback, author_id_fallback).select(&.is_a?(SearchVideo)).map(&.as(SearchVideo)) extracted = extract_items(initial_data, author_fallback, author_id_fallback)
if extracted.is_a?(Category)
target = extracted.contents
else
target = extracted
end
return target.select(&.is_a?(SearchVideo)).map(&.as(SearchVideo))
end end
def extract_selected_tab(tabs) def extract_selected_tab(tabs)
# Extract the selected tab from the array of tabs Youtube returns # Extract the selected tab from the array of tabs Youtube returns
return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0] return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]["tabRenderer"]
end
def fetch_continuation_token(items : Array(JSON::Any))
# Fetches the continuation token from an array of items
return items.last["continuationItemRenderer"]?
.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
end
def fetch_continuation_token(initial_data : Hash(String, JSON::Any))
# Fetches the continuation token from initial data
if initial_data["onResponseReceivedActions"]?
continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
else
tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"]
end
return fetch_continuation_token(continuation_items.as_a)
end end
def check_enum(db, enum_name, struct_type = nil) def check_enum(db, enum_name, struct_type = nil)

258
src/invidious/helpers/invidiousitems.cr Normal file
View File

@ -0,0 +1,258 @@
struct SearchVideo
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property published : Time
property views : Int64
property description_html : String
property length_seconds : Int32
property live_now : Bool
property paid : Bool
property premium : Bool
property premiere_timestamp : Time?
def to_xml(auto_generated, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
else
xml.element("name") { xml.text author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) }
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
xml.element("media:description") { xml.text html_to_content(self.description_html) }
end
xml.element("media:community") do
xml.element("media:statistics", views: self.views)
end
end
end
def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil)
if xml
to_xml(HOST_URL, auto_generated, query_params, xml)
else
XML.build do |json|
to_xml(HOST_URL, auto_generated, query_params, xml)
end
end
end
def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
end
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "viewCount", self.views
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "lengthSeconds", self.length_seconds
json.field "liveNow", self.live_now
json.field "paid", self.paid
json.field "premium", self.premium
json.field "isUpcoming", self.is_upcoming
if self.premiere_timestamp
json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
def is_upcoming
premiere_timestamp ? true : false
end
end
struct SearchPlaylistVideo
include DB::Serializable
property title : String
property id : String
property length_seconds : Int32
end
struct SearchPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property video_count : Int32
property videos : Array(SearchPlaylistVideo)
property thumbnail : String?
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "playlist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "playlistThumbnail", self.thumbnail
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoCount", self.video_count
json.field "videos" do
json.array do
self.videos.each do |video|
json.object do
json.field "title", video.title
json.field "videoId", video.id
json.field "lengthSeconds", video.length_seconds
json.field "videoThumbnails" do
generate_thumbnails(json, video.id)
end
end
end
end
end
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
struct SearchChannel
include DB::Serializable
property author : String
property ucid : String
property author_thumbnail : String
property subscriber_count : Int32
property video_count : Int32
property description_html : String
property auto_generated : Bool
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "channel"
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "autoGenerated", self.auto_generated
json.field "subCount", self.subscriber_count
json.field "videoCount", self.video_count
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
class Category
include DB::Serializable
property title : String
property contents : Array(SearchItem) | SearchItem
property browse_endpoint_data : String?
property continuation_token : String?
property badges : Array(Tuple(String, String))?
def to_json(locale, json : JSON::Builder)
json.object do
json.field "title", self.title
json.field "contents", self.contents
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category

1
src/invidious/routes/channels.cr
View File

@ -102,7 +102,6 @@ class Invidious::Routes::Channels < Invidious::Routes::BaseRoute
return env.redirect "/channel/#{channel.ucid}" return env.redirect "/channel/#{channel.ucid}"
end end
# When a channel only has a single category it lacks the category param option so we'll handle it here.
if continuation if continuation
offset = env.params.query["offset"]? offset = env.params.query["offset"]?
if offset if offset

232
src/invidious/search.cr
View File

@ -1,235 +1,3 @@
struct SearchVideo
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property published : Time
property views : Int64
property description_html : String
property length_seconds : Int32
property live_now : Bool
property paid : Bool
property premium : Bool
property premiere_timestamp : Time?
def to_xml(auto_generated, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
else
xml.element("name") { xml.text author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) }
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
xml.element("media:description") { xml.text html_to_content(self.description_html) }
end
xml.element("media:community") do
xml.element("media:statistics", views: self.views)
end
end
end
def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil)
if xml
to_xml(HOST_URL, auto_generated, query_params, xml)
else
XML.build do |json|
to_xml(HOST_URL, auto_generated, query_params, xml)
end
end
end
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
end
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "viewCount", self.views
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "lengthSeconds", self.length_seconds
json.field "liveNow", self.live_now
json.field "paid", self.paid
json.field "premium", self.premium
json.field "isUpcoming", self.is_upcoming
if self.premiere_timestamp
json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
def is_upcoming
premiere_timestamp ? true : false
end
end
struct SearchPlaylistVideo
include DB::Serializable
property title : String
property id : String
property length_seconds : Int32
end
struct SearchPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property video_count : Int32
property videos : Array(SearchPlaylistVideo)
property thumbnail : String?
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "playlist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "playlistThumbnail", self.thumbnail
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoCount", self.video_count
json.field "videos" do
json.array do
self.videos.each do |video|
json.object do
json.field "title", video.title
json.field "videoId", video.id
json.field "lengthSeconds", video.length_seconds
json.field "videoThumbnails" do
generate_thumbnails(json, video.id)
end
end
end
end
end
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
struct SearchChannel
include DB::Serializable
property author : String
property ucid : String
property author_thumbnail : String
property subscriber_count : Int32
property video_count : Int32
property description_html : String
property auto_generated : Bool
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "channel"
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "autoGenerated", self.auto_generated
json.field "subCount", self.subscriber_count
json.field "videoCount", self.video_count
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist
def channel_search(query, page, channel) def channel_search(query, page, channel)
response = YT_POOL.client &.get("/channel/#{channel}") response = YT_POOL.client &.get("/channel/#{channel}")

17
src/invidious/views/channel/featured_channels.ecr
View File

@ -14,7 +14,7 @@
<details open=""> <details open="">
<summary style="display: revert;"> <summary style="display: revert;">
<h3 class="category-heading"> <h3 class="category-heading">
<% if (category_request_param = category.browse_endpoint_param).is_a?(String) %> <% if (category_request_param = category.browse_endpoint_data).is_a?(String) %>
<a href="/channel/<%=channel.ucid%>/channels/<%=HTML.escape(category_request_param)%>"> <a href="/channel/<%=channel.ucid%>/channels/<%=HTML.escape(category_request_param)%>">
<%= category.title %> <%= category.title %>
</a> </a>
@ -25,8 +25,12 @@
</summary> </summary>
<% contents = category.contents%> <% contents = category.contents%>
<div class="pure-g section-contents"> <div class="pure-g section-contents">
<% if contents.is_a?(Array(FeaturedChannel)) %> <% if contents.is_a?(Array) %>
<% contents.each do |item|%> <% contents.each do |item|%>
<% if !item.is_a?(SearchChannel)%>
<% next %>
<% end %>
<div class="channel-profile pure-u-1 pure-u-sm-1-2 pure-u-md-1-3 pure-u-lg-1-4 pure-u-xl-1-5"> <div class="channel-profile pure-u-1 pure-u-sm-1-2 pure-u-md-1-3 pure-u-lg-1-4 pure-u-xl-1-5">
<a class="featured-channel-icon" href="/channel/<%= item.ucid %>"> <a class="featured-channel-icon" href="/channel/<%= item.ucid %>">
<% if !env.get("preferences").as(Preferences).thin_mode %> <% if !env.get("preferences").as(Preferences).thin_mode %>
@ -47,7 +51,11 @@
</div> </div>
</div> </div>
<%end%> <%end%>
<% elsif contents.is_a?(FeaturedChannel) %> <% elsif contents.is_a?(SearchItem) %>
<% if !contents.is_a?(SearchChannel)%>
<% next %>
<% end %>
<%item = contents %> <%item = contents %>
<div class="channel-profile large-featured-channel pure-u-1"> <div class="channel-profile large-featured-channel pure-u-1">
<a class="featured-channel-icon" href="/channel/<%= item.ucid %>"> <a class="featured-channel-icon" href="/channel/<%= item.ucid %>">
@ -69,11 +77,10 @@
<% sub_count_text = number_to_short_text(contents.subscriber_count) %> <% sub_count_text = number_to_short_text(contents.subscriber_count) %>
<%= rendered "components/subscribe_widget" %> <%= rendered "components/subscribe_widget" %>
</div> </div>
<%end%> <% end %>
</div> </div>
</details> </details>
</div> </div>
</div>
<% end %> <% end %>
<% else %> <% else %>
<h3 class="pure-u-1"> <h3 class="pure-u-1">

1
src/invidious/views/components/item.ecr
View File

@ -97,6 +97,7 @@
<%= item.responds_to?(:views) && item.views ? translate(locale, "`x` views", number_to_short_text(item.views || 0)) : "" %> <%= item.responds_to?(:views) && item.views ? translate(locale, "`x` views", number_to_short_text(item.views || 0)) : "" %>
</div> </div>
</h5> </h5>
<% when Category %>
<% else %> <% else %>
<% if !env.get("preferences").as(Preferences).thin_mode %> <% if !env.get("preferences").as(Preferences).thin_mode %>
<a style="width:100%" href="/watch?v=<%= item.id %>"> <a style="width:100%" href="/watch?v=<%= item.id %>">