Merge pull request #2878 from matthewmcgarvey/migrations

Add custom migration implementation

src/invidious.cr

@@ -27,6 +27,7 @@ require "compress/zip"
 require "protodec/utils"
 
 require "./invidious/database/*"
+require "./invidious/database/migrations/*"
 require "./invidious/helpers/*"
 require "./invidious/yt_backend/*"
 require "./invidious/frontend/*"
@@ -102,6 +103,10 @@ Kemal.config.extra_options do |parser|
     puts SOFTWARE.to_pretty_json
     exit
   end
+  parser.on("--migrate", "Run any migrations") do
+    Invidious::Database::Migrator.new(PG_DB).migrate
+    exit
+  end
 end
 
 Kemal::CLI.new ARGV

src/invidious/database/migration.cr

@@ -0,0 +1,38 @@
+abstract class Invidious::Database::Migration
+  macro inherited
+    Migrator.migrations << self
+  end
+
+  @@version : Int64?
+
+  def self.version(version : Int32 | Int64)
+    @@version = version.to_i64
+  end
+
+  getter? completed = false
+
+  def initialize(@db : DB::Database)
+  end
+
+  abstract def up(conn : DB::Connection)
+
+  def migrate
+    # migrator already ignores completed migrations
+    # but this is an extra check to make sure a migration doesn't run twice
+    return if completed?
+
+    @db.transaction do |txn|
+      up(txn.connection)
+      track(txn.connection)
+      @completed = true
+    end
+  end
+
+  def version : Int64
+    @@version.not_nil!
+  end
+
+  private def track(conn : DB::Connection)
+    conn.exec("INSERT INTO #{Migrator::MIGRATIONS_TABLE} (version) VALUES ($1)", version)
+  end
+end

src/invidious/database/migrations/0001_create_channels_table.cr

@@ -0,0 +1,30 @@
+module Invidious::Database::Migrations
+  class CreateChannelsTable < Migration
+    version 1
+
+    def up(conn : DB::Connection)
+      conn.exec <<-SQL
+      CREATE TABLE IF NOT EXISTS public.channels
+      (
+        id text NOT NULL,
+        author text,
+        updated timestamp with time zone,
+        deleted boolean,
+        subscribed timestamp with time zone,
+        CONSTRAINT channels_id_key UNIQUE (id)
+      );
+      SQL
+
+      conn.exec <<-SQL
+      GRANT ALL ON TABLE public.channels TO current_user;
+      SQL
+
+      conn.exec <<-SQL
+      CREATE INDEX IF NOT EXISTS channels_id_idx
+        ON public.channels
+        USING btree
+        (id COLLATE pg_catalog."default");
+      SQL
+    end
+  end
+end

src/invidious/database/migrations/0002_create_videos_table.cr

@@ -0,0 +1,28 @@
+module Invidious::Database::Migrations
+  class CreateVideosTable < Migration
+    version 2
+
+    def up(conn : DB::Connection)
+      conn.exec <<-SQL
+      CREATE UNLOGGED TABLE IF NOT EXISTS public.videos
+      (
+        id text NOT NULL,
+        info text,
+        updated timestamp with time zone,
+        CONSTRAINT videos_pkey PRIMARY KEY (id)
+      );
+      SQL
+
+      conn.exec <<-SQL
+      GRANT ALL ON TABLE public.videos TO current_user;
+      SQL
+
+      conn.exec <<-SQL
+      CREATE UNIQUE INDEX IF NOT EXISTS id_idx
+        ON public.videos
+        USING btree
+        (id COLLATE pg_catalog."default");
+      SQL
+    end
+  end
+end

src/invidious/database/migrations/0003_create_channel_videos_table.cr

@@ -0,0 +1,35 @@
+module Invidious::Database::Migrations
+  class CreateChannelVideosTable < Migration
+    version 3
+
+    def up(conn : DB::Connection)
+      conn.exec <<-SQL
+      CREATE TABLE IF NOT EXISTS public.channel_videos
+      (
+        id text NOT NULL,
+        title text,
+        published timestamp with time zone,
+        updated timestamp with time zone,
+        ucid text,
+        author text,
+        length_seconds integer,
+        live_now boolean,
+        premiere_timestamp timestamp with time zone,
+        views bigint,
+        CONSTRAINT channel_videos_id_key UNIQUE (id)
+      );
+      SQL
+
+      conn.exec <<-SQL
+      GRANT ALL ON TABLE public.channel_videos TO current_user;
+      SQL
+
+      conn.exec <<-SQL
+      CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx
+        ON public.channel_videos
+        USING btree
+        (ucid COLLATE pg_catalog."default");
+      SQL
+    end
+  end
+end

src/invidious/database/migrations/0004_create_users_table.cr

@@ -0,0 +1,34 @@
+module Invidious::Database::Migrations
+  class CreateUsersTable < Migration
+    version 4
+
+    def up(conn : DB::Connection)
+      conn.exec <<-SQL
+      CREATE TABLE IF NOT EXISTS public.users
+      (
+        updated timestamp with time zone,
+        notifications text[],
+        subscriptions text[],
+        email text NOT NULL,
+        preferences text,
+        password text,
+        token text,
+        watched text[],
+        feed_needs_update boolean,
+        CONSTRAINT users_email_key UNIQUE (email)
+      );
+      SQL
+
+      conn.exec <<-SQL
+      GRANT ALL ON TABLE public.users TO current_user;
+      SQL
+
+      conn.exec <<-SQL
+      CREATE UNIQUE INDEX IF NOT EXISTS email_unique_idx
+        ON public.users
+        USING btree
+        (lower(email) COLLATE pg_catalog."default");
+      SQL
+    end
+  end
+end

src/invidious/database/migrations/0005_create_session_ids_table.cr

@@ -0,0 +1,28 @@
+module Invidious::Database::Migrations
+  class CreateSessionIdsTable < Migration
+    version 5
+
+    def up(conn : DB::Connection)
+      conn.exec <<-SQL
+      CREATE TABLE IF NOT EXISTS public.session_ids
+      (
+        id text NOT NULL,
+        email text,
+        issued timestamp with time zone,
+        CONSTRAINT session_ids_pkey PRIMARY KEY (id)
+      );
+      SQL
+
+      conn.exec <<-SQL
+      GRANT ALL ON TABLE public.session_ids TO current_user;
+      SQL
+
+      conn.exec <<-SQL
+      CREATE INDEX IF NOT EXISTS session_ids_id_idx
+        ON public.session_ids
+        USING btree
+        (id COLLATE pg_catalog."default");
+      SQL
+    end
+  end
+end

src/invidious/database/migrations/0006_create_nonces_table.cr

@@ -0,0 +1,27 @@
+module Invidious::Database::Migrations
+  class CreateNoncesTable < Migration
+    version 6
+
+    def up(conn : DB::Connection)
+      conn.exec <<-SQL
+      CREATE TABLE IF NOT EXISTS public.nonces
+      (
+        nonce text,
+        expire timestamp with time zone,
+        CONSTRAINT nonces_id_key UNIQUE (nonce)
+      );
+      SQL
+
+      conn.exec <<-SQL
+      GRANT ALL ON TABLE public.nonces TO current_user;
+      SQL
+
+      conn.exec <<-SQL
+      CREATE INDEX IF NOT EXISTS nonces_nonce_idx
+        ON public.nonces
+        USING btree
+        (nonce COLLATE pg_catalog."default");
+      SQL
+    end
+  end
+end

src/invidious/database/migrations/0007_create_annotations_table.cr

@@ -0,0 +1,20 @@
+module Invidious::Database::Migrations
+  class CreateAnnotationsTable < Migration
+    version 7
+
+    def up(conn : DB::Connection)
+      conn.exec <<-SQL
+      CREATE TABLE IF NOT EXISTS public.annotations
+      (
+        id text NOT NULL,
+        annotations xml,
+        CONSTRAINT annotations_id_key UNIQUE (id)
+      );
+      SQL
+
+      conn.exec <<-SQL
+      GRANT ALL ON TABLE public.annotations TO current_user;
+      SQL
+    end
+  end
+end

src/invidious/database/migrations/0008_create_playlists_table.cr

@@ -0,0 +1,50 @@
+module Invidious::Database::Migrations
+  class CreatePlaylistsTable < Migration
+    version 8
+
+    def up(conn : DB::Connection)
+      if !privacy_type_exists?(conn)
+        conn.exec <<-SQL
+        CREATE TYPE public.privacy AS ENUM
+        (
+          'Public',
+          'Unlisted',
+          'Private'
+        );
+        SQL
+      end
+
+      conn.exec <<-SQL
+      CREATE TABLE IF NOT EXISTS public.playlists
+      (
+        title text,
+        id text primary key,
+        author text,
+        description text,
+        video_count integer,
+        created timestamptz,
+        updated timestamptz,
+        privacy privacy,
+        index int8[]
+      );
+      SQL
+
+      conn.exec <<-SQL
+      GRANT ALL ON public.playlists TO current_user;
+      SQL
+    end
+
+    private def privacy_type_exists?(conn : DB::Connection) : Bool
+      request = <<-SQL
+        SELECT 1 AS one
+        FROM pg_type
+        INNER JOIN pg_namespace ON pg_namespace.oid = pg_type.typnamespace
+        WHERE pg_namespace.nspname = 'public'
+          AND pg_type.typname = 'privacy'
+        LIMIT 1;
+      SQL
+
+      !conn.query_one?(request, as: Int32).nil?
+    end
+  end
+end

src/invidious/database/migrations/0009_create_playlist_videos_table.cr

@@ -0,0 +1,27 @@
+module Invidious::Database::Migrations
+  class CreatePlaylistVideosTable < Migration
+    version 9
+
+    def up(conn : DB::Connection)
+      conn.exec <<-SQL
+      CREATE TABLE IF NOT EXISTS public.playlist_videos
+      (
+        title text,
+        id text,
+        author text,
+        ucid text,
+        length_seconds integer,
+        published timestamptz,
+        plid text references playlists(id),
+        index int8,
+        live_now boolean,
+        PRIMARY KEY (index,plid)
+      );
+      SQL
+
+      conn.exec <<-SQL
+      GRANT ALL ON TABLE public.playlist_videos TO current_user;
+      SQL
+    end
+  end
+end

src/invidious/database/migrations/0010_make_videos_unlogged.cr

@@ -0,0 +1,11 @@
+module Invidious::Database::Migrations
+  class MakeVideosUnlogged < Migration
+    version 10
+
+    def up(conn : DB::Connection)
+      conn.exec <<-SQL
+      ALTER TABLE public.videos SET UNLOGGED;
+      SQL
+    end
+  end
+end

src/invidious/database/migrator.cr

@@ -0,0 +1,49 @@
+class Invidious::Database::Migrator
+  MIGRATIONS_TABLE = "public.invidious_migrations"
+
+  class_getter migrations = [] of Invidious::Database::Migration.class
+
+  def initialize(@db : DB::Database)
+  end
+
+  def migrate
+    versions = load_versions
+
+    ran_migration = false
+    load_migrations.sort_by(&.version)
+      .each do |migration|
+        next if versions.includes?(migration.version)
+
+        puts "Running migration: #{migration.class.name}"
+        migration.migrate
+        ran_migration = true
+      end
+
+    puts "No migrations to run." unless ran_migration
+  end
+
+  def pending_migrations? : Bool
+    versions = load_versions
+
+    load_migrations.sort_by(&.version)
+      .any? { |migration| !versions.includes?(migration.version) }
+  end
+
+  private def load_migrations : Array(Invidious::Database::Migration)
+    self.class.migrations.map(&.new(@db))
+  end
+
+  private def load_versions : Array(Int64)
+    create_migrations_table
+    @db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64)
+  end
+
+  private def create_migrations_table
+    @db.exec <<-SQL
+      CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} (
+        id bigserial PRIMARY KEY,
+        version bigint NOT NULL
+      )
+    SQL
+  end
+end