Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions internal/catalog/rank.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,50 @@ CREATE TRIGGER catalog_rank_insert AFTER INSERT ON catalog_titles FOR EACH ROW W
CREATE TRIGGER catalog_rank_update AFTER UPDATE OF `+rankWatched+` ON catalog_titles FOR EACH ROW WHEN (NEW.type IN ('movie','show') OR OLD.type IN ('movie','show')) EXECUTE FUNCTION catalog_rank_fn();
CREATE TRIGGER catalog_rank_delete AFTER DELETE ON catalog_titles FOR EACH ROW WHEN (OLD.type IN ('movie','show')) EXECUTE FUNCTION catalog_rank_fn();
INSERT INTO catalog_meta(k,v) VALUES('rank_side','a'),('rank_building','') ON CONFLICT(k) DO UPDATE SET v=excluded.v;`)
// A logical restore (a backup, or ether cluster init copying SQLite into
// PostgreSQL) writes tables, not views: the target keeps the view its
// migrations created, on catalog_rank_a, while the restored rank_side
// may name catalog_rank_b. Point the view at the restored live copy.
store.RegisterRestore("catalog_rank_view", func(context.Context, *sql.Tx) (func(context.Context, *sql.Tx) error, error) {
return func(ctx context.Context, tx *sql.Tx) error {
for _, s := range rankViewSQL(liveRankSide(ctx, tx)) {
if _, err := tx.ExecContext(ctx, s); err != nil {
return fmt.Errorf("catalog: point rank view at the restored copy: %w", err)
}
}
return nil
}, nil
})
}

// liveRankSide is the copy rank_side names, the one the triggers maintain.
func liveRankSide(ctx context.Context, q interface {
QueryRowContext(context.Context, string, ...any) *sql.Row
}) string {
var v string
if err := q.QueryRowContext(ctx, `SELECT v FROM catalog_meta WHERE k='rank_side'`).Scan(&v); err == nil && v == "b" {
return "b"
}
return "a"
}

// rankViewSQL points the catalog_rank view at copy side. Drop and create
// work on both dialects; SQLite has no CREATE OR REPLACE VIEW.
func rankViewSQL(side string) []string {
return []string{`DROP VIEW IF EXISTS catalog_rank`, `CREATE VIEW catalog_rank AS SELECT ` + rankColumns + ` FROM catalog_rank_` + side}
}

// rankViewNames reports whether the catalog_rank view reads copy side.
func (c *Catalog) rankViewNames(ctx context.Context, pg bool, side string) (bool, error) {
q := `SELECT sql FROM sqlite_master WHERE type='view' AND name='catalog_rank'`
if pg {
q = `SELECT pg_get_viewdef('catalog_rank'::regclass)`
}
var def string
if err := c.db.QueryRowContext(ctx, q).Scan(&def); err != nil {
return false, err
}
return strings.Contains(def, "catalog_rank_"+side), nil
}

// rankRebuildEvery spaces the rank rebuilds: TMDB publishes popularity
Expand Down Expand Up @@ -255,6 +299,23 @@ func (c *Catalog) rebuildRank(ctx context.Context) (RankRebuild, error) {
}
defer conn.ExecContext(context.WithoutCancel(ctx), `SELECT pg_advisory_unlock(hashtext('ether:catalog-rank-rebuild'))`)
}
// A database restored before the restore hook pointed the view at the
// live copy may still read the idle one: popularity pages read that
// emptied copy, and PostgreSQL refuses to drop a table a view depends
// on, so every rebuild failed. Point the view back first.
if stale, err := c.rankViewNames(ctx, pg, idle); err != nil {
return out, fmt.Errorf("catalog: rank rebuild: read view: %w", err)
} else if stale {
if pg {
err = c.swapRankViewPG(ctx, "catalog_rank_"+live, nil)
} else {
err = exec(rankViewSQL(live)...)
}
if err != nil {
return out, err
}
c.log.Warn("catalog: popularity rank view read the idle copy; pointed it at the live copy", "live", "catalog_rank_"+live)
}
// From here the triggers leave the idle copy alone.
if err := exec(`DELETE FROM catalog_rank_pending`, `UPDATE catalog_meta SET v='`+idle+`' WHERE k='rank_building'`); err != nil {
return out, err
Expand Down
102 changes: 102 additions & 0 deletions internal/catalog/rank_view_restore_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package catalog

import (
"bytes"
"context"
"database/sql"
"fmt"
"testing"
"time"

"github.com/braedonsaunders/ether/internal/core"
"github.com/braedonsaunders/ether/internal/store"
)

// rankViewFixture stores movies and rebuilds the rank once, so the live
// copy is catalog_rank_b and catalog_rank_a is the emptied, retired one.
func rankViewFixture(t *testing.T, ctx context.Context) (*Catalog, int) {
t.Helper()
c := newChunkTestCatalog(t)
rel := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
var titles []core.Title
for i := int64(1); i <= 25; i++ {
titles = append(titles, core.Title{Key: core.MovieKey(i), Type: core.TypeMovie, Title: fmt.Sprint("M", i), Released: rel, Popularity: float64(i)})
}
if err := c.upsertMany(ctx, titles, upsertOpts{browsable: true}); err != nil {
t.Fatal(err)
}
if _, err := c.RebuildRank(ctx); err != nil {
t.Fatal(err)
}
if side := rankSideOf(t, ctx, c.db); side != "b" {
t.Fatalf("fixture: live rank copy %q after one rebuild, want b", side)
}
return c, len(titles)
}

func rankSideOf(t *testing.T, ctx context.Context, db *sql.DB) string {
t.Helper()
var side string
if err := db.QueryRowContext(ctx, `SELECT v FROM catalog_meta WHERE k='rank_side'`).Scan(&side); err != nil {
t.Fatal(err)
}
return side
}

func rankViewRows(t *testing.T, ctx context.Context, db *sql.DB) int {
t.Helper()
var n int
if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM catalog_rank`).Scan(&n); err != nil {
t.Fatal(err)
}
return n
}

// A logical copy (ether cluster init from SQLite to PostgreSQL, a backup
// restore) restores tables, not views: the target's view is the one its
// migrations created, on catalog_rank_a, while the restored rank_side names
// catalog_rank_b. Popularity pages read the empty retired copy, and on
// PostgreSQL every later rebuild failed to drop catalog_rank_a ("other
// objects depend on it"), so the rank never recovered.
func TestLogicalRestoreKeepsTheRankViewOnTheLiveCopy(t *testing.T) {
ctx := context.Background()
src, titles := rankViewFixture(t, ctx)
var dump bytes.Buffer
if err := store.ExportLogical(ctx, src.db, &dump); err != nil {
t.Fatal(err)
}
dst := newChunkTestCatalog(t)
if err := store.ImportLogical(ctx, dst.db, bytes.NewReader(dump.Bytes())); err != nil {
t.Fatal(err)
}
if got := rankViewRows(t, ctx, dst.db); got != titles {
t.Fatalf("catalog_rank after restore: %d rows, want %d", got, titles)
}
if _, err := dst.RebuildRank(ctx); err != nil {
t.Fatalf("rank rebuild after restore: %v", err)
}
if got := rankViewRows(t, ctx, dst.db); got != titles {
t.Fatalf("catalog_rank after the next rebuild: %d rows, want %d", got, titles)
}
}

// A database already left with the view on the retired copy recovers at
// the next rebuild.
func TestRankRebuildRepairsAViewOnTheIdleCopy(t *testing.T) {
ctx := context.Background()
c, titles := rankViewFixture(t, ctx)
for _, q := range []string{`DROP VIEW catalog_rank`, `CREATE VIEW catalog_rank AS SELECT ` + rankColumns + ` FROM catalog_rank_a`} {
if _, err := c.db.ExecContext(ctx, q); err != nil {
t.Fatal(err)
}
}
if got := rankViewRows(t, ctx, c.db); got != 0 {
t.Fatalf("fixture: view on the retired copy reads %d rows", got)
}
if _, err := c.RebuildRank(ctx); err != nil {
t.Fatalf("rank rebuild with the view on the idle copy: %v", err)
}
if got := rankViewRows(t, ctx, c.db); got != titles {
t.Fatalf("catalog_rank after the rebuild: %d rows, want %d", got, titles)
}
}