Skip to content

perf: SQLite optimization for large databases (1M+ torrents) #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gmonarque merged 5 commits into from
Mar 27, 2026
Merged

perf: SQLite optimization for large databases (1M+ torrents) #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7a79b0c
Add paginated historical fetch to bypass relay's per-subscription eve…
jblemee Mar 5, 2026
e4c0ae0
perf: SQLite indexes, PRAGMAs, and query optimizations
gschaer Mar 25, 2026
085983f
perf: cache trust data and resume historical fetch from last event
gschaer Mar 25, 2026
97cee9e
fix: config persistence, Torznab API keys, and rate limits
gschaer Mar 25, 2026
883352e
fix: address PR review feedback on correctness
gschaer Mar 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions internal/api/handlers/indexer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func StopIndexer(w http.ResponseWriter, r *http.Request) {
})
}

// GetIndexerStatus returns the current indexer status
// GetIndexerStatus returns the current indexer status (lightweight, no heavy queries)
func GetIndexerStatus(w http.ResponseWriter, r *http.Request) {
running := false
if indexerController != nil {
Expand All @@ -107,14 +107,20 @@ func GetIndexerStatus(w http.ResponseWriter, r *http.Request) {

enabled, _ := database.GetSetting("indexer_enabled")

// Get indexing stats
stats, _ := database.GetStats()
// Use lightweight count queries instead of full GetStats()
db := database.Get()

var totalTorrents int64
db.QueryRow("SELECT COUNT(*) FROM torrents").Scan(&totalTorrents)

var connectedRelays int64
db.QueryRow("SELECT COUNT(*) FROM relays WHERE status = 'connected'").Scan(&connectedRelays)

respondJSON(w, http.StatusOK, map[string]interface{}{
"running": running,
"enabled": enabled == "true",
"total_torrents": stats["total_torrents"],
"connected_relays": stats["connected_relays"],
"total_torrents": totalTorrents,
"connected_relays": connectedRelays,
})
}

Expand Down
94 changes: 56 additions & 38 deletions internal/api/handlers/search.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,16 +102,22 @@ func Search(w http.ResponseWriter, r *http.Request) {

var rows *sql.Rows

// Build trust EXISTS subquery (avoids JOIN + DISTINCT overhead)
trustExistsClause := `EXISTS (
SELECT 1 FROM torrent_uploads tu
WHERE tu.torrent_id = t.id
AND tu.uploader_npub IN ` + trustPlaceholders + `
)`

if query != "" {
// Full-text search with trust filtering
sqlQuery := `
SELECT DISTINCT t.id, t.info_hash, t.name, t.size, t.category, t.seeders, t.leechers,
SELECT t.id, t.info_hash, t.name, t.size, t.category, t.seeders, t.leechers,
t.magnet_uri, t.title, t.year, t.poster_url, t.overview, t.trust_score, t.first_seen_at
FROM torrents t
JOIN torrents_fts fts ON t.id = fts.rowid
JOIN torrent_uploads tu ON t.id = tu.torrent_id
WHERE torrents_fts MATCH ?
AND tu.uploader_npub IN ` + trustPlaceholders
AND ` + trustExistsClause

args := []interface{}{query}
args = append(args, trustArgs...)
Expand All @@ -132,32 +138,43 @@ func Search(w http.ResponseWriter, r *http.Request) {
args = append(args, limit, offset)

rows, err = db.Query(sqlQuery, args...)
} else {
// List all (no search query) with trust filtering
} else if category != "" {
// Category filter: use composite index (category, trust_score, first_seen_at)
// to filter by category AND walk in sort order simultaneously.
// Empty categories return instantly (no matching index entries).
// Populated categories find 50 rows by walking the index in order.
sqlQuery := `
SELECT DISTINCT t.id, t.info_hash, t.name, t.size, t.category, t.seeders, t.leechers,
SELECT t.id, t.info_hash, t.name, t.size, t.category, t.seeders, t.leechers,
t.magnet_uri, t.title, t.year, t.poster_url, t.overview, t.trust_score, t.first_seen_at
FROM torrents t
JOIN torrent_uploads tu ON t.id = tu.torrent_id
WHERE tu.uploader_npub IN ` + trustPlaceholders
FROM torrents t INDEXED BY idx_torrents_category_trust_seen
WHERE ` + trustExistsClause

args := append([]interface{}{}, trustArgs...)

if category != "" {
if isBaseCategory {
// Match all subcategories within the base category range
sqlQuery += " AND t.category >= ? AND t.category < ?"
args = append(args, categoryNum, categoryNum+1000)
} else {
// Exact match for subcategory
sqlQuery += " AND t.category = ?"
args = append(args, categoryNum)
}
if isBaseCategory {
sqlQuery += " AND t.category >= ? AND t.category < ?"
args = append(args, categoryNum, categoryNum+1000)
} else {
sqlQuery += " AND t.category = ?"
args = append(args, categoryNum)
}

sqlQuery += " ORDER BY t.trust_score DESC, t.first_seen_at DESC LIMIT ? OFFSET ?"
args = append(args, limit, offset)

rows, err = db.Query(sqlQuery, args...)
} else {
// No filters: walk the sort index directly, check trust for each row, stop at LIMIT
sqlQuery := `
SELECT t.id, t.info_hash, t.name, t.size, t.category, t.seeders, t.leechers,
t.magnet_uri, t.title, t.year, t.poster_url, t.overview, t.trust_score, t.first_seen_at
FROM torrents t INDEXED BY idx_torrents_trust_first_seen
WHERE ` + trustExistsClause + `
ORDER BY t.trust_score DESC, t.first_seen_at DESC LIMIT ? OFFSET ?`

args := append([]interface{}{}, trustArgs...)
args = append(args, limit, offset)

rows, err = db.Query(sqlQuery, args...)
}

Expand Down Expand Up @@ -203,11 +220,10 @@ func Search(w http.ResponseWriter, r *http.Request) {
var total int64
if query != "" {
countQuery := `
SELECT COUNT(DISTINCT t.id) FROM torrents t
SELECT COUNT(*) FROM torrents t
JOIN torrents_fts fts ON t.id = fts.rowid
JOIN torrent_uploads tu ON t.id = tu.torrent_id
WHERE torrents_fts MATCH ?
AND tu.uploader_npub IN ` + trustPlaceholders
AND ` + trustExistsClause

countArgs := []interface{}{query}
countArgs = append(countArgs, trustArgs...)
Expand All @@ -221,23 +237,25 @@ func Search(w http.ResponseWriter, r *http.Request) {
}
}
db.QueryRow(countQuery, countArgs...).Scan(&total)
} else {
countQuery := `
SELECT COUNT(DISTINCT t.id) FROM torrents t
JOIN torrent_uploads tu ON t.id = tu.torrent_id
WHERE tu.uploader_npub IN ` + trustPlaceholders

countArgs := append([]interface{}{}, trustArgs...)
if category != "" {
if isBaseCategory {
countQuery += " AND t.category >= ? AND t.category < ?"
countArgs = append(countArgs, categoryNum, categoryNum+1000)
} else {
countQuery += " AND t.category = ?"
countArgs = append(countArgs, categoryNum)
}
} else if category != "" {
Copy link
Copy Markdown
Owner

@gmonarque gmonarque Mar 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

total doesn’t look aligned with results anymore. This path drops the trust filter for category-only counts, and the unfiltered branch below counts upload rows rather than distinct torrents, so pagination can report more hits than the query can actually return

Copy link
Copy Markdown
Contributor Author

@gschaer gschaer Mar 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed the unfiltered count — now uses COUNT(DISTINCT torrent_id) FROM torrent_uploads which is correct and performs the same as COUNT(*) in benchmarks.

For category-filtered counts, the exact trust-filtered count requires a full JOIN across both tables (O(n) scan), which is prohibitively slow on large databases. We keep the category-index-only count (O(log n)) as an approximation. In practice this is accurate because the indexer already filters by trusted authors at ingest time, so untrusted torrents rarely exist in the table. Added a comment explaining this trade-off.

As a future improvement, the UI could display approximate counts as "~172,000 results" or "1-50 of many" (similar to Gmail's search) instead of an exact total, which would make this trade-off transparent to the user.

// Category count: use the category index directly. An exact trust-filtered
// count requires a JOIN across both tables (O(n) full scan), so we use an
// approximate count from the category index alone (O(log n) index lookup).
// This may slightly overcount if untrusted torrents exist in the category,
// but the indexer already filters by trusted authors at ingest time, so
// the difference is negligible in practice.
if isBaseCategory {
db.QueryRow(`SELECT COUNT(*) FROM torrents WHERE category >= ? AND category < ?`,
categoryNum, categoryNum+1000).Scan(&total)
} else {
db.QueryRow(`SELECT COUNT(*) FROM torrents WHERE category = ?`,
categoryNum).Scan(&total)
}
db.QueryRow(countQuery, countArgs...).Scan(&total)
} else {
// No filters: count distinct torrents from trusted uploaders
countQuery := `SELECT COUNT(DISTINCT torrent_id) FROM torrent_uploads
WHERE uploader_npub IN ` + trustPlaceholders
db.QueryRow(countQuery, trustArgs...).Scan(&total)
}

respondJSON(w, http.StatusOK, map[string]interface{}{
Expand Down
105 changes: 56 additions & 49 deletions internal/api/handlers/stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,36 @@ package handlers

import (
"database/sql"
"encoding/json"
"net/http"
"strconv"
"sync"
"time"

"github.com/gmonarque/lighthouse/internal/database"
"github.com/gmonarque/lighthouse/internal/nostr"
"github.com/gmonarque/lighthouse/internal/trust"
)

// GetStats returns dashboard statistics (filtered by trust)
// statsCache caches the expensive stats response (60s TTL)
var statsCache struct {
mu sync.RWMutex
data []byte
expiry time.Time
}

// GetStats returns dashboard statistics (filtered by trust), cached for 60s
func GetStats(w http.ResponseWriter, r *http.Request) {
// Serve from cache if fresh
statsCache.mu.RLock()
if time.Now().Before(statsCache.expiry) && statsCache.data != nil {
data := statsCache.data
statsCache.mu.RUnlock()
w.Header().Set("Content-Type", "application/json")
w.Write(data)
return
}
statsCache.mu.RUnlock()
db := database.Get()

// Get trusted uploaders for filtering
Expand Down Expand Up @@ -41,7 +61,7 @@ func GetStats(w http.ResponseWriter, r *http.Request) {
stats["categories"] = make(map[int]int64)
stats["recent_torrents"] = []interface{}{}
} else {
// Build trust filter subquery
// Build trust EXISTS subquery (avoids JOIN + DISTINCT overhead)
trustPlaceholders := "("
trustArgs := make([]interface{}, len(trustedHexPubkeys))
for i, u := range trustedHexPubkeys {
Expand All @@ -53,32 +73,28 @@ func GetStats(w http.ResponseWriter, r *http.Request) {
}
trustPlaceholders += ")"

// Total trusted torrents
var totalTorrents int64
countQuery := `SELECT COUNT(DISTINCT t.id) FROM torrents t
JOIN torrent_uploads tu ON t.id = tu.torrent_id
WHERE tu.uploader_npub IN ` + trustPlaceholders
if err := db.QueryRow(countQuery, trustArgs...).Scan(&totalTorrents); err != nil {
respondError(w, http.StatusInternalServerError, "Failed to get stats")
return
}
stats["total_torrents"] = totalTorrents
// Drive queries from torrent_uploads (much smaller than torrents).
// torrent_uploads is filtered by uploader via index, then JOINed to torrents by PK.
// Use COUNT(DISTINCT) to avoid overcounting torrents with multiple uploads.

// Total size (bytes) for trusted torrents
// Count distinct torrents + total size in a single query
var totalTorrents int64
var totalSize sql.NullInt64
sizeQuery := `SELECT SUM(t.size) FROM torrents t
JOIN torrent_uploads tu ON t.id = tu.torrent_id
summaryQuery := `SELECT COUNT(DISTINCT tu.torrent_id), COALESCE(SUM(t.size), 0)
FROM torrents t INNER JOIN torrent_uploads tu ON t.id = tu.torrent_id
WHERE tu.uploader_npub IN ` + trustPlaceholders
if err := db.QueryRow(sizeQuery, trustArgs...).Scan(&totalSize); err != nil {
if err := db.QueryRow(summaryQuery, trustArgs...).Scan(&totalTorrents, &totalSize); err != nil {
respondError(w, http.StatusInternalServerError, "Failed to get stats")
return
}
stats["total_torrents"] = totalTorrents
stats["total_size"] = totalSize.Int64

// Torrents by category (trusted only)
catQuery := `SELECT t.category, COUNT(DISTINCT t.id) as count FROM torrents t
JOIN torrent_uploads tu ON t.id = tu.torrent_id
WHERE tu.uploader_npub IN ` + trustPlaceholders + ` GROUP BY t.category`
// Categories: same approach, drive from torrent_uploads
catQuery := `SELECT t.category, COUNT(DISTINCT tu.torrent_id) as count
FROM torrents t INNER JOIN torrent_uploads tu ON t.id = tu.torrent_id
WHERE tu.uploader_npub IN ` + trustPlaceholders + `
GROUP BY t.category`
rows, err := db.Query(catQuery, trustArgs...)
if err != nil {
respondError(w, http.StatusInternalServerError, "Failed to get stats")
Expand All @@ -97,12 +113,16 @@ func GetStats(w http.ResponseWriter, r *http.Request) {
}
stats["categories"] = categories

// Recent trusted torrents
recentQuery := `SELECT DISTINCT t.id, t.info_hash, t.name, t.size, t.category, t.seeders, t.leechers,
// Recent 10: EXISTS with first_seen_at index is fast for small LIMIT
trustExistsClause := `EXISTS (
SELECT 1 FROM torrent_uploads tu
WHERE tu.torrent_id = t.id
AND tu.uploader_npub IN ` + trustPlaceholders + `
)`
recentQuery := `SELECT t.id, t.info_hash, t.name, t.size, t.category, t.seeders, t.leechers,
t.title, t.year, t.poster_url, t.trust_score, t.first_seen_at
FROM torrents t
JOIN torrent_uploads tu ON t.id = tu.torrent_id
WHERE tu.uploader_npub IN ` + trustPlaceholders + `
WHERE ` + trustExistsClause + `
ORDER BY t.first_seen_at DESC LIMIT 10`
recentRows, err := db.Query(recentQuery, trustArgs...)
if err != nil {
Expand Down Expand Up @@ -163,29 +183,16 @@ func GetStats(w http.ResponseWriter, r *http.Request) {
}
stats["blacklist_count"] = blacklistCount

// Unique uploaders should also be filtered by trust
var uniqueUploaders int64
if len(trustedHexPubkeys) > 0 {
// Build placeholder string for IN clause
uploaderPlaceholders := "("
uploaderArgs := make([]interface{}, len(trustedHexPubkeys))
for i, u := range trustedHexPubkeys {
if i > 0 {
uploaderPlaceholders += ","
}
uploaderPlaceholders += "?"
uploaderArgs[i] = u
}
uploaderPlaceholders += ")"
// Unique uploaders (just count trusted pubkeys directly — no DB query needed)
stats["unique_uploaders"] = int64(len(trustedHexPubkeys))

uploaderQuery := `SELECT COUNT(DISTINCT uploader_npub) FROM torrent_uploads WHERE uploader_npub IN ` + uploaderPlaceholders
if err := db.QueryRow(uploaderQuery, uploaderArgs...).Scan(&uniqueUploaders); err != nil {
uniqueUploaders = 0
}
} else {
uniqueUploaders = 0
// Cache the result for 60 seconds
if jsonData, err := json.Marshal(stats); err == nil {
statsCache.mu.Lock()
statsCache.data = jsonData
statsCache.expiry = time.Now().Add(60 * time.Second)
statsCache.mu.Unlock()
}
stats["unique_uploaders"] = uniqueUploaders

respondJSON(w, http.StatusOK, stats)
}
Expand Down Expand Up @@ -229,7 +236,7 @@ func GetStatsChart(w http.ResponseWriter, r *http.Request) {
return
}

// Build trust filter
// Build trust EXISTS subquery
trustPlaceholders := "("
trustArgs := make([]interface{}, len(trustedHexPubkeys)+1)
trustArgs[0] = days
Expand All @@ -242,9 +249,9 @@ func GetStatsChart(w http.ResponseWriter, r *http.Request) {
}
trustPlaceholders += ")"

query := `SELECT DATE(t.first_seen_at) as date, COUNT(DISTINCT t.id) as count
FROM torrents t
JOIN torrent_uploads tu ON t.id = tu.torrent_id
// Drive from torrent_uploads, join torrents only for first_seen_at
query := `SELECT DATE(t.first_seen_at) as date, COUNT(*) as count
FROM torrents t INNER JOIN torrent_uploads tu ON t.id = tu.torrent_id
WHERE t.first_seen_at >= DATE('now', '-' || ? || ' days')
AND tu.uploader_npub IN ` + trustPlaceholders + `
GROUP BY DATE(t.first_seen_at)
Expand Down
27 changes: 25 additions & 2 deletions internal/api/handlers/torznab.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,40 @@ import (
"net/http"
"strconv"

"github.com/gmonarque/lighthouse/internal/api/apikeys"
"github.com/gmonarque/lighthouse/internal/api/middleware"
"github.com/gmonarque/lighthouse/internal/config"
"github.com/gmonarque/lighthouse/internal/torznab"
)

// Torznab handles all Torznab API requests
func Torznab(w http.ResponseWriter, r *http.Request) {
// Validate API key
// Validate API key: accept legacy config key OR multi-user key with torznab permission
cfg := config.Get()
apiKey := r.URL.Query().Get("apikey")
if apiKey == "" {
apiKey = r.Header.Get("X-API-Key")
}

authenticated := false

// Check legacy single API key
if cfg.Server.APIKey != "" && apiKey == cfg.Server.APIKey {
authenticated = true
}

// Check multi-user API keys with torznab permission
if !authenticated && apiKey != "" {
storage := middleware.GetAPIKeyStorage()
if key, err := storage.ValidateKey(apiKey); err == nil && key != nil {
if key.HasAnyPermission(apikeys.PermissionTorznab, apikeys.PermissionAdmin) {
authenticated = true
}
}
}

if cfg.Server.APIKey != "" && apiKey != cfg.Server.APIKey {
// If auth is required and not authenticated, reject
if !authenticated && (cfg.Server.APIKey != "" || apiKey != "") {
respondTorznabError(w, torznab.ErrorIncorrectUserCreds, "Invalid API key")
return
}
Expand Down
Loading