From 37df7aceff00016aca18dcbabc8c298fb48d749b Mon Sep 17 00:00:00 2001 From: 4gray <4gray@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:02:17 +0200 Subject: [PATCH] perf(database): skip the per-launch duplicate scan and keep playlist payloads last (#1883) deduplicateXtreamCache no longer scans the whole catalog on every launch once both unique indexes exist (7.8 s cold on a 3.9M-title library). The playlists table keeps payload as its last column (playlists-table.ts); ensurePlaylistsPayloadLast rebuilds an existing table once, keeping its indexes and triggers, foreign keys off, committing only if references referencing playlists gain no violations. On a full copy of that profile initDatabase went from 5.2-8.1 s to 39 ms and the playlist metadata read from 1.5-2 s to 22-34 ms; the one-time rebuild costs about 4 s. Co-Authored-By: Claude Fable 5.1 Co-Authored-By: Claude Opus 5.5 --- .changes/database-startup-scans.md | 6 + libs/shared/database/README.md | 32 ++- .../src/lib/connection-migrations.spec.ts | 253 +++++++++++++++++- libs/shared/database/src/lib/connection.ts | 44 +-- .../database/src/lib/playlists-table.spec.ts | 202 ++++++++++++++ .../database/src/lib/playlists-table.ts | 170 ++++++++++++ libs/shared/database/src/lib/schema.ts | 3 +- .../src/lib/testing/connection-upgrade.ts | 41 +++ .../src/lib/xtream-cache-unique-indexes.ts | 24 ++ 9 files changed, 741 insertions(+), 34 deletions(-) create mode 100644 .changes/database-startup-scans.md create mode 100644 libs/shared/database/src/lib/playlists-table.spec.ts create mode 100644 libs/shared/database/src/lib/playlists-table.ts create mode 100644 libs/shared/database/src/lib/xtream-cache-unique-indexes.ts diff --git a/.changes/database-startup-scans.md b/.changes/database-startup-scans.md new file mode 100644 index 000000000..6ddf7add3 --- /dev/null +++ b/.changes/database-startup-scans.md @@ -0,0 +1,6 @@ +--- +type: perf +area: database +--- + +The app starts faster on large libraries: the catalog is no longer scanned for duplicates on every launch, and reading the playlist list no longer walks every M3U playlist's channel data (a one-time rebuild of the playlists table on the first start after the update). diff --git a/libs/shared/database/README.md b/libs/shared/database/README.md index c1b2e0383..14dd7adef 100644 --- a/libs/shared/database/README.md +++ b/libs/shared/database/README.md @@ -65,6 +65,34 @@ When changing initialization: install, and repeated startup. If a release changes migration ordering, cover each distinct affected historical schema. +Two startup costs found on a 4 GB profile (3.9M catalog titles, 182 +playlists) are guarded in `runMigrations()`: + +- `deduplicateXtreamCache()` returns at once when both unique indexes + (`src/lib/xtream-cache-unique-indexes.ts`: + `categories_playlist_type_xtream_unique`, + `content_category_type_xtream_unique`) exist: they make duplicates + impossible, and its two `GROUP BY ... HAVING COUNT(*) > 1` scans cost a full + pass over `content` on every launch otherwise (7.8 s cold on that profile). + A profile whose unique index could not be created yet still gets the repair. +- `payload` must be the last column of `playlists` (`PLAYLISTS_COLUMNS` in + `src/lib/playlists-table.ts`): SQLite + reads a row's columns in order and walks the overflow pages of every column + before the one it needs, so a column after an M3U payload costs the whole + payload on each read (`getAppPlaylistMetas` took 4-22 s for 70 MB of + payloads). Tables created before 0.19 got `payload` appended by `ALTER TABLE` + with the EPG URL columns after it, and fresh installs up to 0.24 created + `last_usage` after it; `ensurePlaylistsPayloadLast()` rebuilds such a table + once (new table, copy, drop, rename, with foreign keys off) and is a no-op + afterwards. It commits only if the tables referencing playlists hold no more + foreign key violations than before; rows a legacy profile already orphaned + do not block it. `src/lib/playlists-table.spec.ts` runs the rebuild and its + rollback on real SQLite. The rebuild rewrites every payload, about 4 s on + that profile; later starts open the database in about 40 ms instead of + 5-8 s. Never append a column after `payload`; add it to `PLAYLISTS_COLUMNS` + before `payload` and to + `COLUMN_MIGRATION_STATEMENTS`, and the rebuild moves it on the next start. + The #1580 index-ordering fix is included in 0.24 through PR #1550. `src/lib/connection-upgrades.spec.ts` exercises `initDatabase()` with real SQLite under the Electron runtime, using fresh-install schema snapshots from @@ -72,7 +100,9 @@ tags 0.19–0.23 and a fresh current database. The `epg_channel_id` column is ab in 0.19, present from 0.20, and indexed from 0.23. Each case checks current Drizzle tables, columns/types, and named indexes/uniqueness, as well as preserved user rows, foreign keys, database integrity, index availability, and repeated startup; -an existing EPG index must keep its definition and root page. Snapshots live in +an existing EPG index must keep its definition and root page, and `payload` must +end up as the last `playlists` column (the 0.19 snapshot reaches the current +schema through the ALTER list, so it reproduces an upgraded profile's order). Snapshots live in `src/lib/testing/fixtures/` and are independent of the current schema, so moving the index ahead of its column migration makes the 0.19 case fail again. diff --git a/libs/shared/database/src/lib/connection-migrations.spec.ts b/libs/shared/database/src/lib/connection-migrations.spec.ts index c7f2de17c..977a13780 100644 --- a/libs/shared/database/src/lib/connection-migrations.spec.ts +++ b/libs/shared/database/src/lib/connection-migrations.spec.ts @@ -1,4 +1,8 @@ import { __databaseConnectionTestHooks } from './connection'; +import { + ensurePlaylistsPayloadLast, + playlistsTableSql, +} from './playlists-table'; const { columnMigrationStatements, @@ -9,6 +13,7 @@ const { cleanupLegacyTmdbSearchCache, upgradeContentTitleFtsTokenizer, contentTitleFtsStatement, + deduplicateXtreamCache, } = __databaseConnectionTestHooks; type SqliteHandle = Parameters[0]; @@ -25,7 +30,11 @@ type StatementHandler = { type HandlerRule = [pattern: string, handler: StatementHandler]; -function createSqliteMock(rules: HandlerRule[], exec: jest.Mock = jest.fn()) { +function createSqliteMock( + rules: HandlerRule[], + exec: jest.Mock = jest.fn(), + pragma: jest.Mock = jest.fn(() => []) +) { const prepare = jest.fn((statement: string) => { const compact = compactSql(statement); const rule = rules.find(([pattern]) => compact.includes(pattern)); @@ -44,7 +53,13 @@ function createSqliteMock(rules: HandlerRule[], exec: jest.Mock = jest.fn()) { return { exec, prepare, - sqlite: { exec, prepare, transaction } as unknown as SqliteHandle, + pragma, + sqlite: { + exec, + prepare, + pragma, + transaction, + } as unknown as SqliteHandle, transaction, }; } @@ -414,3 +429,237 @@ describe('content title FTS tokenizer upgrade', () => { expect(marker.run).not.toHaveBeenCalled(); }); }); + +describe('deduplicateXtreamCache guard', () => { + const uniqueIndexCountRule = (count: number): HandlerRule => [ + "FROM sqlite_master WHERE type = 'index'", + { get: () => ({ count }) }, + ]; + + it('skips the duplicate scans once both unique indexes exist', () => { + const categoryGroups = jest.fn(() => []); + const contentGroups = jest.fn(() => []); + const { sqlite, transaction } = createSqliteMock([ + uniqueIndexCountRule(2), + [ + 'FROM categories GROUP BY playlist_id, type, xtream_id', + { all: categoryGroups }, + ], + [ + 'FROM content GROUP BY category_id, type, xtream_id', + { all: contentGroups }, + ], + ]); + + deduplicateXtreamCache(sqlite); + + expect(transaction).not.toHaveBeenCalled(); + expect(categoryGroups).not.toHaveBeenCalled(); + expect(contentGroups).not.toHaveBeenCalled(); + }); + + it('still scans while one of the unique indexes is missing', () => { + const categoryGroups = jest.fn(() => []); + const contentGroups = jest.fn(() => []); + const { sqlite, transaction } = createSqliteMock([ + uniqueIndexCountRule(1), + [ + 'FROM categories GROUP BY playlist_id, type, xtream_id', + { all: categoryGroups }, + ], + [ + 'FROM content GROUP BY category_id, type, xtream_id', + { all: contentGroups }, + ], + ]); + + deduplicateXtreamCache(sqlite); + + expect(transaction).toHaveBeenCalledTimes(1); + expect(categoryGroups).toHaveBeenCalledTimes(1); + expect(contentGroups).toHaveBeenCalledTimes(1); + }); + + it('names the unique indexes the index migrations create', () => { + for (const name of [ + 'categories_playlist_type_xtream_unique', + 'content_category_type_xtream_unique', + ]) { + expect( + indexMigrationStatements.some((statement) => + statement.includes( + `CREATE UNIQUE INDEX IF NOT EXISTS ${name} ` + ) + ) + ).toBe(true); + } + }); +}); + +describe('ensurePlaylistsPayloadLast', () => { + const canonicalColumns = [ + 'id', + 'name', + 'serverUrl', + 'username', + 'password', + 'date_created', + 'last_updated', + 'type', + 'userAgent', + 'origin', + 'referrer', + 'filePath', + 'epg_urls', + 'detected_epg_urls', + 'manual_epg_urls', + 'disabled_epg_urls', + 'autoRefresh', + 'macAddress', + 'url', + 'portal_url', + 'count', + 'import_date', + 'update_date', + 'position', + 'favorites', + 'recently_viewed', + 'last_usage', + 'payload', + ]; + /** A pre-0.19 table after its ALTER TABLE migrations. */ + const legacyColumns = [ + ...canonicalColumns.filter( + (name) => !name.endsWith('epg_urls') && name !== 'payload' + ), + 'payload', + 'epg_urls', + 'detected_epg_urls', + 'manual_epg_urls', + 'disabled_epg_urls', + ]; + /** Rules answering the column list and the foreign key check. */ + const playlistRules = ( + columns: string[], + foreignKeyCheck: unknown[] = [] + ): HandlerRule[] => [ + [ + "FROM pragma_table_info('playlists')", + { all: () => columns.map((name) => ({ name })) }, + ], + [ + 'FROM sqlite_master AS m, pragma_foreign_key_list(m.name)', + { all: () => [{ name: 'favorites' }, { name: 'categories' }] }, + ], + [ + 'FROM pragma_foreign_key_check(?)', + { + get: (table: unknown) => ({ + count: foreignKeyCheck.filter( + (row) => (row as { table: string }).table === table + ).length, + }), + }, + ], + ]; + const rebuildStatements = (prepare: jest.Mock) => + prepare.mock.calls + .map(([statement]) => compactSql(statement as string)) + .filter((statement) => !statement.includes('pragma_')); + + let warnSpy: jest.SpyInstance; + let logSpy: jest.SpyInstance; + + beforeEach(() => { + warnSpy = jest + .spyOn(console, 'warn') + .mockImplementation(() => undefined); + logSpy = jest.spyOn(console, 'log').mockImplementation(() => undefined); + }); + + afterEach(() => { + warnSpy.mockRestore(); + logSpy.mockRestore(); + }); + + it('creates fresh tables with payload as the last column', () => { + expect(createTableStatements[0]).toMatch( + /CREATE TABLE IF NOT EXISTS playlists \([\s\S]*payload TEXT\s*\)$/ + ); + expect(playlistsTableSql('rebuilt')).toMatch( + /^CREATE TABLE IF NOT EXISTS rebuilt \(/ + ); + }); + + it('leaves a table whose last column is payload alone', () => { + const { sqlite, prepare, pragma, transaction } = createSqliteMock( + playlistRules(canonicalColumns) + ); + + ensurePlaylistsPayloadLast(sqlite); + + expect(transaction).not.toHaveBeenCalled(); + expect(rebuildStatements(prepare)).toEqual([]); + expect(pragma).not.toHaveBeenCalled(); + }); + + it('rebuilds a table whose columns were appended after payload, with foreign keys off', () => { + const { sqlite, prepare, pragma, transaction } = createSqliteMock( + playlistRules(legacyColumns) + ); + + ensurePlaylistsPayloadLast(sqlite); + + expect(transaction).toHaveBeenCalledTimes(1); + const quoted = canonicalColumns.map((name) => `"${name}"`).join(', '); + expect(rebuildStatements(prepare)).toEqual([ + 'DROP TABLE IF EXISTS playlists_payload_rebuild', + expect.stringMatching( + /^CREATE TABLE IF NOT EXISTS playlists_payload_rebuild \( id TEXT PRIMARY KEY,.* last_usage TEXT, payload TEXT \)$/ + ), + `INSERT INTO playlists_payload_rebuild (${quoted}) SELECT ${quoted} FROM playlists`, + "SELECT sql FROM sqlite_master WHERE tbl_name = 'playlists' AND type IN ('index', 'trigger') AND sql IS NOT NULL ORDER BY type = 'trigger', name", + 'DROP TABLE playlists', + 'ALTER TABLE playlists_payload_rebuild RENAME TO playlists', + ]); + // Foreign keys go off before the transaction and come back after it, + // and the check runs inside it, after the rename. + expect(pragma.mock.calls.map(([statement]) => statement)).toEqual([ + 'foreign_keys = OFF', + 'foreign_keys = ON', + ]); + const all = prepare.mock.calls.map(([statement]) => + compactSql(statement as string) + ); + const checks = all.flatMap((statement, index) => + statement.startsWith( + 'SELECT COUNT(*) AS count FROM pragma_foreign_key_check(?)' + ) + ? [index] + : [] + ); + const rename = all.indexOf( + 'ALTER TABLE playlists_payload_rebuild RENAME TO playlists' + ); + // Counted once before the rebuild and once more after the rename. + expect(checks[0]).toBeLessThan(rename); + expect(checks.at(-1)).toBeGreaterThan(rename); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it('does not rebuild while a canonical column is still missing', () => { + const { sqlite, pragma, transaction } = createSqliteMock( + playlistRules( + legacyColumns.filter((name) => name !== 'disabled_epg_urls') + ) + ); + + ensurePlaylistsPayloadLast(sqlite); + + expect(transaction).not.toHaveBeenCalled(); + expect(pragma).not.toHaveBeenCalled(); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('columns missing: disabled_epg_urls') + ); + }); +}); diff --git a/libs/shared/database/src/lib/connection.ts b/libs/shared/database/src/lib/connection.ts index 934ba84bc..03036b764 100644 --- a/libs/shared/database/src/lib/connection.ts +++ b/libs/shared/database/src/lib/connection.ts @@ -3,6 +3,11 @@ import { DOWNLOADS_INDEX_STATEMENTS, ensureDownloadsCatchupSchema, } from './download-schema'; +import { + ensurePlaylistsPayloadLast, + playlistsTableSql, +} from './playlists-table'; +import { xtreamCacheUniqueIndexesExist } from './xtream-cache-unique-indexes'; /** * Database connection and initialization for IPTVnator * Uses Drizzle ORM with better-sqlite3 @@ -133,36 +138,7 @@ const RECORDINGS_INDEX_STATEMENTS = [ ]; const CREATE_TABLE_STATEMENTS = [ - `CREATE TABLE IF NOT EXISTS playlists ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - serverUrl TEXT, - username TEXT, - password TEXT, - date_created TEXT DEFAULT (datetime('now')), - last_updated TEXT, - type TEXT NOT NULL CHECK (type IN ('xtream', 'stalker', 'm3u-file', 'm3u-text', 'm3u-url')), - userAgent TEXT, - origin TEXT, - referrer TEXT, - filePath TEXT, - epg_urls TEXT, - detected_epg_urls TEXT, - manual_epg_urls TEXT, - disabled_epg_urls TEXT, - autoRefresh INTEGER DEFAULT 0, - macAddress TEXT, - url TEXT, - portal_url TEXT, - count INTEGER, - import_date TEXT, - update_date INTEGER, - position INTEGER, - favorites TEXT, - recently_viewed TEXT, - payload TEXT, - last_usage TEXT - )`, + playlistsTableSql('playlists'), `CREATE TABLE IF NOT EXISTS app_state ( key TEXT PRIMARY KEY, value TEXT NOT NULL, @@ -444,6 +420,7 @@ export const __databaseConnectionTestHooks = { columnMigrationStatements: COLUMN_MIGRATION_STATEMENTS, indexMigrationStatements: INDEX_MIGRATION_STATEMENTS, ensureDownloadsPauseResumeSchema, + deduplicateXtreamCache, normalizeXtreamContentAddedEpochs, ensureContentTitleFts, upgradeContentTitleFtsTokenizer, @@ -495,6 +472,12 @@ type XtreamContentCandidate = { }; function deduplicateXtreamCache(sqliteDb: Database.Database): void { + // With both unique indexes in place the cache cannot hold duplicates, + // and the two GROUP BY scans below would only cost a full pass over the + // content table on every launch (7.8 s on a 3.9M-title library). + if (xtreamCacheUniqueIndexesExist(sqliteDb)) { + return; + } const executeCleanup = sqliteDb.transaction(() => { const duplicateCategoryGroups = sqliteDb .prepare( @@ -1189,6 +1172,7 @@ function runMigrations(sqliteDb: Database.Database): void { cleanupLegacyTmdbSearchCache(sqliteDb); ensureDownloadsPauseResumeSchema(sqliteDb); runMigrationStatements(sqliteDb, COLUMN_MIGRATION_STATEMENTS); + ensurePlaylistsPayloadLast(sqliteDb); ensureDownloadsCatchupSchema(sqliteDb); // The tokenizer upgrade recreates and rebuilds the index itself, so the // plain rebuild below would only repeat work it just did. diff --git a/libs/shared/database/src/lib/playlists-table.spec.ts b/libs/shared/database/src/lib/playlists-table.spec.ts new file mode 100644 index 000000000..f4627b919 --- /dev/null +++ b/libs/shared/database/src/lib/playlists-table.spec.ts @@ -0,0 +1,202 @@ +import { execFileSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +interface RebuildOutcome { + warnings: string[]; + columns: string[]; + tableSql: string; + playlists: Record[]; + favorites: Record[]; + leftoverRebuildTable: boolean; + foreignKeys: number; + integrity: string; + dependents: { type: string; name: string }[]; + triggerFires: boolean; +} + +/** + * Runs ensurePlaylistsPayloadLast against a real in-memory SQLite database + * holding a pre-0.19 playlists table (payload appended before the EPG URL + * columns), a favorites table referencing it, and an orphaned favorite a + * legacy profile can carry. better-sqlite3 is built for Electron, so the + * script runs under Electron as Node. + */ +function rebuildLegacyPlaylists(options: { + blockingView: boolean; +}): RebuildOutcome { + const electronPath = createRequire(__filename)('electron') as string; + const moduleUrl = pathToFileURL( + resolve(__dirname, 'playlists-table.ts') + ).href; + const script = ` + const { default: Database } = await import('better-sqlite3'); + const { ensurePlaylistsPayloadLast } = await import(${JSON.stringify(moduleUrl)}); + const sqlite = new Database(':memory:'); + sqlite.pragma('foreign_keys = ON'); + sqlite.exec(\` + CREATE TABLE playlists ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, serverUrl TEXT, + username TEXT, password TEXT, + date_created TEXT DEFAULT (datetime('now')), last_updated TEXT, + type TEXT NOT NULL CHECK (type IN ('xtream', 'stalker', 'm3u-file', 'm3u-text', 'm3u-url')), + userAgent TEXT, origin TEXT, referrer TEXT, filePath TEXT, + autoRefresh INTEGER DEFAULT 0, macAddress TEXT, url TEXT, + last_usage TEXT + ); + ALTER TABLE playlists ADD COLUMN portal_url TEXT; + ALTER TABLE playlists ADD COLUMN count INTEGER; + ALTER TABLE playlists ADD COLUMN import_date TEXT; + ALTER TABLE playlists ADD COLUMN update_date INTEGER; + ALTER TABLE playlists ADD COLUMN position INTEGER; + ALTER TABLE playlists ADD COLUMN favorites TEXT; + ALTER TABLE playlists ADD COLUMN recently_viewed TEXT; + ALTER TABLE playlists ADD COLUMN payload TEXT; + ALTER TABLE playlists ADD COLUMN epg_urls TEXT; + ALTER TABLE playlists ADD COLUMN detected_epg_urls TEXT; + ALTER TABLE playlists ADD COLUMN manual_epg_urls TEXT; + ALTER TABLE playlists ADD COLUMN disabled_epg_urls TEXT; + CREATE TABLE favorites ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content_id INTEGER NOT NULL, + playlist_id TEXT NOT NULL, + FOREIGN KEY (playlist_id) REFERENCES playlists (id) ON DELETE CASCADE + ); + INSERT INTO playlists (id, name, type, url, last_usage, count, payload, + epg_urls, detected_epg_urls, manual_epg_urls, disabled_epg_urls) + VALUES ('m3u', 'Saved list', 'm3u-url', 'https://list.invalid/a.m3u', + '2026-01-01T00:00:00.000Z', 2, '{"channels":[{"name":"One"},{"name":"Two"}]}', + '["https://epg.invalid/a.xml"]', '["https://epg.invalid/b.xml"]', + '["https://epg.invalid/c.xml"]', '[]'); + INSERT INTO playlists (id, name, type, serverUrl, username, password) + VALUES ('xtream', 'Saved source', 'xtream', 'https://source.invalid', 'user', 'secret'); + INSERT INTO favorites (content_id, playlist_id) VALUES (1, 'm3u'), (2, 'xtream'); + \`); + sqlite.pragma('foreign_keys = OFF'); + sqlite.exec("INSERT INTO favorites (content_id, playlist_id) VALUES (3, 'deleted')"); + sqlite.pragma('foreign_keys = ON'); + sqlite.exec(\` + CREATE INDEX playlists_type_idx ON playlists(type); + CREATE TRIGGER playlists_reject_blocked BEFORE INSERT ON playlists + WHEN NEW.id = 'blocked' BEGIN SELECT RAISE(ABORT, 'blocked id'); END; + \`); + if (${options.blockingView}) { + sqlite.exec('CREATE VIEW playlist_names AS SELECT name FROM playlists'); + } + const warnings = []; + console.warn = (...args) => warnings.push(args.map(String).join(' ')); + console.log = () => undefined; + ensurePlaylistsPayloadLast(sqlite); + let triggerFires = false; + try { + sqlite.prepare("INSERT INTO playlists (id, name, type) VALUES ('blocked', 'B', 'm3u-url')").run(); + } catch (error) { + triggerFires = String(error).includes('blocked id'); + } + const byName = (rows) => rows.map((row) => + Object.fromEntries(Object.entries(row).sort(([a], [b]) => a.localeCompare(b)))); + process.stdout.write(JSON.stringify({ + warnings, + columns: sqlite.prepare("SELECT name FROM pragma_table_info('playlists')").all().map((r) => r.name), + tableSql: sqlite.prepare("SELECT sql FROM sqlite_master WHERE name = 'playlists'").get().sql, + playlists: byName(sqlite.prepare('SELECT * FROM playlists ORDER BY id').all()), + favorites: sqlite.prepare('SELECT * FROM favorites ORDER BY id').all(), + leftoverRebuildTable: Boolean(sqlite.prepare( + "SELECT 1 FROM sqlite_master WHERE name = 'playlists_payload_rebuild'").get()), + foreignKeys: sqlite.pragma('foreign_keys', { simple: true }), + dependents: sqlite.prepare( + "SELECT type, name FROM sqlite_master WHERE tbl_name = 'playlists' AND sql IS NOT NULL AND type IN ('index', 'trigger') ORDER BY name").all(), + triggerFires, + integrity: sqlite.pragma('integrity_check', { simple: true }), + })); + sqlite.close(); + `; + const output = execFileSync( + electronPath, + ['--import', 'tsx', '--eval', script], + { + cwd: process.cwd(), + encoding: 'utf8', + timeout: 30_000, + env: { + ...process.env, + ELECTRON_RUN_AS_NODE: '1', + TSX_TSCONFIG_PATH: resolve(process.cwd(), 'tsconfig.base.json'), + }, + } + ); + return JSON.parse(output) as RebuildOutcome; +} + +const expectedPlaylists = [ + expect.objectContaining({ + id: 'm3u', + name: 'Saved list', + url: 'https://list.invalid/a.m3u', + last_usage: '2026-01-01T00:00:00.000Z', + count: 2, + payload: '{"channels":[{"name":"One"},{"name":"Two"}]}', + epg_urls: '["https://epg.invalid/a.xml"]', + detected_epg_urls: '["https://epg.invalid/b.xml"]', + manual_epg_urls: '["https://epg.invalid/c.xml"]', + disabled_epg_urls: '[]', + }), + expect.objectContaining({ + id: 'xtream', + serverUrl: 'https://source.invalid', + username: 'user', + password: 'secret', + payload: null, + }), +]; + +const expectedDependents = [ + { type: 'trigger', name: 'playlists_reject_blocked' }, + { type: 'index', name: 'playlists_type_idx' }, +]; + +const expectedFavorites = [ + { id: 1, content_id: 1, playlist_id: 'm3u' }, + { id: 2, content_id: 2, playlist_id: 'xtream' }, + { id: 3, content_id: 3, playlist_id: 'deleted' }, +]; + +describe('ensurePlaylistsPayloadLast on real SQLite', () => { + it('moves payload last and keeps every row, favorite, pre-existing orphan, index and trigger', () => { + const outcome = rebuildLegacyPlaylists({ blockingView: false }); + + expect(outcome.warnings).toEqual([]); + expect(outcome.columns.at(-1)).toBe('payload'); + expect(outcome.columns).toHaveLength(28); + expect(outcome.playlists).toEqual(expectedPlaylists); + // Not cascaded away by the drop, and the orphan it found does not + // block the rebuild. + expect(outcome.favorites).toEqual(expectedFavorites); + expect(outcome.leftoverRebuildTable).toBe(false); + expect(outcome.foreignKeys).toBe(1); + expect(outcome.integrity).toBe('ok'); + expect(outcome.dependents).toEqual(expectedDependents); + expect(outcome.triggerFires).toBe(true); + }); + + it('rolls everything back when a step fails, and turns foreign keys back on', () => { + // A view on playlists makes the rename fail after the drop. + const outcome = rebuildLegacyPlaylists({ blockingView: true }); + + expect(outcome.warnings).toEqual([ + expect.stringContaining( + '[DB] playlists payload-last rebuild failed:' + ), + ]); + expect(outcome.columns.at(-1)).toBe('disabled_epg_urls'); + expect(outcome.tableSql).toContain('payload TEXT, epg_urls TEXT'); + expect(outcome.playlists).toEqual(expectedPlaylists); + expect(outcome.favorites).toEqual(expectedFavorites); + expect(outcome.leftoverRebuildTable).toBe(false); + expect(outcome.foreignKeys).toBe(1); + expect(outcome.integrity).toBe('ok'); + expect(outcome.dependents).toEqual(expectedDependents); + expect(outcome.triggerFires).toBe(true); + }); +}); diff --git a/libs/shared/database/src/lib/playlists-table.ts b/libs/shared/database/src/lib/playlists-table.ts new file mode 100644 index 000000000..95e10147e --- /dev/null +++ b/libs/shared/database/src/lib/playlists-table.ts @@ -0,0 +1,170 @@ +import type Database from 'better-sqlite3'; + +/** + * The playlists table's columns in creation order. `payload` (an M3U + * playlist's channels, up to several MB per row) must stay LAST: SQLite + * reads a row's columns in order and walks the overflow pages of every + * column before the one it needs, so any column after `payload` costs the + * whole payload on each read (playlist metadata took 4-22 s on a profile + * with 70 MB of payloads). Columns appended by `ALTER TABLE ADD COLUMN` + * land after it; `ensurePlaylistsPayloadLast` rebuilds such tables once. + */ +const PLAYLISTS_COLUMNS: readonly (readonly [ + name: string, + definition: string, +])[] = [ + ['id', 'TEXT PRIMARY KEY'], + ['name', 'TEXT NOT NULL'], + ['serverUrl', 'TEXT'], + ['username', 'TEXT'], + ['password', 'TEXT'], + ['date_created', `TEXT DEFAULT (datetime('now'))`], + ['last_updated', 'TEXT'], + [ + 'type', + `TEXT NOT NULL CHECK (type IN ('xtream', 'stalker', 'm3u-file', 'm3u-text', 'm3u-url'))`, + ], + ['userAgent', 'TEXT'], + ['origin', 'TEXT'], + ['referrer', 'TEXT'], + ['filePath', 'TEXT'], + ['epg_urls', 'TEXT'], + ['detected_epg_urls', 'TEXT'], + ['manual_epg_urls', 'TEXT'], + ['disabled_epg_urls', 'TEXT'], + ['autoRefresh', 'INTEGER DEFAULT 0'], + ['macAddress', 'TEXT'], + ['url', 'TEXT'], + ['portal_url', 'TEXT'], + ['count', 'INTEGER'], + ['import_date', 'TEXT'], + ['update_date', 'INTEGER'], + ['position', 'INTEGER'], + ['favorites', 'TEXT'], + ['recently_viewed', 'TEXT'], + ['last_usage', 'TEXT'], + ['payload', 'TEXT'], +]; + +export function playlistsTableSql(tableName: string): string { + const columns = PLAYLISTS_COLUMNS.map( + ([name, definition]) => ` ${name} ${definition}` + ).join(',\n'); + return `CREATE TABLE IF NOT EXISTS ${tableName} (\n${columns}\n )`; +} + +export const PLAYLISTS_REBUILD_TABLE = 'playlists_payload_rebuild'; + +/** + * Foreign key violations in the tables that reference playlists. Only those: + * a whole-database check would also walk the catalog and the EPG programmes. + */ +function countChildViolations(sqliteDb: Database.Database): number { + const children = sqliteDb + .prepare( + `SELECT DISTINCT m.name AS name + FROM sqlite_master AS m, pragma_foreign_key_list(m.name) AS f + WHERE m.type = 'table' AND f."table" = 'playlists'` + ) + .all() as { name: string }[]; + const checkChild = sqliteDb.prepare( + `SELECT COUNT(*) AS count FROM pragma_foreign_key_check(?)` + ); + return children.reduce( + (total, { name }) => + total + + ((checkChild.get(name) as { count?: number } | undefined)?.count ?? + 0), + 0 + ); +} + +/** + * Rebuilds `playlists` so that `payload` is its last column (see + * PLAYLISTS_COLUMNS). Tables created before 0.19 got `payload` appended by + * ALTER TABLE and the EPG URL columns after it; fresh installs up to 0.24 + * created `last_usage` after it. Runs after the column migrations, so the + * legacy table holds every canonical column, and only when a column follows + * `payload`: a rebuilt table is a no-op on the next start. Foreign keys are + * switched off for the rebuild (categories, favorites, history, positions + * and pins reference playlists ON DELETE CASCADE: the drop must not cascade, + * and the rename of the new table leaves their REFERENCES clauses alone), + * and the transaction commits only if the tables referencing playlists hold + * no more foreign key violations than before. Indexes and triggers on the + * table are recreated on the new one; a view on it makes the rename fail and + * rolls the rebuild back. + */ +export function ensurePlaylistsPayloadLast(sqliteDb: Database.Database): void { + const names = ( + sqliteDb + .prepare(`SELECT name FROM pragma_table_info('playlists')`) + .all() as { name: string }[] + ).map(({ name }) => name); + if (names.length === 0 || names[names.length - 1] === 'payload') { + return; + } + const canonical = PLAYLISTS_COLUMNS.map(([name]) => name); + const missing = canonical.filter((name) => !names.includes(name)); + if (missing.length > 0) { + console.warn( + `[DB] playlists rebuild skipped, columns missing: ${missing.join(', ')}` + ); + return; + } + const columnList = canonical.map((name) => `"${name}"`).join(', '); + const before = countChildViolations(sqliteDb); + sqliteDb.pragma('foreign_keys = OFF'); + try { + const rebuild = sqliteDb.transaction(() => { + sqliteDb + .prepare(`DROP TABLE IF EXISTS ${PLAYLISTS_REBUILD_TABLE}`) + .run(); + sqliteDb.prepare(playlistsTableSql(PLAYLISTS_REBUILD_TABLE)).run(); + sqliteDb + .prepare( + `INSERT INTO ${PLAYLISTS_REBUILD_TABLE} (${columnList}) + SELECT ${columnList} FROM playlists` + ) + .run(); + // DROP TABLE takes the table's own indexes and triggers with it; + // put them back on the new table after the rename. + const dependents = ( + sqliteDb + .prepare( + `SELECT sql FROM sqlite_master + WHERE tbl_name = 'playlists' + AND type IN ('index', 'trigger') + AND sql IS NOT NULL + ORDER BY type = 'trigger', name` + ) + .all() as { sql: string }[] + ).map(({ sql }) => sql); + sqliteDb.prepare(`DROP TABLE playlists`).run(); + sqliteDb + .prepare( + `ALTER TABLE ${PLAYLISTS_REBUILD_TABLE} RENAME TO playlists` + ) + .run(); + for (const statement of dependents) { + sqliteDb.prepare(statement).run(); + } + // The rebuild must not orphan a row. Rows a legacy profile already + // orphaned (written with foreign keys off) are not its doing and + // must not block it on every start. + const after = countChildViolations(sqliteDb); + if (after > before) { + throw new Error( + `${after - before} foreign key violation(s) after the rebuild` + ); + } + }); + rebuild(); + console.log( + '[DB] Rebuilt playlists table with payload as its last column' + ); + } catch (error) { + console.warn('[DB] playlists payload-last rebuild failed:', error); + } finally { + sqliteDb.pragma('foreign_keys = ON'); + } +} diff --git a/libs/shared/database/src/lib/schema.ts b/libs/shared/database/src/lib/schema.ts index 9dd3c2726..045338c04 100644 --- a/libs/shared/database/src/lib/schema.ts +++ b/libs/shared/database/src/lib/schema.ts @@ -52,8 +52,9 @@ export const playlists = sqliteTable('playlists', { position: integer('position'), favorites: text('favorites'), recentlyViewed: text('recently_viewed'), - payload: text('payload'), lastUsage: text('last_usage'), + // Last on purpose: see PLAYLISTS_COLUMNS in connection.ts. + payload: text('payload'), }); // App key-value state table (e.g. one-time migration flags) diff --git a/libs/shared/database/src/lib/testing/connection-upgrade.ts b/libs/shared/database/src/lib/testing/connection-upgrade.ts index dcfaa1c47..faa162cee 100644 --- a/libs/shared/database/src/lib/testing/connection-upgrade.ts +++ b/libs/shared/database/src/lib/testing/connection-upgrade.ts @@ -37,6 +37,38 @@ function seed(sqlite: Database.Database) { if (columns.some(({ name }) => name === 'epg_channel_id')) { sqlite.exec("UPDATE content SET epg_channel_id = 'retained-epg-id'"); } + // An M3U source with a channel payload and every playlist column the + // schema has: the payload-last rebuild of `playlists` must carry them all + // across unchanged. + const playlistColumns = new Set( + (sqlite.pragma('table_info(playlists)') as { name: string }[]).map( + ({ name }) => name + ) + ); + if (playlistColumns.has('payload')) { + const values: Record = { + id: 'm3u', + name: 'Saved list', + type: 'm3u-url', + url: 'https://list.invalid/a.m3u', + payload: '{"channels":[{"name":"One"}]}', + epg_urls: '["https://epg.invalid/a.xml"]', + detected_epg_urls: '["https://epg.invalid/b.xml"]', + favorites: '["one"]', + recently_viewed: '["one"]', + last_usage: '2026-01-01T00:00:00.000Z', + count: 1, + }; + const present = Object.keys(values).filter((name) => + playlistColumns.has(name) + ); + sqlite + .prepare( + `INSERT INTO playlists (${present.join(', ')}) + VALUES (${present.map(() => '?').join(', ')})` + ) + .run(...present.map((name) => values[name])); + } } /** @@ -145,6 +177,15 @@ async function main() { const sqlite = new Database(databasePath, { readonly: true }); try { verifyCurrentSchema(sqlite); + assert.equal( + ( + sqlite.pragma('table_info(playlists)') as { + name: string; + }[] + ).at(-1)?.name, + 'payload', + 'payload must be the last playlists column' + ); for (const { query, rows } of before) { assert.deepEqual(sqlite.prepare(query).all(), rows, query); } diff --git a/libs/shared/database/src/lib/xtream-cache-unique-indexes.ts b/libs/shared/database/src/lib/xtream-cache-unique-indexes.ts new file mode 100644 index 000000000..1e38e6b6f --- /dev/null +++ b/libs/shared/database/src/lib/xtream-cache-unique-indexes.ts @@ -0,0 +1,24 @@ +import type Database from 'better-sqlite3'; + +export const XTREAM_CACHE_UNIQUE_INDEXES = [ + 'categories_playlist_type_xtream_unique', + 'content_category_type_xtream_unique', +] as const; + +/** + * Both unique indexes of INDEX_MIGRATION_STATEMENTS exist, so the Xtream + * cache cannot hold the duplicates `deduplicateXtreamCache` repairs. + */ +export function xtreamCacheUniqueIndexesExist( + sqliteDb: Database.Database +): boolean { + const row = sqliteDb + .prepare( + `SELECT COUNT(*) AS count FROM sqlite_master + WHERE type = 'index' AND name IN (${XTREAM_CACHE_UNIQUE_INDEXES.map( + (name) => `'${name}'` + ).join(', ')})` + ) + .get() as { count?: number } | undefined; + return row?.count === XTREAM_CACHE_UNIQUE_INDEXES.length; +}