perf(database): skip the per-launch duplicate scan and keep playlist payloads last (#1883)

deduplicateXtreamCache no longer scans the whole catalog on every launch once both unique indexes exist (7.8 s cold on a 3.9M-title library). The playlists table keeps payload as its last column (playlists-table.ts); ensurePlaylistsPayloadLast rebuilds an existing table once, keeping its indexes and triggers, foreign keys off, committing only if references referencing playlists gain no violations. On a full copy of that profile initDatabase went from 5.2-8.1 s to 39 ms and the playlist metadata read from 1.5-2 s to 22-34 ms; the one-time rebuild costs about 4 s.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.changes/database-startup-scans.md

@@ -0,0 +1,6 @@
+---
+type: perf
+area: database
+---
+
+The app starts faster on large libraries: the catalog is no longer scanned for duplicates on every launch, and reading the playlist list no longer walks every M3U playlist's channel data (a one-time rebuild of the playlists table on the first start after the update).

libs/shared/database/README.md

@@ -65,6 +65,34 @@ When changing initialization:
   install, and repeated startup. If a release changes migration ordering, cover
   each distinct affected historical schema.
 
+Two startup costs found on a 4 GB profile (3.9M catalog titles, 182
+playlists) are guarded in `runMigrations()`:
+
+- `deduplicateXtreamCache()` returns at once when both unique indexes
+  (`src/lib/xtream-cache-unique-indexes.ts`:
+  `categories_playlist_type_xtream_unique`,
+  `content_category_type_xtream_unique`) exist: they make duplicates
+  impossible, and its two `GROUP BY ... HAVING COUNT(*) > 1` scans cost a full
+  pass over `content` on every launch otherwise (7.8 s cold on that profile).
+  A profile whose unique index could not be created yet still gets the repair.
+- `payload` must be the last column of `playlists` (`PLAYLISTS_COLUMNS` in
+  `src/lib/playlists-table.ts`): SQLite
+  reads a row's columns in order and walks the overflow pages of every column
+  before the one it needs, so a column after an M3U payload costs the whole
+  payload on each read (`getAppPlaylistMetas` took 4-22 s for 70 MB of
+  payloads). Tables created before 0.19 got `payload` appended by `ALTER TABLE`
+  with the EPG URL columns after it, and fresh installs up to 0.24 created
+  `last_usage` after it; `ensurePlaylistsPayloadLast()` rebuilds such a table
+  once (new table, copy, drop, rename, with foreign keys off) and is a no-op
+  afterwards. It commits only if the tables referencing playlists hold no more
+  foreign key violations than before; rows a legacy profile already orphaned
+  do not block it. `src/lib/playlists-table.spec.ts` runs the rebuild and its
+  rollback on real SQLite. The rebuild rewrites every payload, about 4 s on
+  that profile; later starts open the database in about 40 ms instead of
+  5-8 s. Never append a column after `payload`; add it to `PLAYLISTS_COLUMNS`
+  before `payload` and to
+  `COLUMN_MIGRATION_STATEMENTS`, and the rebuild moves it on the next start.
+
 The #1580 index-ordering fix is included in 0.24 through PR #1550.
 `src/lib/connection-upgrades.spec.ts` exercises `initDatabase()` with real
 SQLite under the Electron runtime, using fresh-install schema snapshots from
@@ -72,7 +100,9 @@ tags 0.19–0.23 and a fresh current database. The `epg_channel_id` column is ab
 in 0.19, present from 0.20, and indexed from 0.23. Each case checks current Drizzle
 tables, columns/types, and named indexes/uniqueness, as well as preserved user
 rows, foreign keys, database integrity, index availability, and repeated startup;
-an existing EPG index must keep its definition and root page. Snapshots live in
+an existing EPG index must keep its definition and root page, and `payload` must
+end up as the last `playlists` column (the 0.19 snapshot reaches the current
+schema through the ALTER list, so it reproduces an upgraded profile's order). Snapshots live in
 `src/lib/testing/fixtures/` and are independent of the current schema, so moving
 the index ahead of its column migration makes the 0.19 case fail again.
 

libs/shared/database/src/lib/connection-migrations.spec.ts

@@ -1,4 +1,8 @@
 import { __databaseConnectionTestHooks } from './connection';
+import {
+    ensurePlaylistsPayloadLast,
+    playlistsTableSql,
+} from './playlists-table';
 
 const {
     columnMigrationStatements,
@@ -9,6 +13,7 @@ const {
     cleanupLegacyTmdbSearchCache,
     upgradeContentTitleFtsTokenizer,
     contentTitleFtsStatement,
+    deduplicateXtreamCache,
 } = __databaseConnectionTestHooks;
 
 type SqliteHandle = Parameters<typeof runMigrations>[0];
@@ -25,7 +30,11 @@ type StatementHandler = {
 
 type HandlerRule = [pattern: string, handler: StatementHandler];
 
-function createSqliteMock(rules: HandlerRule[], exec: jest.Mock = jest.fn()) {
+function createSqliteMock(
+    rules: HandlerRule[],
+    exec: jest.Mock = jest.fn(),
+    pragma: jest.Mock = jest.fn(() => [])
+) {
     const prepare = jest.fn((statement: string) => {
         const compact = compactSql(statement);
         const rule = rules.find(([pattern]) => compact.includes(pattern));
@@ -44,7 +53,13 @@ function createSqliteMock(rules: HandlerRule[], exec: jest.Mock = jest.fn()) {
     return {
         exec,
         prepare,
-        sqlite: { exec, prepare, transaction } as unknown as SqliteHandle,
+        pragma,
+        sqlite: {
+            exec,
+            prepare,
+            pragma,
+            transaction,
+        } as unknown as SqliteHandle,
         transaction,
     };
 }
@@ -414,3 +429,237 @@ describe('content title FTS tokenizer upgrade', () => {
         expect(marker.run).not.toHaveBeenCalled();
     });
 });
+
+describe('deduplicateXtreamCache guard', () => {
+    const uniqueIndexCountRule = (count: number): HandlerRule => [
+        "FROM sqlite_master WHERE type = 'index'",
+        { get: () => ({ count }) },
+    ];
+
+    it('skips the duplicate scans once both unique indexes exist', () => {
+        const categoryGroups = jest.fn(() => []);
+        const contentGroups = jest.fn(() => []);
+        const { sqlite, transaction } = createSqliteMock([
+            uniqueIndexCountRule(2),
+            [
+                'FROM categories GROUP BY playlist_id, type, xtream_id',
+                { all: categoryGroups },
+            ],
+            [
+                'FROM content GROUP BY category_id, type, xtream_id',
+                { all: contentGroups },
+            ],
+        ]);
+
+        deduplicateXtreamCache(sqlite);
+
+        expect(transaction).not.toHaveBeenCalled();
+        expect(categoryGroups).not.toHaveBeenCalled();
+        expect(contentGroups).not.toHaveBeenCalled();
+    });
+
+    it('still scans while one of the unique indexes is missing', () => {
+        const categoryGroups = jest.fn(() => []);
+        const contentGroups = jest.fn(() => []);
+        const { sqlite, transaction } = createSqliteMock([
+            uniqueIndexCountRule(1),
+            [
+                'FROM categories GROUP BY playlist_id, type, xtream_id',
+                { all: categoryGroups },
+            ],
+            [
+                'FROM content GROUP BY category_id, type, xtream_id',
+                { all: contentGroups },
+            ],
+        ]);
+
+        deduplicateXtreamCache(sqlite);
+
+        expect(transaction).toHaveBeenCalledTimes(1);
+        expect(categoryGroups).toHaveBeenCalledTimes(1);
+        expect(contentGroups).toHaveBeenCalledTimes(1);
+    });
+
+    it('names the unique indexes the index migrations create', () => {
+        for (const name of [
+            'categories_playlist_type_xtream_unique',
+            'content_category_type_xtream_unique',
+        ]) {
+            expect(
+                indexMigrationStatements.some((statement) =>
+                    statement.includes(
+                        `CREATE UNIQUE INDEX IF NOT EXISTS ${name} `
+                    )
+                )
+            ).toBe(true);
+        }
+    });
+});
+
+describe('ensurePlaylistsPayloadLast', () => {
+    const canonicalColumns = [
+        'id',
+        'name',
+        'serverUrl',
+        'username',
+        'password',
+        'date_created',
+        'last_updated',
+        'type',
+        'userAgent',
+        'origin',
+        'referrer',
+        'filePath',
+        'epg_urls',
+        'detected_epg_urls',
+        'manual_epg_urls',
+        'disabled_epg_urls',
+        'autoRefresh',
+        'macAddress',
+        'url',
+        'portal_url',
+        'count',
+        'import_date',
+        'update_date',
+        'position',
+        'favorites',
+        'recently_viewed',
+        'last_usage',
+        'payload',
+    ];
+    /** A pre-0.19 table after its ALTER TABLE migrations. */
+    const legacyColumns = [
+        ...canonicalColumns.filter(
+            (name) => !name.endsWith('epg_urls') && name !== 'payload'
+        ),
+        'payload',
+        'epg_urls',
+        'detected_epg_urls',
+        'manual_epg_urls',
+        'disabled_epg_urls',
+    ];
+    /** Rules answering the column list and the foreign key check. */
+    const playlistRules = (
+        columns: string[],
+        foreignKeyCheck: unknown[] = []
+    ): HandlerRule[] => [
+        [
+            "FROM pragma_table_info('playlists')",
+            { all: () => columns.map((name) => ({ name })) },
+        ],
+        [
+            'FROM sqlite_master AS m, pragma_foreign_key_list(m.name)',
+            { all: () => [{ name: 'favorites' }, { name: 'categories' }] },
+        ],
+        [
+            'FROM pragma_foreign_key_check(?)',
+            {
+                get: (table: unknown) => ({
+                    count: foreignKeyCheck.filter(
+                        (row) => (row as { table: string }).table === table
+                    ).length,
+                }),
+            },
+        ],
+    ];
+    const rebuildStatements = (prepare: jest.Mock) =>
+        prepare.mock.calls
+            .map(([statement]) => compactSql(statement as string))
+            .filter((statement) => !statement.includes('pragma_'));
+
+    let warnSpy: jest.SpyInstance;
+    let logSpy: jest.SpyInstance;
+
+    beforeEach(() => {
+        warnSpy = jest
+            .spyOn(console, 'warn')
+            .mockImplementation(() => undefined);
+        logSpy = jest.spyOn(console, 'log').mockImplementation(() => undefined);
+    });
+
+    afterEach(() => {
+        warnSpy.mockRestore();
+        logSpy.mockRestore();
+    });
+
+    it('creates fresh tables with payload as the last column', () => {
+        expect(createTableStatements[0]).toMatch(
+            /CREATE TABLE IF NOT EXISTS playlists \([\s\S]*payload TEXT\s*\)$/
+        );
+        expect(playlistsTableSql('rebuilt')).toMatch(
+            /^CREATE TABLE IF NOT EXISTS rebuilt \(/
+        );
+    });
+
+    it('leaves a table whose last column is payload alone', () => {
+        const { sqlite, prepare, pragma, transaction } = createSqliteMock(
+            playlistRules(canonicalColumns)
+        );
+
+        ensurePlaylistsPayloadLast(sqlite);
+
+        expect(transaction).not.toHaveBeenCalled();
+        expect(rebuildStatements(prepare)).toEqual([]);
+        expect(pragma).not.toHaveBeenCalled();
+    });
+
+    it('rebuilds a table whose columns were appended after payload, with foreign keys off', () => {
+        const { sqlite, prepare, pragma, transaction } = createSqliteMock(
+            playlistRules(legacyColumns)
+        );
+
+        ensurePlaylistsPayloadLast(sqlite);
+
+        expect(transaction).toHaveBeenCalledTimes(1);
+        const quoted = canonicalColumns.map((name) => `"${name}"`).join(', ');
+        expect(rebuildStatements(prepare)).toEqual([
+            'DROP TABLE IF EXISTS playlists_payload_rebuild',
+            expect.stringMatching(
+                /^CREATE TABLE IF NOT EXISTS playlists_payload_rebuild \( id TEXT PRIMARY KEY,.* last_usage TEXT, payload TEXT \)$/
+            ),
+            `INSERT INTO playlists_payload_rebuild (${quoted}) SELECT ${quoted} FROM playlists`,
+            "SELECT sql FROM sqlite_master WHERE tbl_name = 'playlists' AND type IN ('index', 'trigger') AND sql IS NOT NULL ORDER BY type = 'trigger', name",
+            'DROP TABLE playlists',
+            'ALTER TABLE playlists_payload_rebuild RENAME TO playlists',
+        ]);
+        // Foreign keys go off before the transaction and come back after it,
+        // and the check runs inside it, after the rename.
+        expect(pragma.mock.calls.map(([statement]) => statement)).toEqual([
+            'foreign_keys = OFF',
+            'foreign_keys = ON',
+        ]);
+        const all = prepare.mock.calls.map(([statement]) =>
+            compactSql(statement as string)
+        );
+        const checks = all.flatMap((statement, index) =>
+            statement.startsWith(
+                'SELECT COUNT(*) AS count FROM pragma_foreign_key_check(?)'
+            )
+                ? [index]
+                : []
+        );
+        const rename = all.indexOf(
+            'ALTER TABLE playlists_payload_rebuild RENAME TO playlists'
+        );
+        // Counted once before the rebuild and once more after the rename.
+        expect(checks[0]).toBeLessThan(rename);
+        expect(checks.at(-1)).toBeGreaterThan(rename);
+        expect(warnSpy).not.toHaveBeenCalled();
+    });
+
+    it('does not rebuild while a canonical column is still missing', () => {
+        const { sqlite, pragma, transaction } = createSqliteMock(
+            playlistRules(
+                legacyColumns.filter((name) => name !== 'disabled_epg_urls')
+            )
+        );
+
+        ensurePlaylistsPayloadLast(sqlite);
+
+        expect(transaction).not.toHaveBeenCalled();
+        expect(pragma).not.toHaveBeenCalled();
+        expect(warnSpy).toHaveBeenCalledWith(
+            expect.stringContaining('columns missing: disabled_epg_urls')
+        );
+    });
+});

libs/shared/database/src/lib/connection.ts

@@ -3,6 +3,11 @@ import {
     DOWNLOADS_INDEX_STATEMENTS,
     ensureDownloadsCatchupSchema,
 } from './download-schema';
+import {
+    ensurePlaylistsPayloadLast,
+    playlistsTableSql,
+} from './playlists-table';
+import { xtreamCacheUniqueIndexesExist } from './xtream-cache-unique-indexes';
 /**
  * Database connection and initialization for IPTVnator
  * Uses Drizzle ORM with better-sqlite3
@@ -133,36 +138,7 @@ const RECORDINGS_INDEX_STATEMENTS = [
 ];
 
 const CREATE_TABLE_STATEMENTS = [
-    `CREATE TABLE IF NOT EXISTS playlists (
-      id TEXT PRIMARY KEY,
-      name TEXT NOT NULL,
-      serverUrl TEXT,
-      username TEXT,
-      password TEXT,
-      date_created TEXT DEFAULT (datetime('now')),
-      last_updated TEXT,
-      type TEXT NOT NULL CHECK (type IN ('xtream', 'stalker', 'm3u-file', 'm3u-text', 'm3u-url')),
-      userAgent TEXT,
-      origin TEXT,
-      referrer TEXT,
-      filePath TEXT,
-      epg_urls TEXT,
-      detected_epg_urls TEXT,
-      manual_epg_urls TEXT,
-      disabled_epg_urls TEXT,
-      autoRefresh INTEGER DEFAULT 0,
-      macAddress TEXT,
-      url TEXT,
-      portal_url TEXT,
-      count INTEGER,
-      import_date TEXT,
-      update_date INTEGER,
-      position INTEGER,
-      favorites TEXT,
-      recently_viewed TEXT,
-      payload TEXT,
-      last_usage TEXT
-  )`,
+    playlistsTableSql('playlists'),
     `CREATE TABLE IF NOT EXISTS app_state (
       key TEXT PRIMARY KEY,
       value TEXT NOT NULL,
@@ -444,6 +420,7 @@ export const __databaseConnectionTestHooks = {
     columnMigrationStatements: COLUMN_MIGRATION_STATEMENTS,
     indexMigrationStatements: INDEX_MIGRATION_STATEMENTS,
     ensureDownloadsPauseResumeSchema,
+    deduplicateXtreamCache,
     normalizeXtreamContentAddedEpochs,
     ensureContentTitleFts,
     upgradeContentTitleFtsTokenizer,
@@ -495,6 +472,12 @@ type XtreamContentCandidate = {
 };
 
 function deduplicateXtreamCache(sqliteDb: Database.Database): void {
+    // With both unique indexes in place the cache cannot hold duplicates,
+    // and the two GROUP BY scans below would only cost a full pass over the
+    // content table on every launch (7.8 s on a 3.9M-title library).
+    if (xtreamCacheUniqueIndexesExist(sqliteDb)) {
+        return;
+    }
     const executeCleanup = sqliteDb.transaction(() => {
         const duplicateCategoryGroups = sqliteDb
             .prepare(
@@ -1189,6 +1172,7 @@ function runMigrations(sqliteDb: Database.Database): void {
     cleanupLegacyTmdbSearchCache(sqliteDb);
     ensureDownloadsPauseResumeSchema(sqliteDb);
     runMigrationStatements(sqliteDb, COLUMN_MIGRATION_STATEMENTS);
+    ensurePlaylistsPayloadLast(sqliteDb);
     ensureDownloadsCatchupSchema(sqliteDb);
     // The tokenizer upgrade recreates and rebuilds the index itself, so the
     // plain rebuild below would only repeat work it just did.

libs/shared/database/src/lib/playlists-table.spec.ts

@@ -0,0 +1,202 @@
+import { execFileSync } from 'node:child_process';
+import { createRequire } from 'node:module';
+import { resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+interface RebuildOutcome {
+    warnings: string[];
+    columns: string[];
+    tableSql: string;
+    playlists: Record<string, unknown>[];
+    favorites: Record<string, unknown>[];
+    leftoverRebuildTable: boolean;
+    foreignKeys: number;
+    integrity: string;
+    dependents: { type: string; name: string }[];
+    triggerFires: boolean;
+}
+
+/**
+ * Runs ensurePlaylistsPayloadLast against a real in-memory SQLite database
+ * holding a pre-0.19 playlists table (payload appended before the EPG URL
+ * columns), a favorites table referencing it, and an orphaned favorite a
+ * legacy profile can carry. better-sqlite3 is built for Electron, so the
+ * script runs under Electron as Node.
+ */
+function rebuildLegacyPlaylists(options: {
+    blockingView: boolean;
+}): RebuildOutcome {
+    const electronPath = createRequire(__filename)('electron') as string;
+    const moduleUrl = pathToFileURL(
+        resolve(__dirname, 'playlists-table.ts')
+    ).href;
+    const script = `
+        const { default: Database } = await import('better-sqlite3');
+        const { ensurePlaylistsPayloadLast } = await import(${JSON.stringify(moduleUrl)});
+        const sqlite = new Database(':memory:');
+        sqlite.pragma('foreign_keys = ON');
+        sqlite.exec(\`
+            CREATE TABLE playlists (
+                id TEXT PRIMARY KEY, name TEXT NOT NULL, serverUrl TEXT,
+                username TEXT, password TEXT,
+                date_created TEXT DEFAULT (datetime('now')), last_updated TEXT,
+                type TEXT NOT NULL CHECK (type IN ('xtream', 'stalker', 'm3u-file', 'm3u-text', 'm3u-url')),
+                userAgent TEXT, origin TEXT, referrer TEXT, filePath TEXT,
+                autoRefresh INTEGER DEFAULT 0, macAddress TEXT, url TEXT,
+                last_usage TEXT
+            );
+            ALTER TABLE playlists ADD COLUMN portal_url TEXT;
+            ALTER TABLE playlists ADD COLUMN count INTEGER;
+            ALTER TABLE playlists ADD COLUMN import_date TEXT;
+            ALTER TABLE playlists ADD COLUMN update_date INTEGER;
+            ALTER TABLE playlists ADD COLUMN position INTEGER;
+            ALTER TABLE playlists ADD COLUMN favorites TEXT;
+            ALTER TABLE playlists ADD COLUMN recently_viewed TEXT;
+            ALTER TABLE playlists ADD COLUMN payload TEXT;
+            ALTER TABLE playlists ADD COLUMN epg_urls TEXT;
+            ALTER TABLE playlists ADD COLUMN detected_epg_urls TEXT;
+            ALTER TABLE playlists ADD COLUMN manual_epg_urls TEXT;
+            ALTER TABLE playlists ADD COLUMN disabled_epg_urls TEXT;
+            CREATE TABLE favorites (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                content_id INTEGER NOT NULL,
+                playlist_id TEXT NOT NULL,
+                FOREIGN KEY (playlist_id) REFERENCES playlists (id) ON DELETE CASCADE
+            );
+            INSERT INTO playlists (id, name, type, url, last_usage, count, payload,
+                epg_urls, detected_epg_urls, manual_epg_urls, disabled_epg_urls)
+            VALUES ('m3u', 'Saved list', 'm3u-url', 'https://list.invalid/a.m3u',
+                '2026-01-01T00:00:00.000Z', 2, '{"channels":[{"name":"One"},{"name":"Two"}]}',
+                '["https://epg.invalid/a.xml"]', '["https://epg.invalid/b.xml"]',
+                '["https://epg.invalid/c.xml"]', '[]');
+            INSERT INTO playlists (id, name, type, serverUrl, username, password)
+            VALUES ('xtream', 'Saved source', 'xtream', 'https://source.invalid', 'user', 'secret');
+            INSERT INTO favorites (content_id, playlist_id) VALUES (1, 'm3u'), (2, 'xtream');
+        \`);
+        sqlite.pragma('foreign_keys = OFF');
+        sqlite.exec("INSERT INTO favorites (content_id, playlist_id) VALUES (3, 'deleted')");
+        sqlite.pragma('foreign_keys = ON');
+        sqlite.exec(\`
+            CREATE INDEX playlists_type_idx ON playlists(type);
+            CREATE TRIGGER playlists_reject_blocked BEFORE INSERT ON playlists
+            WHEN NEW.id = 'blocked' BEGIN SELECT RAISE(ABORT, 'blocked id'); END;
+        \`);
+        if (${options.blockingView}) {
+            sqlite.exec('CREATE VIEW playlist_names AS SELECT name FROM playlists');
+        }
+        const warnings = [];
+        console.warn = (...args) => warnings.push(args.map(String).join(' '));
+        console.log = () => undefined;
+        ensurePlaylistsPayloadLast(sqlite);
+        let triggerFires = false;
+        try {
+            sqlite.prepare("INSERT INTO playlists (id, name, type) VALUES ('blocked', 'B', 'm3u-url')").run();
+        } catch (error) {
+            triggerFires = String(error).includes('blocked id');
+        }
+        const byName = (rows) => rows.map((row) =>
+            Object.fromEntries(Object.entries(row).sort(([a], [b]) => a.localeCompare(b))));
+        process.stdout.write(JSON.stringify({
+            warnings,
+            columns: sqlite.prepare("SELECT name FROM pragma_table_info('playlists')").all().map((r) => r.name),
+            tableSql: sqlite.prepare("SELECT sql FROM sqlite_master WHERE name = 'playlists'").get().sql,
+            playlists: byName(sqlite.prepare('SELECT * FROM playlists ORDER BY id').all()),
+            favorites: sqlite.prepare('SELECT * FROM favorites ORDER BY id').all(),
+            leftoverRebuildTable: Boolean(sqlite.prepare(
+                "SELECT 1 FROM sqlite_master WHERE name = 'playlists_payload_rebuild'").get()),
+            foreignKeys: sqlite.pragma('foreign_keys', { simple: true }),
+            dependents: sqlite.prepare(
+                "SELECT type, name FROM sqlite_master WHERE tbl_name = 'playlists' AND sql IS NOT NULL AND type IN ('index', 'trigger') ORDER BY name").all(),
+            triggerFires,
+            integrity: sqlite.pragma('integrity_check', { simple: true }),
+        }));
+        sqlite.close();
+    `;
+    const output = execFileSync(
+        electronPath,
+        ['--import', 'tsx', '--eval', script],
+        {
+            cwd: process.cwd(),
+            encoding: 'utf8',
+            timeout: 30_000,
+            env: {
+                ...process.env,
+                ELECTRON_RUN_AS_NODE: '1',
+                TSX_TSCONFIG_PATH: resolve(process.cwd(), 'tsconfig.base.json'),
+            },
+        }
+    );
+    return JSON.parse(output) as RebuildOutcome;
+}
+
+const expectedPlaylists = [
+    expect.objectContaining({
+        id: 'm3u',
+        name: 'Saved list',
+        url: 'https://list.invalid/a.m3u',
+        last_usage: '2026-01-01T00:00:00.000Z',
+        count: 2,
+        payload: '{"channels":[{"name":"One"},{"name":"Two"}]}',
+        epg_urls: '["https://epg.invalid/a.xml"]',
+        detected_epg_urls: '["https://epg.invalid/b.xml"]',
+        manual_epg_urls: '["https://epg.invalid/c.xml"]',
+        disabled_epg_urls: '[]',
+    }),
+    expect.objectContaining({
+        id: 'xtream',
+        serverUrl: 'https://source.invalid',
+        username: 'user',
+        password: 'secret',
+        payload: null,
+    }),
+];
+
+const expectedDependents = [
+    { type: 'trigger', name: 'playlists_reject_blocked' },
+    { type: 'index', name: 'playlists_type_idx' },
+];
+
+const expectedFavorites = [
+    { id: 1, content_id: 1, playlist_id: 'm3u' },
+    { id: 2, content_id: 2, playlist_id: 'xtream' },
+    { id: 3, content_id: 3, playlist_id: 'deleted' },
+];
+
+describe('ensurePlaylistsPayloadLast on real SQLite', () => {
+    it('moves payload last and keeps every row, favorite, pre-existing orphan, index and trigger', () => {
+        const outcome = rebuildLegacyPlaylists({ blockingView: false });
+
+        expect(outcome.warnings).toEqual([]);
+        expect(outcome.columns.at(-1)).toBe('payload');
+        expect(outcome.columns).toHaveLength(28);
+        expect(outcome.playlists).toEqual(expectedPlaylists);
+        // Not cascaded away by the drop, and the orphan it found does not
+        // block the rebuild.
+        expect(outcome.favorites).toEqual(expectedFavorites);
+        expect(outcome.leftoverRebuildTable).toBe(false);
+        expect(outcome.foreignKeys).toBe(1);
+        expect(outcome.integrity).toBe('ok');
+        expect(outcome.dependents).toEqual(expectedDependents);
+        expect(outcome.triggerFires).toBe(true);
+    });
+
+    it('rolls everything back when a step fails, and turns foreign keys back on', () => {
+        // A view on playlists makes the rename fail after the drop.
+        const outcome = rebuildLegacyPlaylists({ blockingView: true });
+
+        expect(outcome.warnings).toEqual([
+            expect.stringContaining(
+                '[DB] playlists payload-last rebuild failed:'
+            ),
+        ]);
+        expect(outcome.columns.at(-1)).toBe('disabled_epg_urls');
+        expect(outcome.tableSql).toContain('payload TEXT, epg_urls TEXT');
+        expect(outcome.playlists).toEqual(expectedPlaylists);
+        expect(outcome.favorites).toEqual(expectedFavorites);
+        expect(outcome.leftoverRebuildTable).toBe(false);
+        expect(outcome.foreignKeys).toBe(1);
+        expect(outcome.integrity).toBe('ok');
+        expect(outcome.dependents).toEqual(expectedDependents);
+        expect(outcome.triggerFires).toBe(true);
+    });
+});

libs/shared/database/src/lib/playlists-table.ts

@@ -0,0 +1,170 @@
+import type Database from 'better-sqlite3';
+
+/**
+ * The playlists table's columns in creation order. `payload` (an M3U
+ * playlist's channels, up to several MB per row) must stay LAST: SQLite
+ * reads a row's columns in order and walks the overflow pages of every
+ * column before the one it needs, so any column after `payload` costs the
+ * whole payload on each read (playlist metadata took 4-22 s on a profile
+ * with 70 MB of payloads). Columns appended by `ALTER TABLE ADD COLUMN`
+ * land after it; `ensurePlaylistsPayloadLast` rebuilds such tables once.
+ */
+const PLAYLISTS_COLUMNS: readonly (readonly [
+    name: string,
+    definition: string,
+])[] = [
+    ['id', 'TEXT PRIMARY KEY'],
+    ['name', 'TEXT NOT NULL'],
+    ['serverUrl', 'TEXT'],
+    ['username', 'TEXT'],
+    ['password', 'TEXT'],
+    ['date_created', `TEXT DEFAULT (datetime('now'))`],
+    ['last_updated', 'TEXT'],
+    [
+        'type',
+        `TEXT NOT NULL CHECK (type IN ('xtream', 'stalker', 'm3u-file', 'm3u-text', 'm3u-url'))`,
+    ],
+    ['userAgent', 'TEXT'],
+    ['origin', 'TEXT'],
+    ['referrer', 'TEXT'],
+    ['filePath', 'TEXT'],
+    ['epg_urls', 'TEXT'],
+    ['detected_epg_urls', 'TEXT'],
+    ['manual_epg_urls', 'TEXT'],
+    ['disabled_epg_urls', 'TEXT'],
+    ['autoRefresh', 'INTEGER DEFAULT 0'],
+    ['macAddress', 'TEXT'],
+    ['url', 'TEXT'],
+    ['portal_url', 'TEXT'],
+    ['count', 'INTEGER'],
+    ['import_date', 'TEXT'],
+    ['update_date', 'INTEGER'],
+    ['position', 'INTEGER'],
+    ['favorites', 'TEXT'],
+    ['recently_viewed', 'TEXT'],
+    ['last_usage', 'TEXT'],
+    ['payload', 'TEXT'],
+];
+
+export function playlistsTableSql(tableName: string): string {
+    const columns = PLAYLISTS_COLUMNS.map(
+        ([name, definition]) => `      ${name} ${definition}`
+    ).join(',\n');
+    return `CREATE TABLE IF NOT EXISTS ${tableName} (\n${columns}\n  )`;
+}
+
+export const PLAYLISTS_REBUILD_TABLE = 'playlists_payload_rebuild';
+
+/**
+ * Foreign key violations in the tables that reference playlists. Only those:
+ * a whole-database check would also walk the catalog and the EPG programmes.
+ */
+function countChildViolations(sqliteDb: Database.Database): number {
+    const children = sqliteDb
+        .prepare(
+            `SELECT DISTINCT m.name AS name
+             FROM sqlite_master AS m, pragma_foreign_key_list(m.name) AS f
+             WHERE m.type = 'table' AND f."table" = 'playlists'`
+        )
+        .all() as { name: string }[];
+    const checkChild = sqliteDb.prepare(
+        `SELECT COUNT(*) AS count FROM pragma_foreign_key_check(?)`
+    );
+    return children.reduce(
+        (total, { name }) =>
+            total +
+            ((checkChild.get(name) as { count?: number } | undefined)?.count ??
+                0),
+        0
+    );
+}
+
+/**
+ * Rebuilds `playlists` so that `payload` is its last column (see
+ * PLAYLISTS_COLUMNS). Tables created before 0.19 got `payload` appended by
+ * ALTER TABLE and the EPG URL columns after it; fresh installs up to 0.24
+ * created `last_usage` after it. Runs after the column migrations, so the
+ * legacy table holds every canonical column, and only when a column follows
+ * `payload`: a rebuilt table is a no-op on the next start. Foreign keys are
+ * switched off for the rebuild (categories, favorites, history, positions
+ * and pins reference playlists ON DELETE CASCADE: the drop must not cascade,
+ * and the rename of the new table leaves their REFERENCES clauses alone),
+ * and the transaction commits only if the tables referencing playlists hold
+ * no more foreign key violations than before. Indexes and triggers on the
+ * table are recreated on the new one; a view on it makes the rename fail and
+ * rolls the rebuild back.
+ */
+export function ensurePlaylistsPayloadLast(sqliteDb: Database.Database): void {
+    const names = (
+        sqliteDb
+            .prepare(`SELECT name FROM pragma_table_info('playlists')`)
+            .all() as { name: string }[]
+    ).map(({ name }) => name);
+    if (names.length === 0 || names[names.length - 1] === 'payload') {
+        return;
+    }
+    const canonical = PLAYLISTS_COLUMNS.map(([name]) => name);
+    const missing = canonical.filter((name) => !names.includes(name));
+    if (missing.length > 0) {
+        console.warn(
+            `[DB] playlists rebuild skipped, columns missing: ${missing.join(', ')}`
+        );
+        return;
+    }
+    const columnList = canonical.map((name) => `"${name}"`).join(', ');
+    const before = countChildViolations(sqliteDb);
+    sqliteDb.pragma('foreign_keys = OFF');
+    try {
+        const rebuild = sqliteDb.transaction(() => {
+            sqliteDb
+                .prepare(`DROP TABLE IF EXISTS ${PLAYLISTS_REBUILD_TABLE}`)
+                .run();
+            sqliteDb.prepare(playlistsTableSql(PLAYLISTS_REBUILD_TABLE)).run();
+            sqliteDb
+                .prepare(
+                    `INSERT INTO ${PLAYLISTS_REBUILD_TABLE} (${columnList})
+                     SELECT ${columnList} FROM playlists`
+                )
+                .run();
+            // DROP TABLE takes the table's own indexes and triggers with it;
+            // put them back on the new table after the rename.
+            const dependents = (
+                sqliteDb
+                    .prepare(
+                        `SELECT sql FROM sqlite_master
+                         WHERE tbl_name = 'playlists'
+                           AND type IN ('index', 'trigger')
+                           AND sql IS NOT NULL
+                         ORDER BY type = 'trigger', name`
+                    )
+                    .all() as { sql: string }[]
+            ).map(({ sql }) => sql);
+            sqliteDb.prepare(`DROP TABLE playlists`).run();
+            sqliteDb
+                .prepare(
+                    `ALTER TABLE ${PLAYLISTS_REBUILD_TABLE} RENAME TO playlists`
+                )
+                .run();
+            for (const statement of dependents) {
+                sqliteDb.prepare(statement).run();
+            }
+            // The rebuild must not orphan a row. Rows a legacy profile already
+            // orphaned (written with foreign keys off) are not its doing and
+            // must not block it on every start.
+            const after = countChildViolations(sqliteDb);
+            if (after > before) {
+                throw new Error(
+                    `${after - before} foreign key violation(s) after the rebuild`
+                );
+            }
+        });
+        rebuild();
+        console.log(
+            '[DB] Rebuilt playlists table with payload as its last column'
+        );
+    } catch (error) {
+        console.warn('[DB] playlists payload-last rebuild failed:', error);
+    } finally {
+        sqliteDb.pragma('foreign_keys = ON');
+    }
+}

libs/shared/database/src/lib/schema.ts

@@ -52,8 +52,9 @@ export const playlists = sqliteTable('playlists', {
     position: integer('position'),
     favorites: text('favorites'),
     recentlyViewed: text('recently_viewed'),
-    payload: text('payload'),
     lastUsage: text('last_usage'),
+    // Last on purpose: see PLAYLISTS_COLUMNS in connection.ts.
+    payload: text('payload'),
 });
 
 // App key-value state table (e.g. one-time migration flags)

libs/shared/database/src/lib/testing/connection-upgrade.ts

@@ -37,6 +37,38 @@ function seed(sqlite: Database.Database) {
     if (columns.some(({ name }) => name === 'epg_channel_id')) {
         sqlite.exec("UPDATE content SET epg_channel_id = 'retained-epg-id'");
     }
+    // An M3U source with a channel payload and every playlist column the
+    // schema has: the payload-last rebuild of `playlists` must carry them all
+    // across unchanged.
+    const playlistColumns = new Set(
+        (sqlite.pragma('table_info(playlists)') as { name: string }[]).map(
+            ({ name }) => name
+        )
+    );
+    if (playlistColumns.has('payload')) {
+        const values: Record<string, string | number> = {
+            id: 'm3u',
+            name: 'Saved list',
+            type: 'm3u-url',
+            url: 'https://list.invalid/a.m3u',
+            payload: '{"channels":[{"name":"One"}]}',
+            epg_urls: '["https://epg.invalid/a.xml"]',
+            detected_epg_urls: '["https://epg.invalid/b.xml"]',
+            favorites: '["one"]',
+            recently_viewed: '["one"]',
+            last_usage: '2026-01-01T00:00:00.000Z',
+            count: 1,
+        };
+        const present = Object.keys(values).filter((name) =>
+            playlistColumns.has(name)
+        );
+        sqlite
+            .prepare(
+                `INSERT INTO playlists (${present.join(', ')})
+                 VALUES (${present.map(() => '?').join(', ')})`
+            )
+            .run(...present.map((name) => values[name]));
+    }
 }
 
 /**
@@ -145,6 +177,15 @@ async function main() {
             const sqlite = new Database(databasePath, { readonly: true });
             try {
                 verifyCurrentSchema(sqlite);
+                assert.equal(
+                    (
+                        sqlite.pragma('table_info(playlists)') as {
+                            name: string;
+                        }[]
+                    ).at(-1)?.name,
+                    'payload',
+                    'payload must be the last playlists column'
+                );
                 for (const { query, rows } of before) {
                     assert.deepEqual(sqlite.prepare(query).all(), rows, query);
                 }

libs/shared/database/src/lib/xtream-cache-unique-indexes.ts

@@ -0,0 +1,24 @@
+import type Database from 'better-sqlite3';
+
+export const XTREAM_CACHE_UNIQUE_INDEXES = [
+    'categories_playlist_type_xtream_unique',
+    'content_category_type_xtream_unique',
+] as const;
+
+/**
+ * Both unique indexes of INDEX_MIGRATION_STATEMENTS exist, so the Xtream
+ * cache cannot hold the duplicates `deduplicateXtreamCache` repairs.
+ */
+export function xtreamCacheUniqueIndexesExist(
+    sqliteDb: Database.Database
+): boolean {
+    const row = sqliteDb
+        .prepare(
+            `SELECT COUNT(*) AS count FROM sqlite_master
+             WHERE type = 'index' AND name IN (${XTREAM_CACHE_UNIQUE_INDEXES.map(
+                 (name) => `'${name}'`
+             ).join(', ')})`
+        )
+        .get() as { count?: number } | undefined;
+    return row?.count === XTREAM_CACHE_UNIQUE_INDEXES.length;
+}