From a9f4ae38ea3c56d8fc10601c60a082f53759efdb Mon Sep 17 00:00:00 2001 From: 4gray <4gray@users.noreply.github.com> Date: Mon, 12 Oct 2026 07:37:27 +0200 Subject: [PATCH] perf(database): rebuild the title index once per profile, not again on the second launch (#1900) upgradeContentTitleFtsTokenizer recreates and rebuilds content_title_fts and records migration:content-title-fts-remove-diacritics:v1, but not migration:content-title-fts-trigram:v1. On a new profile the tokenizer upgrade therefore did the first rebuild, and on the second launch ensureContentTitleFts found its own marker missing and rebuilt the whole index again, synchronously inside initDatabase on the main thread, right after the first catalog import. A trigram rebuild costs about 4.5 s per million titles (sqlite3 3.51, synthetic titles). Every J4 iteration paid it too, since each launches a copy of a freshly seeded profile. The upgrade's transaction now records both markers. The skipped-release upgrade harness counts rebuilds across every initialization of a profile and requires exactly one; it fails for all six cases without the fix. Co-authored-by: 4gray Co-authored-by: Claude Opus 5.5 --- .../database-title-index-second-launch.md | 6 ++++ docs/architecture/sqlite-db-worker.md | 4 ++- docs/architecture/vod-multi-source.md | 5 +++- .../src/lib/connection-migrations.spec.ts | 7 ++++- libs/shared/database/src/lib/connection.ts | 22 ++++++++------ .../src/lib/testing/connection-upgrade.ts | 30 +++++++++++++++++++ 6 files changed, 62 insertions(+), 12 deletions(-) create mode 100644 .changes/database-title-index-second-launch.md diff --git a/.changes/database-title-index-second-launch.md b/.changes/database-title-index-second-launch.md new file mode 100644 index 000000000..5ce1b76b4 --- /dev/null +++ b/.changes/database-title-index-second-launch.md @@ -0,0 +1,6 @@ +--- +type: perf +area: database +--- + +The second launch after setting up a new library no longer rebuilds the whole title search index before the app opens, which took several seconds on large catalogs (about 4.5 s per million titles). diff --git a/docs/architecture/sqlite-db-worker.md b/docs/architecture/sqlite-db-worker.md index baa435811..582a0b429 100644 --- a/docs/architecture/sqlite-db-worker.md +++ b/docs/architecture/sqlite-db-worker.md @@ -550,7 +550,9 @@ Xtream-only row shape: as `Café` can still reach rows stored as either `Café` or `Cafe` before the worker's accent-insensitive ranking step. Existing databases rebuild this index once through the `migration:content-title-fts-trigram:v1` app-state - marker before legacy content cleanup/normalization migrations run; fresh + marker before legacy content cleanup/normalization migrations run (the + diacritics rebuild records that marker too, so it is never rebuilt twice; + see [VOD multi-source](vod-multi-source.md#short-titles-and-unicode)); fresh inserts, deletes, and title updates stay synchronized through SQLite triggers. 2. `source_type = "m3u"` rows come from M3U playlist payloads stored in `playlists.payload`. The worker uses SQL `payload LIKE` against channel diff --git a/docs/architecture/vod-multi-source.md b/docs/architecture/vod-multi-source.md index 11c13119b..604aafbb7 100644 --- a/docs/architecture/vod-multi-source.md +++ b/docs/architecture/vod-multi-source.md @@ -833,7 +833,10 @@ titles ("Amélie" → "amelie") against an index built from the raw one — with folding, every accented title was invisible to the FTS path, and two identical `Amélie` entries produced no candidates at all. The tokenizer is fixed at CREATE time, so existing databases are recreated and rebuilt once behind -`migration:content-title-fts-remove-diacritics:v1`. +`migration:content-title-fts-remove-diacritics:v1`. That rebuild also records +`migration:content-title-fts-trigram:v1`, the plain rebuild it supersedes; +before it did, a new profile rebuilt the whole index a second time on its +second launch, on the main thread, right after the first catalog import. `remove_diacritics` needs SQLite 3.45+. The migration probes support on a temp table first and does nothing when the runtime rejects it, leaving the working diff --git a/libs/shared/database/src/lib/connection-migrations.spec.ts b/libs/shared/database/src/lib/connection-migrations.spec.ts index 977a13780..28d81c1f7 100644 --- a/libs/shared/database/src/lib/connection-migrations.spec.ts +++ b/libs/shared/database/src/lib/connection-migrations.spec.ts @@ -358,7 +358,12 @@ describe('content title FTS tokenizer upgrade', () => { expect.stringContaining("tokenize='trigram remove_diacritics 1'") ); expect(rebuild.run).toHaveBeenCalled(); - expect(marker.run).toHaveBeenCalled(); + // The rebuild also covers the plain index migration; without its + // record the next launch rebuilds the whole index again. + expect(marker.run.mock.calls).toEqual([ + ['migration:content-title-fts-remove-diacritics:v1'], + ['migration:content-title-fts-trigram:v1'], + ]); // One transaction, so a rejected CREATE rolls the drop back and the // working index survives. expect(transaction).toHaveBeenCalled(); diff --git a/libs/shared/database/src/lib/connection.ts b/libs/shared/database/src/lib/connection.ts index 03036b764..5f026114b 100644 --- a/libs/shared/database/src/lib/connection.ts +++ b/libs/shared/database/src/lib/connection.ts @@ -755,15 +755,19 @@ function upgradeContentTitleFtsTokenizer(sqliteDb: Database.Database): boolean { ) .run(); - sqliteDb - .prepare( - `INSERT INTO app_state (key, value, updated_at) - VALUES (?, 'done', datetime('now')) - ON CONFLICT(key) DO UPDATE SET - value = excluded.value, - updated_at = excluded.updated_at` - ) - .run(CONTENT_TITLE_FTS_DIACRITICS_MIGRATION_KEY); + // The rebuild is also all that `ensureContentTitleFts` does, so + // its record is written too. Without it the next launch rebuilds + // the whole index again, on the main thread, on the first launch + // after a fresh install has imported its catalog. + const markDone = sqliteDb.prepare( + `INSERT INTO app_state (key, value, updated_at) + VALUES (?, 'done', datetime('now')) + ON CONFLICT(key) DO UPDATE SET + value = excluded.value, + updated_at = excluded.updated_at` + ); + markDone.run(CONTENT_TITLE_FTS_DIACRITICS_MIGRATION_KEY); + markDone.run(CONTENT_TITLE_FTS_MIGRATION_KEY); }); executeMigration(); diff --git a/libs/shared/database/src/lib/testing/connection-upgrade.ts b/libs/shared/database/src/lib/testing/connection-upgrade.ts index faa162cee..2a6b071ff 100644 --- a/libs/shared/database/src/lib/testing/connection-upgrade.ts +++ b/libs/shared/database/src/lib/testing/connection-upgrade.ts @@ -71,6 +71,30 @@ function seed(sqlite: Database.Database) { } } +const titleIndexRebuilds: string[] = []; + +/** + * Every full rebuild of the title FTS index goes through `prepare`. A profile + * needs one; another on a later launch re-indexes the whole catalog on the + * main thread before the app can read anything. + */ +function countTitleIndexRebuilds(connection: Database.Database): void { + const prepare = connection.prepare; + connection.prepare = function countingPrepare( + this: Database.Database, + sql: string + ) { + if ( + /content_title_fts\(content_title_fts\)\s+VALUES\s+\('rebuild'\)/.test( + sql + ) + ) { + titleIndexRebuilds.push(sql); + } + return prepare.call(this, sql); + } as typeof connection.prepare; +} + /** * The performance capture counts one `exec` call as one SQL statement * (apps/electron-backend/src/app/workers/database-worker-sql-statement-count.ts), @@ -81,6 +105,7 @@ function seed(sqlite: Database.Database) { function requireSingleStatementExec(): string[] { const batches: string[] = []; setDatabaseConnectionObserver((connection) => { + countTitleIndexRebuilds(connection); const exec = connection.exec; connection.exec = function singleStatementExec(sql: string) { try { @@ -230,6 +255,11 @@ async function main() { [], 'Initialization must pass one SQL statement per exec call' ); + assert.equal( + titleIndexRebuilds.length, + 1, + 'The title index must be rebuilt once per profile, not on every launch' + ); process.stdout.write('upgrade verified'); }