mirror of
https://github.com/4gray/iptvnator.git
synced 2026-10-11 11:06:16 -08:00
fix(database): verify the title index folds, rather than trust the marker
`createTables` declares content_title_fts with the plain trigram tokenizer, and the diacritics migration declares it again with folding. Two sources of truth for one tokenizer: if the table ever went missing after the marker was written, `CREATE TABLE IF NOT EXISTS` would restore the unfolded form and the migration would skip it on the marker alone. The upgrade now reads the live table's own DDL from sqlite_master and rebuilds unless it really folds. A degraded index is invisible from the outside — discovery just stops finding "Pokémon" for "pokemon" — so the record has to be checked against the thing it describes. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
354df3d851
commit
501282289a
3 files changed
+80
-4
No files matched your search
@@ -531,7 +531,16 @@ CREATE time, so existing databases are recreated and rebuilt once behind
|
||||
`remove_diacritics` needs SQLite 3.45+. The migration probes support on a temp
|
||||
table first and does nothing when the runtime rejects it, leaving the working
|
||||
index in place — and does NOT record itself as done, so a later app version
|
||||
shipping a newer SQLite upgrades then.
|
||||
shipping a newer SQLite upgrades then. (better-sqlite3 currently bundles 3.53,
|
||||
so the fallback is defensive rather than a path anyone is on.)
|
||||
|
||||
The completed marker is not taken as proof. `createTables` declares this table
|
||||
too, with the plain tokenizer, and `CREATE TABLE IF NOT EXISTS` would recreate
|
||||
it unfolded if it ever went missing after the marker was written — leaving two
|
||||
sources of truth for one tokenizer. The migration therefore reads the live
|
||||
table's own DDL out of `sqlite_master` and rebuilds unless it really folds. A
|
||||
degraded index is otherwise invisible: discovery simply stops finding "Pokémon"
|
||||
for "pokemon", with nothing to indicate it should have.
|
||||
|
||||
**Case folding for non-ASCII, on the FTS tier, is still not possible.**
|
||||
`LOWER()` and the trigram tokenizer both fold ASCII only, so a Cyrillic title
|
||||
|
||||
@@ -54,6 +54,16 @@ const completedMigrationStateRule: HandlerRule = [
|
||||
{ get: () => ({ value: 'done' }) },
|
||||
];
|
||||
|
||||
/** The live title index, already carrying the folding tokenizer. */
|
||||
const foldedIndexRule: HandlerRule = [
|
||||
'SELECT sql FROM sqlite_master',
|
||||
{
|
||||
get: () => ({
|
||||
sql: "CREATE VIRTUAL TABLE content_title_fts USING fts5(title, tokenize='trigram remove_diacritics 1')",
|
||||
}),
|
||||
},
|
||||
];
|
||||
|
||||
describe('createTables', () => {
|
||||
it('executes every fresh-install statement against the connection in order', () => {
|
||||
const { exec, sqlite } = createSqliteMock([]);
|
||||
@@ -86,7 +96,7 @@ describe('runMigrations error tolerance', () => {
|
||||
}
|
||||
});
|
||||
const { sqlite } = createSqliteMock(
|
||||
[completedMigrationStateRule],
|
||||
[completedMigrationStateRule, foldedIndexRule],
|
||||
exec
|
||||
);
|
||||
|
||||
@@ -111,7 +121,7 @@ describe('runMigrations error tolerance', () => {
|
||||
}
|
||||
});
|
||||
const { sqlite } = createSqliteMock(
|
||||
[completedMigrationStateRule],
|
||||
[completedMigrationStateRule, foldedIndexRule],
|
||||
exec
|
||||
);
|
||||
|
||||
@@ -299,6 +309,7 @@ describe('content title FTS tokenizer upgrade', () => {
|
||||
it('does nothing once the migration has completed', () => {
|
||||
const { sqlite, exec } = createSqliteMock([
|
||||
completedMigrationStateRule,
|
||||
foldedIndexRule,
|
||||
]);
|
||||
|
||||
upgradeContentTitleFtsTokenizer(sqlite);
|
||||
@@ -306,6 +317,34 @@ describe('content title FTS tokenizer upgrade', () => {
|
||||
expect(exec).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rebuilds when the record says done but the index is not folded', () => {
|
||||
const rebuild = { run: jest.fn() };
|
||||
const { sqlite, exec } = createSqliteMock([
|
||||
completedMigrationStateRule,
|
||||
[
|
||||
'SELECT sql FROM sqlite_master',
|
||||
{
|
||||
get: () => ({
|
||||
sql: "CREATE VIRTUAL TABLE content_title_fts USING fts5(title, tokenize='trigram')",
|
||||
}),
|
||||
},
|
||||
],
|
||||
['INSERT INTO content_title_fts(content_title_fts)', rebuild],
|
||||
]);
|
||||
|
||||
upgradeContentTitleFtsTokenizer(sqlite);
|
||||
|
||||
// `createTables` declares this table too, with the plain tokenizer, so
|
||||
// the marker and the live table can disagree. Trusting the marker
|
||||
// leaves a silently degraded index: discovery simply stops finding
|
||||
// "Pokémon" for "pokemon", with nothing to show that it should have.
|
||||
const statements = exec.mock.calls.map(([sql]) => compactSql(sql));
|
||||
expect(statements).toContainEqual(
|
||||
expect.stringContaining("tokenize='trigram remove_diacritics 1'")
|
||||
);
|
||||
expect(rebuild.run).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('leaves the index alone when the runtime rejects the tokenizer', () => {
|
||||
const marker = { run: jest.fn() };
|
||||
const exec = jest.fn((statement: string) => {
|
||||
|
||||
@@ -647,6 +647,26 @@ function contentTitleFtsStatement(removeDiacritics: boolean): string {
|
||||
}
|
||||
|
||||
/** Whether this SQLite accepts the folding tokenizer, asked without risk. */
|
||||
/**
|
||||
* Whether the title index that actually exists folds diacritics, read from its
|
||||
* own stored DDL rather than from the migration record.
|
||||
*
|
||||
* A missing table answers `false`, which is the useful answer: there is nothing
|
||||
* folded to keep, so the caller rebuilds.
|
||||
*/
|
||||
function contentTitleFtsFoldsDiacritics(sqliteDb: Database.Database): boolean {
|
||||
const row = sqliteDb
|
||||
.prepare(
|
||||
`SELECT sql FROM sqlite_master
|
||||
WHERE type = 'table' AND name = 'content_title_fts'`
|
||||
)
|
||||
.get() as { sql?: unknown } | undefined;
|
||||
|
||||
return (
|
||||
typeof row?.sql === 'string' && row.sql.includes('remove_diacritics')
|
||||
);
|
||||
}
|
||||
|
||||
function supportsTrigramDiacriticFolding(sqliteDb: Database.Database): boolean {
|
||||
try {
|
||||
sqliteDb.exec(
|
||||
@@ -676,7 +696,15 @@ function upgradeContentTitleFtsTokenizer(sqliteDb: Database.Database): boolean {
|
||||
| { value?: unknown }
|
||||
| undefined;
|
||||
|
||||
if (migrationState?.value === 'done') {
|
||||
// The marker alone is not evidence. `createTables` declares this table
|
||||
// too, with the plain tokenizer, so a table recreated by that path
|
||||
// after the marker was written would be silently unfolded — and a
|
||||
// degraded index is invisible: discovery just stops finding "Pokémon"
|
||||
// for "pokemon". Ask the live table instead of trusting the record.
|
||||
if (
|
||||
migrationState?.value === 'done' &&
|
||||
contentTitleFtsFoldsDiacritics(sqliteDb)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user