From 0f8f7be2fe9880b072889ffa5611c81d1b523459 Mon Sep 17 00:00:00 2001 From: 4gray Date: Fri, 24 Jul 2026 07:39:10 +0200 Subject: [PATCH] fix(search): keep non-compound tokens as SQL constraints on compound arms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up (Greptile P1 x2, Codex P2 x3 on #1172): for multi-token compound terms like "A&E HD", the compound substring arms let any title containing "a&e" into the bounded SQL candidate window (200 per-playlist / 5000 global) before scoring enforced the remaining tokens, so enough plain "A&E" rows could push real "US: A&E HD" matches out of the window. - title and M3U payload conditions now compose per word: each word's token conditions AND together, and only a compound word's own arm gains the intact-substring OR — every other word stays an independent constraint - the trigram FTS supplement AND-s the non-compound tokens as contains LIKE conditions next to MATCH '"a&e"' - new SearchWordPlan/getSearchWordPlans/getCompoundResidualTokenGroups helpers in content-search.util.ts Verified on real SQLite: searching "A&E HD" selects "US | A&E HD" but no longer selects plain "A&E"/"US: A&E" rows at the SQL level. Co-Authored-By: Claude Fable 5 --- .../operations/content-search.util.spec.ts | 28 +++ .../operations/content-search.util.ts | 51 +++++ .../operations/content.operations.spec.ts | 62 ++++++ .../database/operations/content.operations.ts | 190 +++++++++++------- docs/architecture/sqlite-db-worker.md | 8 +- 5 files changed, 266 insertions(+), 73 deletions(-) diff --git a/apps/electron-backend/src/app/database/operations/content-search.util.spec.ts b/apps/electron-backend/src/app/database/operations/content-search.util.spec.ts index 550d8e223..c668afcf6 100644 --- a/apps/electron-backend/src/app/database/operations/content-search.util.spec.ts +++ b/apps/electron-backend/src/app/database/operations/content-search.util.spec.ts @@ -2,7 +2,9 @@ import { buildCompoundFtsMatchQuery, buildCompoundLikePatterns, buildM3uPayloadCompoundPatterns, + getCompoundResidualTokenGroups, getCompoundSearchWords, + getSearchWordPlans, scoreSearchTextMatch, shouldUseContentTitlePrefixIndex, } from './content-search.util'; @@ -33,6 +35,32 @@ describe('content-search.util', () => { }); + describe('getSearchWordPlans', () => { + it('keeps each word with its own token groups and compound flag', () => { + expect(getSearchWordPlans('A&E HD')).toEqual([ + { compound: 'A&E', tokenGroups: [['a'], ['e']] }, + { compound: null, tokenGroups: [['hd']] }, + ]); + }); + + it('skips punctuation-only words', () => { + expect(getSearchWordPlans('Tom & Jerry')).toEqual([ + { compound: null, tokenGroups: [['tom']] }, + { compound: null, tokenGroups: [['jerry']] }, + ]); + }); + }); + + describe('getCompoundResidualTokenGroups', () => { + it('returns the token groups of the non-compound words only', () => { + expect(getCompoundResidualTokenGroups('A&E HD')).toEqual([['hd']]); + expect(getCompoundResidualTokenGroups('A&E')).toEqual([]); + expect(getCompoundResidualTokenGroups('History')).toEqual([ + ['history'], + ]); + }); + }); + describe('buildCompoundFtsMatchQuery', () => { it('quotes each compound word as a trigram substring phrase', () => { expect(buildCompoundFtsMatchQuery('A&E')).toBe('"a&e"'); diff --git a/apps/electron-backend/src/app/database/operations/content-search.util.ts b/apps/electron-backend/src/app/database/operations/content-search.util.ts index 8c318692b..dc0d9488b 100644 --- a/apps/electron-backend/src/app/database/operations/content-search.util.ts +++ b/apps/electron-backend/src/app/database/operations/content-search.util.ts @@ -167,6 +167,57 @@ export function getCompoundSearchWords(value: unknown): string[] { return [...words]; } +/** + * One whitespace-delimited search word with its token groups. `compound` is + * the intact word when it contains internal punctuation (see + * `getCompoundSearchWords`), else null. SQL condition builders compose + * per-word conditions from this so a compound word's substring arm stays + * AND-constrained by the other words of the query — `A&E HD` must not let + * plain `A&E` titles flood the SQL candidate window before scoring runs. + */ +export interface SearchWordPlan { + compound: string | null; + tokenGroups: string[][]; +} + +export function getSearchWordPlans(value: unknown): SearchWordPlan[] { + if (typeof value !== 'string') { + return []; + } + + const plans: SearchWordPlan[] = []; + for (const rawWord of value.split(/\s+/)) { + const word = rawWord.replace(WORD_EDGE_PUNCTUATION_REGEXP, ''); + if (!word) { + continue; + } + + const tokenGroups = getSqlSearchTokenGroups(word); + if (tokenGroups.length === 0) { + continue; + } + + const isCompound = + word.length >= MIN_COMPOUND_WORD_LENGTH && + INNER_PUNCTUATION_REGEXP.test(word); + plans.push({ compound: isCompound ? word : null, tokenGroups }); + } + + return plans; +} + +/** + * Token groups of the non-compound words of the term. When the compound FTS + * arm looks up `"a&e"` for `A&E HD`, these groups (`hd`) are re-applied as + * LIKE conditions so the supplement cannot fill the candidate limit with + * titles that match the compound word alone. + */ +export function getCompoundResidualTokenGroups(value: unknown): string[][] { + return getSearchWordPlans(value) + .filter((plan) => plan.compound === null) + .flatMap((plan) => plan.tokenGroups); +} + /** As-typed plus diacritic-stripped forms, both long enough for trigram FTS. */ function getCompoundWordVariants(word: string): string[] { const stripped = word.normalize('NFKD').replace(/[\u0300-\u036f]/g, ''); diff --git a/apps/electron-backend/src/app/database/operations/content.operations.spec.ts b/apps/electron-backend/src/app/database/operations/content.operations.spec.ts index ec52825d8..586dfd065 100644 --- a/apps/electron-backend/src/app/database/operations/content.operations.spec.ts +++ b/apps/electron-backend/src/app/database/operations/content.operations.spec.ts @@ -902,6 +902,68 @@ describe('content.operations', () => { expect(results.map((item) => item.title)).toEqual(['A&E', 'US: A&E']); }); + it('keeps non-compound tokens as conditions on the compound FTS supplement', async () => { + const all = jest.fn().mockResolvedValue([]); + const db = { + all, + select: jest.fn(), + } as unknown as AppDatabase; + + await globalSearch(db, 'A&E HD', ['live'], false, ['xtream'], { + limit: 10, + }); + + expect(all).toHaveBeenCalledTimes(2); + + const matchSqlCall = sqlMock.mock.calls.find(([strings]) => + Array.from(strings as TemplateStringsArray) + .join(' ') + .includes('content_title_fts MATCH') + ); + expect(matchSqlCall?.[1]).toBe('"a&e"'); + + // The FTS arm must AND-in the residual "hd" token so plain "A&E" + // titles cannot exhaust the candidate limit before scoring runs. + const residualPatterns = sqlMock.mock.calls + .filter(([strings]) => + Array.from(strings as TemplateStringsArray) + .join(' ') + .includes('c.title LIKE') + ) + .flatMap(([, ...values]) => values) + .filter((value): value is string => typeof value === 'string'); + expect(residualPatterns).toContain('%hd%'); + }); + + it('requires every word of a multi-token compound term in per-playlist SQL conditions', async () => { + const limit = jest.fn().mockResolvedValue([]); + const where = jest.fn().mockReturnValue({ limit }); + const innerJoin = jest.fn().mockReturnValue({ where }); + const from = jest.fn().mockReturnValue({ innerJoin }); + const select = jest.fn().mockReturnValue({ from }); + const db = { + select, + } as unknown as AppDatabase; + + await searchContent(db, 'playlist-1', 'A&E HD', ['live']); + + // One condition per word: the compound "A&E" word (an OR with the + // intact-substring arm) plus the plain "hd" word — both AND-ed, so + // the compound arm cannot bypass the "hd" requirement. + const conditions = where.mock.calls[0][0].conditions as Array<{ + kind: string; + }>; + expect(conditions).toHaveLength(4); + expect(conditions[2]).toEqual(expect.objectContaining({ kind: 'or' })); + expect(conditions[3]).toEqual(expect.objectContaining({ kind: 'and' })); + + const patterns = sqlMock.mock.calls + .flatMap(([, ...values]) => values) + .filter((value): value is string => typeof value === 'string'); + expect(patterns).toContain('%a&e%'); + expect(patterns).toContain('%hd%'); + }); + it('falls back to a content scan when the compound FTS lookup fails', async () => { const globRow = { id: 1, diff --git a/apps/electron-backend/src/app/database/operations/content.operations.ts b/apps/electron-backend/src/app/database/operations/content.operations.ts index 1efeff4fc..b2eeaf8c8 100644 --- a/apps/electron-backend/src/app/database/operations/content.operations.ts +++ b/apps/electron-backend/src/app/database/operations/content.operations.ts @@ -22,7 +22,8 @@ import { buildLikePatterns, buildM3uPayloadCompoundPatterns, buildM3uPayloadTextFieldPatterns, - getCompoundSearchWords, + getCompoundResidualTokenGroups, + getSearchWordPlans, getSqlSearchTokenGroups, isShortSearchTokenGroup, normalizeSearchMatchText, @@ -156,45 +157,55 @@ function getGlobalSearchCandidateLimit(): number { return MAX_GLOBAL_SEARCH_CANDIDATE_LIMIT; } +/** + * Per-word title conditions, AND-ed by the caller. A word's condition is the + * AND of its token-group LIKE conditions; a punctuation-joined word ("A&E") + * additionally matches as an intact contains pattern anywhere in the title + * (issue #1161). Composing per word keeps the other words' constraints on + * the compound arm, so "A&E HD" cannot fill the SQL candidate limit with + * titles that only contain "A&E". + */ function buildContentTitleSearchConditions(searchTerm: string) { - const tokenGroups = getSqlSearchTokenGroups(searchTerm); - if (tokenGroups.length === 0) { - return []; - } + let groupIndex = 0; - const groupConditions = tokenGroups - .map((tokens, index) => { - const mode = - index === 0 && isShortSearchTokenGroup(tokens) - ? 'prefix' - : 'contains'; - const likeConditions = tokens.flatMap((token) => - buildLikePatterns(token, mode).map( - (pattern) => - sql`${schema.content.title} LIKE ${pattern} ESCAPE '\\'` - ) - ); - return or(...likeConditions); - }) - .filter((condition) => condition !== undefined); + return getSearchWordPlans(searchTerm) + .map((plan) => { + const tokenConditions = plan.tokenGroups + .map((tokens) => { + const mode = + groupIndex === 0 && isShortSearchTokenGroup(tokens) + ? 'prefix' + : 'contains'; + groupIndex += 1; + const likeConditions = tokens.flatMap((token) => + buildLikePatterns(token, mode).map( + (pattern) => + sql`${schema.content.title} LIKE ${pattern} ESCAPE '\\'` + ) + ); + return or(...likeConditions); + }) + .filter((condition) => condition !== undefined); - // Punctuation-joined words ("A&E") tokenize into short fragments whose - // prefix anchoring would miss "US: A&E", so the intact word also matches - // as a contains pattern anywhere in the title (issue #1161). - const compoundConditions = getCompoundSearchWords(searchTerm).flatMap( - (word) => - buildCompoundLikePatterns(word).map( + const tokenArm = + tokenConditions.length > 0 + ? and(...tokenConditions) + : undefined; + if (plan.compound === null) { + return tokenArm; + } + + const compoundConditions = buildCompoundLikePatterns( + plan.compound + ).map( (pattern) => sql`${schema.content.title} LIKE ${pattern} ESCAPE '\\'` - ) - ); - - if (compoundConditions.length === 0 || groupConditions.length === 0) { - return groupConditions; - } - - const combined = or(and(...groupConditions), ...compoundConditions); - return combined === undefined ? groupConditions : [combined]; + ); + return tokenArm === undefined + ? or(...compoundConditions) + : or(tokenArm, ...compoundConditions); + }) + .filter((condition) => condition !== undefined); } function buildRawContentTitleSearchSql(searchTerm: string): SQL[] { @@ -221,6 +232,26 @@ function buildRawContentTitleSearchSql(searchTerm: string): SQL[] { }); } +/** + * Contains-mode LIKE conditions for the non-compound words of the term, + * applied on top of the compound FTS lookup so `A&E HD` only surfaces + * candidates that also contain `hd` (the FTS phrase can only express the + * compound word — trigram tokens need >= 3 chars). + */ +function buildCompoundResidualTitleSql(searchTerm: string): SQL[] { + return getCompoundResidualTokenGroups(searchTerm).map( + (tokens) => + sql`(${sql.join( + tokens.flatMap((token) => + buildLikePatterns(token).map( + (pattern) => sql`c.title LIKE ${pattern} ESCAPE '\\'` + ) + ), + sql` OR ` + )})` + ); +} + function dedupeXtreamCandidatesById( candidates: XtreamGlobalSearchCandidate[] ): XtreamGlobalSearchCandidate[] { @@ -282,7 +313,8 @@ async function selectXtreamGlobalSearchCandidatesWithFts( matchQuery: string, types: string[], excludeHidden: boolean, - candidateLimit: number + candidateLimit: number, + residualTitleConditions: SQL[] = [] ): Promise { if (!matchQuery || types.length === 0) { return []; @@ -313,6 +345,11 @@ async function selectXtreamGlobalSearchCandidatesWithFts( types.map((type) => sql`${type}`), sql`, ` )}) + ${ + residualTitleConditions.length > 0 + ? sql`AND ${sql.join(residualTitleConditions, sql` AND `)}` + : sql`` + } ${excludeHidden ? sql`AND cat.hidden = 0` : sql``} ORDER BY rank, c.title LIMIT ${candidateLimit} @@ -358,46 +395,54 @@ async function selectXtreamGlobalSearchCandidatesWithContentScan( .limit(candidateLimit); } +/** + * Per-word M3U payload prefilter conditions, AND-ed by the caller — the same + * compound-word composition as `buildContentTitleSearchConditions`: "A&E" + * must reach channel names like "US: A&E" (issue #1161), while the other + * words of the query keep constraining the candidate playlists. + */ function buildM3uPayloadSearchConditions(searchTerm: string) { - const tokenGroups = getSqlSearchTokenGroups(searchTerm); - if (tokenGroups.length === 0) { - return []; - } + let groupIndex = 0; - const groupConditions = tokenGroups - .map((tokens, index) => { - const mode = - index === 0 && isShortSearchTokenGroup(tokens) - ? 'prefix' - : 'contains'; - const patterns = tokens.flatMap((token) => - buildM3uPayloadTextFieldPatterns(token, mode) - ); - const likeConditions = patterns.map( + return getSearchWordPlans(searchTerm) + .map((plan) => { + const tokenConditions = plan.tokenGroups + .map((tokens) => { + const mode = + groupIndex === 0 && isShortSearchTokenGroup(tokens) + ? 'prefix' + : 'contains'; + groupIndex += 1; + const patterns = tokens.flatMap((token) => + buildM3uPayloadTextFieldPatterns(token, mode) + ); + const likeConditions = patterns.map( + (pattern) => + sql`${schema.playlists.payload} LIKE ${pattern} ESCAPE '\\'` + ); + return or(...likeConditions); + }) + .filter((condition) => condition !== undefined); + + const tokenArm = + tokenConditions.length > 0 + ? and(...tokenConditions) + : undefined; + if (plan.compound === null) { + return tokenArm; + } + + const compoundConditions = buildM3uPayloadCompoundPatterns( + plan.compound + ).map( (pattern) => sql`${schema.playlists.payload} LIKE ${pattern} ESCAPE '\\'` ); - return or(...likeConditions); + return tokenArm === undefined + ? or(...compoundConditions) + : or(tokenArm, ...compoundConditions); }) .filter((condition) => condition !== undefined); - - // Same compound-word rescue as the content title conditions: "A&E" must - // match channel names like "US: A&E" that the prefix-anchored short - // tokens would exclude at the playlist payload prefilter (issue #1161). - const compoundConditions = getCompoundSearchWords(searchTerm).flatMap( - (word) => - buildM3uPayloadCompoundPatterns(word).map( - (pattern) => - sql`${schema.playlists.payload} LIKE ${pattern} ESCAPE '\\'` - ) - ); - - if (compoundConditions.length === 0 || groupConditions.length === 0) { - return groupConditions; - } - - const combined = or(and(...groupConditions), ...compoundConditions); - return combined === undefined ? groupConditions : [combined]; } function asStringArray(value: unknown): string[] { @@ -1080,7 +1125,9 @@ export async function globalSearch( // The prefix arm only sees titles that start with the short first // token ("A&E" -> GLOB 'a*'), so a compound word like "a&e" is // additionally looked up as a trigram FTS substring to reach - // titles such as "US: A&E" (issue #1161). + // titles such as "US: A&E" (issue #1161). The non-compound words + // of the query stay applied as LIKE conditions so this arm cannot + // fill the candidate limit with compound-only matches. const compoundMatchQuery = buildCompoundFtsMatchQuery(searchTerm); if (compoundMatchQuery) { try { @@ -1090,7 +1137,8 @@ export async function globalSearch( compoundMatchQuery, types, excludeHidden, - candidateLimit + candidateLimit, + buildCompoundResidualTitleSql(searchTerm) ); candidates = dedupeXtreamCandidatesById([ ...candidates, diff --git a/docs/architecture/sqlite-db-worker.md b/docs/architecture/sqlite-db-worker.md index 3c519b4c2..502ca23d3 100644 --- a/docs/architecture/sqlite-db-worker.md +++ b/docs/architecture/sqlite-db-worker.md @@ -248,8 +248,12 @@ Xtream-only row shape: a supplemental trigram FTS `MATCH '"a&e"'` query for the Xtream arm (merged and deduped with the prefix-index candidates), intact-word `LIKE` contains patterns for the per-playlist and M3U payload prefilters, and a - space-bounded whole-phrase check in the ranking step. This lets `A&E` find - `US: A&E` anywhere in the title while single short tokens stay + space-bounded whole-phrase check in the ranking step. All compound arms + keep the remaining words of the query as SQL constraints — the FTS + supplement AND-s the non-compound tokens as `LIKE` conditions and the + `LIKE` prefilters compose per word — so `A&E HD` cannot fill the bounded + candidate window with titles that only contain `A&E`. This lets `A&E` + find `US: A&E` anywhere in the title while single short tokens stay prefix-anchored (issue #1161). 6. `excludeHidden` still filters hidden Xtream categories and also filters M3U channels whose `group.title` is listed in the playlist payload's