fix(search): keep non-compound tokens as SQL constraints on compound arms

Review follow-up (Greptile P1 x2, Codex P2 x3 on #1172): for multi-token
compound terms like "A&E HD", the compound substring arms let any title
containing "a&e" into the bounded SQL candidate window (200 per-playlist /
5000 global) before scoring enforced the remaining tokens, so enough plain
"A&E" rows could push real "US: A&E HD" matches out of the window.

- title and M3U payload conditions now compose per word: each word's token
  conditions AND together, and only a compound word's own arm gains the
  intact-substring OR — every other word stays an independent constraint
- the trigram FTS supplement AND-s the non-compound tokens as contains
  LIKE conditions next to MATCH '"a&e"'
- new SearchWordPlan/getSearchWordPlans/getCompoundResidualTokenGroups
  helpers in content-search.util.ts

Verified on real SQLite: searching "A&E HD" selects "US | A&E HD" but no
longer selects plain "A&E"/"US: A&E" rows at the SQL level.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
4grayandClaude Fable 5 committed 2026-07-24 07:39:10 +02:00
1 parent 37a66a3327
commit 0f8f7be2fe
5 files changed
+266 -73

No files matched your search

@@ -2,7 +2,9 @@ import {
buildCompoundFtsMatchQuery,
buildCompoundLikePatterns,
buildM3uPayloadCompoundPatterns,
getCompoundResidualTokenGroups,
getCompoundSearchWords,
getSearchWordPlans,
scoreSearchTextMatch,
shouldUseContentTitlePrefixIndex,
} from './content-search.util';
@@ -33,6 +35,32 @@ describe('content-search.util', () => {
});
describe('getSearchWordPlans', () => {
it('keeps each word with its own token groups and compound flag', () => {
expect(getSearchWordPlans('A&E HD')).toEqual([
{ compound: 'A&E', tokenGroups: [['a'], ['e']] },
{ compound: null, tokenGroups: [['hd']] },
]);
});
it('skips punctuation-only words', () => {
expect(getSearchWordPlans('Tom & Jerry')).toEqual([
{ compound: null, tokenGroups: [['tom']] },
{ compound: null, tokenGroups: [['jerry']] },
]);
});
});
describe('getCompoundResidualTokenGroups', () => {
it('returns the token groups of the non-compound words only', () => {
expect(getCompoundResidualTokenGroups('A&E HD')).toEqual([['hd']]);
expect(getCompoundResidualTokenGroups('A&E')).toEqual([]);
expect(getCompoundResidualTokenGroups('History')).toEqual([
['history'],
]);
});
});
describe('buildCompoundFtsMatchQuery', () => {
it('quotes each compound word as a trigram substring phrase', () => {
expect(buildCompoundFtsMatchQuery('A&E')).toBe('"a&e"');
@@ -167,6 +167,57 @@ export function getCompoundSearchWords(value: unknown): string[] {
return [...words];
}
/**
* One whitespace-delimited search word with its token groups. `compound` is
* the intact word when it contains internal punctuation (see
* `getCompoundSearchWords`), else null. SQL condition builders compose
* per-word conditions from this so a compound word's substring arm stays
* AND-constrained by the other words of the query — `A&E HD` must not let
* plain `A&E` titles flood the SQL candidate window before scoring runs.
*/
export interface SearchWordPlan {
compound: string | null;
tokenGroups: string[][];
}
export function getSearchWordPlans(value: unknown): SearchWordPlan[] {
if (typeof value !== 'string') {
return [];
}
const plans: SearchWordPlan[] = [];
for (const rawWord of value.split(/\s+/)) {
const word = rawWord.replace(WORD_EDGE_PUNCTUATION_REGEXP, '');
if (!word) {
continue;
}
const tokenGroups = getSqlSearchTokenGroups(word);
if (tokenGroups.length === 0) {
continue;
}
const isCompound =
word.length >= MIN_COMPOUND_WORD_LENGTH &&
INNER_PUNCTUATION_REGEXP.test(word);
plans.push({ compound: isCompound ? word : null, tokenGroups });
}
return plans;
}
/**
* Token groups of the non-compound words of the term. When the compound FTS
* arm looks up `"a&e"` for `A&E HD`, these groups (`hd`) are re-applied as
* LIKE conditions so the supplement cannot fill the candidate limit with
* titles that match the compound word alone.
*/
export function getCompoundResidualTokenGroups(value: unknown): string[][] {
return getSearchWordPlans(value)
.filter((plan) => plan.compound === null)
.flatMap((plan) => plan.tokenGroups);
}
/** As-typed plus diacritic-stripped forms, both long enough for trigram FTS. */
function getCompoundWordVariants(word: string): string[] {
const stripped = word.normalize('NFKD').replace(/[\u0300-\u036f]/g, '');
@@ -902,6 +902,68 @@ describe('content.operations', () => {
expect(results.map((item) => item.title)).toEqual(['A&E', 'US: A&E']);
});
it('keeps non-compound tokens as conditions on the compound FTS supplement', async () => {
const all = jest.fn().mockResolvedValue([]);
const db = {
all,
select: jest.fn(),
} as unknown as AppDatabase;
await globalSearch(db, 'A&E HD', ['live'], false, ['xtream'], {
limit: 10,
});
expect(all).toHaveBeenCalledTimes(2);
const matchSqlCall = sqlMock.mock.calls.find(([strings]) =>
Array.from(strings as TemplateStringsArray)
.join(' ')
.includes('content_title_fts MATCH')
);
expect(matchSqlCall?.[1]).toBe('"a&e"');
// The FTS arm must AND-in the residual "hd" token so plain "A&E"
// titles cannot exhaust the candidate limit before scoring runs.
const residualPatterns = sqlMock.mock.calls
.filter(([strings]) =>
Array.from(strings as TemplateStringsArray)
.join(' ')
.includes('c.title LIKE')
)
.flatMap(([, ...values]) => values)
.filter((value): value is string => typeof value === 'string');
expect(residualPatterns).toContain('%hd%');
});
it('requires every word of a multi-token compound term in per-playlist SQL conditions', async () => {
const limit = jest.fn().mockResolvedValue([]);
const where = jest.fn().mockReturnValue({ limit });
const innerJoin = jest.fn().mockReturnValue({ where });
const from = jest.fn().mockReturnValue({ innerJoin });
const select = jest.fn().mockReturnValue({ from });
const db = {
select,
} as unknown as AppDatabase;
await searchContent(db, 'playlist-1', 'A&E HD', ['live']);
// One condition per word: the compound "A&E" word (an OR with the
// intact-substring arm) plus the plain "hd" word — both AND-ed, so
// the compound arm cannot bypass the "hd" requirement.
const conditions = where.mock.calls[0][0].conditions as Array<{
kind: string;
}>;
expect(conditions).toHaveLength(4);
expect(conditions[2]).toEqual(expect.objectContaining({ kind: 'or' }));
expect(conditions[3]).toEqual(expect.objectContaining({ kind: 'and' }));
const patterns = sqlMock.mock.calls
.flatMap(([, ...values]) => values)
.filter((value): value is string => typeof value === 'string');
expect(patterns).toContain('%a&e%');
expect(patterns).toContain('%hd%');
});
it('falls back to a content scan when the compound FTS lookup fails', async () => {
const globRow = {
id: 1,
@@ -22,7 +22,8 @@ import {
buildLikePatterns,
buildM3uPayloadCompoundPatterns,
buildM3uPayloadTextFieldPatterns,
getCompoundSearchWords,
getCompoundResidualTokenGroups,
getSearchWordPlans,
getSqlSearchTokenGroups,
isShortSearchTokenGroup,
normalizeSearchMatchText,
@@ -156,45 +157,55 @@ function getGlobalSearchCandidateLimit(): number {
return MAX_GLOBAL_SEARCH_CANDIDATE_LIMIT;
}
/**
* Per-word title conditions, AND-ed by the caller. A word's condition is the
* AND of its token-group LIKE conditions; a punctuation-joined word ("A&E")
* additionally matches as an intact contains pattern anywhere in the title
* (issue #1161). Composing per word keeps the other words' constraints on
* the compound arm, so "A&E HD" cannot fill the SQL candidate limit with
* titles that only contain "A&E".
*/
function buildContentTitleSearchConditions(searchTerm: string) {
const tokenGroups = getSqlSearchTokenGroups(searchTerm);
if (tokenGroups.length === 0) {
return [];
}
let groupIndex = 0;
const groupConditions = tokenGroups
.map((tokens, index) => {
const mode =
index === 0 && isShortSearchTokenGroup(tokens)
? 'prefix'
: 'contains';
const likeConditions = tokens.flatMap((token) =>
buildLikePatterns(token, mode).map(
(pattern) =>
sql`${schema.content.title} LIKE ${pattern} ESCAPE '\\'`
)
);
return or(...likeConditions);
})
.filter((condition) => condition !== undefined);
return getSearchWordPlans(searchTerm)
.map((plan) => {
const tokenConditions = plan.tokenGroups
.map((tokens) => {
const mode =
groupIndex === 0 && isShortSearchTokenGroup(tokens)
? 'prefix'
: 'contains';
groupIndex += 1;
const likeConditions = tokens.flatMap((token) =>
buildLikePatterns(token, mode).map(
(pattern) =>
sql`${schema.content.title} LIKE ${pattern} ESCAPE '\\'`
)
);
return or(...likeConditions);
})
.filter((condition) => condition !== undefined);
// Punctuation-joined words ("A&E") tokenize into short fragments whose
// prefix anchoring would miss "US: A&E", so the intact word also matches
// as a contains pattern anywhere in the title (issue #1161).
const compoundConditions = getCompoundSearchWords(searchTerm).flatMap(
(word) =>
buildCompoundLikePatterns(word).map(
const tokenArm =
tokenConditions.length > 0
? and(...tokenConditions)
: undefined;
if (plan.compound === null) {
return tokenArm;
}
const compoundConditions = buildCompoundLikePatterns(
plan.compound
).map(
(pattern) =>
sql`${schema.content.title} LIKE ${pattern} ESCAPE '\\'`
)
);
if (compoundConditions.length === 0 || groupConditions.length === 0) {
return groupConditions;
}
const combined = or(and(...groupConditions), ...compoundConditions);
return combined === undefined ? groupConditions : [combined];
);
return tokenArm === undefined
? or(...compoundConditions)
: or(tokenArm, ...compoundConditions);
})
.filter((condition) => condition !== undefined);
}
function buildRawContentTitleSearchSql(searchTerm: string): SQL[] {
@@ -221,6 +232,26 @@ function buildRawContentTitleSearchSql(searchTerm: string): SQL[] {
});
}
/**
* Contains-mode LIKE conditions for the non-compound words of the term,
* applied on top of the compound FTS lookup so `A&E HD` only surfaces
* candidates that also contain `hd` (the FTS phrase can only express the
* compound word — trigram tokens need >= 3 chars).
*/
function buildCompoundResidualTitleSql(searchTerm: string): SQL[] {
return getCompoundResidualTokenGroups(searchTerm).map(
(tokens) =>
sql`(${sql.join(
tokens.flatMap((token) =>
buildLikePatterns(token).map(
(pattern) => sql`c.title LIKE ${pattern} ESCAPE '\\'`
)
),
sql` OR `
)})`
);
}
function dedupeXtreamCandidatesById(
candidates: XtreamGlobalSearchCandidate[]
): XtreamGlobalSearchCandidate[] {
@@ -282,7 +313,8 @@ async function selectXtreamGlobalSearchCandidatesWithFts(
matchQuery: string,
types: string[],
excludeHidden: boolean,
candidateLimit: number
candidateLimit: number,
residualTitleConditions: SQL[] = []
): Promise<XtreamGlobalSearchCandidate[]> {
if (!matchQuery || types.length === 0) {
return [];
@@ -313,6 +345,11 @@ async function selectXtreamGlobalSearchCandidatesWithFts(
types.map((type) => sql`${type}`),
sql`, `
)})
${
residualTitleConditions.length > 0
? sql`AND ${sql.join(residualTitleConditions, sql` AND `)}`
: sql``
}
${excludeHidden ? sql`AND cat.hidden = 0` : sql``}
ORDER BY rank, c.title
LIMIT ${candidateLimit}
@@ -358,46 +395,54 @@ async function selectXtreamGlobalSearchCandidatesWithContentScan(
.limit(candidateLimit);
}
/**
* Per-word M3U payload prefilter conditions, AND-ed by the caller — the same
* compound-word composition as `buildContentTitleSearchConditions`: "A&E"
* must reach channel names like "US: A&E" (issue #1161), while the other
* words of the query keep constraining the candidate playlists.
*/
function buildM3uPayloadSearchConditions(searchTerm: string) {
const tokenGroups = getSqlSearchTokenGroups(searchTerm);
if (tokenGroups.length === 0) {
return [];
}
let groupIndex = 0;
const groupConditions = tokenGroups
.map((tokens, index) => {
const mode =
index === 0 && isShortSearchTokenGroup(tokens)
? 'prefix'
: 'contains';
const patterns = tokens.flatMap((token) =>
buildM3uPayloadTextFieldPatterns(token, mode)
);
const likeConditions = patterns.map(
return getSearchWordPlans(searchTerm)
.map((plan) => {
const tokenConditions = plan.tokenGroups
.map((tokens) => {
const mode =
groupIndex === 0 && isShortSearchTokenGroup(tokens)
? 'prefix'
: 'contains';
groupIndex += 1;
const patterns = tokens.flatMap((token) =>
buildM3uPayloadTextFieldPatterns(token, mode)
);
const likeConditions = patterns.map(
(pattern) =>
sql`${schema.playlists.payload} LIKE ${pattern} ESCAPE '\\'`
);
return or(...likeConditions);
})
.filter((condition) => condition !== undefined);
const tokenArm =
tokenConditions.length > 0
? and(...tokenConditions)
: undefined;
if (plan.compound === null) {
return tokenArm;
}
const compoundConditions = buildM3uPayloadCompoundPatterns(
plan.compound
).map(
(pattern) =>
sql`${schema.playlists.payload} LIKE ${pattern} ESCAPE '\\'`
);
return or(...likeConditions);
return tokenArm === undefined
? or(...compoundConditions)
: or(tokenArm, ...compoundConditions);
})
.filter((condition) => condition !== undefined);
// Same compound-word rescue as the content title conditions: "A&E" must
// match channel names like "US: A&E" that the prefix-anchored short
// tokens would exclude at the playlist payload prefilter (issue #1161).
const compoundConditions = getCompoundSearchWords(searchTerm).flatMap(
(word) =>
buildM3uPayloadCompoundPatterns(word).map(
(pattern) =>
sql`${schema.playlists.payload} LIKE ${pattern} ESCAPE '\\'`
)
);
if (compoundConditions.length === 0 || groupConditions.length === 0) {
return groupConditions;
}
const combined = or(and(...groupConditions), ...compoundConditions);
return combined === undefined ? groupConditions : [combined];
}
function asStringArray(value: unknown): string[] {
@@ -1080,7 +1125,9 @@ export async function globalSearch(
// The prefix arm only sees titles that start with the short first
// token ("A&E" -> GLOB 'a*'), so a compound word like "a&e" is
// additionally looked up as a trigram FTS substring to reach
// titles such as "US: A&E" (issue #1161).
// titles such as "US: A&E" (issue #1161). The non-compound words
// of the query stay applied as LIKE conditions so this arm cannot
// fill the candidate limit with compound-only matches.
const compoundMatchQuery = buildCompoundFtsMatchQuery(searchTerm);
if (compoundMatchQuery) {
try {
@@ -1090,7 +1137,8 @@ export async function globalSearch(
compoundMatchQuery,
types,
excludeHidden,
candidateLimit
candidateLimit,
buildCompoundResidualTitleSql(searchTerm)
);
candidates = dedupeXtreamCandidatesById([
...candidates,
+6 -2
View File
@@ -248,8 +248,12 @@ Xtream-only row shape:
a supplemental trigram FTS `MATCH '"a&e"'` query for the Xtream arm (merged
and deduped with the prefix-index candidates), intact-word `LIKE` contains
patterns for the per-playlist and M3U payload prefilters, and a
space-bounded whole-phrase check in the ranking step. This lets `A&E` find
`US: A&E` anywhere in the title while single short tokens stay
space-bounded whole-phrase check in the ranking step. All compound arms
keep the remaining words of the query as SQL constraints — the FTS
supplement AND-s the non-compound tokens as `LIKE` conditions and the
`LIKE` prefilters compose per word — so `A&E HD` cannot fill the bounded
candidate window with titles that only contain `A&E`. This lets `A&E`
find `US: A&E` anywhere in the title while single short tokens stay
prefix-anchored (issue #1161).
6. `excludeHidden` still filters hidden Xtream categories and also filters M3U
channels whose `group.title` is listed in the playlist payload's