mirror of
https://github.com/4gray/iptvnator.git
synced 2026-10-12 03:56:15 -08:00
fix(search): keep non-compound tokens as SQL constraints on compound arms
Review follow-up (Greptile P1 x2, Codex P2 x3 on #1172): for multi-token compound terms like "A&E HD", the compound substring arms let any title containing "a&e" into the bounded SQL candidate window (200 per-playlist / 5000 global) before scoring enforced the remaining tokens, so enough plain "A&E" rows could push real "US: A&E HD" matches out of the window. - title and M3U payload conditions now compose per word: each word's token conditions AND together, and only a compound word's own arm gains the intact-substring OR — every other word stays an independent constraint - the trigram FTS supplement AND-s the non-compound tokens as contains LIKE conditions next to MATCH '"a&e"' - new SearchWordPlan/getSearchWordPlans/getCompoundResidualTokenGroups helpers in content-search.util.ts Verified on real SQLite: searching "A&E HD" selects "US | A&E HD" but no longer selects plain "A&E"/"US: A&E" rows at the SQL level. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
37a66a3327
commit
0f8f7be2fe
5 files changed
+266
-73
No files matched your search
@@ -2,7 +2,9 @@ import {
|
||||
buildCompoundFtsMatchQuery,
|
||||
buildCompoundLikePatterns,
|
||||
buildM3uPayloadCompoundPatterns,
|
||||
getCompoundResidualTokenGroups,
|
||||
getCompoundSearchWords,
|
||||
getSearchWordPlans,
|
||||
scoreSearchTextMatch,
|
||||
shouldUseContentTitlePrefixIndex,
|
||||
} from './content-search.util';
|
||||
@@ -33,6 +35,32 @@ describe('content-search.util', () => {
|
||||
|
||||
});
|
||||
|
||||
describe('getSearchWordPlans', () => {
|
||||
it('keeps each word with its own token groups and compound flag', () => {
|
||||
expect(getSearchWordPlans('A&E HD')).toEqual([
|
||||
{ compound: 'A&E', tokenGroups: [['a'], ['e']] },
|
||||
{ compound: null, tokenGroups: [['hd']] },
|
||||
]);
|
||||
});
|
||||
|
||||
it('skips punctuation-only words', () => {
|
||||
expect(getSearchWordPlans('Tom & Jerry')).toEqual([
|
||||
{ compound: null, tokenGroups: [['tom']] },
|
||||
{ compound: null, tokenGroups: [['jerry']] },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getCompoundResidualTokenGroups', () => {
|
||||
it('returns the token groups of the non-compound words only', () => {
|
||||
expect(getCompoundResidualTokenGroups('A&E HD')).toEqual([['hd']]);
|
||||
expect(getCompoundResidualTokenGroups('A&E')).toEqual([]);
|
||||
expect(getCompoundResidualTokenGroups('History')).toEqual([
|
||||
['history'],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildCompoundFtsMatchQuery', () => {
|
||||
it('quotes each compound word as a trigram substring phrase', () => {
|
||||
expect(buildCompoundFtsMatchQuery('A&E')).toBe('"a&e"');
|
||||
|
||||
@@ -167,6 +167,57 @@ export function getCompoundSearchWords(value: unknown): string[] {
|
||||
return [...words];
|
||||
}
|
||||
|
||||
/**
|
||||
* One whitespace-delimited search word with its token groups. `compound` is
|
||||
* the intact word when it contains internal punctuation (see
|
||||
* `getCompoundSearchWords`), else null. SQL condition builders compose
|
||||
* per-word conditions from this so a compound word's substring arm stays
|
||||
* AND-constrained by the other words of the query — `A&E HD` must not let
|
||||
* plain `A&E` titles flood the SQL candidate window before scoring runs.
|
||||
*/
|
||||
export interface SearchWordPlan {
|
||||
compound: string | null;
|
||||
tokenGroups: string[][];
|
||||
}
|
||||
|
||||
export function getSearchWordPlans(value: unknown): SearchWordPlan[] {
|
||||
if (typeof value !== 'string') {
|
||||
return [];
|
||||
}
|
||||
|
||||
const plans: SearchWordPlan[] = [];
|
||||
for (const rawWord of value.split(/\s+/)) {
|
||||
const word = rawWord.replace(WORD_EDGE_PUNCTUATION_REGEXP, '');
|
||||
if (!word) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const tokenGroups = getSqlSearchTokenGroups(word);
|
||||
if (tokenGroups.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const isCompound =
|
||||
word.length >= MIN_COMPOUND_WORD_LENGTH &&
|
||||
INNER_PUNCTUATION_REGEXP.test(word);
|
||||
plans.push({ compound: isCompound ? word : null, tokenGroups });
|
||||
}
|
||||
|
||||
return plans;
|
||||
}
|
||||
|
||||
/**
|
||||
* Token groups of the non-compound words of the term. When the compound FTS
|
||||
* arm looks up `"a&e"` for `A&E HD`, these groups (`hd`) are re-applied as
|
||||
* LIKE conditions so the supplement cannot fill the candidate limit with
|
||||
* titles that match the compound word alone.
|
||||
*/
|
||||
export function getCompoundResidualTokenGroups(value: unknown): string[][] {
|
||||
return getSearchWordPlans(value)
|
||||
.filter((plan) => plan.compound === null)
|
||||
.flatMap((plan) => plan.tokenGroups);
|
||||
}
|
||||
|
||||
/** As-typed plus diacritic-stripped forms, both long enough for trigram FTS. */
|
||||
function getCompoundWordVariants(word: string): string[] {
|
||||
const stripped = word.normalize('NFKD').replace(/[\u0300-\u036f]/g, '');
|
||||
|
||||
@@ -902,6 +902,68 @@ describe('content.operations', () => {
|
||||
expect(results.map((item) => item.title)).toEqual(['A&E', 'US: A&E']);
|
||||
});
|
||||
|
||||
it('keeps non-compound tokens as conditions on the compound FTS supplement', async () => {
|
||||
const all = jest.fn().mockResolvedValue([]);
|
||||
const db = {
|
||||
all,
|
||||
select: jest.fn(),
|
||||
} as unknown as AppDatabase;
|
||||
|
||||
await globalSearch(db, 'A&E HD', ['live'], false, ['xtream'], {
|
||||
limit: 10,
|
||||
});
|
||||
|
||||
expect(all).toHaveBeenCalledTimes(2);
|
||||
|
||||
const matchSqlCall = sqlMock.mock.calls.find(([strings]) =>
|
||||
Array.from(strings as TemplateStringsArray)
|
||||
.join(' ')
|
||||
.includes('content_title_fts MATCH')
|
||||
);
|
||||
expect(matchSqlCall?.[1]).toBe('"a&e"');
|
||||
|
||||
// The FTS arm must AND-in the residual "hd" token so plain "A&E"
|
||||
// titles cannot exhaust the candidate limit before scoring runs.
|
||||
const residualPatterns = sqlMock.mock.calls
|
||||
.filter(([strings]) =>
|
||||
Array.from(strings as TemplateStringsArray)
|
||||
.join(' ')
|
||||
.includes('c.title LIKE')
|
||||
)
|
||||
.flatMap(([, ...values]) => values)
|
||||
.filter((value): value is string => typeof value === 'string');
|
||||
expect(residualPatterns).toContain('%hd%');
|
||||
});
|
||||
|
||||
it('requires every word of a multi-token compound term in per-playlist SQL conditions', async () => {
|
||||
const limit = jest.fn().mockResolvedValue([]);
|
||||
const where = jest.fn().mockReturnValue({ limit });
|
||||
const innerJoin = jest.fn().mockReturnValue({ where });
|
||||
const from = jest.fn().mockReturnValue({ innerJoin });
|
||||
const select = jest.fn().mockReturnValue({ from });
|
||||
const db = {
|
||||
select,
|
||||
} as unknown as AppDatabase;
|
||||
|
||||
await searchContent(db, 'playlist-1', 'A&E HD', ['live']);
|
||||
|
||||
// One condition per word: the compound "A&E" word (an OR with the
|
||||
// intact-substring arm) plus the plain "hd" word — both AND-ed, so
|
||||
// the compound arm cannot bypass the "hd" requirement.
|
||||
const conditions = where.mock.calls[0][0].conditions as Array<{
|
||||
kind: string;
|
||||
}>;
|
||||
expect(conditions).toHaveLength(4);
|
||||
expect(conditions[2]).toEqual(expect.objectContaining({ kind: 'or' }));
|
||||
expect(conditions[3]).toEqual(expect.objectContaining({ kind: 'and' }));
|
||||
|
||||
const patterns = sqlMock.mock.calls
|
||||
.flatMap(([, ...values]) => values)
|
||||
.filter((value): value is string => typeof value === 'string');
|
||||
expect(patterns).toContain('%a&e%');
|
||||
expect(patterns).toContain('%hd%');
|
||||
});
|
||||
|
||||
it('falls back to a content scan when the compound FTS lookup fails', async () => {
|
||||
const globRow = {
|
||||
id: 1,
|
||||
|
||||
@@ -22,7 +22,8 @@ import {
|
||||
buildLikePatterns,
|
||||
buildM3uPayloadCompoundPatterns,
|
||||
buildM3uPayloadTextFieldPatterns,
|
||||
getCompoundSearchWords,
|
||||
getCompoundResidualTokenGroups,
|
||||
getSearchWordPlans,
|
||||
getSqlSearchTokenGroups,
|
||||
isShortSearchTokenGroup,
|
||||
normalizeSearchMatchText,
|
||||
@@ -156,45 +157,55 @@ function getGlobalSearchCandidateLimit(): number {
|
||||
return MAX_GLOBAL_SEARCH_CANDIDATE_LIMIT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-word title conditions, AND-ed by the caller. A word's condition is the
|
||||
* AND of its token-group LIKE conditions; a punctuation-joined word ("A&E")
|
||||
* additionally matches as an intact contains pattern anywhere in the title
|
||||
* (issue #1161). Composing per word keeps the other words' constraints on
|
||||
* the compound arm, so "A&E HD" cannot fill the SQL candidate limit with
|
||||
* titles that only contain "A&E".
|
||||
*/
|
||||
function buildContentTitleSearchConditions(searchTerm: string) {
|
||||
const tokenGroups = getSqlSearchTokenGroups(searchTerm);
|
||||
if (tokenGroups.length === 0) {
|
||||
return [];
|
||||
}
|
||||
let groupIndex = 0;
|
||||
|
||||
const groupConditions = tokenGroups
|
||||
.map((tokens, index) => {
|
||||
const mode =
|
||||
index === 0 && isShortSearchTokenGroup(tokens)
|
||||
? 'prefix'
|
||||
: 'contains';
|
||||
const likeConditions = tokens.flatMap((token) =>
|
||||
buildLikePatterns(token, mode).map(
|
||||
(pattern) =>
|
||||
sql`${schema.content.title} LIKE ${pattern} ESCAPE '\\'`
|
||||
)
|
||||
);
|
||||
return or(...likeConditions);
|
||||
})
|
||||
.filter((condition) => condition !== undefined);
|
||||
return getSearchWordPlans(searchTerm)
|
||||
.map((plan) => {
|
||||
const tokenConditions = plan.tokenGroups
|
||||
.map((tokens) => {
|
||||
const mode =
|
||||
groupIndex === 0 && isShortSearchTokenGroup(tokens)
|
||||
? 'prefix'
|
||||
: 'contains';
|
||||
groupIndex += 1;
|
||||
const likeConditions = tokens.flatMap((token) =>
|
||||
buildLikePatterns(token, mode).map(
|
||||
(pattern) =>
|
||||
sql`${schema.content.title} LIKE ${pattern} ESCAPE '\\'`
|
||||
)
|
||||
);
|
||||
return or(...likeConditions);
|
||||
})
|
||||
.filter((condition) => condition !== undefined);
|
||||
|
||||
// Punctuation-joined words ("A&E") tokenize into short fragments whose
|
||||
// prefix anchoring would miss "US: A&E", so the intact word also matches
|
||||
// as a contains pattern anywhere in the title (issue #1161).
|
||||
const compoundConditions = getCompoundSearchWords(searchTerm).flatMap(
|
||||
(word) =>
|
||||
buildCompoundLikePatterns(word).map(
|
||||
const tokenArm =
|
||||
tokenConditions.length > 0
|
||||
? and(...tokenConditions)
|
||||
: undefined;
|
||||
if (plan.compound === null) {
|
||||
return tokenArm;
|
||||
}
|
||||
|
||||
const compoundConditions = buildCompoundLikePatterns(
|
||||
plan.compound
|
||||
).map(
|
||||
(pattern) =>
|
||||
sql`${schema.content.title} LIKE ${pattern} ESCAPE '\\'`
|
||||
)
|
||||
);
|
||||
|
||||
if (compoundConditions.length === 0 || groupConditions.length === 0) {
|
||||
return groupConditions;
|
||||
}
|
||||
|
||||
const combined = or(and(...groupConditions), ...compoundConditions);
|
||||
return combined === undefined ? groupConditions : [combined];
|
||||
);
|
||||
return tokenArm === undefined
|
||||
? or(...compoundConditions)
|
||||
: or(tokenArm, ...compoundConditions);
|
||||
})
|
||||
.filter((condition) => condition !== undefined);
|
||||
}
|
||||
|
||||
function buildRawContentTitleSearchSql(searchTerm: string): SQL[] {
|
||||
@@ -221,6 +232,26 @@ function buildRawContentTitleSearchSql(searchTerm: string): SQL[] {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Contains-mode LIKE conditions for the non-compound words of the term,
|
||||
* applied on top of the compound FTS lookup so `A&E HD` only surfaces
|
||||
* candidates that also contain `hd` (the FTS phrase can only express the
|
||||
* compound word — trigram tokens need >= 3 chars).
|
||||
*/
|
||||
function buildCompoundResidualTitleSql(searchTerm: string): SQL[] {
|
||||
return getCompoundResidualTokenGroups(searchTerm).map(
|
||||
(tokens) =>
|
||||
sql`(${sql.join(
|
||||
tokens.flatMap((token) =>
|
||||
buildLikePatterns(token).map(
|
||||
(pattern) => sql`c.title LIKE ${pattern} ESCAPE '\\'`
|
||||
)
|
||||
),
|
||||
sql` OR `
|
||||
)})`
|
||||
);
|
||||
}
|
||||
|
||||
function dedupeXtreamCandidatesById(
|
||||
candidates: XtreamGlobalSearchCandidate[]
|
||||
): XtreamGlobalSearchCandidate[] {
|
||||
@@ -282,7 +313,8 @@ async function selectXtreamGlobalSearchCandidatesWithFts(
|
||||
matchQuery: string,
|
||||
types: string[],
|
||||
excludeHidden: boolean,
|
||||
candidateLimit: number
|
||||
candidateLimit: number,
|
||||
residualTitleConditions: SQL[] = []
|
||||
): Promise<XtreamGlobalSearchCandidate[]> {
|
||||
if (!matchQuery || types.length === 0) {
|
||||
return [];
|
||||
@@ -313,6 +345,11 @@ async function selectXtreamGlobalSearchCandidatesWithFts(
|
||||
types.map((type) => sql`${type}`),
|
||||
sql`, `
|
||||
)})
|
||||
${
|
||||
residualTitleConditions.length > 0
|
||||
? sql`AND ${sql.join(residualTitleConditions, sql` AND `)}`
|
||||
: sql``
|
||||
}
|
||||
${excludeHidden ? sql`AND cat.hidden = 0` : sql``}
|
||||
ORDER BY rank, c.title
|
||||
LIMIT ${candidateLimit}
|
||||
@@ -358,46 +395,54 @@ async function selectXtreamGlobalSearchCandidatesWithContentScan(
|
||||
.limit(candidateLimit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-word M3U payload prefilter conditions, AND-ed by the caller — the same
|
||||
* compound-word composition as `buildContentTitleSearchConditions`: "A&E"
|
||||
* must reach channel names like "US: A&E" (issue #1161), while the other
|
||||
* words of the query keep constraining the candidate playlists.
|
||||
*/
|
||||
function buildM3uPayloadSearchConditions(searchTerm: string) {
|
||||
const tokenGroups = getSqlSearchTokenGroups(searchTerm);
|
||||
if (tokenGroups.length === 0) {
|
||||
return [];
|
||||
}
|
||||
let groupIndex = 0;
|
||||
|
||||
const groupConditions = tokenGroups
|
||||
.map((tokens, index) => {
|
||||
const mode =
|
||||
index === 0 && isShortSearchTokenGroup(tokens)
|
||||
? 'prefix'
|
||||
: 'contains';
|
||||
const patterns = tokens.flatMap((token) =>
|
||||
buildM3uPayloadTextFieldPatterns(token, mode)
|
||||
);
|
||||
const likeConditions = patterns.map(
|
||||
return getSearchWordPlans(searchTerm)
|
||||
.map((plan) => {
|
||||
const tokenConditions = plan.tokenGroups
|
||||
.map((tokens) => {
|
||||
const mode =
|
||||
groupIndex === 0 && isShortSearchTokenGroup(tokens)
|
||||
? 'prefix'
|
||||
: 'contains';
|
||||
groupIndex += 1;
|
||||
const patterns = tokens.flatMap((token) =>
|
||||
buildM3uPayloadTextFieldPatterns(token, mode)
|
||||
);
|
||||
const likeConditions = patterns.map(
|
||||
(pattern) =>
|
||||
sql`${schema.playlists.payload} LIKE ${pattern} ESCAPE '\\'`
|
||||
);
|
||||
return or(...likeConditions);
|
||||
})
|
||||
.filter((condition) => condition !== undefined);
|
||||
|
||||
const tokenArm =
|
||||
tokenConditions.length > 0
|
||||
? and(...tokenConditions)
|
||||
: undefined;
|
||||
if (plan.compound === null) {
|
||||
return tokenArm;
|
||||
}
|
||||
|
||||
const compoundConditions = buildM3uPayloadCompoundPatterns(
|
||||
plan.compound
|
||||
).map(
|
||||
(pattern) =>
|
||||
sql`${schema.playlists.payload} LIKE ${pattern} ESCAPE '\\'`
|
||||
);
|
||||
return or(...likeConditions);
|
||||
return tokenArm === undefined
|
||||
? or(...compoundConditions)
|
||||
: or(tokenArm, ...compoundConditions);
|
||||
})
|
||||
.filter((condition) => condition !== undefined);
|
||||
|
||||
// Same compound-word rescue as the content title conditions: "A&E" must
|
||||
// match channel names like "US: A&E" that the prefix-anchored short
|
||||
// tokens would exclude at the playlist payload prefilter (issue #1161).
|
||||
const compoundConditions = getCompoundSearchWords(searchTerm).flatMap(
|
||||
(word) =>
|
||||
buildM3uPayloadCompoundPatterns(word).map(
|
||||
(pattern) =>
|
||||
sql`${schema.playlists.payload} LIKE ${pattern} ESCAPE '\\'`
|
||||
)
|
||||
);
|
||||
|
||||
if (compoundConditions.length === 0 || groupConditions.length === 0) {
|
||||
return groupConditions;
|
||||
}
|
||||
|
||||
const combined = or(and(...groupConditions), ...compoundConditions);
|
||||
return combined === undefined ? groupConditions : [combined];
|
||||
}
|
||||
|
||||
function asStringArray(value: unknown): string[] {
|
||||
@@ -1080,7 +1125,9 @@ export async function globalSearch(
|
||||
// The prefix arm only sees titles that start with the short first
|
||||
// token ("A&E" -> GLOB 'a*'), so a compound word like "a&e" is
|
||||
// additionally looked up as a trigram FTS substring to reach
|
||||
// titles such as "US: A&E" (issue #1161).
|
||||
// titles such as "US: A&E" (issue #1161). The non-compound words
|
||||
// of the query stay applied as LIKE conditions so this arm cannot
|
||||
// fill the candidate limit with compound-only matches.
|
||||
const compoundMatchQuery = buildCompoundFtsMatchQuery(searchTerm);
|
||||
if (compoundMatchQuery) {
|
||||
try {
|
||||
@@ -1090,7 +1137,8 @@ export async function globalSearch(
|
||||
compoundMatchQuery,
|
||||
types,
|
||||
excludeHidden,
|
||||
candidateLimit
|
||||
candidateLimit,
|
||||
buildCompoundResidualTitleSql(searchTerm)
|
||||
);
|
||||
candidates = dedupeXtreamCandidatesById([
|
||||
...candidates,
|
||||
|
||||
@@ -248,8 +248,12 @@ Xtream-only row shape:
|
||||
a supplemental trigram FTS `MATCH '"a&e"'` query for the Xtream arm (merged
|
||||
and deduped with the prefix-index candidates), intact-word `LIKE` contains
|
||||
patterns for the per-playlist and M3U payload prefilters, and a
|
||||
space-bounded whole-phrase check in the ranking step. This lets `A&E` find
|
||||
`US: A&E` anywhere in the title while single short tokens stay
|
||||
space-bounded whole-phrase check in the ranking step. All compound arms
|
||||
keep the remaining words of the query as SQL constraints — the FTS
|
||||
supplement AND-s the non-compound tokens as `LIKE` conditions and the
|
||||
`LIKE` prefilters compose per word — so `A&E HD` cannot fill the bounded
|
||||
candidate window with titles that only contain `A&E`. This lets `A&E`
|
||||
find `US: A&E` anywhere in the title while single short tokens stay
|
||||
prefix-anchored (issue #1161).
|
||||
6. `excludeHidden` still filters hidden Xtream categories and also filters M3U
|
||||
channels whose `group.title` is listed in the playlist payload's
|
||||
|
||||
Reference in new issue
Block a user