mirror of
https://github.com/4gray/iptvnator.git
synced 2026-10-11 02:46:16 -08:00
feat(matching): strip appended language/quality tags in title normalization
Real-world portal catalogs duplicate one show under dozens of tagged
variants ("|ALB| Fallout", "4K-DE - The Pitt (2025) (US)",
"Breaking Bad-eng", "Fallout_esp", "The Last of Us (2023) AF"). A third
of them normalized to polluted keys, silently skipping TMDB enrichment
and staying invisible to cross-portal title matching.
normalizeTitleKeys() now handles, conservatively:
- wrapped pipe tags: "|ALB| X", "|MULTI| X"
- longer/compound leads: "EXYU| X", "4K-DE - X", "AR-SUBS - X",
"4K-OSN+ - X" (dash/pipe only; colon stays
2-3 chars so "NCIS: LA" is untouched)
- underscore suffixes: "X_eng", "(US)_msub" (single-underscore only,
"The_Last_of_Us" stays intact)
- double-dash suffixes: "X--esp"
- joined dash tags: "X-DE", "X-eng" (vocabulary-gated and
case-uniform only; "Spider-Man", "Kick-It",
"Peut-être" are untouched)
- bare trailing tags: "X (2025) DE", "Breaking Bad ES" (UPPERCASE
vocabulary only, skipped for ALL-CAPS titles;
"Rocky II", "Made in USA", "Making It" are
untouched)
Every leading-tag segment must contain a letter, so numeric titles
("1917 - ...") are never treated as tags. The display-side
stripCountryPrefix() learns the same compound/plus-sign prefixes and the
numeric guard.
buildSearchLookupKey() gets a |v2 suffix so cached negative TMDB match
resolutions keyed on old polluted titles are invalidated.
Measured on 248 real catalog names from four shows (The Pitt, Fallout,
The Last of Us, Breaking Bad): clean matching keys 65% -> 99%, display
strip 91% -> 100%. The corpora are committed as spec fixtures.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
4grayandClaude Fable 5 committed
1 parent
b1fc23abbb
commit
beff8a7e33
6 files changed
+268
-16
No files matched your search
@@ -78,10 +78,10 @@ describe('extractYear', () => {
|
||||
describe('lookup keys', () => {
|
||||
it('builds stable search and details keys', () => {
|
||||
expect(buildSearchLookupKey('the matrix', 1999)).toBe(
|
||||
'title:the matrix|year:1999'
|
||||
'title:the matrix|year:1999|v2'
|
||||
);
|
||||
expect(buildSearchLookupKey('the matrix', null)).toBe(
|
||||
'title:the matrix|year:'
|
||||
'title:the matrix|year:|v2'
|
||||
);
|
||||
expect(buildDetailsLookupKey(603)).toBe('id:603|v2');
|
||||
});
|
||||
|
||||
@@ -70,7 +70,10 @@ export function buildSearchLookupKey(
|
||||
normalizedTitle: string,
|
||||
year: number | null
|
||||
): string {
|
||||
return `title:${normalizedTitle}|year:${year ?? ''}`;
|
||||
// v2: normalizeTitleKeys learned to strip appended language/quality
|
||||
// tags; the version suffix invalidates cached (incl. negative) match
|
||||
// resolutions keyed on the old polluted titles
|
||||
return `title:${normalizedTitle}|year:${year ?? ''}|v2`;
|
||||
}
|
||||
|
||||
export function buildDetailsLookupKey(tmdbId: number): string {
|
||||
|
||||
@@ -56,6 +56,127 @@ describe('normalizeTitleKeys', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('provider tag stripping', () => {
|
||||
it('strips wrapped pipe tags', () => {
|
||||
expect(normalizeTitle('|DE| ARD')).toBe('ard');
|
||||
expect(normalizeTitle('|MULTI| Fallout - 4K')).toBe('fallout');
|
||||
expect(normalizeTitle('|EXYU| The Pitt')).toBe('the pitt');
|
||||
});
|
||||
|
||||
it('strips long and compound leading tags', () => {
|
||||
expect(normalizeTitle('EXYU| Fallout')).toBe('fallout');
|
||||
expect(normalizeTitle('MULTI| Breaking Bad')).toBe('breaking bad');
|
||||
expect(normalizeTitle('4K-DE - The Pitt (2025) (US)')).toBe(
|
||||
'the pitt'
|
||||
);
|
||||
expect(normalizeTitle('AR-SUBS - Fallout (2024) (US)')).toBe(
|
||||
'fallout'
|
||||
);
|
||||
expect(normalizeTitle('4K-OSN+ - The Last of Us (2023)')).toBe(
|
||||
'the last of us'
|
||||
);
|
||||
});
|
||||
|
||||
it('never treats numeric fragments as leading tags', () => {
|
||||
expect(normalizeTitle('1917 - Behind the Lines')).toBe(
|
||||
'1917 behind the lines'
|
||||
);
|
||||
});
|
||||
|
||||
it('strips underscore and double-dash suffix tags', () => {
|
||||
expect(normalizeTitle('Fallout_eng')).toBe('fallout');
|
||||
expect(normalizeTitle('Breaking Bad (US)_msub')).toBe('breaking bad');
|
||||
expect(normalizeTitle('The Pitt (2025)_sub')).toBe('the pitt');
|
||||
expect(normalizeTitle('The Last of Us--esp')).toBe('the last of us');
|
||||
});
|
||||
|
||||
it('keeps underscore-as-space titles intact', () => {
|
||||
expect(normalizeTitle('The_Last_of_Us')).toBe('the last of us');
|
||||
});
|
||||
|
||||
it('strips joined dash tags only for case-uniform vocabulary tokens', () => {
|
||||
expect(normalizeTitle('Breaking Bad-eng')).toBe('breaking bad');
|
||||
expect(normalizeTitle('The Last of Us-DE')).toBe('the last of us');
|
||||
expect(normalizeTitle('The Pitt (2025)-it')).toBe('the pitt');
|
||||
expect(normalizeTitle('Spider-Man')).toBe('spider man');
|
||||
expect(normalizeTitle('Kick-It')).toBe('kick it');
|
||||
});
|
||||
|
||||
it('strips bare trailing UPPERCASE vocabulary tags', () => {
|
||||
expect(normalizeTitle('The Pitt (2025) DE')).toBe('the pitt');
|
||||
expect(normalizeTitle('Breaking Bad ES')).toBe('breaking bad');
|
||||
expect(normalizeTitle('EN| Breaking Bad SUB')).toBe('breaking bad');
|
||||
expect(normalizeTitle('The Last of Us (2023) AF')).toBe(
|
||||
'the last of us'
|
||||
);
|
||||
});
|
||||
|
||||
it('never strips trailing tags that could be real endings', () => {
|
||||
expect(normalizeTitle('Rocky II')).toBe('rocky ii');
|
||||
expect(normalizeTitle('Made in USA')).toBe('made in usa');
|
||||
expect(normalizeTitle('NCIS: LA')).toBe('ncis la');
|
||||
expect(normalizeTitle('Making It')).toBe('making it');
|
||||
expect(normalizeTitle('THE LAST OF US')).toBe('the last of us');
|
||||
});
|
||||
|
||||
const pittCorpus = [
|
||||
'The Pitt (2025)_sub', 'The Pitt (2025)-it', 'The Pitt (2025)',
|
||||
'The Pitt (Hindi)', 'The Pitt (2025) 4K', 'The Pitt (2025) DE',
|
||||
'The Pitt (2025) ES', 'The Pitt (2025) FR', 'The Pitt (2025)_eng',
|
||||
'The Pitt [MULTI-SUB]', 'The Pitt (2025) (4K DV)', 'GR - The Pitt',
|
||||
'4K-DE - The Pitt (2025) (US)', '4K-TR - The Pitt (2025) (US)',
|
||||
'AR-SUBS - The Pitt (2025) (US)', 'DE - The Pitt (2025) (US)',
|
||||
'ALB| The Pitt', 'EXYU| The Pitt', '|ALB| The Pitt', '|DE| The Pitt',
|
||||
];
|
||||
|
||||
const falloutCorpus = [
|
||||
'Fallout', 'DE - Fallout (2024)', 'Fallout (2024) - 4K',
|
||||
'Fallout (2024) FR-EN', 'Fallout (2024) Multi', 'Fallout (2024)_fr',
|
||||
'Fallout_esp', 'Fallout (4K)', '4K-AMZ - Fallout (2024) (US)',
|
||||
'AL - Fallout (2024)', 'AMZ - Fallout (2024) (US)',
|
||||
'AR-DE - Fallout (US)', 'LA - Fallout', 'EN| Fallout - 4K',
|
||||
'MULTI| Fallout - 4K', 'Fallout ( مدبلج )', 'Fallout (Telugu)',
|
||||
'|EN| Fallout - 4K', '|MULTI| Fallout', '|TR| Fallout',
|
||||
];
|
||||
|
||||
const lastOfUsCorpus = [
|
||||
'The Last of Us', 'The Last Of Us', 'The Last of Us (2023) 4K',
|
||||
'The Last of Us (2023) AF', 'The Last of Us_tr',
|
||||
'The Last of Us--esp', 'The Last of Us-DE', 'The Last of Us-esp',
|
||||
'The Last of Us [L]', 'The Last of Us ( HD )',
|
||||
'4K-OSN+ - The Last of Us (2023)', 'IS - The Last of Us (2023) (US)',
|
||||
'RU - The Last of Us', 'ALB| The Last of Us',
|
||||
];
|
||||
|
||||
const breakingBadCorpus = [
|
||||
'Breaking Bad', 'Breaking Bad (2008)_fr', 'Breaking Bad (US)_msub',
|
||||
'Breaking Bad_it', 'Breaking Bad-DE', 'Breaking Bad-eng',
|
||||
'Breaking Bad ( عائلي )', 'Breaking Bad (Pure)',
|
||||
'Breaking Bad - Multi', 'Breaking Bad ES', 'AR-DE - Breaking Bad',
|
||||
'EN| Breaking Bad SUB', 'MULTI| Breaking Bad', 'AR| Breaking Bad',
|
||||
];
|
||||
|
||||
it.each([
|
||||
['the pitt', pittCorpus],
|
||||
['fallout', falloutCorpus],
|
||||
['the last of us', lastOfUsCorpus],
|
||||
['breaking bad', breakingBadCorpus],
|
||||
])(
|
||||
'normalizes every observed provider variant of "%s" to one key',
|
||||
(expected, corpus) => {
|
||||
for (const name of corpus) {
|
||||
expect(normalizeTitleKeys(name).base).toBe(expected);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
it('keeps localized subtitles (indistinguishable from real ones)', () => {
|
||||
expect(normalizeTitle('Breaking Bad: A Química do Mal')).toBe(
|
||||
'breaking bad a quimica do mal'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('titleYearsCompatible', () => {
|
||||
it('accepts unknown years and ±1 tolerance', () => {
|
||||
expect(titleYearsCompatible(null, 2049)).toBe(true);
|
||||
|
||||
@@ -28,11 +28,102 @@ const QUALITY_TAGS = new Set([
|
||||
]);
|
||||
|
||||
/**
|
||||
* Leading channel/language prefix like "EN - ", "DE| ", "FR: ".
|
||||
* UPPERCASE-only on purpose: a case-insensitive match would amputate real
|
||||
* title words ("It: Chapter Two" → "Chapter Two").
|
||||
* Wrapped tag at the very start of a provider title: "|DE| ARD",
|
||||
* "|MULTI| Fallout". The lookahead requires a letter in the tag so a
|
||||
* numeric fragment can never be treated as one.
|
||||
*/
|
||||
const LANGUAGE_PREFIX = /^[A-Z]{2,3}\s*[-|:]\s+/;
|
||||
const WRAPPED_TAG_PREFIX = /^\s*\|(?=[0-9+]*[A-Z])[A-Z0-9+]{2,5}\|\s*/;
|
||||
|
||||
/**
|
||||
* Leading channel/language prefix like "EN - ", "DE| ", "FR: ", including
|
||||
* compound provider/quality forms ("4K-DE - ", "AR-SUBS - ", "4K-OSN+ - ")
|
||||
* and longer tags ("EXYU| ", "MULTI| ").
|
||||
* UPPERCASE-only on purpose: a case-insensitive match would amputate real
|
||||
* title words ("It: Chapter Two" → "Chapter Two"). Every segment must
|
||||
* contain a letter so numeric titles ("1917 - ...") are never tags.
|
||||
* Colon separators stay limited to 2–3 chars — longer acronyms before a
|
||||
* colon are franchise titles ("NCIS: LA"), not tags.
|
||||
*/
|
||||
const LANGUAGE_PREFIX =
|
||||
/^(?:(?=[0-9+]*[A-Z])[A-Z0-9+]{2,5}(?:-(?=[0-9+]*[A-Z])[A-Z0-9+]{2,6}){0,2}\s*[-|]\s+|(?=[0-9+]*[A-Z])[A-Z0-9+]{2,3}\s*:\s+)/;
|
||||
|
||||
/**
|
||||
* Curated whitelist for TRAILING language/subtitle tags ("Fallout_eng",
|
||||
* "Breaking Bad-DE", "The Pitt (2025) ES"). Trailing stripping must be
|
||||
* vocabulary-gated: a pattern-only rule would amputate real endings —
|
||||
* roman numerals ("Rocky II"), acronyms ("Made in USA"), franchise
|
||||
* suffixes ("NCIS: LA"). US/USA/UK/LA are deliberately absent.
|
||||
*/
|
||||
const TRAILING_TAG_VOCABULARY = new Set([
|
||||
'AF', 'AL', 'ALB', 'AR', 'BY', 'DE', 'DUB', 'EN', 'ENG', 'ES', 'ESP',
|
||||
'EXYU', 'FR', 'FRA', 'GE', 'GR', 'HU', 'IN', 'IR', 'IS', 'IT', 'ITA',
|
||||
'KA', 'KU', 'LAT', 'ML', 'MSUB', 'MULTI', 'NL', 'PL', 'PT', 'RO', 'RU',
|
||||
'SC', 'SE', 'SUB', 'SUBS', 'SW', 'TA', 'TL', 'TR', 'TUR',
|
||||
]);
|
||||
|
||||
const DOUBLE_DASH_SUFFIX = /[-–]{2}[A-Za-z]{2,5}\s*$/;
|
||||
const UNDERSCORE_SUFFIX = /_[A-Za-z]{2,5}\s*$/;
|
||||
const JOINED_DASH_SUFFIX = /-([A-Za-z]{2,5})\s*$/;
|
||||
const TRAILING_TAG_SUFFIX = /\s([A-Z]{2,5})\s*$/;
|
||||
|
||||
/** Tag tokens are case-uniform; real title words are Capitalized. */
|
||||
function isCaseUniform(token: string): boolean {
|
||||
return token === token.toLowerCase() || token === token.toUpperCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip appended language/subtitle tags. Runs BEFORE lowercasing —
|
||||
* casing is the main false-positive guard: ALL-CAPS titles carry no
|
||||
* casing signal ("THE LAST OF US" must keep its "US"), so caps-gated
|
||||
* rules are skipped for them, and Capitalized endings ("Making It",
|
||||
* "Kick-It") never look like tags. Compound tags ("FR-EN") shed one
|
||||
* token per pass, so stripping repeats to a fixpoint.
|
||||
*/
|
||||
function stripTrailingTags(value: string): string {
|
||||
let result = value;
|
||||
for (let pass = 0; pass < 3; pass++) {
|
||||
const next = stripTrailingTagOnce(result);
|
||||
if (next === result) break;
|
||||
result = next;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function stripTrailingTagOnce(value: string): string {
|
||||
const result = value.trimEnd();
|
||||
const hasLowercase = /\p{Ll}/u.test(result);
|
||||
|
||||
// "The Last of Us--esp": no real title contains a double dash.
|
||||
if (DOUBLE_DASH_SUFFIX.test(result)) {
|
||||
return result.replace(DOUBLE_DASH_SUFFIX, '').trimEnd();
|
||||
}
|
||||
|
||||
// "Fallout_eng" — but not "The_Last_of_Us", where underscores are
|
||||
// space substitutes (only strip when this is the sole underscore).
|
||||
if (
|
||||
UNDERSCORE_SUFFIX.test(result) &&
|
||||
result.indexOf('_') === result.lastIndexOf('_')
|
||||
) {
|
||||
return result.replace(UNDERSCORE_SUFFIX, '').trimEnd();
|
||||
}
|
||||
|
||||
const joined = result.match(JOINED_DASH_SUFFIX);
|
||||
if (
|
||||
joined &&
|
||||
TRAILING_TAG_VOCABULARY.has(joined[1].toUpperCase()) &&
|
||||
isCaseUniform(joined[1]) &&
|
||||
(joined[1] !== joined[1].toUpperCase() || hasLowercase)
|
||||
) {
|
||||
return result.replace(JOINED_DASH_SUFFIX, '').trimEnd();
|
||||
}
|
||||
|
||||
const trailing = result.match(TRAILING_TAG_SUFFIX);
|
||||
if (trailing && hasLowercase && TRAILING_TAG_VOCABULARY.has(trailing[1])) {
|
||||
return result.replace(TRAILING_TAG_SUFFIX, '').trimEnd();
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
const YEAR_PATTERN = /\b(19\d{2}|20\d{2})\b/;
|
||||
|
||||
@@ -74,11 +165,14 @@ export function normalizeTitleKeys(
|
||||
return { exact: '', base: '', trailingYear: null };
|
||||
}
|
||||
|
||||
const cleaned = raw
|
||||
// Inner classes exclude the opening delimiter too, so runaway
|
||||
// inputs like "[[[[[..." backtrack linearly (CodeQL js/polynomial-redos)
|
||||
.replace(/\[[^\][]*\]|\([^()]*\)|\{[^{}]*\}/g, ' ')
|
||||
.replace(LANGUAGE_PREFIX, '')
|
||||
const cleaned = stripTrailingTags(
|
||||
raw
|
||||
.replace(WRAPPED_TAG_PREFIX, '')
|
||||
// Inner classes exclude the opening delimiter too, so runaway
|
||||
// inputs like "[[[[[..." backtrack linearly (CodeQL js/polynomial-redos)
|
||||
.replace(/\[[^\][]*\]|\([^()]*\)|\{[^{}]*\}/g, ' ')
|
||||
.replace(LANGUAGE_PREFIX, '')
|
||||
)
|
||||
.normalize('NFD')
|
||||
.replace(/[\u0300-\u036F]/g, '')
|
||||
.toLowerCase()
|
||||
|
||||
@@ -43,6 +43,27 @@ describe('stripCountryPrefix', () => {
|
||||
expect(stripCountryPrefix('US: CNN')).toBe('CNN');
|
||||
});
|
||||
|
||||
it('strips compound quality/provider tags', () => {
|
||||
expect(stripCountryPrefix('4K-DE - The Pitt (2025)')).toBe(
|
||||
'The Pitt (2025)'
|
||||
);
|
||||
expect(stripCountryPrefix('AR-SUBS - Fallout')).toBe('Fallout');
|
||||
expect(stripCountryPrefix('4K-OSN+ - The Last of Us')).toBe(
|
||||
'The Last of Us'
|
||||
);
|
||||
});
|
||||
|
||||
it('strips longer uppercase tags', () => {
|
||||
expect(stripCountryPrefix('EXYU - News')).toBe('News');
|
||||
expect(stripCountryPrefix('MULTI - Movies')).toBe('Movies');
|
||||
});
|
||||
|
||||
it('never treats numeric fragments as tags', () => {
|
||||
expect(stripCountryPrefix('1917 - Documentary')).toBe(
|
||||
'1917 - Documentary'
|
||||
);
|
||||
});
|
||||
|
||||
it('only strips the first tag segment', () => {
|
||||
expect(stripCountryPrefix('ES - A3 - Sports')).toBe('A3 - Sports');
|
||||
});
|
||||
|
||||
@@ -3,9 +3,18 @@
|
||||
* country/group tag — they routinely appear inside real channel names
|
||||
* ("Sky - Sports F1", "Discovery - Science"). Pipes are conventionally
|
||||
* used only as tag separators, so they always count.
|
||||
*
|
||||
* A tag is 1–3 short UPPERCASE alphanumeric segments ("US", "4K-DE",
|
||||
* "AR-SUBS", "OSN+"). Every segment must contain a letter so numeric
|
||||
* titles ("1917 - ...") are never treated as tags. Colon separators stay
|
||||
* limited to plain 2–3 char tags — longer acronyms before a colon are
|
||||
* franchise titles ("NCIS: LA"), not tags.
|
||||
*/
|
||||
const DASH_OR_COLON_SEPARATORS = [' - ', '- ', ' -', ': '];
|
||||
const TAG_PREFIX_PATTERN = /^[A-Z0-9]{2,3}$/;
|
||||
const DASH_SEPARATORS = [' - ', '- ', ' -'];
|
||||
const COLON_SEPARATOR = ': ';
|
||||
const TAG_PREFIX_PATTERN =
|
||||
/^(?=[0-9+]*[A-Z])[A-Z0-9+]{2,5}(?:-(?=[0-9+]*[A-Z])[A-Z0-9+]{2,6}){0,2}$/;
|
||||
const COLON_TAG_PATTERN = /^(?=[0-9+]*[A-Z])[A-Z0-9+]{2,3}$/;
|
||||
|
||||
interface SeparatorMatch {
|
||||
index: number;
|
||||
@@ -68,10 +77,14 @@ function findTagSeparator(name: string): SeparatorMatch | null {
|
||||
best = { index: pipeIndex, length: 1 };
|
||||
}
|
||||
|
||||
for (const separator of DASH_OR_COLON_SEPARATORS) {
|
||||
for (const separator of [...DASH_SEPARATORS, COLON_SEPARATOR]) {
|
||||
const index = name.indexOf(separator);
|
||||
if (index === -1 || (best && best.index <= index)) continue;
|
||||
if (!TAG_PREFIX_PATTERN.test(name.slice(0, index).trim())) continue;
|
||||
const pattern =
|
||||
separator === COLON_SEPARATOR
|
||||
? COLON_TAG_PATTERN
|
||||
: TAG_PREFIX_PATTERN;
|
||||
if (!pattern.test(name.slice(0, index).trim())) continue;
|
||||
best = { index, length: separator.length };
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user