feat(matching): strip appended language/quality tags in title normalization (#1211)

* feat(matching): strip appended language/quality tags in title normalization Real-world portal catalogs duplicate one show under dozens of tagged variants ("|ALB| Fallout", "4K-DE - The Pitt (2025) (US)", "Breaking Bad-eng", "Fallout_esp", "The Last of Us (2023) AF"). A third of them normalized to polluted keys, silently skipping TMDB enrichment and staying invisible to cross-portal title matching. normalizeTitleKeys() now handles, conservatively: - wrapped pipe tags: "|ALB| X", "|MULTI| X" - longer/compound leads: "EXYU| X", "4K-DE - X", "AR-SUBS - X", "4K-OSN+ - X" (dash/pipe only; colon stays 2-3 chars so "NCIS: LA" is untouched) - underscore suffixes: "X_eng", "(US)_msub" (single-underscore only, "The_Last_of_Us" stays intact) - double-dash suffixes: "X--esp" - joined dash tags: "X-DE", "X-eng" (vocabulary-gated and case-uniform only; "Spider-Man", "Kick-It", "Peut-être" are untouched) - bare trailing tags: "X (2025) DE", "Breaking Bad ES" (UPPERCASE vocabulary only, skipped for ALL-CAPS titles; "Rocky II", "Made in USA", "Making It" are untouched) Every leading-tag segment must contain a letter, so numeric titles ("1917 - ...") are never treated as tags. The display-side stripCountryPrefix() learns the same compound/plus-sign prefixes and the numeric guard. buildSearchLookupKey() gets a |v2 suffix so cached negative TMDB match resolutions keyed on old polluted titles are invalidated. Measured on 248 real catalog names from four shows (The Pitt, Fallout, The Last of Us, Breaking Bad): clean matching keys 65% -> 99%, display strip 91% -> 100%. The corpora are committed as spec fixtures. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(matching): use ES2015-safe trailing trim in title normalization String.prototype.trimEnd is ES2019; the shared-interfaces lib compiles against an older lib target (TS2550 in typecheck:web). Replace with a regex-based trimRight helper. Jest uses its own tsconfig, so this only surfaced in the CI typecheck, not local unit runs. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(matching): guard tag stripping against real-title false positives Address code-review findings on the tag-stripping rules: - underscore suffix is now vocabulary-gated, so "Mr_Robot", "Cowboy_Bebop", "Mrs_Davis" keep their second word - leading single-segment tags before a spaced dash stay 2-3 chars (only hyphen-compounds like "4K-DE" and pipe-tags like "EXYU|" may be wider), so "DUNE - Part Two" and "ALIEN - Covenant" are left intact - "IN" is excluded from the weak joined-dash/underscore paths so "drive-in" and "Plug-in" are not truncated (India still strips via the strong "IN| " / "IN - " forms) The display-side stripCountryPrefix() mirrors the narrowed dash rule. Corpus coverage is unchanged at 99% (245/248); new counter-example tests lock in the guards. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-10-08 17:06:15 -08:00 · 2026-07-19 19:42:20 +02:00
1 parent aa1941cf2c
commit 402382421c
@@ -78,10 +78,10 @@ describe('extractYear', () => {
 describe('lookup keys', () => {
    it('builds stable search and details keys', () => {
        expect(buildSearchLookupKey('the matrix', 1999)).toBe(
-            'title:the matrix|year:1999'
+            'title:the matrix|year:1999|v2'
        );
        expect(buildSearchLookupKey('the matrix', null)).toBe(
-            'title:the matrix|year:'
+            'title:the matrix|year:|v2'
        );
        expect(buildDetailsLookupKey(603)).toBe('id:603|v2');
    });
@@ -70,7 +70,10 @@ export function buildSearchLookupKey(
    normalizedTitle: string,
    year: number | null
 ): string {
-    return `title:${normalizedTitle}|year:${year ?? ''}`;
+    // v2: normalizeTitleKeys learned to strip appended language/quality
+    // tags; the version suffix invalidates cached (incl. negative) match
+    // resolutions keyed on the old polluted titles
+    return `title:${normalizedTitle}|year:${year ?? ''}|v2`;
 }

 export function buildDetailsLookupKey(tmdbId: number): string {
@@ -56,6 +56,143 @@ describe('normalizeTitleKeys', () => {
    });
 });

+describe('provider tag stripping', () => {
+    it('strips wrapped pipe tags', () => {
+        expect(normalizeTitle('|DE| ARD')).toBe('ard');
+        expect(normalizeTitle('|MULTI| Fallout - 4K')).toBe('fallout');
+        expect(normalizeTitle('|EXYU| The Pitt')).toBe('the pitt');
+    });
+
+    it('strips long and compound leading tags', () => {
+        expect(normalizeTitle('EXYU| Fallout')).toBe('fallout');
+        expect(normalizeTitle('MULTI| Breaking Bad')).toBe('breaking bad');
+        expect(normalizeTitle('4K-DE - The Pitt (2025) (US)')).toBe(
+            'the pitt'
+        );
+        expect(normalizeTitle('AR-SUBS - Fallout (2024) (US)')).toBe(
+            'fallout'
+        );
+        expect(normalizeTitle('4K-OSN+ - The Last of Us (2023)')).toBe(
+            'the last of us'
+        );
+    });
+
+    it('never treats numeric fragments as leading tags', () => {
+        expect(normalizeTitle('1917 - Behind the Lines')).toBe(
+            '1917 behind the lines'
+        );
+    });
+
+    it('keeps bare 4-5 char words before a spaced dash (real titles)', () => {
+        expect(normalizeTitle('DUNE - Part Two')).toBe('dune part two');
+        expect(normalizeTitle('ALIEN - Covenant')).toBe('alien covenant');
+    });
+
+    it('strips underscore and double-dash suffix tags', () => {
+        expect(normalizeTitle('Fallout_eng')).toBe('fallout');
+        expect(normalizeTitle('Breaking Bad (US)_msub')).toBe('breaking bad');
+        expect(normalizeTitle('The Pitt (2025)_sub')).toBe('the pitt');
+        expect(normalizeTitle('The Last of Us--esp')).toBe('the last of us');
+    });
+
+    it('keeps underscore-as-space titles intact', () => {
+        expect(normalizeTitle('The_Last_of_Us')).toBe('the last of us');
+    });
+
+    it('keeps sole-underscore titles whose tail is not a known tag', () => {
+        expect(normalizeTitle('Mr_Robot')).toBe('mr robot');
+        expect(normalizeTitle('Cowboy_Bebop')).toBe('cowboy bebop');
+        expect(normalizeTitle('Mrs_Davis')).toBe('mrs davis');
+    });
+
+    it('strips joined dash tags only for case-uniform vocabulary tokens', () => {
+        expect(normalizeTitle('Breaking Bad-eng')).toBe('breaking bad');
+        expect(normalizeTitle('The Last of Us-DE')).toBe('the last of us');
+        expect(normalizeTitle('The Pitt (2025)-it')).toBe('the pitt');
+        expect(normalizeTitle('Spider-Man')).toBe('spider man');
+        expect(normalizeTitle('Kick-It')).toBe('kick it');
+    });
+
+    it('keeps English hyphenated word endings that collide with codes', () => {
+        expect(normalizeTitle('drive-in')).toBe('drive in');
+        expect(normalizeTitle('Plug-in')).toBe('plug in');
+    });
+
+    it('strips bare trailing UPPERCASE vocabulary tags', () => {
+        expect(normalizeTitle('The Pitt (2025) DE')).toBe('the pitt');
+        expect(normalizeTitle('Breaking Bad ES')).toBe('breaking bad');
+        expect(normalizeTitle('EN| Breaking Bad SUB')).toBe('breaking bad');
+        expect(normalizeTitle('The Last of Us (2023) AF')).toBe(
+            'the last of us'
+        );
+    });
+
+    it('never strips trailing tags that could be real endings', () => {
+        expect(normalizeTitle('Rocky II')).toBe('rocky ii');
+        expect(normalizeTitle('Made in USA')).toBe('made in usa');
+        expect(normalizeTitle('NCIS: LA')).toBe('ncis la');
+        expect(normalizeTitle('Making It')).toBe('making it');
+        expect(normalizeTitle('THE LAST OF US')).toBe('the last of us');
+    });
+
+    const pittCorpus = [
+        'The Pitt (2025)_sub', 'The Pitt (2025)-it', 'The Pitt (2025)',
+        'The Pitt (Hindi)', 'The Pitt (2025) 4K', 'The Pitt (2025) DE',
+        'The Pitt (2025) ES', 'The Pitt (2025) FR', 'The Pitt (2025)_eng',
+        'The Pitt [MULTI-SUB]', 'The Pitt (2025) (4K DV)', 'GR - The Pitt',
+        '4K-DE - The Pitt (2025) (US)', '4K-TR - The Pitt (2025) (US)',
+        'AR-SUBS - The Pitt (2025) (US)', 'DE - The Pitt (2025) (US)',
+        'ALB| The Pitt', 'EXYU| The Pitt', '|ALB| The Pitt', '|DE| The Pitt',
+    ];
+
+    const falloutCorpus = [
+        'Fallout', 'DE - Fallout (2024)', 'Fallout (2024) - 4K',
+        'Fallout (2024) FR-EN', 'Fallout (2024) Multi', 'Fallout (2024)_fr',
+        'Fallout_esp', 'Fallout (4K)', '4K-AMZ - Fallout (2024) (US)',
+        'AL - Fallout (2024)', 'AMZ - Fallout (2024) (US)',
+        'AR-DE - Fallout (US)', 'LA - Fallout', 'EN| Fallout - 4K',
+        'MULTI| Fallout - 4K', 'Fallout ( مدبلج )', 'Fallout (Telugu)',
+        '|EN| Fallout - 4K', '|MULTI| Fallout', '|TR| Fallout',
+    ];
+
+    const lastOfUsCorpus = [
+        'The Last of Us', 'The Last Of Us', 'The Last of Us (2023) 4K',
+        'The Last of Us (2023) AF', 'The Last of Us_tr',
+        'The Last of Us--esp', 'The Last of Us-DE', 'The Last of Us-esp',
+        'The Last of Us [L]', 'The Last of Us ( HD )',
+        '4K-OSN+ - The Last of Us (2023)', 'IS - The Last of Us (2023) (US)',
+        'RU - The Last of Us', 'ALB| The Last of Us',
+    ];
+
+    const breakingBadCorpus = [
+        'Breaking Bad', 'Breaking Bad (2008)_fr', 'Breaking Bad (US)_msub',
+        'Breaking Bad_it', 'Breaking Bad-DE', 'Breaking Bad-eng',
+        'Breaking Bad ( عائلي )', 'Breaking Bad (Pure)',
+        'Breaking Bad - Multi', 'Breaking Bad ES', 'AR-DE - Breaking Bad',
+        'EN| Breaking Bad SUB', 'MULTI| Breaking Bad', 'AR| Breaking Bad',
+    ];
+
+    it.each([
+        ['the pitt', pittCorpus],
+        ['fallout', falloutCorpus],
+        ['the last of us', lastOfUsCorpus],
+        ['breaking bad', breakingBadCorpus],
+    ])(
+        'normalizes every observed provider variant of "%s" to one key',
+        (expected, corpus) => {
+            for (const name of corpus) {
+                expect(normalizeTitleKeys(name).base).toBe(expected);
+            }
+        }
+    );
+
+    it('keeps localized subtitles (indistinguishable from real ones)', () => {
+        expect(normalizeTitle('Breaking Bad: A Química do Mal')).toBe(
+            'breaking bad a quimica do mal'
+        );
+    });
+});
+
 describe('titleYearsCompatible', () => {
    it('accepts unknown years and ±1 tolerance', () => {
        expect(titleYearsCompatible(null, 2049)).toBe(true);
@@ -28,11 +28,142 @@ const QUALITY_TAGS = new Set([
 ]);

 /**
- * Leading channel/language prefix like "EN - ", "DE| ", "FR: ".
- * UPPERCASE-only on purpose: a case-insensitive match would amputate real
- * title words ("It: Chapter Two" → "Chapter Two").
+ * Wrapped tag at the very start of a provider title: "|DE| ARD",
+ * "|MULTI| Fallout". The lookahead requires a letter in the tag so a
+ * numeric fragment can never be treated as one.
 */
-const LANGUAGE_PREFIX = /^[A-Z]{2,3}\s*[-|:]\s+/;
+const WRAPPED_TAG_PREFIX = /^\s*\|(?=[0-9+]*[A-Z])[A-Z0-9+]{2,5}\|\s*/;
+
+/**
+ * Leading channel/language prefix like "EN - ", "DE| ", "FR: ", including
+ * compound provider/quality forms ("4K-DE - ", "AR-SUBS - ", "4K-OSN+ - ")
+ * and longer pipe-tagged forms ("EXYU| ", "MULTI| ").
+ * UPPERCASE-only on purpose: a case-insensitive match would amputate real
+ * title words ("It: Chapter Two" → "Chapter Two"). Every segment must
+ * contain a letter so numeric titles ("1917 - ...") are never tags.
+ *
+ * Separator strength gates how wide a single segment may be:
+ *   - dash ("EN - "): compound OR 2–3 chars — a bare 4–5 char word before
+ *     a spaced dash is a real title ("DUNE - Part Two", "ALIEN - Covenant")
+ *   - pipe ("EXYU| "): compound OR 2–5 chars — a pipe is a strong tag signal
+ *   - colon ("EN: "): 2–3 chars — longer acronyms are franchise titles
+ *     ("NCIS: LA")
+ */
+const SEG = '(?=[0-9+]*[A-Z])[A-Z0-9+]';
+const COMPOUND_TAG = `${SEG}{2,5}(?:-${SEG}{2,6}){1,2}`;
+const LANGUAGE_PREFIX = new RegExp(
+    '^(?:' +
+        `(?:${COMPOUND_TAG}|${SEG}{2,3})\\s*-\\s+` +
+        `|(?:${COMPOUND_TAG}|${SEG}{2,5})\\s*\\|\\s+` +
+        `|${SEG}{2,3}\\s*:\\s+` +
+        ')'
+);
+
+/**
+ * Curated whitelist for TRAILING language/subtitle tags ("Fallout_eng",
+ * "Breaking Bad-DE", "The Pitt (2025) ES"). Trailing stripping must be
+ * vocabulary-gated: a pattern-only rule would amputate real endings —
+ * roman numerals ("Rocky II"), acronyms ("Made in USA"), franchise
+ * suffixes ("NCIS: LA"). US/USA/UK/LA are deliberately absent.
+ */
+const TRAILING_TAG_VOCABULARY = new Set([
+    'AF', 'AL', 'ALB', 'AR', 'BY', 'DE', 'DUB', 'EN', 'ENG', 'ES', 'ESP',
+    'EXYU', 'FR', 'FRA', 'GE', 'GR', 'HU', 'IN', 'IR', 'IS', 'IT', 'ITA',
+    'KA', 'KU', 'LAT', 'ML', 'MSUB', 'MULTI', 'NL', 'PL', 'PT', 'RO', 'RU',
+    'SC', 'SE', 'SUB', 'SUBS', 'SW', 'TA', 'TL', 'TR', 'TUR',
+]);
+
+/**
+ * Vocabulary tags that are also common English hyphenated-word endings
+ * ("drive-in", "plug-in"). Accepted only after a STRONG separator (bare
+ * spaced/uppercase form), never in the weak joined-dash/underscore forms
+ * where "X-in"/"X_in" reads as a title, not a tag.
+ */
+const WEAK_JOIN_EXCLUSIONS = new Set(['IN']);
+
+const DOUBLE_DASH_SUFFIX = /[-–]{2}[A-Za-z]{2,5}\s*$/;
+const UNDERSCORE_SUFFIX = /_([A-Za-z]{2,5})\s*$/;
+const JOINED_DASH_SUFFIX = /-([A-Za-z]{2,5})\s*$/;
+const TRAILING_TAG_SUFFIX = /\s([A-Z]{2,5})\s*$/;
+
+/** Tag tokens are case-uniform; real title words are Capitalized. */
+function isCaseUniform(token: string): boolean {
+    return token === token.toLowerCase() || token === token.toUpperCase();
+}
+
+/**
+ * A captured joined-dash/underscore token is provider metadata only when
+ * it is a known vocabulary tag, case-uniform, and not one of the English
+ * word-forming exclusions. This keeps "Mr_Robot", "Cowboy_Bebop",
+ * "drive-in", and "Plug-in" intact while stripping "_eng", "-DE", "-it".
+ */
+function isJoinedTag(token: string): boolean {
+    const upper = token.toUpperCase();
+    return (
+        TRAILING_TAG_VOCABULARY.has(upper) &&
+        !WEAK_JOIN_EXCLUSIONS.has(upper) &&
+        isCaseUniform(token)
+    );
+}
+
+/** ES2015-safe trailing-whitespace trim (the lib target predates trimEnd). */
+function trimRight(value: string): string {
+    return value.replace(/\s+$/, '');
+}
+
+/**
+ * Strip appended language/subtitle tags. Runs BEFORE lowercasing —
+ * casing is the main false-positive guard: ALL-CAPS titles carry no
+ * casing signal ("THE LAST OF US" must keep its "US"), so caps-gated
+ * rules are skipped for them, and Capitalized endings ("Making It",
+ * "Kick-It") never look like tags. Compound tags ("FR-EN") shed one
+ * token per pass, so stripping repeats to a fixpoint.
+ */
+function stripTrailingTags(value: string): string {
+    let result = value;
+    for (let pass = 0; pass < 3; pass++) {
+        const next = stripTrailingTagOnce(result);
+        if (next === result) break;
+        result = next;
+    }
+    return result;
+}
+
+function stripTrailingTagOnce(value: string): string {
+    const result = trimRight(value);
+    const hasLowercase = /\p{Ll}/u.test(result);
+
+    // "The Last of Us--esp": no real title contains a double dash.
+    if (DOUBLE_DASH_SUFFIX.test(result)) {
+        return trimRight(result.replace(DOUBLE_DASH_SUFFIX, ''));
+    }
+
+    // "Fallout_eng" — but not "The_Last_of_Us" (underscores as spaces, only
+    // strip a sole underscore) and not "Mr_Robot"/"Cowboy_Bebop" (the tail
+    // must be a known tag, so the segment is real-title evidence otherwise).
+    const underscore = result.match(UNDERSCORE_SUFFIX);
+    if (
+        underscore &&
+        result.indexOf('_') === result.lastIndexOf('_') &&
+        isJoinedTag(underscore[1])
+    ) {
+        return trimRight(result.replace(UNDERSCORE_SUFFIX, ''));
+    }
+
+    // "Breaking Bad-eng", "The Last of Us-DE" — but not "drive-in"/"Plug-in".
+    // hasLowercase gates ALL-CAPS titles out (no casing signal to trust).
+    const joined = result.match(JOINED_DASH_SUFFIX);
+    if (joined && hasLowercase && isJoinedTag(joined[1])) {
+        return trimRight(result.replace(JOINED_DASH_SUFFIX, ''));
+    }
+
+    const trailing = result.match(TRAILING_TAG_SUFFIX);
+    if (trailing && hasLowercase && TRAILING_TAG_VOCABULARY.has(trailing[1])) {
+        return trimRight(result.replace(TRAILING_TAG_SUFFIX, ''));
+    }
+
+    return result;
+}

 const YEAR_PATTERN = /\b(19\d{2}|20\d{2})\b/;

@@ -74,11 +205,14 @@ export function normalizeTitleKeys(
        return { exact: '', base: '', trailingYear: null };
    }

-    const cleaned = raw
-        // Inner classes exclude the opening delimiter too, so runaway
-        // inputs like "[[[[[..." backtrack linearly (CodeQL js/polynomial-redos)
-        .replace(/\[[^\][]*\]|\([^()]*\)|\{[^{}]*\}/g, ' ')
-        .replace(LANGUAGE_PREFIX, '')
+    const cleaned = stripTrailingTags(
+        raw
+            .replace(WRAPPED_TAG_PREFIX, '')
+            // Inner classes exclude the opening delimiter too, so runaway
+            // inputs like "[[[[[..." backtrack linearly (CodeQL js/polynomial-redos)
+            .replace(/\[[^\][]*\]|\([^()]*\)|\{[^{}]*\}/g, ' ')
+            .replace(LANGUAGE_PREFIX, '')
+    )
        .normalize('NFD')
        .replace(/[\u0300-\u036F]/g, '')
        .toLowerCase()
@@ -43,6 +43,36 @@ describe('stripCountryPrefix', () => {
            expect(stripCountryPrefix('US: CNN')).toBe('CNN');
        });

+        it('strips compound quality/provider tags', () => {
+            expect(stripCountryPrefix('4K-DE - The Pitt (2025)')).toBe(
+                'The Pitt (2025)'
+            );
+            expect(stripCountryPrefix('AR-SUBS - Fallout')).toBe('Fallout');
+            expect(stripCountryPrefix('4K-OSN+ - The Last of Us')).toBe(
+                'The Last of Us'
+            );
+        });
+
+        it('strips longer pipe-tagged prefixes', () => {
+            expect(stripCountryPrefix('EXYU| News')).toBe('News');
+            expect(stripCountryPrefix('MULTI| Movies')).toBe('Movies');
+        });
+
+        it('never treats numeric fragments as tags', () => {
+            expect(stripCountryPrefix('1917 - Documentary')).toBe(
+                '1917 - Documentary'
+            );
+        });
+
+        it('keeps bare 4-5 char words before a spaced dash (real titles)', () => {
+            expect(stripCountryPrefix('DUNE - Part Two')).toBe(
+                'DUNE - Part Two'
+            );
+            expect(stripCountryPrefix('ALIEN - Covenant')).toBe(
+                'ALIEN - Covenant'
+            );
+        });
+
        it('only strips the first tag segment', () => {
            expect(stripCountryPrefix('ES - A3 - Sports')).toBe('A3 - Sports');
        });
@@ -3,9 +3,19 @@
 * country/group tag — they routinely appear inside real channel names
 * ("Sky - Sports F1", "Discovery - Science"). Pipes are conventionally
 * used only as tag separators, so they always count.
+ *
+ * Before a dash a tag is either a compound ("4K-DE", "AR-SUBS", "4K-OSN+"
+ * — the inner hyphen is the tag signal) or a plain 2–3 char code ("US",
+ * "4K"). A bare 4–5 char word before a spaced dash is a real title
+ * ("DUNE - Part Two", "ALIEN - Covenant"), so it is not a tag. Colon tags
+ * stay 2–3 chars — longer acronyms are franchise titles ("NCIS: LA").
+ * Every segment must contain a letter so numbers ("1917 - ...") are safe.
 */
-const DASH_OR_COLON_SEPARATORS = [' - ', '- ', ' -', ': '];
-const TAG_PREFIX_PATTERN = /^[A-Z0-9]{2,3}$/;
+const DASH_SEPARATORS = [' - ', '- ', ' -'];
+const COLON_SEPARATOR = ': ';
+const TAG_PREFIX_PATTERN =
+    /^(?:(?=[0-9+]*[A-Z])[A-Z0-9+]{2,5}(?:-(?=[0-9+]*[A-Z])[A-Z0-9+]{2,6}){1,2}|(?=[0-9+]*[A-Z])[A-Z0-9+]{2,3})$/;
+const COLON_TAG_PATTERN = /^(?=[0-9+]*[A-Z])[A-Z0-9+]{2,3}$/;

 interface SeparatorMatch {
    index: number;
@@ -68,10 +78,14 @@ function findTagSeparator(name: string): SeparatorMatch | null {
        best = { index: pipeIndex, length: 1 };
    }

-    for (const separator of DASH_OR_COLON_SEPARATORS) {
+    for (const separator of [...DASH_SEPARATORS, COLON_SEPARATOR]) {
        const index = name.indexOf(separator);
        if (index === -1 || (best && best.index <= index)) continue;
-        if (!TAG_PREFIX_PATTERN.test(name.slice(0, index).trim())) continue;
+        const pattern =
+            separator === COLON_SEPARATOR
+                ? COLON_TAG_PATTERN
+                : TAG_PREFIX_PATTERN;
+        if (!pattern.test(name.slice(0, index).trim())) continue;
        best = { index, length: separator.length };
    }