diff --git a/libs/services/src/lib/tmdb/tmdb-matcher.spec.ts b/libs/services/src/lib/tmdb/tmdb-matcher.spec.ts index 394bc89f4..f61f7ce86 100644 --- a/libs/services/src/lib/tmdb/tmdb-matcher.spec.ts +++ b/libs/services/src/lib/tmdb/tmdb-matcher.spec.ts @@ -78,10 +78,10 @@ describe('extractYear', () => { describe('lookup keys', () => { it('builds stable search and details keys', () => { expect(buildSearchLookupKey('the matrix', 1999)).toBe( - 'title:the matrix|year:1999' + 'title:the matrix|year:1999|v2' ); expect(buildSearchLookupKey('the matrix', null)).toBe( - 'title:the matrix|year:' + 'title:the matrix|year:|v2' ); expect(buildDetailsLookupKey(603)).toBe('id:603|v2'); }); diff --git a/libs/services/src/lib/tmdb/tmdb-matcher.ts b/libs/services/src/lib/tmdb/tmdb-matcher.ts index a07d2e16f..2c60cca56 100644 --- a/libs/services/src/lib/tmdb/tmdb-matcher.ts +++ b/libs/services/src/lib/tmdb/tmdb-matcher.ts @@ -70,7 +70,10 @@ export function buildSearchLookupKey( normalizedTitle: string, year: number | null ): string { - return `title:${normalizedTitle}|year:${year ?? ''}`; + // v2: normalizeTitleKeys learned to strip appended language/quality + // tags; the version suffix invalidates cached (incl. negative) match + // resolutions keyed on the old polluted titles + return `title:${normalizedTitle}|year:${year ?? ''}|v2`; } export function buildDetailsLookupKey(tmdbId: number): string { diff --git a/libs/shared/interfaces/src/lib/title-normalization.util.spec.ts b/libs/shared/interfaces/src/lib/title-normalization.util.spec.ts index 3c60b3d8a..6566ae95c 100644 --- a/libs/shared/interfaces/src/lib/title-normalization.util.spec.ts +++ b/libs/shared/interfaces/src/lib/title-normalization.util.spec.ts @@ -56,6 +56,143 @@ describe('normalizeTitleKeys', () => { }); }); +describe('provider tag stripping', () => { + it('strips wrapped pipe tags', () => { + expect(normalizeTitle('|DE| ARD')).toBe('ard'); + expect(normalizeTitle('|MULTI| Fallout - 4K')).toBe('fallout'); + expect(normalizeTitle('|EXYU| The Pitt')).toBe('the pitt'); + }); + + it('strips long and compound leading tags', () => { + expect(normalizeTitle('EXYU| Fallout')).toBe('fallout'); + expect(normalizeTitle('MULTI| Breaking Bad')).toBe('breaking bad'); + expect(normalizeTitle('4K-DE - The Pitt (2025) (US)')).toBe( + 'the pitt' + ); + expect(normalizeTitle('AR-SUBS - Fallout (2024) (US)')).toBe( + 'fallout' + ); + expect(normalizeTitle('4K-OSN+ - The Last of Us (2023)')).toBe( + 'the last of us' + ); + }); + + it('never treats numeric fragments as leading tags', () => { + expect(normalizeTitle('1917 - Behind the Lines')).toBe( + '1917 behind the lines' + ); + }); + + it('keeps bare 4-5 char words before a spaced dash (real titles)', () => { + expect(normalizeTitle('DUNE - Part Two')).toBe('dune part two'); + expect(normalizeTitle('ALIEN - Covenant')).toBe('alien covenant'); + }); + + it('strips underscore and double-dash suffix tags', () => { + expect(normalizeTitle('Fallout_eng')).toBe('fallout'); + expect(normalizeTitle('Breaking Bad (US)_msub')).toBe('breaking bad'); + expect(normalizeTitle('The Pitt (2025)_sub')).toBe('the pitt'); + expect(normalizeTitle('The Last of Us--esp')).toBe('the last of us'); + }); + + it('keeps underscore-as-space titles intact', () => { + expect(normalizeTitle('The_Last_of_Us')).toBe('the last of us'); + }); + + it('keeps sole-underscore titles whose tail is not a known tag', () => { + expect(normalizeTitle('Mr_Robot')).toBe('mr robot'); + expect(normalizeTitle('Cowboy_Bebop')).toBe('cowboy bebop'); + expect(normalizeTitle('Mrs_Davis')).toBe('mrs davis'); + }); + + it('strips joined dash tags only for case-uniform vocabulary tokens', () => { + expect(normalizeTitle('Breaking Bad-eng')).toBe('breaking bad'); + expect(normalizeTitle('The Last of Us-DE')).toBe('the last of us'); + expect(normalizeTitle('The Pitt (2025)-it')).toBe('the pitt'); + expect(normalizeTitle('Spider-Man')).toBe('spider man'); + expect(normalizeTitle('Kick-It')).toBe('kick it'); + }); + + it('keeps English hyphenated word endings that collide with codes', () => { + expect(normalizeTitle('drive-in')).toBe('drive in'); + expect(normalizeTitle('Plug-in')).toBe('plug in'); + }); + + it('strips bare trailing UPPERCASE vocabulary tags', () => { + expect(normalizeTitle('The Pitt (2025) DE')).toBe('the pitt'); + expect(normalizeTitle('Breaking Bad ES')).toBe('breaking bad'); + expect(normalizeTitle('EN| Breaking Bad SUB')).toBe('breaking bad'); + expect(normalizeTitle('The Last of Us (2023) AF')).toBe( + 'the last of us' + ); + }); + + it('never strips trailing tags that could be real endings', () => { + expect(normalizeTitle('Rocky II')).toBe('rocky ii'); + expect(normalizeTitle('Made in USA')).toBe('made in usa'); + expect(normalizeTitle('NCIS: LA')).toBe('ncis la'); + expect(normalizeTitle('Making It')).toBe('making it'); + expect(normalizeTitle('THE LAST OF US')).toBe('the last of us'); + }); + + const pittCorpus = [ + 'The Pitt (2025)_sub', 'The Pitt (2025)-it', 'The Pitt (2025)', + 'The Pitt (Hindi)', 'The Pitt (2025) 4K', 'The Pitt (2025) DE', + 'The Pitt (2025) ES', 'The Pitt (2025) FR', 'The Pitt (2025)_eng', + 'The Pitt [MULTI-SUB]', 'The Pitt (2025) (4K DV)', 'GR - The Pitt', + '4K-DE - The Pitt (2025) (US)', '4K-TR - The Pitt (2025) (US)', + 'AR-SUBS - The Pitt (2025) (US)', 'DE - The Pitt (2025) (US)', + 'ALB| The Pitt', 'EXYU| The Pitt', '|ALB| The Pitt', '|DE| The Pitt', + ]; + + const falloutCorpus = [ + 'Fallout', 'DE - Fallout (2024)', 'Fallout (2024) - 4K', + 'Fallout (2024) FR-EN', 'Fallout (2024) Multi', 'Fallout (2024)_fr', + 'Fallout_esp', 'Fallout (4K)', '4K-AMZ - Fallout (2024) (US)', + 'AL - Fallout (2024)', 'AMZ - Fallout (2024) (US)', + 'AR-DE - Fallout (US)', 'LA - Fallout', 'EN| Fallout - 4K', + 'MULTI| Fallout - 4K', 'Fallout ( مدبلج )', 'Fallout (Telugu)', + '|EN| Fallout - 4K', '|MULTI| Fallout', '|TR| Fallout', + ]; + + const lastOfUsCorpus = [ + 'The Last of Us', 'The Last Of Us', 'The Last of Us (2023) 4K', + 'The Last of Us (2023) AF', 'The Last of Us_tr', + 'The Last of Us--esp', 'The Last of Us-DE', 'The Last of Us-esp', + 'The Last of Us [L]', 'The Last of Us ( HD )', + '4K-OSN+ - The Last of Us (2023)', 'IS - The Last of Us (2023) (US)', + 'RU - The Last of Us', 'ALB| The Last of Us', + ]; + + const breakingBadCorpus = [ + 'Breaking Bad', 'Breaking Bad (2008)_fr', 'Breaking Bad (US)_msub', + 'Breaking Bad_it', 'Breaking Bad-DE', 'Breaking Bad-eng', + 'Breaking Bad ( عائلي )', 'Breaking Bad (Pure)', + 'Breaking Bad - Multi', 'Breaking Bad ES', 'AR-DE - Breaking Bad', + 'EN| Breaking Bad SUB', 'MULTI| Breaking Bad', 'AR| Breaking Bad', + ]; + + it.each([ + ['the pitt', pittCorpus], + ['fallout', falloutCorpus], + ['the last of us', lastOfUsCorpus], + ['breaking bad', breakingBadCorpus], + ])( + 'normalizes every observed provider variant of "%s" to one key', + (expected, corpus) => { + for (const name of corpus) { + expect(normalizeTitleKeys(name).base).toBe(expected); + } + } + ); + + it('keeps localized subtitles (indistinguishable from real ones)', () => { + expect(normalizeTitle('Breaking Bad: A Química do Mal')).toBe( + 'breaking bad a quimica do mal' + ); + }); +}); + describe('titleYearsCompatible', () => { it('accepts unknown years and ±1 tolerance', () => { expect(titleYearsCompatible(null, 2049)).toBe(true); diff --git a/libs/shared/interfaces/src/lib/title-normalization.util.ts b/libs/shared/interfaces/src/lib/title-normalization.util.ts index 3b746683b..b06c73694 100644 --- a/libs/shared/interfaces/src/lib/title-normalization.util.ts +++ b/libs/shared/interfaces/src/lib/title-normalization.util.ts @@ -28,11 +28,142 @@ const QUALITY_TAGS = new Set([ ]); /** - * Leading channel/language prefix like "EN - ", "DE| ", "FR: ". - * UPPERCASE-only on purpose: a case-insensitive match would amputate real - * title words ("It: Chapter Two" → "Chapter Two"). + * Wrapped tag at the very start of a provider title: "|DE| ARD", + * "|MULTI| Fallout". The lookahead requires a letter in the tag so a + * numeric fragment can never be treated as one. */ -const LANGUAGE_PREFIX = /^[A-Z]{2,3}\s*[-|:]\s+/; +const WRAPPED_TAG_PREFIX = /^\s*\|(?=[0-9+]*[A-Z])[A-Z0-9+]{2,5}\|\s*/; + +/** + * Leading channel/language prefix like "EN - ", "DE| ", "FR: ", including + * compound provider/quality forms ("4K-DE - ", "AR-SUBS - ", "4K-OSN+ - ") + * and longer pipe-tagged forms ("EXYU| ", "MULTI| "). + * UPPERCASE-only on purpose: a case-insensitive match would amputate real + * title words ("It: Chapter Two" → "Chapter Two"). Every segment must + * contain a letter so numeric titles ("1917 - ...") are never tags. + * + * Separator strength gates how wide a single segment may be: + * - dash ("EN - "): compound OR 2–3 chars — a bare 4–5 char word before + * a spaced dash is a real title ("DUNE - Part Two", "ALIEN - Covenant") + * - pipe ("EXYU| "): compound OR 2–5 chars — a pipe is a strong tag signal + * - colon ("EN: "): 2–3 chars — longer acronyms are franchise titles + * ("NCIS: LA") + */ +const SEG = '(?=[0-9+]*[A-Z])[A-Z0-9+]'; +const COMPOUND_TAG = `${SEG}{2,5}(?:-${SEG}{2,6}){1,2}`; +const LANGUAGE_PREFIX = new RegExp( + '^(?:' + + `(?:${COMPOUND_TAG}|${SEG}{2,3})\\s*-\\s+` + + `|(?:${COMPOUND_TAG}|${SEG}{2,5})\\s*\\|\\s+` + + `|${SEG}{2,3}\\s*:\\s+` + + ')' +); + +/** + * Curated whitelist for TRAILING language/subtitle tags ("Fallout_eng", + * "Breaking Bad-DE", "The Pitt (2025) ES"). Trailing stripping must be + * vocabulary-gated: a pattern-only rule would amputate real endings — + * roman numerals ("Rocky II"), acronyms ("Made in USA"), franchise + * suffixes ("NCIS: LA"). US/USA/UK/LA are deliberately absent. + */ +const TRAILING_TAG_VOCABULARY = new Set([ + 'AF', 'AL', 'ALB', 'AR', 'BY', 'DE', 'DUB', 'EN', 'ENG', 'ES', 'ESP', + 'EXYU', 'FR', 'FRA', 'GE', 'GR', 'HU', 'IN', 'IR', 'IS', 'IT', 'ITA', + 'KA', 'KU', 'LAT', 'ML', 'MSUB', 'MULTI', 'NL', 'PL', 'PT', 'RO', 'RU', + 'SC', 'SE', 'SUB', 'SUBS', 'SW', 'TA', 'TL', 'TR', 'TUR', +]); + +/** + * Vocabulary tags that are also common English hyphenated-word endings + * ("drive-in", "plug-in"). Accepted only after a STRONG separator (bare + * spaced/uppercase form), never in the weak joined-dash/underscore forms + * where "X-in"/"X_in" reads as a title, not a tag. + */ +const WEAK_JOIN_EXCLUSIONS = new Set(['IN']); + +const DOUBLE_DASH_SUFFIX = /[-–]{2}[A-Za-z]{2,5}\s*$/; +const UNDERSCORE_SUFFIX = /_([A-Za-z]{2,5})\s*$/; +const JOINED_DASH_SUFFIX = /-([A-Za-z]{2,5})\s*$/; +const TRAILING_TAG_SUFFIX = /\s([A-Z]{2,5})\s*$/; + +/** Tag tokens are case-uniform; real title words are Capitalized. */ +function isCaseUniform(token: string): boolean { + return token === token.toLowerCase() || token === token.toUpperCase(); +} + +/** + * A captured joined-dash/underscore token is provider metadata only when + * it is a known vocabulary tag, case-uniform, and not one of the English + * word-forming exclusions. This keeps "Mr_Robot", "Cowboy_Bebop", + * "drive-in", and "Plug-in" intact while stripping "_eng", "-DE", "-it". + */ +function isJoinedTag(token: string): boolean { + const upper = token.toUpperCase(); + return ( + TRAILING_TAG_VOCABULARY.has(upper) && + !WEAK_JOIN_EXCLUSIONS.has(upper) && + isCaseUniform(token) + ); +} + +/** ES2015-safe trailing-whitespace trim (the lib target predates trimEnd). */ +function trimRight(value: string): string { + return value.replace(/\s+$/, ''); +} + +/** + * Strip appended language/subtitle tags. Runs BEFORE lowercasing — + * casing is the main false-positive guard: ALL-CAPS titles carry no + * casing signal ("THE LAST OF US" must keep its "US"), so caps-gated + * rules are skipped for them, and Capitalized endings ("Making It", + * "Kick-It") never look like tags. Compound tags ("FR-EN") shed one + * token per pass, so stripping repeats to a fixpoint. + */ +function stripTrailingTags(value: string): string { + let result = value; + for (let pass = 0; pass < 3; pass++) { + const next = stripTrailingTagOnce(result); + if (next === result) break; + result = next; + } + return result; +} + +function stripTrailingTagOnce(value: string): string { + const result = trimRight(value); + const hasLowercase = /\p{Ll}/u.test(result); + + // "The Last of Us--esp": no real title contains a double dash. + if (DOUBLE_DASH_SUFFIX.test(result)) { + return trimRight(result.replace(DOUBLE_DASH_SUFFIX, '')); + } + + // "Fallout_eng" — but not "The_Last_of_Us" (underscores as spaces, only + // strip a sole underscore) and not "Mr_Robot"/"Cowboy_Bebop" (the tail + // must be a known tag, so the segment is real-title evidence otherwise). + const underscore = result.match(UNDERSCORE_SUFFIX); + if ( + underscore && + result.indexOf('_') === result.lastIndexOf('_') && + isJoinedTag(underscore[1]) + ) { + return trimRight(result.replace(UNDERSCORE_SUFFIX, '')); + } + + // "Breaking Bad-eng", "The Last of Us-DE" — but not "drive-in"/"Plug-in". + // hasLowercase gates ALL-CAPS titles out (no casing signal to trust). + const joined = result.match(JOINED_DASH_SUFFIX); + if (joined && hasLowercase && isJoinedTag(joined[1])) { + return trimRight(result.replace(JOINED_DASH_SUFFIX, '')); + } + + const trailing = result.match(TRAILING_TAG_SUFFIX); + if (trailing && hasLowercase && TRAILING_TAG_VOCABULARY.has(trailing[1])) { + return trimRight(result.replace(TRAILING_TAG_SUFFIX, '')); + } + + return result; +} const YEAR_PATTERN = /\b(19\d{2}|20\d{2})\b/; @@ -74,11 +205,14 @@ export function normalizeTitleKeys( return { exact: '', base: '', trailingYear: null }; } - const cleaned = raw - // Inner classes exclude the opening delimiter too, so runaway - // inputs like "[[[[[..." backtrack linearly (CodeQL js/polynomial-redos) - .replace(/\[[^\][]*\]|\([^()]*\)|\{[^{}]*\}/g, ' ') - .replace(LANGUAGE_PREFIX, '') + const cleaned = stripTrailingTags( + raw + .replace(WRAPPED_TAG_PREFIX, '') + // Inner classes exclude the opening delimiter too, so runaway + // inputs like "[[[[[..." backtrack linearly (CodeQL js/polynomial-redos) + .replace(/\[[^\][]*\]|\([^()]*\)|\{[^{}]*\}/g, ' ') + .replace(LANGUAGE_PREFIX, '') + ) .normalize('NFD') .replace(/[\u0300-\u036F]/g, '') .toLowerCase() diff --git a/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.spec.ts b/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.spec.ts index abbf90037..3633d4e2e 100644 --- a/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.spec.ts +++ b/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.spec.ts @@ -43,6 +43,36 @@ describe('stripCountryPrefix', () => { expect(stripCountryPrefix('US: CNN')).toBe('CNN'); }); + it('strips compound quality/provider tags', () => { + expect(stripCountryPrefix('4K-DE - The Pitt (2025)')).toBe( + 'The Pitt (2025)' + ); + expect(stripCountryPrefix('AR-SUBS - Fallout')).toBe('Fallout'); + expect(stripCountryPrefix('4K-OSN+ - The Last of Us')).toBe( + 'The Last of Us' + ); + }); + + it('strips longer pipe-tagged prefixes', () => { + expect(stripCountryPrefix('EXYU| News')).toBe('News'); + expect(stripCountryPrefix('MULTI| Movies')).toBe('Movies'); + }); + + it('never treats numeric fragments as tags', () => { + expect(stripCountryPrefix('1917 - Documentary')).toBe( + '1917 - Documentary' + ); + }); + + it('keeps bare 4-5 char words before a spaced dash (real titles)', () => { + expect(stripCountryPrefix('DUNE - Part Two')).toBe( + 'DUNE - Part Two' + ); + expect(stripCountryPrefix('ALIEN - Covenant')).toBe( + 'ALIEN - Covenant' + ); + }); + it('only strips the first tag segment', () => { expect(stripCountryPrefix('ES - A3 - Sports')).toBe('A3 - Sports'); }); diff --git a/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.ts b/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.ts index f1213996f..d0b8b74dc 100644 --- a/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.ts +++ b/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.ts @@ -3,9 +3,19 @@ * country/group tag — they routinely appear inside real channel names * ("Sky - Sports F1", "Discovery - Science"). Pipes are conventionally * used only as tag separators, so they always count. + * + * Before a dash a tag is either a compound ("4K-DE", "AR-SUBS", "4K-OSN+" + * — the inner hyphen is the tag signal) or a plain 2–3 char code ("US", + * "4K"). A bare 4–5 char word before a spaced dash is a real title + * ("DUNE - Part Two", "ALIEN - Covenant"), so it is not a tag. Colon tags + * stay 2–3 chars — longer acronyms are franchise titles ("NCIS: LA"). + * Every segment must contain a letter so numbers ("1917 - ...") are safe. */ -const DASH_OR_COLON_SEPARATORS = [' - ', '- ', ' -', ': ']; -const TAG_PREFIX_PATTERN = /^[A-Z0-9]{2,3}$/; +const DASH_SEPARATORS = [' - ', '- ', ' -']; +const COLON_SEPARATOR = ': '; +const TAG_PREFIX_PATTERN = + /^(?:(?=[0-9+]*[A-Z])[A-Z0-9+]{2,5}(?:-(?=[0-9+]*[A-Z])[A-Z0-9+]{2,6}){1,2}|(?=[0-9+]*[A-Z])[A-Z0-9+]{2,3})$/; +const COLON_TAG_PATTERN = /^(?=[0-9+]*[A-Z])[A-Z0-9+]{2,3}$/; interface SeparatorMatch { index: number; @@ -68,10 +78,14 @@ function findTagSeparator(name: string): SeparatorMatch | null { best = { index: pipeIndex, length: 1 }; } - for (const separator of DASH_OR_COLON_SEPARATORS) { + for (const separator of [...DASH_SEPARATORS, COLON_SEPARATOR]) { const index = name.indexOf(separator); if (index === -1 || (best && best.index <= index)) continue; - if (!TAG_PREFIX_PATTERN.test(name.slice(0, index).trim())) continue; + const pattern = + separator === COLON_SEPARATOR + ? COLON_TAG_PATTERN + : TAG_PREFIX_PATTERN; + if (!pattern.test(name.slice(0, index).trim())) continue; best = { index, length: separator.length }; }