diff --git a/libs/services/src/lib/tmdb/tmdb-matcher.spec.ts b/libs/services/src/lib/tmdb/tmdb-matcher.spec.ts index 394bc89f4..f61f7ce86 100644 --- a/libs/services/src/lib/tmdb/tmdb-matcher.spec.ts +++ b/libs/services/src/lib/tmdb/tmdb-matcher.spec.ts @@ -78,10 +78,10 @@ describe('extractYear', () => { describe('lookup keys', () => { it('builds stable search and details keys', () => { expect(buildSearchLookupKey('the matrix', 1999)).toBe( - 'title:the matrix|year:1999' + 'title:the matrix|year:1999|v2' ); expect(buildSearchLookupKey('the matrix', null)).toBe( - 'title:the matrix|year:' + 'title:the matrix|year:|v2' ); expect(buildDetailsLookupKey(603)).toBe('id:603|v2'); }); diff --git a/libs/services/src/lib/tmdb/tmdb-matcher.ts b/libs/services/src/lib/tmdb/tmdb-matcher.ts index a07d2e16f..2c60cca56 100644 --- a/libs/services/src/lib/tmdb/tmdb-matcher.ts +++ b/libs/services/src/lib/tmdb/tmdb-matcher.ts @@ -70,7 +70,10 @@ export function buildSearchLookupKey( normalizedTitle: string, year: number | null ): string { - return `title:${normalizedTitle}|year:${year ?? ''}`; + // v2: normalizeTitleKeys learned to strip appended language/quality + // tags; the version suffix invalidates cached (incl. negative) match + // resolutions keyed on the old polluted titles + return `title:${normalizedTitle}|year:${year ?? ''}|v2`; } export function buildDetailsLookupKey(tmdbId: number): string { diff --git a/libs/shared/interfaces/src/lib/title-normalization.util.spec.ts b/libs/shared/interfaces/src/lib/title-normalization.util.spec.ts index 3c60b3d8a..2cc08901e 100644 --- a/libs/shared/interfaces/src/lib/title-normalization.util.spec.ts +++ b/libs/shared/interfaces/src/lib/title-normalization.util.spec.ts @@ -56,6 +56,127 @@ describe('normalizeTitleKeys', () => { }); }); +describe('provider tag stripping', () => { + it('strips wrapped pipe tags', () => { + expect(normalizeTitle('|DE| ARD')).toBe('ard'); + expect(normalizeTitle('|MULTI| Fallout - 4K')).toBe('fallout'); + expect(normalizeTitle('|EXYU| The Pitt')).toBe('the pitt'); + }); + + it('strips long and compound leading tags', () => { + expect(normalizeTitle('EXYU| Fallout')).toBe('fallout'); + expect(normalizeTitle('MULTI| Breaking Bad')).toBe('breaking bad'); + expect(normalizeTitle('4K-DE - The Pitt (2025) (US)')).toBe( + 'the pitt' + ); + expect(normalizeTitle('AR-SUBS - Fallout (2024) (US)')).toBe( + 'fallout' + ); + expect(normalizeTitle('4K-OSN+ - The Last of Us (2023)')).toBe( + 'the last of us' + ); + }); + + it('never treats numeric fragments as leading tags', () => { + expect(normalizeTitle('1917 - Behind the Lines')).toBe( + '1917 behind the lines' + ); + }); + + it('strips underscore and double-dash suffix tags', () => { + expect(normalizeTitle('Fallout_eng')).toBe('fallout'); + expect(normalizeTitle('Breaking Bad (US)_msub')).toBe('breaking bad'); + expect(normalizeTitle('The Pitt (2025)_sub')).toBe('the pitt'); + expect(normalizeTitle('The Last of Us--esp')).toBe('the last of us'); + }); + + it('keeps underscore-as-space titles intact', () => { + expect(normalizeTitle('The_Last_of_Us')).toBe('the last of us'); + }); + + it('strips joined dash tags only for case-uniform vocabulary tokens', () => { + expect(normalizeTitle('Breaking Bad-eng')).toBe('breaking bad'); + expect(normalizeTitle('The Last of Us-DE')).toBe('the last of us'); + expect(normalizeTitle('The Pitt (2025)-it')).toBe('the pitt'); + expect(normalizeTitle('Spider-Man')).toBe('spider man'); + expect(normalizeTitle('Kick-It')).toBe('kick it'); + }); + + it('strips bare trailing UPPERCASE vocabulary tags', () => { + expect(normalizeTitle('The Pitt (2025) DE')).toBe('the pitt'); + expect(normalizeTitle('Breaking Bad ES')).toBe('breaking bad'); + expect(normalizeTitle('EN| Breaking Bad SUB')).toBe('breaking bad'); + expect(normalizeTitle('The Last of Us (2023) AF')).toBe( + 'the last of us' + ); + }); + + it('never strips trailing tags that could be real endings', () => { + expect(normalizeTitle('Rocky II')).toBe('rocky ii'); + expect(normalizeTitle('Made in USA')).toBe('made in usa'); + expect(normalizeTitle('NCIS: LA')).toBe('ncis la'); + expect(normalizeTitle('Making It')).toBe('making it'); + expect(normalizeTitle('THE LAST OF US')).toBe('the last of us'); + }); + + const pittCorpus = [ + 'The Pitt (2025)_sub', 'The Pitt (2025)-it', 'The Pitt (2025)', + 'The Pitt (Hindi)', 'The Pitt (2025) 4K', 'The Pitt (2025) DE', + 'The Pitt (2025) ES', 'The Pitt (2025) FR', 'The Pitt (2025)_eng', + 'The Pitt [MULTI-SUB]', 'The Pitt (2025) (4K DV)', 'GR - The Pitt', + '4K-DE - The Pitt (2025) (US)', '4K-TR - The Pitt (2025) (US)', + 'AR-SUBS - The Pitt (2025) (US)', 'DE - The Pitt (2025) (US)', + 'ALB| The Pitt', 'EXYU| The Pitt', '|ALB| The Pitt', '|DE| The Pitt', + ]; + + const falloutCorpus = [ + 'Fallout', 'DE - Fallout (2024)', 'Fallout (2024) - 4K', + 'Fallout (2024) FR-EN', 'Fallout (2024) Multi', 'Fallout (2024)_fr', + 'Fallout_esp', 'Fallout (4K)', '4K-AMZ - Fallout (2024) (US)', + 'AL - Fallout (2024)', 'AMZ - Fallout (2024) (US)', + 'AR-DE - Fallout (US)', 'LA - Fallout', 'EN| Fallout - 4K', + 'MULTI| Fallout - 4K', 'Fallout ( مدبلج )', 'Fallout (Telugu)', + '|EN| Fallout - 4K', '|MULTI| Fallout', '|TR| Fallout', + ]; + + const lastOfUsCorpus = [ + 'The Last of Us', 'The Last Of Us', 'The Last of Us (2023) 4K', + 'The Last of Us (2023) AF', 'The Last of Us_tr', + 'The Last of Us--esp', 'The Last of Us-DE', 'The Last of Us-esp', + 'The Last of Us [L]', 'The Last of Us ( HD )', + '4K-OSN+ - The Last of Us (2023)', 'IS - The Last of Us (2023) (US)', + 'RU - The Last of Us', 'ALB| The Last of Us', + ]; + + const breakingBadCorpus = [ + 'Breaking Bad', 'Breaking Bad (2008)_fr', 'Breaking Bad (US)_msub', + 'Breaking Bad_it', 'Breaking Bad-DE', 'Breaking Bad-eng', + 'Breaking Bad ( عائلي )', 'Breaking Bad (Pure)', + 'Breaking Bad - Multi', 'Breaking Bad ES', 'AR-DE - Breaking Bad', + 'EN| Breaking Bad SUB', 'MULTI| Breaking Bad', 'AR| Breaking Bad', + ]; + + it.each([ + ['the pitt', pittCorpus], + ['fallout', falloutCorpus], + ['the last of us', lastOfUsCorpus], + ['breaking bad', breakingBadCorpus], + ])( + 'normalizes every observed provider variant of "%s" to one key', + (expected, corpus) => { + for (const name of corpus) { + expect(normalizeTitleKeys(name).base).toBe(expected); + } + } + ); + + it('keeps localized subtitles (indistinguishable from real ones)', () => { + expect(normalizeTitle('Breaking Bad: A Química do Mal')).toBe( + 'breaking bad a quimica do mal' + ); + }); +}); + describe('titleYearsCompatible', () => { it('accepts unknown years and ±1 tolerance', () => { expect(titleYearsCompatible(null, 2049)).toBe(true); diff --git a/libs/shared/interfaces/src/lib/title-normalization.util.ts b/libs/shared/interfaces/src/lib/title-normalization.util.ts index 3b746683b..883dd0d7a 100644 --- a/libs/shared/interfaces/src/lib/title-normalization.util.ts +++ b/libs/shared/interfaces/src/lib/title-normalization.util.ts @@ -28,11 +28,102 @@ const QUALITY_TAGS = new Set([ ]); /** - * Leading channel/language prefix like "EN - ", "DE| ", "FR: ". - * UPPERCASE-only on purpose: a case-insensitive match would amputate real - * title words ("It: Chapter Two" → "Chapter Two"). + * Wrapped tag at the very start of a provider title: "|DE| ARD", + * "|MULTI| Fallout". The lookahead requires a letter in the tag so a + * numeric fragment can never be treated as one. */ -const LANGUAGE_PREFIX = /^[A-Z]{2,3}\s*[-|:]\s+/; +const WRAPPED_TAG_PREFIX = /^\s*\|(?=[0-9+]*[A-Z])[A-Z0-9+]{2,5}\|\s*/; + +/** + * Leading channel/language prefix like "EN - ", "DE| ", "FR: ", including + * compound provider/quality forms ("4K-DE - ", "AR-SUBS - ", "4K-OSN+ - ") + * and longer tags ("EXYU| ", "MULTI| "). + * UPPERCASE-only on purpose: a case-insensitive match would amputate real + * title words ("It: Chapter Two" → "Chapter Two"). Every segment must + * contain a letter so numeric titles ("1917 - ...") are never tags. + * Colon separators stay limited to 2–3 chars — longer acronyms before a + * colon are franchise titles ("NCIS: LA"), not tags. + */ +const LANGUAGE_PREFIX = + /^(?:(?=[0-9+]*[A-Z])[A-Z0-9+]{2,5}(?:-(?=[0-9+]*[A-Z])[A-Z0-9+]{2,6}){0,2}\s*[-|]\s+|(?=[0-9+]*[A-Z])[A-Z0-9+]{2,3}\s*:\s+)/; + +/** + * Curated whitelist for TRAILING language/subtitle tags ("Fallout_eng", + * "Breaking Bad-DE", "The Pitt (2025) ES"). Trailing stripping must be + * vocabulary-gated: a pattern-only rule would amputate real endings — + * roman numerals ("Rocky II"), acronyms ("Made in USA"), franchise + * suffixes ("NCIS: LA"). US/USA/UK/LA are deliberately absent. + */ +const TRAILING_TAG_VOCABULARY = new Set([ + 'AF', 'AL', 'ALB', 'AR', 'BY', 'DE', 'DUB', 'EN', 'ENG', 'ES', 'ESP', + 'EXYU', 'FR', 'FRA', 'GE', 'GR', 'HU', 'IN', 'IR', 'IS', 'IT', 'ITA', + 'KA', 'KU', 'LAT', 'ML', 'MSUB', 'MULTI', 'NL', 'PL', 'PT', 'RO', 'RU', + 'SC', 'SE', 'SUB', 'SUBS', 'SW', 'TA', 'TL', 'TR', 'TUR', +]); + +const DOUBLE_DASH_SUFFIX = /[-–]{2}[A-Za-z]{2,5}\s*$/; +const UNDERSCORE_SUFFIX = /_[A-Za-z]{2,5}\s*$/; +const JOINED_DASH_SUFFIX = /-([A-Za-z]{2,5})\s*$/; +const TRAILING_TAG_SUFFIX = /\s([A-Z]{2,5})\s*$/; + +/** Tag tokens are case-uniform; real title words are Capitalized. */ +function isCaseUniform(token: string): boolean { + return token === token.toLowerCase() || token === token.toUpperCase(); +} + +/** + * Strip appended language/subtitle tags. Runs BEFORE lowercasing — + * casing is the main false-positive guard: ALL-CAPS titles carry no + * casing signal ("THE LAST OF US" must keep its "US"), so caps-gated + * rules are skipped for them, and Capitalized endings ("Making It", + * "Kick-It") never look like tags. Compound tags ("FR-EN") shed one + * token per pass, so stripping repeats to a fixpoint. + */ +function stripTrailingTags(value: string): string { + let result = value; + for (let pass = 0; pass < 3; pass++) { + const next = stripTrailingTagOnce(result); + if (next === result) break; + result = next; + } + return result; +} + +function stripTrailingTagOnce(value: string): string { + const result = value.trimEnd(); + const hasLowercase = /\p{Ll}/u.test(result); + + // "The Last of Us--esp": no real title contains a double dash. + if (DOUBLE_DASH_SUFFIX.test(result)) { + return result.replace(DOUBLE_DASH_SUFFIX, '').trimEnd(); + } + + // "Fallout_eng" — but not "The_Last_of_Us", where underscores are + // space substitutes (only strip when this is the sole underscore). + if ( + UNDERSCORE_SUFFIX.test(result) && + result.indexOf('_') === result.lastIndexOf('_') + ) { + return result.replace(UNDERSCORE_SUFFIX, '').trimEnd(); + } + + const joined = result.match(JOINED_DASH_SUFFIX); + if ( + joined && + TRAILING_TAG_VOCABULARY.has(joined[1].toUpperCase()) && + isCaseUniform(joined[1]) && + (joined[1] !== joined[1].toUpperCase() || hasLowercase) + ) { + return result.replace(JOINED_DASH_SUFFIX, '').trimEnd(); + } + + const trailing = result.match(TRAILING_TAG_SUFFIX); + if (trailing && hasLowercase && TRAILING_TAG_VOCABULARY.has(trailing[1])) { + return result.replace(TRAILING_TAG_SUFFIX, '').trimEnd(); + } + + return result; +} const YEAR_PATTERN = /\b(19\d{2}|20\d{2})\b/; @@ -74,11 +165,14 @@ export function normalizeTitleKeys( return { exact: '', base: '', trailingYear: null }; } - const cleaned = raw - // Inner classes exclude the opening delimiter too, so runaway - // inputs like "[[[[[..." backtrack linearly (CodeQL js/polynomial-redos) - .replace(/\[[^\][]*\]|\([^()]*\)|\{[^{}]*\}/g, ' ') - .replace(LANGUAGE_PREFIX, '') + const cleaned = stripTrailingTags( + raw + .replace(WRAPPED_TAG_PREFIX, '') + // Inner classes exclude the opening delimiter too, so runaway + // inputs like "[[[[[..." backtrack linearly (CodeQL js/polynomial-redos) + .replace(/\[[^\][]*\]|\([^()]*\)|\{[^{}]*\}/g, ' ') + .replace(LANGUAGE_PREFIX, '') + ) .normalize('NFD') .replace(/[\u0300-\u036F]/g, '') .toLowerCase() diff --git a/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.spec.ts b/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.spec.ts index abbf90037..a91d4f9ee 100644 --- a/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.spec.ts +++ b/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.spec.ts @@ -43,6 +43,27 @@ describe('stripCountryPrefix', () => { expect(stripCountryPrefix('US: CNN')).toBe('CNN'); }); + it('strips compound quality/provider tags', () => { + expect(stripCountryPrefix('4K-DE - The Pitt (2025)')).toBe( + 'The Pitt (2025)' + ); + expect(stripCountryPrefix('AR-SUBS - Fallout')).toBe('Fallout'); + expect(stripCountryPrefix('4K-OSN+ - The Last of Us')).toBe( + 'The Last of Us' + ); + }); + + it('strips longer uppercase tags', () => { + expect(stripCountryPrefix('EXYU - News')).toBe('News'); + expect(stripCountryPrefix('MULTI - Movies')).toBe('Movies'); + }); + + it('never treats numeric fragments as tags', () => { + expect(stripCountryPrefix('1917 - Documentary')).toBe( + '1917 - Documentary' + ); + }); + it('only strips the first tag segment', () => { expect(stripCountryPrefix('ES - A3 - Sports')).toBe('A3 - Sports'); }); diff --git a/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.ts b/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.ts index f1213996f..8d382f82a 100644 --- a/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.ts +++ b/libs/shared/m3u-utils/src/lib/strip-country-prefix.util.ts @@ -3,9 +3,18 @@ * country/group tag — they routinely appear inside real channel names * ("Sky - Sports F1", "Discovery - Science"). Pipes are conventionally * used only as tag separators, so they always count. + * + * A tag is 1–3 short UPPERCASE alphanumeric segments ("US", "4K-DE", + * "AR-SUBS", "OSN+"). Every segment must contain a letter so numeric + * titles ("1917 - ...") are never treated as tags. Colon separators stay + * limited to plain 2–3 char tags — longer acronyms before a colon are + * franchise titles ("NCIS: LA"), not tags. */ -const DASH_OR_COLON_SEPARATORS = [' - ', '- ', ' -', ': ']; -const TAG_PREFIX_PATTERN = /^[A-Z0-9]{2,3}$/; +const DASH_SEPARATORS = [' - ', '- ', ' -']; +const COLON_SEPARATOR = ': '; +const TAG_PREFIX_PATTERN = + /^(?=[0-9+]*[A-Z])[A-Z0-9+]{2,5}(?:-(?=[0-9+]*[A-Z])[A-Z0-9+]{2,6}){0,2}$/; +const COLON_TAG_PATTERN = /^(?=[0-9+]*[A-Z])[A-Z0-9+]{2,3}$/; interface SeparatorMatch { index: number; @@ -68,10 +77,14 @@ function findTagSeparator(name: string): SeparatorMatch | null { best = { index: pipeIndex, length: 1 }; } - for (const separator of DASH_OR_COLON_SEPARATORS) { + for (const separator of [...DASH_SEPARATORS, COLON_SEPARATOR]) { const index = name.indexOf(separator); if (index === -1 || (best && best.index <= index)) continue; - if (!TAG_PREFIX_PATTERN.test(name.slice(0, index).trim())) continue; + const pattern = + separator === COLON_SEPARATOR + ? COLON_TAG_PATTERN + : TAG_PREFIX_PATTERN; + if (!pattern.test(name.slice(0, index).trim())) continue; best = { index, length: separator.length }; }