From 78df3e7dbb6d7b189b3d2887459c36fc6d80ad89 Mon Sep 17 00:00:00 2001 From: 4gray <4gray@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:14:59 +0200 Subject: [PATCH] fix(portals): match Greek titles whichever sigma the provider typed (#1310) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greek Σ has two lowercase forms — medial σ and word-final ς — and neither the candidate query nor the confirmation treated them as one letter. The GLOB scan built each character's class from a one-way reach that only arrived at ς when it started from ς, so a request for "ΑΣ" never admitted a stored "Ας". Classes are now built from a fold group — every character sharing an uppercase form — derived by scanning the cased ranges at module load the way ACCENTED_BY_BASE already is. It generalises past sigma on its own: dotless ı folds with i, long ſ with s, historic Cyrillic letterforms with В Д О С Т Ъ Ѣ. Only the 24 groups of 767 that a per-character fold would miss are kept. Admitting the row was only half of it. normalizeTitleKeys then compared "ασ" against "ας" and discarded it, because toLowerCase picks the sigma form by position. Both SQL tiers already folded them together — SQLite's trigram tokenizer does full Unicode folding natively, unlike LOWER() — so the JS confirmation was the only tier that did not, making this a pre-existing gap on the FTS path as well. Normalization now rewrites ς to σ after lowercasing, which is what Unicode case folding does. Guards unchanged: a case mapping that changes length (ß → SS, İ) or a GLOB metacharacter still returns null rather than a partial pattern. --- .changes/portals-greek-sigma-matching.md | 10 + .../operations/title-sources-matching.spec.ts | 36 ++++ .../title-sources.operations.spec.ts | 16 +- .../operations/title-token-glob.spec.ts | 65 ++++++- .../database/operations/title-token-glob.ts | 101 +++++++++-- docs/architecture/vod-multi-source.md | 171 ++++++++++-------- .../src/lib/title-normalization.util.spec.ts | 19 ++ .../src/lib/title-normalization.util.ts | 9 + 8 files changed, 332 insertions(+), 95 deletions(-) create mode 100644 .changes/portals-greek-sigma-matching.md diff --git a/.changes/portals-greek-sigma-matching.md b/.changes/portals-greek-sigma-matching.md new file mode 100644 index 000000000..e61fa18c9 --- /dev/null +++ b/.changes/portals-greek-sigma-matching.md @@ -0,0 +1,10 @@ +--- +type: fix +area: portals +--- + +Greek titles now match no matter which sigma the provider typed. A word-final +`ς` and a medial `σ` at the end of a word are treated as the same letter, so +finding a film in your other playlists, the "Similar" rail, actor filmographies +and TMDB enrichment no longer miss a Greek movie purely because two catalogues +spelled its last letter differently. diff --git a/apps/electron-backend/src/app/database/operations/title-sources-matching.spec.ts b/apps/electron-backend/src/app/database/operations/title-sources-matching.spec.ts index 4bd780071..18af42c84 100644 --- a/apps/electron-backend/src/app/database/operations/title-sources-matching.spec.ts +++ b/apps/electron-backend/src/app/database/operations/title-sources-matching.spec.ts @@ -139,6 +139,42 @@ describe('title-sources.operations — confirmation and scoping', () => { expect(matches).toHaveLength(1); expect(matches[0].matchConfidence).toBe('fuzzy'); }); + + it('confirms a Greek title whichever sigma either side used', async () => { + // End to end, not just at the GLOB tier: the widened character + // class admits the row, and the confirmation has to agree that it + // is the same film. Greek Σ has two lowercase forms and + // `toLowerCase` picks by position, so the request and the stored + // title can disagree without either being wrong. + const spellings = ['ΑΣ', 'Ας', 'ασ', 'ας']; + + for (const requested of spellings) { + const rows = spellings.map((stored, index) => ({ + ...duneRow, + title: stored, + xtream_id: 600 + index, + })); + const { db } = createDbMock(rows); + + const matches = await findTitleSources(db, { + title: requested, + }); + + expect( + `${requested} confirms ${matches.length} of ${rows.length}` + ).toBe(`${requested} confirms ${rows.length} of ${rows.length}`); + } + }); + + it('still rejects a different Greek word', async () => { + // The sigma fold must not turn into "any short Greek title + // matches any other". + const { db } = createDbMock([{ ...duneRow, title: 'ΑΝ' }]); + + await expect( + findTitleSources(db, { title: 'ΑΣ' }) + ).resolves.toEqual([]); + }); }); describe('playlist scoping and duplicates', () => { diff --git a/apps/electron-backend/src/app/database/operations/title-sources.operations.spec.ts b/apps/electron-backend/src/app/database/operations/title-sources.operations.spec.ts index 8a7cc29ca..1cf2da513 100644 --- a/apps/electron-backend/src/app/database/operations/title-sources.operations.spec.ts +++ b/apps/electron-backend/src/app/database/operations/title-sources.operations.spec.ts @@ -37,7 +37,10 @@ describe('title-sources.operations', () => { expect(all).toHaveBeenCalledTimes(1); expect(result).toHaveLength(1); expect(result[0]).toEqual( - expect.objectContaining({ xtreamId: 77, matchConfidence: 'exact' }) + expect.objectContaining({ + xtreamId: 77, + matchConfidence: 'exact', + }) ); }); @@ -83,7 +86,16 @@ describe('title-sources.operations', () => { // code points — so the case is folded in JavaScript and handed // over one class per character. Without this a title stored in // any case other than the two below is simply never found. - expect(scanQuery.params).toContain('*[Оо][нН]*'); + // Asserted by shape rather than by literal: the class also carries + // every other character folding to the same uppercase, so pinning + // the exact members here would make this spec fail for a change + // that only ever widens the fold. What it owes is a class per + // character holding BOTH cases of that character. + expect(scanQuery.params).toContainEqual( + expect.stringMatching( + /^\*(\[[^\]]*О[^\]]*о[^\]]*\])(\[[^\]]*н[^\]]*Н[^\]]*\])\*$/ + ) + ); expect(scanQuery.sql).toContain('instr'); // Both the folded and the as-typed form, since the class is built // from the raw token and cannot cover a diacritic difference. diff --git a/apps/electron-backend/src/app/database/operations/title-token-glob.spec.ts b/apps/electron-backend/src/app/database/operations/title-token-glob.spec.ts index de9a80922..308792f84 100644 --- a/apps/electron-backend/src/app/database/operations/title-token-glob.spec.ts +++ b/apps/electron-backend/src/app/database/operations/title-token-glob.spec.ts @@ -40,7 +40,10 @@ const describeWithSqlite = hasSqlite() ? describe : describe.skip; describe('caseInsensitiveGlobPattern', () => { it('builds a case class for each cased character', () => { - expect(caseInsensitiveGlobPattern('Он')).toBe('*[Оо][нН]*'); + // ᲂ is U+1C82, the historic narrow o, which also uppercases to О. It + // is in the class for the same reason ς is in sigma's: the fold groups + // every character sharing an uppercase, rather than listing pairs. + expect(caseInsensitiveGlobPattern('Он')).toBe('*[Ооᲂ][нН]*'); }); it('leaves uncased characters alone', () => { @@ -49,18 +52,24 @@ describe('caseInsensitiveGlobPattern', () => { it('covers a letter with two lowercase spellings', () => { // Greek Σ lowercases to σ, but a word-final sigma is written ς and is - // just as much a lowercase of it. Going back down from the uppercase - // form reaches the spelling the character in hand does not name. + // just as much a lowercase of it. expect(caseInsensitiveGlobPattern('ς')).toBe('*[ςΣσ]*'); }); - it('reaches the second spelling in one direction only', () => { - // σ → Σ → σ never arrives at ς, so a request spelled with a medial - // sigma does not match a stored final one. Left as is: ς is only ever - // correct at the end of a word, which is exactly where the request's - // own last character sits, so the pair that occurs in real titles is - // the one above. Closing the other direction needs a fold table. - expect(caseInsensitiveGlobPattern('σ')).toBe('*[σΣ]*'); + it('reaches the second spelling from every spelling', () => { + // Case mapping alone only arrives at ς when it starts from ς: both + // σ → Σ → σ and Σ → σ miss it, so a request for "ΑΣ" or "ασ" did not + // match a stored "Ας". The fold group is keyed on the shared uppercase + // instead, so every spelling reaches every other one. + expect(caseInsensitiveGlobPattern('σ')).toBe('*[σΣς]*'); + expect(caseInsensitiveGlobPattern('Σ')).toBe('*[Σσς]*'); + }); + + it('groups characters beyond sigma', () => { + // The group is derived from the code point ranges, not tabulated, so + // the dotless ı — which uppercases to I like i does — is covered by + // the same rule without anyone having to think of it. + expect(caseInsensitiveGlobPattern('i')).toBe('*[iIı]*'); }); it('refuses a token holding a GLOB metacharacter', () => { @@ -120,6 +129,42 @@ describeWithSqlite('caseInsensitiveGlobPattern against SQLite', () => { expect(sqliteGlob('οσ', pattern)).toBe(true); }); + it('matches every sigma spelling from every other', () => { + // The bug this closes: a request for "ΑΣ" did not match a stored + // "Ας", because case mapping reaches the word-final ς only when it + // starts from ς. Upper and lower case are both requests a user can + // type, so the reach has to be symmetric — asserted as the full + // matrix rather than the one direction that used to work. + const spellings = ['ΑΣ', 'Ας', 'ασ', 'ας']; + const matrix = (found: (a: string, b: string) => boolean) => + spellings.flatMap((requested) => + spellings.map( + (stored) => + `${requested} finds ${stored}: ${found(requested, stored)}` + ) + ); + + // One assertion over the whole matrix, so a regression names the exact + // pair that stopped matching instead of just "false". + expect( + matrix((requested, stored) => + sqliteGlob( + stored, + caseInsensitiveGlobPattern(requested) as string + ) + ) + ).toEqual(matrix(() => true)); + }); + + it('still tells two different Greek words apart', () => { + // The fold widens each class; it must not make every Greek word match + // every other. Confirms the widening did not turn into a wildcard. + const pattern = caseInsensitiveGlobPattern('ΑΣ') as string; + + expect(sqliteGlob('ΑΝ', pattern)).toBe(false); + expect(sqliteGlob('ΟΣ', pattern)).toBe(false); + }); + it('finds an accented title from its folded ASCII token', () => { // The token has already had its own diacritics folded, so "ca" is what // a search for "Ça" arrives as. Without the accented forms this diff --git a/apps/electron-backend/src/app/database/operations/title-token-glob.ts b/apps/electron-backend/src/app/database/operations/title-token-glob.ts index 299743ab6..3dd106b08 100644 --- a/apps/electron-backend/src/app/database/operations/title-token-glob.ts +++ b/apps/electron-backend/src/app/database/operations/title-token-glob.ts @@ -42,6 +42,84 @@ const ACCENTED_BY_BASE = ((): Map => { return byBase; })(); +/** + * The forms a class MUST carry for one character: its own case pair, plus the + * uppercase form's own lowercase. + * + * Kept as one function because it is also what decides which fold groups below + * are worth storing — a group that adds nothing to this set is dead weight. + */ +function caseForms(character: string): Set { + const upper = character.toUpperCase(); + return new Set([ + character, + character.toLowerCase(), + upper, + upper.toLowerCase(), + ]); +} + +/** + * Every character that folds to the same uppercase, keyed by it. + * + * Derived by scanning the cased ranges once, exactly like `ACCENTED_BY_BASE`, + * rather than tabulating the pairs by hand. A letter can have more lowercase + * spellings than case mapping alone will reach from any one of them: Greek `Σ` + * lowercases to `σ`, but a word-final sigma is written `ς` and is just as much + * a lowercase of it, and neither `Σ` nor `σ` can arrive at `ς`. Grouping by the + * shared uppercase reaches every spelling from every other one, and does it for + * the whole alphabet instead of the one pair someone thought to write down. + */ +const CASE_FOLD_GROUPS = ((): Map => { + const byUpper = new Map(); + // Wide enough to hold the cased scripts that appear in titles; the filter + // below decides what actually earns a place. Greek Extended is in for + // U+1FBE, whose uppercase is a plain Greek iota, and Cyrillic Extended-C + // for the historic letterforms that fold onto В Д О С Т Ъ Ѣ. + const ranges: ReadonlyArray<[number, number]> = [ + [0x0041, 0x024f], + [0x0370, 0x03ff], + [0x0400, 0x04ff], + [0x1c80, 0x1c8f], + [0x1e00, 0x1fff], + ]; + for (const [first, last] of ranges) { + for (let codePoint = first; codePoint <= last; codePoint += 1) { + const character = String.fromCodePoint(codePoint); + const upper = character.toUpperCase(); + // Both of these are really in the ranges above: 89 characters + // uppercase to more than one code point (`ß` → `SS`, `ǰ`, `ΐ`), + // which names no single class member and cannot key the map, and + // `[`, `]` and `^` are letters' neighbours in Basic Latin. The + // map's whole value is that anything in it is safe to put in a + // class, so the filtering happens here rather than at every use. + if ( + [...upper].length !== 1 || + GLOB_METACHARACTERS.test(character) + ) { + continue; + } + + const members = byUpper.get(upper) ?? []; + members.push(character); + byUpper.set(upper, members); + } + } + + // Almost every group is just {upper, lower}, which `caseForms` already + // returns from either one. Keeping only the groups that reach a spelling + // it cannot leaves 24 entries out of 767 — and says so as a rule, so + // widening a range later cannot quietly refill the map with pairs that + // were never needed. + return new Map( + [...byUpper].filter(([, members]) => + members.some((member) => + members.some((from) => !caseForms(from).has(member)) + ) + ) + ); +})(); + /** Files one character under the plain ASCII letter it decomposes to. */ function addAccentedForm( byBase: Map, @@ -91,17 +169,7 @@ export function caseInsensitiveGlobBody( let pattern = ''; for (const character of token) { - const upper = character.toUpperCase(); - // Going back down from the uppercase form catches letters with more - // than one lowercase spelling: Greek Σ lowercases to σ, but ς is an - // equally valid lowercase of it, and a class built only from the - // character in hand would know just one of the two. - const forms = new Set([ - character, - character.toLowerCase(), - upper, - upper.toLowerCase(), - ]); + const forms = caseForms(character); // These are the forms the pattern MUST carry, so one it cannot express // means there is no honest pattern to build. @@ -114,6 +182,17 @@ export function caseInsensitiveGlobBody( return null; } + // Every character sharing this uppercase, which is what covers a + // letter spelled more than one way in lower case. Case mapping alone + // only reaches the second spelling from the second spelling: `ς` finds + // `Σ` and `σ`, while a request for `ΑΣ` or `ασ` never arrives at `ς` + // and missed a stored `Ας`. Every member of the group is a single + // code point and class-safe by construction. + for (const member of CASE_FOLD_GROUPS.get(character.toUpperCase()) ?? + []) { + forms.add(member); + } + // A normalized token has already had its diacritics folded away, so // "ca" has to be able to find a stored "Ça" — otherwise discovery runs // one way only, depending on which playlist happens to be open. diff --git a/docs/architecture/vod-multi-source.md b/docs/architecture/vod-multi-source.md index 423aac3ec..d33efbace 100644 --- a/docs/architecture/vod-multi-source.md +++ b/docs/architecture/vod-multi-source.md @@ -9,14 +9,14 @@ current source is dead, serves an unsupported codec, or buffers badly. ## Scope (v1) -| | | -|---|---| -| Source types | **Xtream ↔ Xtream only** | -| Content | Movies only — series are not offered a source chip | -| Environment | **Electron only** — the chip renders nothing in the PWA | +| | | +| ------------- | ------------------------------------------------------------------------------------------------- | +| Source types | **Xtream ↔ Xtream only** | +| Content | Movies only — series are not offered a source chip | +| Environment | **Electron only** — the chip renders nothing in the PWA | | Auto-failover | Opt-in, **off by default** (`Settings.vodAutoFailover`); offered only on the built-in web players | -| Pin scope | Per movie (a global portal priority is out of scope) | -| Stream probe | HEAD → reachable + latency, sent with the playlist's own playback headers. **No codec probing** | +| Pin scope | Per movie (a global portal priority is out of scope) | +| Stream probe | HEAD → reachable + latency, sent with the playlist's own playback headers. **No codec probing** | Stalker never reaches the `content` table (it would need a live authenticated `get_ordered_list&search=` per portal), and M3U playlists are stored as a JSON @@ -24,7 +24,7 @@ blob whose search path forces `content_type: 'live'`. Both are additive later without changing the contracts — `VodSourceCandidate.portalType` already carries `'xtream' | 'stalker' | 'm3u'` and discovery sits behind a service interface. -A probe answer is cached per *request*, not per URL: two playlists can share a +A probe answer is cached per _request_, not per URL: two playlists can share a stream URL and require different headers, and one of them answering 403 says nothing about the other. @@ -39,12 +39,12 @@ This is the part to preserve if anything here is refactored. Every metadata value carries **where it came from**: -| provenance | produced by | rendered as | -|---|---|---| -| `api` | `get_vod_info` — container, codec, audio, dimensions | plain tag | -| `parsed` | regex over the title/filename | tag prefixed `~`, warn colour | -| `probe` | HEAD → reachable + latency (retried as a ranged GET when the server answers 405/501, since plenty serve media over GET while refusing HEAD) | `ok` / `fail` status tag | -| *absent* | — | **no tag at all** + a `check` chip | +| provenance | produced by | rendered as | +| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------- | +| `api` | `get_vod_info` — container, codec, audio, dimensions | plain tag | +| `parsed` | regex over the title/filename | tag prefixed `~`, warn colour | +| `probe` | HEAD → reachable + latency (retried as a ranged GET when the server answers 405/501, since plenty serve media over GET while refusing HEAD) | `ok` / `fail` status tag | +| _absent_ | — | **no tag at all** + a `check` chip | Three rules follow, and each is enforced in code rather than by convention: @@ -57,7 +57,7 @@ Three rules follow, and each is enforced in code rather than by convention: (contacted and refused) from `unknown` (timed out, blocked by the redirect policy, or no probe capability). A probe returning HTTP status `0` maps to `unknown`; the UI keeps offering a check and never shows the source as dead. -3. **Empty beats wrong.** Quality is derived from pixel *width*, because +3. **Empty beats wrong.** Quality is derived from pixel _width_, because letterboxed masters are cropped vertically — a 2.39:1 1080p film is 1920×800, and 800 alone is indistinguishable from a 1280×800 encode. With no width, a height is trusted only within 5% of a standard frame height; otherwise no tag @@ -68,13 +68,13 @@ Three rules follow, and each is enforced in code rather than by convention: unrecognised shape returns nothing rather than a label that would be published as an `api` fact. - A known height has to be consistent with the width **on every tier**, - ranges included. Cropping only ever *removes* lines, so a shorter frame is a - letterboxed master of that format, while a taller one is a different shape: - 640×480 against 640×360 below, and 1440×1080 (anamorphic 1080) or 1600×900 - against the 720p range above. All of those get no tag. Bucketing HD widths is - otherwise sound — the standard widths really are far apart — but only once - the height is allowed to veto the answer. + A known height has to be consistent with the width **on every tier**, + ranges included. Cropping only ever _removes_ lines, so a shorter frame is a + letterboxed master of that format, while a taller one is a different shape: + 640×480 against 640×360 below, and 1440×1080 (anamorphic 1080) or 1600×900 + against the 720p range above. All of those get no tag. Bucketing HD widths is + otherwise sound — the standard widths really are far apart — but only once + the height is allowed to veto the answer. Provenance is per-field and changes over time: at discovery a row has only `parsed` tags, because the `content` table stores no container, codec or audio. @@ -102,17 +102,17 @@ and so its ranking logic is testable without TestBed. ### Ownership -| Concern | Location | -|---|---| -| DTOs, match key | `libs/shared/interfaces/src/lib/vod-source*.ts` | -| Pin table | `libs/shared/database/src/lib/vod-source-pins.schema.ts` | -| Discovery SQL | `apps/electron-backend/.../operations/title-sources.operations.ts` | -| Pin CRUD | `apps/electron-backend/.../operations/vod-source-pin.operations.ts` | -| Probe handler | `apps/electron-backend/src/app/events/stream-probe.ts` | -| Discovery / resolve / rank | `libs/portal/shared/data-access/src/lib/multi-source/` | -| Probe + pin clients | `libs/services/src/lib/{stream-probe,vod-source-pin}.service.ts` | -| Row / popover / chip | `libs/ui/components/src/lib/vod-sources/` | -| Page wiring | `libs/portal/xtream/feature/src/lib/vod-details/vod-multi-source-*.ts` | +| Concern | Location | +| -------------------------- | ---------------------------------------------------------------------- | +| DTOs, match key | `libs/shared/interfaces/src/lib/vod-source*.ts` | +| Pin table | `libs/shared/database/src/lib/vod-source-pins.schema.ts` | +| Discovery SQL | `apps/electron-backend/.../operations/title-sources.operations.ts` | +| Pin CRUD | `apps/electron-backend/.../operations/vod-source-pin.operations.ts` | +| Probe handler | `apps/electron-backend/src/app/events/stream-probe.ts` | +| Discovery / resolve / rank | `libs/portal/shared/data-access/src/lib/multi-source/` | +| Probe + pin clients | `libs/services/src/lib/{stream-probe,vod-source-pin}.service.ts` | +| Row / popover / chip | `libs/ui/components/src/lib/vod-sources/` | +| Page wiring | `libs/portal/xtream/feature/src/lib/vod-details/vod-multi-source-*.ts` | ### One picker, two places, two counts @@ -139,11 +139,11 @@ otherwise `title:{normalizedBase}:{year}` via the shared `normalizeTitleKeys`. Enrichment adds BOTH identifying fields late, so the same movie can already be pinned under any poorer form of itself: -| pinned when | stored under | -|---|---| -| after enrichment | `tmdb:{id}` | -| before the TMDB id | `title:{base}:{year}` | -| before the year too | `title:{base}:` | +| pinned when | stored under | +| ------------------- | --------------------- | +| after enrichment | `tmdb:{id}` | +| before the TMDB id | `title:{base}:{year}` | +| before the year too | `title:{base}:` | `buildVodSourceMatchKeyCandidates` returns all three, most-trusted first. Reading every alias is what keeps a late TMDB id from orphaning an earlier pin. @@ -151,11 +151,11 @@ Reading every alias is what keeps a late TMDB id from orphaning an earlier pin. Three key sets, because reading, writing and deleting are different questions (`pinKeysFor` builds all three so they cannot disagree): -| set | contents | why | -|---|---|---| -| `lookup` | every alias above | a pin may sit under any poorer form of the movie | -| `write` | `tmdb:` + `title:{base}:{year}` | keys that name exactly ONE film | -| `loaded` | the key the pin on screen came from | the only ambiguous row this session may retire | +| set | contents | why | +| -------- | ----------------------------------- | ------------------------------------------------ | +| `lookup` | every alias above | a pin may sit under any poorer form of the movie | +| `write` | `tmdb:` + `title:{base}:{year}` | keys that name exactly ONE film | +| `loaded` | the key the pin on screen came from | the only ambiguous row this session may retire | A write stores the decision under **every** key in `write`, and clears whatever stale row is left over. Each half rules out the other's shortcut: @@ -168,7 +168,7 @@ stale row is left over. Each half rules out the other's shortcut: makes it readable at every stage of the same film's identity, and overwriting the poorer key is also what stops it from still naming the source the user just replaced. -- Spreading it across *every* alias is not the fix either: the yearless form is +- Spreading it across _every_ alias is not the fix either: the yearless form is shared by every remake, so a known-year pin stored there would answer for a different film — pin Dune (2021), open Dune (1984) before its year arrives, and it starts the 2021 source. That form is deliberately absent from `write` @@ -179,7 +179,7 @@ The renderer passes `write[0]` as the pin's own `matchKey` and the rest as inside the same transaction, so no key list can half-apply. Known limit, inherent to addressing rows by key alone: a write can only touch -keys the renderer can *name*. Re-pin a film during a pre-enrichment window and +keys the renderer can _name_. Re-pin a film during a pre-enrichment window and the `tmdb:{id}` row from an earlier, enriched session is not among them, so it survives pointing at the old source — and outranks the title key once the id arrives again. Nothing can enumerate it from the page's side; closing it needs a @@ -198,7 +198,7 @@ the user makes in the meantime, leaving the row and the primary Play naming a source the database no longer holds. Applying it first makes the later write simply win. -For the same reason a write or an unpin never *deletes* the yearless alias on +For the same reason a write or an unpin never _deletes_ the yearless alias on spec: that row may hold another remake's preference. The single exception is the row this session actually read, because the user is acting on the pin they can see — and leaving that one would make an unpin come back. @@ -217,7 +217,7 @@ when enrichment lands: any of them **re-runs discovery** — that is how a yearless search gets its year and a `tmdb:`-keyed pin becomes findable. - `vodMultiSourceSessionKey` is `playlistId:contentId` — the film itself. It is - what decides whether that rerun is a *refresh* or a *new session*. + what decides whether that rerun is a _refresh_ or a _new session_. A refresh keeps the controller: the source the user switched to stays active (with the facts its resolve produced, rather than the catalog's guesses), the @@ -227,7 +227,7 @@ actually streaming, and hand failover a clean slate for sources it has already spent. Only a different film resets — including the tried set, which is what makes failover terminate. -A rerun can also legitimately *drop* the playing row: the year the enrichment +A rerun can also legitimately _drop_ the playing row: the year the enrichment supplies makes the year gate reject a copy the yearless search had admitted ("Dune" 1984 while watching the 2021 film). Off the list is right — it is not the same film. Off the screen is not, so `applyDiscoveredSources` keeps it as a @@ -255,19 +255,19 @@ one that does not exist — the chip simply does not appear. So: (`' ' || LOWER(title) || ' ' GLOB '*[^a-z0-9]it[^a-z0-9]*'`) and orders by title length (a match is the title plus decoration: "IT (2017) 1080p"). - The FTS path keeps its 60-row window because it ranks by relevance — the best - rows are the ones it keeps. A scan cannot rank, so a window there would - silently decide which valid sources the user may see, and it would not even - buy anything: the GLOB cannot use an index, so SQLite reads every row either - way and a `LIMIT` saves transfer, not work. What bounds the scan instead is - its predicate: reaching it means NO token cleared the trigram minimum — a - one-or-two-character title like "It", but also an all-short multiword one - like "I Am" — and EVERY token must then appear as a word. Matching on the - first token alone would return most of a large catalog for the confirmation - pass to discard, which on the single database worker is real blocking. + The FTS path keeps its 60-row window because it ranks by relevance — the best + rows are the ones it keeps. A scan cannot rank, so a window there would + silently decide which valid sources the user may see, and it would not even + buy anything: the GLOB cannot use an index, so SQLite reads every row either + way and a `LIMIT` saves transfer, not work. What bounds the scan instead is + its predicate: reaching it means NO token cleared the trigram minimum — a + one-or-two-character title like "It", but also an all-short multiword one + like "I Am" — and EVERY token must then appear as a word. Matching on the + first token alone would return most of a large catalog for the confirmation + pass to discard, which on the single database worker is real blocking. - Like the `LIKE` it replaces it compares ASCII-lowercased text, so a non-ASCII - short title is no better and no worse served than before. + Like the `LIKE` it replaces it compares ASCII-lowercased text, so a non-ASCII + short title is no better and no worse served than before. Both remain necessary-not-sufficient filters: the two-tier normalized confirmation still runs afterwards, so the looser query never admits "Upgrade" @@ -277,7 +277,7 @@ One row inside the excluded playlist is kept when the caller names it (`keepContentId`): a pin can point at another copy of the film in the playlist being viewed, and dropping that row would leave the preference pointing at nothing. The host therefore reads the pin BEFORE discovery. When that pin is -the route's *own* row, the kept copy and `currentSourceRow()` are the same +the route's _own_ row, the kept copy and `currentSourceRow()` are the same stream, so `applyDiscoveredSources` drops the duplicate rather than listing it twice. @@ -287,7 +287,7 @@ The year gate applies to **both** match tiers, not just the year-stripped one. `normalizeTitleKeys` strips bracketed segments wholesale — they usually hold quality and language tags — so "Dune (1984)" and "Dune" normalize to the same string and the exact tier would accept the remake without ever consulting a -year, ranked *above* every fuzzy match. Discovery reads a bracketed year out of +year, ranked _above_ every fuzzy match. Discovery reads a bracketed year out of the raw title first, and when both sides state a year and they disagree, it is not the same movie. An unknown year still never blocks. @@ -335,7 +335,7 @@ the player component, `WebPlayerView` and the engine all survive, and Three details make the position survive: - The carried position is the **live** one. `handleInlineTimeUpdate` reports to - `VodMultiSourceHostService.reportPosition()` *before* the 15-second + `VodMultiSourceHostService.reportPosition()` _before_ the 15-second persistence throttle, so a switch does not rewind by up to 15 seconds. External players have no `timeupdate` at all, so for them the polled `playback_positions` value IS the live one and is reported directly — @@ -346,7 +346,7 @@ Three details make the position survive: a switch changes the key. The resolved playback carries the **new** source's `contentInfo`, and that source's row takes over. -The position also has to exist *before* anything plays. Nothing reports a live +The position also has to exist _before_ anything plays. Nothing reports a live one until the first `timeupdate`, so a pinned source started straight off the Resume button would resolve at zero and restart the film. The controller is therefore seeded from the persisted position (`seedResumeSeconds`), one-way: @@ -565,17 +565,44 @@ stored with different capitalisation in two playlists ("ОН" vs "Он") cannot folded by the indexed path. Closing that needs a stored normalized-title column, which is deliberately out of scope here. -The **scan** tier does fold it, because GLOB character classes are *not* +The **scan** tier does fold it, because GLOB character classes are _not_ ASCII-only — `patternCompare` reads them as UTF-8 code points, and `'Он' GLOB '*[Оо][Нн]*'` is true. `caseInsensitiveGlobPattern` therefore folds the case in JavaScript, where Unicode case mapping is real, and hands SQLite one -class per character. Each class also carries the uppercase form's OWN lowercase, -which is what covers a letter spelled two ways in lower case: Greek `Σ` -lowercases to `σ`, but a word-final sigma is written `ς` and is equally a -lowercase of it. That reach is one-way — `σ → Σ → σ` never arrives at `ς` — and -left so deliberately, because `ς` is only correct at the end of a word, exactly -where the request's own last character sits; closing the other direction needs a -fold table. +class per character. + +A letter can have more lowercase spellings than case mapping reaches from any +one of them: Greek `Σ` lowercases to `σ`, but a word-final sigma is written `ς` +and is equally a lowercase of it, and neither `Σ` nor `σ` arrives at `ς`. Each +class is therefore built from a **fold group** — every character sharing an +uppercase form — so every spelling reaches every other one. A request for `ΑΣ` +finds a stored `Ας`, which the earlier one-way reach (`ς` found `Σ` and `σ`, but +never the reverse) did not: uppercase is just as typeable as lowercase, so +"`ς` only occurs word-finally" never justified the asymmetry. + +`CASE_FOLD_GROUPS` is derived by scanning the cased ranges at module load, the +same way `ACCENTED_BY_BASE` is, rather than tabulating pairs by hand — so it +generalises past sigma on its own: dotless `ı` folds with `i`, the long `ſ` with +`s`, and the historic Cyrillic letterforms with `В Д О С Т Ъ Ѣ`. Only the groups +a per-character fold would miss are kept, 24 of 767, since the rest are just +{upper, lower} and add nothing. Widening a class only ever costs candidate rows: +the scan is a necessary-not-sufficient filter and `normalizeTitleKeys` is what +confirms a match. + +**Admitting a candidate is only half of it.** A class that admits a row the +confirmation then rejects finds nothing, so sigma has to be folded on both +tiers. `normalizeTitleKeys` therefore rewrites `ς` to `σ` after lowercasing. +This is not symmetry for its own sake: `toLowerCase` picks the form by +position, so `"ΑΣ"` arrives as `"ας"` while an already-lowercase `"ασ"` stays +medial, and the same word reaches the comparison spelled two ways. + +Both SQL tiers were already folding them together — SQLite's trigram tokenizer +does full Unicode folding natively (unlike `LOWER()`, which is ASCII-only), and +the scan's GLOB classes now do it in JavaScript. The confirmation was the only +tier that did not, which made this a **pre-existing gap on the FTS path too**, +not just on the scan: a stored `"ο αρχοντασ"` was returned as a candidate for +`"ο αρχοντας"` and then discarded. Folding to the medial form is what Unicode +case folding does. It returns `null` — leaving the substring tests as the whole answer — for a token holding a GLOB metacharacter (SQLite GLOB has no escape character, so an diff --git a/libs/shared/interfaces/src/lib/title-normalization.util.spec.ts b/libs/shared/interfaces/src/lib/title-normalization.util.spec.ts index 8f8df8318..f365e454c 100644 --- a/libs/shared/interfaces/src/lib/title-normalization.util.spec.ts +++ b/libs/shared/interfaces/src/lib/title-normalization.util.spec.ts @@ -69,6 +69,25 @@ describe('normalizeTitleKeys', () => { it('keeps plural "seasons" endings — only singular markers are tags', () => { expect(normalizeTitle('Best of 2 Seasons')).toBe('best of 2 seasons'); }); + + it('folds both lowercase spellings of Greek sigma together', () => { + // `toLowerCase` picks the form by position — "ΑΣ" becomes "ας" while + // an already-lowercase "ασ" stays medial — so one word arrives here + // spelled two ways. Both SQL tiers fold them (SQLite's trigram + // tokenizer natively, the scan's GLOB classes in JavaScript), so + // leaving them apart here admits a candidate and then discards it. + expect(normalizeTitle('ΑΣ')).toBe(normalizeTitle('Ας')); + expect(normalizeTitle('ασ')).toBe(normalizeTitle('ας')); + expect(normalizeTitle('ΑΣ')).toBe(normalizeTitle('ασ')); + expect(normalizeTitle('Ο Άρχοντας')).toBe(normalizeTitle('ο αρχοντασ')); + }); + + it('does not fold sigma into unrelated Greek letters', () => { + // The fold is one letter's two lowercase forms, not a general + // loosening of Greek — these must stay different titles. + expect(normalizeTitle('ΑΣ')).not.toBe(normalizeTitle('ΑΝ')); + expect(normalizeTitle('ΟΣ')).not.toBe(normalizeTitle('ΑΣ')); + }); }); describe('provider tag stripping', () => { diff --git a/libs/shared/interfaces/src/lib/title-normalization.util.ts b/libs/shared/interfaces/src/lib/title-normalization.util.ts index 5e95dc2e2..b77104612 100644 --- a/libs/shared/interfaces/src/lib/title-normalization.util.ts +++ b/libs/shared/interfaces/src/lib/title-normalization.util.ts @@ -228,6 +228,15 @@ export function normalizeTitleKeys( .normalize('NFD') .replace(/[\u0300-\u036F]/g, '') .toLowerCase() + // Greek \u03A3 has two lowercase forms and `toLowerCase` picks by position: + // "\u0391\u03A3" becomes "\u03B1\u03C2" while an already-lowercase "\u03B1\u03C3" stays medial, so + // the same word reaches this line spelled two ways. Both SQL tiers + // fold them together \u2014 SQLite's trigram tokenizer does it natively, + // and the scan's GLOB classes do it in `caseInsensitiveGlobPattern` \u2014 + // so without this the candidate is admitted by the query and then + // thrown away by the confirmation. Folding to the medial form is what + // Unicode case folding does. + .replace(/\u03C2/g, '\u03C3') .replace(/[^\p{L}\p{N}]+/gu, ' ') .split(' ') .filter((token) => token !== '' && !QUALITY_TAGS.has(token))