diff --git a/.changes/tmdb-title-match-requests.md b/.changes/tmdb-title-match-requests.md new file mode 100644 index 000000000..3e38764a9 --- /dev/null +++ b/.changes/tmdb-title-match-requests.md @@ -0,0 +1,6 @@ +--- +type: perf +area: tmdb +--- + +On large libraries the TMDB trending and recommendation rails, and the "In your library" lookups on Discover, actor and Similar views, no longer hold up the rest of the app while they search your playlists: the search now runs in small steps that let other work in between. diff --git a/docs/architecture/tmdb-metadata-enrichment.md b/docs/architecture/tmdb-metadata-enrichment.md index 74a9e72a8..606eb72fa 100644 --- a/docs/architecture/tmdb-metadata-enrichment.md +++ b/docs/architecture/tmdb-metadata-enrichment.md @@ -208,7 +208,8 @@ component for detail→detail navigation. `CrossPortalSimilarService` (`libs/services`) extends the rail across portals: recommendations are matched against ALL imported Xtream -playlists with one batched `DB_MATCH_TITLES` request (Electron only, +playlists with one batched match (`DB_MATCH_TITLES`, sent in small +requests, see "Sending a batched match"; Electron only, same two-tier + year rule). Stalker detail views — where the local catalog is server-paginated and unmatchable — get their "Similar" rail purely from these cross-portal matches (shared `VodDetailsComponent` for @@ -353,7 +354,8 @@ Filmography has two scopes: rest open the portal search prefilled with the title (`?q=`). Stalker has no local catalog, so every title goes through the portal search. - **All portals** (Electron only, toggle hidden in the PWA): one batched - `DB_MATCH_TITLES` worker request runs a trigram-FTS lookup per title + match (`DB_MATCH_TITLES` worker requests of five titles, see "Sending a + batched match") runs a trigram-FTS lookup per title over ALL imported Xtream playlists (`operations/title-match.operations.ts`), confirming candidates with the same two-tier normalized-title matching the renderer uses @@ -363,6 +365,24 @@ Filmography has two scopes: This also works from Stalker actor pages — the one place the Stalker catalog limitation is lifted. +### Sending a batched match + +`CatalogTitleMatchService.matchTitles` deduplicates the titles, keeps the +first `TITLE_MATCH_TITLE_LIMIT` (200, the worker's own cap) and sends them +in `DB_MATCH_TITLES` requests of `TITLE_MATCH_REQUEST_SIZE` (5), one after +another, concatenating the answers in order. The DB worker answers one +request at a time, so a whole rail's batch in one request held every other +read behind it: on a 3.9M-title library one recommendations batch took +95 s cold and kept the dashboard's playback positions (and with them the +Continue Watching rail) waiting; now other reads wait for five titles at +most. The answer stays all-or-nothing: a failed +request, or a parental lock that starts withholding everything midway, +returns no matches, as one failed request did before. One request ran +under one parental-lock state, so a batch does too: when +`ParentalLockService.version()` changes between requests (a relock, an +unlock, a changed lock list) the batch starts over, and after +`TITLE_MATCH_LOCK_ATTEMPTS` (3) changing attempts it returns no matches. + ### Resolving a batched match Every `DB_MATCH_TITLES` consumer — the Trending rail, the "Because you @@ -645,7 +665,8 @@ since shipped.) toggle): `DashboardTrendingService` (`libs/workspace/dashboard/data-access`) pulls the weekly TMDB trending lists (cached one day per language) and runs ONE batched - `CatalogTitleMatchService.matchTitles` request against the imported + `CatalogTitleMatchService.matchTitles` call (worker requests of five + titles, see "Sending a batched match") against the imported Xtream playlists — matched cards navigate straight to their detail view and show the playlist name; unmatched cards open the global search prefilled. Requires both the TMDB opt-in and the Electron DB worker @@ -665,7 +686,8 @@ since shipped.) interleaved round-robin, deduplicated by TMDB id, stripped of anything already watched or favorited, and matched against the imported libraries with ONE batched - `CatalogTitleMatchService.matchTitles` request. The watched/favorited + `CatalogTitleMatchService.matchTitles` call (worker requests of five + titles). The watched/favorited exclusion index is built through the same lookup-attempt builder the seeds use, so an activity row is indexed under more than its display title: under the media type the detail view enriched with (a Stalker @@ -719,7 +741,7 @@ since shipped.) on its behalf would drop its recommendations for the session; a seed that has no TMDB match never resolves either, so that user's rail re-runs each visit, which is bounded work (cached enrichment misses - plus one batched worker call). Latched loads are keyed by the TMDB language + plus one batched title match). Latched loads are keyed by the TMDB language (payloads are localized; the facade exposes `language()` for this), the seed set, the watched/favorited exclusion set AND the imported-playlist id set — watching something new re-seeds on the next dashboard visit, diff --git a/libs/services/src/lib/catalog-title-match.service.spec.ts b/libs/services/src/lib/catalog-title-match.service.spec.ts index b4f24bd68..5b4b85795 100644 --- a/libs/services/src/lib/catalog-title-match.service.spec.ts +++ b/libs/services/src/lib/catalog-title-match.service.spec.ts @@ -3,6 +3,9 @@ import { TestBed } from '@angular/core/testing'; import { CatalogTitleMatch } from '@iptvnator/shared/interfaces'; import { CatalogTitleMatchService, + TITLE_MATCH_LOCK_ATTEMPTS, + TITLE_MATCH_REQUEST_SIZE, + TITLE_MATCH_TITLE_LIMIT, groupTitleMatchesByKey, pickTitleMatch, } from './catalog-title-match.service'; @@ -207,7 +210,10 @@ describe('CatalogTitleMatchService parental lock', () => { providers: [ { provide: ParentalLockService, - useValue: { withholdsEverything: signal(withholds) }, + useValue: { + withholdsEverything: signal(withholds), + version: signal(0), + }, }, ], }); @@ -253,3 +259,164 @@ describe('CatalogTitleMatchService parental lock', () => { expect(visible()).toEqual([show]); }); }); + +describe('CatalogTitleMatchService requests', () => { + const match = (queryTitle: string) => ({ + queryTitle, + playlistId: 'pl-1', + playlistName: 'Portal', + categoryId: 7, + xtreamId: 1, + type: 'movie' as const, + trailingYear: null, + }); + const titles = (count: number) => + Array.from({ length: count }, (_, index) => `Title ${index}`); + + function setUp( + dbMatchTitles: jest.Mock, + withholdsEverything = signal(false), + version = signal(0) + ) { + (window as { electron?: unknown }).electron = { dbMatchTitles }; + TestBed.configureTestingModule({ + providers: [ + { + provide: ParentalLockService, + useValue: { withholdsEverything, version }, + }, + ], + }); + return TestBed.inject(CatalogTitleMatchService); + } + + afterEach(() => { + delete (window as { electron?: unknown }).electron; + }); + + it('asks in small requests, one after another, and keeps the result order', async () => { + let inFlight = 0; + let maxInFlight = 0; + const dbMatchTitles = jest.fn(async (batch: string[]) => { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + await Promise.resolve(); + inFlight--; + return batch.map(match); + }); + const service = setUp(dbMatchTitles); + + const count = TITLE_MATCH_REQUEST_SIZE * 2 + 3; + const result = await service.matchTitles(titles(count)); + + expect(dbMatchTitles.mock.calls.map(([batch]) => batch.length)).toEqual( + [TITLE_MATCH_REQUEST_SIZE, TITLE_MATCH_REQUEST_SIZE, 3] + ); + expect(maxInFlight).toBe(1); + expect(result.map(({ queryTitle }) => queryTitle)).toEqual( + titles(count) + ); + }); + + it('drops duplicates and blanks and keeps the overall title cap', async () => { + const dbMatchTitles = jest.fn(async (batch: string[]) => + batch.map(match) + ); + const service = setUp(dbMatchTitles); + + await service.matchTitles([ + 'Dune', + ' ', + 'Dune', + ...titles(TITLE_MATCH_TITLE_LIMIT + 5), + ]); + + const sent = dbMatchTitles.mock.calls.flatMap(([batch]) => batch); + expect(sent).toHaveLength(TITLE_MATCH_TITLE_LIMIT); + expect(sent[0]).toBe('Dune'); + expect(new Set(sent).size).toBe(sent.length); + }); + + it('answers nothing when one request fails, and stops asking', async () => { + const warn = jest + .spyOn(console, 'warn') + .mockImplementation(() => undefined); + const dbMatchTitles = jest + .fn() + .mockImplementationOnce(async (batch: string[]) => batch.map(match)) + .mockRejectedValueOnce(new Error('worker gone')); + const service = setUp(dbMatchTitles); + + await expect( + service.matchTitles(titles(TITLE_MATCH_REQUEST_SIZE * 3)) + ).resolves.toEqual([]); + + expect(dbMatchTitles).toHaveBeenCalledTimes(2); + warn.mockRestore(); + }); + + it.each([ + ['the first', 1], + ['the last', 3], + ])( + 'answers nothing when the parental lock starts withholding everything during %s request', + async (_label, closingRequest) => { + const withholds = signal(false); + let requests = 0; + const dbMatchTitles = jest.fn(async (batch: string[]) => { + requests++; + if (requests === closingRequest) { + withholds.set(true); + } + return batch.map(match); + }); + const service = setUp(dbMatchTitles, withholds); + + await expect( + service.matchTitles(titles(TITLE_MATCH_REQUEST_SIZE * 3)) + ).resolves.toEqual([]); + + // No request after the lock closed. + expect(dbMatchTitles).toHaveBeenCalledTimes(closingRequest); + } + ); + + it('starts over when the lock state changes between requests', async () => { + const version = signal(0); + let requests = 0; + const dbMatchTitles = jest.fn(async (batch: string[]) => { + requests++; + if (requests === 2) { + // Relocked while the second request ran: the first + // answer was filtered under the old state. + version.set(1); + } + return batch.map(match); + }); + const service = setUp(dbMatchTitles, signal(false), version); + const count = TITLE_MATCH_REQUEST_SIZE * 3; + + const result = await service.matchTitles(titles(count)); + + // Two requests under the old state, then all three again. + expect(dbMatchTitles).toHaveBeenCalledTimes(5); + expect(result.map(({ queryTitle }) => queryTitle)).toEqual( + titles(count) + ); + }); + + it('answers nothing when the lock state keeps changing', async () => { + const version = signal(0); + const dbMatchTitles = jest.fn(async (batch: string[]) => { + version.update((value) => value + 1); + return batch.map(match); + }); + const service = setUp(dbMatchTitles, signal(false), version); + + await expect( + service.matchTitles(titles(TITLE_MATCH_REQUEST_SIZE * 3)) + ).resolves.toEqual([]); + + expect(dbMatchTitles).toHaveBeenCalledTimes(TITLE_MATCH_LOCK_ATTEMPTS); + }); +}); diff --git a/libs/services/src/lib/catalog-title-match.service.ts b/libs/services/src/lib/catalog-title-match.service.ts index 9b9c23135..2eed76648 100644 --- a/libs/services/src/lib/catalog-title-match.service.ts +++ b/libs/services/src/lib/catalog-title-match.service.ts @@ -92,9 +92,31 @@ export function pickTitleMatch( ); } +/** + * Titles per `DB_MATCH_TITLES` request. The DB worker answers one request + * at a time and runs a ranked FTS query per title, a second or more for a + * common word on a large catalog; a whole rail's batch in one request held + * every other read (the dashboard's playback positions among them) behind + * it for up to a minute and a half. Small requests, sent one after + * another, let those reads in between. + */ +export const TITLE_MATCH_REQUEST_SIZE = 5; + +/** Distinct titles matched per call: the worker's MAX_TITLES_PER_REQUEST. */ +export const TITLE_MATCH_TITLE_LIMIT = 200; + +/** + * Times a batch starts over when the parental lock changes while it runs; + * a lock that keeps changing yields no matches rather than a mixed answer. + */ +export const TITLE_MATCH_LOCK_ATTEMPTS = 3; + +const LOCK_CHANGED = Symbol('lock changed'); + /** * Cross-playlist title matching via the Electron DB worker - * (`DB_MATCH_TITLES`, trigram FTS over all imported Xtream playlists). + * (`DB_MATCH_TITLES`, trigram FTS over all imported Xtream playlists), in + * requests of TITLE_MATCH_REQUEST_SIZE titles. * Unavailable in the PWA — `isAvailable` gates the actor page's * "All portals" scope. */ @@ -121,12 +143,59 @@ export class CatalogTitleMatchService { return []; } - try { - return await window.electron.dbMatchTitles(titles); - } catch (error) { - console.warn('Cross-playlist title matching failed:', error); - return []; + // The worker dedupes and caps each request; doing both here first + // keeps the overall cap and the result order of one big request. + const unique = [ + ...new Set( + titles.filter( + (title) => typeof title === 'string' && title.trim() !== '' + ) + ), + ].slice(0, TITLE_MATCH_TITLE_LIMIT); + // One request ran under one lock state; a batch must too. If the + // lock changes between requests (a relock, an unlock, a changed + // lock list), the earlier answers were filtered under the old state + // and the batch starts over. + for (let attempt = 0; attempt < TITLE_MATCH_LOCK_ATTEMPTS; attempt++) { + const matches = await this.matchUnderOneLockState(unique); + if (matches !== LOCK_CHANGED) { + return matches; + } } + return []; + } + + private async matchUnderOneLockState( + titles: readonly string[] + ): Promise { + const lockVersion = this.parentalLock.version(); + const matches: CatalogTitleMatch[] = []; + for ( + let start = 0; + start < titles.length; + start += TITLE_MATCH_REQUEST_SIZE + ) { + try { + matches.push( + ...(await window.electron.dbMatchTitles( + titles.slice(start, start + TITLE_MATCH_REQUEST_SIZE) + )) + ); + } catch (error) { + // All or nothing, as with one request: callers treat an + // empty answer as a failed lookup they may run again. + console.warn('Cross-playlist title matching failed:', error); + return []; + } + // The lock can close while a request runs, the last one too. + if (this.parentalLock.withholdsEverything()) { + return []; + } + if (this.parentalLock.version() !== lockVersion) { + return LOCK_CHANGED; + } + } + return matches; } /** diff --git a/libs/services/src/lib/cross-portal-similar.service.ts b/libs/services/src/lib/cross-portal-similar.service.ts index c2fe31d41..92f7124ac 100644 --- a/libs/services/src/lib/cross-portal-similar.service.ts +++ b/libs/services/src/lib/cross-portal-similar.service.ts @@ -30,7 +30,8 @@ const DEFAULT_LIMIT = 12; /** * Matches TMDB recommendations against ALL imported Xtream playlists via - * one batched DB-worker request — powers the "Similar" rail for portals + * one batched `matchTitles` call (DB-worker requests of five titles) — + * powers the "Similar" rail for portals * without a matchable local catalog (Stalker) and supplements the Xtream * rail with titles available in the user's other portals. Electron-only: * `isAvailable` is false in the PWA and every call resolves to []. diff --git a/libs/workspace/dashboard/data-access/src/lib/dashboard-recommendations.service.ts b/libs/workspace/dashboard/data-access/src/lib/dashboard-recommendations.service.ts index 46c87951c..c5f0335a9 100644 --- a/libs/workspace/dashboard/data-access/src/lib/dashboard-recommendations.service.ts +++ b/libs/workspace/dashboard/data-access/src/lib/dashboard-recommendations.service.ts @@ -51,8 +51,8 @@ const MAX_ITEMS = 18; * the enrichment facade (items whose detail view was opened come straight * from the SQLite cache, where `recommendations` ride along with every * details payload), the per-seed lists are interleaved and deduplicated, - * already watched/favorited titles are dropped, and ONE batched worker - * request keeps only titles that exist in an imported library. + * already watched/favorited titles are dropped, and ONE batched + * `matchTitles` call keeps only titles that exist in an imported library. * * Requires the TMDB opt-in and the Electron DB worker — hidden in the PWA. * A load is keyed by the seed set AND the watched/favorited exclusion set, @@ -173,7 +173,8 @@ export class DashboardRecommendationsService { // match never resolves either, so this rail re-runs on // each visit for that user — bounded work, since the // enrichment misses are cached and the catalog match is - // one batched worker call. + // one batched `matchTitles` call (worker requests of + // five titles, other reads in between). this.loadedKey = perSeed.every((seed) => seed.resolved) ? loadKey : null; @@ -182,7 +183,8 @@ export class DashboardRecommendationsService { // latched: an empty match result is indistinguishable // from a transient worker failure at this layer // (matchTitles maps failures to []), and re-running is - // cheap (cached enrichment + one batched worker call). + // cheap (cached enrichment + one batched `matchTitles` + // call). // Mirrors the trending rail's retry-on-empty semantics. // The PREVIOUS key must reset too: it described the // rail that was just cleared, and returning to those @@ -449,8 +451,8 @@ export class DashboardRecommendationsService { if (candidates.length === 0) { return EMPTY_LOAD; } - // Both aliases go into the ONE batched request; the index lookup - // prefers the localized form. Built with a loop rather than + // Both aliases go into the ONE batched `matchTitles` call; the index + // lookup prefers the localized form. Built with a loop rather than // flatMap — the web app compiles this lib against `lib: es2018`, // which predates Array.prototype.flatMap. const queryTitles: string[] = []; diff --git a/libs/workspace/dashboard/data-access/src/lib/dashboard-trending.service.ts b/libs/workspace/dashboard/data-access/src/lib/dashboard-trending.service.ts index 857eb9a87..3bc5f02a4 100644 --- a/libs/workspace/dashboard/data-access/src/lib/dashboard-trending.service.ts +++ b/libs/workspace/dashboard/data-access/src/lib/dashboard-trending.service.ts @@ -18,7 +18,7 @@ export interface DashboardTrendingItem extends TmdbTrendingEntry { * "Trending this week" dashboard rail data. Requires the TMDB opt-in and * (for availability matching / navigation) the Electron DB worker — the * rail is hidden in the PWA. Trending lists are cached for a day, and the - * title match runs as ONE batched worker request, fired only after the + * title match runs as ONE batched `matchTitles` call, fired only after the * dashboard's own data has loaded so it never competes for the worker. */ @Injectable({ providedIn: 'root' })