From 4252bea2e0e4bb8ff55dbd97f6ac94cf3b67d8a2 Mon Sep 17 00:00:00 2001 From: 4gray Date: Fri, 31 Jul 2026 14:06:18 +0200 Subject: [PATCH] fix(downloads): throttle sparse metadata refreshes --- docs/architecture/download-manager.md | 6 +- .../download-metadata-refresh.spec.ts | 60 ++++++++ .../download-metadata-refresh.ts | 133 ++++++++++++++++-- .../download-metadata.mapper.spec.ts | 21 +++ .../download-metadata.mapper.ts | 1 + .../download-offline-metadata.service.spec.ts | 113 +++++++++++++++ .../download-offline-metadata.service.ts | 44 +++--- 7 files changed, 346 insertions(+), 32 deletions(-) create mode 100644 libs/portal/downloads/feature/src/lib/offline-detail/download-metadata-refresh.spec.ts diff --git a/docs/architecture/download-manager.md b/docs/architecture/download-manager.md index 52474eb52..3421bb331 100644 --- a/docs/architecture/download-manager.md +++ b/docs/architecture/download-manager.md @@ -123,8 +123,10 @@ variants, contextual buttons, and theme-aware styling. typed provider catalog and optional TMDB enrichment. A successful refresh advances its freshness timestamp; transient TMDB failures may retain improved provider fields without advancing the timestamp or language, so a later visit - retries. Concurrent movie and grouped-series refreshes are request-ordered so - only the newest generation may persist. + retries. Legitimately sparse successful results are remembered in a bounded, + language-scoped runtime throttle for the same TTL, preventing persistence + reloads from starting an enrichment loop. Concurrent movie and grouped-series + refreshes are request-ordered so only the newest generation may persist. - Snapshot artwork accepts only safe HTTP(S) image URLs. Renderer normalization and main-process persistence independently reject credential-shaped path and query keys, including `username`, so provider credentials cannot be cached in diff --git a/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata-refresh.spec.ts b/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata-refresh.spec.ts new file mode 100644 index 000000000..c914c1b82 --- /dev/null +++ b/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata-refresh.spec.ts @@ -0,0 +1,60 @@ +import { TMDB_DETAILS_CACHE_TTL_MS } from '@iptvnator/services'; +import { + COMPLETED_METADATA_REFRESH_MAX_ENTRIES, + CompletedMetadataRefreshThrottle, +} from './download-metadata-refresh'; + +describe('CompletedMetadataRefreshThrottle', () => { + beforeEach(() => { + jest.useFakeTimers(); + jest.setSystemTime(new Date('2026-07-31T08:30:00.000Z')); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('expires completed attempts with the TMDB details TTL', () => { + const throttle = new CompletedMetadataRefreshThrottle(); + throttle.mark('download:17', 'en', 1); + + expect(throttle.isCompleted('download:17', 'en')).toBe(true); + + jest.advanceTimersByTime(TMDB_DETAILS_CACHE_TTL_MS + 1); + + expect(throttle.isCompleted('download:17', 'en')).toBe(false); + }); + + it('bounds completed attempts by evicting the oldest group and language', () => { + const throttle = new CompletedMetadataRefreshThrottle(); + for ( + let index = 0; + index <= COMPLETED_METADATA_REFRESH_MAX_ENTRIES; + index += 1 + ) { + throttle.mark(`download:${index}`, 'en', index + 1); + } + + expect(throttle.isCompleted('download:0', 'en')).toBe(false); + expect( + throttle.isCompleted( + `download:${COMPLETED_METADATA_REFRESH_MAX_ENTRIES}`, + 'en' + ) + ).toBe(true); + }); + + it('does not let older cleanup erase a newer completed generation', () => { + const throttle = new CompletedMetadataRefreshThrottle(); + throttle.mark('series:playlist-a:93', 'en', 1); + throttle.mark('series:playlist-a:93', 'en', 2); + + throttle.clear('series:playlist-a:93', 'en', 1); + + expect(throttle.isCompleted('series:playlist-a:93', 'en')).toBe(true); + + throttle.clear('series:playlist-a:93', 'en', 2); + + expect(throttle.isCompleted('series:playlist-a:93', 'en')).toBe(false); + }); +}); diff --git a/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata-refresh.ts b/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata-refresh.ts index 55a50e752..6d13fc0a7 100644 --- a/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata-refresh.ts +++ b/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata-refresh.ts @@ -1,4 +1,7 @@ -import { TMDB_DETAILS_CACHE_TTL_MS } from '@iptvnator/services'; +import { + TMDB_DETAILS_CACHE_TTL_MS, + type SettingsStore, +} from '@iptvnator/services'; import type { DownloadMetadataSnapshot } from '@iptvnator/shared/interfaces'; export const TMDB_REFRESH_STATE = { @@ -24,10 +27,26 @@ interface FinalizeMetadataRefreshInput { } export interface FinalizedMetadataRefresh { + completedAttempt: boolean; shouldPersist: boolean; snapshot: DownloadMetadataSnapshot; } +export const COMPLETED_METADATA_REFRESH_MAX_ENTRIES = 256; + +interface CompletedMetadataRefresh { + expiresAt: number; + generation: number; +} + +interface PersistMetadataRefreshInput { + completedAttempt: boolean; + generation: number; + groupKey: string; + language: string; + persist(): Promise<{ success: boolean }>; +} + function isSparse(snapshot: DownloadMetadataSnapshot): boolean { return !snapshot.plot || (!snapshot.posterUrl && !snapshot.backdropUrl); } @@ -42,16 +61,28 @@ function isFresh(snapshot: DownloadMetadataSnapshot): boolean { ); } +export function currentMetadataLanguage( + settings: Pick +): string { + try { + return settings.language().trim() || 'en'; + } catch { + return 'en'; + } +} + export function needsMetadataRefresh( snapshot: DownloadMetadataSnapshot | undefined, - language: string + language: string, + completedSparseAttempt = false ): boolean { - return ( - !snapshot || - snapshot.language !== language || - isSparse(snapshot) || - !isFresh(snapshot) - ); + if (!snapshot || snapshot.language !== language) { + return true; + } + if (isSparse(snapshot)) { + return !completedSparseAttempt; + } + return !isFresh(snapshot); } export function withoutMetadataFreshness( @@ -117,11 +148,57 @@ export function finalizeMetadataRefresh({ const shouldPersist = refreshSucceeded || !materiallyEqual(local, finalized); return { + completedAttempt: refreshSucceeded, shouldPersist, snapshot: shouldPersist ? finalized : local, }; } +function completedRefreshKey(groupKey: string, language: string): string { + return `${groupKey}\u0000${language}`; +} + +export class CompletedMetadataRefreshThrottle { + private readonly attempts = new Map(); + + isCompleted(groupKey: string, language: string): boolean { + const now = Date.now(); + this.deleteExpired(now); + return this.attempts.has(completedRefreshKey(groupKey, language)); + } + + mark(groupKey: string, language: string, generation: number): void { + const now = Date.now(); + const key = completedRefreshKey(groupKey, language); + this.deleteExpired(now); + this.attempts.delete(key); + while (this.attempts.size >= COMPLETED_METADATA_REFRESH_MAX_ENTRIES) { + const oldestKey = this.attempts.keys().next().value; + if (oldestKey === undefined) break; + this.attempts.delete(oldestKey); + } + this.attempts.set(key, { + expiresAt: now + TMDB_DETAILS_CACHE_TTL_MS, + generation, + }); + } + + clear(groupKey: string, language: string, generation: number): void { + const key = completedRefreshKey(groupKey, language); + if (this.attempts.get(key)?.generation === generation) { + this.attempts.delete(key); + } + } + + private deleteExpired(now: number): void { + for (const [key, attempt] of this.attempts) { + if (attempt.expiresAt < now) { + this.attempts.delete(key); + } + } + } +} + export class LatestMetadataWriteGuard { private generation = 0; private readonly latestByKey = new Map(); @@ -142,3 +219,43 @@ export class LatestMetadataWriteGuard { } } } + +export class MetadataRefreshCoordinator { + private readonly completed = new CompletedMetadataRefreshThrottle(); + private readonly writes = new LatestMetadataWriteGuard(); + + begin(groupKey: string): number { + return this.writes.begin(groupKey); + } + + finish(groupKey: string, generation: number): void { + this.writes.finish(groupKey, generation); + } + + isCompleted(groupKey: string, language: string): boolean { + return this.completed.isCompleted(groupKey, language); + } + + async persistLatest({ + completedAttempt, + generation, + groupKey, + language, + persist, + }: PersistMetadataRefreshInput): Promise { + if (!this.writes.isLatest(groupKey, generation)) return; + if (completedAttempt) { + this.completed.mark(groupKey, language, generation); + } + try { + const result = await persist(); + if (!result.success && completedAttempt) { + this.completed.clear(groupKey, language, generation); + } + } catch { + if (completedAttempt) { + this.completed.clear(groupKey, language, generation); + } + } + } +} diff --git a/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata.mapper.spec.ts b/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata.mapper.spec.ts index 13d34ef3f..1a25f2669 100644 --- a/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata.mapper.spec.ts +++ b/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata.mapper.spec.ts @@ -283,6 +283,27 @@ describe('download metadata mapper', () => { ); }); + it('falls back from blank nested Stalker artwork to root poster_url', () => { + const mapped = mapProviderToDownloadSnapshot({ + source: 'stalker', + language: 'en', + mediaKind: 'movie', + fallback: providerSnapshot, + provider: { + poster_url: + 'https://images.example.test/posters/canonical-recent.jpg', + info: { + name: 'Nested title', + movie_image: '', + }, + }, + }); + + expect(mapped.posterUrl).toBe( + 'https://images.example.test/posters/canonical-recent.jpg' + ); + }); + it('bounds mapped people and genres to the persisted DTO limits', () => { const mapped = mapProviderToDownloadSnapshot({ source: 'xtream', diff --git a/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata.mapper.ts b/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata.mapper.ts index 60e9c4f02..6124b40cd 100644 --- a/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata.mapper.ts +++ b/libs/portal/downloads/feature/src/lib/offline-detail/download-metadata.mapper.ts @@ -185,6 +185,7 @@ export function mapProviderToDownloadSnapshot({ ? (string(info?.['movie_image']) ?? string(root['cover']) ?? string(root['screenshot_uri']) ?? + string(root['poster_url']) ?? string(root['logo'])) : string( first(editorial, [ diff --git a/libs/portal/downloads/feature/src/lib/offline-detail/download-offline-metadata.service.spec.ts b/libs/portal/downloads/feature/src/lib/offline-detail/download-offline-metadata.service.spec.ts index 1b44b76b6..09435c79b 100644 --- a/libs/portal/downloads/feature/src/lib/offline-detail/download-offline-metadata.service.spec.ts +++ b/libs/portal/downloads/feature/src/lib/offline-detail/download-offline-metadata.service.spec.ts @@ -5,6 +5,7 @@ import { DownloadsService, PlaylistsService, SettingsStore, + TMDB_DETAILS_CACHE_TTL_MS, TmdbEnrichmentService, type TmdbMovieDetails, type TmdbTvDetails, @@ -510,6 +511,118 @@ describe('DownloadOfflineMetadataService', () => { } }); + it('throttles a successful sparse provider refresh per language during reload', async () => { + jest.useFakeTimers(); + jest.setSystemTime(new Date(NOW)); + try { + db.getContentByXtreamId.mockResolvedValue({ + ...providerContent('Local title', 41), + category_id: 4, + poster_url: '', + }); + const local = snapshot({ + plot: undefined, + posterUrl: undefined, + backdropUrl: undefined, + enrichedAt: '2020-01-01T00:00:00.000Z', + }); + downloads.updateMetadata.mockImplementationOnce( + async (_downloadId, refreshed) => { + await service.resolve(movieDetail({}, refreshed)); + return { success: true }; + } + ); + + const refreshed = await service.resolve(movieDetail({}, local)); + + expect(refreshed.plot).toBeUndefined(); + expect(refreshed.posterUrl).toBeUndefined(); + expect(db.getContentByXtreamId).toHaveBeenCalledTimes(1); + expect(downloads.updateMetadata).toHaveBeenCalledTimes(1); + + currentLanguage = 'de'; + const translated = await service.resolve( + movieDetail({}, refreshed) + ); + + expect(translated.language).toBe('de'); + expect(db.getContentByXtreamId).toHaveBeenCalledTimes(2); + expect(downloads.updateMetadata).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }); + + it('throttles a successful sparse materially identical TMDB refresh', async () => { + jest.useFakeTimers(); + jest.setSystemTime(new Date(NOW)); + try { + tmdb.isEnabled.mockReturnValue(true); + tmdb.enrichMovie.mockResolvedValue({ id: 603 }); + const local = snapshot({ + plot: undefined, + posterUrl: undefined, + backdropUrl: undefined, + tmdbId: 603, + rating: 0, + enrichedAt: '2020-01-01T00:00:00.000Z', + }); + + const refreshed = await service.resolve(movieDetail({}, local)); + await service.resolve(movieDetail({}, refreshed)); + + expect(tmdb.enrichMovie).toHaveBeenCalledTimes(1); + expect(downloads.updateMetadata).toHaveBeenCalledTimes(1); + } finally { + jest.useRealTimers(); + } + }); + + it('retries a sparse transient TMDB failure', async () => { + tmdb.isEnabled.mockReturnValue(true); + tmdb.enrichMovie.mockRejectedValue(new Error('TMDB unavailable')); + const local = snapshot({ + plot: undefined, + posterUrl: undefined, + backdropUrl: undefined, + enrichedAt: '2020-01-01T00:00:00.000Z', + }); + + await service.resolve(movieDetail({}, local)); + await service.resolve(movieDetail({}, local)); + + expect(tmdb.enrichMovie).toHaveBeenCalledTimes(2); + expect(downloads.updateMetadata).not.toHaveBeenCalled(); + }); + + it('expires a completed sparse refresh throttle with the TMDB TTL', async () => { + jest.useFakeTimers(); + jest.setSystemTime(new Date(NOW)); + try { + db.getContentByXtreamId.mockResolvedValue({ + ...providerContent('Local title', 41), + category_id: 4, + poster_url: '', + }); + const local = snapshot({ + plot: undefined, + posterUrl: undefined, + backdropUrl: undefined, + enrichedAt: '2020-01-01T00:00:00.000Z', + }); + + const refreshed = await service.resolve(movieDetail({}, local)); + await service.resolve(movieDetail({}, refreshed)); + jest.advanceTimersByTime(TMDB_DETAILS_CACHE_TTL_MS + 1); + await service.resolve(movieDetail({}, refreshed)); + + expect(db.getContentByXtreamId).toHaveBeenCalledTimes(2); + expect(downloads.updateMetadata).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }); + it('persists provider metadata without marking a failed TMDB refresh current', async () => { jest.useFakeTimers(); jest.setSystemTime(new Date(NOW)); diff --git a/libs/portal/downloads/feature/src/lib/offline-detail/download-offline-metadata.service.ts b/libs/portal/downloads/feature/src/lib/offline-detail/download-offline-metadata.service.ts index 5bd169a05..1a9010d66 100644 --- a/libs/portal/downloads/feature/src/lib/offline-detail/download-offline-metadata.service.ts +++ b/libs/portal/downloads/feature/src/lib/offline-detail/download-offline-metadata.service.ts @@ -22,8 +22,9 @@ import { type DownloadMetadataProviderSource, } from './download-metadata.mapper'; import { + currentMetadataLanguage, finalizeMetadataRefresh, - LatestMetadataWriteGuard, + MetadataRefreshCoordinator, needsMetadataRefresh, TMDB_REFRESH_STATE, type TmdbRefreshResult, @@ -151,17 +152,17 @@ export class DownloadOfflineMetadataService { private readonly playlists = inject(PlaylistsService); private readonly settings = inject(SettingsStore); private readonly tmdb = inject(TmdbEnrichmentService); - private readonly writeGuard = new LatestMetadataWriteGuard(); + private readonly refreshes = new MetadataRefreshCoordinator(); async resolve( detail: DownloadOfflineDetail ): Promise { const writeKey = metadataWriteKey(detail); - const generation = this.writeGuard.begin(writeKey); + const generation = this.refreshes.begin(writeKey); try { return await this.resolveGeneration(detail, writeKey, generation); } finally { - this.writeGuard.finish(writeKey, generation); + this.refreshes.finish(writeKey, generation); } } @@ -170,10 +171,16 @@ export class DownloadOfflineMetadataService { writeKey: string, generation: number ): Promise { - const language = this.currentLanguage(); + const language = currentMetadataLanguage(this.settings); const persisted = existingSnapshot(detail); const local = persisted ?? fallbackSnapshot(detail, language); - if (!needsMetadataRefresh(persisted, language)) { + const completedSparseAttempt = this.refreshes.isCompleted( + writeKey, + language + ); + if ( + !needsMetadataRefresh(persisted, language, completedSparseAttempt) + ) { return local; } @@ -210,27 +217,20 @@ export class DownloadOfflineMetadataService { return finalized.snapshot; } - if (this.writeGuard.isLatest(writeKey, generation)) { - try { - await this.downloads.updateMetadata( + await this.refreshes.persistLatest({ + completedAttempt: finalized.completedAttempt, + generation, + groupKey: writeKey, + language, + persist: () => + this.downloads.updateMetadata( representative(detail).id, finalized.snapshot - ); - } catch { - // Enriched metadata remains useful without backfill. - } - } + ), + }); return finalized.snapshot; } - private currentLanguage(): string { - try { - return this.settings.language().trim() || 'en'; - } catch { - return 'en'; - } - } - private async loadProviderContext( detail: DownloadOfflineDetail ): Promise {