fix(downloads): throttle sparse metadata refreshes

This commit is contained in:
4gray committed 2026-08-01 16:29:50 +02:00
1 parent 082c90caa1
commit 4252bea2e0
7 files changed
+346 -32

No files matched your search

+4 -2
View File
@@ -123,8 +123,10 @@ variants, contextual buttons, and theme-aware styling.
typed provider catalog and optional TMDB enrichment. A successful refresh
advances its freshness timestamp; transient TMDB failures may retain improved
provider fields without advancing the timestamp or language, so a later visit
retries. Concurrent movie and grouped-series refreshes are request-ordered so
only the newest generation may persist.
retries. Legitimately sparse successful results are remembered in a bounded,
language-scoped runtime throttle for the same TTL, preventing persistence
reloads from starting an enrichment loop. Concurrent movie and grouped-series
refreshes are request-ordered so only the newest generation may persist.
- Snapshot artwork accepts only safe HTTP(S) image URLs. Renderer normalization
and main-process persistence independently reject credential-shaped path and
query keys, including `username`, so provider credentials cannot be cached in
@@ -0,0 +1,60 @@
import { TMDB_DETAILS_CACHE_TTL_MS } from '@iptvnator/services';
import {
COMPLETED_METADATA_REFRESH_MAX_ENTRIES,
CompletedMetadataRefreshThrottle,
} from './download-metadata-refresh';
describe('CompletedMetadataRefreshThrottle', () => {
beforeEach(() => {
jest.useFakeTimers();
jest.setSystemTime(new Date('2026-07-31T08:30:00.000Z'));
});
afterEach(() => {
jest.useRealTimers();
});
it('expires completed attempts with the TMDB details TTL', () => {
const throttle = new CompletedMetadataRefreshThrottle();
throttle.mark('download:17', 'en', 1);
expect(throttle.isCompleted('download:17', 'en')).toBe(true);
jest.advanceTimersByTime(TMDB_DETAILS_CACHE_TTL_MS + 1);
expect(throttle.isCompleted('download:17', 'en')).toBe(false);
});
it('bounds completed attempts by evicting the oldest group and language', () => {
const throttle = new CompletedMetadataRefreshThrottle();
for (
let index = 0;
index <= COMPLETED_METADATA_REFRESH_MAX_ENTRIES;
index += 1
) {
throttle.mark(`download:${index}`, 'en', index + 1);
}
expect(throttle.isCompleted('download:0', 'en')).toBe(false);
expect(
throttle.isCompleted(
`download:${COMPLETED_METADATA_REFRESH_MAX_ENTRIES}`,
'en'
)
).toBe(true);
});
it('does not let older cleanup erase a newer completed generation', () => {
const throttle = new CompletedMetadataRefreshThrottle();
throttle.mark('series:playlist-a:93', 'en', 1);
throttle.mark('series:playlist-a:93', 'en', 2);
throttle.clear('series:playlist-a:93', 'en', 1);
expect(throttle.isCompleted('series:playlist-a:93', 'en')).toBe(true);
throttle.clear('series:playlist-a:93', 'en', 2);
expect(throttle.isCompleted('series:playlist-a:93', 'en')).toBe(false);
});
});
@@ -1,4 +1,7 @@
import { TMDB_DETAILS_CACHE_TTL_MS } from '@iptvnator/services';
import {
TMDB_DETAILS_CACHE_TTL_MS,
type SettingsStore,
} from '@iptvnator/services';
import type { DownloadMetadataSnapshot } from '@iptvnator/shared/interfaces';
export const TMDB_REFRESH_STATE = {
@@ -24,10 +27,26 @@ interface FinalizeMetadataRefreshInput {
}
export interface FinalizedMetadataRefresh {
completedAttempt: boolean;
shouldPersist: boolean;
snapshot: DownloadMetadataSnapshot;
}
export const COMPLETED_METADATA_REFRESH_MAX_ENTRIES = 256;
interface CompletedMetadataRefresh {
expiresAt: number;
generation: number;
}
interface PersistMetadataRefreshInput {
completedAttempt: boolean;
generation: number;
groupKey: string;
language: string;
persist(): Promise<{ success: boolean }>;
}
function isSparse(snapshot: DownloadMetadataSnapshot): boolean {
return !snapshot.plot || (!snapshot.posterUrl && !snapshot.backdropUrl);
}
@@ -42,16 +61,28 @@ function isFresh(snapshot: DownloadMetadataSnapshot): boolean {
);
}
export function currentMetadataLanguage(
settings: Pick<SettingsStore, 'language'>
): string {
try {
return settings.language().trim() || 'en';
} catch {
return 'en';
}
}
export function needsMetadataRefresh(
snapshot: DownloadMetadataSnapshot | undefined,
language: string
language: string,
completedSparseAttempt = false
): boolean {
return (
!snapshot ||
snapshot.language !== language ||
isSparse(snapshot) ||
!isFresh(snapshot)
);
if (!snapshot || snapshot.language !== language) {
return true;
}
if (isSparse(snapshot)) {
return !completedSparseAttempt;
}
return !isFresh(snapshot);
}
export function withoutMetadataFreshness(
@@ -117,11 +148,57 @@ export function finalizeMetadataRefresh({
const shouldPersist =
refreshSucceeded || !materiallyEqual(local, finalized);
return {
completedAttempt: refreshSucceeded,
shouldPersist,
snapshot: shouldPersist ? finalized : local,
};
}
function completedRefreshKey(groupKey: string, language: string): string {
return `${groupKey}\u0000${language}`;
}
export class CompletedMetadataRefreshThrottle {
private readonly attempts = new Map<string, CompletedMetadataRefresh>();
isCompleted(groupKey: string, language: string): boolean {
const now = Date.now();
this.deleteExpired(now);
return this.attempts.has(completedRefreshKey(groupKey, language));
}
mark(groupKey: string, language: string, generation: number): void {
const now = Date.now();
const key = completedRefreshKey(groupKey, language);
this.deleteExpired(now);
this.attempts.delete(key);
while (this.attempts.size >= COMPLETED_METADATA_REFRESH_MAX_ENTRIES) {
const oldestKey = this.attempts.keys().next().value;
if (oldestKey === undefined) break;
this.attempts.delete(oldestKey);
}
this.attempts.set(key, {
expiresAt: now + TMDB_DETAILS_CACHE_TTL_MS,
generation,
});
}
clear(groupKey: string, language: string, generation: number): void {
const key = completedRefreshKey(groupKey, language);
if (this.attempts.get(key)?.generation === generation) {
this.attempts.delete(key);
}
}
private deleteExpired(now: number): void {
for (const [key, attempt] of this.attempts) {
if (attempt.expiresAt < now) {
this.attempts.delete(key);
}
}
}
}
export class LatestMetadataWriteGuard {
private generation = 0;
private readonly latestByKey = new Map<string, number>();
@@ -142,3 +219,43 @@ export class LatestMetadataWriteGuard {
}
}
}
export class MetadataRefreshCoordinator {
private readonly completed = new CompletedMetadataRefreshThrottle();
private readonly writes = new LatestMetadataWriteGuard();
begin(groupKey: string): number {
return this.writes.begin(groupKey);
}
finish(groupKey: string, generation: number): void {
this.writes.finish(groupKey, generation);
}
isCompleted(groupKey: string, language: string): boolean {
return this.completed.isCompleted(groupKey, language);
}
async persistLatest({
completedAttempt,
generation,
groupKey,
language,
persist,
}: PersistMetadataRefreshInput): Promise<void> {
if (!this.writes.isLatest(groupKey, generation)) return;
if (completedAttempt) {
this.completed.mark(groupKey, language, generation);
}
try {
const result = await persist();
if (!result.success && completedAttempt) {
this.completed.clear(groupKey, language, generation);
}
} catch {
if (completedAttempt) {
this.completed.clear(groupKey, language, generation);
}
}
}
}
@@ -283,6 +283,27 @@ describe('download metadata mapper', () => {
);
});
it('falls back from blank nested Stalker artwork to root poster_url', () => {
const mapped = mapProviderToDownloadSnapshot({
source: 'stalker',
language: 'en',
mediaKind: 'movie',
fallback: providerSnapshot,
provider: {
poster_url:
'https://images.example.test/posters/canonical-recent.jpg',
info: {
name: 'Nested title',
movie_image: '',
},
},
});
expect(mapped.posterUrl).toBe(
'https://images.example.test/posters/canonical-recent.jpg'
);
});
it('bounds mapped people and genres to the persisted DTO limits', () => {
const mapped = mapProviderToDownloadSnapshot({
source: 'xtream',
@@ -185,6 +185,7 @@ export function mapProviderToDownloadSnapshot({
? (string(info?.['movie_image']) ??
string(root['cover']) ??
string(root['screenshot_uri']) ??
string(root['poster_url']) ??
string(root['logo']))
: string(
first(editorial, [
@@ -5,6 +5,7 @@ import {
DownloadsService,
PlaylistsService,
SettingsStore,
TMDB_DETAILS_CACHE_TTL_MS,
TmdbEnrichmentService,
type TmdbMovieDetails,
type TmdbTvDetails,
@@ -510,6 +511,118 @@ describe('DownloadOfflineMetadataService', () => {
}
});
it('throttles a successful sparse provider refresh per language during reload', async () => {
jest.useFakeTimers();
jest.setSystemTime(new Date(NOW));
try {
db.getContentByXtreamId.mockResolvedValue({
...providerContent('Local title', 41),
category_id: 4,
poster_url: '',
});
const local = snapshot({
plot: undefined,
posterUrl: undefined,
backdropUrl: undefined,
enrichedAt: '2020-01-01T00:00:00.000Z',
});
downloads.updateMetadata.mockImplementationOnce(
async (_downloadId, refreshed) => {
await service.resolve(movieDetail({}, refreshed));
return { success: true };
}
);
const refreshed = await service.resolve(movieDetail({}, local));
expect(refreshed.plot).toBeUndefined();
expect(refreshed.posterUrl).toBeUndefined();
expect(db.getContentByXtreamId).toHaveBeenCalledTimes(1);
expect(downloads.updateMetadata).toHaveBeenCalledTimes(1);
currentLanguage = 'de';
const translated = await service.resolve(
movieDetail({}, refreshed)
);
expect(translated.language).toBe('de');
expect(db.getContentByXtreamId).toHaveBeenCalledTimes(2);
expect(downloads.updateMetadata).toHaveBeenCalledTimes(2);
} finally {
jest.useRealTimers();
}
});
it('throttles a successful sparse materially identical TMDB refresh', async () => {
jest.useFakeTimers();
jest.setSystemTime(new Date(NOW));
try {
tmdb.isEnabled.mockReturnValue(true);
tmdb.enrichMovie.mockResolvedValue({ id: 603 });
const local = snapshot({
plot: undefined,
posterUrl: undefined,
backdropUrl: undefined,
tmdbId: 603,
rating: 0,
enrichedAt: '2020-01-01T00:00:00.000Z',
});
const refreshed = await service.resolve(movieDetail({}, local));
await service.resolve(movieDetail({}, refreshed));
expect(tmdb.enrichMovie).toHaveBeenCalledTimes(1);
expect(downloads.updateMetadata).toHaveBeenCalledTimes(1);
} finally {
jest.useRealTimers();
}
});
it('retries a sparse transient TMDB failure', async () => {
tmdb.isEnabled.mockReturnValue(true);
tmdb.enrichMovie.mockRejectedValue(new Error('TMDB unavailable'));
const local = snapshot({
plot: undefined,
posterUrl: undefined,
backdropUrl: undefined,
enrichedAt: '2020-01-01T00:00:00.000Z',
});
await service.resolve(movieDetail({}, local));
await service.resolve(movieDetail({}, local));
expect(tmdb.enrichMovie).toHaveBeenCalledTimes(2);
expect(downloads.updateMetadata).not.toHaveBeenCalled();
});
it('expires a completed sparse refresh throttle with the TMDB TTL', async () => {
jest.useFakeTimers();
jest.setSystemTime(new Date(NOW));
try {
db.getContentByXtreamId.mockResolvedValue({
...providerContent('Local title', 41),
category_id: 4,
poster_url: '',
});
const local = snapshot({
plot: undefined,
posterUrl: undefined,
backdropUrl: undefined,
enrichedAt: '2020-01-01T00:00:00.000Z',
});
const refreshed = await service.resolve(movieDetail({}, local));
await service.resolve(movieDetail({}, refreshed));
jest.advanceTimersByTime(TMDB_DETAILS_CACHE_TTL_MS + 1);
await service.resolve(movieDetail({}, refreshed));
expect(db.getContentByXtreamId).toHaveBeenCalledTimes(2);
expect(downloads.updateMetadata).toHaveBeenCalledTimes(2);
} finally {
jest.useRealTimers();
}
});
it('persists provider metadata without marking a failed TMDB refresh current', async () => {
jest.useFakeTimers();
jest.setSystemTime(new Date(NOW));
@@ -22,8 +22,9 @@ import {
type DownloadMetadataProviderSource,
} from './download-metadata.mapper';
import {
currentMetadataLanguage,
finalizeMetadataRefresh,
LatestMetadataWriteGuard,
MetadataRefreshCoordinator,
needsMetadataRefresh,
TMDB_REFRESH_STATE,
type TmdbRefreshResult,
@@ -151,17 +152,17 @@ export class DownloadOfflineMetadataService {
private readonly playlists = inject(PlaylistsService);
private readonly settings = inject(SettingsStore);
private readonly tmdb = inject(TmdbEnrichmentService);
private readonly writeGuard = new LatestMetadataWriteGuard();
private readonly refreshes = new MetadataRefreshCoordinator();
async resolve(
detail: DownloadOfflineDetail
): Promise<DownloadMetadataSnapshot> {
const writeKey = metadataWriteKey(detail);
const generation = this.writeGuard.begin(writeKey);
const generation = this.refreshes.begin(writeKey);
try {
return await this.resolveGeneration(detail, writeKey, generation);
} finally {
this.writeGuard.finish(writeKey, generation);
this.refreshes.finish(writeKey, generation);
}
}
@@ -170,10 +171,16 @@ export class DownloadOfflineMetadataService {
writeKey: string,
generation: number
): Promise<DownloadMetadataSnapshot> {
const language = this.currentLanguage();
const language = currentMetadataLanguage(this.settings);
const persisted = existingSnapshot(detail);
const local = persisted ?? fallbackSnapshot(detail, language);
if (!needsMetadataRefresh(persisted, language)) {
const completedSparseAttempt = this.refreshes.isCompleted(
writeKey,
language
);
if (
!needsMetadataRefresh(persisted, language, completedSparseAttempt)
) {
return local;
}
@@ -210,27 +217,20 @@ export class DownloadOfflineMetadataService {
return finalized.snapshot;
}
if (this.writeGuard.isLatest(writeKey, generation)) {
try {
await this.downloads.updateMetadata(
await this.refreshes.persistLatest({
completedAttempt: finalized.completedAttempt,
generation,
groupKey: writeKey,
language,
persist: () =>
this.downloads.updateMetadata(
representative(detail).id,
finalized.snapshot
);
} catch {
// Enriched metadata remains useful without backfill.
}
}
),
});
return finalized.snapshot;
}
private currentLanguage(): string {
try {
return this.settings.language().trim() || 'en';
} catch {
return 'en';
}
}
private async loadProviderContext(
detail: DownloadOfflineDetail
): Promise<ProviderContext> {