mirror of
https://github.com/4gray/iptvnator.git
synced 2026-10-09 17:36:15 -08:00
fix(downloads): throttle sparse metadata refreshes
This commit is contained in:
1 parent
082c90caa1
commit
4252bea2e0
7 files changed
+346
-32
No files matched your search
@@ -123,8 +123,10 @@ variants, contextual buttons, and theme-aware styling.
|
||||
typed provider catalog and optional TMDB enrichment. A successful refresh
|
||||
advances its freshness timestamp; transient TMDB failures may retain improved
|
||||
provider fields without advancing the timestamp or language, so a later visit
|
||||
retries. Concurrent movie and grouped-series refreshes are request-ordered so
|
||||
only the newest generation may persist.
|
||||
retries. Legitimately sparse successful results are remembered in a bounded,
|
||||
language-scoped runtime throttle for the same TTL, preventing persistence
|
||||
reloads from starting an enrichment loop. Concurrent movie and grouped-series
|
||||
refreshes are request-ordered so only the newest generation may persist.
|
||||
- Snapshot artwork accepts only safe HTTP(S) image URLs. Renderer normalization
|
||||
and main-process persistence independently reject credential-shaped path and
|
||||
query keys, including `username`, so provider credentials cannot be cached in
|
||||
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
import { TMDB_DETAILS_CACHE_TTL_MS } from '@iptvnator/services';
|
||||
import {
|
||||
COMPLETED_METADATA_REFRESH_MAX_ENTRIES,
|
||||
CompletedMetadataRefreshThrottle,
|
||||
} from './download-metadata-refresh';
|
||||
|
||||
describe('CompletedMetadataRefreshThrottle', () => {
|
||||
beforeEach(() => {
|
||||
jest.useFakeTimers();
|
||||
jest.setSystemTime(new Date('2026-07-31T08:30:00.000Z'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('expires completed attempts with the TMDB details TTL', () => {
|
||||
const throttle = new CompletedMetadataRefreshThrottle();
|
||||
throttle.mark('download:17', 'en', 1);
|
||||
|
||||
expect(throttle.isCompleted('download:17', 'en')).toBe(true);
|
||||
|
||||
jest.advanceTimersByTime(TMDB_DETAILS_CACHE_TTL_MS + 1);
|
||||
|
||||
expect(throttle.isCompleted('download:17', 'en')).toBe(false);
|
||||
});
|
||||
|
||||
it('bounds completed attempts by evicting the oldest group and language', () => {
|
||||
const throttle = new CompletedMetadataRefreshThrottle();
|
||||
for (
|
||||
let index = 0;
|
||||
index <= COMPLETED_METADATA_REFRESH_MAX_ENTRIES;
|
||||
index += 1
|
||||
) {
|
||||
throttle.mark(`download:${index}`, 'en', index + 1);
|
||||
}
|
||||
|
||||
expect(throttle.isCompleted('download:0', 'en')).toBe(false);
|
||||
expect(
|
||||
throttle.isCompleted(
|
||||
`download:${COMPLETED_METADATA_REFRESH_MAX_ENTRIES}`,
|
||||
'en'
|
||||
)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('does not let older cleanup erase a newer completed generation', () => {
|
||||
const throttle = new CompletedMetadataRefreshThrottle();
|
||||
throttle.mark('series:playlist-a:93', 'en', 1);
|
||||
throttle.mark('series:playlist-a:93', 'en', 2);
|
||||
|
||||
throttle.clear('series:playlist-a:93', 'en', 1);
|
||||
|
||||
expect(throttle.isCompleted('series:playlist-a:93', 'en')).toBe(true);
|
||||
|
||||
throttle.clear('series:playlist-a:93', 'en', 2);
|
||||
|
||||
expect(throttle.isCompleted('series:playlist-a:93', 'en')).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,7 @@
|
||||
import { TMDB_DETAILS_CACHE_TTL_MS } from '@iptvnator/services';
|
||||
import {
|
||||
TMDB_DETAILS_CACHE_TTL_MS,
|
||||
type SettingsStore,
|
||||
} from '@iptvnator/services';
|
||||
import type { DownloadMetadataSnapshot } from '@iptvnator/shared/interfaces';
|
||||
|
||||
export const TMDB_REFRESH_STATE = {
|
||||
@@ -24,10 +27,26 @@ interface FinalizeMetadataRefreshInput {
|
||||
}
|
||||
|
||||
export interface FinalizedMetadataRefresh {
|
||||
completedAttempt: boolean;
|
||||
shouldPersist: boolean;
|
||||
snapshot: DownloadMetadataSnapshot;
|
||||
}
|
||||
|
||||
export const COMPLETED_METADATA_REFRESH_MAX_ENTRIES = 256;
|
||||
|
||||
interface CompletedMetadataRefresh {
|
||||
expiresAt: number;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
interface PersistMetadataRefreshInput {
|
||||
completedAttempt: boolean;
|
||||
generation: number;
|
||||
groupKey: string;
|
||||
language: string;
|
||||
persist(): Promise<{ success: boolean }>;
|
||||
}
|
||||
|
||||
function isSparse(snapshot: DownloadMetadataSnapshot): boolean {
|
||||
return !snapshot.plot || (!snapshot.posterUrl && !snapshot.backdropUrl);
|
||||
}
|
||||
@@ -42,16 +61,28 @@ function isFresh(snapshot: DownloadMetadataSnapshot): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
export function currentMetadataLanguage(
|
||||
settings: Pick<SettingsStore, 'language'>
|
||||
): string {
|
||||
try {
|
||||
return settings.language().trim() || 'en';
|
||||
} catch {
|
||||
return 'en';
|
||||
}
|
||||
}
|
||||
|
||||
export function needsMetadataRefresh(
|
||||
snapshot: DownloadMetadataSnapshot | undefined,
|
||||
language: string
|
||||
language: string,
|
||||
completedSparseAttempt = false
|
||||
): boolean {
|
||||
return (
|
||||
!snapshot ||
|
||||
snapshot.language !== language ||
|
||||
isSparse(snapshot) ||
|
||||
!isFresh(snapshot)
|
||||
);
|
||||
if (!snapshot || snapshot.language !== language) {
|
||||
return true;
|
||||
}
|
||||
if (isSparse(snapshot)) {
|
||||
return !completedSparseAttempt;
|
||||
}
|
||||
return !isFresh(snapshot);
|
||||
}
|
||||
|
||||
export function withoutMetadataFreshness(
|
||||
@@ -117,11 +148,57 @@ export function finalizeMetadataRefresh({
|
||||
const shouldPersist =
|
||||
refreshSucceeded || !materiallyEqual(local, finalized);
|
||||
return {
|
||||
completedAttempt: refreshSucceeded,
|
||||
shouldPersist,
|
||||
snapshot: shouldPersist ? finalized : local,
|
||||
};
|
||||
}
|
||||
|
||||
function completedRefreshKey(groupKey: string, language: string): string {
|
||||
return `${groupKey}\u0000${language}`;
|
||||
}
|
||||
|
||||
export class CompletedMetadataRefreshThrottle {
|
||||
private readonly attempts = new Map<string, CompletedMetadataRefresh>();
|
||||
|
||||
isCompleted(groupKey: string, language: string): boolean {
|
||||
const now = Date.now();
|
||||
this.deleteExpired(now);
|
||||
return this.attempts.has(completedRefreshKey(groupKey, language));
|
||||
}
|
||||
|
||||
mark(groupKey: string, language: string, generation: number): void {
|
||||
const now = Date.now();
|
||||
const key = completedRefreshKey(groupKey, language);
|
||||
this.deleteExpired(now);
|
||||
this.attempts.delete(key);
|
||||
while (this.attempts.size >= COMPLETED_METADATA_REFRESH_MAX_ENTRIES) {
|
||||
const oldestKey = this.attempts.keys().next().value;
|
||||
if (oldestKey === undefined) break;
|
||||
this.attempts.delete(oldestKey);
|
||||
}
|
||||
this.attempts.set(key, {
|
||||
expiresAt: now + TMDB_DETAILS_CACHE_TTL_MS,
|
||||
generation,
|
||||
});
|
||||
}
|
||||
|
||||
clear(groupKey: string, language: string, generation: number): void {
|
||||
const key = completedRefreshKey(groupKey, language);
|
||||
if (this.attempts.get(key)?.generation === generation) {
|
||||
this.attempts.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
private deleteExpired(now: number): void {
|
||||
for (const [key, attempt] of this.attempts) {
|
||||
if (attempt.expiresAt < now) {
|
||||
this.attempts.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class LatestMetadataWriteGuard {
|
||||
private generation = 0;
|
||||
private readonly latestByKey = new Map<string, number>();
|
||||
@@ -142,3 +219,43 @@ export class LatestMetadataWriteGuard {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class MetadataRefreshCoordinator {
|
||||
private readonly completed = new CompletedMetadataRefreshThrottle();
|
||||
private readonly writes = new LatestMetadataWriteGuard();
|
||||
|
||||
begin(groupKey: string): number {
|
||||
return this.writes.begin(groupKey);
|
||||
}
|
||||
|
||||
finish(groupKey: string, generation: number): void {
|
||||
this.writes.finish(groupKey, generation);
|
||||
}
|
||||
|
||||
isCompleted(groupKey: string, language: string): boolean {
|
||||
return this.completed.isCompleted(groupKey, language);
|
||||
}
|
||||
|
||||
async persistLatest({
|
||||
completedAttempt,
|
||||
generation,
|
||||
groupKey,
|
||||
language,
|
||||
persist,
|
||||
}: PersistMetadataRefreshInput): Promise<void> {
|
||||
if (!this.writes.isLatest(groupKey, generation)) return;
|
||||
if (completedAttempt) {
|
||||
this.completed.mark(groupKey, language, generation);
|
||||
}
|
||||
try {
|
||||
const result = await persist();
|
||||
if (!result.success && completedAttempt) {
|
||||
this.completed.clear(groupKey, language, generation);
|
||||
}
|
||||
} catch {
|
||||
if (completedAttempt) {
|
||||
this.completed.clear(groupKey, language, generation);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -283,6 +283,27 @@ describe('download metadata mapper', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back from blank nested Stalker artwork to root poster_url', () => {
|
||||
const mapped = mapProviderToDownloadSnapshot({
|
||||
source: 'stalker',
|
||||
language: 'en',
|
||||
mediaKind: 'movie',
|
||||
fallback: providerSnapshot,
|
||||
provider: {
|
||||
poster_url:
|
||||
'https://images.example.test/posters/canonical-recent.jpg',
|
||||
info: {
|
||||
name: 'Nested title',
|
||||
movie_image: '',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(mapped.posterUrl).toBe(
|
||||
'https://images.example.test/posters/canonical-recent.jpg'
|
||||
);
|
||||
});
|
||||
|
||||
it('bounds mapped people and genres to the persisted DTO limits', () => {
|
||||
const mapped = mapProviderToDownloadSnapshot({
|
||||
source: 'xtream',
|
||||
|
||||
@@ -185,6 +185,7 @@ export function mapProviderToDownloadSnapshot({
|
||||
? (string(info?.['movie_image']) ??
|
||||
string(root['cover']) ??
|
||||
string(root['screenshot_uri']) ??
|
||||
string(root['poster_url']) ??
|
||||
string(root['logo']))
|
||||
: string(
|
||||
first(editorial, [
|
||||
|
||||
+113
@@ -5,6 +5,7 @@ import {
|
||||
DownloadsService,
|
||||
PlaylistsService,
|
||||
SettingsStore,
|
||||
TMDB_DETAILS_CACHE_TTL_MS,
|
||||
TmdbEnrichmentService,
|
||||
type TmdbMovieDetails,
|
||||
type TmdbTvDetails,
|
||||
@@ -510,6 +511,118 @@ describe('DownloadOfflineMetadataService', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('throttles a successful sparse provider refresh per language during reload', async () => {
|
||||
jest.useFakeTimers();
|
||||
jest.setSystemTime(new Date(NOW));
|
||||
try {
|
||||
db.getContentByXtreamId.mockResolvedValue({
|
||||
...providerContent('Local title', 41),
|
||||
category_id: 4,
|
||||
poster_url: '',
|
||||
});
|
||||
const local = snapshot({
|
||||
plot: undefined,
|
||||
posterUrl: undefined,
|
||||
backdropUrl: undefined,
|
||||
enrichedAt: '2020-01-01T00:00:00.000Z',
|
||||
});
|
||||
downloads.updateMetadata.mockImplementationOnce(
|
||||
async (_downloadId, refreshed) => {
|
||||
await service.resolve(movieDetail({}, refreshed));
|
||||
return { success: true };
|
||||
}
|
||||
);
|
||||
|
||||
const refreshed = await service.resolve(movieDetail({}, local));
|
||||
|
||||
expect(refreshed.plot).toBeUndefined();
|
||||
expect(refreshed.posterUrl).toBeUndefined();
|
||||
expect(db.getContentByXtreamId).toHaveBeenCalledTimes(1);
|
||||
expect(downloads.updateMetadata).toHaveBeenCalledTimes(1);
|
||||
|
||||
currentLanguage = 'de';
|
||||
const translated = await service.resolve(
|
||||
movieDetail({}, refreshed)
|
||||
);
|
||||
|
||||
expect(translated.language).toBe('de');
|
||||
expect(db.getContentByXtreamId).toHaveBeenCalledTimes(2);
|
||||
expect(downloads.updateMetadata).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('throttles a successful sparse materially identical TMDB refresh', async () => {
|
||||
jest.useFakeTimers();
|
||||
jest.setSystemTime(new Date(NOW));
|
||||
try {
|
||||
tmdb.isEnabled.mockReturnValue(true);
|
||||
tmdb.enrichMovie.mockResolvedValue({ id: 603 });
|
||||
const local = snapshot({
|
||||
plot: undefined,
|
||||
posterUrl: undefined,
|
||||
backdropUrl: undefined,
|
||||
tmdbId: 603,
|
||||
rating: 0,
|
||||
enrichedAt: '2020-01-01T00:00:00.000Z',
|
||||
});
|
||||
|
||||
const refreshed = await service.resolve(movieDetail({}, local));
|
||||
await service.resolve(movieDetail({}, refreshed));
|
||||
|
||||
expect(tmdb.enrichMovie).toHaveBeenCalledTimes(1);
|
||||
expect(downloads.updateMetadata).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('retries a sparse transient TMDB failure', async () => {
|
||||
tmdb.isEnabled.mockReturnValue(true);
|
||||
tmdb.enrichMovie.mockRejectedValue(new Error('TMDB unavailable'));
|
||||
const local = snapshot({
|
||||
plot: undefined,
|
||||
posterUrl: undefined,
|
||||
backdropUrl: undefined,
|
||||
enrichedAt: '2020-01-01T00:00:00.000Z',
|
||||
});
|
||||
|
||||
await service.resolve(movieDetail({}, local));
|
||||
await service.resolve(movieDetail({}, local));
|
||||
|
||||
expect(tmdb.enrichMovie).toHaveBeenCalledTimes(2);
|
||||
expect(downloads.updateMetadata).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('expires a completed sparse refresh throttle with the TMDB TTL', async () => {
|
||||
jest.useFakeTimers();
|
||||
jest.setSystemTime(new Date(NOW));
|
||||
try {
|
||||
db.getContentByXtreamId.mockResolvedValue({
|
||||
...providerContent('Local title', 41),
|
||||
category_id: 4,
|
||||
poster_url: '',
|
||||
});
|
||||
const local = snapshot({
|
||||
plot: undefined,
|
||||
posterUrl: undefined,
|
||||
backdropUrl: undefined,
|
||||
enrichedAt: '2020-01-01T00:00:00.000Z',
|
||||
});
|
||||
|
||||
const refreshed = await service.resolve(movieDetail({}, local));
|
||||
await service.resolve(movieDetail({}, refreshed));
|
||||
jest.advanceTimersByTime(TMDB_DETAILS_CACHE_TTL_MS + 1);
|
||||
await service.resolve(movieDetail({}, refreshed));
|
||||
|
||||
expect(db.getContentByXtreamId).toHaveBeenCalledTimes(2);
|
||||
expect(downloads.updateMetadata).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('persists provider metadata without marking a failed TMDB refresh current', async () => {
|
||||
jest.useFakeTimers();
|
||||
jest.setSystemTime(new Date(NOW));
|
||||
|
||||
+22
-22
@@ -22,8 +22,9 @@ import {
|
||||
type DownloadMetadataProviderSource,
|
||||
} from './download-metadata.mapper';
|
||||
import {
|
||||
currentMetadataLanguage,
|
||||
finalizeMetadataRefresh,
|
||||
LatestMetadataWriteGuard,
|
||||
MetadataRefreshCoordinator,
|
||||
needsMetadataRefresh,
|
||||
TMDB_REFRESH_STATE,
|
||||
type TmdbRefreshResult,
|
||||
@@ -151,17 +152,17 @@ export class DownloadOfflineMetadataService {
|
||||
private readonly playlists = inject(PlaylistsService);
|
||||
private readonly settings = inject(SettingsStore);
|
||||
private readonly tmdb = inject(TmdbEnrichmentService);
|
||||
private readonly writeGuard = new LatestMetadataWriteGuard();
|
||||
private readonly refreshes = new MetadataRefreshCoordinator();
|
||||
|
||||
async resolve(
|
||||
detail: DownloadOfflineDetail
|
||||
): Promise<DownloadMetadataSnapshot> {
|
||||
const writeKey = metadataWriteKey(detail);
|
||||
const generation = this.writeGuard.begin(writeKey);
|
||||
const generation = this.refreshes.begin(writeKey);
|
||||
try {
|
||||
return await this.resolveGeneration(detail, writeKey, generation);
|
||||
} finally {
|
||||
this.writeGuard.finish(writeKey, generation);
|
||||
this.refreshes.finish(writeKey, generation);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,10 +171,16 @@ export class DownloadOfflineMetadataService {
|
||||
writeKey: string,
|
||||
generation: number
|
||||
): Promise<DownloadMetadataSnapshot> {
|
||||
const language = this.currentLanguage();
|
||||
const language = currentMetadataLanguage(this.settings);
|
||||
const persisted = existingSnapshot(detail);
|
||||
const local = persisted ?? fallbackSnapshot(detail, language);
|
||||
if (!needsMetadataRefresh(persisted, language)) {
|
||||
const completedSparseAttempt = this.refreshes.isCompleted(
|
||||
writeKey,
|
||||
language
|
||||
);
|
||||
if (
|
||||
!needsMetadataRefresh(persisted, language, completedSparseAttempt)
|
||||
) {
|
||||
return local;
|
||||
}
|
||||
|
||||
@@ -210,27 +217,20 @@ export class DownloadOfflineMetadataService {
|
||||
return finalized.snapshot;
|
||||
}
|
||||
|
||||
if (this.writeGuard.isLatest(writeKey, generation)) {
|
||||
try {
|
||||
await this.downloads.updateMetadata(
|
||||
await this.refreshes.persistLatest({
|
||||
completedAttempt: finalized.completedAttempt,
|
||||
generation,
|
||||
groupKey: writeKey,
|
||||
language,
|
||||
persist: () =>
|
||||
this.downloads.updateMetadata(
|
||||
representative(detail).id,
|
||||
finalized.snapshot
|
||||
);
|
||||
} catch {
|
||||
// Enriched metadata remains useful without backfill.
|
||||
}
|
||||
}
|
||||
),
|
||||
});
|
||||
return finalized.snapshot;
|
||||
}
|
||||
|
||||
private currentLanguage(): string {
|
||||
try {
|
||||
return this.settings.language().trim() || 'en';
|
||||
} catch {
|
||||
return 'en';
|
||||
}
|
||||
}
|
||||
|
||||
private async loadProviderContext(
|
||||
detail: DownloadOfflineDetail
|
||||
): Promise<ProviderContext> {
|
||||
|
||||
Reference in new issue
Block a user