From 37fac60d92d6ffc3e1964e74397f32c7192869f5 Mon Sep 17 00:00:00 2001 From: 4gray Date: Wed, 14 Oct 2026 15:31:29 +0200 Subject: [PATCH] fix(server-storage): M3U favorites by URL and history channel ids in the move; restores wait for the server The move's M3U remap swapped references for server channel ids, but the app looks favorites up by stream URL and history entries carry the URL as their id: legacy favorites became invisible and history duplicated. Legacy id favorites now become their channel's URL on both sides, and history keeps its URL id with the server's channel id as channel_id. A restore in server mode reported success once its lists were queued: a list the server refused (413) left the old data while the parked restore was consumed. Both restore paths now wait for that playlist's writes (UserDataStorage.flushed, a few flush rounds for conflict replays) and stay parked when the server did not take them. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/playlist-backup-restore.md | 3 +- docs/architecture/pwa-self-hosted.md | 8 ++-- .../pwa-xtream-data-source.spec.ts | 37 ++++++++++++++++++ .../data-sources/pwa-xtream-data-source.ts | 7 ++++ ...playlist-backup.service.pwa-xtream.spec.ts | 28 ++++++++++++++ .../lib/pwa-xtream-backup-state.service.ts | 8 ++++ .../server-migration-merge.spec.ts | 30 +++++++++------ .../server-storage/server-migration-merge.ts | 38 +++++++++---------- .../server-migration.service.spec.ts | 12 ++++-- .../server-migration.service.ts | 7 +++- .../server-user-data-adopt.spec.ts | 35 +++++++++++++++++ .../server-storage/server-user-data-adopt.ts | 24 ++++++++++++ .../server-user-data-storage.ts | 9 +++-- libs/services/src/lib/user-data-storage.ts | 13 +++++-- 14 files changed, 211 insertions(+), 48 deletions(-) diff --git a/docs/architecture/playlist-backup-restore.md b/docs/architecture/playlist-backup-restore.md index 5c28aa9be..52ac6db09 100644 --- a/docs/architecture/playlist-backup-restore.md +++ b/docs/architecture/playlist-backup-restore.md @@ -255,7 +255,8 @@ self-hosted server storage is covered too. So does server mode before its first read of the server's user data: the export waits for a read in flight (`UserDataStorage.ready`) and fails when none succeeded. A restore does the same and stays parked, since its - replacement lists would otherwise merge into the server's. + replacement lists would otherwise merge into the server's; it also stays + parked until the server took that playlist's lists. - A pending restore snapshot still wins over the stored maps, as on desktop. - Import parks the restore state like the desktop does, then applies it at once through `XtreamPendingRestoreService.applyAndConsume`: the browser has diff --git a/docs/architecture/pwa-self-hosted.md b/docs/architecture/pwa-self-hosted.md index 276ae0735..ed1292677 100644 --- a/docs/architecture/pwa-self-hosted.md +++ b/docs/architecture/pwa-self-hosted.md @@ -265,7 +265,8 @@ settings and guards read them in every build. restore (also its replay when the playlist opens) fails while no read succeeded: unread maps would export as empty lists, and a restore's lists would merge into the server's instead of replacing them (the restore - stays parked). Replayed edits keep recently viewed lists newest first, + stays parked). A restore also counts only once the server took that + playlist's lists (`UserDataStorage.flushed`); a refused one stays parked. Replayed edits keep recently viewed lists newest first, as the app cuts them at their limit from the end. Also, re-reads run one at a time and keep entries edited while they were in flight or not yet saved, and entries another tab saved meanwhile (the @@ -363,8 +364,9 @@ settings and guards read them in every build. user, Stalker portal and MAC, M3U URL), merges into the server's copy; the others upload with their ids. A merged playlist gains the browser document's favorites and history (M3U favorites name channels by stream - URL; older ones by a channel id that is random per import and is - translated through the URL; recently viewed newest first by `added_at`, + URL, as the app looks them up; older ones by a channel id that is random + per import, which becomes its URL on both sides; history entries keep + their URL id and get the server's channel id as `channel_id`; recently viewed newest first by `added_at`, one per item with its newest view; Stalker items by id, category and series flag, since one provider category can hold a movie and a series of one id; nothing is dropped), and the legacy Xtream maps follow under diff --git a/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.spec.ts b/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.spec.ts index d31e13a8d..646359f1d 100644 --- a/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.spec.ts +++ b/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.spec.ts @@ -1047,5 +1047,42 @@ describe('PwaXtreamDataSource', () => { JSON.stringify({ p1: ['live:1'] }) ); }); + + it('keeps a restore parked while the server refuses its lists', async () => { + const values = new Map(); + const flushed = jest.fn(async () => false); + const storage = { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => { + values.set(key, value); + }, + removeItem: (key: string) => { + values.delete(key); + }, + ready: async () => true, + flushed, + }; + TestBed.resetTestingModule(); + TestBed.configureTestingModule({ + providers: [ + PwaXtreamDataSource, + { provide: USER_DATA_STORAGE, useValue: storage }, + { provide: ParentalLockService, useValue: parentalLock }, + { provide: XtreamApiService, useValue: apiService }, + { provide: PlaylistsService, useValue: playlistsService }, + ], + }); + const source = TestBed.inject(PwaXtreamDataSource); + + await expect( + source.restoreUserData('p1', { + hiddenCategories: [], + favorites: [], + recentlyViewed: [], + playbackPositions: [], + } as never) + ).rejects.toThrow(/did not take the restored lists/); + expect(flushed).toHaveBeenCalledWith('p1'); + }); }); }); diff --git a/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.ts b/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.ts index 424f88789..0941c175b 100644 --- a/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.ts +++ b/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.ts @@ -1450,6 +1450,13 @@ export class PwaXtreamDataSource implements IXtreamDataSource { }) ); this.savePlaybackPositionsToStorage(playbackPositions); + // Server mode: done once the server took the lists, else it stays + // parked for the next open. + if ((await this.userDataStorage.flushed?.(playlistId)) === false) { + throw new Error( + 'The server did not take the restored lists; the restore waits.' + ); + } } // ========================================================================= diff --git a/libs/services/src/lib/playlist-backup.service.pwa-xtream.spec.ts b/libs/services/src/lib/playlist-backup.service.pwa-xtream.spec.ts index d593cb85e..e02f6a0d8 100644 --- a/libs/services/src/lib/playlist-backup.service.pwa-xtream.spec.ts +++ b/libs/services/src/lib/playlist-backup.service.pwa-xtream.spec.ts @@ -537,6 +537,34 @@ describe('PlaylistBackupService Xtream user state in the PWA', () => { ).not.toBeNull(); }); + it('keeps the restore parked until the server took the lists', async () => { + const manifest = await exportSeeded(); + localStorage.clear(); + const flushed = jest.fn(async () => false); + const service = createPlaylistBackupService({ + playlistsService: playlistsService([existingXtreamPlaylist]), + pwaXtream: createPwaXtreamBackupState({ + getItem: (key) => localStorage.getItem(key), + setItem: (key, value) => localStorage.setItem(key, value), + removeItem: (key) => localStorage.removeItem(key), + ready: async () => true, + flushed, + }), + }); + + const summary = await service.importBackup(xtreamOnly(manifest)); + + expect(flushed).toHaveBeenCalledWith('xtream-1'); + expect(summary.failed).toBe(1); + expect( + pendingRestore(service).getOrThrow('xtream-1') + ).not.toBeNull(); + // Titles are merged regardless: the replay carries none. + expect(read(KEYS.collectionItems)).toEqual({ + 'xtream-1': { 'movie:501': heat, 'series:77': legacyShow }, + }); + }); + it('replaces a damaged title cache but refuses an unreadable storage', async () => { const manifest = await exportSeeded(); localStorage.setItem(KEYS.collectionItems, '{"xtream-1": {'); diff --git a/libs/services/src/lib/pwa-xtream-backup-state.service.ts b/libs/services/src/lib/pwa-xtream-backup-state.service.ts index c3b1dfa74..b6baeb322 100644 --- a/libs/services/src/lib/pwa-xtream-backup-state.service.ts +++ b/libs/services/src/lib/pwa-xtream-backup-state.service.ts @@ -135,7 +135,15 @@ export class PwaXtreamBackupStateService { } throw error; } + // Server mode: the lists count once the server took them; a refused + // one keeps the restore parked (titles merged regardless, as above). + const saved = (await this.storage.flushed?.(playlistId)) !== false; this.mergeSnapshots(playlistId, snapshots); + if (!saved) { + throw new Error( + 'The server did not take the restored lists; the restore waits.' + ); + } } private replaceLists( diff --git a/libs/services/src/lib/server-storage/server-migration-merge.spec.ts b/libs/services/src/lib/server-storage/server-migration-merge.spec.ts index 3c2a2f607..9bcd7cbd8 100644 --- a/libs/services/src/lib/server-storage/server-migration-merge.spec.ts +++ b/libs/services/src/lib/server-storage/server-migration-merge.spec.ts @@ -137,30 +137,36 @@ describe('remapChannelReferences', () => { { id: 's2', url: 'http://s/2' }, ]; - it('names the server channel with the same stream URL', () => { + it('turns legacy channel-id favorites into their stream URL', () => { expect( - remapChannelReferences(['b2', 'b1', 's1'], browser, server) - ).toEqual(['s2', 's1', 's1']); + remapChannelReferences(['b2', 'b1', 'http://s/1'], browser, server) + ).toEqual(['http://s/2', 'http://s/1', 'http://s/1']); + // The server's own legacy ids, against its own channels. + expect(remapChannelReferences(['s1'], server, server)).toEqual([ + 'http://s/1', + ]); }); it('keeps URL favorites and anything it cannot resolve', () => { expect( remapChannelReferences( - ['http://s/1', 'http://elsewhere/9', 'b3'], + ['http://elsewhere/9', 'b9', 'b3'], browser, server ) - ).toEqual(['http://s/1', 'http://elsewhere/9', 'b3']); + ).toEqual(['http://elsewhere/9', 'b9', 'http://s/gone']); }); - it('updates M3U history entries and passes portal items through', () => { + it("keeps a history entry's URL id and names the server's channel", () => { const portal = { id: 7, category_id: 'itv' }; + const entry = { + source: 'm3u', + id: 'http://s/1', + url: 'http://s/1', + channel_id: 'b1', + }; expect( - remapChannelReferences( - [{ source: 'm3u', id: 'b1', url: 'http://s/1' }, portal], - browser, - server - ) - ).toEqual([{ source: 'm3u', id: 's1', url: 'http://s/1' }, portal]); + remapChannelReferences([entry, portal], browser, server) + ).toEqual([{ ...entry, channel_id: 's1' }, portal]); }); }); diff --git a/libs/services/src/lib/server-storage/server-migration-merge.ts b/libs/services/src/lib/server-storage/server-migration-merge.ts index ece71880b..5901344be 100644 --- a/libs/services/src/lib/server-storage/server-migration-merge.ts +++ b/libs/services/src/lib/server-storage/server-migration-merge.ts @@ -80,38 +80,36 @@ interface ChannelRef { } /** - * M3U favorites name channels by stream URL (older ones by channel id), - * and every import gives channels new random ids. A browser channel id is - * translated to the server channel with the same URL; URLs and anything - * the server cannot resolve are kept as they are, never dropped. M3U - * history entries get the server's channel id; portal items pass through. + * M3U favorites and history as the app reads them, moved to the server's + * copy of a playlist. Favorites name channels by stream URL; older ones by + * a channel id that every import makes up anew, which becomes that + * channel's URL here (`ownChannels`: the playlist the favorite came from). + * History entries keep their URL id and get the server's channel id as + * `channel_id`. Anything else passes through, never dropped. */ export function remapChannelReferences( references: readonly T[], - browserChannels: readonly ChannelRef[], + ownChannels: readonly ChannelRef[], serverChannels: readonly ChannelRef[] ): T[] { - const urlByBrowserId = new Map( - browserChannels.map((channel) => [channel.id, channel.url]) + const urlById = new Map( + ownChannels.map((channel) => [channel.id, channel.url]) ); - const serverIds = new Set(serverChannels.map((channel) => channel.id)); const idByUrl = new Map( serverChannels.map((channel) => [channel.url, channel.id]) ); return references.map((reference) => { if (typeof reference === 'string') { - if (serverIds.has(reference) || idByUrl.has(reference)) { - return reference; - } - const id = idByUrl.get(urlByBrowserId.get(reference)); - return (typeof id === 'string' ? id : reference) as T; + const url = urlById.get(reference); + return (typeof url === 'string' ? url : reference) as T; } - if ( - isPlainObject(reference) && - reference['source'] === 'm3u' && - typeof idByUrl.get(reference['url']) === 'string' - ) { - return { ...reference, id: idByUrl.get(reference['url']) } as T; + if (isPlainObject(reference) && reference['source'] === 'm3u') { + const channelId = idByUrl.get(reference['url']); + return ( + typeof channelId === 'string' + ? { ...reference, channel_id: channelId } + : reference + ) as T; } return reference; }); diff --git a/libs/services/src/lib/server-storage/server-migration.service.spec.ts b/libs/services/src/lib/server-storage/server-migration.service.spec.ts index e960e902b..472b3d952 100644 --- a/libs/services/src/lib/server-storage/server-migration.service.spec.ts +++ b/libs/services/src/lib/server-storage/server-migration.service.spec.ts @@ -293,9 +293,10 @@ describe('ServerMigrationService', () => { ).moveBrowserData(); expect(result).toEqual({ imported: 0, merged: 1, failed: 0 }); + // By stream URL, as the app looks favorites up. expect(session.documentStore.dbUpsertAppPlaylist).toHaveBeenCalledWith({ ...stored, - favorites: ['s1', 's2'], + favorites: ['http://s/1', 'http://s/2'], recentlyViewed: [], }); }); @@ -303,9 +304,10 @@ describe('ServerMigrationService', () => { it("keeps a document's newest view of an item both copies hold", async () => { const view = (day: number, extra = {}) => ({ source: 'm3u', - id: 'b1', + id: 'http://s/1', url: 'http://s/1', title: 'One', + channel_id: 'b1', category_id: 'live', added_at: `2026-10-0${day}T20:00:00.000Z`, ...extra, @@ -335,7 +337,7 @@ describe('ServerMigrationService', () => { url: browserPlaylists[0].url, favorites: [], // The same channel, watched there before this browser's view. - recentlyViewed: [view(1, { id: 's1' })], + recentlyViewed: [view(1, { channel_id: 's1' })], playlist: { items: [{ id: 's1', url: 'http://s/1' }] }, }; session.documentStore.dbGetAppPlaylist.mockResolvedValue(stored); @@ -345,7 +347,9 @@ describe('ServerMigrationService', () => { // Not skipped for an unchanged length: the newer view replaces it. expect(session.documentStore.dbUpsertAppPlaylist).toHaveBeenCalledWith({ ...stored, - recentlyViewed: [view(3, { id: 's1', poster_url: 'new.png' })], + recentlyViewed: [ + view(3, { channel_id: 's1', poster_url: 'new.png' }), + ], }); }); diff --git a/libs/services/src/lib/server-storage/server-migration.service.ts b/libs/services/src/lib/server-storage/server-migration.service.ts index d7c7ac5d7..f82d16383 100644 --- a/libs/services/src/lib/server-storage/server-migration.service.ts +++ b/libs/services/src/lib/server-storage/server-migration.service.ts @@ -171,8 +171,13 @@ export class ServerMigrationService { const serverChannels = channelsOf(stored); const next = { ...stored, + // By stream URL on both sides, so one channel is one favorite. favorites: unionItems( - stored.favorites ?? [], + remapChannelReferences( + stored.favorites ?? [], + serverChannels, + serverChannels + ), remapChannelReferences( favorites, browserChannels, diff --git a/libs/services/src/lib/server-storage/server-user-data-adopt.spec.ts b/libs/services/src/lib/server-storage/server-user-data-adopt.spec.ts index dd644989e..9761d5536 100644 --- a/libs/services/src/lib/server-storage/server-user-data-adopt.spec.ts +++ b/libs/services/src/lib/server-storage/server-user-data-adopt.spec.ts @@ -317,4 +317,39 @@ describe('ServerUserDataStorage taking over other page loads', () => { expect(server.state['xtream-favorites']['p1']).toEqual(['S']); expect(suspended.resumed()).toBe(false); }); + + it("tells whether one playlist's writes were all taken", async () => { + const server = fakeServer({ 'xtream-favorites': {} }); + const send = server.api.sendJson.getMockImplementation(); + if (!send) { + throw new Error('The fake server answers sendJson'); + } + const tab = new ServerUserDataStorage( + { + ...server.api, + // The server refuses p2's entry as too large. + sendJson: jest.fn(async (...args: Parameters) => + args[1].endsWith('/p2') + ? Promise.reject( + new ServerApiError(413, 'too_large', 'big') + ) + : send(...args) + ), + } as never, + memoryStorage(), + jest.fn(), + 'default', + memoryStorage() + ); + await tab.hydrate(); + tab.setItem( + 'xtream-favorites', + JSON.stringify({ p1: ['live:1'], p2: ['live:2'] }) + ); + + await expect(tab.flushed('p1')).resolves.toBe(true); + await expect(tab.flushed('p2')).resolves.toBe(false); + await expect(tab.flushed()).resolves.toBe(false); + expect(server.state['xtream-favorites']['p1']).toEqual(['live:1']); + }); }); diff --git a/libs/services/src/lib/server-storage/server-user-data-adopt.ts b/libs/services/src/lib/server-storage/server-user-data-adopt.ts index 82e70f591..756f187ec 100644 --- a/libs/services/src/lib/server-storage/server-user-data-adopt.ts +++ b/libs/services/src/lib/server-storage/server-user-data-adopt.ts @@ -2,6 +2,7 @@ import type { ServerUserStateNamespace } from '@iptvnator/shared/interfaces'; import { behind, type EditStep, + splitEditKey, UNKNOWN_TAG, yielding, } from './server-user-data-entries'; @@ -104,3 +105,26 @@ export function restorePage( deferred.set(`${edit.namespace}|${edit.playlistId}`, edit); } } + +/** + * Flushes until nothing (of `playlistId`) is queued or travelling, a few + * rounds for conflict replays: false when some stays unsent (refused, or + * offline). + */ +export async function settleWrites( + writes: UserDataWriteQueue, + playlistId?: string +): Promise { + const unsent = () => + writes + .unconfirmedKeys() + .some( + (key) => + playlistId === undefined || + splitEditKey(key)[1] === playlistId + ); + for (let round = 0; round < 3 && unsent(); round += 1) { + await writes.flush(); + } + return !unsent(); +} diff --git a/libs/services/src/lib/server-storage/server-user-data-storage.ts b/libs/services/src/lib/server-storage/server-user-data-storage.ts index 9db37e0d3..2a2c37a55 100644 --- a/libs/services/src/lib/server-storage/server-user-data-storage.ts +++ b/libs/services/src/lib/server-storage/server-user-data-storage.ts @@ -17,6 +17,7 @@ import { adoptEdits, extendUnsent, restorePage, + settleWrites, } from './server-user-data-adopt'; import { UserDataCache } from './server-user-data-cache'; import { @@ -304,9 +305,11 @@ export class ServerUserDataStorage implements UserDataStorage { } /** Sends the queued entries after any batch still travelling. */ - flush(): Promise { - return this.writes.flush(); - } + readonly flush = (): Promise => this.writes.flush(); + + /** See `UserDataStorage.flushed`. */ + readonly flushed = (playlistId?: string) => + settleWrites(this.writes, playlistId); /** Page hide: sends the queue with requests that survive the page. */ flushWithKeepalive(): void { diff --git a/libs/services/src/lib/user-data-storage.ts b/libs/services/src/lib/user-data-storage.ts index 29d4b91f8..d085fa7de 100644 --- a/libs/services/src/lib/user-data-storage.ts +++ b/libs/services/src/lib/user-data-storage.ts @@ -2,14 +2,19 @@ import { InjectionToken } from '@angular/core'; /** * The slice of the Web Storage API that PWA user-data stores rely on, plus - * the stored keys where the store can list them, and `ready` where reads - * show a source's data only once it was read (server mode): it waits for - * a first read in flight and tells whether one succeeded. + * the stored keys where the store can list them, and where writes reach a + * source later (server mode): `ready`, which waits for a first read in + * flight and tells whether one succeeded, and `flushed`, which sends what + * is queued and tells whether all of it (of one playlist) was taken. */ export type UserDataStorage = Pick< Storage, 'getItem' | 'setItem' | 'removeItem' -> & { keys?(): string[]; ready?(): Promise }; +> & { + keys?(): string[]; + ready?(): Promise; + flushed?(playlistId?: string): Promise; +}; /** * `localStorage` behind an adapter that reads the global only inside each