diff --git a/docs/architecture/playlist-backup-restore.md b/docs/architecture/playlist-backup-restore.md index 5c28aa9be..52ac6db09 100644 --- a/docs/architecture/playlist-backup-restore.md +++ b/docs/architecture/playlist-backup-restore.md @@ -255,7 +255,8 @@ self-hosted server storage is covered too. So does server mode before its first read of the server's user data: the export waits for a read in flight (`UserDataStorage.ready`) and fails when none succeeded. A restore does the same and stays parked, since its - replacement lists would otherwise merge into the server's. + replacement lists would otherwise merge into the server's; it also stays + parked until the server took that playlist's lists. - A pending restore snapshot still wins over the stored maps, as on desktop. - Import parks the restore state like the desktop does, then applies it at once through `XtreamPendingRestoreService.applyAndConsume`: the browser has diff --git a/docs/architecture/pwa-self-hosted.md b/docs/architecture/pwa-self-hosted.md index 276ae0735..ed1292677 100644 --- a/docs/architecture/pwa-self-hosted.md +++ b/docs/architecture/pwa-self-hosted.md @@ -265,7 +265,8 @@ settings and guards read them in every build. restore (also its replay when the playlist opens) fails while no read succeeded: unread maps would export as empty lists, and a restore's lists would merge into the server's instead of replacing them (the restore - stays parked). Replayed edits keep recently viewed lists newest first, + stays parked). A restore also counts only once the server took that + playlist's lists (`UserDataStorage.flushed`); a refused one stays parked. Replayed edits keep recently viewed lists newest first, as the app cuts them at their limit from the end. Also, re-reads run one at a time and keep entries edited while they were in flight or not yet saved, and entries another tab saved meanwhile (the @@ -363,8 +364,9 @@ settings and guards read them in every build. user, Stalker portal and MAC, M3U URL), merges into the server's copy; the others upload with their ids. A merged playlist gains the browser document's favorites and history (M3U favorites name channels by stream - URL; older ones by a channel id that is random per import and is - translated through the URL; recently viewed newest first by `added_at`, + URL, as the app looks them up; older ones by a channel id that is random + per import, which becomes its URL on both sides; history entries keep + their URL id and get the server's channel id as `channel_id`; recently viewed newest first by `added_at`, one per item with its newest view; Stalker items by id, category and series flag, since one provider category can hold a movie and a series of one id; nothing is dropped), and the legacy Xtream maps follow under diff --git a/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.spec.ts b/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.spec.ts index d31e13a8d..646359f1d 100644 --- a/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.spec.ts +++ b/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.spec.ts @@ -1047,5 +1047,42 @@ describe('PwaXtreamDataSource', () => { JSON.stringify({ p1: ['live:1'] }) ); }); + + it('keeps a restore parked while the server refuses its lists', async () => { + const values = new Map(); + const flushed = jest.fn(async () => false); + const storage = { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => { + values.set(key, value); + }, + removeItem: (key: string) => { + values.delete(key); + }, + ready: async () => true, + flushed, + }; + TestBed.resetTestingModule(); + TestBed.configureTestingModule({ + providers: [ + PwaXtreamDataSource, + { provide: USER_DATA_STORAGE, useValue: storage }, + { provide: ParentalLockService, useValue: parentalLock }, + { provide: XtreamApiService, useValue: apiService }, + { provide: PlaylistsService, useValue: playlistsService }, + ], + }); + const source = TestBed.inject(PwaXtreamDataSource); + + await expect( + source.restoreUserData('p1', { + hiddenCategories: [], + favorites: [], + recentlyViewed: [], + playbackPositions: [], + } as never) + ).rejects.toThrow(/did not take the restored lists/); + expect(flushed).toHaveBeenCalledWith('p1'); + }); }); }); diff --git a/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.ts b/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.ts index 424f88789..0941c175b 100644 --- a/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.ts +++ b/libs/portal/xtream/data-access/src/lib/data-sources/pwa-xtream-data-source.ts @@ -1450,6 +1450,13 @@ export class PwaXtreamDataSource implements IXtreamDataSource { }) ); this.savePlaybackPositionsToStorage(playbackPositions); + // Server mode: done once the server took the lists, else it stays + // parked for the next open. + if ((await this.userDataStorage.flushed?.(playlistId)) === false) { + throw new Error( + 'The server did not take the restored lists; the restore waits.' + ); + } } // ========================================================================= diff --git a/libs/services/src/lib/playlist-backup.service.pwa-xtream.spec.ts b/libs/services/src/lib/playlist-backup.service.pwa-xtream.spec.ts index d593cb85e..e02f6a0d8 100644 --- a/libs/services/src/lib/playlist-backup.service.pwa-xtream.spec.ts +++ b/libs/services/src/lib/playlist-backup.service.pwa-xtream.spec.ts @@ -537,6 +537,34 @@ describe('PlaylistBackupService Xtream user state in the PWA', () => { ).not.toBeNull(); }); + it('keeps the restore parked until the server took the lists', async () => { + const manifest = await exportSeeded(); + localStorage.clear(); + const flushed = jest.fn(async () => false); + const service = createPlaylistBackupService({ + playlistsService: playlistsService([existingXtreamPlaylist]), + pwaXtream: createPwaXtreamBackupState({ + getItem: (key) => localStorage.getItem(key), + setItem: (key, value) => localStorage.setItem(key, value), + removeItem: (key) => localStorage.removeItem(key), + ready: async () => true, + flushed, + }), + }); + + const summary = await service.importBackup(xtreamOnly(manifest)); + + expect(flushed).toHaveBeenCalledWith('xtream-1'); + expect(summary.failed).toBe(1); + expect( + pendingRestore(service).getOrThrow('xtream-1') + ).not.toBeNull(); + // Titles are merged regardless: the replay carries none. + expect(read(KEYS.collectionItems)).toEqual({ + 'xtream-1': { 'movie:501': heat, 'series:77': legacyShow }, + }); + }); + it('replaces a damaged title cache but refuses an unreadable storage', async () => { const manifest = await exportSeeded(); localStorage.setItem(KEYS.collectionItems, '{"xtream-1": {'); diff --git a/libs/services/src/lib/pwa-xtream-backup-state.service.ts b/libs/services/src/lib/pwa-xtream-backup-state.service.ts index c3b1dfa74..b6baeb322 100644 --- a/libs/services/src/lib/pwa-xtream-backup-state.service.ts +++ b/libs/services/src/lib/pwa-xtream-backup-state.service.ts @@ -135,7 +135,15 @@ export class PwaXtreamBackupStateService { } throw error; } + // Server mode: the lists count once the server took them; a refused + // one keeps the restore parked (titles merged regardless, as above). + const saved = (await this.storage.flushed?.(playlistId)) !== false; this.mergeSnapshots(playlistId, snapshots); + if (!saved) { + throw new Error( + 'The server did not take the restored lists; the restore waits.' + ); + } } private replaceLists( diff --git a/libs/services/src/lib/server-storage/server-migration-merge.spec.ts b/libs/services/src/lib/server-storage/server-migration-merge.spec.ts index 3c2a2f607..9bcd7cbd8 100644 --- a/libs/services/src/lib/server-storage/server-migration-merge.spec.ts +++ b/libs/services/src/lib/server-storage/server-migration-merge.spec.ts @@ -137,30 +137,36 @@ describe('remapChannelReferences', () => { { id: 's2', url: 'http://s/2' }, ]; - it('names the server channel with the same stream URL', () => { + it('turns legacy channel-id favorites into their stream URL', () => { expect( - remapChannelReferences(['b2', 'b1', 's1'], browser, server) - ).toEqual(['s2', 's1', 's1']); + remapChannelReferences(['b2', 'b1', 'http://s/1'], browser, server) + ).toEqual(['http://s/2', 'http://s/1', 'http://s/1']); + // The server's own legacy ids, against its own channels. + expect(remapChannelReferences(['s1'], server, server)).toEqual([ + 'http://s/1', + ]); }); it('keeps URL favorites and anything it cannot resolve', () => { expect( remapChannelReferences( - ['http://s/1', 'http://elsewhere/9', 'b3'], + ['http://elsewhere/9', 'b9', 'b3'], browser, server ) - ).toEqual(['http://s/1', 'http://elsewhere/9', 'b3']); + ).toEqual(['http://elsewhere/9', 'b9', 'http://s/gone']); }); - it('updates M3U history entries and passes portal items through', () => { + it("keeps a history entry's URL id and names the server's channel", () => { const portal = { id: 7, category_id: 'itv' }; + const entry = { + source: 'm3u', + id: 'http://s/1', + url: 'http://s/1', + channel_id: 'b1', + }; expect( - remapChannelReferences( - [{ source: 'm3u', id: 'b1', url: 'http://s/1' }, portal], - browser, - server - ) - ).toEqual([{ source: 'm3u', id: 's1', url: 'http://s/1' }, portal]); + remapChannelReferences([entry, portal], browser, server) + ).toEqual([{ ...entry, channel_id: 's1' }, portal]); }); }); diff --git a/libs/services/src/lib/server-storage/server-migration-merge.ts b/libs/services/src/lib/server-storage/server-migration-merge.ts index ece71880b..5901344be 100644 --- a/libs/services/src/lib/server-storage/server-migration-merge.ts +++ b/libs/services/src/lib/server-storage/server-migration-merge.ts @@ -80,38 +80,36 @@ interface ChannelRef { } /** - * M3U favorites name channels by stream URL (older ones by channel id), - * and every import gives channels new random ids. A browser channel id is - * translated to the server channel with the same URL; URLs and anything - * the server cannot resolve are kept as they are, never dropped. M3U - * history entries get the server's channel id; portal items pass through. + * M3U favorites and history as the app reads them, moved to the server's + * copy of a playlist. Favorites name channels by stream URL; older ones by + * a channel id that every import makes up anew, which becomes that + * channel's URL here (`ownChannels`: the playlist the favorite came from). + * History entries keep their URL id and get the server's channel id as + * `channel_id`. Anything else passes through, never dropped. */ export function remapChannelReferences( references: readonly T[], - browserChannels: readonly ChannelRef[], + ownChannels: readonly ChannelRef[], serverChannels: readonly ChannelRef[] ): T[] { - const urlByBrowserId = new Map( - browserChannels.map((channel) => [channel.id, channel.url]) + const urlById = new Map( + ownChannels.map((channel) => [channel.id, channel.url]) ); - const serverIds = new Set(serverChannels.map((channel) => channel.id)); const idByUrl = new Map( serverChannels.map((channel) => [channel.url, channel.id]) ); return references.map((reference) => { if (typeof reference === 'string') { - if (serverIds.has(reference) || idByUrl.has(reference)) { - return reference; - } - const id = idByUrl.get(urlByBrowserId.get(reference)); - return (typeof id === 'string' ? id : reference) as T; + const url = urlById.get(reference); + return (typeof url === 'string' ? url : reference) as T; } - if ( - isPlainObject(reference) && - reference['source'] === 'm3u' && - typeof idByUrl.get(reference['url']) === 'string' - ) { - return { ...reference, id: idByUrl.get(reference['url']) } as T; + if (isPlainObject(reference) && reference['source'] === 'm3u') { + const channelId = idByUrl.get(reference['url']); + return ( + typeof channelId === 'string' + ? { ...reference, channel_id: channelId } + : reference + ) as T; } return reference; }); diff --git a/libs/services/src/lib/server-storage/server-migration.service.spec.ts b/libs/services/src/lib/server-storage/server-migration.service.spec.ts index e960e902b..472b3d952 100644 --- a/libs/services/src/lib/server-storage/server-migration.service.spec.ts +++ b/libs/services/src/lib/server-storage/server-migration.service.spec.ts @@ -293,9 +293,10 @@ describe('ServerMigrationService', () => { ).moveBrowserData(); expect(result).toEqual({ imported: 0, merged: 1, failed: 0 }); + // By stream URL, as the app looks favorites up. expect(session.documentStore.dbUpsertAppPlaylist).toHaveBeenCalledWith({ ...stored, - favorites: ['s1', 's2'], + favorites: ['http://s/1', 'http://s/2'], recentlyViewed: [], }); }); @@ -303,9 +304,10 @@ describe('ServerMigrationService', () => { it("keeps a document's newest view of an item both copies hold", async () => { const view = (day: number, extra = {}) => ({ source: 'm3u', - id: 'b1', + id: 'http://s/1', url: 'http://s/1', title: 'One', + channel_id: 'b1', category_id: 'live', added_at: `2026-10-0${day}T20:00:00.000Z`, ...extra, @@ -335,7 +337,7 @@ describe('ServerMigrationService', () => { url: browserPlaylists[0].url, favorites: [], // The same channel, watched there before this browser's view. - recentlyViewed: [view(1, { id: 's1' })], + recentlyViewed: [view(1, { channel_id: 's1' })], playlist: { items: [{ id: 's1', url: 'http://s/1' }] }, }; session.documentStore.dbGetAppPlaylist.mockResolvedValue(stored); @@ -345,7 +347,9 @@ describe('ServerMigrationService', () => { // Not skipped for an unchanged length: the newer view replaces it. expect(session.documentStore.dbUpsertAppPlaylist).toHaveBeenCalledWith({ ...stored, - recentlyViewed: [view(3, { id: 's1', poster_url: 'new.png' })], + recentlyViewed: [ + view(3, { channel_id: 's1', poster_url: 'new.png' }), + ], }); }); diff --git a/libs/services/src/lib/server-storage/server-migration.service.ts b/libs/services/src/lib/server-storage/server-migration.service.ts index d7c7ac5d7..f82d16383 100644 --- a/libs/services/src/lib/server-storage/server-migration.service.ts +++ b/libs/services/src/lib/server-storage/server-migration.service.ts @@ -171,8 +171,13 @@ export class ServerMigrationService { const serverChannels = channelsOf(stored); const next = { ...stored, + // By stream URL on both sides, so one channel is one favorite. favorites: unionItems( - stored.favorites ?? [], + remapChannelReferences( + stored.favorites ?? [], + serverChannels, + serverChannels + ), remapChannelReferences( favorites, browserChannels, diff --git a/libs/services/src/lib/server-storage/server-user-data-adopt.spec.ts b/libs/services/src/lib/server-storage/server-user-data-adopt.spec.ts index dd644989e..9761d5536 100644 --- a/libs/services/src/lib/server-storage/server-user-data-adopt.spec.ts +++ b/libs/services/src/lib/server-storage/server-user-data-adopt.spec.ts @@ -317,4 +317,39 @@ describe('ServerUserDataStorage taking over other page loads', () => { expect(server.state['xtream-favorites']['p1']).toEqual(['S']); expect(suspended.resumed()).toBe(false); }); + + it("tells whether one playlist's writes were all taken", async () => { + const server = fakeServer({ 'xtream-favorites': {} }); + const send = server.api.sendJson.getMockImplementation(); + if (!send) { + throw new Error('The fake server answers sendJson'); + } + const tab = new ServerUserDataStorage( + { + ...server.api, + // The server refuses p2's entry as too large. + sendJson: jest.fn(async (...args: Parameters) => + args[1].endsWith('/p2') + ? Promise.reject( + new ServerApiError(413, 'too_large', 'big') + ) + : send(...args) + ), + } as never, + memoryStorage(), + jest.fn(), + 'default', + memoryStorage() + ); + await tab.hydrate(); + tab.setItem( + 'xtream-favorites', + JSON.stringify({ p1: ['live:1'], p2: ['live:2'] }) + ); + + await expect(tab.flushed('p1')).resolves.toBe(true); + await expect(tab.flushed('p2')).resolves.toBe(false); + await expect(tab.flushed()).resolves.toBe(false); + expect(server.state['xtream-favorites']['p1']).toEqual(['live:1']); + }); }); diff --git a/libs/services/src/lib/server-storage/server-user-data-adopt.ts b/libs/services/src/lib/server-storage/server-user-data-adopt.ts index 82e70f591..756f187ec 100644 --- a/libs/services/src/lib/server-storage/server-user-data-adopt.ts +++ b/libs/services/src/lib/server-storage/server-user-data-adopt.ts @@ -2,6 +2,7 @@ import type { ServerUserStateNamespace } from '@iptvnator/shared/interfaces'; import { behind, type EditStep, + splitEditKey, UNKNOWN_TAG, yielding, } from './server-user-data-entries'; @@ -104,3 +105,26 @@ export function restorePage( deferred.set(`${edit.namespace}|${edit.playlistId}`, edit); } } + +/** + * Flushes until nothing (of `playlistId`) is queued or travelling, a few + * rounds for conflict replays: false when some stays unsent (refused, or + * offline). + */ +export async function settleWrites( + writes: UserDataWriteQueue, + playlistId?: string +): Promise { + const unsent = () => + writes + .unconfirmedKeys() + .some( + (key) => + playlistId === undefined || + splitEditKey(key)[1] === playlistId + ); + for (let round = 0; round < 3 && unsent(); round += 1) { + await writes.flush(); + } + return !unsent(); +} diff --git a/libs/services/src/lib/server-storage/server-user-data-storage.ts b/libs/services/src/lib/server-storage/server-user-data-storage.ts index 9db37e0d3..2a2c37a55 100644 --- a/libs/services/src/lib/server-storage/server-user-data-storage.ts +++ b/libs/services/src/lib/server-storage/server-user-data-storage.ts @@ -17,6 +17,7 @@ import { adoptEdits, extendUnsent, restorePage, + settleWrites, } from './server-user-data-adopt'; import { UserDataCache } from './server-user-data-cache'; import { @@ -304,9 +305,11 @@ export class ServerUserDataStorage implements UserDataStorage { } /** Sends the queued entries after any batch still travelling. */ - flush(): Promise { - return this.writes.flush(); - } + readonly flush = (): Promise => this.writes.flush(); + + /** See `UserDataStorage.flushed`. */ + readonly flushed = (playlistId?: string) => + settleWrites(this.writes, playlistId); /** Page hide: sends the queue with requests that survive the page. */ flushWithKeepalive(): void { diff --git a/libs/services/src/lib/user-data-storage.ts b/libs/services/src/lib/user-data-storage.ts index 29d4b91f8..d085fa7de 100644 --- a/libs/services/src/lib/user-data-storage.ts +++ b/libs/services/src/lib/user-data-storage.ts @@ -2,14 +2,19 @@ import { InjectionToken } from '@angular/core'; /** * The slice of the Web Storage API that PWA user-data stores rely on, plus - * the stored keys where the store can list them, and `ready` where reads - * show a source's data only once it was read (server mode): it waits for - * a first read in flight and tells whether one succeeded. + * the stored keys where the store can list them, and where writes reach a + * source later (server mode): `ready`, which waits for a first read in + * flight and tells whether one succeeded, and `flushed`, which sends what + * is queued and tells whether all of it (of one playlist) was taken. */ export type UserDataStorage = Pick< Storage, 'getItem' | 'setItem' | 'removeItem' -> & { keys?(): string[]; ready?(): Promise }; +> & { + keys?(): string[]; + ready?(): Promise; + flushed?(playlistId?: string): Promise; +}; /** * `localStorage` behind an adapter that reads the global only inside each