From 72c84a3ebadca6916d9bd3acabf1baee3fdfeff9 Mon Sep 17 00:00:00 2001 From: 4gray Date: Mon, 12 Oct 2026 07:00:07 +0200 Subject: [PATCH] fix(server-storage): keep URL favorites, flush before moving, cache-full and cut-off bodies - Moving M3U favorites keeps URL favorites and anything the server cannot resolve; only older channel ids are translated. - Queued user-data edits are sent before the move writes combined values. - A re-read that does not fit the cache keeps the old cache and baseline together. - A response body cut off after its headers counts as offline. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/pwa-self-hosted.md | 10 +++-- .../server-storage/server-api-client.spec.ts | 27 ++++++++++++++ .../lib/server-storage/server-api-client.ts | 22 ++++++++++- .../server-migration-merge.spec.ts | 12 +++++- .../server-storage/server-migration-merge.ts | 37 ++++++++----------- .../server-migration.service.spec.ts | 11 +++++- .../server-migration.service.ts | 2 + .../server-playlist-document-store.spec.ts | 1 + .../server-playlist-document-store.ts | 7 +++- .../server-user-data-storage.spec.ts | 33 +++++++++++++++++ .../server-user-data-storage.ts | 14 +++++-- 11 files changed, 141 insertions(+), 35 deletions(-) diff --git a/docs/architecture/pwa-self-hosted.md b/docs/architecture/pwa-self-hosted.md index 972dec327..bb7c66294 100644 --- a/docs/architecture/pwa-self-hosted.md +++ b/docs/architecture/pwa-self-hosted.md @@ -202,7 +202,9 @@ settings and guards read them in every build. (the page-hide flush waits for a batch still travelling), a failed write, including a refused `keepalive` one, stays queued for the next flush, and re-reads run one at a time and keep entries edited while they were in - flight or not yet saved. Before the first read finishes, an edit is + flight or not yet saved. A re-read whose result does not fit the cache + keeps the old cache and baseline together. A response body cut off after + its headers counts as offline. Before the first read finishes, an edit is compared with the cache, so it sends only its own entry. A re-read updates the cache; views already open show another device's changes on their next load (no live push). Start-up waits @@ -230,9 +232,9 @@ settings and guards read them in every build. playlist the server already has, by id or by source (Xtream server and user, Stalker portal and MAC, M3U URL), merges into the server's copy; the others upload with their ids. A merged playlist gains the browser - document's favorites and history (M3U channel ids are random per import, - so they are translated through the stream URL; unmatched ones are - dropped), and the legacy Xtream maps follow under + document's favorites and history (M3U favorites name channels by stream + URL; older ones by a channel id that is random per import and is + translated through the URL; nothing is dropped), and the legacy Xtream maps follow under the server id; `combineUserData` unions lists (playback positions by content type and Xtream id, renamed to the server playlist) and merges maps, the server's value winning on a conflict. Browser data is never deleted, and diff --git a/libs/services/src/lib/server-storage/server-api-client.spec.ts b/libs/services/src/lib/server-storage/server-api-client.spec.ts index 0869289b3..fd6fb6a52 100644 --- a/libs/services/src/lib/server-storage/server-api-client.spec.ts +++ b/libs/services/src/lib/server-storage/server-api-client.spec.ts @@ -99,6 +99,33 @@ describe('ServerApiClient', () => { ).resolves.toBe(false); }); + it('reports offline when a body is cut off after the headers', async () => { + const cut = { + ...response(200), + json: async () => { + throw new TypeError('network error'); + }, + } as unknown as Response; + + await expect( + client(jest.fn().mockResolvedValue(cut)).getJson('/user-state') + ).rejects.toMatchObject({ status: 0, code: 'network' }); + expect(status.reportOffline).toHaveBeenCalled(); + + // Malformed JSON is an answer, not an outage. + status.reportOffline.mockClear(); + const malformed = { + ...response(200), + json: async () => { + throw new SyntaxError('Unexpected token'); + }, + } as unknown as Response; + await expect( + client(jest.fn().mockResolvedValue(malformed)).getJson('/x') + ).rejects.toBeInstanceOf(SyntaxError); + expect(status.reportOffline).not.toHaveBeenCalled(); + }); + it('clears the token and reports unauthorized on 401', async () => { localStorage.setItem(SERVER_ACCESS_TOKEN_STORAGE_KEY, 'old'); const fetchFn = jest diff --git a/libs/services/src/lib/server-storage/server-api-client.ts b/libs/services/src/lib/server-storage/server-api-client.ts index 774b41807..1c3b49f40 100644 --- a/libs/services/src/lib/server-storage/server-api-client.ts +++ b/libs/services/src/lib/server-storage/server-api-client.ts @@ -107,7 +107,27 @@ export class ServerApiClient { } async getJson(path: string): Promise { - return (await this.request(path)).json() as Promise; + return this.readJson(await this.request(path)); + } + + /** + * A body cut off after the headers arrived is an outage too; malformed + * JSON is not. + */ + async readJson(response: Response): Promise { + try { + return (await response.json()) as T; + } catch (error) { + if (error instanceof SyntaxError) { + throw error; + } + this.options.status.reportOffline(); + throw new ServerApiError( + 0, + 'network', + 'The server is not reachable.' + ); + } } sendJson( diff --git a/libs/services/src/lib/server-storage/server-migration-merge.spec.ts b/libs/services/src/lib/server-storage/server-migration-merge.spec.ts index cca870431..dae860f1c 100644 --- a/libs/services/src/lib/server-storage/server-migration-merge.spec.ts +++ b/libs/services/src/lib/server-storage/server-migration-merge.spec.ts @@ -119,10 +119,20 @@ describe('remapChannelReferences', () => { it('names the server channel with the same stream URL', () => { expect( - remapChannelReferences(['b2', 'b1', 'b3', 's1'], browser, server) + remapChannelReferences(['b2', 'b1', 's1'], browser, server) ).toEqual(['s2', 's1', 's1']); }); + it('keeps URL favorites and anything it cannot resolve', () => { + expect( + remapChannelReferences( + ['http://s/1', 'http://elsewhere/9', 'b3'], + browser, + server + ) + ).toEqual(['http://s/1', 'http://elsewhere/9', 'b3']); + }); + it('updates M3U history entries and passes portal items through', () => { const portal = { id: 7, category_id: 'itv' }; expect( diff --git a/libs/services/src/lib/server-storage/server-migration-merge.ts b/libs/services/src/lib/server-storage/server-migration-merge.ts index fcfd07d9a..9d4266f5e 100644 --- a/libs/services/src/lib/server-storage/server-migration-merge.ts +++ b/libs/services/src/lib/server-storage/server-migration-merge.ts @@ -77,10 +77,11 @@ interface ChannelRef { } /** - * M3U favorites and history name channels by id, and every import gives - * channels new random ids: a browser favorite is translated to the server - * channel with the same stream URL, and dropped when there is none. - * Portal favorites (objects) pass through. + * M3U favorites name channels by stream URL (older ones by channel id), + * and every import gives channels new random ids. A browser channel id is + * translated to the server channel with the same URL; URLs and anything + * the server cannot resolve are kept as they are, never dropped. M3U + * history entries get the server's channel id; portal items pass through. */ export function remapChannelReferences( references: readonly T[], @@ -94,29 +95,23 @@ export function remapChannelReferences( const idByUrl = new Map( serverChannels.map((channel) => [channel.url, channel.id]) ); - const result: T[] = []; - for (const reference of references) { + return references.map((reference) => { if (typeof reference === 'string') { - const id = serverIds.has(reference) - ? reference - : idByUrl.get(urlByBrowserId.get(reference)); - if (typeof id === 'string') { - result.push(id as T); + if (serverIds.has(reference) || idByUrl.has(reference)) { + return reference; } - } else if ( + const id = idByUrl.get(urlByBrowserId.get(reference)); + return (typeof id === 'string' ? id : reference) as T; + } + if ( isPlainObject(reference) && reference['source'] === 'm3u' && - idByUrl.has(reference['url']) + typeof idByUrl.get(reference['url']) === 'string' ) { - result.push({ - ...reference, - id: idByUrl.get(reference['url']), - } as T); - } else { - result.push(reference); + return { ...reference, id: idByUrl.get(reference['url']) } as T; } - } - return result; + return reference; + }); } /** diff --git a/libs/services/src/lib/server-storage/server-migration.service.spec.ts b/libs/services/src/lib/server-storage/server-migration.service.spec.ts index 131aaa991..b8c036103 100644 --- a/libs/services/src/lib/server-storage/server-migration.service.spec.ts +++ b/libs/services/src/lib/server-storage/server-migration.service.spec.ts @@ -23,7 +23,7 @@ describe('ServerMigrationService', () => { dbUpsertAppPlaylist: jest.Mock; }; api: { getJson: jest.Mock; sendJson: jest.Mock }; - userData: { hydrate: jest.Mock }; + userData: { flush: jest.Mock; hydrate: jest.Mock }; }; beforeEach(() => { @@ -40,7 +40,10 @@ describe('ServerMigrationService', () => { getJson: jest.fn().mockResolvedValue({ entries: {} }), sendJson: jest.fn().mockResolvedValue(undefined), }, - userData: { hydrate: jest.fn().mockResolvedValue(undefined) }, + userData: { + flush: jest.fn().mockResolvedValue(undefined), + hydrate: jest.fn().mockResolvedValue(undefined), + }, }; TestBed.configureTestingModule({ providers: [ @@ -104,6 +107,10 @@ describe('ServerMigrationService', () => { { value: ['live:1'] } ); expect(session.api.sendJson).toHaveBeenCalledTimes(1); + // Queued edits were sent before the combined values were read. + expect(session.userData.flush.mock.invocationCallOrder[0]).toBeLessThan( + session.api.getJson.mock.invocationCallOrder[0] + ); expect(session.userData.hydrate).toHaveBeenCalled(); expect(migration.browserDataMoved()).toBe(true); }); diff --git a/libs/services/src/lib/server-storage/server-migration.service.ts b/libs/services/src/lib/server-storage/server-migration.service.ts index 61a282500..b47a1be2a 100644 --- a/libs/services/src/lib/server-storage/server-migration.service.ts +++ b/libs/services/src/lib/server-storage/server-migration.service.ts @@ -119,6 +119,8 @@ export class ServerMigrationService { } } + // Queued edits go out first, so none lands after the combined values. + await this.session.userData.flush(); failed += await this.copyXtreamUserData(targets); await this.session.userData.hydrate(); if (failed === 0) { diff --git a/libs/services/src/lib/server-storage/server-playlist-document-store.spec.ts b/libs/services/src/lib/server-storage/server-playlist-document-store.spec.ts index 0939b4ff2..bdeb43bc0 100644 --- a/libs/services/src/lib/server-storage/server-playlist-document-store.spec.ts +++ b/libs/services/src/lib/server-storage/server-playlist-document-store.spec.ts @@ -32,6 +32,7 @@ describe('ServerPlaylistDocumentStore', () => { const api = { request: jest.fn(), getJson: jest.fn(), + readJson: jest.fn((response: Response) => response.json()), sendJson: jest.fn(), }; return { api, store: new ServerPlaylistDocumentStore(api as never) }; diff --git a/libs/services/src/lib/server-storage/server-playlist-document-store.ts b/libs/services/src/lib/server-storage/server-playlist-document-store.ts index 037d7d892..d7691bef1 100644 --- a/libs/services/src/lib/server-storage/server-playlist-document-store.ts +++ b/libs/services/src/lib/server-storage/server-playlist-document-store.ts @@ -13,7 +13,10 @@ interface CachedDocument { readonly document: Playlist; } -type ServerApi = Pick; +type ServerApi = Pick< + ServerApiClient, + 'request' | 'getJson' | 'readJson' | 'sendJson' +>; /** * `PlaylistDocumentStore` over the server storage API. Full documents are @@ -56,7 +59,7 @@ export class ServerPlaylistDocumentStore implements PlaylistDocumentStore { if (response.status === 304 && cached) { return this.serve(cached.document, cached.etag); } - const document = (await response.json()) as Playlist; + const document = await this.api.readJson(response); const etag = this.remember(playlistId, response, document); return this.serve(document, etag); } diff --git a/libs/services/src/lib/server-storage/server-user-data-storage.spec.ts b/libs/services/src/lib/server-storage/server-user-data-storage.spec.ts index 4035857de..1fcd8d149 100644 --- a/libs/services/src/lib/server-storage/server-user-data-storage.spec.ts +++ b/libs/services/src/lib/server-storage/server-user-data-storage.spec.ts @@ -373,4 +373,37 @@ describe('ServerUserDataStorage', () => { await storage.flush(); expect(api.sendJson).not.toHaveBeenCalled(); }); + + it('keeps cache and baseline together when the cache is full', async () => { + const { api, local, storage } = setup(); + await storage.hydrate(); + // The server now has a second playlist the cache cannot take. + api.getJson.mockResolvedValueOnce({ + entries: { + 'xtream-favorites': { p1: ['live:1'], p2: ['live:2'] }, + }, + }); + const setItem = local.setItem; + local.setItem = (key: string, value: string) => { + if (key === 'iptvnator-server:xtream-favorites') { + throw new DOMException('full', 'QuotaExceededError'); + } + setItem(key, value); + }; + await storage.hydrate(); + local.setItem = setItem; + + storage.setItem( + 'xtream-favorites', + JSON.stringify({ p1: ['live:1', 'live:3'] }) + ); + await storage.flush(); + + expect(api.request).not.toHaveBeenCalled(); + expect(api.sendJson).toHaveBeenCalledWith( + 'PUT', + '/user-state/xtream-favorites/p1', + { value: ['live:1', 'live:3'] } + ); + }); }); diff --git a/libs/services/src/lib/server-storage/server-user-data-storage.ts b/libs/services/src/lib/server-storage/server-user-data-storage.ts index 90abfa62d..3244e9180 100644 --- a/libs/services/src/lib/server-storage/server-user-data-storage.ts +++ b/libs/services/src/lib/server-storage/server-user-data-storage.ts @@ -128,11 +128,17 @@ export class ServerUserDataStorage implements UserDataStorage { delete value[playlistId]; } } + try { + this.local.setItem( + SERVER_CACHE_PREFIX + namespace, + JSON.stringify(value) + ); + } catch { + // Storage full: keep the old cache and the baseline it was + // diffed against together, or an edit would read as deletes. + continue; + } this.synced.set(namespace, value); - this.local.setItem( - SERVER_CACHE_PREFIX + namespace, - JSON.stringify(value) - ); } for (const [key, edit] of this.edits) { if (