diff --git a/docs/architecture/playlist-backup-restore.md b/docs/architecture/playlist-backup-restore.md index 6e8014478..148d679e0 100644 --- a/docs/architecture/playlist-backup-restore.md +++ b/docs/architecture/playlist-backup-restore.md @@ -252,6 +252,9 @@ self-hosted server storage is covered too. browser tracks neither. - An unreadable map (storage error, malformed JSON, a playlist entry that is not a list) fails the export instead of being written as an empty list. + So does server mode before its first read of the server's user data: + the export waits for that read (`UserDataStorage.ready`) and fails when + it does not succeed. - A pending restore snapshot still wins over the stored maps, as on desktop. - Import parks the restore state like the desktop does, then applies it at once through `XtreamPendingRestoreService.applyAndConsume`: the browser has diff --git a/docs/architecture/pwa-self-hosted.md b/docs/architecture/pwa-self-hosted.md index cc0e8be61..b34d6e425 100644 --- a/docs/architecture/pwa-self-hosted.md +++ b/docs/architecture/pwa-self-hosted.md @@ -217,8 +217,10 @@ settings and guards read them in every build. keeps its copy only when `X-Replaced-ETag` names the version its payload came from. - **Xtream user data.** `USER_DATA_STORAGE` becomes `ServerUserDataStorage`: - the four synced namespaces are cached under the `iptvnator-server:` key - prefix (the legacy unprefixed keys are never written in server mode) and + the four synced namespaces are cached per server instance, under + `iptvnator-server:cache::` (the legacy unprefixed + keys are never written in server mode; a tab still open on another + server never writes over this one's cache) and each change sends only the playlist entries that differ, debounced, with a `keepalive` flush when the page hides. A change is compared with the map the caller read: when the cache changed since (another tab, a re-read), @@ -238,15 +240,18 @@ settings and guards read them in every build. tab's: its edits go ahead of this page's newer ones, and without a server version this page read they are sent so that they conflict and replay; the writes of a tab closed before it - could send them are queued, ahead of its own, by the next re-read in - another tab once that tab's record has gone stale; the result replays the + could send them are queued, ahead of its own, by another tab once that + tab's record has gone stale (at its next re-read, or its 30 s check when + it has nothing to sync and never re-reads); the result replays the closed tab's edits first and this tab's newer ones last. An adopted edit changes only what is still as its page saw it (`EditStep.yields`): an item or key changed since, by this tab or another device, keeps its newer value. Tabs never clear each other's. Cache and saved writes belong to one server database (`iptvnator-server:cache-owner`); another one behind the same address - starts clean. Also, + starts clean. A backup export waits for the first read + (`UserDataStorage.ready`) and fails if it cannot happen: maps that never + read the server would export as empty lists. Also, re-reads run one at a time and keep entries edited while they were in flight or not yet saved. A response body cut off after its headers counts as offline. Each write is sent on the tag of the entry it was made diff --git a/libs/services/src/lib/playlist-backup.service.pwa-xtream.spec.ts b/libs/services/src/lib/playlist-backup.service.pwa-xtream.spec.ts index 7a1599452..f656edb1b 100644 --- a/libs/services/src/lib/playlist-backup.service.pwa-xtream.spec.ts +++ b/libs/services/src/lib/playlist-backup.service.pwa-xtream.spec.ts @@ -269,6 +269,41 @@ describe('PlaylistBackupService Xtream user state in the PWA', () => { await expect(service.exportBackup()).rejects.toThrow('denied'); }); + + it('waits for server user data before it reads the lists', async () => { + // Server mode: the maps hold the server's data once it was read. + const ready = jest.fn(async () => seed()); + const service = createPlaylistBackupService({ + playlistsService: playlistsService([existingXtreamPlaylist]), + pwaXtream: createPwaXtreamBackupState({ + getItem: (key) => localStorage.getItem(key), + setItem: (key, value) => localStorage.setItem(key, value), + removeItem: (key) => localStorage.removeItem(key), + ready, + }), + }); + + const { manifest } = await service.exportBackup(); + + expect(ready).toHaveBeenCalledTimes(1); + expect(xtreamEntry(manifest).userState.favorites).not.toEqual([]); + }); + + it('refuses to export server user data it could not read', async () => { + const service = createPlaylistBackupService({ + playlistsService: playlistsService([existingXtreamPlaylist]), + pwaXtream: createPwaXtreamBackupState({ + getItem: () => null, + setItem: jest.fn(), + removeItem: jest.fn(), + ready: () => Promise.reject(new Error('offline')), + }), + }); + + await expect(service.exportBackup()).rejects.toThrow( + /user data could not be read/ + ); + }); }); describe('import', () => { diff --git a/libs/services/src/lib/playlist-backup.service.ts b/libs/services/src/lib/playlist-backup.service.ts index 85498b79b..87ac6a48e 100644 --- a/libs/services/src/lib/playlist-backup.service.ts +++ b/libs/services/src/lib/playlist-backup.service.ts @@ -91,6 +91,9 @@ export class PlaylistBackupService { 'The parental lock store could not be read; the backup would omit its locks.' ); } + if (!this.runtime.supportsXtreamSqliteDataSource) { + await this.pwaXtream.ready(); + } const playlists = await firstValueFrom( this.playlistsService.getAllData() ); diff --git a/libs/services/src/lib/pwa-xtream-backup-state.service.ts b/libs/services/src/lib/pwa-xtream-backup-state.service.ts index 36287dbbd..e893281be 100644 --- a/libs/services/src/lib/pwa-xtream-backup-state.service.ts +++ b/libs/services/src/lib/pwa-xtream-backup-state.service.ts @@ -31,6 +31,21 @@ const CONTENT_TYPES: readonly unknown[] = ['live', 'movie', 'series']; export class PwaXtreamBackupStateService { private readonly storage = inject(USER_DATA_STORAGE); + /** + * Before an export: in server mode the maps show the server's data only + * after the first read, which may still run or have failed at start, + * and an unread map would be written as empty lists. + */ + async ready(): Promise { + try { + await this.storage.ready?.(); + } catch { + throw new Error( + "The server's user data could not be read; the backup would omit it." + ); + } + } + /** * The entry with the playlist's stored lists and snapshots filled in; * hidden categories and pins, which the browser build does not track, diff --git a/libs/services/src/lib/server-storage/server-unsent-writes.ts b/libs/services/src/lib/server-storage/server-unsent-writes.ts index 4b8a3a8f3..fab8bf29a 100644 --- a/libs/services/src/lib/server-storage/server-unsent-writes.ts +++ b/libs/services/src/lib/server-storage/server-unsent-writes.ts @@ -4,6 +4,7 @@ import { type ServerUserStateNamespace, } from '@iptvnator/shared/interfaces'; import type { UserDataStorage } from '../user-data-storage'; +import { cacheKey } from './server-user-data-cache'; /** * User-data writes the server has not confirmed, kept in `localStorage` @@ -49,7 +50,11 @@ export function claimCache(local: UserDataStorage, instanceId: string): void { return; } for (const namespace of SERVER_USER_STATE_NAMESPACES) { + // Also the unscoped keys of earlier builds. local.removeItem(`iptvnator-server:${namespace}`); + if (owner) { + local.removeItem(cacheKey(owner, namespace)); + } } if (owner) { for (const prefix of PAGE_PREFIXES) { diff --git a/libs/services/src/lib/server-storage/server-user-data-cache.ts b/libs/services/src/lib/server-storage/server-user-data-cache.ts index 422f74bb1..b0693473f 100644 --- a/libs/services/src/lib/server-storage/server-user-data-cache.ts +++ b/libs/services/src/lib/server-storage/server-user-data-cache.ts @@ -6,7 +6,18 @@ import type { UserDataStorage } from '../user-data-storage'; import { parseMap } from './server-user-data-entries'; /** Server-mode cache keys; the legacy unprefixed keys are never touched. */ -export const SERVER_CACHE_PREFIX = 'iptvnator-server:'; +export const SERVER_CACHE_PREFIX = 'iptvnator-server:cache'; + +/** + * One namespace's cache of one server instance: a tab still open on + * another server (a changed backend URL) never writes over it. + */ +export function cacheKey( + instanceId: string, + namespace: ServerUserStateNamespace +): string { + return `${SERVER_CACHE_PREFIX}:${instanceId}:${namespace}`; +} /** * The synced namespaces as this browser shows them: in browser storage, @@ -16,7 +27,10 @@ export const SERVER_CACHE_PREFIX = 'iptvnator-server:'; export class UserDataCache { private readonly overflow = new Map(); - constructor(private readonly local: UserDataStorage) {} + constructor( + private readonly local: UserDataStorage, + private readonly instanceId: string + ) {} read(namespace: ServerUserStateNamespace): string | null { const kept = this.overflow.get(namespace); @@ -24,7 +38,7 @@ export class UserDataCache { return kept; } try { - return this.local.getItem(SERVER_CACHE_PREFIX + namespace); + return this.local.getItem(cacheKey(this.instanceId, namespace)); } catch { return null; } @@ -33,10 +47,11 @@ export class UserDataCache { /** `null` empties the namespace. */ write(namespace: ServerUserStateNamespace, value: string | null): void { try { + const key = cacheKey(this.instanceId, namespace); if (value === null) { - this.local.removeItem(SERVER_CACHE_PREFIX + namespace); + this.local.removeItem(key); } else { - this.local.setItem(SERVER_CACHE_PREFIX + namespace, value); + this.local.setItem(key, value); } this.overflow.delete(namespace); } catch { diff --git a/libs/services/src/lib/server-storage/server-user-data-queue.ts b/libs/services/src/lib/server-storage/server-user-data-queue.ts index 3b72e1c19..df1f84da1 100644 --- a/libs/services/src/lib/server-storage/server-user-data-queue.ts +++ b/libs/services/src/lib/server-storage/server-user-data-queue.ts @@ -25,7 +25,8 @@ import { newestWrites } from './server-unsent-writes'; const FLUSH_DELAY_MS = 500; -type ServerApi = Pick< +/** What the queue needs of the API client. */ +export type ServerApi = Pick< ServerApiClient, 'readJson' | 'sendJson' | 'request' | 'sendKeepalive' >; diff --git a/libs/services/src/lib/server-storage/server-user-data-storage.spec.ts b/libs/services/src/lib/server-storage/server-user-data-storage.spec.ts index 35bdcbfc4..78c94774d 100644 --- a/libs/services/src/lib/server-storage/server-user-data-storage.spec.ts +++ b/libs/services/src/lib/server-storage/server-user-data-storage.spec.ts @@ -1,6 +1,8 @@ import type { KeepaliveOutcome } from './server-api-client'; import { ServerUserDataStorage } from './server-user-data-storage'; +const FAVORITES_CACHE = 'iptvnator-server:cache:default:xtream-favorites'; + function memoryStorage() { const values = new Map(); return { @@ -66,9 +68,7 @@ describe('ServerUserDataStorage', () => { await storage.hydrate(); expect(storage.getItem('xtream-favorites')).toBe('{"p1":["live:1"]}'); - expect(local.values.get('iptvnator-server:xtream-favorites')).toBe( - '{"p1":["live:1"]}' - ); + expect(local.values.get(FAVORITES_CACHE)).toBe('{"p1":["live:1"]}'); expect(local.values.has('xtream-favorites')).toBe(false); }); @@ -351,7 +351,7 @@ describe('ServerUserDataStorage', () => { const { api, local, storage } = setup(); // The cache from the last visit; no read from the server yet. local.setItem( - 'iptvnator-server:xtream-favorites', + FAVORITES_CACHE, JSON.stringify({ p1: ['a'], p2: ['b'] }) ); @@ -445,7 +445,7 @@ describe('ServerUserDataStorage', () => { }); const setItem = local.setItem; local.setItem = (key: string, value: string) => { - if (key === 'iptvnator-server:xtream-favorites') { + if (key === FAVORITES_CACHE) { throw new DOMException('full', 'QuotaExceededError'); } setItem(key, value); @@ -535,6 +535,11 @@ describe('ServerUserDataStorage', () => { it('drops cache and saved writes of another server database', () => { const local = memoryStorage(); local.setItem('iptvnator-server:cache-owner', 'old-db'); + local.setItem( + 'iptvnator-server:cache:old-db:xtream-favorites', + '{"p1":["a"]}' + ); + // An earlier build's unscoped key. local.setItem('iptvnator-server:xtream-favorites', '{"p1":["a"]}'); local.setItem( 'iptvnator-server:unconfirmed:old-db:tab', @@ -562,6 +567,9 @@ describe('ServerUserDataStorage', () => { expect(local.values.has('iptvnator-server:xtream-favorites')).toBe( false ); + expect( + local.values.has('iptvnator-server:cache:old-db:xtream-favorites') + ).toBe(false); expect(savedWrites(local, 'old-db')).toEqual([]); expect( local.values.has('iptvnator-server:unconfirmed:old-db:tab') @@ -569,6 +577,40 @@ describe('ServerUserDataStorage', () => { expect(local.values.get('iptvnator-server:cache-owner')).toBe('new-db'); }); + it('is ready once it read the server, and not while it cannot', async () => { + const { api, storage } = setup(); + api.getJson.mockRejectedValueOnce(new Error('offline')); + + await expect(storage.ready()).rejects.toThrow('offline'); + await storage.ready(); + await storage.ready(); + + expect(api.getJson).toHaveBeenCalledTimes(2); + }); + + it("never writes over another server instance's cache", async () => { + const local = memoryStorage(); + const tab = (instanceId: string, favorites: unknown) => + new ServerUserDataStorage( + { + getJson: jest.fn().mockResolvedValue({ + entries: { 'xtream-favorites': favorites }, + }), + } as never, + local, + jest.fn(), + instanceId + ); + // The tab of the old backend stays open after a new one started. + const old = tab('old-db', { p1: ['old'] }); + const current = tab('new-db', { p1: ['new'] }); + await current.hydrate(); + + await old.hydrate(); + + expect(current.getItem('xtream-favorites')).toBe('{"p1":["new"]}'); + }); + it('keeps a newer travelling write when an older one is answered', async () => { const { api, local, storage } = setup(); await storage.hydrate(); @@ -688,14 +730,14 @@ describe('ServerUserDataStorage', () => { it('keeps server favorites through a cache-full first read and a later edit', async () => { const { api, local, storage } = setup(); - local.setItem('iptvnator-server:xtream-favorites', '{"p1":["A"]}'); + local.setItem(FAVORITES_CACHE, '{"p1":["A"]}'); storage.setItem('xtream-favorites', '{"p1":["A","B"]}'); api.getJson.mockResolvedValueOnce({ entries: { 'xtream-favorites': { p1: ['S'] } }, }); const setItem = local.setItem; local.setItem = (key: string, value: string) => { - if (key === 'iptvnator-server:xtream-favorites') { + if (key === FAVORITES_CACHE) { throw new DOMException('full', 'QuotaExceededError'); } setItem(key, value); diff --git a/libs/services/src/lib/server-storage/server-user-data-storage.ts b/libs/services/src/lib/server-storage/server-user-data-storage.ts index 7afa73e34..ca8ed6c6d 100644 --- a/libs/services/src/lib/server-storage/server-user-data-storage.ts +++ b/libs/services/src/lib/server-storage/server-user-data-storage.ts @@ -21,7 +21,7 @@ import { refreshedNamespace, withoutDeleted, } from './server-user-data-edit'; -import { UserDataWriteQueue } from './server-user-data-queue'; +import { type ServerApi, UserDataWriteQueue } from './server-user-data-queue'; import { claimCache, markClosed, @@ -37,13 +37,6 @@ import { swapPage, } from './server-unsent-writes'; -export { SERVER_CACHE_PREFIX } from './server-user-data-cache'; - -type ServerApi = Pick< - ServerApiClient, - 'getJson' | 'readJson' | 'sendJson' | 'request' | 'sendKeepalive' ->; - /** * Write-through `USER_DATA_STORAGE` for server mode. Synced namespaces are * maps keyed by playlist id; a write compares the new map with the one the @@ -86,7 +79,7 @@ export class ServerUserDataStorage implements UserDataStorage { private hydration: Promise = Promise.resolve(); constructor( - private readonly api: ServerApi, + private readonly api: ServerApi & Pick, private readonly local: UserDataStorage, onWriteFailure: (error: unknown) => void, /** Scopes the cache and saved writes to one server database. */ @@ -97,7 +90,7 @@ export class ServerUserDataStorage implements UserDataStorage { */ tabStore: UserDataStorage = local ) { - this.cache = new UserDataCache(local); + this.cache = new UserDataCache(local, instanceId); this.writes = new UserDataWriteQueue(api, { changed: () => this.saveUnsent(), show: (namespace, playlistId, value) => @@ -204,6 +197,11 @@ export class ServerUserDataStorage implements UserDataStorage { this.awaited = null; this.adoptPage(page); } + // A tab with nothing to sync never re-reads, which is where closed + // tabs' saved writes are otherwise picked up. + if (this.hydrated) { + this.adoptClosedTabs(); + } if (this.hasUnsentWrites()) { this.publishUnconfirmed(); } @@ -214,6 +212,11 @@ export class ServerUserDataStorage implements UserDataStorage { return !this.hydrated || this.hasUnsentWrites(); } + /** Reads show the server's data: read once, or rejects. */ + ready(): Promise { + return this.hydrated ? Promise.resolve() : this.hydrate(); + } + /** Whether a write is queued, being retried, or still travelling. */ hasUnsentWrites(): boolean { return this.writes.hasUnsent() || this.deferred.size > 0; diff --git a/libs/services/src/lib/server-storage/server-user-data-sync.spec.ts b/libs/services/src/lib/server-storage/server-user-data-sync.spec.ts index 313592641..90f1595ea 100644 --- a/libs/services/src/lib/server-storage/server-user-data-sync.spec.ts +++ b/libs/services/src/lib/server-storage/server-user-data-sync.spec.ts @@ -681,6 +681,46 @@ describe('ServerUserDataStorage with other devices', () => { expect(server.state['xtream-playback-positions']['p1']).toEqual(at(30)); }); + it("sends a closed tab's saved write from an idle tab", async () => { + const server = fakeServer({ 'xtream-favorites': { p1: ['S'] } }); + const shared = memoryStorage(); + const clock = jest.spyOn(Date, 'now').mockReturnValue(1_000); + // Read, and with nothing to sync it never reads again. + const idle = new ServerUserDataStorage( + server.api as never, + shared, + jest.fn(), + 'default', + memoryStorage() + ); + await idle.hydrate(); + const closed = new ServerUserDataStorage( + { + ...server.api, + sendJson: jest.fn(() => + Promise.reject(new ServerApiError(0, 'network', 'offline')) + ), + } as never, + shared, + jest.fn(), + 'default', + memoryStorage() + ); + await closed.hydrate(); + closed.setItem('xtream-favorites', JSON.stringify({ p1: ['S', 'X'] })); + await closed.flush(); + closed.close(); + + clock.mockReturnValue(1_000 + UNCONFIRMED_TTL_MS + 1); + expect(idle.needsSync()).toBe(false); + idle.touch(); + await idle.flush(); + await idle.flush(); + clock.mockRestore(); + + expect(server.state['xtream-favorites']['p1']).toEqual(['S', 'X']); + }); + it("keeps progress this tab saved over a closed tab's older edit", async () => { const at = (seconds: number) => [ { diff --git a/libs/services/src/lib/user-data-storage.ts b/libs/services/src/lib/user-data-storage.ts index e1f931d17..f5710b482 100644 --- a/libs/services/src/lib/user-data-storage.ts +++ b/libs/services/src/lib/user-data-storage.ts @@ -2,12 +2,14 @@ import { InjectionToken } from '@angular/core'; /** * The slice of the Web Storage API that PWA user-data stores rely on, plus - * the stored keys where the store can list them. + * the stored keys where the store can list them, and `ready` where reads + * show a source's data only once it was read (server mode): it rejects + * when that read fails. */ export type UserDataStorage = Pick< Storage, 'getItem' | 'setItem' | 'removeItem' -> & { keys?(): string[] }; +> & { keys?(): string[]; ready?(): Promise }; /** * `localStorage` behind an adapter that reads the global only inside each