From 9bcdbc0efb25d4fa734b8e5c48ff0519a76c0f87 Mon Sep 17 00:00:00 2001 From: 4gray <4gray@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:45:35 +0200 Subject: [PATCH] fix(migration): preserve and recover legacy desktop sources (#1550) * fix(migration): recover legacy desktop sources without replacing current data * test(migration): cover legacy recovery IPC contracts * test(migration): use static legacy Electron bootstrap --- .../migration-legacy-electron-profiles.md | 7 + AGENTS.md | 11 + CLAUDE.md | 11 + .../src/fixtures/v019-schema.sql | 155 +++++++ .../src/legacy-playlist-migration.e2e.ts | 410 ++++++++++++++++++ .../src/legacy-profile-fixture.ts | 231 ++++++++++ .../src/app/api/main.preload.spec-data.ts | 12 + .../src/app/api/main.preload.ts | 4 + .../playlist-migration.operations.ts | 76 ++++ .../operations/playlist.operations.ts | 2 +- .../app/events/database/playlist.events.ts | 6 + .../database/worker-ipc-contract.spec-data.ts | 5 + .../database/worker-ipc-contract.spec.ts | 11 +- .../services/electron-profile-bootstrap.ts | 11 + .../services/legacy-profile-recovery.spec.ts | 130 ++++++ .../app/services/legacy-profile-recovery.ts | 133 ++++++ .../src/app/services/legacy-profile.spec.ts | 79 ++++ .../src/app/services/legacy-profile.ts | 34 ++ .../src/app/workers/database-worker.types.ts | 1 + .../src/app/workers/database.worker.ts | 9 + apps/electron-backend/src/main.ts | 9 +- docs/architecture/m3u-playlist-module.md | 71 +++ .../src/lib/playlist-migration.spec.ts | 68 +++ libs/services/src/lib/playlists.service.ts | 50 +-- libs/shared/database/src/lib/connection.ts | 3 +- .../src/lib/electron-api.interface.ts | 4 + 26 files changed, 1505 insertions(+), 38 deletions(-) create mode 100644 .changes/migration-legacy-electron-profiles.md create mode 100644 apps/electron-backend-e2e/src/fixtures/v019-schema.sql create mode 100644 apps/electron-backend-e2e/src/legacy-playlist-migration.e2e.ts create mode 100644 apps/electron-backend-e2e/src/legacy-profile-fixture.ts create mode 100644 apps/electron-backend/src/app/database/operations/playlist-migration.operations.ts create mode 100644 apps/electron-backend/src/app/services/electron-profile-bootstrap.ts create mode 100644 apps/electron-backend/src/app/services/legacy-profile-recovery.spec.ts create mode 100644 apps/electron-backend/src/app/services/legacy-profile-recovery.ts create mode 100644 apps/electron-backend/src/app/services/legacy-profile.spec.ts create mode 100644 apps/electron-backend/src/app/services/legacy-profile.ts create mode 100644 libs/services/src/lib/playlist-migration.spec.ts diff --git a/.changes/migration-legacy-electron-profiles.md b/.changes/migration-legacy-electron-profiles.md new file mode 100644 index 000000000..d8d3b00dc --- /dev/null +++ b/.changes/migration-legacy-electron-profiles.md @@ -0,0 +1,7 @@ +--- +type: fix +area: migration +issues: [1504] +--- + +Upgrades from older desktop versions retain all sources from the legacy profile. Already-upgraded users can choose to recover missing sources without replacing current sources or settings. Migration keeps the original data and retries safely after a failed write. diff --git a/AGENTS.md b/AGENTS.md index fadea455e..9e438c0fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -102,6 +102,17 @@ name or passphrase you entered is not correct`. Keep the patch until - Electron-specific changes affecting IPC, SQLite, packaged runtime, external players, native file access, or Electron-only routes require Electron E2E coverage where available, or CDP/manual verification with `agent-browser` and the tracing flags documented below. - Final task summaries must list tests added or updated, validation commands run with results, and any skipped validation with the reason. For docs-only changes, state that unit/E2E validation was not required and verify the changed Markdown instead. +## Legacy Desktop Profile Migration + +`electron-profile-bootstrap.ts` selects the known v0.19 `electron-backend` +profile before eager main-process imports only when current Chromium storage +is unused. Existing profiles retain their settings and offer explicit recovery +of missing sources from a disposable legacy snapshot. Playlist rows and a +completion receipt commit atomically in the DB worker; original IndexedDB is +retained, current payload rows are preserved, and completed imports never +replay deleted sources. Contract and recovery limits: +`docs/architecture/m3u-playlist-module.md` (Desktop upgrades from legacy profiles). + ## Electron Debugging (CDP) - Start the Electron development app with: `nx serve electron-backend` diff --git a/CLAUDE.md b/CLAUDE.md index f588108e8..700c1eb7c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -316,6 +316,17 @@ The target still reports success either way, so a broken glob hides missing coverage instead of failing. After changing such a target, compare the linted file count against `find -name '*.ts' | wc -l`. +## Legacy Desktop Profile Migration + +`electron-profile-bootstrap.ts` selects the known v0.19 `electron-backend` +profile before eager main-process imports only when current Chromium storage +is unused. Existing profiles retain their settings and offer explicit recovery +of missing sources from a disposable legacy snapshot. Playlist rows and a +completion receipt commit atomically in the DB worker; original IndexedDB is +retained, current payload rows are preserved, and completed imports never +replay deleted sources. Contract and recovery limits: +`docs/architecture/m3u-playlist-module.md` (Desktop upgrades from legacy profiles). + ## Architecture ### Monorepo Structure (Nx Workspace) diff --git a/apps/electron-backend-e2e/src/fixtures/v019-schema.sql b/apps/electron-backend-e2e/src/fixtures/v019-schema.sql new file mode 100644 index 000000000..3019dec42 --- /dev/null +++ b/apps/electron-backend-e2e/src/fixtures/v019-schema.sql @@ -0,0 +1,155 @@ +-- Verbatim CREATE statements from v0.19.0 libs/shared/database/src/lib/connection.ts +CREATE TABLE IF NOT EXISTS playlists ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + serverUrl TEXT, + username TEXT, + password TEXT, + date_created TEXT DEFAULT (datetime('now')), + last_updated TEXT, + type TEXT NOT NULL CHECK (type IN ('xtream', 'stalker', 'm3u-file', 'm3u-text', 'm3u-url')), + userAgent TEXT, + origin TEXT, + referrer TEXT, + filePath TEXT, + autoRefresh INTEGER DEFAULT 0, + macAddress TEXT, + url TEXT, + last_usage TEXT + ); +CREATE TABLE IF NOT EXISTS categories ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + playlist_id TEXT NOT NULL, + name TEXT NOT NULL, + type TEXT NOT NULL CHECK (type IN ('live', 'movies', 'series')), + xtream_id INTEGER NOT NULL, + hidden INTEGER DEFAULT 0, + FOREIGN KEY (playlist_id) REFERENCES playlists (id) ON DELETE CASCADE + ); +CREATE TABLE IF NOT EXISTS content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + category_id INTEGER NOT NULL, + title TEXT NOT NULL, + rating TEXT, + added TEXT, + poster_url TEXT, + xtream_id INTEGER NOT NULL, + type TEXT NOT NULL CHECK (type IN ('live', 'movie', 'series')), + FOREIGN KEY (category_id) REFERENCES categories (id) ON DELETE CASCADE + ); +CREATE TABLE IF NOT EXISTS recently_viewed ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content_id INTEGER NOT NULL, + playlist_id TEXT NOT NULL, + viewed_at TEXT DEFAULT (datetime('now')), + FOREIGN KEY (content_id) REFERENCES content (id) ON DELETE CASCADE, + FOREIGN KEY (playlist_id) REFERENCES playlists (id) ON DELETE CASCADE + ); +CREATE TABLE IF NOT EXISTS favorites ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content_id INTEGER NOT NULL, + playlist_id TEXT NOT NULL, + added_at TEXT DEFAULT (datetime('now')), + FOREIGN KEY (content_id) REFERENCES content(id) ON DELETE CASCADE, + FOREIGN KEY (playlist_id) REFERENCES playlists(id) ON DELETE CASCADE + ); +CREATE INDEX IF NOT EXISTS idx_content_type ON content(type); +CREATE INDEX IF NOT EXISTS idx_content_category ON content(category_id); +CREATE INDEX IF NOT EXISTS idx_categories_playlist ON categories(playlist_id); +CREATE INDEX IF NOT EXISTS idx_content_title ON content(title); +CREATE INDEX IF NOT EXISTS idx_content_xtream ON content(xtream_id); +CREATE INDEX IF NOT EXISTS idx_categories_type ON categories(type); +CREATE UNIQUE INDEX IF NOT EXISTS favorites_content_playlist_unique ON favorites(content_id, playlist_id); +CREATE INDEX IF NOT EXISTS favorites_playlist_idx ON favorites(playlist_id); +CREATE INDEX IF NOT EXISTS favorites_content_idx ON favorites(content_id); +CREATE UNIQUE INDEX IF NOT EXISTS recently_viewed_content_playlist_unique ON recently_viewed(content_id, playlist_id); +CREATE INDEX IF NOT EXISTS recently_viewed_playlist_idx ON recently_viewed(playlist_id); +CREATE INDEX IF NOT EXISTS recently_viewed_viewed_at_idx ON recently_viewed(viewed_at); +CREATE TABLE IF NOT EXISTS epg_channels ( + id TEXT PRIMARY KEY, + display_name TEXT NOT NULL, + icon_url TEXT, + url TEXT, + source_url TEXT NOT NULL, + updated_at TEXT DEFAULT (datetime('now')) + ); +CREATE TABLE IF NOT EXISTS epg_programs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + channel_id TEXT NOT NULL, + start TEXT NOT NULL, + stop TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT, + category TEXT, + icon_url TEXT, + rating TEXT, + episode_num TEXT, + FOREIGN KEY (channel_id) REFERENCES epg_channels(id) ON DELETE CASCADE + ); +CREATE INDEX IF NOT EXISTS idx_epg_channels_source ON epg_channels(source_url); +CREATE INDEX IF NOT EXISTS idx_epg_channels_name ON epg_channels(display_name); +CREATE INDEX IF NOT EXISTS idx_epg_programs_channel ON epg_programs(channel_id); +CREATE INDEX IF NOT EXISTS idx_epg_programs_start ON epg_programs(start); +CREATE INDEX IF NOT EXISTS idx_epg_programs_time_range ON epg_programs(channel_id, start, stop); +CREATE VIRTUAL TABLE IF NOT EXISTS epg_programs_fts USING fts5( + title, + description, + category, + content='epg_programs', + content_rowid='id' + ); +CREATE TRIGGER IF NOT EXISTS epg_programs_ai AFTER INSERT ON epg_programs BEGIN + INSERT INTO epg_programs_fts(rowid, title, description, category) + VALUES (new.id, new.title, new.description, new.category); + END; +CREATE TRIGGER IF NOT EXISTS epg_programs_ad AFTER DELETE ON epg_programs BEGIN + INSERT INTO epg_programs_fts(epg_programs_fts, rowid, title, description, category) + VALUES ('delete', old.id, old.title, old.description, old.category); + END; +CREATE TRIGGER IF NOT EXISTS epg_programs_au AFTER UPDATE ON epg_programs BEGIN + INSERT INTO epg_programs_fts(epg_programs_fts, rowid, title, description, category) + VALUES ('delete', old.id, old.title, old.description, old.category); + INSERT INTO epg_programs_fts(rowid, title, description, category) + VALUES (new.id, new.title, new.description, new.category); + END; +CREATE TABLE IF NOT EXISTS playback_positions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + playlist_id TEXT NOT NULL, + content_xtream_id INTEGER NOT NULL, + content_type TEXT NOT NULL CHECK (content_type IN ('vod', 'episode')), + series_xtream_id INTEGER, + season_number INTEGER, + episode_number INTEGER, + position_seconds INTEGER NOT NULL DEFAULT 0, + duration_seconds INTEGER, + updated_at TEXT DEFAULT (datetime('now')), + FOREIGN KEY (playlist_id) REFERENCES playlists (id) ON DELETE CASCADE + ); +CREATE UNIQUE INDEX IF NOT EXISTS playback_positions_content_playlist_unique ON playback_positions(content_xtream_id, playlist_id, content_type); +CREATE INDEX IF NOT EXISTS playback_positions_playlist_idx ON playback_positions(playlist_id); +CREATE INDEX IF NOT EXISTS playback_positions_series_idx ON playback_positions(series_xtream_id); +CREATE INDEX IF NOT EXISTS playback_positions_updated_idx ON playback_positions(updated_at); +CREATE TABLE IF NOT EXISTS downloads ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + playlist_id TEXT NOT NULL, + xtream_id INTEGER NOT NULL, + content_type TEXT NOT NULL CHECK (content_type IN ('vod', 'episode')), + series_xtream_id INTEGER, + season_number INTEGER, + episode_number INTEGER, + title TEXT NOT NULL, + url TEXT NOT NULL, + file_name TEXT, + file_path TEXT, + poster_url TEXT, + status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'downloading', 'completed', 'failed', 'canceled')), + bytes_downloaded INTEGER DEFAULT 0, + total_bytes INTEGER, + error_message TEXT, + created_at TEXT DEFAULT (datetime('now')), + updated_at TEXT DEFAULT (datetime('now')), + FOREIGN KEY (playlist_id) REFERENCES playlists (id) ON DELETE CASCADE + ); +CREATE UNIQUE INDEX IF NOT EXISTS downloads_xtream_playlist_unique ON downloads(xtream_id, playlist_id, content_type); +CREATE INDEX IF NOT EXISTS downloads_playlist_idx ON downloads(playlist_id); +CREATE INDEX IF NOT EXISTS downloads_status_idx ON downloads(status); diff --git a/apps/electron-backend-e2e/src/legacy-playlist-migration.e2e.ts b/apps/electron-backend-e2e/src/legacy-playlist-migration.e2e.ts new file mode 100644 index 000000000..3ca9e6fa4 --- /dev/null +++ b/apps/electron-backend-e2e/src/legacy-playlist-migration.e2e.ts @@ -0,0 +1,410 @@ +import { + _electron as electron, + ElectronApplication, + Page, +} from '@playwright/test'; +import { join } from 'path'; +import { writeFile } from 'fs/promises'; +import { + test, + expect, + launchElectronApp, + closeElectronApp, + buildElectronLaunchEnvironment, + electronMainPath, + workspaceRoot, + openSources, +} from './electron-test-fixtures'; +import { seedLegacyProfile, legacyPlaylists } from './legacy-profile-fixture'; + +const migrationKey = 'm3u-playlists-indexeddb-to-sqlite-v1'; +const recoveryKey = 'playlists-electron-backend-profile-v1'; + +async function sql( + app: ElectronApplication, + dataDir: string, + statement: string +) { + return app.evaluate( + (_electron, { file, dependency, statement }) => { + const Database = process + .getBuiltinModule('module') + .createRequire(dependency)(dependency), + db = new Database(file); + try { + return db.prepare(statement).reader + ? db.prepare(statement).all() + : db.prepare(statement).run(); + } finally { + db.close(); + } + }, + { + file: join(dataDir, 'databases/iptvnator.db'), + dependency: join(workspaceRoot, 'node_modules/better-sqlite3'), + statement, + } + ); +} + +async function sourceCount(page: Page) { + return page.evaluate( + async () => (await window.electron.dbGetAppPlaylists()).length + ); +} + +async function launchWithRecoveryChoice( + dataDir: string, + response: number, + retry = false +) { + const wrapper = join(dataDir, 'recovery-launch.cjs'); + await writeFile( + wrapper, + `const {dialog}=require('electron'); dialog.showMessageBox=async()=>({response:${response},checkboxChecked:false}); require(${JSON.stringify(electronMainPath)});` + ); + const app = await electron.launch({ + args: [ + ...(process.platform === 'linux' && process.env['CI'] + ? ['--no-sandbox', '--disable-gpu'] + : []), + wrapper, + ...(retry ? ['--recover-legacy-playlists'] : []), + ], + env: buildElectronLaunchEnvironment(dataDir), + }); + const page = await app.firstWindow(); + await page.waitForSelector('app-root'); + await page.waitForFunction( + () => typeof window.electron?.dbGetAppPlaylists === 'function' + ); + return { app, page }; +} + +test.describe('v0.19 profile migration', () => { + for (const active of ['xtream-2', 'stalker-60', 'm3u']) { + test(`imports all 65 sources with ${active} last active, offline`, async ({ + dataDir, + }) => { + await seedLegacyProfile(dataDir, active); + let launched = await launchElectronApp(dataDir); + try { + const page = launched.mainWindow; + await expect.poll(() => sourceCount(page)).toBe(65); + expect( + await launched.electronApp.evaluate(({ app }) => + app.getPath('userData') + ) + ).toBe(join(dataDir, 'electron-backend')); + await openSources(page); + await expect( + page.getByText('Legacy Stalker 0', { exact: true }) + ).toBeVisible(); + await expect( + page.getByText('Legacy Xtream 0', { exact: true }) + ).toBeVisible(); + const sources = await page.evaluate(() => + window.electron.dbGetAppPlaylists() + ); + expect(sources.filter((p) => p.macAddress)).toHaveLength(61); + expect(sources.filter((p) => p.serverUrl)).toHaveLength(3); + expect(sources.find((p) => p._id === active)?.lastUsage).toBe( + '2025-12-31T00:00:00.000Z' + ); + expect(sources.find((p) => p._id === 'xtream-2')).toMatchObject( + { + autoRefresh: true, + serverUrl: 'https://xtream-2.invalid', + username: 'synthetic-2', + password: 'synthetic-only', + } + ); + expect( + sources.find((p) => p._id === 'stalker-60') + ).toMatchObject({ + stalkerSerialNumber: 'synthetic-serial-60', + stalkerDeviceId1: 'synthetic-device-60', + favorites: legacyPlaylists().find( + (p) => p._id === 'stalker-60' + )?.['favorites'], + }); + expect(sources.find((p) => p._id === 'm3u')).toMatchObject({ + favorites: ['channel-1'], + userAgent: 'LegacySyntheticAgent', + count: 1, + }); + expect( + await page.evaluate(() => localStorage.getItem('volume')) + ).toBe('0.37'); + expect( + await page.evaluate( + () => + new Promise((resolve, reject) => { + const r = indexedDB.open('ngStorage', 1); + r.onerror = () => reject(r.error); + r.onsuccess = () => { + const db = r.result, + q = db + .transaction('localStorage') + .objectStore('localStorage') + .get('settings'); + q.onsuccess = () => { + db.close(); + resolve(q.result); + }; + }; + }) + ) + ).toMatchObject({ language: 'en', theme: 'dark' }); + expect( + await sql( + launched.electronApp, + dataDir, + 'SELECT count(*) AS count FROM content' + ) + ).toEqual([{ count: 1 }]); + expect( + await sql( + launched.electronApp, + dataDir, + 'SELECT count(*) AS count FROM favorites' + ) + ).toEqual([{ count: 1 }]); + expect( + await sql( + launched.electronApp, + dataDir, + 'SELECT count(*) AS count FROM recently_viewed' + ) + ).toEqual([{ count: 1 }]); + // Retained source database is independent of current deletion. + expect( + await page.evaluate( + () => + new Promise((resolve, reject) => { + const r = indexedDB.open('iptvnator', 1); + r.onerror = () => reject(r.error); + r.onsuccess = () => { + const db = r.result, + q = db + .transaction('playlists') + .objectStore('playlists') + .count(); + q.onsuccess = () => { + db.close(); + resolve(q.result); + }; + }; + }) + ) + ).toBe(65); + await page.evaluate(() => + window.electron.dbDeletePlaylist('stalker-0') + ); + } finally { + await closeElectronApp(launched); + } + launched = await launchElectronApp(dataDir); + try { + await expect + .poll(() => sourceCount(launched.mainWindow)) + .toBe(64); + } finally { + await closeElectronApp(launched); + } + }); + } + + test('rolls back all rows and the receipt on one failed SQLite write, then retries safely', async ({ + dataDir, + }) => { + const { electronApp: app, mainWindow: page } = + await launchElectronApp(dataDir); + try { + await page.evaluate( + async (key) => window.electron.dbSetAppState(key, ''), + migrationKey + ); + await sql( + app, + dataDir, + "CREATE TRIGGER synthetic_migration_failure BEFORE INSERT ON playlists WHEN NEW.id = 'xtream-1' BEGIN SELECT RAISE(ABORT, 'synthetic write failure'); END" + ); + const result = await page.evaluate(async (sources) => { + try { + await window.electron.dbMigrateAppPlaylists( + sources as Parameters< + typeof window.electron.dbMigrateAppPlaylists + >[0] + ); + return 'unexpected success'; + } catch { + return 'failed'; + } + }, legacyPlaylists()); + expect(result).toBe('failed'); + expect(await sourceCount(page)).toBe(0); + expect( + await page.evaluate( + (key) => window.electron.dbGetAppState(key), + migrationKey + ) + ).not.toBe('1'); + await sql(app, dataDir, 'DROP TRIGGER synthetic_migration_failure'); + await page.evaluate( + (sources) => + window.electron.dbMigrateAppPlaylists( + sources as Parameters< + typeof window.electron.dbMigrateAppPlaylists + >[0] + ), + legacyPlaylists() + ); + expect(await sourceCount(page)).toBe(65); + await page.evaluate( + (sources) => + window.electron.dbMigrateAppPlaylists( + sources as Parameters< + typeof window.electron.dbMigrateAppPlaylists + >[0] + ), + legacyPlaylists() + ); + expect(await sourceCount(page)).toBe(65); + } finally { + await closeElectronApp({ electronApp: app, mainWindow: page }); + } + }); + test('requires explicit recovery for an upgraded profile and preserves current sources/settings and deletions', async ({ + dataDir, + }) => { + const initial = await launchElectronApp(dataDir); + try { + await initial.mainWindow.evaluate(async () => { + await window.electron.dbUpsertAppPlaylist({ + _id: 'xtream-0', + title: 'Current edited source', + serverUrl: 'https://current.invalid', + username: 'current-user', + password: 'current-password', + count: 0, + importDate: '2026-01-01', + lastUsage: '2026-01-01', + autoRefresh: false, + }); + localStorage.setItem('volume', '0.73'); + }); + } finally { + await closeElectronApp(initial); + } + await seedLegacyProfile(dataDir, 'xtream-2', false); + let recovered = await launchWithRecoveryChoice(dataDir, 0); + try { + await expect + .poll(() => + recovered.page.evaluate( + (key) => window.electron.dbGetAppState(key), + recoveryKey + ) + ) + .toBe('declined'); + expect(await sourceCount(recovered.page)).toBe(1); + expect( + await recovered.app.evaluate(({ app }) => + app.getPath('userData') + ) + ).toBe(join(dataDir, 'user-data')); + } finally { + await closeElectronApp({ + electronApp: recovered.app, + mainWindow: recovered.page, + }); + } + recovered = await launchWithRecoveryChoice(dataDir, 1, true); + try { + await expect.poll(() => sourceCount(recovered.page)).toBe(65); + expect( + await recovered.page.evaluate(() => + window.electron.dbGetAppPlaylist('xtream-0') + ) + ).toMatchObject({ + title: 'Current edited source', + serverUrl: 'https://current.invalid', + username: 'current-user', + }); + expect( + await recovered.page.evaluate(() => + localStorage.getItem('volume') + ) + ).toBe('0.73'); + await recovered.page.evaluate(() => + window.electron.dbDeletePlaylist('stalker-0') + ); + } finally { + await closeElectronApp({ + electronApp: recovered.app, + mainWindow: recovered.page, + }); + } + recovered = await launchWithRecoveryChoice(dataDir, 1, true); + try { + await expect.poll(() => sourceCount(recovered.page)).toBe(64); + expect( + await recovered.page.evaluate(() => + window.electron.dbGetAppPlaylist('stalker-0') + ) + ).toBeNull(); + } finally { + await closeElectronApp({ + electronApp: recovered.app, + mainWindow: recovered.page, + }); + } + }); + test('restarts a failed real legacy import without losing the original store or cached data', async ({ + dataDir, + }) => { + await seedLegacyProfile(dataDir, 'xtream-2', true, true); + let launched = await launchElectronApp(dataDir); + try { + await expect( + launched.mainWindow.evaluate( + (sources) => + window.electron.dbMigrateAppPlaylists( + sources as Parameters< + typeof window.electron.dbMigrateAppPlaylists + >[0] + ), + legacyPlaylists() + ) + ).rejects.toThrow('Legacy playlist migration failed'); + expect(await sourceCount(launched.mainWindow)).toBe(1); + expect( + await launched.mainWindow.evaluate( + (key) => window.electron.dbGetAppState(key), + migrationKey + ) + ).not.toBe('1'); + expect( + await sql( + launched.electronApp, + dataDir, + 'SELECT count(*) AS count FROM favorites' + ) + ).toEqual([{ count: 1 }]); + await sql( + launched.electronApp, + dataDir, + 'DROP TRIGGER synthetic_migration_failure' + ); + } finally { + await closeElectronApp(launched); + } + launched = await launchElectronApp(dataDir); + try { + await expect.poll(() => sourceCount(launched.mainWindow)).toBe(65); + } finally { + await closeElectronApp(launched); + } + }); +}); diff --git a/apps/electron-backend-e2e/src/legacy-profile-fixture.ts b/apps/electron-backend-e2e/src/legacy-profile-fixture.ts new file mode 100644 index 000000000..e4fb31904 --- /dev/null +++ b/apps/electron-backend-e2e/src/legacy-profile-fixture.ts @@ -0,0 +1,231 @@ +import { _electron as electron } from '@playwright/test'; +import { mkdir, writeFile, readFile } from 'fs/promises'; +import { join } from 'path'; +import { workspaceRoot } from './electron-test-fixtures'; + +interface LegacyPlaylist extends Record { + _id: string; + title: string; + count: number; + importDate: string; + lastUsage: string; + autoRefresh: boolean; +} + +/** v0.19 Playlist shape, with no type discriminator; endpoints never resolve. */ +export function legacyPlaylists(active = 'xtream-2') { + const base = { + count: 0, + importDate: '2025-01-01T00:00:00.000Z', + lastUsage: '2025-01-01T00:00:00.000Z', + autoRefresh: false, + favorites: [], + }; + const portals: LegacyPlaylist[] = Array.from({ length: 61 }, (_, i) => ({ + ...base, + _id: `stalker-${i}`, + title: `Legacy Stalker ${i}`, + portalUrl: `https://stalker-${i}.invalid/stalker_portal/c/`, + macAddress: `00:1A:79:00:00:${i.toString(16).padStart(2, '0')}`, + stalkerSerialNumber: `synthetic-serial-${i}`, + stalkerDeviceId1: `synthetic-device-${i}`, + favorites: [ + { + id: '7', + name: 'Synthetic favorite', + cmd: 'ffmpeg http://synthetic.invalid/7', + stream_type: 'live', + }, + ], + recentlyViewed: [ + { id: '8', name: 'Synthetic recent', added_at: '2025-01-02' }, + ], + })); + for (let i = 0; i < 3; i++) + portals.push({ + ...base, + _id: `xtream-${i}`, + title: `Legacy Xtream ${i}`, + serverUrl: `https://xtream-${i}.invalid`, + username: `synthetic-${i}`, + password: 'synthetic-only', + autoRefresh: true, + }); + portals.push({ + ...base, + _id: 'm3u', + title: 'Legacy M3U', + count: 1, + favorites: ['channel-1'], + userAgent: 'LegacySyntheticAgent', + playlist: { + header: { raw: '#EXTM3U' }, + items: [ + { + id: 'channel-1', + name: 'Synthetic channel', + url: 'https://channel.invalid/live', + group: { title: 'Test' }, + }, + ], + }, + }); + return portals.map((p) => + p._id === active ? { ...p, lastUsage: '2025-12-31T00:00:00.000Z' } : p + ); +} + +export async function seedLegacyProfile( + dataDir: string, + active = 'xtream-2', + seedSqlite = true, + failWrite = false +) { + const fixture = join(dataDir, 'seed'); + const profile = join(dataDir, 'electron-backend'); + await mkdir(fixture, { recursive: true }); + await mkdir(profile, { recursive: true }); + const index = join(fixture, 'index.html'); + await writeFile( + index, + 'Synthetic v0.19 profile' + ); + await writeFile( + join(fixture, 'package.json'), + JSON.stringify({ + name: 'electron-backend', + version: '0.0.1', + main: 'main.cjs', + }) + ); + await writeFile( + join(fixture, 'main.cjs'), + `const { app, BrowserWindow } = require('electron'); +const { join } = require('path'); +app.setPath('userData', join(__dirname, '..', 'electron-backend')); +app.whenReady().then(() => new BrowserWindow({ + show: false, + webPreferences: { sandbox: true }, +}).loadFile(join(__dirname, 'index.html')));` + ); + const app = await electron.launch({ + args: [ + ...(process.platform === 'linux' && process.env['CI'] + ? ['--no-sandbox', '--disable-gpu'] + : []), + fixture, + ], + env: { ...process.env, NODE_ENV: 'test' }, + }); + try { + const page = await app.firstWindow(); + await page.waitForLoadState(); + await page.evaluate(async (playlists) => { + await new Promise((resolve, reject) => { + const req = indexedDB.open('iptvnator', 1); + req.onupgradeneeded = () => { + const store = req.result.createObjectStore('playlists', { + keyPath: '_id', + autoIncrement: false, + }); + // Exact v0.19 indexed-db.config.ts schema. + for (const field of [ + '_id', + 'filename', + 'title', + 'count', + 'playlist', + 'importDate', + 'lastUsage', + 'favorites', + 'recentlyViewed', + 'autoRefresh', + 'url', + 'filePath', + ]) + store.createIndex(field, field, { unique: false }); + }; + req.onerror = () => reject(req.error); + req.onsuccess = () => { + const db = req.result, + tx = db.transaction('playlists', 'readwrite'); + for (const p of playlists) + tx.objectStore('playlists').put(p); + tx.oncomplete = () => { + db.close(); + resolve(); + }; + tx.onabort = () => reject(tx.error); + }; + }); + // ngx-pwa/local-storage v0.19 defaults; unrelated preferences survive. + await new Promise((resolve, reject) => { + const req = indexedDB.open('ngStorage', 1); + req.onupgradeneeded = () => + req.result.createObjectStore('localStorage'); + req.onerror = () => reject(req.error); + req.onsuccess = () => { + const db = req.result, + tx = db.transaction('localStorage', 'readwrite'); + tx.objectStore('localStorage').put( + { language: 'en', player: 'html5', theme: 'dark' }, + 'settings' + ); + tx.oncomplete = () => { + db.close(); + resolve(); + }; + }; + }); + localStorage.setItem('volume', '0.37'); + }, legacyPlaylists(active)); + if (seedSqlite) { + await mkdir(join(dataDir, 'databases'), { recursive: true }); + const sql = await readFile( + join( + workspaceRoot, + 'apps/electron-backend-e2e/src/fixtures/v019-schema.sql' + ), + 'utf8' + ); + await app.evaluate( + (_electron, { path, dependency, sql, failWrite }) => { + const Database = process + .getBuiltinModule('module') + .createRequire(dependency)(dependency); + const db = new Database(path); + db.exec(sql); + db.prepare( + 'INSERT INTO playlists (id,name,type,serverUrl,username,password) VALUES (?,?,?,?,?,?)' + ).run( + 'xtream-2', + 'Legacy Xtream 2', + 'xtream', + 'https://stale-cache.invalid', + 'stale-user', + 'stale-password' + ); + db.exec( + "INSERT INTO categories (id,playlist_id,name,type,xtream_id) VALUES (1,'xtream-2','Synthetic','live',1); INSERT INTO content (id,category_id,title,xtream_id,type) VALUES (1,1,'Cached channel',7,'live'); INSERT INTO favorites (content_id,playlist_id) VALUES (1,'xtream-2'); INSERT INTO recently_viewed (content_id,playlist_id) VALUES (1,'xtream-2');" + ); + if (failWrite) + db.exec( + "CREATE TRIGGER synthetic_migration_failure BEFORE INSERT ON playlists WHEN NEW.id = 'xtream-1' BEGIN SELECT RAISE(ABORT, 'synthetic write failure'); END" + ); + db.close(); + }, + { + path: join(dataDir, 'databases', 'iptvnator.db'), + dependency: join( + workspaceRoot, + 'node_modules/better-sqlite3' + ), + sql, + failWrite, + } + ); + } + } finally { + await app.close(); + } +} diff --git a/apps/electron-backend/src/app/api/main.preload.spec-data.ts b/apps/electron-backend/src/app/api/main.preload.spec-data.ts index 16429250b..1f2db2db2 100644 --- a/apps/electron-backend/src/app/api/main.preload.spec-data.ts +++ b/apps/electron-backend/src/app/api/main.preload.spec-data.ts @@ -82,6 +82,18 @@ export const dbPreloadCases: PreloadInvokeCase[] = [ channel: 'DB_UPSERT_APP_PLAYLIST', forwardedArgs: [playlist], }, + { + method: 'dbRecoverLegacyPlaylists', + args: [], + channel: 'DB_RECOVER_LEGACY_PLAYLISTS', + forwardedArgs: [], + }, + { + method: 'dbMigrateAppPlaylists', + args: [playlists], + channel: 'DB_MIGRATE_APP_PLAYLISTS', + forwardedArgs: [playlists], + }, { method: 'dbUpsertAppPlaylists', args: [playlists], diff --git a/apps/electron-backend/src/app/api/main.preload.ts b/apps/electron-backend/src/app/api/main.preload.ts index b1720151b..4e59298e3 100644 --- a/apps/electron-backend/src/app/api/main.preload.ts +++ b/apps/electron-backend/src/app/api/main.preload.ts @@ -750,6 +750,10 @@ const electronApi: ElectronBridgeApi = { ipcRenderer.invoke('DB_GET_PLAYLIST', playlistId), dbUpsertAppPlaylist: (playlist: Playlist, _operationId?: string) => ipcRenderer.invoke('DB_UPSERT_APP_PLAYLIST', playlist), + dbMigrateAppPlaylists: (playlists: Playlist[]) => + ipcRenderer.invoke('DB_MIGRATE_APP_PLAYLISTS', playlists), + dbRecoverLegacyPlaylists: () => + ipcRenderer.invoke('DB_RECOVER_LEGACY_PLAYLISTS'), dbUpsertAppPlaylists: (playlists: Playlist[]) => ipcRenderer.invoke('DB_UPSERT_APP_PLAYLISTS', playlists), dbGetAppPlaylists: () => ipcRenderer.invoke('DB_GET_APP_PLAYLISTS'), diff --git a/apps/electron-backend/src/app/database/operations/playlist-migration.operations.ts b/apps/electron-backend/src/app/database/operations/playlist-migration.operations.ts new file mode 100644 index 000000000..6c2bf3cbd --- /dev/null +++ b/apps/electron-backend/src/app/database/operations/playlist-migration.operations.ts @@ -0,0 +1,76 @@ +import { eq } from 'drizzle-orm'; +import * as schema from '@iptvnator/shared/database/schema'; +import type { AppDatabase } from '../database.types'; +import { buildPlaylistRow } from './playlist.operations'; + +export const PLAYLIST_MIGRATION_KEY = 'm3u-playlists-indexeddb-to-sqlite-v1'; +export const LEGACY_PROFILE_MIGRATION_KEY = + 'playlists-electron-backend-profile-v1'; + +/** All rows and the receipt commit together; retries never replay a finished import. */ +export function migrateAppPlaylists( + db: AppDatabase, + playlists: Record[], + key = PLAYLIST_MIGRATION_KEY +): { success: boolean; count: number } { + try { + return db.transaction((tx) => { + if ( + tx + .select() + .from(schema.appState) + .where(eq(schema.appState.key, key)) + .get()?.value === '1' + ) { + return { success: true, count: 0 }; + } + if (!Array.isArray(playlists)) + throw new Error('Invalid legacy playlists'); + const ids = new Set(); + let count = 0; + for (const playlist of playlists) { + const row = buildPlaylistRow(playlist); + if (!row || ids.has(row.id)) + throw new Error('Invalid legacy playlist'); + ids.add(row.id); + const existing = tx + .select() + .from(schema.playlists) + .where(eq(schema.playlists.id, row.id)) + .get(); + // A payload identifies an authoritative application source. A + // v0.19 cache-only row can gain its legacy metadata without + // replacing any catalog/favorite/history rows. + if (existing?.payload) continue; + const merged = { + ...row, + // Cache-only credentials can be stale: v0.19 edited the + // authoritative source in IndexedDB without updating SQLite. + dateCreated: existing?.dateCreated ?? row.dateCreated, + lastUpdated: row.lastUpdated ?? existing?.lastUpdated, + }; + tx.insert(schema.playlists) + .values(merged) + .onConflictDoUpdate({ + target: schema.playlists.id, + set: merged, + }) + .run(); + count++; + } + tx.insert(schema.appState) + .values({ key, value: '1' }) + .onConflictDoUpdate({ + target: schema.appState.key, + set: { value: '1' }, + }) + .run(); + return { success: true, count }; + }); + } catch { + // Drizzle errors can include SQL parameter values (portal credentials). + throw new Error( + 'Legacy playlist migration failed; original data was retained' + ); + } +} diff --git a/apps/electron-backend/src/app/database/operations/playlist.operations.ts b/apps/electron-backend/src/app/database/operations/playlist.operations.ts index 1f036f5e9..b15c0988f 100644 --- a/apps/electron-backend/src/app/database/operations/playlist.operations.ts +++ b/apps/electron-backend/src/app/database/operations/playlist.operations.ts @@ -166,7 +166,7 @@ function inferPlaylistType(playlist: Record): PlaylistType { return PLAYLIST_TYPES.M3U_TEXT; } -function buildPlaylistRow( +export function buildPlaylistRow( playlist: Record ): schema.NewPlaylist | null { const id = getStringValue(playlist._id) ?? getStringValue(playlist.id); diff --git a/apps/electron-backend/src/app/events/database/playlist.events.ts b/apps/electron-backend/src/app/events/database/playlist.events.ts index 58d75af18..eb9683c75 100644 --- a/apps/electron-backend/src/app/events/database/playlist.events.ts +++ b/apps/electron-backend/src/app/events/database/playlist.events.ts @@ -4,12 +4,18 @@ */ import { ipcMain } from 'electron'; +import { recoverLegacyProfile } from '../../services/legacy-profile-recovery'; import { databaseWorkerClient } from '../../services/database-worker-client'; import { handleWorkerRequest, requestWorkerWithEvents, } from './worker-events.utils'; +ipcMain.handle('DB_RECOVER_LEGACY_PLAYLISTS', () => recoverLegacyProfile()); +handleWorkerRequest( + 'DB_MIGRATE_APP_PLAYLISTS', + (playlists: Record[]) => ({ playlists }) +); handleWorkerRequest('DB_CREATE_PLAYLIST', (playlist: Record) => playlist); handleWorkerRequest( 'DB_UPSERT_APP_PLAYLIST', diff --git a/apps/electron-backend/src/app/events/database/worker-ipc-contract.spec-data.ts b/apps/electron-backend/src/app/events/database/worker-ipc-contract.spec-data.ts index 00ae5f697..49a51080e 100644 --- a/apps/electron-backend/src/app/events/database/worker-ipc-contract.spec-data.ts +++ b/apps/electron-backend/src/app/events/database/worker-ipc-contract.spec-data.ts @@ -73,6 +73,11 @@ export const workerIpcContractCases: WorkerIpcContractCase[] = [ args: [playlist], payload: playlist, }, + { + operation: 'DB_MIGRATE_APP_PLAYLISTS', + args: [playlists], + payload: { playlists }, + }, { operation: 'DB_UPSERT_APP_PLAYLISTS', args: [playlists], diff --git a/apps/electron-backend/src/app/events/database/worker-ipc-contract.spec.ts b/apps/electron-backend/src/app/events/database/worker-ipc-contract.spec.ts index 0729cd338..4998d870b 100644 --- a/apps/electron-backend/src/app/events/database/worker-ipc-contract.spec.ts +++ b/apps/electron-backend/src/app/events/database/worker-ipc-contract.spec.ts @@ -92,11 +92,20 @@ describe('database worker IPC contract', () => { it('registers an IPC handler for every worker operation', () => { const registeredDbChannels = [...mockRegisteredHandlers.keys()] - .filter((channel) => channel !== 'DB_CANCEL_OPERATION') + .filter( + (channel) => + ![ + 'DB_CANCEL_OPERATION', + 'DB_RECOVER_LEGACY_PLAYLISTS', + ].includes(channel) + ) .sort(); expect(registeredDbChannels).toEqual([...DB_WORKER_OPERATIONS].sort()); expect(mockRegisteredHandlers.has('DB_CANCEL_OPERATION')).toBe(true); + expect(mockRegisteredHandlers.has('DB_RECOVER_LEGACY_PLAYLISTS')).toBe( + true + ); }); it.each(workerIpcContractCases)( diff --git a/apps/electron-backend/src/app/services/electron-profile-bootstrap.ts b/apps/electron-backend/src/app/services/electron-profile-bootstrap.ts new file mode 100644 index 000000000..059f46cfd --- /dev/null +++ b/apps/electron-backend/src/app/services/electron-profile-bootstrap.ts @@ -0,0 +1,11 @@ +import { app } from 'electron'; +import { getElectronUserDataPath } from '@iptvnator/shared/database'; +import { selectLegacyProfile } from './legacy-profile'; + +const override = getElectronUserDataPath(); +const current = override ?? app.getPath('userData'); +const legacy = selectLegacyProfile( + process.env.IPTVNATOR_E2E_DATA_DIR?.trim() || app.getPath('appData'), + current +); +if (legacy || override) app.setPath('userData', legacy ?? current); diff --git a/apps/electron-backend/src/app/services/legacy-profile-recovery.spec.ts b/apps/electron-backend/src/app/services/legacy-profile-recovery.spec.ts new file mode 100644 index 000000000..6c4dd810d --- /dev/null +++ b/apps/electron-backend/src/app/services/legacy-profile-recovery.spec.ts @@ -0,0 +1,130 @@ +const mockRequest = jest.fn(); +const mockDialog = jest.fn(); +const mockRead = jest.fn(); +const mockDestroy = jest.fn(); +const mockAccess = jest.fn(); +const mockCopy = jest.fn(); +const mockRemove = jest.fn(); + +jest.mock('electron', () => ({ + app: { + getPath: (key: string) => + key === 'appData' ? '/synthetic' : '/synthetic/current', + commandLine: { hasSwitch: () => false }, + }, + dialog: { showMessageBox: (...args: unknown[]) => mockDialog(...args) }, + session: { fromPath: () => ({}) }, + BrowserWindow: jest.fn(() => ({ + loadFile: async () => undefined, + destroy: mockDestroy, + webContents: { executeJavaScript: mockRead }, + })), +})); +jest.mock('fs/promises', () => ({ + access: (...args: unknown[]) => mockAccess(...args), + cp: (...args: unknown[]) => mockCopy(...args), + mkdtemp: async () => '/synthetic/snapshot', + writeFile: async () => undefined, + rm: (...args: unknown[]) => mockRemove(...args), +})); +jest.mock('../database/connection', () => ({ getDatabase: async () => ({}) })); +jest.mock('./database-worker-client', () => ({ + databaseWorkerClient: { + request: (...args: unknown[]) => mockRequest(...args), + }, +})); + +const key = 'playlists-electron-backend-profile-v1'; +describe('optional legacy profile recovery', () => { + beforeEach(() => { + jest.resetModules(); + jest.clearAllMocks(); + mockAccess.mockResolvedValue(undefined); + mockCopy.mockResolvedValue(undefined); + mockRemove.mockResolvedValue(undefined); + mockRequest.mockResolvedValue(null); + mockDialog.mockResolvedValue({ response: 0 }); + mockRead.mockResolvedValue([{ _id: 'synthetic-source' }]); + }); + it('defaults to keeping current sources and never opens the legacy DB before consent', async () => { + const { recoverLegacyProfile } = + await import('./legacy-profile-recovery'); + await recoverLegacyProfile(); + expect(mockDialog).toHaveBeenCalledWith( + expect.objectContaining({ + defaultId: 0, + cancelId: 0, + detail: expect.stringContaining('intentionally deleted'), + }) + ); + expect(mockCopy).not.toHaveBeenCalled(); + expect(mockRequest).toHaveBeenCalledWith('DB_SET_APP_STATE', { + key, + value: 'declined', + }); + }); + it('reads a disposable copy only after consent and commits through the worker', async () => { + mockDialog.mockResolvedValue({ response: 1 }); + const { recoverLegacyProfile } = + await import('./legacy-profile-recovery'); + await recoverLegacyProfile(); + expect(mockDialog.mock.invocationCallOrder[0]).toBeLessThan( + mockCopy.mock.invocationCallOrder[0] + ); + expect(mockCopy).toHaveBeenCalledWith( + '/synthetic/electron-backend/IndexedDB', + '/synthetic/snapshot/IndexedDB', + { recursive: true, dereference: true } + ); + expect(mockRequest).toHaveBeenCalledWith('DB_MIGRATE_APP_PLAYLISTS', { + key, + playlists: [{ _id: 'synthetic-source' }], + }); + expect(mockDestroy).toHaveBeenCalled(); + expect(mockRemove).toHaveBeenCalled(); + }); + it('keeps current sources usable on a corrupt legacy read, without a successful receipt', async () => { + const warning = jest + .spyOn(console, 'warn') + .mockImplementation(() => undefined); + mockDialog.mockResolvedValue({ response: 1 }); + mockRead.mockRejectedValueOnce(new Error('synthetic corruption')); + const { recoverLegacyProfile } = + await import('./legacy-profile-recovery'); + await expect(recoverLegacyProfile()).resolves.toBeUndefined(); + expect(mockRequest).not.toHaveBeenCalledWith( + 'DB_MIGRATE_APP_PLAYLISTS', + expect.anything() + ); + expect(mockRequest).not.toHaveBeenCalledWith( + 'DB_SET_APP_STATE', + expect.anything() + ); + expect(mockDialog).toHaveBeenLastCalledWith( + expect.objectContaining({ type: 'error' }) + ); + warning.mockRestore(); + }); + it('never reimports completed recovery, including after source deletion', async () => { + mockRequest.mockResolvedValue('1'); + const { recoverLegacyProfile } = + await import('./legacy-profile-recovery'); + await recoverLegacyProfile(); + expect(mockDialog).not.toHaveBeenCalled(); + expect(mockCopy).not.toHaveBeenCalled(); + }); + it('bounds a stalled reader and destroys it without committing a receipt', async () => { + jest.useFakeTimers(); + mockRead.mockReturnValueOnce(new Promise(() => undefined)); + const { readLegacyProfilePlaylists } = + await import('./legacy-profile-recovery'); + const read = readLegacyProfilePlaylists('/synthetic/electron-backend'); + const rejected = expect(read).rejects.toThrow('Legacy read timed out'); + await jest.advanceTimersByTimeAsync(15000); + await rejected; + expect(mockDestroy).toHaveBeenCalled(); + expect(mockRemove).toHaveBeenCalled(); + expect(mockRequest).not.toHaveBeenCalled(); + jest.useRealTimers(); + }); +}); diff --git a/apps/electron-backend/src/app/services/legacy-profile-recovery.ts b/apps/electron-backend/src/app/services/legacy-profile-recovery.ts new file mode 100644 index 000000000..defd68cbf --- /dev/null +++ b/apps/electron-backend/src/app/services/legacy-profile-recovery.ts @@ -0,0 +1,133 @@ +import { app, BrowserWindow, dialog, session } from 'electron'; +import { mkdtemp, rm, writeFile, access } from 'fs/promises'; +import { copyLegacyIndexedDb } from './legacy-profile'; +import { tmpdir } from 'os'; +import { join, resolve } from 'path'; +import { getDatabase } from '../database/connection'; +import { databaseWorkerClient } from './database-worker-client'; +import { LEGACY_PROFILE_MIGRATION_KEY } from '../database/operations/playlist-migration.operations'; + +/** Read only a disposable copy: Chromium must never open the original LevelDB. */ +export async function readLegacyProfilePlaylists( + profile: string +): Promise[]> { + const snapshot = await mkdtemp( + join(tmpdir(), 'iptvnator-legacy-recovery-') + ); + let reader: BrowserWindow | undefined; + let timer: ReturnType | undefined; + try { + await copyLegacyIndexedDb(profile, snapshot); + const index = join(snapshot, 'index.html'); + await writeFile( + index, + 'Legacy playlist reader' + ); + reader = new BrowserWindow({ + show: false, + webPreferences: { + session: session.fromPath(snapshot), + sandbox: true, + contextIsolation: true, + nodeIntegration: false, + }, + }); + const read = reader.loadFile(index).then(() => + reader!.webContents + .executeJavaScript(`new Promise((resolve, reject) => { + const request = indexedDB.open('iptvnator', 1); + request.onupgradeneeded = () => { request.transaction.abort(); reject(new Error('Legacy database is absent')); }; + request.onerror = () => reject(new Error('Legacy database could not be opened')); + request.onsuccess = () => { + const db = request.result; + if (!db.objectStoreNames.contains('playlists')) { db.close(); reject(new Error('Legacy store is absent')); return; } + const tx = db.transaction('playlists', 'readonly'); + const all = tx.objectStore('playlists').getAll(); + tx.oncomplete = () => { db.close(); resolve(all.result); }; + tx.onabort = tx.onerror = () => { db.close(); reject(new Error('Legacy read failed')); }; + }; + })`) + ); + return await Promise.race([ + read, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error('Legacy read timed out')), + 15000 + ); + }), + ]); + } finally { + clearTimeout(timer); + reader?.destroy(); + // Chromium can keep the disposable session's files open on Windows. + await rm(snapshot, { + recursive: true, + force: true, + maxRetries: 3, + }).catch(() => undefined); + } +} + +let recovery: Promise | undefined; +export function recoverLegacyProfile(): Promise { + return (recovery ??= recover().catch(() => { + recovery = undefined; + console.warn( + 'Legacy profile recovery failed; original data was retained' + ); + return dialog + .showMessageBox({ + type: 'error', + title: 'Legacy recovery could not finish', + message: + 'Your current sources are still available. Close the old IPTVnator version and restart to retry. The original data was retained.', + }) + .then(() => undefined); + })); +} + +async function recover(): Promise { + const root = + process.env.IPTVNATOR_E2E_DATA_DIR?.trim() || app.getPath('appData'); + const legacy = join(root, 'electron-backend'); + if (resolve(legacy) === resolve(app.getPath('userData'))) return; + try { + await access(join(legacy, 'IndexedDB', 'file__0.indexeddb.leveldb')); + } catch { + return; + } + await getDatabase(); + const state = await databaseWorkerClient.request( + 'DB_GET_APP_STATE', + { key: LEGACY_PROFILE_MIGRATION_KEY } + ); + if (state === '1') return; + if ( + state === 'declined' && + !app.commandLine.hasSwitch('recover-legacy-playlists') + ) + return; + const { response } = await dialog.showMessageBox({ + type: 'question', + title: 'Recover legacy IPTVnator sources', + message: 'An older IPTVnator profile was found.', + detail: 'Recover all missing sources from this profile? This can also restore sources you intentionally deleted after upgrading. Existing sources and current settings will be kept. The original profile will not be erased. Close the old IPTVnator version before continuing.', + buttons: ['Keep current sources', 'Recover all missing sources'], + defaultId: 0, + cancelId: 0, + noLink: true, + }); + if (response !== 1) { + await databaseWorkerClient.request('DB_SET_APP_STATE', { + key: LEGACY_PROFILE_MIGRATION_KEY, + value: 'declined', + }); + return; + } + const playlists = await readLegacyProfilePlaylists(legacy); + await databaseWorkerClient.request('DB_MIGRATE_APP_PLAYLISTS', { + playlists, + key: LEGACY_PROFILE_MIGRATION_KEY, + }); +} diff --git a/apps/electron-backend/src/app/services/legacy-profile.spec.ts b/apps/electron-backend/src/app/services/legacy-profile.spec.ts new file mode 100644 index 000000000..128e722d6 --- /dev/null +++ b/apps/electron-backend/src/app/services/legacy-profile.spec.ts @@ -0,0 +1,79 @@ +import { + mkdtempSync, + mkdirSync, + rmSync, + symlinkSync, + writeFileSync, + readFileSync, + lstatSync, +} from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { selectLegacyProfile, copyLegacyIndexedDb } from './legacy-profile'; + +describe('packaged legacy profile compatibility', () => { + let root: string; + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'iptvnator-profile-test-')); + }); + afterEach(() => rmSync(root, { recursive: true, force: true })); + it('finds v0.19 storage before a fresh packaged renderer creates an empty database', () => { + const legacy = join(root, 'electron-backend'); + mkdirSync(join(legacy, 'IndexedDB', 'file__0.indexeddb.leveldb'), { + recursive: true, + }); + expect(selectLegacyProfile(root, join(root, 'IPTVnator'))).toBe(legacy); + }); + it('preserves an already used current profile, even when it contains no playlists', () => { + mkdirSync( + join( + root, + 'electron-backend', + 'IndexedDB', + 'file__0.indexeddb.leveldb' + ), + { recursive: true } + ); + const current = join(root, 'IPTVnator'); + mkdirSync(join(current, 'IndexedDB'), { recursive: true }); + expect(selectLegacyProfile(root, current)).toBeNull(); + }); + it('does not select an unrelated or absent profile', () => { + expect(selectLegacyProfile(root, join(root, 'IPTVnator'))).toBeNull(); + }); + it('copies symlinked legacy storage into independent regular files', async () => { + const legacy = join(root, 'legacy'), + original = join(root, 'original'); + const snapshot = join(root, 'snapshot'); + mkdirSync(legacy); + mkdirSync(original); + writeFileSync(join(original, 'synthetic.ldb'), 'original data'); + symlinkSync(original, join(legacy, 'IndexedDB'), 'junction'); + await copyLegacyIndexedDb(legacy, snapshot); + expect(lstatSync(join(snapshot, 'IndexedDB')).isSymbolicLink()).toBe( + false + ); + writeFileSync( + join(snapshot, 'IndexedDB', 'synthetic.ldb'), + 'Chromium snapshot write' + ); + expect(readFileSync(join(original, 'synthetic.ldb'), 'utf8')).toBe( + 'original data' + ); + }); + it('keeps a current main-process config even before Chromium storage exists', () => { + mkdirSync( + join( + root, + 'electron-backend', + 'IndexedDB', + 'file__0.indexeddb.leveldb' + ), + { recursive: true } + ); + const current = join(root, 'IPTVnator'); + mkdirSync(current); + writeFileSync(join(current, 'config.json'), '{}'); + expect(selectLegacyProfile(root, current)).toBeNull(); + }); +}); diff --git a/apps/electron-backend/src/app/services/legacy-profile.ts b/apps/electron-backend/src/app/services/legacy-profile.ts new file mode 100644 index 000000000..f062103b3 --- /dev/null +++ b/apps/electron-backend/src/app/services/legacy-profile.ts @@ -0,0 +1,34 @@ +import { existsSync } from 'fs'; +import { cp } from 'fs/promises'; +import { join, resolve } from 'path'; + +/** v0.19 packaged package.json used Nx's project name, electron-backend. */ +export function selectLegacyProfile( + appData: string, + current: string +): string | null { + const legacy = join(appData, 'electron-backend'); + if (resolve(legacy) === resolve(current)) return null; + // Any current Chromium persistence is authoritative, including an empty + // playlist store after a user deliberately deleted their sources. + if ( + ['IndexedDB', 'Local Storage', 'Preferences', 'config.json'].some( + (entry) => existsSync(join(current, entry)) + ) + ) + return null; + return existsSync(join(legacy, 'IndexedDB', 'file__0.indexeddb.leveldb')) + ? legacy + : null; +} + +/** Materialize symlink targets so Chromium cannot write through to the original. */ +export async function copyLegacyIndexedDb( + profile: string, + snapshot: string +): Promise { + await cp(join(profile, 'IndexedDB'), join(snapshot, 'IndexedDB'), { + recursive: true, + dereference: true, + }); +} diff --git a/apps/electron-backend/src/app/workers/database-worker.types.ts b/apps/electron-backend/src/app/workers/database-worker.types.ts index 273376835..0eae7ea29 100644 --- a/apps/electron-backend/src/app/workers/database-worker.types.ts +++ b/apps/electron-backend/src/app/workers/database-worker.types.ts @@ -18,6 +18,7 @@ export const DB_WORKER_OPERATIONS = [ 'DB_CREATE_PLAYLIST', 'DB_UPSERT_APP_PLAYLIST', 'DB_UPSERT_APP_PLAYLISTS', + 'DB_MIGRATE_APP_PLAYLISTS', 'DB_GET_APP_PLAYLISTS', 'DB_GET_APP_PLAYLIST_METAS', 'DB_GET_APP_PLAYLIST', diff --git a/apps/electron-backend/src/app/workers/database.worker.ts b/apps/electron-backend/src/app/workers/database.worker.ts index 3315c7929..790604981 100644 --- a/apps/electron-backend/src/app/workers/database.worker.ts +++ b/apps/electron-backend/src/app/workers/database.worker.ts @@ -1,3 +1,4 @@ +import { migrateAppPlaylists } from '../database/operations/playlist-migration.operations'; import { closeWorkerDatabase, getWorkerDatabase, @@ -679,6 +680,14 @@ async function executeRequest( ); } + case 'DB_MIGRATE_APP_PLAYLISTS': { + const payload = message.payload as { + playlists: Record[]; + key?: string; + }; + return migrateAppPlaylists(db, payload.playlists, payload.key); + } + case 'DB_UPSERT_APP_PLAYLISTS': { return upsertAppPlaylists( db, diff --git a/apps/electron-backend/src/main.ts b/apps/electron-backend/src/main.ts index 367ca55ad..0b50087df 100644 --- a/apps/electron-backend/src/main.ts +++ b/apps/electron-backend/src/main.ts @@ -1,5 +1,6 @@ +// Select persistence before eager imports (notably electron-conf) cache userData. +import './app/services/electron-profile-bootstrap'; import { app, BrowserWindow } from 'electron'; -import { getElectronUserDataPath } from '@iptvnator/shared/database'; import { autoUpdater } from 'electron-updater'; import fixPath from 'fix-path'; import App from './app/app'; @@ -68,12 +69,6 @@ if ( app.commandLine.appendSwitch('ozone-platform', 'x11'); } -const electronUserDataPath = getElectronUserDataPath(); - -if (electronUserDataPath) { - app.setPath('userData', electronUserDataPath); -} - // The frame-copy embedded MPV engine must be decided before the main window // exists: it relaxes the window sandbox for the preload frame pump, and // webPreferences are fixed at window creation. The Settings toggle persists diff --git a/docs/architecture/m3u-playlist-module.md b/docs/architecture/m3u-playlist-module.md index 26f6c01a3..c6b46ae24 100644 --- a/docs/architecture/m3u-playlist-module.md +++ b/docs/architecture/m3u-playlist-module.md @@ -13,6 +13,77 @@ The M3U playlist module provides: - Per-playlist group visibility management in the groups view - Video playback with multiple player backends +## Desktop upgrades from legacy profiles + +v0.19.0 stores the complete source list in IndexedDB database `iptvnator`, +version 1, store `playlists`, key path `_id`. Source type is inferred from +`serverUrl` (Xtream), `macAddress` (Stalker), or the M3U fields; the last-used +source does not select which records are migrated. Settings live separately +in `ngStorage`, and player preferences also use localStorage. SQLite already +holds the Xtream catalogs opened in v0.19, including favorites and history; +it is not the authoritative inventory of configured sources. + +The published v0.19 Linux DEB's `app.asar/package.json` names the app +`electron-backend`. Later packages set `productName: IPTVnator`. Electron +resolves different `userData` directories from those identities; a later +`app.setName('iptvnator')` does not reset the cached path. The shared +`~/.iptvnator/databases/iptvnator.db` is independent of that path, so cached +Xtream sources can remain visible while the complete IndexedDB list is in +the older profile. Linux usually places these profiles below `~/.config/`. +Snap/Flatpak confinement and a changed installation type can give them +different roots; recovery does not scan unrelated profiles or sandbox roots. + +`electron-profile-bootstrap.ts` runs before eager main-process imports such +as `electron-conf`. If the current profile has no IndexedDB, Local Storage, +Preferences, or `config.json` yet, it reuses the known `electron-backend` profile containing +`IndexedDB/file__0.indexeddb.leveldb`, preserving its settings as well. It never +switches an already-used current profile. E2E overrides keep both profiles and +SQLite under the isolated test root. + +`PlaylistsService` sends the complete current-profile IndexedDB list through +`DB_MIGRATE_APP_PLAYLISTS`. The worker validates every ID (including duplicates) +and commits all rows plus `m3u-playlists-indexeddb-to-sqlite-v1` in one synchronous +transaction. Any malformed source or failed write rolls back the entire batch; +restarting retries it. Original IndexedDB records remain in place. Full SQLite +rows with a `payload` are authoritative and are skipped. Cache-only Xtream rows +receive the IndexedDB source configuration (cached credentials can be stale), +retaining creation/import timestamps and all linked catalogs, favorites, +history and playback positions. No provider connection is needed. Migration +preserves stored data; it does not create missing provider catalogs or translate +unrelated historical formats into new features. + +For an already-used current profile, a separate native recovery offer defaults +to **Keep current sources**. **Recover all missing sources** explicitly explains +that it may also restore sources intentionally deleted after upgrading. There +is no pre-existing deletion ledger that can distinguish these from omissions, +so recovery is never automatic. Close the old version before accepting. +Recovery opens a disposable copy of the old IndexedDB, never the original, +and keeps current settings and full source records. Its separate atomic receipt +`playlists-electron-backend-profile-v1` prevents any subsequent replay, including +after source deletion. Declining records `declined`; starting the app with +`--recover-legacy-playlists` offers it again. That switch does not bypass consent +or replay a completed recovery. A recovery read/write error leaves current +sources usable, reports the failure, writes no successful receipt, and can be +retried on restart. + +If the old profile is absent, unreadable, in another sandbox, or was already +cleared by an earlier successful migration, missing source data cannot be +reconstructed from the current SQLite cache. Restore a backup or recreate those +sources; do not clear migration flags to force a blind replay. + +SQL initialization also creates indexes on added columns only after the column +migrations. In particular, `idx_content_epg_channel` must follow the addition of +`content.epg_channel_id`; creating it in the initial CREATE TABLE pass aborts +initialization on the v0.19 schema before the playlist migration can run. + +Validation: `electron-backend-e2e:e2e-ci--src/legacy-playlist-migration.e2e.ts` +seeds the exact v0.19 IndexedDB schema and verbatim SQL CREATE statements with +61 Stalker sources, three Xtream sources, M3U, and linked cached user data. +It exercises source-list UI, alternate last-used sources, retained settings, +recovery consent, existing data, write failure, restart, and deletion after +migration. Local macOS Electron verification does not substitute for installed +Linux Mint MATE, Snap, or Flatpak upgrade testing. + ## Module Structure ``` diff --git a/libs/services/src/lib/playlist-migration.spec.ts b/libs/services/src/lib/playlist-migration.spec.ts new file mode 100644 index 000000000..a68e974ee --- /dev/null +++ b/libs/services/src/lib/playlist-migration.spec.ts @@ -0,0 +1,68 @@ +import { firstValueFrom, of } from 'rxjs'; +import { PlaylistsService } from './playlists.service'; + +describe('Electron legacy playlist migration', () => { + const original = window.electron; + afterEach(() => { + window.electron = original; + jest.restoreAllMocks(); + }); + function setup() { + const playlists = [ + { _id: 'stalker-1', macAddress: '00:1A:79:00:00:01' }, + { _id: 'xtream-1', serverUrl: 'https://synthetic.invalid' }, + { _id: 'm3u-1', playlist: { items: [] } }, + ]; + const dbService = { + getAll: jest.fn(() => of(playlists)), + clear: jest.fn(() => of(undefined)), + }; + const electron = { + dbGetAppState: jest.fn(async () => null), + dbSetAppState: jest.fn(async () => undefined), + dbGetAppPlaylists: jest.fn(async () => []), + dbUpsertAppPlaylists: jest.fn(async () => ({ + success: true, + count: playlists.length, + })), + dbMigrateAppPlaylists: jest.fn(async () => ({ + success: true, + count: playlists.length, + })), + }; + window.electron = electron as unknown as typeof window.electron; + const service = Object.create( + PlaylistsService.prototype + ) as PlaylistsService; + Object.assign(service, { + dbService, + runtime: { supportsSqlite: true }, + electronMigrationPromise: null, + }); + return { playlists, dbService, electron, service }; + } + it('retains legacy IndexedDB and uses the atomic non-overwriting migration', async () => { + const { playlists, dbService, electron, service } = setup(); + await firstValueFrom(service.getAllPlaylists()); + expect(electron.dbMigrateAppPlaylists).toHaveBeenCalledWith(playlists); + expect(dbService.clear).not.toHaveBeenCalled(); + expect(electron.dbSetAppState).not.toHaveBeenCalledWith( + 'm3u-playlists-indexeddb-to-sqlite-v1', + '1' + ); + }); + it('retries failed migration and does not present an incomplete source list as success', async () => { + const { electron, service } = setup(); + jest.spyOn(console, 'error').mockImplementation(() => undefined); + electron.dbMigrateAppPlaylists.mockRejectedValueOnce( + new Error('synthetic failure') + ); + await expect(firstValueFrom(service.getAllPlaylists())).rejects.toThrow( + 'synthetic failure' + ); + await expect( + firstValueFrom(service.getAllPlaylists()) + ).resolves.toEqual([]); + expect(electron.dbMigrateAppPlaylists).toHaveBeenCalledTimes(2); + }); +}); diff --git a/libs/services/src/lib/playlists.service.ts b/libs/services/src/lib/playlists.service.ts index c26a6142e..d667f2129 100644 --- a/libs/services/src/lib/playlists.service.ts +++ b/libs/services/src/lib/playlists.service.ts @@ -72,6 +72,8 @@ type PlaylistStorageElectronApi = { playlist: Playlist, operationId?: string ) => Promise; + dbMigrateAppPlaylists: (playlists: Playlist[]) => Promise; + dbRecoverLegacyPlaylists?: () => Promise; dbUpsertAppPlaylists: (playlists: Playlist[]) => Promise; }; @@ -141,8 +143,12 @@ export class PlaylistsService { if (!this.electronMigrationPromise) { this.electronMigrationPromise = (async () => { await this.migrateIndexedDbPlaylistsToSqlite(); + await this.electronApi?.dbRecoverLegacyPlaylists?.(); await this.migrateStalkerPlaylistMetadataInSqlite(); - })(); + })().catch((error) => { + this.electronMigrationPromise = null; + throw error; + }); } return this.electronMigrationPromise; @@ -176,34 +182,22 @@ export class PlaylistsService { } private async migrateIndexedDbPlaylistsToSqlite(): Promise { - try { - const electron = this.electronApi; - if (!electron) { - return; - } - - const alreadyMigrated = await electron.dbGetAppState( - SQLITE_PLAYLIST_MIGRATION_FLAG - ); - if (alreadyMigrated === '1') { - return; - } - - const indexedDbPlaylists = await firstValueFrom( - this.dbService.getAll(DbStores.Playlists) - ); - - if (indexedDbPlaylists.length > 0) { - await electron.dbUpsertAppPlaylists(indexedDbPlaylists); - await firstValueFrom(this.dbService.clear(DbStores.Playlists)); - } - + const electron = this.electronApi; + if (!electron) return; + if ( + (await electron.dbGetAppState(SQLITE_PLAYLIST_MIGRATION_FLAG)) === + '1' + ) + return; + const playlists = await firstValueFrom( + this.dbService.getAll(DbStores.Playlists) + ); + if (playlists.length) { + // The worker commits rows and the receipt atomically. Keep the + // original IndexedDB as a recovery source, even after success. + await electron.dbMigrateAppPlaylists(playlists); + } else { await electron.dbSetAppState(SQLITE_PLAYLIST_MIGRATION_FLAG, '1'); - } catch (error) { - console.error( - 'Failed to migrate IndexedDB playlists to SQLite:', - error - ); } } diff --git a/libs/shared/database/src/lib/connection.ts b/libs/shared/database/src/lib/connection.ts index 01fc74b92..304f47588 100644 --- a/libs/shared/database/src/lib/connection.ts +++ b/libs/shared/database/src/lib/connection.ts @@ -239,7 +239,6 @@ const CREATE_TABLE_STATEMENTS = [ `CREATE INDEX IF NOT EXISTS idx_categories_playlist ON categories(playlist_id)`, `CREATE INDEX IF NOT EXISTS idx_content_title ON content(title)`, `CREATE INDEX IF NOT EXISTS idx_content_xtream ON content(xtream_id)`, - `CREATE INDEX IF NOT EXISTS idx_content_epg_channel ON content(epg_channel_id)`, `CREATE INDEX IF NOT EXISTS idx_content_type_added ON content(type, added)`, `CREATE INDEX IF NOT EXISTS idx_categories_type ON categories(type)`, // Partial covering index for visible categories — supports the dashboard's @@ -429,6 +428,8 @@ const COLUMN_MIGRATION_STATEMENTS = [ ]; const INDEX_MIGRATION_STATEMENTS = [ + // Existing v0.19 content tables gain this column above, after CREATE TABLE. + `CREATE INDEX IF NOT EXISTS idx_content_epg_channel ON content(epg_channel_id)`, // v1.3.0 -> v1.4.0: Prevent duplicate Xtream categories/content rows `CREATE UNIQUE INDEX IF NOT EXISTS categories_playlist_type_xtream_unique ON categories(playlist_id, type, xtream_id)`, `CREATE UNIQUE INDEX IF NOT EXISTS content_category_type_xtream_unique ON content(category_id, type, xtream_id)`, diff --git a/libs/shared/interfaces/src/lib/electron-api.interface.ts b/libs/shared/interfaces/src/lib/electron-api.interface.ts index 2fcd2bfdb..4229bf3ef 100644 --- a/libs/shared/interfaces/src/lib/electron-api.interface.ts +++ b/libs/shared/interfaces/src/lib/electron-api.interface.ts @@ -889,6 +889,10 @@ export interface ElectronBridgeApi { */ operationId?: string ) => Promise; + dbMigrateAppPlaylists: ( + playlists: Playlist[] + ) => Promise<{ success: boolean; count: number }>; + dbRecoverLegacyPlaylists: () => Promise; dbUpsertAppPlaylists: ( playlists: Playlist[] ) => Promise;