From afa4d8c33a97172c5f88a75d8569d501da9cbc14 Mon Sep 17 00:00:00 2001 From: 4gray Date: Tue, 8 Sep 2026 02:24:36 +0200 Subject: [PATCH] fix(downloads): protect archive removal and missing-file recovery --- .../src/xtream-catchup-timezone.e2e.ts | 60 ++++++- .../database/download-catchup-removal.spec.ts | 69 ++++++++ .../database/download-catchup-removal.ts | 66 ++++++++ .../database/download-redownload.spec.ts | 35 +++++ .../events/database/download-redownload.ts | 12 +- .../database/download-removal-requests.ts | 147 ++++++++++++++++++ .../app/events/database/download-runtime.ts | 7 + .../events/database/downloads.events.spec.ts | 39 ++++- .../app/events/database/downloads.events.ts | 131 ++-------------- .../events/database/downloads.test-helpers.ts | 20 ++- docs/architecture/download-manager.md | 10 +- tools/coverage/coverage-policy.json | 11 +- 12 files changed, 477 insertions(+), 130 deletions(-) create mode 100644 apps/electron-backend/src/app/events/database/download-catchup-removal.spec.ts create mode 100644 apps/electron-backend/src/app/events/database/download-catchup-removal.ts create mode 100644 apps/electron-backend/src/app/events/database/download-removal-requests.ts diff --git a/apps/electron-backend-e2e/src/xtream-catchup-timezone.e2e.ts b/apps/electron-backend-e2e/src/xtream-catchup-timezone.e2e.ts index 0be651cd0..ac80a84b0 100644 --- a/apps/electron-backend-e2e/src/xtream-catchup-timezone.e2e.ts +++ b/apps/electron-backend-e2e/src/xtream-catchup-timezone.e2e.ts @@ -2,7 +2,13 @@ import { getDownloadPlayPaths, installDownloadPlayCapture, } from './downloads.e2e-support'; -import { mkdirSync, readFileSync, readdirSync } from 'node:fs'; +import { + mkdirSync, + readFileSync, + readdirSync, + unlinkSync, + writeFileSync, +} from 'node:fs'; import { join } from 'node:path'; import type { Page } from '@playwright/test'; import { @@ -500,6 +506,58 @@ test('@downloads @epg @xtream @electron downloads a completed archive into the l expect( readdirSync(folder).filter((name) => name.endsWith('.ts')) ).toHaveLength(1); + // Missing-file recovery must reserve fresh output and preserve a foreign + // entry at the old partial name, even with a completed journal present. + unlinkSync(row.filePath); + writeFileSync(row.filePath + '.part', 'unrelated retained file'); + expect( + await app.mainWindow.evaluate( + (id) => window.electron.downloadsRedownloadMissing(id), + row.id + ) + ).toEqual({ success: true }); + await expect + .poll( + async () => + ( + await app.mainWindow.evaluate(() => + window.electron.downloadsGetList() + ) + ).find((entry) => entry.id === row.id)?.status + ) + .toBe('completed'); + const redownloaded = ( + await app.mainWindow.evaluate(() => + window.electron.downloadsGetList() + ) + ).find((entry) => entry.id === row.id); + if (!redownloaded?.filePath) + throw new Error('Missing re-downloaded archive'); + expect(redownloaded.filePath).not.toBe(row.filePath); + expect(readFileSync(row.filePath + '.part', 'utf8')).toBe( + 'unrelated retained file' + ); + expect(readFileSync(redownloaded.filePath)).toEqual( + readFileSync('apps/xtream-mock-server/src/fixtures/live.mpegts') + ); + writeFileSync( + redownloaded.filePath + '.part', + 'unrelated terminal file' + ); + expect( + await app.mainWindow.evaluate( + (id) => window.electron.downloadsRemove(id), + row.id + ) + ).toEqual({ success: true }); + expect(readFileSync(redownloaded.filePath + '.part', 'utf8')).toBe( + 'unrelated terminal file' + ); + expect( + await app.mainWindow.evaluate(() => + window.electron.downloadsGetList() + ) + ).toEqual([]); } finally { await closeElectronApp(app); } diff --git a/apps/electron-backend/src/app/events/database/download-catchup-removal.spec.ts b/apps/electron-backend/src/app/events/database/download-catchup-removal.spec.ts new file mode 100644 index 000000000..dd40fb453 --- /dev/null +++ b/apps/electron-backend/src/app/events/database/download-catchup-removal.spec.ts @@ -0,0 +1,69 @@ +import { + mkdtempSync, + lstatSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, + unlinkSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { removeJournaledCatchupPartial } from './download-catchup-removal'; +import type { ArchivePartialProof } from './download-catchup-journal'; +jest.mock('node:fs', () => { + const actual = jest.requireActual('node:fs'); + return { + ...actual, + renameSync: jest.fn(actual.renameSync), + unlinkSync: jest.fn(actual.unlinkSync), + }; +}); +const actual = jest.requireActual('node:fs'); +let directory: string, filePath: string, proof: ArchivePartialProof; +beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), 'archive-remove-')); + filePath = join(directory, 'show.ts'); + writeFileSync(filePath + '.part', 'owned bytes'); + proof = { + version: 1, + phase: 'transfer', + filePath, + partialIdentity: lstatSync(filePath + '.part'), + }; + jest.mocked(renameSync).mockReset().mockImplementation(actual.renameSync); + jest.mocked(unlinkSync).mockReset().mockImplementation(actual.unlinkSync); +}); +afterEach(() => rmSync(directory, { recursive: true, force: true })); +it('removes only the journaled partial', () => { + removeJournaledCatchupPartial(filePath, proof); + expect(() => lstatSync(filePath + '.part')).toThrow(); +}); +it('preserves entries without proof', () => { + removeJournaledCatchupPartial(filePath, undefined); + expect(readFileSync(filePath + '.part', 'utf8')).toBe('owned bytes'); +}); +it('preserves a replaced regular partial in place', () => { + renameSync(filePath + '.part', join(directory, 'original')); + writeFileSync(filePath + '.part', 'unrelated bytes'); + removeJournaledCatchupPartial(filePath, proof); + expect(readFileSync(filePath + '.part', 'utf8')).toBe('unrelated bytes'); +}); +it('restores a replacement captured at the cleanup boundary', () => { + jest.mocked(renameSync).mockImplementationOnce((from, to) => { + actual.renameSync(from, join(directory, 'original')); + writeFileSync(from, 'unrelated bytes'); + actual.renameSync(from, to); + }); + removeJournaledCatchupPartial(filePath, proof); + expect(readFileSync(filePath + '.part', 'utf8')).toBe('unrelated bytes'); +}); +it('restores an owned partial and reports an I/O error for retry', () => { + jest.mocked(unlinkSync).mockImplementationOnce(() => { + throw Object.assign(new Error('locked'), { code: 'EACCES' }); + }); + expect(() => removeJournaledCatchupPartial(filePath, proof)).toThrow( + 'locked' + ); + expect(readFileSync(filePath + '.part', 'utf8')).toBe('owned bytes'); +}); diff --git a/apps/electron-backend/src/app/events/database/download-catchup-removal.ts b/apps/electron-backend/src/app/events/database/download-catchup-removal.ts new file mode 100644 index 000000000..f851b1fce --- /dev/null +++ b/apps/electron-backend/src/app/events/database/download-catchup-removal.ts @@ -0,0 +1,66 @@ +import { + lstatSync, + type Stats, + mkdtempSync, + renameSync, + linkSync, + unlinkSync, + rmdirSync, +} from 'node:fs'; +import { dirname, join } from 'node:path'; +import type { ArchiveDownloadProof } from './download-catchup-journal'; +import type { ArchiveFileIdentity } from './download-catchup-output'; + +/** IPC removal stays synchronous after its runtime guard, like VOD cleanup. */ +export function removeJournaledCatchupPartial( + filePath: string | null, + proof: ArchiveDownloadProof | undefined +): void { + // No proof means no authority to remove the retained entry. + if (!filePath || !proof || proof.filePath !== filePath) return; + const path = `${filePath}.part`; + const matches = (file: Stats, identity: ArchiveFileIdentity) => + file.isFile() && file.dev === identity.dev && file.ino === identity.ino; + try { + if (!matches(lstatSync(path), proof.partialIdentity)) return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw error; + } + const directory = mkdtempSync(join(dirname(path), '.iptvnator-cleanup-')); + const captured = join(directory, 'entry'); + try { + renameSync(path, captured); + if (matches(lstatSync(captured), proof.partialIdentity)) { + unlinkSync(captured); + } else { + try { + linkSync(captured, path); + unlinkSync(captured); + } catch { + console.warn( + '[Downloads] Replaced file retained for recovery:', + captured + ); + } + } + } catch (error) { + // Keep owned files reachable for another Remove attempt after I/O errors. + try { + linkSync(captured, path); + unlinkSync(captured); + } catch { + console.warn( + '[Downloads] Partial retained for recovery:', + captured + ); + } + throw error; + } finally { + try { + rmdirSync(directory); + } catch { + /* never recursively remove a capture */ + } + } +} diff --git a/apps/electron-backend/src/app/events/database/download-redownload.spec.ts b/apps/electron-backend/src/app/events/database/download-redownload.spec.ts index 1a1d0b91a..cd12f9cf1 100644 --- a/apps/electron-backend/src/app/events/database/download-redownload.spec.ts +++ b/apps/electron-backend/src/app/events/database/download-redownload.spec.ts @@ -67,6 +67,8 @@ async function setup(options: SetupOptions = {}) { } }); const enqueueDownload = jest.fn(); + const proof = { version: 1, phase: 'transfer' }; + const removeJournaledCatchupPartial = jest.fn(); jest.doMock('node:fs', () => ({ ...jest.requireActual('node:fs'), @@ -78,6 +80,14 @@ async function setup(options: SetupOptions = {}) { })); jest.doMock('../url-safety', () => ({ assertRemoteUrlAllowed })); jest.doMock('./download-file-path', () => ({ removePartialDownloadFile })); + jest.doMock('./download-catchup-journal', () => ({ + readArchiveFinalizations: jest + .fn() + .mockResolvedValue(new Map([[42, proof]])), + })); + jest.doMock('./download-catchup-removal', () => ({ + removeJournaledCatchupPartial, + })); jest.doMock('./download-runtime', () => ({ enqueueDownload })); const { redownloadMissingRequest } = await import('./download-redownload'); @@ -85,6 +95,8 @@ async function setup(options: SetupOptions = {}) { accessSync, assertRemoteUrlAllowed, enqueueDownload, + removeJournaledCatchupPartial, + proof, lstatSync, redownloadMissingRequest, removePartialDownloadFile, @@ -124,6 +136,29 @@ describe('redownload missing completed file', () => { }); }); + it('reserves a fresh archive path and uses journal-backed previous-partial cleanup', async () => { + const catchup = { + channelName: 'News', + startTimestamp: 100, + stopTimestamp: 200, + }; + const h = await setup({ row: { contentType: 'catchup', catchup } }); + await expect(h.redownloadMissingRequest(42)).resolves.toEqual({ + success: true, + }); + expect(h.removeJournaledCatchupPartial).toHaveBeenCalledWith( + '/downloads/movie.mp4', + h.proof + ); + expect(h.removePartialDownloadFile).not.toHaveBeenCalled(); + expect(h.set).toHaveBeenCalledWith( + expect.objectContaining({ status: 'queued', filePath: null }) + ); + expect(h.enqueueDownload).toHaveBeenCalledWith( + expect.objectContaining({ catchup, filePath: null }) + ); + }); + it('adds the Xtream fallback User-Agent to a legacy row', async () => { const harness = await setup({ playlistType: 'xtream', diff --git a/apps/electron-backend/src/app/events/database/download-redownload.ts b/apps/electron-backend/src/app/events/database/download-redownload.ts index d99393ca5..a430fffd0 100644 --- a/apps/electron-backend/src/app/events/database/download-redownload.ts +++ b/apps/electron-backend/src/app/events/database/download-redownload.ts @@ -1,3 +1,5 @@ +import { readArchiveFinalizations } from './download-catchup-journal'; +import { removeJournaledCatchupPartial } from './download-catchup-removal'; import { catchupForDownload } from './download-catchup'; import { and, eq, sql } from 'drizzle-orm'; import { accessSync, constants } from 'node:fs'; @@ -66,7 +68,12 @@ export async function redownloadMissingRequest( const headers = await resolveStoredDownloadHeaders(db, item); try { - removePartialDownloadFile(item.filePath); + if (catchup) { + const proof = (await readArchiveFinalizations(db, [item.id])).get( + item.id + ); + removeJournaledCatchupPartial(item.filePath, proof); + } else removePartialDownloadFile(item.filePath); } catch (error) { console.error( '[Downloads] Failed to delete partial before missing-file recovery:', @@ -82,6 +89,7 @@ export async function redownloadMissingRequest( .update(schema.downloads) .set({ bytesDownloaded: 0, + ...(catchup ? { filePath: null } : {}), errorMessage: null, resumeValidator: null, status: 'queued', @@ -105,7 +113,7 @@ export async function redownloadMissingRequest( catchup, directory: dirname(item.filePath), fileName: basename(item.filePath), - filePath: item.filePath, + filePath: catchup ? null : item.filePath, headers, id: item.id, resumeValidator: null, diff --git a/apps/electron-backend/src/app/events/database/download-removal-requests.ts b/apps/electron-backend/src/app/events/database/download-removal-requests.ts new file mode 100644 index 000000000..794ced90c --- /dev/null +++ b/apps/electron-backend/src/app/events/database/download-removal-requests.ts @@ -0,0 +1,147 @@ +import { and, eq, inArray } from 'drizzle-orm'; +import { getDatabase } from '../../database/connection'; +import * as schema from '../../database/schema'; +import { readArchiveFinalizations } from './download-catchup-journal'; +import { removeJournaledCatchupPartial } from './download-catchup-removal'; +import { removePartialDownloadFile } from './download-file-path'; +import { + broadcastDownloadUpdate, + isDownloadCommitting, + hasRuntimeDownload, + removeDownloadFromRuntime, +} from './download-runtime'; + +const removablePartialStatuses = new Set([ + 'queued', + 'paused', + 'completed', + 'failed', + 'canceled', +]); + +export async function removeDownloadRequest(downloadId: number) { + try { + console.log('[Downloads] Remove download:', downloadId); + const db = await getDatabase(); + const rows = await db + .select({ + filePath: schema.downloads.filePath, + contentType: schema.downloads.contentType, + status: schema.downloads.status, + }) + .from(schema.downloads) + .where(eq(schema.downloads.id, downloadId)) + .limit(1); + const row = rows[0]; + const proof = + row?.contentType === 'catchup' + ? (await readArchiveFinalizations(db, [downloadId])).get( + downloadId + ) + : undefined; + if (isDownloadCommitting(downloadId)) + return { + success: false, + error: 'Download is completing; try again shortly', + }; + if (row?.filePath && removablePartialStatuses.has(row.status)) { + try { + if (row.contentType === 'catchup') + removeJournaledCatchupPartial(row.filePath, proof); + else removePartialDownloadFile(row.filePath); + } catch (cleanupError) { + // Keep the row (and its runtime entry) so the .part is never + // orphaned, but answer with a structured failure the UI can + // surface instead of an opaque IPC rejection. Retrying the + // remove re-attempts the deletion. + console.error( + '[Downloads] Failed to delete partial file on remove:', + row.filePath, + cleanupError + ); + return { + error: 'Could not delete the partial file', + success: false, + }; + } + } + if (removeDownloadFromRuntime(downloadId) === false) + return { + success: false, + error: 'Download is completing; try again shortly', + }; + await db + .delete(schema.downloads) + .where(eq(schema.downloads.id, downloadId)); + broadcastDownloadUpdate(); + return { success: true }; + } catch (error) { + console.error('[Downloads] Error removing download:', error); + throw error; + } +} + +export async function clearCompletedDownloadsRequest(playlistId?: string) { + try { + const db = await getDatabase(); + const terminalStatus = inArray(schema.downloads.status, [ + 'completed', + 'failed', + 'canceled', + ]); + const terminalFilter = playlistId + ? and(eq(schema.downloads.playlistId, playlistId), terminalStatus) + : terminalStatus; + const rows = await db + .select({ + id: schema.downloads.id, + filePath: schema.downloads.filePath, + contentType: schema.downloads.contentType, + status: schema.downloads.status, + }) + .from(schema.downloads) + .where(terminalFilter); + const proofs = await readArchiveFinalizations( + db, + rows + .filter((row) => row.contentType === 'catchup') + .map((row) => row.id) + ); + const downloadIdsToDelete: number[] = []; + for (const row of rows) { + if (hasRuntimeDownload(row.id)) continue; + if (row.filePath && removablePartialStatuses.has(row.status)) { + try { + if (row.contentType === 'catchup') + removeJournaledCatchupPartial( + row.filePath, + proofs.get(row.id) + ); + else removePartialDownloadFile(row.filePath); + } catch (error) { + console.error( + '[Downloads] Retaining download after partial cleanup failed:', + error + ); + continue; + } + } + downloadIdsToDelete.push(row.id); + } + if (downloadIdsToDelete.length > 0) { + await db + .delete(schema.downloads) + .where( + and( + terminalFilter, + inArray(schema.downloads.id, downloadIdsToDelete) + ) + ); + broadcastDownloadUpdate(); + } + return { success: true }; + } catch (error) { + console.error('[Downloads] Error clearing completed:', error); + throw error; + } +} diff --git a/apps/electron-backend/src/app/events/database/download-runtime.ts b/apps/electron-backend/src/app/events/database/download-runtime.ts index 02207b6f1..d306c6388 100644 --- a/apps/electron-backend/src/app/events/database/download-runtime.ts +++ b/apps/electron-backend/src/app/events/database/download-runtime.ts @@ -134,6 +134,13 @@ export async function cancelDownload(downloadId: number): Promise { return true; } +export function hasRuntimeDownload(downloadId: number): boolean { + return ( + activeDownload?.id === downloadId || + downloadQueue.some((task) => task.id === downloadId) + ); +} + export function isDownloadCommitting(downloadId: number): boolean { return ( activeDownload?.id === downloadId && diff --git a/apps/electron-backend/src/app/events/database/downloads.events.spec.ts b/apps/electron-backend/src/app/events/database/downloads.events.spec.ts index d2351d9e0..0fdb47cc5 100644 --- a/apps/electron-backend/src/app/events/database/downloads.events.spec.ts +++ b/apps/electron-backend/src/app/events/database/downloads.events.spec.ts @@ -5,6 +5,9 @@ import { mockDownloadRow, mockRemoveDownloadFromRuntime, mockIsDownloadCommitting, + mockHasRuntimeDownload, + mockRemoveJournaledPartial, + mockArchiveProofs, mockRemovePartialDownloadFile, mockTerminalRows, setupDownloadsEventsHarness, @@ -35,7 +38,7 @@ describe('downloads events: partial-file cleanup', () => { const { deleteWhere } = mockTerminalRows([ createDownloadRow('completed'), ]); - mockIsDownloadCommitting.mockReturnValue(true); + mockHasRuntimeDownload.mockReturnValue(true); await expect( getHandler('DOWNLOADS_CLEAR_COMPLETED')(null) ).resolves.toEqual({ success: true }); @@ -43,6 +46,40 @@ describe('downloads events: partial-file cleanup', () => { expect(deleteWhere).not.toHaveBeenCalled(); }); + it('routes archive Remove through durable cleanup instead of pathname unlink', async () => { + const { deleteWhere } = mockDownloadRow({ + ...createDownloadRow('failed'), + contentType: 'catchup', + }); + const proof = { version: 1, phase: 'transfer' }; + mockArchiveProofs.mockResolvedValue(new Map([[42, proof]])); + await expect(getHandler('DOWNLOADS_REMOVE')(null, 42)).resolves.toEqual( + { success: true } + ); + expect(mockRemoveJournaledPartial).toHaveBeenCalledWith( + '/downloads/resume.mp4', + proof + ); + expect(mockRemovePartialDownloadFile).not.toHaveBeenCalled(); + expect(deleteWhere).toHaveBeenCalled(); + }); + + it('routes archive Clear completed through durable cleanup', async () => { + mockTerminalRows([ + { ...createDownloadRow('canceled'), contentType: 'catchup' }, + ]); + const proof = { version: 1, phase: 'transfer' }; + mockArchiveProofs.mockResolvedValue(new Map([[1, proof]])); + await expect( + getHandler('DOWNLOADS_CLEAR_COMPLETED')(null) + ).resolves.toEqual({ success: true }); + expect(mockRemoveJournaledPartial).toHaveBeenCalledWith( + '/downloads/resume.mp4', + proof + ); + expect(mockRemovePartialDownloadFile).not.toHaveBeenCalled(); + }); + it('removes queued resumed partial files before deleting the row', async () => { const { deleteWhere } = mockDownloadRow(createDownloadRow('queued')); diff --git a/apps/electron-backend/src/app/events/database/downloads.events.ts b/apps/electron-backend/src/app/events/database/downloads.events.ts index 8ba41254a..d974eb6d9 100644 --- a/apps/electron-backend/src/app/events/database/downloads.events.ts +++ b/apps/electron-backend/src/app/events/database/downloads.events.ts @@ -1,5 +1,9 @@ +import { + clearCompletedDownloadsRequest, + removeDownloadRequest, +} from './download-removal-requests'; import type { DownloadMetadataSnapshot } from '@iptvnator/shared/interfaces'; -import { and, eq, inArray } from 'drizzle-orm'; +import { eq } from 'drizzle-orm'; import { app, dialog, ipcMain, shell } from 'electron'; import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; @@ -10,7 +14,6 @@ import { decorateDownloadItemAsync, isAvailableDownloadFile, } from './download-file-availability'; -import { removePartialDownloadFile } from './download-file-path'; import { updateDownloadMetadataRequest } from './download-metadata-update'; import { resumeDownloadRequest, @@ -23,20 +26,10 @@ import { resetStaleDownloads } from './download-recovery'; import { broadcastDownloadUpdate, cancelDownload, - isDownloadCommitting, pauseDownload, - removeDownloadFromRuntime, setMainWindow, } from './download-runtime'; -const removablePartialStatuses = new Set([ - 'queued', - 'paused', - 'completed', - 'failed', - 'canceled', -]); - function getDownloadAuthorizationPath(): string { return join( app.getPath('userData'), @@ -177,58 +170,9 @@ ipcMain.handle( } ); -ipcMain.handle('DOWNLOADS_REMOVE', async (_event, downloadId: number) => { - try { - console.log('[Downloads] Remove download:', downloadId); - const db = await getDatabase(); - const rows = await db - .select({ - filePath: schema.downloads.filePath, - status: schema.downloads.status, - }) - .from(schema.downloads) - .where(eq(schema.downloads.id, downloadId)) - .limit(1); - const row = rows[0]; - if (isDownloadCommitting(downloadId)) - return { - success: false, - error: 'Download is completing; try again shortly', - }; - if (row?.filePath && removablePartialStatuses.has(row.status)) { - try { - removePartialDownloadFile(row.filePath); - } catch (cleanupError) { - // Keep the row (and its runtime entry) so the .part is never - // orphaned, but answer with a structured failure the UI can - // surface instead of an opaque IPC rejection. Retrying the - // remove re-attempts the deletion. - console.error( - '[Downloads] Failed to delete partial file on remove:', - row.filePath, - cleanupError - ); - return { - error: 'Could not delete the partial file', - success: false, - }; - } - } - if (removeDownloadFromRuntime(downloadId) === false) - return { - success: false, - error: 'Download is completing; try again shortly', - }; - await db - .delete(schema.downloads) - .where(eq(schema.downloads.id, downloadId)); - broadcastDownloadUpdate(); - return { success: true }; - } catch (error) { - console.error('[Downloads] Error removing download:', error); - throw error; - } -}); +ipcMain.handle('DOWNLOADS_REMOVE', (_event, downloadId: number) => + removeDownloadRequest(downloadId) +); ipcMain.handle('DOWNLOADS_GET_LIST', async (_event, playlistId?: string) => { try { @@ -315,63 +259,8 @@ ipcMain.handle('DOWNLOADS_PLAY_FILE', async (_event, filePath: string) => { : { error: 'File not found', success: false }; }); -ipcMain.handle( - 'DOWNLOADS_CLEAR_COMPLETED', - async (_event, playlistId?: string) => { - try { - const db = await getDatabase(); - const terminalStatus = inArray(schema.downloads.status, [ - 'completed', - 'failed', - 'canceled', - ]); - const terminalFilter = playlistId - ? and( - eq(schema.downloads.playlistId, playlistId), - terminalStatus - ) - : terminalStatus; - const rows = await db - .select({ - id: schema.downloads.id, - filePath: schema.downloads.filePath, - status: schema.downloads.status, - }) - .from(schema.downloads) - .where(terminalFilter); - const downloadIdsToDelete: number[] = []; - for (const row of rows) { - if (isDownloadCommitting(row.id)) continue; - if (row.filePath && removablePartialStatuses.has(row.status)) { - try { - removePartialDownloadFile(row.filePath); - } catch (error) { - console.error( - '[Downloads] Retaining download after partial cleanup failed:', - error - ); - continue; - } - } - downloadIdsToDelete.push(row.id); - } - if (downloadIdsToDelete.length > 0) { - await db - .delete(schema.downloads) - .where( - and( - terminalFilter, - inArray(schema.downloads.id, downloadIdsToDelete) - ) - ); - broadcastDownloadUpdate(); - } - return { success: true }; - } catch (error) { - console.error('[Downloads] Error clearing completed:', error); - throw error; - } - } +ipcMain.handle('DOWNLOADS_CLEAR_COMPLETED', (_event, playlistId?: string) => + clearCompletedDownloadsRequest(playlistId) ); export { resetStaleDownloads, setMainWindow }; diff --git a/apps/electron-backend/src/app/events/database/downloads.test-helpers.ts b/apps/electron-backend/src/app/events/database/downloads.test-helpers.ts index d751aeffc..dfc298fbd 100644 --- a/apps/electron-backend/src/app/events/database/downloads.test-helpers.ts +++ b/apps/electron-backend/src/app/events/database/downloads.test-helpers.ts @@ -12,6 +12,9 @@ export const mockRegisteredHandlers = new Map(); export const mockGetDatabase = jest.fn(); export const mockRemoveDownloadFromRuntime = jest.fn(); export const mockIsDownloadCommitting = jest.fn(); +export const mockHasRuntimeDownload = jest.fn(); +export const mockArchiveProofs = jest.fn(); +export const mockRemoveJournaledPartial = jest.fn(); export const mockBroadcastDownloadUpdate = jest.fn(); export const mockRemovePartialDownloadFile = jest.fn(); export const mockPauseDownload = jest.fn(); @@ -55,6 +58,9 @@ export async function setupDownloadsEventsHarness(): Promise { mockGetDatabase.mockReset(); mockRemoveDownloadFromRuntime.mockReset(); mockIsDownloadCommitting.mockReset().mockReturnValue(false); + mockHasRuntimeDownload.mockReset().mockReturnValue(false); + mockArchiveProofs.mockReset().mockResolvedValue(new Map()); + mockRemoveJournaledPartial.mockReset(); mockBroadcastDownloadUpdate.mockReset(); mockRemovePartialDownloadFile.mockReset(); mockPauseDownload.mockReset(); @@ -113,10 +119,17 @@ export async function setupDownloadsEventsHarness(): Promise { jest.doMock('./download-redownload', () => ({ redownloadMissingRequest: mockRedownloadMissingRequest, })); + jest.doMock('./download-catchup-journal', () => ({ + readArchiveFinalizations: mockArchiveProofs, + })); + jest.doMock('./download-catchup-removal', () => ({ + removeJournaledCatchupPartial: mockRemoveJournaledPartial, + })); jest.doMock('./download-runtime', () => ({ broadcastDownloadUpdate: mockBroadcastDownloadUpdate, cancelDownload: jest.fn(), isDownloadCommitting: mockIsDownloadCommitting, + hasRuntimeDownload: mockHasRuntimeDownload, pauseDownload: mockPauseDownload, removeDownloadFromRuntime: mockRemoveDownloadFromRuntime, setMainWindow: jest.fn(), @@ -171,6 +184,7 @@ export function expectManagedPathLookup( } export function mockDownloadRow(row: { + contentType?: string; filePath: string | null; status: string; }) { @@ -190,7 +204,11 @@ export function mockDownloadRow(row: { } export function mockTerminalRows( - rows: Array<{ filePath: string | null; status: string }> + rows: Array<{ + filePath: string | null; + status: string; + contentType?: string; + }> ) { const deleteWhere = jest.fn().mockResolvedValue(undefined); const selectWhere = jest diff --git a/docs/architecture/download-manager.md b/docs/architecture/download-manager.md index 88468d319..c22df9cef 100644 --- a/docs/architecture/download-manager.md +++ b/docs/architecture/download-manager.md @@ -87,7 +87,12 @@ before awaited partial cleanup and the SQLite completion write. Pause/cancel the return false without setting flags; a command is never accepted and subsequently overwritten by completion. Remove rejects this committing row before partial cleanup or deletion, and Clear completed skips it, preserving the cascading -journal until completion finishes. +journal until completion finishes. Remove, Clear completed and missing-file +re-download use journal-backed private capture for archive partial cleanup; +unknown or replaced entries are preserved. Cleanup remains synchronous after the +runtime guard, so a completion transition cannot interleave with unlink. +Missing archive re-downloads claim a fresh reservation instead of inheriting the +completed file's identity. A kill between exclusive copy-file creation and its identity journal commit can leave an unowned **empty** destination: no bytes are written before the commit. Recovery preserves that file rather than guessing ownership; Retry uses a @@ -125,7 +130,8 @@ available after restart and after the source archive expires. ## Backend responsibilities - **Queue control (`apps/electron-backend/src/app/events/database/download-runtime.ts`)** - `DownloadTask` mirrors a row of the shared `downloads` table (type `Download` in `libs/shared/database/src/lib/schema.ts`) plus transient cancel/pause/progress helpers (shared task types live in `download-task.ts`). Request validation and row creation live in `download-requests.ts`, retry/resume flows in `download-resume-requests.ts`, while `downloads.events.ts` stays focused on IPC registration. `enqueueDownload()` pushes the task onto `downloadQueue` and triggers `processQueue()`. `processQueue()` keeps one active download, updates the row to `downloading`, and calls `startDownload()`. The byte transfer itself lives in `download-transfer.ts`, finalization and retained-partial persistence in `download-finalize.ts` (ordinary file promotion in `download-file-finalize.ts`), cancellation/pause persistence in `download-runtime-persistence.ts`, and the renderer update broadcast in `download-broadcast.ts`. + `DownloadTask` mirrors a row of the shared `downloads` table (type `Download` in `libs/shared/database/src/lib/schema.ts`) plus transient cancel/pause/progress helpers (shared task types live in `download-task.ts`). Request validation and row creation live in `download-requests.ts`, retry/resume flows in `download-resume-requests.ts`, removal/terminal cleanup in + `download-removal-requests.ts`, while `downloads.events.ts` stays focused on IPC registration. `enqueueDownload()` pushes the task onto `downloadQueue` and triggers `processQueue()`. `processQueue()` keeps one active download, updates the row to `downloading`, and calls `startDownload()`. The byte transfer itself lives in `download-transfer.ts`, finalization and retained-partial persistence in `download-finalize.ts` (ordinary file promotion in `download-file-finalize.ts`), cancellation/pause persistence in `download-runtime-persistence.ts`, and the renderer update broadcast in `download-broadcast.ts`. - **Range-aware transfer (`download-transfer.ts`)** The transfer streams the response through the backend's validated Axios redirect helper instead of `electron-dl`, and always requests `Accept-Encoding: identity`: Range offsets, totals, and the persisted `.part` must describe the same representation, and Axios's transparent gzip/brotli decoding would put decoded bytes on disk while every counter speaks encoded bytes. Headers (user agent, referer, origin) are persisted in `request_headers` and re-applied through the same allowlist when read back on retry/resume. Fresh Xtream movie and series-episode downloads propagate the playlist's configured headers, using its User-Agent when present and otherwise sharing the provider-compatible `XTREAM_CLIENT_USER_AGENT` used by Xtream API requests and stream probes. Retry, resume, and missing-file recovery resolve the owning playlist type and add that fallback to legacy Xtream rows without a stored User-Agent; known Stalker rows are left unchanged. Download rows deliberately outlive individually deleted playlists, so a headerless legacy row whose source no longer exists receives the same IPTV-player fallback because its original provider type cannot be recovered. Active pause/cancel operations abort the current request with `AbortController`; pause keeps the partial file and cancel removes it. Resume checks the existing `.part` size (rejecting anything that is not a regular file, so a symlink planted while paused is never followed). The first response's strong `ETag` (or `Last-Modified`) is persisted in `resume_validator`; a partial carrying that validator resumes with `Range: bytes=-` plus `If-Range`, so the server itself proves the entity is unchanged. A retained partial **without** a validator resumes through overlap verification instead (`download-overlap.ts`): the `Range` request rewinds by up to 256 KiB (`OVERLAP_VERIFICATION_BYTES`) and a transform stream compares that replayed window byte-for-byte against the partial's tail before anything is appended — a self-made validator for the many Xtream panels that send neither header. A mismatching overlap truncates the `.part` and restarts the transfer from byte zero (`OverlapMismatchError`); a partial smaller than the overlap window is verified in full from byte zero over a plain request and appended to — never rewritten in place, so a reconnect that dies early can only grow the file. Success requires the verifier to have consumed its ENTIRE window: a response that ends inside the overlap is an ordinary retained interruption when the stream died early, but a response that delivered its complete AUTHORITATIVE total inside the window — whether it then closed cleanly or reset — proves the remote entity shrank and restarts from scratch; the old suffix is never finalized as a completed file. An HTTP 416 answer to a resume request is classified by `classifyRangeNotSatisfiable()`: it COMPLETES only an exact-EOF request with identity proof (`If-Range`-backed, or the EOF probe that follows a fully verified overlap replay) whose stated `bytes */N` equals the partial — a bare length match on a rewound request proves nothing about whose bytes are on disk; it RESTARTS only when a STATED total proves the entity shrank — the total sits below a rewound request's first byte, or at it (the rewound range beginning exactly at the new EOF), or below the partial at an exact-EOF request; every length-less, ambiguous, or contradictory 416 RETAINS the partial, and none of these paths ever reaches generic cleanup. A validator promoted by a complete overlap match survives mid-append failures too: the promotion also runs on the error path, and retained-failure and pause persistence write `resume_validator` from the task, so later attempts resume via `If-Range` instead of replaying the window — without this, a server whose per-connection cap barely exceeds the window would stall out on sub-threshold progress. A verify-append attempt promotes the response's `ETag`/`Last-Modified` onto the row only after the complete overlap matched; until then the retained bytes are unproven and blessing them with a validator would let the next resume `If-Range`-append onto a foreign prefix. The response's TOTAL stays equally uncommitted (task and row) until the overlap matched — a persisted total equal to the unverified partial's size would let the completed-partial shortcut finalize unproven bytes after a pause, crash, or retained failure. Retained-interruption persistence keeps the live task in sync with the row (a stale falsified total would make the next reconnect's resume-offset guard reject the partial). Overlap replay re-counts bytes from the rewound offset, so reported progress is floored at the partial's retained size whenever the transfer appends — a response that ends inside the overlap can never move displayed progress backwards. A `206 Partial Content` answer must start at the requested offset (`Content-Range` is verified) before bytes are appended; any other 2xx answer — the server ignoring `Range`, or `If-Range` detecting that the remote file changed — restarts the transfer from byte zero over the same `.part` instead of failing the download. - **Destination collision policy** diff --git a/tools/coverage/coverage-policy.json b/tools/coverage/coverage-policy.json index f6303ffa1..c667640f5 100644 --- a/tools/coverage/coverage-policy.json +++ b/tools/coverage/coverage-policy.json @@ -36,8 +36,15 @@ { "path": "apps/electron-backend/src/app/events/database/downloads.events.ts", "statements": { - "minimumCovered": 86, - "minimumPercent": 58.5 + "minimumCovered": 79, + "minimumPercent": 62.69 + } + }, + { + "path": "apps/electron-backend/src/app/events/database/download-removal-requests.ts", + "statements": { + "minimumCovered": 53, + "minimumPercent": 91.37 } } ]