diff --git a/.changes/downloads-vod-reliability.md b/.changes/downloads-vod-reliability.md new file mode 100644 index 000000000..dcb1fb7ea --- /dev/null +++ b/.changes/downloads-vod-reliability.md @@ -0,0 +1,7 @@ +--- +type: fix +area: downloads +issues: [897, 1289] +--- + +Xtream movie downloads now use the same provider-compatible client identity as portal requests. If a connection drops after data has arrived, IPTVnator keeps the partial file, shows a credential-safe interruption code, and Retry resumes it with Range validation instead of starting over. diff --git a/CLAUDE.md b/CLAUDE.md index 120fb100e..5eecde25c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -911,6 +911,13 @@ engine` (restart required) or **Download Manager**: +- Xtream VOD downloads default to the same provider-compatible + `XTREAM_CLIENT_USER_AGENT` used by API requests and stream probes, while an + explicit playlist User-Agent still wins. Retry, resume, and missing-file + recovery also add the fallback to legacy Xtream rows that have no stored + User-Agent. Allowlisted connection resets after bytes reach disk retain the partial and show a credential-safe + `DOWNLOAD_NETWORK_INTERRUPTED` code; Retry continues with Range/If-Range + instead of starting from zero. - The desktop-only manager shares one global download store across the global, Xtream-scoped, and Stalker-scoped routes. Completed movie and grouped-series cards use the global Small/Medium/Large cover-grid tokens; missing completed diff --git a/apps/electron-backend-e2e/src/download-reliability.e2e.ts b/apps/electron-backend-e2e/src/download-reliability.e2e.ts new file mode 100644 index 000000000..13b69fe11 --- /dev/null +++ b/apps/electron-backend-e2e/src/download-reliability.e2e.ts @@ -0,0 +1,117 @@ +import { mkdirSync, readFileSync, statSync } from 'fs'; +import { join } from 'path'; +import type { Page } from '@playwright/test'; +import { + addXtreamPortal, + closeElectronApp, + expect, + launchElectronApp, + resetMockServers, + test, + waitForXtreamWorkspaceReady, +} from './electron-test-fixtures'; +import { + createInterruptedRangeServer, + INTERRUPTED_RANGE_SERVER_ETAG, + startDownload, +} from './downloads.e2e-support'; + +async function openDownloadsPage(page: Page): Promise { + await page.getByRole('button', { name: 'Open downloads' }).click(); + await page.waitForURL(/\/workspace\/downloads(?:\?.*)?$/); +} + +async function getPlaylistId(page: Page, title: string): Promise { + const playlists = await page.evaluate( + async () => (await window.electron?.dbGetAppPlaylistMetas?.()) ?? [] + ); + const playlist = playlists.find((entry) => entry.title === title); + expect(playlist, `playlist "${title}" should exist`).toBeDefined(); + return playlist?._id ?? ''; +} + +test.describe('Electron download reliability', () => { + test('@downloads @electron retains a network-interrupted partial and retries it with HTTP Range', async ({ + dataDir, + request, + }) => { + await resetMockServers(request, ['xtream']); + const rangeServer = await createInterruptedRangeServer(); + const app = await launchElectronApp(dataDir); + + try { + await addXtreamPortal(app.mainWindow, { + name: 'Reset Portal', + username: 'user1', + password: 'pass1', + }); + await waitForXtreamWorkspaceReady(app.mainWindow); + await openDownloadsPage(app.mainWindow); + + const downloadsDir = join(dataDir, 'e2e-reset-downloads'); + mkdirSync(downloadsDir, { recursive: true }); + await app.electronApp.evaluate(({ dialog }, folder) => { + dialog.showOpenDialog = async () => + ({ + canceled: false, + filePaths: [folder], + }) as Awaited>; + }, downloadsDir); + await app.mainWindow + .getByRole('button', { name: 'Change Folder' }) + .click(); + + const playlistId = await getPlaylistId( + app.mainWindow, + 'Reset Portal' + ); + const downloadId = await startDownload(app.mainWindow, { + playlistId, + xtreamId: 9801, + contentType: 'vod', + title: 'E2E Reset Movie', + url: rangeServer.url, + downloadFolder: downloadsDir, + }); + const item = app.mainWindow.getByTestId( + `download-queue-item-${downloadId}` + ); + await expect(item.locator('.download-queue__status')).toContainText( + 'Failed', + { timeout: 30000 } + ); + await expect(item.locator('.download-queue__error')).toContainText( + 'DOWNLOAD_NETWORK_INTERRUPTED (ECONNRESET)' + ); + + const partialPath = join(downloadsDir, 'E2E Reset Movie.mp4.part'); + expect(statSync(partialPath).size).toBe( + rangeServer.interruptedBytes + ); + + await item + .getByRole('button', { name: 'Retry E2E Reset Movie' }) + .click(); + await expect( + app.mainWindow.getByTestId( + `download-library-movie-${downloadId}` + ) + ).toBeVisible({ timeout: 30000 }); + + const resumeRequest = rangeServer.requests.find( + (entry) => entry.range + ); + expect(resumeRequest?.range).toBe( + `bytes=${rangeServer.interruptedBytes}-` + ); + expect(resumeRequest?.ifRange).toBe(INTERRUPTED_RANGE_SERVER_ETAG); + const finalFile = readFileSync( + join(downloadsDir, 'E2E Reset Movie.mp4') + ); + expect(finalFile.equals(rangeServer.payload)).toBe(true); + } finally { + await closeElectronApp(app); + await rangeServer.close(); + } + }); +}); diff --git a/apps/electron-backend-e2e/src/downloads.e2e-support.ts b/apps/electron-backend-e2e/src/downloads.e2e-support.ts index fc7997f78..2ec21c4ab 100644 --- a/apps/electron-backend-e2e/src/downloads.e2e-support.ts +++ b/apps/electron-backend-e2e/src/downloads.e2e-support.ts @@ -31,6 +31,10 @@ interface ThrottledRangeServer { url: string; } +interface InterruptedRangeServer extends ThrottledRangeServer { + interruptedBytes: number; +} + interface TruncatedDownloadServer { close: () => Promise; payload: Buffer; @@ -39,6 +43,7 @@ interface TruncatedDownloadServer { } export const RANGE_SERVER_ETAG = '"e2e-range-etag"'; +export const INTERRUPTED_RANGE_SERVER_ETAG = '"e2e-reset-etag"'; const downloadPlayCaptureKey = '__iptvnatorE2eDownloadPlayPaths'; export function getStalkerSeriesDownloadTarget( @@ -172,6 +177,63 @@ export async function createThrottledRangeServer( }; } +/** + * Resets the first full response after writing a valid prefix, then serves the + * remainder to a Range retry. This matches a provider/proxy connection drop + * without manufacturing a clean EOF. + */ +export async function createInterruptedRangeServer(): Promise { + const payload = Buffer.alloc(64 * 1024, 9); + const interruptedBytes = 16 * 1024; + const requests: RangeServerRequest[] = []; + + const server = createServer((req, res) => { + const range = req.headers.range; + const ifRange = req.headers['if-range']; + requests.push({ + ifRange: typeof ifRange === 'string' ? ifRange : undefined, + range: typeof range === 'string' ? range : undefined, + }); + + const offset = range + ? Number(/^bytes=(\d+)-$/.exec(range)?.[1] ?? Number.NaN) + : 0; + if (range && Number.isFinite(offset)) { + res.writeHead(206, { + 'Content-Length': payload.length - offset, + 'Content-Range': `bytes ${offset}-${payload.length - 1}/${payload.length}`, + 'Content-Type': 'video/mp4', + ETag: INTERRUPTED_RANGE_SERVER_ETAG, + }); + res.end(payload.subarray(offset)); + return; + } + + res.writeHead(200, { + 'Content-Length': payload.length, + 'Content-Type': 'video/mp4', + ETag: INTERRUPTED_RANGE_SERVER_ETAG, + }); + res.write(payload.subarray(0, interruptedBytes), () => { + setTimeout(() => res.socket?.destroy(), 20); + }); + }); + + await new Promise((resolve) => + server.listen(0, '127.0.0.1', resolve) + ); + const { port } = server.address() as AddressInfo; + + return { + close: () => + new Promise((resolve) => server.close(() => resolve())), + interruptedBytes, + payload, + requests, + url: `http://127.0.0.1:${port}/media/e2e-reset-movie.mp4`, + }; +} + /** * Ends a chunked response cleanly while Content-Range advertises a larger * representation. The runtime therefore retains the valid .part for a Range diff --git a/apps/electron-backend/src/app/events/database/download-finalize.ts b/apps/electron-backend/src/app/events/database/download-finalize.ts index 955fe1ac4..152d1ebff 100644 --- a/apps/electron-backend/src/app/events/database/download-finalize.ts +++ b/apps/electron-backend/src/app/events/database/download-finalize.ts @@ -14,12 +14,16 @@ import type { DownloadTask, TransferProgress, } from './download-task'; -import { describeError, TruncatedTransferError } from './download-transfer'; +import { + describeError, + InterruptedTransferError, + TruncatedTransferError, +} from './download-transfer'; /** * Persistence for a failed startDownload() attempt, after its cancel/pause - * checkpoints have been ruled out. Chooses between: retaining a truncated - * transfer for a Range retry, committing an already-finalized file, + * checkpoints have been ruled out. Chooses between: retaining a recoverable + * partial for a Range retry, committing an already-finalized file, * retaining a completed partial, or the generic delete-partial failure. */ export async function handleDownloadFailure( @@ -28,8 +32,12 @@ export async function handleDownloadFailure( reservation: ReservedPartialDownloadFile | undefined, error: unknown ): Promise { - if (error instanceof TruncatedTransferError && reservation) { - // The short response is retained so a retry can continue the + if ( + (error instanceof TruncatedTransferError || + error instanceof InterruptedTransferError) && + reservation + ) { + // The recoverable partial is retained so a retry can continue the // transfer via Range instead of starting over. await persistCompletedPartialFailure( db, @@ -338,7 +346,9 @@ function canCopyCompletedPartialAfterLinkFailure(error: unknown): boolean { } /** @returns false when a .part exists but could not be deleted. */ -export function removePartialFile(filePath: string | null | undefined): boolean { +export function removePartialFile( + filePath: string | null | undefined +): boolean { try { removePartialDownloadFile(filePath); return true; diff --git a/apps/electron-backend/src/app/events/database/download-redownload.spec.ts b/apps/electron-backend/src/app/events/database/download-redownload.spec.ts index 1514e50b2..1a1d0b91a 100644 --- a/apps/electron-backend/src/app/events/database/download-redownload.spec.ts +++ b/apps/electron-backend/src/app/events/database/download-redownload.spec.ts @@ -3,6 +3,7 @@ interface SetupOptions { cleanupError?: Error; claim?: { changes: number }; fileReappeared?: boolean; + playlistType?: 'xtream' | 'stalker'; row?: Record | null; urlError?: Error; } @@ -25,13 +26,18 @@ async function setup(options: SetupOptions = {}) { url: 'https://example.test/movie.mp4', ...options.row, }; - const limit = jest.fn().mockResolvedValue(row ? [row] : []); + const downloadLimit = jest.fn().mockResolvedValue(row ? [row] : []); + const playlistLimit = jest + .fn() + .mockResolvedValue([{ type: options.playlistType ?? 'stalker' }]); const where = jest.fn().mockResolvedValue(options.claim ?? { changes: 1 }); const set = jest.fn(() => ({ where })); const db = { - select: jest.fn(() => ({ + select: jest.fn((selection?: unknown) => ({ from: jest.fn(() => ({ - where: jest.fn(() => ({ limit })), + where: jest.fn(() => ({ + limit: selection ? playlistLimit : downloadLimit, + })), })), })), update: jest.fn(() => ({ set })), @@ -118,6 +124,27 @@ describe('redownload missing completed file', () => { }); }); + it('adds the Xtream fallback User-Agent to a legacy row', async () => { + const harness = await setup({ + playlistType: 'xtream', + row: { + playlistId: 'playlist-1', + requestHeaders: null, + }, + }); + + await expect(harness.redownloadMissingRequest(42)).resolves.toEqual({ + success: true, + }); + expect(harness.enqueueDownload).toHaveBeenCalledWith( + expect.objectContaining({ + headers: { + 'User-Agent': 'VLC/3.0.18 LibVLC/3.0.18', + }, + }) + ); + }); + it('recovers a file that reappeared without updating or enqueueing', async () => { const harness = await setup({ fileReappeared: true }); diff --git a/apps/electron-backend/src/app/events/database/download-redownload.ts b/apps/electron-backend/src/app/events/database/download-redownload.ts index b34f49f2c..fd9aa8d44 100644 --- a/apps/electron-backend/src/app/events/database/download-redownload.ts +++ b/apps/electron-backend/src/app/events/database/download-redownload.ts @@ -6,7 +6,7 @@ import * as schema from '../../database/schema'; import { assertRemoteUrlAllowed } from '../url-safety'; import { isAvailableDownloadFile } from './download-file-availability'; import { removePartialDownloadFile } from './download-file-path'; -import { parseStoredHeaders } from './download-requests'; +import { resolveStoredDownloadHeaders } from './download-request-headers'; import { enqueueDownload } from './download-runtime'; export interface RedownloadMissingResult { @@ -61,6 +61,7 @@ export async function redownloadMissingRequest( } await assertRemoteUrlAllowed(item.url, { allowPrivateNetworks: true }); + const headers = await resolveStoredDownloadHeaders(db, item); try { removePartialDownloadFile(item.filePath); @@ -102,7 +103,7 @@ export async function redownloadMissingRequest( directory: dirname(item.filePath), fileName: basename(item.filePath), filePath: item.filePath, - headers: parseStoredHeaders(item.requestHeaders), + headers, id: item.id, resumeValidator: null, totalBytes: null, diff --git a/apps/electron-backend/src/app/events/database/download-request-headers.spec.ts b/apps/electron-backend/src/app/events/database/download-request-headers.spec.ts new file mode 100644 index 000000000..9ef125d90 --- /dev/null +++ b/apps/electron-backend/src/app/events/database/download-request-headers.spec.ts @@ -0,0 +1,63 @@ +import type { DownloadsDatabase } from './download-task'; + +function createDatabase(playlistType: 'xtream' | 'stalker') { + const limit = jest.fn().mockResolvedValue([{ type: playlistType }]); + const db = { + select: jest.fn(() => ({ + from: jest.fn(() => ({ + where: jest.fn(() => ({ limit })), + })), + })), + } as unknown as DownloadsDatabase; + return { db, select: db.select }; +} + +describe('stored download request headers', () => { + it('adds the provider-compatible fallback for a legacy Xtream row', async () => { + const { db } = createDatabase('xtream'); + const { resolveStoredDownloadHeaders } = + await import('./download-request-headers'); + + await expect( + resolveStoredDownloadHeaders(db, { + playlistId: 'playlist-1', + requestHeaders: JSON.stringify({ + Authorization: 'not-allowed', + Referer: 'https://example.test/', + }), + }) + ).resolves.toEqual({ + Referer: 'https://example.test/', + 'User-Agent': 'VLC/3.0.18 LibVLC/3.0.18', + }); + }); + + it('does not add the Xtream fallback to a Stalker row', async () => { + const { db } = createDatabase('stalker'); + const { resolveStoredDownloadHeaders } = + await import('./download-request-headers'); + + await expect( + resolveStoredDownloadHeaders(db, { + playlistId: 'playlist-1', + requestHeaders: null, + }) + ).resolves.toBeUndefined(); + }); + + it('preserves an explicit stored User-Agent without querying the playlist', async () => { + const { db, select } = createDatabase('xtream'); + const { resolveStoredDownloadHeaders } = + await import('./download-request-headers'); + + await expect( + resolveStoredDownloadHeaders(db, { + playlistId: 'playlist-1', + requestHeaders: JSON.stringify({ + 'User-Agent': 'Custom/1.0', + }), + }) + ).resolves.toEqual({ 'User-Agent': 'Custom/1.0' }); + expect(select).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/electron-backend/src/app/events/database/download-request-headers.ts b/apps/electron-backend/src/app/events/database/download-request-headers.ts new file mode 100644 index 000000000..709b9b4d8 --- /dev/null +++ b/apps/electron-backend/src/app/events/database/download-request-headers.ts @@ -0,0 +1,62 @@ +import { XTREAM_CLIENT_USER_AGENT } from '@iptvnator/shared/interfaces'; +import { eq } from 'drizzle-orm'; +import * as schema from '../../database/schema'; +import type { DownloadsDatabase } from './download-task'; + +const STORED_HEADER_ALLOWLIST = ['User-Agent', 'Origin', 'Referer'] as const; + +export function parseStoredHeaders( + value: string | null | undefined +): Record | undefined { + if (!value) { + return undefined; + } + + try { + const parsed = JSON.parse(value) as unknown; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + return undefined; + } + + // Re-apply the write-time allowlist so a tampered or imported + // database row cannot smuggle arbitrary headers into requests. + const entries = parsed as Record; + const headers = STORED_HEADER_ALLOWLIST.reduce>( + (acc, key) => { + const headerValue = entries[key]; + if (typeof headerValue === 'string') { + acc[key] = headerValue; + } + return acc; + }, + {} + ); + return Object.keys(headers).length > 0 ? headers : undefined; + } catch { + return undefined; + } +} + +export async function resolveStoredDownloadHeaders( + db: DownloadsDatabase, + item: { playlistId: string; requestHeaders: string | null | undefined } +): Promise | undefined> { + const headers = parseStoredHeaders(item.requestHeaders); + if (headers?.['User-Agent']?.trim()) { + return headers; + } + + const playlists = await db + .select({ type: schema.playlists.type }) + .from(schema.playlists) + .where(eq(schema.playlists.id, item.playlistId)) + .limit(1); + if (playlists[0]?.type !== 'xtream') { + return headers; + } + + return { + ...headers, + 'User-Agent': XTREAM_CLIENT_USER_AGENT, + }; +} diff --git a/apps/electron-backend/src/app/events/database/download-requests.spec.ts b/apps/electron-backend/src/app/events/database/download-requests.spec.ts index 4f6cd4650..8e4af8d69 100644 --- a/apps/electron-backend/src/app/events/database/download-requests.spec.ts +++ b/apps/electron-backend/src/app/events/database/download-requests.spec.ts @@ -384,6 +384,66 @@ describe('download request metadata snapshots', () => { }); describe('download requests resume', () => { + it('adds the Xtream fallback User-Agent to a legacy paused row', async () => { + jest.resetModules(); + const schema = await import('../../database/schema'); + const row = { + filePath: '/downloads/movie.mp4', + id: 42, + playlistId: 'playlist-1', + requestHeaders: null, + resumeValidator: '"etag-9"', + status: 'paused', + title: 'Movie', + totalBytes: 100, + url: 'https://example.test/movie.mp4', + }; + const downloadLimit = jest.fn().mockResolvedValue([row]); + const playlistLimit = jest.fn().mockResolvedValue([{ type: 'xtream' }]); + const db = { + select: jest.fn(() => ({ + from: jest.fn((table: unknown) => ({ + where: jest.fn(() => ({ + limit: + table === schema.playlists + ? playlistLimit + : downloadLimit, + })), + })), + })), + update: jest.fn(() => ({ + set: jest.fn(() => ({ + where: jest.fn().mockResolvedValue({ changes: 1 }), + })), + })), + }; + const enqueueDownload = jest.fn(); + const authorizer = { + requireAuthorized: jest.fn(async (directory: string) => directory), + } as unknown as DownloadDirectoryAuthorizer; + + jest.doMock('../../database/connection', () => ({ + getDatabase: jest.fn().mockResolvedValue(db), + })); + jest.doMock('../url-safety', () => ({ + assertRemoteUrlAllowed: jest.fn().mockResolvedValue(undefined), + })); + jest.doMock('./download-runtime', () => ({ enqueueDownload })); + + const { resumeDownloadRequest } = await import('./download-requests'); + await expect( + resumeDownloadRequest(42, '/unused', authorizer) + ).resolves.toEqual({ success: true }); + + expect(enqueueDownload).toHaveBeenCalledWith( + expect.objectContaining({ + headers: { + 'User-Agent': 'VLC/3.0.18 LibVLC/3.0.18', + }, + }) + ); + }); + it('enqueues a paused download with stored headers and original target path', async () => { jest.resetModules(); diff --git a/apps/electron-backend/src/app/events/database/download-requests.ts b/apps/electron-backend/src/app/events/database/download-requests.ts index 5793ea0a4..5acc8d25a 100644 --- a/apps/electron-backend/src/app/events/database/download-requests.ts +++ b/apps/electron-backend/src/app/events/database/download-requests.ts @@ -6,6 +6,7 @@ import * as schema from '../../database/schema'; import { assertRemoteUrlAllowed } from '../url-safety'; import { DownloadDirectoryAuthorizer } from './download-directory-authorization'; import { removePartialDownloadFile } from './download-file-path'; +import { resolveStoredDownloadHeaders } from './download-request-headers'; import { assertDownloadMetadataArtworkDiffersFromStream, assertDownloadMetadataMatchesContentType, @@ -80,40 +81,6 @@ function serializeHeaders( return headers ? JSON.stringify(headers) : null; } -const STORED_HEADER_ALLOWLIST = ['User-Agent', 'Origin', 'Referer'] as const; - -export function parseStoredHeaders( - value: string | null -): Record | undefined { - if (!value) { - return undefined; - } - - try { - const parsed = JSON.parse(value) as unknown; - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - return undefined; - } - - // Re-apply the write-time allowlist so a tampered or imported - // database row cannot smuggle arbitrary headers into requests. - const entries = parsed as Record; - const headers = STORED_HEADER_ALLOWLIST.reduce>( - (acc, key) => { - const headerValue = entries[key]; - if (typeof headerValue === 'string') { - acc[key] = headerValue; - } - return acc; - }, - {} - ); - return Object.keys(headers).length > 0 ? headers : undefined; - } catch { - return undefined; - } -} - export async function startDownloadRequest( data: StartDownloadRequest, authorizer: DownloadDirectoryAuthorizer @@ -320,6 +287,7 @@ export async function retryDownloadRequest( const fileName = retainedFilePath ? basename(retainedFilePath) : createFileName(item.title, item.url); + const headers = await resolveStoredDownloadHeaders(db, item); const queuedUpdate = retainedFilePath ? { errorMessage: null, @@ -345,7 +313,7 @@ export async function retryDownloadRequest( directory, fileName, filePath: retainedFilePath, - headers: parseStoredHeaders(item.requestHeaders), + headers, id: item.id, resumeValidator: retainedFilePath ? item.resumeValidator : null, totalBytes: retainedFilePath ? item.totalBytes : null, @@ -388,6 +356,7 @@ export async function resumeDownloadRequest( const fileName = item.filePath ? basename(item.filePath) : createFileName(item.title, item.url); + const headers = await resolveStoredDownloadHeaders(db, item); // Claim the row atomically: a concurrent resume for the same id loses // this conditional update and must not enqueue a second task. @@ -416,7 +385,7 @@ export async function resumeDownloadRequest( directory, fileName, filePath: item.filePath, - headers: parseStoredHeaders(item.requestHeaders), + headers, id: item.id, resumeValidator: item.resumeValidator, totalBytes: item.totalBytes, diff --git a/apps/electron-backend/src/app/events/database/download-resume.spec.ts b/apps/electron-backend/src/app/events/database/download-resume.spec.ts index 5688fa4cc..9a3699550 100644 --- a/apps/electron-backend/src/app/events/database/download-resume.spec.ts +++ b/apps/electron-backend/src/app/events/database/download-resume.spec.ts @@ -11,6 +11,7 @@ interface ResumeHarness { interface ResumeHarnessOptions { partialSize: number; + partialSizeAfterTransferError?: number; response: { data: Readable; headers: Record; @@ -82,7 +83,12 @@ async function setupResumeHarness( })); jest.doMock('./download-file-path', () => ({ getPartialDownloadPath: (filePath: string) => `${filePath}.part`, - getPartialDownloadSize: jest.fn(() => options.partialSize), + getPartialDownloadSize: jest + .fn() + .mockReturnValueOnce(options.partialSize) + .mockReturnValue( + options.partialSizeAfterTransferError ?? options.partialSize + ), removePartialDownloadFile, reserveAvailablePartialDownloadFile: jest.fn( (directory: string, filename: string) => ({ @@ -264,6 +270,185 @@ describe('download resume validation', () => { } }); + it('retains received bytes when the connection resets mid-transfer', async () => { + const body = new PassThrough(); + const harness = await setupResumeHarness({ + finalSize: 'enoent', + partialSize: 0, + partialSizeAfterTransferError: 20, + response: { + data: body, + headers: { + 'content-length': '100', + etag: '"etag-reset"', + }, + status: 200, + }, + }); + const consoleError = jest + .spyOn(console, 'error') + .mockImplementation(() => undefined); + + try { + harness.runtime.enqueueDownload(createTask()); + while ( + harness.requestWithValidatedRedirects.mock.calls.length < 1 + ) { + await new Promise((resolve) => setImmediate(resolve)); + } + + body.write(Buffer.alloc(20, 'r')); + const resetError = new Error( + 'socket hang up' + ) as NodeJS.ErrnoException; + resetError.code = 'ECONNRESET'; + body.destroy(resetError); + await waitForStatus(harness.set, 'failed'); + + expect(harness.set).toHaveBeenCalledWith( + expect.objectContaining({ + bytesDownloaded: 20, + errorMessage: + 'DOWNLOAD_NETWORK_INTERRUPTED (ECONNRESET): Retry to continue from the saved partial file', + filePath: '/downloads/movie.mp4', + status: 'failed', + totalBytes: 100, + }) + ); + expect(harness.set).toHaveBeenCalledWith( + expect.objectContaining({ + resumeValidator: '"etag-reset"', + }) + ); + expect(harness.removePartialDownloadFile).not.toHaveBeenCalled(); + } finally { + consoleError.mockRestore(); + } + }); + + it('retains an existing partial when a resumed response resets before another byte', async () => { + const body = new PassThrough(); + const harness = await setupResumeHarness({ + finalSize: 'enoent', + partialSize: 40, + partialSizeAfterTransferError: 40, + response: { + data: body, + headers: { 'content-range': 'bytes 40-99/100' }, + status: 206, + }, + }); + const consoleError = jest + .spyOn(console, 'error') + .mockImplementation(() => undefined); + + try { + harness.runtime.enqueueDownload( + createTask({ + filePath: '/downloads/movie.mp4', + resumeValidator: '"etag-reset"', + totalBytes: 100, + }) + ); + while ( + harness.requestWithValidatedRedirects.mock.calls.length < 1 + ) { + await new Promise((resolve) => setImmediate(resolve)); + } + + const resetError = new Error( + 'socket hang up' + ) as NodeJS.ErrnoException; + resetError.code = 'ECONNRESET'; + body.destroy(resetError); + await waitForStatus(harness.set, 'failed'); + + expect(harness.set).toHaveBeenCalledWith( + expect.objectContaining({ + bytesDownloaded: 40, + errorMessage: + 'DOWNLOAD_NETWORK_INTERRUPTED (ECONNRESET): Retry to continue from the saved partial file', + filePath: '/downloads/movie.mp4', + status: 'failed', + totalBytes: 100, + }) + ); + expect(harness.removePartialDownloadFile).not.toHaveBeenCalled(); + } finally { + consoleError.mockRestore(); + } + }); + + it.each([ + { + code: 'EUNKNOWN', + headers: { 'content-length': '100' }, + label: 'unknown stream error', + partialSizeAfterTransferError: 20, + }, + { + code: 'ECONNRESET', + headers: {}, + label: 'response without an advertised total', + partialSizeAfterTransferError: 20, + }, + { + code: 'ECONNRESET', + headers: { 'content-length': '100' }, + label: 'fresh zero-byte failure', + partialSizeAfterTransferError: 0, + }, + { + code: 'ECONNRESET', + headers: { 'content-length': '100' }, + label: 'partial larger than the advertised total', + partialSizeAfterTransferError: 101, + }, + ])( + 'uses generic cleanup for $label', + async ({ code, headers, partialSizeAfterTransferError }) => { + const body = new PassThrough(); + const harness = await setupResumeHarness({ + finalSize: 'enoent', + partialSize: 0, + partialSizeAfterTransferError, + response: { data: body, headers, status: 200 }, + }); + const consoleError = jest + .spyOn(console, 'error') + .mockImplementation(() => undefined); + + try { + harness.runtime.enqueueDownload(createTask()); + while ( + harness.requestWithValidatedRedirects.mock.calls.length < 1 + ) { + await new Promise((resolve) => setImmediate(resolve)); + } + + const streamError = new Error( + 'socket hang up' + ) as NodeJS.ErrnoException; + streamError.code = code; + body.destroy(streamError); + await waitForStatus(harness.set, 'failed'); + + expect(harness.removePartialDownloadFile).toHaveBeenCalledWith( + '/downloads/movie.mp4' + ); + expect(harness.set).toHaveBeenCalledWith( + expect.objectContaining({ + errorMessage: 'socket hang up', + filePath: null, + status: 'failed', + }) + ); + } finally { + consoleError.mockRestore(); + } + } + ); + it('captures a strong ETag from the first response for later resumes', async () => { const harness = await setupResumeHarness({ finalSize: 4, diff --git a/apps/electron-backend/src/app/events/database/download-transfer.ts b/apps/electron-backend/src/app/events/database/download-transfer.ts index cb8df4a81..abb19f7b2 100644 --- a/apps/electron-backend/src/app/events/database/download-transfer.ts +++ b/apps/electron-backend/src/app/events/database/download-transfer.ts @@ -34,6 +34,27 @@ export class TruncatedTransferError extends Error { } } +const RETAINABLE_NETWORK_ERROR_CODES = new Set([ + 'ECONNABORTED', + 'ECONNRESET', + 'EPIPE', + 'ETIMEDOUT', + 'ERR_HTTP2_STREAM_CANCEL', + 'ERR_HTTP2_STREAM_ERROR', + 'ERR_STREAM_PREMATURE_CLOSE', +]); + +export class InterruptedTransferError extends Error { + constructor( + readonly progress: TransferProgress, + networkCode: string + ) { + super( + `DOWNLOAD_NETWORK_INTERRUPTED (${networkCode}): Retry to continue from the saved partial file` + ); + } +} + export async function transferToPartialFile( db: DownloadsDatabase, task: DownloadTask, @@ -130,6 +151,21 @@ export async function transferToPartialFile( try { await pipeline(readable, output); + } catch (error) { + const interruptedProgress = getInterruptedTransferProgress( + error, + reservation, + effectiveOffset, + totalBytes + ); + if (interruptedProgress) { + await persistProgress(db, task, interruptedProgress.progress); + throw new InterruptedTransferError( + interruptedProgress.progress, + interruptedProgress.networkCode + ); + } + throw error; } finally { abortController.signal.removeEventListener('abort', abortStream); } @@ -141,6 +177,38 @@ export async function transferToPartialFile( return { bytesDownloaded, totalBytes }; } +function getInterruptedTransferProgress( + error: unknown, + reservation: ReservedPartialDownloadFile, + initialBytes: number, + totalBytes: number | null +): { networkCode: string; progress: TransferProgress } | null { + const networkCode = + error && typeof error === 'object' && 'code' in error + ? String(error.code) + : ''; + if ( + !RETAINABLE_NETWORK_ERROR_CODES.has(networkCode) || + totalBytes === null + ) { + return null; + } + + const bytesDownloaded = getPartialDownloadSize(reservation.path); + if ( + bytesDownloaded === 0 || + bytesDownloaded < initialBytes || + bytesDownloaded >= totalBytes + ) { + return null; + } + + return { + networkCode, + progress: { bytesDownloaded, totalBytes }, + }; +} + function getResumeOffset( task: DownloadTask, reservation: ReservedPartialDownloadFile diff --git a/apps/electron-backend/src/app/events/stream-probe.ts b/apps/electron-backend/src/app/events/stream-probe.ts index 760b6f20f..1870c773f 100644 --- a/apps/electron-backend/src/app/events/stream-probe.ts +++ b/apps/electron-backend/src/app/events/stream-probe.ts @@ -13,13 +13,10 @@ import axios, { AxiosRequestConfig } from 'axios'; import { ipcMain } from 'electron'; +import { XTREAM_CLIENT_USER_AGENT } from '@iptvnator/shared/interfaces'; import { UnsafeUrlError } from './url-safety'; import { requestWithValidatedRedirects } from '../util/validated-axios'; -// Some Xtream panels sit behind Cloudflare (or similar WAFs) configured to -// challenge generic browser-looking User-Agents while allowlisting known -// IPTV player clients. A VLC-style User-Agent reliably passes those checks. -const PROBE_CLIENT_USER_AGENT = 'VLC/3.0.18 LibVLC/3.0.18'; const PROBE_TIMEOUT_MS = 10000; export interface StreamProbePayload { @@ -64,7 +61,7 @@ export async function runStreamProbe( // The playlist's own User-Agent wins when it has one: the default // below is a guess that merely gets past most WAFs, while that // one is what the server was configured to expect. - 'User-Agent': payload.userAgent?.trim() || PROBE_CLIENT_USER_AGENT, + 'User-Agent': payload.userAgent?.trim() || XTREAM_CLIENT_USER_AGENT, ...(payload.referer?.trim() ? { Referer: payload.referer.trim() } : {}), @@ -95,8 +92,7 @@ export async function runStreamProbe( ); // A ranged GET opens a stream we never read — release it immediately. const responseBody = response.data as - | { destroy?: () => void } - | undefined; + { destroy?: () => void } | undefined; responseBody?.destroy?.(); return { status: response.status, diff --git a/apps/electron-backend/src/app/events/xtream.events.ts b/apps/electron-backend/src/app/events/xtream.events.ts index 6b43a3975..30f7349f4 100644 --- a/apps/electron-backend/src/app/events/xtream.events.ts +++ b/apps/electron-backend/src/app/events/xtream.events.ts @@ -8,6 +8,7 @@ import { ipcMain } from 'electron'; import { PortalDebugEvent, XTREAM_CANCEL_SESSION, + XTREAM_CLIENT_USER_AGENT, XTREAM_MAIN_PERFORMANCE_PHASE, normalizeXtreamServerUrl, } from '@iptvnator/shared/interfaces'; @@ -20,11 +21,6 @@ import { } from './xtream-performance'; import { cancelXtreamSessionRequests } from './xtream-session-cancellation'; -// Some Xtream panels sit behind Cloudflare (or similar WAFs) configured to -// challenge generic browser-looking User-Agents while allowlisting known -// IPTV player clients. A VLC-style User-Agent reliably passes those checks. -const XTREAM_CLIENT_USER_AGENT = 'VLC/3.0.18 LibVLC/3.0.18'; - export default class XtreamEvents { static bootstrapXtreamEvents(): Electron.IpcMain { return ipcMain; diff --git a/docs/architecture/download-manager.md b/docs/architecture/download-manager.md index 42825f6fc..b7e0527ff 100644 --- a/docs/architecture/download-manager.md +++ b/docs/architecture/download-manager.md @@ -12,7 +12,7 @@ variants, contextual buttons, and theme-aware styling. - **Queue control (`apps/electron-backend/src/app/events/database/download-runtime.ts`)** `DownloadTask` mirrors a row of the shared `downloads` table (type `Download` in `libs/shared/database/src/lib/schema.ts`) plus transient cancel/pause/progress helpers (shared task types live in `download-task.ts`). Request validation and row creation live in `download-requests.ts`, while `downloads.events.ts` stays focused on IPC registration. `enqueueDownload()` pushes the task onto `downloadQueue` and triggers `processQueue()`. `processQueue()` keeps one active download, updates the row to `downloading`, and calls `startDownload()`. The byte transfer itself lives in `download-transfer.ts`, finalization and retained-partial persistence in `download-finalize.ts`, and the renderer update broadcast in `download-broadcast.ts`. - **Range-aware transfer (`download-transfer.ts`)** - The transfer streams the response through the backend's validated Axios redirect helper instead of `electron-dl`. Headers (user agent, referer, origin) are persisted in `request_headers` and re-applied through the same allowlist when read back on retry/resume. Active pause/cancel operations abort the current request with `AbortController`; pause keeps the partial file and cancel removes it. Resume checks the existing `.part` size (rejecting anything that is not a regular file, so a symlink planted while paused is never followed) and sends `Range: bytes=-` plus `If-Range` with the stored entity validator. The first response's strong `ETag` (or `Last-Modified`) is persisted in `resume_validator` for exactly this purpose. A `206 Partial Content` answer must start at the requested offset (`Content-Range` is verified) before bytes are appended; any other 2xx answer — the server ignoring `Range`, or `If-Range` detecting that the remote file changed — restarts the transfer from byte zero over the same `.part` instead of failing the download. + The transfer streams the response through the backend's validated Axios redirect helper instead of `electron-dl`. Headers (user agent, referer, origin) are persisted in `request_headers` and re-applied through the same allowlist when read back on retry/resume. Xtream VOD downloads use the playlist's configured User-Agent when present and otherwise share the provider-compatible `XTREAM_CLIENT_USER_AGENT` used by Xtream API requests and stream probes. Retry, resume, and missing-file recovery resolve the owning playlist type and add that fallback to legacy Xtream rows without a stored User-Agent; Stalker rows are left unchanged. Active pause/cancel operations abort the current request with `AbortController`; pause keeps the partial file and cancel removes it. Resume checks the existing `.part` size (rejecting anything that is not a regular file, so a symlink planted while paused is never followed) and sends `Range: bytes=-` plus `If-Range` with the stored entity validator. The first response's strong `ETag` (or `Last-Modified`) is persisted in `resume_validator` for exactly this purpose. A `206 Partial Content` answer must start at the requested offset (`Content-Range` is verified) before bytes are appended; any other 2xx answer — the server ignoring `Range`, or `If-Range` detecting that the remote file changed — restarts the transfer from byte zero over the same `.part` instead of failing the download. - **Destination collision policy** Existing destination files are never overwritten, inspected, or deleted. Before starting a new transfer, the backend atomically reserves a free @@ -23,11 +23,12 @@ variants, contextual buttons, and theme-aware styling. retained `.part` is renamed aside and finalized to the next free numbered destination (`Movie (1).mp4`) instead of resolving the collision by size or `unlink()`. Completion creates the final `filePath` from the `.part` without - overwriting an existing file; cancel and ordinary transfer failures remove - the `.part`, while finalization failures and completed-partial failures - deliberately retain it (the row keeps `filePath` so a later retry can finish - without re-downloading); pause and restart recovery keep it for a later - resume. Re-downloading such a failed row from a detail page + overwriting an existing file; cancel and non-recoverable transfer failures + remove the `.part`, while finalization failures, completed-partial failures, + and allowlisted network interruptions after bytes reached disk deliberately + retain it (the row keeps `filePath` so a later retry can finish without + re-downloading); pause and restart recovery keep it for a later resume. + Re-downloading such a failed row from a detail page (`DOWNLOADS_START`) deletes the retained `.part` before the row is reset. - **Derived file readiness and recovery** `DOWNLOADS_GET_LIST` and `DOWNLOADS_GET` inspect completed destinations on @@ -184,6 +185,7 @@ variants, contextual buttons, and theme-aware styling. - A `.part` that cannot be deleted (locked, permission denied) never loses its database path: cancel persists `canceled` while retaining `filePath` for later cleanup, and `DOWNLOADS_REMOVE` keeps the row and answers `success: false` (surfaced as a snackbar) so retrying the remove re-attempts the deletion once the lock is released. - Resume claims the row atomically (`paused` → `queued` as a conditional update) and the runtime queue rejects duplicate ids, so two rapid Resume clicks racing the status refresh can never produce two transfers for the same download. - A response that ends cleanly before the advertised representation size (for example a proxy that caps each response) is never committed as completed: the transfer fails with `Transfer ended before the advertised size` while retaining the `.part` and `filePath`, so a retry continues via Range from where it stopped. +- An allowlisted mid-response network failure such as `ECONNRESET` is recoverable only when the response advertised a larger total and the `.part` contains valid incomplete bytes. This includes a validated `206` resume that drops before adding another byte. The failed row retains that partial and exposes a stable `DOWNLOAD_NETWORK_INTERRUPTED ()` message without a URL; Retry continues through the same Range/If-Range validation. Pre-response failures, unknown stream errors, filesystem errors, empty fresh failures, and responses without a trustworthy total keep the generic failure path. - Retained `filePath`s recorded in the database stay usable after the user switches download folders — resume/retry of a retained row does not re-require the folder to be the current selection. Fresh downloads still authorize against the currently selected folder. - Startup recovery recognizes a finalization that crashed between creating the final file and committing the row (`downloading` row, no partial, final file present with the recorded size) and marks it `completed` instead of failing it and orphaning the file. - Pause/resume is covered end to end by `apps/electron-backend-e2e/src/downloads.e2e.ts`: a throttled Range-capable mock server verifies the paused `.part` on disk, the `Range`/`If-Range` resume request, and byte-exact assembly of the final file. diff --git a/docs/architecture/xtream-portal-compatibility.md b/docs/architecture/xtream-portal-compatibility.md index d0eb29d88..a983b9921 100644 --- a/docs/architecture/xtream-portal-compatibility.md +++ b/docs/architecture/xtream-portal-compatibility.md @@ -98,9 +98,11 @@ and private-network checks. ## User-Agent -Electron's `XTREAM_REQUEST` and `XTREAM_PROBE_URL` handlers -(`apps/electron-backend/src/app/events/xtream.events.ts`) send a shared -`XTREAM_CLIENT_USER_AGENT` constant on every outgoing request. Some Xtream +Electron's `XTREAM_REQUEST` and stream-probe handlers plus Xtream VOD download +requests share the exported `XTREAM_CLIENT_USER_AGENT` fallback. A playlist's +explicit User-Agent still wins for its stream probe and download. Legacy +download rows without a stored User-Agent receive the fallback when retrying, +resuming, or recovering a missing completed file. Some Xtream panels sit behind a WAF (e.g. Cloudflare) configured to challenge generic/incomplete browser-looking User-Agents while allowlisting known IPTV player clients; a player-style User-Agent (currently a VLC signature) avoids diff --git a/libs/portal/xtream/feature/src/lib/vod-details/vod-details-downloads.service.ts b/libs/portal/xtream/feature/src/lib/vod-details/vod-details-downloads.service.ts index b7fc6a8a1..720006b3b 100644 --- a/libs/portal/xtream/feature/src/lib/vod-details/vod-details-downloads.service.ts +++ b/libs/portal/xtream/feature/src/lib/vod-details/vod-details-downloads.service.ts @@ -5,6 +5,7 @@ import { DownloadsService } from '@iptvnator/services'; import { resolveXtreamVodPlaybackSource } from '@iptvnator/portal/xtream/data-access'; import { getXtreamVodInfo, + XTREAM_CLIENT_USER_AGENT, XtreamVodDetails, type XtreamVodInfo, } from '@iptvnator/shared/interfaces'; @@ -179,7 +180,8 @@ export class VodDetailsDownloadsService { : textList(info?.director)?.map((name) => ({ name })), }), headers: { - userAgent: playlist.userAgent, + userAgent: + playlist.userAgent?.trim() || XTREAM_CLIENT_USER_AGENT, referer: playlist.referrer, origin: playlist.origin, }, diff --git a/libs/portal/xtream/feature/src/lib/vod-details/vod-details-route-playback.spec.ts b/libs/portal/xtream/feature/src/lib/vod-details/vod-details-route-playback.spec.ts index fbc5bfe85..130bdf0dc 100644 --- a/libs/portal/xtream/feature/src/lib/vod-details/vod-details-route-playback.spec.ts +++ b/libs/portal/xtream/feature/src/lib/vod-details/vod-details-route-playback.spec.ts @@ -462,6 +462,9 @@ describe('VodDetailsRouteComponent — playback actions', () => { // came from — so the download would fetch the wrong movie. expect(stubs.startDownload).toHaveBeenCalledWith( expect.objectContaining({ + headers: expect.objectContaining({ + userAgent: 'VLC/3.0.18 LibVLC/3.0.18', + }), playlistId: 'playlist-1', xtreamId: 650020, metadataSnapshot: expect.objectContaining({ diff --git a/libs/portal/xtream/feature/src/lib/vod-details/vod-details-route.actions.spec.ts b/libs/portal/xtream/feature/src/lib/vod-details/vod-details-route.actions.spec.ts index 1e89f598a..9987eaccd 100644 --- a/libs/portal/xtream/feature/src/lib/vod-details/vod-details-route.actions.spec.ts +++ b/libs/portal/xtream/feature/src/lib/vod-details/vod-details-route.actions.spec.ts @@ -399,6 +399,11 @@ describe('VodDetailsRouteComponent fallback actions', () => { ); expect(startDownload).toHaveBeenCalledWith( expect.objectContaining({ + headers: { + origin: 'https://origin.example', + referer: 'https://referrer.example', + userAgent: 'IPTVnator', + }, title: 'Catalog movie', posterUrl: 'https://example.com/catalog-poster.jpg', url: 'http://example.com/movie/650020.mp4', diff --git a/libs/shared/interfaces/src/lib/xtream-portal.utils.ts b/libs/shared/interfaces/src/lib/xtream-portal.utils.ts index afb76e303..3eaf73a67 100644 --- a/libs/shared/interfaces/src/lib/xtream-portal.utils.ts +++ b/libs/shared/interfaces/src/lib/xtream-portal.utils.ts @@ -1,8 +1,5 @@ export type XtreamPortalStatusType = - | 'active' - | 'inactive' - | 'expired' - | 'unavailable'; + 'active' | 'inactive' | 'expired' | 'unavailable'; export interface XtreamPortalStatusResponseLike { user_info?: { @@ -19,6 +16,10 @@ export interface XtreamCredentialsFromUrl { const XTREAM_API_ENDPOINT_PATTERN = /\/(?:get|player_api)\.php$/i; +// Some Xtream panels challenge generic Node HTTP clients while allowing +// established IPTV players. Keep API, probe, and download requests aligned. +export const XTREAM_CLIENT_USER_AGENT = 'VLC/3.0.18 LibVLC/3.0.18'; + export function normalizeXtreamServerUrl(value: string): string { const trimmed = value.trim(); if (!trimmed) {