mirror of
https://github.com/4gray/iptvnator.git
synced 2026-10-08 17:06:15 -08:00
fix(downloads): resume interrupted Xtream VOD transfers (#1329)
* fix(downloads): resume interrupted Xtream VOD transfers * fix(downloads): validate partials before resuming * fix(downloads): propagate headers to episode transfers
This commit is contained in:
1 parent
d44045de31
commit
aba89d64cf
29 files changed
+894
-92
No files matched your search
@@ -0,0 +1,7 @@
|
||||
---
|
||||
type: fix
|
||||
area: downloads
|
||||
issues: [897, 1289]
|
||||
---
|
||||
|
||||
Xtream movie and episode downloads now keep their provider-compatible identity from the first request through legacy retries after source removal. Recoverable connection drops retain validated partials and show a credential-safe code; Retry resumes only with ETag or Last-Modified, otherwise it safely restarts.
|
||||
@@ -911,6 +911,19 @@ engine` (restart required) or
|
||||
|
||||
**Download Manager**:
|
||||
|
||||
- Fresh Xtream movie and series-episode downloads propagate the playlist's
|
||||
User-Agent, Referer, and Origin, defaulting User-Agent to the same
|
||||
provider-compatible `XTREAM_CLIENT_USER_AGENT` used by API requests and
|
||||
stream probes. Retry, resume, and missing-file
|
||||
recovery also add the fallback to legacy Xtream rows that have no stored
|
||||
User-Agent. Because download rows survive source deletion, a headerless
|
||||
legacy row whose playlist is already absent receives the same IPTV-player
|
||||
fallback; a known Stalker row remains unchanged. Allowlisted connection
|
||||
resets after bytes reach disk retain the partial and show a credential-safe
|
||||
`DOWNLOAD_NETWORK_INTERRUPTED` code only when the response supplied a strong
|
||||
ETag or Last-Modified validator. Retry then continues with Range/If-Range;
|
||||
without a validator it starts from byte zero and overwrites the unverified
|
||||
partial instead of risking mixed-representation corruption.
|
||||
- The desktop-only manager shares one global download store across the global,
|
||||
Xtream-scoped, and Stalker-scoped routes. Completed movie and grouped-series
|
||||
cards use the global Small/Medium/Large cover-grid tokens; missing completed
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import { mkdirSync, readFileSync, statSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import type { Page } from '@playwright/test';
|
||||
import {
|
||||
addXtreamPortal,
|
||||
closeElectronApp,
|
||||
expect,
|
||||
launchElectronApp,
|
||||
resetMockServers,
|
||||
test,
|
||||
waitForXtreamWorkspaceReady,
|
||||
} from './electron-test-fixtures';
|
||||
import {
|
||||
createInterruptedRangeServer,
|
||||
INTERRUPTED_RANGE_SERVER_ETAG,
|
||||
startDownload,
|
||||
} from './downloads.e2e-support';
|
||||
|
||||
async function openDownloadsPage(page: Page): Promise<void> {
|
||||
await page.getByRole('button', { name: 'Open downloads' }).click();
|
||||
await page.waitForURL(/\/workspace\/downloads(?:\?.*)?$/);
|
||||
}
|
||||
|
||||
async function getPlaylistId(page: Page, title: string): Promise<string> {
|
||||
const playlists = await page.evaluate(
|
||||
async () => (await window.electron?.dbGetAppPlaylistMetas?.()) ?? []
|
||||
);
|
||||
const playlist = playlists.find((entry) => entry.title === title);
|
||||
expect(playlist, `playlist "${title}" should exist`).toBeDefined();
|
||||
return playlist?._id ?? '';
|
||||
}
|
||||
|
||||
test.describe('Electron download reliability', () => {
|
||||
test('@downloads @electron retains a network-interrupted partial and retries it with HTTP Range', async ({
|
||||
dataDir,
|
||||
request,
|
||||
}) => {
|
||||
await resetMockServers(request, ['xtream']);
|
||||
const rangeServer = await createInterruptedRangeServer();
|
||||
const app = await launchElectronApp(dataDir);
|
||||
|
||||
try {
|
||||
await addXtreamPortal(app.mainWindow, {
|
||||
name: 'Reset Portal',
|
||||
username: 'user1',
|
||||
password: 'pass1',
|
||||
});
|
||||
await waitForXtreamWorkspaceReady(app.mainWindow);
|
||||
await openDownloadsPage(app.mainWindow);
|
||||
|
||||
const downloadsDir = join(dataDir, 'e2e-reset-downloads');
|
||||
mkdirSync(downloadsDir, { recursive: true });
|
||||
await app.electronApp.evaluate(({ dialog }, folder) => {
|
||||
dialog.showOpenDialog = async () =>
|
||||
({
|
||||
canceled: false,
|
||||
filePaths: [folder],
|
||||
}) as Awaited<ReturnType<typeof dialog.showOpenDialog>>;
|
||||
}, downloadsDir);
|
||||
await app.mainWindow
|
||||
.getByRole('button', { name: 'Change Folder' })
|
||||
.click();
|
||||
|
||||
const playlistId = await getPlaylistId(
|
||||
app.mainWindow,
|
||||
'Reset Portal'
|
||||
);
|
||||
const downloadId = await startDownload(app.mainWindow, {
|
||||
playlistId,
|
||||
xtreamId: 9801,
|
||||
contentType: 'vod',
|
||||
title: 'E2E Reset Movie',
|
||||
url: rangeServer.url,
|
||||
downloadFolder: downloadsDir,
|
||||
});
|
||||
const item = app.mainWindow.getByTestId(
|
||||
`download-queue-item-${downloadId}`
|
||||
);
|
||||
await expect(item.locator('.download-queue__status')).toContainText(
|
||||
'Failed',
|
||||
{ timeout: 30000 }
|
||||
);
|
||||
await expect(item.locator('.download-queue__error')).toContainText(
|
||||
'DOWNLOAD_NETWORK_INTERRUPTED (ECONNRESET)'
|
||||
);
|
||||
|
||||
const partialPath = join(downloadsDir, 'E2E Reset Movie.mp4.part');
|
||||
expect(statSync(partialPath).size).toBe(
|
||||
rangeServer.interruptedBytes
|
||||
);
|
||||
|
||||
await item
|
||||
.getByRole('button', { name: 'Retry E2E Reset Movie' })
|
||||
.click();
|
||||
await expect(
|
||||
app.mainWindow.getByTestId(
|
||||
`download-library-movie-${downloadId}`
|
||||
)
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
|
||||
const resumeRequest = rangeServer.requests.find(
|
||||
(entry) => entry.range
|
||||
);
|
||||
expect(resumeRequest?.range).toBe(
|
||||
`bytes=${rangeServer.interruptedBytes}-`
|
||||
);
|
||||
expect(resumeRequest?.ifRange).toBe(INTERRUPTED_RANGE_SERVER_ETAG);
|
||||
const finalFile = readFileSync(
|
||||
join(downloadsDir, 'E2E Reset Movie.mp4')
|
||||
);
|
||||
expect(finalFile.equals(rangeServer.payload)).toBe(true);
|
||||
} finally {
|
||||
await closeElectronApp(app);
|
||||
await rangeServer.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -31,6 +31,10 @@ interface ThrottledRangeServer {
|
||||
url: string;
|
||||
}
|
||||
|
||||
interface InterruptedRangeServer extends ThrottledRangeServer {
|
||||
interruptedBytes: number;
|
||||
}
|
||||
|
||||
interface TruncatedDownloadServer {
|
||||
close: () => Promise<void>;
|
||||
payload: Buffer;
|
||||
@@ -39,6 +43,7 @@ interface TruncatedDownloadServer {
|
||||
}
|
||||
|
||||
export const RANGE_SERVER_ETAG = '"e2e-range-etag"';
|
||||
export const INTERRUPTED_RANGE_SERVER_ETAG = '"e2e-reset-etag"';
|
||||
const downloadPlayCaptureKey = '__iptvnatorE2eDownloadPlayPaths';
|
||||
|
||||
export function getStalkerSeriesDownloadTarget(
|
||||
@@ -172,6 +177,63 @@ export async function createThrottledRangeServer(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the first full response after writing a valid prefix, then serves the
|
||||
* remainder to a Range retry. This matches a provider/proxy connection drop
|
||||
* without manufacturing a clean EOF.
|
||||
*/
|
||||
export async function createInterruptedRangeServer(): Promise<InterruptedRangeServer> {
|
||||
const payload = Buffer.alloc(64 * 1024, 9);
|
||||
const interruptedBytes = 16 * 1024;
|
||||
const requests: RangeServerRequest[] = [];
|
||||
|
||||
const server = createServer((req, res) => {
|
||||
const range = req.headers.range;
|
||||
const ifRange = req.headers['if-range'];
|
||||
requests.push({
|
||||
ifRange: typeof ifRange === 'string' ? ifRange : undefined,
|
||||
range: typeof range === 'string' ? range : undefined,
|
||||
});
|
||||
|
||||
const offset = range
|
||||
? Number(/^bytes=(\d+)-$/.exec(range)?.[1] ?? Number.NaN)
|
||||
: 0;
|
||||
if (range && Number.isFinite(offset)) {
|
||||
res.writeHead(206, {
|
||||
'Content-Length': payload.length - offset,
|
||||
'Content-Range': `bytes ${offset}-${payload.length - 1}/${payload.length}`,
|
||||
'Content-Type': 'video/mp4',
|
||||
ETag: INTERRUPTED_RANGE_SERVER_ETAG,
|
||||
});
|
||||
res.end(payload.subarray(offset));
|
||||
return;
|
||||
}
|
||||
|
||||
res.writeHead(200, {
|
||||
'Content-Length': payload.length,
|
||||
'Content-Type': 'video/mp4',
|
||||
ETag: INTERRUPTED_RANGE_SERVER_ETAG,
|
||||
});
|
||||
res.write(payload.subarray(0, interruptedBytes), () => {
|
||||
setTimeout(() => res.socket?.destroy(), 20);
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) =>
|
||||
server.listen(0, '127.0.0.1', resolve)
|
||||
);
|
||||
const { port } = server.address() as AddressInfo;
|
||||
|
||||
return {
|
||||
close: () =>
|
||||
new Promise<void>((resolve) => server.close(() => resolve())),
|
||||
interruptedBytes,
|
||||
payload,
|
||||
requests,
|
||||
url: `http://127.0.0.1:${port}/media/e2e-reset-movie.mp4`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Ends a chunked response cleanly while Content-Range advertises a larger
|
||||
* representation. The runtime therefore retains the valid .part for a Range
|
||||
|
||||
@@ -14,12 +14,16 @@ import type {
|
||||
DownloadTask,
|
||||
TransferProgress,
|
||||
} from './download-task';
|
||||
import { describeError, TruncatedTransferError } from './download-transfer';
|
||||
import {
|
||||
describeError,
|
||||
InterruptedTransferError,
|
||||
TruncatedTransferError,
|
||||
} from './download-transfer';
|
||||
|
||||
/**
|
||||
* Persistence for a failed startDownload() attempt, after its cancel/pause
|
||||
* checkpoints have been ruled out. Chooses between: retaining a truncated
|
||||
* transfer for a Range retry, committing an already-finalized file,
|
||||
* checkpoints have been ruled out. Chooses between: retaining a recoverable
|
||||
* partial for a Range retry, committing an already-finalized file,
|
||||
* retaining a completed partial, or the generic delete-partial failure.
|
||||
*/
|
||||
export async function handleDownloadFailure(
|
||||
@@ -28,8 +32,12 @@ export async function handleDownloadFailure(
|
||||
reservation: ReservedPartialDownloadFile | undefined,
|
||||
error: unknown
|
||||
): Promise<void> {
|
||||
if (error instanceof TruncatedTransferError && reservation) {
|
||||
// The short response is retained so a retry can continue the
|
||||
if (
|
||||
(error instanceof TruncatedTransferError ||
|
||||
error instanceof InterruptedTransferError) &&
|
||||
reservation
|
||||
) {
|
||||
// The recoverable partial is retained so a retry can continue the
|
||||
// transfer via Range instead of starting over.
|
||||
await persistCompletedPartialFailure(
|
||||
db,
|
||||
@@ -338,7 +346,9 @@ function canCopyCompletedPartialAfterLinkFailure(error: unknown): boolean {
|
||||
}
|
||||
|
||||
/** @returns false when a .part exists but could not be deleted. */
|
||||
export function removePartialFile(filePath: string | null | undefined): boolean {
|
||||
export function removePartialFile(
|
||||
filePath: string | null | undefined
|
||||
): boolean {
|
||||
try {
|
||||
removePartialDownloadFile(filePath);
|
||||
return true;
|
||||
|
||||
@@ -3,6 +3,7 @@ interface SetupOptions {
|
||||
cleanupError?: Error;
|
||||
claim?: { changes: number };
|
||||
fileReappeared?: boolean;
|
||||
playlistType?: 'xtream' | 'stalker';
|
||||
row?: Record<string, unknown> | null;
|
||||
urlError?: Error;
|
||||
}
|
||||
@@ -25,13 +26,18 @@ async function setup(options: SetupOptions = {}) {
|
||||
url: 'https://example.test/movie.mp4',
|
||||
...options.row,
|
||||
};
|
||||
const limit = jest.fn().mockResolvedValue(row ? [row] : []);
|
||||
const downloadLimit = jest.fn().mockResolvedValue(row ? [row] : []);
|
||||
const playlistLimit = jest
|
||||
.fn()
|
||||
.mockResolvedValue([{ type: options.playlistType ?? 'stalker' }]);
|
||||
const where = jest.fn().mockResolvedValue(options.claim ?? { changes: 1 });
|
||||
const set = jest.fn(() => ({ where }));
|
||||
const db = {
|
||||
select: jest.fn(() => ({
|
||||
select: jest.fn((selection?: unknown) => ({
|
||||
from: jest.fn(() => ({
|
||||
where: jest.fn(() => ({ limit })),
|
||||
where: jest.fn(() => ({
|
||||
limit: selection ? playlistLimit : downloadLimit,
|
||||
})),
|
||||
})),
|
||||
})),
|
||||
update: jest.fn(() => ({ set })),
|
||||
@@ -118,6 +124,27 @@ describe('redownload missing completed file', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('adds the Xtream fallback User-Agent to a legacy row', async () => {
|
||||
const harness = await setup({
|
||||
playlistType: 'xtream',
|
||||
row: {
|
||||
playlistId: 'playlist-1',
|
||||
requestHeaders: null,
|
||||
},
|
||||
});
|
||||
|
||||
await expect(harness.redownloadMissingRequest(42)).resolves.toEqual({
|
||||
success: true,
|
||||
});
|
||||
expect(harness.enqueueDownload).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
headers: {
|
||||
'User-Agent': 'VLC/3.0.18 LibVLC/3.0.18',
|
||||
},
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('recovers a file that reappeared without updating or enqueueing', async () => {
|
||||
const harness = await setup({ fileReappeared: true });
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import * as schema from '../../database/schema';
|
||||
import { assertRemoteUrlAllowed } from '../url-safety';
|
||||
import { isAvailableDownloadFile } from './download-file-availability';
|
||||
import { removePartialDownloadFile } from './download-file-path';
|
||||
import { parseStoredHeaders } from './download-requests';
|
||||
import { resolveStoredDownloadHeaders } from './download-request-headers';
|
||||
import { enqueueDownload } from './download-runtime';
|
||||
|
||||
export interface RedownloadMissingResult {
|
||||
@@ -61,6 +61,7 @@ export async function redownloadMissingRequest(
|
||||
}
|
||||
|
||||
await assertRemoteUrlAllowed(item.url, { allowPrivateNetworks: true });
|
||||
const headers = await resolveStoredDownloadHeaders(db, item);
|
||||
|
||||
try {
|
||||
removePartialDownloadFile(item.filePath);
|
||||
@@ -102,7 +103,7 @@ export async function redownloadMissingRequest(
|
||||
directory: dirname(item.filePath),
|
||||
fileName: basename(item.filePath),
|
||||
filePath: item.filePath,
|
||||
headers: parseStoredHeaders(item.requestHeaders),
|
||||
headers,
|
||||
id: item.id,
|
||||
resumeValidator: null,
|
||||
totalBytes: null,
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import type { DownloadsDatabase } from './download-task';
|
||||
|
||||
function createDatabase(playlistType?: 'xtream' | 'stalker') {
|
||||
const limit = jest
|
||||
.fn()
|
||||
.mockResolvedValue(playlistType ? [{ type: playlistType }] : []);
|
||||
const db = {
|
||||
select: jest.fn(() => ({
|
||||
from: jest.fn(() => ({
|
||||
where: jest.fn(() => ({ limit })),
|
||||
})),
|
||||
})),
|
||||
} as unknown as DownloadsDatabase;
|
||||
return { db, select: db.select };
|
||||
}
|
||||
|
||||
describe('stored download request headers', () => {
|
||||
it('adds the provider-compatible fallback for a legacy Xtream row', async () => {
|
||||
const { db } = createDatabase('xtream');
|
||||
const { resolveStoredDownloadHeaders } =
|
||||
await import('./download-request-headers');
|
||||
|
||||
await expect(
|
||||
resolveStoredDownloadHeaders(db, {
|
||||
playlistId: 'playlist-1',
|
||||
requestHeaders: JSON.stringify({
|
||||
Authorization: 'not-allowed',
|
||||
Referer: 'https://example.test/',
|
||||
}),
|
||||
})
|
||||
).resolves.toEqual({
|
||||
Referer: 'https://example.test/',
|
||||
'User-Agent': 'VLC/3.0.18 LibVLC/3.0.18',
|
||||
});
|
||||
});
|
||||
|
||||
it('does not add the Xtream fallback to a Stalker row', async () => {
|
||||
const { db } = createDatabase('stalker');
|
||||
const { resolveStoredDownloadHeaders } =
|
||||
await import('./download-request-headers');
|
||||
|
||||
await expect(
|
||||
resolveStoredDownloadHeaders(db, {
|
||||
playlistId: 'playlist-1',
|
||||
requestHeaders: null,
|
||||
})
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('adds the player fallback when a legacy row outlives its deleted source', async () => {
|
||||
const { db } = createDatabase();
|
||||
const { resolveStoredDownloadHeaders } =
|
||||
await import('./download-request-headers');
|
||||
|
||||
await expect(
|
||||
resolveStoredDownloadHeaders(db, {
|
||||
playlistId: 'deleted-playlist',
|
||||
requestHeaders: null,
|
||||
})
|
||||
).resolves.toEqual({
|
||||
'User-Agent': 'VLC/3.0.18 LibVLC/3.0.18',
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves an explicit stored User-Agent without querying the playlist', async () => {
|
||||
const { db, select } = createDatabase('xtream');
|
||||
const { resolveStoredDownloadHeaders } =
|
||||
await import('./download-request-headers');
|
||||
|
||||
await expect(
|
||||
resolveStoredDownloadHeaders(db, {
|
||||
playlistId: 'playlist-1',
|
||||
requestHeaders: JSON.stringify({
|
||||
'User-Agent': 'Custom/1.0',
|
||||
}),
|
||||
})
|
||||
).resolves.toEqual({ 'User-Agent': 'Custom/1.0' });
|
||||
expect(select).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
import { XTREAM_CLIENT_USER_AGENT } from '@iptvnator/shared/interfaces';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import * as schema from '../../database/schema';
|
||||
import type { DownloadsDatabase } from './download-task';
|
||||
|
||||
const STORED_HEADER_ALLOWLIST = ['User-Agent', 'Origin', 'Referer'] as const;
|
||||
|
||||
export function parseStoredHeaders(
|
||||
value: string | null | undefined
|
||||
): Record<string, string> | undefined {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Re-apply the write-time allowlist so a tampered or imported
|
||||
// database row cannot smuggle arbitrary headers into requests.
|
||||
const entries = parsed as Record<string, unknown>;
|
||||
const headers = STORED_HEADER_ALLOWLIST.reduce<Record<string, string>>(
|
||||
(acc, key) => {
|
||||
const headerValue = entries[key];
|
||||
if (typeof headerValue === 'string') {
|
||||
acc[key] = headerValue;
|
||||
}
|
||||
return acc;
|
||||
},
|
||||
{}
|
||||
);
|
||||
return Object.keys(headers).length > 0 ? headers : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveStoredDownloadHeaders(
|
||||
db: DownloadsDatabase,
|
||||
item: { playlistId: string; requestHeaders: string | null | undefined }
|
||||
): Promise<Record<string, string> | undefined> {
|
||||
const headers = parseStoredHeaders(item.requestHeaders);
|
||||
if (headers?.['User-Agent']?.trim()) {
|
||||
return headers;
|
||||
}
|
||||
|
||||
const playlists = await db
|
||||
.select({ type: schema.playlists.type })
|
||||
.from(schema.playlists)
|
||||
.where(eq(schema.playlists.id, item.playlistId))
|
||||
.limit(1);
|
||||
const playlistType = playlists[0]?.type;
|
||||
if (playlistType !== undefined && playlistType !== 'xtream') {
|
||||
return headers;
|
||||
}
|
||||
|
||||
// Download rows intentionally survive individual playlist deletion. Older
|
||||
// rows have no stored provider marker, so a missing source must use the
|
||||
// IPTV-player fallback as the only recoverable identity-compatible default.
|
||||
return {
|
||||
...headers,
|
||||
'User-Agent': XTREAM_CLIENT_USER_AGENT,
|
||||
};
|
||||
}
|
||||
@@ -384,6 +384,66 @@ describe('download request metadata snapshots', () => {
|
||||
});
|
||||
|
||||
describe('download requests resume', () => {
|
||||
it('adds the Xtream fallback User-Agent to a legacy paused row', async () => {
|
||||
jest.resetModules();
|
||||
const schema = await import('../../database/schema');
|
||||
const row = {
|
||||
filePath: '/downloads/movie.mp4',
|
||||
id: 42,
|
||||
playlistId: 'playlist-1',
|
||||
requestHeaders: null,
|
||||
resumeValidator: '"etag-9"',
|
||||
status: 'paused',
|
||||
title: 'Movie',
|
||||
totalBytes: 100,
|
||||
url: 'https://example.test/movie.mp4',
|
||||
};
|
||||
const downloadLimit = jest.fn().mockResolvedValue([row]);
|
||||
const playlistLimit = jest.fn().mockResolvedValue([{ type: 'xtream' }]);
|
||||
const db = {
|
||||
select: jest.fn(() => ({
|
||||
from: jest.fn((table: unknown) => ({
|
||||
where: jest.fn(() => ({
|
||||
limit:
|
||||
table === schema.playlists
|
||||
? playlistLimit
|
||||
: downloadLimit,
|
||||
})),
|
||||
})),
|
||||
})),
|
||||
update: jest.fn(() => ({
|
||||
set: jest.fn(() => ({
|
||||
where: jest.fn().mockResolvedValue({ changes: 1 }),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
const enqueueDownload = jest.fn();
|
||||
const authorizer = {
|
||||
requireAuthorized: jest.fn(async (directory: string) => directory),
|
||||
} as unknown as DownloadDirectoryAuthorizer;
|
||||
|
||||
jest.doMock('../../database/connection', () => ({
|
||||
getDatabase: jest.fn().mockResolvedValue(db),
|
||||
}));
|
||||
jest.doMock('../url-safety', () => ({
|
||||
assertRemoteUrlAllowed: jest.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
jest.doMock('./download-runtime', () => ({ enqueueDownload }));
|
||||
|
||||
const { resumeDownloadRequest } = await import('./download-requests');
|
||||
await expect(
|
||||
resumeDownloadRequest(42, '/unused', authorizer)
|
||||
).resolves.toEqual({ success: true });
|
||||
|
||||
expect(enqueueDownload).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
headers: {
|
||||
'User-Agent': 'VLC/3.0.18 LibVLC/3.0.18',
|
||||
},
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('enqueues a paused download with stored headers and original target path', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import * as schema from '../../database/schema';
|
||||
import { assertRemoteUrlAllowed } from '../url-safety';
|
||||
import { DownloadDirectoryAuthorizer } from './download-directory-authorization';
|
||||
import { removePartialDownloadFile } from './download-file-path';
|
||||
import { resolveStoredDownloadHeaders } from './download-request-headers';
|
||||
import {
|
||||
assertDownloadMetadataArtworkDiffersFromStream,
|
||||
assertDownloadMetadataMatchesContentType,
|
||||
@@ -80,40 +81,6 @@ function serializeHeaders(
|
||||
return headers ? JSON.stringify(headers) : null;
|
||||
}
|
||||
|
||||
const STORED_HEADER_ALLOWLIST = ['User-Agent', 'Origin', 'Referer'] as const;
|
||||
|
||||
export function parseStoredHeaders(
|
||||
value: string | null
|
||||
): Record<string, string> | undefined {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Re-apply the write-time allowlist so a tampered or imported
|
||||
// database row cannot smuggle arbitrary headers into requests.
|
||||
const entries = parsed as Record<string, unknown>;
|
||||
const headers = STORED_HEADER_ALLOWLIST.reduce<Record<string, string>>(
|
||||
(acc, key) => {
|
||||
const headerValue = entries[key];
|
||||
if (typeof headerValue === 'string') {
|
||||
acc[key] = headerValue;
|
||||
}
|
||||
return acc;
|
||||
},
|
||||
{}
|
||||
);
|
||||
return Object.keys(headers).length > 0 ? headers : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export async function startDownloadRequest(
|
||||
data: StartDownloadRequest,
|
||||
authorizer: DownloadDirectoryAuthorizer
|
||||
@@ -320,6 +287,7 @@ export async function retryDownloadRequest(
|
||||
const fileName = retainedFilePath
|
||||
? basename(retainedFilePath)
|
||||
: createFileName(item.title, item.url);
|
||||
const headers = await resolveStoredDownloadHeaders(db, item);
|
||||
const queuedUpdate = retainedFilePath
|
||||
? {
|
||||
errorMessage: null,
|
||||
@@ -345,7 +313,7 @@ export async function retryDownloadRequest(
|
||||
directory,
|
||||
fileName,
|
||||
filePath: retainedFilePath,
|
||||
headers: parseStoredHeaders(item.requestHeaders),
|
||||
headers,
|
||||
id: item.id,
|
||||
resumeValidator: retainedFilePath ? item.resumeValidator : null,
|
||||
totalBytes: retainedFilePath ? item.totalBytes : null,
|
||||
@@ -388,6 +356,7 @@ export async function resumeDownloadRequest(
|
||||
const fileName = item.filePath
|
||||
? basename(item.filePath)
|
||||
: createFileName(item.title, item.url);
|
||||
const headers = await resolveStoredDownloadHeaders(db, item);
|
||||
|
||||
// Claim the row atomically: a concurrent resume for the same id loses
|
||||
// this conditional update and must not enqueue a second task.
|
||||
@@ -416,7 +385,7 @@ export async function resumeDownloadRequest(
|
||||
directory,
|
||||
fileName,
|
||||
filePath: item.filePath,
|
||||
headers: parseStoredHeaders(item.requestHeaders),
|
||||
headers,
|
||||
id: item.id,
|
||||
resumeValidator: item.resumeValidator,
|
||||
totalBytes: item.totalBytes,
|
||||
|
||||
@@ -62,6 +62,7 @@ describe('destination collision handling', () => {
|
||||
runtime.enqueueDownload({
|
||||
...createTask(),
|
||||
filePath: '/downloads/movie.mp4',
|
||||
resumeValidator: '"etag-1"',
|
||||
totalBytes: 100,
|
||||
});
|
||||
await waitForStatus(set, 'completed');
|
||||
|
||||
@@ -11,6 +11,7 @@ interface ResumeHarness {
|
||||
|
||||
interface ResumeHarnessOptions {
|
||||
partialSize: number;
|
||||
partialSizeAfterTransferError?: number;
|
||||
response: {
|
||||
data: Readable;
|
||||
headers: Record<string, string>;
|
||||
@@ -82,7 +83,12 @@ async function setupResumeHarness(
|
||||
}));
|
||||
jest.doMock('./download-file-path', () => ({
|
||||
getPartialDownloadPath: (filePath: string) => `${filePath}.part`,
|
||||
getPartialDownloadSize: jest.fn(() => options.partialSize),
|
||||
getPartialDownloadSize: jest
|
||||
.fn()
|
||||
.mockReturnValueOnce(options.partialSize)
|
||||
.mockReturnValue(
|
||||
options.partialSizeAfterTransferError ?? options.partialSize
|
||||
),
|
||||
removePartialDownloadFile,
|
||||
reserveAvailablePartialDownloadFile: jest.fn(
|
||||
(directory: string, filename: string) => ({
|
||||
@@ -145,6 +151,34 @@ describe('download resume validation', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('restarts without Range when a retained partial has no validator', async () => {
|
||||
const harness = await setupResumeHarness({
|
||||
finalSize: 4,
|
||||
partialSize: 50,
|
||||
response: {
|
||||
data: Readable.from([Buffer.from('full')]),
|
||||
headers: { 'content-length': '4', etag: '"etag-new"' },
|
||||
status: 200,
|
||||
},
|
||||
});
|
||||
|
||||
harness.runtime.enqueueDownload(
|
||||
createTask({
|
||||
filePath: '/downloads/movie.mp4',
|
||||
totalBytes: 54,
|
||||
})
|
||||
);
|
||||
await waitForStatus(harness.set, 'completed');
|
||||
|
||||
const requestOptions =
|
||||
harness.requestWithValidatedRedirects.mock.calls[0][1];
|
||||
expect(requestOptions.headers).toEqual({});
|
||||
expect(harness.createWriteStream).toHaveBeenCalledWith(
|
||||
'/downloads/movie.mp4.part',
|
||||
{ flags: 'w' }
|
||||
);
|
||||
});
|
||||
|
||||
it('restarts from byte zero when a resume request is answered with 200', async () => {
|
||||
const harness = await setupResumeHarness({
|
||||
finalSize: 4,
|
||||
@@ -209,6 +243,7 @@ describe('download resume validation', () => {
|
||||
harness.runtime.enqueueDownload(
|
||||
createTask({
|
||||
filePath: '/downloads/movie.mp4',
|
||||
resumeValidator: '"etag-1"',
|
||||
totalBytes: 54,
|
||||
})
|
||||
);
|
||||
@@ -245,6 +280,7 @@ describe('download resume validation', () => {
|
||||
harness.runtime.enqueueDownload(
|
||||
createTask({
|
||||
filePath: '/downloads/movie.mp4',
|
||||
resumeValidator: '"etag-1"',
|
||||
totalBytes: 100,
|
||||
})
|
||||
);
|
||||
@@ -264,6 +300,191 @@ describe('download resume validation', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('retains received bytes when the connection resets mid-transfer', async () => {
|
||||
const body = new PassThrough();
|
||||
const harness = await setupResumeHarness({
|
||||
finalSize: 'enoent',
|
||||
partialSize: 0,
|
||||
partialSizeAfterTransferError: 20,
|
||||
response: {
|
||||
data: body,
|
||||
headers: {
|
||||
'content-length': '100',
|
||||
etag: '"etag-reset"',
|
||||
},
|
||||
status: 200,
|
||||
},
|
||||
});
|
||||
const consoleError = jest
|
||||
.spyOn(console, 'error')
|
||||
.mockImplementation(() => undefined);
|
||||
|
||||
try {
|
||||
harness.runtime.enqueueDownload(createTask());
|
||||
while (
|
||||
harness.requestWithValidatedRedirects.mock.calls.length < 1
|
||||
) {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
}
|
||||
|
||||
body.write(Buffer.alloc(20, 'r'));
|
||||
const resetError = new Error(
|
||||
'socket hang up'
|
||||
) as NodeJS.ErrnoException;
|
||||
resetError.code = 'ECONNRESET';
|
||||
body.destroy(resetError);
|
||||
await waitForStatus(harness.set, 'failed');
|
||||
|
||||
expect(harness.set).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
bytesDownloaded: 20,
|
||||
errorMessage:
|
||||
'DOWNLOAD_NETWORK_INTERRUPTED (ECONNRESET): Retry to continue from the saved partial file',
|
||||
filePath: '/downloads/movie.mp4',
|
||||
status: 'failed',
|
||||
totalBytes: 100,
|
||||
})
|
||||
);
|
||||
expect(harness.set).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
resumeValidator: '"etag-reset"',
|
||||
})
|
||||
);
|
||||
expect(harness.removePartialDownloadFile).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
consoleError.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('retains an existing partial when a resumed response resets before another byte', async () => {
|
||||
const body = new PassThrough();
|
||||
const harness = await setupResumeHarness({
|
||||
finalSize: 'enoent',
|
||||
partialSize: 40,
|
||||
partialSizeAfterTransferError: 40,
|
||||
response: {
|
||||
data: body,
|
||||
headers: { 'content-range': 'bytes 40-99/100' },
|
||||
status: 206,
|
||||
},
|
||||
});
|
||||
const consoleError = jest
|
||||
.spyOn(console, 'error')
|
||||
.mockImplementation(() => undefined);
|
||||
|
||||
try {
|
||||
harness.runtime.enqueueDownload(
|
||||
createTask({
|
||||
filePath: '/downloads/movie.mp4',
|
||||
resumeValidator: '"etag-reset"',
|
||||
totalBytes: 100,
|
||||
})
|
||||
);
|
||||
while (
|
||||
harness.requestWithValidatedRedirects.mock.calls.length < 1
|
||||
) {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
}
|
||||
|
||||
const resetError = new Error(
|
||||
'socket hang up'
|
||||
) as NodeJS.ErrnoException;
|
||||
resetError.code = 'ECONNRESET';
|
||||
body.destroy(resetError);
|
||||
await waitForStatus(harness.set, 'failed');
|
||||
|
||||
expect(harness.set).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
bytesDownloaded: 40,
|
||||
errorMessage:
|
||||
'DOWNLOAD_NETWORK_INTERRUPTED (ECONNRESET): Retry to continue from the saved partial file',
|
||||
filePath: '/downloads/movie.mp4',
|
||||
status: 'failed',
|
||||
totalBytes: 100,
|
||||
})
|
||||
);
|
||||
expect(harness.removePartialDownloadFile).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
consoleError.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
code: 'EUNKNOWN',
|
||||
headers: { 'content-length': '100' },
|
||||
label: 'unknown stream error',
|
||||
partialSizeAfterTransferError: 20,
|
||||
},
|
||||
{
|
||||
code: 'ECONNRESET',
|
||||
headers: {},
|
||||
label: 'response without an advertised total',
|
||||
partialSizeAfterTransferError: 20,
|
||||
},
|
||||
{
|
||||
code: 'ECONNRESET',
|
||||
headers: { 'content-length': '100' },
|
||||
label: 'response without a representation validator',
|
||||
partialSizeAfterTransferError: 20,
|
||||
},
|
||||
{
|
||||
code: 'ECONNRESET',
|
||||
headers: { 'content-length': '100' },
|
||||
label: 'fresh zero-byte failure',
|
||||
partialSizeAfterTransferError: 0,
|
||||
},
|
||||
{
|
||||
code: 'ECONNRESET',
|
||||
headers: { 'content-length': '100' },
|
||||
label: 'partial larger than the advertised total',
|
||||
partialSizeAfterTransferError: 101,
|
||||
},
|
||||
])(
|
||||
'uses generic cleanup for $label',
|
||||
async ({ code, headers, partialSizeAfterTransferError }) => {
|
||||
const body = new PassThrough();
|
||||
const harness = await setupResumeHarness({
|
||||
finalSize: 'enoent',
|
||||
partialSize: 0,
|
||||
partialSizeAfterTransferError,
|
||||
response: { data: body, headers, status: 200 },
|
||||
});
|
||||
const consoleError = jest
|
||||
.spyOn(console, 'error')
|
||||
.mockImplementation(() => undefined);
|
||||
|
||||
try {
|
||||
harness.runtime.enqueueDownload(createTask());
|
||||
while (
|
||||
harness.requestWithValidatedRedirects.mock.calls.length < 1
|
||||
) {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
}
|
||||
|
||||
const streamError = new Error(
|
||||
'socket hang up'
|
||||
) as NodeJS.ErrnoException;
|
||||
streamError.code = code;
|
||||
body.destroy(streamError);
|
||||
await waitForStatus(harness.set, 'failed');
|
||||
|
||||
expect(harness.removePartialDownloadFile).toHaveBeenCalledWith(
|
||||
'/downloads/movie.mp4'
|
||||
);
|
||||
expect(harness.set).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
errorMessage: 'socket hang up',
|
||||
filePath: null,
|
||||
status: 'failed',
|
||||
})
|
||||
);
|
||||
} finally {
|
||||
consoleError.mockRestore();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
it('captures a strong ETag from the first response for later resumes', async () => {
|
||||
const harness = await setupResumeHarness({
|
||||
finalSize: 4,
|
||||
|
||||
@@ -42,17 +42,16 @@ describe('download runtime pause and resume', () => {
|
||||
it('persists active pause without deleting the partial file', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
const set = jest.fn(() => ({ where: jest.fn().mockResolvedValue(undefined) }));
|
||||
const set = jest.fn(() => ({
|
||||
where: jest.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
const db = {
|
||||
update: jest.fn(() => ({ set })),
|
||||
};
|
||||
const removePartialDownloadFile = jest.fn();
|
||||
const stream = new PassThrough();
|
||||
const requestWithValidatedRedirects = jest.fn(
|
||||
async (
|
||||
_url: string,
|
||||
options: { signal?: AbortSignal }
|
||||
) => {
|
||||
async (_url: string, options: { signal?: AbortSignal }) => {
|
||||
options.signal?.addEventListener('abort', () => {
|
||||
stream.destroy(new Error('aborted'));
|
||||
});
|
||||
@@ -119,10 +118,12 @@ describe('download runtime pause and resume', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('uses an HTTP Range header when a partial file already exists', async () => {
|
||||
it('uses an HTTP Range header when a validated partial file already exists', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
const set = jest.fn(() => ({ where: jest.fn().mockResolvedValue(undefined) }));
|
||||
const set = jest.fn(() => ({
|
||||
where: jest.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
const db = {
|
||||
update: jest.fn(() => ({ set })),
|
||||
};
|
||||
@@ -168,13 +169,17 @@ describe('download runtime pause and resume', () => {
|
||||
runtime.enqueueDownload({
|
||||
...createTask(),
|
||||
filePath: '/downloads/movie.mp4',
|
||||
resumeValidator: '"etag-1"',
|
||||
});
|
||||
await waitForCallCount(requestWithValidatedRedirects, 1);
|
||||
|
||||
expect(requestWithValidatedRedirects).toHaveBeenCalledWith(
|
||||
'https://example.test/movie.mp4',
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({ Range: 'bytes=50-' }),
|
||||
headers: expect.objectContaining({
|
||||
'If-Range': '"etag-1"',
|
||||
Range: 'bytes=50-',
|
||||
}),
|
||||
}),
|
||||
{ allowPrivateNetworks: true }
|
||||
);
|
||||
@@ -184,7 +189,9 @@ describe('download runtime pause and resume', () => {
|
||||
it('deletes a queued resumed partial file when the queued task is canceled', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
const set = jest.fn(() => ({ where: jest.fn().mockResolvedValue(undefined) }));
|
||||
const set = jest.fn(() => ({
|
||||
where: jest.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
const db = {
|
||||
update: jest.fn(() => ({ set })),
|
||||
};
|
||||
@@ -271,7 +278,9 @@ describe('download runtime pause and resume', () => {
|
||||
it('retains the partial path when canceling a queued task whose partial cannot be deleted', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
const set = jest.fn(() => ({ where: jest.fn().mockResolvedValue(undefined) }));
|
||||
const set = jest.fn(() => ({
|
||||
where: jest.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
const db = { update: jest.fn(() => ({ set })) };
|
||||
const removePartialDownloadFile = jest.fn(() => {
|
||||
throw new Error('EPERM: locked');
|
||||
@@ -338,7 +347,9 @@ describe('download runtime pause and resume', () => {
|
||||
it('retains the partial path when canceling a paused row whose partial cannot be deleted', async () => {
|
||||
jest.resetModules();
|
||||
|
||||
const set = jest.fn(() => ({ where: jest.fn().mockResolvedValue(undefined) }));
|
||||
const set = jest.fn(() => ({
|
||||
where: jest.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
const db = {
|
||||
select: jest.fn(() => ({
|
||||
from: jest.fn(() => ({
|
||||
|
||||
@@ -34,12 +34,39 @@ export class TruncatedTransferError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
const RETAINABLE_NETWORK_ERROR_CODES = new Set([
|
||||
'ECONNABORTED',
|
||||
'ECONNRESET',
|
||||
'EPIPE',
|
||||
'ETIMEDOUT',
|
||||
'ERR_HTTP2_STREAM_CANCEL',
|
||||
'ERR_HTTP2_STREAM_ERROR',
|
||||
'ERR_STREAM_PREMATURE_CLOSE',
|
||||
]);
|
||||
|
||||
export class InterruptedTransferError extends Error {
|
||||
constructor(
|
||||
readonly progress: TransferProgress,
|
||||
networkCode: string
|
||||
) {
|
||||
super(
|
||||
`DOWNLOAD_NETWORK_INTERRUPTED (${networkCode}): Retry to continue from the saved partial file`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function transferToPartialFile(
|
||||
db: DownloadsDatabase,
|
||||
task: DownloadTask,
|
||||
reservation: ReservedPartialDownloadFile
|
||||
): Promise<TransferProgress> {
|
||||
const resumeOffset = getResumeOffset(task, reservation);
|
||||
const retainedOffset = getResumeOffset(task, reservation);
|
||||
const resumeOffset = task.resumeValidator ? retainedOffset : 0;
|
||||
if (retainedOffset > 0 && resumeOffset === 0) {
|
||||
console.warn(
|
||||
`[Downloads] Restarting ${reservation.filename} from the beginning (saved partial has no ETag or Last-Modified validator)`
|
||||
);
|
||||
}
|
||||
|
||||
const headers = {
|
||||
...(task.headers ?? {}),
|
||||
@@ -130,6 +157,22 @@ export async function transferToPartialFile(
|
||||
|
||||
try {
|
||||
await pipeline(readable, output);
|
||||
} catch (error) {
|
||||
const interruptedProgress = getInterruptedTransferProgress(
|
||||
error,
|
||||
reservation,
|
||||
effectiveOffset,
|
||||
totalBytes,
|
||||
task.resumeValidator
|
||||
);
|
||||
if (interruptedProgress) {
|
||||
await persistProgress(db, task, interruptedProgress.progress);
|
||||
throw new InterruptedTransferError(
|
||||
interruptedProgress.progress,
|
||||
interruptedProgress.networkCode
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
abortController.signal.removeEventListener('abort', abortStream);
|
||||
}
|
||||
@@ -141,6 +184,40 @@ export async function transferToPartialFile(
|
||||
return { bytesDownloaded, totalBytes };
|
||||
}
|
||||
|
||||
function getInterruptedTransferProgress(
|
||||
error: unknown,
|
||||
reservation: ReservedPartialDownloadFile,
|
||||
initialBytes: number,
|
||||
totalBytes: number | null,
|
||||
resumeValidator: string | null | undefined
|
||||
): { networkCode: string; progress: TransferProgress } | null {
|
||||
const networkCode =
|
||||
error && typeof error === 'object' && 'code' in error
|
||||
? String(error.code)
|
||||
: '';
|
||||
if (
|
||||
!RETAINABLE_NETWORK_ERROR_CODES.has(networkCode) ||
|
||||
totalBytes === null ||
|
||||
!resumeValidator
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const bytesDownloaded = getPartialDownloadSize(reservation.path);
|
||||
if (
|
||||
bytesDownloaded === 0 ||
|
||||
bytesDownloaded < initialBytes ||
|
||||
bytesDownloaded >= totalBytes
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
networkCode,
|
||||
progress: { bytesDownloaded, totalBytes },
|
||||
};
|
||||
}
|
||||
|
||||
function getResumeOffset(
|
||||
task: DownloadTask,
|
||||
reservation: ReservedPartialDownloadFile
|
||||
|
||||
@@ -13,13 +13,10 @@
|
||||
|
||||
import axios, { AxiosRequestConfig } from 'axios';
|
||||
import { ipcMain } from 'electron';
|
||||
import { XTREAM_CLIENT_USER_AGENT } from '@iptvnator/shared/interfaces';
|
||||
import { UnsafeUrlError } from './url-safety';
|
||||
import { requestWithValidatedRedirects } from '../util/validated-axios';
|
||||
|
||||
// Some Xtream panels sit behind Cloudflare (or similar WAFs) configured to
|
||||
// challenge generic browser-looking User-Agents while allowlisting known
|
||||
// IPTV player clients. A VLC-style User-Agent reliably passes those checks.
|
||||
const PROBE_CLIENT_USER_AGENT = 'VLC/3.0.18 LibVLC/3.0.18';
|
||||
const PROBE_TIMEOUT_MS = 10000;
|
||||
|
||||
export interface StreamProbePayload {
|
||||
@@ -64,7 +61,7 @@ export async function runStreamProbe(
|
||||
// The playlist's own User-Agent wins when it has one: the default
|
||||
// below is a guess that merely gets past most WAFs, while that
|
||||
// one is what the server was configured to expect.
|
||||
'User-Agent': payload.userAgent?.trim() || PROBE_CLIENT_USER_AGENT,
|
||||
'User-Agent': payload.userAgent?.trim() || XTREAM_CLIENT_USER_AGENT,
|
||||
...(payload.referer?.trim()
|
||||
? { Referer: payload.referer.trim() }
|
||||
: {}),
|
||||
@@ -95,8 +92,7 @@ export async function runStreamProbe(
|
||||
);
|
||||
// A ranged GET opens a stream we never read — release it immediately.
|
||||
const responseBody = response.data as
|
||||
| { destroy?: () => void }
|
||||
| undefined;
|
||||
{ destroy?: () => void } | undefined;
|
||||
responseBody?.destroy?.();
|
||||
return {
|
||||
status: response.status,
|
||||
|
||||
@@ -8,6 +8,7 @@ import { ipcMain } from 'electron';
|
||||
import {
|
||||
PortalDebugEvent,
|
||||
XTREAM_CANCEL_SESSION,
|
||||
XTREAM_CLIENT_USER_AGENT,
|
||||
XTREAM_MAIN_PERFORMANCE_PHASE,
|
||||
normalizeXtreamServerUrl,
|
||||
} from '@iptvnator/shared/interfaces';
|
||||
@@ -20,11 +21,6 @@ import {
|
||||
} from './xtream-performance';
|
||||
import { cancelXtreamSessionRequests } from './xtream-session-cancellation';
|
||||
|
||||
// Some Xtream panels sit behind Cloudflare (or similar WAFs) configured to
|
||||
// challenge generic browser-looking User-Agents while allowlisting known
|
||||
// IPTV player clients. A VLC-style User-Agent reliably passes those checks.
|
||||
const XTREAM_CLIENT_USER_AGENT = 'VLC/3.0.18 LibVLC/3.0.18';
|
||||
|
||||
export default class XtreamEvents {
|
||||
static bootstrapXtreamEvents(): Electron.IpcMain {
|
||||
return ipcMain;
|
||||
|
||||
@@ -12,7 +12,7 @@ variants, contextual buttons, and theme-aware styling.
|
||||
- **Queue control (`apps/electron-backend/src/app/events/database/download-runtime.ts`)**
|
||||
`DownloadTask` mirrors a row of the shared `downloads` table (type `Download` in `libs/shared/database/src/lib/schema.ts`) plus transient cancel/pause/progress helpers (shared task types live in `download-task.ts`). Request validation and row creation live in `download-requests.ts`, while `downloads.events.ts` stays focused on IPC registration. `enqueueDownload()` pushes the task onto `downloadQueue` and triggers `processQueue()`. `processQueue()` keeps one active download, updates the row to `downloading`, and calls `startDownload()`. The byte transfer itself lives in `download-transfer.ts`, finalization and retained-partial persistence in `download-finalize.ts`, and the renderer update broadcast in `download-broadcast.ts`.
|
||||
- **Range-aware transfer (`download-transfer.ts`)**
|
||||
The transfer streams the response through the backend's validated Axios redirect helper instead of `electron-dl`. Headers (user agent, referer, origin) are persisted in `request_headers` and re-applied through the same allowlist when read back on retry/resume. Active pause/cancel operations abort the current request with `AbortController`; pause keeps the partial file and cancel removes it. Resume checks the existing `.part` size (rejecting anything that is not a regular file, so a symlink planted while paused is never followed) and sends `Range: bytes=<offset>-` plus `If-Range` with the stored entity validator. The first response's strong `ETag` (or `Last-Modified`) is persisted in `resume_validator` for exactly this purpose. A `206 Partial Content` answer must start at the requested offset (`Content-Range` is verified) before bytes are appended; any other 2xx answer — the server ignoring `Range`, or `If-Range` detecting that the remote file changed — restarts the transfer from byte zero over the same `.part` instead of failing the download.
|
||||
The transfer streams the response through the backend's validated Axios redirect helper instead of `electron-dl`. Headers (user agent, referer, origin) are persisted in `request_headers` and re-applied through the same allowlist when read back on retry/resume. Fresh Xtream movie and series-episode downloads propagate the playlist's configured headers, using its User-Agent when present and otherwise sharing the provider-compatible `XTREAM_CLIENT_USER_AGENT` used by Xtream API requests and stream probes. Retry, resume, and missing-file recovery resolve the owning playlist type and add that fallback to legacy Xtream rows without a stored User-Agent; known Stalker rows are left unchanged. Download rows deliberately outlive individually deleted playlists, so a headerless legacy row whose source no longer exists receives the same IPTV-player fallback because its original provider type cannot be recovered. Active pause/cancel operations abort the current request with `AbortController`; pause keeps the partial file and cancel removes it. Resume checks the existing `.part` size (rejecting anything that is not a regular file, so a symlink planted while paused is never followed). The first response's strong `ETag` (or `Last-Modified`) is persisted in `resume_validator`; only a partial carrying that validator may send `Range: bytes=<offset>-` plus `If-Range` and append bytes. A retained partial without a validator restarts from byte zero and overwrites its `.part`, so a changed remote representation can never be joined to an unverified prefix. A `206 Partial Content` answer must start at the requested offset (`Content-Range` is verified) before bytes are appended; any other 2xx answer — the server ignoring `Range`, or `If-Range` detecting that the remote file changed — restarts the transfer from byte zero over the same `.part` instead of failing the download.
|
||||
- **Destination collision policy**
|
||||
Existing destination files are never overwritten, inspected, or deleted.
|
||||
Before starting a new transfer, the backend atomically reserves a free
|
||||
@@ -23,11 +23,14 @@ variants, contextual buttons, and theme-aware styling.
|
||||
retained `.part` is renamed aside and finalized to the next free numbered
|
||||
destination (`Movie (1).mp4`) instead of resolving the collision by size or
|
||||
`unlink()`. Completion creates the final `filePath` from the `.part` without
|
||||
overwriting an existing file; cancel and ordinary transfer failures remove
|
||||
the `.part`, while finalization failures and completed-partial failures
|
||||
deliberately retain it (the row keeps `filePath` so a later retry can finish
|
||||
without re-downloading); pause and restart recovery keep it for a later
|
||||
resume. Re-downloading such a failed row from a detail page
|
||||
overwriting an existing file; cancel and non-recoverable transfer failures
|
||||
remove the `.part`, while finalization failures, completed-partial failures,
|
||||
and allowlisted network interruptions after bytes reached disk with a stored
|
||||
representation validator deliberately retain it (the row keeps `filePath`
|
||||
so a later retry can finish without re-downloading); pause and restart
|
||||
recovery keep partials, but a later retry starts over when no validator was
|
||||
available.
|
||||
Re-downloading such a failed row from a detail page
|
||||
(`DOWNLOADS_START`) deletes the retained `.part` before the row is reset.
|
||||
- **Derived file readiness and recovery**
|
||||
`DOWNLOADS_GET_LIST` and `DOWNLOADS_GET` inspect completed destinations on
|
||||
@@ -184,6 +187,7 @@ variants, contextual buttons, and theme-aware styling.
|
||||
- A `.part` that cannot be deleted (locked, permission denied) never loses its database path: cancel persists `canceled` while retaining `filePath` for later cleanup, and `DOWNLOADS_REMOVE` keeps the row and answers `success: false` (surfaced as a snackbar) so retrying the remove re-attempts the deletion once the lock is released.
|
||||
- Resume claims the row atomically (`paused` → `queued` as a conditional update) and the runtime queue rejects duplicate ids, so two rapid Resume clicks racing the status refresh can never produce two transfers for the same download.
|
||||
- A response that ends cleanly before the advertised representation size (for example a proxy that caps each response) is never committed as completed: the transfer fails with `Transfer ended before the advertised size` while retaining the `.part` and `filePath`, so a retry continues via Range from where it stopped.
|
||||
- An allowlisted mid-response network failure such as `ECONNRESET` is recoverable only when the response advertised a larger total and the `.part` contains valid incomplete bytes. This includes a validated `206` resume that drops before adding another byte. The failed row retains that partial and exposes a stable `DOWNLOAD_NETWORK_INTERRUPTED (<code>)` message without a URL; Retry continues through the same Range/If-Range validation. Pre-response failures, unknown stream errors, filesystem errors, empty fresh failures, and responses without a trustworthy total keep the generic failure path.
|
||||
- Retained `filePath`s recorded in the database stay usable after the user switches download folders — resume/retry of a retained row does not re-require the folder to be the current selection. Fresh downloads still authorize against the currently selected folder.
|
||||
- Startup recovery recognizes a finalization that crashed between creating the final file and committing the row (`downloading` row, no partial, final file present with the recorded size) and marks it `completed` instead of failing it and orphaning the file.
|
||||
- Pause/resume is covered end to end by `apps/electron-backend-e2e/src/downloads.e2e.ts`: a throttled Range-capable mock server verifies the paused `.part` on disk, the `Range`/`If-Range` resume request, and byte-exact assembly of the final file.
|
||||
|
||||
@@ -98,9 +98,17 @@ and private-network checks.
|
||||
|
||||
## User-Agent
|
||||
|
||||
Electron's `XTREAM_REQUEST` and `XTREAM_PROBE_URL` handlers
|
||||
(`apps/electron-backend/src/app/events/xtream.events.ts`) send a shared
|
||||
`XTREAM_CLIENT_USER_AGENT` constant on every outgoing request. Some Xtream
|
||||
Electron's `XTREAM_REQUEST` and stream-probe handlers plus fresh Xtream movie
|
||||
and series-episode download requests share the exported
|
||||
`XTREAM_CLIENT_USER_AGENT` fallback. A playlist's explicit User-Agent,
|
||||
Referer, and Origin are propagated to either download kind; the explicit
|
||||
User-Agent still wins over the fallback. Legacy
|
||||
download rows without a stored User-Agent receive the fallback when retrying,
|
||||
resuming, or recovering a missing completed file. Download rows intentionally
|
||||
survive individual source deletion; when the playlist row is already gone and
|
||||
its type can no longer be recovered, a headerless legacy download receives the
|
||||
same IPTV-player fallback, while a still-identifiable Stalker row remains
|
||||
unchanged. Some Xtream
|
||||
panels sit behind a WAF (e.g. Cloudflare) configured to challenge
|
||||
generic/incomplete browser-looking User-Agents while allowlisting known IPTV
|
||||
player clients; a player-style User-Agent (currently a VLC signature) avoids
|
||||
|
||||
@@ -88,6 +88,9 @@ describe('SerialDetailsComponent', () => {
|
||||
serverUrl: 'http://xtream.example',
|
||||
username: 'user',
|
||||
password: 'pass',
|
||||
userAgent: 'Provider Player/1.0',
|
||||
referrer: 'https://provider.test/player',
|
||||
origin: 'https://provider.test',
|
||||
});
|
||||
const fetchSerialDetailsWithMetadata = jest.fn();
|
||||
const cancelDetailsRequest = jest.fn();
|
||||
@@ -356,6 +359,14 @@ describe('SerialDetailsComponent', () => {
|
||||
],
|
||||
});
|
||||
expect(seasonContainer?.downloadsEnabled()).toBe(true);
|
||||
expect(seasonContainer?.xtreamDownloadContext()).toEqual({
|
||||
serverUrl: 'http://xtream.example',
|
||||
username: 'user',
|
||||
password: 'pass',
|
||||
userAgent: 'Provider Player/1.0',
|
||||
referrer: 'https://provider.test/player',
|
||||
origin: 'https://provider.test',
|
||||
});
|
||||
expect(seasonContainer?.downloadMetadataContext()).toEqual(
|
||||
expect.objectContaining({
|
||||
language: 'en',
|
||||
|
||||
@@ -260,6 +260,9 @@ export class SerialDetailsComponent implements OnInit, OnDestroy {
|
||||
serverUrl: playlist.serverUrl,
|
||||
username: playlist.username,
|
||||
password: playlist.password,
|
||||
userAgent: playlist.userAgent,
|
||||
referrer: playlist.referrer,
|
||||
origin: playlist.origin,
|
||||
}
|
||||
: null
|
||||
);
|
||||
|
||||
@@ -5,6 +5,7 @@ import { DownloadsService } from '@iptvnator/services';
|
||||
import { resolveXtreamVodPlaybackSource } from '@iptvnator/portal/xtream/data-access';
|
||||
import {
|
||||
getXtreamVodInfo,
|
||||
XTREAM_CLIENT_USER_AGENT,
|
||||
XtreamVodDetails,
|
||||
type XtreamVodInfo,
|
||||
} from '@iptvnator/shared/interfaces';
|
||||
@@ -179,7 +180,8 @@ export class VodDetailsDownloadsService {
|
||||
: textList(info?.director)?.map((name) => ({ name })),
|
||||
}),
|
||||
headers: {
|
||||
userAgent: playlist.userAgent,
|
||||
userAgent:
|
||||
playlist.userAgent?.trim() || XTREAM_CLIENT_USER_AGENT,
|
||||
referer: playlist.referrer,
|
||||
origin: playlist.origin,
|
||||
},
|
||||
|
||||
@@ -462,6 +462,9 @@ describe('VodDetailsRouteComponent — playback actions', () => {
|
||||
// came from — so the download would fetch the wrong movie.
|
||||
expect(stubs.startDownload).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
userAgent: 'VLC/3.0.18 LibVLC/3.0.18',
|
||||
}),
|
||||
playlistId: 'playlist-1',
|
||||
xtreamId: 650020,
|
||||
metadataSnapshot: expect.objectContaining({
|
||||
|
||||
@@ -399,6 +399,11 @@ describe('VodDetailsRouteComponent fallback actions', () => {
|
||||
);
|
||||
expect(startDownload).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
headers: {
|
||||
origin: 'https://origin.example',
|
||||
referer: 'https://referrer.example',
|
||||
userAgent: 'IPTVnator',
|
||||
},
|
||||
title: 'Catalog movie',
|
||||
posterUrl: 'https://example.com/catalog-poster.jpg',
|
||||
url: 'http://example.com/movie/650020.mp4',
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
export type XtreamPortalStatusType =
|
||||
| 'active'
|
||||
| 'inactive'
|
||||
| 'expired'
|
||||
| 'unavailable';
|
||||
'active' | 'inactive' | 'expired' | 'unavailable';
|
||||
|
||||
export interface XtreamPortalStatusResponseLike {
|
||||
user_info?: {
|
||||
@@ -19,6 +16,10 @@ export interface XtreamCredentialsFromUrl {
|
||||
|
||||
const XTREAM_API_ENDPOINT_PATTERN = /\/(?:get|player_api)\.php$/i;
|
||||
|
||||
// Some Xtream panels challenge generic Node HTTP clients while allowing
|
||||
// established IPTV players. Keep API, probe, and download requests aligned.
|
||||
export const XTREAM_CLIENT_USER_AGENT = 'VLC/3.0.18 LibVLC/3.0.18';
|
||||
|
||||
export function normalizeXtreamServerUrl(value: string): string {
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import {
|
||||
DownloadMetadataSnapshot,
|
||||
XTREAM_CLIENT_USER_AGENT,
|
||||
XtreamSerieEpisode,
|
||||
XtreamSerieEpisodeInfo,
|
||||
} from '@iptvnator/shared/interfaces';
|
||||
@@ -19,6 +20,9 @@ export interface XtreamEpisodeDownloadContext {
|
||||
serverUrl?: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
userAgent?: string;
|
||||
referrer?: string;
|
||||
origin?: string;
|
||||
}
|
||||
|
||||
/** Provider-neutral metadata already loaded by the detail host. */
|
||||
@@ -34,6 +38,11 @@ export interface XtreamEpisodeDownloadRequest {
|
||||
seriesXtreamId: number;
|
||||
seasonNumber: number;
|
||||
episodeNumber: number;
|
||||
headers: {
|
||||
userAgent: string;
|
||||
referer?: string;
|
||||
origin?: string;
|
||||
};
|
||||
metadataSnapshot?: DownloadMetadataSnapshot;
|
||||
}
|
||||
|
||||
@@ -150,6 +159,11 @@ export function buildXtreamEpisodeDownloadRequest(options: {
|
||||
seriesXtreamId: seriesId,
|
||||
seasonNumber,
|
||||
episodeNumber,
|
||||
headers: {
|
||||
userAgent: context.userAgent?.trim() || XTREAM_CLIENT_USER_AGENT,
|
||||
referer: context.referrer,
|
||||
origin: context.origin,
|
||||
},
|
||||
...(options.metadataContext
|
||||
? {
|
||||
metadataSnapshot: buildMetadataSnapshot(
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { XtreamSerieEpisode } from '@iptvnator/shared/interfaces';
|
||||
import {
|
||||
XTREAM_CLIENT_USER_AGENT,
|
||||
XtreamSerieEpisode,
|
||||
} from '@iptvnator/shared/interfaces';
|
||||
import {
|
||||
buildXtreamEpisodeDownloadRequest,
|
||||
getEpisodeDownloadId,
|
||||
@@ -76,6 +79,35 @@ describe('episode-download.util', () => {
|
||||
seriesXtreamId: 900,
|
||||
seasonNumber: 2,
|
||||
episodeNumber: 3,
|
||||
headers: {
|
||||
userAgent: XTREAM_CLIENT_USER_AGENT,
|
||||
referer: undefined,
|
||||
origin: undefined,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('forwards playlist request headers for the first episode transfer', () => {
|
||||
const request = buildXtreamEpisodeDownloadRequest({
|
||||
episode: episode({ id: '55' }),
|
||||
context: {
|
||||
serverUrl: 'http://host',
|
||||
username: 'u',
|
||||
password: 'p',
|
||||
userAgent: 'Provider Player/1.0',
|
||||
referrer: 'https://provider.test/player',
|
||||
origin: 'https://provider.test',
|
||||
},
|
||||
playlistId: 'pl-1',
|
||||
seriesId: 900,
|
||||
seriesTitle: 'Show',
|
||||
fallbackSeasonKey: '1',
|
||||
});
|
||||
|
||||
expect(request.headers).toEqual({
|
||||
userAgent: 'Provider Player/1.0',
|
||||
referer: 'https://provider.test/player',
|
||||
origin: 'https://provider.test',
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -157,6 +157,9 @@ describe('SeasonContainerComponent', () => {
|
||||
serverUrl: 'http://host',
|
||||
username: 'u',
|
||||
password: 'p',
|
||||
userAgent: 'Provider Player/1.0',
|
||||
referrer: 'https://provider.test/player',
|
||||
origin: 'https://provider.test',
|
||||
});
|
||||
fixture.componentRef.setInput('downloadMetadataContext', {
|
||||
language: 'en',
|
||||
@@ -170,6 +173,11 @@ describe('SeasonContainerComponent', () => {
|
||||
expect(downloadsServiceStub.startDownload).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
title: 'Signal House - S01E01 - Pilot',
|
||||
headers: {
|
||||
userAgent: 'Provider Player/1.0',
|
||||
referer: 'https://provider.test/player',
|
||||
origin: 'https://provider.test',
|
||||
},
|
||||
metadataSnapshot: expect.objectContaining({
|
||||
mediaKind: 'series',
|
||||
title: 'Signal House',
|
||||
|
||||
@@ -36,6 +36,7 @@ import {
|
||||
getEpisodeDownloadId,
|
||||
isStalkerEpisode,
|
||||
type SeasonContainerDownloadMetadataContext,
|
||||
type XtreamEpisodeDownloadContext,
|
||||
} from './episode-download.util';
|
||||
import {
|
||||
EPISODE_INFO_PLAY,
|
||||
@@ -51,11 +52,7 @@ import { SeasonTabsComponent } from './season-tabs.component';
|
||||
type EpisodeViewMode = 'grid' | 'list';
|
||||
const EPISODE_VIEW_MODE_KEY = 'iptvnator_episode_view_mode';
|
||||
|
||||
export interface SeasonContainerXtreamDownloadContext {
|
||||
serverUrl?: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
}
|
||||
export type SeasonContainerXtreamDownloadContext = XtreamEpisodeDownloadContext;
|
||||
|
||||
export type { SeasonContainerDownloadMetadataContext } from './episode-download.util';
|
||||
|
||||
|
||||
Reference in new issue
Block a user