fix(downloads): preserve 64-bit archive file identities on Windows

This commit is contained in:
4gray committed 2026-09-08 07:47:29 +02:00
1 parent 8da2631336
commit 3112dd6b4c
18 files changed
+216 -51

No files matched your search

+2 -1
View File
@@ -1051,7 +1051,8 @@ The same journal stores transfer-phase descriptor identity before truncation;
Resume checks it at open, and rejected replacements are preserved and detached
so Retry can reserve a fresh path. A synchronous completion-commit boundary
rejects late pause/cancel commands before awaited cleanup and persistence.
Archive ownership includes device, inode and positive creation time to reject
Archive ownership reads device/inode as BigInt and journals decimal strings
without losing 64-bit Windows file references, alongside positive creation time to reject
reused inodes after unlink; old proofs without creation time remain untrusted.
Fresh reservations atomically commit their row path/name and captured ownership
before the initial HTTP wait;
+2 -1
View File
@@ -1862,7 +1862,8 @@ The same journal stores transfer-phase descriptor identity before truncation;
Resume checks it at open, and rejected replacements are preserved and detached
so Retry can reserve a fresh path. A synchronous completion-commit boundary
rejects late pause/cancel commands before awaited cleanup and persistence.
Archive ownership includes device, inode and positive creation time to reject
Archive ownership reads device/inode as BigInt and journals decimal strings
without losing 64-bit Windows file references, alongside positive creation time to reject
reused inodes after unlink; old proofs without creation time remain untrusted.
Fresh reservations atomically commit their row path/name and captured ownership
before the initial HTTP wait;
@@ -400,6 +400,42 @@ test('@downloads @epg @xtream @electron downloads a completed archive into the l
)
);
expect(captured).toEqual([]);
const ownership = await app.electronApp.evaluate(
(_electron, { dependency, file, id, mediaPath }) => {
const Database = process
.getBuiltinModule('module')
.createRequire(dependency)(dependency);
const db = new Database(file);
try {
const proof = JSON.parse(
db
.prepare(
'SELECT proof FROM download_archive_finalizations WHERE download_id=?'
)
.get(id).proof
);
const stats = process
.getBuiltinModule('fs')
.lstatSync(mediaPath, { bigint: true });
return {
recorded: proof.finalIdentity,
actual: {
dev: String(stats.dev),
ino: String(stats.ino),
},
};
} finally {
db.close();
}
},
{
dependency: join(workspaceRoot, 'node_modules/better-sqlite3'),
file: join(dataDir, 'databases/iptvnator.db'),
id: row.id,
mediaPath: row.filePath,
}
);
expect(ownership.recorded).toMatchObject(ownership.actual);
// A failed completion status write must recover the proven file in place
// on a repeated EPG submission, without a duplicate transfer.
await app.electronApp.evaluate(
@@ -1,4 +1,5 @@
import { linkSync, lstatSync, rmdirSync, unlinkSync } from 'node:fs';
import { readArchiveStatsSync } from './download-catchup-stats';
import { linkSync, rmdirSync, unlinkSync } from 'node:fs';
import { dirname } from 'node:path';
import {
sameArchiveFileIdentity,
@@ -32,7 +33,7 @@ function cleanupCapture(
if (!path) return;
let file;
try {
file = lstatSync(path);
file = readArchiveStatsSync(path);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
}
@@ -45,7 +46,10 @@ function cleanupCapture(
} catch (error) {
if (
(error as NodeJS.ErrnoException).code !== 'EEXIST' ||
!sameArchiveFileIdentity(lstatSync(publicPath), file)
!sameArchiveFileIdentity(
readArchiveStatsSync(publicPath),
file
)
)
throw error;
}
@@ -1,4 +1,5 @@
import { link, lstat, mkdtemp, rename, rmdir, unlink } from 'node:fs/promises';
import { readArchiveStats } from './download-catchup-stats';
import { link, mkdtemp, rename, rmdir, unlink } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import {
sameArchiveFileIdentity,
@@ -16,7 +17,7 @@ export async function cleanupCatchupFile(
const captured = join(directory, 'entry');
try {
await rename(path, captured);
const stats = await lstat(captured);
const stats = await readArchiveStats(captured);
if (stats.isFile() && sameArchiveFileIdentity(stats, identity)) {
await unlink(captured);
} else {
@@ -49,7 +50,7 @@ export async function cleanupCatchupPartial(
const path = filePath + '.part';
try {
if (identity) await cleanupCatchupFile(path, identity);
await lstat(path);
await readArchiveStats(path);
return false;
} catch (error) {
return (error as NodeJS.ErrnoException).code === 'ENOENT';
@@ -1,3 +1,4 @@
import { archiveFileStats } from './download-catchup-stats';
import {
mkdtemp,
lstat,
@@ -103,8 +104,8 @@ describe('archive file promotion', () => {
).resolves.toEqual({
size: identity.size,
identity: expect.objectContaining({
dev: expect.any(Number),
ino: expect.any(Number),
dev: expect.any(String),
ino: expect.any(String),
}),
});
expect(await readFile(reservation.path, 'utf8')).toBe(
@@ -122,8 +123,8 @@ describe('archive file promotion', () => {
).resolves.toEqual({
size: identity.size,
identity: expect.objectContaining({
dev: expect.any(Number),
ino: expect.any(Number),
dev: expect.any(String),
ino: expect.any(String),
}),
});
expect(await readFile(reservation.path, 'utf8')).toBe(
@@ -159,7 +160,9 @@ describe('archive file promotion', () => {
lstat(reservation.path)
).rejects.toMatchObject({ code: 'ENOENT' });
} else {
const created = await lstat(reservation.path);
const created = archiveFileStats(
await lstat(reservation.path)
);
expect(created).toEqual(
expect.objectContaining({
dev: expected.dev,
@@ -1,7 +1,12 @@
import {
archiveFileStats,
readArchiveStats,
type ArchiveFileStats,
} from './download-catchup-stats';
import type { DownloadTask, CompletedPartialProgress } from './download-task';
import { cleanupCatchupFile } from './download-catchup-cleanup';
import { constants, type Stats } from 'node:fs';
import { link, lstat, open } from 'node:fs/promises';
import { constants } from 'node:fs';
import { link, open } from 'node:fs/promises';
import {
archiveFileIdentity,
sameArchiveFileIdentity,
@@ -10,7 +15,7 @@ import {
import type { ReservedPartialDownloadFile } from './download-file-path';
function verify(
stats: Stats,
stats: ArchiveFileStats,
identity: ArchiveFileIdentity,
size: number
): void {
@@ -41,7 +46,7 @@ export async function finalizeCatchupPartial(
};
checkInterruption();
if (!identity) throw new Error('Archive transfer identity is unavailable');
verify(await lstat(reservation.partialPath), identity, size);
verify(await readArchiveStats(reservation.partialPath), identity, size);
const source = await open(
reservation.partialPath,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)
@@ -49,7 +54,11 @@ export async function finalizeCatchupPartial(
let created: ArchiveFileIdentity | undefined;
let sourceClosed = false;
try {
verify(await source.stat(), identity, size);
verify(
archiveFileStats(await source.stat({ bigint: true })),
identity,
size
);
// A hardlink can become complete immediately: persist its expected
// identity before publishing it, while the verified partial still exists.
await recordProof?.(identity);
@@ -59,7 +68,7 @@ export async function finalizeCatchupPartial(
// The link we created belongs to the verified source, even if
// another writer replaces its public name before lstat completes.
created = identity;
const promoted = await lstat(reservation.path);
const promoted = await readArchiveStats(reservation.path);
verify(promoted, identity, size);
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
@@ -79,7 +88,7 @@ export async function finalizeCatchupPartial(
// source by pathname for this copy: it could have been replaced.
const target = await open(reservation.path, 'wx', 0o600);
try {
created = await target.stat();
created = archiveFileStats(await target.stat({ bigint: true }));
// For a copy, record the exclusively created target identity
// before the first byte, so a complete file never lacks proof.
await recordProof?.(created);
@@ -121,7 +130,7 @@ export async function finalizeCatchupPartial(
await source.close();
sourceClosed = true;
checkInterruption();
verify(await lstat(reservation.path), created, size);
verify(await readArchiveStats(reservation.path), created, size);
checkInterruption();
// Publication is verified. Fence new commands before awaited cleanup
// and the completion write; accepted commands were handled above.
@@ -153,7 +162,7 @@ export async function recoverCatchupCompletion(
const proof = task.catchupFinalized;
if (!proof || proof.filePath !== task.filePath) return null;
try {
const file = await lstat(proof.filePath);
const file = await readArchiveStats(proof.filePath);
return file.isFile() &&
sameArchiveFileIdentity(file, proof.identity) &&
file.size === proof.size
@@ -62,7 +62,7 @@ it('commits the reservation path and ownership together before transfer setup',
expect(JSON.parse(result.proofs[0].proof)).toMatchObject({
phase: 'transfer',
filePath: result.row.file_path,
partialIdentity: { dev: 1, ino: 2, birthtimeMs: 1000 },
partialIdentity: { dev: '1', ino: '2', birthtimeMs: 1000 },
});
});
@@ -1,10 +1,11 @@
import { readArchiveStatsSync } from './download-catchup-stats';
import { cleanupArchiveCapture } from './download-catchup-capture';
import { eq, inArray, sql } from 'drizzle-orm';
import { lstatSync } from 'node:fs';
import { isAbsolute } from 'node:path';
import * as schema from '../../database/schema';
import {
archiveFileIdentity,
isArchiveFileId,
sameArchiveFileIdentity,
type ArchiveFileIdentity,
} from './download-catchup-output';
@@ -147,8 +148,8 @@ function identity(value: unknown): value is ArchiveFileIdentity {
if (!value || typeof value !== 'object') return false;
const candidate = value as ArchiveFileIdentity;
return (
Number.isSafeInteger(candidate.dev) &&
Number.isSafeInteger(candidate.ino) &&
isArchiveFileId(candidate.dev) &&
isArchiveFileId(candidate.ino) &&
Number.isFinite(candidate.birthtimeMs) &&
candidate.birthtimeMs > 0
);
@@ -220,7 +221,7 @@ export function verifiedArchiveSize(
if (!proof || proof.phase === 'transfer' || proof.filePath !== filePath)
return null;
try {
const file = lstatSync(proof.filePath);
const file = readArchiveStatsSync(proof.filePath);
return file.isFile() &&
sameArchiveFileIdentity(file, proof.finalIdentity) &&
file.size === proof.size
@@ -1,20 +1,34 @@
import { archiveFileStats, readArchiveStats } from './download-catchup-stats';
import { constants, type WriteStream } from 'node:fs';
import { lstat, open } from 'node:fs/promises';
import { open } from 'node:fs/promises';
export interface ArchiveFileIdentity {
readonly dev: number;
readonly ino: number;
readonly dev: number | string;
readonly ino: number | string;
readonly birthtimeMs: number;
}
export function isArchiveFileId(value: unknown): value is number | string {
return typeof value === 'string'
? /^(0|[1-9][0-9]{0,19})$/.test(value) &&
BigInt(value) <= BigInt('18446744073709551615')
: typeof value === 'number' &&
Number.isSafeInteger(value) &&
value >= 0;
}
/** Inodes can be reused after unlink; creation time identifies the generation. */
export function sameArchiveFileIdentity(
a: ArchiveFileIdentity,
b: ArchiveFileIdentity
): boolean {
return (
a.dev === b.dev &&
a.ino === b.ino &&
isArchiveFileId(a.dev) &&
isArchiveFileId(a.ino) &&
isArchiveFileId(b.dev) &&
isArchiveFileId(b.ino) &&
String(a.dev) === String(b.dev) &&
String(a.ino) === String(b.ino) &&
Number.isFinite(a.birthtimeMs) &&
a.birthtimeMs > 0 &&
a.birthtimeMs === b.birthtimeMs
@@ -24,9 +38,15 @@ export function sameArchiveFileIdentity(
export function archiveFileIdentity(
file: ArchiveFileIdentity
): ArchiveFileIdentity {
if (!isArchiveFileId(file.dev) || !isArchiveFileId(file.ino))
throw new Error('Archive filesystem identity is unavailable');
if (!Number.isFinite(file.birthtimeMs) || file.birthtimeMs <= 0)
throw new Error('Archive filesystem creation time is unavailable');
return { dev: file.dev, ino: file.ino, birthtimeMs: file.birthtimeMs };
return {
dev: String(file.dev),
ino: String(file.ino),
birthtimeMs: file.birthtimeMs,
};
}
export class ArchivePartialReplacedError extends Error {
@@ -41,7 +61,7 @@ export async function openCatchupOutput(
expectedIdentity?: ArchiveFileIdentity,
beforeTruncate?: (identity: ArchiveFileIdentity) => Promise<void>
): Promise<{ stream: WriteStream; identity: ArchiveFileIdentity }> {
const before = await lstat(partialPath).catch(
const before = await readArchiveStats(partialPath).catch(
(error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return undefined;
throw error;
@@ -58,7 +78,7 @@ export async function openCatchupOutput(
0o600
);
try {
const current = await handle.stat();
const current = archiveFileStats(await handle.stat({ bigint: true }));
const identity = archiveFileIdentity(current);
// Descriptor identity also protects platforms without O_NOFOLLOW from
// a link/file replacement between lstat and open. Missing files use wx.
@@ -1,7 +1,9 @@
import {
readArchiveStatsSync,
type ArchiveFileStats,
} from './download-catchup-stats';
import { cleanupArchiveCapture } from './download-catchup-capture';
import {
lstatSync,
type Stats,
mkdtempSync,
renameSync,
linkSync,
@@ -45,10 +47,10 @@ function removeOwnedEntry(
identity: ArchiveFileIdentity,
recordCapture: (path: string) => void
): void {
const matches = (file: Stats, identity: ArchiveFileIdentity) =>
const matches = (file: ArchiveFileStats, identity: ArchiveFileIdentity) =>
file.isFile() && sameArchiveFileIdentity(file, identity);
try {
if (!matches(lstatSync(path), identity)) return;
if (!matches(readArchiveStatsSync(path), identity)) return;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return;
throw error;
@@ -58,7 +60,7 @@ function removeOwnedEntry(
try {
recordCapture(captured);
renameSync(path, captured);
if (matches(lstatSync(captured), identity)) {
if (matches(readArchiveStatsSync(captured), identity)) {
unlinkSync(captured);
} else {
try {
@@ -1,3 +1,4 @@
import { archiveFileStats } from './download-catchup-stats';
import { openCatchupOutput } from './download-catchup-output';
import {
mkdtemp,
@@ -87,7 +88,9 @@ it.each([false, true])(
code: 'ENOENT',
});
expect(
await lstat(join(directory, 'show (1).ts.part'))
archiveFileStats(
await lstat(join(directory, 'show (1).ts.part'))
)
).toEqual(
expect.objectContaining(
task.catchupExpectedPartialIdentity!
@@ -1,5 +1,5 @@
import { archiveFileStats, readArchiveStats } from './download-catchup-stats';
import { closeSync, fstatSync, openSync } from 'node:fs';
import { lstat } from 'node:fs/promises';
import { cleanupStoredCatchupPartial } from './download-catchup-removal';
import {
clearArchiveFinalization,
@@ -19,7 +19,7 @@ export async function reserveFreshCatchupTarget(
task: DownloadTask
) {
if (task.filePath) {
const partial = await lstat(`${task.filePath}.part`).catch(
const partial = await readArchiveStats(`${task.filePath}.part`).catch(
(error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return undefined;
throw error;
@@ -55,7 +55,7 @@ export async function reserveOwnedCatchupTarget(
const descriptor = openSync(partialPath, 'wx', 0o600);
try {
task.catchupExpectedPartialIdentity = archiveFileIdentity(
fstatSync(descriptor)
archiveFileStats(fstatSync(descriptor, { bigint: true }))
);
} finally {
closeSync(descriptor);
@@ -0,0 +1,52 @@
import { type BigIntStats } from 'node:fs';
import { archiveFileStats } from './download-catchup-stats';
import {
archiveFileIdentity,
sameArchiveFileIdentity,
} from './download-catchup-output';
import { parseArchiveFinalization } from './download-catchup-journal';
function stats(ino: string) {
return archiveFileStats({
dev: BigInt(1),
ino: BigInt(ino),
birthtimeMs: BigInt(1000),
birthtimeNs: BigInt('1000123456'),
size: BigInt(188),
nlink: BigInt(1),
isFile: () => true,
} as BigIntStats);
}
it('round-trips 64-bit Windows file references without losing ownership proof', () => {
const file = stats('18446744073709551000');
const identity = archiveFileIdentity(file);
const proof = parseArchiveFinalization(
JSON.stringify({
version: 1,
phase: 'transfer',
filePath: '/archive.ts',
partialIdentity: identity,
})
);
expect(proof?.partialIdentity.ino).toBe('18446744073709551000');
expect(sameArchiveFileIdentity(file, proof!.partialIdentity)).toBe(true);
expect(identity.birthtimeMs).toBe(1000.123456);
});
it('distinguishes adjacent file references that collide when converted to Number', () => {
const a = stats('18446744073709551000');
const b = stats('18446744073709551001');
expect(Number(a.ino)).toBe(Number(b.ino));
expect(sameArchiveFileIdentity(a, b)).toBe(false);
});
it('rejects already-rounded numeric ownership evidence', () => {
const rounded = {
dev: 1,
ino: Number('18446744073709551000'),
birthtimeMs: 1000,
};
expect(sameArchiveFileIdentity(rounded, rounded)).toBe(false);
expect(() => archiveFileIdentity(rounded)).toThrow(
'identity is unavailable'
);
});
@@ -0,0 +1,31 @@
import { lstatSync, type BigIntStats, type Stats } from 'node:fs';
import { lstat } from 'node:fs/promises';
/** Preserve 64-bit file IDs; Number stats can round Windows file references. */
export function archiveFileStats(file: BigIntStats | Stats) {
if (
(typeof file.dev === 'number' && !Number.isSafeInteger(file.dev)) ||
(typeof file.ino === 'number' && !Number.isSafeInteger(file.ino))
)
throw new Error('Archive filesystem identity was rounded');
const birthtimeMs =
'birthtimeNs' in file
? Number(file.birthtimeNs / BigInt(1000000000)) * 1000 +
Number(file.birthtimeNs % BigInt(1000000000)) / 1000000
: file.birthtimeMs;
return {
dev: String(file.dev),
ino: String(file.ino),
birthtimeMs,
size: Number(file.size),
nlink: Number(file.nlink),
isFile: () => file.isFile(),
};
}
export type ArchiveFileStats = ReturnType<typeof archiveFileStats>;
export async function readArchiveStats(path: string) {
return archiveFileStats(await lstat(path, { bigint: true }));
}
export function readArchiveStatsSync(path: string) {
return archiveFileStats(lstatSync(path, { bigint: true }));
}
@@ -129,7 +129,7 @@ describe('TS archive transfer', () => {
expect(filePath).toBe(path);
expect(identity).toEqual(
expect.objectContaining({
ino: (await stat(path + '.part')).ino,
ino: String((await stat(path + '.part')).ino),
})
);
expect(id).toBe(task.id);
@@ -192,8 +192,8 @@ describe('TS archive transfer', () => {
task.id,
path,
expect.objectContaining({
dev: expect.any(Number),
ino: expect.any(Number),
dev: expect.any(String),
ino: expect.any(String),
})
);
} finally {
@@ -1,3 +1,4 @@
import { readArchiveStatsSync } from './download-catchup-stats';
import { sameArchiveFileIdentity } from './download-catchup-output';
import {
cleanupStoredCatchupPartial,
@@ -10,7 +11,7 @@ import {
type ArchiveDownloadProof,
} from './download-catchup-journal';
import { inArray, sql } from 'drizzle-orm';
import { lstatSync, statSync } from 'node:fs';
import { statSync } from 'node:fs';
import { getDatabase } from '../../database/connection';
import * as schema from '../../database/schema';
import {
@@ -36,7 +37,7 @@ function hasReplacedArchivePartial(download: StaleDownload): boolean {
)
return false;
try {
const file = lstatSync(`${download.filePath}.part`);
const file = readArchiveStatsSync(`${download.filePath}.part`);
const expected = download.proof.partialIdentity;
return !file.isFile() || !sameArchiveFileIdentity(file, expected);
} catch (error) {
+2 -2
View File
@@ -100,8 +100,8 @@ that apply only to another remote transfer; retained cleanup failures keep their
journal. Remove, Clear completed and missing-file
re-download and repeated programme submissions use journal-backed private
capture for archive partial cleanup;
unknown or replaced entries are preserved. Ownership includes a positive file
creation timestamp as well as device/inode, so inode reuse after unlink cannot
unknown or replaced entries are preserved. Ownership reads device/inode as BigInt and journals decimal strings without
losing 64-bit Windows file references, alongside a positive file creation timestamp, so inode reuse after unlink cannot
bless a new entry; proofs lacking creation time remain untrusted. Fresh
reservations capture this identity from their exclusive creation descriptor and
commit it together with the downloads row path/name in one SQLite transaction