test(performance): harden formal capture validity

This commit is contained in:
4gray committed 2026-07-27 10:05:49 +02:00
1 parent 5891ac0673
commit 77e8fd92e3
14 files changed
+1127 -100

No files matched your search

@@ -0,0 +1,127 @@
import assert from 'node:assert/strict';
import { describe, it } from 'node:test';
import {
assessDatabaseWorkerRequestMetricsValidity,
DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON,
} from './database-worker-request-metrics-validity';
import { createCompleteTestMainCapture } from './m3u-import-report.test-helpers';
import type {
WorkerCaptureMetrics,
WorkerRequestPerformanceMetrics,
} from './m3u-refresh-cancellation-contract';
describe('database worker request-metrics validity', () => {
it('requires exactly two complete request samples in every measured run', () => {
const worker = completeWorker();
const validity = assessDatabaseWorkerRequestMetricsValidity([
iteration('warmup', 'warmup', [worker]),
iteration('run-01', 'measured', [worker]),
iteration('run-02', 'measured', [worker]),
iteration('diagnostic', 'diagnostic', [worker]),
]);
assert.deepEqual(validity, {
expectedRequestCount: 4,
invalidRequests: [],
measuredRunCount: 2,
validForComparison: true,
validMeasuredRunCount: 2,
validRequestCount: 4,
});
});
it('fails each measured run with missing ELD, ELU, CPU, or raw capture metrics', () => {
const validity = assessDatabaseWorkerRequestMetricsValidity([
invalidRequestIteration('run-eld', {
eventLoopDelay: null,
eventLoopDelayUnavailableReason:
'event-loop-delay-capture-unavailable',
histogramFlushedEpochMs: null,
}),
invalidRequestIteration('run-elu', {
eventLoopUtilization: null,
eventLoopUtilizationUnavailableReason:
'event-loop-utilization-unavailable',
}),
invalidRequestIteration('run-cpu', {
threadCpuSystemMicros: null,
threadCpuUnavailableReason: 'thread-cpu-usage-unavailable',
threadCpuUserMicros: null,
}),
invalidRequestIteration('run-capture', {
performanceCaptureUnavailableReason:
'worker-performance-capture-invalid',
}),
]);
assert.equal(validity.expectedRequestCount, 8);
assert.equal(validity.validRequestCount, 4);
assert.equal(validity.validMeasuredRunCount, 0);
assert.equal(validity.validForComparison, false);
assert.deepEqual(
validity.invalidRequests.map(({ reason }) => reason),
[
DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.EVENT_LOOP_DELAY_INVALID,
DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.EVENT_LOOP_UTILIZATION_INVALID,
DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.THREAD_CPU_INVALID,
DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.WORKER_CAPTURE_INVALID,
]
);
});
it('fails closed for missing, multiple, and wrong request sets', () => {
const worker = completeWorker();
const validity = assessDatabaseWorkerRequestMetricsValidity([
iteration('missing', 'measured', []),
iteration('multiple', 'measured', [worker, worker]),
iteration('wrong-requests', 'measured', [
{
...worker,
requests: worker.requests.slice(0, 1),
},
]),
]);
assert.equal(validity.expectedRequestCount, 6);
assert.equal(validity.validRequestCount, 0);
assert.equal(validity.validForComparison, false);
assert.deepEqual(
validity.invalidRequests.map(({ reason }) => reason),
[
DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.DATABASE_WORKER_MISSING,
DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.MULTIPLE_DATABASE_WORKERS,
DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.REQUEST_SET_INVALID,
]
);
});
});
function completeWorker(): WorkerCaptureMetrics {
const worker = createCompleteTestMainCapture().workers[0];
assert.ok(worker);
return worker;
}
function invalidRequestIteration(
runId: string,
updates: Partial<WorkerRequestPerformanceMetrics>
) {
const worker = completeWorker();
const get = worker.requests[1];
assert.ok(get);
return iteration(runId, 'measured', [
{
...worker,
requests: [worker.requests[0], { ...get, ...updates }],
},
]);
}
function iteration(
runId: string,
kind: string,
workers: readonly WorkerCaptureMetrics[]
) {
return { kind, main: { workers }, runId };
}
@@ -0,0 +1,197 @@
import type {
WorkerCaptureMetrics,
WorkerRequestPerformanceMetrics,
} from './m3u-refresh-cancellation-contract';
import {
PERFORMANCE_ITERATION_KIND,
PERFORMANCE_WORKER_KIND,
} from './m3u-refresh-cancellation-contract';
export const DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON = {
DATABASE_WORKER_MISSING: 'database-worker-missing',
EVENT_LOOP_DELAY_INVALID: 'event-loop-delay-invalid',
EVENT_LOOP_UTILIZATION_INVALID: 'event-loop-utilization-invalid',
MULTIPLE_DATABASE_WORKERS: 'multiple-database-workers',
REQUEST_SET_INVALID: 'initial-import-request-set-invalid',
THREAD_CPU_INVALID: 'thread-cpu-invalid',
WORKER_CAPTURE_INVALID: 'worker-performance-capture-invalid',
} as const;
export type DatabaseWorkerRequestMetricsInvalidReason =
(typeof DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON)[keyof typeof DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON];
export interface DatabaseWorkerRequestMetricsValidity {
readonly expectedRequestCount: number;
readonly invalidRequests: readonly {
readonly operation: string | null;
readonly reason: DatabaseWorkerRequestMetricsInvalidReason;
readonly runId: string;
}[];
readonly measuredRunCount: number;
readonly validForComparison: boolean;
readonly validMeasuredRunCount: number;
readonly validRequestCount: number;
}
interface RequestMetricsIteration {
readonly kind: string;
readonly main: {
readonly workers: readonly WorkerCaptureMetrics[];
};
readonly runId: string;
}
export function assessDatabaseWorkerRequestMetricsValidity(
iterations: readonly RequestMetricsIteration[]
): DatabaseWorkerRequestMetricsValidity {
const measured = iterations.filter(
(iteration) =>
iteration.kind === PERFORMANCE_ITERATION_KIND.MEASURED
);
const invalidRequests: {
operation: string | null;
reason: DatabaseWorkerRequestMetricsInvalidReason;
runId: string;
}[] = [];
let validMeasuredRunCount = 0;
let validRequestCount = 0;
for (const iteration of measured) {
const workers = iteration.main.workers.filter(
(worker) => worker.kind === PERFORMANCE_WORKER_KIND.DATABASE
);
if (workers.length === 0) {
invalidRequests.push({
operation: null,
reason: DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.DATABASE_WORKER_MISSING,
runId: iteration.runId,
});
continue;
}
const worker = workers.length === 1 ? workers[0] : null;
if (!worker) {
invalidRequests.push({
operation: null,
reason: DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.MULTIPLE_DATABASE_WORKERS,
runId: iteration.runId,
});
continue;
}
const requests = requireInitialImportRequestPair(worker);
if (requests === null) {
invalidRequests.push({
operation: null,
reason: DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.REQUEST_SET_INVALID,
runId: iteration.runId,
});
continue;
}
let runIsValid = true;
for (const request of requests) {
const reason = assessRequest(request);
if (reason === null) {
validRequestCount += 1;
} else {
runIsValid = false;
invalidRequests.push({
operation: request.operation,
reason,
runId: iteration.runId,
});
}
}
if (runIsValid) {
validMeasuredRunCount += 1;
}
}
const expectedRequestCount = measured.length * 2;
return Object.freeze({
expectedRequestCount,
invalidRequests: Object.freeze(invalidRequests),
measuredRunCount: measured.length,
validForComparison:
measured.length > 0 &&
validMeasuredRunCount === measured.length &&
validRequestCount === expectedRequestCount,
validMeasuredRunCount,
validRequestCount,
});
}
function requireInitialImportRequestPair(
worker: WorkerCaptureMetrics
): readonly [
WorkerRequestPerformanceMetrics,
WorkerRequestPerformanceMetrics,
] | null {
const upserts = worker.requests.filter(
(request) =>
request.operation === 'DB_UPSERT_APP_PLAYLIST' && request.success
);
const gets = worker.requests.filter(
(request) =>
request.operation === 'DB_GET_APP_PLAYLIST' && request.success
);
return worker.requests.length === 2 &&
upserts.length === 1 &&
upserts[0] &&
gets.length === 1 &&
gets[0]
? [upserts[0], gets[0]]
: null;
}
function assessRequest(
request: WorkerRequestPerformanceMetrics
): DatabaseWorkerRequestMetricsInvalidReason | null {
if (
request.performanceCaptureUnavailableReason !== null ||
request.invalidReason !== null
) {
return DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.WORKER_CAPTURE_INVALID;
}
if (
request.eventLoopDelayUnavailableReason !== null ||
request.histogramFlushedEpochMs === null ||
!isEventLoopDelay(request.eventLoopDelay)
) {
return DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.EVENT_LOOP_DELAY_INVALID;
}
if (
request.eventLoopUtilizationUnavailableReason !== null ||
!isUtilization(request.eventLoopUtilization)
) {
return DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.EVENT_LOOP_UTILIZATION_INVALID;
}
if (
request.threadCpuUnavailableReason !== null ||
!isFiniteNonNegative(request.threadCpuSystemMicros) ||
!isFiniteNonNegative(request.threadCpuUserMicros)
) {
return DATABASE_WORKER_REQUEST_METRICS_INVALID_REASON.THREAD_CPU_INVALID;
}
return null;
}
function isEventLoopDelay(
value: WorkerRequestPerformanceMetrics['eventLoopDelay']
): value is NonNullable<WorkerRequestPerformanceMetrics['eventLoopDelay']> {
return (
value !== null &&
isFiniteNonNegative(value.maxMs) &&
isFiniteNonNegative(value.p95Ms) &&
isFiniteNonNegative(value.p99Ms) &&
value.p95Ms <= value.p99Ms &&
value.p99Ms <= value.maxMs
);
}
function isUtilization(value: number | null): value is number {
return isFiniteNonNegative(value) && value <= 1;
}
function isFiniteNonNegative(value: unknown): value is number {
return typeof value === 'number' && Number.isFinite(value) && value >= 0;
}
@@ -1,9 +1,12 @@
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { mkdir, mkdtemp, rm, symlink } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { describe, it } from 'node:test';
import {
assertM3uImportOutputPathHasNoSymlinks,
assertM3uImportSourceState,
M3U_IMPORT_RENDERER_CDP_PORT,
resolveM3uImportRendererCdpPort,
@@ -73,6 +76,40 @@ describe('initial M3U import benchmark lifecycle', () => {
);
});
it('rejects an existing output ancestor symlink before artifact creation', async () => {
const fixtureRoot = await mkdtemp(
join(tmpdir(), 'iptvnator-m3u-output-path-')
);
const repositoryRoot = join(fixtureRoot, 'repository');
const performanceRoot = join(
repositoryRoot,
'dist',
'performance'
);
const outside = join(fixtureRoot, 'outside');
await mkdir(performanceRoot, { recursive: true });
await mkdir(outside);
await symlink(outside, join(performanceRoot, 'redirect'));
try {
await assert.rejects(
assertM3uImportOutputPathHasNoSymlinks(
repositoryRoot,
join(performanceRoot, 'redirect', 'run')
),
/Performance output path contains a symbolic link/
);
await assert.doesNotReject(
assertM3uImportOutputPathHasNoSymlinks(
repositoryRoot,
join(performanceRoot, 'ordinary', 'run')
)
);
} finally {
await rm(fixtureRoot, { force: true, recursive: true });
}
});
it('keeps formal CDP capture on 9222 while allowing an isolated smoke port', () => {
assert.equal(
resolveM3uImportRendererCdpPort(false, undefined),
@@ -122,4 +159,25 @@ describe('initial M3U import benchmark lifecycle', () => {
assert.match(source, /status\.databaseGetsCompleted\s*===\s*1/);
assert.match(source, /status\.databasePending\s*===\s*0/);
});
it('always disposes renderer capture before closing Electron on iteration failure', () => {
const source = readFileSync(
resolve(
process.cwd(),
'src/performance/m3u-import.benchmark.ts'
),
'utf8'
);
const runIteration = source.indexOf('async function runIteration');
const finallyBlock = source.indexOf('} finally {', runIteration);
const rendererDispose = source.indexOf(
'await renderer.dispose()',
finallyBlock
);
const appClose = source.indexOf('await closeElectronApp(app)', finallyBlock);
assert.ok(finallyBlock > runIteration);
assert.ok(rendererDispose > finallyBlock);
assert.ok(appClose > rendererDispose);
});
});
@@ -1,6 +1,6 @@
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { access, mkdir, readFile, writeFile } from 'node:fs/promises';
import { access, lstat, mkdir, readFile, writeFile } from 'node:fs/promises';
import { createServer as createTcpServer } from 'node:net';
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
@@ -63,6 +63,10 @@ export async function resolveM3uImportBenchmarkConfiguration(): Promise<M3uImpor
if (typeof electronPackage.version !== 'string') {
throw new Error('Unable to resolve the Electron runtime version');
}
await assertM3uImportOutputPathHasNoSymlinks(
workspaceRoot,
layout.outputRoot
);
await assertMissing(layout.variantDirectory);
await assertPerformanceArtifactCapacity(layout.variantDirectory);
await mkdir(layout.variantDirectory, { recursive: true });
@@ -102,6 +106,36 @@ export function resolveM3uImportOutputLayout(
return Object.freeze({ outputRoot, variantDirectory });
}
export async function assertM3uImportOutputPathHasNoSymlinks(
repositoryRoot: string,
outputRoot: string
): Promise<void> {
const root = resolve(repositoryRoot);
const output = resolve(outputRoot);
if (!isStrictDescendant(root, output)) {
throw new Error('Performance output path escapes the repository');
}
const components = relative(root, output).split(sep);
let current = root;
for (const component of ['', ...components]) {
current = component === '' ? current : join(current, component);
try {
const stats = await lstat(current);
if (stats.isSymbolicLink()) {
throw new Error(
`Performance output path contains a symbolic link: ${current}`
);
}
} catch (error) {
if (isMissingPath(error)) {
return;
}
throw error;
}
}
}
export function assertM3uImportSourceState(
smoke: boolean,
state: M3uImportGitSourceState
@@ -255,3 +289,12 @@ async function assertMissing(path: string): Promise<void> {
}
throw new Error(`Performance output already exists: ${path}`);
}
function isMissingPath(error: unknown): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
error.code === 'ENOENT'
);
}
@@ -10,12 +10,13 @@ interface TraceData {
}
export interface RendererArtifactCapture {
beginTraceOutput(): void;
readonly cpuProfilePath: string | null;
finishTraceOutput(): Promise<void>;
readonly heapSnapshotPath: string | null;
readonly onTraceComplete: () => void;
readonly onTraceData: (data: TraceData) => void;
readonly traceComplete: Promise<void>;
readonly traceOutput: ReturnType<typeof createWriteStream> | null;
readonly tracePath: string | null;
}
@@ -27,15 +28,46 @@ export function createRendererArtifactCapture(options: {
? join(options.outputDirectory, 'renderer.trace.json')
: null;
const traceOutput = tracePath ? createWriteStream(tracePath) : null;
let traceOutputOpened = false;
let traceOutputFinish: Promise<void> | null = null;
let traceFirstEvent = true;
let completeTrace: (() => void) | null = null;
const traceComplete = new Promise<void>((resolve) => {
completeTrace = resolve;
});
return {
beginTraceOutput: () => {
if (
traceOutput &&
!traceOutputOpened &&
traceOutputFinish === null
) {
traceOutput.write('{"traceEvents":[');
traceOutputOpened = true;
}
},
cpuProfilePath: options.diagnostic
? join(options.outputDirectory, 'renderer.cpuprofile')
: null,
finishTraceOutput: () => {
traceOutputFinish ??= (async () => {
if (!traceOutput) {
return;
}
if (!traceOutputOpened) {
traceOutput.write('{"traceEvents":[');
traceOutputOpened = true;
}
if (!traceOutput.writableEnded) {
traceOutput.write(']}');
traceOutput.end();
}
if (!traceOutput.writableFinished) {
await once(traceOutput, 'finish');
}
})();
return traceOutputFinish;
},
heapSnapshotPath: options.diagnostic
? join(options.outputDirectory, 'renderer.heapsnapshot')
: null,
@@ -44,6 +76,9 @@ export function createRendererArtifactCapture(options: {
completeTrace = null;
},
onTraceData: (data) => {
if (!traceOutputOpened || traceOutput?.writableEnded) {
return;
}
for (const event of data.value) {
traceOutput?.write(
`${traceFirstEvent ? '' : ','}${JSON.stringify(event)}`
@@ -52,7 +87,6 @@ export function createRendererArtifactCapture(options: {
}
},
traceComplete,
traceOutput,
tracePath,
};
}
@@ -63,7 +97,7 @@ export async function startRendererDiagnosticCapture(
): Promise<void> {
await session.send('Profiler.enable');
await session.send('Profiler.start');
artifacts.traceOutput?.write('{"traceEvents":[');
artifacts.beginTraceOutput();
session.on('Tracing.dataCollected', artifacts.onTraceData);
session.on('Tracing.tracingComplete', artifacts.onTraceComplete);
await session.send('Tracing.start', {
@@ -83,23 +117,29 @@ export async function stopRendererDiagnosticCapture(
session: CDPSession,
artifacts: RendererArtifactCapture
): Promise<void> {
const cpuProfile = await session.send('Profiler.stop');
await writeFile(
requirePath(artifacts.cpuProfilePath, 'renderer CPU profile'),
JSON.stringify(cpuProfile['profile']),
'utf8'
);
await session.send('Tracing.end');
await artifacts.traceComplete;
session.off('Tracing.dataCollected', artifacts.onTraceData);
session.off('Tracing.tracingComplete', artifacts.onTraceComplete);
artifacts.traceOutput?.write(']}');
artifacts.traceOutput?.end();
if (artifacts.traceOutput) {
await once(artifacts.traceOutput, 'finish');
try {
const cpuProfile = await session.send('Profiler.stop');
await writeFile(
requirePath(artifacts.cpuProfilePath, 'renderer CPU profile'),
JSON.stringify(cpuProfile['profile']),
'utf8'
);
await session.send('Tracing.end');
await artifacts.traceComplete;
} finally {
disposeRendererDiagnosticListeners(session, artifacts);
await artifacts.finishTraceOutput();
}
}
export async function disposeRendererDiagnosticCapture(
session: CDPSession,
artifacts: RendererArtifactCapture
): Promise<void> {
disposeRendererDiagnosticListeners(session, artifacts);
await artifacts.finishTraceOutput();
}
export async function takeRendererHeapSnapshot(
session: CDPSession,
path: string | null
@@ -111,12 +151,26 @@ export async function takeRendererHeapSnapshot(
output.write(chunk);
};
session.on('HeapProfiler.addHeapSnapshotChunk', onChunk);
await session.send('HeapProfiler.takeHeapSnapshot', {
reportProgress: false,
});
session.off('HeapProfiler.addHeapSnapshotChunk', onChunk);
output.end();
await once(output, 'finish');
try {
await session.send('HeapProfiler.takeHeapSnapshot', {
reportProgress: false,
});
} finally {
session.off('HeapProfiler.addHeapSnapshotChunk', onChunk);
output.end();
if (!output.writableFinished) {
await once(output, 'finish');
}
}
}
function disposeRendererDiagnosticListeners(
session: CDPSession,
artifacts: RendererArtifactCapture
): void {
session.off('Tracing.dataCollected', artifacts.onTraceData);
session.off('Tracing.tracingComplete', artifacts.onTraceComplete);
artifacts.onTraceComplete();
}
function requirePath(value: string | null, label: string): string {
@@ -0,0 +1,313 @@
/* eslint-disable playwright/expect-expect -- This is a Node assertion-based lifecycle test. */
import { RENDERER_PERFORMANCE_PHASE_HOOK_KEY } from '@iptvnator/shared/logging';
import assert from 'node:assert/strict';
import { EventEmitter } from 'node:events';
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import test from 'node:test';
import { startM3uImportRendererCapture } from './m3u-import-renderer-capture';
import { M3U_IMPORT_RENDERER_STATE_KEY } from './m3u-import-renderer-probe';
test('idempotently disposes failed-iteration timers, probe, trace stream, listeners, and CDP session', async () => {
const browser = installFakeProbeEnvironment();
const outputDirectory = await mkdtemp(
join(tmpdir(), 'iptvnator-renderer-capture-dispose-')
);
const session = new FakeCdpSession();
const control = {
click: async () => undefined,
fill: async () => undefined,
first: () => control,
getByRole: () => control,
isDisabled: async () => false,
last: () => control,
waitFor: async () => undefined,
};
const page = {
...browser.page,
context: () => ({
newCDPSession: async () => session,
}),
getByRole: () => control,
locator: () => control,
};
try {
const capture = await startM3uImportRendererCapture(
page as never,
{
diagnostic: true,
outputDirectory,
playlistTitle: 'Synthetic',
playlistUrl:
'http://127.0.0.1:43210/synthetic-performance.m3u',
}
);
assert.equal(session.listenerCount('Tracing.dataCollected'), 1);
assert.equal(session.listenerCount('Tracing.tracingComplete'), 1);
await capture.dispose();
await capture.dispose();
assert.equal(session.detachCount, 1);
assert.equal(session.listenerCount('Tracing.dataCollected'), 0);
assert.equal(session.listenerCount('Tracing.tracingComplete'), 0);
assert.equal(browser.readState(), undefined);
const heapSamplesAfterDispose = session.heapUsageCalls;
await new Promise((resolve) => setTimeout(resolve, 50));
assert.equal(session.heapUsageCalls, heapSamplesAfterDispose);
assert.deepEqual(
JSON.parse(
await readFile(
join(outputDirectory, 'renderer.trace.json'),
'utf8'
)
),
{ traceEvents: [] }
);
} finally {
browser.cleanup();
await rm(outputDirectory, { force: true, recursive: true });
}
});
test('failed-iteration disposal does not await a wedged renderer or CDP detach', async () => {
const browser = installFakeProbeEnvironment();
const outputDirectory = await mkdtemp(
join(tmpdir(), 'iptvnator-renderer-capture-wedged-')
);
const session = new FakeCdpSession({ hangDetach: true });
let evaluateCalls = 0;
const page = createFakePage(browser, session, async (callback, argument) => {
evaluateCalls += 1;
if (evaluateCalls > 1) {
return new Promise<never>(() => undefined);
}
return browser.page.evaluate(callback, argument);
});
try {
const capture = await startM3uImportRendererCapture(page as never, {
diagnostic: false,
outputDirectory,
playlistTitle: 'Synthetic',
playlistUrl:
'http://127.0.0.1:43210/synthetic-performance.m3u',
});
const outcome = await Promise.race([
capture.dispose().then(() => 'disposed'),
new Promise<'timeout'>((resolve) =>
setTimeout(() => resolve('timeout'), 150)
),
]);
assert.equal(outcome, 'disposed');
assert.equal(session.detachCount, 1);
const heapSamplesAfterDispose = session.heapUsageCalls;
await new Promise((resolve) => setTimeout(resolve, 50));
assert.equal(session.heapUsageCalls, heapSamplesAfterDispose);
} finally {
browser.cleanup();
await rm(outputDirectory, { force: true, recursive: true });
}
});
test('rolls back partially started capture when Chromium tracing rejects', async () => {
const browser = installFakeProbeEnvironment();
const outputDirectory = await mkdtemp(
join(tmpdir(), 'iptvnator-renderer-capture-startup-')
);
const session = new FakeCdpSession({ failMethod: 'Tracing.start' });
const intervalToken = Object.freeze({});
const setIntervalDescriptor = Object.getOwnPropertyDescriptor(
globalThis,
'setInterval'
);
const clearIntervalDescriptor = Object.getOwnPropertyDescriptor(
globalThis,
'clearInterval'
);
let clearIntervalCalls = 0;
Object.defineProperty(globalThis, 'setInterval', {
configurable: true,
value: () => intervalToken,
writable: true,
});
Object.defineProperty(globalThis, 'clearInterval', {
configurable: true,
value: (token: unknown) => {
if (token === intervalToken) {
clearIntervalCalls += 1;
}
},
writable: true,
});
try {
await assert.rejects(
startM3uImportRendererCapture(
createFakePage(browser, session) as never,
{
diagnostic: true,
outputDirectory,
playlistTitle: 'Synthetic',
playlistUrl:
'http://127.0.0.1:43210/synthetic-performance.m3u',
}
),
/synthetic Tracing\.start failure/
);
assert.equal(clearIntervalCalls, 1);
assert.equal(session.detachCount, 1);
assert.equal(session.listenerCount('Tracing.dataCollected'), 0);
assert.equal(session.listenerCount('Tracing.tracingComplete'), 0);
assert.equal(browser.readState(), undefined);
assert.deepEqual(
JSON.parse(
await readFile(
join(outputDirectory, 'renderer.trace.json'),
'utf8'
)
),
{ traceEvents: [] }
);
} finally {
restoreGlobal('setInterval', setIntervalDescriptor);
restoreGlobal('clearInterval', clearIntervalDescriptor);
browser.cleanup();
await rm(outputDirectory, { force: true, recursive: true });
}
});
class FakeCdpSession extends EventEmitter {
detachCount = 0;
heapUsageCalls = 0;
constructor(
private readonly options: {
readonly failMethod?: string;
readonly hangDetach?: boolean;
} = {}
) {
super();
}
async detach(): Promise<void> {
this.detachCount += 1;
if (this.options.hangDetach) {
return new Promise<never>(() => undefined);
}
}
async send(method: string): Promise<Record<string, unknown>> {
if (method === this.options.failMethod) {
throw new Error(`synthetic ${method} failure`);
}
if (method === 'Runtime.getHeapUsage') {
this.heapUsageCalls += 1;
return { usedSize: 1_024 };
}
return {};
}
}
function installFakeProbeEnvironment() {
const descriptors = new Map<string, PropertyDescriptor | undefined>();
let nextFrameId = 1;
const replaceGlobal = (key: string, value: unknown): void => {
descriptors.set(key, Object.getOwnPropertyDescriptor(globalThis, key));
Object.defineProperty(globalThis, key, {
configurable: true,
value,
writable: true,
});
};
class FakePerformanceObserver {
static readonly supportedEntryTypes = ['longtask'];
disconnect(): undefined {
return undefined;
}
observe(): undefined {
return undefined;
}
takeRecords(): PerformanceEntry[] {
return [];
}
}
replaceGlobal('performance', { now: () => 0, timeOrigin: 0 });
replaceGlobal('PerformanceObserver', FakePerformanceObserver);
replaceGlobal('cancelAnimationFrame', () => undefined);
replaceGlobal('requestAnimationFrame', () => nextFrameId++);
replaceGlobal('location', { pathname: '/' });
replaceGlobal('document', { querySelector: () => null });
return {
cleanup: () => {
for (const [key, descriptor] of descriptors) {
if (descriptor) {
Object.defineProperty(globalThis, key, descriptor);
} else {
Reflect.deleteProperty(globalThis, key);
}
}
Reflect.deleteProperty(
globalThis,
Symbol.for(RENDERER_PERFORMANCE_PHASE_HOOK_KEY)
);
Reflect.deleteProperty(
globalThis,
M3U_IMPORT_RENDERER_STATE_KEY
);
},
page: {
evaluate: async (
callback: (argument: never) => unknown,
argument: never
) => callback(argument),
},
readState: () =>
(globalThis as Record<string, unknown>)[
M3U_IMPORT_RENDERER_STATE_KEY
],
};
}
function createFakePage(
browser: ReturnType<typeof installFakeProbeEnvironment>,
session: FakeCdpSession,
evaluate = browser.page.evaluate
) {
const control = {
click: async () => undefined,
fill: async () => undefined,
first: () => control,
getByRole: () => control,
isDisabled: async () => false,
last: () => control,
waitFor: async () => undefined,
};
return {
...browser.page,
context: () => ({
newCDPSession: async () => session,
}),
evaluate,
getByRole: () => control,
locator: () => control,
};
}
function restoreGlobal(
key: 'clearInterval' | 'setInterval',
descriptor: PropertyDescriptor | undefined
): void {
if (descriptor) {
Object.defineProperty(globalThis, key, descriptor);
} else {
Reflect.deleteProperty(globalThis, key);
}
}
@@ -377,8 +377,13 @@ test('fails closed when any terminal proof or the probe itself is missing', asyn
test('clears the harness heap sampler even when the renderer probe is missing', () => {
const source = readSource(captureUrl);
assert.ok(source);
assert.match(
source,
/try \{\s*probe = await stopM3uImportRendererProbe\(page\);[\s\S]*finally \{\s*clearInterval\(heapSampleTimer\)/
const backgroundStop = source.indexOf(
'await stopBackgroundWork()',
source.indexOf('const stopCapture')
);
const probeStop = source.indexOf('probe = await stopProbe()', backgroundStop);
assert.match(source, /clearInterval\(heapSampleTimer\)/);
assert.ok(backgroundStop >= 0);
assert.ok(probeStop > backgroundStop);
});
@@ -3,6 +3,8 @@ import type { CDPSession, Page } from '@playwright/test';
import type { NumericDistribution } from './m3u-refresh-cancellation-contract';
import {
createRendererArtifactCapture,
disposeRendererDiagnosticCapture,
type RendererArtifactCapture,
startRendererDiagnosticCapture,
stopRendererDiagnosticCapture,
takeRendererHeapSnapshot,
@@ -50,6 +52,7 @@ export interface M3uImportRendererCaptureMetrics {
export interface RunningM3uImportRendererCapture {
readonly session: CDPSession;
dispose(): Promise<void>;
stop(): Promise<M3uImportRendererCaptureMetrics>;
triggerImport(): Promise<number>;
waitForTerminal(): Promise<void>;
@@ -65,10 +68,9 @@ export async function startM3uImportRendererCapture(
): Promise<RunningM3uImportRendererCapture> {
await prepareM3uImportDialog(page, options);
const session = await page.context().newCDPSession(page);
await session.send('Runtime.enable');
await session.send('Performance.enable');
await installM3uImportRendererProbe(page);
let probeInstalled = false;
let heapSampleTimer: NodeJS.Timeout | null = null;
let artifacts: RendererArtifactCapture | null = null;
const heapSamples: number[] = [];
let heapSampleError: unknown = null;
let heapSamplePromise: Promise<void> | null = null;
@@ -89,13 +91,38 @@ export async function startM3uImportRendererCapture(
})();
return heapSamplePromise;
};
await sampleHeap();
throwCaptureError(heapSampleError, 'Renderer heap sampling failed');
const heapSampleTimer = setInterval(() => void sampleHeap(), 20);
const artifacts = createRendererArtifactCapture(options);
if (options.diagnostic) {
await startRendererDiagnosticCapture(session, artifacts);
try {
await session.send('Runtime.enable');
await session.send('Performance.enable');
await installM3uImportRendererProbe(page);
probeInstalled = true;
await sampleHeap();
throwCaptureError(heapSampleError, 'Renderer heap sampling failed');
heapSampleTimer = setInterval(() => void sampleHeap(), 20);
artifacts = createRendererArtifactCapture(options);
if (options.diagnostic) {
await startRendererDiagnosticCapture(session, artifacts);
}
} catch (error) {
if (heapSampleTimer) {
clearInterval(heapSampleTimer);
heapSampleTimer = null;
}
if (artifacts && options.diagnostic) {
await disposeRendererDiagnosticCapture(
session,
artifacts
).catch(() => undefined);
}
if (probeInstalled) {
continueBestEffort(() => stopM3uImportRendererProbe(page));
}
continueBestEffort(() => session.detach());
throw error;
}
if (!artifacts || !heapSampleTimer) {
throw new Error('M3U import renderer capture initialization failed');
}
let heartbeatDeadlineEpochMs: number | null = null;
@@ -103,6 +130,24 @@ export async function startM3uImportRendererCapture(
let heartbeatInFlight: Promise<void> | null = null;
let heartbeatTimer: NodeJS.Timeout | null = null;
let stopped = false;
let backgroundStopPromise: Promise<void> | null = null;
let probeStopPromise: Promise<M3uImportRendererProbeMetrics> | null = null;
let sessionDetachPromise: Promise<void> | null = null;
let abortPromise: Promise<void> | null = null;
let stopPromise: Promise<M3uImportRendererCaptureMetrics> | null = null;
let stopRequested = false;
let disposeRequested = false;
const stopSchedulingBackgroundWork = (): void => {
stopped = true;
if (heartbeatTimer) {
clearTimeout(heartbeatTimer);
heartbeatTimer = null;
}
if (heapSampleTimer) {
clearInterval(heapSampleTimer);
heapSampleTimer = null;
}
};
const scheduleHeartbeat = (): void => {
if (stopped || heartbeatDeadlineEpochMs === null) {
return;
@@ -131,67 +176,118 @@ export async function startM3uImportRendererCapture(
Math.max(0, deadline - Date.now())
);
};
const stopBackgroundWork = (): Promise<void> => {
backgroundStopPromise ??= (async () => {
stopSchedulingBackgroundWork();
await heartbeatInFlight;
await heapSamplePromise;
await sampleHeap();
})();
return backgroundStopPromise;
};
const stopProbe = (): Promise<M3uImportRendererProbeMetrics> => {
probeStopPromise ??= stopM3uImportRendererProbe(page);
return probeStopPromise;
};
const detachSession = (): Promise<void> => {
sessionDetachPromise ??= session.detach();
return sessionDetachPromise;
};
const abortResources = (): Promise<void> => {
abortPromise ??= (async () => {
stopSchedulingBackgroundWork();
continueBestEffort(() => heartbeatInFlight ?? Promise.resolve());
continueBestEffort(() => heapSamplePromise ?? Promise.resolve());
continueBestEffort(stopProbe);
if (options.diagnostic) {
await disposeRendererDiagnosticCapture(
session,
artifacts
).catch(() => undefined);
}
continueBestEffort(detachSession);
})();
return abortPromise;
};
const stopCapture =
async (): Promise<M3uImportRendererCaptureMetrics> => {
try {
await stopBackgroundWork();
let probe: M3uImportRendererProbeMetrics | null = null;
let probeError: unknown = null;
try {
probe = await stopProbe();
} catch (error) {
probeError = error;
}
if (options.diagnostic) {
await stopRendererDiagnosticCapture(session, artifacts);
}
await session.send('HeapProfiler.enable');
await session.send('HeapProfiler.collectGarbage');
const postGc = (await session.send(
'Runtime.getHeapUsage'
)) as HeapUsageResult;
if (options.diagnostic) {
await takeRendererHeapSnapshot(
session,
artifacts.heapSnapshotPath
);
}
await detachSession();
throwCaptureError(
heapSampleError,
'Renderer heap sampling failed'
);
throwCaptureError(heartbeatError, 'Renderer heartbeat failed');
throwCaptureError(probeError, 'Renderer probe stop failed');
assertCompleteM3uImportRendererProbe(probe);
return Object.freeze({
cpuProfilePath: artifacts.cpuProfilePath,
frameGap: summarizeNumbers(probe.frameGapsMs),
heapSnapshotPath: artifacts.heapSnapshotPath,
heartbeatDelay: summarizeNumbers(
probe.heartbeatDelaysMs
),
longTask: summarizeNumbers(probe.longTasksMs),
peakHeapUsedBytes: Math.max(0, ...heapSamples),
phaseTimestamps: Object.freeze({
firstChannelVisibleEpochMs:
probe.firstChannelVisibleEpochMs,
operationStartEpochMs: probe.operationStartEpochMs,
routeReadyEpochMs: probe.routeReadyEpochMs,
terminalEpochMs: probe.terminalEpochMs,
uiPaintedEpochMs: probe.uiPaintedEpochMs,
}),
postGcHeapUsedBytes: postGc.usedSize,
probe,
tracePath: artifacts.tracePath,
});
} catch (error) {
await abortResources();
throw error;
}
};
return Object.freeze({
dispose: async () => {
disposeRequested = true;
if (stopPromise) {
continueBestEffort(() => stopPromise as Promise<unknown>);
}
await abortResources();
},
session,
stop: async () => {
if (stopped) {
throw new Error('M3U import renderer capture already stopped');
}
stopped = true;
if (heartbeatTimer) {
clearTimeout(heartbeatTimer);
}
await heartbeatInFlight;
let probe: M3uImportRendererProbeMetrics | null = null;
let probeError: unknown = null;
try {
probe = await stopM3uImportRendererProbe(page);
} catch (error) {
probeError = error;
} finally {
clearInterval(heapSampleTimer);
await sampleHeap();
}
if (options.diagnostic) {
await stopRendererDiagnosticCapture(session, artifacts);
}
await session.send('HeapProfiler.enable');
await session.send('HeapProfiler.collectGarbage');
const postGc = (await session.send(
'Runtime.getHeapUsage'
)) as HeapUsageResult;
if (options.diagnostic) {
await takeRendererHeapSnapshot(
session,
artifacts.heapSnapshotPath
stop: () => {
if (stopRequested || disposeRequested) {
return Promise.reject(
new Error('M3U import renderer capture already stopped')
);
}
await session.detach();
throwCaptureError(heapSampleError, 'Renderer heap sampling failed');
throwCaptureError(heartbeatError, 'Renderer heartbeat failed');
throwCaptureError(probeError, 'Renderer probe stop failed');
assertCompleteM3uImportRendererProbe(probe);
return Object.freeze({
cpuProfilePath: artifacts.cpuProfilePath,
frameGap: summarizeNumbers(probe.frameGapsMs),
heapSnapshotPath: artifacts.heapSnapshotPath,
heartbeatDelay: summarizeNumbers(probe.heartbeatDelaysMs),
longTask: summarizeNumbers(probe.longTasksMs),
peakHeapUsedBytes: Math.max(0, ...heapSamples),
phaseTimestamps: Object.freeze({
firstChannelVisibleEpochMs:
probe.firstChannelVisibleEpochMs,
operationStartEpochMs: probe.operationStartEpochMs,
routeReadyEpochMs: probe.routeReadyEpochMs,
terminalEpochMs: probe.terminalEpochMs,
uiPaintedEpochMs: probe.uiPaintedEpochMs,
}),
postGcHeapUsedBytes: postGc.usedSize,
probe,
tracePath: artifacts.tracePath,
});
stopRequested = true;
stopPromise = stopCapture();
return stopPromise;
},
triggerImport: async () => {
const operationStartEpochMs = await triggerM3uImport(page);
@@ -257,6 +353,14 @@ function throwCaptureError(error: unknown, label: string): void {
}
}
function continueBestEffort(work: () => Promise<unknown>): void {
try {
void work().catch(() => undefined);
} catch {
// The owning Electron app is closed immediately after abort cleanup.
}
}
function isPositiveFinite(value: unknown): value is number {
return typeof value === 'number' && Number.isFinite(value) && value > 0;
}
@@ -298,7 +298,10 @@ export function createCompleteTestMainCapture(
eventLoopDelayUnavailableReason: null,
eventLoopUtilization: 0.8,
eventLoopUtilizationUnavailableReason: null,
histogramFlushedEpochMs: 1134,
invalidReason: null,
threadCpuSystemMicros: 40,
threadCpuUnavailableReason: null,
threadCpuUserMicros: 50,
},
{
@@ -307,7 +310,10 @@ export function createCompleteTestMainCapture(
eventLoopDelayUnavailableReason: null,
eventLoopUtilization: 0.9,
eventLoopUtilizationUnavailableReason: null,
histogramFlushedEpochMs: 1163,
invalidReason: null,
threadCpuSystemMicros: 60,
threadCpuUnavailableReason: null,
threadCpuUserMicros: 70,
},
],
@@ -155,3 +155,81 @@ it('summarizes measured runs only while preserving per-process memory boundaries
/m3u-import-summary-schedule-invalid/
);
});
it('fails comparison when any measured worker request omits mandatory event-loop or CPU metrics', () => {
const completeMain = createCompleteTestMainCapture();
const databaseWorker = completeMain.workers[0];
const get = databaseWorker?.requests[1];
assert.ok(databaseWorker);
assert.ok(get);
const invalidMain = {
...completeMain,
workers: [
{
...databaseWorker,
requests: [
databaseWorker.requests[0],
{
...get,
eventLoopDelay: null,
eventLoopDelayUnavailableReason:
'event-loop-delay-capture-unavailable',
histogramFlushedEpochMs: null,
},
],
},
],
};
const warmup = createM3uImportIterationResult({
channelCount: TEST_M3U_CHANNEL_COUNT,
fixtureBytes: TEST_M3U_FIXTURE_BYTES,
kind: PERFORMANCE_ITERATION_KIND.WARMUP,
main: createCompleteTestMainCapture(),
renderer: createCompleteTestRendererCapture(),
runId: 'warmup-01',
scenarioId: 'm3u-import-10k',
});
const measured = createM3uImportIterationResult({
channelCount: TEST_M3U_CHANNEL_COUNT,
fixtureBytes: TEST_M3U_FIXTURE_BYTES,
kind: PERFORMANCE_ITERATION_KIND.MEASURED,
main: invalidMain,
renderer: createCompleteTestRendererCapture(),
runId: 'run-01',
scenarioId: 'm3u-import-10k',
});
const diagnostic = createM3uImportIterationResult({
channelCount: TEST_M3U_CHANNEL_COUNT,
fixtureBytes: TEST_M3U_FIXTURE_BYTES,
kind: PERFORMANCE_ITERATION_KIND.DIAGNOSTIC,
main: createCompleteTestMainCapture(),
renderer: createCompleteTestRendererCapture(),
runId: 'diagnostic',
scenarioId: 'm3u-import-10k',
});
const summary = createM3uImportBenchmarkSummary(
createTestM3uImportManifest(),
[warmup, measured, diagnostic]
);
assert.equal(
summary.validity.databaseWorkerRequestMetrics.validForComparison,
false
);
assert.equal(
summary.validity.databaseWorkerRequestMetrics.expectedRequestCount,
2
);
assert.equal(
summary.validity.databaseWorkerRequestMetrics.validRequestCount,
1
);
assert.equal(
summary.measured.databaseWorkerGetEventLoopDelayP95Ms.count,
0
);
assert.throws(
() => assertM3uImportComparisonValidity(summary),
/Database worker request metrics are invalid/
);
});
@@ -24,6 +24,10 @@ import {
type DatabaseWorkerPeakMemoryValidity,
} from './database-worker-peak-memory-validity';
import { assessDatabaseWorkerPostGcValidity } from './database-worker-post-gc-validity';
import {
assessDatabaseWorkerRequestMetricsValidity,
type DatabaseWorkerRequestMetricsValidity,
} from './database-worker-request-metrics-validity';
import {
assessRendererRssValidity,
type RendererRssValidity,
@@ -140,6 +144,7 @@ export interface M3uImportBenchmarkSummary {
readonly validity: {
readonly databaseWorkerPeakMemory: DatabaseWorkerPeakMemoryValidity;
readonly databaseWorkerPostGc: DatabaseWorkerPostGcValidity;
readonly databaseWorkerRequestMetrics: DatabaseWorkerRequestMetricsValidity;
readonly rendererRss: RendererRssValidity;
};
}
@@ -390,6 +395,8 @@ export function createM3uImportBenchmarkSummary(
assessDatabaseWorkerPeakMemoryValidity(iterations),
databaseWorkerPostGc:
assessDatabaseWorkerPostGcValidity(iterations),
databaseWorkerRequestMetrics:
assessDatabaseWorkerRequestMetricsValidity(iterations),
rendererRss: assessRendererRssValidity(iterations),
}),
});
@@ -31,7 +31,10 @@ import {
type M3uImportBenchmarkSummary,
type M3uImportIterationResult,
} from './m3u-import-summary';
import { startM3uImportRendererCapture } from './m3u-import-renderer-capture';
import {
startM3uImportRendererCapture,
type RunningM3uImportRendererCapture,
} from './m3u-import-renderer-capture';
import {
installMainCapture,
readMainCaptureStatus,
@@ -142,6 +145,13 @@ export function assertM3uImportComparisonValidity(
)}`
);
}
if (!summary.validity.databaseWorkerRequestMetrics.validForComparison) {
throw new Error(
`Database worker request metrics are invalid for comparison: ${JSON.stringify(
summary.validity.databaseWorkerRequestMetrics.invalidRequests
)}`
);
}
}
async function runIteration(
@@ -158,6 +168,7 @@ async function runIteration(
join(tmpdir(), 'iptvnator-m3u-import-performance-')
);
let app: LaunchedElectronApp | null = null;
let renderer: RunningM3uImportRendererCapture | null = null;
try {
await assertTcpPortAvailable(config.rendererCdpPort);
@@ -186,7 +197,7 @@ async function runIteration(
const requestsBefore = server.requestCount();
const diagnostic =
definition.kind === PERFORMANCE_ITERATION_KIND.DIAGNOSTIC;
const renderer = await startM3uImportRendererCapture(app.mainWindow, {
renderer = await startM3uImportRendererCapture(app.mainWindow, {
diagnostic,
outputDirectory: iterationDirectory,
playlistTitle: `Synthetic M3U import ${definition.channelCount}`,
@@ -229,6 +240,14 @@ async function runIteration(
await writeJson(join(iterationDirectory, 'result.json'), result);
return result;
} finally {
if (renderer) {
await renderer.dispose().catch((error: unknown) => {
console.error(
'[performance] Renderer capture cleanup failed',
error
);
});
}
if (app) {
await closeElectronApp(app).catch((error: unknown) => {
console.error('[performance] Electron cleanup failed', error);
+13 -4
View File
@@ -89,9 +89,10 @@ Smoke runs may use a dirty worktree and validate only the harness; they cannot
support a performance claim. If another local application owns port 9222, smoke
only may set `IPTVNATOR_PERF_CDP_PORT` to an unused loopback port; formal runs
fail closed unless CDP uses `127.0.0.1:9222`. Raw captures and JSON results stay
under the gitignored `dist/performance/` tree. Headline distributions contain
only the five measured runs: warm-up and diagnostic profiles are never mixed
into them.
under the gitignored `dist/performance/` tree; preflight rejects a symbolic
link in any existing output-path component before creating artifacts. Headline
distributions contain only the five measured runs: warm-up and diagnostic
profiles are never mixed into them.
The benchmark attributes these non-additive intervals:
@@ -129,7 +130,15 @@ must not scan or log playlist payloads to generate metadata. Renderer
long-task, frame-gap, and heartbeat samples are clipped to the measured
operation boundary. Formal comparison fails closed after writing raw results
if exact-window renderer RSS, database-worker peak heap/external samples, or
the database worker's explicit post-GC heap is missing or incoherent.
the database worker's explicit post-GC heap is missing or incoherent. Both
initial-import database requests in every measured run must also contain
coherent event-loop delay, event-loop utilization, and thread-CPU metrics;
the validity record exposes the exact expected and valid request counts rather
than silently dropping nullable samples. An iteration failure idempotently
stops renderer timers, closes trace listeners/output, and starts best-effort
probe/session teardown without waiting on a wedged renderer before Electron is
closed. A partial capture-start failure performs the same rollback before it
escapes to the benchmark lifecycle.
Diagnostic artifacts require separate renderer, main, and database-worker CPU
profiles plus renderer/main/database-worker heap snapshots and a Chromium
trace; raw profiles remain ignored.
+7
View File
@@ -172,6 +172,13 @@ explicit sum is `ipcStructuredCloneProxyMs`. See
[M3U Playlist Module Architecture](./m3u-playlist-module.md#initial-url-import-performance-benchmark-electron)
for the complete cross-process attribution.
Formal initial-import comparison also requires both request-scoped captures in
every measured run to have coherent event-loop delay, event-loop utilization,
and worker-thread CPU values with no unavailable or invalid reason. Summary
validity records the exact expected and valid request counts; nullable metrics
remain in raw results but cannot be silently omitted from comparison
distributions.
The main-process benchmark samples the database worker's V8
`used_heap_size` and `external_memory` independently. Raw output includes a
valid-sample count for each metric. A peak is numeric only after at least one