diff --git a/apps/electron-backend-e2e/src/electron-process-lifecycle.ts b/apps/electron-backend-e2e/src/electron-process-lifecycle.ts new file mode 100644 index 000000000..51841fcd1 --- /dev/null +++ b/apps/electron-backend-e2e/src/electron-process-lifecycle.ts @@ -0,0 +1,171 @@ +import type { ChildProcess } from 'node:child_process'; + +type ElectronChildProcess = Pick< + ChildProcess, + 'exitCode' | 'kill' | 'once' | 'removeListener' | 'signalCode' +>; + +export interface ClosableElectronApplication { + close(): Promise; + process(): ElectronChildProcess; +} + +export interface ElectronExitConfirmationOptions { + readonly closeTimeoutMs: number; + readonly exitTimeoutMs: number; +} + +export interface PrepareElectronApplicationOptions { + readonly application: Application; + readonly dispose: (application: Application) => Promise; + readonly prepare: (application: Application) => Promise; +} + +export class ElectronApplicationDisposalError extends AggregateError { + constructor(failure: unknown, disposeFailure: unknown) { + super( + [failure, disposeFailure], + 'Electron launch preparation and disposal both failed', + { cause: failure } + ); + this.name = 'ElectronApplicationDisposalError'; + } +} + +export async function prepareElectronApplication( + options: PrepareElectronApplicationOptions +): Promise { + try { + return await options.prepare(options.application); + } catch (failure) { + try { + await options.dispose(options.application); + } catch (disposeFailure) { + throw new ElectronApplicationDisposalError(failure, disposeFailure); + } + throw failure; + } +} + +export async function closeElectronApplicationAndConfirmExit( + application: ClosableElectronApplication, + options: ElectronExitConfirmationOptions +): Promise { + assertTimeout(options.closeTimeoutMs); + assertTimeout(options.exitTimeoutMs); + const child = application.process(); + if (hasExited(child)) return; + + const exit = observeProcessExit(child); + let failure: unknown; + try { + const close = Promise.resolve() + .then(() => application.close()) + .then( + () => ({ kind: 'close-resolved' as const }), + (closeFailure: unknown) => ({ + failure: closeFailure, + kind: 'close-rejected' as const, + }) + ); + const first = await raceCloseAndExit( + close, + exit.promise, + options.closeTimeoutMs + ); + if (first.kind === 'exit') return; + if ( + first.kind === 'close-resolved' && + (await resolvesWithin(exit.promise, options.exitTimeoutMs)) + ) { + return; + } + if (first.kind === 'close-rejected') failure = first.failure; + + try { + child.kill(); + } catch (killFailure) { + failure = failure + ? new AggregateError([failure, killFailure]) + : killFailure; + } + if (await resolvesWithin(exit.promise, options.exitTimeoutMs)) return; + } finally { + exit.cancel(); + } + throw new Error('electron-process-exit-unconfirmed', { cause: failure }); +} + +function hasExited(child: ElectronChildProcess): boolean { + return child.exitCode !== null || child.signalCode !== null; +} + +function observeProcessExit(child: ElectronChildProcess): { + readonly cancel: () => void; + readonly promise: Promise; +} { + let resolveExit: (() => void) | undefined; + const onExit = () => resolveExit?.(); + const promise = new Promise((resolvePromise) => { + resolveExit = resolvePromise; + child.once('exit', onExit); + if (hasExited(child)) resolvePromise(); + }); + return { + cancel: () => child.removeListener('exit', onExit), + promise, + }; +} + +async function resolvesWithin( + promise: Promise, + timeoutMs: number +): Promise { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise.then(() => true), + new Promise((resolvePromise) => { + timeout = setTimeout(() => resolvePromise(false), timeoutMs); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +async function raceCloseAndExit( + close: Promise< + | { readonly kind: 'close-resolved' } + | { readonly failure: unknown; readonly kind: 'close-rejected' } + >, + exit: Promise, + timeoutMs: number +): Promise< + | { readonly kind: 'close-resolved' } + | { readonly failure: unknown; readonly kind: 'close-rejected' } + | { readonly kind: 'exit' } + | { readonly kind: 'timeout' } +> { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + close, + exit.then(() => ({ kind: 'exit' as const })), + new Promise<{ readonly kind: 'timeout' }>((resolvePromise) => { + timeout = setTimeout( + () => resolvePromise({ kind: 'timeout' }), + timeoutMs + ); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +function assertTimeout(value: number): void { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error('electron-process-exit-timeout-invalid'); + } +} diff --git a/apps/electron-backend-e2e/src/electron-test-fixtures.ts b/apps/electron-backend-e2e/src/electron-test-fixtures.ts index 920427dcf..818546505 100644 --- a/apps/electron-backend-e2e/src/electron-test-fixtures.ts +++ b/apps/electron-backend-e2e/src/electron-test-fixtures.ts @@ -27,6 +27,10 @@ import { reapOrphanedDataDirs, writeDataDirOwnerMarker, } from './data-dir-reaper'; +import { + closeElectronApplicationAndConfirmExit, + prepareElectronApplication, +} from './electron-process-lifecycle'; export const workspaceRoot = resolve(__dirname, '../../..'); export const electronMainPath = join( @@ -223,18 +227,26 @@ export async function launchElectronApp( args, env: buildElectronLaunchEnvironment(dataDir, options), }); - attachElectronProcessDiagnostics(electronApp); - - const mainWindow = await findMainWindow(electronApp); - await waitForAppReady(mainWindow); - await startPortalDebugCapture(mainWindow); - await startDbOperationCapture(mainWindow); - await startRendererFrameCapture(mainWindow); - - return { - electronApp, - mainWindow, - }; + return prepareElectronApplication({ + application: electronApp, + dispose: (application) => + closeElectronApplicationAndConfirmExit(application, { + closeTimeoutMs: electronAppCloseTimeoutMs, + exitTimeoutMs: electronAppKillWaitMs, + }), + prepare: async (application) => { + attachElectronProcessDiagnostics(application); + const mainWindow = await findMainWindow(application); + await waitForAppReady(mainWindow); + await startPortalDebugCapture(mainWindow); + await startDbOperationCapture(mainWindow); + await startRendererFrameCapture(mainWindow); + return { + electronApp: application, + mainWindow, + }; + }, + }); } export function buildElectronLaunchEnvironment( @@ -495,15 +507,19 @@ export async function launchCompetingElectronInstance( const { appArgs = [], timeoutMs = 30000 } = options; // In a Node context the `electron` package resolves to its binary path. const electronBinaryPath = require('electron') as unknown as string; - const child = spawn(electronBinaryPath, buildElectronLaunchArgs([], appArgs), { - env: { - ...process.env, - ELECTRON_IS_DEV: '0', - IPTVNATOR_E2E_DATA_DIR: dataDir, - NODE_ENV: 'test', - }, - stdio: ['ignore', 'ignore', 'pipe'], - }); + const child = spawn( + electronBinaryPath, + buildElectronLaunchArgs([], appArgs), + { + env: { + ...process.env, + ELECTRON_IS_DEV: '0', + IPTVNATOR_E2E_DATA_DIR: dataDir, + NODE_ENV: 'test', + }, + stdio: ['ignore', 'ignore', 'pipe'], + } + ); // Kept for the assertion message: a competing launch that dies for an // unrelated reason (missing sandbox, missing GPU) looks exactly like a @@ -570,6 +586,15 @@ export async function closeElectronApp( } } +export async function closeElectronAppAndConfirmExit( + app: LaunchedElectronApp +): Promise { + await closeElectronApplicationAndConfirmExit(app.electronApp, { + closeTimeoutMs: electronAppCloseTimeoutMs, + exitTimeoutMs: electronAppKillWaitMs, + }); +} + async function waitForPromiseWithTimeout( promise: Promise, timeoutMs: number diff --git a/apps/electron-backend-e2e/src/performance/electron-launch-environment.spec.ts b/apps/electron-backend-e2e/src/performance/electron-launch-environment.spec.ts index c6c82239f..4ab626254 100644 --- a/apps/electron-backend-e2e/src/performance/electron-launch-environment.spec.ts +++ b/apps/electron-backend-e2e/src/performance/electron-launch-environment.spec.ts @@ -1,4 +1,6 @@ import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; import { describe, it } from 'node:test'; import { @@ -8,6 +10,33 @@ import { import { assertXtreamBenchmarkEnvironmentIsolated } from './xtream-benchmark-iteration-support'; describe('Electron launch environment isolation', () => { + it('makes post-spawn fixture preparation exception-safe', () => { + const fixture = readFileSync( + resolve(process.cwd(), 'src/electron-test-fixtures.ts'), + 'utf8' + ); + const launch = fixture.indexOf( + 'const electronApp = await electron.launch' + ); + const guardedPreparation = fixture.indexOf( + 'return prepareElectronApplication', + launch + ); + const strictDisposal = fixture.indexOf( + 'closeElectronApplicationAndConfirmExit', + guardedPreparation + ); + const mainWindow = fixture.indexOf( + 'const mainWindow = await findMainWindow', + guardedPreparation + ); + + assert.ok(launch >= 0); + assert.ok(guardedPreparation > launch); + assert.ok(strictDisposal > guardedPreparation); + assert.ok(mainWindow > strictDisposal); + }); + it('inherits only portable runtime keys for a profiled benchmark child', () => { const environment = buildElectronLaunchEnvironment( '/tmp/iptvnator-performance', diff --git a/apps/electron-backend-e2e/src/performance/electron-process-lifecycle.spec.ts b/apps/electron-backend-e2e/src/performance/electron-process-lifecycle.spec.ts new file mode 100644 index 000000000..15a6d003b --- /dev/null +++ b/apps/electron-backend-e2e/src/performance/electron-process-lifecycle.spec.ts @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { describe, it } from 'node:test'; + +import { + closeElectronApplicationAndConfirmExit, + ElectronApplicationDisposalError, + prepareElectronApplication, +} from '../electron-process-lifecycle'; + +class FakeChildProcess extends EventEmitter { + exitCode: number | null = null; + signalCode: NodeJS.Signals | null = null; + killCalls = 0; + shouldExitOnKill = true; + + kill(): boolean { + this.killCalls += 1; + if (this.shouldExitOnKill) { + this.signalCode = 'SIGTERM'; + queueMicrotask(() => this.emit('exit', null, 'SIGTERM')); + } + return true; + } +} + +describe('Electron process lifecycle', () => { + it('closes and confirms exit when post-spawn launch preparation fails', async () => { + const child = new FakeChildProcess(); + const launchFailure = new Error('renderer readiness failed'); + let closeCalls = 0; + const application = { + close: async () => { + closeCalls += 1; + child.exitCode = 0; + child.emit('exit', 0, null); + }, + process: () => child, + }; + + await assert.rejects( + prepareElectronApplication({ + application, + dispose: (app) => + closeElectronApplicationAndConfirmExit(app, { + closeTimeoutMs: 10, + exitTimeoutMs: 10, + }), + prepare: async () => { + throw launchFailure; + }, + }), + (error: unknown) => error === launchFailure + ); + assert.equal(closeCalls, 1); + assert.equal(child.exitCode, 0); + }); + + it('fails when process exit remains unconfirmed after forced teardown', async () => { + const child = new FakeChildProcess(); + child.shouldExitOnKill = false; + const application = { + close: () => new Promise(() => undefined), + process: () => child, + }; + + await assert.rejects( + closeElectronApplicationAndConfirmExit(application, { + closeTimeoutMs: 1, + exitTimeoutMs: 1, + }), + /electron-process-exit-unconfirmed/ + ); + assert.equal(child.killCalls, 1); + }); + + it('preserves both preparation and unconfirmed-disposal failures', async () => { + const preparationFailure = new Error('renderer readiness failed'); + const disposalFailure = new Error('electron-process-exit-unconfirmed'); + + await assert.rejects( + prepareElectronApplication({ + application: {}, + dispose: async () => { + throw disposalFailure; + }, + prepare: async () => { + throw preparationFailure; + }, + }), + (error: unknown) => + error instanceof ElectronApplicationDisposalError && + error.cause === preparationFailure && + error.errors.includes(disposalFailure) + ); + }); + + it('observes process exit promptly even when Playwright close never settles', async () => { + const child = new FakeChildProcess(); + const application = { + close: () => { + queueMicrotask(() => { + child.exitCode = 0; + child.emit('exit', 0, null); + }); + return new Promise(() => undefined); + }, + process: () => child, + }; + + await Promise.race([ + closeElectronApplicationAndConfirmExit(application, { + closeTimeoutMs: 1_000, + exitTimeoutMs: 10, + }), + new Promise((_, rejectPromise) => { + setTimeout( + () => rejectPromise(new Error('exit-observation-delayed')), + 50 + ); + }), + ]); + assert.equal(child.killCalls, 0); + }); + + it('confirms the exit caused by forced teardown', async () => { + const child = new FakeChildProcess(); + const application = { + close: () => new Promise(() => undefined), + process: () => child, + }; + + await closeElectronApplicationAndConfirmExit(application, { + closeTimeoutMs: 1, + exitTimeoutMs: 10, + }); + assert.equal(child.killCalls, 1); + assert.equal(child.signalCode, 'SIGTERM'); + }); +}); diff --git a/apps/electron-backend-e2e/src/performance/xtream-benchmark-app-startup.ts b/apps/electron-backend-e2e/src/performance/xtream-benchmark-app-startup.ts new file mode 100644 index 000000000..71620578f --- /dev/null +++ b/apps/electron-backend-e2e/src/performance/xtream-benchmark-app-startup.ts @@ -0,0 +1,168 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { + closeElectronAppAndConfirmExit, + launchElectronApp, + type LaunchedElectronApp, +} from '../electron-test-fixtures'; +import { ElectronApplicationDisposalError } from '../electron-process-lifecycle'; +import type { XtreamBenchmarkConfiguration } from './xtream-benchmark-configuration'; +import { + installMainCapture, + type MainCaptureStartOptions, +} from './m3u-refresh-main-capture'; +import { + assertRendererCdpTarget, + assertTcpPortAvailable, + resolveRendererWindowIdentity, +} from './m3u-import-benchmark-support'; +import { + assertXtreamBenchmarkEnvironmentIsolated, + prearmXtreamDatabaseWorker, +} from './xtream-benchmark-iteration-support'; +import { disableGenericElectronFixtureProbes } from './xtream-benchmark-runtime'; +import { + captureXtreamLaunchIdentity, + type XtreamLaunchIdentity, +} from './xtream-process-evidence'; +import { + classifyXtreamStartupFailure, + runXtreamStartupWithRetry, +} from './xtream-startup-retry'; +import type { XtreamStartupRetryReason } from './xtream-summary-contract'; + +interface XtreamAppResource { + readonly app: LaunchedElectronApp; + readonly dataDirectory: string; +} + +interface PreparedXtreamApp { + readonly launch: XtreamLaunchIdentity; + readonly rendererWindowIdentity: MainCaptureStartOptions['rendererWindowIdentity']; +} + +export interface LaunchedXtreamBenchmarkApp + extends XtreamAppResource, PreparedXtreamApp { + readonly startupEvidence: { + readonly attemptCount: number; + readonly retryReasons: readonly XtreamStartupRetryReason[]; + readonly status: 'ready'; + }; +} + +export interface LaunchXtreamBenchmarkAppOptions { + readonly configuration: XtreamBenchmarkConfiguration; + readonly seenElectronPids: Set; +} + +async function disposeXtreamAppResource( + resource: XtreamAppResource +): Promise { + await closeElectronAppAndConfirmExit(resource.app); + await removeXtreamDataDirectory(resource.dataDirectory); +} + +async function createXtreamAppResource( + configuration: XtreamBenchmarkConfiguration +): Promise { + await assertTcpPortAvailable(configuration.cdpPort); + const dataDirectory = await mkdtemp( + join(tmpdir(), 'iptvnator-xtream-performance-') + ); + try { + const app = await launchElectronApp(dataDirectory, { + args: [ + '--js-flags=--expose-gc', + '--remote-debugging-address=127.0.0.1', + `--remote-debugging-port=${configuration.cdpPort}`, + `--user-data-dir=${join(dataDirectory, 'user-data')}`, + ], + environmentInheritance: 'runtime-only', + env: { + IPTVNATOR_DB_WORKER_BATCH_DELAY_MS: '0', + IPTVNATOR_PERF_CAPTURE: '1', + IPTVNATOR_PERF_WORKER_PROFILING: '1', + IPTVNATOR_TRACE_RENDERER_CONSOLE: '0', + }, + omitEnvKeys: [ + 'IPTVNATOR_XTREAM_MOCK_CONTROL', + 'IPTVNATOR_XTREAM_MOCK_CONTROL_TOKEN', + ], + }); + return { app, dataDirectory }; + } catch (failure) { + if (failure instanceof ElectronApplicationDisposalError) { + throw failure; + } + try { + await removeXtreamDataDirectory(dataDirectory); + } catch (cleanupFailure) { + throw new AggregateError( + [failure, cleanupFailure], + 'Xtream app launch and profile cleanup both failed', + { cause: failure } + ); + } + throw failure; + } +} + +function removeXtreamDataDirectory(dataDirectory: string): Promise { + return rm(dataDirectory, { + force: true, + maxRetries: 20, + recursive: true, + retryDelay: 250, + }); +} + +async function prepareXtreamAppResource( + resource: XtreamAppResource, + options: LaunchXtreamBenchmarkAppOptions +): Promise { + const { app } = resource; + const { configuration } = options; + app.mainWindow.setDefaultTimeout(10 * 60 * 1_000); + await assertXtreamBenchmarkEnvironmentIsolated(app); + await disableGenericElectronFixtureProbes(app.mainWindow); + await assertRendererCdpTarget(app.mainWindow, configuration.cdpPort); + await installMainCapture(app.electronApp); + await prearmXtreamDatabaseWorker(app); + const launch = await captureXtreamLaunchIdentity( + app.electronApp, + app.electronApp.process().pid, + options.seenElectronPids + ); + const rendererWindowIdentity = await resolveRendererWindowIdentity(app); + return { launch, rendererWindowIdentity }; +} + +export async function launchXtreamBenchmarkApp( + options: LaunchXtreamBenchmarkAppOptions +): Promise { + const started = await runXtreamStartupWithRetry({ + classifyFailure: classifyXtreamStartupFailure, + create: () => createXtreamAppResource(options.configuration), + dispose: disposeXtreamAppResource, + maxAttempts: 2, + prepare: (resource) => prepareXtreamAppResource(resource, options), + }); + return Object.freeze({ + ...started.resource, + ...started.prepared, + startupEvidence: Object.freeze({ + attemptCount: started.attemptCount, + retryReasons: started.retryReasons, + status: 'ready' as const, + }), + }); +} + +export async function disposeXtreamBenchmarkApp( + app: LaunchedElectronApp, + dataDirectory: string +): Promise { + await disposeXtreamAppResource({ app, dataDirectory }); +} diff --git a/apps/electron-backend-e2e/src/performance/xtream-benchmark-iteration.ts b/apps/electron-backend-e2e/src/performance/xtream-benchmark-iteration.ts index 45d83ea69..03868de83 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-benchmark-iteration.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-benchmark-iteration.ts @@ -1,12 +1,11 @@ -import { mkdir, mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; +import { mkdir } from 'node:fs/promises'; import { join } from 'node:path'; +import type { LaunchedElectronApp } from '../electron-test-fixtures'; import { - closeElectronApp, - launchElectronApp, - type LaunchedElectronApp, -} from '../electron-test-fixtures'; + disposeXtreamBenchmarkApp, + launchXtreamBenchmarkApp, +} from './xtream-benchmark-app-startup'; import { type XtreamIterationDefinition, type XtreamPerformanceManifest, @@ -24,18 +23,12 @@ import { readXtreamCatalogDatabaseSnapshot } from './xtream-catalog-database-rea import { assembleXtreamRawIteration } from './xtream-iteration-assembler'; import { beginXtreamMainMeasurement, - installMainCapture, readXtreamMainCaptureStatus, rolloverXtreamMainCapture, startMainCapture, stopXtreamMainCapture, type MainCaptureStartOptions, } from './m3u-refresh-main-capture'; -import { - assertRendererCdpTarget, - assertTcpPortAvailable, - resolveRendererWindowIdentity, -} from './m3u-import-benchmark-support'; import { assertPerformanceArtifactCapacity } from './performance-artifact-preflight'; import { createXtreamArtifactSafetyContext, @@ -46,25 +39,21 @@ import { import { createXtreamSingleAttemptCapture, persistXtreamIterationFailureEvidenceAndThrow, + runXtreamFinalTeardown, type XtreamBenchmarkFailureStage, + type XtreamPropagatedFailure, } from './xtream-benchmark-lifecycle'; import { - assertXtreamBenchmarkEnvironmentIsolated, createXtreamDiagnosticEvidence, createXtreamSourceTitle, finalizeXtreamCaptureAfterRendererTerminal, - prearmXtreamDatabaseWorker, seedXtreamExistingPortal, } from './xtream-benchmark-iteration-support'; import { deriveXtreamMeasuredPlaylistId, - disableGenericElectronFixtureProbes, xtreamScenarioRequiresSeed, } from './xtream-benchmark-runtime'; -import { - captureXtreamLaunchIdentity, - createXtreamIterationProcessEvidence, -} from './xtream-process-evidence'; +import { createXtreamIterationProcessEvidence } from './xtream-process-evidence'; import { startXtreamRendererCapture, type RunningXtreamRendererCapture, @@ -95,13 +84,12 @@ export async function runXtreamBenchmarkIteration( mode: 0o700, recursive: false, }); - const dataDirectory = await mkdtemp( - join(tmpdir(), 'iptvnator-xtream-performance-') - ); const safety = createXtreamArtifactSafetyContext( configuration.controlToken ); let app: LaunchedElectronApp | null = null; + let dataDirectory: string | null = null; + let propagatedFailure: XtreamPropagatedFailure | null = null; let renderer: RunningXtreamRendererCapture | null = null; let prepared: PreparedXtreamScenario | null = null; let controlState: Awaited> | null = @@ -117,44 +105,23 @@ export async function runXtreamBenchmarkIteration( await control.resetAll(); const fixture = await control.prepare(); assertXtreamManifestIdentity(context.expectedFixture, fixture); - await assertTcpPortAvailable(configuration.cdpPort); - app = await launchElectronApp(dataDirectory, { - args: [ - '--js-flags=--expose-gc', - '--remote-debugging-address=127.0.0.1', - `--remote-debugging-port=${configuration.cdpPort}`, - `--user-data-dir=${join(dataDirectory, 'user-data')}`, - ], - environmentInheritance: 'runtime-only', - env: { - IPTVNATOR_DB_WORKER_BATCH_DELAY_MS: '0', - IPTVNATOR_PERF_CAPTURE: '1', - IPTVNATOR_PERF_WORKER_PROFILING: '1', - IPTVNATOR_TRACE_RENDERER_CONSOLE: '0', - }, - omitEnvKeys: [ - 'IPTVNATOR_XTREAM_MOCK_CONTROL', - 'IPTVNATOR_XTREAM_MOCK_CONTROL_TOKEN', - ], + const started = await launchXtreamBenchmarkApp({ + configuration, + seenElectronPids: context.seenElectronPids, }); - app.mainWindow.setDefaultTimeout(10 * 60 * 1_000); - await assertXtreamBenchmarkEnvironmentIsolated(app); - await disableGenericElectronFixtureProbes(app.mainWindow); - await assertRendererCdpTarget(app.mainWindow, configuration.cdpPort); - await installMainCapture(app.electronApp); - await prearmXtreamDatabaseWorker(app); - const electronApp = app.electronApp; - const launch = await captureXtreamLaunchIdentity( - app.electronApp, - app.electronApp.process().pid, - context.seenElectronPids + app = started.app; + dataDirectory = started.dataDirectory; + await writeXtreamSafeJson( + join(iterationDirectory, 'startup-evidence.json'), + started.startupEvidence, + safety ); - const rendererWindowIdentity = await resolveRendererWindowIdentity(app); + const electronApp = app.electronApp; const captureOptions: MainCaptureStartOptions = { deferMeasurement: true, diagnostic: definition.kind === 'diagnostic', outputDirectory: iterationDirectory, - rendererWindowIdentity, + rendererWindowIdentity: started.rendererWindowIdentity, }; const sourceTitle = createXtreamSourceTitle(definition); const seeded = xtreamScenarioRequiresSeed(definition.scenarioId); @@ -260,7 +227,7 @@ export async function runXtreamBenchmarkIteration( const finalized = await finalizeXtreamCaptureAfterRendererTerminal({ captureGeneration, driverTerminal: terminalPromise, - readStatus: () => readXtreamMainCaptureStatus(app.electronApp), + readStatus: () => readXtreamMainCaptureStatus(electronApp), rendererTerminal: renderer.waitForTerminal(), scenarioId: definition.scenarioId, stopMain: stopMainCaptureOnce, @@ -313,7 +280,7 @@ export async function runXtreamBenchmarkIteration( app.electronApp, { captureStoppedEpochMs: mainCapture.captureStoppedEpochMs, - dataDirectory, + dataDirectory: started.dataDirectory, playlistId, requireFromPath: join( configuration.workspaceRoot, @@ -338,10 +305,12 @@ export async function runXtreamBenchmarkIteration( iterationDirectory ); const processEvidence = await createXtreamIterationProcessEvidence({ - ...launch, + ...started.launch, diagnosticDirectorySha256: diagnosticEvidence?.directorySha256 ?? null, iterationDirectory, + startupAttemptCount: started.startupEvidence.attemptCount, + startupRetryReasons: started.startupEvidence.retryReasons, }); const raw = assembleXtreamRawIteration({ catalogVerification, @@ -362,38 +331,48 @@ export async function runXtreamBenchmarkIteration( ); } catch (failure) { const electronApp = app?.electronApp; - await persistXtreamIterationFailureEvidenceAndThrow(failure, { - evidence: { - control: async () => controlState ?? control.state(), - ...(electronApp - ? { - mainStatus: () => - readXtreamMainCaptureStatus(electronApp), - } - : {}), - ...(stopMainCaptureOnce - ? { mainCapture: stopMainCaptureOnce } - : {}), - ...(stopRendererCaptureOnce - ? { rendererCapture: stopRendererCaptureOnce } - : {}), - ...(terminal ? { terminal: async () => terminal } : {}), - ...(trigger ? { trigger: async () => trigger } : {}), - }, - persist: (filename, value) => - writeXtreamSafeJson( - join(iterationDirectory, filename), - value, - safety - ), - stage: failureStage, - }); + try { + return await persistXtreamIterationFailureEvidenceAndThrow( + failure, + { + evidence: { + control: async () => controlState ?? control.state(), + ...(electronApp + ? { + mainStatus: () => + readXtreamMainCaptureStatus(electronApp), + } + : {}), + ...(stopMainCaptureOnce + ? { mainCapture: stopMainCaptureOnce } + : {}), + ...(stopRendererCaptureOnce + ? { rendererCapture: stopRendererCaptureOnce } + : {}), + ...(terminal ? { terminal: async () => terminal } : {}), + ...(trigger ? { trigger: async () => trigger } : {}), + }, + persist: (filename, value) => + writeXtreamSafeJson( + join(iterationDirectory, filename), + value, + safety + ), + stage: failureStage, + } + ); + } catch (failureToPropagate) { + propagatedFailure = { failure: failureToPropagate }; + throw failureToPropagate; + } } finally { await prepared?.dispose().catch(() => undefined); await renderer?.dispose().catch(() => undefined); - if (app) await closeElectronApp(app); - await rm(dataDirectory, { force: true, recursive: true }).catch( - () => undefined - ); + if (app && dataDirectory) { + await runXtreamFinalTeardown( + () => disposeXtreamBenchmarkApp(app, dataDirectory), + propagatedFailure + ); + } } } diff --git a/apps/electron-backend-e2e/src/performance/xtream-benchmark-lifecycle.spec.ts b/apps/electron-backend-e2e/src/performance/xtream-benchmark-lifecycle.spec.ts index be7038c6d..ffa3e8749 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-benchmark-lifecycle.spec.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-benchmark-lifecycle.spec.ts @@ -7,6 +7,7 @@ import { createXtreamSingleAttemptCapture, persistXtreamIterationFailureEvidence, persistXtreamIterationFailureEvidenceAndThrow, + runXtreamFinalTeardown, } from './xtream-benchmark-lifecycle'; const performanceDirectory = resolve(process.cwd(), 'src/performance'); @@ -190,6 +191,40 @@ describe('Xtream benchmark failure lifecycle', () => { ); }); + it('preserves both the propagated iteration and final teardown failures', async () => { + const iterationFailure = new Error('fixed-iteration-failure'); + const teardownFailure = new Error('fixed-teardown-failure'); + + await assert.rejects( + runXtreamFinalTeardown( + async () => { + throw teardownFailure; + }, + { failure: iterationFailure } + ), + (failure: unknown) => { + assert.ok(failure instanceof AggregateError); + assert.deepEqual(failure.errors, [ + iterationFailure, + teardownFailure, + ]); + assert.strictEqual(failure.cause, iterationFailure); + return true; + } + ); + }); + + it('preserves an exact teardown failure when the iteration succeeded', async () => { + const teardownFailure = new Error('fixed-teardown-failure'); + + await assert.rejects( + runXtreamFinalTeardown(async () => { + throw teardownFailure; + }, null), + (failure: unknown) => failure === teardownFailure + ); + }); + it('collects failure evidence before teardown and writes summary before its guard', () => { const iteration = readFileSync( resolve(performanceDirectory, 'xtream-benchmark-iteration.ts'), @@ -201,6 +236,10 @@ describe('Xtream benchmark failure lifecycle', () => { failureCatch ); const teardown = iteration.indexOf('} finally {', failureCatch); + const preservingTeardown = iteration.indexOf( + 'runXtreamFinalTeardown(', + teardown + ); assert.ok(failureCatch >= 0, 'iteration failure catch is missing'); assert.ok( @@ -211,6 +250,10 @@ describe('Xtream benchmark failure lifecycle', () => { teardown > failurePersistence, 'teardown occurs before failure evidence persistence' ); + assert.ok( + preservingTeardown > teardown, + 'final teardown does not preserve an iteration failure' + ); assert.doesNotMatch( iteration.slice(failureCatch, teardown), /if \(captureInstalled\)/u, diff --git a/apps/electron-backend-e2e/src/performance/xtream-benchmark-lifecycle.ts b/apps/electron-backend-e2e/src/performance/xtream-benchmark-lifecycle.ts index d41d35f49..bd8cca6b1 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-benchmark-lifecycle.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-benchmark-lifecycle.ts @@ -29,6 +29,10 @@ interface PersistXtreamIterationFailureEvidenceOptions { readonly stage: XtreamBenchmarkFailureStage; } +export interface XtreamPropagatedFailure { + readonly failure: unknown; +} + const EVIDENCE_ARTIFACTS: readonly [FailureEvidenceName, string][] = Object.freeze([ ['mainStatus', 'failure-main-status.json'], @@ -91,3 +95,19 @@ export async function persistXtreamIterationFailureEvidenceAndThrow( } throw failure; } + +export async function runXtreamFinalTeardown( + teardown: () => Promise, + propagatedFailure: XtreamPropagatedFailure | null +): Promise { + try { + await teardown(); + } catch (teardownFailure) { + if (!propagatedFailure) throw teardownFailure; + throw new AggregateError( + [propagatedFailure.failure, teardownFailure], + 'Xtream benchmark iteration and final teardown both failed', + { cause: propagatedFailure.failure } + ); + } +} diff --git a/apps/electron-backend-e2e/src/performance/xtream-benchmark-report-provenance.fixtures.ts b/apps/electron-backend-e2e/src/performance/xtream-benchmark-report-provenance.fixtures.ts index ad770f04d..b4c182e7c 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-benchmark-report-provenance.fixtures.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-benchmark-report-provenance.fixtures.ts @@ -61,12 +61,24 @@ export interface XtreamReportFixture { readonly iterations: readonly XtreamValidatedIterationResult[]; } -export async function formalReportFixture(): Promise { - return reportFixture(createXtreamIterationDefinitions(false), 10, true); +export async function formalReportFixture( + startupRetryIndex: number | null = null +): Promise { + return reportFixture( + createXtreamIterationDefinitions(false), + 10, + true, + startupRetryIndex + ); } export async function smokeReportFixture(): Promise { - return reportFixture(createXtreamIterationDefinitions(true), 100, false); + return reportFixture( + createXtreamIterationDefinitions(true), + 100, + false, + null + ); } export async function clearXtreamProvenanceFixtureDirectories(): Promise { @@ -80,7 +92,8 @@ export async function clearXtreamProvenanceFixtureDirectories(): Promise { async function reportFixture( definitions: readonly XtreamIterationDefinition[], identityOffset: number, - mixedInitialOverlap: boolean + mixedInitialOverlap: boolean, + startupRetryIndex: number | null ): Promise { const entries = await Promise.all( definitions.map(async (definition, index) => { @@ -105,6 +118,16 @@ async function reportFixture( ) { raw = withWorkerRequestOverlap(raw, 2); } + if (index === startupRetryIndex) { + raw = { + ...raw, + processIdentity: { + ...raw.processIdentity, + startupAttemptCount: 2, + startupRetryReasons: ['sqlite-database-locked'], + }, + }; + } return { evidence, iteration: await persist(raw, identity + identityOffset), diff --git a/apps/electron-backend-e2e/src/performance/xtream-benchmark-report.fixtures.ts b/apps/electron-backend-e2e/src/performance/xtream-benchmark-report.fixtures.ts index 19c6c50f5..0210c31d7 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-benchmark-report.fixtures.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-benchmark-report.fixtures.ts @@ -33,6 +33,12 @@ import { import { bindFixtureWorkerEvidence } from './xtream-worker-phase-evidence.fixture'; import type { WorkerRequestPerformanceMetrics } from './m3u-refresh-cancellation-contract'; +export const sha256 = (value: string): string => + createHash('sha256').update(value).digest('hex'); +const hash = (character: string): string => character.repeat(64); +const numberedHash = (value: number, salt: number): string => + sha256(`${salt}:${value}`); + export const TOKEN = 'random-control-token-that-must-never-persist'; const buildFile = (path: string, marker: string): XtreamBuildFileIdentity => ({ bytes: marker.length, @@ -216,6 +222,8 @@ export function iteration( generationIdentitySha256: numberedHash(identity, 3), launchId: `launch-${identity}`, profileDirectorySha256: numberedHash(identity, 4), + startupAttemptCount: 1, + startupRetryReasons: [], }, phaseCapture, phases, @@ -386,15 +394,3 @@ function restoreValidRequestMetrics( threadCpuUserMicros: 20, }; } - -export function sha256(value: string): string { - return createHash('sha256').update(value).digest('hex'); -} - -function hash(character: string): string { - return character.repeat(64); -} - -function numberedHash(value: number, salt: number): string { - return sha256(`${salt}:${value}`); -} diff --git a/apps/electron-backend-e2e/src/performance/xtream-benchmark-report.spec.ts b/apps/electron-backend-e2e/src/performance/xtream-benchmark-report.spec.ts index 9c1aeb0b8..c7336be71 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-benchmark-report.spec.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-benchmark-report.spec.ts @@ -188,6 +188,22 @@ describe('Xtream benchmark report contract', () => { ); }); + it('keeps a bounded startup retry in receipt-bound summary iterations', async () => { + const { diagnosticEvidence, iterations } = await formalReportFixture(1); + const summary = createXtreamBenchmarkSummary( + MANIFEST, + iterations, + diagnosticEvidence + ); + + const retryIdentity = itemAt(summary.iterations, 1).processIdentity; + assert.equal(retryIdentity.startupAttemptCount, 2); + assert.deepEqual(retryIdentity.startupRetryReasons, [ + 'sqlite-database-locked', + ]); + assert.equal(summary.validForComparison, true); + }); + it('rejects forged validated objects and excludes diagnostic CPU profiles from headline metrics', async () => { const { diagnosticEvidence, iterations } = await formalReportFixture(); const forged = [...iterations]; diff --git a/apps/electron-backend-e2e/src/performance/xtream-exact-schema.ts b/apps/electron-backend-e2e/src/performance/xtream-exact-schema.ts index 9e2882836..1b139526c 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-exact-schema.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-exact-schema.ts @@ -8,6 +8,7 @@ import { } from './xtream-benchmark-manifest-schema'; import { XTREAM_NOT_APPLICABLE, + XTREAM_STARTUP_RETRY_REASON, type XtreamRawIterationResult, type XtreamValidatedIterationResult, } from './xtream-summary-contract'; @@ -317,7 +318,17 @@ export function isExactXtreamProcessIdentity(value: unknown): boolean { 'generationIdentitySha256', 'launchId', 'profileDirectorySha256', + 'startupAttemptCount', + 'startupRetryReasons', ] as const; + const startupAttemptCount = isPositiveSafeInteger( + isExactRecord(value, keys) ? value['startupAttemptCount'] : null + ) + ? Number(value['startupAttemptCount']) + : 0; + const startupRetryReasons = isExactRecord(value, keys) + ? value['startupRetryReasons'] + : null; return ( isExactRecord(value, keys) && isPositiveSafeInteger(value['captureGeneration']) && @@ -328,7 +339,17 @@ export function isExactXtreamProcessIdentity(value: unknown): boolean { typeof value['generationIdentitySha256'] === 'string' && XTREAM_SHA256.test(value['generationIdentitySha256']) && typeof value['profileDirectorySha256'] === 'string' && - XTREAM_SHA256.test(value['profileDirectorySha256']) + XTREAM_SHA256.test(value['profileDirectorySha256']) && + (startupAttemptCount === 1 || startupAttemptCount === 2) && + Array.isArray(startupRetryReasons) && + startupRetryReasons.length === startupAttemptCount - 1 && + startupRetryReasons.every( + (reason) => + typeof reason === 'string' && + Object.values(XTREAM_STARTUP_RETRY_REASON).includes( + reason as never + ) + ) ); } diff --git a/apps/electron-backend-e2e/src/performance/xtream-iteration-assembler.fixture.ts b/apps/electron-backend-e2e/src/performance/xtream-iteration-assembler.fixture.ts index 360fcf8f7..fe2d6815b 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-iteration-assembler.fixture.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-iteration-assembler.fixture.ts @@ -136,6 +136,8 @@ export function createXtreamAssemblerFixture( profileDirectorySha256: String(scenarioOrdinal(scenarioId)).repeat( 64 ), + startupAttemptCount: 1, + startupRetryReasons: [], }, rendererCapture, terminal: createTerminal(raw, background), diff --git a/apps/electron-backend-e2e/src/performance/xtream-iteration-assembler.spec.ts b/apps/electron-backend-e2e/src/performance/xtream-iteration-assembler.spec.ts index 31a47bf63..02a73af95 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-iteration-assembler.spec.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-iteration-assembler.spec.ts @@ -23,6 +23,14 @@ describe('Xtream raw iteration assembler', () => { result.processIdentity.captureGeneration, input.mainCapture.captureGeneration ); + assert.equal( + result.processIdentity.startupAttemptCount, + input.process.startupAttemptCount + ); + assert.deepEqual( + result.processIdentity.startupRetryReasons, + input.process.startupRetryReasons + ); assert.equal( result.databaseWorker.requests.evidence.length, input.mainCapture.requests.length diff --git a/apps/electron-backend-e2e/src/performance/xtream-iteration-assembler.ts b/apps/electron-backend-e2e/src/performance/xtream-iteration-assembler.ts index e2c6ddb53..5cc8a7a2c 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-iteration-assembler.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-iteration-assembler.ts @@ -33,9 +33,11 @@ import { } from './xtream-scenario-driver'; import { XTREAM_NOT_APPLICABLE, + XTREAM_STARTUP_RETRY_REASON, type XtreamCatalogVerification as XtreamSummaryCatalogVerification, type XtreamIterationProcessIdentity, type XtreamRawIterationResult, + type XtreamStartupRetryReason, } from './xtream-summary-contract'; export interface XtreamIterationProcessEvidence { @@ -43,6 +45,8 @@ export interface XtreamIterationProcessEvidence { readonly freshProcessVerified: true; readonly launchId: string; readonly profileDirectorySha256: string; + readonly startupAttemptCount: 1 | 2; + readonly startupRetryReasons: readonly XtreamStartupRetryReason[]; } export interface XtreamIterationAssemblyInput { @@ -215,7 +219,8 @@ function createProcessIdentity( !Number.isSafeInteger(process.electronPid) || process.electronPid <= 0 || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(process.launchId) || - !/^[a-f0-9]{64}$/.test(process.profileDirectorySha256) + !/^[a-f0-9]{64}$/.test(process.profileDirectorySha256) || + !validStartupEvidence(process) ) { invalid(); } @@ -228,6 +233,8 @@ function createProcessIdentity( electronPid: process.electronPid, launchId: process.launchId, profileDirectorySha256: process.profileDirectorySha256, + startupAttemptCount: process.startupAttemptCount, + startupRetryReasons: process.startupRetryReasons, rendererTargetId: input.rendererCapture.targetId, browserWindowId: input.mainCapture.capture.rendererWindow.windowIdentity @@ -245,9 +252,26 @@ function createProcessIdentity( .digest('hex'), launchId: process.launchId, profileDirectorySha256: process.profileDirectorySha256, + startupAttemptCount: process.startupAttemptCount, + startupRetryReasons: [...process.startupRetryReasons], }; } +function validStartupEvidence( + process: XtreamIterationProcessEvidence +): boolean { + return ( + (process.startupAttemptCount === 1 || + process.startupAttemptCount === 2) && + Array.isArray(process.startupRetryReasons) && + process.startupRetryReasons.length === + process.startupAttemptCount - 1 && + process.startupRetryReasons.every((reason) => + Object.values(XTREAM_STARTUP_RETRY_REASON).includes(reason) + ) + ); +} + function requireEpoch(value: unknown): number { if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) invalid(); diff --git a/apps/electron-backend-e2e/src/performance/xtream-iteration-validity.spec.ts b/apps/electron-backend-e2e/src/performance/xtream-iteration-validity.spec.ts index 759cc3b06..37b914919 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-iteration-validity.spec.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-iteration-validity.spec.ts @@ -118,6 +118,18 @@ describe('Xtream raw iteration validity', () => { }), /invalid Xtream iteration/ ); + assert.throws( + () => + assertXtreamIterationResult({ + ...raw, + processIdentity: { + ...raw.processIdentity, + startupAttemptCount: 2, + startupRetryReasons: [], + }, + }), + /invalid Xtream iteration/ + ); const { processIdentity: _removed, ...missingIdentity } = raw; void _removed; assert.throws( diff --git a/apps/electron-backend-e2e/src/performance/xtream-process-evidence.spec.ts b/apps/electron-backend-e2e/src/performance/xtream-process-evidence.spec.ts index 186ee5f7e..7541c2f28 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-process-evidence.spec.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-process-evidence.spec.ts @@ -72,10 +72,14 @@ describe('Xtream process evidence', () => { electronPid: 41_001, iterationDirectory, launchId: 'launch-00000000-0000-4000-8000-000000000001', + startupAttemptCount: 1, + startupRetryReasons: [], }); assert.equal(evidence.freshProcessVerified, true); assert.match(evidence.profileDirectorySha256, /^[a-f0-9]{64}$/u); + assert.equal(evidence.startupAttemptCount, 1); + assert.deepEqual(evidence.startupRetryReasons, []); }); it('uses validated diagnostic directory evidence verbatim', async () => { @@ -89,9 +93,33 @@ describe('Xtream process evidence', () => { electronPid: 41_001, iterationDirectory, launchId: 'launch-00000000-0000-4000-8000-000000000001', + startupAttemptCount: 2, + startupRetryReasons: ['sqlite-database-locked'], }); assert.equal(evidence.profileDirectorySha256, directorySha256); + assert.equal(evidence.startupAttemptCount, 2); + assert.deepEqual(evidence.startupRetryReasons, [ + 'sqlite-database-locked', + ]); + }); + + it('rejects inconsistent startup attempt evidence', async () => { + const root = await temporaryDirectory(); + const iterationDirectory = join(root, 'run-01'); + await mkdir(iterationDirectory); + + await assert.rejects( + createXtreamIterationProcessEvidence({ + diagnosticDirectorySha256: null, + electronPid: 41_001, + iterationDirectory, + launchId: 'launch-00000000-0000-4000-8000-000000000001', + startupAttemptCount: 2, + startupRetryReasons: [], + }), + /xtream-electron-process-identity-invalid/ + ); }); }); diff --git a/apps/electron-backend-e2e/src/performance/xtream-process-evidence.ts b/apps/electron-backend-e2e/src/performance/xtream-process-evidence.ts index 3b25a80ec..bbc8480c2 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-process-evidence.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-process-evidence.ts @@ -4,6 +4,10 @@ import { lstat, realpath } from 'node:fs/promises'; import type { ElectronApplication } from '@playwright/test'; import type { XtreamIterationProcessEvidence } from './xtream-iteration-assembler'; +import { + XTREAM_STARTUP_RETRY_REASON, + type XtreamStartupRetryReason, +} from './xtream-summary-contract'; export interface XtreamLaunchIdentity { readonly electronPid: number; @@ -13,6 +17,8 @@ export interface XtreamLaunchIdentity { export interface XtreamIterationProcessEvidenceOptions extends XtreamLaunchIdentity { readonly diagnosticDirectorySha256: string | null; readonly iterationDirectory: string; + readonly startupAttemptCount: number; + readonly startupRetryReasons: readonly XtreamStartupRetryReason[]; } export async function captureXtreamLaunchIdentity( @@ -58,7 +64,11 @@ export async function createXtreamIterationProcessEvidence( if ( !Number.isSafeInteger(options.electronPid) || options.electronPid <= 0 || - !/^launch-[a-f0-9-]{36}$/u.test(options.launchId) + !/^launch-[a-f0-9-]{36}$/u.test(options.launchId) || + !validStartupEvidence( + options.startupAttemptCount, + options.startupRetryReasons + ) ) { invalidProcess(); } @@ -78,6 +88,10 @@ export async function createXtreamIterationProcessEvidence( freshProcessVerified: true, launchId: options.launchId, profileDirectorySha256, + startupAttemptCount: options.startupAttemptCount as 1 | 2, + startupRetryReasons: Object.freeze([ + ...options.startupRetryReasons, + ]), }); } catch (error) { if ( @@ -90,6 +104,20 @@ export async function createXtreamIterationProcessEvidence( } } +function validStartupEvidence( + attemptCount: number, + retryReasons: readonly XtreamStartupRetryReason[] +): boolean { + return ( + (attemptCount === 1 || attemptCount === 2) && + Array.isArray(retryReasons) && + retryReasons.length === attemptCount - 1 && + retryReasons.every((reason) => + Object.values(XTREAM_STARTUP_RETRY_REASON).includes(reason) + ) + ); +} + function sha256(value: string): string { return createHash('sha256').update(value).digest('hex'); } diff --git a/apps/electron-backend-e2e/src/performance/xtream-scenario-driver-wiring.spec.ts b/apps/electron-backend-e2e/src/performance/xtream-scenario-driver-wiring.spec.ts index 504049d1d..2b0543746 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-scenario-driver-wiring.spec.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-scenario-driver-wiring.spec.ts @@ -130,13 +130,13 @@ describe('Xtream scenario driver source wiring', () => { it('prearms the persistent database worker outside the measured capture', () => { const iteration = source('xtream-benchmark-iteration.ts'); + const startup = source('xtream-benchmark-app-startup.ts'); const support = source('xtream-benchmark-iteration-support.ts'); - const install = iteration.indexOf( + const install = startup.indexOf( 'await installMainCapture(app.electronApp)' ); - const prearm = iteration.indexOf( - 'await prearmXtreamDatabaseWorker(app)' - ); + const prearm = startup.indexOf('await prearmXtreamDatabaseWorker(app)'); + const launch = iteration.indexOf('await launchXtreamBenchmarkApp'); const captureOptions = iteration.indexOf( 'const captureOptions: MainCaptureStartOptions' ); @@ -155,7 +155,22 @@ describe('Xtream scenario driver source wiring', () => { assert.ok(workerPrearm > mainDatabaseReady); assert.ok(install >= 0); assert.ok(prearm > install); - assert.ok(captureOptions > prearm); + assert.ok(launch >= 0); + assert.ok(captureOptions > launch); + assert.match(startup, /maxAttempts:\s*2/); + assert.match(startup, /classifyXtreamStartupFailure/); + assert.match(startup, /disposeXtreamAppResource/); + assert.match(startup, /closeElectronAppAndConfirmExit/); + assert.match( + startup, + /failure instanceof ElectronApplicationDisposalError/ + ); + assert.match(startup, /maxRetries:\s*20/); + assert.match(startup, /retryDelay:\s*250/); + assert.doesNotMatch( + startup, + /removeXtreamDataDirectory\([^)]*\)\.catch/ + ); }); it('subscribes the renderer probe before the cancellation observer is armed', () => { diff --git a/apps/electron-backend-e2e/src/performance/xtream-startup-evidence.spec.ts b/apps/electron-backend-e2e/src/performance/xtream-startup-evidence.spec.ts new file mode 100644 index 000000000..d58020fcc --- /dev/null +++ b/apps/electron-backend-e2e/src/performance/xtream-startup-evidence.spec.ts @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { XTREAM_SCENARIO_ID } from './xtream-benchmark-contract'; +import { createXtreamAssemblerFixture } from './xtream-iteration-assembler.fixture'; +import { assembleXtreamRawIteration } from './xtream-iteration-assembler'; + +describe('Xtream startup evidence', () => { + it('rejects an attempt count that does not match its retry reasons', () => { + const input = createXtreamAssemblerFixture( + XTREAM_SCENARIO_ID.INITIAL_IMPORT_LARGE + ); + + assert.throws( + () => + assembleXtreamRawIteration({ + ...input, + process: { + ...input.process, + startupAttemptCount: 2, + startupRetryReasons: [], + }, + }), + /xtream-iteration-assembly-invalid/ + ); + }); + + it('binds retry evidence into the capture generation identity', () => { + const input = createXtreamAssemblerFixture( + XTREAM_SCENARIO_ID.INITIAL_IMPORT_LARGE + ); + const firstAttempt = assembleXtreamRawIteration(input); + const secondAttempt = assembleXtreamRawIteration({ + ...input, + process: { + ...input.process, + startupAttemptCount: 2, + startupRetryReasons: ['sqlite-database-locked'], + }, + }); + + assert.notEqual( + firstAttempt.processIdentity.generationIdentitySha256, + secondAttempt.processIdentity.generationIdentitySha256 + ); + }); +}); diff --git a/apps/electron-backend-e2e/src/performance/xtream-startup-retry.spec.ts b/apps/electron-backend-e2e/src/performance/xtream-startup-retry.spec.ts new file mode 100644 index 000000000..fc508d2c0 --- /dev/null +++ b/apps/electron-backend-e2e/src/performance/xtream-startup-retry.spec.ts @@ -0,0 +1,145 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { + classifyXtreamStartupFailure, + runXtreamStartupWithRetry, +} from './xtream-startup-retry'; + +interface FakeResource { + readonly id: number; +} + +describe('Xtream benchmark startup retry', () => { + it('relaunches once with a fresh resource after a locked cold database', async () => { + const created: number[] = []; + const disposed: number[] = []; + const result = await runXtreamStartupWithRetry({ + classifyFailure: classifyXtreamStartupFailure, + create: async (attempt) => { + if (attempt === 2) { + assert.deepEqual(disposed, [1]); + } + created.push(attempt); + return { id: attempt }; + }, + dispose: async ({ id }) => { + disposed.push(id); + }, + maxAttempts: 2, + prepare: async ({ id }) => { + if (id === 1) { + throw new Error( + 'Error invoking remote method: SqliteError: database is locked' + ); + } + return `ready-${id}`; + }, + }); + + assert.deepEqual(created, [1, 2]); + assert.deepEqual(disposed, [1]); + assert.equal(result.attemptCount, 2); + assert.equal(result.prepared, 'ready-2'); + assert.deepEqual(result.resource, { id: 2 }); + assert.deepEqual(result.retryReasons, ['sqlite-database-locked']); + }); + + it('does not retry an unrelated startup failure', async () => { + const failure = new Error('renderer preload contract invalid'); + const created: number[] = []; + const disposed: number[] = []; + + await assert.rejects( + runXtreamStartupWithRetry({ + classifyFailure: classifyXtreamStartupFailure, + create: async (attempt) => { + created.push(attempt); + return { id: attempt }; + }, + dispose: async ({ id }) => { + disposed.push(id); + }, + maxAttempts: 2, + prepare: async () => { + throw failure; + }, + }), + (error: unknown) => error === failure + ); + assert.deepEqual(created, [1]); + assert.deepEqual(disposed, [1]); + }); + + it('does not create a second app when failed-attempt teardown is unproven', async () => { + const created: number[] = []; + const teardownFailure = new Error('electron-process-exit-unconfirmed'); + + await assert.rejects( + runXtreamStartupWithRetry({ + classifyFailure: classifyXtreamStartupFailure, + create: async (attempt) => { + created.push(attempt); + return { id: attempt }; + }, + dispose: async () => { + throw teardownFailure; + }, + maxAttempts: 2, + prepare: async () => { + throw new Error('SqliteError: database is locked'); + }, + }), + (error: unknown) => + error instanceof AggregateError && + error.errors.includes(teardownFailure) + ); + assert.deepEqual(created, [1]); + }); + + it('bounds repeated readiness failures and disposes every failed app', async () => { + const created: number[] = []; + const disposed: number[] = []; + + await assert.rejects( + runXtreamStartupWithRetry({ + classifyFailure: classifyXtreamStartupFailure, + create: async (attempt) => { + created.push(attempt); + return { id: attempt }; + }, + dispose: async ({ id }) => { + disposed.push(id); + }, + maxAttempts: 2, + prepare: async ({ id }) => { + throw new Error(`no such table on attempt ${id}`); + }, + }), + /no such table on attempt 2/ + ); + assert.deepEqual(created, [1, 2]); + assert.deepEqual(disposed, [1, 2]); + }); + + it('classifies only the known cold-start SQLite readiness failures', () => { + assert.equal( + classifyXtreamStartupFailure( + new Error('SqliteError: database is locked') + ), + 'sqlite-database-locked' + ); + assert.equal( + classifyXtreamStartupFailure( + new Error('SqliteError: no such table: playlists') + ), + 'sqlite-schema-not-ready' + ); + assert.equal( + classifyXtreamStartupFailure( + new Error('database disk image malformed') + ), + null + ); + }); +}); diff --git a/apps/electron-backend-e2e/src/performance/xtream-startup-retry.ts b/apps/electron-backend-e2e/src/performance/xtream-startup-retry.ts new file mode 100644 index 000000000..4c1ece1cc --- /dev/null +++ b/apps/electron-backend-e2e/src/performance/xtream-startup-retry.ts @@ -0,0 +1,92 @@ +import { + XTREAM_STARTUP_RETRY_REASON, + type XtreamStartupRetryReason, +} from './xtream-summary-contract'; + +export interface XtreamStartupRetryOptions { + readonly classifyFailure: ( + failure: unknown + ) => XtreamStartupRetryReason | null; + readonly create: (attempt: number) => Promise; + readonly dispose: (resource: Resource) => Promise; + readonly maxAttempts: number; + readonly prepare: ( + resource: Resource, + attempt: number + ) => Promise; +} + +export interface XtreamStartupRetryResult { + readonly attemptCount: number; + readonly prepared: Prepared; + readonly resource: Resource; + readonly retryReasons: readonly XtreamStartupRetryReason[]; +} + +function readFailureMessage(failure: unknown): string { + return failure instanceof Error ? failure.message : String(failure); +} + +export function classifyXtreamStartupFailure( + failure: unknown +): XtreamStartupRetryReason | null { + const message = readFailureMessage(failure); + if (/database is locked/i.test(message)) { + return XTREAM_STARTUP_RETRY_REASON.SQLITE_DATABASE_LOCKED; + } + if (/no such table/i.test(message)) { + return XTREAM_STARTUP_RETRY_REASON.SQLITE_SCHEMA_NOT_READY; + } + return null; +} + +async function disposeFailedStartup( + dispose: (resource: Resource) => Promise, + failure: unknown, + resource: Resource +): Promise { + try { + await dispose(resource); + } catch (disposeFailure) { + throw new AggregateError( + [failure, disposeFailure], + 'Xtream startup and failed-attempt disposal both failed', + { cause: failure } + ); + } +} + +export async function runXtreamStartupWithRetry( + options: XtreamStartupRetryOptions +): Promise> { + if ( + !Number.isInteger(options.maxAttempts) || + options.maxAttempts < 1 || + options.maxAttempts > 2 + ) { + throw new Error('xtream-startup-attempt-limit-invalid'); + } + + const retryReasons: XtreamStartupRetryReason[] = []; + for (let attempt = 1; attempt <= options.maxAttempts; attempt += 1) { + const resource = await options.create(attempt); + try { + const prepared = await options.prepare(resource, attempt); + return Object.freeze({ + attemptCount: attempt, + prepared, + resource, + retryReasons: Object.freeze([...retryReasons]), + }); + } catch (failure) { + const retryReason = options.classifyFailure(failure); + await disposeFailedStartup(options.dispose, failure, resource); + if (!retryReason || attempt === options.maxAttempts) { + throw failure; + } + retryReasons.push(retryReason); + } + } + + throw new Error('xtream-startup-attempts-exhausted'); +} diff --git a/apps/electron-backend-e2e/src/performance/xtream-summary-contract.ts b/apps/electron-backend-e2e/src/performance/xtream-summary-contract.ts index bcaf428ca..7ff73b6cc 100644 --- a/apps/electron-backend-e2e/src/performance/xtream-summary-contract.ts +++ b/apps/electron-backend-e2e/src/performance/xtream-summary-contract.ts @@ -69,6 +69,14 @@ export const XTREAM_WORKER_WHOLE_METRIC_SCOPE = { OPERATION_WINDOW: 'whole-worker-operation-window', } as const; +export const XTREAM_STARTUP_RETRY_REASON = { + SQLITE_DATABASE_LOCKED: 'sqlite-database-locked', + SQLITE_SCHEMA_NOT_READY: 'sqlite-schema-not-ready', +} as const; + +export type XtreamStartupRetryReason = + (typeof XTREAM_STARTUP_RETRY_REASON)[keyof typeof XTREAM_STARTUP_RETRY_REASON]; + export const XTREAM_STRUCTURED_CLONE_ATTRIBUTION = { SELECTED_IPC_PROXY: 'selected-ipc-proxy-includes-queueing-and-scheduling', } as const; @@ -215,6 +223,8 @@ export interface XtreamIterationProcessIdentity { readonly generationIdentitySha256: string; readonly launchId: string; readonly profileDirectorySha256: string; + readonly startupAttemptCount: 1 | 2; + readonly startupRetryReasons: readonly XtreamStartupRetryReason[]; } export interface XtreamRawIterationResult extends XtreamIterationDefinition { diff --git a/docs/architecture/sqlite-db-worker.md b/docs/architecture/sqlite-db-worker.md index 1ed1dc64f..9cd127449 100644 --- a/docs/architecture/sqlite-db-worker.md +++ b/docs/architecture/sqlite-db-worker.md @@ -229,6 +229,21 @@ validity records the exact expected and valid request counts; nullable metrics remain in raw results but cannot be silently omitted from comparison distributions. +The formal Xtream benchmark launches every iteration with a fresh temporary +profile. On that cold profile, the renderer can request the persistent database +worker while the main process is still initializing the shared SQLite schema. +The harness treats only `database is locked` and `no such table` during its +pre-measurement readiness checks as this startup race. It closes the failed app, +confirms that its Electron process exited, removes that profile, and permits one +relaunch with another fresh profile. Unrelated failures are never retried, +unconfirmed teardown or profile cleanup is fatal, a second readiness failure is +fatal, and the measured import, refresh, delete, cancel, or background-UI +operation still has exactly one attempt. Each successful iteration persists +`startup-evidence.json` for immediate diagnostics and also binds the attempt +count and fixed retry reason into the receipted `result.json` process identity. +The validated identity is copied into `summary.json`, so comparison output +cannot hide startup instability. + The main-process benchmark samples the database worker's V8 `used_heap_size` and `external_memory` independently. Raw output includes a valid-sample count for each metric. A peak is numeric only after at least one