diff --git a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Change app language.png b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Change app language.png index 9b09f0443..906358a9d 100644 Binary files a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Change app language.png and b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Change app language.png differ diff --git a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Change app theme.png b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Change app theme.png index ae5559a5e..cd0c24610 100644 Binary files a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Change app theme.png and b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Change app theme.png differ diff --git a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Change video player.png b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Change video player.png index 9e4c06376..db81e79f1 100644 Binary files a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Change video player.png and b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Change video player.png differ diff --git a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Check settings page.png b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Check settings page.png index 34a05696c..0ad8614a3 100644 Binary files a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Check settings page.png and b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Check settings page.png differ diff --git a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Deep links open one section page and unknown sections redirect.png b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Deep links open one section page and unknown sections redirect.png index 18f7992d2..5d9426b3b 100644 Binary files a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Deep links open one section page and unknown sections redirect.png and b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Deep links open one section page and unknown sections redirect.png differ diff --git a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Leaving with unsaved edits asks for confirmation.png b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Leaving with unsaved edits asks for confirmation.png index b63a79e02..c3fa7f9ff 100644 Binary files a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Leaving with unsaved edits asks for confirmation.png and b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Leaving with unsaved edits asks for confirmation.png differ diff --git a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Save-and-leave persists the staged edit.png b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Save-and-leave persists the staged edit.png index 06cb9eb42..6b1ad7296 100644 Binary files a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Save-and-leave persists the staged edit.png and b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Save-and-leave persists the staged edit.png differ diff --git a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Unsaved bar survives section switches and discard reverts.png b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Unsaved bar survives section switches and discard reverts.png index 9f075ac73..885f67d76 100644 Binary files a/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Unsaved bar survives section switches and discard reverts.png and b/apps/web-e2e/dist/.playwright/apps/web-e2e/screenshots/settings/@settings @web Unsaved bar survives section switches and discard reverts.png differ diff --git a/docs/architecture/performance-journeys.md b/docs/architecture/performance-journeys.md index e2352362b..e32387ce2 100644 --- a/docs/architecture/performance-journeys.md +++ b/docs/architecture/performance-journeys.md @@ -57,6 +57,58 @@ The measurement script is `tools/performance/measure-initial-bytes.mjs`; its Node tests run with `pnpm nx test performance-tools` (Tier B in the coverage policy) and lint with `pnpm nx lint performance-tools`. +## Ratchet + +`tools/performance/journey-baselines.json` holds one entry per journey and +counter: + +```json +{ + "journeys": { + "launch": { + "renderer.initialBytes": { + "value": 2739510, + "unit": "bytes", + "updatedAt": "2026-09-26", + "evidencePr": 1693, + "measuredWith": "pnpm nx build web && pnpm run perf:initial-bytes" + } + } + } +} +``` + +`tools/performance/check-journey-ratchet.mjs` compares a journey summary with +that file: + +- a counter above its `value` fails; counters are exact, there is no slack; +- a wall-clock entry carries `toleranceRatio` and fails above + `value × toleranceRatio`; +- a baseline with no measurement in the summary fails, so dropping a + measurement cannot disable the ratchet; a counter is read only from + `journeys..counters` and a wall-clock entry (one with + `toleranceRatio`) only from `journeys..wallClock`, so a value in + the wrong section also counts as missing; +- a measurement below its baseline passes and prints a "tighten" hint; +- a measured counter without a baseline is noted, not failed; +- checking nothing fails: an empty baselines file, or `--only` naming an + entry that does not exist, cannot exit 0. + +`--only /` (repeatable) restricts the check to the named +baselines. A script that measures one counter writes its own summary file +and checks only its counter, so it neither overwrites another measurement's +summary nor fails the other baselines as unmeasured. + +```bash +pnpm run perf:initial-bytes:check # measure dist/apps/web into dist/performance/initial-bytes.summary.json, check only that counter +pnpm run perf:ratchet:check # check every baseline against dist/performance/journey-summary.json +``` + +Baselines only move down. Lower `value` in the same PR as the change that +earned it, set `updatedAt` and `evidencePr`, and paste the measurement output +into the PR. Never raise a value to make a PR pass: if growth is a deliberate +trade-off, say so in the PR and let the maintainer decide. + ## Adding a counter 1. Produce the value from the built output or from a deterministic probe, not diff --git a/docs/architecture/validation-map.md b/docs/architecture/validation-map.md index 76944e692..94bc34017 100644 --- a/docs/architecture/validation-map.md +++ b/docs/architecture/validation-map.md @@ -157,12 +157,15 @@ audit. ```bash pnpm nx build web -pnpm run perf:initial-bytes +pnpm run perf:initial-bytes # breakdown only +pnpm run perf:initial-bytes:check # measure, then compare with the committed baseline pnpm nx test performance-tools ``` `perf:initial-bytes` reads the built `dist/apps/web/index.html` and sums the -bytes on the initial path (the J1 counter `renderer.initialBytes`). The +bytes on the initial path (the J1 counter `renderer.initialBytes`). +`perf:initial-bytes:check` then fails if the value exceeds +`tools/performance/journey-baselines.json`; baselines only move down. The contract, what counts and how to add a counter are in the [performance journeys](performance-journeys.md) document. diff --git a/package.json b/package.json index 3069235f5..aadc8eb5d 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,9 @@ "build:website": "nx build website", "i18n:check": "node tools/i18n/check-drift.mjs", "perf:initial-bytes": "node tools/performance/measure-initial-bytes.mjs", - "perf:tools:test": "node --test tools/performance/measure-initial-bytes.test.mjs", + "perf:initial-bytes:check": "node tools/performance/measure-initial-bytes.mjs --summary dist/performance/initial-bytes.summary.json && node tools/performance/check-journey-ratchet.mjs --summary dist/performance/initial-bytes.summary.json --only launch/renderer.initialBytes", + "perf:ratchet:check": "node tools/performance/check-journey-ratchet.mjs --summary dist/performance/journey-summary.json", + "perf:tools:test": "node --test tools/performance/measure-initial-bytes.test.mjs tools/performance/check-journey-ratchet.test.mjs", "agents:validate": "node tools/skills/validate-agent-guidance.mjs", "skills:validate": "node tools/skills/validate-repository-skills.mjs", "release:artwork:dry-run": "tsx --tsconfig tsconfig.base.json tools/release/generate-marketing-artwork.ts --dry-run", diff --git a/tools/performance/check-journey-ratchet.mjs b/tools/performance/check-journey-ratchet.mjs new file mode 100644 index 000000000..916566497 --- /dev/null +++ b/tools/performance/check-journey-ratchet.mjs @@ -0,0 +1,297 @@ +/** + * Compares a journey summary (written by the measurement scripts) against the + * committed baselines in tools/performance/journey-baselines.json. + * + * Rules, from docs/architecture/performance-journeys.md: + * - A counter above its baseline fails. Counters are exact; there is no slack. + * - An entry with `toleranceRatio` (wall-clock) fails above + * `value * toleranceRatio`. + * - A baseline without a measurement fails, so removing a measurement can + * never silently disable the ratchet. + * - A measurement below its baseline prints a "can tighten" hint. Baselines + * are lowered by hand, with the measured output as evidence, never raised. + * - Checking nothing is a failure: an empty baselines file, or `--only` + * naming an entry that does not exist, must not exit 0. + * + * `--only /` (repeatable) restricts the check to the named + * baselines, so a script that measures one counter can check that counter + * without every other baseline failing as unmeasured. + * + * Usage: + * node tools/performance/check-journey-ratchet.mjs --summary + * [--baselines tools/performance/journey-baselines.json] + * [--only launch/renderer.initialBytes ...] + */ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export const DEFAULT_BASELINES_PATH = + 'tools/performance/journey-baselines.json'; + +function formatNumber(value) { + return Number.isInteger(value) + ? value.toLocaleString('en-US') + : value.toLocaleString('en-US', { maximumFractionDigits: 2 }); +} + +function isPlainObject(value) { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function validateBaselines(baselines) { + if (!isPlainObject(baselines) || !isPlainObject(baselines.journeys)) { + throw new Error( + 'Baselines file must be an object with a "journeys" map.' + ); + } + for (const [journey, entries] of Object.entries(baselines.journeys)) { + if (!isPlainObject(entries)) { + throw new Error( + `Baseline journey "${journey}" must map counter names to entries.` + ); + } + for (const [name, entry] of Object.entries(entries)) { + const label = `${journey}/${name}`; + if (!isPlainObject(entry) || !Number.isFinite(entry.value)) { + throw new Error( + `Baseline ${label} needs a finite numeric "value".` + ); + } + if ( + entry.toleranceRatio !== undefined && + !( + Number.isFinite(entry.toleranceRatio) && + entry.toleranceRatio >= 1 + ) + ) { + throw new Error( + `Baseline ${label} has an invalid "toleranceRatio" (must be >= 1).` + ); + } + } + } + return baselines; +} + +/** + * A baseline entry is either a counter (exact) or a wall-clock entry (carries + * `toleranceRatio`), and each is read only from its own summary section. A + * value that moved to the other section is treated as missing, so a summary + * schema regression fails the ratchet instead of slipping through it. + */ +function summarySection(entry) { + return entry.toleranceRatio !== undefined ? 'wallClock' : 'counters'; +} + +function measuredValue(summaryJourney, name, entry) { + if (!isPlainObject(summaryJourney)) return undefined; + return summaryJourney[summarySection(entry)]?.[name]; +} + +/** + * Pure comparison. Returns every outcome so the CLI and tests can render it; + * `failures` non-empty means the ratchet is broken. + */ +export function compareToBaselines({ baselines, summary, only = [] }) { + validateBaselines(baselines); + const summaryJourneys = isPlainObject(summary?.journeys) + ? summary.journeys + : {}; + const result = { + failures: [], + tightenable: [], + passed: [], + unbaselined: [], + }; + const selected = new Set(only); + let evaluated = 0; + + for (const target of only) { + const [journey, ...rest] = target.split('/'); + if (baselines.journeys[journey]?.[rest.join('/')] === undefined) { + result.failures.push( + `${target}: --only names a baseline that does not exist in the baselines file.` + ); + } + } + + for (const [journey, entries] of Object.entries(baselines.journeys)) { + for (const [name, entry] of Object.entries(entries)) { + const label = `${journey}/${name}`; + if (selected.size > 0 && !selected.has(label)) continue; + evaluated += 1; + const unit = entry.unit ? ` ${entry.unit}` : ''; + const measured = measuredValue( + summaryJourneys[journey], + name, + entry + ); + + if (measured === undefined) { + result.failures.push( + `${label}: baseline ${formatNumber(entry.value)}${unit} has no measurement under journeys.${journey}.${summarySection(entry)} in the summary. The ratchet cannot be bypassed by dropping a measurement; restore it.` + ); + continue; + } + if (typeof measured !== 'number' || !Number.isFinite(measured)) { + result.failures.push( + `${label}: measured value ${JSON.stringify(measured)} is not a finite number.` + ); + continue; + } + + const limit = + entry.toleranceRatio !== undefined + ? entry.value * entry.toleranceRatio + : entry.value; + const limitText = + entry.toleranceRatio !== undefined + ? `${formatNumber(limit)} (baseline ${formatNumber(entry.value)} × ${entry.toleranceRatio})` + : `baseline ${formatNumber(entry.value)}`; + + if (measured > limit) { + result.failures.push( + `${label}: ${formatNumber(measured)}${unit} exceeds ${limitText} by ${formatNumber(measured - limit)}${unit}. Bring the value back down; baselines only move down. If the growth is a deliberate trade-off, say so in the PR and let the maintainer decide.` + ); + } else if (measured < entry.value) { + result.tightenable.push( + `${label}: ${formatNumber(measured)}${unit} is below baseline ${formatNumber(entry.value)} by ${formatNumber(entry.value - measured)}${unit}. Lower the baseline in ${DEFAULT_BASELINES_PATH} with this run as evidence.` + ); + } else { + result.passed.push( + `${label}: ${formatNumber(measured)}${unit} within ${limitText}.` + ); + } + } + } + + for (const [journey, summaryJourney] of Object.entries(summaryJourneys)) { + const measuredNames = [ + ...Object.keys(summaryJourney?.counters ?? {}), + ...Object.keys(summaryJourney?.wallClock ?? {}), + ]; + for (const name of measuredNames) { + if (baselines.journeys[journey]?.[name] === undefined) { + result.unbaselined.push( + `${journey}/${name}: measured but has no baseline yet. Add one to ${DEFAULT_BASELINES_PATH} once the counter is validated.` + ); + } + } + } + + if (evaluated === 0) { + result.failures.push( + `No baselines were checked. ${DEFAULT_BASELINES_PATH} must keep at least one entry; a ratchet that guards nothing must not pass.` + ); + } + + return result; +} + +export function formatResult(result) { + const lines = []; + for (const line of result.passed) lines.push(`ok ${line}`); + for (const line of result.tightenable) lines.push(`tighten ${line}`); + for (const line of result.unbaselined) lines.push(`note ${line}`); + for (const line of result.failures) lines.push(`FAIL ${line}`); + const checked = + result.passed.length + + result.tightenable.length + + result.failures.length; + lines.push( + result.failures.length > 0 + ? `Journey ratchet failed: ${result.failures.length} of ${checked} baselines exceeded.` + : `Journey ratchet OK: ${checked} baselines checked, ${result.tightenable.length} can be tightened.` + ); + return lines.join('\n'); +} + +export function parseArgs(argv) { + const options = { + summary: null, + baselines: DEFAULT_BASELINES_PATH, + only: [], + }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === '--') continue; + if (argument === '--summary') { + options.summary = argv[++index]; + } else if (argument.startsWith('--summary=')) { + options.summary = argument.slice('--summary='.length); + } else if (argument === '--baselines') { + options.baselines = argv[++index]; + } else if (argument.startsWith('--baselines=')) { + options.baselines = argument.slice('--baselines='.length); + } else if (argument === '--only') { + options.only.push(argv[++index]); + } else if (argument.startsWith('--only=')) { + options.only.push(argument.slice('--only='.length)); + } else { + throw new Error(`Unknown argument: ${argument}`); + } + if ( + options.summary === undefined || + options.baselines === undefined || + options.only.includes(undefined) + ) { + throw new Error(`Missing value for ${argument}`); + } + } + for (const target of options.only) { + if (!/^[^/]+\/.+$/.test(target)) { + throw new Error( + `--only expects /, received "${target}".` + ); + } + } + if (!options.summary) { + throw new Error('--summary is required.'); + } + return options; +} + +async function readJson(filePath, description) { + try { + return JSON.parse(await readFile(filePath, 'utf8')); + } catch (error) { + throw new Error( + `Cannot read ${description} at ${filePath}: ${error.message}` + ); + } +} + +const isMain = + process.argv[1] && + path.resolve(process.argv[1]) === + path.resolve(fileURLToPath(import.meta.url)); + +if (isMain) { + try { + const options = parseArgs(process.argv.slice(2)); + const baselines = await readJson( + path.resolve(options.baselines), + 'baselines' + ); + const summary = await readJson( + path.resolve(options.summary), + 'journey summary' + ); + const result = compareToBaselines({ + baselines, + summary, + only: options.only, + }); + const output = formatResult(result); + if (result.failures.length > 0) { + console.error(output); + process.exitCode = 1; + } else { + console.log(output); + } + } catch (error) { + console.error(`check-journey-ratchet: ${error.message}`); + process.exitCode = 1; + } +} diff --git a/tools/performance/check-journey-ratchet.test.mjs b/tools/performance/check-journey-ratchet.test.mjs new file mode 100644 index 000000000..2f3d65cf9 --- /dev/null +++ b/tools/performance/check-journey-ratchet.test.mjs @@ -0,0 +1,413 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { after, before, test } from 'node:test'; + +import { + DEFAULT_BASELINES_PATH, + compareToBaselines, + formatResult, + parseArgs, + validateBaselines, +} from './check-journey-ratchet.mjs'; + +const scriptPath = fileURLToPath( + new URL('./check-journey-ratchet.mjs', import.meta.url) +); +const committedBaselinesPath = fileURLToPath( + new URL('./journey-baselines.json', import.meta.url) +); + +const baselines = { + version: 1, + journeys: { + launch: { + 'renderer.initialBytes': { value: 2750491, unit: 'bytes' }, + spawnToFirstCardMs: { + value: 1000, + unit: 'ms', + toleranceRatio: 1.25, + }, + }, + }, +}; + +const summaryWith = (initialBytes, wallClockMs = 1000) => ({ + journeys: { + launch: { + counters: { 'renderer.initialBytes': initialBytes }, + wallClock: { spawnToFirstCardMs: wallClockMs }, + }, + }, +}); + +let workDir; + +before(async () => { + workDir = await mkdtemp(path.join(os.tmpdir(), 'check-journey-ratchet-')); +}); + +after(async () => { + await rm(workDir, { recursive: true, force: true }); +}); + +test('a counter equal to its baseline passes', () => { + const result = compareToBaselines({ + baselines, + summary: summaryWith(2750491), + }); + assert.deepEqual(result.failures, []); + assert.deepEqual(result.tightenable, []); + assert.equal(result.passed.length, 2); + assert.match( + result.passed[0], + /launch\/renderer\.initialBytes: 2,750,491 bytes within baseline 2,750,491/ + ); +}); + +test('a counter one byte above its baseline fails with the delta', () => { + const result = compareToBaselines({ + baselines, + summary: summaryWith(2750492), + }); + assert.equal(result.failures.length, 1); + assert.match( + result.failures[0], + /launch\/renderer\.initialBytes: 2,750,492 bytes exceeds baseline 2,750,491 by 1 bytes/ + ); + assert.match(result.failures[0], /baselines only move down/); +}); + +test('a counter below its baseline passes and asks to tighten', () => { + const result = compareToBaselines({ + baselines, + summary: summaryWith(2600000), + }); + assert.deepEqual(result.failures, []); + assert.equal(result.tightenable.length, 1); + assert.match( + result.tightenable[0], + /below baseline 2,750,491 by 150,491 bytes/ + ); + assert.match( + result.tightenable[0], + new RegExp(DEFAULT_BASELINES_PATH.replace(/\//g, '\\/')) + ); +}); + +test('wall-clock entries fail only above value × toleranceRatio', () => { + const within = compareToBaselines({ + baselines, + summary: summaryWith(2750491, 1250), + }); + assert.deepEqual(within.failures, []); + assert.match( + within.passed[1], + /1,250 ms within 1,250 \(baseline 1,000 × 1\.25\)/ + ); + + const above = compareToBaselines({ + baselines, + summary: summaryWith(2750491, 1251), + }); + assert.equal(above.failures.length, 1); + assert.match( + above.failures[0], + /spawnToFirstCardMs: 1,251 ms exceeds 1,250 \(baseline 1,000 × 1\.25\) by 1 ms/ + ); + + const below = compareToBaselines({ + baselines, + summary: summaryWith(2750491, 900), + }); + assert.equal(below.tightenable.length, 1); + assert.match( + below.tightenable[0], + /spawnToFirstCardMs: 900 ms is below baseline 1,000/ + ); +}); + +test('a baseline without a measurement fails instead of being skipped', () => { + const result = compareToBaselines({ + baselines, + summary: { journeys: { launch: { counters: {} } } }, + }); + assert.equal(result.failures.length, 2); + assert.match( + result.failures[0], + /renderer\.initialBytes: baseline 2,750,491 bytes has no measurement/ + ); + assert.match( + result.failures[0], + /cannot be bypassed by dropping a measurement/ + ); +}); + +test('a value in the wrong summary section counts as missing', () => { + const counterInWallClock = compareToBaselines({ + baselines, + summary: { + journeys: { + launch: { + counters: {}, + wallClock: { + 'renderer.initialBytes': 2750491, + spawnToFirstCardMs: 1000, + }, + }, + }, + }, + }); + assert.equal(counterInWallClock.failures.length, 1); + assert.match( + counterInWallClock.failures[0], + /renderer\.initialBytes: baseline 2,750,491 bytes has no measurement under journeys\.launch\.counters/ + ); + + const wallClockInCounters = compareToBaselines({ + baselines, + summary: { + journeys: { + launch: { + counters: { + 'renderer.initialBytes': 2750491, + spawnToFirstCardMs: 1000, + }, + }, + }, + }, + }); + assert.equal(wallClockInCounters.failures.length, 1); + assert.match( + wallClockInCounters.failures[0], + /spawnToFirstCardMs: baseline 1,000 ms has no measurement under journeys\.launch\.wallClock/ + ); +}); + +test('an empty or malformed summary fails every baseline', () => { + assert.equal( + compareToBaselines({ baselines, summary: {} }).failures.length, + 2 + ); + assert.equal( + compareToBaselines({ baselines, summary: null }).failures.length, + 2 + ); + const nonNumeric = compareToBaselines({ + baselines, + summary: { + journeys: { + launch: { counters: { 'renderer.initialBytes': '2750491' } }, + }, + }, + }); + assert.match(nonNumeric.failures[0], /"2750491" is not a finite number/); +}); + +test('a measured counter without a baseline is reported but does not fail', () => { + const summary = summaryWith(2750491); + summary.journeys.launch.counters['renderer.cdTicksToFirstCard'] = 12; + summary.journeys.search = { counters: { sqlStatementsPerKeystroke: 3 } }; + const result = compareToBaselines({ baselines, summary }); + assert.deepEqual(result.failures, []); + assert.deepEqual( + result.unbaselined.map((line) => line.split(':')[0]), + [ + 'launch/renderer.cdTicksToFirstCard', + 'search/sqlStatementsPerKeystroke', + ] + ); +}); + +test('an empty baselines file fails instead of passing vacuously', () => { + const result = compareToBaselines({ + baselines: { journeys: {} }, + summary: summaryWith(2750491), + }); + assert.equal(result.failures.length, 1); + assert.match(result.failures[0], /No baselines were checked/); + assert.match( + formatResult(result), + /Journey ratchet failed: 1 of 1 baselines exceeded/ + ); +}); + +test('--only restricts the check to the named baselines', () => { + const result = compareToBaselines({ + baselines, + summary: { + journeys: { + launch: { counters: { 'renderer.initialBytes': 2750491 } }, + }, + }, + only: ['launch/renderer.initialBytes'], + }); + assert.deepEqual(result.failures, []); + assert.equal(result.passed.length, 1); + assert.match(result.passed[0], /launch\/renderer\.initialBytes/); +}); + +test('--only naming a missing baseline fails rather than checking nothing', () => { + const result = compareToBaselines({ + baselines, + summary: summaryWith(2750491), + only: ['launch/does.notExist'], + }); + assert.equal(result.failures.length, 2); + assert.match( + result.failures[0], + /launch\/does\.notExist: --only names a baseline that does not exist/ + ); + assert.match(result.failures[1], /No baselines were checked/); +}); + +test('rejects malformed baseline files', () => { + assert.throws(() => validateBaselines({}), /object with a "journeys" map/); + assert.throws( + () => validateBaselines({ journeys: { launch: [] } }), + /journey "launch" must map/ + ); + assert.throws( + () => + validateBaselines({ + journeys: { launch: { x: { value: 'big' } } }, + }), + /Baseline launch\/x needs a finite numeric "value"/ + ); + assert.throws( + () => + validateBaselines({ + journeys: { launch: { x: { value: 1, toleranceRatio: 0.5 } } }, + }), + /invalid "toleranceRatio"/ + ); +}); + +test('formats a summary line for both outcomes', () => { + const ok = formatResult( + compareToBaselines({ baselines, summary: summaryWith(2600000) }) + ); + assert.match(ok, /^ok {7}launch\/spawnToFirstCardMs/m); + assert.match(ok, /^tighten {2}launch\/renderer\.initialBytes/m); + assert.match( + ok, + /Journey ratchet OK: 2 baselines checked, 1 can be tightened\.$/ + ); + + const failed = formatResult( + compareToBaselines({ baselines, summary: summaryWith(3000000) }) + ); + assert.match(failed, /^FAIL {5}launch\/renderer\.initialBytes/m); + assert.match( + failed, + /Journey ratchet failed: 1 of 2 baselines exceeded\.$/ + ); +}); + +test('parses CLI arguments and requires --summary', () => { + assert.deepEqual(parseArgs(['--summary', 's.json']), { + summary: 's.json', + baselines: DEFAULT_BASELINES_PATH, + only: [], + }); + assert.deepEqual( + parseArgs([ + '--', + '--summary=s.json', + '--baselines=b.json', + '--only', + 'launch/a', + '--only=search/b', + ]), + { + summary: 's.json', + baselines: 'b.json', + only: ['launch/a', 'search/b'], + } + ); + assert.throws( + () => parseArgs(['--summary', 's', '--only', 'launch']), + /--only expects \// + ); + assert.throws( + () => parseArgs(['--summary', 's', '--only']), + /Missing value for --only/ + ); + assert.throws( + () => parseArgs([]), + /--summary is required/ + ); + assert.throws( + () => parseArgs(['--summary']), + /Missing value for --summary/ + ); + assert.throws( + () => parseArgs(['--summary', 's', '--strict']), + /Unknown argument: --strict/ + ); +}); + +test('the committed baselines file is valid and every entry names its evidence fields', async () => { + const committed = JSON.parse( + await readFile(committedBaselinesPath, 'utf8') + ); + validateBaselines(committed); + assert.ok( + committed.journeys.launch?.['renderer.initialBytes'], + 'the launch/renderer.initialBytes baseline must stay committed; the CI ratchet guards it' + ); + for (const entries of Object.values(committed.journeys)) { + for (const entry of Object.values(entries)) { + assert.match(entry.updatedAt, /^\d{4}-\d{2}-\d{2}$/); + assert.ok( + 'evidencePr' in entry, + 'evidencePr must be present (null before the first PR)' + ); + assert.equal(typeof entry.measuredWith, 'string'); + } + } +}); + +async function runCli(summary, extraBaselines = baselines) { + const summaryPath = path.join( + workDir, + `summary-${Date.now()}-${Math.random()}.json` + ); + const baselinesPath = path.join( + workDir, + `baselines-${Date.now()}-${Math.random()}.json` + ); + await writeFile(summaryPath, JSON.stringify(summary)); + await writeFile(baselinesPath, JSON.stringify(extraBaselines)); + return spawnSync( + process.execPath, + [scriptPath, '--summary', summaryPath, '--baselines', baselinesPath], + { encoding: 'utf8' } + ); +} + +test('CLI exits 0 when within baselines and 1 when a counter grew', async () => { + const ok = await runCli(summaryWith(2750491)); + assert.equal(ok.status, 0, ok.stderr); + assert.match(ok.stdout, /Journey ratchet OK/); + + const grew = await runCli(summaryWith(2750492)); + assert.equal(grew.status, 1); + assert.match(grew.stderr, /FAIL {5}launch\/renderer\.initialBytes/); +}); + +test('CLI exits 1 with a readable message when the summary is missing', () => { + const result = spawnSync( + process.execPath, + [scriptPath, '--summary', path.join(workDir, 'nope.json')], + { encoding: 'utf8' } + ); + assert.equal(result.status, 1); + assert.match( + result.stderr, + /check-journey-ratchet: Cannot read journey summary at/ + ); +}); diff --git a/tools/performance/journey-baselines.json b/tools/performance/journey-baselines.json new file mode 100644 index 000000000..b2c0a34d5 --- /dev/null +++ b/tools/performance/journey-baselines.json @@ -0,0 +1,14 @@ +{ + "version": 1, + "journeys": { + "launch": { + "renderer.initialBytes": { + "value": 2739510, + "unit": "bytes", + "updatedAt": "2026-09-26", + "evidencePr": 1693, + "measuredWith": "pnpm nx build web && pnpm run perf:initial-bytes" + } + } + } +} diff --git a/tools/performance/project.json b/tools/performance/project.json index 44afb96eb..ca364bde7 100644 --- a/tools/performance/project.json +++ b/tools/performance/project.json @@ -14,7 +14,7 @@ { "externalDependencies": ["parse5"] } ], "options": { - "command": "node --test tools/performance/measure-initial-bytes.test.mjs", + "command": "node --test tools/performance/measure-initial-bytes.test.mjs tools/performance/check-journey-ratchet.test.mjs", "cwd": "{workspaceRoot}" } },