/** * Compares a journey summary (written by the measurement scripts) against the * committed baselines in tools/performance/journey-baselines.json. * * Rules, from docs/architecture/performance-journeys.md: * - A counter above `value + slack` fails. Counters are exact; `slack` * (default 0, in the entry's unit) absorbs bundler noise and concurrent * merges without letting growth accumulate past it. * - An entry with `toleranceRatio` (wall-clock) fails above * `value * toleranceRatio`. * - A baseline without a measurement fails, so removing a measurement can * never silently disable the ratchet. * - A measurement below its baseline prints a "can tighten" hint. Baselines * are lowered by hand, with the measured output as evidence, never raised. * - Checking nothing is a failure: an empty baselines file, or `--only` * naming an entry that does not exist, must not exit 0. * - An optional `note` (a string) says why an entry is enforced, such as a * guard whose link to wall-clock is not validated (Principle 3). It is * printed with a failure, so the author sees what the counter stands for. * * `--only /` (repeatable) restricts the check to the named * baselines, so a script that measures one counter can check that counter * without every other baseline failing as unmeasured. * * Usage: * node tools/performance/check-journey-ratchet.mjs --summary * [--baselines tools/performance/journey-baselines.json] * [--only launch/renderer.initialBytes ...] */ import { readFile } from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; export const DEFAULT_BASELINES_PATH = 'tools/performance/journey-baselines.json'; /** The PR label that lets check-baseline-direction.mjs accept a weakening. */ export const BASELINE_INCREASE_LABEL = 'perf-baseline-increase'; // Three decimals: journey summaries round layout-shift scores to three. function formatNumber(value) { return Number.isInteger(value) ? value.toLocaleString('en-US') : value.toLocaleString('en-US', { maximumFractionDigits: 3 }); } function isPlainObject(value) { return typeof value === 'object' && value !== null && !Array.isArray(value); } export function validateBaselines(baselines) { if (!isPlainObject(baselines) || !isPlainObject(baselines.journeys)) { throw new Error( 'Baselines file must be an object with a "journeys" map.' ); } for (const [journey, entries] of Object.entries(baselines.journeys)) { if (!isPlainObject(entries)) { throw new Error( `Baseline journey "${journey}" must map counter names to entries.` ); } for (const [name, entry] of Object.entries(entries)) { const label = `${journey}/${name}`; if (!isPlainObject(entry) || !Number.isFinite(entry.value)) { throw new Error( `Baseline ${label} needs a finite numeric "value".` ); } if ( entry.toleranceRatio !== undefined && !( Number.isFinite(entry.toleranceRatio) && entry.toleranceRatio >= 1 ) ) { throw new Error( `Baseline ${label} has an invalid "toleranceRatio" (must be >= 1).` ); } if ( entry.slack !== undefined && !(Number.isInteger(entry.slack) && entry.slack >= 0) ) { throw new Error( `Baseline ${label} has an invalid "slack" (must be an integer >= 0).` ); } if (entry.note !== undefined && typeof entry.note !== 'string') { throw new Error(`Baseline ${label} has a non-string "note".`); } if ( entry.slack !== undefined && entry.toleranceRatio !== undefined ) { throw new Error( `Baseline ${label} sets both "slack" and "toleranceRatio"; slack is for counters, toleranceRatio for wall-clock entries.` ); } } } return baselines; } /** * A baseline entry is either a counter (exact) or a wall-clock entry (carries * `toleranceRatio`), and each is read only from its own summary section. A * value that moved to the other section is treated as missing, so a summary * schema regression fails the ratchet instead of slipping through it. */ function summarySection(entry) { return entry.toleranceRatio !== undefined ? 'wallClock' : 'counters'; } /** * What the ratchet enforces: `value × toleranceRatio` for a wall-clock entry, * `value + slack` for a counter. check-baseline-direction.mjs compares this, * not the bare value, so a PR cannot lower a value while widening a tolerance. */ export function enforcedLimit(entry) { return entry.value * (entry.toleranceRatio ?? 1) + (entry.slack ?? 0); } function describeLimit(entry, unit) { if (entry.toleranceRatio !== undefined) { return `${formatNumber(enforcedLimit(entry))} (baseline ${formatNumber(entry.value)} × ${entry.toleranceRatio})`; } if (entry.slack) { return `${formatNumber(enforcedLimit(entry))} (baseline ${formatNumber(entry.value)} + slack ${formatNumber(entry.slack)}${unit})`; } return `baseline ${formatNumber(entry.value)}`; } function measuredValue(summaryJourney, name, entry) { if (!isPlainObject(summaryJourney)) return undefined; return summaryJourney[summarySection(entry)]?.[name]; } /** * Pure comparison. Returns every outcome so the CLI and tests can render it; * `failures` non-empty means the ratchet is broken. */ export function compareToBaselines({ baselines, summary, only = [] }) { validateBaselines(baselines); const summaryJourneys = isPlainObject(summary?.journeys) ? summary.journeys : {}; const result = { failures: [], tightenable: [], passed: [], unbaselined: [], }; const selected = new Set(only); let evaluated = 0; for (const target of only) { const [journey, ...rest] = target.split('/'); if (baselines.journeys[journey]?.[rest.join('/')] === undefined) { result.failures.push( `${target}: --only names a baseline that does not exist in the baselines file.` ); } } for (const [journey, entries] of Object.entries(baselines.journeys)) { for (const [name, entry] of Object.entries(entries)) { const label = `${journey}/${name}`; if (selected.size > 0 && !selected.has(label)) continue; evaluated += 1; const unit = entry.unit ? ` ${entry.unit}` : ''; const measured = measuredValue( summaryJourneys[journey], name, entry ); if (measured === undefined) { result.failures.push( `${label}: baseline ${formatNumber(entry.value)}${unit} has no measurement under journeys.${journey}.${summarySection(entry)} in the summary. The ratchet cannot be bypassed by dropping a measurement; restore it.` ); continue; } if (typeof measured !== 'number' || !Number.isFinite(measured)) { result.failures.push( `${label}: measured value ${JSON.stringify(measured)} is not a finite number.` ); continue; } const limit = enforcedLimit(entry); const limitText = describeLimit(entry, unit); if (measured > limit) { const note = entry.note ? ` Note: ${entry.note}` : ''; result.failures.push( `${label}: ${formatNumber(measured)}${unit} exceeds ${limitText} by ${formatNumber(measured - limit)}${unit}. Bring the value back down; baselines only move down. If the growth is a deliberate trade-off, raise the baseline in ${DEFAULT_BASELINES_PATH}, make the case in the PR, and ask a maintainer to add the ${BASELINE_INCREASE_LABEL} label.${note}` ); } else if (entry.slack && measured > entry.value) { result.passed.push( `${label}: ${formatNumber(measured)}${unit} within ${limitText}; uses ${formatNumber(measured - entry.value)} of ${formatNumber(limit - entry.value)}${unit} slack.` ); } else if (measured < entry.value) { result.tightenable.push( `${label}: ${formatNumber(measured)}${unit} is below baseline ${formatNumber(entry.value)} by ${formatNumber(entry.value - measured)}${unit}. Lower the baseline in ${DEFAULT_BASELINES_PATH} with this run as evidence.` ); } else { result.passed.push( `${label}: ${formatNumber(measured)}${unit} within ${limitText}.` ); } } } for (const [journey, summaryJourney] of Object.entries(summaryJourneys)) { const measuredNames = [ ...Object.keys(summaryJourney?.counters ?? {}), ...Object.keys(summaryJourney?.wallClock ?? {}), ]; for (const name of measuredNames) { if (baselines.journeys[journey]?.[name] === undefined) { result.unbaselined.push( `${journey}/${name}: measured but has no baseline yet. Add one to ${DEFAULT_BASELINES_PATH} once the counter is validated.` ); } } } if (evaluated === 0) { result.failures.push( `No baselines were checked. ${DEFAULT_BASELINES_PATH} must keep at least one entry; a ratchet that guards nothing must not pass.` ); } return result; } export function formatResult(result) { const lines = []; for (const line of result.passed) lines.push(`ok ${line}`); for (const line of result.tightenable) lines.push(`tighten ${line}`); for (const line of result.unbaselined) lines.push(`note ${line}`); for (const line of result.failures) lines.push(`FAIL ${line}`); const checked = result.passed.length + result.tightenable.length + result.failures.length; lines.push( result.failures.length > 0 ? `Journey ratchet failed: ${result.failures.length} of ${checked} baselines exceeded.` : `Journey ratchet OK: ${checked} baselines checked, ${result.tightenable.length} can be tightened.` ); return lines.join('\n'); } export function parseArgs(argv) { const options = { summary: null, baselines: DEFAULT_BASELINES_PATH, only: [], }; for (let index = 0; index < argv.length; index += 1) { const argument = argv[index]; if (argument === '--') continue; if (argument === '--summary') { options.summary = argv[++index]; } else if (argument.startsWith('--summary=')) { options.summary = argument.slice('--summary='.length); } else if (argument === '--baselines') { options.baselines = argv[++index]; } else if (argument.startsWith('--baselines=')) { options.baselines = argument.slice('--baselines='.length); } else if (argument === '--only') { options.only.push(argv[++index]); } else if (argument.startsWith('--only=')) { options.only.push(argument.slice('--only='.length)); } else { throw new Error(`Unknown argument: ${argument}`); } if ( options.summary === undefined || options.baselines === undefined || options.only.includes(undefined) ) { throw new Error(`Missing value for ${argument}`); } } for (const target of options.only) { if (!/^[^/]+\/.+$/.test(target)) { throw new Error( `--only expects /, received "${target}".` ); } } if (!options.summary) { throw new Error('--summary is required.'); } return options; } async function readJson(filePath, description) { try { return JSON.parse(await readFile(filePath, 'utf8')); } catch (error) { throw new Error( `Cannot read ${description} at ${filePath}: ${error.message}` ); } } const isMain = process.argv[1] && path.resolve(process.argv[1]) === path.resolve(fileURLToPath(import.meta.url)); if (isMain) { try { const options = parseArgs(process.argv.slice(2)); const baselines = await readJson( path.resolve(options.baselines), 'baselines' ); const summary = await readJson( path.resolve(options.summary), 'journey summary' ); const result = compareToBaselines({ baselines, summary, only: options.only, }); const output = formatResult(result); if (result.failures.length > 0) { console.error(output); process.exitCode = 1; } else { console.log(output); } } catch (error) { console.error(`check-journey-ratchet: ${error.message}`); process.exitCode = 1; } }