/** * Refuses a change that weakens tools/performance/journey-baselines.json. * * The ratchet job compares a measurement with the baselines file of the same * commit, so on its own it cannot tell a genuine payload reduction from a PR * that grows the payload and raises the baseline by the same amount. This * check closes that gap: given the baselines file of the target branch and * the one of the PR, any entry whose enforced limit (`value × toleranceRatio`) * went up, whose tolerance widened, or that disappeared, is a failure. New * entries and lowered limits pass. * * Usage: * node tools/performance/check-baseline-direction.mjs \ * --base \ * --head tools/performance/journey-baselines.json * * A missing --base file means the target branch has no baselines yet, so * there is nothing that could have been weakened. */ import { existsSync } from 'node:fs'; import { readFile } from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { validateBaselines } from './check-journey-ratchet.mjs'; export const DEFAULT_HEAD_PATH = 'tools/performance/journey-baselines.json'; function entries(baselines) { const flat = new Map(); for (const [journey, counters] of Object.entries(baselines.journeys)) { for (const [name, entry] of Object.entries(counters)) { flat.set(`${journey}/${name}`, entry); } } return flat; } function formatNumber(value) { return value.toLocaleString('en-US', { maximumFractionDigits: 2 }); } /** * What the ratchet actually enforces: `value × toleranceRatio` for a * wall-clock entry, the bare value for a counter. Comparing values alone would * let a PR lower a value while widening the tolerance. */ function effectiveLimit(entry) { return entry.value * (entry.toleranceRatio ?? 1); } /** Pure comparison; `failures` non-empty means the change weakens the ratchet. */ export function compareBaselineDirection({ base, head }) { validateBaselines(base); validateBaselines(head); const result = { failures: [], lowered: [], unchanged: [], added: [] }; const headEntries = entries(head); for (const [label, baseEntry] of entries(base)) { const headEntry = headEntries.get(label); const unit = baseEntry.unit ? ` ${baseEntry.unit}` : ''; if (!headEntry) { result.failures.push( `${label}: baseline ${formatNumber(baseEntry.value)}${unit} was removed. Baselines are retired only by a maintainer decision recorded in the PR, not by deleting the entry.` ); continue; } const baseTolerance = baseEntry.toleranceRatio ?? 1; const headTolerance = headEntry.toleranceRatio ?? 1; const baseLimit = effectiveLimit(baseEntry); const headLimit = effectiveLimit(headEntry); if (headTolerance > baseTolerance) { result.failures.push( `${label}: toleranceRatio widened from ${baseTolerance} to ${headTolerance}. Tolerances are a maintainer decision; a PR may only narrow them.` ); } else if (headLimit > baseLimit) { result.failures.push( `${label}: baseline raised from ${formatNumber(baseLimit)} to ${formatNumber(headLimit)}${unit}. Baselines only move down; bring the measurement back under ${formatNumber(baseLimit)} or make the case for the increase in the PR.` ); } else if (headLimit < baseLimit) { result.lowered.push( `${label}: ${formatNumber(baseLimit)} -> ${formatNumber(headLimit)}${unit}.` ); } else { result.unchanged.push(label); } } for (const label of headEntries.keys()) { if (!entries(base).has(label)) result.added.push(label); } return result; } export function formatDirectionResult(result) { const lines = []; for (const line of result.lowered) lines.push(`lowered ${line}`); for (const label of result.added) lines.push(`added ${label}`); for (const line of result.failures) lines.push(`FAIL ${line}`); lines.push( result.failures.length > 0 ? `Baseline direction check failed: ${result.failures.length} entries raised or removed.` : `Baseline direction OK: ${result.unchanged.length} unchanged, ${result.lowered.length} lowered, ${result.added.length} added.` ); return lines.join('\n'); } export function parseArgs(argv) { const options = { base: null, head: DEFAULT_HEAD_PATH }; for (let index = 0; index < argv.length; index += 1) { const argument = argv[index]; if (argument === '--') continue; if (argument === '--base') { options.base = argv[++index]; } else if (argument.startsWith('--base=')) { options.base = argument.slice('--base='.length); } else if (argument === '--head') { options.head = argv[++index]; } else if (argument.startsWith('--head=')) { options.head = argument.slice('--head='.length); } else { throw new Error(`Unknown argument: ${argument}`); } if (options.base === undefined || options.head === undefined) { throw new Error(`Missing value for ${argument}`); } } if (!options.base) { throw new Error( '--base is required.' ); } return options; } async function readJson(filePath, description) { try { return JSON.parse(await readFile(filePath, 'utf8')); } catch (error) { throw new Error( `Cannot read ${description} at ${filePath}: ${error.message}` ); } } const isMain = process.argv[1] && path.resolve(process.argv[1]) === path.resolve(fileURLToPath(import.meta.url)); if (isMain) { try { const options = parseArgs(process.argv.slice(2)); const basePath = path.resolve(options.base); const base = existsSync(basePath) ? await readJson(basePath, 'target-branch baselines') : { version: 1, journeys: {} }; if (!existsSync(basePath)) { console.log( `No baselines file at ${options.base} on the target branch; nothing to weaken.` ); } const head = await readJson(path.resolve(options.head), 'baselines'); const result = compareBaselineDirection({ base, head }); const output = formatDirectionResult(result); if (result.failures.length > 0) { console.error(output); process.exitCode = 1; } else { console.log(output); } } catch (error) { console.error(`check-baseline-direction: ${error.message}`); process.exitCode = 1; } }