diff --git a/docs/architecture/performance-journeys.md b/docs/architecture/performance-journeys.md index 163e7dba2..4fbd9449a 100644 --- a/docs/architecture/performance-journeys.md +++ b/docs/architecture/performance-journeys.md @@ -1228,7 +1228,9 @@ that file: `journeys..counters` and a wall-clock entry (one with `toleranceRatio`) only from `journeys..wallClock`, so a value in the wrong section also counts as missing; -- a measurement below its baseline passes and prints a "tighten" hint; +- a measurement below its baseline passes and prints a "tighten" hint, + except for an entry with `"autoTighten": false`, whose band one low run + does not move; - a measured counter without a baseline is noted, not failed; - checking nothing fails: an empty baselines file, or `--only` naming an entry that does not exist, cannot exit 0; diff --git a/tools/performance/check-journey-ratchet.mjs b/tools/performance/check-journey-ratchet.mjs index 20bf9da0d..370deadae 100644 --- a/tools/performance/check-journey-ratchet.mjs +++ b/tools/performance/check-journey-ratchet.mjs @@ -12,6 +12,8 @@ * never silently disable the ratchet. * - A measurement below its baseline prints a "can tighten" hint. Baselines * are lowered by hand, with the measured output as evidence, never raised. + * An entry with `"autoTighten": false` gets no hint: its slack covers a band + * of normal values, so one low run is no evidence for a lower `value`. * - Checking nothing is a failure: an empty baselines file, or `--only` * naming an entry that does not exist, must not exit 0. * - An optional `note` (a string) says why an entry is enforced, such as a @@ -208,6 +210,10 @@ export function compareToBaselines({ baselines, summary, only = [] }) { result.passed.push( `${label}: ${formatNumber(measured)}${unit} within ${limitText}; uses ${formatNumber(measured - entry.value)} of ${formatNumber(limit - entry.value)}${unit} slack.` ); + } else if (measured < entry.value && entry.autoTighten === false) { + result.passed.push( + `${label}: ${formatNumber(measured)}${unit} within ${limitText}; below the baseline, but autoTighten is false: lower it by hand only when the top of its band fell too (docs/architecture/performance-journeys.md).` + ); } else if (measured < entry.value) { result.tightenable.push( `${label}: ${formatNumber(measured)}${unit} is below baseline ${formatNumber(entry.value)} by ${formatNumber(entry.value - measured)}${unit}. Lower the baseline in ${DEFAULT_BASELINES_PATH} with this run as evidence.` diff --git a/tools/performance/check-journey-ratchet.test.mjs b/tools/performance/check-journey-ratchet.test.mjs index 4d0386b13..a424ae0de 100644 --- a/tools/performance/check-journey-ratchet.test.mjs +++ b/tools/performance/check-journey-ratchet.test.mjs @@ -151,6 +151,42 @@ test('a counter with slack passes up to value + slack and reports the slack used assert.match(below.tightenable[0], /below baseline 1,000 by 10 bytes/); }); +test('an entry with autoTighten false gets no tightening hint below its value', () => { + const baselines = { + version: 1, + journeys: { + playback: { + 'renderer.domMutationsToPlaying': { + value: 733, + unit: 'mutations', + slack: 20, + autoTighten: false, + }, + }, + }, + }; + const result = compareToBaselines({ + baselines, + summary: { + journeys: { + playback: { + counters: { 'renderer.domMutationsToPlaying': 731 }, + }, + }, + }, + }); + assert.deepEqual(result.failures, []); + assert.deepEqual(result.tightenable, []); + assert.match( + result.passed[0], + /731 mutations within 753 .*autoTighten is false: lower it by hand/ + ); + assert.match( + formatResult(result), + /1 baselines checked, 0 can be tightened/ + ); +}); + test('wall-clock entries fail only above value × toleranceRatio', () => { const within = compareToBaselines({ baselines,