From 363afe2ec7d427d4328d5e9364ccef0040a4df8e Mon Sep 17 00:00:00 2001 From: 4gray Date: Tue, 13 Oct 2026 18:19:05 +0200 Subject: [PATCH] test(perf): keep the weekly tightener off the J3 mutation band Greptile: lowering value to the largest of three runs while keeping slack 16 can drop the limit below the band's top (three runs at 731 give 747, below the normal 748 fifth-call iteration). A baseline may now carry "autoTighten": false; the tightener keeps it and says why, and the validator rejects a non-boolean flag. The J3 entry sets it and is lowered by hand only with evidence that the top of the band fell too. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/performance-journeys.md | 14 +++++++++++--- tools/performance/check-journey-ratchet.mjs | 8 ++++++++ tools/performance/check-journey-ratchet.test.mjs | 7 +++++++ tools/performance/journey-baselines.json | 1 + tools/performance/tighten-baselines.mjs | 9 +++++++++ tools/performance/tighten-baselines.test.mjs | 16 ++++++++++++++++ 6 files changed, 52 insertions(+), 3 deletions(-) diff --git a/docs/architecture/performance-journeys.md b/docs/architecture/performance-journeys.md index f535a0eaa..3b0a70e95 100644 --- a/docs/architecture/performance-journeys.md +++ b/docs/architecture/performance-journeys.md @@ -1389,9 +1389,14 @@ between commits: 731, 732, 733, 734 or 736 since #1817, and 748 in an iteration where a fifth bridge call (`updateRemoteControlStatus`) lands before `playing`. `slack` 16 covers that band; the regression it guards, the timeline rendering the whole schedule again, is about 5,000 mutations above -it. `evidenceRun` 38311314484 (f60d79701) read 733 in every iteration. Lower -`value` (keeping the slack) when the counter drops; remove the slack once -the fifth call's race is gone and `master` runs agree. +it. `evidenceRun` 38311314484 (f60d79701) read 733 in every iteration. + +The entry carries `"autoTighten": false`, so the weekly tightening leaves it +alone: three runs that all read 731 would lower `value` to 731 and the limit +to 747, below the 748 of a normal fifth-call iteration. Lower it by hand, and +only with runs showing the top of the band fell too (a fifth-call iteration +at or below the new `value` + `slack`). Remove the slack and the flag once the +fifth call's race is gone and `master` runs agree. The two J4 entries, `search/renderer.ipcCallsPerSearch` 1 and `search/renderer.sqlStatementsPerSearch` 2 (#1899, `evidenceRun` @@ -1463,6 +1468,9 @@ its job; its entries are then unmeasured in that run. A final job runs P50 for a `.p50` entry); - a counter marked `counterStability..stable: false` in any run is kept, and the report says which run and which iterations disagreed; +- an entry with `"autoTighten": false` is kept: its slack covers a band of + normal values, and the largest of three runs can be the bottom of that band + (see `playback/renderer.domMutationsToPlaying` above); - `value` never goes up, `slack` and `toleranceRatio` never change, and no entry is added or removed: the result must pass `check-baseline-direction.mjs` without `--allow-increase`, which both the diff --git a/tools/performance/check-journey-ratchet.mjs b/tools/performance/check-journey-ratchet.mjs index 3459b5506..20bf9da0d 100644 --- a/tools/performance/check-journey-ratchet.mjs +++ b/tools/performance/check-journey-ratchet.mjs @@ -89,6 +89,14 @@ export function validateBaselines(baselines) { if (entry.note !== undefined && typeof entry.note !== 'string') { throw new Error(`Baseline ${label} has a non-string "note".`); } + if ( + entry.autoTighten !== undefined && + typeof entry.autoTighten !== 'boolean' + ) { + throw new Error( + `Baseline ${label} has a non-boolean "autoTighten".` + ); + } if ( entry.slack !== undefined && entry.toleranceRatio !== undefined diff --git a/tools/performance/check-journey-ratchet.test.mjs b/tools/performance/check-journey-ratchet.test.mjs index 2b4a04c08..4d0386b13 100644 --- a/tools/performance/check-journey-ratchet.test.mjs +++ b/tools/performance/check-journey-ratchet.test.mjs @@ -368,6 +368,13 @@ test('rejects malformed baseline files', () => { }), /non-string "note"/ ); + assert.throws( + () => + validateBaselines({ + journeys: { launch: { x: { value: 1, autoTighten: 'no' } } }, + }), + /non-boolean "autoTighten"/ + ); }); test('a failing entry prints its note; a fractional score is exact', () => { diff --git a/tools/performance/journey-baselines.json b/tools/performance/journey-baselines.json index ecc3846ab..8b1863ac2 100644 --- a/tools/performance/journey-baselines.json +++ b/tools/performance/journey-baselines.json @@ -96,6 +96,7 @@ "value": 733, "unit": "mutations", "slack": 16, + "autoTighten": false, "updatedAt": "2026-10-13", "evidencePr": 1817, "evidenceRun": "https://github.com/4gray/iptvnator/actions/runs/38311314484", diff --git a/tools/performance/tighten-baselines.mjs b/tools/performance/tighten-baselines.mjs index 8ae7786d6..d7e78be71 100644 --- a/tools/performance/tighten-baselines.mjs +++ b/tools/performance/tighten-baselines.mjs @@ -15,6 +15,9 @@ * `.p50` entry is the largest P50. * - A counter marked `counterStability..stable === false` in any run is * skipped: its summary value hides iterations that disagreed. + * - An entry with `"autoTighten": false` is kept: its slack covers a band of + * normal values, and lowering `value` to the largest of three runs could + * drop the limit below the top of that band. It is lowered by hand. * - `value` never goes up, `slack` and `toleranceRatio` never change, and no * entry is added or removed; check-baseline-direction.mjs must accept the * result without --allow-increase, and this script checks that itself. @@ -112,6 +115,12 @@ function decide(entry, name, journey, runs) { }; } } + if (entry.autoTighten === false) { + return { + measured, + reason: 'autoTighten is false: lowered by hand, see the entry in performance-journeys.md.', + }; + } if (counter) { for (const [index, run] of runs.entries()) { const stability = run.journeys[journey]?.counterStability?.[name]; diff --git a/tools/performance/tighten-baselines.test.mjs b/tools/performance/tighten-baselines.test.mjs index ccd3e5b25..67ba9f552 100644 --- a/tools/performance/tighten-baselines.test.mjs +++ b/tools/performance/tighten-baselines.test.mjs @@ -167,6 +167,22 @@ test('one run at or above the value keeps the baseline untouched', () => { } }); +test('an entry with autoTighten false is kept even when every run is lower', () => { + // Its slack covers a band of normal values; lowering value to the largest + // of three runs could put the limit below the top of that band. + const baselines = baselinesFile(); + baselines.journeys.launch['renderer.initialBytes'].autoTighten = false; + const result = tighten(baselines, [ + bytesRun(990), + bytesRun(995), + bytesRun(980), + ]); + assert.deepEqual(result.baselines, baselines); + assert.deepEqual(result.tightened, []); + assert.match(result.rows[0].reason, /autoTighten is false/); + assert.deepEqual(result.rows[0].measured, [990, 995, 980]); +}); + test('an entry missing or non-numeric in any run is kept', () => { const missing = tighten(baselinesFile(), [ bytesRun(990),