diff --git a/docs/architecture/performance-journeys.md b/docs/architecture/performance-journeys.md index f535a0eaa..3b0a70e95 100644 --- a/docs/architecture/performance-journeys.md +++ b/docs/architecture/performance-journeys.md @@ -1389,9 +1389,14 @@ between commits: 731, 732, 733, 734 or 736 since #1817, and 748 in an iteration where a fifth bridge call (`updateRemoteControlStatus`) lands before `playing`. `slack` 16 covers that band; the regression it guards, the timeline rendering the whole schedule again, is about 5,000 mutations above -it. `evidenceRun` 38311314484 (f60d79701) read 733 in every iteration. Lower -`value` (keeping the slack) when the counter drops; remove the slack once -the fifth call's race is gone and `master` runs agree. +it. `evidenceRun` 38311314484 (f60d79701) read 733 in every iteration. + +The entry carries `"autoTighten": false`, so the weekly tightening leaves it +alone: three runs that all read 731 would lower `value` to 731 and the limit +to 747, below the 748 of a normal fifth-call iteration. Lower it by hand, and +only with runs showing the top of the band fell too (a fifth-call iteration +at or below the new `value` + `slack`). Remove the slack and the flag once the +fifth call's race is gone and `master` runs agree. The two J4 entries, `search/renderer.ipcCallsPerSearch` 1 and `search/renderer.sqlStatementsPerSearch` 2 (#1899, `evidenceRun` @@ -1463,6 +1468,9 @@ its job; its entries are then unmeasured in that run. A final job runs P50 for a `.p50` entry); - a counter marked `counterStability..stable: false` in any run is kept, and the report says which run and which iterations disagreed; +- an entry with `"autoTighten": false` is kept: its slack covers a band of + normal values, and the largest of three runs can be the bottom of that band + (see `playback/renderer.domMutationsToPlaying` above); - `value` never goes up, `slack` and `toleranceRatio` never change, and no entry is added or removed: the result must pass `check-baseline-direction.mjs` without `--allow-increase`, which both the diff --git a/tools/performance/check-journey-ratchet.mjs b/tools/performance/check-journey-ratchet.mjs index 3459b5506..20bf9da0d 100644 --- a/tools/performance/check-journey-ratchet.mjs +++ b/tools/performance/check-journey-ratchet.mjs @@ -89,6 +89,14 @@ export function validateBaselines(baselines) { if (entry.note !== undefined && typeof entry.note !== 'string') { throw new Error(`Baseline ${label} has a non-string "note".`); } + if ( + entry.autoTighten !== undefined && + typeof entry.autoTighten !== 'boolean' + ) { + throw new Error( + `Baseline ${label} has a non-boolean "autoTighten".` + ); + } if ( entry.slack !== undefined && entry.toleranceRatio !== undefined diff --git a/tools/performance/check-journey-ratchet.test.mjs b/tools/performance/check-journey-ratchet.test.mjs index 2b4a04c08..4d0386b13 100644 --- a/tools/performance/check-journey-ratchet.test.mjs +++ b/tools/performance/check-journey-ratchet.test.mjs @@ -368,6 +368,13 @@ test('rejects malformed baseline files', () => { }), /non-string "note"/ ); + assert.throws( + () => + validateBaselines({ + journeys: { launch: { x: { value: 1, autoTighten: 'no' } } }, + }), + /non-boolean "autoTighten"/ + ); }); test('a failing entry prints its note; a fractional score is exact', () => { diff --git a/tools/performance/journey-baselines.json b/tools/performance/journey-baselines.json index ecc3846ab..8b1863ac2 100644 --- a/tools/performance/journey-baselines.json +++ b/tools/performance/journey-baselines.json @@ -96,6 +96,7 @@ "value": 733, "unit": "mutations", "slack": 16, + "autoTighten": false, "updatedAt": "2026-10-13", "evidencePr": 1817, "evidenceRun": "https://github.com/4gray/iptvnator/actions/runs/38311314484", diff --git a/tools/performance/tighten-baselines.mjs b/tools/performance/tighten-baselines.mjs index 8ae7786d6..d7e78be71 100644 --- a/tools/performance/tighten-baselines.mjs +++ b/tools/performance/tighten-baselines.mjs @@ -15,6 +15,9 @@ * `.p50` entry is the largest P50. * - A counter marked `counterStability..stable === false` in any run is * skipped: its summary value hides iterations that disagreed. + * - An entry with `"autoTighten": false` is kept: its slack covers a band of + * normal values, and lowering `value` to the largest of three runs could + * drop the limit below the top of that band. It is lowered by hand. * - `value` never goes up, `slack` and `toleranceRatio` never change, and no * entry is added or removed; check-baseline-direction.mjs must accept the * result without --allow-increase, and this script checks that itself. @@ -112,6 +115,12 @@ function decide(entry, name, journey, runs) { }; } } + if (entry.autoTighten === false) { + return { + measured, + reason: 'autoTighten is false: lowered by hand, see the entry in performance-journeys.md.', + }; + } if (counter) { for (const [index, run] of runs.entries()) { const stability = run.journeys[journey]?.counterStability?.[name]; diff --git a/tools/performance/tighten-baselines.test.mjs b/tools/performance/tighten-baselines.test.mjs index ccd3e5b25..67ba9f552 100644 --- a/tools/performance/tighten-baselines.test.mjs +++ b/tools/performance/tighten-baselines.test.mjs @@ -167,6 +167,22 @@ test('one run at or above the value keeps the baseline untouched', () => { } }); +test('an entry with autoTighten false is kept even when every run is lower', () => { + // Its slack covers a band of normal values; lowering value to the largest + // of three runs could put the limit below the top of that band. + const baselines = baselinesFile(); + baselines.journeys.launch['renderer.initialBytes'].autoTighten = false; + const result = tighten(baselines, [ + bytesRun(990), + bytesRun(995), + bytesRun(980), + ]); + assert.deepEqual(result.baselines, baselines); + assert.deepEqual(result.tightened, []); + assert.match(result.rows[0].reason, /autoTighten is false/); + assert.deepEqual(result.rows[0].measured, [990, 995, 980]); +}); + test('an entry missing or non-numeric in any run is kept', () => { const missing = tighten(baselinesFile(), [ bytesRun(990),