mirror of
https://github.com/4gray/iptvnator.git
synced 2026-10-13 20:56:19 -08:00
test(perf): keep the weekly tightener off the J3 mutation band
Greptile: lowering value to the largest of three runs while keeping slack 16 can drop the limit below the band's top (three runs at 731 give 747, below the normal 748 fifth-call iteration). A baseline may now carry "autoTighten": false; the tightener keeps it and says why, and the validator rejects a non-boolean flag. The J3 entry sets it and is lowered by hand only with evidence that the top of the band fell too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c4178463e1
commit
363afe2ec7
6 files changed
+52
-3
No files matched your search
@@ -1389,9 +1389,14 @@ between commits: 731, 732, 733, 734 or 736 since #1817, and 748 in an
|
||||
iteration where a fifth bridge call (`updateRemoteControlStatus`) lands
|
||||
before `playing`. `slack` 16 covers that band; the regression it guards, the
|
||||
timeline rendering the whole schedule again, is about 5,000 mutations above
|
||||
it. `evidenceRun` 38311314484 (f60d79701) read 733 in every iteration. Lower
|
||||
`value` (keeping the slack) when the counter drops; remove the slack once
|
||||
the fifth call's race is gone and `master` runs agree.
|
||||
it. `evidenceRun` 38311314484 (f60d79701) read 733 in every iteration.
|
||||
|
||||
The entry carries `"autoTighten": false`, so the weekly tightening leaves it
|
||||
alone: three runs that all read 731 would lower `value` to 731 and the limit
|
||||
to 747, below the 748 of a normal fifth-call iteration. Lower it by hand, and
|
||||
only with runs showing the top of the band fell too (a fifth-call iteration
|
||||
at or below the new `value` + `slack`). Remove the slack and the flag once the
|
||||
fifth call's race is gone and `master` runs agree.
|
||||
|
||||
The two J4 entries, `search/renderer.ipcCallsPerSearch` 1 and
|
||||
`search/renderer.sqlStatementsPerSearch` 2 (#1899, `evidenceRun`
|
||||
@@ -1463,6 +1468,9 @@ its job; its entries are then unmeasured in that run. A final job runs
|
||||
P50 for a `.p50` entry);
|
||||
- a counter marked `counterStability.<name>.stable: false` in any run is
|
||||
kept, and the report says which run and which iterations disagreed;
|
||||
- an entry with `"autoTighten": false` is kept: its slack covers a band of
|
||||
normal values, and the largest of three runs can be the bottom of that band
|
||||
(see `playback/renderer.domMutationsToPlaying` above);
|
||||
- `value` never goes up, `slack` and `toleranceRatio` never change, and no
|
||||
entry is added or removed: the result must pass
|
||||
`check-baseline-direction.mjs` without `--allow-increase`, which both the
|
||||
|
||||
@@ -89,6 +89,14 @@ export function validateBaselines(baselines) {
|
||||
if (entry.note !== undefined && typeof entry.note !== 'string') {
|
||||
throw new Error(`Baseline ${label} has a non-string "note".`);
|
||||
}
|
||||
if (
|
||||
entry.autoTighten !== undefined &&
|
||||
typeof entry.autoTighten !== 'boolean'
|
||||
) {
|
||||
throw new Error(
|
||||
`Baseline ${label} has a non-boolean "autoTighten".`
|
||||
);
|
||||
}
|
||||
if (
|
||||
entry.slack !== undefined &&
|
||||
entry.toleranceRatio !== undefined
|
||||
|
||||
@@ -368,6 +368,13 @@ test('rejects malformed baseline files', () => {
|
||||
}),
|
||||
/non-string "note"/
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
validateBaselines({
|
||||
journeys: { launch: { x: { value: 1, autoTighten: 'no' } } },
|
||||
}),
|
||||
/non-boolean "autoTighten"/
|
||||
);
|
||||
});
|
||||
|
||||
test('a failing entry prints its note; a fractional score is exact', () => {
|
||||
|
||||
@@ -96,6 +96,7 @@
|
||||
"value": 733,
|
||||
"unit": "mutations",
|
||||
"slack": 16,
|
||||
"autoTighten": false,
|
||||
"updatedAt": "2026-10-13",
|
||||
"evidencePr": 1817,
|
||||
"evidenceRun": "https://github.com/4gray/iptvnator/actions/runs/38311314484",
|
||||
|
||||
@@ -15,6 +15,9 @@
|
||||
* `.p50` entry is the largest P50.
|
||||
* - A counter marked `counterStability.<name>.stable === false` in any run is
|
||||
* skipped: its summary value hides iterations that disagreed.
|
||||
* - An entry with `"autoTighten": false` is kept: its slack covers a band of
|
||||
* normal values, and lowering `value` to the largest of three runs could
|
||||
* drop the limit below the top of that band. It is lowered by hand.
|
||||
* - `value` never goes up, `slack` and `toleranceRatio` never change, and no
|
||||
* entry is added or removed; check-baseline-direction.mjs must accept the
|
||||
* result without --allow-increase, and this script checks that itself.
|
||||
@@ -112,6 +115,12 @@ function decide(entry, name, journey, runs) {
|
||||
};
|
||||
}
|
||||
}
|
||||
if (entry.autoTighten === false) {
|
||||
return {
|
||||
measured,
|
||||
reason: 'autoTighten is false: lowered by hand, see the entry in performance-journeys.md.',
|
||||
};
|
||||
}
|
||||
if (counter) {
|
||||
for (const [index, run] of runs.entries()) {
|
||||
const stability = run.journeys[journey]?.counterStability?.[name];
|
||||
|
||||
@@ -167,6 +167,22 @@ test('one run at or above the value keeps the baseline untouched', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('an entry with autoTighten false is kept even when every run is lower', () => {
|
||||
// Its slack covers a band of normal values; lowering value to the largest
|
||||
// of three runs could put the limit below the top of that band.
|
||||
const baselines = baselinesFile();
|
||||
baselines.journeys.launch['renderer.initialBytes'].autoTighten = false;
|
||||
const result = tighten(baselines, [
|
||||
bytesRun(990),
|
||||
bytesRun(995),
|
||||
bytesRun(980),
|
||||
]);
|
||||
assert.deepEqual(result.baselines, baselines);
|
||||
assert.deepEqual(result.tightened, []);
|
||||
assert.match(result.rows[0].reason, /autoTighten is false/);
|
||||
assert.deepEqual(result.rows[0].measured, [990, 995, 980]);
|
||||
});
|
||||
|
||||
test('an entry missing or non-numeric in any run is kept', () => {
|
||||
const missing = tighten(baselinesFile(), [
|
||||
bytesRun(990),
|
||||
|
||||
Reference in new issue
Block a user