mirror of
https://github.com/4gray/iptvnator.git
synced 2026-10-08 09:01:03 -08:00
ci(performance): give the initial-bytes ratchet slack and a labelled override
The exact renderer.initialBytes counter failed PRs for reasons outside their diff: two concurrent merges left master 108 bytes over the baseline for hours, and bundler identifier renaming moves the counter by hundreds of bytes. PRs growing it by 243 and 302 bytes had no way to pass at all, because the direction check refuses any raised baseline. - Counter entries accept `slack` (integer, entry unit): the ratchet enforces `value + slack`, reports how much slack a measurement uses, and still prints the tighten hint below `value`. renderer.initialBytes gets 4096 bytes, so growth can accumulate at most 4 KiB past the last lowered baseline while regressions such as +35 KB still fail. - check-baseline-direction.mjs compares `value + slack`, treats widened slack like a widened tolerance, and takes `--allow-increase`, which reports weakened entries as ALLOWED instead of failing. - CI passes `--allow-increase` only when the pull request (or, for a master push, the pull request merged as the pushed commit) carries the perf-baseline-increase label, read from the API so a job re-run picks up a label added later. This change widens the slack itself, so its own PR needs the label. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0166PWobUjpoiWt8E9bBdsCs
This commit is contained in:
7 files changed
+343
-51
No files matched your search
@@ -130,6 +130,10 @@ jobs:
|
||||
name: Initial bytes ratchet
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
# pull-requests: read lets the direction check read the PR's labels.
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -153,12 +157,21 @@ jobs:
|
||||
# the revision this one is measured against instead: the target
|
||||
# branch for a pull request, the previous head for a master push,
|
||||
# master for a manual dispatch. Any raised limit, widened tolerance
|
||||
# or removed entry fails.
|
||||
# or slack, or removed entry fails, unless a maintainer put the
|
||||
# perf-baseline-increase label on the pull request (for a master
|
||||
# push: on the pull request merged as the pushed commit). Labels
|
||||
# are read from the API, not the event payload, so re-running this
|
||||
# job after adding the label picks it up.
|
||||
- name: Refuse baseline increases against the previous revision
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
BEFORE_SHA: ${{ github.event.before }}
|
||||
HEAD_SHA: ${{ github.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
INCREASE_LABEL: perf-baseline-increase
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "$EVENT_NAME" in
|
||||
@@ -178,9 +191,26 @@ jobs:
|
||||
esac
|
||||
git show "FETCH_HEAD:tools/performance/journey-baselines.json" > /tmp/base-journey-baselines.json 2>/dev/null ||
|
||||
rm -f /tmp/base-journey-baselines.json
|
||||
case "$EVENT_NAME" in
|
||||
pull_request)
|
||||
labels="$(gh api "repos/$REPOSITORY/issues/$PR_NUMBER/labels" --jq '.[].name')"
|
||||
;;
|
||||
push)
|
||||
labels="$(gh api "repos/$REPOSITORY/commits/$HEAD_SHA/pulls" --jq '.[].labels[].name')"
|
||||
;;
|
||||
*)
|
||||
labels=""
|
||||
;;
|
||||
esac
|
||||
allow=()
|
||||
if grep -qxF "$INCREASE_LABEL" <<< "$labels"; then
|
||||
echo "The $INCREASE_LABEL label is set; weakened baselines are reported, not failed."
|
||||
allow=(--allow-increase)
|
||||
fi
|
||||
node tools/performance/check-baseline-direction.mjs \
|
||||
--base /tmp/base-journey-baselines.json \
|
||||
--head tools/performance/journey-baselines.json
|
||||
--head tools/performance/journey-baselines.json \
|
||||
"${allow[@]}"
|
||||
|
||||
# The production configuration is what users download; measuring
|
||||
# any other build would ratchet a number nobody ships.
|
||||
@@ -190,8 +220,8 @@ jobs:
|
||||
CI: true
|
||||
NX_TASKS_RUNNER_DYNAMIC_OUTPUT: false
|
||||
|
||||
# Fails when renderer.initialBytes exceeds the value committed in
|
||||
# tools/performance/journey-baselines.json. Baselines only move
|
||||
# Fails when renderer.initialBytes exceeds value + slack committed
|
||||
# in tools/performance/journey-baselines.json. Baselines only move
|
||||
# down, with the printed measurement as evidence; the contract is
|
||||
# docs/architecture/performance-journeys.md.
|
||||
- name: Check renderer.initialBytes against the baseline
|
||||
|
||||
@@ -276,6 +276,7 @@ counter:
|
||||
"renderer.initialBytes": {
|
||||
"value": 2739510,
|
||||
"unit": "bytes",
|
||||
"slack": 4096,
|
||||
"updatedAt": "2026-09-26",
|
||||
"evidencePr": 1693,
|
||||
"measuredWith": "pnpm nx build web && pnpm run perf:initial-bytes"
|
||||
@@ -288,7 +289,9 @@ counter:
|
||||
`tools/performance/check-journey-ratchet.mjs` compares a journey summary with
|
||||
that file:
|
||||
|
||||
- a counter above its `value` fails; counters are exact, there is no slack;
|
||||
- a counter above `value + slack` fails; counters are exact, and `slack`
|
||||
(default 0, in the entry's unit) is the only allowance, printed as "uses N
|
||||
of S slack" whenever a measurement is above `value`;
|
||||
- a wall-clock entry carries `toleranceRatio` and fails above
|
||||
`value × toleranceRatio`;
|
||||
- a baseline with no measurement in the summary fails, so dropping a
|
||||
@@ -338,19 +341,43 @@ chunk-level identifier renaming shifts when a module enters or leaves
|
||||
call sites, eating about 320 of the bytes saved. Judge a small change by the
|
||||
`--stats-json` input sizes, not only by the counter.
|
||||
|
||||
Both effects are why `renderer.initialBytes` carries `"slack": 4096`. With a
|
||||
zero allowance the +108 above failed every PR for hours on 2026-09-27, and
|
||||
PRs growing the counter by 243 and 302 bytes, each through one service
|
||||
change, had no way to pass. The
|
||||
slack is fixed, not a ratio, and sits on top of `value`, which still only
|
||||
moves down: growth accumulates at most 4 KiB past the last lowered baseline
|
||||
before the job fails again, while a regression such as #1601's +35,435 bytes
|
||||
fails as before. Lower `value` to the measured number as usual; the slack
|
||||
stays and is not part of the evidence.
|
||||
|
||||
The job also refuses a weakened baselines file:
|
||||
`tools/performance/check-baseline-direction.mjs` compares
|
||||
`journey-baselines.json` with the revision the change is measured against
|
||||
(the target branch of a pull request, the previous head of a `master` push,
|
||||
`master` for a manual dispatch) and fails when any
|
||||
entry's enforced limit (`value × toleranceRatio`) went up, a tolerance widened
|
||||
or an entry disappeared, so a PR cannot grow the payload and raise the
|
||||
baseline to match. Lowered limits and new entries pass.
|
||||
entry's enforced limit (`value × toleranceRatio` or `value + slack`) went up,
|
||||
a tolerance or slack widened or an entry disappeared, so a PR cannot grow the
|
||||
payload and raise the baseline to match. Lowered limits and new entries pass.
|
||||
|
||||
Baselines only move down. Lower `value` in the same PR as the change that
|
||||
earned it, set `updatedAt` and `evidencePr`, and paste the measurement output
|
||||
into the PR. Never raise a value to make a PR pass: if growth is a deliberate
|
||||
trade-off, say so in the PR and let the maintainer decide.
|
||||
trade-off (a framework upgrade, a feature that must be on the initial path),
|
||||
raise `value` to the runner's measurement in the PR, make the case with the
|
||||
per-file breakdown, and ask a maintainer to add the `perf-baseline-increase`
|
||||
label. With the label the direction check prints the weakened entries as
|
||||
`ALLOWED` and passes; the job reads labels from the API when it runs, so
|
||||
re-run the job after the label is added. For a `master` push the label is
|
||||
read from the pull request merged as the pushed commit, so a squash merge of
|
||||
a labelled PR passes and a direct push that raises a baseline still fails.
|
||||
Only people with triage access can set labels, so the label is the
|
||||
maintainer decision.
|
||||
|
||||
A PR merged while this job is red makes every later PR fail it with the same
|
||||
numbers until `master` is fixed: #1601 merged at +35,435 bytes and failed
|
||||
the job for every PR until #1734. Treat the job as blocking before merging;
|
||||
making it a required check is a maintainer decision.
|
||||
|
||||
The runtime counters come from the `Performance journeys` job of the same
|
||||
workflow, on `ubuntu-latest` only. It runs `pnpm run perf:journeys` under
|
||||
|
||||
@@ -5,14 +5,19 @@
|
||||
* commit, so on its own it cannot tell a genuine payload reduction from a PR
|
||||
* that grows the payload and raises the baseline by the same amount. This
|
||||
* check closes that gap: given the baselines file of the target branch and
|
||||
* the one of the PR, any entry whose enforced limit (`value × toleranceRatio`)
|
||||
* went up, whose tolerance widened, or that disappeared, is a failure. New
|
||||
* entries and lowered limits pass.
|
||||
* the one of the PR, any entry whose enforced limit (`value × toleranceRatio`
|
||||
* or `value + slack`) went up, whose tolerance or slack widened, or that
|
||||
* disappeared, is a failure. New entries and lowered limits pass.
|
||||
*
|
||||
* `--allow-increase` turns those failures into printed "allowed" lines. CI
|
||||
* passes it only when a maintainer put the perf-baseline-increase label on
|
||||
* the pull request, so a deliberate increase stays visible and needs a
|
||||
* decision instead of being impossible.
|
||||
*
|
||||
* Usage:
|
||||
* node tools/performance/check-baseline-direction.mjs \
|
||||
* --base <target-branch-journey-baselines.json> \
|
||||
* --head tools/performance/journey-baselines.json
|
||||
* --head tools/performance/journey-baselines.json [--allow-increase]
|
||||
*
|
||||
* A missing --base file means the target branch has no baselines yet, so
|
||||
* there is nothing that could have been weakened.
|
||||
@@ -22,7 +27,11 @@ import { readFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { validateBaselines } from './check-journey-ratchet.mjs';
|
||||
import {
|
||||
BASELINE_INCREASE_LABEL,
|
||||
enforcedLimit,
|
||||
validateBaselines,
|
||||
} from './check-journey-ratchet.mjs';
|
||||
|
||||
export const DEFAULT_HEAD_PATH = 'tools/performance/journey-baselines.json';
|
||||
|
||||
@@ -41,41 +50,52 @@ function formatNumber(value) {
|
||||
}
|
||||
|
||||
/**
|
||||
* What the ratchet actually enforces: `value × toleranceRatio` for a
|
||||
* wall-clock entry, the bare value for a counter. Comparing values alone would
|
||||
* let a PR lower a value while widening the tolerance.
|
||||
* Pure comparison; `failures` non-empty means the change weakens the ratchet.
|
||||
* With `allowIncrease` every weakening lands in `allowed` instead.
|
||||
*/
|
||||
function effectiveLimit(entry) {
|
||||
return entry.value * (entry.toleranceRatio ?? 1);
|
||||
}
|
||||
|
||||
/** Pure comparison; `failures` non-empty means the change weakens the ratchet. */
|
||||
export function compareBaselineDirection({ base, head }) {
|
||||
export function compareBaselineDirection({
|
||||
base,
|
||||
head,
|
||||
allowIncrease = false,
|
||||
}) {
|
||||
validateBaselines(base);
|
||||
validateBaselines(head);
|
||||
const result = { failures: [], lowered: [], unchanged: [], added: [] };
|
||||
const result = {
|
||||
failures: [],
|
||||
allowed: [],
|
||||
lowered: [],
|
||||
unchanged: [],
|
||||
added: [],
|
||||
};
|
||||
const weakened = allowIncrease ? result.allowed : result.failures;
|
||||
const headEntries = entries(head);
|
||||
|
||||
for (const [label, baseEntry] of entries(base)) {
|
||||
const headEntry = headEntries.get(label);
|
||||
const unit = baseEntry.unit ? ` ${baseEntry.unit}` : '';
|
||||
if (!headEntry) {
|
||||
result.failures.push(
|
||||
weakened.push(
|
||||
`${label}: baseline ${formatNumber(baseEntry.value)}${unit} was removed. Baselines are retired only by a maintainer decision recorded in the PR, not by deleting the entry.`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const baseTolerance = baseEntry.toleranceRatio ?? 1;
|
||||
const headTolerance = headEntry.toleranceRatio ?? 1;
|
||||
const baseLimit = effectiveLimit(baseEntry);
|
||||
const headLimit = effectiveLimit(headEntry);
|
||||
const baseSlack = baseEntry.slack ?? 0;
|
||||
const headSlack = headEntry.slack ?? 0;
|
||||
const baseLimit = enforcedLimit(baseEntry);
|
||||
const headLimit = enforcedLimit(headEntry);
|
||||
if (headTolerance > baseTolerance) {
|
||||
result.failures.push(
|
||||
`${label}: toleranceRatio widened from ${baseTolerance} to ${headTolerance}. Tolerances are a maintainer decision; a PR may only narrow them.`
|
||||
weakened.push(
|
||||
`${label}: toleranceRatio widened from ${baseTolerance} to ${headTolerance}. Tolerances are a maintainer decision (the ${BASELINE_INCREASE_LABEL} label); a PR may only narrow them.`
|
||||
);
|
||||
} else if (headSlack > baseSlack) {
|
||||
weakened.push(
|
||||
`${label}: slack widened from ${formatNumber(baseSlack)} to ${formatNumber(headSlack)}${unit}. Slack is a maintainer decision (the ${BASELINE_INCREASE_LABEL} label); a PR may only narrow it.`
|
||||
);
|
||||
} else if (headLimit > baseLimit) {
|
||||
result.failures.push(
|
||||
`${label}: baseline raised from ${formatNumber(baseLimit)} to ${formatNumber(headLimit)}${unit}. Baselines only move down; bring the measurement back under ${formatNumber(baseLimit)} or make the case for the increase in the PR.`
|
||||
weakened.push(
|
||||
`${label}: baseline raised from ${formatNumber(baseLimit)} to ${formatNumber(headLimit)}${unit}. Baselines only move down; bring the measurement back under ${formatNumber(baseLimit)}, or make the case for the increase in the PR and ask a maintainer to add the ${BASELINE_INCREASE_LABEL} label.`
|
||||
);
|
||||
} else if (headLimit < baseLimit) {
|
||||
result.lowered.push(
|
||||
@@ -95,17 +115,26 @@ export function formatDirectionResult(result) {
|
||||
const lines = [];
|
||||
for (const line of result.lowered) lines.push(`lowered ${line}`);
|
||||
for (const label of result.added) lines.push(`added ${label}`);
|
||||
for (const line of result.allowed) lines.push(`ALLOWED ${line}`);
|
||||
for (const line of result.failures) lines.push(`FAIL ${line}`);
|
||||
const allowed =
|
||||
result.allowed.length > 0
|
||||
? `, ${result.allowed.length} weakened with the ${BASELINE_INCREASE_LABEL} label`
|
||||
: '';
|
||||
lines.push(
|
||||
result.failures.length > 0
|
||||
? `Baseline direction check failed: ${result.failures.length} entries raised or removed.`
|
||||
: `Baseline direction OK: ${result.unchanged.length} unchanged, ${result.lowered.length} lowered, ${result.added.length} added.`
|
||||
: `Baseline direction OK: ${result.unchanged.length} unchanged, ${result.lowered.length} lowered, ${result.added.length} added${allowed}.`
|
||||
);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export function parseArgs(argv) {
|
||||
const options = { base: null, head: DEFAULT_HEAD_PATH };
|
||||
const options = {
|
||||
base: null,
|
||||
head: DEFAULT_HEAD_PATH,
|
||||
allowIncrease: false,
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const argument = argv[index];
|
||||
if (argument === '--') continue;
|
||||
@@ -117,6 +146,8 @@ export function parseArgs(argv) {
|
||||
options.head = argv[++index];
|
||||
} else if (argument.startsWith('--head=')) {
|
||||
options.head = argument.slice('--head='.length);
|
||||
} else if (argument === '--allow-increase') {
|
||||
options.allowIncrease = true;
|
||||
} else {
|
||||
throw new Error(`Unknown argument: ${argument}`);
|
||||
}
|
||||
@@ -160,7 +191,11 @@ if (isMain) {
|
||||
);
|
||||
}
|
||||
const head = await readJson(path.resolve(options.head), 'baselines');
|
||||
const result = compareBaselineDirection({ base, head });
|
||||
const result = compareBaselineDirection({
|
||||
base,
|
||||
head,
|
||||
allowIncrease: options.allowIncrease,
|
||||
});
|
||||
const output = formatDirectionResult(result);
|
||||
if (result.failures.length > 0) {
|
||||
console.error(output);
|
||||
|
||||
@@ -142,6 +142,83 @@ test('the enforced limit is what is compared for wall-clock entries', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('a widened or newly added slack fails; a narrowed one lowers the limit', () => {
|
||||
const withSlack = (value, slack) => ({
|
||||
version: 1,
|
||||
journeys: {
|
||||
launch: {
|
||||
'renderer.initialBytes': { value, unit: 'bytes', slack },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const added = compareBaselineDirection({
|
||||
base: file(100),
|
||||
head: withSlack(90, 20),
|
||||
});
|
||||
assert.equal(added.failures.length, 1);
|
||||
assert.match(
|
||||
added.failures[0],
|
||||
/renderer\.initialBytes: slack widened from 0 to 20 bytes/
|
||||
);
|
||||
|
||||
const widened = compareBaselineDirection({
|
||||
base: withSlack(100, 10),
|
||||
head: withSlack(95, 12),
|
||||
});
|
||||
assert.equal(widened.failures.length, 1);
|
||||
assert.match(widened.failures[0], /slack widened from 10 to 12 bytes/);
|
||||
|
||||
const narrowed = compareBaselineDirection({
|
||||
base: withSlack(100, 10),
|
||||
head: withSlack(100, 4),
|
||||
});
|
||||
assert.deepEqual(narrowed.failures, []);
|
||||
assert.match(narrowed.lowered[0], /110 -> 104 bytes/);
|
||||
|
||||
const raised = compareBaselineDirection({
|
||||
base: withSlack(100, 10),
|
||||
head: withSlack(101, 10),
|
||||
});
|
||||
assert.match(raised.failures[0], /baseline raised from 110 to 111 bytes/);
|
||||
assert.match(raised.failures[0], /perf-baseline-increase label/);
|
||||
});
|
||||
|
||||
test('allowIncrease reports every weakening as allowed instead of failing', () => {
|
||||
const result = compareBaselineDirection({
|
||||
base: file(100, { cdTicks: { value: 5 } }),
|
||||
head: {
|
||||
version: 1,
|
||||
journeys: {
|
||||
launch: {
|
||||
'renderer.initialBytes': {
|
||||
value: 120,
|
||||
unit: 'bytes',
|
||||
slack: 8,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
allowIncrease: true,
|
||||
});
|
||||
assert.deepEqual(result.failures, []);
|
||||
assert.equal(result.allowed.length, 2);
|
||||
assert.match(result.allowed[0], /slack widened from 0 to 8 bytes/);
|
||||
assert.match(result.allowed[1], /cdTicks: baseline 5 was removed/);
|
||||
assert.match(
|
||||
formatDirectionResult(result),
|
||||
/^ALLOWED {2}launch\/renderer\.initialBytes.*\nALLOWED {2}launch\/cdTicks.*\nBaseline direction OK: 0 unchanged, 0 lowered, 0 added, 2 weakened with the perf-baseline-increase label\.$/
|
||||
);
|
||||
|
||||
const lowered = compareBaselineDirection({
|
||||
base: file(100),
|
||||
head: file(90),
|
||||
allowIncrease: true,
|
||||
});
|
||||
assert.deepEqual(lowered.allowed, []);
|
||||
assert.equal(lowered.lowered.length, 1);
|
||||
});
|
||||
|
||||
test('an empty target-branch file cannot be weakened', () => {
|
||||
const result = compareBaselineDirection({
|
||||
base: { journeys: {} },
|
||||
@@ -170,11 +247,16 @@ test('parses arguments and requires --base', () => {
|
||||
assert.deepEqual(parseArgs(['--base', 'b.json']), {
|
||||
base: 'b.json',
|
||||
head: DEFAULT_HEAD_PATH,
|
||||
allowIncrease: false,
|
||||
});
|
||||
assert.deepEqual(parseArgs(['--', '--base=b.json', '--head=h.json']), {
|
||||
base: 'b.json',
|
||||
head: 'h.json',
|
||||
});
|
||||
assert.deepEqual(
|
||||
parseArgs(['--', '--base=b.json', '--head=h.json', '--allow-increase']),
|
||||
{
|
||||
base: 'b.json',
|
||||
head: 'h.json',
|
||||
allowIncrease: true,
|
||||
}
|
||||
);
|
||||
assert.throws(
|
||||
() => parseArgs([]),
|
||||
/--base <target-branch-journey-baselines\.json> is required/
|
||||
@@ -186,7 +268,7 @@ test('parses arguments and requires --base', () => {
|
||||
);
|
||||
});
|
||||
|
||||
async function runCli(base, head) {
|
||||
async function runCli(base, head, extraArgs = []) {
|
||||
const basePath =
|
||||
base === null
|
||||
? path.join(workDir, 'absent.json')
|
||||
@@ -196,14 +278,14 @@ async function runCli(base, head) {
|
||||
await writeFile(headPath, JSON.stringify(head));
|
||||
return spawnSync(
|
||||
process.execPath,
|
||||
[scriptPath, '--base', basePath, '--head', headPath],
|
||||
[scriptPath, '--base', basePath, '--head', headPath, ...extraArgs],
|
||||
{
|
||||
encoding: 'utf8',
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
test('CLI exits 0 for a lowered baseline, 1 for a raised one, 0 when the target branch has no file', async () => {
|
||||
test('CLI exits 0 for a lowered baseline, 1 for a raised one, 0 for a raised one with --allow-increase, 0 when the target branch has no file', async () => {
|
||||
const lowered = await runCli(file(100), file(90));
|
||||
assert.equal(lowered.status, 0, lowered.stderr);
|
||||
assert.match(lowered.stdout, /Baseline direction OK/);
|
||||
@@ -215,6 +297,13 @@ test('CLI exits 0 for a lowered baseline, 1 for a raised one, 0 when the target
|
||||
/FAIL {5}launch\/renderer\.initialBytes: baseline raised/
|
||||
);
|
||||
|
||||
const allowed = await runCli(file(100), file(101), ['--allow-increase']);
|
||||
assert.equal(allowed.status, 0, allowed.stderr);
|
||||
assert.match(
|
||||
allowed.stdout,
|
||||
/ALLOWED {2}launch\/renderer\.initialBytes: baseline raised/
|
||||
);
|
||||
|
||||
const noBase = await runCli(null, file(100));
|
||||
assert.equal(noBase.status, 0, noBase.stderr);
|
||||
assert.match(noBase.stdout, /nothing to weaken/);
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
* committed baselines in tools/performance/journey-baselines.json.
|
||||
*
|
||||
* Rules, from docs/architecture/performance-journeys.md:
|
||||
* - A counter above its baseline fails. Counters are exact; there is no slack.
|
||||
* - A counter above `value + slack` fails. Counters are exact; `slack`
|
||||
* (default 0, in the entry's unit) absorbs bundler noise and concurrent
|
||||
* merges without letting growth accumulate past it.
|
||||
* - An entry with `toleranceRatio` (wall-clock) fails above
|
||||
* `value * toleranceRatio`.
|
||||
* - A baseline without a measurement fails, so removing a measurement can
|
||||
@@ -29,6 +31,9 @@ import { fileURLToPath } from 'node:url';
|
||||
export const DEFAULT_BASELINES_PATH =
|
||||
'tools/performance/journey-baselines.json';
|
||||
|
||||
/** The PR label that lets check-baseline-direction.mjs accept a weakening. */
|
||||
export const BASELINE_INCREASE_LABEL = 'perf-baseline-increase';
|
||||
|
||||
function formatNumber(value) {
|
||||
return Number.isInteger(value)
|
||||
? value.toLocaleString('en-US')
|
||||
@@ -69,6 +74,22 @@ export function validateBaselines(baselines) {
|
||||
`Baseline ${label} has an invalid "toleranceRatio" (must be >= 1).`
|
||||
);
|
||||
}
|
||||
if (
|
||||
entry.slack !== undefined &&
|
||||
!(Number.isInteger(entry.slack) && entry.slack >= 0)
|
||||
) {
|
||||
throw new Error(
|
||||
`Baseline ${label} has an invalid "slack" (must be an integer >= 0).`
|
||||
);
|
||||
}
|
||||
if (
|
||||
entry.slack !== undefined &&
|
||||
entry.toleranceRatio !== undefined
|
||||
) {
|
||||
throw new Error(
|
||||
`Baseline ${label} sets both "slack" and "toleranceRatio"; slack is for counters, toleranceRatio for wall-clock entries.`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return baselines;
|
||||
@@ -84,6 +105,25 @@ function summarySection(entry) {
|
||||
return entry.toleranceRatio !== undefined ? 'wallClock' : 'counters';
|
||||
}
|
||||
|
||||
/**
|
||||
* What the ratchet enforces: `value × toleranceRatio` for a wall-clock entry,
|
||||
* `value + slack` for a counter. check-baseline-direction.mjs compares this,
|
||||
* not the bare value, so a PR cannot lower a value while widening a tolerance.
|
||||
*/
|
||||
export function enforcedLimit(entry) {
|
||||
return entry.value * (entry.toleranceRatio ?? 1) + (entry.slack ?? 0);
|
||||
}
|
||||
|
||||
function describeLimit(entry, unit) {
|
||||
if (entry.toleranceRatio !== undefined) {
|
||||
return `${formatNumber(enforcedLimit(entry))} (baseline ${formatNumber(entry.value)} × ${entry.toleranceRatio})`;
|
||||
}
|
||||
if (entry.slack) {
|
||||
return `${formatNumber(enforcedLimit(entry))} (baseline ${formatNumber(entry.value)} + slack ${formatNumber(entry.slack)}${unit})`;
|
||||
}
|
||||
return `baseline ${formatNumber(entry.value)}`;
|
||||
}
|
||||
|
||||
function measuredValue(summaryJourney, name, entry) {
|
||||
if (!isPlainObject(summaryJourney)) return undefined;
|
||||
return summaryJourney[summarySection(entry)]?.[name];
|
||||
@@ -141,18 +181,16 @@ export function compareToBaselines({ baselines, summary, only = [] }) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const limit =
|
||||
entry.toleranceRatio !== undefined
|
||||
? entry.value * entry.toleranceRatio
|
||||
: entry.value;
|
||||
const limitText =
|
||||
entry.toleranceRatio !== undefined
|
||||
? `${formatNumber(limit)} (baseline ${formatNumber(entry.value)} × ${entry.toleranceRatio})`
|
||||
: `baseline ${formatNumber(entry.value)}`;
|
||||
const limit = enforcedLimit(entry);
|
||||
const limitText = describeLimit(entry, unit);
|
||||
|
||||
if (measured > limit) {
|
||||
result.failures.push(
|
||||
`${label}: ${formatNumber(measured)}${unit} exceeds ${limitText} by ${formatNumber(measured - limit)}${unit}. Bring the value back down; baselines only move down. If the growth is a deliberate trade-off, say so in the PR and let the maintainer decide.`
|
||||
`${label}: ${formatNumber(measured)}${unit} exceeds ${limitText} by ${formatNumber(measured - limit)}${unit}. Bring the value back down; baselines only move down. If the growth is a deliberate trade-off, raise the baseline in ${DEFAULT_BASELINES_PATH}, make the case in the PR, and ask a maintainer to add the ${BASELINE_INCREASE_LABEL} label.`
|
||||
);
|
||||
} else if (measured > entry.value) {
|
||||
result.passed.push(
|
||||
`${label}: ${formatNumber(measured)}${unit} within ${limitText}; uses ${formatNumber(measured - entry.value)} of ${formatNumber(limit - entry.value)}${unit} slack.`
|
||||
);
|
||||
} else if (measured < entry.value) {
|
||||
result.tightenable.push(
|
||||
|
||||
@@ -98,6 +98,55 @@ test('a counter below its baseline passes and asks to tighten', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('a counter with slack passes up to value + slack and reports the slack used', () => {
|
||||
const withSlack = {
|
||||
version: 1,
|
||||
journeys: {
|
||||
launch: {
|
||||
'renderer.initialBytes': {
|
||||
value: 1000,
|
||||
unit: 'bytes',
|
||||
slack: 100,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const check = (initialBytes) =>
|
||||
compareToBaselines({
|
||||
baselines: withSlack,
|
||||
summary: {
|
||||
journeys: {
|
||||
launch: {
|
||||
counters: { 'renderer.initialBytes': initialBytes },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const within = check(1100);
|
||||
assert.deepEqual(within.failures, []);
|
||||
assert.deepEqual(within.tightenable, []);
|
||||
assert.match(
|
||||
within.passed[0],
|
||||
/1,100 bytes within 1,100 \(baseline 1,000 \+ slack 100 bytes\); uses 100 of 100 bytes slack/
|
||||
);
|
||||
|
||||
const above = check(1101);
|
||||
assert.equal(above.failures.length, 1);
|
||||
assert.match(
|
||||
above.failures[0],
|
||||
/1,101 bytes exceeds 1,100 \(baseline 1,000 \+ slack 100 bytes\) by 1 bytes/
|
||||
);
|
||||
assert.match(above.failures[0], /add the perf-baseline-increase label/);
|
||||
|
||||
const exact = check(1000);
|
||||
assert.match(exact.passed[0], /1,000 bytes within 1,100/);
|
||||
assert.doesNotMatch(exact.passed[0], /uses/);
|
||||
|
||||
const below = check(990);
|
||||
assert.match(below.tightenable[0], /below baseline 1,000 by 10 bytes/);
|
||||
});
|
||||
|
||||
test('wall-clock entries fail only above value × toleranceRatio', () => {
|
||||
const within = compareToBaselines({
|
||||
baselines,
|
||||
@@ -284,6 +333,29 @@ test('rejects malformed baseline files', () => {
|
||||
}),
|
||||
/invalid "toleranceRatio"/
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
validateBaselines({
|
||||
journeys: { launch: { x: { value: 1, slack: -1 } } },
|
||||
}),
|
||||
/invalid "slack"/
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
validateBaselines({
|
||||
journeys: { launch: { x: { value: 1, slack: 1.5 } } },
|
||||
}),
|
||||
/invalid "slack"/
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
validateBaselines({
|
||||
journeys: {
|
||||
launch: { x: { value: 1, slack: 1, toleranceRatio: 1.1 } },
|
||||
},
|
||||
}),
|
||||
/sets both "slack" and "toleranceRatio"/
|
||||
);
|
||||
});
|
||||
|
||||
test('formats a summary line for both outcomes', () => {
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
"renderer.initialBytes": {
|
||||
"value": 1598232,
|
||||
"unit": "bytes",
|
||||
"slack": 4096,
|
||||
"updatedAt": "2026-09-27",
|
||||
"evidencePr": 1734,
|
||||
"measuredWith": "pnpm nx build web && pnpm run perf:initial-bytes"
|
||||
|
||||
Reference in new issue
Block a user