ci(perf): enforce the journey counters stable on master (#1829)

* ci(perf): enforce the journey counters stable on master Promote the J1, J2 and J3 counters that were identical in all 55 measured iterations of the 11 master runs from 2026-10-03 to 2026-10-04 to journey-baselines.json, and check them in the Performance journeys job (still warn-only), in one step together with #1828's two validated J1 entries. None of the new ones has Principle 3 evidence, so each carries a "guard only, not validated" note that the checker prints with a failure. A performance-tools test keeps the job's --only list equal to the journey entries. Number formatting uses three decimals, the precision of the layout-shift scores. J2 renderer.layoutShiftScore is 0.233, not the window's 0.222: every master run from #1814 (page Back buttons in the header) on reads 0.233. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * ci(perf): check the journey counters whenever a summary was written Review follow-up (Greptile): the check ran only after the composite action succeeded, so a failed job-summary report after a written summary.json skipped every baseline. It now runs unless the job was cancelled, as long as the action produced a summary path. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: 4gray <fourgray@proton.me> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 17:06:15 -08:00 · 2026-10-06 11:43:55 +02:00
1 parent de2e1b19d7
commit 0a006cb027
@@ -49,8 +49,9 @@ function entries(baselines) {
    return flat;
 }

+// Three decimals: journey summaries round layout-shift scores to three.
 function formatNumber(value) {
-    return value.toLocaleString('en-US', { maximumFractionDigits: 2 });
+    return value.toLocaleString('en-US', { maximumFractionDigits: 3 });
 }

 /**
@@ -14,6 +14,9 @@
 *   are lowered by hand, with the measured output as evidence, never raised.
 * - Checking nothing is a failure: an empty baselines file, or `--only`
 *   naming an entry that does not exist, must not exit 0.
+ * - An optional `note` (a string) says why an entry is enforced, such as a
+ *   guard whose link to wall-clock is not validated (Principle 3). It is
+ *   printed with a failure, so the author sees what the counter stands for.
 *
 * `--only <journey>/<counter>` (repeatable) restricts the check to the named
 * baselines, so a script that measures one counter can check that counter
@@ -34,10 +37,11 @@ export const DEFAULT_BASELINES_PATH =
 /** The PR label that lets check-baseline-direction.mjs accept a weakening. */
 export const BASELINE_INCREASE_LABEL = 'perf-baseline-increase';

+// Three decimals: journey summaries round layout-shift scores to three.
 function formatNumber(value) {
    return Number.isInteger(value)
        ? value.toLocaleString('en-US')
-        : value.toLocaleString('en-US', { maximumFractionDigits: 2 });
+        : value.toLocaleString('en-US', { maximumFractionDigits: 3 });
 }

 function isPlainObject(value) {
@@ -82,6 +86,9 @@ export function validateBaselines(baselines) {
                    `Baseline ${label} has an invalid "slack" (must be an integer >= 0).`
                );
            }
+            if (entry.note !== undefined && typeof entry.note !== 'string') {
+                throw new Error(`Baseline ${label} has a non-string "note".`);
+            }
            if (
                entry.slack !== undefined &&
                entry.toleranceRatio !== undefined
@@ -185,8 +192,9 @@ export function compareToBaselines({ baselines, summary, only = [] }) {
            const limitText = describeLimit(entry, unit);

            if (measured > limit) {
+                const note = entry.note ? ` Note: ${entry.note}` : '';
                result.failures.push(
-                    `${label}: ${formatNumber(measured)}${unit} exceeds ${limitText} by ${formatNumber(measured - limit)}${unit}. Bring the value back down; baselines only move down. If the growth is a deliberate trade-off, raise the baseline in ${DEFAULT_BASELINES_PATH}, make the case in the PR, and ask a maintainer to add the ${BASELINE_INCREASE_LABEL} label.`
+                    `${label}: ${formatNumber(measured)}${unit} exceeds ${limitText} by ${formatNumber(measured - limit)}${unit}. Bring the value back down; baselines only move down. If the growth is a deliberate trade-off, raise the baseline in ${DEFAULT_BASELINES_PATH}, make the case in the PR, and ask a maintainer to add the ${BASELINE_INCREASE_LABEL} label.${note}`
                );
            } else if (entry.slack && measured > entry.value) {
                result.passed.push(
@@ -21,6 +21,10 @@ const committedBaselinesPath = fileURLToPath(
    new URL('./journey-baselines.json', import.meta.url)
 );

+const ciWorkflowPath = fileURLToPath(
+    new URL('../../.github/workflows/ci.yml', import.meta.url)
+);
+
 const baselines = {
    version: 1,
    journeys: {
@@ -357,6 +361,49 @@ test('rejects malformed baseline files', () => {
            }),
        /sets both "slack" and "toleranceRatio"/
    );
+    assert.throws(
+        () =>
+            validateBaselines({
+                journeys: { launch: { x: { value: 1, note: true } } },
+            }),
+        /non-string "note"/
+    );
+});
+
+test('a failing entry prints its note; a fractional score is exact', () => {
+    const guarded = {
+        journeys: {
+            'open-source': {
+                'renderer.layoutShiftScore': {
+                    value: 0.222,
+                    unit: 'score',
+                    slack: 0,
+                    note: 'guard only, not validated',
+                },
+            },
+        },
+    };
+    const check = (score) =>
+        compareToBaselines({
+            baselines: guarded,
+            summary: {
+                journeys: {
+                    'open-source': {
+                        counters: { 'renderer.layoutShiftScore': score },
+                    },
+                },
+            },
+        });
+
+    assert.deepEqual(check(0.222).failures, []);
+    const above = check(0.223);
+    assert.equal(above.failures.length, 1);
+    assert.match(
+        above.failures[0],
+        /0\.223 score exceeds baseline 0\.222 by 0\.001 score/
+    );
+    assert.match(above.failures[0], /Note: guard only, not validated$/);
+    assert.equal(check(0.221).tightenable.length, 1);
 });

 test('formats a summary line for both outcomes', () => {
@@ -444,6 +491,27 @@ test('the committed baselines file is valid and every entry names its evidence f
    }
 });

+test('the Performance journeys job checks every journey-run baseline', async () => {
+    const committed = JSON.parse(
+        await readFile(committedBaselinesPath, 'utf8')
+    );
+    const workflow = await readFile(ciWorkflowPath, 'utf8');
+    const step = workflow.match(
+        /- name: Check the journey counters against the baselines\n([\s\S]*?)(?=\n\s*- name: )/
+    );
+    assert.ok(step, 'ci.yml must keep the journey counter check step');
+    const only = [...step[1].matchAll(/--only (\S+)/g)].map((m) => m[1]);
+    // renderer.initialBytes is measured from the web build by the Initial
+    // bytes ratchet job (perf:initial-bytes:check); every other baseline comes
+    // from the journeys and must be enforced by this step, or it guards nothing.
+    const expected = Object.entries(committed.journeys)
+        .flatMap(([journey, entries]) =>
+            Object.keys(entries).map((name) => `${journey}/${name}`)
+        )
+        .filter((label) => label !== 'launch/renderer.initialBytes');
+    assert.deepEqual([...only].sort(), [...expected].sort());
+});
+
 async function runCli(summary, extraBaselines = baselines) {
    const summaryPath = path.join(
        workDir,
@@ -27,6 +27,90 @@
                "evidencePr": 1828,
                "evidenceRun": "https://github.com/4gray/iptvnator/actions/runs/37192092882",
                "measuredWith": "pnpm run perf:journeys (ubuntu-latest, xvfb), identical in 3 runs"
+            },
+            "main.modulesRegisteredBeforeWindow": {
+                "value": 2,
+                "unit": "phases",
+                "slack": 0,
+                "note": "guard only, not validated: no PR has yet shown that lowering this counter lowers the journey wall-clock (Principle 3)",
+                "updatedAt": "2026-10-04",
+                "evidencePr": null,
+                "evidenceRun": "https://github.com/4gray/iptvnator/actions/runs/37184230956",
+                "measuredWith": "pnpm run perf:journeys (ubuntu-latest, xvfb), identical in all 55 measured iterations of 11 master runs, 2026-10-03 to 2026-10-04"
+            },
+            "renderer.layoutShiftScore": {
+                "value": 0,
+                "unit": "score",
+                "slack": 0,
+                "note": "guard only, not validated: no PR has yet shown that lowering this counter lowers the journey wall-clock (Principle 3)",
+                "updatedAt": "2026-10-04",
+                "evidencePr": null,
+                "evidenceRun": "https://github.com/4gray/iptvnator/actions/runs/37184230956",
+                "measuredWith": "pnpm run perf:journeys (ubuntu-latest, xvfb), identical in all 55 measured iterations of 11 master runs, 2026-10-03 to 2026-10-04"
+            },
+            "renderer.layoutShiftScoreSettled": {
+                "value": 0,
+                "unit": "score",
+                "slack": 0,
+                "note": "guard only, not validated: no PR has yet shown that lowering this counter lowers the journey wall-clock (Principle 3)",
+                "updatedAt": "2026-10-04",
+                "evidencePr": null,
+                "evidenceRun": "https://github.com/4gray/iptvnator/actions/runs/37184230956",
+                "measuredWith": "pnpm run perf:journeys (ubuntu-latest, xvfb), identical in all 55 measured iterations of 11 master runs, 2026-10-03 to 2026-10-04"
+            }
+        },
+        "open-source": {
+            "main.mockHttpRequestsToSettled": {
+                "value": 1,
+                "unit": "requests",
+                "slack": 0,
+                "note": "guard only, not validated: no PR has yet shown that lowering this counter lowers the journey wall-clock (Principle 3)",
+                "updatedAt": "2026-10-04",
+                "evidencePr": null,
+                "evidenceRun": "https://github.com/4gray/iptvnator/actions/runs/37184230956",
+                "measuredWith": "pnpm run perf:journeys (ubuntu-latest, xvfb), identical in all 55 measured iterations of 11 master runs, 2026-10-03 to 2026-10-04"
+            },
+            "renderer.ipcCallsToFirstPage": {
+                "value": 17,
+                "unit": "calls",
+                "slack": 0,
+                "note": "guard only, not validated: no PR has yet shown that lowering this counter lowers the journey wall-clock (Principle 3)",
+                "updatedAt": "2026-10-04",
+                "evidencePr": null,
+                "evidenceRun": "https://github.com/4gray/iptvnator/actions/runs/37184230956",
+                "measuredWith": "pnpm run perf:journeys (ubuntu-latest, xvfb), identical in all 55 measured iterations of 11 master runs, 2026-10-03 to 2026-10-04"
+            },
+            "renderer.layoutShiftScore": {
+                "value": 0.233,
+                "unit": "score",
+                "slack": 0,
+                "note": "guard only, not validated: no PR has yet shown that lowering this counter lowers the journey wall-clock (Principle 3)",
+                "updatedAt": "2026-10-06",
+                "evidencePr": null,
+                "evidenceRun": "https://github.com/4gray/iptvnator/actions/runs/37372780064",
+                "measuredWith": "pnpm run perf:journeys (ubuntu-latest, xvfb), identical in every measured iteration of the 7 master runs from 84aef83a6 (#1814, which moved it from 0.222) to b78224376"
+            }
+        },
+        "playback": {
+            "renderer.httpRequestsToPlaying": {
+                "value": 2,
+                "unit": "requests",
+                "slack": 0,
+                "note": "guard only, not validated: no PR has yet shown that lowering this counter lowers the journey wall-clock (Principle 3)",
+                "updatedAt": "2026-10-04",
+                "evidencePr": null,
+                "evidenceRun": "https://github.com/4gray/iptvnator/actions/runs/37184230956",
+                "measuredWith": "pnpm run perf:journeys (ubuntu-latest, xvfb), identical in all 55 measured iterations of 11 master runs, 2026-10-03 to 2026-10-04"
+            },
+            "renderer.layoutShiftScore": {
+                "value": 0.001,
+                "unit": "score",
+                "slack": 0,
+                "note": "guard only, not validated: no PR has yet shown that lowering this counter lowers the journey wall-clock (Principle 3)",
+                "updatedAt": "2026-10-04",
+                "evidencePr": null,
+                "evidenceRun": "https://github.com/4gray/iptvnator/actions/runs/37184230956",
+                "measuredWith": "pnpm run perf:journeys (ubuntu-latest, xvfb), identical in all 55 measured iterations of 11 master runs, 2026-10-03 to 2026-10-04"
            }
        }
    }
@@ -11,6 +11,7 @@
            "inputs": [
                "{projectRoot}/*.mjs",
                "{projectRoot}/*.json",
+                "{workspaceRoot}/.github/workflows/ci.yml",
                { "externalDependencies": ["parse5"] }
            ],
            "options": {
@@ -54,8 +54,9 @@ function isPlainObject(value) {
    return typeof value === 'object' && value !== null && !Array.isArray(value);
 }

+// Three decimals: journey summaries round layout-shift scores to three.
 function formatNumber(value) {
-    return value.toLocaleString('en-US', { maximumFractionDigits: 2 });
+    return value.toLocaleString('en-US', { maximumFractionDigits: 3 });
 }

 /**
@@ -93,6 +93,66 @@ test('a counter below its value in every run drops to the largest run', () => {
    assert.match(direction.lowered[0], /1,064 -> 1,059 bytes/);
 });

+test('a journey-run entry keeps its note and is lowered to three decimals', () => {
+    const score = {
+        value: 0.222,
+        unit: 'score',
+        slack: 0,
+        note: 'guard only, not validated',
+        updatedAt: '2026-10-04',
+        evidencePr: 1,
+        measuredWith: 'pnpm run perf:journeys',
+    };
+    const baselines = {
+        version: 1,
+        journeys: {
+            ...baselinesFile().journeys,
+            'open-source': { 'renderer.layoutShiftScore': score },
+            playback: { 'renderer.layoutShiftScore': { ...score, value: 0 } },
+        },
+    };
+    // The weekly job merges each runner's initial-bytes and journey summaries.
+    const runs = [0.221, 0.22, 0.221].map((value, index) =>
+        mergeRunSummaries(
+            [
+                bytesRun(1000),
+                {
+                    journeys: {
+                        'open-source': {
+                            counters: { 'renderer.layoutShiftScore': value },
+                        },
+                        playback: {
+                            counters: { 'renderer.layoutShiftScore': 0 },
+                        },
+                    },
+                },
+            ],
+            `run ${index + 1}`
+        )
+    );
+    const result = tighten(baselines, runs);
+    assert.deepEqual(
+        result.baselines.journeys['open-source']['renderer.layoutShiftScore'],
+        {
+            ...score,
+            value: 0.221,
+            updatedAt: '2026-10-05',
+            measuredWith:
+                '.github/workflows/performance-ratchet.yml, max of 3 runs',
+            evidenceRun: RUN_URL,
+        }
+    );
+    assert.deepEqual(
+        result.baselines.journeys.playback,
+        baselines.journeys.playback,
+        'a counter already at 0 is kept'
+    );
+    assert.match(
+        formatReport(result),
+        /\| `open-source\/renderer\.layoutShiftScore` \| 0\.222 score \| 0\.221 \| 0\.22 \| 0\.221 \| lowered to 0\.221 score \|/
+    );
+});
+
 test('one run at or above the value keeps the baseline untouched', () => {
    for (const measured of [1000, 1010]) {
        const baselines = baselinesFile();