iptvnator/tools/performance/check-journey-ratchet.test.mjs

import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { after, before, test } from 'node:test';

import {
    DEFAULT_BASELINES_PATH,
    compareToBaselines,
    formatResult,
    parseArgs,
    validateBaselines,
} from './check-journey-ratchet.mjs';

const scriptPath = fileURLToPath(
    new URL('./check-journey-ratchet.mjs', import.meta.url)
);
const committedBaselinesPath = fileURLToPath(
    new URL('./journey-baselines.json', import.meta.url)
);

const ciWorkflowPath = fileURLToPath(
    new URL('../../.github/workflows/ci.yml', import.meta.url)
);

const baselines = {
    version: 1,
    journeys: {
        launch: {
            'renderer.initialBytes': { value: 2750491, unit: 'bytes' },
            spawnToFirstCardMs: {
                value: 1000,
                unit: 'ms',
                toleranceRatio: 1.25,
            },
        },
    },
};

const summaryWith = (initialBytes, wallClockMs = 1000) => ({
    journeys: {
        launch: {
            counters: { 'renderer.initialBytes': initialBytes },
            wallClock: { spawnToFirstCardMs: wallClockMs },
        },
    },
});

let workDir;

before(async () => {
    workDir = await mkdtemp(path.join(os.tmpdir(), 'check-journey-ratchet-'));
});

after(async () => {
    await rm(workDir, { recursive: true, force: true });
});

test('a counter equal to its baseline passes', () => {
    const result = compareToBaselines({
        baselines,
        summary: summaryWith(2750491),
    });
    assert.deepEqual(result.failures, []);
    assert.deepEqual(result.tightenable, []);
    assert.equal(result.passed.length, 2);
    assert.match(
        result.passed[0],
        /launch\/renderer\.initialBytes: 2,750,491 bytes within baseline 2,750,491/
    );
});

test('a counter one byte above its baseline fails with the delta', () => {
    const result = compareToBaselines({
        baselines,
        summary: summaryWith(2750492),
    });
    assert.equal(result.failures.length, 1);
    assert.match(
        result.failures[0],
        /launch\/renderer\.initialBytes: 2,750,492 bytes exceeds baseline 2,750,491 by 1 bytes/
    );
    assert.match(result.failures[0], /baselines only move down/);
});

test('a counter below its baseline passes and asks to tighten', () => {
    const result = compareToBaselines({
        baselines,
        summary: summaryWith(2600000),
    });
    assert.deepEqual(result.failures, []);
    assert.equal(result.tightenable.length, 1);
    assert.match(
        result.tightenable[0],
        /below baseline 2,750,491 by 150,491 bytes/
    );
    assert.match(
        result.tightenable[0],
        new RegExp(DEFAULT_BASELINES_PATH.replace(/\//g, '\\/'))
    );
});

test('a counter with slack passes up to value + slack and reports the slack used', () => {
    const withSlack = {
        version: 1,
        journeys: {
            launch: {
                'renderer.initialBytes': {
                    value: 1000,
                    unit: 'bytes',
                    slack: 100,
                },
            },
        },
    };
    const check = (initialBytes) =>
        compareToBaselines({
            baselines: withSlack,
            summary: {
                journeys: {
                    launch: {
                        counters: { 'renderer.initialBytes': initialBytes },
                    },
                },
            },
        });

    const within = check(1100);
    assert.deepEqual(within.failures, []);
    assert.deepEqual(within.tightenable, []);
    assert.match(
        within.passed[0],
        /1,100 bytes within 1,100 \(baseline 1,000 \+ slack 100 bytes\); uses 100 of 100 bytes slack/
    );

    const above = check(1101);
    assert.equal(above.failures.length, 1);
    assert.match(
        above.failures[0],
        /1,101 bytes exceeds 1,100 \(baseline 1,000 \+ slack 100 bytes\) by 1 bytes/
    );
    assert.match(above.failures[0], /add the perf-baseline-increase label/);

    const exact = check(1000);
    assert.match(exact.passed[0], /1,000 bytes within 1,100/);
    assert.doesNotMatch(exact.passed[0], /uses/);

    const below = check(990);
    assert.match(below.tightenable[0], /below baseline 1,000 by 10 bytes/);
});

test('wall-clock entries fail only above value × toleranceRatio', () => {
    const within = compareToBaselines({
        baselines,
        summary: summaryWith(2750491, 1250),
    });
    assert.deepEqual(within.failures, []);
    assert.match(
        within.passed[1],
        /1,250 ms within 1,250 \(baseline 1,000 × 1\.25\)\.$/
    );
    assert.doesNotMatch(within.passed[1], /slack/);

    const above = compareToBaselines({
        baselines,
        summary: summaryWith(2750491, 1251),
    });
    assert.equal(above.failures.length, 1);
    assert.match(
        above.failures[0],
        /spawnToFirstCardMs: 1,251 ms exceeds 1,250 \(baseline 1,000 × 1\.25\) by 1 ms/
    );

    const below = compareToBaselines({
        baselines,
        summary: summaryWith(2750491, 900),
    });
    assert.equal(below.tightenable.length, 1);
    assert.match(
        below.tightenable[0],
        /spawnToFirstCardMs: 900 ms is below baseline 1,000/
    );
});

test('a baseline without a measurement fails instead of being skipped', () => {
    const result = compareToBaselines({
        baselines,
        summary: { journeys: { launch: { counters: {} } } },
    });
    assert.equal(result.failures.length, 2);
    assert.match(
        result.failures[0],
        /renderer\.initialBytes: baseline 2,750,491 bytes has no measurement/
    );
    assert.match(
        result.failures[0],
        /cannot be bypassed by dropping a measurement/
    );
});

test('a value in the wrong summary section counts as missing', () => {
    const counterInWallClock = compareToBaselines({
        baselines,
        summary: {
            journeys: {
                launch: {
                    counters: {},
                    wallClock: {
                        'renderer.initialBytes': 2750491,
                        spawnToFirstCardMs: 1000,
                    },
                },
            },
        },
    });
    assert.equal(counterInWallClock.failures.length, 1);
    assert.match(
        counterInWallClock.failures[0],
        /renderer\.initialBytes: baseline 2,750,491 bytes has no measurement under journeys\.launch\.counters/
    );

    const wallClockInCounters = compareToBaselines({
        baselines,
        summary: {
            journeys: {
                launch: {
                    counters: {
                        'renderer.initialBytes': 2750491,
                        spawnToFirstCardMs: 1000,
                    },
                },
            },
        },
    });
    assert.equal(wallClockInCounters.failures.length, 1);
    assert.match(
        wallClockInCounters.failures[0],
        /spawnToFirstCardMs: baseline 1,000 ms has no measurement under journeys\.launch\.wallClock/
    );
});

test('an empty or malformed summary fails every baseline', () => {
    assert.equal(
        compareToBaselines({ baselines, summary: {} }).failures.length,
        2
    );
    assert.equal(
        compareToBaselines({ baselines, summary: null }).failures.length,
        2
    );
    const nonNumeric = compareToBaselines({
        baselines,
        summary: {
            journeys: {
                launch: { counters: { 'renderer.initialBytes': '2750491' } },
            },
        },
    });
    assert.match(nonNumeric.failures[0], /"2750491" is not a finite number/);
});

test('a measured counter without a baseline is reported but does not fail', () => {
    const summary = summaryWith(2750491);
    summary.journeys.launch.counters['renderer.cdTicksToFirstCard'] = 12;
    summary.journeys.search = { counters: { sqlStatementsPerKeystroke: 3 } };
    const result = compareToBaselines({ baselines, summary });
    assert.deepEqual(result.failures, []);
    assert.deepEqual(
        result.unbaselined.map((line) => line.split(':')[0]),
        [
            'launch/renderer.cdTicksToFirstCard',
            'search/sqlStatementsPerKeystroke',
        ]
    );
});

test('an empty baselines file fails instead of passing vacuously', () => {
    const result = compareToBaselines({
        baselines: { journeys: {} },
        summary: summaryWith(2750491),
    });
    assert.equal(result.failures.length, 1);
    assert.match(result.failures[0], /No baselines were checked/);
    assert.match(
        formatResult(result),
        /Journey ratchet failed: 1 of 1 baselines exceeded/
    );
});

test('--only restricts the check to the named baselines', () => {
    const result = compareToBaselines({
        baselines,
        summary: {
            journeys: {
                launch: { counters: { 'renderer.initialBytes': 2750491 } },
            },
        },
        only: ['launch/renderer.initialBytes'],
    });
    assert.deepEqual(result.failures, []);
    assert.equal(result.passed.length, 1);
    assert.match(result.passed[0], /launch\/renderer\.initialBytes/);
});

test('--only naming a missing baseline fails rather than checking nothing', () => {
    const result = compareToBaselines({
        baselines,
        summary: summaryWith(2750491),
        only: ['launch/does.notExist'],
    });
    assert.equal(result.failures.length, 2);
    assert.match(
        result.failures[0],
        /launch\/does\.notExist: --only names a baseline that does not exist/
    );
    assert.match(result.failures[1], /No baselines were checked/);
});

test('rejects malformed baseline files', () => {
    assert.throws(() => validateBaselines({}), /object with a "journeys" map/);
    assert.throws(
        () => validateBaselines({ journeys: { launch: [] } }),
        /journey "launch" must map/
    );
    assert.throws(
        () =>
            validateBaselines({
                journeys: { launch: { x: { value: 'big' } } },
            }),
        /Baseline launch\/x needs a finite numeric "value"/
    );
    assert.throws(
        () =>
            validateBaselines({
                journeys: { launch: { x: { value: 1, toleranceRatio: 0.5 } } },
            }),
        /invalid "toleranceRatio"/
    );
    assert.throws(
        () =>
            validateBaselines({
                journeys: { launch: { x: { value: 1, slack: -1 } } },
            }),
        /invalid "slack"/
    );
    assert.throws(
        () =>
            validateBaselines({
                journeys: { launch: { x: { value: 1, slack: 1.5 } } },
            }),
        /invalid "slack"/
    );
    assert.throws(
        () =>
            validateBaselines({
                journeys: {
                    launch: { x: { value: 1, slack: 1, toleranceRatio: 1.1 } },
                },
            }),
        /sets both "slack" and "toleranceRatio"/
    );
    assert.throws(
        () =>
            validateBaselines({
                journeys: { launch: { x: { value: 1, note: true } } },
            }),
        /non-string "note"/
    );
});

test('a failing entry prints its note; a fractional score is exact', () => {
    const guarded = {
        journeys: {
            'open-source': {
                'renderer.layoutShiftScore': {
                    value: 0.222,
                    unit: 'score',
                    slack: 0,
                    note: 'guard only, not validated',
                },
            },
        },
    };
    const check = (score) =>
        compareToBaselines({
            baselines: guarded,
            summary: {
                journeys: {
                    'open-source': {
                        counters: { 'renderer.layoutShiftScore': score },
                    },
                },
            },
        });

    assert.deepEqual(check(0.222).failures, []);
    const above = check(0.223);
    assert.equal(above.failures.length, 1);
    assert.match(
        above.failures[0],
        /0\.223 score exceeds baseline 0\.222 by 0\.001 score/
    );
    assert.match(above.failures[0], /Note: guard only, not validated$/);
    assert.equal(check(0.221).tightenable.length, 1);
});

test('formats a summary line for both outcomes', () => {
    const ok = formatResult(
        compareToBaselines({ baselines, summary: summaryWith(2600000) })
    );
    assert.match(ok, /^ok {7}launch\/spawnToFirstCardMs/m);
    assert.match(ok, /^tighten {2}launch\/renderer\.initialBytes/m);
    assert.match(
        ok,
        /Journey ratchet OK: 2 baselines checked, 1 can be tightened\.$/
    );

    const failed = formatResult(
        compareToBaselines({ baselines, summary: summaryWith(3000000) })
    );
    assert.match(failed, /^FAIL {5}launch\/renderer\.initialBytes/m);
    assert.match(
        failed,
        /Journey ratchet failed: 1 of 2 baselines exceeded\.$/
    );
});

test('parses CLI arguments and requires --summary', () => {
    assert.deepEqual(parseArgs(['--summary', 's.json']), {
        summary: 's.json',
        baselines: DEFAULT_BASELINES_PATH,
        only: [],
    });
    assert.deepEqual(
        parseArgs([
            '--',
            '--summary=s.json',
            '--baselines=b.json',
            '--only',
            'launch/a',
            '--only=search/b',
        ]),
        {
            summary: 's.json',
            baselines: 'b.json',
            only: ['launch/a', 'search/b'],
        }
    );
    assert.throws(
        () => parseArgs(['--summary', 's', '--only', 'launch']),
        /--only expects <journey>\/<counter>/
    );
    assert.throws(
        () => parseArgs(['--summary', 's', '--only']),
        /Missing value for --only/
    );
    assert.throws(
        () => parseArgs([]),
        /--summary <journey-summary\.json> is required/
    );
    assert.throws(
        () => parseArgs(['--summary']),
        /Missing value for --summary/
    );
    assert.throws(
        () => parseArgs(['--summary', 's', '--strict']),
        /Unknown argument: --strict/
    );
});

test('the committed baselines file is valid and every entry names its evidence fields', async () => {
    const committed = JSON.parse(
        await readFile(committedBaselinesPath, 'utf8')
    );
    validateBaselines(committed);
    assert.ok(
        committed.journeys.launch?.['renderer.initialBytes'],
        'the launch/renderer.initialBytes baseline must stay committed; the CI ratchet guards it'
    );
    for (const entries of Object.values(committed.journeys)) {
        for (const entry of Object.values(entries)) {
            assert.match(entry.updatedAt, /^\d{4}-\d{2}-\d{2}$/);
            assert.ok(
                'evidencePr' in entry,
                'evidencePr must be present (null before the first PR)'
            );
            assert.equal(typeof entry.measuredWith, 'string');
        }
    }
});

test('the Performance journeys job checks every journey-run baseline', async () => {
    const committed = JSON.parse(
        await readFile(committedBaselinesPath, 'utf8')
    );
    const workflow = await readFile(ciWorkflowPath, 'utf8');
    const step = workflow.match(
        /- name: Check the journey counters against the baselines\n([\s\S]*?)(?=\n\s*- name: )/
    );
    assert.ok(step, 'ci.yml must keep the journey counter check step');
    const only = [...step[1].matchAll(/--only (\S+)/g)].map((m) => m[1]);
    // renderer.initialBytes is measured from the web build by the Initial
    // bytes ratchet job (perf:initial-bytes:check); every other baseline comes
    // from the journeys and must be enforced by this step, or it guards nothing.
    const expected = Object.entries(committed.journeys)
        .flatMap(([journey, entries]) =>
            Object.keys(entries).map((name) => `${journey}/${name}`)
        )
        .filter((label) => label !== 'launch/renderer.initialBytes');
    assert.deepEqual([...only].sort(), [...expected].sort());
});

async function runCli(summary, extraBaselines = baselines) {
    const summaryPath = path.join(
        workDir,
        `summary-${Date.now()}-${Math.random()}.json`
    );
    const baselinesPath = path.join(
        workDir,
        `baselines-${Date.now()}-${Math.random()}.json`
    );
    await writeFile(summaryPath, JSON.stringify(summary));
    await writeFile(baselinesPath, JSON.stringify(extraBaselines));
    return spawnSync(
        process.execPath,
        [scriptPath, '--summary', summaryPath, '--baselines', baselinesPath],
        { encoding: 'utf8' }
    );
}

test('CLI exits 0 when within baselines and 1 when a counter grew', async () => {
    const ok = await runCli(summaryWith(2750491));
    assert.equal(ok.status, 0, ok.stderr);
    assert.match(ok.stdout, /Journey ratchet OK/);

    const grew = await runCli(summaryWith(2750492));
    assert.equal(grew.status, 1);
    assert.match(grew.stderr, /FAIL {5}launch\/renderer\.initialBytes/);
});

test('CLI exits 1 with a readable message when the summary is missing', () => {
    const result = spawnSync(
        process.execPath,
        [scriptPath, '--summary', path.join(workDir, 'nope.json')],
        { encoding: 'utf8' }
    );
    assert.equal(result.status, 1);
    assert.match(
        result.stderr,
        /check-journey-ratchet: Cannot read journey summary at/
    );
});