ci(e2e): shard the Electron Playwright suite per OS (#1700)

Run the sequential Electron E2E suite as three Playwright shards per OS
(one runner each) and summarize all shards of an OS in one follow-up job.
The semantic summary script accepts a directory of shard reports, merges
them and refuses to write when a shard is missing, duplicated or
malformed, or when an explicit input does not exist.

Slowest shard per OS in the final run: ubuntu 12.5 min (was 26),
macOS 13.7 min (was 34), Windows 24.5 min (was 35).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tools/coverage/e2e-semantic-summary.mjs

@@ -11,6 +11,13 @@ import {
 import path from 'node:path';
 import process from 'node:process';
 
+import {
+    describeShardReports,
+    findPlaywrightJsonReports,
+    loadPlaywrightReports,
+    verifyShardReports,
+} from './e2e-shard-reports.mjs';
+
 const workspaceRoot = process.cwd();
 const args = process.argv.slice(2);
 const projectArg = valueFor('--project');
@@ -18,7 +25,12 @@ const inputArg = valueFor('--input');
 const policy = JSON.parse(
     readFileSync(path.join(workspaceRoot, 'tools/coverage/coverage-policy.json'), 'utf8')
 );
-const outputDir = path.join(workspaceRoot, policy.reporting.e2eSummaryDir);
+const outputDirArg = valueFor('--output-dir');
+const outputDir = path.resolve(
+    workspaceRoot,
+    outputDirArg ?? policy.reporting.e2eSummaryDir
+);
+const outputDirLabel = outputDirArg ?? policy.reporting.e2eSummaryDir;
 
 function valueFor(flag) {
     const prefixed = args.find((arg) => arg.startsWith(`${flag}=`));
@@ -57,8 +69,19 @@ function tagsFromTitle(title) {
     );
 }
 
-function collectFromPlaywrightJson(filePath, projectName) {
-    const report = JSON.parse(readFileSync(filePath, 'utf8'));
+function fail(message) {
+    console.error(`e2e-semantic-summary: ${message}`);
+    if (process.env.GITHUB_STEP_SUMMARY) {
+        writeFileSync(
+            process.env.GITHUB_STEP_SUMMARY,
+            `\n> **E2E semantic summary not written:** ${message}\n`,
+            { flag: 'a' }
+        );
+    }
+    process.exit(1);
+}
+
+function collectFromPlaywrightJson(report, projectName) {
     const tests = [];
 
     function walkSuite(suite, inheritedFile) {
@@ -137,16 +160,58 @@ function defaultInputFor(projectName) {
     return path.join(workspaceRoot, 'dist/test-results', projectName, 'results.json');
 }
 
-function collectTests(projectName) {
+/**
+ * `--output-dir` overrides the policy's summary directory so several runs
+ * (one per OS in CI) can be summarized side by side in one job.
+ *
+ * `--input` may name one Playwright JSON report or a directory that holds the
+ * `results.json` of every shard (as downloaded from the per-shard CI
+ * artifacts). An explicit input that does not exist, a directory without any
+ * report, or an incomplete or duplicated shard set aborts instead of writing
+ * a partial summary. Only the implicit default falls back to scanning the
+ * spec sources.
+ */
+function resolveReportPaths(projectName) {
     const inputPath = inputArg
         ? path.resolve(workspaceRoot, inputArg)
         : defaultInputFor(projectName);
 
-    if (existsSync(inputPath)) {
-        return collectFromPlaywrightJson(inputPath, projectName);
+    if (!existsSync(inputPath)) {
+        if (inputArg) {
+            fail(`Playwright JSON report input does not exist: ${inputPath}`);
+        }
+        return [];
+    }
+    if (!statSync(inputPath).isDirectory()) {
+        return [inputPath];
+    }
+    const found = findPlaywrightJsonReports(inputPath);
+    if (found.length === 0) {
+        fail(`no Playwright JSON reports (results.json) found under ${inputPath}`);
+    }
+    return found;
+}
+
+function collectTests(projectName) {
+    const reportPaths = resolveReportPaths(projectName);
+    if (reportPaths.length === 0) {
+        return { tests: collectFromSource(projectName), reports: [] };
     }
 
-    return collectFromSource(projectName);
+    const reports = loadPlaywrightReports(reportPaths);
+    const verification = verifyShardReports(reports);
+    if (!verification.ok) {
+        fail(
+            `${projectName} reports do not form one complete run: ${verification.problems.join('; ')}`
+        );
+    }
+
+    return {
+        tests: reports.flatMap((entry) =>
+            collectFromPlaywrightJson(entry.report, projectName)
+        ),
+        reports,
+    };
 }
 
 function statusCounts(tests) {
@@ -174,7 +239,7 @@ function journeyMatches(journey, tests) {
     );
 }
 
-function markdownFor(projectName, tests) {
+function markdownFor(projectName, tests, reportsLabel) {
     const counts = statusCounts(tests);
     const countsText = Object.entries(counts)
         .map(([status, count]) => `${status}: ${count}`)
@@ -197,6 +262,8 @@ function markdownFor(projectName, tests) {
 
 Source: ${tests.some((test) => test.status === 'not-run') ? 'spec source scan' : 'Playwright JSON report'}
 
+Reports: ${reportsLabel}
+
 Total tracked tests: ${tests.length}
 
 Statuses: ${countsText || 'none'}
@@ -216,12 +283,23 @@ ${journeys || '| _none_ | _n/a_ | 0 | missing |'}
 }
 
 const projects = projectArg ? [projectArg] : ['web-e2e', 'electron-backend-e2e'];
-const allTests = projects.flatMap((projectName) => collectTests(projectName));
+const collected = projects.map((projectName) => ({
+    projectName,
+    ...collectTests(projectName),
+}));
+const allTests = collected.flatMap((entry) => entry.tests);
+const reportsLabel = collected
+    .map((entry) =>
+        projectArg
+            ? describeShardReports(entry.reports)
+            : `${entry.projectName}: ${describeShardReports(entry.reports)}`
+    )
+    .join('; ');
 
 mkdirSync(outputDir, { recursive: true });
 
 if (projectArg) {
-    const content = markdownFor(projectArg, allTests);
+    const content = markdownFor(projectArg, allTests, reportsLabel);
     writeFileSync(path.join(outputDir, `${projectArg}-semantic-summary.md`), content);
     writeFileSync(
         path.join(outputDir, `${projectArg}-semantic-summary.json`),
@@ -230,9 +308,9 @@ if (projectArg) {
     if (process.env.GITHUB_STEP_SUMMARY) {
         writeFileSync(process.env.GITHUB_STEP_SUMMARY, `\n${content}\n`, { flag: 'a' });
     }
-    console.log(`Wrote ${policy.reporting.e2eSummaryDir}/${projectArg}-semantic-summary.md`);
+    console.log(`Wrote ${outputDirLabel}/${projectArg}-semantic-summary.md`);
 } else {
-    const content = markdownFor(undefined, allTests);
+    const content = markdownFor(undefined, allTests, reportsLabel);
     writeFileSync(path.join(outputDir, 'semantic-summary.md'), content);
     writeFileSync(
         path.join(outputDir, 'semantic-summary.json'),
@@ -241,5 +319,5 @@ if (projectArg) {
     if (process.env.GITHUB_STEP_SUMMARY) {
         writeFileSync(process.env.GITHUB_STEP_SUMMARY, `\n${content}\n`, { flag: 'a' });
     }
-    console.log(`Wrote ${policy.reporting.e2eSummaryDir}/semantic-summary.md`);
+    console.log(`Wrote ${outputDirLabel}/semantic-summary.md`);
 }

tools/coverage/e2e-shard-reports.mjs

@@ -0,0 +1,148 @@
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import path from 'node:path';
+
+/**
+ * Helpers for reading one or more Playwright JSON reports (`results.json`).
+ *
+ * The Electron E2E workflow runs the suite as several Playwright shards, one
+ * per runner. Each shard writes its own `results.json` that carries
+ * `config.shard = { current, total }` and only that shard's tests. A semantic
+ * summary is only honest when every shard is present exactly once, so callers
+ * verify the set before merging.
+ */
+
+export const playwrightReportFileName = 'results.json';
+
+/** Recursively lists every `results.json` below `rootDir`, sorted by path. */
+export function findPlaywrightJsonReports(rootDir) {
+    if (!existsSync(rootDir) || !statSync(rootDir).isDirectory()) {
+        return [];
+    }
+    const found = [];
+    for (const entry of readdirSync(rootDir)) {
+        const fullPath = path.join(rootDir, entry);
+        if (statSync(fullPath).isDirectory()) {
+            found.push(...findPlaywrightJsonReports(fullPath));
+        } else if (entry === playwrightReportFileName) {
+            found.push(fullPath);
+        }
+    }
+    return found.sort();
+}
+
+/**
+ * Parses each report and extracts its shard descriptor: `null` when the run
+ * was not sharded (`config.shard` absent or `null`). A descriptor that is
+ * present but malformed is kept as `malformedShard` so verification rejects
+ * it instead of mistaking a partial run for a complete unsharded one.
+ */
+export function loadPlaywrightReports(reportPaths) {
+    return reportPaths.map((reportPath) => {
+        const report = JSON.parse(readFileSync(reportPath, 'utf8'));
+        const shard = report.config?.shard ?? null;
+        const wellFormed =
+            shard !== null &&
+            Number.isInteger(shard.current) &&
+            Number.isInteger(shard.total);
+        return {
+            path: reportPath,
+            report,
+            shard: wellFormed ? { current: shard.current, total: shard.total } : null,
+            malformedShard: shard !== null && !wellFormed,
+        };
+    });
+}
+
+/**
+ * Checks that the loaded reports form exactly one complete run: either a
+ * single unsharded report, or every shard `1..total` exactly once.
+ */
+export function verifyShardReports(reports) {
+    const problems = [];
+    if (reports.length === 0) {
+        problems.push('no Playwright JSON reports were provided');
+        return { ok: false, problems };
+    }
+
+    const malformed = reports.filter((entry) => entry.malformedShard);
+    if (malformed.length > 0) {
+        problems.push(`malformed config.shard descriptor: ${describePaths(malformed)}`);
+    }
+
+    const unsharded = reports.filter(
+        (entry) => entry.shard === null && !entry.malformedShard
+    );
+    const sharded = reports.filter((entry) => entry.shard !== null);
+
+    if (unsharded.length > 0 && sharded.length > 0) {
+        problems.push(
+            `mixed sharded and unsharded reports: ${describePaths(unsharded)} carry no shard descriptor`
+        );
+    }
+    if (unsharded.length > 1) {
+        problems.push(
+            `${unsharded.length} unsharded reports would count every test more than once: ${describePaths(unsharded)}`
+        );
+    }
+
+    if (sharded.length > 0) {
+        const totals = new Set(sharded.map((entry) => entry.shard.total));
+        if (totals.size > 1) {
+            problems.push(
+                `shard totals disagree: ${Array.from(totals).sort().join(', ')}`
+            );
+        } else {
+            const [total] = totals;
+            const seen = new Map();
+            for (const entry of sharded) {
+                const list = seen.get(entry.shard.current) ?? [];
+                list.push(entry);
+                seen.set(entry.shard.current, list);
+            }
+            const missing = [];
+            for (let index = 1; index <= total; index += 1) {
+                if (!seen.has(index)) {
+                    missing.push(`${index}/${total}`);
+                }
+            }
+            if (missing.length > 0) {
+                problems.push(`missing shards: ${missing.join(', ')}`);
+            }
+            for (const [current, entries] of seen) {
+                if (entries.length > 1) {
+                    problems.push(
+                        `shard ${current}/${total} appears ${entries.length} times: ${describePaths(entries)}`
+                    );
+                }
+                if (current < 1 || current > total) {
+                    problems.push(`shard ${current}/${total} is outside 1..${total}`);
+                }
+            }
+        }
+    }
+
+    return { ok: problems.length === 0, problems };
+}
+
+/** Human-readable label for the summary, e.g. `3/3 shards (1/3, 2/3, 3/3)`. */
+export function describeShardReports(reports) {
+    if (reports.length === 0) {
+        return 'none';
+    }
+    const sharded = reports.filter((entry) => entry.shard !== null);
+    if (sharded.length === 0) {
+        return reports.length === 1
+            ? '1 report (unsharded)'
+            : `${reports.length} reports (unsharded)`;
+    }
+    const total = Math.max(...sharded.map((entry) => entry.shard.total));
+    const labels = sharded
+        .map((entry) => entry.shard)
+        .sort((left, right) => left.current - right.current)
+        .map((shard) => `${shard.current}/${shard.total}`);
+    return `${sharded.length}/${total} shards (${labels.join(', ')})`;
+}
+
+function describePaths(entries) {
+    return entries.map((entry) => entry.path).join(', ');
+}

tools/coverage/e2e-shard-reports.test.mjs

@@ -0,0 +1,407 @@
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import {
+    copyFileSync,
+    existsSync,
+    mkdirSync,
+    mkdtempSync,
+    readFileSync,
+    rmSync,
+    writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { afterEach, describe, it } from 'node:test';
+
+import {
+    describeShardReports,
+    findPlaywrightJsonReports,
+    loadPlaywrightReports,
+    verifyShardReports,
+} from './e2e-shard-reports.mjs';
+
+const toolsDir = path.dirname(fileURLToPath(import.meta.url));
+const repositoryRoot = path.resolve(toolsDir, '../..');
+const summaryScript = path.join(toolsDir, 'e2e-semantic-summary.mjs');
+const temporaryRoots = [];
+
+afterEach(() => {
+    for (const root of temporaryRoots.splice(0)) {
+        rmSync(root, { recursive: true, force: true });
+    }
+});
+
+function makeTemporaryDir(prefix) {
+    const root = mkdtempSync(path.join(tmpdir(), prefix));
+    temporaryRoots.push(root);
+    return root;
+}
+
+function playwrightReport({ shard, file, titles, status = 'passed' }) {
+    return {
+        config: { shard },
+        suites: [
+            {
+                file,
+                specs: titles.map((title) => ({
+                    title,
+                    tags: [],
+                    tests: [{ results: [{ status }] }],
+                })),
+            },
+        ],
+        errors: [],
+        stats: {},
+    };
+}
+
+function writeReport(root, relativeDir, report) {
+    const directory = path.join(root, relativeDir);
+    mkdirSync(directory, { recursive: true });
+    const reportPath = path.join(directory, 'results.json');
+    writeFileSync(reportPath, JSON.stringify(report));
+    return reportPath;
+}
+
+function entry(shard, reportPath = `report-${shard?.current ?? 'single'}.json`) {
+    return { path: reportPath, shard, report: {} };
+}
+
+describe('findPlaywrightJsonReports', () => {
+    it('lists every nested results.json in sorted order', () => {
+        const root = makeTemporaryDir('iptvnator-shard-find-');
+        const second = writeReport(root, 'b-shard-2/dist/test-results/x', {});
+        const first = writeReport(root, 'a-shard-1/dist/test-results/x', {});
+        writeFileSync(path.join(root, 'a-shard-1', 'other.json'), '{}');
+
+        assert.deepEqual(findPlaywrightJsonReports(root), [first, second]);
+    });
+
+    it('returns nothing for a missing directory or a file path', () => {
+        const root = makeTemporaryDir('iptvnator-shard-find-');
+        const reportPath = writeReport(root, 'single', {});
+
+        assert.deepEqual(findPlaywrightJsonReports(path.join(root, 'nope')), []);
+        assert.deepEqual(findPlaywrightJsonReports(reportPath), []);
+    });
+});
+
+describe('loadPlaywrightReports', () => {
+    it('extracts a shard descriptor and flags malformed ones', () => {
+        const root = makeTemporaryDir('iptvnator-shard-load-');
+        const sharded = writeReport(root, 'sharded', {
+            config: { shard: { current: 2, total: 3 } },
+        });
+        const unsharded = writeReport(root, 'unsharded', { config: { shard: null } });
+        const malformed = writeReport(root, 'malformed', {
+            config: { shard: { current: '2', total: 3 } },
+        });
+
+        const loaded = loadPlaywrightReports([sharded, unsharded, malformed]);
+
+        assert.deepEqual(loaded[0].shard, { current: 2, total: 3 });
+        assert.equal(loaded[0].malformedShard, false);
+        assert.equal(loaded[1].shard, null);
+        assert.equal(loaded[1].malformedShard, false);
+        assert.equal(loaded[2].shard, null);
+        assert.equal(loaded[2].malformedShard, true);
+    });
+});
+
+describe('verifyShardReports', () => {
+    it('accepts a single unsharded report', () => {
+        assert.deepEqual(verifyShardReports([entry(null)]), { ok: true, problems: [] });
+    });
+
+    it('accepts a complete shard set in any order', () => {
+        const reports = [
+            entry({ current: 3, total: 3 }),
+            entry({ current: 1, total: 3 }),
+            entry({ current: 2, total: 3 }),
+        ];
+
+        assert.deepEqual(verifyShardReports(reports), { ok: true, problems: [] });
+    });
+
+    it('rejects a malformed shard descriptor even as the only report', () => {
+        const result = verifyShardReports([
+            { ...entry(null, 'broken.json'), malformedShard: true },
+        ]);
+
+        assert.equal(result.ok, false);
+        assert.deepEqual(result.problems, [
+            'malformed config.shard descriptor: broken.json',
+        ]);
+    });
+
+    it('rejects an empty set', () => {
+        const result = verifyShardReports([]);
+
+        assert.equal(result.ok, false);
+        assert.match(result.problems.join('\n'), /no Playwright JSON reports/);
+    });
+
+    it('names missing shards', () => {
+        const result = verifyShardReports([
+            entry({ current: 1, total: 3 }),
+            entry({ current: 3, total: 3 }),
+        ]);
+
+        assert.equal(result.ok, false);
+        assert.deepEqual(result.problems, ['missing shards: 2/3']);
+    });
+
+    it('rejects duplicate shards even when the set looks complete', () => {
+        const result = verifyShardReports([
+            entry({ current: 1, total: 2 }, 'a.json'),
+            entry({ current: 1, total: 2 }, 'b.json'),
+            entry({ current: 2, total: 2 }, 'c.json'),
+        ]);
+
+        assert.equal(result.ok, false);
+        assert.deepEqual(result.problems, ['shard 1/2 appears 2 times: a.json, b.json']);
+    });
+
+    it('rejects disagreeing shard totals', () => {
+        const result = verifyShardReports([
+            entry({ current: 1, total: 2 }),
+            entry({ current: 2, total: 3 }),
+        ]);
+
+        assert.equal(result.ok, false);
+        assert.deepEqual(result.problems, ['shard totals disagree: 2, 3']);
+    });
+
+    it('rejects mixed sharded and unsharded reports and repeated unsharded runs', () => {
+        const mixed = verifyShardReports([
+            entry({ current: 1, total: 1 }),
+            entry(null, 'plain.json'),
+        ]);
+        const repeated = verifyShardReports([entry(null, 'a.json'), entry(null, 'b.json')]);
+
+        assert.equal(mixed.ok, false);
+        assert.match(mixed.problems.join('\n'), /mixed sharded and unsharded.*plain\.json/);
+        assert.equal(repeated.ok, false);
+        assert.match(repeated.problems.join('\n'), /2 unsharded reports.*a\.json, b\.json/);
+    });
+});
+
+describe('describeShardReports', () => {
+    it('labels empty, unsharded and sharded sets', () => {
+        assert.equal(describeShardReports([]), 'none');
+        assert.equal(describeShardReports([entry(null)]), '1 report (unsharded)');
+        assert.equal(
+            describeShardReports([
+                entry({ current: 2, total: 3 }),
+                entry({ current: 1, total: 3 }),
+            ]),
+            '2/3 shards (1/3, 2/3)'
+        );
+    });
+});
+
+describe('e2e-semantic-summary CLI', () => {
+    function makeWorkspace() {
+        const root = makeTemporaryDir('iptvnator-e2e-summary-');
+        mkdirSync(path.join(root, 'tools/coverage'), { recursive: true });
+        copyFileSync(
+            path.join(repositoryRoot, 'tools/coverage/coverage-policy.json'),
+            path.join(root, 'tools/coverage/coverage-policy.json')
+        );
+        return root;
+    }
+
+    function runSummary(root, args) {
+        const stepSummary = path.join(root, 'step-summary.md');
+        const result = spawnSync(
+            process.execPath,
+            [summaryScript, '--project=electron-backend-e2e', ...args],
+            { cwd: root, encoding: 'utf8', env: { ...process.env, GITHUB_STEP_SUMMARY: stepSummary } }
+        );
+        return {
+            ...result,
+            stepSummary: existsSync(stepSummary) ? readFileSync(stepSummary, 'utf8') : '',
+            summaryPath: path.join(root, 'coverage/e2e/electron-backend-e2e-semantic-summary.md'),
+            jsonPath: path.join(root, 'coverage/e2e/electron-backend-e2e-semantic-summary.json'),
+        };
+    }
+
+    it('merges a complete shard directory into one summary', () => {
+        const root = makeWorkspace();
+        const shards = path.join(root, 'shards');
+        writeReport(
+            shards,
+            'electron-ubuntu-1/dist/test-results/electron-backend-e2e',
+            playwrightReport({
+                shard: { current: 1, total: 3 },
+                file: 'src/smoke.e2e.ts',
+                titles: ['boots @critical @electron'],
+            })
+        );
+        writeReport(
+            shards,
+            'electron-ubuntu-2/dist/test-results/electron-backend-e2e',
+            playwrightReport({
+                shard: { current: 2, total: 3 },
+                file: 'src/search.e2e.ts',
+                titles: ['finds @search', 'sorts @search'],
+                status: 'failed',
+            })
+        );
+        writeReport(
+            shards,
+            'electron-ubuntu-3/dist/test-results/electron-backend-e2e',
+            playwrightReport({
+                shard: { current: 3, total: 3 },
+                file: 'src/xtream.e2e.ts',
+                titles: ['browses @xtream'],
+            })
+        );
+
+        const result = runSummary(root, ['--input=shards']);
+
+        assert.equal(result.status, 0, result.stderr);
+        const markdown = readFileSync(result.summaryPath, 'utf8');
+        assert.match(markdown, /Source: Playwright JSON report/);
+        assert.match(markdown, /Reports: 3\/3 shards \(1\/3, 2\/3, 3\/3\)/);
+        assert.match(markdown, /Total tracked tests: 4/);
+        assert.match(markdown, /\| @search \| 2 \|/);
+        assert.match(markdown, /\| Workspace search across providers \| @search \| 2 \| failing \|/);
+        assert.match(markdown, /\| Electron app starts and renders workspace \| @critical \| 1 \| covered \|/);
+        const tests = JSON.parse(readFileSync(result.jsonPath, 'utf8'));
+        assert.deepEqual(
+            tests.map((test) => test.file).sort(),
+            ['src/search.e2e.ts', 'src/search.e2e.ts', 'src/smoke.e2e.ts', 'src/xtream.e2e.ts']
+        );
+        assert.match(result.stepSummary, /Reports: 3\/3 shards/);
+    });
+
+    it('refuses to summarize an incomplete shard set', () => {
+        const root = makeWorkspace();
+        const shards = path.join(root, 'shards');
+        writeReport(
+            shards,
+            'shard-1',
+            playwrightReport({
+                shard: { current: 1, total: 3 },
+                file: 'src/smoke.e2e.ts',
+                titles: ['boots @critical'],
+            })
+        );
+        writeReport(
+            shards,
+            'shard-3',
+            playwrightReport({
+                shard: { current: 3, total: 3 },
+                file: 'src/xtream.e2e.ts',
+                titles: ['browses @xtream'],
+            })
+        );
+
+        const result = runSummary(root, ['--input=shards']);
+
+        assert.equal(result.status, 1);
+        assert.match(result.stderr, /missing shards: 2\/3/);
+        assert.match(result.stepSummary, /not written.*missing shards: 2\/3/);
+        assert.equal(existsSync(result.summaryPath), false);
+    });
+
+    it('refuses an explicit input path that does not exist', () => {
+        const root = makeWorkspace();
+
+        const result = runSummary(root, ['--input=dist/e2e-shards']);
+
+        assert.equal(result.status, 1);
+        assert.match(result.stderr, /input does not exist/);
+        assert.match(result.stepSummary, /not written.*input does not exist/);
+        assert.equal(existsSync(result.summaryPath), false);
+    });
+
+    it('refuses a single report whose shard descriptor is malformed', () => {
+        const root = makeWorkspace();
+        writeReport(root, 'shards/one', {
+            config: { shard: { current: 'x', total: 3 } },
+            suites: [],
+        });
+
+        const result = runSummary(root, ['--input=shards']);
+
+        assert.equal(result.status, 1);
+        assert.match(result.stderr, /malformed config\.shard descriptor/);
+        assert.equal(existsSync(result.summaryPath), false);
+    });
+
+    it('refuses a report directory without any results.json', () => {
+        const root = makeWorkspace();
+        mkdirSync(path.join(root, 'shards/empty'), { recursive: true });
+
+        const result = runSummary(root, ['--input=shards']);
+
+        assert.equal(result.status, 1);
+        assert.match(result.stderr, /no Playwright JSON reports/);
+        assert.equal(existsSync(result.summaryPath), false);
+    });
+
+    it('still accepts a single report file and labels it unsharded', () => {
+        const root = makeWorkspace();
+        const reportPath = writeReport(
+            root,
+            'dist/test-results/electron-backend-e2e',
+            playwrightReport({
+                shard: null,
+                file: 'src/smoke.e2e.ts',
+                titles: ['boots @critical'],
+            })
+        );
+
+        const explicit = runSummary(root, [`--input=${reportPath}`]);
+        assert.equal(explicit.status, 0, explicit.stderr);
+        assert.match(readFileSync(explicit.summaryPath, 'utf8'), /Reports: 1 report \(unsharded\)/);
+
+        const implicit = runSummary(root, []);
+        assert.equal(implicit.status, 0, implicit.stderr);
+        assert.match(readFileSync(implicit.summaryPath, 'utf8'), /Total tracked tests: 1/);
+    });
+
+    it('writes into --output-dir instead of the policy directory', () => {
+        const root = makeWorkspace();
+        writeReport(
+            root,
+            'dist/test-results/electron-backend-e2e',
+            playwrightReport({
+                shard: null,
+                file: 'src/smoke.e2e.ts',
+                titles: ['boots @critical'],
+            })
+        );
+
+        const result = runSummary(root, ['--output-dir=coverage/e2e/macos-latest']);
+
+        assert.equal(result.status, 0, result.stderr);
+        assert.equal(existsSync(result.summaryPath), false);
+        const summaryPath = path.join(
+            root,
+            'coverage/e2e/macos-latest/electron-backend-e2e-semantic-summary.md'
+        );
+        assert.match(readFileSync(summaryPath, 'utf8'), /Total tracked tests: 1/);
+        assert.match(result.stdout, /Wrote coverage\/e2e\/macos-latest\//);
+    });
+
+    it('falls back to the spec source scan when no report exists', () => {
+        const root = makeWorkspace();
+        mkdirSync(path.join(root, 'apps/electron-backend-e2e/src'), { recursive: true });
+        writeFileSync(
+            path.join(root, 'apps/electron-backend-e2e/src/smoke.e2e.ts'),
+            "test('boots @critical', async () => {});\n"
+        );
+
+        const result = runSummary(root, []);
+
+        assert.equal(result.status, 0, result.stderr);
+        const markdown = readFileSync(result.summaryPath, 'utf8');
+        assert.match(markdown, /Source: spec source scan/);
+        assert.match(markdown, /Reports: none/);
+        assert.match(markdown, /Statuses: not-run: 1/);
+    });
+});