diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 74ff1e198..dd4031464 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -37,11 +37,13 @@ permissions: jobs: electron-e2e-tests: - name: Electron E2E on ${{ matrix.os }} + name: Electron E2E on ${{ matrix.os }} (${{ matrix.shard }}/${{ matrix.shard-total }}) runs-on: ${{ matrix.os }} - # macOS's sequential Electron suite exceeded 45m while still passing - # tests; retain the full suite with room for setup and retries. - timeout-minutes: ${{ matrix.os == 'macos-latest' && 60 || 45 }} + # The sequential Electron suite is split into Playwright shards (one + # runner each, split by spec file). Measured on 2026-09-26, a third of + # the suite is 6-12 minutes of test time on top of 3-7 minutes of + # setup, so 30 minutes leaves room for retries on every OS. + timeout-minutes: 30 env: IPTVNATOR_ALLOW_PRIVATE_NETWORK_URLS: '1' NX_SKIP_NX_CACHE: true @@ -49,6 +51,8 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] + shard: [1, 2, 3] + shard-total: [3] steps: - uses: actions/checkout@v7 @@ -68,7 +72,10 @@ jobs: - name: Build Backend run: pnpm nx build electron-backend + # Process-lifecycle checks are independent of the shard split; + # run them once per OS. - name: Verify Electron process cleanup + if: matrix.shard == 1 run: pnpm exec tsx --test apps/electron-backend-e2e/src/performance/electron-process-lifecycle.spec.ts apps/electron-backend-e2e/src/performance/electron-process-termination.spec.ts - name: Install Playwright Browsers @@ -76,29 +83,81 @@ jobs: - name: Run Electron E2E Tests (Linux) if: runner.os == 'Linux' - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" pnpm nx run electron-backend-e2e:e2e + run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" pnpm nx run electron-backend-e2e:e2e -- --shard=${{ matrix.shard }}/${{ matrix.shard-total }} env: CI: true - name: Run Electron E2E Tests (Windows/Mac) if: runner.os != 'Linux' - run: pnpm nx run electron-backend-e2e:e2e + run: pnpm nx run electron-backend-e2e:e2e -- --shard=${{ matrix.shard }}/${{ matrix.shard-total }} env: CI: true - - name: Summarize Electron E2E semantic coverage - if: always() - run: pnpm run coverage:e2e:summary -- --project=electron-backend-e2e - + # Each shard's results.json carries config.shard; the summary job + # below merges the shards of one OS into a single semantic summary. - name: Upload Electron Test Results if: always() uses: actions/upload-artifact@v7 with: - name: playwright-report-electron-${{ matrix.os }} + name: playwright-report-electron-${{ matrix.os }}-${{ matrix.shard }} path: | dist/playwright-report/electron-backend-e2e/ dist/test-results/electron-backend-e2e/ - coverage/e2e/ + retention-days: 7 + + electron-e2e-summary: + name: Electron E2E summary + runs-on: ubuntu-latest + needs: electron-e2e-tests + # Summarize failed shards too, but not a cancelled (superseded) run. + if: ${{ !cancelled() }} + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v7 + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version-file: '.nvmrc' + + # One download per OS keeps each OS's shards in their own directory + # (the artifact paths inside the shards are identical). + - name: Download shard reports (Ubuntu) + uses: actions/download-artifact@v8 + with: + pattern: playwright-report-electron-ubuntu-latest-* + path: dist/e2e-shards/ubuntu-latest + + - name: Download shard reports (macOS) + uses: actions/download-artifact@v8 + with: + pattern: playwright-report-electron-macos-latest-* + path: dist/e2e-shards/macos-latest + + - name: Download shard reports (Windows) + uses: actions/download-artifact@v8 + with: + pattern: playwright-report-electron-windows-latest-* + path: dist/e2e-shards/windows-latest + + # The script fails when a shard's results.json is missing or + # duplicated, so a partial run is never summarized as complete. + - name: Summarize Electron E2E semantic coverage per OS + run: | + status=0 + for os in ubuntu-latest macos-latest windows-latest; do + echo "## Electron E2E on $os" >> "$GITHUB_STEP_SUMMARY" + node tools/coverage/e2e-semantic-summary.mjs --project=electron-backend-e2e --input="dist/e2e-shards/$os" --output-dir="coverage/e2e/$os" || status=1 + done + exit "$status" + + - name: Upload Electron semantic summaries + if: always() + uses: actions/upload-artifact@v7 + with: + name: e2e-semantic-summary-electron + path: coverage/e2e/ retention-days: 7 web-e2e-tests: diff --git a/docs/architecture/validation-map.md b/docs/architecture/validation-map.md index 9a22c587c..0e0e6e767 100644 --- a/docs/architecture/validation-map.md +++ b/docs/architecture/validation-map.md @@ -135,6 +135,16 @@ After an E2E run, generate the semantic summary with: pnpm run coverage:e2e:summary ``` +CI runs the Electron suite as three Playwright shards per OS +(`--shard=/3`, split by spec file because the suite is sequential). Each +shard uploads `playwright-report-electron--`; the follow-up +`Electron E2E summary` job downloads the shards of each OS into their own +directory and runs the summary per OS with `--input=` and +`--output-dir=coverage/e2e/`. A directory input merges every +`results.json` beneath it and fails when a shard is missing or duplicated, so +the summary never reports a partial run as complete. Tests for that merge live +in `tools/coverage/e2e-shard-reports.test.mjs` (`pnpm run coverage:tools:test`). + For local investigation only, Chromium browser V8 coverage can be explored with: ```bash diff --git a/package.json b/package.json index aa1379a0f..9254f378f 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "styles:inputs:test": "node --test tools/nx/check-stylesheet-inputs.test.mjs", "styles:inputs:check": "node tools/nx/check-stylesheet-inputs.mjs", "styles:inputs:validate": "pnpm run styles:inputs:test && pnpm run styles:inputs:check", - "coverage:tools:test": "node --test tools/coverage/coverage-integrity.test.mjs", + "coverage:tools:test": "node --test tools/coverage/coverage-integrity.test.mjs tools/coverage/e2e-shard-reports.test.mjs", "coverage:unit:ci": "node tools/coverage/run-tier-a-coverage.mjs", "coverage:merge": "node tools/coverage/merge-coverage.mjs", "coverage:health": "node tools/coverage/coverage-health.mjs", diff --git a/tools/coverage/e2e-semantic-summary.mjs b/tools/coverage/e2e-semantic-summary.mjs index cac4b1dd0..3b7a112dc 100644 --- a/tools/coverage/e2e-semantic-summary.mjs +++ b/tools/coverage/e2e-semantic-summary.mjs @@ -11,6 +11,13 @@ import { import path from 'node:path'; import process from 'node:process'; +import { + describeShardReports, + findPlaywrightJsonReports, + loadPlaywrightReports, + verifyShardReports, +} from './e2e-shard-reports.mjs'; + const workspaceRoot = process.cwd(); const args = process.argv.slice(2); const projectArg = valueFor('--project'); @@ -18,7 +25,12 @@ const inputArg = valueFor('--input'); const policy = JSON.parse( readFileSync(path.join(workspaceRoot, 'tools/coverage/coverage-policy.json'), 'utf8') ); -const outputDir = path.join(workspaceRoot, policy.reporting.e2eSummaryDir); +const outputDirArg = valueFor('--output-dir'); +const outputDir = path.resolve( + workspaceRoot, + outputDirArg ?? policy.reporting.e2eSummaryDir +); +const outputDirLabel = outputDirArg ?? policy.reporting.e2eSummaryDir; function valueFor(flag) { const prefixed = args.find((arg) => arg.startsWith(`${flag}=`)); @@ -57,8 +69,19 @@ function tagsFromTitle(title) { ); } -function collectFromPlaywrightJson(filePath, projectName) { - const report = JSON.parse(readFileSync(filePath, 'utf8')); +function fail(message) { + console.error(`e2e-semantic-summary: ${message}`); + if (process.env.GITHUB_STEP_SUMMARY) { + writeFileSync( + process.env.GITHUB_STEP_SUMMARY, + `\n> **E2E semantic summary not written:** ${message}\n`, + { flag: 'a' } + ); + } + process.exit(1); +} + +function collectFromPlaywrightJson(report, projectName) { const tests = []; function walkSuite(suite, inheritedFile) { @@ -137,16 +160,58 @@ function defaultInputFor(projectName) { return path.join(workspaceRoot, 'dist/test-results', projectName, 'results.json'); } -function collectTests(projectName) { +/** + * `--output-dir` overrides the policy's summary directory so several runs + * (one per OS in CI) can be summarized side by side in one job. + * + * `--input` may name one Playwright JSON report or a directory that holds the + * `results.json` of every shard (as downloaded from the per-shard CI + * artifacts). An explicit input that does not exist, a directory without any + * report, or an incomplete or duplicated shard set aborts instead of writing + * a partial summary. Only the implicit default falls back to scanning the + * spec sources. + */ +function resolveReportPaths(projectName) { const inputPath = inputArg ? path.resolve(workspaceRoot, inputArg) : defaultInputFor(projectName); - if (existsSync(inputPath)) { - return collectFromPlaywrightJson(inputPath, projectName); + if (!existsSync(inputPath)) { + if (inputArg) { + fail(`Playwright JSON report input does not exist: ${inputPath}`); + } + return []; + } + if (!statSync(inputPath).isDirectory()) { + return [inputPath]; + } + const found = findPlaywrightJsonReports(inputPath); + if (found.length === 0) { + fail(`no Playwright JSON reports (results.json) found under ${inputPath}`); + } + return found; +} + +function collectTests(projectName) { + const reportPaths = resolveReportPaths(projectName); + if (reportPaths.length === 0) { + return { tests: collectFromSource(projectName), reports: [] }; } - return collectFromSource(projectName); + const reports = loadPlaywrightReports(reportPaths); + const verification = verifyShardReports(reports); + if (!verification.ok) { + fail( + `${projectName} reports do not form one complete run: ${verification.problems.join('; ')}` + ); + } + + return { + tests: reports.flatMap((entry) => + collectFromPlaywrightJson(entry.report, projectName) + ), + reports, + }; } function statusCounts(tests) { @@ -174,7 +239,7 @@ function journeyMatches(journey, tests) { ); } -function markdownFor(projectName, tests) { +function markdownFor(projectName, tests, reportsLabel) { const counts = statusCounts(tests); const countsText = Object.entries(counts) .map(([status, count]) => `${status}: ${count}`) @@ -197,6 +262,8 @@ function markdownFor(projectName, tests) { Source: ${tests.some((test) => test.status === 'not-run') ? 'spec source scan' : 'Playwright JSON report'} +Reports: ${reportsLabel} + Total tracked tests: ${tests.length} Statuses: ${countsText || 'none'} @@ -216,12 +283,23 @@ ${journeys || '| _none_ | _n/a_ | 0 | missing |'} } const projects = projectArg ? [projectArg] : ['web-e2e', 'electron-backend-e2e']; -const allTests = projects.flatMap((projectName) => collectTests(projectName)); +const collected = projects.map((projectName) => ({ + projectName, + ...collectTests(projectName), +})); +const allTests = collected.flatMap((entry) => entry.tests); +const reportsLabel = collected + .map((entry) => + projectArg + ? describeShardReports(entry.reports) + : `${entry.projectName}: ${describeShardReports(entry.reports)}` + ) + .join('; '); mkdirSync(outputDir, { recursive: true }); if (projectArg) { - const content = markdownFor(projectArg, allTests); + const content = markdownFor(projectArg, allTests, reportsLabel); writeFileSync(path.join(outputDir, `${projectArg}-semantic-summary.md`), content); writeFileSync( path.join(outputDir, `${projectArg}-semantic-summary.json`), @@ -230,9 +308,9 @@ if (projectArg) { if (process.env.GITHUB_STEP_SUMMARY) { writeFileSync(process.env.GITHUB_STEP_SUMMARY, `\n${content}\n`, { flag: 'a' }); } - console.log(`Wrote ${policy.reporting.e2eSummaryDir}/${projectArg}-semantic-summary.md`); + console.log(`Wrote ${outputDirLabel}/${projectArg}-semantic-summary.md`); } else { - const content = markdownFor(undefined, allTests); + const content = markdownFor(undefined, allTests, reportsLabel); writeFileSync(path.join(outputDir, 'semantic-summary.md'), content); writeFileSync( path.join(outputDir, 'semantic-summary.json'), @@ -241,5 +319,5 @@ if (projectArg) { if (process.env.GITHUB_STEP_SUMMARY) { writeFileSync(process.env.GITHUB_STEP_SUMMARY, `\n${content}\n`, { flag: 'a' }); } - console.log(`Wrote ${policy.reporting.e2eSummaryDir}/semantic-summary.md`); + console.log(`Wrote ${outputDirLabel}/semantic-summary.md`); } diff --git a/tools/coverage/e2e-shard-reports.mjs b/tools/coverage/e2e-shard-reports.mjs new file mode 100644 index 000000000..fe0ace957 --- /dev/null +++ b/tools/coverage/e2e-shard-reports.mjs @@ -0,0 +1,148 @@ +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import path from 'node:path'; + +/** + * Helpers for reading one or more Playwright JSON reports (`results.json`). + * + * The Electron E2E workflow runs the suite as several Playwright shards, one + * per runner. Each shard writes its own `results.json` that carries + * `config.shard = { current, total }` and only that shard's tests. A semantic + * summary is only honest when every shard is present exactly once, so callers + * verify the set before merging. + */ + +export const playwrightReportFileName = 'results.json'; + +/** Recursively lists every `results.json` below `rootDir`, sorted by path. */ +export function findPlaywrightJsonReports(rootDir) { + if (!existsSync(rootDir) || !statSync(rootDir).isDirectory()) { + return []; + } + const found = []; + for (const entry of readdirSync(rootDir)) { + const fullPath = path.join(rootDir, entry); + if (statSync(fullPath).isDirectory()) { + found.push(...findPlaywrightJsonReports(fullPath)); + } else if (entry === playwrightReportFileName) { + found.push(fullPath); + } + } + return found.sort(); +} + +/** + * Parses each report and extracts its shard descriptor: `null` when the run + * was not sharded (`config.shard` absent or `null`). A descriptor that is + * present but malformed is kept as `malformedShard` so verification rejects + * it instead of mistaking a partial run for a complete unsharded one. + */ +export function loadPlaywrightReports(reportPaths) { + return reportPaths.map((reportPath) => { + const report = JSON.parse(readFileSync(reportPath, 'utf8')); + const shard = report.config?.shard ?? null; + const wellFormed = + shard !== null && + Number.isInteger(shard.current) && + Number.isInteger(shard.total); + return { + path: reportPath, + report, + shard: wellFormed ? { current: shard.current, total: shard.total } : null, + malformedShard: shard !== null && !wellFormed, + }; + }); +} + +/** + * Checks that the loaded reports form exactly one complete run: either a + * single unsharded report, or every shard `1..total` exactly once. + */ +export function verifyShardReports(reports) { + const problems = []; + if (reports.length === 0) { + problems.push('no Playwright JSON reports were provided'); + return { ok: false, problems }; + } + + const malformed = reports.filter((entry) => entry.malformedShard); + if (malformed.length > 0) { + problems.push(`malformed config.shard descriptor: ${describePaths(malformed)}`); + } + + const unsharded = reports.filter( + (entry) => entry.shard === null && !entry.malformedShard + ); + const sharded = reports.filter((entry) => entry.shard !== null); + + if (unsharded.length > 0 && sharded.length > 0) { + problems.push( + `mixed sharded and unsharded reports: ${describePaths(unsharded)} carry no shard descriptor` + ); + } + if (unsharded.length > 1) { + problems.push( + `${unsharded.length} unsharded reports would count every test more than once: ${describePaths(unsharded)}` + ); + } + + if (sharded.length > 0) { + const totals = new Set(sharded.map((entry) => entry.shard.total)); + if (totals.size > 1) { + problems.push( + `shard totals disagree: ${Array.from(totals).sort().join(', ')}` + ); + } else { + const [total] = totals; + const seen = new Map(); + for (const entry of sharded) { + const list = seen.get(entry.shard.current) ?? []; + list.push(entry); + seen.set(entry.shard.current, list); + } + const missing = []; + for (let index = 1; index <= total; index += 1) { + if (!seen.has(index)) { + missing.push(`${index}/${total}`); + } + } + if (missing.length > 0) { + problems.push(`missing shards: ${missing.join(', ')}`); + } + for (const [current, entries] of seen) { + if (entries.length > 1) { + problems.push( + `shard ${current}/${total} appears ${entries.length} times: ${describePaths(entries)}` + ); + } + if (current < 1 || current > total) { + problems.push(`shard ${current}/${total} is outside 1..${total}`); + } + } + } + } + + return { ok: problems.length === 0, problems }; +} + +/** Human-readable label for the summary, e.g. `3/3 shards (1/3, 2/3, 3/3)`. */ +export function describeShardReports(reports) { + if (reports.length === 0) { + return 'none'; + } + const sharded = reports.filter((entry) => entry.shard !== null); + if (sharded.length === 0) { + return reports.length === 1 + ? '1 report (unsharded)' + : `${reports.length} reports (unsharded)`; + } + const total = Math.max(...sharded.map((entry) => entry.shard.total)); + const labels = sharded + .map((entry) => entry.shard) + .sort((left, right) => left.current - right.current) + .map((shard) => `${shard.current}/${shard.total}`); + return `${sharded.length}/${total} shards (${labels.join(', ')})`; +} + +function describePaths(entries) { + return entries.map((entry) => entry.path).join(', '); +} diff --git a/tools/coverage/e2e-shard-reports.test.mjs b/tools/coverage/e2e-shard-reports.test.mjs new file mode 100644 index 000000000..b7a58a867 --- /dev/null +++ b/tools/coverage/e2e-shard-reports.test.mjs @@ -0,0 +1,407 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, describe, it } from 'node:test'; + +import { + describeShardReports, + findPlaywrightJsonReports, + loadPlaywrightReports, + verifyShardReports, +} from './e2e-shard-reports.mjs'; + +const toolsDir = path.dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = path.resolve(toolsDir, '../..'); +const summaryScript = path.join(toolsDir, 'e2e-semantic-summary.mjs'); +const temporaryRoots = []; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } +}); + +function makeTemporaryDir(prefix) { + const root = mkdtempSync(path.join(tmpdir(), prefix)); + temporaryRoots.push(root); + return root; +} + +function playwrightReport({ shard, file, titles, status = 'passed' }) { + return { + config: { shard }, + suites: [ + { + file, + specs: titles.map((title) => ({ + title, + tags: [], + tests: [{ results: [{ status }] }], + })), + }, + ], + errors: [], + stats: {}, + }; +} + +function writeReport(root, relativeDir, report) { + const directory = path.join(root, relativeDir); + mkdirSync(directory, { recursive: true }); + const reportPath = path.join(directory, 'results.json'); + writeFileSync(reportPath, JSON.stringify(report)); + return reportPath; +} + +function entry(shard, reportPath = `report-${shard?.current ?? 'single'}.json`) { + return { path: reportPath, shard, report: {} }; +} + +describe('findPlaywrightJsonReports', () => { + it('lists every nested results.json in sorted order', () => { + const root = makeTemporaryDir('iptvnator-shard-find-'); + const second = writeReport(root, 'b-shard-2/dist/test-results/x', {}); + const first = writeReport(root, 'a-shard-1/dist/test-results/x', {}); + writeFileSync(path.join(root, 'a-shard-1', 'other.json'), '{}'); + + assert.deepEqual(findPlaywrightJsonReports(root), [first, second]); + }); + + it('returns nothing for a missing directory or a file path', () => { + const root = makeTemporaryDir('iptvnator-shard-find-'); + const reportPath = writeReport(root, 'single', {}); + + assert.deepEqual(findPlaywrightJsonReports(path.join(root, 'nope')), []); + assert.deepEqual(findPlaywrightJsonReports(reportPath), []); + }); +}); + +describe('loadPlaywrightReports', () => { + it('extracts a shard descriptor and flags malformed ones', () => { + const root = makeTemporaryDir('iptvnator-shard-load-'); + const sharded = writeReport(root, 'sharded', { + config: { shard: { current: 2, total: 3 } }, + }); + const unsharded = writeReport(root, 'unsharded', { config: { shard: null } }); + const malformed = writeReport(root, 'malformed', { + config: { shard: { current: '2', total: 3 } }, + }); + + const loaded = loadPlaywrightReports([sharded, unsharded, malformed]); + + assert.deepEqual(loaded[0].shard, { current: 2, total: 3 }); + assert.equal(loaded[0].malformedShard, false); + assert.equal(loaded[1].shard, null); + assert.equal(loaded[1].malformedShard, false); + assert.equal(loaded[2].shard, null); + assert.equal(loaded[2].malformedShard, true); + }); +}); + +describe('verifyShardReports', () => { + it('accepts a single unsharded report', () => { + assert.deepEqual(verifyShardReports([entry(null)]), { ok: true, problems: [] }); + }); + + it('accepts a complete shard set in any order', () => { + const reports = [ + entry({ current: 3, total: 3 }), + entry({ current: 1, total: 3 }), + entry({ current: 2, total: 3 }), + ]; + + assert.deepEqual(verifyShardReports(reports), { ok: true, problems: [] }); + }); + + it('rejects a malformed shard descriptor even as the only report', () => { + const result = verifyShardReports([ + { ...entry(null, 'broken.json'), malformedShard: true }, + ]); + + assert.equal(result.ok, false); + assert.deepEqual(result.problems, [ + 'malformed config.shard descriptor: broken.json', + ]); + }); + + it('rejects an empty set', () => { + const result = verifyShardReports([]); + + assert.equal(result.ok, false); + assert.match(result.problems.join('\n'), /no Playwright JSON reports/); + }); + + it('names missing shards', () => { + const result = verifyShardReports([ + entry({ current: 1, total: 3 }), + entry({ current: 3, total: 3 }), + ]); + + assert.equal(result.ok, false); + assert.deepEqual(result.problems, ['missing shards: 2/3']); + }); + + it('rejects duplicate shards even when the set looks complete', () => { + const result = verifyShardReports([ + entry({ current: 1, total: 2 }, 'a.json'), + entry({ current: 1, total: 2 }, 'b.json'), + entry({ current: 2, total: 2 }, 'c.json'), + ]); + + assert.equal(result.ok, false); + assert.deepEqual(result.problems, ['shard 1/2 appears 2 times: a.json, b.json']); + }); + + it('rejects disagreeing shard totals', () => { + const result = verifyShardReports([ + entry({ current: 1, total: 2 }), + entry({ current: 2, total: 3 }), + ]); + + assert.equal(result.ok, false); + assert.deepEqual(result.problems, ['shard totals disagree: 2, 3']); + }); + + it('rejects mixed sharded and unsharded reports and repeated unsharded runs', () => { + const mixed = verifyShardReports([ + entry({ current: 1, total: 1 }), + entry(null, 'plain.json'), + ]); + const repeated = verifyShardReports([entry(null, 'a.json'), entry(null, 'b.json')]); + + assert.equal(mixed.ok, false); + assert.match(mixed.problems.join('\n'), /mixed sharded and unsharded.*plain\.json/); + assert.equal(repeated.ok, false); + assert.match(repeated.problems.join('\n'), /2 unsharded reports.*a\.json, b\.json/); + }); +}); + +describe('describeShardReports', () => { + it('labels empty, unsharded and sharded sets', () => { + assert.equal(describeShardReports([]), 'none'); + assert.equal(describeShardReports([entry(null)]), '1 report (unsharded)'); + assert.equal( + describeShardReports([ + entry({ current: 2, total: 3 }), + entry({ current: 1, total: 3 }), + ]), + '2/3 shards (1/3, 2/3)' + ); + }); +}); + +describe('e2e-semantic-summary CLI', () => { + function makeWorkspace() { + const root = makeTemporaryDir('iptvnator-e2e-summary-'); + mkdirSync(path.join(root, 'tools/coverage'), { recursive: true }); + copyFileSync( + path.join(repositoryRoot, 'tools/coverage/coverage-policy.json'), + path.join(root, 'tools/coverage/coverage-policy.json') + ); + return root; + } + + function runSummary(root, args) { + const stepSummary = path.join(root, 'step-summary.md'); + const result = spawnSync( + process.execPath, + [summaryScript, '--project=electron-backend-e2e', ...args], + { cwd: root, encoding: 'utf8', env: { ...process.env, GITHUB_STEP_SUMMARY: stepSummary } } + ); + return { + ...result, + stepSummary: existsSync(stepSummary) ? readFileSync(stepSummary, 'utf8') : '', + summaryPath: path.join(root, 'coverage/e2e/electron-backend-e2e-semantic-summary.md'), + jsonPath: path.join(root, 'coverage/e2e/electron-backend-e2e-semantic-summary.json'), + }; + } + + it('merges a complete shard directory into one summary', () => { + const root = makeWorkspace(); + const shards = path.join(root, 'shards'); + writeReport( + shards, + 'electron-ubuntu-1/dist/test-results/electron-backend-e2e', + playwrightReport({ + shard: { current: 1, total: 3 }, + file: 'src/smoke.e2e.ts', + titles: ['boots @critical @electron'], + }) + ); + writeReport( + shards, + 'electron-ubuntu-2/dist/test-results/electron-backend-e2e', + playwrightReport({ + shard: { current: 2, total: 3 }, + file: 'src/search.e2e.ts', + titles: ['finds @search', 'sorts @search'], + status: 'failed', + }) + ); + writeReport( + shards, + 'electron-ubuntu-3/dist/test-results/electron-backend-e2e', + playwrightReport({ + shard: { current: 3, total: 3 }, + file: 'src/xtream.e2e.ts', + titles: ['browses @xtream'], + }) + ); + + const result = runSummary(root, ['--input=shards']); + + assert.equal(result.status, 0, result.stderr); + const markdown = readFileSync(result.summaryPath, 'utf8'); + assert.match(markdown, /Source: Playwright JSON report/); + assert.match(markdown, /Reports: 3\/3 shards \(1\/3, 2\/3, 3\/3\)/); + assert.match(markdown, /Total tracked tests: 4/); + assert.match(markdown, /\| @search \| 2 \|/); + assert.match(markdown, /\| Workspace search across providers \| @search \| 2 \| failing \|/); + assert.match(markdown, /\| Electron app starts and renders workspace \| @critical \| 1 \| covered \|/); + const tests = JSON.parse(readFileSync(result.jsonPath, 'utf8')); + assert.deepEqual( + tests.map((test) => test.file).sort(), + ['src/search.e2e.ts', 'src/search.e2e.ts', 'src/smoke.e2e.ts', 'src/xtream.e2e.ts'] + ); + assert.match(result.stepSummary, /Reports: 3\/3 shards/); + }); + + it('refuses to summarize an incomplete shard set', () => { + const root = makeWorkspace(); + const shards = path.join(root, 'shards'); + writeReport( + shards, + 'shard-1', + playwrightReport({ + shard: { current: 1, total: 3 }, + file: 'src/smoke.e2e.ts', + titles: ['boots @critical'], + }) + ); + writeReport( + shards, + 'shard-3', + playwrightReport({ + shard: { current: 3, total: 3 }, + file: 'src/xtream.e2e.ts', + titles: ['browses @xtream'], + }) + ); + + const result = runSummary(root, ['--input=shards']); + + assert.equal(result.status, 1); + assert.match(result.stderr, /missing shards: 2\/3/); + assert.match(result.stepSummary, /not written.*missing shards: 2\/3/); + assert.equal(existsSync(result.summaryPath), false); + }); + + it('refuses an explicit input path that does not exist', () => { + const root = makeWorkspace(); + + const result = runSummary(root, ['--input=dist/e2e-shards']); + + assert.equal(result.status, 1); + assert.match(result.stderr, /input does not exist/); + assert.match(result.stepSummary, /not written.*input does not exist/); + assert.equal(existsSync(result.summaryPath), false); + }); + + it('refuses a single report whose shard descriptor is malformed', () => { + const root = makeWorkspace(); + writeReport(root, 'shards/one', { + config: { shard: { current: 'x', total: 3 } }, + suites: [], + }); + + const result = runSummary(root, ['--input=shards']); + + assert.equal(result.status, 1); + assert.match(result.stderr, /malformed config\.shard descriptor/); + assert.equal(existsSync(result.summaryPath), false); + }); + + it('refuses a report directory without any results.json', () => { + const root = makeWorkspace(); + mkdirSync(path.join(root, 'shards/empty'), { recursive: true }); + + const result = runSummary(root, ['--input=shards']); + + assert.equal(result.status, 1); + assert.match(result.stderr, /no Playwright JSON reports/); + assert.equal(existsSync(result.summaryPath), false); + }); + + it('still accepts a single report file and labels it unsharded', () => { + const root = makeWorkspace(); + const reportPath = writeReport( + root, + 'dist/test-results/electron-backend-e2e', + playwrightReport({ + shard: null, + file: 'src/smoke.e2e.ts', + titles: ['boots @critical'], + }) + ); + + const explicit = runSummary(root, [`--input=${reportPath}`]); + assert.equal(explicit.status, 0, explicit.stderr); + assert.match(readFileSync(explicit.summaryPath, 'utf8'), /Reports: 1 report \(unsharded\)/); + + const implicit = runSummary(root, []); + assert.equal(implicit.status, 0, implicit.stderr); + assert.match(readFileSync(implicit.summaryPath, 'utf8'), /Total tracked tests: 1/); + }); + + it('writes into --output-dir instead of the policy directory', () => { + const root = makeWorkspace(); + writeReport( + root, + 'dist/test-results/electron-backend-e2e', + playwrightReport({ + shard: null, + file: 'src/smoke.e2e.ts', + titles: ['boots @critical'], + }) + ); + + const result = runSummary(root, ['--output-dir=coverage/e2e/macos-latest']); + + assert.equal(result.status, 0, result.stderr); + assert.equal(existsSync(result.summaryPath), false); + const summaryPath = path.join( + root, + 'coverage/e2e/macos-latest/electron-backend-e2e-semantic-summary.md' + ); + assert.match(readFileSync(summaryPath, 'utf8'), /Total tracked tests: 1/); + assert.match(result.stdout, /Wrote coverage\/e2e\/macos-latest\//); + }); + + it('falls back to the spec source scan when no report exists', () => { + const root = makeWorkspace(); + mkdirSync(path.join(root, 'apps/electron-backend-e2e/src'), { recursive: true }); + writeFileSync( + path.join(root, 'apps/electron-backend-e2e/src/smoke.e2e.ts'), + "test('boots @critical', async () => {});\n" + ); + + const result = runSummary(root, []); + + assert.equal(result.status, 0, result.stderr); + const markdown = readFileSync(result.summaryPath, 'utf8'); + assert.match(markdown, /Source: spec source scan/); + assert.match(markdown, /Reports: none/); + assert.match(markdown, /Statuses: not-run: 1/); + }); +});