From ae3649a38789584b79440c48ba5f9446ba91bcfe Mon Sep 17 00:00:00 2001 From: 4gray Date: Sat, 25 Jul 2026 23:29:45 +0200 Subject: [PATCH] fix(coverage): fail on incomplete instrumentation --- libs/m3u-state/tsconfig.spec.json | 6 +- package.json | 3 +- tools/coverage/coverage-health.mjs | 170 ++++++++++++++++++++++--- tools/coverage/merge-coverage.mjs | 27 ++-- tools/coverage/run-tier-a-coverage.mjs | 84 +++++++++--- 5 files changed, 242 insertions(+), 48 deletions(-) diff --git a/libs/m3u-state/tsconfig.spec.json b/libs/m3u-state/tsconfig.spec.json index cc2617958..19497995e 100644 --- a/libs/m3u-state/tsconfig.spec.json +++ b/libs/m3u-state/tsconfig.spec.json @@ -2,12 +2,12 @@ "extends": "./tsconfig.json", "compilerOptions": { "outDir": "../../dist/out-tsc", - "module": "commonjs", + "module": "preserve", "target": "es2016", "types": ["jest", "node"], - "moduleResolution": "node10" + "moduleResolution": "bundler" }, - "files": ["src/test-setup.ts"], + "files": ["src/test-setup.ts", "../../global.d.ts"], "include": [ "jest.config.ts", "src/**/*.test.ts", diff --git a/package.json b/package.json index 45b2170bc..2d7af5b4c 100644 --- a/package.json +++ b/package.json @@ -33,11 +33,12 @@ "test:backend": "nx test electron-backend", "test:unit:all": "nx run-many --target=test --all --parallel=3", "test:unit:ci": "nx run-many --target=test --all --parallel=3 --output-style=static", + "coverage:tools:test": "node --test tools/coverage/coverage-integrity.test.mjs", "coverage:unit:ci": "node tools/coverage/run-tier-a-coverage.mjs", "coverage:merge": "node tools/coverage/merge-coverage.mjs", "coverage:health": "node tools/coverage/coverage-health.mjs", "coverage:policy:check": "node tools/coverage/check-coverage-policy.mjs", - "coverage:ci": "pnpm run coverage:policy:check && pnpm run coverage:unit:ci && pnpm run coverage:merge && node tools/coverage/coverage-health.mjs --require-report", + "coverage:ci": "pnpm run coverage:tools:test && pnpm run coverage:policy:check && pnpm run coverage:unit:ci && pnpm run coverage:merge && node tools/coverage/coverage-health.mjs --require-report", "coverage:e2e:summary": "node tools/coverage/e2e-semantic-summary.mjs", "coverage:e2e:v8:web": "node tools/coverage/e2e-v8-web.mjs", "typecheck:web": "tsc -p apps/web/tsconfig.app.json --noEmit", diff --git a/tools/coverage/coverage-health.mjs b/tools/coverage/coverage-health.mjs index 555b75b32..b4a4522fe 100644 --- a/tools/coverage/coverage-health.mjs +++ b/tools/coverage/coverage-health.mjs @@ -7,9 +7,26 @@ import { readdirSync, statSync, } from 'node:fs'; +import { createRequire } from 'node:module'; import path from 'node:path'; import process from 'node:process'; +import { + evaluateCoverageRatchets, + validateProjectCoverage, + validateRequiredProjectReports, +} from './coverage-integrity.mjs'; + +const require = createRequire(import.meta.url); +const { createCoverageMap } = require('istanbul-lib-coverage'); +const COVERAGE_METRICS = [ + 'statements', + 'branches', + 'functions', + 'lines', +]; +const COVERAGE_SUMMARY_FIELDS = ['covered', 'total', 'pct']; + const workspaceRoot = process.cwd(); const args = new Set(process.argv.slice(2)); const requireReport = args.has('--require-report'); @@ -37,6 +54,15 @@ function projectJsonPath(project) { return path.join(workspaceRoot, project.root, 'project.json'); } +function projectCoveragePath(project) { + return path.join( + workspaceRoot, + 'coverage', + project.root, + 'coverage-final.json' + ); +} + function verifyTierAProjects() { for (const project of policy.unitCoverage.tierA) { const filePath = projectJsonPath(project); @@ -58,14 +84,63 @@ function verifyTierAProjects() { } } +function verifyProjectCoverageReports() { + if (requireReport) { + const validation = validateRequiredProjectReports({ + projects: policy.unitCoverage.tierA, + workspaceRoot, + }); + errors.push(...validation.errors); + return; + } + + for (const project of policy.unitCoverage.tierA) { + if (!existsSync(projectCoveragePath(project))) { + continue; + } + + const validation = validateProjectCoverage({ + project, + workspaceRoot, + }); + errors.push(...validation.errors); + } +} + +function reportCoverageSummaryMismatches(reportedSummary, computedSummary) { + for (const metric of COVERAGE_METRICS) { + const reported = reportedSummary[metric]; + const computed = computedSummary[metric]; + if ( + COVERAGE_SUMMARY_FIELDS.every( + (field) => reported?.[field] === computed?.[field] + ) + ) { + continue; + } + + errors.push( + `Merged coverage summary mismatch for ${metric}: coverage-summary.json reports covered ${reported?.covered}, total ${reported?.total}, pct ${reported?.pct}; coverage-final.json computes covered ${computed?.covered}, total ${computed?.total}, pct ${computed?.pct}.` + ); + } +} + function verifyCoverageReport() { const summaryPath = path.join( workspaceRoot, policy.reporting.mergedCoverageDir, 'coverage-summary.json' ); + const ratchet = policy.reporting.coverageRatchet; + const coveragePath = path.join( + workspaceRoot, + policy.reporting.mergedCoverageDir, + 'coverage-final.json' + ); + const summaryExists = existsSync(summaryPath); + const coverageExists = existsSync(coveragePath); - if (!existsSync(summaryPath)) { + if (!summaryExists && !coverageExists) { const message = `Merged coverage summary not found at ${path.relative(workspaceRoot, summaryPath)}.`; if (requireReport) { errors.push(message); @@ -75,25 +150,83 @@ function verifyCoverageReport() { return; } - const summary = readJson(summaryPath).total; - console.log( - `Merged coverage: statements ${summary.statements.pct}%, branches ${summary.branches.pct}%, functions ${summary.functions.pct}%, lines ${summary.lines.pct}%.` - ); + if (summaryExists !== coverageExists) { + const existingPath = summaryExists ? summaryPath : coveragePath; + const missingPath = summaryExists ? coveragePath : summaryPath; + errors.push( + `Merged coverage artifacts are incomplete: found ${path.relative(workspaceRoot, existingPath)} but ${path.relative(workspaceRoot, missingPath)} is missing.` + ); + return; + } - if (requireReport) { - for (const project of policy.unitCoverage.tierA) { - const projectCoveragePath = path.join( + let summaryDocument; + try { + summaryDocument = readJson(summaryPath); + } catch (error) { + errors.push( + `Merged coverage summary ${path.relative(workspaceRoot, summaryPath)} contains invalid JSON: ${error.message}` + ); + return; + } + + const summary = summaryDocument?.total; + if ( + !summary || + COVERAGE_METRICS.some( + (metric) => + !summary[metric] || + !Number.isFinite(summary[metric].pct) + ) + ) { + errors.push( + `Merged coverage summary ${path.relative(workspaceRoot, summaryPath)} does not contain usable total coverage metrics.` + ); + return; + } + + let coverageData; + try { + coverageData = readJson(coveragePath); + } catch (error) { + errors.push( + `Merged coverage report ${path.relative(workspaceRoot, coveragePath)} contains invalid JSON: ${error.message}` + ); + return; + } + + let computedSummary; + try { + const coverageMap = createCoverageMap(coverageData); + computedSummary = coverageMap.getCoverageSummary().toJSON(); + } catch (error) { + errors.push( + `Merged coverage report ${path.relative(workspaceRoot, coveragePath)} is not valid Istanbul coverage: ${error.message}` + ); + return; + } + + console.log( + `Merged coverage: statements ${computedSummary.statements.pct}%, branches ${computedSummary.branches.pct}%, functions ${computedSummary.functions.pct}%, lines ${computedSummary.lines.pct}%.` + ); + reportCoverageSummaryMismatches(summary, computedSummary); + + if (ratchet === undefined) { + return; + } + + try { + errors.push( + ...evaluateCoverageRatchets({ + coverageData, + mergedSummary: computedSummary, + ratchet, workspaceRoot, - 'coverage', - project.root, - 'coverage-final.json' - ); - if (!existsSync(projectCoveragePath)) { - errors.push( - `Tier A project ${project.name} did not produce ${path.relative(workspaceRoot, projectCoveragePath)}.` - ); - } - } + }) + ); + } catch (error) { + errors.push( + `Merged coverage report ${path.relative(workspaceRoot, coveragePath)} is not valid Istanbul coverage: ${error.message}` + ); } } @@ -216,6 +349,7 @@ function reportChangedCriticalFiles() { } verifyTierAProjects(); +verifyProjectCoverageReports(); verifyCoverageReport(); scanE2ETags(); reportChangedCriticalFiles(); diff --git a/tools/coverage/merge-coverage.mjs b/tools/coverage/merge-coverage.mjs index 36b006dfb..901907005 100644 --- a/tools/coverage/merge-coverage.mjs +++ b/tools/coverage/merge-coverage.mjs @@ -2,7 +2,6 @@ import { createRequire } from 'node:module'; import { - existsSync, mkdirSync, readFileSync, rmSync, @@ -11,6 +10,8 @@ import { import path from 'node:path'; import process from 'node:process'; +import { validateRequiredProjectReports } from './coverage-integrity.mjs'; + const require = createRequire(import.meta.url); const { createCoverageMap } = require('istanbul-lib-coverage'); const libReport = require('istanbul-lib-report'); @@ -22,11 +23,20 @@ const policy = JSON.parse( ); const outputDir = path.join(workspaceRoot, policy.reporting.mergedCoverageDir); -const coverageFiles = policy.unitCoverage.tierA - .map((project) => path.join(workspaceRoot, 'coverage', project.root, 'coverage-final.json')) - .filter((coverageFile) => existsSync(coverageFile)); +const validation = validateRequiredProjectReports({ + projects: policy.unitCoverage.tierA, + workspaceRoot, +}); -if (coverageFiles.length === 0) { +if (validation.errors.length > 0) { + for (const error of validation.errors) { + console.error(`Error: ${error}`); + } + process.exit(1); +} + +const coverageInputs = validation.reports; +if (coverageInputs.length === 0) { console.error('No Tier A coverage-final.json files found under coverage/.'); process.exit(1); } @@ -36,9 +46,8 @@ mkdirSync(outputDir, { recursive: true }); const coverageMap = createCoverageMap({}); -for (const coverageFile of coverageFiles) { - const data = JSON.parse(readFileSync(coverageFile, 'utf8')); - coverageMap.merge(data); +for (const input of coverageInputs) { + coverageMap.merge(input.data); } const context = libReport.createContext({ @@ -56,7 +65,7 @@ writeFileSync( `${JSON.stringify(summary, null, 4)}\n` ); -console.log(`Merged ${coverageFiles.length} coverage files into ${policy.reporting.mergedCoverageDir}`); +console.log(`Merged ${coverageInputs.length} coverage files into ${policy.reporting.mergedCoverageDir}`); console.log( `Statements: ${summary.statements.pct}% | Branches: ${summary.branches.pct}% | Functions: ${summary.functions.pct}% | Lines: ${summary.lines.pct}%` ); diff --git a/tools/coverage/run-tier-a-coverage.mjs b/tools/coverage/run-tier-a-coverage.mjs index ecf553363..c9dda4974 100644 --- a/tools/coverage/run-tier-a-coverage.mjs +++ b/tools/coverage/run-tier-a-coverage.mjs @@ -1,10 +1,15 @@ #!/usr/bin/env node -import { spawnSync } from 'node:child_process'; +import { spawn } from 'node:child_process'; import { existsSync, readFileSync, rmSync } from 'node:fs'; import path from 'node:path'; import process from 'node:process'; +import { + createCoverageOutputScanner, + validateProjectCoverage, +} from './coverage-integrity.mjs'; + const workspaceRoot = process.cwd(); const policyPath = path.join(workspaceRoot, 'tools/coverage/coverage-policy.json'); const policy = JSON.parse(readFileSync(policyPath, 'utf8')); @@ -120,6 +125,64 @@ function buildNxArgs(project) { ); } +function spawnCoverage(args, scanner) { + return new Promise((resolve, reject) => { + const child = spawn('pnpm', args, { + cwd: workspaceRoot, + env: { + ...process.env, + CI: process.env.CI ?? 'true', + NX_TASKS_RUNNER_DYNAMIC_OUTPUT: 'false', + }, + stdio: ['inherit', 'pipe', 'pipe'], + }); + + child.stdout.on('data', (chunk) => { + scanner.push(chunk); + process.stdout.write(chunk); + }); + child.stderr.on('data', (chunk) => { + scanner.push(chunk); + process.stderr.write(chunk); + }); + child.once('error', reject); + child.once('close', (code, signal) => { + resolve({ code, signal }); + }); + }); +} + +async function collectProjectCoverage(project) { + const args = buildNxArgs(project); + console.log(`\n==> Collecting coverage for ${project.name}`); + console.log(`pnpm ${args.join(' ')}`); + + const scanner = createCoverageOutputScanner(); + const result = await spawnCoverage(args, scanner); + let failed = result.code !== 0 || result.signal !== null; + + if (scanner.collectionFailed) { + console.error( + `Coverage collection failed while testing ${project.name}.` + ); + failed = true; + } + + if (failed) { + return result.code && result.code !== 0 ? result.code : 1; + } + + const validation = validateProjectCoverage({ + project, + workspaceRoot, + }); + for (const error of validation.errors) { + console.error(`Error: ${error}`); + } + + return validation.errors.length === 0 ? 0 : 1; +} + for (const project of tierAProjects) { const coverageDir = path.join(workspaceRoot, 'coverage', project.root); rmSync(coverageDir, { recursive: true, force: true }); @@ -136,21 +199,8 @@ if (requestedProjects.size === 0) { } for (const project of tierAProjects) { - const args = buildNxArgs(project); - console.log(`\n==> Collecting coverage for ${project.name}`); - console.log(`pnpm ${args.join(' ')}`); - - const result = spawnSync('pnpm', args, { - cwd: workspaceRoot, - env: { - ...process.env, - CI: process.env.CI ?? 'true', - NX_TASKS_RUNNER_DYNAMIC_OUTPUT: 'false', - }, - stdio: 'inherit', - }); - - if (result.status !== 0) { - process.exit(result.status ?? 1); + const status = await collectProjectCoverage(project); + if (status !== 0) { + process.exit(status); } }