From 2bf7cea3d9856d4df7e0d72fdf02ae749bb1015d Mon Sep 17 00:00:00 2001 From: 4gray Date: Sun, 26 Jul 2026 02:30:04 +0200 Subject: [PATCH] fix(coverage): validate merged completeness --- tools/coverage/coverage-health.mjs | 25 ++-- tools/coverage/coverage-integrity.mjs | 93 ++++++++++++ tools/coverage/coverage-integrity.test.mjs | 165 +++++++++++++++++++++ 3 files changed, 271 insertions(+), 12 deletions(-) diff --git a/tools/coverage/coverage-health.mjs b/tools/coverage/coverage-health.mjs index b4a4522fe..705623e72 100644 --- a/tools/coverage/coverage-health.mjs +++ b/tools/coverage/coverage-health.mjs @@ -7,18 +7,16 @@ import { readdirSync, statSync, } from 'node:fs'; -import { createRequire } from 'node:module'; import path from 'node:path'; import process from 'node:process'; import { evaluateCoverageRatchets, + validateMergedCoverage, validateProjectCoverage, validateRequiredProjectReports, } from './coverage-integrity.mjs'; -const require = createRequire(import.meta.url); -const { createCoverageMap } = require('istanbul-lib-coverage'); const COVERAGE_METRICS = [ 'statements', 'branches', @@ -194,23 +192,26 @@ function verifyCoverageReport() { return; } - let computedSummary; - try { - const coverageMap = createCoverageMap(coverageData); - computedSummary = coverageMap.getCoverageSummary().toJSON(); - } catch (error) { - errors.push( - `Merged coverage report ${path.relative(workspaceRoot, coveragePath)} is not valid Istanbul coverage: ${error.message}` - ); + const mergedValidation = validateMergedCoverage({ + coverageData, + projects: policy.unitCoverage.tierA, + reportPath: coveragePath, + workspaceRoot, + }); + errors.push(...mergedValidation.errors); + if (!mergedValidation.coverageMap) { return; } + const computedSummary = mergedValidation.coverageMap + .getCoverageSummary() + .toJSON(); console.log( `Merged coverage: statements ${computedSummary.statements.pct}%, branches ${computedSummary.branches.pct}%, functions ${computedSummary.functions.pct}%, lines ${computedSummary.lines.pct}%.` ); reportCoverageSummaryMismatches(summary, computedSummary); - if (ratchet === undefined) { + if (mergedValidation.errors.length > 0 || ratchet === undefined) { return; } diff --git a/tools/coverage/coverage-integrity.mjs b/tools/coverage/coverage-integrity.mjs index ca74632c0..b73e035b4 100644 --- a/tools/coverage/coverage-integrity.mjs +++ b/tools/coverage/coverage-integrity.mjs @@ -339,6 +339,99 @@ export function validateRequiredProjectReports({ projects, workspaceRoot }) { return { errors, reports }; } +export function validateMergedCoverage({ + coverageData, + projects, + reportPath, + workspaceRoot, +}) { + const relativeReportPath = toPosix( + path.relative(workspaceRoot, reportPath) + ); + const reportDescription = `Merged coverage report ${relativeReportPath}`; + + if ( + !isObjectRecord(coverageData) || + Object.values(coverageData).some( + (entry) => !isObjectRecord(entry) + ) + ) { + return { + coverageMap: undefined, + errors: [ + `${reportDescription} must contain a JSON object mapping source paths to JSON objects.`, + ], + }; + } + + let coverageMap; + try { + coverageMap = createCoverageMap(coverageData); + for (const filePath of coverageMap.files()) { + coverageMap.fileCoverageFor(filePath).toSummary(); + } + } catch (error) { + return { + coverageMap: undefined, + errors: [ + `${reportDescription} is not valid Istanbul coverage: ${error.message}`, + ], + }; + } + + const { + duplicateAbsolutePaths, + filesByAbsolutePath: reportedFiles, + } = indexCoverageFilesByAbsolutePath(coverageMap, workspaceRoot); + if (duplicateAbsolutePaths.length > 0) { + return { + coverageMap: undefined, + errors: duplicateAbsolutePaths.map( + (filePath) => + `${reportDescription} contains a duplicate entry for ${toPosix( + path.relative(workspaceRoot, filePath) + )} after path normalization.` + ), + }; + } + + const errors = []; + for (const project of projects) { + for (const filePath of runtimeOwningSourceFiles( + workspaceRoot, + project.sourceRoot + )) { + const relativeSourcePath = toPosix( + path.relative(workspaceRoot, filePath) + ); + const reportedPath = reportedFiles.get(filePath); + if (reportedPath === undefined) { + errors.push( + `${reportDescription} is missing runtime-owning source ${relativeSourcePath} from Tier A project ${project.name}.` + ); + continue; + } + + const summary = coverageMap + .fileCoverageFor(reportedPath) + .toSummary() + .toJSON(); + const hasUsableInstrumentation = [ + summary.statements, + summary.functions, + summary.branches, + ].some((metric) => metric.total > 0); + if (!hasUsableInstrumentation) { + errors.push( + `${reportDescription} has no usable instrumentation for runtime-owning source ${relativeSourcePath} from Tier A project ${project.name}.` + ); + } + } + } + + return { coverageMap, errors }; +} + function formatConfigurationValue(value) { return typeof value === 'string' ? JSON.stringify(value) : String(value); } diff --git a/tools/coverage/coverage-integrity.test.mjs b/tools/coverage/coverage-integrity.test.mjs index e1f553d04..5dcc87304 100644 --- a/tools/coverage/coverage-integrity.test.mjs +++ b/tools/coverage/coverage-integrity.test.mjs @@ -8,6 +8,7 @@ import { createCoverageOutputScanner, evaluateCoverageRatchets, hasRuntimeOwnedStatement, + validateMergedCoverage, validateProjectCoverage, validateRequiredProjectReports, } from './coverage-integrity.mjs'; @@ -424,6 +425,170 @@ describe('validateProjectCoverage', () => { }); }); +describe('validateMergedCoverage', () => { + it('reports the exact runtime source omitted from an otherwise valid merged map', () => { + const included = makeProjectFixture({ + name: 'included', + projectRoot: 'libs/included', + }); + const omitted = makeProjectFixture({ + name: 'omitted', + projectRoot: 'libs/omitted', + workspaceRoot: included.workspaceRoot, + }); + writeCompleteReport(included); + writeCompleteReport(omitted); + + assert.deepEqual(validateProjectCoverage(included).errors, []); + assert.deepEqual(validateProjectCoverage(omitted).errors, []); + + const result = validateMergedCoverage({ + coverageData: { + [included.sourcePath]: coverageEntry(included.sourcePath), + }, + projects: [included.project, omitted.project], + reportPath: path.join( + included.workspaceRoot, + 'coverage/merged/coverage-final.json' + ), + workspaceRoot: included.workspaceRoot, + }); + + assert.deepEqual(result.errors, [ + 'Merged coverage report coverage/merged/coverage-final.json is missing runtime-owning source libs/omitted/src/runtime.ts from Tier A project omitted.', + ]); + }); + + it('fails completeness when permissive ratchets accept a recomputed summary', () => { + const fixture = makeProjectFixture({ + name: 'm3u-state', + projectRoot: 'libs/m3u-state', + }); + const effectsPath = path.join( + fixture.workspaceRoot, + fixture.project.sourceRoot, + 'effects.ts' + ); + writeFileSync( + effectsPath, + 'export const effects = () => "runtime";\n' + ); + writeFileSync( + fixture.reportPath, + JSON.stringify({ + [fixture.sourcePath]: coverageEntry(fixture.sourcePath, [1]), + [effectsPath]: coverageEntry(effectsPath, [0]), + }) + ); + assert.deepEqual(validateProjectCoverage(fixture).errors, []); + + const coverageData = { + [fixture.sourcePath]: coverageEntry(fixture.sourcePath, [1]), + }; + const completeness = validateMergedCoverage({ + coverageData, + projects: [fixture.project], + reportPath: path.join( + fixture.workspaceRoot, + 'coverage/merged/coverage-final.json' + ), + workspaceRoot: fixture.workspaceRoot, + }); + const recomputedSummary = completeness.coverageMap + .getCoverageSummary() + .toJSON(); + + assert.deepEqual( + evaluateCoverageRatchets({ + coverageData, + mergedSummary: recomputedSummary, + ratchet: ratchetWithCriticalFiles([]), + workspaceRoot: fixture.workspaceRoot, + }), + [] + ); + assert.deepEqual(completeness.errors, [ + 'Merged coverage report coverage/merged/coverage-final.json is missing runtime-owning source libs/m3u-state/src/effects.ts from Tier A project m3u-state.', + ]); + }); + + it('accepts a complete merged map with normalized relative and absolute paths', () => { + const relative = makeProjectFixture({ + name: 'relative', + projectRoot: 'libs/relative', + }); + const absolute = makeProjectFixture({ + name: 'absolute', + projectRoot: 'libs/absolute', + workspaceRoot: relative.workspaceRoot, + }); + const relativePath = path.relative( + relative.workspaceRoot, + relative.sourcePath + ); + + const result = validateMergedCoverage({ + coverageData: { + [relativePath]: coverageEntry(relativePath), + [absolute.sourcePath]: coverageEntry(absolute.sourcePath), + }, + projects: [relative.project, absolute.project], + reportPath: path.join( + relative.workspaceRoot, + 'coverage/merged/coverage-final.json' + ), + workspaceRoot: relative.workspaceRoot, + }); + + assert.deepEqual(result.errors, []); + }); + + it('rejects unusable merged instrumentation for a runtime source', () => { + const fixture = makeProjectFixture(); + + const result = validateMergedCoverage({ + coverageData: { + [fixture.sourcePath]: coverageEntry(fixture.sourcePath, []), + }, + projects: [fixture.project], + reportPath: path.join( + fixture.workspaceRoot, + 'coverage/merged/coverage-final.json' + ), + workspaceRoot: fixture.workspaceRoot, + }); + + assert.deepEqual(result.errors, [ + 'Merged coverage report coverage/merged/coverage-final.json has no usable instrumentation for runtime-owning source libs/example/src/runtime.ts from Tier A project example.', + ]); + }); + + it('rejects duplicate merged paths after normalization', () => { + const fixture = makeProjectFixture(); + const relativePath = path.relative( + fixture.workspaceRoot, + fixture.sourcePath + ); + + const result = validateMergedCoverage({ + coverageData: { + [relativePath]: coverageEntry(relativePath), + [fixture.sourcePath]: coverageEntry(fixture.sourcePath), + }, + projects: [fixture.project], + reportPath: path.join( + fixture.workspaceRoot, + 'coverage/merged/coverage-final.json' + ), + workspaceRoot: fixture.workspaceRoot, + }); + + assert.deepEqual(result.errors, [ + 'Merged coverage report coverage/merged/coverage-final.json contains a duplicate entry for libs/example/src/runtime.ts after path normalization.', + ]); + }); +}); + describe('evaluateCoverageRatchets', () => { it('reports all aggregate percentage regressions', () => { const errors = evaluateCoverageRatchets({