fix(coverage): fail on incomplete instrumentation

libs/m3u-state/tsconfig.spec.json

@@ -2,12 +2,12 @@
     "extends": "./tsconfig.json",
     "compilerOptions": {
         "outDir": "../../dist/out-tsc",
-        "module": "commonjs",
+        "module": "preserve",
         "target": "es2016",
         "types": ["jest", "node"],
-        "moduleResolution": "node10"
+        "moduleResolution": "bundler"
     },
-    "files": ["src/test-setup.ts"],
+    "files": ["src/test-setup.ts", "../../global.d.ts"],
     "include": [
         "jest.config.ts",
         "src/**/*.test.ts",

package.json

@@ -33,11 +33,12 @@
         "test:backend": "nx test electron-backend",
         "test:unit:all": "nx run-many --target=test --all --parallel=3",
         "test:unit:ci": "nx run-many --target=test --all --parallel=3 --output-style=static",
+        "coverage:tools:test": "node --test tools/coverage/coverage-integrity.test.mjs",
         "coverage:unit:ci": "node tools/coverage/run-tier-a-coverage.mjs",
         "coverage:merge": "node tools/coverage/merge-coverage.mjs",
         "coverage:health": "node tools/coverage/coverage-health.mjs",
         "coverage:policy:check": "node tools/coverage/check-coverage-policy.mjs",
-        "coverage:ci": "pnpm run coverage:policy:check && pnpm run coverage:unit:ci && pnpm run coverage:merge && node tools/coverage/coverage-health.mjs --require-report",
+        "coverage:ci": "pnpm run coverage:tools:test && pnpm run coverage:policy:check && pnpm run coverage:unit:ci && pnpm run coverage:merge && node tools/coverage/coverage-health.mjs --require-report",
         "coverage:e2e:summary": "node tools/coverage/e2e-semantic-summary.mjs",
         "coverage:e2e:v8:web": "node tools/coverage/e2e-v8-web.mjs",
         "typecheck:web": "tsc -p apps/web/tsconfig.app.json --noEmit",

tools/coverage/coverage-health.mjs

@@ -7,9 +7,26 @@ import {
     readdirSync,
     statSync,
 } from 'node:fs';
+import { createRequire } from 'node:module';
 import path from 'node:path';
 import process from 'node:process';
 
+import {
+    evaluateCoverageRatchets,
+    validateProjectCoverage,
+    validateRequiredProjectReports,
+} from './coverage-integrity.mjs';
+
+const require = createRequire(import.meta.url);
+const { createCoverageMap } = require('istanbul-lib-coverage');
+const COVERAGE_METRICS = [
+    'statements',
+    'branches',
+    'functions',
+    'lines',
+];
+const COVERAGE_SUMMARY_FIELDS = ['covered', 'total', 'pct'];
+
 const workspaceRoot = process.cwd();
 const args = new Set(process.argv.slice(2));
 const requireReport = args.has('--require-report');
@@ -37,6 +54,15 @@ function projectJsonPath(project) {
     return path.join(workspaceRoot, project.root, 'project.json');
 }
 
+function projectCoveragePath(project) {
+    return path.join(
+        workspaceRoot,
+        'coverage',
+        project.root,
+        'coverage-final.json'
+    );
+}
+
 function verifyTierAProjects() {
     for (const project of policy.unitCoverage.tierA) {
         const filePath = projectJsonPath(project);
@@ -58,14 +84,63 @@ function verifyTierAProjects() {
     }
 }
 
+function verifyProjectCoverageReports() {
+    if (requireReport) {
+        const validation = validateRequiredProjectReports({
+            projects: policy.unitCoverage.tierA,
+            workspaceRoot,
+        });
+        errors.push(...validation.errors);
+        return;
+    }
+
+    for (const project of policy.unitCoverage.tierA) {
+        if (!existsSync(projectCoveragePath(project))) {
+            continue;
+        }
+
+        const validation = validateProjectCoverage({
+            project,
+            workspaceRoot,
+        });
+        errors.push(...validation.errors);
+    }
+}
+
+function reportCoverageSummaryMismatches(reportedSummary, computedSummary) {
+    for (const metric of COVERAGE_METRICS) {
+        const reported = reportedSummary[metric];
+        const computed = computedSummary[metric];
+        if (
+            COVERAGE_SUMMARY_FIELDS.every(
+                (field) => reported?.[field] === computed?.[field]
+            )
+        ) {
+            continue;
+        }
+
+        errors.push(
+            `Merged coverage summary mismatch for ${metric}: coverage-summary.json reports covered ${reported?.covered}, total ${reported?.total}, pct ${reported?.pct}; coverage-final.json computes covered ${computed?.covered}, total ${computed?.total}, pct ${computed?.pct}.`
+        );
+    }
+}
+
 function verifyCoverageReport() {
     const summaryPath = path.join(
         workspaceRoot,
         policy.reporting.mergedCoverageDir,
         'coverage-summary.json'
     );
+    const ratchet = policy.reporting.coverageRatchet;
+    const coveragePath = path.join(
+        workspaceRoot,
+        policy.reporting.mergedCoverageDir,
+        'coverage-final.json'
+    );
+    const summaryExists = existsSync(summaryPath);
+    const coverageExists = existsSync(coveragePath);
 
-    if (!existsSync(summaryPath)) {
+    if (!summaryExists && !coverageExists) {
         const message = `Merged coverage summary not found at ${path.relative(workspaceRoot, summaryPath)}.`;
         if (requireReport) {
             errors.push(message);
@@ -75,25 +150,83 @@ function verifyCoverageReport() {
         return;
     }
 
-    const summary = readJson(summaryPath).total;
-    console.log(
-        `Merged coverage: statements ${summary.statements.pct}%, branches ${summary.branches.pct}%, functions ${summary.functions.pct}%, lines ${summary.lines.pct}%.`
-    );
+    if (summaryExists !== coverageExists) {
+        const existingPath = summaryExists ? summaryPath : coveragePath;
+        const missingPath = summaryExists ? coveragePath : summaryPath;
+        errors.push(
+            `Merged coverage artifacts are incomplete: found ${path.relative(workspaceRoot, existingPath)} but ${path.relative(workspaceRoot, missingPath)} is missing.`
+        );
+        return;
+    }
 
-    if (requireReport) {
-        for (const project of policy.unitCoverage.tierA) {
-            const projectCoveragePath = path.join(
+    let summaryDocument;
+    try {
+        summaryDocument = readJson(summaryPath);
+    } catch (error) {
+        errors.push(
+            `Merged coverage summary ${path.relative(workspaceRoot, summaryPath)} contains invalid JSON: ${error.message}`
+        );
+        return;
+    }
+
+    const summary = summaryDocument?.total;
+    if (
+        !summary ||
+        COVERAGE_METRICS.some(
+            (metric) =>
+                !summary[metric] ||
+                !Number.isFinite(summary[metric].pct)
+        )
+    ) {
+        errors.push(
+            `Merged coverage summary ${path.relative(workspaceRoot, summaryPath)} does not contain usable total coverage metrics.`
+        );
+        return;
+    }
+
+    let coverageData;
+    try {
+        coverageData = readJson(coveragePath);
+    } catch (error) {
+        errors.push(
+            `Merged coverage report ${path.relative(workspaceRoot, coveragePath)} contains invalid JSON: ${error.message}`
+        );
+        return;
+    }
+
+    let computedSummary;
+    try {
+        const coverageMap = createCoverageMap(coverageData);
+        computedSummary = coverageMap.getCoverageSummary().toJSON();
+    } catch (error) {
+        errors.push(
+            `Merged coverage report ${path.relative(workspaceRoot, coveragePath)} is not valid Istanbul coverage: ${error.message}`
+        );
+        return;
+    }
+
+    console.log(
+        `Merged coverage: statements ${computedSummary.statements.pct}%, branches ${computedSummary.branches.pct}%, functions ${computedSummary.functions.pct}%, lines ${computedSummary.lines.pct}%.`
+    );
+    reportCoverageSummaryMismatches(summary, computedSummary);
+
+    if (ratchet === undefined) {
+        return;
+    }
+
+    try {
+        errors.push(
+            ...evaluateCoverageRatchets({
+                coverageData,
+                mergedSummary: computedSummary,
+                ratchet,
                 workspaceRoot,
-                'coverage',
-                project.root,
-                'coverage-final.json'
-            );
-            if (!existsSync(projectCoveragePath)) {
-                errors.push(
-                    `Tier A project ${project.name} did not produce ${path.relative(workspaceRoot, projectCoveragePath)}.`
-                );
-            }
-        }
+            })
+        );
+    } catch (error) {
+        errors.push(
+            `Merged coverage report ${path.relative(workspaceRoot, coveragePath)} is not valid Istanbul coverage: ${error.message}`
+        );
     }
 }
 
@@ -216,6 +349,7 @@ function reportChangedCriticalFiles() {
 }
 
 verifyTierAProjects();
+verifyProjectCoverageReports();
 verifyCoverageReport();
 scanE2ETags();
 reportChangedCriticalFiles();

tools/coverage/merge-coverage.mjs

@@ -2,7 +2,6 @@
 
 import { createRequire } from 'node:module';
 import {
-    existsSync,
     mkdirSync,
     readFileSync,
     rmSync,
@@ -11,6 +10,8 @@ import {
 import path from 'node:path';
 import process from 'node:process';
 
+import { validateRequiredProjectReports } from './coverage-integrity.mjs';
+
 const require = createRequire(import.meta.url);
 const { createCoverageMap } = require('istanbul-lib-coverage');
 const libReport = require('istanbul-lib-report');
@@ -22,11 +23,20 @@ const policy = JSON.parse(
 );
 const outputDir = path.join(workspaceRoot, policy.reporting.mergedCoverageDir);
 
-const coverageFiles = policy.unitCoverage.tierA
-    .map((project) => path.join(workspaceRoot, 'coverage', project.root, 'coverage-final.json'))
-    .filter((coverageFile) => existsSync(coverageFile));
+const validation = validateRequiredProjectReports({
+    projects: policy.unitCoverage.tierA,
+    workspaceRoot,
+});
 
-if (coverageFiles.length === 0) {
+if (validation.errors.length > 0) {
+    for (const error of validation.errors) {
+        console.error(`Error: ${error}`);
+    }
+    process.exit(1);
+}
+
+const coverageInputs = validation.reports;
+if (coverageInputs.length === 0) {
     console.error('No Tier A coverage-final.json files found under coverage/.');
     process.exit(1);
 }
@@ -36,9 +46,8 @@ mkdirSync(outputDir, { recursive: true });
 
 const coverageMap = createCoverageMap({});
 
-for (const coverageFile of coverageFiles) {
-    const data = JSON.parse(readFileSync(coverageFile, 'utf8'));
-    coverageMap.merge(data);
+for (const input of coverageInputs) {
+    coverageMap.merge(input.data);
 }
 
 const context = libReport.createContext({
@@ -56,7 +65,7 @@ writeFileSync(
     `${JSON.stringify(summary, null, 4)}\n`
 );
 
-console.log(`Merged ${coverageFiles.length} coverage files into ${policy.reporting.mergedCoverageDir}`);
+console.log(`Merged ${coverageInputs.length} coverage files into ${policy.reporting.mergedCoverageDir}`);
 console.log(
     `Statements: ${summary.statements.pct}% | Branches: ${summary.branches.pct}% | Functions: ${summary.functions.pct}% | Lines: ${summary.lines.pct}%`
 );

tools/coverage/run-tier-a-coverage.mjs

@@ -1,10 +1,15 @@
 #!/usr/bin/env node
 
-import { spawnSync } from 'node:child_process';
+import { spawn } from 'node:child_process';
 import { existsSync, readFileSync, rmSync } from 'node:fs';
 import path from 'node:path';
 import process from 'node:process';
 
+import {
+    createCoverageOutputScanner,
+    validateProjectCoverage,
+} from './coverage-integrity.mjs';
+
 const workspaceRoot = process.cwd();
 const policyPath = path.join(workspaceRoot, 'tools/coverage/coverage-policy.json');
 const policy = JSON.parse(readFileSync(policyPath, 'utf8'));
@@ -120,6 +125,64 @@ function buildNxArgs(project) {
     );
 }
 
+function spawnCoverage(args, scanner) {
+    return new Promise((resolve, reject) => {
+        const child = spawn('pnpm', args, {
+            cwd: workspaceRoot,
+            env: {
+                ...process.env,
+                CI: process.env.CI ?? 'true',
+                NX_TASKS_RUNNER_DYNAMIC_OUTPUT: 'false',
+            },
+            stdio: ['inherit', 'pipe', 'pipe'],
+        });
+
+        child.stdout.on('data', (chunk) => {
+            scanner.push(chunk);
+            process.stdout.write(chunk);
+        });
+        child.stderr.on('data', (chunk) => {
+            scanner.push(chunk);
+            process.stderr.write(chunk);
+        });
+        child.once('error', reject);
+        child.once('close', (code, signal) => {
+            resolve({ code, signal });
+        });
+    });
+}
+
+async function collectProjectCoverage(project) {
+    const args = buildNxArgs(project);
+    console.log(`\n==> Collecting coverage for ${project.name}`);
+    console.log(`pnpm ${args.join(' ')}`);
+
+    const scanner = createCoverageOutputScanner();
+    const result = await spawnCoverage(args, scanner);
+    let failed = result.code !== 0 || result.signal !== null;
+
+    if (scanner.collectionFailed) {
+        console.error(
+            `Coverage collection failed while testing ${project.name}.`
+        );
+        failed = true;
+    }
+
+    if (failed) {
+        return result.code && result.code !== 0 ? result.code : 1;
+    }
+
+    const validation = validateProjectCoverage({
+        project,
+        workspaceRoot,
+    });
+    for (const error of validation.errors) {
+        console.error(`Error: ${error}`);
+    }
+
+    return validation.errors.length === 0 ? 0 : 1;
+}
+
 for (const project of tierAProjects) {
     const coverageDir = path.join(workspaceRoot, 'coverage', project.root);
     rmSync(coverageDir, { recursive: true, force: true });
@@ -136,21 +199,8 @@ if (requestedProjects.size === 0) {
 }
 
 for (const project of tierAProjects) {
-    const args = buildNxArgs(project);
-    console.log(`\n==> Collecting coverage for ${project.name}`);
-    console.log(`pnpm ${args.join(' ')}`);
-
-    const result = spawnSync('pnpm', args, {
-        cwd: workspaceRoot,
-        env: {
-            ...process.env,
-            CI: process.env.CI ?? 'true',
-            NX_TASKS_RUNNER_DYNAMIC_OUTPUT: 'false',
-        },
-        stdio: 'inherit',
-    });
-
-    if (result.status !== 0) {
-        process.exit(result.status ?? 1);
+    const status = await collectProjectCoverage(project);
+    if (status !== 0) {
+        process.exit(status);
     }
 }