From ef900d0f2ad7f264051cc926ef2781a0c0c6a4d6 Mon Sep 17 00:00:00 2001 From: 4gray <4gray@users.noreply.github.com> Date: Sat, 6 Jun 2026 17:41:40 +0200 Subject: [PATCH] [codex] Add scoped coverage reporting (#1024) * add scoped coverage reporting * fix coverage review feedback --------- Co-authored-by: 4gray --- .github/workflows/ci.yml | 27 +- .github/workflows/e2e-tests.yaml | 22 +- .../electron-backend-e2e/playwright.config.ts | 7 + apps/electron-backend-e2e/src/epg.e2e.ts | 4 +- .../electron-backend-e2e/src/favorites.e2e.ts | 6 +- .../src/file-import.e2e.ts | 2 +- .../electron-backend-e2e/src/providers.e2e.ts | 6 +- apps/electron-backend-e2e/src/recent.e2e.ts | 6 +- apps/electron-backend-e2e/src/settings.e2e.ts | 6 +- apps/electron-backend-e2e/src/smoke.e2e.ts | 6 +- .../src/xtream-epg.e2e.ts | 2 +- apps/web-e2e/playwright.config.ts | 15 + apps/web-e2e/src/basic.e2e.ts | 6 +- apps/web-e2e/src/fixtures.ts | 41 +++ apps/web-e2e/src/pwa-styles.e2e.ts | 4 +- apps/web-e2e/src/self-hosted.e2e.ts | 3 +- apps/web-e2e/src/settings.e2e.ts | 11 +- apps/web-e2e/src/stalker.e2e.ts | 3 +- apps/web-e2e/src/xtream.e2e.ts | 3 +- apps/web/jest.config.ts | 17 +- docs/architecture/validation-map.md | 45 +++ jest.preset.js | 18 +- jest.web-esm.workspace.ts | 4 +- package.json | 9 + pnpm-lock.yaml | 9 + tools/coverage/coverage-health.mjs | 234 ++++++++++++ tools/coverage/coverage-policy.json | 346 ++++++++++++++++++ tools/coverage/e2e-semantic-summary.mjs | 245 +++++++++++++ tools/coverage/e2e-v8-web.mjs | 153 ++++++++ tools/coverage/merge-coverage.mjs | 62 ++++ tools/coverage/run-tier-a-coverage.mjs | 156 ++++++++ 31 files changed, 1433 insertions(+), 45 deletions(-) create mode 100644 apps/web-e2e/src/fixtures.ts create mode 100644 tools/coverage/coverage-health.mjs create mode 100644 tools/coverage/coverage-policy.json create mode 100644 tools/coverage/e2e-semantic-summary.mjs create mode 100644 tools/coverage/e2e-v8-web.mjs create mode 100644 tools/coverage/merge-coverage.mjs create mode 100644 tools/coverage/run-tier-a-coverage.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c29a78130..6a73d3f6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: unit-and-typecheck: name: Unit Tests and Typechecks runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 steps: - name: Checkout code @@ -37,8 +37,29 @@ jobs: - name: Check i18n drift run: pnpm run i18n:check - - name: Run curated Nx unit test suite - run: pnpm run test:unit:ci + - name: Run Tier A unit coverage suite + run: pnpm run coverage:ci env: CI: true NX_TASKS_RUNNER_DYNAMIC_OUTPUT: false + + - name: Upload unit coverage artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: unit-coverage + path: | + coverage/merged/ + retention-days: 14 + + - name: Upload unit coverage to Codecov + if: always() + uses: codecov/codecov-action@v6 + with: + files: ./coverage/merged/lcov.info,./coverage/merged/cobertura-coverage.xml + flags: unit + name: iptvnator-unit + fail_ci_if_error: false + handle_no_reports_found: true + disable_search: true + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 3f7876809..b3ef0012c 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -52,12 +52,19 @@ jobs: env: CI: true + - name: Summarize Electron E2E semantic coverage + if: always() + run: pnpm run coverage:e2e:summary -- --project=electron-backend-e2e + - name: Upload Electron Test Results - if: failure() + if: always() uses: actions/upload-artifact@v4 with: name: playwright-report-electron-${{ matrix.os }} - path: dist/playwright-report/electron-backend-e2e/ + path: | + dist/playwright-report/electron-backend-e2e/ + dist/test-results/electron-backend-e2e/ + coverage/e2e/ retention-days: 7 web-e2e-tests: @@ -89,10 +96,17 @@ jobs: env: CI: true + - name: Summarize Web E2E semantic coverage + if: always() + run: pnpm run coverage:e2e:summary -- --project=web-e2e + - name: Upload Web Test Results - if: failure() + if: always() uses: actions/upload-artifact@v4 with: name: playwright-report-web-ubuntu - path: dist/.playwright/apps/web-e2e/playwright-report/ + path: | + dist/playwright-report/web-e2e/ + dist/test-results/web-e2e/ + coverage/e2e/ retention-days: 7 diff --git a/apps/electron-backend-e2e/playwright.config.ts b/apps/electron-backend-e2e/playwright.config.ts index 94137651e..9446117e2 100644 --- a/apps/electron-backend-e2e/playwright.config.ts +++ b/apps/electron-backend-e2e/playwright.config.ts @@ -27,6 +27,13 @@ export default defineConfig({ '../../dist/playwright-report/electron-backend-e2e', }, ], + [ + 'json', + { + outputFile: + '../../dist/test-results/electron-backend-e2e/results.json', + }, + ], ], /* Shared settings for all the projects below */ use: { diff --git a/apps/electron-backend-e2e/src/epg.e2e.ts b/apps/electron-backend-e2e/src/epg.e2e.ts index 267539b94..de27fe91d 100644 --- a/apps/electron-backend-e2e/src/epg.e2e.ts +++ b/apps/electron-backend-e2e/src/epg.e2e.ts @@ -29,7 +29,7 @@ const epgFixtureXml = ` `; test.describe('Electron EPG', () => { - test('adds an EPG source, fetches guide data, removes the source row, and clears stored EPG data', async ({ + test('@epg @electron adds an EPG source, fetches guide data, removes the source row, and clears stored EPG data', async ({ dataDir, }) => { const epgServer = await createMutableTextServer(epgFixtureXml, { @@ -85,7 +85,7 @@ test.describe('Electron EPG', () => { } }); - test('uses the XMLTV channel icon as a fallback when the playlist has no tvg-logo', async ({ + test('@epg @electron uses the XMLTV channel icon as a fallback when the playlist has no tvg-logo', async ({ dataDir, }) => { const playlistServer = await createMutableTextServer( diff --git a/apps/electron-backend-e2e/src/favorites.e2e.ts b/apps/electron-backend-e2e/src/favorites.e2e.ts index ddd37a443..4db9c3edc 100644 --- a/apps/electron-backend-e2e/src/favorites.e2e.ts +++ b/apps/electron-backend-e2e/src/favorites.e2e.ts @@ -43,7 +43,7 @@ import { } from './portal-mock-fixtures'; test.describe('Electron Favorites', () => { - test('shows M3U favorites in playlist and all-playlists scope, and preserves them after restart', async ({ + test('@persistence @m3u @electron shows M3U favorites in playlist and all-playlists scope, and preserves them after restart', async ({ dataDir, }) => { const playlistTitle = 'm3u-favorites-source.m3u'; @@ -172,7 +172,7 @@ test.describe('Electron Favorites', () => { } }); - test('shows Xtream live, movie, and series favorites in playlist and all-playlists scope, and preserves them after restart', async ({ + test('@persistence @xtream @electron shows Xtream live, movie, and series favorites in playlist and all-playlists scope, and preserves them after restart', async ({ dataDir, request, }) => { @@ -374,7 +374,7 @@ test.describe('Electron Favorites', () => { } }); - test('shows Stalker live, movie, and series favorites in playlist and all-playlists scope, and preserves them after restart', async ({ + test('@persistence @stalker @electron shows Stalker live, movie, and series favorites in playlist and all-playlists scope, and preserves them after restart', async ({ dataDir, request, }) => { diff --git a/apps/electron-backend-e2e/src/file-import.e2e.ts b/apps/electron-backend-e2e/src/file-import.e2e.ts index 579cb6dd9..03d3c8f81 100644 --- a/apps/electron-backend-e2e/src/file-import.e2e.ts +++ b/apps/electron-backend-e2e/src/file-import.e2e.ts @@ -8,7 +8,7 @@ import { } from './electron-test-fixtures'; test.describe('Electron Native Playlist Import', () => { - test('imports an M3U playlist via the native file picker path', async ({ + test('@m3u @electron imports an M3U playlist via the native file picker path', async ({ dataDir, }) => { const app = await launchElectronApp(dataDir); diff --git a/apps/electron-backend-e2e/src/providers.e2e.ts b/apps/electron-backend-e2e/src/providers.e2e.ts index c9ebd4eb7..0ff51b833 100644 --- a/apps/electron-backend-e2e/src/providers.e2e.ts +++ b/apps/electron-backend-e2e/src/providers.e2e.ts @@ -19,7 +19,7 @@ import { } from './electron-test-fixtures'; test.describe('Electron Provider Smoke Tests', () => { - test('loads Xtream content through the Electron IPC path', async ({ + test('@xtream @electron loads Xtream content through the Electron IPC path', async ({ dataDir, request, }) => { @@ -42,7 +42,7 @@ test.describe('Electron Provider Smoke Tests', () => { } }); - test('loads Stalker content through the Electron IPC path', async ({ + test('@stalker @electron loads Stalker content through the Electron IPC path', async ({ dataDir, request, }) => { @@ -67,7 +67,7 @@ test.describe('Electron Provider Smoke Tests', () => { } }); - test('shows refresh overlay immediately from the dashboard Xtream source menu', async ({ + test('@xtream @electron shows refresh overlay immediately from the dashboard Xtream source menu', async ({ dataDir, request, }) => { diff --git a/apps/electron-backend-e2e/src/recent.e2e.ts b/apps/electron-backend-e2e/src/recent.e2e.ts index 37a1329d0..66d9e6322 100644 --- a/apps/electron-backend-e2e/src/recent.e2e.ts +++ b/apps/electron-backend-e2e/src/recent.e2e.ts @@ -89,7 +89,7 @@ test.describe('Electron Recently Viewed', () => { } }); - test('tracks M3U recent channels in newest-first order, supports all-playlists scope, and persists favorites after restart', async ({ + test('@persistence @m3u @electron tracks M3U recent channels in newest-first order, supports all-playlists scope, and persists favorites after restart', async ({ dataDir, }) => { const playlistTitle = 'm3u-recent-source.m3u'; @@ -219,7 +219,7 @@ test.describe('Electron Recently Viewed', () => { } }); - test('tracks Xtream live, movie, and series history across playlist and all-playlists scope, persists after restart, and supports clearing', async ({ + test('@persistence @xtream @electron tracks Xtream live, movie, and series history across playlist and all-playlists scope, persists after restart, and supports clearing', async ({ dataDir, request, }) => { @@ -425,7 +425,7 @@ test.describe('Electron Recently Viewed', () => { } }); - test('tracks Stalker live, movie, and series history across playlist and all-playlists scope, and preserves it after restart', async ({ + test('@persistence @stalker @electron tracks Stalker live, movie, and series history across playlist and all-playlists scope, and preserves it after restart', async ({ dataDir, request, }) => { diff --git a/apps/electron-backend-e2e/src/settings.e2e.ts b/apps/electron-backend-e2e/src/settings.e2e.ts index cc5e1de3e..31cf08999 100644 --- a/apps/electron-backend-e2e/src/settings.e2e.ts +++ b/apps/electron-backend-e2e/src/settings.e2e.ts @@ -41,7 +41,7 @@ const epgFixtureXml = ` `; test.describe('Electron Settings', () => { - test('gates external MPV playback behind double-click when enabled', async ({ + test('@settings @electron gates external MPV playback behind double-click when enabled', async ({ dataDir, }) => { const app = await launchElectronApp(dataDir); @@ -121,7 +121,7 @@ test.describe('Electron Settings', () => { } }); - test('persists changed desktop settings across app restart', async ({ + test('@settings @persistence @electron persists changed desktop settings across app restart', async ({ dataDir, }) => { const epgServer = await createMutableTextServer(epgFixtureXml, { @@ -207,7 +207,7 @@ test.describe('Electron Settings', () => { } }); - test('starts on sources when dashboard is disabled', async ({ dataDir }) => { + test('@settings @electron starts on sources when dashboard is disabled', async ({ dataDir }) => { const firstLaunch = await launchElectronApp(dataDir); try { diff --git a/apps/electron-backend-e2e/src/smoke.e2e.ts b/apps/electron-backend-e2e/src/smoke.e2e.ts index 5924bb4b9..cfd1af80e 100644 --- a/apps/electron-backend-e2e/src/smoke.e2e.ts +++ b/apps/electron-backend-e2e/src/smoke.e2e.ts @@ -6,7 +6,7 @@ import { } from './electron-test-fixtures'; test.describe('Electron App Smoke Test', () => { - test('app should start and display the dashboard', async ({ dataDir }) => { + test('@critical @electron app should start and display the dashboard', async ({ dataDir }) => { const app = await launchElectronApp(dataDir); try { @@ -27,7 +27,7 @@ test.describe('Electron App Smoke Test', () => { } }); - test('app should expose the expected main window properties', async ({ + test('@critical @electron app should expose the expected main window properties', async ({ dataDir, }) => { const app = await launchElectronApp(dataDir); @@ -57,7 +57,7 @@ test.describe('Electron App Smoke Test', () => { } }); - test('app should render workspace content', async ({ dataDir }) => { + test('@critical @electron app should render workspace content', async ({ dataDir }) => { const app = await launchElectronApp(dataDir); try { diff --git a/apps/electron-backend-e2e/src/xtream-epg.e2e.ts b/apps/electron-backend-e2e/src/xtream-epg.e2e.ts index 6d2fb6506..435b545ce 100644 --- a/apps/electron-backend-e2e/src/xtream-epg.e2e.ts +++ b/apps/electron-backend-e2e/src/xtream-epg.e2e.ts @@ -22,7 +22,7 @@ const epgCredentials = { }; for (const timeZone of ['UTC', 'Europe/Berlin'] as const) { - test(`renders Xtream EPG previews and selected-channel schedule in ${timeZone}`, async ({ + test(`@epg @xtream @electron renders Xtream EPG previews and selected-channel schedule in ${timeZone}`, async ({ dataDir, request, }) => { diff --git a/apps/web-e2e/playwright.config.ts b/apps/web-e2e/playwright.config.ts index cb60a5824..adee8a21c 100644 --- a/apps/web-e2e/playwright.config.ts +++ b/apps/web-e2e/playwright.config.ts @@ -28,6 +28,21 @@ const reuseExistingWebServer = isStaticPwaE2E ? false : !process.env['CI']; export default defineConfig({ ...nxE2EPreset(__filename, { testDir: './src' }), testMatch: ['**/*.e2e.ts'], + reporter: [ + ['list'], + [ + 'html', + { + outputFolder: '../../dist/playwright-report/web-e2e', + }, + ], + [ + 'json', + { + outputFile: '../../dist/test-results/web-e2e/results.json', + }, + ], + ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { baseURL, diff --git a/apps/web-e2e/src/basic.e2e.ts b/apps/web-e2e/src/basic.e2e.ts index 462d5f53a..8fa6d4d13 100644 --- a/apps/web-e2e/src/basic.e2e.ts +++ b/apps/web-e2e/src/basic.e2e.ts @@ -1,9 +1,9 @@ -import { expect, test } from '@playwright/test'; import { join } from 'path'; +import { expect, test } from './fixtures'; const fixturePath = join(__dirname, 'fixtures/test.m3u'); -test('basic test', async ({ page }) => { +test('@web @m3u basic playlist import flow', async ({ page }) => { await page.goto('/'); // Basic checks @@ -31,7 +31,7 @@ test('basic test', async ({ page }) => { await expect(page.getByText('4. HappyKids TV')).toBeVisible(); }); -test('keyboard shortcuts help opens from question mark', async ({ page }) => { +test('@web keyboard shortcuts help opens from question mark', async ({ page }) => { await page.goto('/'); await page.getByRole('button', { name: 'Open keyboard shortcuts' }).focus(); diff --git a/apps/web-e2e/src/fixtures.ts b/apps/web-e2e/src/fixtures.ts new file mode 100644 index 000000000..b2272acbd --- /dev/null +++ b/apps/web-e2e/src/fixtures.ts @@ -0,0 +1,41 @@ +import { test as base, expect, type Page } from '@playwright/test'; +import { mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +function safeArtifactName(value: string): string { + return value + .replace(/[^a-z0-9]+/gi, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 120); +} + +export const test = base.extend<{ page: Page }>({ + page: async ({ page }, use, testInfo) => { + if (process.env['IPTVNATOR_E2E_V8_COVERAGE'] !== '1') { + await use(page); + return; + } + + await page.coverage.startJSCoverage({ resetOnNavigation: false }); + await page.coverage.startCSSCoverage({ resetOnNavigation: false }); + + try { + await use(page); + } finally { + const [js, css] = await Promise.all([ + page.coverage.stopJSCoverage(), + page.coverage.stopCSSCoverage(), + ]); + const coverageDir = path.join(testInfo.outputDir, 'v8-coverage'); + const artifactName = safeArtifactName(testInfo.titlePath.join(' ')); + + await mkdir(coverageDir, { recursive: true }); + await writeFile( + path.join(coverageDir, `${artifactName || 'test'}.json`), + `${JSON.stringify({ css, js }, null, 4)}\n` + ); + } + }, +}); + +export { expect }; diff --git a/apps/web-e2e/src/pwa-styles.e2e.ts b/apps/web-e2e/src/pwa-styles.e2e.ts index 3d91ec4c5..5413355a5 100644 --- a/apps/web-e2e/src/pwa-styles.e2e.ts +++ b/apps/web-e2e/src/pwa-styles.e2e.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { expect, test } from './fixtures'; // eslint-disable-next-line playwright/no-skipped-test -- Static PWA assertions require the built service worker artifact. test.skip( @@ -6,7 +6,7 @@ test.skip( 'Static PWA stylesheet regression test only runs against the built PWA output.' ); -test('@pwa-static PWA build applies the full stylesheet under CSP', async ({ +test('@pwa @pwa-static PWA build applies the full stylesheet under CSP', async ({ page, }) => { const cspConsoleErrors: string[] = []; diff --git a/apps/web-e2e/src/self-hosted.e2e.ts b/apps/web-e2e/src/self-hosted.e2e.ts index 85047cab0..74d3e634f 100644 --- a/apps/web-e2e/src/self-hosted.e2e.ts +++ b/apps/web-e2e/src/self-hosted.e2e.ts @@ -1,4 +1,5 @@ -import { expect, type Locator, type Page, test } from '@playwright/test'; +import type { Locator, Page } from '@playwright/test'; +import { expect, test } from './fixtures'; const WEB_BACKEND_URL = 'http://localhost:3333'; const XTREAM_MOCK_PORT = process.env['XTREAM_MOCK_PORT'] ?? '3211'; diff --git a/apps/web-e2e/src/settings.e2e.ts b/apps/web-e2e/src/settings.e2e.ts index f7ba49b2b..2b687b91b 100644 --- a/apps/web-e2e/src/settings.e2e.ts +++ b/apps/web-e2e/src/settings.e2e.ts @@ -1,5 +1,6 @@ -import { expect, Page, test } from '@playwright/test'; +import type { Page } from '@playwright/test'; import { join } from 'path'; +import { expect, test } from './fixtures'; async function openSettings(page: Page) { await page.locator('a[href$="/workspace/settings"]').click(); @@ -22,12 +23,12 @@ test.describe('Settings', () => { await page.goto('/'); }); - test('Check settings page', async ({ page }) => { + test('@settings @web Check settings page', async ({ page }) => { await openSettings(page); await page.locator('.settings-back-button').click(); }); - test('Change video player', async ({ page }) => { + test('@settings @web Change video player', async ({ page }) => { await openSettings(page); const playerSelect = page.locator('[data-test-id="select-video-player"]'); @@ -47,7 +48,7 @@ test.describe('Settings', () => { ); }); - test('Change app theme', async ({ page }) => { + test('@settings @web Change app theme', async ({ page }) => { await openSettings(page); // v0.22 compact theme picker exposes the segmented control as a // radiogroup with options labelled just "Light"/"Dark"/"System". @@ -72,7 +73,7 @@ test.describe('Settings', () => { ).toHaveAttribute('aria-checked', 'true'); }); - test('Change app language', async ({ page }) => { + test('@settings @web Change app language', async ({ page }) => { await openSettings(page); const languageSelect = page.locator('[data-test-id="select-language"]'); diff --git a/apps/web-e2e/src/stalker.e2e.ts b/apps/web-e2e/src/stalker.e2e.ts index 084c8e579..f2371a150 100644 --- a/apps/web-e2e/src/stalker.e2e.ts +++ b/apps/web-e2e/src/stalker.e2e.ts @@ -1,10 +1,9 @@ import { - expect, type APIRequestContext, type Locator, type Page, - test, } from '@playwright/test'; +import { expect, test } from './fixtures'; import { getRegisteredProviderUrl, interceptProviderTargetRegistration, diff --git a/apps/web-e2e/src/xtream.e2e.ts b/apps/web-e2e/src/xtream.e2e.ts index 2cf44e57b..908abf780 100644 --- a/apps/web-e2e/src/xtream.e2e.ts +++ b/apps/web-e2e/src/xtream.e2e.ts @@ -1,4 +1,5 @@ -import { APIRequestContext, test, expect, Page } from '@playwright/test'; +import type { APIRequestContext, Page } from '@playwright/test'; +import { expect, test } from './fixtures'; import { getRegisteredProviderUrl, interceptProviderTargetRegistration, diff --git a/apps/web/jest.config.ts b/apps/web/jest.config.ts index 5cfd40cac..6145bb3e0 100644 --- a/apps/web/jest.config.ts +++ b/apps/web/jest.config.ts @@ -6,6 +6,18 @@ const angularEsmPreset = createEsmPreset({ tsconfig: '/tsconfig.spec.json', }); +const coverageReporters = ['json', 'json-summary', 'lcovonly', 'text-summary']; + +const collectCoverageFrom = [ + 'src/**/*.{ts,js,mjs,html}', + '!src/**/*.{spec,test}.ts', + '!src/**/test-setup.ts', + '!src/**/test-stubs/**', + '!src/**/*.generated.*', + '!src/**/environments/**', + '!src/**/index.ts', +]; + export default { ...nxPreset, ...angularEsmPreset, @@ -33,7 +45,6 @@ export default { transform: angularEsmPreset.transform, transformIgnorePatterns: [], extensionsToTreatAsEsm: angularEsmPreset.extensionsToTreatAsEsm, - coverageReporters: [ - ...(nxPreset.coverageReporters ?? []), - ], + coverageReporters, + collectCoverageFrom, }; diff --git a/docs/architecture/validation-map.md b/docs/architecture/validation-map.md index 21633a269..7df36a6de 100644 --- a/docs/architecture/validation-map.md +++ b/docs/architecture/validation-map.md @@ -23,6 +23,34 @@ pnpm nx show projects --withTarget e2e | Shared SQLite schema/connection | `pnpm nx test database` | | Packaging metadata | `pnpm nx test packaging` | +## Coverage Tiers + +Use `tools/coverage/coverage-policy.json` as the source of truth for coverage +ownership. + +| Tier | Rule | Validation | +| --- | --- | --- | +| A | Product/runtime Angular, Electron, backend, data-access, portal, playlist, workspace, playback, EPG, and shared UI code collects source coverage. | `pnpm run coverage:ci` | +| B | Validate behavior without percentage coverage, such as `website`, `packaging`, and Playwright E2E projects. | `pnpm nx test website`, `pnpm nx test packaging`, or the closest E2E target | +| C | Excluded from the source coverage baseline, such as mock servers, test helper libraries, and untested feature shells. | Validate through dependent flows, or add focused tests when changing behavior directly | + +`apps/website` is an Astro marketing site. Its useful signal is a successful +static build plus targeted output checks, not a merged code coverage percentage. +Projects with a test target but no specs, such as `remote-control-web` and +`remote-control` today, should not be in Tier A until focused specs exist. + +For local coverage inspection: + +```bash +pnpm run coverage:unit:ci +pnpm run coverage:merge +pnpm run coverage:health +``` + +The merged report is written to `coverage/merged/` as HTML, LCOV, Cobertura, +and JSON summary output. CI uploads the merged Tier A report to Codecov with the +`unit` flag and keeps the HTML report as a GitHub artifact. + ## E2E | Area | Command | @@ -33,6 +61,23 @@ pnpm nx show projects --withTarget e2e Use atomized E2E targets when available, for example `pnpm nx run web-e2e:e2e-ci--src/xtream.e2e.ts`. +Playwright coverage is measured semantically by tags and critical journeys, not +by a source-line percentage. E2E reports should use tags such as `@critical`, +`@electron`, `@web`, `@xtream`, `@stalker`, `@m3u`, `@search`, `@epg`, +`@persistence`, `@settings`, `@pwa`, and `@self-hosted`. + +After an E2E run, generate the semantic summary with: + +```bash +pnpm run coverage:e2e:summary +``` + +For local investigation only, Chromium browser V8 coverage can be explored with: + +```bash +pnpm run coverage:e2e:v8:web +``` + ## I18n ```bash diff --git a/jest.preset.js b/jest.preset.js index f078ddcec..0f7d1350d 100644 --- a/jest.preset.js +++ b/jest.preset.js @@ -1,3 +1,19 @@ const nxPreset = require('@nx/jest/preset').default; -module.exports = { ...nxPreset }; +const coverageReporters = ['json', 'json-summary', 'lcovonly', 'text-summary']; + +const collectCoverageFrom = [ + 'src/**/*.{ts,js,mjs,html}', + '!src/**/*.{spec,test}.ts', + '!src/**/test-setup.ts', + '!src/**/test-stubs/**', + '!src/**/*.generated.*', + '!src/**/environments/**', + '!src/**/index.ts', +]; + +module.exports = { + ...nxPreset, + coverageReporters, + collectCoverageFrom, +}; diff --git a/jest.web-esm.workspace.ts b/jest.web-esm.workspace.ts index b8e315466..d10d1b943 100644 --- a/jest.web-esm.workspace.ts +++ b/jest.web-esm.workspace.ts @@ -6,6 +6,8 @@ const angularEsmPreset = createEsmPreset({ tsconfig: '/apps/web/tsconfig.spec.json', }); +const coverageReporters = ['json', 'json-summary', 'lcovonly', 'text-summary']; + export default { ...nxPreset, ...angularEsmPreset, @@ -37,5 +39,5 @@ export default { extensionsToTreatAsEsm: angularEsmPreset.extensionsToTreatAsEsm, modulePathIgnorePatterns: ['/dist/', '/.nx/'], watchPathIgnorePatterns: ['/dist/', '/.nx/'], - coverageReporters: [...(nxPreset.coverageReporters ?? [])], + coverageReporters, }; diff --git a/package.json b/package.json index a2c2a9258..3c9b4b191 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,12 @@ "test:backend": "nx test electron-backend", "test:unit:all": "nx run-many --target=test --all --parallel=3", "test:unit:ci": "nx run-many --target=test --projects=electron-backend,web,services,m3u-state,portal-stalker-data-access,portal-shared-util,playlist-shared-ui,portal-xtream-data-access,portal-xtream-feature,workspace-dashboard-data-access,components,@iptvnator/pipes,packaging,epg-data-access,workspace-shell-util,database --parallel=3 --output-style=static", + "coverage:unit:ci": "node tools/coverage/run-tier-a-coverage.mjs", + "coverage:merge": "node tools/coverage/merge-coverage.mjs", + "coverage:health": "node tools/coverage/coverage-health.mjs", + "coverage:ci": "pnpm run coverage:unit:ci && pnpm run coverage:merge && node tools/coverage/coverage-health.mjs --require-report", + "coverage:e2e:summary": "node tools/coverage/e2e-semantic-summary.mjs", + "coverage:e2e:v8:web": "node tools/coverage/e2e-v8-web.mjs", "typecheck:web": "tsc -p apps/web/tsconfig.app.json --noEmit", "typecheck:backend": "tsc -p apps/electron-backend/tsconfig.app.json --noEmit", "typecheck:ci": "pnpm run typecheck:web && pnpm run typecheck:backend", @@ -184,6 +190,9 @@ "eslint-plugin-playwright": "^1.6.2", "express": "5.2.1", "globals": "15.9.0", + "istanbul-lib-coverage": "3.2.2", + "istanbul-lib-report": "3.0.1", + "istanbul-reports": "3.2.0", "jest": "^30.0.2", "jest-environment-jsdom": "^30.0.2", "jest-environment-node": "^30.0.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 234913856..bc4d35bca 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -383,6 +383,15 @@ importers: globals: specifier: 15.9.0 version: 15.9.0 + istanbul-lib-coverage: + specifier: 3.2.2 + version: 3.2.2 + istanbul-lib-report: + specifier: 3.0.1 + version: 3.0.1 + istanbul-reports: + specifier: 3.2.0 + version: 3.2.0 jest: specifier: ^30.0.2 version: 30.2.0(@types/node@20.19.9)(babel-plugin-macros@3.1.0)(esbuild-register@3.6.0(esbuild@0.27.0))(ts-node@10.9.2(@swc/core@1.15.8(@swc/helpers@0.5.18))(@types/node@20.19.9)(typescript@5.9.3)) diff --git a/tools/coverage/coverage-health.mjs b/tools/coverage/coverage-health.mjs new file mode 100644 index 000000000..555b75b32 --- /dev/null +++ b/tools/coverage/coverage-health.mjs @@ -0,0 +1,234 @@ +#!/usr/bin/env node + +import { execFileSync } from 'node:child_process'; +import { + existsSync, + readFileSync, + readdirSync, + statSync, +} from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; + +const workspaceRoot = process.cwd(); +const args = new Set(process.argv.slice(2)); +const requireReport = args.has('--require-report'); +const policy = JSON.parse( + readFileSync(path.join(workspaceRoot, 'tools/coverage/coverage-policy.json'), 'utf8') +); + +const warnings = []; +const errors = []; + +function warn(message) { + warnings.push(message); + if (process.env.GITHUB_ACTIONS) { + console.log(`::warning::${message}`); + } else { + console.warn(`Warning: ${message}`); + } +} + +function readJson(filePath) { + return JSON.parse(readFileSync(filePath, 'utf8')); +} + +function projectJsonPath(project) { + return path.join(workspaceRoot, project.root, 'project.json'); +} + +function verifyTierAProjects() { + for (const project of policy.unitCoverage.tierA) { + const filePath = projectJsonPath(project); + if (!existsSync(filePath)) { + errors.push(`Tier A project ${project.name} is missing ${project.root}/project.json.`); + continue; + } + + const projectJson = readJson(filePath); + if (!projectJson.targets?.test) { + errors.push(`Tier A project ${project.name} must have a test target.`); + } + if (!project.sourceRoot || !existsSync(path.join(workspaceRoot, project.sourceRoot))) { + errors.push(`Tier A project ${project.name} has an invalid sourceRoot: ${project.sourceRoot}`); + } + if (project.sourceRoot && !hasSpecUnder(project.sourceRoot)) { + errors.push(`Tier A project ${project.name} has no unit specs under ${project.sourceRoot}. Move it to Tier B/C or add focused tests.`); + } + } +} + +function verifyCoverageReport() { + const summaryPath = path.join( + workspaceRoot, + policy.reporting.mergedCoverageDir, + 'coverage-summary.json' + ); + + if (!existsSync(summaryPath)) { + const message = `Merged coverage summary not found at ${path.relative(workspaceRoot, summaryPath)}.`; + if (requireReport) { + errors.push(message); + } else { + warn(message); + } + return; + } + + const summary = readJson(summaryPath).total; + console.log( + `Merged coverage: statements ${summary.statements.pct}%, branches ${summary.branches.pct}%, functions ${summary.functions.pct}%, lines ${summary.lines.pct}%.` + ); + + if (requireReport) { + for (const project of policy.unitCoverage.tierA) { + const projectCoveragePath = path.join( + workspaceRoot, + 'coverage', + project.root, + 'coverage-final.json' + ); + if (!existsSync(projectCoveragePath)) { + errors.push( + `Tier A project ${project.name} did not produce ${path.relative(workspaceRoot, projectCoveragePath)}.` + ); + } + } + } +} + +function listFiles(directory, predicate) { + if (!existsSync(directory)) { + return []; + } + + const files = []; + for (const entry of readdirSync(directory)) { + const fullPath = path.join(directory, entry); + const stats = statSync(fullPath); + if (stats.isDirectory()) { + files.push(...listFiles(fullPath, predicate)); + } else if (predicate(fullPath)) { + files.push(fullPath); + } + } + return files; +} + +function hasSpecUnder(sourceRoot) { + return listFiles(path.join(workspaceRoot, sourceRoot), (file) => + /\.(spec|test)\.ts$/.test(file) + ).length > 0; +} + +function scanE2ETags() { + const e2eFiles = [ + ...listFiles(path.join(workspaceRoot, 'apps/web-e2e/src'), (file) => file.endsWith('.e2e.ts')), + ...listFiles(path.join(workspaceRoot, 'apps/electron-backend-e2e/src'), (file) => + file.endsWith('.e2e.ts') + ), + ]; + const tags = new Set(); + + for (const file of e2eFiles) { + const contents = readFileSync(file, 'utf8'); + for (const match of contents.matchAll(/@[a-z0-9-]+/gi)) { + tags.add(match[0]); + } + } + + for (const tag of policy.e2eSemanticCoverage.trackedTags) { + if (!tags.has(tag)) { + warn(`Tracked E2E tag ${tag} is not present in current Playwright specs yet.`); + } + } +} + +function changedFiles() { + const changedFileSet = new Set(); + const commands = []; + if (process.env.GITHUB_BASE_REF) { + commands.push(['git', ['diff', '--name-only', `origin/${process.env.GITHUB_BASE_REF}...HEAD`]]); + } else { + commands.push(['git', ['diff', '--name-only', 'HEAD']]); + commands.push(['git', ['ls-files', '--others', '--exclude-standard']]); + commands.push(['git', ['diff', '--name-only', 'HEAD~1...HEAD']]); + } + + for (const [command, commandArgs] of commands) { + try { + const output = execFileSync(command, commandArgs, { + cwd: workspaceRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + const outputFiles = output + .split('\n') + .map((file) => file.trim()) + .filter(Boolean); + for (const file of outputFiles) { + changedFileSet.add(file); + } + } catch { + // Try the next diff source. + } + } + + return Array.from(changedFileSet); +} + +function sourceOwner(file) { + return policy.unitCoverage.tierA.find((project) => + file.startsWith(`${project.sourceRoot}/`) + ); +} + +function hasNearbySpec(file) { + const parsed = path.parse(file); + const candidateNames = [ + path.join(parsed.dir, `${parsed.name}.spec.ts`), + path.join(parsed.dir, `${parsed.name}.test.ts`), + ]; + + return candidateNames.some((candidate) => existsSync(path.join(workspaceRoot, candidate))); +} + +function reportChangedCriticalFiles() { + const files = changedFiles().filter( + (file) => + /\.(ts|html)$/.test(file) && + !/\.(spec|test)\.ts$/.test(file) && + !file.endsWith('test-setup.ts') + ); + + for (const file of files) { + const owner = sourceOwner(file); + if (!owner) { + continue; + } + + if (!hasNearbySpec(file)) { + warn( + `${file} is Tier A source without a same-name unit spec. Preferred validation: ${owner.validationCommand}; related E2E tags: ${owner.e2eTags.join(', ')}.` + ); + } + } +} + +verifyTierAProjects(); +verifyCoverageReport(); +scanE2ETags(); +reportChangedCriticalFiles(); + +if (errors.length > 0) { + for (const error of errors) { + console.error(`Error: ${error}`); + } + process.exit(1); +} + +console.log( + warnings.length === 0 + ? 'Coverage health checks passed.' + : `Coverage health completed with ${warnings.length} warning(s).` +); diff --git a/tools/coverage/coverage-policy.json b/tools/coverage/coverage-policy.json new file mode 100644 index 000000000..d97aa5185 --- /dev/null +++ b/tools/coverage/coverage-policy.json @@ -0,0 +1,346 @@ +{ + "version": 1, + "reporting": { + "mergedCoverageDir": "coverage/merged", + "e2eSummaryDir": "coverage/e2e", + "unitCodecovFlag": "unit" + }, + "unitCoverage": { + "description": "Tier A projects collect and trend source coverage. Tier B projects are validated without percentage coverage. Tier C projects are excluded from the source coverage baseline.", + "tierA": [ + { + "name": "web", + "root": "apps/web", + "sourceRoot": "apps/web/src", + "jestRootMode": "project", + "validationCommand": "pnpm nx test web", + "e2eTags": ["@web", "@m3u", "@pwa", "@settings"] + }, + { + "name": "electron-backend", + "root": "apps/electron-backend", + "sourceRoot": "apps/electron-backend/src", + "validationCommand": "pnpm nx test electron-backend", + "e2eTags": ["@electron", "@critical", "@persistence"] + }, + { + "name": "web-backend", + "root": "apps/web-backend", + "sourceRoot": "apps/web-backend/src", + "validationCommand": "pnpm nx test web-backend", + "e2eTags": ["@self-hosted", "@web"] + }, + { + "name": "services", + "root": "libs/services", + "sourceRoot": "libs/services/src", + "validationCommand": "pnpm nx test services", + "e2eTags": ["@m3u", "@persistence", "@settings"] + }, + { + "name": "epg-data-access", + "root": "libs/epg/data-access", + "sourceRoot": "libs/epg/data-access/src", + "validationCommand": "pnpm nx test epg-data-access", + "e2eTags": ["@epg"] + }, + { + "name": "m3u-state", + "root": "libs/m3u-state", + "sourceRoot": "libs/m3u-state/src", + "validationCommand": "pnpm nx test m3u-state", + "e2eTags": ["@m3u", "@persistence"] + }, + { + "name": "playlist-import-feature", + "root": "libs/playlist/import/feature", + "sourceRoot": "libs/playlist/import/feature/src", + "validationCommand": "pnpm nx test playlist-import-feature", + "e2eTags": ["@m3u", "@file-import"] + }, + { + "name": "playlist-m3u-feature-player", + "root": "libs/playlist/m3u/feature-player", + "sourceRoot": "libs/playlist/m3u/feature-player/src", + "validationCommand": "pnpm nx test playlist-m3u-feature-player", + "e2eTags": ["@m3u", "@playback"] + }, + { + "name": "playlist-shared-ui", + "root": "libs/playlist/shared/ui", + "sourceRoot": "libs/playlist/shared/ui/src", + "validationCommand": "pnpm nx test playlist-shared-ui", + "e2eTags": ["@m3u"] + }, + { + "name": "playlist-shared-util", + "root": "libs/playlist/shared/util", + "sourceRoot": "libs/playlist/shared/util/src", + "validationCommand": "pnpm nx test playlist-shared-util", + "e2eTags": ["@m3u"] + }, + { + "name": "portal-catalog-feature", + "root": "libs/portal/catalog/feature", + "sourceRoot": "libs/portal/catalog/feature/src", + "validationCommand": "pnpm nx test portal-catalog-feature", + "e2eTags": ["@xtream", "@stalker"] + }, + { + "name": "portal-shared-data-access", + "root": "libs/portal/shared/data-access", + "sourceRoot": "libs/portal/shared/data-access/src", + "validationCommand": "pnpm nx test portal-shared-data-access", + "e2eTags": ["@xtream", "@stalker", "@persistence"] + }, + { + "name": "portal-shared-ui", + "root": "libs/portal/shared/ui", + "sourceRoot": "libs/portal/shared/ui/src", + "validationCommand": "pnpm nx test portal-shared-ui", + "e2eTags": ["@xtream", "@stalker"] + }, + { + "name": "portal-shared-util", + "root": "libs/portal/shared/util", + "sourceRoot": "libs/portal/shared/util/src", + "validationCommand": "pnpm nx test portal-shared-util", + "e2eTags": ["@xtream", "@stalker"] + }, + { + "name": "portal-stalker-data-access", + "root": "libs/portal/stalker/data-access", + "sourceRoot": "libs/portal/stalker/data-access/src", + "validationCommand": "pnpm nx test portal-stalker-data-access", + "e2eTags": ["@stalker"] + }, + { + "name": "portal-stalker-feature", + "root": "libs/portal/stalker/feature", + "sourceRoot": "libs/portal/stalker/feature/src", + "validationCommand": "pnpm nx test portal-stalker-feature", + "e2eTags": ["@stalker"] + }, + { + "name": "portal-xtream-data-access", + "root": "libs/portal/xtream/data-access", + "sourceRoot": "libs/portal/xtream/data-access/src", + "validationCommand": "pnpm nx test portal-xtream-data-access", + "e2eTags": ["@xtream"] + }, + { + "name": "portal-xtream-feature", + "root": "libs/portal/xtream/feature", + "sourceRoot": "libs/portal/xtream/feature/src", + "validationCommand": "pnpm nx test portal-xtream-feature", + "e2eTags": ["@xtream"] + }, + { + "name": "database", + "root": "libs/shared/database", + "sourceRoot": "libs/shared/database/src", + "validationCommand": "pnpm nx test database", + "e2eTags": ["@electron", "@persistence"] + }, + { + "name": "shared-interfaces", + "root": "libs/shared/interfaces", + "sourceRoot": "libs/shared/interfaces/src", + "validationCommand": "pnpm nx test shared-interfaces", + "e2eTags": ["@m3u", "@xtream", "@stalker"] + }, + { + "name": "m3u-utils", + "root": "libs/shared/m3u-utils", + "sourceRoot": "libs/shared/m3u-utils/src", + "validationCommand": "pnpm nx test m3u-utils", + "e2eTags": ["@m3u"] + }, + { + "name": "components", + "root": "libs/ui/components", + "sourceRoot": "libs/ui/components/src", + "validationCommand": "pnpm nx test components", + "e2eTags": ["@m3u", "@search", "@persistence"] + }, + { + "name": "ui-epg", + "root": "libs/ui/epg", + "sourceRoot": "libs/ui/epg/src", + "validationCommand": "pnpm nx test ui-epg", + "e2eTags": ["@epg"] + }, + { + "name": "@iptvnator/pipes", + "root": "libs/ui/pipes", + "sourceRoot": "libs/ui/pipes/src", + "validationCommand": "pnpm nx test @iptvnator/pipes", + "e2eTags": ["@epg", "@settings"] + }, + { + "name": "ui-playback", + "root": "libs/ui/playback", + "sourceRoot": "libs/ui/playback/src", + "validationCommand": "pnpm nx test ui-playback", + "e2eTags": ["@playback", "@m3u"] + }, + { + "name": "shared-portals", + "root": "libs/ui/shared-portals", + "sourceRoot": "libs/ui/shared-portals/src", + "validationCommand": "pnpm nx test shared-portals", + "e2eTags": ["@xtream", "@stalker"] + }, + { + "name": "workspace-dashboard-data-access", + "root": "libs/workspace/dashboard/data-access", + "sourceRoot": "libs/workspace/dashboard/data-access/src", + "validationCommand": "pnpm nx test workspace-dashboard-data-access", + "e2eTags": ["@dashboard", "@persistence"] + }, + { + "name": "workspace-dashboard-feature", + "root": "libs/workspace/dashboard/feature", + "sourceRoot": "libs/workspace/dashboard/feature/src", + "validationCommand": "pnpm nx test workspace-dashboard-feature", + "e2eTags": ["@dashboard", "@persistence"] + }, + { + "name": "workspace-shell-feature", + "root": "libs/workspace/shell/feature", + "sourceRoot": "libs/workspace/shell/feature/src", + "validationCommand": "pnpm nx test workspace-shell-feature", + "e2eTags": ["@search", "@settings", "@persistence"] + }, + { + "name": "workspace-shell-util", + "root": "libs/workspace/shell/util", + "sourceRoot": "libs/workspace/shell/util/src", + "validationCommand": "pnpm nx test workspace-shell-util", + "e2eTags": ["@search", "@settings"] + } + ], + "tierB": [ + { + "name": "website", + "root": "apps/website", + "validationCommand": "pnpm nx test website", + "reason": "Astro marketing site; build output and targeted static checks are more meaningful than source percentage coverage." + }, + { + "name": "packaging", + "root": "tools/packaging", + "validationCommand": "pnpm nx test packaging", + "reason": "Node package identity checks validate generated packaging metadata." + }, + { + "name": "remote-control-web", + "root": "apps/remote-control-web", + "validationCommand": "pnpm nx build remote-control-web", + "reason": "Product surface without unit specs today; validate build output and add focused tests when behavior changes." + }, + { + "name": "web-e2e", + "root": "apps/web-e2e", + "validationCommand": "pnpm nx run web-e2e:e2e -- --project=chromium", + "reason": "Playwright semantic flow coverage is reported separately." + }, + { + "name": "electron-backend-e2e", + "root": "apps/electron-backend-e2e", + "validationCommand": "pnpm nx run electron-backend-e2e:e2e", + "reason": "Playwright semantic flow coverage is reported separately." + } + ], + "tierC": [ + { + "name": "xtream-mock-server", + "root": "apps/xtream-mock-server", + "reason": "Validated through Playwright API and user-flow tests unless mock behavior changes directly." + }, + { + "name": "stalker-mock-server", + "root": "apps/stalker-mock-server", + "reason": "Validated through Playwright API and user-flow tests unless mock behavior changes directly." + }, + { + "name": "shared-testing", + "root": "libs/shared/testing", + "reason": "Test helper library." + }, + { + "name": "portal-downloads-feature", + "root": "libs/portal/downloads/feature", + "reason": "No test target yet; add focused tests when feature work changes this area." + }, + { + "name": "remote-control", + "root": "libs/ui/remote-control", + "reason": "No unit specs yet; validate through remote-control flows or add focused tests when behavior changes directly." + } + ] + }, + "e2eSemanticCoverage": { + "trackedTags": [ + "@critical", + "@electron", + "@web", + "@xtream", + "@stalker", + "@m3u", + "@search", + "@epg", + "@persistence", + "@settings", + "@pwa", + "@self-hosted" + ], + "criticalJourneys": [ + { + "name": "Electron app starts and renders workspace", + "projects": ["electron-backend-e2e"], + "matchAnyTags": ["@critical"] + }, + { + "name": "Xtream portal import, browse, and cached access", + "projects": ["web-e2e", "electron-backend-e2e"], + "matchAnyTags": ["@xtream"] + }, + { + "name": "Stalker portal import and browse", + "projects": ["web-e2e", "electron-backend-e2e"], + "matchAnyTags": ["@stalker"] + }, + { + "name": "M3U import, playback, and playlist state", + "projects": ["web-e2e", "electron-backend-e2e"], + "matchAnyTags": ["@m3u"] + }, + { + "name": "Workspace search across providers", + "projects": ["electron-backend-e2e"], + "matchAnyTags": ["@search"] + }, + { + "name": "EPG rendering and source management", + "projects": ["electron-backend-e2e"], + "matchAnyTags": ["@epg"] + }, + { + "name": "Favorites, recent items, and restart persistence", + "projects": ["electron-backend-e2e"], + "matchAnyTags": ["@persistence"] + }, + { + "name": "Settings persistence and playback options", + "projects": ["web-e2e", "electron-backend-e2e"], + "matchAnyTags": ["@settings"] + }, + { + "name": "Self-hosted PWA/backend integration", + "projects": ["web-e2e"], + "matchAnyTags": ["@self-hosted", "@pwa"] + } + ] + } +} diff --git a/tools/coverage/e2e-semantic-summary.mjs b/tools/coverage/e2e-semantic-summary.mjs new file mode 100644 index 000000000..cac4b1dd0 --- /dev/null +++ b/tools/coverage/e2e-semantic-summary.mjs @@ -0,0 +1,245 @@ +#!/usr/bin/env node + +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + statSync, + writeFileSync, +} from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; + +const workspaceRoot = process.cwd(); +const args = process.argv.slice(2); +const projectArg = valueFor('--project'); +const inputArg = valueFor('--input'); +const policy = JSON.parse( + readFileSync(path.join(workspaceRoot, 'tools/coverage/coverage-policy.json'), 'utf8') +); +const outputDir = path.join(workspaceRoot, policy.reporting.e2eSummaryDir); + +function valueFor(flag) { + const prefixed = args.find((arg) => arg.startsWith(`${flag}=`)); + if (prefixed) { + return prefixed.slice(flag.length + 1); + } + const index = args.indexOf(flag); + return index >= 0 ? args[index + 1] : undefined; +} + +function listFiles(directory, predicate) { + if (!existsSync(directory)) { + return []; + } + const files = []; + for (const entry of readdirSync(directory)) { + const fullPath = path.join(directory, entry); + const stats = statSync(fullPath); + if (stats.isDirectory()) { + files.push(...listFiles(fullPath, predicate)); + } else if (predicate(fullPath)) { + files.push(fullPath); + } + } + return files; +} + +function normalizeTag(tag) { + const normalized = tag.startsWith('@') ? tag : `@${tag}`; + return normalized.toLowerCase(); +} + +function tagsFromTitle(title) { + return Array.from( + new Set((title.match(/@[a-z0-9-]+/gi) ?? []).map(normalizeTag).sort()) + ); +} + +function collectFromPlaywrightJson(filePath, projectName) { + const report = JSON.parse(readFileSync(filePath, 'utf8')); + const tests = []; + + function walkSuite(suite, inheritedFile) { + const file = suite.file ?? inheritedFile; + for (const spec of suite.specs ?? []) { + const title = [...(spec.titlePath ?? []), spec.title].filter(Boolean).join(' '); + const tags = new Set([ + ...(spec.tags ?? []).map(normalizeTag), + ...tagsFromTitle(title), + ]); + const statuses = (spec.tests ?? []).flatMap((test) => + (test.results ?? []).map((result) => result.status) + ); + const status = statuses.includes('failed') + ? 'failed' + : statuses.includes('timedOut') + ? 'failed' + : statuses.includes('skipped') + ? 'skipped' + : statuses.length > 1 && statuses.includes('passed') + ? 'flaky' + : statuses[0] ?? 'unknown'; + + tests.push({ + project: projectName, + file, + title, + status, + tags: Array.from(tags).sort(), + }); + } + + for (const child of suite.suites ?? []) { + walkSuite(child, file); + } + } + + for (const suite of report.suites ?? []) { + walkSuite(suite, suite.file); + } + + return tests; +} + +function collectFromSource(projectName) { + const root = + projectName === 'electron-backend-e2e' + ? 'apps/electron-backend-e2e/src' + : 'apps/web-e2e/src'; + const files = listFiles(path.join(workspaceRoot, root), (file) => file.endsWith('.e2e.ts')); + const tests = []; + + for (const file of files) { + const relativeFile = path.relative(workspaceRoot, file); + const contents = readFileSync(file, 'utf8'); + const regex = /\btest(?:\.describe)?\s*\(\s*(['"`])([\s\S]*?)\1/g; + for (const match of contents.matchAll(regex)) { + const title = match[2].replace(/\s+/g, ' ').trim(); + const tags = tagsFromTitle(title); + if (tags.length > 0) { + tests.push({ + project: projectName, + file: relativeFile, + title, + status: 'not-run', + tags, + }); + } + } + } + + return tests; +} + +function defaultInputFor(projectName) { + return path.join(workspaceRoot, 'dist/test-results', projectName, 'results.json'); +} + +function collectTests(projectName) { + const inputPath = inputArg + ? path.resolve(workspaceRoot, inputArg) + : defaultInputFor(projectName); + + if (existsSync(inputPath)) { + return collectFromPlaywrightJson(inputPath, projectName); + } + + return collectFromSource(projectName); +} + +function statusCounts(tests) { + return tests.reduce((counts, test) => { + counts[test.status] = (counts[test.status] ?? 0) + 1; + return counts; + }, {}); +} + +function tagCounts(tests) { + const counts = {}; + for (const test of tests) { + for (const tag of test.tags) { + counts[tag] = (counts[tag] ?? 0) + 1; + } + } + return counts; +} + +function journeyMatches(journey, tests) { + return tests.filter( + (test) => + journey.projects.includes(test.project) && + journey.matchAnyTags.some((tag) => test.tags.includes(tag)) + ); +} + +function markdownFor(projectName, tests) { + const counts = statusCounts(tests); + const countsText = Object.entries(counts) + .map(([status, count]) => `${status}: ${count}`) + .join(', '); + const tagRows = Object.entries(tagCounts(tests)) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([tag, count]) => `| ${tag} | ${count} |`) + .join('\n'); + const journeys = policy.e2eSemanticCoverage.criticalJourneys + .filter((journey) => !projectName || journey.projects.includes(projectName)) + .map((journey) => { + const matches = journeyMatches(journey, tests); + const failed = matches.some((test) => test.status === 'failed'); + const status = matches.length === 0 ? 'missing' : failed ? 'failing' : 'covered'; + return `| ${journey.name} | ${journey.matchAnyTags.join(', ')} | ${matches.length} | ${status} |`; + }) + .join('\n'); + + return `# E2E Semantic Coverage${projectName ? `: ${projectName}` : ''} + +Source: ${tests.some((test) => test.status === 'not-run') ? 'spec source scan' : 'Playwright JSON report'} + +Total tracked tests: ${tests.length} + +Statuses: ${countsText || 'none'} + +## Tags + +| Tag | Tests | +| --- | ---: | +${tagRows || '| _none_ | 0 |'} + +## Critical Journeys + +| Journey | Matching tags | Tests | Status | +| --- | --- | ---: | --- | +${journeys || '| _none_ | _n/a_ | 0 | missing |'} +`; +} + +const projects = projectArg ? [projectArg] : ['web-e2e', 'electron-backend-e2e']; +const allTests = projects.flatMap((projectName) => collectTests(projectName)); + +mkdirSync(outputDir, { recursive: true }); + +if (projectArg) { + const content = markdownFor(projectArg, allTests); + writeFileSync(path.join(outputDir, `${projectArg}-semantic-summary.md`), content); + writeFileSync( + path.join(outputDir, `${projectArg}-semantic-summary.json`), + `${JSON.stringify(allTests, null, 4)}\n` + ); + if (process.env.GITHUB_STEP_SUMMARY) { + writeFileSync(process.env.GITHUB_STEP_SUMMARY, `\n${content}\n`, { flag: 'a' }); + } + console.log(`Wrote ${policy.reporting.e2eSummaryDir}/${projectArg}-semantic-summary.md`); +} else { + const content = markdownFor(undefined, allTests); + writeFileSync(path.join(outputDir, 'semantic-summary.md'), content); + writeFileSync( + path.join(outputDir, 'semantic-summary.json'), + `${JSON.stringify(allTests, null, 4)}\n` + ); + if (process.env.GITHUB_STEP_SUMMARY) { + writeFileSync(process.env.GITHUB_STEP_SUMMARY, `\n${content}\n`, { flag: 'a' }); + } + console.log(`Wrote ${policy.reporting.e2eSummaryDir}/semantic-summary.md`); +} diff --git a/tools/coverage/e2e-v8-web.mjs b/tools/coverage/e2e-v8-web.mjs new file mode 100644 index 000000000..4a23c49c4 --- /dev/null +++ b/tools/coverage/e2e-v8-web.mjs @@ -0,0 +1,153 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + statSync, + writeFileSync, +} from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; + +console.warn( + 'Experimental: browser V8 coverage for existing Playwright specs requires Chromium and is intended for local investigation, not CI gating.' +); + +const workspaceRoot = process.cwd(); + +const result = spawnSync( + 'pnpm', + [ + 'nx', + 'run', + 'web-e2e:e2e', + '--', + '--project=chromium', + ], + { + cwd: process.cwd(), + env: { + ...process.env, + IPTVNATOR_E2E_V8_COVERAGE: '1', + }, + stdio: 'inherit', + } +); + +if (result.status !== 0) { + process.exit(result.status ?? 1); +} + +function listFiles(directory, predicate) { + if (!existsSync(directory)) { + return []; + } + + const files = []; + for (const entry of readdirSync(directory)) { + const fullPath = path.join(directory, entry); + const stats = statSync(fullPath); + if (stats.isDirectory()) { + files.push(...listFiles(fullPath, predicate)); + } else if (predicate(fullPath)) { + files.push(fullPath); + } + } + return files; +} + +function mergeRanges(ranges) { + const sorted = ranges + .filter((range) => range.count > 0) + .map((range) => [range.startOffset, range.endOffset]) + .sort(([leftStart], [rightStart]) => leftStart - rightStart); + const merged = []; + + for (const [start, end] of sorted) { + const previous = merged.at(-1); + if (previous && start <= previous[1]) { + previous[1] = Math.max(previous[1], end); + } else { + merged.push([start, end]); + } + } + + return merged; +} + +function summarizeJsEntry(entry) { + const sourceLength = entry.source?.length ?? 0; + const ranges = mergeRanges((entry.functions ?? []).flatMap((fn) => fn.ranges ?? [])); + const usedBytes = ranges.reduce((total, [start, end]) => total + Math.max(0, end - start), 0); + return { + url: entry.url, + sourceLength, + usedBytes, + pct: sourceLength > 0 ? Math.round((usedBytes / sourceLength) * 10000) / 100 : 0, + }; +} + +const rawCoverageRoot = path.join(workspaceRoot, 'apps/web-e2e/test-results'); +const rawFiles = listFiles(rawCoverageRoot, (file) => + file.endsWith('.json') && file.includes(`${path.sep}v8-coverage${path.sep}`) +); +const entriesByUrl = new Map(); + +for (const file of rawFiles) { + const raw = JSON.parse(readFileSync(file, 'utf8')); + for (const entry of raw.js ?? []) { + if (!entry.url || entry.url.startsWith('extensions::')) { + continue; + } + const summary = summarizeJsEntry(entry); + const existing = entriesByUrl.get(summary.url); + if (!existing || summary.usedBytes > existing.usedBytes) { + entriesByUrl.set(summary.url, summary); + } + } +} + +const summaries = Array.from(entriesByUrl.values()).sort((left, right) => + left.url.localeCompare(right.url) +); +const outputDir = path.join(workspaceRoot, 'coverage/e2e-v8/web'); +mkdirSync(outputDir, { recursive: true }); +writeFileSync( + path.join(outputDir, 'summary.json'), + `${JSON.stringify( + { + rawCoverageRoot: path.relative(workspaceRoot, rawCoverageRoot), + rawFiles: rawFiles.length, + scripts: summaries, + }, + null, + 4 + )}\n` +); +writeFileSync( + path.join(outputDir, 'summary.md'), + `# Web E2E V8 Coverage + +Raw coverage root: ${path.relative(workspaceRoot, rawCoverageRoot)} + +Raw coverage files: ${rawFiles.length} + +This experimental report is Chromium/V8 byte coverage from Playwright's browser +coverage API. Use it for local investigation; rely on Tier A unit coverage and +E2E semantic coverage for CI gates. + +| Script URL | Used bytes | Source bytes | Byte coverage | +| --- | ---: | ---: | ---: | +${summaries + .map( + (entry) => + `| ${entry.url || '_anonymous_'} | ${entry.usedBytes} | ${entry.sourceLength} | ${entry.pct}% |` + ) + .join('\n') || '| _none_ | 0 | 0 | 0% |'} +` +); + +console.log(`Wrote coverage/e2e-v8/web/summary.md from ${rawFiles.length} raw coverage files.`); diff --git a/tools/coverage/merge-coverage.mjs b/tools/coverage/merge-coverage.mjs new file mode 100644 index 000000000..36b006dfb --- /dev/null +++ b/tools/coverage/merge-coverage.mjs @@ -0,0 +1,62 @@ +#!/usr/bin/env node + +import { createRequire } from 'node:module'; +import { + existsSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; + +const require = createRequire(import.meta.url); +const { createCoverageMap } = require('istanbul-lib-coverage'); +const libReport = require('istanbul-lib-report'); +const reports = require('istanbul-reports'); + +const workspaceRoot = process.cwd(); +const policy = JSON.parse( + readFileSync(path.join(workspaceRoot, 'tools/coverage/coverage-policy.json'), 'utf8') +); +const outputDir = path.join(workspaceRoot, policy.reporting.mergedCoverageDir); + +const coverageFiles = policy.unitCoverage.tierA + .map((project) => path.join(workspaceRoot, 'coverage', project.root, 'coverage-final.json')) + .filter((coverageFile) => existsSync(coverageFile)); + +if (coverageFiles.length === 0) { + console.error('No Tier A coverage-final.json files found under coverage/.'); + process.exit(1); +} + +rmSync(outputDir, { recursive: true, force: true }); +mkdirSync(outputDir, { recursive: true }); + +const coverageMap = createCoverageMap({}); + +for (const coverageFile of coverageFiles) { + const data = JSON.parse(readFileSync(coverageFile, 'utf8')); + coverageMap.merge(data); +} + +const context = libReport.createContext({ + dir: outputDir, + coverageMap, +}); + +for (const reporter of ['json', 'json-summary', 'lcovonly', 'cobertura', 'html', 'text-summary']) { + reports.create(reporter).execute(context); +} + +const summary = coverageMap.getCoverageSummary().toJSON(); +writeFileSync( + path.join(outputDir, 'coverage-summary.pretty.json'), + `${JSON.stringify(summary, null, 4)}\n` +); + +console.log(`Merged ${coverageFiles.length} coverage files into ${policy.reporting.mergedCoverageDir}`); +console.log( + `Statements: ${summary.statements.pct}% | Branches: ${summary.branches.pct}% | Functions: ${summary.functions.pct}% | Lines: ${summary.lines.pct}%` +); diff --git a/tools/coverage/run-tier-a-coverage.mjs b/tools/coverage/run-tier-a-coverage.mjs new file mode 100644 index 000000000..ecf553363 --- /dev/null +++ b/tools/coverage/run-tier-a-coverage.mjs @@ -0,0 +1,156 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { existsSync, readFileSync, rmSync } from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; + +const workspaceRoot = process.cwd(); +const policyPath = path.join(workspaceRoot, 'tools/coverage/coverage-policy.json'); +const policy = JSON.parse(readFileSync(policyPath, 'utf8')); +const requestedProjects = new Set( + process.argv + .slice(2) + .flatMap((arg) => + arg.startsWith('--projects=') + ? arg.slice('--projects='.length).split(',') + : [] + ) + .map((project) => project.trim()) + .filter(Boolean) +); + +const tierAProjects = policy.unitCoverage.tierA.filter( + (project) => requestedProjects.size === 0 || requestedProjects.has(project.name) +); + +if (tierAProjects.length === 0) { + console.error('No Tier A coverage projects matched the requested filters.'); + process.exit(1); +} + +function readProjectJson(project) { + const projectJsonPath = path.join(workspaceRoot, project.root, 'project.json'); + if (!existsSync(projectJsonPath)) { + throw new Error(`Missing project.json for ${project.name}: ${projectJsonPath}`); + } + + return JSON.parse(readFileSync(projectJsonPath, 'utf8')); +} + +function toPosix(filePath) { + return filePath.split(path.sep).join('/'); +} + +function coverageDirFor(project, jestRootMode) { + const workspaceCoverageDir = path.join('coverage', project.root); + if (jestRootMode === 'project') { + return toPosix(path.relative(project.root, workspaceCoverageDir)); + } + + return toPosix(workspaceCoverageDir); +} + +function collectCoverageArgs(project, jestRootMode) { + const sourceGlob = + jestRootMode === 'project' + ? 'src/**/*.{ts,js,mjs,html}' + : `${project.sourceRoot}/**/*.{ts,js,mjs,html}`; + const sourcePrefix = + jestRootMode === 'project' ? 'src' : project.sourceRoot; + + return [ + '--coverage', + `--coverageDirectory=${coverageDirFor(project, jestRootMode)}`, + `--collectCoverageFrom=${sourceGlob}`, + `--collectCoverageFrom=!${sourcePrefix}/**/*.{spec,test}.ts`, + `--collectCoverageFrom=!${sourcePrefix}/**/test-setup.ts`, + `--collectCoverageFrom=!${sourcePrefix}/**/test-stubs/**`, + `--collectCoverageFrom=!${sourcePrefix}/**/*.generated.*`, + `--collectCoverageFrom=!${sourcePrefix}/**/environments/**`, + `--collectCoverageFrom=!${sourcePrefix}/**/index.ts`, + ]; +} + +function jestRootModeFor(project) { + const mode = project.jestRootMode ?? 'workspace'; + if (mode !== 'workspace' && mode !== 'project') { + throw new Error( + `Tier A project ${project.name} has unsupported jestRootMode: ${mode}` + ); + } + + return mode; +} + +function buildNxArgs(project) { + const projectJson = readProjectJson(project); + const testTarget = projectJson.targets?.test; + if (!testTarget) { + throw new Error(`Tier A project ${project.name} has no test target.`); + } + + if (testTarget.executor === '@nx/jest:jest') { + return [ + 'nx', + 'run', + `${project.name}:test`, + '--configuration=ci', + '--codeCoverage', + `--coverageDirectory=${coverageDirFor(project, 'workspace')}`, + '--output-style=static', + ]; + } + + if (testTarget.executor === 'nx:run-commands') { + const jestRootMode = jestRootModeFor(project); + + return [ + 'nx', + 'run', + `${project.name}:test`, + '--output-style=static', + '--', + ...collectCoverageArgs(project, jestRootMode), + ]; + } + + throw new Error( + `Tier A project ${project.name} uses unsupported test executor: ${testTarget.executor}` + ); +} + +for (const project of tierAProjects) { + const coverageDir = path.join(workspaceRoot, 'coverage', project.root); + rmSync(coverageDir, { recursive: true, force: true }); +} + +if (requestedProjects.size === 0) { + for (const project of [...policy.unitCoverage.tierB, ...policy.unitCoverage.tierC]) { + if (!project.root) { + continue; + } + const coverageDir = path.join(workspaceRoot, 'coverage', project.root); + rmSync(coverageDir, { recursive: true, force: true }); + } +} + +for (const project of tierAProjects) { + const args = buildNxArgs(project); + console.log(`\n==> Collecting coverage for ${project.name}`); + console.log(`pnpm ${args.join(' ')}`); + + const result = spawnSync('pnpm', args, { + cwd: workspaceRoot, + env: { + ...process.env, + CI: process.env.CI ?? 'true', + NX_TASKS_RUNNER_DYNAMIC_OUTPUT: 'false', + }, + stdio: 'inherit', + }); + + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +}