diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b05476d9c..60e1a14a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -296,8 +296,8 @@ jobs: if: needs.performance-journeys-scope.outputs.run == 'true' runs-on: ubuntu-latest # The electron-performance build is the bulk of the time; each journey - # (launch, open-source) is six fresh Electron processes plus one - # seeding run. + # (launch, open-source, playback) is six fresh Electron processes plus + # one seeding run. timeout-minutes: 30 # Warn-only for the first two weeks of plan item B3: a failure is # visible on the run but does not fail the workflow. diff --git a/README.md b/README.md index 25bcbb9d8..ae7f09d20 100644 --- a/README.md +++ b/README.md @@ -388,8 +388,9 @@ $ pnpm run perf:initial-bytes The contract behind that number is in [docs/architecture/performance-journeys.md](docs/architecture/performance-journeys.md). -To benchmark the "launch to usable" journey (fresh Electron process on a -seeded profile, exact renderer counters plus wall-clock), run: +To benchmark the "launch to usable", "open a source" and "start playback" +journeys (fresh Electron processes on a seeded profile against the local +Xtream mock, exact renderer counters plus wall-clock), run: ``` $ pnpm run perf:journeys diff --git a/apps/electron-backend-e2e/src/journeys/launch-journey-app.ts b/apps/electron-backend-e2e/src/journeys/launch-journey-app.ts index b69633e70..10f1bcb12 100644 --- a/apps/electron-backend-e2e/src/journeys/launch-journey-app.ts +++ b/apps/electron-backend-e2e/src/journeys/launch-journey-app.ts @@ -85,12 +85,21 @@ function removeDirectory(directory: string): Promise { }); } +/** What a journey changes in the seeded profile; J1 and J2 use neither. */ +export interface LaunchJourneySeedOptions { + /** Portal credentials; default: the mock's default account. */ + readonly portal?: Parameters[1]; + /** Runs after both sources are imported, e.g. to change settings. */ + readonly configure?: (page: Page) => Promise; +} + /** * Seeds one M3U source and one Xtream portal through the app's own dialogs * and returns the data directory to copy for every measured launch. */ export async function seedLaunchJourneyProfile( - mockOrigin: string + mockOrigin: string, + options: LaunchJourneySeedOptions = {} ): Promise { const templateDirectory = await mkdtemp( join(tmpdir(), 'iptvnator-journey-launch-seed-') @@ -103,8 +112,12 @@ export async function seedLaunchJourneyProfile( `${mockOrigin}/playlist.m3u` ); await waitForM3uCatalog(app.mainWindow); - await addXtreamPortal(app.mainWindow, { serverUrl: mockOrigin }); + await addXtreamPortal(app.mainWindow, { + ...options.portal, + serverUrl: mockOrigin, + }); await waitForXtreamCatalog(app.mainWindow); + await options.configure?.(app.mainWindow); } finally { await closeElectronAppAndConfirmExit(app); } diff --git a/apps/electron-backend-e2e/src/journeys/open-source-journey-app.ts b/apps/electron-backend-e2e/src/journeys/open-source-journey-app.ts index 15a7295ed..45c9cc074 100644 --- a/apps/electron-backend-e2e/src/journeys/open-source-journey-app.ts +++ b/apps/electron-backend-e2e/src/journeys/open-source-journey-app.ts @@ -1,161 +1,44 @@ -import type { ElectronApplication, Page } from '@playwright/test'; - import { defaultXtreamPortalName } from '../electron-test-fixtures'; import { - countJourneyMainIpcInFlight, + JOURNEY_CLICK_QUIET_MS, + waitForJourneyClickQuiet, + waitForJourneyMockQuiet, +} from '../performance/journey-click-settle'; +import { detachJourneyMainIpcCapture, installJourneyMainIpcCapture, JOURNEY_MAIN_IPC_STATE_KEY, JOURNEY_RENDERER_API_TRACE_CHANNEL, - peekJourneyMainIpcCaptures, readJourneyMainIpcCapture, } from '../performance/journey-main-ipc-capture'; import type { JourneyMockRequestLedger } from '../performance/journey-mock-request-ledger'; -import { waitForJourneyQuiet } from '../performance/journey-quiet-wait'; import { armJourneyRendererProbe, createOpenSourceJourneyProbeOptions, waitForJourneyRendererProbe, type JourneyRendererProbeState, } from '../performance/journey-renderer-probe'; -import type { - OpenSourceJourneyMeasurement, - OpenSourceJourneySettle, -} from '../performance/open-source-journey-record'; +import type { OpenSourceJourneyMeasurement } from '../performance/open-source-journey-record'; import type { LaunchJourneySession } from './launch-journey-app'; /** * J2 "Open a source": runs inside a process that J1 has just launched, after * J1's counters are final. The app is first allowed to settle (no DOM - * mutation, bridge call or mock request for `QUIET_MS`), so leftovers of the - * startup are not attributed to the click. Then the Xtream portal card on + * mutation, bridge call or mock request for `JOURNEY_CLICK_QUIET_MS`), so + * leftovers of the startup are not attributed to the click. Then the Xtream portal card on * the dashboard is clicked and the probe, the IPC capture and the mock * request ledger measure until the category list and the first page of * items are painted. */ export const OPEN_SOURCE_JOURNEY_MAIN_IPC_STATE_KEY = '__iptvnatorJourneyOpenSourceMainIpcCapture'; -const QUIET_MS = 1_000; -const POLL_MS = 100; -const SETTLE_TIMEOUT_MS = 30_000; +const ERROR_PREFIX = 'open-source-journey'; /** * After the mock settled, the ledger is watched this much longer before it * is read, so requests that arrive after the accepted quiet sample show up * in `httpRequestsAfterSettledByRoute` instead of vanishing unseen. */ -const LATE_REQUEST_OBSERVATION_MS = QUIET_MS; - -interface ActivitySample { - readonly domMutations: number; - readonly httpInFlight: number; - readonly httpRequests: number; - readonly ipcCalls: number; - readonly ipcInFlight: number; -} - -async function readPreStartMutations( - page: Page, - stateKey: string -): Promise { - return page.evaluate((key) => { - const state = (globalThis as unknown as Record)[ - key - ] as { preStart?: { domMutations?: number } } | undefined; - const count = state?.preStart?.domMutations; - if (typeof count !== 'number') { - throw new Error('journey-renderer-probe-not-armed'); - } - return count; - }, stateKey); -} - -/** - * Waits until DOM, bridge and mock traffic have all been unchanged for - * `QUIET_MS` with no mock request and no bridge call in flight: a slow - * response or a pending bridge call can still change the DOM or trigger - * follow-up work after the click. Pending bridge calls come from J1's - * capture, which was installed before the document loaded and so has seen - * every call start. An app that never settles fails the iteration instead - * of producing a count that includes its background work. - */ -async function waitForQuiet( - electronApp: ElectronApplication, - page: Page, - ledger: JourneyMockRequestLedger, - probeStateKey: string -): Promise<{ - /** Ledger position read by the accepted quiet sample itself. */ - readonly ledgerMark: number; - readonly settle: OpenSourceJourneySettle; -}> { - const armMark = ledger.mark(); - const sample = async (): Promise => { - // Both captures in one snapshot: a call counted by J2's capture is - // then also pending in J1's, never counted with a stale in-flight 0. - const [launchCapture, openSourceCapture] = - await peekJourneyMainIpcCaptures(electronApp, [ - JOURNEY_MAIN_IPC_STATE_KEY, - OPEN_SOURCE_JOURNEY_MAIN_IPC_STATE_KEY, - ]); - if (launchCapture.unmatchedCompletions > 0) { - throw new Error('open-source-journey-bridge-completions-unmatched'); - } - return { - domMutations: await readPreStartMutations(page, probeStateKey), - httpInFlight: ledger.inFlight(), - httpRequests: ledger.mark(), - ipcCalls: openSourceCapture.callsBeforeStart, - ipcInFlight: countJourneyMainIpcInFlight(launchCapture), - }; - }; - const { sample: quiet, waitedMs } = await waitForJourneyQuiet({ - inFlight: (activity) => activity.httpInFlight + activity.ipcInFlight, - pollMs: POLL_MS, - quietMs: QUIET_MS, - sample, - timeoutError: (activity) => - new Error( - `open-source-journey-not-quiet: ${JSON.stringify(activity)}` - ), - timeoutMs: SETTLE_TIMEOUT_MS, - }); - return { - ledgerMark: quiet.httpRequests, - settle: { - preStartDomMutations: quiet.domMutations, - preStartHttpRequests: quiet.httpRequests - armMark, - preStartIpcCalls: quiet.ipcCalls, - quietMs: QUIET_MS, - waitedMs, - }, - }; -} - -/** - * Waits until the mock has seen no new request for `QUIET_MS` and none is - * in flight, so responses slower than the quiet interval and the requests - * they trigger stay inside the measured window. - */ -async function waitForMockQuiet( - ledger: JourneyMockRequestLedger -): Promise { - const { sample: quiet } = await waitForJourneyQuiet({ - inFlight: (activity) => activity.inFlight, - pollMs: POLL_MS, - quietMs: QUIET_MS, - sample: async () => ({ - inFlight: ledger.inFlight(), - requests: ledger.mark(), - }), - timeoutError: () => new Error('open-source-journey-mock-not-quiet'), - timeoutMs: SETTLE_TIMEOUT_MS, - }); - // The window ends at the ledger position this accepted sample read. A - // request that arrives after it was never seen in flight, so its - // response and follow-ups are not waited for; counting it would make - // the counter depend on when the ledger is read. - return quiet.requests; -} +const LATE_REQUEST_OBSERVATION_MS = JOURNEY_CLICK_QUIET_MS; /** * `spawnLedgerMark` is the ledger position taken before the process was @@ -194,12 +77,12 @@ export async function measureOpenSourceJourney( // reading the IPC capture. Requests from that position on but before // the renderer's click stamp arrived after the app settled, and the // record rejects such an iteration. - const { ledgerMark: settledLedgerMark, settle } = await waitForQuiet( - electronApp, - mainWindow, - ledger, - probeOptions.stateKey - ); + const { ledgerMark: settledLedgerMark, settle } = + await waitForJourneyClickQuiet(electronApp, mainWindow, ledger, { + errorPrefix: ERROR_PREFIX, + ipcCaptureStateKey: OPEN_SOURCE_JOURNEY_MAIN_IPC_STATE_KEY, + probeStateKey: probeOptions.stateKey, + }); // J1's capture was only needed to see pending launch calls while // settling; detached, it no longer runs for every J2 bridge call. await detachJourneyMainIpcCapture(electronApp, JOURNEY_MAIN_IPC_STATE_KEY); @@ -215,7 +98,10 @@ export async function measureOpenSourceJourney( OPEN_SOURCE_JOURNEY_MAIN_IPC_STATE_KEY, 10_000 ); - const settledAfterLedgerMark = await waitForMockQuiet(ledger); + const settledAfterLedgerMark = await waitForJourneyMockQuiet( + ledger, + ERROR_PREFIX + ); await new Promise((resolve) => setTimeout(resolve, LATE_REQUEST_OBSERVATION_MS) ); diff --git a/apps/electron-backend-e2e/src/journeys/playback-journey-app.ts b/apps/electron-backend-e2e/src/journeys/playback-journey-app.ts new file mode 100644 index 000000000..1068f9f83 --- /dev/null +++ b/apps/electron-backend-e2e/src/journeys/playback-journey-app.ts @@ -0,0 +1,224 @@ +import type { ElectronApplication, Page } from '@playwright/test'; + +import { configureLiveFormat } from '../xtream-live-format.fixture'; +import { + JOURNEY_CLICK_QUIET_MS, + waitForJourneyClickQuiet, +} from '../performance/journey-click-settle'; +import { + detachJourneyMainIpcCapture, + installJourneyMainIpcCapture, + JOURNEY_MAIN_IPC_STATE_KEY, + JOURNEY_RENDERER_API_TRACE_CHANNEL, + readJourneyMainIpcCapture, +} from '../performance/journey-main-ipc-capture'; +import type { JourneyMockRequestLedger } from '../performance/journey-mock-request-ledger'; +import { + armJourneyRendererProbe, + createPlaybackJourneyProbeOptions, + JOURNEY_OPEN_SOURCE_START_SELECTOR, + waitForJourneyRendererProbe, +} from '../performance/journey-renderer-probe'; +import { + PLAYBACK_JOURNEY_AFTER_PLAYING_WINDOW_MS, + type PlaybackJourneyMeasurement, +} from '../performance/playback-journey-record'; +import type { + LaunchJourneySeedOptions, + LaunchJourneySession, +} from './launch-journey-app'; + +/** + * J3 "Playback": runs inside a process that J1 has just launched. The test + * opens the Xtream portal's live section and its first category (not + * measured), lets the app settle like J2, then clicks the first live channel + * and measures until the HTML5 player's video element fires `playing`. + * + * The portal is the mock's `live-fallback` account, whose `.ts` live URLs + * serve a local six-second H.264 baseline + AAC MPEG-TS fixture + * (`apps/xtream-mock-server/src/fixtures/live.mpegts`). The HTML5 player + * plays it through mpegts.js and Media Source Extensions, which Electron's + * Chromium supports on every platform. The marketing accounts' live URLs + * serve zero-filled bytes that no player can decode, and the other accounts + * redirect to a public HLS stream. Contract: + * docs/architecture/performance-journeys.md. + */ +export const PLAYBACK_JOURNEY_MAIN_IPC_STATE_KEY = + '__iptvnatorJourneyPlaybackMainIpcCapture'; +export const PLAYBACK_JOURNEY_PORTAL_NAME = 'Journey live portal'; +const ERROR_PREFIX = 'playback-journey'; +const EXTERNAL_ARTWORK_STATE_KEY = '__iptvnatorJourneyExternalArtwork'; +/** + * The generated live catalog's channel and category logos point at + * picsum.photos. They are cancelled in the main process, so no request of + * the journey leaves the machine and a logo never loads, or fails, at a + * different moment on a runner with a different network. + */ +const EXTERNAL_ARTWORK_URLS = ['*://picsum.photos/*', '*://*.picsum.photos/*']; + +/** Seeds J2's profile with the local-media portal and the HTML5 player. */ +export const PLAYBACK_JOURNEY_SEED: LaunchJourneySeedOptions = { + // The built-in HTML5 player with the `ts` stream format: live URLs end + // in `.ts`, which the mock serves from the local fixture. + configure: (page) => configureLiveFormat(page, 'html5', 'ts'), + portal: { + name: PLAYBACK_JOURNEY_PORTAL_NAME, + password: 'live-fallback', + username: 'live-fallback', + }, +}; + +async function blockExternalArtwork( + electronApp: ElectronApplication +): Promise { + await electronApp.evaluate( + ({ session }, input) => { + const target = globalThis as unknown as Record; + if (target[input.key] !== undefined) { + throw new Error('playback-journey-artwork-block-installed'); + } + const state = { cancelled: 0 }; + target[input.key] = state; + // The app registers no onBeforeRequest listener of its own + // (only onBeforeSendHeaders), so this replaces nothing. + session.defaultSession.webRequest.onBeforeRequest( + { urls: input.urls }, + (_details, callback) => { + state.cancelled += 1; + callback({ cancel: true }); + } + ); + }, + { key: EXTERNAL_ARTWORK_STATE_KEY, urls: EXTERNAL_ARTWORK_URLS } + ); +} + +async function readCancelledExternalArtwork( + electronApp: ElectronApplication +): Promise { + return electronApp.evaluate( + (_electron, key) => + ( + (globalThis as unknown as Record)[key] as { + cancelled: number; + } + ).cancelled, + EXTERNAL_ARTWORK_STATE_KEY + ); +} + +/** Dashboard card → live section → first category, as a user would. */ +async function openLiveCategory(page: Page, timeoutMs: number): Promise { + await page + .locator(JOURNEY_OPEN_SOURCE_START_SELECTOR) + .filter({ hasText: PLAYBACK_JOURNEY_PORTAL_NAME }) + .first() + .click({ timeout: timeoutMs }); + await page.waitForURL(/\/workspace\/xtreams\/[^/]+\/vod/, { + timeout: timeoutMs, + }); + await page + .getByRole('link', { name: 'Live TV', exact: true }) + .click({ timeout: timeoutMs }); + await page + .locator('app-workspace-context-panel .category-item') + .first() + .click({ timeout: timeoutMs }); +} + +/** + * `spawnLedgerMark` is the ledger position taken before the process was + * spawned, so the launch's and the navigation's mock traffic is kept as + * evidence. + */ +export async function measurePlaybackJourney( + session: LaunchJourneySession, + ledger: JourneyMockRequestLedger, + spawnLedgerMark: number, + timeoutMs: number +): Promise { + const { electronApp, mainWindow } = session; + const probeOptions = createPlaybackJourneyProbeOptions(); + const startClick = probeOptions.startClick; + if (!startClick) { + throw new Error('playback-journey-probe-without-start'); + } + await blockExternalArtwork(electronApp); + await openLiveCategory(mainWindow, timeoutMs); + const channel = mainWindow.locator(startClick.selector).first(); + await channel.waitFor({ state: 'visible', timeout: timeoutMs }); + await installJourneyMainIpcCapture(electronApp, { + channel: JOURNEY_RENDERER_API_TRACE_CHANNEL, + sentinelId: probeOptions.sentinelId, + sentinelMethod: probeOptions.sentinelMethod, + startSentinelId: startClick.sentinelId, + stateKey: PLAYBACK_JOURNEY_MAIN_IPC_STATE_KEY, + }); + await armJourneyRendererProbe(mainWindow, probeOptions); + // Hover first so hover effects happen before the app settles. + await channel.hover({ timeout: timeoutMs }); + const { ledgerMark: settledLedgerMark, settle } = + await waitForJourneyClickQuiet(electronApp, mainWindow, ledger, { + errorPrefix: ERROR_PREFIX, + ipcCaptureStateKey: PLAYBACK_JOURNEY_MAIN_IPC_STATE_KEY, + probeStateKey: probeOptions.stateKey, + }); + await detachJourneyMainIpcCapture(electronApp, JOURNEY_MAIN_IPC_STATE_KEY); + await channel.click({ timeout: timeoutMs }); + const renderer = await waitForJourneyRendererProbe( + mainWindow, + probeOptions.stateKey, + timeoutMs + ); + const ipc = await readJourneyMainIpcCapture( + electronApp, + PLAYBACK_JOURNEY_MAIN_IPC_STATE_KEY, + 10_000 + ); + const clickEpochMs = renderer.start?.epochMs; + const playingEpochMs = renderer.terminal?.epochMs; + if (clickEpochMs === undefined || playingEpochMs === undefined) { + throw new Error('playback-journey-probe-incomplete'); + } + // A live stream never leaves the mock quiet, so instead of J2's quiet + // wait the ledger is watched for a fixed window after `playing`. + const afterPlayingUntilEpochMs = + playingEpochMs + PLAYBACK_JOURNEY_AFTER_PLAYING_WINDOW_MS; + const remainingMs = + afterPlayingUntilEpochMs - (performance.timeOrigin + performance.now()); + if (remainingMs > 0) { + await new Promise((resolve) => setTimeout(resolve, remainingMs)); + } + // The ledger stamps arrivals with this process's clock, the probe with + // the renderer's; both read the same host clock (as in J2). The record + // keeps the distance of the nearest request to each boundary, so a + // count that a small clock difference could flip is visible. + const sinceSpawn = ledger.since(spawnLedgerMark); + const beforeClick = sinceSpawn.filter( + (entry) => entry.epochMs < clickEpochMs + ); + return { + externalArtworkCancelled: + await readCancelledExternalArtwork(electronApp), + http: { + afterPlaying: sinceSpawn.filter( + (entry) => + entry.epochMs >= playingEpochMs && + entry.epochMs < afterPlayingUntilEpochMs + ), + afterSettleBeforeClick: beforeClick.filter( + (entry) => entry.sequence >= settledLedgerMark + ).length, + beforeClick, + toPlaying: sinceSpawn.filter( + (entry) => + entry.epochMs >= clickEpochMs && + entry.epochMs < playingEpochMs + ), + }, + ipc, + pid: session.launch.pid, + renderer, + settle, + }; +} diff --git a/apps/electron-backend-e2e/src/journeys/playback.journey.ts b/apps/electron-backend-e2e/src/journeys/playback.journey.ts new file mode 100644 index 000000000..a4c8c42e5 --- /dev/null +++ b/apps/electron-backend-e2e/src/journeys/playback.journey.ts @@ -0,0 +1,91 @@ +import { test } from '@playwright/test'; + +import { startJourneyMockRequestLedger } from '../performance/journey-mock-request-ledger'; +import type { JourneyIterationRecord } from '../performance/journey-summary'; +import { + PLAYBACK_JOURNEY_ID, + PLAYBACK_JOURNEY_UNAVAILABLE_COUNTERS, + toPlaybackIterationRecord, +} from '../performance/playback-journey-record'; +import { + JOURNEY_ITERATION_TIMEOUT_MS, + JOURNEY_MEASURED_ITERATIONS, + JOURNEY_WARMUP_ITERATIONS, + logJourneyIteration, + writeJourneyRunEntry, +} from './journey-run'; +import { + LAUNCH_JOURNEY_MOCK_ORIGIN, + removeLaunchJourneyProfile, + runLaunchJourney, + seedLaunchJourneyProfile, +} from './launch-journey-app'; +import { + measurePlaybackJourney, + PLAYBACK_JOURNEY_SEED, +} from './playback-journey-app'; + +/** + * J3 "Playback": click on a live channel of an Xtream portal until the + * built-in HTML5 player's video element fires `playing`. Every iteration is + * a fresh J1 launch on a copy of the seeded profile (J2's profile with the + * portal on the mock's local-media `live-fallback` account and the HTML5 + * player selected); the click happens after the app has settled in the + * portal's first live category. + * Contract: docs/architecture/performance-journeys.md. + */ +test.describe.configure({ mode: 'serial' }); + +test('J3 start playback', async () => { + const ledger = await startJourneyMockRequestLedger( + LAUNCH_JOURNEY_MOCK_ORIGIN + ); + const iterations: JourneyIterationRecord[] = []; + let electronVersion = 'unknown'; + try { + const templateDirectory = await seedLaunchJourneyProfile( + ledger.origin, + PLAYBACK_JOURNEY_SEED + ); + try { + const total = + JOURNEY_WARMUP_ITERATIONS + JOURNEY_MEASURED_ITERATIONS; + for (let index = 0; index < total; index += 1) { + const warmup = index < JOURNEY_WARMUP_ITERATIONS; + const spawnLedgerMark = ledger.mark(); + const { continuation, launch } = await runLaunchJourney( + templateDirectory, + JOURNEY_ITERATION_TIMEOUT_MS, + // Like J2: no main-process counters or SQL hook. + { mainCounters: false }, + (session) => + measurePlaybackJourney( + session, + ledger, + spawnLedgerMark, + JOURNEY_ITERATION_TIMEOUT_MS + ) + ); + electronVersion = launch.electronVersion; + const record = toPlaybackIterationRecord( + index, + warmup, + continuation + ); + iterations.push(record); + logJourneyIteration(PLAYBACK_JOURNEY_ID, record); + } + } finally { + await removeLaunchJourneyProfile(templateDirectory); + } + } finally { + await ledger.close(); + } + + await writeJourneyRunEntry( + PLAYBACK_JOURNEY_ID, + iterations, + PLAYBACK_JOURNEY_UNAVAILABLE_COUNTERS, + electronVersion + ); +}); diff --git a/apps/electron-backend-e2e/src/performance/journey-click-settle.ts b/apps/electron-backend-e2e/src/performance/journey-click-settle.ts new file mode 100644 index 000000000..2b1f6fa69 --- /dev/null +++ b/apps/electron-backend-e2e/src/performance/journey-click-settle.ts @@ -0,0 +1,179 @@ +import type { ElectronApplication, Page } from '@playwright/test'; + +import { + countJourneyMainIpcInFlight, + JOURNEY_MAIN_IPC_STATE_KEY, + peekJourneyMainIpcCaptures, +} from './journey-main-ipc-capture'; +import type { JourneyMockRequestLedger } from './journey-mock-request-ledger'; +import { waitForJourneyQuiet } from './journey-quiet-wait'; + +/** + * Settling for the click-started journeys (J2 "Open a source", J3 + * "Playback"). Before the click the app must be quiet, so leftovers of the + * startup or of the navigation to the start screen are not attributed to + * the click. Contract: docs/architecture/performance-journeys.md. + */ +export const JOURNEY_CLICK_QUIET_MS = 1_000; +const POLL_MS = 100; +const SETTLE_TIMEOUT_MS = 30_000; + +/** How long the app was left alone before the click, and what it did. */ +export interface JourneyClickSettle { + readonly preStartDomMutations: number; + readonly preStartHttpRequests: number; + readonly preStartIpcCalls: number; + readonly quietMs: number; + readonly waitedMs: number; +} + +export interface JourneyClickSettleOptions { + /** Prefix of the timeout errors, e.g. `open-source-journey`. */ + readonly errorPrefix: string; + /** State key of the journey's own IPC capture (with a start sentinel). */ + readonly ipcCaptureStateKey: string; + readonly probeStateKey: string; +} + +interface ActivitySample { + readonly domMutations: number; + readonly httpInFlight: number; + readonly httpRequests: number; + readonly ipcCalls: number; + readonly ipcInFlight: number; +} + +async function readPreStartMutations( + page: Page, + stateKey: string +): Promise { + return page.evaluate((key) => { + const state = (globalThis as unknown as Record)[ + key + ] as { preStart?: { domMutations?: number } } | undefined; + const count = state?.preStart?.domMutations; + if (typeof count !== 'number') { + throw new Error('journey-renderer-probe-not-armed'); + } + return count; + }, stateKey); +} + +/** + * Waits until DOM, bridge and mock traffic have all been unchanged for + * `JOURNEY_CLICK_QUIET_MS` with no mock request and no bridge call in + * flight: a slow response or a pending bridge call can still change the DOM + * or trigger follow-up work after the click. Pending bridge calls come from + * J1's capture, which was installed before the document loaded and so has + * seen every call start. An app that never settles fails the iteration + * instead of producing a count that includes its background work. + */ +export async function waitForJourneyClickQuiet( + electronApp: ElectronApplication, + page: Page, + ledger: JourneyMockRequestLedger, + options: JourneyClickSettleOptions +): Promise<{ + /** Ledger position read by the accepted quiet sample itself. */ + readonly ledgerMark: number; + readonly settle: JourneyClickSettle; +}> { + const armMark = ledger.mark(); + const sample = async (): Promise => { + // Both captures in one snapshot: a call counted by the journey's + // capture is then also pending in J1's, never counted with a stale + // in-flight 0. + const [launchCapture, journeyCapture] = + await peekJourneyMainIpcCaptures(electronApp, [ + JOURNEY_MAIN_IPC_STATE_KEY, + options.ipcCaptureStateKey, + ]); + if (launchCapture.unmatchedCompletions > 0) { + throw new Error( + `${options.errorPrefix}-bridge-completions-unmatched` + ); + } + return { + domMutations: await readPreStartMutations( + page, + options.probeStateKey + ), + httpInFlight: ledger.inFlight(), + httpRequests: ledger.mark(), + ipcCalls: journeyCapture.callsBeforeStart, + ipcInFlight: countJourneyMainIpcInFlight(launchCapture), + }; + }; + const { sample: quiet, waitedMs } = await waitForJourneyQuiet({ + inFlight: (activity) => activity.httpInFlight + activity.ipcInFlight, + pollMs: POLL_MS, + quietMs: JOURNEY_CLICK_QUIET_MS, + sample, + timeoutError: (activity) => + new Error( + `${options.errorPrefix}-not-quiet: ${JSON.stringify(activity)}` + ), + timeoutMs: SETTLE_TIMEOUT_MS, + }); + return { + ledgerMark: quiet.httpRequests, + settle: { + preStartDomMutations: quiet.domMutations, + preStartHttpRequests: quiet.httpRequests - armMark, + preStartIpcCalls: quiet.ipcCalls, + quietMs: JOURNEY_CLICK_QUIET_MS, + waitedMs, + }, + }; +} + +/** + * Waits until the mock has seen no new request for `JOURNEY_CLICK_QUIET_MS` + * and none is in flight, so responses slower than the quiet interval and + * the requests they trigger stay inside the measured window. + */ +export async function waitForJourneyMockQuiet( + ledger: JourneyMockRequestLedger, + errorPrefix: string +): Promise { + const { sample: quiet } = await waitForJourneyQuiet({ + inFlight: (activity) => activity.inFlight, + pollMs: POLL_MS, + quietMs: JOURNEY_CLICK_QUIET_MS, + sample: async () => ({ + inFlight: ledger.inFlight(), + requests: ledger.mark(), + }), + timeoutError: () => new Error(`${errorPrefix}-mock-not-quiet`), + timeoutMs: SETTLE_TIMEOUT_MS, + }); + // The window ends at the ledger position this accepted sample read. A + // request that arrives after it was never seen in flight, so its + // response and follow-ups are not waited for; counting it would make + // the counter depend on when the ledger is read. + return quiet.requests; +} + +/** + * Rejects an iteration whose activity moved after the settle snapshot but + * before the click (while Playwright ran its actionability checks): it + * could complete after the click and be counted as the journey's. The probe + * and the capture keep counting until the click itself, so they must still + * match the snapshot. Returns the kinds that moved. + */ +export function journeyActivityBeforeClick( + settle: JourneyClickSettle, + observed: { + readonly httpAfterSettleBeforeClick: number; + readonly ipcCallsBeforeStart: number; + readonly preStartDomMutations: number; + } +): string[] { + return [ + observed.preStartDomMutations !== settle.preStartDomMutations + ? 'dom' + : null, + observed.ipcCallsBeforeStart !== settle.preStartIpcCalls ? 'ipc' : null, + observed.httpAfterSettleBeforeClick > 0 ? 'http' : null, + ].filter((kind): kind is string => kind !== null); +} diff --git a/apps/electron-backend-e2e/src/performance/journey-renderer-probe-media.spec.ts b/apps/electron-backend-e2e/src/performance/journey-renderer-probe-media.spec.ts new file mode 100644 index 000000000..857126a03 --- /dev/null +++ b/apps/electron-backend-e2e/src/performance/journey-renderer-probe-media.spec.ts @@ -0,0 +1,233 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { JSDOM } from 'jsdom'; + +import { + assertJourneyRendererProbeState, + createPlaybackJourneyProbeOptions, + JOURNEY_IPC_SENTINEL_ID, + JOURNEY_IPC_SENTINEL_METHOD, + JOURNEY_OPEN_SOURCE_PROBE_STATE_KEY, + JOURNEY_PLAYBACK_END_SENTINEL_ID, + JOURNEY_PLAYBACK_PROBE_STATE_KEY, + JOURNEY_PLAYBACK_START_SENTINEL_ID, + JOURNEY_PROBE_STATE_KEY, + type JourneyRendererProbeOptions, +} from './journey-renderer-probe'; +import { + createFixtureFromDom, + settle, + type FakeObserver, + type Fixture, +} from './journey-renderer-probe.test-helpers'; + +const LIVE_URL = 'http://localhost/workspace/xtreams/playlist-1/live'; + +interface PlaybackFixture extends Fixture { + readonly channel: HTMLElement; + readonly player: HTMLElement; +} + +function createPlaybackFixture( + overrides: Partial = {} +): PlaybackFixture { + const dom = new JSDOM( + ` + +
Channel 1
+ +`, + { pretendToBeVisual: true, runScripts: 'outside-only', url: LIVE_URL } + ); + const fixture = createFixtureFromDom( + dom, + { ...createPlaybackJourneyProbeOptions(), ...overrides }, + true + ); + const { document } = fixture.window; + return { + ...fixture, + channel: document.querySelector( + '[data-test-id="channel-item"]' + ) as HTMLElement, + player: document.querySelector('app-web-player-view') as HTMLElement, + }; +} + +/** What the player does on the click: mount a video element. */ +function mountVideo(fixture: PlaybackFixture): HTMLVideoElement { + const video = fixture.window.document.createElement('video'); + fixture.player.append(video); + return video; +} + +function fire(fixture: Fixture, target: EventTarget, type: string): void { + // Media events do not bubble; the probe must see them in capture. + target.dispatchEvent(new fixture.window.Event(type, { bubbles: false })); +} + +test('playback options start at a live channel row and end on the player video', () => { + const options = createPlaybackJourneyProbeOptions(); + assert.equal(options.journey, 'playback'); + assert.equal(options.stateKey, JOURNEY_PLAYBACK_PROBE_STATE_KEY); + assert.notEqual(options.stateKey, JOURNEY_PROBE_STATE_KEY); + assert.notEqual(options.stateKey, JOURNEY_OPEN_SOURCE_PROBE_STATE_KEY); + assert.equal(options.sentinelId, JOURNEY_PLAYBACK_END_SENTINEL_ID); + assert.notEqual(options.sentinelId, JOURNEY_IPC_SENTINEL_ID); + assert.equal(options.sentinelMethod, JOURNEY_IPC_SENTINEL_METHOD); + assert.equal( + options.startClick?.sentinelId, + JOURNEY_PLAYBACK_START_SENTINEL_ID + ); + assert.match(options.startClick?.selector ?? '', /channel-item/); + assert.equal(options.cardSelector, 'app-web-player-view video'); + assert.deepEqual(options.media, { + endEvent: 'playing', + phaseEvents: ['loadedmetadata'], + }); +}); + +test('ends at the playing event, not when the video becomes visible, and records loadedmetadata on the way', async () => { + const fixture = createPlaybackFixture(); + let video: HTMLVideoElement | null = null; + fixture.channel.addEventListener('click', () => { + video = mountVideo(fixture); + }); + (fixture.channel.querySelector('.name') as HTMLElement).click(); + await settle(); + assert.ok(video, 'the app mounted a video'); + const mounted = video as HTMLVideoElement; + assert.equal( + fixture.state().terminal, + null, + 'a visible video is not enough' + ); + assert.deepEqual(fixture.bridgeCalls, [JOURNEY_PLAYBACK_START_SENTINEL_ID]); + + fire(fixture, mounted, 'loadedmetadata'); + await settle(); + const afterMetadata = fixture.state(); + assert.equal(afterMetadata.terminal, null); + assert.equal( + typeof afterMetadata.media?.phases['loadedmetadata'], + 'number' + ); + + // Mutations queued in the same task as the event still count: the + // probe takes them synchronously at the event. + mounted.setAttribute('data-state', 'playing'); + fire(fixture, mounted, 'playing'); + mounted.setAttribute('data-state', 'after'); + await settle(); + const state = fixture.state(); + assert.deepEqual(fixture.bridgeCalls, [ + JOURNEY_PLAYBACK_START_SENTINEL_ID, + JOURNEY_PLAYBACK_END_SENTINEL_ID, + ]); + assert.ok(state.start && state.terminal && state.media?.element); + assert.equal(state.start.targetTestId, 'channel-item'); + assert.equal(state.terminal.cardTag, 'video'); + assert.equal(state.terminal.cardCount, 1); + assert.deepEqual(state.terminal.companionCounts, []); + assert.equal(state.terminal.pathname, '/workspace/xtreams/playlist-1/live'); + // video mounted (1) + data-state=playing (1); the later change is not. + assert.equal(state.counters.domMutations, 2); + const metadataEpochMs = state.media.phases['loadedmetadata'] as number; + assert.ok(state.start.epochMs <= metadataEpochMs); + assert.ok(metadataEpochMs <= state.terminal.epochMs); + assert.equal(state.media.phases['playing'], state.terminal.epochMs); + assert.equal(state.media.element.paused, true); + assert.equal(state.final, true); + assert.equal(state.settle.status, 'disabled'); + assert.doesNotThrow(() => assertJourneyRendererProbeState(state)); +}); + +test('ignores media events before the click and on other media elements', async () => { + const fixture = createPlaybackFixture(); + const early = mountVideo(fixture); + fire(fixture, early, 'loadedmetadata'); + fire(fixture, early, 'playing'); + await settle(); + assert.equal(fixture.state().start, null); + assert.equal(fixture.state().terminal, null); + + fixture.channel.click(); + const preview = fixture.window.document.createElement('video'); + fixture.window.document.body.append(preview); + fire(fixture, preview, 'playing'); + await settle(); + const state = fixture.state(); + assert.ok(state.start); + assert.equal(state.terminal, null); + assert.deepEqual(state.media?.phases, {}); + assert.deepEqual(fixture.bridgeCalls, [JOURNEY_PLAYBACK_START_SENTINEL_ID]); +}); + +test('counts shifts and long tasks up to the playing event, not to the post-paint cutoff', async () => { + const fixture = createPlaybackFixture(); + const [layoutShift, longTask] = fixture.observers as [ + FakeObserver, + FakeObserver, + ]; + const now = () => fixture.window.performance.now(); + fixture.channel.click(); + const video = mountVideo(fixture); + layoutShift.emit([ + { + entryType: 'layout-shift', + hadRecentInput: true, + startTime: now(), + value: 0.004, + }, + ]); + longTask.emit([{ duration: 80, entryType: 'longtask', startTime: now() }]); + // Leave room between the click and the event for the entries below. + await new Promise((resolve) => setTimeout(resolve, 20)); + fire(fixture, video, 'playing'); + const playingMs = + (fixture.rawState().terminal?.epochMs ?? 0) - + fixture.window.performance.timeOrigin; + // Delivered after the event: one started before it, one after. + layoutShift.emit([ + { + entryType: 'layout-shift', + hadRecentInput: false, + startTime: playingMs - 1, + value: 0.002, + }, + { + entryType: 'layout-shift', + hadRecentInput: false, + startTime: playingMs + 5, + value: 0.5, + }, + ]); + longTask.emit([ + // The task that dispatched `playing` began before it and counts. + { duration: 60, entryType: 'longtask', startTime: playingMs - 10 }, + { duration: 300, entryType: 'longtask', startTime: playingMs + 5 }, + ]); + await settle(); + const state = fixture.state(); + assert.equal(state.final, true); + assert.equal(state.counters.recentInputLayoutShiftScore, 0.004); + assert.equal(state.counters.layoutShiftScore, 0.002); + assert.equal(state.counters.longTasks, 2); + assert.deepEqual(state.longTaskDurationsMs, [80, 60]); + assert.ok((state.firstCardPaintEpochMs ?? 0) >= state.terminal!.epochMs); +}); + +test('a media terminal without a click start is invalid', () => { + const fixture = createPlaybackFixture({ startClick: undefined }); + assert.ok( + fixture + .state() + .invalidReasons.includes('media-terminal-needs-start-click') + ); +}); + +test('other journeys carry no media block', () => { + const fixture = createPlaybackFixture({ media: undefined }); + assert.equal(fixture.state().media, null); +}); diff --git a/apps/electron-backend-e2e/src/performance/journey-renderer-probe.spec.ts b/apps/electron-backend-e2e/src/performance/journey-renderer-probe.spec.ts index 9dfa35301..58e64fb77 100644 --- a/apps/electron-backend-e2e/src/performance/journey-renderer-probe.spec.ts +++ b/apps/electron-backend-e2e/src/performance/journey-renderer-probe.spec.ts @@ -18,38 +18,14 @@ import { JOURNEY_SETTLE_ROOT_SELECTOR, journeyRendererProbeScript, type JourneyRendererProbeOptions, - type JourneyRendererProbeState, } from './journey-renderer-probe'; - -interface FakeEntry { - duration?: number; - entryType: string; - hadRecentInput?: boolean; - sources?: { - currentRect: { height: number; y: number }; - node: unknown; - previousRect: { height: number; y: number }; - }[]; - startTime: number; - value?: number; -} - -interface FakeObserver { - disconnected: boolean; - emit(entries: FakeEntry[]): void; - queue: FakeEntry[]; - type: string | null; -} - -interface Fixture { - readonly bridgeCalls: unknown[]; - readonly observers: FakeObserver[]; - /** The live state object inside the jsdom realm. */ - readonly rawState: () => JourneyRendererProbeState; - /** A JSON clone, so assertions compare values across realms. */ - readonly state: () => JourneyRendererProbeState; - readonly window: JSDOM['window']; -} +import { + createFixtureFromDom, + settle, + type FakeEntry, + type FakeObserver, + type Fixture, +} from './journey-renderer-probe.test-helpers'; const PAGE = `
IPTVnator
@@ -62,55 +38,6 @@ const FAST_SETTLE = { rootSelector: JOURNEY_SETTLE_ROOT_SELECTOR, } as const; -function installFakePerformance( - window: JSDOM['window'], - observers: FakeObserver[] -): void { - class FakePerformanceObserver implements FakeObserver { - disconnected = false; - queue: FakeEntry[] = []; - type: string | null = null; - constructor( - private readonly callback: (list: { - getEntries(): FakeEntry[]; - }) => void - ) { - observers.push(this); - } - observe(options: { type: string }): void { - this.type = options.type; - } - takeRecords(): FakeEntry[] { - const queued = this.queue; - this.queue = []; - return queued; - } - disconnect(): void { - this.disconnected = true; - } - emit(entries: FakeEntry[]): void { - this.callback({ getEntries: () => entries }); - } - } - Object.defineProperty(window, 'PerformanceObserver', { - configurable: true, - value: FakePerformanceObserver, - }); - Object.defineProperty(window.performance, 'getEntriesByType', { - configurable: true, - value: (type: string) => - type === 'navigation' - ? [{ domContentLoadedEventEnd: 100, loadEventEnd: 120 }] - : [], - }); - // jsdom never lays out, so visibility is "connected to the document". - window.HTMLElement.prototype.getClientRects = function getClientRects( - this: HTMLElement - ) { - return (this.isConnected ? [{}] : []) as unknown as DOMRectList; - }; -} - function createFixture( overrides: Partial & { bridge?: boolean; @@ -138,79 +65,6 @@ function createFixture( ); } -function createFixtureFromDom( - dom: JSDOM, - options: JourneyRendererProbeOptions, - bridge: boolean -): Fixture { - const { window } = dom; - const observers: FakeObserver[] = []; - const bridgeCalls: unknown[] = []; - installFakePerformance(window, observers); - // tsx (esbuild keepNames) rewrites named inner functions as - // `__name(fn, 'name')` when it transpiles the probe for this test runner. - // Playwright's Babel transform, which serializes the probe for the real - // browser, does not, so the shim is a test-runner concern only. - Object.defineProperty(window, '__name', { - configurable: true, - value: (target: unknown) => target, - }); - if (bridge) { - Object.defineProperty(window, 'electron', { - configurable: true, - value: Object.freeze({ - [JOURNEY_IPC_SENTINEL_METHOD]: (id: unknown) => { - bridgeCalls.push(id); - return Promise.resolve(null); - }, - onSomething: () => undefined, - }), - }); - } - window.eval( - `(${journeyRendererProbeScript.toString()})(${JSON.stringify(options)})` - ); - const rawState = (): JourneyRendererProbeState => - (window as unknown as Record)[ - options.stateKey - ] as JourneyRendererProbeState; - liveStates.push(rawState); - return { - bridgeCalls, - observers, - rawState, - state: () => - JSON.parse(JSON.stringify(rawState())) as JourneyRendererProbeState, - window, - }; -} - -/** Probes created by this file, so `settle` can wait for their cutoff. */ -const liveStates: (() => JourneyRendererProbeState)[] = []; - -/** - * Waits `ms`, then until every probe that reached its terminal batch has also - * passed the post-paint cutoff (a rAF plus a timer) and closed its settle - * window. A fixed delay alone flakes when the harness runs all spec files in - * parallel. - */ -async function settle(ms = 40): Promise { - await new Promise((resolve) => setTimeout(resolve, ms)); - const deadline = Date.now() + 3_000; - while ( - liveStates.some((read) => { - const state = read(); - return ( - state.terminal !== null && - (!state.final || state.settle.status === 'pending') - ); - }) && - Date.now() < deadline - ) { - await new Promise((resolve) => setTimeout(resolve, 10)); - } -} - function renderFirstCard(fixture: Fixture): void { const { document } = fixture.window; document.getElementById('initial-splash')?.remove(); diff --git a/apps/electron-backend-e2e/src/performance/journey-renderer-probe.test-helpers.ts b/apps/electron-backend-e2e/src/performance/journey-renderer-probe.test-helpers.ts new file mode 100644 index 000000000..d1cb41e7d --- /dev/null +++ b/apps/electron-backend-e2e/src/performance/journey-renderer-probe.test-helpers.ts @@ -0,0 +1,165 @@ +import type { JSDOM } from 'jsdom'; + +import { + JOURNEY_IPC_SENTINEL_METHOD, + journeyRendererProbeScript, + type JourneyRendererProbeOptions, + type JourneyRendererProbeState, +} from './journey-renderer-probe'; + +/** + * jsdom fixtures shared by the renderer probe specs: fake performance + * observers, a frozen bridge that records sentinel calls, and a wait for + * every probe's post-paint cutoff. + */ +export interface FakeEntry { + duration?: number; + entryType: string; + hadRecentInput?: boolean; + sources?: { + currentRect: { height: number; y: number }; + node: unknown; + previousRect: { height: number; y: number }; + }[]; + startTime: number; + value?: number; +} + +export interface FakeObserver { + disconnected: boolean; + emit(entries: FakeEntry[]): void; + queue: FakeEntry[]; + type: string | null; +} + +export interface Fixture { + readonly bridgeCalls: unknown[]; + readonly observers: FakeObserver[]; + /** The live state object inside the jsdom realm. */ + readonly rawState: () => JourneyRendererProbeState; + /** A JSON clone, so assertions compare values across realms. */ + readonly state: () => JourneyRendererProbeState; + readonly window: JSDOM['window']; +} + +export function installFakePerformance( + window: JSDOM['window'], + observers: FakeObserver[] +): void { + class FakePerformanceObserver implements FakeObserver { + disconnected = false; + queue: FakeEntry[] = []; + type: string | null = null; + constructor( + private readonly callback: (list: { + getEntries(): FakeEntry[]; + }) => void + ) { + observers.push(this); + } + observe(options: { type: string }): void { + this.type = options.type; + } + takeRecords(): FakeEntry[] { + const queued = this.queue; + this.queue = []; + return queued; + } + disconnect(): void { + this.disconnected = true; + } + emit(entries: FakeEntry[]): void { + this.callback({ getEntries: () => entries }); + } + } + Object.defineProperty(window, 'PerformanceObserver', { + configurable: true, + value: FakePerformanceObserver, + }); + Object.defineProperty(window.performance, 'getEntriesByType', { + configurable: true, + value: (type: string) => + type === 'navigation' + ? [{ domContentLoadedEventEnd: 100, loadEventEnd: 120 }] + : [], + }); + // jsdom never lays out, so visibility is "connected to the document". + window.HTMLElement.prototype.getClientRects = function getClientRects( + this: HTMLElement + ) { + return (this.isConnected ? [{}] : []) as unknown as DOMRectList; + }; +} + +export function createFixtureFromDom( + dom: JSDOM, + options: JourneyRendererProbeOptions, + bridge: boolean +): Fixture { + const { window } = dom; + const observers: FakeObserver[] = []; + const bridgeCalls: unknown[] = []; + installFakePerformance(window, observers); + // tsx (esbuild keepNames) rewrites named inner functions as + // `__name(fn, 'name')` when it transpiles the probe for this test runner. + // Playwright's Babel transform, which serializes the probe for the real + // browser, does not, so the shim is a test-runner concern only. + Object.defineProperty(window, '__name', { + configurable: true, + value: (target: unknown) => target, + }); + if (bridge) { + Object.defineProperty(window, 'electron', { + configurable: true, + value: Object.freeze({ + [JOURNEY_IPC_SENTINEL_METHOD]: (id: unknown) => { + bridgeCalls.push(id); + return Promise.resolve(null); + }, + onSomething: () => undefined, + }), + }); + } + window.eval( + `(${journeyRendererProbeScript.toString()})(${JSON.stringify(options)})` + ); + const rawState = (): JourneyRendererProbeState => + (window as unknown as Record)[ + options.stateKey + ] as JourneyRendererProbeState; + liveStates.push(rawState); + return { + bridgeCalls, + observers, + rawState, + state: () => + JSON.parse(JSON.stringify(rawState())) as JourneyRendererProbeState, + window, + }; +} + +/** Probes created by this file, so `settle` can wait for their cutoff. */ +const liveStates: (() => JourneyRendererProbeState)[] = []; + +/** + * Waits `ms`, then until every probe that reached its terminal batch has also + * passed the post-paint cutoff (a rAF plus a timer) and closed its settle + * window. A fixed delay alone flakes when the harness runs all spec files in + * parallel. + */ +export async function settle(ms = 40): Promise { + await new Promise((resolve) => setTimeout(resolve, ms)); + const deadline = Date.now() + 3_000; + while ( + liveStates.some((read) => { + const state = read(); + return ( + state.terminal !== null && + (!state.final || state.settle.status === 'pending') + ); + }) && + Date.now() < deadline + ) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} diff --git a/apps/electron-backend-e2e/src/performance/journey-renderer-probe.ts b/apps/electron-backend-e2e/src/performance/journey-renderer-probe.ts index 0326119b0..86af2e664 100644 --- a/apps/electron-backend-e2e/src/performance/journey-renderer-probe.ts +++ b/apps/electron-backend-e2e/src/performance/journey-renderer-probe.ts @@ -15,6 +15,9 @@ import type { Page } from '@playwright/test'; * then emits one JSON blob under `options.stateKey`. With `options.settle` * (J1) it keeps summing layout shifts after the first card until the page * has settled, for shifts such as collapsing skeletons that land later. + * With `options.media` (J3 "Playback") the terminal condition is a media + * event (`playing`) on an element matching `cardSelector` instead of that + * element becoming visible. * * IPC invocations are not counted here: the bridge object exposed by * `contextBridge` is frozen, so the probe cannot wrap it. Instead the probe @@ -33,6 +36,17 @@ export const JOURNEY_OPEN_SOURCE_START_SENTINEL_ID = '__iptvnator-journey-open-source-start__'; export const JOURNEY_OPEN_SOURCE_END_SENTINEL_ID = '__iptvnator-journey-open-source-end__'; +export const JOURNEY_PLAYBACK_PROBE_STATE_KEY = + '__iptvnatorJourneyPlaybackProbe'; +export const JOURNEY_PLAYBACK_START_SENTINEL_ID = + '__iptvnator-journey-playback-start__'; +export const JOURNEY_PLAYBACK_END_SENTINEL_ID = + '__iptvnator-journey-playback-end__'; +/** A live channel row in the Xtream live layout. */ +export const JOURNEY_PLAYBACK_START_SELECTOR = + 'app-live-stream-layout [data-test-id="channel-item"]'; +/** The HTML5 player's video element inside the web player view. */ +export const JOURNEY_PLAYBACK_VIDEO_SELECTOR = 'app-web-player-view video'; /** The Xtream portal card on the dashboard or its row on /workspace/sources. */ export const JOURNEY_OPEN_SOURCE_START_SELECTOR = '[data-test-id="dashboard-recent-sources-rail-card"], app-playlist-item'; @@ -64,6 +78,17 @@ export interface JourneyRendererProbeSettleOptions { readonly rootSelector: string; } +/** + * A journey that ends on a media event rather than on visibility (J3). The + * first `endEvent` after the start on an element matching `cardSelector` + * is the terminal moment; the first of each `phaseEvents` after the start + * is recorded under `media.phases`. + */ +export interface JourneyRendererProbeMediaOptions { + readonly endEvent: string; + readonly phaseEvents: readonly string[]; +} + export const JOURNEY_SETTLE_QUIET_MS = 500; export const JOURNEY_SETTLE_CAP_MS = 3_000; /** The workspace shell's content pane; the rail and header stay outside. */ @@ -75,6 +100,8 @@ export interface JourneyRendererProbeOptions { /** Further selectors that must each match a visible element as well. */ readonly companionSelectors?: readonly string[]; readonly journey: string; + /** Absent: the journey ends when `cardSelector` becomes visible. */ + readonly media?: JourneyRendererProbeMediaOptions; /** Pathname fragment the terminal route must contain. */ readonly routeFragment: string; readonly sentinelId: string; @@ -137,6 +164,20 @@ export interface JourneyRendererProbeState { readonly invalidReasons: string[]; readonly journey: string; readonly longTaskDurationsMs: number[]; + /** Media-terminated journeys only; null otherwise. */ + readonly media: { + /** At the terminal event, the element it fired on. */ + element: { + readonly currentSrcScheme: string; + readonly currentTime: number; + readonly paused: boolean; + readonly readyState: number; + readonly videoHeight: number; + readonly videoWidth: number; + } | null; + /** Event type → epoch of its first occurrence after the start. */ + readonly phases: Record; + } | null; navigation: { readonly domContentLoadedEpochMs: number; readonly loadEventEndEpochMs: number; @@ -197,6 +238,7 @@ export function journeyRendererProbeScript( } const epoch = (): number => performance.timeOrigin + performance.now(); const startClick = options.startClick ?? null; + const mediaOptions = options.media ?? null; const settleOptions = options.settle ?? null; const companionSelectors = options.companionSelectors ?? []; const bridge = target['electron'] as Record | undefined; @@ -228,6 +270,7 @@ export function journeyRendererProbeScript( invalidReasons: [], journey: options.journey, longTaskDurationsMs: [], + media: mediaOptions === null ? null : { element: null, phases: {} }, navigation: null, preStart: { domMutations: 0, lastMutationEpochMs: null }, schemaVersion: 1, @@ -251,6 +294,9 @@ export function journeyRendererProbeScript( ) { state.invalidReasons.push('probe-installed-after-document-start'); } + if (mediaOptions !== null && startClick === null) { + state.invalidReasons.push('media-terminal-needs-start-click'); + } // Performance entries before the journey's start belong to an earlier // journey (buffered entries included) and are dropped. let fromEpochMs = @@ -463,20 +509,27 @@ export function journeyRendererProbeScript( armQuiet(); }; + // A media journey's counters stop at the terminal event itself (the + // task that dispatched it still overlaps and counts); the others count + // until the post-paint cutoff. const finalize = (untilEpochMs: number): void => { + const countUntilEpochMs = + mediaOptions !== null && state.terminal !== null + ? state.terminal.epochMs + : untilEpochMs; if (layoutShiftObserver) { const records = layoutShiftObserver.takeRecords(); collectSettleShifts(records); acceptLayoutShift( [...pendingLayoutShifts, ...records], - untilEpochMs + countUntilEpochMs ); if (settleOptions === null) layoutShiftObserver.disconnect(); } if (longTaskObserver) { acceptLongTasks( [...pendingLongTasks, ...longTaskObserver.takeRecords()], - untilEpochMs + countUntilEpochMs ); longTaskObserver.disconnect(); } @@ -535,6 +588,7 @@ export function journeyRendererProbeScript( } state.counters.domMutations += records.length; if ( + mediaOptions !== null || !location.pathname.includes(options.routeFragment) || document.getElementById(options.splashId) !== null ) { @@ -547,12 +601,19 @@ export function journeyRendererProbeScript( if (!isVisible(document.querySelector(selector))) return; companionCounts.push(document.querySelectorAll(selector).length); } + end(card, companionCounts, epoch()); + }); + const end = ( + card: Element, + companionCounts: number[], + epochMs: number + ): void => { state.terminal = { cardCount: document.querySelectorAll(options.cardSelector).length, cardTag: card.tagName.toLowerCase(), cardTestId: card.getAttribute('data-test-id'), companionCounts, - epochMs: epoch(), + epochMs, pathname: location.pathname, }; mutationObserver.disconnect(); @@ -577,7 +638,7 @@ export function journeyRendererProbeScript( requestAnimationFrame(() => { setTimeout(() => finalize(epoch()), 0); }); - }); + }; mutationObserver.observe(document.documentElement ?? document, { attributes: true, characterData: true, @@ -615,6 +676,43 @@ export function journeyRendererProbeScript( window.removeEventListener('click', onClick, true); }; window.addEventListener('click', onClick, true); + if (mediaOptions === null) return; + + // Media events do not bubble, but a capture listener on window still + // sees them before any listener of the app. Mutations up to the event + // are taken synchronously, so the count ends exactly at the event. + const onMediaEvent = (event: Event): void => { + const element = event.target; + if ( + state.start === null || + state.terminal !== null || + state.media === null || + !(element instanceof HTMLMediaElement) || + !element.matches(options.cardSelector) + ) { + return; + } + const at = epoch(); + state.media.phases[event.type] ??= at; + if (event.type !== mediaOptions.endEvent) return; + state.counters.domMutations += mutationObserver.takeRecords().length; + const video = element as HTMLMediaElement & { + videoHeight?: number; + videoWidth?: number; + }; + state.media.element = { + currentSrcScheme: element.currentSrc.split(':')[0] ?? '', + currentTime: element.currentTime, + paused: element.paused, + readyState: element.readyState, + videoHeight: video.videoHeight ?? 0, + videoWidth: video.videoWidth ?? 0, + }; + end(element, [], at); + }; + for (const type of [mediaOptions.endEvent, ...mediaOptions.phaseEvents]) { + window.addEventListener(type, onMediaEvent, true); + } } export function createLaunchJourneyProbeOptions(): JourneyRendererProbeOptions { @@ -660,6 +758,28 @@ export function createOpenSourceJourneyProbeOptions(): JourneyRendererProbeOptio }; } +/** + * Options for J3 "Playback": the click on a live channel row starts the + * journey; it ends at the first `playing` event of the HTML5 player's video + * element, with `loadedmetadata` recorded as an intermediate phase. + */ +export function createPlaybackJourneyProbeOptions(): JourneyRendererProbeOptions { + return { + cardSelector: JOURNEY_PLAYBACK_VIDEO_SELECTOR, + journey: 'playback', + media: { endEvent: 'playing', phaseEvents: ['loadedmetadata'] }, + routeFragment: '/workspace/xtreams/', + sentinelId: JOURNEY_PLAYBACK_END_SENTINEL_ID, + sentinelMethod: JOURNEY_IPC_SENTINEL_METHOD, + splashId: 'initial-splash', + startClick: { + selector: JOURNEY_PLAYBACK_START_SELECTOR, + sentinelId: JOURNEY_PLAYBACK_START_SENTINEL_ID, + }, + stateKey: JOURNEY_PLAYBACK_PROBE_STATE_KEY, + }; +} + /** * Registers the probe on a page that is still parked on `about:blank` by the * journey gate, so it is guaranteed to run at the start of the next document. diff --git a/apps/electron-backend-e2e/src/performance/launch-journey-record.spec.ts b/apps/electron-backend-e2e/src/performance/launch-journey-record.spec.ts index e88dc2a9b..e2b81858a 100644 --- a/apps/electron-backend-e2e/src/performance/launch-journey-record.spec.ts +++ b/apps/electron-backend-e2e/src/performance/launch-journey-record.spec.ts @@ -38,6 +38,7 @@ function measurement( invalidReasons: [], journey: 'launch', longTaskDurationsMs: [71.26, 120.04], + media: null, navigation: { domContentLoadedEpochMs: 1_300, loadEventEndEpochMs: 1_400.26, diff --git a/apps/electron-backend-e2e/src/performance/open-source-journey-record.spec.ts b/apps/electron-backend-e2e/src/performance/open-source-journey-record.spec.ts index d4067df5c..779eb4178 100644 --- a/apps/electron-backend-e2e/src/performance/open-source-journey-record.spec.ts +++ b/apps/electron-backend-e2e/src/performance/open-source-journey-record.spec.ts @@ -46,6 +46,7 @@ function measurement( invalidReasons: [], journey: 'open-source', longTaskDurationsMs: [61.26], + media: null, navigation: null, preStart: { domMutations: 4, lastMutationEpochMs: 9_100 }, schemaVersion: 1, diff --git a/apps/electron-backend-e2e/src/performance/open-source-journey-record.ts b/apps/electron-backend-e2e/src/performance/open-source-journey-record.ts index 5ff3bd307..3d4241f16 100644 --- a/apps/electron-backend-e2e/src/performance/open-source-journey-record.ts +++ b/apps/electron-backend-e2e/src/performance/open-source-journey-record.ts @@ -1,3 +1,7 @@ +import { + journeyActivityBeforeClick, + type JourneyClickSettle, +} from './journey-click-settle'; import type { JourneyMainIpcCaptureState } from './journey-main-ipc-capture'; import { countJourneyMockRoutes, @@ -38,13 +42,7 @@ export const OPEN_SOURCE_JOURNEY_UNAVAILABLE_COUNTERS: Readonly< }); /** How long the app was left alone before the click, and what it did. */ -export interface OpenSourceJourneySettle { - readonly preStartDomMutations: number; - readonly preStartHttpRequests: number; - readonly preStartIpcCalls: number; - readonly quietMs: number; - readonly waitedMs: number; -} +export type OpenSourceJourneySettle = JourneyClickSettle; export interface OpenSourceJourneyMeasurement { readonly http: { @@ -109,13 +107,11 @@ export function toOpenSourceIterationRecord( // the click and be counted as J2. The probe and the capture keep // counting until the click itself, so they must still match the // snapshot; otherwise the iteration is rejected. - const lateActivity = [ - renderer.preStart.domMutations !== settle.preStartDomMutations - ? 'dom' - : null, - ipc.callsBeforeStart !== settle.preStartIpcCalls ? 'ipc' : null, - http.afterSettleBeforeClick > 0 ? 'http' : null, - ].filter((kind) => kind !== null); + const lateActivity = journeyActivityBeforeClick(settle, { + httpAfterSettleBeforeClick: http.afterSettleBeforeClick, + ipcCallsBeforeStart: ipc.callsBeforeStart, + preStartDomMutations: renderer.preStart.domMutations, + }); if (lateActivity.length > 0) { throw new Error( `open-source-journey-record-activity-before-click-${lateActivity.join('-')}` diff --git a/apps/electron-backend-e2e/src/performance/playback-journey-record.spec.ts b/apps/electron-backend-e2e/src/performance/playback-journey-record.spec.ts new file mode 100644 index 000000000..21bd8df3c --- /dev/null +++ b/apps/electron-backend-e2e/src/performance/playback-journey-record.spec.ts @@ -0,0 +1,344 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import type { JourneyMainIpcCaptureState } from './journey-main-ipc-capture'; +import type { JourneyMockRequest } from './journey-mock-request-ledger'; +import type { JourneyRendererProbeState } from './journey-renderer-probe'; +import { summarizeJourneyIterations } from './journey-summary'; +import { + PLAYBACK_JOURNEY_UNAVAILABLE_COUNTERS, + toPlaybackIterationRecord, + type PlaybackJourneyMeasurement, +} from './playback-journey-record'; + +const LIVE_PATH = '/dist/apps/web/workspace/xtreams/playlist-1/live'; +const STREAM_ROUTE = '/live/:username/:password/10000.ts'; + +function request(sequence: number, route: string, epochMs: number) { + return { epochMs, method: 'GET', route, sequence } as JourneyMockRequest; +} + +type Start = NonNullable; +type Terminal = NonNullable; +type Media = NonNullable; + +function measurement( + overrides: Partial = {} +): PlaybackJourneyMeasurement { + const renderer: JourneyRendererProbeState = { + capabilities: { + changeDetectionTicks: 'unavailable-ng-global-not-published', + layoutShift: true, + longTask: true, + observedTarget: 'documentElement', + }, + counters: { + domMutations: 6_188, + layoutShiftScore: 0, + layoutShiftScoreSettled: 0, + longTasks: 0, + recentInputLayoutShiftScore: 0.00061, + }, + final: true, + firstCardPaintEpochMs: 10_360, + installed: { + bridgePresent: true, + documentElementPresent: true, + epochMs: 9_000, + readyState: 'complete', + scriptCount: 12, + }, + invalidReasons: [], + journey: 'playback', + longTaskDurationsMs: [], + media: { + element: { + currentSrcScheme: 'blob', + currentTime: 0.02133, + paused: false, + readyState: 4, + videoHeight: 90, + videoWidth: 160, + }, + phases: { loadedmetadata: 10_095.04, playing: 10_349.84 }, + }, + navigation: null, + preStart: { domMutations: 53, lastMutationEpochMs: 8_900 }, + schemaVersion: 1, + sentinel: { epochMs: 10_350, status: 'sent' }, + settle: { + domMutations: 0, + epochMs: null, + lastMutationEpochMs: null, + lateShifts: [], + observedTarget: null, + status: 'disabled', + }, + start: { + epochMs: 10_000, + listenerEpochMs: 10_000.5, + pathname: LIVE_PATH, + sentinelStatus: 'sent', + targetTag: 'div', + targetTestId: 'channel-item', + }, + terminal: { + cardCount: 1, + cardTag: 'video', + cardTestId: null, + companionCounts: [], + epochMs: 10_349.84, + pathname: LIVE_PATH, + }, + }; + const ipc: JourneyMainIpcCaptureState = { + callsAfterSentinel: 3, + callsBeforeStart: 6, + callsBeforeSentinel: 4, + callsByMethod: { + getEpgMapping: 1, + setUserAgent: 1, + updateRemoteControlStatus: 1, + xtreamRequest: 1, + }, + inFlightByMethod: {}, + installedEpochMs: 9_500, + malformedEvents: 0, + processStartEpochMs: 1_000, + senderIds: [1], + sentinel: { occurrences: 1, receivedEpochMs: 10_351 }, + start: { occurrences: 1, receivedEpochMs: 10_001 }, + unmatchedCompletions: 0, + }; + return { + externalArtworkCancelled: 8, + http: { + afterPlaying: [ + request(9, '/player_api.php?action=get_short_epg', 10_400), + ], + afterSettleBeforeClick: 0, + beforeClick: [ + request(0, '/player_api.php?action=get_live_streams', 8_000), + ], + toPlaying: [ + request(7, STREAM_ROUTE, 10_020), + request( + 8, + '/player_api.php?action=get_simple_data_table', + 10_030 + ), + ], + }, + ipc, + pid: 5151, + renderer, + settle: { + preStartDomMutations: 53, + preStartHttpRequests: 4, + preStartIpcCalls: 6, + quietMs: 1_000, + waitedMs: 1_809, + }, + ...overrides, + }; +} + +function withRenderer( + patch: Partial +): PlaybackJourneyMeasurement { + const base = measurement(); + return { ...base, renderer: { ...base.renderer, ...patch } }; +} + +test('maps the media-terminated probe, IPC window and mock ledger to exact counters', () => { + const record = toPlaybackIterationRecord(2, false, measurement()); + assert.equal(record.index, 2); + assert.equal(record.warmup, false); + assert.equal(record.pid, 5151); + assert.deepEqual(record.counters, { + 'renderer.domMutationsToPlaying': 6_188, + 'renderer.httpRequestsToPlaying': 2, + 'renderer.ipcCallsToPlaying': 4, + 'renderer.layoutShiftScore': 0.001, + 'renderer.longTasks': 0, + }); + assert.deepEqual(record.wallClock, { + clickToLoadedMetadataMs: 95, + clickToPlayingMs: 349.8, + }); + assert.deepEqual(record.evidence['httpRequestsByRoute'], { + '/live/:username/:password/10000.ts': 1, + '/player_api.php?action=get_simple_data_table': 1, + }); + assert.deepEqual(record.evidence['httpRequestsAfterPlayingByRoute'], { + '/player_api.php?action=get_short_epg': 1, + }); + assert.deepEqual(record.evidence['httpRequestsBeforeClickByRoute'], { + '/player_api.php?action=get_live_streams': 1, + }); + assert.deepEqual(record.evidence['media'], { + currentSrcScheme: 'blob', + currentTime: 0.021, + paused: false, + readyState: 4, + videoElements: 1, + videoHeight: 90, + videoWidth: 160, + }); + assert.deepEqual(record.evidence['epochs'], { + click: 10_000, + clickListener: 10_000.5, + loadedMetadata: 10_095.04, + mainIpcSentinel: 10_351, + mainIpcStart: 10_001, + playing: 10_349.84, + }); + // Click 10,000: last request before at 8,000, first after at 10,020. + // Playing 10,349.84: last before at 10,030, first after at 10,400. + assert.deepEqual(record.evidence['httpBoundaryMarginsMs'], { + click: 20, + playing: 50.2, + }); + assert.equal(record.evidence['externalArtworkCancelled'], 8); + assert.equal(record.evidence['ipcCallsAfterPlaying'], 3); +}); + +test('rejects measurements that did not start at a live channel or did not play a video', () => { + const base = measurement(); + const start = base.renderer.start as Start; + const terminal = base.renderer.terminal as Terminal; + const media = base.renderer.media as Media; + const cases: [PlaybackJourneyMeasurement, RegExp][] = [ + [withRenderer({ start: null }), /incomplete-probe/], + [withRenderer({ terminal: null }), /incomplete-probe/], + [withRenderer({ media: null }), /incomplete-probe/], + [{ ...base, ipc: { ...base.ipc, start: null } }, /ipc-without-start/], + [ + withRenderer({ + start: { ...start, pathname: '/workspace/dashboard' }, + }), + /start-route/, + ], + [ + withRenderer({ + start: { + ...start, + pathname: '/workspace/xtreams/playlist-1/vod', + }, + }), + /start-route/, + ], + [ + withRenderer({ terminal: { ...terminal, cardTag: 'div' } }), + /not-a-video/, + ], + [withRenderer({ media: { ...media, element: null } }), /not-a-video/], + [withRenderer({ media: { ...media, phases: {} } }), /clock-order/], + [ + withRenderer({ + media: { + ...media, + phases: { loadedmetadata: terminal.epochMs + 1 }, + }, + }), + /clock-order/, + ], + [ + withRenderer({ + media: { + ...media, + phases: { loadedmetadata: start.epochMs - 1 }, + }, + }), + /clock-order/, + ], + [ + withRenderer({ + terminal: { ...terminal, epochMs: start.epochMs }, + media: { ...media, phases: { loadedmetadata: start.epochMs } }, + }), + /clock-order/, + ], + [ + withRenderer({ + capabilities: { + ...base.renderer.capabilities, + changeDetectionTicks: 'hook-present-not-counted', + }, + }), + /cd-hook-hook-present-not-counted/, + ], + ]; + for (const [input, error] of cases) { + assert.throws(() => toPlaybackIterationRecord(0, false, input), error); + } +}); + +test('requires the stream from the local fixture inside the window', () => { + const base = measurement(); + const withoutStream = { + ...base, + http: { + ...base.http, + toPlaying: base.http.toPlaying.filter( + (entry) => entry.route !== STREAM_ROUTE + ), + }, + }; + assert.throws( + () => toPlaybackIterationRecord(0, false, withoutStream), + /no-local-stream/ + ); + const hls = { + ...base, + http: { + ...base.http, + toPlaying: [request(7, '/live/:username/:password/10000.m3u8', 1)], + }, + }; + assert.throws( + () => toPlaybackIterationRecord(0, false, hls), + /no-local-stream/ + ); +}); + +test('rejects activity that moved between the settle snapshot and the click', () => { + const base = measurement(); + assert.throws( + () => + toPlaybackIterationRecord(0, false, { + ...base, + http: { ...base.http, afterSettleBeforeClick: 1 }, + ipc: { ...base.ipc, callsBeforeStart: 7 }, + renderer: { + ...base.renderer, + preStart: { domMutations: 54, lastMutationEpochMs: 1 }, + }, + }), + /activity-before-click-dom-ipc-http/ + ); +}); + +test('summarizes playback iterations with J3 counters and unavailable reasons', () => { + const iterations = [0, 1, 2].map((index) => + toPlaybackIterationRecord(index, index === 0, { + ...measurement(), + pid: 100 + index, + }) + ); + const entry = summarizeJourneyIterations( + iterations, + PLAYBACK_JOURNEY_UNAVAILABLE_COUNTERS + ); + assert.equal(entry.counters['renderer.ipcCallsToPlaying'], 4); + assert.deepEqual(entry.counterStability['renderer.ipcCallsToPlaying'], { + stable: true, + values: [4, 4], + }); + assert.equal(entry.wallClock['clickToPlayingMs.p50'], 349.8); + assert.equal(entry.wallClock['clickToLoadedMetadataMs.p90'], 95); + assert.deepEqual(Object.keys(entry.unavailable).sort(), [ + 'renderer.cdTicksToPlaying', + 'renderer.ipcSerialDepthToPlaying', + ]); +}); diff --git a/apps/electron-backend-e2e/src/performance/playback-journey-record.ts b/apps/electron-backend-e2e/src/performance/playback-journey-record.ts new file mode 100644 index 000000000..ce0f59a44 --- /dev/null +++ b/apps/electron-backend-e2e/src/performance/playback-journey-record.ts @@ -0,0 +1,231 @@ +import { + journeyActivityBeforeClick, + type JourneyClickSettle, +} from './journey-click-settle'; +import type { JourneyMainIpcCaptureState } from './journey-main-ipc-capture'; +import { + countJourneyMockRoutes, + type JourneyMockRequest, +} from './journey-mock-request-ledger'; +import type { JourneyRendererProbeState } from './journey-renderer-probe'; +import type { JourneyIterationRecord } from './journey-summary'; + +/** + * Maps one measured "start playback" (renderer probe armed at the click and + * ended by the video element's `playing` event, main IPC capture between the + * start and end sentinels, mock request ledger) to the journey summary's + * iteration record for J3. + */ +export const PLAYBACK_JOURNEY_ID = 'playback'; + +export const PLAYBACK_JOURNEY_COUNTER = { + DOM_MUTATIONS: 'renderer.domMutationsToPlaying', + HTTP_REQUESTS: 'renderer.httpRequestsToPlaying', + IPC_CALLS: 'renderer.ipcCallsToPlaying', + LAYOUT_SHIFT_SCORE: 'renderer.layoutShiftScore', + LONG_TASKS: 'renderer.longTasks', +} as const; + +export const PLAYBACK_JOURNEY_WALL_CLOCK = { + /** Click until the video element's first `loadedmetadata`. */ + CLICK_TO_LOADED_METADATA: 'clickToLoadedMetadataMs', + /** Click until its first `playing`. */ + CLICK_TO_PLAYING: 'clickToPlayingMs', +} as const; + +/** How long requests after `playing` are observed, for evidence only. */ +export const PLAYBACK_JOURNEY_AFTER_PLAYING_WINDOW_MS = 1_000; + +export const PLAYBACK_JOURNEY_UNAVAILABLE_COUNTERS: Readonly< + Record +> = Object.freeze({ + 'renderer.cdTicksToPlaying': + 'The electron-performance build optimizes scripts (ngDevMode=false), so Angular does not publish window.ng and ɵsetProfiler is unavailable.', + 'renderer.ipcSerialDepthToPlaying': + 'The serial-depth helper is being added for J1 in a separate thread and is not on master yet; J3 adopts it once it lands.', +}); + +export interface PlaybackJourneyMeasurement { + /** Logo requests to picsum.photos cancelled in the main process. */ + readonly externalArtworkCancelled: number; + readonly http: { + /** Requests in the first second after `playing`. */ + readonly afterPlaying: readonly JourneyMockRequest[]; + /** Mock requests after the app settled but before the click stamp. */ + readonly afterSettleBeforeClick: number; + /** Mock requests from the spawn (J1, navigation, settling). */ + readonly beforeClick: readonly JourneyMockRequest[]; + /** Mock requests from the click stamp until `playing`. */ + readonly toPlaying: readonly JourneyMockRequest[]; + }; + readonly ipc: JourneyMainIpcCaptureState; + readonly pid: number; + readonly renderer: JourneyRendererProbeState; + readonly settle: JourneyClickSettle; +} + +const ROUTE_FRAGMENT = '/workspace/xtreams/'; +const LIVE_STREAM_ROUTE = /^\/live\/:username\/:password\/[^/]+\.ts$/; + +/** + * Distance of the nearest ledger request to a boundary on either side + * (null when there is none). The ledger and the renderer stamp with + * different processes' clocks; a small margin flags a count that a clock + * difference could move across the boundary. + */ +function boundaryMarginMs( + before: readonly JourneyMockRequest[], + after: readonly JourneyMockRequest[], + boundaryEpochMs: number +): number | null { + const distances = [ + ...before.map((entry) => boundaryEpochMs - entry.epochMs), + ...after.map((entry) => entry.epochMs - boundaryEpochMs), + ]; + return distances.length === 0 ? null : roundTenth(Math.min(...distances)); +} + +function roundTenth(value: number): number { + return Math.round(value * 10) / 10; +} + +function roundThousandth(value: number): number { + return Math.round(value * 1_000) / 1_000; +} + +export function toPlaybackIterationRecord( + index: number, + warmup: boolean, + measurement: PlaybackJourneyMeasurement +): JourneyIterationRecord { + const { http, ipc, renderer, settle } = measurement; + const { media, start, terminal } = renderer; + if (start === null || terminal === null || media === null) { + throw new Error('playback-journey-record-incomplete-probe'); + } + if (ipc.start === null) { + throw new Error('playback-journey-record-ipc-without-start'); + } + if ( + !start.pathname.includes(ROUTE_FRAGMENT) || + !start.pathname.includes('/live') + ) { + throw new Error('playback-journey-record-start-route'); + } + if (terminal.cardTag !== 'video' || media.element === null) { + throw new Error('playback-journey-record-not-a-video'); + } + const loadedMetadataEpochMs = media.phases['loadedmetadata']; + const clickToPlayingMs = terminal.epochMs - start.epochMs; + if ( + loadedMetadataEpochMs === undefined || + loadedMetadataEpochMs < start.epochMs || + loadedMetadataEpochMs > terminal.epochMs || + clickToPlayingMs <= 0 + ) { + throw new Error('playback-journey-record-clock-order'); + } + // The stream itself must have come from the mock's local fixture; a + // player that played something else did not measure this journey. + if (!http.toPlaying.some((entry) => LIVE_STREAM_ROUTE.test(entry.route))) { + throw new Error('playback-journey-record-no-local-stream'); + } + const lateActivity = journeyActivityBeforeClick(settle, { + httpAfterSettleBeforeClick: http.afterSettleBeforeClick, + ipcCallsBeforeStart: ipc.callsBeforeStart, + preStartDomMutations: renderer.preStart.domMutations, + }); + if (lateActivity.length > 0) { + throw new Error( + `playback-journey-record-activity-before-click-${lateActivity.join('-')}` + ); + } + if ( + renderer.capabilities.changeDetectionTicks !== + 'unavailable-ng-global-not-published' + ) { + throw new Error( + `playback-journey-record-cd-hook-${renderer.capabilities.changeDetectionTicks}` + ); + } + return Object.freeze({ + counters: Object.freeze({ + [PLAYBACK_JOURNEY_COUNTER.DOM_MUTATIONS]: + renderer.counters.domMutations, + [PLAYBACK_JOURNEY_COUNTER.HTTP_REQUESTS]: http.toPlaying.length, + [PLAYBACK_JOURNEY_COUNTER.IPC_CALLS]: ipc.callsBeforeSentinel, + // The click's 500 ms input window covers the start of the + // journey, so shifts flagged hadRecentInput are included, as + // in J2. + [PLAYBACK_JOURNEY_COUNTER.LAYOUT_SHIFT_SCORE]: roundThousandth( + renderer.counters.layoutShiftScore + + renderer.counters.recentInputLayoutShiftScore + ), + [PLAYBACK_JOURNEY_COUNTER.LONG_TASKS]: renderer.counters.longTasks, + }), + evidence: Object.freeze({ + capabilities: renderer.capabilities, + epochs: Object.freeze({ + click: start.epochMs, + clickListener: start.listenerEpochMs, + loadedMetadata: loadedMetadataEpochMs, + mainIpcSentinel: ipc.sentinel.receivedEpochMs, + mainIpcStart: ipc.start.receivedEpochMs, + playing: terminal.epochMs, + }), + externalArtworkCancelled: measurement.externalArtworkCancelled, + httpRequestsAfterPlayingByRoute: countJourneyMockRoutes( + http.afterPlaying + ), + httpRequestsBeforeClickByRoute: countJourneyMockRoutes( + http.beforeClick + ), + httpRequestsByRoute: countJourneyMockRoutes(http.toPlaying), + httpBoundaryMarginsMs: Object.freeze({ + click: boundaryMarginMs( + http.beforeClick, + http.toPlaying, + start.epochMs + ), + playing: boundaryMarginMs( + http.toPlaying, + http.afterPlaying, + terminal.epochMs + ), + }), + ipcCallsAfterPlaying: ipc.callsAfterSentinel, + ipcCallsByMethod: ipc.callsByMethod, + layoutShift: Object.freeze({ + recentInput: roundThousandth( + renderer.counters.recentInputLayoutShiftScore + ), + withoutRecentInput: roundThousandth( + renderer.counters.layoutShiftScore + ), + }), + longTaskDurationsMs: renderer.longTaskDurationsMs.map(roundTenth), + media: Object.freeze({ + ...media.element, + currentTime: roundThousandth(media.element.currentTime), + videoElements: terminal.cardCount, + }), + settle, + start: Object.freeze({ + pathname: start.pathname, + targetTag: start.targetTag, + targetTestId: start.targetTestId, + }), + terminalPathname: terminal.pathname, + }), + index, + pid: measurement.pid, + wallClock: Object.freeze({ + [PLAYBACK_JOURNEY_WALL_CLOCK.CLICK_TO_LOADED_METADATA]: roundTenth( + loadedMetadataEpochMs - start.epochMs + ), + [PLAYBACK_JOURNEY_WALL_CLOCK.CLICK_TO_PLAYING]: + roundTenth(clickToPlayingMs), + }), + warmup, + }); +} diff --git a/docs/architecture/performance-journeys.md b/docs/architecture/performance-journeys.md index 66fc6c15d..26c7cb0ab 100644 --- a/docs/architecture/performance-journeys.md +++ b/docs/architecture/performance-journeys.md @@ -19,9 +19,9 @@ live in `tools/performance/`. | J4 `search` | six-character query typed into global search | results list settled | J1 is instrumented: `renderer.initialBytes` from the built output, and the -runtime counters of the launch benchmark below. J2 is instrumented by its own -spec (below). J3 and J4 follow the plan in `.plans/` and are added one thread -at a time; each thread names its journey and counter in the PR description. +runtime counters of the launch benchmark below. J2 and J3 are instrumented by +their own specs (below). J4 follows the plan in `.plans/` and is added in its +own thread; each thread names its journey and counter in the PR description. ## Running the journeys @@ -367,6 +367,13 @@ values, so a new counter needs no schema change. A J1 runtime baseline is added once its counter is deterministic on the CI runner; the launch counters are not yet (see [Ratchet](#ratchet)), so the summary is evidence only. +J3 adds the `journeys.playback` entry with the same shape and no schema +version change: `counters` and `wallClock` hold only plain numbers, and its +iterations carry `evidence.media` (the video element at `playing`) and +`evidence.epochs.loadedMetadata` / `.playing`. The renderer probe blob gained +a `media` field (`null` for J1 and J2), which the probe's +`schemaVersion` 1 readers ignore. + ## J2 `open-source`: open a source to a browsable list `open-source.journey.ts` reuses the J1 profile and process pattern: the @@ -469,6 +476,132 @@ give a click-to-settled count; that is left to a follow-up. All epochs are taken in the renderer, so neither entry crosses a process clock. +## J3 `playback`: start playback to the first frame + +`playback.journey.ts` follows J2: the profile is seeded once through the +"Add playlist" dialogs (`seedLaunchJourneyProfile` with +`PLAYBACK_JOURNEY_SEED`), every iteration copies it, spawns a fresh process +through `runLaunchJourney` without main-process counters, and hands the +running app to `measurePlaybackJourney` in +`src/journeys/playback-journey-app.ts`. One warm-up and five measured +iterations. + +**Profile.** J2's M3U source plus an Xtream portal ("Journey live portal") on +the mock's `live-fallback:live-fallback` account, behind the same request +ledger proxy. Seeding also selects **Settings > Playback > Video player > +HTML5 video player** and **Stream format > ts** through the settings page +(`configureLiveFormat`), so live URLs end in `.ts`. Embedded MPV and external +players are out of scope. + +**Stream.** `/live/live-fallback/live-fallback/10000.ts` returns +`apps/xtream-mock-server/src/fixtures/live.mpegts` from disk: six seconds of +160x90 H.264 baseline video and AAC audio in MPEG-TS, about 300 KB. The HTML5 +player plays `.ts` through mpegts.js, which transmuxes to fragmented MP4 for +Media Source Extensions; H.264 and AAC are among the codecs Electron's +Chromium decodes on every platform, including the Linux runner, and the +Electron E2E for the live-format fallback already plays this fixture there. +Two other choices were rejected: the `marketing` and `marketing2` accounts +serve live URLs from local bytes, but those bytes are zero-filled (a fixture +for download screenshots, not media), so no player ever fires `playing`; every +other account redirects streams to a public HLS test stream. The record +fails an iteration whose click-to-`playing` window has no `.ts` request for +a live stream, so a player that played something else is never measured. + +**No request leaves the machine.** The generated live catalog's channel and +category logos point at `picsum.photos`. Before navigating, the journey +registers `session.defaultSession.webRequest.onBeforeRequest` for +`*://picsum.photos/*` from the test side and cancels those requests (the app +registers no `onBeforeRequest` listener of its own, so none is replaced). A +logo therefore never loads, or fails, at a moment that depends on the +runner's network; the number cancelled is kept as +`evidence.externalArtworkCancelled`. Every other request goes to the mock +through the ledger. + +**Start.** After J1 has ended, the test clicks the portal's dashboard card, +the **Live TV** link and the first category (not measured), then installs the +IPC capture with a start sentinel, arms the probe, hovers the first +`app-live-stream-layout [data-test-id="channel-item"]` and waits for the same +1 s quiet as J2 (`src/performance/journey-click-settle.ts`, shared with J2). +J1's capture is detached and the channel is clicked. The probe's capture-phase +`click` listener stamps the start and sends +`cancelSourceProbe('__iptvnator-journey-playback-start__')`. The record +rejects activity between the settle snapshot and the click exactly as J2 +does. + +**End.** The probe runs with `media: { endEvent: 'playing', phaseEvents: +['loadedmetadata'] }`. Media events do not bubble, but a capture-phase +listener on `window` sees them before any listener of the app. The first +`playing` event after the start on an element matching +`app-web-player-view video` ends the journey: pending mutation records are +taken synchronously, the end sentinel +`cancelSourceProbe('__iptvnator-journey-playback-end__')` is sent, and the +element's state is recorded (`evidence.media`: `readyState`, `paused`, +`currentTime`, intrinsic size, and `currentSrcScheme`, which is `blob` for +Media Source playback). The first `loadedmetadata` on such an element after +the start is recorded as a phase. A visible video element does not end the +journey, and media events before the click or on other elements are +ignored. + +### Counters + +| Counter | Source | +| -------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `renderer.ipcCallsToPlaying` | Bridge `start` trace events between the start and end sentinels, as `renderer.ipcCallsToFirstPage` in J2. | +| `renderer.httpRequestsToPlaying` | Requests the ledger proxy received from the click stamp until the `playing` stamp, from either process (the stream request comes from the renderer, Xtream API calls from the main process). Both stamps are `performance.timeOrigin + performance.now()` of processes on the same host clock, as in J2. Unlike J2's counter the window ends at the terminal, not at a quiet mock: a live stream has no quiet end. Later requests are kept as `evidence.httpRequestsAfterPlayingByRoute`, the ones in the window as `evidence.httpRequestsByRoute`. | +| `renderer.domMutationsToPlaying` | `MutationRecord`s from the click until the `playing` event, including records still queued when it fires. | +| `renderer.layoutShiftScore` | All `layout-shift` entries from the click until the `playing` event, including `hadRecentInput` ones (as J2), rounded to three decimals. Entries delivered up to the post-paint cutoff are read, but only those that started by the event count. | +| `renderer.longTasks` | `longtask` entries over 50 ms whose time range overlaps the window from the click to the `playing` event, so the task that dispatched the event counts. Evidence until shown to be stable on the runner. | + +`renderer.httpRequestsToPlaying` compares the ledger's arrival stamps +(test process) with the renderer's click and `playing` stamps. Both are +`performance.timeOrigin + performance.now()` on the same host clock, but the +two processes' time origins can differ by a fraction of a millisecond, so +every iteration records `evidence.httpBoundaryMarginsMs`: the distance of +the nearest request on either side of the click and of `playing`. A margin +of a few milliseconds means a clock difference could move that request +across the boundary. Locally the first request after the click arrives 3-6 +ms after its stamp (the click causes it, so it cannot precede the click) and +the nearest request to `playing` is more than 170 ms away; both are well +above a sub-millisecond origin difference. `evidence.httpRequestsAfterPlayingByRoute` covers a fixed +window of 1 s after `playing` (the test waits that long before reading the +ledger), not a quiet mock as in J2: a live stream has no quiet end. + +Two counters are listed under `unavailable`. `renderer.cdTicksToPlaying` is +missing for the same reason as in J1 and J2. +`renderer.ipcSerialDepthToPlaying` is missing because the serial-depth +helper (see [Startup work before the first card](#startup-work-before-the-first-card)) +was not on `master` when J3 landed; J3 adopts it once the J1 thread adds it. +`main.sqlStatementsToPlaying` is not measured for the same reason as J2's +SQL counter. + +### Wall-clock + +| Entry | Derivation | +| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| `clickToPlayingMs.p50/.p90` | `playing` epoch minus start epoch. | +| `clickToLoadedMetadataMs.p50/.p90` | First `loadedmetadata` epoch minus start epoch: player setup, the stream request and the first transmuxed init segment. | + +The difference of the two is stream start: buffering until the element can +play. All epochs are taken in the renderer. + +### First measurement + +Local, macOS, 2026-09-30 (two `perf:journeys` runs, five measured iterations +each): every counter identical in all ten, `renderer.ipcCallsToPlaying` 4 +(`getEpgMapping`, `xtreamRequest`, `updateRemoteControlStatus`, +`setUserAgent`), `renderer.httpRequestsToPlaying` 2 (the `.ts` stream and +`get_simple_data_table`), `renderer.domMutationsToPlaying` 6,188, +`renderer.layoutShiftScore` 0.001, `renderer.longTasks` 0; P50 +click→`loadedmetadata` 92-94 ms and click→`playing` 239-257 ms. The warm-up +iteration of the first run took the cold path (888 ms to `playing`, one more +`updateRemoteControlStatus` call); warm-ups are excluded. About 6,000 of the +mutations come from the EPG timeline rendering about 240 programme blocks +from the `get_simple_data_table` response before the first frame. Whether +that response and its render land before `playing` is a race on a slower +machine, so check the runner's `counterStability` before trusting the +mutation and request counts. No J3 baseline exists yet; J3 counters join the +ratchet once three runner runs agree. + ## `renderer.initialBytes` The bytes a browser fetches before Angular can bootstrap, read from the built @@ -764,10 +897,12 @@ reports slow imports of non-Latin playlists. continues from J1 with `runLaunchJourney` and lets the app settle first, as `open-source-journey-app.ts` does. 2. Give the journey its own probe options (`cardSelector`, - `companionSelectors`, `routeFragment`, `startClick` for a click start) or - extend `journey-renderer-probe.ts` when the end condition is not "elements - became visible". Use a state key and sentinel ids of its own. Keep the - probe self-contained: Playwright serializes it with `toString()`. + `companionSelectors`, `routeFragment`, `startClick` for a click start, + `media` for a media-event end such as J3's `playing`) or extend + `journey-renderer-probe.ts` when the end condition is neither. Use a state + key and sentinel ids of its own. Keep the probe self-contained: Playwright + serializes it with `toString()`. A click-started journey settles with + `waitForJourneyClickQuiet` from `journey-click-settle.ts`. 3. Map the measurement to a `JourneyIterationRecord` in a `-journey-record.ts` under `src/performance/`; name counters `renderer.*` or `main.*`, and list counters you cannot measure under