From f4da4d6074077b1e77e6c1970c0afaa3b5f80ced Mon Sep 17 00:00:00 2001 From: 4gray Date: Sat, 11 Jul 2026 00:00:25 +0200 Subject: [PATCH] feat(embedded-mpv): frame-copy canvas mode in the player component + docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EmbeddedMpvPlayerComponent renders when support reports engine 'frame-copy' and the session controller starts/stops the preload frame pump around the session lifecycle. The bounds provider skips HIDDEN_BOUNDS and the popover cutout for this engine — the canvas is ordinary DOM, dialogs and popovers stack above it natively; bounds sync still drives the helper's render size. Adapter unit tests cover spawn args, snapshot caching, shm generations, protocol encoding, unexpected-exit mapping, and dispose escalation. Architecture doc and CLAUDE.md describe the engine, its flag, and the sandbox trade-off. Verified end to end in the built app (M1 Pro): engine detection, helper spawn, lavfi playback onto the canvas via CDP-injected smoke — including an orientation fix (helper FLIP_Y already yields texture-order rows; the pump shader must not flip uv again). Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 1 + .../src/app/api/embedded-mpv-frame-pump.ts | 6 +- .../embedded-mpv-frame-copy.adapter.spec.ts | 176 ++++++++++++++++++ docs/architecture/embedded-mpv-native.md | 53 ++++++ .../embedded-mpv-player.component.html | 9 + .../embedded-mpv-player.component.scss | 12 ++ .../embedded-mpv-player.component.ts | 10 + .../embedded-mpv-session-controller.ts | 15 ++ 8 files changed, 281 insertions(+), 1 deletion(-) create mode 100644 apps/electron-backend/src/app/services/embedded-mpv-frame-copy.adapter.spec.ts diff --git a/CLAUDE.md b/CLAUDE.md index a68c0c124..e5f494d8b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -617,6 +617,7 @@ This project uses modern Angular signal-based APIs and patterns. **ALWAYS** use - Built-in HTML5 player with HLS.js or Video.js - External players: MPV, VLC (via IPC to Electron backend) - Embedded MPV (experimental, macOS/Windows/Linux): renders mpv video inside the Electron window through a native addon. macOS uses the libmpv render API in an `NSOpenGLView`; Windows uses in-process libmpv with `--wid` against an app-owned child `HWND`; Linux spawns an out-of-process `mpv --wid=` controlled over a JSON IPC socket (X11/XWayland only, requires system `mpv` on PATH; subtitles/speed/aspect/recording are not exported there). mpv's own screensaver inhibition does not apply to any of these paths, so `EmbeddedMpvNativeService` holds an Electron `powerSaveBlocker` (`prevent-display-sleep`) whenever any session's status is `playing`, and releases it on pause, dispose, or shutdown. Service: `apps/electron-backend/src/app/services/embedded-mpv-native.service.ts`; full architecture: `docs/architecture/embedded-mpv-native.md`. +- Embedded MPV frame-copy engine (experimental, macOS Apple Silicon only, `IPTVNATOR_ENABLE_EMBEDDED_MPV_FRAME_COPY=1` on top of the embedded MPV experiment flag): a per-session `iptvnator_mpv_helper` process renders mpv offscreen at viewport size and publishes BGRA frames into a shm ring; the preload frame pump uploads them onto a renderer ``, so controls/dialogs are ordinary DOM above the video (no native-surface compositing workarounds; the flag relaxes the window sandbox for the preload's native reader addon). Adapter: `apps/electron-backend/src/app/services/embedded-mpv-frame-copy.adapter.ts`; helper: `apps/electron-backend/native/helper/`; details in `docs/architecture/embedded-mpv-native.md` ("Frame-Copy Engine"). **VOD/Series Detail Pages (two-state layout)**: diff --git a/apps/electron-backend/src/app/api/embedded-mpv-frame-pump.ts b/apps/electron-backend/src/app/api/embedded-mpv-frame-pump.ts index c6192d413..2f13d030d 100644 --- a/apps/electron-backend/src/app/api/embedded-mpv-frame-pump.ts +++ b/apps/electron-backend/src/app/api/embedded-mpv-frame-pump.ts @@ -46,11 +46,15 @@ const CANVAS_SELECTOR = 'canvas[data-embedded-mpv-frame]'; const ATTACH_TIMEOUT_MS = 5000; const ATTACH_POLL_MS = 100; +// Note on orientation: the helper renders with MPV_RENDER_PARAM_FLIP_Y and +// glReadPixels reads rows bottom-up, so the shm buffer arrives in texture +// order already — sampling uses the un-flipped uv (a second flip here would +// show the video upside down). const VERTEX_SHADER = `#version 300 es out vec2 v_uv; void main() { vec2 corner = vec2((gl_VertexID << 1) & 2, gl_VertexID & 2); - v_uv = vec2(corner.x, 1.0 - corner.y); + v_uv = corner; gl_Position = vec4(corner * 2.0 - 1.0, 0.0, 1.0); }`; diff --git a/apps/electron-backend/src/app/services/embedded-mpv-frame-copy.adapter.spec.ts b/apps/electron-backend/src/app/services/embedded-mpv-frame-copy.adapter.spec.ts new file mode 100644 index 000000000..52188b1f6 --- /dev/null +++ b/apps/electron-backend/src/app/services/embedded-mpv-frame-copy.adapter.spec.ts @@ -0,0 +1,176 @@ +import { EventEmitter } from 'events'; + +const spawnMock = jest.fn(); +jest.mock('child_process', () => ({ + spawn: (...args: unknown[]) => spawnMock(...args), +})); + +import { EmbeddedMpvFrameCopyAdapter } from './embedded-mpv-frame-copy.adapter'; + +class FakeHelperProcess extends EventEmitter { + exitCode: number | null = null; + readonly stdout = new EventEmitter(); + readonly stderr = new EventEmitter(); + readonly stdin = { + writable: true, + written: [] as string[], + write(line: string) { + this.written.push(line); + return true; + }, + }; + readonly kill = jest.fn((signal?: string) => { + this.exitCode = 0; + this.emit('exit', 0, signal ?? null); + return true; + }); + + emitStdout(payload: object): void { + this.stdout.emit('data', Buffer.from(`${JSON.stringify(payload)}\n`)); + } +} + +describe('EmbeddedMpvFrameCopyAdapter', () => { + let child: FakeHelperProcess; + let frameSourceChanges: Array<{ sessionId: string; shmName: string }>; + let adapter: EmbeddedMpvFrameCopyAdapter; + + const createAdapter = (helperPath: string | null = '/native/helper') => { + frameSourceChanges = []; + return new EmbeddedMpvFrameCopyAdapter({ + resolveHelperPath: () => helperPath, + getScaleFactor: () => 2, + onFrameSourceChanged: (sessionId, source) => + frameSourceChanges.push({ sessionId, shmName: source.shmName }), + }); + }; + + beforeEach(() => { + jest.useFakeTimers(); + child = new FakeHelperProcess(); + spawnMock.mockReset(); + spawnMock.mockReturnValue(child); + adapter = createAdapter(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + const createSession = () => + adapter.createSession( + Buffer.alloc(0), + { x: 0, y: 0, width: 640, height: 360 }, + 'Title', + 0.8 + ); + + it('spawns the helper with device-pixel size and initial volume', () => { + const sessionId = createSession(); + expect(sessionId).toMatch(/^impv-fc-/); + const [helperPath, args] = spawnMock.mock.calls[0]; + expect(helperPath).toBe('/native/helper'); + expect(args).toEqual([ + '--shm-base', + `/${sessionId}`, + '--width', + '1280', + '--height', + '720', + '--volume', + '0.8', + ]); + }); + + it('caches helper snapshot events for getSessionSnapshot', () => { + const sessionId = createSession(); + child.emitStdout({ + event: 'snapshot', + status: 'playing', + positionSeconds: 12.5, + durationSeconds: 60, + volume: 0.8, + streamUrl: 'http://stream', + audioTracks: [], + selectedAudioTrackId: null, + subtitleTracks: [], + selectedSubtitleTrackId: null, + playbackSpeed: 1, + aspectOverride: 'no', + recording: { active: false }, + }); + const snapshot = adapter.getSessionSnapshot(sessionId); + expect(snapshot?.status).toBe('playing'); + expect(snapshot?.positionSeconds).toBe(12.5); + expect(snapshot?.streamUrl).toBe('http://stream'); + }); + + it('publishes shm generations through onFrameSourceChanged', () => { + const sessionId = createSession(); + child.emitStdout({ + event: 'shm', + name: `/${sessionId}-g1`, + width: 1280, + height: 720, + generation: 1, + }); + expect(frameSourceChanges).toEqual([ + { sessionId, shmName: `/${sessionId}-g1` }, + ]); + expect(adapter.getFrameSource(sessionId)?.readerPath).toBe( + '/native/embedded_mpv_frame_reader.node' + ); + }); + + it('encodes loadfile options with percent-escaping', () => { + const sessionId = createSession(); + adapter.loadPlayback(sessionId, { + streamUrl: 'http://host/live.m3u8', + title: 'Tab\there', + userAgent: 'UA 1.0', + startTime: 42, + headers: { 'X-Token': 'abc' }, + }); + const line = child.stdin.written.at(-1) ?? ''; + expect(line.startsWith('load\turl=http://host/live.m3u8\t')).toBe(true); + expect(line).toContain('opt.force-media-title=Tab%09here'); + expect(line).toContain('opt.user-agent=UA 1.0'); + expect(line).toContain('opt.start=42'); + expect(line).toContain('opt.http-header-fields=X-Token: abc'); + }); + + it('scales bounds and ignores hidden/degenerate bounds', () => { + const sessionId = createSession(); + adapter.setBounds(sessionId, { x: 0, y: 0, width: 800, height: 450 }); + expect(child.stdin.written.at(-1)).toBe( + 'size\twidth=1600\theight=900\n' + ); + const writesBefore = child.stdin.written.length; + adapter.setBounds(sessionId, { x: -10000, y: -10000, width: 1, height: 1 }); + expect(child.stdin.written.length).toBe(writesBefore); + }); + + it('maps an unexpected helper exit to a session error', () => { + const sessionId = createSession(); + child.exitCode = 1; + child.emit('exit', 1, null); + const snapshot = adapter.getSessionSnapshot(sessionId); + expect(snapshot?.status).toBe('error'); + expect(snapshot?.error).toContain('exited unexpectedly'); + }); + + it('disposes with quit and escalates to SIGTERM', () => { + const sessionId = createSession(); + adapter.disposeSession(sessionId); + expect(child.stdin.written.at(-1)).toBe('quit\n'); + expect(adapter.getSessionSnapshot(sessionId)).toBeNull(); + child.exitCode = null; // helper ignored quit + jest.advanceTimersByTime(600); + expect(child.kill).toHaveBeenCalledWith('SIGTERM'); + }); + + it('reports unsupported without a helper binary', () => { + const withoutHelper = createAdapter(null); + expect(withoutHelper.isSupported()).toBe(false); + }); +}); diff --git a/docs/architecture/embedded-mpv-native.md b/docs/architecture/embedded-mpv-native.md index 16725080f..63054ecc5 100644 --- a/docs/architecture/embedded-mpv-native.md +++ b/docs/architecture/embedded-mpv-native.md @@ -89,6 +89,59 @@ The MPV video surface is a native platform view/window, not a normal DOM element The dock has a stable reserved height while embedded controls are enabled. Controls fade in and out inside that fixed dock, so normal show/hide behavior does not resize the native MPV viewport or make the video jump. Volume and audio-track panels replace the default transport controls inside the same dock and provide a back button to return to the default controls. Popovers and menus must stay inside that dock unless the native layering strategy changes. The native MPV view deliberately ignores hit testing so mouse movement passes through to Chromium and can reveal Angular controls even when the pointer moves quickly across the video area. +## Frame-Copy Engine (Experimental, Apple Silicon Only) + +`IPTVNATOR_ENABLE_EMBEDDED_MPV_FRAME_COPY=1` (on top of the regular +embedded MPV experiment flag) switches macOS/arm64 to a second rendering +engine that replaces the native-view compositing entirely: + +- `apps/electron-backend/native/helper/` — `iptvnator_mpv_helper`, a + one-process-per-session libmpv host. It decodes (hwdec), renders + offscreen at viewport size (headless CGL + async PBO readback ring), + publishes BGRA frames into a POSIX shm seqlock ring + (`frame_shm.h`, 3 slots, resize creates a new `-g` generation), and + plays audio directly. Control protocol: tab-separated commands on stdin, + JSON events on stdout; the `snapshot` event mirrors + `NativeEmbeddedMpvSessionSnapshot`. Status semantics are ported from + `embedded_mpv.mm`. +- `apps/electron-backend/src/app/services/embedded-mpv-frame-copy.adapter.ts` — + implements the same `NativeEmbeddedMpvAddon` surface over the helper + process, so `EmbeddedMpvNativeService` (polling, diffing, power blocker, + recording paths) is reused unchanged. The flag routes `getAddon()` to the + adapter and support reports `engine: 'frame-copy'`. +- `apps/electron-backend/native/src/embedded_mpv_frame_reader.c` — N-API + shm reader loaded by the preload frame pump + (`apps/electron-backend/src/app/api/embedded-mpv-frame-pump.ts`): copy + the newest complete frame into a reused ArrayBuffer once per rAF and + upload it to a WebGL2 texture on the renderer's + `` (BGRA swizzle in the shader). Frame + data never crosses the contextBridge; the bridge only exposes + `attachEmbeddedMpvFrameView`/`detachEmbeddedMpvFrameView`. +- Renderer: `EmbeddedMpvPlayerComponent` renders the canvas when + `support.engine === 'frame-copy'` and skips the compositor workarounds — + no `HIDDEN_BOUNDS` when dialogs open, no popover bottom cutout; dialogs + and controls stack above the canvas as ordinary DOM. Bounds sync still + runs: the helper re-renders at the new viewport size (device pixels via + the display scale factor). + +Trade-offs and constraints: + +- The experiment flag relaxes the BrowserWindow sandbox (preload must + `require` the reader addon); `contextIsolation` and + `nodeIntegration:false` stay on. The sandbox story must be revisited + before this engine can become a default — candidates: utilityProcess + + MessagePort (costs one extra copy + GC churn since Electron ports clone + ArrayBuffers) or a WebCodecs-based path. +- Scope: Apple Silicon only by owner decision (2026-07-10); Intel Macs + keep the native-view engine. Windows/Linux ports of the helper (WGL/EGL) + are future work — the shm protocol and adapter are platform-agnostic. +- Measured baseline (M1 Pro, spikes/mpv-frame-copy/RESULTS.md): 4K60 HEVC + sustained end to end, ~1.2 ms shm copy + ~3.5 ms texture upload, ~10 ms + produce-to-upload latency, zero torn frames over a 10-minute run. +- Helper crash isolation: an unexpected helper exit surfaces as a session + `error` (renderer falls back); it can never take down the Electron main + process, unlike in-process libmpv. + ## Resume And Track Handling `ResolvedPortalPlayback.startTime` is treated as a media offset in seconds for VOD and episodes. The native addon passes it as the `start` option in one MPV `loadfile` options map together with title, user agent, referrer, and HTTP headers. diff --git a/libs/ui/playback/src/lib/embedded-mpv-player/embedded-mpv-player.component.html b/libs/ui/playback/src/lib/embedded-mpv-player/embedded-mpv-player.component.html index 7921f2b09..ab234334c 100644 --- a/libs/ui/playback/src/lib/embedded-mpv-player/embedded-mpv-player.component.html +++ b/libs/ui/playback/src/lib/embedded-mpv-player/embedded-mpv-player.component.html @@ -18,6 +18,15 @@ class="embedded-mpv-player__viewport" (click)="onViewportClick($event)" > + @if (isFrameCopyEngine()) { + + + } @if (isErrored()) {
this.support()?.supported ?? false); + readonly isFrameCopyEngine = computed( + () => this.support()?.engine === 'frame-copy' + ); readonly capabilities = computed( () => this.support()?.capabilities ?? { @@ -343,6 +346,13 @@ export class EmbeddedMpvPlayerComponent implements OnDestroy { } this.controller.setBoundsProvider((host) => { + // The frame-copy engine paints into an ordinary DOM canvas: + // dialogs and popovers stack above it natively, so the + // hide-offscreen and popover-cutout compositor workarounds + // must not shrink its render size. + if (this.isFrameCopyEngine()) { + return measureBounds(host); + } if (this.overlayVisibility.overlayActive()) { return HIDDEN_BOUNDS; } diff --git a/libs/ui/playback/src/lib/embedded-mpv-player/embedded-mpv-session-controller.ts b/libs/ui/playback/src/lib/embedded-mpv-player/embedded-mpv-session-controller.ts index 09beb8b80..740174eeb 100644 --- a/libs/ui/playback/src/lib/embedded-mpv-player/embedded-mpv-session-controller.ts +++ b/libs/ui/playback/src/lib/embedded-mpv-player/embedded-mpv-session-controller.ts @@ -31,6 +31,10 @@ export class EmbeddedMpvSessionController { readonly stalled = signal(false); readonly retryToken = signal(0); + readonly isFrameCopyEngine = computed( + () => this.support()?.engine === 'frame-copy' + ); + private readonly sessionStatus = computed( () => this.session()?.status ?? null ); @@ -182,6 +186,14 @@ export class EmbeddedMpvSessionController { this.sessionId.set(created.id); this.session.set(created); await electron.loadEmbeddedMpvPlayback(created.id, playback); + if (untracked(() => this.isFrameCopyEngine())) { + // Frame-copy engine: start the preload frame pump that + // paints helper frames onto the component's canvas. Failure + // is non-fatal here — the session error/stall paths cover it. + void electron + .attachEmbeddedMpvFrameView?.(created.id) + .catch(() => undefined); + } scheduleBoundsSync(); }; @@ -217,6 +229,9 @@ export class EmbeddedMpvSessionController { this.session.set(null); if (id) { + if (untracked(() => this.isFrameCopyEngine())) { + window.electron?.detachEmbeddedMpvFrameView?.(); + } void window.electron?.disposeEmbeddedMpvSession(id); } };