diff --git a/.changes/playback-frame-copy-software-black-video.md b/.changes/playback-frame-copy-software-black-video.md new file mode 100644 index 000000000..f2404fb0d --- /dev/null +++ b/.changes/playback-frame-copy-software-black-video.md @@ -0,0 +1,6 @@ +--- +type: fix +area: playback +--- + +The experimental Embedded MPV frame-copy engine no longer shows a black picture at random on computers that render graphics in software (for example without GPU drivers or in a virtual machine). diff --git a/apps/electron-backend-e2e/src/embedded-mpv-frame-copy-packaged-diagnostics.ts b/apps/electron-backend-e2e/src/embedded-mpv-frame-copy-packaged-diagnostics.ts new file mode 100644 index 000000000..f276dec33 --- /dev/null +++ b/apps/electron-backend-e2e/src/embedded-mpv-frame-copy-packaged-diagnostics.ts @@ -0,0 +1,172 @@ +import { existsSync, readdirSync, readFileSync } from 'fs'; +import { join } from 'path'; +import { + expect, + test, + type LaunchedElectronApp, +} from './electron-test-fixtures'; +import { + getLatestSession, + renderedFrameSignal, +} from './embedded-mpv-frame-copy-packaged-fixtures'; + +/** + * Failure diagnostics for the packaged frame-copy smoke. + * + * A black canvas alone cannot tell whether the helper never published a + * frame, published a black one, or published a real frame that the preload + * pump failed to draw. The helper's POSIX shared-memory rings live in + * /dev/shm on Linux, so the test runner can read them directly. + */ + +const SHM_DIRECTORY = '/dev/shm'; +const FRAME_SHM_MAGIC = 0x564d5046; +const FRAME_SHM_RING_SLOTS = 3; +const HEADER_BYTES = 56 + FRAME_SHM_RING_SLOTS * 16; + +export interface FrameRingDiagnostics { + name: string; + valid: boolean; + width?: number; + height?: number; + generation?: number; + latestSeq?: number; + slotSeqs?: number[]; + /** Per slot: pixels whose B+G+R exceeds 30 (null past the segment). */ + slotSignals?: Array; + heartbeatAgeMs?: number | null; + /** `slotSignals` entry of the newest published frame. */ + latestFrameSignal?: number | null; +} + +function countVisiblePixels( + segment: Buffer, + start: number, + bytes: number +): number | null { + if (start + bytes > segment.length) { + return null; + } + let signal = 0; + for (let pixel = start; pixel + 3 < start + bytes; pixel += 4) { + if (segment[pixel] + segment[pixel + 1] + segment[pixel + 2] > 30) { + signal += 1; + } + } + return signal; +} + +/** + * Parses one frame ring (layout: FrameShmHeader in native/helper/frame_shm.h). + * `nowNs` must come from CLOCK_MONOTONIC, as the helper's heartbeat does. + */ +export function describeFrameRing( + name: string, + segment: Buffer, + nowNs: bigint +): FrameRingDiagnostics { + if ( + segment.length < HEADER_BYTES || + segment.readUInt32LE(0) !== FRAME_SHM_MAGIC + ) { + return { name, valid: false }; + } + + const frameBytes = Number(segment.readBigUInt64LE(24)); + const dataOffset = Number(segment.readBigUInt64LE(32)); + const latestSeq = segment.readBigUInt64LE(40); + const heartbeatNs = segment.readBigUInt64LE(48); + const slotSeqs = Array.from({ length: FRAME_SHM_RING_SLOTS }, (_, slot) => + Number(segment.readBigUInt64LE(56 + slot * 16)) + ); + + const slotSignals = slotSeqs.map((_, slot) => + countVisiblePixels(segment, dataOffset + slot * frameBytes, frameBytes) + ); + + return { + name, + valid: true, + width: segment.readUInt32LE(8), + height: segment.readUInt32LE(12), + generation: segment.readUInt32LE(20), + latestSeq: Number(latestSeq), + slotSeqs, + slotSignals, + heartbeatAgeMs: + heartbeatNs > BigInt(0) ? Number(nowNs - heartbeatNs) / 1e6 : null, + latestFrameSignal: + latestSeq > BigInt(0) + ? slotSignals[Number(latestSeq % BigInt(FRAME_SHM_RING_SLOTS))] + : null, + }; +} + +/** + * The helper names its rings `/-g`, so the prefix + * keeps stale or concurrent sessions out of the report. + */ +export function describeFrameRings( + sessionId: string +): FrameRingDiagnostics[] | string { + try { + return readdirSync(SHM_DIRECTORY) + .filter((entry) => entry.startsWith(`${sessionId}-g`)) + .map((entry) => + describeFrameRing( + entry, + readFileSync(join(SHM_DIRECTORY, entry)), + process.hrtime.bigint() + ) + ); + } catch (error) { + return error instanceof Error ? error.message : String(error); + } +} + +/** + * Waits for a visible frame on the smoke canvas; on timeout, attaches the + * session snapshot, the helper's frame rings and mpv's own log (the session + * sets `log-file`) before rethrowing. + */ +export async function expectRenderedFrame( + app: LaunchedElectronApp, + sessionId: string, + timeout: number, + mpvLogPath?: string +): Promise { + try { + await expect + .poll(() => renderedFrameSignal(app), { timeout }) + .toBeGreaterThan(0); + } catch (error) { + const session = await getLatestSession(app, sessionId).catch( + () => null + ); + const diagnostics = { + session: session && { + status: session.status, + positionSeconds: session.positionSeconds, + videoWidth: session.videoWidth, + videoHeight: session.videoHeight, + stats: session.stats, + }, + frameRings: describeFrameRings(sessionId), + }; + const body = JSON.stringify(diagnostics, null, 2); + console.log(`[frame-copy smoke diagnostics] ${body}`); + await test.info().attach('frame-copy-diagnostics', { + body, + contentType: 'application/json', + }); + if (mpvLogPath && existsSync(mpvLogPath)) { + const mpvLog = readFileSync(mpvLogPath, 'utf8'); + console.log(`[frame-copy smoke mpv log]\n${mpvLog}`); + await test.info().attach('mpv-log', { + body: mpvLog, + contentType: 'text/plain', + }); + } + throw error; + } +} diff --git a/apps/electron-backend-e2e/src/embedded-mpv-frame-copy-packaged-fixtures.spec.ts b/apps/electron-backend-e2e/src/embedded-mpv-frame-copy-packaged-fixtures.spec.ts index 9dc9c7f5f..4e174e52a 100644 --- a/apps/electron-backend-e2e/src/embedded-mpv-frame-copy-packaged-fixtures.spec.ts +++ b/apps/electron-backend-e2e/src/embedded-mpv-frame-copy-packaged-fixtures.spec.ts @@ -6,6 +6,7 @@ import { createWavFixture, isMeaningfulNativePlaybackSnapshot, } from './embedded-mpv-frame-copy-packaged-fixtures'; +import { describeFrameRing } from './embedded-mpv-frame-copy-packaged-diagnostics'; import './embedded-mpv-frame-copy-packaged-filesystem.tests'; import { resolvePackagedElectronLaunchArgs } from './electron-test-fixtures'; import packagedPlaywrightConfig from '../playwright.packaged.config'; @@ -206,3 +207,74 @@ describe('dedicated packaged smoke target', () => { assert.deepEqual(packagedPlaywrightConfig.webServer, []); }); }); + +describe('frame ring diagnostics', () => { + // FrameShmHeader (native/helper/frame_shm.h) for a 4x2 ring whose newest + // frame (seq 4, slot 1) has three non-black BGRA pixels. + function createRing(): Buffer { + const width = 4; + const height = 2; + const frameBytes = width * height * 4; + const dataOffset = 4096; + const ring = Buffer.alloc(dataOffset + 3 * frameBytes); + ring.writeUInt32LE(0x564d5046, 0); + ring.writeUInt32LE(1, 4); + ring.writeUInt32LE(width, 8); + ring.writeUInt32LE(height, 12); + ring.writeUInt32LE(width * 4, 16); + ring.writeUInt32LE(2, 20); + ring.writeBigUInt64LE(BigInt(frameBytes), 24); + ring.writeBigUInt64LE(BigInt(dataOffset), 32); + ring.writeBigUInt64LE(BigInt(4), 40); + ring.writeBigUInt64LE(BigInt(1000000000), 48); + [BigInt(3), BigInt(4), BigInt(2)].forEach((seq, slot) => + ring.writeBigUInt64LE(seq, 56 + slot * 16) + ); + const latest = dataOffset + frameBytes; + for (const pixel of [0, 3, 7]) { + ring[latest + pixel * 4 + 2] = 255; // red channel of BGRA + } + // Slot 0 holds a fully visible older frame; it must not leak into the + // newest frame's signal. + ring.fill(255, dataOffset, dataOffset + frameBytes); + return ring; + } + + it('reports the newest published frame and its visible signal', () => { + assert.deepEqual( + describeFrameRing( + 'impv-fc-test-g2', + createRing(), + BigInt(1250000000) + ), + { + name: 'impv-fc-test-g2', + valid: true, + width: 4, + height: 2, + generation: 2, + latestSeq: 4, + slotSeqs: [3, 4, 2], + slotSignals: [8, 3, 0], + heartbeatAgeMs: 250, + latestFrameSignal: 3, + } + ); + }); + + it('distinguishes an unpublished or uninitialised ring', () => { + const unpublished = createRing(); + unpublished.writeBigUInt64LE(BigInt(0), 40); + assert.equal( + describeFrameRing('ring', unpublished, BigInt(0)).latestFrameSignal, + null + ); + assert.deepEqual( + describeFrameRing('ring', Buffer.alloc(8), BigInt(0)), + { + name: 'ring', + valid: false, + } + ); + }); +}); diff --git a/apps/electron-backend-e2e/src/embedded-mpv-frame-copy-packaged.e2e.ts b/apps/electron-backend-e2e/src/embedded-mpv-frame-copy-packaged.e2e.ts index c9ba81236..ac13e51d4 100644 --- a/apps/electron-backend-e2e/src/embedded-mpv-frame-copy-packaged.e2e.ts +++ b/apps/electron-backend-e2e/src/embedded-mpv-frame-copy-packaged.e2e.ts @@ -18,9 +18,9 @@ import { installEmbeddedMpvSessionCapture, installFrameCanvasAndSessionCapture, isMeaningfulNativePlaybackSnapshot, - renderedFrameSignal, type LocalMediaServer, } from './embedded-mpv-frame-copy-packaged-fixtures'; +import { expectRenderedFrame } from './embedded-mpv-frame-copy-packaged-diagnostics'; import { createDisposablePackagedLinuxApp, createPackagedEntryGuard, @@ -120,18 +120,21 @@ test.describe('Packaged Linux embedded MPV frame-copy runtime', () => { }); await installFrameCanvasAndSessionCapture(launchedFrameCopyApp); + const mpvLogPath = join(dataDir, 'mpv-frame-copy.log'); const created = await launchedFrameCopyApp.mainWindow.evaluate( - async () => { - // Decode audio without requiring a sound device on CI. + async (logFile) => { + // Decode audio without requiring a sound device on CI; + // keep mpv's verbose log for failure diagnostics. await window.electron.updateSettings({ - embeddedMpvExtraOptions: 'ao=null', + embeddedMpvExtraOptions: `ao=null\nlog-file=${logFile}`, }); return window.electron.createEmbeddedMpvSession( { x: 0, y: 0, width: 320, height: 180 }, 'Packaged frame-copy smoke', 0 ); - } + }, + mpvLogPath ); await launchedFrameCopyApp.mainWindow.evaluate( @@ -179,11 +182,12 @@ test.describe('Packaged Linux embedded MPV frame-copy runtime', () => { 'packaged-embedded-mpv-frame' ) ).toHaveAttribute('height', '180'); - await expect - .poll(() => renderedFrameSignal(launchedFrameCopyApp), { - timeout: 15000, - }) - .toBeGreaterThan(0); + await expectRenderedFrame( + launchedFrameCopyApp, + created.id, + 15000, + mpvLogPath + ); // Relative seek steps (arrow keys, ±10 s buttons) must reach mpv // as `seek relative+exact` through the real preload → diff --git a/apps/electron-backend/native/helper/frame_helper_gl.h b/apps/electron-backend/native/helper/frame_helper_gl.h index a0eec7098..2651d5c7f 100644 --- a/apps/electron-backend/native/helper/frame_helper_gl.h +++ b/apps/electron-backend/native/helper/frame_helper_gl.h @@ -28,6 +28,8 @@ */ #pragma once +#include +#include #include #if defined(__APPLE__) @@ -126,6 +128,20 @@ namespace frame_helper { /* Matches mpv_opengl_init_params.get_proc_address. */ using GlGetProcAddressFn = void* (*)(void* ctx, const char* name); +/* GL_RENDERER strings of CPU rasterizers (Mesa llvmpipe/softpipe/swrast). */ +inline bool isSoftwareGlRenderer(const std::string& renderer) { + std::string normalized = renderer; + std::transform(normalized.begin(), normalized.end(), normalized.begin(), + [](unsigned char value) { + return static_cast(std::tolower(value)); + }); + for (const char* marker : {"llvmpipe", "softpipe", "swrast", + "software rasterizer", "lavapipe"}) { + if (normalized.find(marker) != std::string::npos) return true; + } + return false; +} + #if defined(__APPLE__) inline void* glDlsymGetProcAddress(void* ctx, const char* name) { @@ -382,19 +398,6 @@ private: candidate.gbmFd >= 0 || candidate.current; } - static bool isSoftwareRenderer(const std::string& renderer) { - std::string normalized = renderer; - std::transform(normalized.begin(), normalized.end(), normalized.begin(), - [](unsigned char value) { - return static_cast(std::tolower(value)); - }); - for (const char* marker : {"llvmpipe", "softpipe", "swrast", - "software rasterizer", "lavapipe"}) { - if (normalized.find(marker) != std::string::npos) return true; - } - return false; - } - static bool openGbmDevice(Candidate& candidate) { for (int node = 128; node <= 131; node++) { char devicePath[32]; @@ -508,7 +511,7 @@ private: const GLubyte* renderer = glGetString(GL_RENDERER); if (!renderer) return fail("GL_RENDERER unavailable"); candidate.renderer = reinterpret_cast(renderer); - candidate.softwareRenderer = isSoftwareRenderer(candidate.renderer); + candidate.softwareRenderer = isSoftwareGlRenderer(candidate.renderer); if (eglMakeCurrent(candidate.display, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT) != EGL_TRUE) { diff --git a/apps/electron-backend/native/helper/frame_helper_render.h b/apps/electron-backend/native/helper/frame_helper_render.h index 1f0dcaf32..a24268573 100644 --- a/apps/electron-backend/native/helper/frame_helper_render.h +++ b/apps/electron-backend/native/helper/frame_helper_render.h @@ -144,6 +144,9 @@ public: std::function onGenerationChanged; + /* Called from the render thread before the mpv render context exists + * when the bound context is a CPU rasterizer (see setupGl()). */ + std::function onSoftwareRenderer; bool start(mpv_handle* mpv, const std::string& shmBaseName, int width, int height, std::string& errorOut); @@ -205,8 +208,11 @@ inline bool RenderPipeline::setupGl(std::string& errorOut) { * software tiers when a later hardware-backed EGL candidate was usable. */ const GLubyte* renderer = glGetString(GL_RENDERER); if (renderer) { - std::fprintf(stderr, "gl renderer: %s\n", - reinterpret_cast(renderer)); + const char* rendererName = reinterpret_cast(renderer); + std::fprintf(stderr, "gl renderer: %s\n", rendererName); + if (onSoftwareRenderer && isSoftwareGlRenderer(rendererName)) { + onSoftwareRenderer(); + } } if (!rebuildTargets(width_, height_)) { diff --git a/apps/electron-backend/native/helper/mpv_frame_helper.cpp b/apps/electron-backend/native/helper/mpv_frame_helper.cpp index e7fab3032..f2cfbf762 100644 --- a/apps/electron-backend/native/helper/mpv_frame_helper.cpp +++ b/apps/electron-backend/native/helper/mpv_frame_helper.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -788,6 +789,35 @@ void onRenderUpdate(void* pipeline) { static_cast(pipeline)->notifyUpdate(); } +/* mpv's LUT scalers (the default lanczos/spline family) upload their weight + * texture with uninitialized row padding (reinit_scaler in + * video/out/gpu/video.c, still present in mpv 0.41 and master). Mesa's CPU + * rasterizers carry NaN/Inf from that padding through zero-weight linear + * filtering, so a session randomly renders pure black depending on heap + * contents. Software GL gets mpv's LUT-free bilinear scalers instead, which + * is also what it can afford; a session option for the same key wins. */ +void applySoftwareRendererOptions(const std::set& sessionKeys) { + static const std::pair kOptions[] = { + {"scale", "bilinear"}, + {"cscale", "bilinear"}, + {"dscale", "bilinear"}, + {"sigmoid-upscaling", "no"}, + }; + std::string applied; + std::string failed; + for (const auto& [key, value] : kOptions) { + if (sessionKeys.count(key) != 0) continue; + const int result = mpv_set_property_string(g_state.mpv, key, value); + std::string& list = result < 0 ? failed : applied; + list += (list.empty() ? "" : ", ") + std::string(key) + "=" + value; + if (result < 0) { + list += std::string(" (") + mpv_error_string(result) + ")"; + } + } + std::fprintf(stderr, "software renderer: applied [%s]; failed [%s]\n", + applied.c_str(), failed.c_str()); +} + struct HelperArgs { std::string shmBase = "/impv"; std::string hwdec = "auto"; @@ -987,6 +1017,7 @@ int main(int argc, char** argv) { mpv_set_option_string(g_state.mpv, "audio-delay", args.audioDelay.c_str()); } + std::set sessionOptionKeys; if (args.mpvOptionsOnStdin) { /* `mpv-options\to000=\to001=...` in the regular command * encoding; zero-padded keys keep the application order (network @@ -1022,7 +1053,9 @@ int main(int argc, char** argv) { const std::string value = option.substr(separator + 1); const int optionResult = mpv_set_option_string( g_state.mpv, key.c_str(), value.c_str()); - if (optionResult < 0) { + if (optionResult >= 0) { + sessionOptionKeys.insert(key); + } else { emitLine(JsonWriter() .str("event", "log") .str("level", "warn") @@ -1095,6 +1128,10 @@ int main(int argc, char** argv) { .finish()); }; + g_state.pipeline.onSoftwareRenderer = [sessionOptionKeys] { + applySoftwareRendererOptions(sessionOptionKeys); + }; + g_state.viewportWidth = args.width; g_state.viewportHeight = args.height; diff --git a/docs/architecture/embedded-mpv-native.md b/docs/architecture/embedded-mpv-native.md index c8541e829..69f46a5d3 100644 --- a/docs/architecture/embedded-mpv-native.md +++ b/docs/architecture/embedded-mpv-native.md @@ -443,7 +443,12 @@ Trade-offs and constraints: string to stderr. If an early tier selects Mesa software rendering (for example, while a proprietary NVIDIA driver is reachable through the default display or GBM), it probes the remaining tiers and uses software only when - no hardware-backed context works. Windows (any arch with a helper, in + no hardware-backed context works. On a software renderer the helper sets + `scale`, `cscale` and `dscale` to `bilinear` and `sigmoid-upscaling=no` + unless a session option sets the same key: mpv's LUT scalers upload their + weight texture with uninitialized row padding (`reinit_scaler`, mpv 0.41), + and llvmpipe carries NaN/Inf from it through zero-weight filtering, so + sessions otherwise render pure black at random. Windows (any arch with a helper, in practice x64) is ported: WGL renders offscreen against a hidden window, the shm ring is a session-local named file mapping, and the reader addon compiles as C++ there (MSVC has no C11 ``). The helper diff --git a/docs/development/electron-debugging.md b/docs/development/electron-debugging.md index 1d88fc503..f4c594095 100644 --- a/docs/development/electron-debugging.md +++ b/docs/development/electron-debugging.md @@ -91,7 +91,12 @@ exit. The retained handle still provides the actual exit code and signal. The Linux portable build uploads `packaged-frame-copy-smoke` reports and traces even when the smoke fails. Check the paused-frame screenshot and trace before -classifying a zero rendered-frame signal as an infrastructure flake. +classifying a zero rendered-frame signal as an infrastructure flake. A zero +signal also attaches `frame-copy-diagnostics` (the session's stream stats, +including mpv's drop counter, and its helper rings read from `/dev/shm`) and +`mpv-log`, the session's verbose mpv log. A `latestSeq` of 0 means the helper +never published; a `latestFrameSignal` of 0 means mpv rendered black; a +visible ring frame behind a black canvas points at the preload pump. The Snap and Flatpak `--embedded-mpv-runtime-probe` launches in `build-and-make.yaml` run under GNU `timeout --verbose -k 10 300` with `ELECTRON_ENABLE_LOGGING=1`, so a main process that throws before app ready