From 91e5a88a5bd64877f2862cb1b5bd003188ee7ad0 Mon Sep 17 00:00:00 2001 From: 4gray Date: Sat, 11 Jul 2026 01:01:22 +0200 Subject: [PATCH] feat(embedded-mpv): aspect-fit rendering in the frame-copy helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The helper now observes dwidth/dheight and renders its FBO at the aspect-fit size of the video inside the requested viewport, bumping a shm generation on change — letterbox bars are never baked into frames (the VOD watch shell's ~2:1 box no longer shows black side bars; the canvas background is transparent so the sides show the app surface, while fullscreen keeps its black backdrop). Frames also get smaller than the viewport when aspects differ, trimming copy cost. Aspect override changes refit automatically. Snapshots now carry videoWidth/videoHeight, and the adapter forwards IPTVNATOR_EMBEDDED_MPV_AUDIO_DELAY to mpv's audio-delay for lip-sync tuning until proper calibration lands. Co-Authored-By: Claude Fable 5 --- .../native/helper/mpv_frame_helper.cpp | 66 ++++++++++++++++++- .../embedded-mpv-frame-copy.adapter.ts | 9 +++ .../services/embedded-mpv-native.service.ts | 9 +++ .../src/lib/embedded-mpv-session.interface.ts | 3 + .../embedded-mpv-player.component.scss | 9 ++- 5 files changed, 92 insertions(+), 4 deletions(-) diff --git a/apps/electron-backend/native/helper/mpv_frame_helper.cpp b/apps/electron-backend/native/helper/mpv_frame_helper.cpp index 0a4068fb7..cff588dd6 100644 --- a/apps/electron-backend/native/helper/mpv_frame_helper.cpp +++ b/apps/electron-backend/native/helper/mpv_frame_helper.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "frame_helper_io.h" @@ -55,6 +56,8 @@ struct SnapshotState { double positionSeconds = 0; double durationSeconds = -1; /* <0 => null */ double volume = 1; /* 0..1 */ + int64_t videoWidth = 0; /* mpv dwidth/dheight; 0 = unknown */ + int64_t videoHeight = 0; std::string streamUrl; std::vector audioTracks; int64_t selectedAudioTrackId = -1; @@ -80,6 +83,11 @@ struct HelperState { bool dirty = true; std::string lastEmittedStatus; uint64_t lastEmitNs = 0; + /* Requested viewport in device pixels; the render target is the + * aspect-fit of the video inside it so letterbox bars are never + * rendered (and frames stay as small as possible). */ + int viewportWidth = 0; + int viewportHeight = 0; std::atomic running{true}; }; @@ -124,6 +132,10 @@ std::string composeSnapshotLocked() { writer.str("event", "snapshot"); writer.str("status", s.status); writer.num("positionSeconds", std::max(0.0, s.positionSeconds)); + if (s.videoWidth > 0 && s.videoHeight > 0) { + writer.num("videoWidth", (double)s.videoWidth); + writer.num("videoHeight", (double)s.videoHeight); + } if (s.durationSeconds >= 0) { writer.num("durationSeconds", s.durationSeconds); } else { @@ -183,6 +195,33 @@ void maybeEmitSnapshot(bool force = false) { void markDirtyLocked() { g_state.dirty = true; } +/* Aspect-fit of the current video inside the requested viewport. Caller + * holds g_state.mutex. Letterbox bars are handled by the renderer layout, + * not baked into frames. */ +std::pair computeRenderSizeLocked() { + const int vpW = g_state.viewportWidth; + const int vpH = g_state.viewportHeight; + const int64_t vw = g_state.snapshot.videoWidth; + const int64_t vh = g_state.snapshot.videoHeight; + if (vw < 16 || vh < 16 || vpW < 16 || vpH < 16) { + return {vpW, vpH}; + } + const double scale = + std::min(vpW / (double)vw, vpH / (double)vh); + const int width = + std::max(16, (int)llround((double)vw * scale)) & ~1; + const int height = + std::max(16, (int)llround((double)vh * scale)) & ~1; + return {width, height}; +} + +void applyRenderSizeLocked() { + const auto [width, height] = computeRenderSizeLocked(); + if (width >= 16 && height >= 16) { + g_state.pipeline.requestResize(width, height); + } +} + /* ---- track-list parsing (ported from embedded_mpv.mm) ------------------- */ void parseTrackNode(const mpv_node& trackNode, const char* wantedType, @@ -299,6 +338,15 @@ void handlePropertyChange(const mpv_event_property& property) { } else if (name == "speed" && property.format == MPV_FORMAT_DOUBLE && property.data) { s.playbackSpeed = *static_cast(property.data); + } else if ((name == "dwidth" || name == "dheight") && + property.format == MPV_FORMAT_INT64 && property.data) { + const int64_t value = *static_cast(property.data); + if (name == "dwidth") { + s.videoWidth = value; + } else { + s.videoHeight = value; + } + applyRenderSizeLocked(); } else if (name == "video-aspect-override" && property.format == MPV_FORMAT_STRING && property.data) { const char* value = *static_cast(property.data); @@ -526,7 +574,10 @@ void handleCommand(const Command& command) { const int width = (int)command.getDouble("width", 0); const int height = (int)command.getDouble("height", 0); if (width >= 16 && height >= 16) { - g_state.pipeline.requestResize(width, height); + std::lock_guard lock(g_state.mutex); + g_state.viewportWidth = width; + g_state.viewportHeight = height; + applyRenderSizeLocked(); } } else if (command.name == "quit") { g_state.running.store(false); @@ -552,6 +603,7 @@ void onRenderUpdate(void* pipeline) { struct HelperArgs { std::string shmBase = "/impv"; std::string hwdec = "auto"; + std::string audioDelay; /* seconds, mpv audio-delay passthrough */ int width = 1280; int height = 720; double volume = 1; @@ -569,6 +621,7 @@ HelperArgs parseArgs(int argc, char** argv) { else if (arg == "--height") args.height = std::atoi(next().c_str()); else if (arg == "--volume") args.volume = std::atof(next().c_str()); else if (arg == "--hwdec") args.hwdec = next(); + else if (arg == "--audio-delay") args.audioDelay = next(); } args.width = std::max(16, args.width); args.height = std::max(16, args.height); @@ -597,6 +650,12 @@ int main(int argc, char** argv) { mpv_set_option_string(g_state.mpv, "idle", "yes"); mpv_set_option_string(g_state.mpv, "input-default-bindings", "no"); mpv_set_option_string(g_state.mpv, "osc", "no"); + if (!args.audioDelay.empty()) { + /* Compensates the frame-copy video-path latency to restore + * lip-sync; calibrated per platform, see the architecture doc. */ + mpv_set_option_string(g_state.mpv, "audio-delay", + args.audioDelay.c_str()); + } if (mpv_initialize(g_state.mpv) < 0) { emitLine(JsonWriter() .str("event", "fatal") @@ -623,6 +682,8 @@ int main(int argc, char** argv) { mpv_observe_property(g_state.mpv, 10, "video-aspect-override", MPV_FORMAT_STRING); mpv_observe_property(g_state.mpv, 11, "eof-reached", MPV_FORMAT_FLAG); + mpv_observe_property(g_state.mpv, 12, "dwidth", MPV_FORMAT_INT64); + mpv_observe_property(g_state.mpv, 13, "dheight", MPV_FORMAT_INT64); g_state.pipeline.onGenerationChanged = [](const std::string& name, int width, int height, @@ -636,6 +697,9 @@ int main(int argc, char** argv) { .finish()); }; + g_state.viewportWidth = args.width; + g_state.viewportHeight = args.height; + std::string startError; if (!g_state.pipeline.start(g_state.mpv, args.shmBase, args.width, args.height, startError)) { diff --git a/apps/electron-backend/src/app/services/embedded-mpv-frame-copy.adapter.ts b/apps/electron-backend/src/app/services/embedded-mpv-frame-copy.adapter.ts index 121442bff..2762a8536 100644 --- a/apps/electron-backend/src/app/services/embedded-mpv-frame-copy.adapter.ts +++ b/apps/electron-backend/src/app/services/embedded-mpv-frame-copy.adapter.ts @@ -116,6 +116,15 @@ export class EmbeddedMpvFrameCopyAdapter implements NativeEmbeddedMpvAddon { String(height), '--volume', String(Math.min(Math.max(initialVolume ?? 1, 0), 1)), + // Lip-sync compensation for the video path's added latency + // (~10 ms measured on M1 Pro); tunable until calibration + // lands, see the architecture doc. + ...(process.env.IPTVNATOR_EMBEDDED_MPV_AUDIO_DELAY + ? [ + '--audio-delay', + process.env.IPTVNATOR_EMBEDDED_MPV_AUDIO_DELAY, + ] + : []), ], { stdio: ['pipe', 'pipe', 'pipe'] } ); diff --git a/apps/electron-backend/src/app/services/embedded-mpv-native.service.ts b/apps/electron-backend/src/app/services/embedded-mpv-native.service.ts index 22e85be22..39f17b671 100644 --- a/apps/electron-backend/src/app/services/embedded-mpv-native.service.ts +++ b/apps/electron-backend/src/app/services/embedded-mpv-native.service.ts @@ -41,6 +41,8 @@ export interface NativeEmbeddedMpvSessionSnapshot { selectedSubtitleTrackId?: number | null; playbackSpeed?: number; aspectOverride?: string; + videoWidth?: number; + videoHeight?: number; recording?: EmbeddedMpvRecordingState; error?: string; } @@ -768,6 +770,13 @@ export class EmbeddedMpvNativeService { typeof snapshot.aspectOverride === 'string' ? snapshot.aspectOverride : 'no', + ...(typeof snapshot.videoWidth === 'number' && + typeof snapshot.videoHeight === 'number' + ? { + videoWidth: snapshot.videoWidth, + videoHeight: snapshot.videoHeight, + } + : {}), recording: snapshot.recording ?? { active: false }, startedAt: session.startedAt, updatedAt: new Date().toISOString(), diff --git a/libs/shared/interfaces/src/lib/embedded-mpv-session.interface.ts b/libs/shared/interfaces/src/lib/embedded-mpv-session.interface.ts index 0a02540bd..9f9a3330f 100644 --- a/libs/shared/interfaces/src/lib/embedded-mpv-session.interface.ts +++ b/libs/shared/interfaces/src/lib/embedded-mpv-session.interface.ts @@ -93,6 +93,9 @@ export interface EmbeddedMpvSession { selectedSubtitleTrackId: number | null; playbackSpeed: number; aspectOverride: string; + /** Source video size (mpv dwidth/dheight); frame-copy engine only. */ + videoWidth?: number; + videoHeight?: number; recording?: EmbeddedMpvRecordingState; startedAt: string; updatedAt: string; diff --git a/libs/ui/playback/src/lib/embedded-mpv-player/embedded-mpv-player.component.scss b/libs/ui/playback/src/lib/embedded-mpv-player/embedded-mpv-player.component.scss index bf8d561fe..23ffa5560 100644 --- a/libs/ui/playback/src/lib/embedded-mpv-player/embedded-mpv-player.component.scss +++ b/libs/ui/playback/src/lib/embedded-mpv-player/embedded-mpv-player.component.scss @@ -52,15 +52,18 @@ } // Frame-copy engine: video frames are painted onto this canvas by the -// preload frame pump. The backing store is sized to the viewport in device -// pixels; CSS letterboxes it inside the viewport. +// preload frame pump. The helper renders at the aspect-fit size of the +// video inside the viewport (no letterbox bars are baked into frames), so +// object-fit: contain centers the content and the sides show the app +// surface instead of TV-style black bars. Fullscreen keeps a black +// backdrop via the viewport rule above. .embedded-mpv-player__frame-canvas { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: contain; - background: #000; + background: transparent; } .embedded-mpv-player--controls-enabled .embedded-mpv-player__viewport {