refactor(mediabunny): [PKG-MB-05] own video iterators, frame pools and rendering

This commit is contained in:
Harvey Zhao committed 2026-09-12 19:47:46 +08:00
1 parent 5d0b128483
commit 9d037dcbf9
27 files changed
+3652 -595

No files matched your search

@@ -45,6 +45,15 @@ export default class MediaBunnyEngine implements EnginePort {
dropLateFrames: option.dropLateFrames ?? false,
poster: option.poster ?? '',
preflightRange: option.preflightRange ?? false,
onError: (error) => {
try {
this.pause()
}
catch (failure) {
console.warn('MediaBunny video failure cleanup:', failure)
}
this.reportError(error)
},
})
this.paused = true
this.ended = false
@@ -72,6 +81,7 @@ export default class MediaBunnyEngine implements EnginePort {
return
const id = ++this.loadSeq
this.#playback.invalidate()
this.video.cancelPending()
this.loadSession?.cancel()
if (id !== this.loadSeq || this.destroyed)
return
@@ -96,6 +106,7 @@ export default class MediaBunnyEngine implements EnginePort {
return
const failed = ++this.loadSeq
this.#playback.invalidate()
this.video.cancelPending()
session.cancel()
if (failed !== this.loadSeq || this.destroyed)
return
@@ -1,5 +0,0 @@
import type { VideoOptions, VideoPort } from './engine-ports'
// Exact coordinator-facing port; implementation migration and frame ownership are MB-05.
declare const VideoEngine: new (options: VideoOptions) => VideoPort
export default VideoEngine
@@ -1,272 +0,0 @@
/**
* Video Engine for MediaBunny
* Handles video frame rendering and synchronization
*/
import {
CanvasSink,
} from 'mediabunny'
import { preflightRange } from './preflight'
export default class VideoEngine {
constructor({
canvas,
ctx,
events,
timeupdateInterval = 250,
avSyncTolerance = 0.12,
dropLateFrames = false,
poster = '',
preflightRange = false,
}) {
this.canvas = canvas
this.ctx = ctx
this.events = events
this.timeupdateInterval = timeupdateInterval
this.avSyncTolerance = avSyncTolerance
this.dropLateFrames = dropLateFrames
this.poster = poster
this.preflightRange = preflightRange
// MediaBunny instances
this.input = null
this.videoSink = null
this.videoIterator = null
// Frame rendering
this.nextFrame = null
this.rafId = 0
this.asyncId = 0
// Video properties
this.width = 0
this.height = 0
this.duration = Number.NaN
// Playback state
this.audioClock = null
this.lastTimeUpdate = 0
this.stalled = false
this.playbackRate = 1
this.posterDrawn = false
this.isFetching = false
}
async preflight(url, signal, isCurrent) {
return preflightRange(url, this.preflightRange, this.events, signal, isCurrent)
}
drawPoster() {
if (!this.poster || this.posterDrawn)
return
const img = new Image()
img.onload = () => {
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height)
this.canvas.width = img.naturalWidth || this.canvas.width
this.canvas.height = img.naturalHeight || this.canvas.height
this.ctx.drawImage(img, 0, 0, this.canvas.width, this.canvas.height)
this.posterDrawn = true
}
img.src = this.poster
}
async stopIterator() {
await this.videoIterator?.return()
this.videoIterator = null
}
clear() {
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height)
}
async load(media, onMetadata) {
++this.asyncId
await this.stopIterator()
this.clear()
this.posterDrawn = false
const { input, videoTrack, duration } = media
this.input = input
this.duration = duration
if (!videoTrack) {
this.handleNoVideoTrack()
onMetadata?.()
return
}
if (videoTrack.codec === null || !(await videoTrack.canDecode())) {
this.handleNoVideoTrack()
onMetadata?.()
return
}
const transparent = await videoTrack.canBeTransparent()
this.videoSink = new CanvasSink(videoTrack, {
poolSize: 2,
fit: 'contain',
alpha: transparent,
})
this.width = videoTrack.displayWidth
this.height = videoTrack.displayHeight
this.canvas.width = this.width
this.canvas.height = this.height
onMetadata?.()
await this.resetIterator(0)
}
handleNoVideoTrack() {
this.videoSink = null
this.width = 0
this.height = 0
this.canvas.width = 0
this.canvas.height = 0
this.clear()
this.drawPoster()
}
async resetIterator(time) {
await this.stopIterator()
if (!this.videoSink)
return
this.videoIterator = this.videoSink.canvases(time)
const first = (await this.videoIterator.next()).value ?? null
const second = (await this.videoIterator.next()).value ?? null
this.nextFrame = second
if (first) {
this.ctx.drawImage(first.canvas, 0, 0)
}
else {
this.drawPoster()
}
}
async updateNextFrame(localId) {
if (!this.videoIterator || this.isFetching)
return
this.isFetching = true
try {
while (true) {
const frame = (await this.videoIterator.next()).value ?? null
if (!frame || localId !== this.asyncId)
return
const t = this.audioClock.currentTime
const tolerance = this.dropLateFrames
? Math.max(0.06, this.avSyncTolerance / Math.max(1, this.playbackRate))
: 0
if (this.dropLateFrames && frame.timestamp < t - tolerance) {
// Skip late frame
continue
}
if (frame.timestamp <= t + tolerance) {
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height)
this.ctx.drawImage(frame.canvas, 0, 0)
if (!this.dropLateFrames && frame.timestamp > t) {
this.nextFrame = null
return
}
}
else {
this.nextFrame = frame
return
}
}
}
finally {
this.isFetching = false
}
}
render() {
if (!this.audioClock)
return
const t = this.audioClock.currentTime
const now = Date.now()
// Emit timeupdate event
if (now - this.lastTimeUpdate >= this.timeupdateInterval) {
this.events.emit('timeupdate')
this.lastTimeUpdate = now
}
// Check if reached end
if (Number.isFinite(this.duration) && t >= this.duration) {
this.stop()
this.stalled = false
this.events.emit('ended')
this.events.emit('pause')
this.events.emit('canplay')
return
}
// Render next frame if ready
if (this.nextFrame && this.nextFrame.timestamp <= t) {
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height)
this.ctx.drawImage(this.nextFrame.canvas, 0, 0)
this.nextFrame = null
this.updateNextFrame(this.asyncId)
if (this.stalled) {
this.events.emit('canplay')
this.events.emit('playing')
this.stalled = false
}
}
else if (!this.nextFrame) {
this.updateNextFrame(this.asyncId)
if (!this.nextFrame && Number.isFinite(this.duration) && t < this.duration && !this.stalled) {
this.stalled = true
this.events.emit('waiting')
}
}
this.rafId = requestAnimationFrame(() => this.render())
}
start(audioEngine) {
this.audioClock = audioEngine
this.asyncId++
this.stalled = false
this.updateNextFrame(this.asyncId)
this.rafId = requestAnimationFrame(() => this.render())
}
stop() {
cancelAnimationFrame(this.rafId)
}
async seek(time) {
this.asyncId++
await this.resetIterator(time)
}
setPlaybackRate(rate) {
this.playbackRate = Math.max(0.1, Number(rate) || 1)
}
destroy() {
this.asyncId++
this.stop()
this.stopIterator()
this.posterDrawn = false
this.videoSink = null
this.input = null
}
}
@@ -0,0 +1,316 @@
import type { Input, WrappedCanvas } from 'mediabunny'
import type { VideoOptions, VideoPort } from './engine-ports'
import type EventTarget from './EventTarget'
import type { PlaybackMedia } from './media'
import type { FrameIterator } from './video-frames'
import { CanvasSink } from 'mediabunny'
import { preflightRange } from './preflight'
import { readCanvas, releaseIterator } from './video-frames'
import Poster from './video-poster'
import Renderer from './video-renderer'
export default class VideoEngine implements VideoPort {
declare canvas: HTMLCanvasElement
declare ctx: CanvasRenderingContext2D | null
declare events: EventTarget
declare timeupdateInterval: number
declare avSyncTolerance: number
declare dropLateFrames: boolean
declare poster: string
declare preflightRange: boolean
declare input: Input | null
declare videoSink: CanvasSink | null
declare videoIterator: FrameIterator | null
declare nextFrame: WrappedCanvas | null
declare rafId: number
declare asyncId: number
declare width: number
declare height: number
declare duration: number
declare audioClock: { readonly currentTime: number } | null
declare lastTimeUpdate: number
declare stalled: boolean
declare playbackRate: number
declare posterDrawn: boolean
declare isFetching: boolean
#generation = 0
#iteration = 0
#loading: Promise<void> | null = null
#createSink: (() => CanvasSink) | null = null
#usedSink: CanvasSink | null = null
#destroyed = false
#preparing = false
#poster = new Poster()
#renderer: Renderer
#onError: ((error: unknown) => void) | undefined
constructor({ canvas, ctx, events, timeupdateInterval = 250, avSyncTolerance = 0.12, dropLateFrames = false, poster = '', preflightRange = false, onError }: VideoOptions) {
this.canvas = canvas
this.ctx = ctx
this.events = events
this.timeupdateInterval = timeupdateInterval
this.avSyncTolerance = avSyncTolerance
this.dropLateFrames = dropLateFrames
this.poster = poster
this.preflightRange = preflightRange
this.input = null
this.videoSink = null
this.videoIterator = null
this.nextFrame = null
this.rafId = 0
this.asyncId = 0
this.width = 0
this.height = 0
this.duration = Number.NaN
this.audioClock = null
this.lastTimeUpdate = 0
this.stalled = false
this.playbackRate = 1
this.posterDrawn = false
this.isFetching = false
this.#onError = onError
this.#renderer = new Renderer(this, () => !this.#preparing, frame => this.#draw(frame, true), error => this.#report(error))
}
#current(generation: number): boolean {
return !this.#destroyed && this.#generation === generation
}
#context(): CanvasRenderingContext2D {
if (!this.ctx)
throw new Error('Canvas 2D context is unavailable.')
return this.ctx
}
#report(error: unknown): void {
if (this.#destroyed)
return
this.#release()
if (this.#onError)
this.#onError(error)
else
this.events.emit('error', error)
}
#release(): void {
void this.stopIterator().catch(error => console.warn('MediaBunny iterator cleanup error:', error))
}
#draw(frame: WrappedCanvas, clear: boolean): void {
this.#poster.cancel()
if (clear)
this.clear()
this.#context().drawImage(frame.canvas, 0, 0)
}
async preflight(source: unknown, signal?: AbortSignal, current?: () => boolean): Promise<boolean> {
return preflightRange(source, this.preflightRange, this.events, signal, current)
}
drawPoster(): void {
if (this.#destroyed || !this.poster || this.posterDrawn)
return
const generation = this.#generation
this.#poster.draw(this.poster, () => this.#current(generation), (image) => {
this.clear()
this.canvas.width = image.naturalWidth || this.canvas.width
this.canvas.height = image.naturalHeight || this.canvas.height
this.#context().drawImage(image, 0, 0, this.canvas.width, this.canvas.height)
this.posterDrawn = true
}, error => this.#report(error))
}
async stopIterator(): Promise<void> {
const iterator = this.videoIterator
this.videoIterator = null
this.nextFrame = null
await releaseIterator(iterator)
}
clear(): void {
this.#context().clearRect(0, 0, this.canvas.width, this.canvas.height)
}
cancelPending(): void {
this.#generation++
this.#iteration++
this.stop()
this.#release()
this.#poster.cancel()
this.#preparing = false
this.#loading = null
this.#createSink = null
this.#usedSink = null
this.videoSink = null
this.input = null
}
async load(media: PlaybackMedia, onMetadata?: () => void): Promise<void> {
if (this.#destroyed)
return
this.cancelPending()
const generation = this.#generation
const iteration = this.#iteration
this.#preparing = true
const pending = Promise.resolve().then(() => this.#load(media, generation, iteration, onMetadata))
this.#loading = pending
try {
await pending
}
finally {
if (this.#loading === pending)
this.#loading = null
}
}
async #load(media: PlaybackMedia, generation: number, iteration: number, onMetadata?: () => void): Promise<void> {
if (!this.#current(generation))
return
try {
this.clear()
this.posterDrawn = false
const { input, videoTrack, duration } = media
this.input = input
this.duration = duration
if (!videoTrack) {
this.handleNoVideoTrack()
onMetadata?.()
return
}
const decodable = videoTrack.codec !== null && await videoTrack.canDecode()
if (!this.#current(generation))
return
if (!decodable) {
this.handleNoVideoTrack()
onMetadata?.()
return
}
const transparent = await videoTrack.canBeTransparent()
if (!this.#current(generation))
return
this.#createSink = () => new CanvasSink(videoTrack, { poolSize: 2, fit: 'contain', alpha: transparent })
this.videoSink = this.#createSink()
this.width = videoTrack.displayWidth
this.height = videoTrack.displayHeight
this.canvas.width = this.width
this.canvas.height = this.height
onMetadata?.()
if (this.#current(generation))
await this.#reset(0, generation, iteration)
}
catch (error) {
if (this.#current(generation)) {
this.#release()
throw error
}
}
finally {
if (this.#current(generation) && this.#iteration === iteration)
this.#preparing = false
}
}
handleNoVideoTrack(): void {
if (this.#destroyed)
return
this.videoSink = null
this.width = 0
this.height = 0
this.canvas.width = 0
this.canvas.height = 0
this.clear()
this.drawPoster()
}
async #reset(time: number, generation: number, iteration: number): Promise<void> {
if (!this.#current(generation) || this.#iteration !== iteration)
return
this.#release()
if (this.#createSink && this.videoSink === this.#usedSink)
this.videoSink = this.#createSink()
const sink = this.videoSink
if (!sink || !this.#current(generation))
return
this.#usedSink = sink
const iterator = sink.canvases(time)
this.videoIterator = iterator
const current = () => this.#current(generation) && this.#iteration === iteration && this.videoIterator === iterator
const first = await readCanvas(iterator)
if (!current())
return
const second = await readCanvas(iterator)
if (!current())
return
this.nextFrame = second
if (first)
this.#draw(first, false)
else
this.drawPoster()
}
async resetIterator(time: number): Promise<void> {
if (this.#destroyed)
return
const generation = this.#generation
const iteration = ++this.#iteration
const loading = this.#loading
this.stop()
this.#poster.cancel()
this.#preparing = true
try {
if (loading)
await loading
await this.#reset(time, generation, iteration)
}
catch (error) {
if (this.#current(generation) && this.#iteration === iteration) {
this.#release()
throw error
}
}
finally {
if (this.#current(generation) && this.#iteration === iteration)
this.#preparing = false
}
}
async updateNextFrame(localId: number): Promise<void> {
await this.#renderer.update(localId)
}
render(): void {
this.#renderer.render()
}
start(audio: { readonly currentTime: number }): void {
this.#renderer.start(audio)
}
stop(): void {
this.#renderer.stop()
}
async seek(time: number): Promise<void> {
await this.resetIterator(time)
}
setPlaybackRate(rate: number): void {
this.playbackRate = Math.max(0.1, Number(rate) || 1)
}
destroy(): void {
if (this.#destroyed)
return
this.#destroyed = true
this.#generation++
this.#renderer.destroy()
this.#poster.cancel()
this.#release()
this.posterDrawn = false
this.#createSink = null
this.#usedSink = null
this.videoSink = null
this.input = null
this.audioClock = null
}
}
@@ -18,6 +18,7 @@ export interface AudioPort {
}
export interface VideoPort {
cancelPending: () => void
width: number
height: number
duration: number
@@ -32,6 +33,7 @@ export interface VideoPort {
}
export interface VideoOptions {
onError?: (error: unknown) => void
canvas: HTMLCanvasElement
ctx: CanvasRenderingContext2D | null
events: EventTarget
@@ -0,0 +1,37 @@
import type { WrappedCanvas } from 'mediabunny'
export type FrameIterator = Pick<AsyncGenerator<WrappedCanvas, void, unknown>, 'next' | 'return'>
const closing = new WeakMap<FrameIterator, Promise<void>>()
const reading = new WeakMap<FrameIterator, Promise<WrappedCanvas | null>>()
export function releaseIterator(iterator: FrameIterator | null): Promise<void> {
if (!iterator)
return Promise.resolve()
const previous = closing.get(iterator)
if (previous)
return previous
const pending = Promise.resolve().then(() => iterator.return()).then(() => {})
closing.set(iterator, pending)
return pending
}
export function readCanvas(iterator: FrameIterator): Promise<WrappedCanvas | null> {
const previous = reading.get(iterator)
if (previous)
return previous
const pending = Promise.resolve().then(() => iterator.next()).then(result => result.value ?? null)
reading.set(iterator, pending)
const clear = () => {
if (reading.get(iterator) === pending)
reading.delete(iterator)
}
pending.then(clear, clear)
return pending
}
export function frameAction(timestamp: number, time: number, dropLateFrames: boolean, avSyncTolerance: number, playbackRate: number): 'skip' | 'draw' | 'queue' {
const tolerance = dropLateFrames ? Math.max(0.06, avSyncTolerance / Math.max(1, playbackRate)) : 0
if (dropLateFrames && timestamp < time - tolerance)
return 'skip'
return timestamp <= time + tolerance ? 'draw' : 'queue'
}
@@ -0,0 +1,33 @@
export default class Poster {
private image: HTMLImageElement | null = null
cancel(): void {
if (!this.image)
return
this.image.onload = null
this.image.onerror = null
this.image = null
}
draw(source: string, current: () => boolean, draw: (image: HTMLImageElement) => void, failure: (error: unknown) => void): void {
this.cancel()
const image = new Image()
this.image = image
image.onload = () => {
if (this.image !== image || !current())
return
this.cancel()
try {
draw(image)
}
catch (error) {
failure(error)
}
}
image.onerror = () => {
if (this.image === image)
this.cancel()
}
image.src = source
}
}
@@ -0,0 +1,192 @@
import type { WrappedCanvas } from 'mediabunny'
import type EventTarget from './EventTarget'
import type { FrameIterator } from './video-frames'
import { publish } from './readiness'
import { frameAction, readCanvas } from './video-frames'
export interface RenderHost {
events: EventTarget
audioClock: { readonly currentTime: number } | null
videoIterator: FrameIterator | null
nextFrame: WrappedCanvas | null
rafId: number
asyncId: number
duration: number
lastTimeUpdate: number
timeupdateInterval: number
avSyncTolerance: number
dropLateFrames: boolean
playbackRate: number
stalled: boolean
isFetching: boolean
}
export default class Renderer {
private stopped = false
private closed = false
private scheduled: number | null = null
private fetching: object | null = null
constructor(private host: RenderHost, private ready: () => boolean, private draw: (frame: WrappedCanvas) => void, private failure: (error: unknown) => void) {}
private current(id: number): boolean {
return !this.closed && !this.stopped && this.host.asyncId === id
}
async update(id: number): Promise<void> {
const iterator = this.host.videoIterator
if (!iterator || !this.ready() || !this.current(id) || this.host.isFetching)
return
const request = {}
this.fetching = request
this.host.isFetching = true
const current = () => this.current(id) && this.host.videoIterator === iterator
try {
while (current()) {
const frame = await readCanvas(iterator)
if (!current() || !frame)
return
const time = this.host.audioClock?.currentTime
if (time === undefined)
return
const action = frameAction(frame.timestamp, time, this.host.dropLateFrames, this.host.avSyncTolerance, this.host.playbackRate)
if (action === 'skip')
continue
if (action === 'draw') {
this.draw(frame)
}
else {
this.host.nextFrame = frame
return
}
}
}
catch (error) {
if (current())
throw error
}
finally {
if (this.fetching === request) {
this.fetching = null
this.host.isFetching = false
}
}
}
private request(id: number): void {
void this.update(id).catch((error) => {
if (!this.current(id))
return
this.stop()
this.report(error)
})
}
private report(error: unknown): void {
try {
this.failure(error)
}
catch (failure) {
console.warn('MediaBunny video error listener:', failure)
}
}
private cancelFrame(): void {
if (this.scheduled === null)
return
cancelAnimationFrame(this.scheduled)
this.scheduled = null
}
private schedule(id: number): void {
if (!this.current(id))
return
this.cancelFrame()
const frame = requestAnimationFrame(() => {
if (this.scheduled !== frame || !this.current(id))
return
this.scheduled = null
try {
this.render()
}
catch (error) {
this.stop()
this.report(error)
}
})
this.scheduled = frame
this.host.rafId = frame
}
render(): void {
const id = this.host.asyncId
const current = () => this.current(id)
if (!current() || !this.host.audioClock)
return
const time = this.host.audioClock.currentTime
const now = Date.now()
if (now - this.host.lastTimeUpdate >= this.host.timeupdateInterval) {
this.host.events.emit('timeupdate')
if (!current())
return
this.host.lastTimeUpdate = now
}
if (!this.ready()) {
this.schedule(id)
return
}
if (Number.isFinite(this.host.duration) && time >= this.host.duration) {
this.stop()
this.host.stalled = false
const ended = this.host.asyncId
publish(this.host.events, () => !this.closed && this.host.asyncId === ended, ['ended', 'pause', 'canplay'])
return
}
if (this.host.nextFrame && this.host.nextFrame.timestamp <= time) {
this.draw(this.host.nextFrame)
this.host.nextFrame = null
this.request(id)
if (this.host.stalled) {
publish(this.host.events, current, ['canplay', 'playing'])
if (!current())
return
this.host.stalled = false
}
}
else if (!this.host.nextFrame) {
this.request(id)
if (Number.isFinite(this.host.duration) && time < this.host.duration && !this.host.stalled) {
this.host.stalled = true
this.host.events.emit('waiting')
}
}
this.schedule(id)
}
start(clock: { readonly currentTime: number }): void {
if (this.closed)
return
this.stop()
this.stopped = false
this.host.audioClock = clock
this.host.stalled = false
if (!this.host.nextFrame)
this.request(this.host.asyncId)
this.schedule(this.host.asyncId)
}
stop(): void {
this.stopped = true
this.host.asyncId++
this.cancelFrame()
this.fetching = null
this.host.isFetching = false
}
destroy(): void {
if (this.closed)
return
this.stop()
this.closed = true
}
}