refactor(mask): [PKG-MASK-03] isolate model runs and release stale work

packages/artplayer-plugin-danmuku-mask/src/config.js

@@ -0,0 +1,14 @@
+export default function maskConfig(option) {
+  return {
+    solutionPath: option.solutionPath || 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation',
+    modelSelection: option.modelSelection || 1,
+    smoothSegmentation: option.smoothSegmentation !== undefined ? option.smoothSegmentation : true,
+    minDetectionConfidence: option.minDetectionConfidence || 0.5,
+    minTrackingConfidence: option.minTrackingConfidence || 0.5,
+    selfieMode: option.selfieMode || false,
+    drawContour: option.drawContour || false,
+    foregroundThreshold: option.foregroundThreshold || 0.5,
+    opacity: option.opacity || 1,
+    maskBlurAmount: option.maskBlurAmount || 3,
+  }
+}

packages/artplayer-plugin-danmuku-mask/src/controller.js

@@ -0,0 +1,188 @@
+import { createOutput, releaseOutput, renderMask } from './output'
+import { loadSegmenter, releaseSegmenter } from './sdk'
+
+export default class MaskController {
+  constructor(art, config, { $video, $danmuku }) {
+    this.art = art
+    this.config = config
+    this.video = $video
+    this.layer = $danmuku
+    this.closed = art.isDestroy
+    this.run = null
+    this.tail = Promise.resolve()
+    this.start = this.start.bind(this)
+    this.stop = this.stop.bind(this)
+    this.destroy = this.destroy.bind(this)
+    this.ready = () => {
+      this.start().catch(error => console.error('Failed to start danmuku mask:', error))
+    }
+    if (!this.closed) {
+      try {
+        art.on('destroy', this.destroy)
+        if (!this.closed)
+          art.on('ready', this.ready)
+      }
+      catch (error) {
+        this.closed = true
+        this.halt()
+        try {
+          art.off('ready', this.ready)
+        }
+        catch {}
+        try {
+          art.off('destroy', this.destroy)
+        }
+        catch {}
+        throw error
+      }
+    }
+  }
+
+  active(run) {
+    return run === this.run && run.running && !this.closed && !this.art.isDestroy
+  }
+
+  release(run) {
+    if (run.releasing)
+      return run.releasing
+    const segmenter = run.segmenter
+    run.segmenter = null
+    const output = run.output
+    run.output = null
+    run.releasing = (async () => {
+      await Promise.resolve()
+      try {
+        // Frame work has settled; an SDK close must not retain this independent bitmap.
+        releaseOutput(output)
+      }
+      finally {
+        if (segmenter)
+          await releaseSegmenter(segmenter)
+      }
+    })()
+    return run.releasing
+  }
+
+  schedule(run) {
+    if (!this.active(run) || run.frame !== null || run.busy)
+      return
+    run.frame = requestAnimationFrame(() => {
+      run.frame = null
+      this.tick(run)
+    })
+  }
+
+  tick(run) {
+    if (!this.active(run) || run.busy)
+      return
+    const video = this.video
+    if (video.paused || video.ended || !(video.videoWidth > 0 && video.videoHeight > 0)) {
+      this.schedule(run)
+      return
+    }
+    run.busy = true
+    const work = (async () => {
+      try {
+        // Reserve the work slot before SDK code can synchronously reenter start/stop.
+        await Promise.resolve()
+        if (this.active(run))
+          await renderMask(run.output, video, this.layer, run.segmenter, this.config, () => this.active(run))
+      }
+      catch (error) {
+        if (this.active(run))
+          console.error('Error in segmentBody:', error)
+      }
+      finally {
+        run.busy = false
+        if (!this.active(run))
+          await this.release(run)
+        else this.schedule(run)
+      }
+    })()
+    // A restart waits for uncancellable SDK work and disposal; it never overlaps it.
+    this.tail = work.catch(error => console.warn('Failed to release danmuku mask resources:', error))
+  }
+
+  async start() {
+    if (this.closed || this.art.isDestroy)
+      return
+    if (this.run?.running)
+      return this.run.started
+    let cancel
+    const cancelled = new Promise(resolve => cancel = resolve)
+    const run = { running: true, initializing: true, busy: false, frame: null, segmenter: null, output: null, cancel, started: null }
+    const previous = this.tail
+    this.run = run
+    const initialize = (async () => {
+      try {
+        await previous
+        if (!this.active(run))
+          return
+        if (!this.video || !this.layer?.style)
+          throw new Error('Danmuku mask requires core video and danmuku template nodes')
+        run.segmenter = await loadSegmenter(this.config, () => this.active(run))
+        if (!this.active(run))
+          return
+        if (!run.segmenter) {
+          run.running = false
+          return
+        }
+        run.output = createOutput(this.layer)
+      }
+      catch (error) {
+        if (this.active(run)) {
+          run.running = false
+          throw error
+        }
+      }
+      finally {
+        run.initializing = false
+        if (!this.active(run))
+          await this.release(run)
+      }
+      if (this.active(run))
+        this.tick(run)
+    })()
+    this.tail = initialize.catch(() => {})
+    // stop settles public start promptly; eventual SDK results remain observed by initialize.
+    run.started = Promise.race([initialize, cancelled])
+    return run.started
+  }
+
+  halt() {
+    const run = this.run
+    if (run) {
+      run.running = false
+      run.cancel()
+      if (run.frame !== null) {
+        cancelAnimationFrame(run.frame)
+        run.frame = null
+      }
+      if (!run.initializing && !run.busy)
+        this.tail = this.release(run).catch(error => console.warn('Failed to release danmuku mask resources:', error))
+    }
+  }
+
+  stop() {
+    this.halt()
+    if (this.layer?.style)
+      this.layer.style.maskImage = 'none'
+  }
+
+  destroy() {
+    if (this.closed)
+      return
+    this.closed = true
+    try {
+      this.stop()
+    }
+    finally {
+      try {
+        this.art.off('ready', this.ready)
+      }
+      finally {
+        this.art.off('destroy', this.destroy)
+      }
+    }
+  }
+}

packages/artplayer-plugin-danmuku-mask/src/index.js

@@ -1,155 +1,16 @@
-import * as bodySegmentation from '@tensorflow-models/body-segmentation'
-import * as tf from '@tensorflow/tfjs-core'
-import '@tensorflow/tfjs-backend-webgl'
-import '@tensorflow/tfjs-backend-cpu'
+import maskConfig from './config'
+import MaskController from './controller'
 
 export default function artplayerPluginDanmukuMask(option = {}) {
   return (art) => {
-    const {
-      template: { $video, $danmuku },
-    } = art
-
-    let segmenter = null
-    let canvas = null
-    let ctx = null
-    let animationFrameId = null
-    let isInitialized = false
-
-    const config = {
-      solutionPath: option.solutionPath || 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation',
-      modelSelection: option.modelSelection || 1,
-      smoothSegmentation: option.smoothSegmentation !== undefined ? option.smoothSegmentation : true,
-      minDetectionConfidence: option.minDetectionConfidence || 0.5,
-      minTrackingConfidence: option.minTrackingConfidence || 0.5,
-      selfieMode: option.selfieMode || false,
-      drawContour: option.drawContour || false,
-      foregroundThreshold: option.foregroundThreshold || 0.5,
-      opacity: option.opacity || 1,
-      maskBlurAmount: option.maskBlurAmount || 3,
-    }
-
-    async function initTensorFlow() {
-      try {
-        await tf.setBackend('webgl')
-      }
-      catch (error) {
-        console.warn('WebGL backend not available, falling back to CPU', error.message)
-        await tf.setBackend('cpu')
-      }
-    }
-
-    async function initSegmenter() {
-      await initTensorFlow()
-      const model = bodySegmentation.SupportedModels.MediaPipeSelfieSegmentation
-
-      const segmenterConfig = {
-        runtime: 'mediapipe',
-        modelType: 'general',
-        solutionPath: config.solutionPath,
-        modelSelection: config.modelSelection,
-        smoothSegmentation: config.smoothSegmentation,
-        minDetectionConfidence: config.minDetectionConfidence,
-        minTrackingConfidence: config.minTrackingConfidence,
-        selfieMode: config.selfieMode,
-      }
-
-      try {
-        segmenter = await bodySegmentation.createSegmenter(model, segmenterConfig)
-        isInitialized = true
-      }
-      catch (error) {
-        console.error('Error initializing segmenter:', error)
-        isInitialized = false
-      }
-    }
-
-    function setupDanmukuStyle() {
-      Object.assign($danmuku.style, {
-        maskMode: 'alpha',
-        maskSize: 'contain',
-        maskRepeat: 'no-repeat',
-        backgroundSize: 'contain',
-        backgroundRepeat: 'no-repeat',
-      })
-    }
-
-    function createCanvas() {
-      canvas = document.createElement('canvas')
-      ctx = canvas.getContext('2d')
-    }
-
-    function makeWhiteTransparent(imageData) {
-      const data = imageData.data
-      for (let i = 0; i < data.length; i += 4) {
-        if (data[i] > 250 && data[i + 1] > 250 && data[i + 2] > 250) {
-          data[i + 3] = 0
-        }
-      }
-      return imageData
-    }
-
-    async function segmentBody() {
-      if (!isInitialized || $video.paused || $video.ended) {
-        animationFrameId = requestAnimationFrame(segmentBody)
-        return
-      }
-
-      try {
-        canvas.width = $video.videoWidth
-        canvas.height = $video.videoHeight
-
-        const segmentation = await segmenter.segmentPeople($video)
-
-        if (!segmentation || segmentation.length === 0) {
-          animationFrameId = requestAnimationFrame(segmentBody)
-          return
-        }
-
-        const foregroundColor = { r: 255, g: 255, b: 255, a: 255 }
-        const backgroundColor = { r: 0, g: 0, b: 0, a: 255 }
-        const mask = await bodySegmentation.toBinaryMask(
-          segmentation,
-          foregroundColor,
-          backgroundColor,
-          config.drawContour,
-          config.foregroundThreshold,
-        )
-
-        await bodySegmentation.drawMask(canvas, $video, mask, config.opacity, config.maskBlurAmount)
-
-        const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height)
-        ctx.putImageData(makeWhiteTransparent(imageData), 0, 0)
-        $danmuku.style.maskImage = `url(${canvas.toDataURL()})`
-      }
-      catch (error) {
-        console.error('Error in segmentBody:', error)
-      }
-
-      animationFrameId = requestAnimationFrame(segmentBody)
-    }
-
+    const { template: { $video, $danmuku } } = art
+    const controller = new MaskController(art, maskConfig(option), { $video, $danmuku })
     async function startSegmentation() {
-      if (!isInitialized) {
-        await initSegmenter()
-      }
-      if (!canvas) {
-        createCanvas()
-      }
-      setupDanmukuStyle()
-      segmentBody()
+      await controller.start()
     }
-
     function stopSegmentation() {
-      $danmuku.style.maskImage = 'none'
-      if (animationFrameId) {
-        cancelAnimationFrame(animationFrameId)
-        animationFrameId = null
-      }
+      controller.stop()
     }
-
-    art.on('ready', startSegmentation)
-    art.on('destroy', stopSegmentation)
-
     return {
       name: 'artplayerPluginDanmukuMask',
       start: startSegmentation,

packages/artplayer-plugin-danmuku-mask/src/output.js

@@ -0,0 +1,65 @@
+import { drawMask, toBinaryMask } from './sdk'
+
+export function makeWhiteTransparent(imageData) {
+  const data = imageData.data
+  for (let i = 0; i < data.length; i += 4) {
+    if (data[i] > 250 && data[i + 1] > 250 && data[i + 2] > 250)
+      data[i + 3] = 0
+  }
+  return imageData
+}
+
+export function createOutput(layer) {
+  const canvas = document.createElement('canvas')
+  try {
+    const ctx = canvas.getContext('2d')
+    if (!ctx)
+      throw new Error('Danmuku mask requires a 2D canvas context')
+    Object.assign(layer.style, {
+      maskMode: 'alpha',
+      maskSize: 'contain',
+      maskRepeat: 'no-repeat',
+      backgroundSize: 'contain',
+      backgroundRepeat: 'no-repeat',
+    })
+    return { canvas, ctx }
+  }
+  catch (error) {
+    canvas.width = 0
+    canvas.height = 0
+    throw error
+  }
+}
+
+export async function renderMask(output, video, layer, segmenter, config, active) {
+  const { canvas, ctx } = output
+  canvas.width = video.videoWidth
+  canvas.height = video.videoHeight
+  const segmentation = await segmenter.segmentPeople(video)
+  if (!active() || !segmentation || !segmentation.length)
+    return
+  const mask = await toBinaryMask(
+    segmentation,
+    { r: 255, g: 255, b: 255, a: 255 },
+    { r: 0, g: 0, b: 0, a: 255 },
+    config.drawContour,
+    config.foregroundThreshold,
+  )
+  if (!active())
+    return
+  await drawMask(canvas, video, mask, config.opacity, config.maskBlurAmount)
+  if (!active())
+    return
+  const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height)
+  ctx.putImageData(makeWhiteTransparent(imageData), 0, 0)
+  const url = canvas.toDataURL()
+  if (active())
+    layer.style.maskImage = `url(${url})`
+}
+
+export function releaseOutput(output) {
+  if (output) {
+    output.canvas.width = 0
+    output.canvas.height = 0
+  }
+}

packages/artplayer-plugin-danmuku-mask/src/sdk.js

@@ -0,0 +1,47 @@
+import * as bodySegmentation from '@tensorflow-models/body-segmentation'
+import * as tf from '@tensorflow/tfjs-core'
+import '@tensorflow/tfjs-backend-webgl'
+import '@tensorflow/tfjs-backend-cpu'
+
+export async function loadSegmenter(config, active) {
+  try {
+    await tf.setBackend('webgl')
+  }
+  catch (error) {
+    if (!active())
+      return null
+    console.warn('WebGL backend not available, falling back to CPU', error.message)
+    await tf.setBackend('cpu')
+  }
+  if (!active())
+    return null
+  try {
+    return await bodySegmentation.createSegmenter(bodySegmentation.SupportedModels.MediaPipeSelfieSegmentation, {
+      runtime: 'mediapipe',
+      modelType: 'general',
+      solutionPath: config.solutionPath,
+      modelSelection: config.modelSelection,
+      smoothSegmentation: config.smoothSegmentation,
+      minDetectionConfidence: config.minDetectionConfidence,
+      minTrackingConfidence: config.minTrackingConfidence,
+      selfieMode: config.selfieMode,
+    })
+  }
+  catch (error) {
+    if (active())
+      console.error('Error initializing segmenter:', error)
+    return null
+  }
+}
+
+export async function releaseSegmenter(segmenter) {
+  try {
+    await segmenter.dispose()
+  }
+  catch (error) {
+    console.warn('Failed to dispose danmuku mask segmenter:', error)
+  }
+}
+
+export const toBinaryMask = (...args) => bodySegmentation.toBinaryMask(...args)
+export const drawMask = (...args) => bodySegmentation.drawMask(...args)