test(danmuku): [PKG-DANMUKU-08] cover old plugin combinations and expose model load failures

test/browser/danmuku-mask-native.spec.js

@@ -1,7 +1,8 @@
 import assert from 'node:assert/strict'
 import fs from 'node:fs'
 import process from 'node:process'
-import { hash } from '../../refactor/scripts/releases.mjs'
+import { ensureArchive, hash, readMember } from '../../refactor/scripts/releases.mjs'
+import { observeDanmukuLoad } from '../helpers/danmuku-combination-load.js'
 import { compilePackage } from '../helpers/load.js'
 import { expect, test } from './fixtures.js'
 
@@ -20,11 +21,18 @@ const assets = baseline.sdk.assets.map(({ file, sha256 }) => {
 const media = 'docs/assets/sample/steve-jobs.mp4'
 const danmukuFile = process.env.ARTPLAYER_DANMUKU_ARTIFACT || 'source UMD'
 let danmukuCode
+const danmukuImplementations = new Map()
 test.beforeAll(async () => {
   ;[code, danmukuCode] = await Promise.all([
     process.env.ARTPLAYER_MASK_ARTIFACT ? fs.readFileSync(artifact, 'utf8') : compilePackage('artplayer-plugin-danmuku-mask', 'umd'),
     process.env.ARTPLAYER_DANMUKU_ARTIFACT ? fs.readFileSync(danmukuFile, 'utf8') : compilePackage('artplayer-plugin-danmuku', 'umd'),
   ])
+  const release = JSON.parse(fs.readFileSync('refactor/baselines/danmuku-release.json', 'utf8')).release
+  const member = 'package/dist/artplayer-plugin-danmuku.js'
+  const published = readMember(await ensureArchive(release), member)
+  assert.equal(hash(published), release.files[member])
+  danmukuImplementations.set('published', { file: `npm ${release.version}`, code: published.toString() })
+  danmukuImplementations.set('candidate', { file: danmukuFile, code: danmukuCode })
 })
 
 for (const core of ['candidate', 'published', 'published-5.3.1-beta.1']) {
@@ -132,138 +140,146 @@ for (const core of ['candidate', 'published', 'published-5.3.1-beta.1']) {
     }
   })
 
-  test(`Mask actual model and Danmuku share the layer through pause, seek and web fullscreen with ${core} core`, async ({ page }, testInfo) => {
-    test.setTimeout(90000)
-    await page.goto(`/test/player.html?core=${core}`)
-    await page.addScriptTag({ content: danmukuCode })
-    await page.addScriptTag({ content: code })
-    const unexpected = []
-    const origin = new URL(testInfo.project.use.baseURL).origin
-    await page.route('**/*', (route) => {
-      if (new URL(route.request().url()).origin !== origin) {
-        unexpected.push(route.request().url())
-        return route.abort()
-      }
-      return route.continue()
-    })
-    await page.evaluate(() => {
-      window.maskCombination = { times: [], visible: [], snapshots: [], samples: [] }
-      const encode = HTMLCanvasElement.prototype.toDataURL
-      HTMLCanvasElement.prototype.toDataURL = function (...args) {
-        const result = encode.apply(this, args)
-        window.maskCombination.times.push(window.art?.currentTime)
-        return result
-      }
-      window.art = new window.Artplayer({
-        container: '.player',
-        url: '/assets/sample/steve-jobs.mp4',
-        muted: true,
-        fullscreenWeb: true,
-        plugins: [window.artplayerPluginDanmuku({ danmuku: [{ text: 'Native mask combination', time: 6, mode: 1 }], heatmap: false }), window.artplayerPluginDanmukuMask({ solutionPath: '/assets/@mediapipe/selfie_segmentation' })],
+  for (const implementation of ['published', 'candidate']) {
+    test(`Mask actual model and Danmuku share the layer through pause, seek and web fullscreen with ${core} core / ${implementation} plugin`, async ({ page }, testInfo) => {
+      const danmuku = danmukuImplementations.get(implementation)
+      test.setTimeout(90000)
+      await page.goto(`/test/player.html?core=${core}`)
+      await page.addScriptTag({ content: danmuku.code })
+      await page.addScriptTag({ content: code })
+      const unexpected = []
+      let load
+      const origin = new URL(testInfo.project.use.baseURL).origin
+      await page.route('**/*', (route) => {
+        if (new URL(route.request().url()).origin !== origin) {
+          unexpected.push(route.request().url())
+          return route.abort()
+        }
+        return route.continue()
       })
-      window.maskCombination.layer = window.art.template.$danmuku
-      const owner = window.art.plugins.artplayerPluginDanmuku.config({})
-      const ready = Object.getOwnPropertyDescriptor(Object.getPrototypeOf(owner), 'readys').get
-      Object.defineProperty(owner, 'readys', { configurable: true, get() {
-        const rows = ready.call(this)
-        window.maskCombination.samples.push({ time: window.art.currentTime, wall: performance.now(), rows: rows.map(row => ({ time: row.time, text: row.text })) })
-        return rows
-      } })
-      window.art.on('artplayerPluginDanmuku:visible', (row) => {
-        const node = row.$ref
-        const rect = node?.getBoundingClientRect()
-        window.maskCombination.visible.push({ text: row.text, connected: node?.isConnected, inLayer: node ? window.maskCombination.layer.contains(node) : false, width: rect?.width, height: rect?.height, display: node ? getComputedStyle(node).display : null })
+      await page.evaluate(() => {
+        window.maskCombination = { times: [], visible: [], snapshots: [], samples: [] }
+        const encode = HTMLCanvasElement.prototype.toDataURL
+        HTMLCanvasElement.prototype.toDataURL = function (...args) {
+          const result = encode.apply(this, args)
+          window.maskCombination.times.push(window.art?.currentTime)
+          return result
+        }
+        window.art = new window.Artplayer({
+          container: '.player',
+          url: '/assets/sample/steve-jobs.mp4',
+          muted: true,
+          fullscreenWeb: true,
+          plugins: [window.artplayerPluginDanmuku({ danmuku: [{ text: 'Native mask combination', time: 6, mode: 1 }], heatmap: false }), window.artplayerPluginDanmukuMask({ solutionPath: '/assets/@mediapipe/selfie_segmentation' })],
+        })
+        window.maskCombination.layer = window.art.template.$danmuku
+        const owner = window.art.plugins.artplayerPluginDanmuku.config({})
+        const ready = Object.getOwnPropertyDescriptor(Object.getPrototypeOf(owner), 'readys').get
+        Object.defineProperty(owner, 'readys', { configurable: true, get() {
+          const rows = ready.call(this)
+          window.maskCombination.samples.push({ time: window.art.currentTime, wall: performance.now(), rows: rows.map(row => ({ time: row.time, text: row.text })) })
+          return rows
+        } })
+        window.art.on('artplayerPluginDanmuku:visible', (row) => {
+          const node = row.$ref
+          const rect = node?.getBoundingClientRect()
+          window.maskCombination.visible.push({ text: row.text, connected: node?.isConnected, inLayer: node ? window.maskCombination.layer.contains(node) : false, width: rect?.width, height: rect?.height, display: node ? getComputedStyle(node).display : null })
+        })
+        document.querySelector('#play').onclick = () => window.art.play()
+        document.querySelector('#pause').onclick = () => window.art.pause()
+        window.maskSnapshot = (phase) => {
+          const layer = window.art.template.$danmuku
+          const rect = layer.getBoundingClientRect()
+          const snapshot = { phase, time: window.art.currentTime, outputs: window.maskCombination.times.length, sameLayer: layer === window.maskCombination.layer, mask: layer.style.maskImage.startsWith('url('), mode: getComputedStyle(layer).maskMode, width: rect.width, height: rect.height, fullscreenWeb: window.art.fullscreenWeb }
+          window.maskCombination.snapshots.push(snapshot)
+          return snapshot
+        }
       })
-      document.querySelector('#play').onclick = () => window.art.play()
-      document.querySelector('#pause').onclick = () => window.art.pause()
-      window.maskSnapshot = (phase) => {
-        const layer = window.art.template.$danmuku
-        const rect = layer.getBoundingClientRect()
-        const snapshot = { phase, time: window.art.currentTime, outputs: window.maskCombination.times.length, sameLayer: layer === window.maskCombination.layer, mask: layer.style.maskImage.startsWith('url('), mode: getComputedStyle(layer).maskMode, width: rect.width, height: rect.height, fullscreenWeb: window.art.fullscreenWeb }
-        window.maskCombination.snapshots.push(snapshot)
-        return snapshot
+      try {
+        await expect.poll(() => page.evaluate(() => window.art.duration)).toBeGreaterThan(20)
+        await page.evaluate(() => {
+          window.art.currentTime = 5
+        })
+        await page.locator('#play').click()
+        await expect.poll(() => page.evaluate(() => window.maskCombination.times.length), { timeout: 30000 }).toBeGreaterThan(2)
+        // Keep the startup row and sampling history as diagnostics: native model
+        // initialization may block its narrow existing timestamp window. Exercise
+        // the ongoing combination after the real model has become operational.
+        await page.evaluate(async () => {
+          const time = window.art.currentTime + 1
+          window.maskCombination.scheduled = time
+          await window.art.plugins.artplayerPluginDanmuku.load([{ text: 'Ready mask combination', time, mode: 1 }])
+        })
+        await expect.poll(() => page.evaluate(() => window.maskCombination.visible.map(row => row.text))).toContain('Ready mask combination')
+        const row = await page.evaluate(() => window.maskCombination.visible.find(row => row.text === 'Ready mask combination'))
+        expect(row.connected && row.inLayer).toBe(true)
+        expect(row.width).toBeGreaterThan(0)
+        expect(row.height).toBeGreaterThan(0)
+        expect(row.display).not.toBe('none')
+        const initial = await page.evaluate(() => window.maskSnapshot('playing'))
+        expect(initial.mask).toBe(true)
+        expect(initial.sameLayer).toBe(true)
+        await page.locator('#pause').click()
+        await expect.poll(() => page.evaluate(() => window.art.video.paused)).toBe(true)
+        // Wait native frames for the already-running SDK frame to finish. Compare
+        // its media timestamp, rather than forbidding one legitimate late encode.
+        const paused = await page.evaluate(async () => {
+          const time = window.art.currentTime
+          const beforeOutputs = window.maskCombination.times.length
+          await new Promise(resolve => setTimeout(resolve, 500))
+          return { before: time, after: window.art.currentTime, extraOutputs: window.maskCombination.times.length - beforeOutputs, snapshot: window.maskSnapshot('paused') }
+        })
+        expect(Math.abs(paused.after - paused.before)).toBeLessThan(0.01)
+        expect(paused.extraOutputs).toBeLessThanOrEqual(1)
+        await page.evaluate(async () => {
+          const seeked = new Promise(resolve => window.art.video.addEventListener('seeked', resolve, { once: true }))
+          window.art.currentTime = 20
+          window.art.fullscreenWeb = true
+          await seeked
+          window.maskCombination.seeked = { time: window.art.currentTime, outputs: window.maskCombination.times.length }
+        })
+        await expect.poll(() => page.evaluate(() => window.art.currentTime)).toBeGreaterThanOrEqual(19.9)
+        const full = await page.evaluate(() => window.maskSnapshot('fullscreen-paused'))
+        expect(full.sameLayer).toBe(true)
+        expect(full.fullscreenWeb).toBe(true)
+        expect(full.width).toBeGreaterThan(initial.width)
+        await page.evaluate(() => window.art.play())
+        await expect.poll(() => page.evaluate(() => window.maskCombination.times.length - window.maskCombination.seeked.outputs), { timeout: 30000 }).toBeGreaterThanOrEqual(2)
+        expect(await page.evaluate(() => window.maskCombination.times.at(-1))).toBeGreaterThan(20)
+        await page.evaluate(async () => {
+          window.art.fullscreenWeb = false
+          window.art.plugins.artplayerPluginDanmuku.hide()
+          window.art.plugins.artplayerPluginDanmuku.show()
+          await window.art.plugins.artplayerPluginDanmuku.load([{ text: 'After seek and visibility', time: window.art.currentTime + 1, mode: 1 }])
+        })
+        await expect.poll(() => page.evaluate(() => window.maskCombination.visible.map(row => row.text))).toContain('After seek and visibility')
+        const afterRow = await page.evaluate(() => window.maskCombination.visible.find(row => row.text === 'After seek and visibility'))
+        expect(afterRow.connected && afterRow.inLayer).toBe(true)
+        expect(afterRow.width).toBeGreaterThan(0)
+        expect(afterRow.height).toBeGreaterThan(0)
+        expect(afterRow.display).not.toBe('none')
+        const restored = await page.evaluate(() => window.maskSnapshot('restored'))
+        expect(restored.sameLayer).toBe(true)
+        expect(restored.mask).toBe(true)
+        expect(restored.width).toBe(initial.width)
+        expect(restored.mode).toBe('alpha')
+        await testInfo.attach('actual-combination-page', { body: await page.screenshot(), contentType: 'image/png' })
+        const outputsBeforeLoad = await page.evaluate(() => window.maskCombination.times.length)
+        load = await observeDanmukuLoad(page, { label: 'Mask load', requireComplete: implementation === 'candidate' })
+        load.modelOutputs = await page.evaluate(before => window.maskCombination.times.length - before, outputsBeforeLoad)
+        expect(load.modelOutputs).toBeGreaterThan(1)
+        await page.evaluate(() => window.art.destroy(false))
+        expect(await page.evaluate(() => window.maskCombination.layer.style.maskImage)).toBe('none')
+        expect(unexpected).toEqual([])
+      }
+      finally {
+        const state = await page.evaluate(() => ({ times: window.maskCombination.times, visible: window.maskCombination.visible, snapshots: window.maskCombination.snapshots, scheduled: window.maskCombination.scheduled, samples: window.maskCombination.samples, seeked: window.maskCombination.seeked, incompleteLoad: window.danmukuCombinationLoadResult }))
+        await testInfo.attach('actual-combination-evidence', { body: JSON.stringify({ core, implementation, mask: { file: artifact, sha256: hash(code) }, danmuku: { file: danmuku.file, sha256: hash(danmuku.code) }, unexpected, state, load, limitation: 'Actual model, native media, timestamp-delivered Danmuku and CSS web fullscreen. Bounded 20 rows/s after model readiness. No OS fullscreen, GPU completion or device coverage claim.' }, null, 2), contentType: 'application/json' })
+        await page.evaluate(() => {
+          if (window.art && !window.art.isDestroy)
+            window.art.destroy()
+        })
       }
     })
-    try {
-      await expect.poll(() => page.evaluate(() => window.art.duration)).toBeGreaterThan(20)
-      await page.evaluate(() => {
-        window.art.currentTime = 5
-      })
-      await page.locator('#play').click()
-      await expect.poll(() => page.evaluate(() => window.maskCombination.times.length), { timeout: 30000 }).toBeGreaterThan(2)
-      // Keep the startup row and sampling history as diagnostics: native model
-      // initialization may block its narrow existing timestamp window. Exercise
-      // the ongoing combination after the real model has become operational.
-      await page.evaluate(async () => {
-        const time = window.art.currentTime + 1
-        window.maskCombination.scheduled = time
-        await window.art.plugins.artplayerPluginDanmuku.load([{ text: 'Ready mask combination', time, mode: 1 }])
-      })
-      await expect.poll(() => page.evaluate(() => window.maskCombination.visible.map(row => row.text))).toContain('Ready mask combination')
-      const row = await page.evaluate(() => window.maskCombination.visible.find(row => row.text === 'Ready mask combination'))
-      expect(row.connected && row.inLayer).toBe(true)
-      expect(row.width).toBeGreaterThan(0)
-      expect(row.height).toBeGreaterThan(0)
-      expect(row.display).not.toBe('none')
-      const initial = await page.evaluate(() => window.maskSnapshot('playing'))
-      expect(initial.mask).toBe(true)
-      expect(initial.sameLayer).toBe(true)
-      await page.locator('#pause').click()
-      await expect.poll(() => page.evaluate(() => window.art.video.paused)).toBe(true)
-      // Wait native frames for the already-running SDK frame to finish. Compare
-      // its media timestamp, rather than forbidding one legitimate late encode.
-      const paused = await page.evaluate(async () => {
-        const time = window.art.currentTime
-        const beforeOutputs = window.maskCombination.times.length
-        await new Promise(resolve => setTimeout(resolve, 500))
-        return { before: time, after: window.art.currentTime, extraOutputs: window.maskCombination.times.length - beforeOutputs, snapshot: window.maskSnapshot('paused') }
-      })
-      expect(Math.abs(paused.after - paused.before)).toBeLessThan(0.01)
-      expect(paused.extraOutputs).toBeLessThanOrEqual(1)
-      await page.evaluate(async () => {
-        const seeked = new Promise(resolve => window.art.video.addEventListener('seeked', resolve, { once: true }))
-        window.art.currentTime = 20
-        window.art.fullscreenWeb = true
-        await seeked
-        window.maskCombination.seeked = { time: window.art.currentTime, outputs: window.maskCombination.times.length }
-      })
-      await expect.poll(() => page.evaluate(() => window.art.currentTime)).toBeGreaterThanOrEqual(19.9)
-      const full = await page.evaluate(() => window.maskSnapshot('fullscreen-paused'))
-      expect(full.sameLayer).toBe(true)
-      expect(full.fullscreenWeb).toBe(true)
-      expect(full.width).toBeGreaterThan(initial.width)
-      await page.evaluate(() => window.art.play())
-      await expect.poll(() => page.evaluate(() => window.maskCombination.times.length - window.maskCombination.seeked.outputs), { timeout: 30000 }).toBeGreaterThanOrEqual(2)
-      expect(await page.evaluate(() => window.maskCombination.times.at(-1))).toBeGreaterThan(20)
-      await page.evaluate(async () => {
-        window.art.fullscreenWeb = false
-        window.art.plugins.artplayerPluginDanmuku.hide()
-        window.art.plugins.artplayerPluginDanmuku.show()
-        await window.art.plugins.artplayerPluginDanmuku.load([{ text: 'After seek and visibility', time: window.art.currentTime + 1, mode: 1 }])
-      })
-      await expect.poll(() => page.evaluate(() => window.maskCombination.visible.map(row => row.text))).toContain('After seek and visibility')
-      const afterRow = await page.evaluate(() => window.maskCombination.visible.find(row => row.text === 'After seek and visibility'))
-      expect(afterRow.connected && afterRow.inLayer).toBe(true)
-      expect(afterRow.width).toBeGreaterThan(0)
-      expect(afterRow.height).toBeGreaterThan(0)
-      expect(afterRow.display).not.toBe('none')
-      const restored = await page.evaluate(() => window.maskSnapshot('restored'))
-      expect(restored.sameLayer).toBe(true)
-      expect(restored.mask).toBe(true)
-      expect(restored.width).toBe(initial.width)
-      expect(restored.mode).toBe('alpha')
-      await testInfo.attach('actual-combination-page', { body: await page.screenshot(), contentType: 'image/png' })
-      await page.evaluate(() => window.art.destroy(false))
-      expect(await page.evaluate(() => window.maskCombination.layer.style.maskImage)).toBe('none')
-      expect(unexpected).toEqual([])
-    }
-    finally {
-      const state = await page.evaluate(() => ({ times: window.maskCombination.times, visible: window.maskCombination.visible, snapshots: window.maskCombination.snapshots, scheduled: window.maskCombination.scheduled, samples: window.maskCombination.samples, seeked: window.maskCombination.seeked }))
-      await testInfo.attach('actual-combination-evidence', { body: JSON.stringify({ core, mask: { file: artifact, sha256: hash(code) }, danmuku: { file: danmukuFile, sha256: hash(danmukuCode) }, unexpected, state, limitation: 'Actual model, native media, timestamp-delivered Danmuku and CSS web fullscreen. No OS fullscreen, GPU completion or device coverage claim.' }, null, 2), contentType: 'application/json' })
-      await page.evaluate(() => {
-        if (window.art && !window.art.isDestroy)
-          window.art.destroy()
-      })
-    }
-  })
+  }
 }