refactor(auto-thumbnail): [PKG-AUTO-THUMB-03] wait for native frame presentation

test/auto-thumbnail-frames.test.js

@@ -0,0 +1,264 @@
+import assert from 'node:assert/strict'
+// eslint-disable-next-line test/no-import-node-test -- Repository Node frame ownership regression runner.
+import test from 'node:test'
+import { autoThumbnailCandidate, autoThumbnailEnvironment } from './helpers/auto-thumbnail.js'
+
+const implementation = await autoThumbnailCandidate()
+
+async function setup({ readyState = 4, cancel = true } = {}) {
+  const env = autoThumbnailEnvironment(implementation)
+  const frames = new Map()
+  const canceled = []
+  let nextFrame = 0
+  await env.factory({ width: 80, number: 2 })(env.art)
+  env.art.emit('video:loadedmetadata')
+  const video = env.videos[0]
+  video.readyState = readyState
+  video.requestVideoFrameCallback = (callback) => {
+    const id = nextFrame++
+    frames.set(id, callback)
+    return id
+  }
+  if (cancel) {
+    video.cancelVideoFrameCallback = (id) => {
+      canceled.push(id)
+      frames.delete(id)
+    }
+  }
+  return Object.assign(env, { video, frames, canceled, present(id = frames.keys().next().value) {
+    const callback = frames.get(id)
+    frames.delete(id)
+    callback(0, { mediaTime: video.currentTime })
+  } })
+}
+
+function closed(env) {
+  assert.equal(env.attached.size, 0)
+  assert.equal(env.frames.size, 0)
+  assert.equal(env.timers.size, 0)
+  assert.equal(env.video.onloadeddata, null)
+  assert.equal(env.video.onseeked, null)
+  assert.equal(env.video.src, undefined)
+}
+
+test('Auto-thumbnail frames wait for loadeddata before starting presentation-aware seeking', async () => {
+  const env = await setup({ readyState: 1 })
+  env.metadata()
+  assert.equal(env.frames.size, 0)
+  assert.equal(env.operations.some(item => item.name === 'seek'), false)
+  assert.equal(env.timers.size, 1)
+  const loaded = env.video.onloadeddata
+  env.video.readyState = 4
+  loaded()
+  loaded()
+  assert.equal(env.frames.size, 1)
+  assert.equal(env.operations.filter(item => item.name === 'seek').length, 1)
+  env.art.emit('destroy')
+  closed(env)
+})
+
+for (const order of ['seek-first', 'frame-first']) {
+  test(`Auto-thumbnail frames need both seek completion and presentation: ${order}`, async () => {
+    const env = await setup()
+    env.metadata()
+    const seeked = env.video.onseeked
+    const frame = env.frames.get(0)
+    if (order === 'seek-first')
+      env.seeked()
+    else env.present()
+    assert.equal(env.blobs.length, 0)
+    if (order === 'seek-first')
+      env.present()
+    else env.seeked()
+    assert.equal(env.blobs.length, 1)
+    assert.equal(env.timers.size, 0)
+    seeked()
+    frame()
+    assert.equal(env.blobs.length, 1)
+    env.finish()
+    seeked()
+    frame()
+    assert.equal(env.blobs.length, 1, 'Old callbacks cannot complete the next frame')
+    env.seeked()
+    env.present()
+    env.finish(1)
+    closed(env)
+    assert.equal(env.updates.length, 2)
+    env.art.emit('destroy')
+    assert.equal(env.urls.size, 0)
+  })
+}
+
+for (const phase of ['data', 'seek', 'frame']) {
+  test(`Auto-thumbnail frames release ${phase} waits and ignore callbacks after destroy`, async () => {
+    const env = await setup({ readyState: phase === 'data' ? 1 : 4 })
+    env.metadata()
+    const loaded = env.video.onloadeddata
+    const frame = env.frames.get(0)
+    const seeked = env.video.onseeked
+    const timer = [...env.timers.values()][0].callback
+    if (phase === 'frame')
+      env.seeked()
+    env.art.emit('destroy')
+    loaded?.()
+    frame?.()
+    seeked?.()
+    timer()
+    assert.equal(env.blobs.length, 0)
+    assert.equal(env.warnings.length, 0)
+    closed(env)
+  })
+}
+
+test('Auto-thumbnail frames invalidate canceled presentation callbacks when retrying the same target', async () => {
+  const env = await setup()
+  env.metadata()
+  const first = env.frames.get(0)
+  env.video.timeOffset = 1
+  env.seeked()
+  assert.deepEqual(env.canceled, [0])
+  first()
+  env.video.timeOffset = 0
+  env.seeked()
+  assert.equal(env.blobs.length, 0)
+  env.present(1)
+  assert.equal(env.blobs.length, 1)
+  env.art.emit('destroy')
+  closed(env)
+})
+
+for (const phase of ['data', 'seek', 'frame']) {
+  test(`Auto-thumbnail frames time out a stalled ${phase} wait without leaking callbacks`, async () => {
+    const env = await setup({ readyState: phase === 'data' ? 1 : 4 })
+    env.metadata()
+    if (phase === 'frame')
+      env.seeked()
+    const timer = [...env.timers.values()][0]
+    assert.equal(timer.delay, 30000)
+    timer.callback()
+    assert.equal(env.warnings.length, 1)
+    assert.match(env.warnings[0][1].message, /frame readiness timed out/)
+    assert.equal(env.blobs.length, 0)
+    closed(env)
+    env.art.emit('destroy')
+  })
+}
+
+test('Auto-thumbnail frames clean up the decoder even if native callback cancellation throws', async () => {
+  const env = await setup()
+  env.metadata()
+  const cancel = env.video.cancelVideoFrameCallback
+  const failure = new Error('Frame cancellation failed')
+  env.video.cancelVideoFrameCallback = (id) => {
+    cancel(id)
+    throw failure
+  }
+  env.art.emit('destroy')
+  closed(env)
+  assert.equal(env.warnings[0][1], failure)
+})
+
+test('Auto-thumbnail frames recover a callback handle registered across synchronous destruction', async () => {
+  const env = await setup()
+  const request = env.video.requestVideoFrameCallback
+  env.video.requestVideoFrameCallback = (callback) => {
+    env.art.emit('destroy')
+    return request(callback)
+  }
+  env.metadata()
+  closed(env)
+  assert.deepEqual(env.canceled, [0])
+  assert.equal(env.operations.some(item => item.name === 'seek'), false)
+})
+
+test('Auto-thumbnail frames recover a timer handle created across synchronous destruction', async () => {
+  const env = await setup()
+  const schedule = env.box.setTimeout
+  env.box.setTimeout = (...args) => {
+    env.art.emit('destroy')
+    return schedule(...args)
+  }
+  env.metadata()
+  closed(env)
+  assert.equal(env.operations.some(item => item.name === 'seek'), false)
+})
+
+test('Auto-thumbnail frames preserve fallback seeking when native callback cancellation is absent', async () => {
+  const env = await setup({ cancel: false })
+  env.metadata()
+  assert.equal(env.frames.size, 0)
+  env.seeked()
+  assert.equal(env.blobs.length, 1)
+  env.art.emit('destroy')
+  closed(env)
+})
+
+test('Auto-thumbnail frames preserve an earlier preview if subsequent presentation times out', async () => {
+  const env = await setup()
+  env.metadata()
+  env.seeked()
+  env.present()
+  env.finish()
+  const previous = env.art.thumbnails
+  env.seeked()
+  const timer = [...env.timers.values()][0]
+  timer.callback()
+  assert.equal(env.art.thumbnails, previous)
+  assert.equal(env.urls.has(previous.url), true)
+  closed(env)
+  env.art.emit('destroy')
+  assert.equal(env.urls.size, 0)
+})
+
+test('Auto-thumbnail frames accept synchronous presentation without leaking its returned handle', async () => {
+  const env = await setup()
+  const request = env.video.requestVideoFrameCallback
+  env.video.requestVideoFrameCallback = (callback) => {
+    const id = request(callback)
+    callback()
+    return id
+  }
+  env.metadata()
+  assert.equal(env.frames.size, 0)
+  assert.equal(env.blobs.length, 0)
+  env.seeked()
+  assert.equal(env.blobs.length, 1)
+  env.art.emit('destroy')
+  closed(env)
+})
+
+test('Auto-thumbnail frames release callbacks and the decoder when clearing its timer throws', async () => {
+  const env = await setup()
+  env.metadata()
+  const clear = env.box.clearTimeout
+  const failure = new Error('Timer cleanup failed')
+  env.box.clearTimeout = (id) => {
+    clear(id)
+    throw failure
+  }
+  env.art.emit('destroy')
+  closed(env)
+  assert.equal(env.warnings[0][1], failure)
+})
+
+test('Auto-thumbnail frames own native registration errors and clear the deadline', async () => {
+  const env = await setup()
+  const failure = new Error('Frame registration failed')
+  env.video.requestVideoFrameCallback = () => {
+    throw failure
+  }
+  env.metadata()
+  closed(env)
+  assert.equal(env.warnings[0][1], failure)
+})
+
+test('Auto-thumbnail frames cannot restore a loadeddata handler after readiness getter destruction', async () => {
+  const env = await setup()
+  Object.defineProperty(env.video, 'readyState', { get() {
+    env.art.emit('destroy')
+    return 1
+  } })
+  env.metadata()
+  closed(env)
+  assert.equal(env.operations.some(item => item.name === 'seek'), false)
+})

test/browser/auto-thumbnail-lifecycle.spec.js

@@ -3,21 +3,29 @@ import { autoThumbnailCandidate } from '../helpers/auto-thumbnail.js'
 import { expect, test } from './fixtures.js'
 
 const implementation = await autoThumbnailCandidate()
-for (const scenario of ['destroy', 'restart', 'complete']) {
+for (const scenario of ['destroy', 'restart', 'complete', 'frame-destroy', 'frame-restart']) {
   test(`Auto-thumbnail candidate native ${scenario} owns decoder and encoded URLs`, async ({ page }, testInfo) => {
     await page.goto('/test/player.html?core=published')
     await page.setContent('<!doctype html><div></div>')
     await page.addScriptTag({ content: implementation.code })
-    await page.evaluate(async (hold) => {
-      const probe = window.probe = { videos: [], updates: [], urls: new Set(), blobs: [], listeners: new Map(), hold }
+    await page.evaluate(async (scenario) => {
+      const probe = window.probe = { videos: [], updates: [], urls: new Set(), blobs: [], frames: [], frameSupport: false, listeners: new Map(), hold: scenario !== 'complete' }
       const createElement = document.createElement.bind(document)
       const createUrl = URL.createObjectURL.bind(URL)
       const revokeUrl = URL.revokeObjectURL.bind(URL)
       const toBlob = HTMLCanvasElement.prototype.toBlob
       document.createElement = function (name, ...args) {
         const element = createElement(name, ...args)
-        if (name === 'video')
+        if (name === 'video') {
           probe.videos.push(element)
+          const request = element.requestVideoFrameCallback?.bind(element)
+          probe.frameSupport = Boolean(request && element.cancelVideoFrameCallback)
+          if (probe.frameSupport && scenario.startsWith('frame-')) {
+            element.requestVideoFrameCallback = callback => request((...args) => {
+              probe.frames.push({ callback, args })
+            })
+          }
+        }
         return element
       }
       HTMLCanvasElement.prototype.toBlob = function (callback, type) {
@@ -63,32 +71,41 @@ for (const scenario of ['destroy', 'restart', 'complete']) {
       }
       probe.result = await window.artplayerPluginAutoThumbnail({ width: 80, number: 2 })(probe.art)
       probe.art.emit('video:loadedmetadata')
-    }, scenario !== 'complete')
-    await expect.poll(() => page.evaluate(() => window.probe.blobs.length)).toBe(scenario === 'complete' ? 2 : 1)
+    }, scenario)
+    const holdFrame = scenario.startsWith('frame-') && await page.evaluate(() => window.probe.frameSupport)
+    await expect.poll(() => page.evaluate(holdFrame => holdFrame ? window.probe.frames.length : window.probe.blobs.length, holdFrame)).toBe(scenario === 'complete' ? 2 : 1)
     const state = await page.evaluate((scenario) => {
       const probe = window.probe
       const before = { urls: probe.urls.size, updates: probe.updates.length }
       if (scenario !== 'complete') {
-        probe.art.emit(scenario)
-        probe.blobs[0].callback(probe.blobs[0].blob)
-        probe.blobs[0].callback(probe.blobs[0].blob)
+        probe.art.emit(scenario.replace('frame-', ''))
+        if (probe.blobs[0]) {
+          probe.blobs[0].callback(probe.blobs[0].blob)
+          probe.blobs[0].callback(probe.blobs[0].blob)
+        }
+        if (probe.frames[0]) {
+          probe.frames[0].callback(...probe.frames[0].args)
+          probe.frames[0].callback(...probe.frames[0].args)
+        }
       }
       const after = { urls: probe.urls.size, updates: probe.updates.length }
       const decoder = probe.videos[0]
-      const video = { connected: decoder.isConnected, src: decoder.getAttribute('src'), readyState: decoder.readyState, paused: decoder.paused, handlers: ['onloadedmetadata', 'onseeked', 'onerror'].map(key => decoder[key] === null) }
+      const video = { connected: decoder.isConnected, src: decoder.getAttribute('src'), readyState: decoder.readyState, paused: decoder.paused, handlers: ['onloadedmetadata', 'onloadeddata', 'onseeked', 'onerror'].map(key => decoder[key] === null) }
       probe.art.emit('destroy')
       probe.art.emit('destroy')
       const final = { urls: probe.urls.size, listeners: [...probe.listeners.values()].reduce((sum, set) => sum + set.size, 0), updates: probe.updates.length }
       const blobs = probe.blobs.map(({ blob }) => ({ type: blob.type, bytes: blob.size }))
       probe.restore()
-      return { before, after, video, final, blobs, result: probe.result }
+      return { before, after, video, final, blobs, heldFrames: probe.frames.length, frameSupport: probe.frameSupport, result: probe.result }
     }, scenario)
-    await testInfo.attach('auto-thumbnail-candidate-native-lifecycle', { contentType: 'application/json', body: JSON.stringify({ scenario, sha256: hash(implementation.code), scope: 'Actual HTTP decoding/seek/JPEG lifecycle with a stub player host. This checkpoint does not establish accurate extracted pixels; AUTO-THUMB-PIXEL-01 remains open.', ...state }) })
+    await testInfo.attach('auto-thumbnail-candidate-native-lifecycle', { contentType: 'application/json', body: JSON.stringify({ scenario, sha256: hash(implementation.code), scope: 'Actual HTTP decoding/seek/JPEG and held native presentation callback lifecycle with a stub player host. Frame scenarios use a held Blob fallback only when native frame callbacks are absent. Pixel acceptance is separate; AUTO-THUMB-PIXEL-01 remains open.', ...state }) })
     expect(state.result).toEqual({ name: 'artplayerPluginAutoThumbnail' })
     expect(state.before).toEqual(scenario === 'complete' ? { urls: 1, updates: 2 } : { urls: 0, updates: 0 })
     expect(state.after).toEqual(state.before)
-    expect(state.video).toEqual({ connected: false, src: null, readyState: 0, paused: true, handlers: [true, true, true] })
+    expect(state.video).toEqual({ connected: false, src: null, readyState: 0, paused: true, handlers: [true, true, true, true] })
     expect(state.final).toEqual({ urls: 0, listeners: 0, updates: scenario === 'complete' ? 2 : 0 })
     expect(state.blobs.every(blob => blob.type === 'image/jpeg' && blob.bytes > 0)).toBe(true)
+    expect(state.heldFrames).toBe(holdFrame ? 1 : 0)
+    expect(state.blobs).toHaveLength(holdFrame ? 0 : scenario === 'complete' ? 2 : 1)
   })
 }

test/browser/auto-thumbnail-pixels.spec.js

@@ -5,14 +5,14 @@ import { expect, test } from './fixtures.js'
 const implementation = await autoThumbnailCandidate()
 
 for (const media of ['auto-thumbnail-timeline', 'pattern']) {
-  test(`Auto-thumbnail candidate retains cells 2-4 ${media} pixels in a hidden decoder`, async ({ page, browserName }, testInfo) => {
+  test(`Auto-thumbnail candidate verifies presented ${media} pixels and records fallback limits`, async ({ page, browserName }, testInfo) => {
     await page.goto('/test/player.html?core=published')
     await page.setContent('<!doctype html><style>video { width: 1px; height: 1px; max-width: 1px; max-height: 1px; display: none }</style>')
     await page.addScriptTag({ content: implementation.code })
     await page.evaluate(async (media) => {
       const createElement = document.createElement.bind(document)
       const drawImage = CanvasRenderingContext2D.prototype.drawImage
-      const probe = window.probe = { videos: [], updates: [], draws: [], playing: 0, warnings: [] }
+      const probe = window.probe = { videos: [], updates: [], draws: [], presented: [], playing: 0, warnings: [] }
       const warn = console.warn
       console.warn = (...args) => probe.warnings.push(args.map(String))
       document.createElement = function (tag, ...args) {
@@ -20,6 +20,13 @@ for (const media of ['auto-thumbnail-timeline', 'pattern']) {
         if (tag === 'video') {
           probe.videos.push(element)
           element.addEventListener('playing', () => probe.playing++)
+          const request = element.requestVideoFrameCallback?.bind(element)
+          if (request) {
+            element.requestVideoFrameCallback = callback => request((now, metadata) => {
+              probe.presented.push({ mediaTime: metadata.mediaTime, currentTime: element.currentTime })
+              callback(now, metadata)
+            })
+          }
         }
         return element
       }
@@ -79,15 +86,17 @@ for (const media of ['auto-thumbnail-timeline', 'pattern']) {
       const final = probe.videos.map(video => ({ connected: video.isConnected, src: video.getAttribute('src'), paused: video.paused }))
       probe.art.emit('destroy')
       probe.restore()
-      return { cells, spatial, draws: probe.draws, final, playing: probe.playing, warnings: probe.warnings }
+      const supportsPresentation = typeof probe.videos[0].requestVideoFrameCallback === 'function' && typeof probe.videos[0].cancelVideoFrameCallback === 'function'
+      return { cells, spatial, draws: probe.draws, presented: probe.presented, supportsPresentation, final, playing: probe.playing, warnings: probe.warnings }
     })
     await testInfo.attach('auto-thumbnail-candidate-hidden-pixels', { contentType: 'application/json', body: JSON.stringify({
       sha256: hash(implementation.code),
       media,
-      scope: 'Native decoding and JPEG pixels for cells 2-4, hidden intrinsic-sized rendering and cleanup. Cells 0-1 are diagnostic only: initial drawing/readiness/timestamp behavior remains open under AUTO-THUMB-PIXEL-01. Stub player host, not a core integration.',
+      scope: 'All five cells are asserted where native frame presentation callbacks are available. Without that capability only cells 2-4 are asserted, with cells 0-1 recorded as unresolved diagnostics under AUTO-THUMB-PIXEL-01. Native HTTP/JPEG and a stub player host, not a core integration.',
       ...state,
     }) })
-    testInfo.annotations.push({ type: 'remaining-initial-frame-risk', description: `${browserName}: cells 0-1 are evidence only; this test cannot close AUTO-THUMB-PIXEL-01 or PKG-AUTO-THUMB-03.` })
+    if (!state.supportsPresentation)
+      testInfo.annotations.push({ type: 'remaining-initial-frame-risk', description: `${browserName}: no native presentation callback; cells 0-1 remain evidence only and cannot close AUTO-THUMB-PIXEL-01 or PKG-AUTO-THUMB-03.` })
     expect(state.warnings).toEqual([])
     expect(state.playing).toBe(0)
     expect(state.final).toEqual([{ connected: false, src: null, paused: true }])
@@ -98,12 +107,25 @@ for (const media of ['auto-thumbnail-timeline', 'pattern']) {
       expect(draw.display).toBe('block')
       expect(draw.width).toBeGreaterThanOrEqual(80)
       expect(draw.height).toBeGreaterThanOrEqual(45)
-      if (index > 1) {
+      if (state.supportsPresentation || index > 1) {
         expect(Math.abs(draw.time - draw.duration * index / 5)).toBeLessThan(0.05)
         expect(draw.raw[3]).toBe(255)
       }
     }
+    if (state.supportsPresentation) {
+      expect(state.presented).toHaveLength(5)
+      for (const [index, frame] of state.presented.entries())
+        expect(Math.abs(frame.mediaTime - state.draws[index].duration * index / 5)).toBeLessThan(0.06)
+    }
     if (media === 'auto-thumbnail-timeline') {
+      if (state.supportsPresentation) {
+        const [purple, red] = state.cells
+        expect(Math.min(purple[0], purple[2])).toBeGreaterThan(100)
+        expect(Math.abs(purple[0] - purple[2])).toBeLessThan(30)
+        expect(purple[1]).toBeLessThan(40)
+        expect(red[0]).toBeGreaterThan(200)
+        expect(Math.max(red[1], red[2])).toBeLessThan(40)
+      }
       const [black, blue, yellow] = state.cells.slice(2)
       expect(Math.max(...black.slice(0, 3))).toBeLessThan(30)
       expect(blue[2]).toBeGreaterThan(200)
@@ -112,7 +134,7 @@ for (const media of ['auto-thumbnail-timeline', 'pattern']) {
       expect(yellow[2]).toBeLessThan(40)
     }
     else {
-      for (const pixel of state.cells.slice(2, 4)) {
+      for (const pixel of state.cells.slice(state.supportsPresentation ? 0 : 2, 4)) {
         expect(pixel[2] - Math.max(pixel[0], pixel[1])).toBeGreaterThan(80)
       }
       expect(state.cells[4][0] - Math.max(state.cells[4][1], state.cells[4][2])).toBeGreaterThan(80)

test/helpers/auto-thumbnail.js

@@ -41,6 +41,8 @@ export function autoThumbnailEnvironment(implementation, { script = false } = {}
   const listeners = new Map()
   const updates = []
   const warnings = []
+  const timers = new Map()
+  let nextTimer = 0
   let nextUrl = 0
   const controls = { nullContext: false, drawError: null, encodeError: null, updateError: null, onUpdate: null, beforeAppend: null, afterAppend: null }
   const attached = new Set()
@@ -68,6 +70,7 @@ export function autoThumbnailEnvironment(implementation, { script = false } = {}
           videoWidth: 1920,
           videoHeight: 1080,
           onloadedmetadata: null,
+          onloadeddata: null,
           onseeked: null,
           onerror: null,
           pause() { operations.push({ name: 'pause', video }) },
@@ -116,7 +119,11 @@ export function autoThumbnailEnvironment(implementation, { script = false } = {}
     },
   }
   const module = { exports: {} }
-  const box = { document, URL: {
+  const box = { document, setTimeout(callback, delay) {
+    const id = nextTimer++
+    timers.set(id, { callback, delay })
+    return id
+  }, clearTimeout(id) { timers.delete(id) }, URL: {
     createObjectURL(blob) {
       if (blob == null)
         throw new TypeError('Blob required')
@@ -174,6 +181,7 @@ export function autoThumbnailEnvironment(implementation, { script = false } = {}
     listeners,
     updates,
     warnings,
+    timers,
     controls,
     attached,
     metadata(video = videos.at(-1)) { video.onloadedmetadata?.() },