refactor(auto-thumbnail): [PKG-AUTO-THUMB-03] wait for native frame presentation

test/browser/auto-thumbnail-lifecycle.spec.js

@@ -3,21 +3,29 @@ import { autoThumbnailCandidate } from '../helpers/auto-thumbnail.js'
 import { expect, test } from './fixtures.js'
 
 const implementation = await autoThumbnailCandidate()
-for (const scenario of ['destroy', 'restart', 'complete']) {
+for (const scenario of ['destroy', 'restart', 'complete', 'frame-destroy', 'frame-restart']) {
   test(`Auto-thumbnail candidate native ${scenario} owns decoder and encoded URLs`, async ({ page }, testInfo) => {
     await page.goto('/test/player.html?core=published')
     await page.setContent('<!doctype html><div></div>')
     await page.addScriptTag({ content: implementation.code })
-    await page.evaluate(async (hold) => {
-      const probe = window.probe = { videos: [], updates: [], urls: new Set(), blobs: [], listeners: new Map(), hold }
+    await page.evaluate(async (scenario) => {
+      const probe = window.probe = { videos: [], updates: [], urls: new Set(), blobs: [], frames: [], frameSupport: false, listeners: new Map(), hold: scenario !== 'complete' }
       const createElement = document.createElement.bind(document)
       const createUrl = URL.createObjectURL.bind(URL)
       const revokeUrl = URL.revokeObjectURL.bind(URL)
       const toBlob = HTMLCanvasElement.prototype.toBlob
       document.createElement = function (name, ...args) {
         const element = createElement(name, ...args)
-        if (name === 'video')
+        if (name === 'video') {
           probe.videos.push(element)
+          const request = element.requestVideoFrameCallback?.bind(element)
+          probe.frameSupport = Boolean(request && element.cancelVideoFrameCallback)
+          if (probe.frameSupport && scenario.startsWith('frame-')) {
+            element.requestVideoFrameCallback = callback => request((...args) => {
+              probe.frames.push({ callback, args })
+            })
+          }
+        }
         return element
       }
       HTMLCanvasElement.prototype.toBlob = function (callback, type) {
@@ -63,32 +71,41 @@ for (const scenario of ['destroy', 'restart', 'complete']) {
       }
       probe.result = await window.artplayerPluginAutoThumbnail({ width: 80, number: 2 })(probe.art)
       probe.art.emit('video:loadedmetadata')
-    }, scenario !== 'complete')
-    await expect.poll(() => page.evaluate(() => window.probe.blobs.length)).toBe(scenario === 'complete' ? 2 : 1)
+    }, scenario)
+    const holdFrame = scenario.startsWith('frame-') && await page.evaluate(() => window.probe.frameSupport)
+    await expect.poll(() => page.evaluate(holdFrame => holdFrame ? window.probe.frames.length : window.probe.blobs.length, holdFrame)).toBe(scenario === 'complete' ? 2 : 1)
     const state = await page.evaluate((scenario) => {
       const probe = window.probe
       const before = { urls: probe.urls.size, updates: probe.updates.length }
       if (scenario !== 'complete') {
-        probe.art.emit(scenario)
-        probe.blobs[0].callback(probe.blobs[0].blob)
-        probe.blobs[0].callback(probe.blobs[0].blob)
+        probe.art.emit(scenario.replace('frame-', ''))
+        if (probe.blobs[0]) {
+          probe.blobs[0].callback(probe.blobs[0].blob)
+          probe.blobs[0].callback(probe.blobs[0].blob)
+        }
+        if (probe.frames[0]) {
+          probe.frames[0].callback(...probe.frames[0].args)
+          probe.frames[0].callback(...probe.frames[0].args)
+        }
       }
       const after = { urls: probe.urls.size, updates: probe.updates.length }
       const decoder = probe.videos[0]
-      const video = { connected: decoder.isConnected, src: decoder.getAttribute('src'), readyState: decoder.readyState, paused: decoder.paused, handlers: ['onloadedmetadata', 'onseeked', 'onerror'].map(key => decoder[key] === null) }
+      const video = { connected: decoder.isConnected, src: decoder.getAttribute('src'), readyState: decoder.readyState, paused: decoder.paused, handlers: ['onloadedmetadata', 'onloadeddata', 'onseeked', 'onerror'].map(key => decoder[key] === null) }
       probe.art.emit('destroy')
       probe.art.emit('destroy')
       const final = { urls: probe.urls.size, listeners: [...probe.listeners.values()].reduce((sum, set) => sum + set.size, 0), updates: probe.updates.length }
       const blobs = probe.blobs.map(({ blob }) => ({ type: blob.type, bytes: blob.size }))
       probe.restore()
-      return { before, after, video, final, blobs, result: probe.result }
+      return { before, after, video, final, blobs, heldFrames: probe.frames.length, frameSupport: probe.frameSupport, result: probe.result }
     }, scenario)
-    await testInfo.attach('auto-thumbnail-candidate-native-lifecycle', { contentType: 'application/json', body: JSON.stringify({ scenario, sha256: hash(implementation.code), scope: 'Actual HTTP decoding/seek/JPEG lifecycle with a stub player host. This checkpoint does not establish accurate extracted pixels; AUTO-THUMB-PIXEL-01 remains open.', ...state }) })
+    await testInfo.attach('auto-thumbnail-candidate-native-lifecycle', { contentType: 'application/json', body: JSON.stringify({ scenario, sha256: hash(implementation.code), scope: 'Actual HTTP decoding/seek/JPEG and held native presentation callback lifecycle with a stub player host. Frame scenarios use a held Blob fallback only when native frame callbacks are absent. Pixel acceptance is separate; AUTO-THUMB-PIXEL-01 remains open.', ...state }) })
     expect(state.result).toEqual({ name: 'artplayerPluginAutoThumbnail' })
     expect(state.before).toEqual(scenario === 'complete' ? { urls: 1, updates: 2 } : { urls: 0, updates: 0 })
     expect(state.after).toEqual(state.before)
-    expect(state.video).toEqual({ connected: false, src: null, readyState: 0, paused: true, handlers: [true, true, true] })
+    expect(state.video).toEqual({ connected: false, src: null, readyState: 0, paused: true, handlers: [true, true, true, true] })
     expect(state.final).toEqual({ urls: 0, listeners: 0, updates: scenario === 'complete' ? 2 : 0 })
     expect(state.blobs.every(blob => blob.type === 'image/jpeg' && blob.bytes > 0)).toBe(true)
+    expect(state.heldFrames).toBe(holdFrame ? 1 : 0)
+    expect(state.blobs).toHaveLength(holdFrame ? 0 : scenario === 'complete' ? 2 : 1)
   })
 }

test/browser/auto-thumbnail-pixels.spec.js

@@ -5,14 +5,14 @@ import { expect, test } from './fixtures.js'
 const implementation = await autoThumbnailCandidate()
 
 for (const media of ['auto-thumbnail-timeline', 'pattern']) {
-  test(`Auto-thumbnail candidate retains cells 2-4 ${media} pixels in a hidden decoder`, async ({ page, browserName }, testInfo) => {
+  test(`Auto-thumbnail candidate verifies presented ${media} pixels and records fallback limits`, async ({ page, browserName }, testInfo) => {
     await page.goto('/test/player.html?core=published')
     await page.setContent('<!doctype html><style>video { width: 1px; height: 1px; max-width: 1px; max-height: 1px; display: none }</style>')
     await page.addScriptTag({ content: implementation.code })
     await page.evaluate(async (media) => {
       const createElement = document.createElement.bind(document)
       const drawImage = CanvasRenderingContext2D.prototype.drawImage
-      const probe = window.probe = { videos: [], updates: [], draws: [], playing: 0, warnings: [] }
+      const probe = window.probe = { videos: [], updates: [], draws: [], presented: [], playing: 0, warnings: [] }
       const warn = console.warn
       console.warn = (...args) => probe.warnings.push(args.map(String))
       document.createElement = function (tag, ...args) {
@@ -20,6 +20,13 @@ for (const media of ['auto-thumbnail-timeline', 'pattern']) {
         if (tag === 'video') {
           probe.videos.push(element)
           element.addEventListener('playing', () => probe.playing++)
+          const request = element.requestVideoFrameCallback?.bind(element)
+          if (request) {
+            element.requestVideoFrameCallback = callback => request((now, metadata) => {
+              probe.presented.push({ mediaTime: metadata.mediaTime, currentTime: element.currentTime })
+              callback(now, metadata)
+            })
+          }
         }
         return element
       }
@@ -79,15 +86,17 @@ for (const media of ['auto-thumbnail-timeline', 'pattern']) {
       const final = probe.videos.map(video => ({ connected: video.isConnected, src: video.getAttribute('src'), paused: video.paused }))
       probe.art.emit('destroy')
       probe.restore()
-      return { cells, spatial, draws: probe.draws, final, playing: probe.playing, warnings: probe.warnings }
+      const supportsPresentation = typeof probe.videos[0].requestVideoFrameCallback === 'function' && typeof probe.videos[0].cancelVideoFrameCallback === 'function'
+      return { cells, spatial, draws: probe.draws, presented: probe.presented, supportsPresentation, final, playing: probe.playing, warnings: probe.warnings }
     })
     await testInfo.attach('auto-thumbnail-candidate-hidden-pixels', { contentType: 'application/json', body: JSON.stringify({
       sha256: hash(implementation.code),
       media,
-      scope: 'Native decoding and JPEG pixels for cells 2-4, hidden intrinsic-sized rendering and cleanup. Cells 0-1 are diagnostic only: initial drawing/readiness/timestamp behavior remains open under AUTO-THUMB-PIXEL-01. Stub player host, not a core integration.',
+      scope: 'All five cells are asserted where native frame presentation callbacks are available. Without that capability only cells 2-4 are asserted, with cells 0-1 recorded as unresolved diagnostics under AUTO-THUMB-PIXEL-01. Native HTTP/JPEG and a stub player host, not a core integration.',
       ...state,
     }) })
-    testInfo.annotations.push({ type: 'remaining-initial-frame-risk', description: `${browserName}: cells 0-1 are evidence only; this test cannot close AUTO-THUMB-PIXEL-01 or PKG-AUTO-THUMB-03.` })
+    if (!state.supportsPresentation)
+      testInfo.annotations.push({ type: 'remaining-initial-frame-risk', description: `${browserName}: no native presentation callback; cells 0-1 remain evidence only and cannot close AUTO-THUMB-PIXEL-01 or PKG-AUTO-THUMB-03.` })
     expect(state.warnings).toEqual([])
     expect(state.playing).toBe(0)
     expect(state.final).toEqual([{ connected: false, src: null, paused: true }])
@@ -98,12 +107,25 @@ for (const media of ['auto-thumbnail-timeline', 'pattern']) {
       expect(draw.display).toBe('block')
       expect(draw.width).toBeGreaterThanOrEqual(80)
       expect(draw.height).toBeGreaterThanOrEqual(45)
-      if (index > 1) {
+      if (state.supportsPresentation || index > 1) {
         expect(Math.abs(draw.time - draw.duration * index / 5)).toBeLessThan(0.05)
         expect(draw.raw[3]).toBe(255)
       }
     }
+    if (state.supportsPresentation) {
+      expect(state.presented).toHaveLength(5)
+      for (const [index, frame] of state.presented.entries())
+        expect(Math.abs(frame.mediaTime - state.draws[index].duration * index / 5)).toBeLessThan(0.06)
+    }
     if (media === 'auto-thumbnail-timeline') {
+      if (state.supportsPresentation) {
+        const [purple, red] = state.cells
+        expect(Math.min(purple[0], purple[2])).toBeGreaterThan(100)
+        expect(Math.abs(purple[0] - purple[2])).toBeLessThan(30)
+        expect(purple[1]).toBeLessThan(40)
+        expect(red[0]).toBeGreaterThan(200)
+        expect(Math.max(red[1], red[2])).toBeLessThan(40)
+      }
       const [black, blue, yellow] = state.cells.slice(2)
       expect(Math.max(...black.slice(0, 3))).toBeLessThan(30)
       expect(blue[2]).toBeGreaterThan(200)
@@ -112,7 +134,7 @@ for (const media of ['auto-thumbnail-timeline', 'pattern']) {
       expect(yellow[2]).toBeLessThan(40)
     }
     else {
-      for (const pixel of state.cells.slice(2, 4)) {
+      for (const pixel of state.cells.slice(state.supportsPresentation ? 0 : 2, 4)) {
         expect(pixel[2] - Math.max(pixel[0], pixel[1])).toBeGreaterThan(80)
       }
       expect(state.cells[4][0] - Math.max(state.cells[4][1], state.cells[4][2])).toBeGreaterThan(80)