test(asr): [PKG-ASR-02] reproduce audio queue and lifecycle failures

This commit is contained in:
Harvey Zhao committed 2026-09-13 14:09:20 +08:00
1 parent a22586a172
commit 53c295819d
11 files changed
+1200 -12

No files matched your search

+319
View File
@@ -0,0 +1,319 @@
import assert from 'node:assert/strict'
import { Buffer } from 'node:buffer'
// eslint-disable-next-line test/no-import-node-test -- Historical ASR audio uses the repository runner.
import test from 'node:test'
import { asrAudioEnvironment, deferredAudio, runAsrWorklet } from './helpers/asr-audio.js'
import { asrHistorical } from './helpers/asr.js'
// These fixtures are immutable historical implementations, never the candidate.
// Defect observations must become separate red/green candidate regressions when fixed.
const implementations = await asrHistorical()
function samples(pcm) {
const view = new DataView(pcm)
return Array.from({ length: pcm.byteLength / 2 }, (_, index) => view.getInt16(index * 2, true))
}
for (const implementation of implementations) {
const label = `ASR audio history ${implementation.name}`
test(`${label}: default PCM clamping and mono WAV bytes`, async () => {
const chunks = []
const environment = asrAudioEnvironment(implementation, { onAudioChunk: chunk => chunks.push(chunk) })
await environment.emit('play')
assert.equal(environment.contexts[0].options.sampleRate, 16000)
assert.equal([...environment.intervals.values()][0].delay, 100)
const input = new Float32Array(1600)
input.set([-2, -1, -0.5, 0, 0.5, 1, 2, Number.NaN])
environment.send(input)
await environment.tick()
assert.equal(chunks.length, 1)
assert.deepEqual(Object.keys(chunks[0]), ['pcm', 'wav'])
assert.equal(chunks[0].pcm.byteLength, 3200)
assert.deepEqual(samples(chunks[0].pcm).slice(0, 8), [-32768, -32768, -16384, 0, 16383, 32767, 32767, 0])
const wav = Buffer.from(chunks[0].wav)
assert.equal(wav.length, 3244)
assert.equal(wav.toString('ascii', 0, 4), 'RIFF')
assert.equal(wav.readUInt32LE(4), 3236)
assert.equal(wav.toString('ascii', 8, 16), 'WAVEfmt ')
assert.equal(wav.readUInt32LE(16), 16)
assert.equal(wav.readUInt16LE(20), 1)
assert.equal(wav.readUInt16LE(22), 1)
assert.equal(wav.readUInt32LE(24), 16000)
assert.equal(wav.readUInt32LE(28), 32000)
assert.equal(wav.readUInt16LE(32), 2)
assert.equal(wav.readUInt16LE(34), 16)
assert.equal(wav.toString('ascii', 36, 40), 'data')
assert.equal(wav.readUInt32LE(40), 3200)
assert.deepEqual(wav.subarray(44), Buffer.from(chunks[0].pcm))
await environment.plugin.stop()
})
test(`${label}: custom sampling concatenates worklet messages in order`, async () => {
const chunks = []
const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: chunk => chunks.push(chunk) })
await environment.emit('play')
environment.send([-1, -0.5, 0])
environment.send([0.5, 1, 0, -1, 1])
await environment.tick()
assert.equal(chunks.length, 1)
assert.deepEqual(samples(chunks[0].pcm), [-32768, -16384, 0, 16383, 32767, 0, -32768, 32767])
assert.equal(new DataView(chunks[0].wav).getUint32(24, true), 8000)
assert.equal([...environment.intervals.values()][0].delay, 1)
await environment.plugin.stop()
})
test(`${label}: actual worklet emits first channel and survives empty input`, async () => {
const environment = asrAudioEnvironment(implementation)
await environment.emit('play')
const { processor, messages } = await runAsrWorklet(environment.modules[0].blob)
const left = Float32Array.of(0.1, 0.2)
const right = Float32Array.of(0.9, 0.8)
assert.equal(processor.process([]), true)
assert.equal(processor.process([[]]), true)
assert.equal(processor.process([[left, right]]), true)
assert.equal(messages.length, 1)
assert.equal(messages[0], left)
assert.equal(environment.blobs.size, 0)
assert.equal(environment.revoked.length, 1)
await environment.plugin.stop()
})
test(`${label}: historical defect drops sub-threshold audio each tick and excess samples`, async () => {
const chunks = []
const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: chunk => chunks.push(chunk) })
await environment.emit('play')
for (let index = 0; index < 2; index++) {
environment.send([1, 1, 1, 1])
await environment.tick()
}
assert.equal(chunks.length, 0, 'Eight samples across two ticks were lost')
environment.send(new Float32Array(12).fill(0.5))
await environment.tick()
environment.send(new Float32Array(4).fill(-0.5))
await environment.tick()
assert.equal(chunks.length, 1, 'Four excess samples were not retained for the next four')
assert.deepEqual(samples(chunks[0].pcm), Array.from({ length: 8 }, () => 16383))
await environment.plugin.stop()
})
test(`${label}: historical defect overlaps slow callbacks and paints stale completion`, async () => {
const pending = []
const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: () => {
const call = deferredAudio()
pending.push(call)
return call.promise
} })
await environment.emit('play')
environment.send(new Float32Array(8))
const first = environment.tick()
environment.send(new Float32Array(8))
const second = environment.tick()
assert.equal(pending.length, 2, 'No backpressure while the first callback is pending')
pending[1].resolve('New result.')
await second
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">New result.</div>')
pending[0].resolve('Old result.')
await first
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">Old result.</div>')
await environment.plugin.stop()
})
test(`${label}: historical defect rejects the detached interval callback`, async () => {
const failure = new Error('recognizer unavailable')
const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: () => Promise.reject(failure) })
await environment.emit('play')
environment.send(new Float32Array(8))
// Capture the real returned Promise; do not create an unhandled rejection in node:test.
await assert.rejects(environment.tick(), error => error === failure)
assert.equal(environment.intervals.size, 1)
assert.equal(environment.errors.length, 0)
assert.equal(environment.layer.innerHTML, '')
await environment.plugin.stop()
})
for (const event of ['pause', 'destroy', 'stop']) {
test(`${label}: historical defect accepts callback completion after ${event}`, async () => {
const pending = deferredAudio()
const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: () => pending.promise })
await environment.emit('play')
environment.send(new Float32Array(8))
const work = environment.tick()
await (event === 'stop' ? environment.plugin.stop() : environment.emit(event))
assert.equal(environment.intervals.size, 0)
assert.equal(environment.recorders[0].port.onmessage, null)
assert.equal(environment.contexts[0].closes, event === 'pause' ? 0 : 1)
pending.resolve('Late result.')
await work
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">Late result.</div>')
assert.equal(environment.timeouts.size, 1, 'Late completion creates a new hide timer')
await environment.plugin.stop()
})
}
test(`${label}: historical defect resume replaces element source with captured stream`, async () => {
const environment = asrAudioEnvironment(implementation, {}, { suspended: true, webkit: true, moz: true })
await environment.emit('play')
assert.equal(environment.contexts[0].resumes, 1)
assert.equal(environment.captures(), 0)
assert.equal(environment.nodes.find(node => node.kind === 'gain').gain.value, 1, 'Initial gain ignores art.volume=0.5')
await environment.emit('video:volumechange')
assert.equal(environment.nodes.find(node => node.kind === 'gain').gain.value, 0.5)
await environment.emit('pause')
assert.equal(environment.recorders[0].disconnects, 1)
assert.equal(environment.nodes.find(node => node.kind === 'element').disconnects, 0)
await environment.emit('play')
assert.equal(environment.contexts.length, 1)
assert.equal(environment.modules.length, 1)
assert.equal(environment.captures(), 1)
assert.equal(environment.nodes.filter(node => node.kind === 'stream').length, 1)
await environment.plugin.stop()
assert.equal(environment.tracks[0].stops, 1)
assert.equal(environment.contexts[0].closes, 1)
})
test(`${label}: historical defect cannot resume direct source without captureStream`, async () => {
const environment = asrAudioEnvironment(implementation, {}, { capture: false })
await environment.emit('play')
await environment.emit('pause')
await environment.emit('play')
assert.equal(environment.recorders.length, 1)
assert.equal(environment.intervals.size, 0)
assert.equal(environment.errors.length, 1)
assert.equal(environment.errors[0][1].message, 'Could not establish audio source')
await environment.plugin.stop()
})
test(`${label}: fallback stops captured tracks and disconnects on destroy`, async () => {
const environment = asrAudioEnvironment(implementation, {}, { directError: new Error('already attached') })
await environment.emit('play')
assert.equal(environment.captures(), 1)
assert.equal(environment.warnings.length, 1)
const source = environment.nodes.find(node => node.kind === 'stream')
assert.equal(source.connections.length, 2)
await environment.emit('destroy')
assert.equal(source.disconnects, 1)
assert.equal(environment.tracks[0].stops, 1)
assert.equal(environment.contexts[0].closes, 1)
await environment.plugin.stop()
assert.equal(environment.tracks[0].stops, 1)
})
test(`${label}: play after successful initialization is idempotent`, async () => {
const environment = asrAudioEnvironment(implementation)
await environment.emit('play')
await environment.emit('play')
assert.equal(environment.contexts.length, 1)
assert.equal(environment.modules.length, 1)
assert.equal(environment.recorders.length, 1)
assert.equal(environment.intervals.size, 1)
assert.equal(environment.captures(), 0)
await environment.plugin.stop()
})
test(`${label}: historical defect source restart keeps queued audio and stale recognizer result`, async () => {
const calls = []
const pending = deferredAudio()
const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: (chunk) => {
calls.push(chunk)
return calls.length === 1 ? pending.promise : undefined
} })
await environment.emit('play')
environment.send(new Float32Array(8).fill(0.5))
const recognizing = environment.tick()
environment.send(new Float32Array(8).fill(-0.5))
environment.art.video.src = 'local-new-source.mp4'
await environment.emit('restart')
await environment.tick()
assert.equal(calls.length, 2)
assert.deepEqual(samples(calls[1].pcm), Array.from({ length: 8 }, () => -16384))
pending.resolve('Previous source.')
await recognizing
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">Previous source.</div>')
await environment.plugin.stop()
})
test(`${label}: explicit stop followed by play uses capture fallback for already attached video`, async () => {
const environment = asrAudioEnvironment(implementation)
await environment.emit('play')
await environment.plugin.stop()
await environment.emit('play')
assert.equal(environment.contexts.length, 2)
assert.equal(environment.contexts[0].closes, 1)
assert.equal(environment.contexts[1].closes, 0)
assert.equal(environment.captures(), 1)
assert.equal(environment.modules.length, 2)
assert.equal(environment.recorders.length, 2)
assert.equal(environment.warnings[0][1].message, 'HTMLMediaElement already connected to a MediaElementSourceNode')
assert.equal(environment.intervals.size, 1)
await environment.plugin.stop()
})
test(`${label}: historical defect concurrent play creates two graphs and leaks one timer`, async () => {
const gate = deferredAudio()
const environment = asrAudioEnvironment(implementation, {}, { moduleGate: gate })
const first = environment.emit('play')
const second = environment.emit('play')
gate.resolve()
await Promise.all([first, second])
assert.equal(environment.contexts.length, 1)
assert.equal(environment.modules.length, 2)
assert.equal(environment.recorders.length, 2)
assert.equal(environment.intervals.size, 2)
await environment.plugin.stop()
assert.equal(environment.intervals.size, 1, 'Only the most recently assigned interval is cleared')
assert.equal(typeof environment.recorders[0].port.onmessage, 'function')
assert.equal(environment.recorders[0].disconnects, 0)
assert.equal(environment.recorders[1].port.onmessage, null)
})
test(`${label}: historical defect pause during worklet loading does not cancel start`, async () => {
const gate = deferredAudio()
const entered = deferredAudio()
const environment = asrAudioEnvironment(implementation, {}, { moduleGate: gate, moduleEntered: entered })
const starting = environment.emit('play')
await entered.promise
await environment.emit('pause')
assert.equal(environment.intervals.size, 0)
gate.resolve()
await starting
assert.equal(environment.intervals.size, 1)
assert.equal(environment.recorders.length, 1)
await environment.plugin.stop()
})
test(`${label}: historical defect destroy during pending worklet reports late initialization error`, async () => {
const gate = deferredAudio()
const entered = deferredAudio()
const environment = asrAudioEnvironment(implementation, {}, { moduleGate: gate, moduleEntered: entered })
const starting = environment.emit('play')
await entered.promise
await environment.emit('destroy')
assert.equal(environment.contexts[0].closes, 1)
gate.resolve()
await starting
assert.equal(environment.intervals.size, 0)
assert.equal(environment.recorders.length, 0)
assert.equal(environment.revoked.length, 1)
assert.equal(environment.errors.length, 1)
assert.equal(environment.errors[0][1].name, 'TypeError')
assert.match(environment.errors[0][1].message, /createGain/u)
})
test(`${label}: historical defect failed worklet load retains context, source and blob URL`, async () => {
const failure = new Error('worklet unavailable')
const environment = asrAudioEnvironment(implementation, {}, { moduleError: failure })
await environment.emit('play')
assert.equal(environment.errors[0][1], failure)
assert.equal(environment.intervals.size, 0)
assert.equal(environment.contexts[0].closes, 0)
assert.equal(environment.blobs.size, 1)
assert.equal(environment.revoked.length, 0)
const source = environment.nodes.find(node => node.kind === 'element')
assert.equal(source.disconnects, 0)
await environment.plugin.stop()
assert.equal(source.disconnects, 1)
assert.equal(environment.contexts[0].closes, 1)
assert.equal(environment.blobs.size, 1, 'Explicit stop also leaves the failed module URL allocated')
})
}
+69
View File
@@ -0,0 +1,69 @@
import fs from 'node:fs'
import process from 'node:process'
import { verifyAsrContract } from '../../refactor/scripts/asr-contract.mjs'
import { hash, readMember } from '../../refactor/scripts/releases.mjs'
import { expect, test } from './fixtures.js'
let contract
test.beforeAll(async () => {
contract = await verifyAsrContract()
})
for (const version of ['2.0.0', '2.1.0']) {
for (const core of ['published', 'candidate']) {
test(`ASR ${version} + ${core}: native Worklet supplies local PCM and stop closes AudioContext`, async ({ page }, testInfo) => {
const release = [contract.baseline.release, ...contract.baseline.previous].find(release => release.version === version)
const code = readMember(contract.archives.get(version), 'package/dist/artplayer-plugin-asr.js')
const media = fs.readFileSync(new URL('./media/audio-tone.m4a', import.meta.url))
await testInfo.attach('asr-inputs', { contentType: 'application/json', body: JSON.stringify({ version, core, pluginSha256: hash(code), integrity: release.integrity, mediaSha256: hash(media), media: 'local mono AAC 440 Hz tone', networkService: false }) })
await page.goto(`/test/player.html?core=${core}`)
const capabilities = await page.evaluate(() => ({ AudioContext: typeof window.AudioContext, webkitAudioContext: typeof window.webkitAudioContext, AudioWorkletNode: typeof window.AudioWorkletNode }))
await testInfo.attach('asr-audio-capabilities', { contentType: 'application/json', body: JSON.stringify(capabilities) })
const missing = capabilities.AudioContext === 'undefined' && capabilities.webkitAudioContext === 'undefined'
if (missing) {
expect(process.platform).toBe('win32')
expect(testInfo.project.name).toBe('webkit')
expect(capabilities.AudioWorkletNode).toBe('undefined')
test.skip(true, 'Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open')
}
expect(capabilities.AudioWorkletNode).toBe('function')
await page.addScriptTag({ content: code.toString() })
await page.evaluate(() => {
const NativeContext = window.AudioContext || window.webkitAudioContext
window.asrContexts = []
window.AudioContext = class extends NativeContext {
constructor(options) {
super(options)
window.asrContexts.push(this)
}
}
window.asrChunks = []
window.art = new window.Artplayer({
container: '.player',
url: '/test/audio-tone.m4a',
volume: 0.2,
plugins: [window.artplayerPluginAsr({
onAudioChunk({ pcm, wav }) {
const samples = new Int16Array(pcm)
let max = 0
for (const sample of samples) max = Math.max(max, Math.abs(sample))
window.asrChunks.push({ samples: samples.length, max, wavBytes: wav.byteLength, sampleRate: new DataView(wav).getUint32(24, true) })
return 'Local transcript.'
},
})],
})
document.querySelector('#play').onclick = () => window.art.play()
})
await expect.poll(() => page.evaluate(() => window.art.isReady)).toBe(true)
await page.locator('#play').click()
await expect.poll(() => page.evaluate(() => window.asrChunks.filter(chunk => chunk.max > 0).length)).toBeGreaterThanOrEqual(3)
const chunks = await page.evaluate(() => window.asrChunks)
expect(chunks.every(chunk => chunk.samples === 1600 && chunk.wavBytes === 3244 && chunk.sampleRate === 16000)).toBe(true)
await expect(page.locator('.art-layer-asr')).toContainText('Local transcript.')
await page.evaluate(() => window.art.plugins.artplayerPluginAsr.stop())
expect(await page.evaluate(() => window.asrContexts.map(context => context.state))).toEqual(['closed'])
await testInfo.attach('asr-native-chunks', { contentType: 'application/json', body: JSON.stringify(chunks) })
await page.evaluate(() => window.art.destroy())
})
}
}
+183
View File
@@ -0,0 +1,183 @@
import assert from 'node:assert/strict'
import vm from 'node:vm'
import { asrEnvironment } from './asr.js'
export function deferredAudio() {
let resolve
let reject
const promise = new Promise((yes, no) => {
resolve = yes
reject = no
})
return { promise, resolve, reject }
}
export function asrAudioEnvironment(implementation, option = {}, capabilities = {}) {
const environment = asrEnvironment(implementation)
const intervals = new Map()
const contexts = []
const recorders = []
const nodes = []
const modules = []
const blobs = new Map()
const revoked = []
const warnings = []
const errors = []
const tracks = [{
stops: 0,
stop() { this.stops++ },
}]
const stream = { getTracks: () => tracks }
const attachedElements = new WeakSet()
let nextTimer = 0
let nextUrl = 0
let captures = 0
function node(kind) {
const value = {
kind,
connections: [],
disconnects: 0,
connect(target) { this.connections.push(target) },
disconnect() {
this.disconnects++
this.connections = []
},
}
nodes.push(value)
return value
}
class ControlledAudioContext {
constructor(options) {
this.options = options
this.state = capabilities.suspended ? 'suspended' : 'running'
this.destination = node('destination')
this.resumes = 0
this.closes = 0
this.audioWorklet = {
addModule: async (url) => {
const blob = blobs.get(url)
assert.ok(blob, 'The actual plugin worklet must have an object URL')
modules.push({ url, blob })
capabilities.moduleEntered?.resolve()
if (capabilities.moduleGate)
await capabilities.moduleGate.promise
if (capabilities.moduleError)
throw capabilities.moduleError
},
}
contexts.push(this)
}
async resume() {
this.resumes++
this.state = 'running'
}
async close() {
this.closes++
this.state = 'closed'
}
createMediaElementSource(video) {
assert.equal(video, environment.art.video)
if (capabilities.directError)
throw capabilities.directError
if (attachedElements.has(video))
throw new Error('HTMLMediaElement already connected to a MediaElementSourceNode')
attachedElements.add(video)
return node('element')
}
createMediaStreamSource(value) {
assert.equal(value, stream)
return node('stream')
}
createGain() {
return Object.assign(node('gain'), { gain: { value: 0 } })
}
}
class ControlledAudioWorkletNode {
constructor(context, name) {
assert.ok(contexts.includes(context))
assert.equal(context.state, 'running')
assert.equal(name, 'recorder-processor')
Object.assign(this, node('recorder'), { port: { onmessage: null } })
recorders.push(this)
}
}
Object.assign(environment.context, {
Blob,
URL: {
createObjectURL(blob) {
const url = `blob:controlled-asr-${++nextUrl}`
blobs.set(url, blob)
return url
},
revokeObjectURL(url) {
assert.ok(blobs.has(url))
revoked.push(url)
blobs.delete(url)
},
},
AudioWorkletNode: ControlledAudioWorkletNode,
console: { warn: (...args) => warnings.push(args), error: (...args) => errors.push(args) },
setInterval(callback, delay) {
intervals.set(++nextTimer, { callback, delay })
return nextTimer
},
clearInterval(timer) { intervals.delete(timer) },
})
environment.context[capabilities.webkit ? 'webkitAudioContext' : 'AudioContext'] = ControlledAudioContext
if (capabilities.capture !== false) {
environment.art.video[capabilities.moz ? 'mozCaptureStream' : 'captureStream'] = () => {
captures++
return stream
}
}
const plugin = environment.factory(option)(environment.art)
return {
...environment,
plugin,
intervals,
contexts,
recorders,
nodes,
modules,
blobs,
revoked,
tracks,
warnings,
errors,
captures: () => captures,
send(samples, recorder = recorders.at(-1)) {
assert.equal(typeof recorder?.port.onmessage, 'function')
recorder.port.onmessage({ data: Float32Array.from(samples) })
},
tick(timer = [...intervals.keys()][0]) {
assert.ok(intervals.has(timer), 'No live capture interval')
return intervals.get(timer).callback()
},
}
}
export async function runAsrWorklet(blob) {
const messages = []
let Processor
const context = vm.createContext({
AudioWorkletProcessor: class {
port = { postMessage: value => messages.push(value) }
},
registerProcessor(name, implementation) {
assert.equal(name, 'recorder-processor')
Processor = implementation
},
})
vm.runInContext(await blob.text(), context, { timeout: 1000 })
assert.equal(typeof Processor, 'function')
return { processor: new Processor(), messages }
}