fix(asr): [PKG-ASR-08] keep captured fallback off the audible route

test/asr-fallback-routing.test.js

@@ -0,0 +1,130 @@
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import process from 'node:process'
+// eslint-disable-next-line test/no-import-node-test -- Candidate ASR fallback uses the repository runner.
+import test from 'node:test'
+import { asrAudioEnvironment } from './helpers/asr-audio.js'
+import { compilePackage } from './helpers/load.js'
+
+const artifact = process.env.ARTPLAYER_ASR_ARTIFACT
+const implementation = {
+  name: artifact || 'candidate-asr',
+  code: artifact ? fs.readFileSync(artifact, 'utf8') : await compilePackage('artplayer-plugin-asr', 'umd'),
+}
+
+function create(capabilities = {}) {
+  const chunks = []
+  const environment = asrAudioEnvironment(implementation, {
+    sampleRate: 8000,
+    interval: 1,
+    onAudioChunk(chunk) {
+      chunks.push(chunk)
+      return 'Captured.'
+    },
+  }, { directError: new Error('Video belongs to another audio context'), ...capabilities })
+  return { ...environment, chunks }
+}
+
+function assertSilentCaptureRoute(environment) {
+  const source = environment.nodes.filter(node => node.kind === 'stream').at(-1)
+  const recorder = environment.recorders.at(-1)
+  assert.ok(source, 'The occupied media element must use its captured stream')
+  assert.deepEqual(source.connections, [recorder], 'Captured audio must not have a parallel speaker route')
+  assert.equal(recorder.connections.length, 1)
+  const sink = recorder.connections[0]
+  assert.equal(sink.kind, 'gain')
+  assert.equal(sink.gain.value, 0, 'The worklet output sink must stay inaudible')
+  assert.deepEqual(sink.connections, [environment.contexts.at(-1).destination])
+  return { source, recorder, sink }
+}
+
+async function assertCaptures(environment) {
+  const before = environment.chunks.length
+  environment.send(new Float32Array(8).fill(0.5))
+  await environment.tick()
+  assert.equal(environment.chunks.length, before + 1)
+  const chunk = environment.chunks.at(-1)
+  assert.equal(chunk.pcm.byteLength, 16)
+  assert.equal(new DataView(chunk.pcm).getInt16(0, true), 16383)
+  assert.equal(chunk.wav.byteLength, 60)
+  assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">Captured.</div>')
+}
+
+for (const moz of [false, true]) {
+  test(`ASR fallback ${moz ? 'mozCaptureStream' : 'captureStream'} captures PCM through an inaudible sink`, async () => {
+    const environment = create({ moz })
+    await environment.emit('play')
+    assert.equal(environment.captures(), 1)
+    assertSilentCaptureRoute(environment)
+    await assertCaptures(environment)
+    await environment.emit('destroy')
+    assert.equal(environment.contexts[0].closes, 1)
+    assert.equal(environment.tracks[0].stops, 1)
+    assert.equal(environment.intervals.size, 0)
+    assert.ok(environment.nodes.every(node => node.connections.length === 0))
+  })
+}
+
+test('ASR fallback pause disconnects its sink without restoring a raw speaker route', async () => {
+  const environment = create()
+  await environment.emit('play')
+  const retired = assertSilentCaptureRoute(environment)
+  await assertCaptures(environment)
+  await environment.emit('pause')
+  assert.equal(environment.intervals.size, 0)
+  assert.equal(environment.contexts[0].closes, 0)
+  assert.equal(environment.tracks[0].stops, 0)
+  assert.equal(retired.recorder.port.onmessage, null)
+  for (const node of Object.values(retired))
+    assert.deepEqual(node.connections, [])
+  await environment.emit('play')
+  assert.equal(environment.captures(), 1)
+  assert.equal(environment.contexts.length, 1)
+  const resumed = assertSilentCaptureRoute(environment)
+  assert.equal(resumed.source, retired.source)
+  assert.notEqual(resumed.recorder, retired.recorder)
+  await assertCaptures(environment)
+  await environment.emit('destroy')
+  assert.equal(environment.tracks[0].stops, 1)
+})
+
+test('ASR fallback explicit stop releases its graph and subsequent play captures without a speaker route', async () => {
+  const environment = create()
+  await environment.emit('play')
+  const retired = assertSilentCaptureRoute(environment)
+  await environment.plugin.stop()
+  assert.equal(environment.contexts[0].closes, 1)
+  assert.equal(environment.tracks[0].stops, 1)
+  assert.equal(environment.intervals.size, 0)
+  for (const node of Object.values(retired))
+    assert.deepEqual(node.connections, [])
+  await environment.emit('play')
+  assert.equal(environment.captures(), 2)
+  assert.equal(environment.contexts.length, 2)
+  assert.notEqual(assertSilentCaptureRoute(environment).source, retired.source)
+  await assertCaptures(environment)
+  await environment.emit('destroy')
+  assert.equal(environment.contexts[1].closes, 1)
+  assert.equal(environment.tracks[0].stops, 2)
+  assert.ok(environment.nodes.every(node => node.connections.length === 0))
+})
+
+test('ASR fallback source restart replaces its capture and keeps the new graph inaudible', async () => {
+  const environment = create()
+  await environment.emit('play')
+  const retired = assertSilentCaptureRoute(environment)
+  await environment.emit('restart')
+  assert.equal(environment.captures(), 2)
+  assert.equal(environment.contexts[0].closes, 1)
+  assert.equal(environment.contexts[1].closes, 0)
+  assert.equal(environment.tracks[0].stops, 1)
+  assert.equal(retired.recorder.port.onmessage, null)
+  for (const node of Object.values(retired))
+    assert.deepEqual(node.connections, [])
+  assert.notEqual(assertSilentCaptureRoute(environment).source, retired.source)
+  await assertCaptures(environment)
+  await environment.emit('destroy')
+  assert.equal(environment.contexts[1].closes, 1)
+  assert.equal(environment.tracks[0].stops, 2)
+  assert.equal(environment.intervals.size, 0)
+})

test/browser/asr-fallback.spec.js

@@ -0,0 +1,173 @@
+import fs from 'node:fs'
+import process from 'node:process'
+import { verifyAsrContract } from '../../refactor/scripts/asr-contract.mjs'
+import { hash, readMember } from '../../refactor/scripts/releases.mjs'
+import { compilePackage } from '../helpers/load.js'
+import { expect, test } from './fixtures.js'
+
+let contract
+let candidateCode
+test.beforeAll(async () => {
+  contract = await verifyAsrContract()
+  candidateCode = process.env.ARTPLAYER_ASR_ARTIFACT ? fs.readFileSync(process.env.ARTPLAYER_ASR_ARTIFACT) : await compilePackage('artplayer-plugin-asr', 'umd')
+})
+
+for (const version of ['2.1.0', 'candidate']) {
+  for (const core of ['published', 'candidate']) {
+    test(`ASR ${version} + ${core}: native capture fallback does not create a second audible route`, async ({ page }, testInfo) => {
+      const code = version === 'candidate' ? candidateCode : readMember(contract.archives.get(version), 'package/dist/artplayer-plugin-asr.js')
+      await page.goto(`/test/player.html?core=${core}`)
+      const capabilities = await page.evaluate(() => ({ context: typeof AudioContext, worklet: typeof AudioWorkletNode, capture: typeof HTMLMediaElement.prototype.captureStream, mozCapture: typeof HTMLMediaElement.prototype.mozCaptureStream }))
+      const forceRejection = testInfo.project.name === 'firefox'
+      await testInfo.attach('asr-fallback-inputs', { contentType: 'application/json', body: JSON.stringify({ version, core, pluginSha256: hash(code), capabilities, forcedException: forceRejection, reason: forceRejection ? 'Firefox permits a second native media source; force only binding rejection to reach real captureStream/Worklet' : 'Native duplicate binding rejection', externalContext: 'Native media-element source remains owned and running', physicalOutput: false }) })
+      if (capabilities.context !== 'function') {
+        expect(process.platform).toBe('win32')
+        expect(testInfo.project.name).toBe('webkit')
+        test.skip(true, 'Windows WebKit has no native WebAudio')
+      }
+      expect(capabilities.worklet).toBe('function')
+      expect([capabilities.capture, capabilities.mozCapture]).toContain('function')
+      await page.addScriptTag({ content: code.toString() })
+      await page.evaluate((forceRejection) => {
+        const NativeContext = window.AudioContext
+        window.asrContexts = []
+        window.asrMeters = []
+        window.asrStreams = []
+        window.asrChunks = []
+        window.asrDirectFailures = []
+        window.AudioContext = class extends NativeContext {
+          constructor(option) {
+            super(option)
+            window.asrContexts.push(this)
+          }
+
+          createMediaElementSource(video) {
+            try {
+              if (forceRejection)
+                throw new DOMException('Controlled duplicate binding rejection for native capture test', 'InvalidStateError')
+              return super.createMediaElementSource(video)
+            }
+            catch (error) {
+              window.asrDirectFailures.push({ name: error.name, message: error.message })
+              throw error
+            }
+          }
+
+          createGain() {
+            const gain = super.createGain()
+            const analyser = super.createAnalyser()
+            analyser.fftSize = 2048
+            const connect = gain.connect.bind(gain)
+            const disconnect = gain.disconnect.bind(gain)
+            gain.connect = (destination, ...args) => {
+              if (destination === this.destination) {
+                connect(analyser)
+                analyser.connect(destination)
+                return destination
+              }
+              return connect(destination, ...args)
+            }
+            gain.disconnect = (...args) => {
+              analyser.disconnect()
+              return disconnect(...args)
+            }
+            window.asrMeters.push(analyser)
+            return gain
+          }
+        }
+        window.art = new window.Artplayer({
+          container: '.player',
+          url: '/test/audio-tone.m4a',
+          volume: 1,
+          loop: true,
+          plugins: [window.artplayerPluginAsr({ onAudioChunk({ pcm }) {
+            const samples = new Int16Array(pcm)
+            const max = samples.reduce((value, sample) => Math.max(value, Math.abs(sample)), 0)
+            window.asrChunks.push({ max, source: window.art.video.currentSrc, samples: samples.length })
+          } })],
+        })
+        const capture = window.art.video.captureStream || window.art.video.mozCaptureStream
+        window.art.video.captureStream = function () {
+          const stream = capture.call(this)
+          window.asrStreams.push({ stream, tracks: stream.getTracks() })
+          return stream
+        }
+        window.externalContext = new NativeContext({ sampleRate: 16000 })
+        const source = window.externalContext.createMediaElementSource(window.art.video)
+        window.externalMeter = window.externalContext.createAnalyser()
+        window.externalMeter.fftSize = 2048
+        source.connect(window.externalMeter)
+        window.externalMeter.connect(window.externalContext.destination)
+        document.querySelector('#play').onclick = async () => {
+          await window.externalContext.resume()
+          await window.art.play()
+        }
+        window.readFallback = () => {
+          const rms = (analyser) => {
+            const samples = new Float32Array(analyser.fftSize)
+            analyser.getFloatTimeDomainData(samples)
+            return Math.sqrt(samples.reduce((sum, value) => sum + value * value, 0) / samples.length)
+          }
+          return { external: rms(window.externalMeter), asr: window.asrMeters.map(rms), contexts: window.asrContexts.map(c => c.state), owner: window.externalContext.state, streams: window.asrStreams.map(s => s.tracks.map(t => ({ kind: t.kind, state: t.readyState }))), chunks: window.asrChunks.slice(-3), failures: window.asrDirectFailures }
+        }
+      }, forceRejection)
+      await expect.poll(() => page.evaluate(() => window.art.isReady)).toBe(true)
+      await page.locator('#play').click()
+      await expect.poll(() => page.evaluate(() => window.asrChunks.filter(c => c.max > 0).length)).toBeGreaterThanOrEqual(3)
+      const playing = await page.evaluate(() => window.readFallback())
+      await page.evaluate(() => {
+        window.art.muted = true
+      })
+      await page.waitForTimeout(450)
+      const muted = await page.evaluate(() => window.readFallback())
+      await testInfo.attach('asr-fallback-output', { contentType: 'application/json', body: JSON.stringify({ playing, muted }) })
+      expect(playing.failures.length).toBe(1)
+      expect(playing.external).toBeGreaterThan(0.01)
+      expect(muted.external).toBe(0)
+      if (version === '2.1.0') {
+        expect(playing.asr.at(-1)).toBeGreaterThan(0.01)
+        expect(muted.asr.at(-1)).toBeGreaterThan(0.01)
+      }
+      else {
+        expect(playing.asr.at(-1)).toBe(0)
+        expect(muted.asr.at(-1)).toBe(0)
+      }
+      expect(muted.chunks.every(chunk => chunk.max > 0)).toBe(true)
+      await page.evaluate(() => {
+        window.art.muted = false
+      })
+      await page.evaluate(() => window.art.plugins.artplayerPluginAsr.stop())
+      await page.waitForTimeout(200)
+      const stopped = await page.evaluate(() => window.readFallback())
+      expect(stopped.contexts).toEqual(['closed'])
+      expect(stopped.owner).toBe('running')
+      expect(stopped.external).toBeGreaterThan(0.01)
+      expect(stopped.streams.flat().every(t => t.state === 'ended')).toBe(true)
+      await testInfo.attach('asr-fallback-stop', { contentType: 'application/json', body: JSON.stringify(stopped) })
+      if (version === 'candidate') {
+        const count = await page.evaluate(() => window.asrChunks.length)
+        await page.evaluate(() => window.art.pause())
+        await page.locator('#play').click()
+        await expect.poll(() => page.evaluate(() => window.asrChunks.filter(c => c.max > 0).length)).toBeGreaterThan(count + 2)
+        expect(await page.evaluate(() => window.asrStreams[1].tracks.every(track => !window.asrStreams[0].tracks.includes(track)))).toBe(true)
+        await page.evaluate(() => {
+          window.art.url = '/test/audio-tone.m4a?asr=fallback-second'
+        })
+        await expect.poll(() => page.evaluate(() => window.art.video.currentSrc.includes('fallback-second') && window.art.video.readyState >= 2)).toBe(true)
+        await page.locator('#play').click()
+        await expect.poll(() => page.evaluate(() => window.asrChunks.filter(c => c.source.includes('fallback-second') && c.max > 0).length)).toBeGreaterThanOrEqual(3)
+        const restarted = await page.evaluate(() => window.readFallback())
+        expect(restarted.owner).toBe('running')
+        expect(restarted.contexts).toEqual(['closed', 'closed', 'running'])
+        expect(restarted.asr.at(-1)).toBe(0)
+        expect(restarted.streams.slice(0, -1).flat().every(track => track.state === 'ended')).toBe(true)
+        expect(restarted.streams.at(-1).every(track => track.state === 'live')).toBe(true)
+        await testInfo.attach('asr-fallback-restart', { contentType: 'application/json', body: JSON.stringify(restarted) })
+      }
+      await page.evaluate(() => window.art.destroy())
+      await expect.poll(() => page.evaluate(() => window.asrContexts.every(context => context.state === 'closed'))).toBe(true)
+      expect(await page.evaluate(() => window.externalContext.state)).toBe('running')
+      await page.evaluate(() => window.externalContext.close())
+    })
+  }
+}

test/helpers/asr-audio.js

@@ -33,8 +33,8 @@ export function asrAudioEnvironment(implementation, option = {}, capabilities =
   let nextUrl = 0
   let captures = 0
 
-  function node(kind) {
-    const value = {
+  function node(kind, value = {}) {
+    Object.assign(value, {
       kind,
       connections: [],
       disconnects: 0,
@@ -43,7 +43,7 @@ export function asrAudioEnvironment(implementation, option = {}, capabilities =
         this.disconnects++
         this.connections = []
       },
-    }
+    })
     nodes.push(value)
     return value
   }
@@ -105,7 +105,8 @@ export function asrAudioEnvironment(implementation, option = {}, capabilities =
       assert.ok(contexts.includes(context))
       assert.equal(context.state, 'running')
       assert.equal(name, 'recorder-processor')
-      Object.assign(this, node('recorder'), { port: { onmessage: null } })
+      node('recorder', this)
+      this.port = { onmessage: null }
       recorders.push(this)
     }
   }