From 53c295819db004e507b35689a7bdec1955e08396 Mon Sep 17 00:00:00 2001 From: Harvey Zhao Date: Sun, 13 Sep 2026 14:09:20 +0800 Subject: [PATCH] test(asr): [PKG-ASR-02] reproduce audio queue and lifecycle failures --- package.json | 4 +- refactor/baselines/asr-audio-validation.json | 551 ++++++++++++++++++ .../2026-09-13-PKG-ASR-02-audio-baseline.md | 45 ++ refactor/plan.md | 5 +- refactor/progress.md | 11 + refactor/risk-table.md | 4 +- refactor/risks.json | 14 +- refactor/tasks.json | 7 +- test/asr-audio.test.js | 319 ++++++++++ test/browser/asr-audio.spec.js | 69 +++ test/helpers/asr-audio.js | 183 ++++++ 11 files changed, 1200 insertions(+), 12 deletions(-) create mode 100644 refactor/baselines/asr-audio-validation.json create mode 100644 refactor/changes/2026-09-13-PKG-ASR-02-audio-baseline.md create mode 100644 test/asr-audio.test.js create mode 100644 test/browser/asr-audio.spec.js create mode 100644 test/helpers/asr-audio.js diff --git a/package.json b/package.json index 4d5d93099..570e5936b 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "ci:build": "yarn build:types && yarn build all && yarn build:i18n && yarn build:ts && yarn build:docs && yarn test:imports", "test:imports": "node --test test/esm.test.js test/i18n.test.js test/ssr.test.js", "typecheck": "node scripts/typecheck.mjs", - "test:unit": "node --test test/asr.test.js test/jassub.test.js test/multiple-subtitles-merge.test.js test/multiple-subtitles-lifecycle.test.js test/multiple-subtitles-failures.test.js test/multiple-subtitles.test.js test/multiple-subtitles-vendor.test.js test/vtt-thumbnail-parser.test.js test/vtt-thumbnail-lifecycle.test.js test/vtt-thumbnail.test.js test/auto-thumbnail-frames.test.js test/auto-thumbnail-lifecycle.test.js test/auto-thumbnail.test.js test/thumbnail-emitter.test.js test/thumbnail-runtime.test.js test/thumbnail-vendor.test.js test/thumbnail-lifecycle.test.js test/thumbnail-input.test.js test/thumbnail.test.js test/iframe-navigation.test.js test/iframe-boundaries.test.js test/iframe-lifecycle.test.js test/iframe.test.js test/mediabunny.test.js test/mediabunny-shim.test.js test/mediabunny-coordination.test.js test/mediabunny-video.test.js test/mediabunny-audio.test.js test/mediabunny-hls.test.js test/mediabunny-entry.test.js test/mediabunny-capability.test.js test/mediabunny-load.test.js test/mediabunny-input.test.js test/dpip.test.js test/dpip-lifecycle.test.js test/canvas.test.js test/canvas-lifecycle.test.js test/ambilight.test.js test/ambilight-lifecycle.test.js test/ambilight-proxy.test.js test/vast.test.js test/vast-lifecycle.test.js test/ads.test.js test/ads-lifecycle.test.js test/playback.test.js test/dash-control.test.js test/dash-contract.test.js test/dash-lifecycle.test.js test/dash-events.test.js test/hls-control.test.js test/audio-track.test.js test/public-behavior.test.js test/helpers.test.js test/chapter.test.js test/utils.test.js test/resource-scope.test.js test/instance-lifecycle.test.js test/options.test.js test/media-hosts.test.js test/plugins.test.js test/source.test.js test/playback-properties.test.js test/media-events.test.js test/template-resources.test.js test/core-vendor.test.js test/component-resources.test.js test/setting-model.test.js test/setting-layout.test.js test/setting-resources.test.js test/subtitle.test.js test/display-native.test.js test/display-video-fullscreen.test.js test/display-pip.test.js test/display-mini.test.js test/display-sizing.test.js test/display-orientation.test.js test/hotkey.test.js test/listener-registry.test.js test/global-events.test.js test/pointer-events.test.js test/gesture.test.js test/event-scheduling.test.js test/notice.test.js test/fast-forward.test.js test/auto-playback.test.js test/builtin-layers.test.js test/prompt-components.test.js test/screenshot.test.js test/thumbnails.test.js test/progress.test.js test/environment.test.js test/storage.test.js test/facade-properties.test.js test/dom-boundaries.test.js test/initialization.test.js test/entry.test.js test/accessibility-button.test.js test/accessibility-focus.test.js test/accessibility-slider.test.js", + "test:unit": "node --test test/asr-audio.test.js test/asr.test.js test/jassub.test.js test/multiple-subtitles-merge.test.js test/multiple-subtitles-lifecycle.test.js test/multiple-subtitles-failures.test.js test/multiple-subtitles.test.js test/multiple-subtitles-vendor.test.js test/vtt-thumbnail-parser.test.js test/vtt-thumbnail-lifecycle.test.js test/vtt-thumbnail.test.js test/auto-thumbnail-frames.test.js test/auto-thumbnail-lifecycle.test.js test/auto-thumbnail.test.js test/thumbnail-emitter.test.js test/thumbnail-runtime.test.js test/thumbnail-vendor.test.js test/thumbnail-lifecycle.test.js test/thumbnail-input.test.js test/thumbnail.test.js test/iframe-navigation.test.js test/iframe-boundaries.test.js test/iframe-lifecycle.test.js test/iframe.test.js test/mediabunny.test.js test/mediabunny-shim.test.js test/mediabunny-coordination.test.js test/mediabunny-video.test.js test/mediabunny-audio.test.js test/mediabunny-hls.test.js test/mediabunny-entry.test.js test/mediabunny-capability.test.js test/mediabunny-load.test.js test/mediabunny-input.test.js test/dpip.test.js test/dpip-lifecycle.test.js test/canvas.test.js test/canvas-lifecycle.test.js test/ambilight.test.js test/ambilight-lifecycle.test.js test/ambilight-proxy.test.js test/vast.test.js test/vast-lifecycle.test.js test/ads.test.js test/ads-lifecycle.test.js test/playback.test.js test/dash-control.test.js test/dash-contract.test.js test/dash-lifecycle.test.js test/dash-events.test.js test/hls-control.test.js test/audio-track.test.js test/public-behavior.test.js test/helpers.test.js test/chapter.test.js test/utils.test.js test/resource-scope.test.js test/instance-lifecycle.test.js test/options.test.js test/media-hosts.test.js test/plugins.test.js test/source.test.js test/playback-properties.test.js test/media-events.test.js test/template-resources.test.js test/core-vendor.test.js test/component-resources.test.js test/setting-model.test.js test/setting-layout.test.js test/setting-resources.test.js test/subtitle.test.js test/display-native.test.js test/display-video-fullscreen.test.js test/display-pip.test.js test/display-mini.test.js test/display-sizing.test.js test/display-orientation.test.js test/hotkey.test.js test/listener-registry.test.js test/global-events.test.js test/pointer-events.test.js test/gesture.test.js test/event-scheduling.test.js test/notice.test.js test/fast-forward.test.js test/auto-playback.test.js test/builtin-layers.test.js test/prompt-components.test.js test/screenshot.test.js test/thumbnails.test.js test/progress.test.js test/environment.test.js test/storage.test.js test/facade-properties.test.js test/dom-boundaries.test.js test/initialization.test.js test/entry.test.js test/accessibility-button.test.js test/accessibility-focus.test.js test/accessibility-slider.test.js", "test:coverage": "node --test test/coverage.test.js && node scripts/coverage.mjs", "test": "yarn test:node && yarn test:baseline", "test:browser": "playwright test", @@ -94,7 +94,7 @@ "check:ci": "node refactor/scripts/ci-workflow.mjs", "test:ci": "node --test test/ci-summary.test.js refactor/scripts/ci-workflow.test.mjs refactor/scripts/impact.test.mjs test/package-runtime.test.js", "test:package:runtime": "node scripts/package-runtime.mjs --canonical", - "test:asr": "node --test test/asr.test.js" + "test:asr": "node --test test/asr.test.js test/asr-audio.test.js" }, "browserslist": "last 1 Chrome version", "devDependencies": { diff --git a/refactor/baselines/asr-audio-validation.json b/refactor/baselines/asr-audio-validation.json new file mode 100644 index 000000000..19ac8c537 --- /dev/null +++ b/refactor/baselines/asr-audio-validation.json @@ -0,0 +1,551 @@ +{ + "schemaVersion": 1, + "task": "PKG-ASR-02", + "baseCommit": "a22586a172a4f061205c514267473189c6ce759d", + "capturedAt": "2026-09-13T06:08:57.980Z", + "unit": { + "historicalImplementations": 7, + "audioScenariosPerImplementation": 19, + "audioPassed": 133, + "publicPassed": 28, + "provenancePassed": 4, + "command": "node --test test/asr.test.js test/asr-audio.test.js refactor/scripts/asr-contract.test.mjs" + }, + "browser": { + "passed": 8, + "skipped": 4, + "failed": 0, + "rows": [ + { + "title": "ASR 2.0.0 + published: native Worklet supplies local PCM and stop closes AudioContext", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "2.0.0", + "core": "published", + "pluginSha256": "30c7b1fc92a262cda8c3aba63afea9c6c038f52c057039d1c6aa957baed155b6", + "integrity": "sha512-nXTf2drMSKnHRbyU9CQ+3o7Jsf1Meam2O9T8LqYakIr18ag/+i71p2x9zdFrLz16Xa6jfQgKeddmU5r3sXp4LA==", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "chunks": [ + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 852, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 849, + "wavBytes": 3244, + "sampleRate": 16000 + } + ], + "annotations": [] + }, + { + "title": "ASR 2.0.0 + candidate: native Worklet supplies local PCM and stop closes AudioContext", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "2.0.0", + "core": "candidate", + "pluginSha256": "30c7b1fc92a262cda8c3aba63afea9c6c038f52c057039d1c6aa957baed155b6", + "integrity": "sha512-nXTf2drMSKnHRbyU9CQ+3o7Jsf1Meam2O9T8LqYakIr18ag/+i71p2x9zdFrLz16Xa6jfQgKeddmU5r3sXp4LA==", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "chunks": [ + { + "samples": 1600, + "max": 837, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 849, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 850, + "wavBytes": 3244, + "sampleRate": 16000 + } + ], + "annotations": [] + }, + { + "title": "ASR 2.1.0 + published: native Worklet supplies local PCM and stop closes AudioContext", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "2.1.0", + "core": "published", + "pluginSha256": "7f975b131a9ea207f88a157aafec778bc4906230500db9b30f5bbfbf327b251d", + "integrity": "sha512-15gXff458PAx9LeJxKzZEvmpFhfwKUfSLgb1Mfr7Xb/ZGsnE+6EA7ezTnO5S55W5mFExjv/e95iOIXP4ijuruQ==", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "chunks": [ + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 850, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + } + ], + "annotations": [] + }, + { + "title": "ASR 2.1.0 + candidate: native Worklet supplies local PCM and stop closes AudioContext", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "2.1.0", + "core": "candidate", + "pluginSha256": "7f975b131a9ea207f88a157aafec778bc4906230500db9b30f5bbfbf327b251d", + "integrity": "sha512-15gXff458PAx9LeJxKzZEvmpFhfwKUfSLgb1Mfr7Xb/ZGsnE+6EA7ezTnO5S55W5mFExjv/e95iOIXP4ijuruQ==", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "chunks": [ + { + "samples": 1600, + "max": 860, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 850, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 852, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + } + ], + "annotations": [] + }, + { + "title": "ASR 2.0.0 + published: native Worklet supplies local PCM and stop closes AudioContext", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "2.0.0", + "core": "published", + "pluginSha256": "30c7b1fc92a262cda8c3aba63afea9c6c038f52c057039d1c6aa957baed155b6", + "integrity": "sha512-nXTf2drMSKnHRbyU9CQ+3o7Jsf1Meam2O9T8LqYakIr18ag/+i71p2x9zdFrLz16Xa6jfQgKeddmU5r3sXp4LA==", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "chunks": [ + { + "samples": 1600, + "max": 899, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 852, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 838, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 849, + "wavBytes": 3244, + "sampleRate": 16000 + } + ], + "annotations": [] + }, + { + "title": "ASR 2.0.0 + candidate: native Worklet supplies local PCM and stop closes AudioContext", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "2.0.0", + "core": "candidate", + "pluginSha256": "30c7b1fc92a262cda8c3aba63afea9c6c038f52c057039d1c6aa957baed155b6", + "integrity": "sha512-nXTf2drMSKnHRbyU9CQ+3o7Jsf1Meam2O9T8LqYakIr18ag/+i71p2x9zdFrLz16Xa6jfQgKeddmU5r3sXp4LA==", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "chunks": [ + { + "samples": 1600, + "max": 0, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 852, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 849, + "wavBytes": 3244, + "sampleRate": 16000 + } + ], + "annotations": [] + }, + { + "title": "ASR 2.1.0 + published: native Worklet supplies local PCM and stop closes AudioContext", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "2.1.0", + "core": "published", + "pluginSha256": "7f975b131a9ea207f88a157aafec778bc4906230500db9b30f5bbfbf327b251d", + "integrity": "sha512-15gXff458PAx9LeJxKzZEvmpFhfwKUfSLgb1Mfr7Xb/ZGsnE+6EA7ezTnO5S55W5mFExjv/e95iOIXP4ijuruQ==", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "chunks": [ + { + "samples": 1600, + "max": 0, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 840, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 852, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + } + ], + "annotations": [] + }, + { + "title": "ASR 2.1.0 + candidate: native Worklet supplies local PCM and stop closes AudioContext", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "2.1.0", + "core": "candidate", + "pluginSha256": "7f975b131a9ea207f88a157aafec778bc4906230500db9b30f5bbfbf327b251d", + "integrity": "sha512-15gXff458PAx9LeJxKzZEvmpFhfwKUfSLgb1Mfr7Xb/ZGsnE+6EA7ezTnO5S55W5mFExjv/e95iOIXP4ijuruQ==", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "chunks": [ + { + "samples": 1600, + "max": 0, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 852, + "wavBytes": 3244, + "sampleRate": 16000 + }, + { + "samples": 1600, + "max": 851, + "wavBytes": 3244, + "sampleRate": 16000 + } + ], + "annotations": [] + }, + { + "title": "ASR 2.0.0 + published: native Worklet supplies local PCM and stop closes AudioContext", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "2.0.0", + "core": "published", + "pluginSha256": "30c7b1fc92a262cda8c3aba63afea9c6c038f52c057039d1c6aa957baed155b6", + "integrity": "sha512-nXTf2drMSKnHRbyU9CQ+3o7Jsf1Meam2O9T8LqYakIr18ag/+i71p2x9zdFrLz16Xa6jfQgKeddmU5r3sXp4LA==", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "chunks": [], + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 27, + "column": 14 + } + } + ] + }, + { + "title": "ASR 2.0.0 + candidate: native Worklet supplies local PCM and stop closes AudioContext", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "2.0.0", + "core": "candidate", + "pluginSha256": "30c7b1fc92a262cda8c3aba63afea9c6c038f52c057039d1c6aa957baed155b6", + "integrity": "sha512-nXTf2drMSKnHRbyU9CQ+3o7Jsf1Meam2O9T8LqYakIr18ag/+i71p2x9zdFrLz16Xa6jfQgKeddmU5r3sXp4LA==", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "chunks": [], + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 27, + "column": 14 + } + } + ] + }, + { + "title": "ASR 2.1.0 + published: native Worklet supplies local PCM and stop closes AudioContext", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "2.1.0", + "core": "published", + "pluginSha256": "7f975b131a9ea207f88a157aafec778bc4906230500db9b30f5bbfbf327b251d", + "integrity": "sha512-15gXff458PAx9LeJxKzZEvmpFhfwKUfSLgb1Mfr7Xb/ZGsnE+6EA7ezTnO5S55W5mFExjv/e95iOIXP4ijuruQ==", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "chunks": [], + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 27, + "column": 14 + } + } + ] + }, + { + "title": "ASR 2.1.0 + candidate: native Worklet supplies local PCM and stop closes AudioContext", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "2.1.0", + "core": "candidate", + "pluginSha256": "7f975b131a9ea207f88a157aafec778bc4906230500db9b30f5bbfbf327b251d", + "integrity": "sha512-15gXff458PAx9LeJxKzZEvmpFhfwKUfSLgb1Mfr7Xb/ZGsnE+6EA7ezTnO5S55W5mFExjv/e95iOIXP4ijuruQ==", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "chunks": [], + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 27, + "column": 14 + } + } + ] + } + ], + "reportPath": "refactor/.cache/asr02-browser-final/report.json", + "reportSha256": "e6c6de70406c39a5d24c55505aa77837054c681c613baebcc30d38fcb9fce993", + "firstRun": { + "passed": 8, + "failed": 4, + "reason": "Fixture extended an unavailable AudioContext constructor; independent native probe found all three WebAudio globals undefined in Windows WebKit 26.6.", + "reportPath": "refactor/.cache/asr02-browser-before/report.json" + } + }, + "limits": [ + "Controlled audio is not native audio evidence.", + "Windows WebKit has no WebAudio; Safari/device acceptance remains pending.", + "Native tests cover initial capture and stop only; candidate resource fixes need new regressions.", + "No external ASR service or microphone request." + ] +} diff --git a/refactor/changes/2026-09-13-PKG-ASR-02-audio-baseline.md b/refactor/changes/2026-09-13-PKG-ASR-02-audio-baseline.md new file mode 100644 index 000000000..594f866b7 --- /dev/null +++ b/refactor/changes/2026-09-13-PKG-ASR-02-audio-baseline.md @@ -0,0 +1,45 @@ +# PKG-ASR-02：ASR 音频与错误基线 + +在 ASR-01 固定的 7 个实现上运行 19 类场景：npm 2.0.0/2.1.0 main/legacy、 +冻结工作区源码/main/legacy。133 项受控音频测试通过；这些测试明确观察历史 +缺陷，不能接到候选上要求保留缺陷。32 项既有公开契约仍通过。 + +## 可重跑证据 + +- `yarn test:asr` 运行公开契约及音频历史场景，161 项；归档校验另有 4 项。 +- `yarn test:browser test/browser/asr-audio.spec.js` 使用实际发布包及本地 AAC 样本， + 两版插件乘新旧核心在 Chromium/Firefox 共 8 项通过；Windows WebKit 26.6 的 + AudioContext、webkitAudioContext、AudioWorkletNode 均不存在，4 项明确跳过。 +- 初次浏览器执行因假设 AudioContext 可用而在 WebKit 失败；独立原生探针确认 + 能力缺失后先记录能力再做精确平台断言，不放宽 Chromium/Firefox 要求。 +- 每个真正采集通过的场景至少 3 块非零音频，每块 1600 个样本、3244 字节 WAV、 + 16000Hz，并渲染本地回调字幕，显式 stop 后实际 AudioContext 为 closed。 +- 报告、引擎版本、输入哈希和限制见 [持久证据](../baselines/asr-audio-validation.json)。 + 没有调用外部 ASR、没有真实麦克风授权、没有真机 Safari 验收。 + +## 复现结果与修复职责 + +| 行为 | 实际观察 | 后续 | +| --- | --- | --- | +| PCM/WAV | 钳位、PCM16LE、mono WAV 头、首通道、定制采样字节精确匹配 | 保持编码契约 | +| 队列 | 两次半块输入全部丢弃；超出阈值的尾部也丢弃 | ASR-CHUNK-01 / ASR-03 | +| 异步回调 | 并发无背压、乱序旧字幕覆盖新字幕，拒绝透出 async interval | ASR-CHUNK-01 / ASR-03 | +| 停止 | pause/stop/destroy 后在途结果仍 append 并创建 hide timer | ASR-LIFE-01 / ASR-03 | +| 恢复 | 已有 direct source 未复用，改走捕获流；无 captureStream 时恢复失败 | ASR-LIFE-01 / ASR-03 | +| 初始化 | 并发 play 产生两图两 interval，stop 只清最后一组；pause 不取消在途 setup | ASR-LIFE-01 / ASR-03 | +| 异常 | addModule 拒绝留下 context/source/URL；显式 stop 仍未回收 URL | ASR-LIFE-01 / ASR-03 | +| 终止竞态 | destroy 后 addModule 完成仍继续访问已清空 context | ASR-LIFE-01 / ASR-03 | +| 切源 | restart 不清旧 PCM、不使旧识别结果失效 | ASR-LIFE-01 / ASR-03 | +| 增益 | 初始 gain=1，首次 volumechange 才跟随 art.volume | 原生音量对照后修复，避免重复衰减 | + +受控环境执行实际 inline Worklet 源码；显式 Promise gate 控制初始化和识别顺序。 +AudioContext/节点/捕获流由测试替代，无法证明音量、设备采样率、CORS 或音源恢复 +在所有真实引擎中正常。实际浏览器目前只证明初次采集、字幕与 stop，恢复/切源/ +并发修复后还需原生回归。ASR-05 保持未完成。 + +## 协作、兼容与回退 + +子代理独立编写受控 helper/tests，主代理审查并接入任务、浏览器与报告。没有生产 +源码、类型、版本或依赖改变。候选修复将增加独立断言，历史缺陷不会被删掉或改绿。 +只运行本包测试、定向 ESLint 和文档台账检查，未重复全量 CI。 +独立提交回退可移除新增测试、证据和命令接入，不改变播放器行为。 diff --git a/refactor/plan.md b/refactor/plan.md index 1b5d43e89..9efe68b5b 100644 --- a/refactor/plan.md +++ b/refactor/plan.md @@ -4,7 +4,7 @@ 基线：`40fcda6a37d0049d42e49c1e64e70d4fd9ba5f7f`。总任务 225 项，范围 22 个包及工作区/示例。 -状态：todo 85 / doing 16 / blocked 0 / done 124 / deferred 0。风险 L/M/H 表示兼容风险，不表示工期。 +状态：todo 84 / doing 16 / blocked 0 / done 125 / deferred 0。风险 L/M/H 表示兼容风险，不表示工期。 前置依赖是启动条件；验收是完成条件。任务可以继续拆分，但不能复用或悄悄删除旧 ID。 @@ -252,7 +252,7 @@ | ID | 范围 / 步骤 | 前置依赖 | 交付物 | 验收条件 | 风险 | 状态 | | --- | --- | --- | --- | --- | --- | --- | | PKG-ASR-01 | artplayer-plugin-asr
核对包契约与历史用法 | BASE-05 | length/interval/sampleRate/onAudioChunk、append/hide/stop 的真实能力 | 源码/声明/README/demo/发布包差异已登记；公开形状和版本范围冻结 | H | done | -| PKG-ASR-02 | artplayer-plugin-asr
建立特有行为与错误测试 | PKG-ASR-01, ENG-03, ENG-05 | PCM/WAV、chunk 时序、回调慢/拒绝、重复初始化与停止 | 旧版本行为可重跑，成功/失败/切源/销毁有必要断言 | H | todo | +| PKG-ASR-02 | artplayer-plugin-asr
建立特有行为与错误测试 | PKG-ASR-01, ENG-03, ENG-05 | PCM/WAV、chunk 时序、回调慢/拒绝、重复初始化与停止 | 旧版本行为可重跑，成功/失败/切源/销毁有必要断言 | H | done | | PKG-ASR-03 | artplayer-plugin-asr
整理内部职责与资源 | PKG-ASR-02, CORE-10, CORE-18 | AudioContext/Worklet/Stream 生命周期和背压分离 | 结构变化和缺陷修复分开记录；原 API/事件/资源生命周期通过 | H | todo | | PKG-ASR-04 | artplayer-plugin-asr
迁移自有源码和公开类型 | PKG-ASR-03, ENG-04, ENG-06, CORE-07 | 主线程/Worklet/音频 buffer 与异步回调类型 | 严格类型检查、旧消费样例通过；声明路径/导出和同步异步兼容 | H | todo | | PKG-ASR-05 | artplayer-plugin-asr
验证新旧核心和组合 | PKG-ASR-04, CORE-22 | 真实 WebAudio 分块、播放暂停/切源/销毁，不引入网络 ASR | 最终核心与原支持范围核心分别通过；设备/SDK 缺证据不能标完成 | H | todo | @@ -527,6 +527,7 @@ - PKG-MULTI-SUB-07: [记录](baselines/multiple-subtitles-timestamps.json) [记录](changes/2026-09-13-PKG-MULTI-SUB-07-timestamps.md) - PKG-JASSUB-01: [记录](baselines/jassub-release.json) [记录](baselines/jassub-vendor.json) [记录](baselines/jassub-font-metadata.json) [记录](baselines/jassub-contract.md) [记录](baselines/jassub-contract-validation.json) [记录](changes/2026-09-13-PKG-JASSUB-01-baseline.md) - PKG-ASR-01: [记录](changes/2026-09-13-PKG-ASR-01-contracts.md) [记录](baselines/asr-contract.md) [记录](baselines/asr-release.json) +- PKG-ASR-02: [记录](changes/2026-09-13-PKG-ASR-02-audio-baseline.md) [记录](baselines/asr-audio-validation.json) - PKG-ADS-01: [记录](baselines/ads-release.json) [记录](baselines/ads-contract.md) [记录](baselines/ads-contract-validation.json) [记录](changes/2026-09-12-PKG-ADS-01-contract.md) [记录](scripts/ads-contract.test.mjs) - PKG-ADS-02: [记录](changes/2026-09-12-PKG-ADS-02-tests.md) [记录](ads-validation.md) [记录](baselines/ads-validation.json) - PKG-ADS-03: [记录](changes/2026-09-12-PKG-ADS-03-lifecycle.md) [记录](ads-validation.md) [记录](baselines/ads-lifecycle-validation.json) diff --git a/refactor/progress.md b/refactor/progress.md index 6902a8e59..6d2f2d278 100644 --- a/refactor/progress.md +++ b/refactor/progress.md @@ -1,5 +1,16 @@ # 进度与证据 +## PKG-ASR-02 音频与错误基线完成 + +子代理交付并由主代理审查的 133 项音频历史测试通过，连同公开契约/来源共 165 项。 +复现丢样本、异步乱序、停止后字幕回写、初始化竞态、URL/节点残留和切源旧数据。 +真实 Chromium/Firefox 共 8 组合通过；Windows WebKit 26.6 原生 WebAudio 全局缺失， +4 项明确跳过，未计入采集通过；首次失败及能力诊断保留。未请求外部识别服务。 +见[变更](changes/2026-09-13-PKG-ASR-02-audio-baseline.md)和[证据](baselines/asr-audio-validation.json)。 +225 项：125 done、16 doing、84 todo；205 风险，音频队列/生命周期风险升级为已复现。 +下一步 ASR-03 拆分生产源码并逐项修复，历史测试保留，候选正确行为另建回归。 + + ## PKG-ASR-01 发布契约完成 冻结实际 npm 2.0.0/2.1.0 共 12 成员及工作区 9 个 Git 输入；32 项契约测试通过， diff --git a/refactor/risk-table.md b/refactor/risk-table.md index fa1ed810d..51d9d0679 100644 --- a/refactor/risk-table.md +++ b/refactor/risk-table.md @@ -207,5 +207,5 @@ | CORE-DEFAULTS-SSR-01 | resolved / 已复现 | Reading static defaults throws when navigator is absent | CORE-25 | | ENG-DIST-PUBLIC-01 | resolved / 已复现 | Vite copied authored declarations into library distribution | ENG-12 | | ASR-TYPE-01 | open / 源码/产物事实 | ASR declarations omit asynchronous stop and callback text results | PKG-ASR-04 | -| ASR-CHUNK-01 | open / 源码/产物事实 | ASR tick discards underfilled chunks and overrun samples in source | PKG-ASR-02, PKG-ASR-03 | -| ASR-LIFE-01 | open / 源码/产物事实 | ASR partial initialization, pause resume and stale callback cleanup need controlled evidence | PKG-ASR-02, PKG-ASR-03, PKG-ASR-05 | +| ASR-CHUNK-01 | open / 已复现 | ASR tick discards underfilled chunks and overrun samples in source | PKG-ASR-02, PKG-ASR-03 | +| ASR-LIFE-01 | open / 已复现 | ASR partial initialization, pause resume and stale callback cleanup need controlled evidence | PKG-ASR-02, PKG-ASR-03, PKG-ASR-05 | diff --git a/refactor/risks.json b/refactor/risks.json index 2dcb0290d..b450d3ec4 100644 --- a/refactor/risks.json +++ b/refactor/risks.json @@ -4560,7 +4560,7 @@ { "id": "ASR-CHUNK-01", "title": "ASR tick discards underfilled chunks and overrun samples in source", - "confirmation": "source-observed", + "confirmation": "reproduced", "status": "open", "owners": [ "PKG-ASR-02", @@ -4568,7 +4568,10 @@ ], "evidence": [ "refactor/baselines/asr-contract.md", - "refactor/baselines/asr-release.json" + "refactor/baselines/asr-release.json", + "test/asr-audio.test.js", + "refactor/changes/2026-09-13-PKG-ASR-02-audio-baseline.md", + "refactor/baselines/asr-audio-validation.json" ], "compatibleResolution": "Reproduce exact sample loss and retain queue remainders with bounded, ordered callback delivery.", "closureCriteria": "Exact PCM/WAV byte sequence and slow/rejected callback regression tests pass without losing or reordering accepted input." @@ -4576,7 +4579,7 @@ { "id": "ASR-LIFE-01", "title": "ASR partial initialization, pause resume and stale callback cleanup need controlled evidence", - "confirmation": "source-observed", + "confirmation": "reproduced", "status": "open", "owners": [ "PKG-ASR-02", @@ -4585,7 +4588,10 @@ ], "evidence": [ "refactor/baselines/asr-contract.md", - "refactor/baselines/asr-release.json" + "refactor/baselines/asr-release.json", + "test/asr-audio.test.js", + "refactor/changes/2026-09-13-PKG-ASR-02-audio-baseline.md", + "refactor/baselines/asr-audio-validation.json" ], "compatibleResolution": "Own pending setup, URLs, source nodes and callbacks across pause, stop, restart and terminal destruction.", "closureCriteria": "Failure/race regression tests plus native WebAudio playback and cleanup evidence prove resource ownership." diff --git a/refactor/tasks.json b/refactor/tasks.json index 006f6d3bc..b468ecf9e 100644 --- a/refactor/tasks.json +++ b/refactor/tasks.json @@ -2639,11 +2639,14 @@ "ENG-03", "ENG-05" ], - "status": "todo", + "status": "done", "risk": "H", "deliverable": "PCM/WAV、chunk 时序、回调慢/拒绝、重复初始化与停止", "acceptance": "旧版本行为可重跑，成功/失败/切源/销毁有必要断言", - "evidence": [] + "evidence": [ + "changes/2026-09-13-PKG-ASR-02-audio-baseline.md", + "baselines/asr-audio-validation.json" + ] }, { "id": "PKG-ASR-03", diff --git a/test/asr-audio.test.js b/test/asr-audio.test.js new file mode 100644 index 000000000..93592cff0 --- /dev/null +++ b/test/asr-audio.test.js @@ -0,0 +1,319 @@ +import assert from 'node:assert/strict' +import { Buffer } from 'node:buffer' +// eslint-disable-next-line test/no-import-node-test -- Historical ASR audio uses the repository runner. +import test from 'node:test' +import { asrAudioEnvironment, deferredAudio, runAsrWorklet } from './helpers/asr-audio.js' +import { asrHistorical } from './helpers/asr.js' + +// These fixtures are immutable historical implementations, never the candidate. +// Defect observations must become separate red/green candidate regressions when fixed. +const implementations = await asrHistorical() + +function samples(pcm) { + const view = new DataView(pcm) + return Array.from({ length: pcm.byteLength / 2 }, (_, index) => view.getInt16(index * 2, true)) +} + +for (const implementation of implementations) { + const label = `ASR audio history ${implementation.name}` + + test(`${label}: default PCM clamping and mono WAV bytes`, async () => { + const chunks = [] + const environment = asrAudioEnvironment(implementation, { onAudioChunk: chunk => chunks.push(chunk) }) + await environment.emit('play') + assert.equal(environment.contexts[0].options.sampleRate, 16000) + assert.equal([...environment.intervals.values()][0].delay, 100) + const input = new Float32Array(1600) + input.set([-2, -1, -0.5, 0, 0.5, 1, 2, Number.NaN]) + environment.send(input) + await environment.tick() + assert.equal(chunks.length, 1) + assert.deepEqual(Object.keys(chunks[0]), ['pcm', 'wav']) + assert.equal(chunks[0].pcm.byteLength, 3200) + assert.deepEqual(samples(chunks[0].pcm).slice(0, 8), [-32768, -32768, -16384, 0, 16383, 32767, 32767, 0]) + const wav = Buffer.from(chunks[0].wav) + assert.equal(wav.length, 3244) + assert.equal(wav.toString('ascii', 0, 4), 'RIFF') + assert.equal(wav.readUInt32LE(4), 3236) + assert.equal(wav.toString('ascii', 8, 16), 'WAVEfmt ') + assert.equal(wav.readUInt32LE(16), 16) + assert.equal(wav.readUInt16LE(20), 1) + assert.equal(wav.readUInt16LE(22), 1) + assert.equal(wav.readUInt32LE(24), 16000) + assert.equal(wav.readUInt32LE(28), 32000) + assert.equal(wav.readUInt16LE(32), 2) + assert.equal(wav.readUInt16LE(34), 16) + assert.equal(wav.toString('ascii', 36, 40), 'data') + assert.equal(wav.readUInt32LE(40), 3200) + assert.deepEqual(wav.subarray(44), Buffer.from(chunks[0].pcm)) + await environment.plugin.stop() + }) + + test(`${label}: custom sampling concatenates worklet messages in order`, async () => { + const chunks = [] + const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: chunk => chunks.push(chunk) }) + await environment.emit('play') + environment.send([-1, -0.5, 0]) + environment.send([0.5, 1, 0, -1, 1]) + await environment.tick() + assert.equal(chunks.length, 1) + assert.deepEqual(samples(chunks[0].pcm), [-32768, -16384, 0, 16383, 32767, 0, -32768, 32767]) + assert.equal(new DataView(chunks[0].wav).getUint32(24, true), 8000) + assert.equal([...environment.intervals.values()][0].delay, 1) + await environment.plugin.stop() + }) + + test(`${label}: actual worklet emits first channel and survives empty input`, async () => { + const environment = asrAudioEnvironment(implementation) + await environment.emit('play') + const { processor, messages } = await runAsrWorklet(environment.modules[0].blob) + const left = Float32Array.of(0.1, 0.2) + const right = Float32Array.of(0.9, 0.8) + assert.equal(processor.process([]), true) + assert.equal(processor.process([[]]), true) + assert.equal(processor.process([[left, right]]), true) + assert.equal(messages.length, 1) + assert.equal(messages[0], left) + assert.equal(environment.blobs.size, 0) + assert.equal(environment.revoked.length, 1) + await environment.plugin.stop() + }) + + test(`${label}: historical defect drops sub-threshold audio each tick and excess samples`, async () => { + const chunks = [] + const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: chunk => chunks.push(chunk) }) + await environment.emit('play') + for (let index = 0; index < 2; index++) { + environment.send([1, 1, 1, 1]) + await environment.tick() + } + assert.equal(chunks.length, 0, 'Eight samples across two ticks were lost') + environment.send(new Float32Array(12).fill(0.5)) + await environment.tick() + environment.send(new Float32Array(4).fill(-0.5)) + await environment.tick() + assert.equal(chunks.length, 1, 'Four excess samples were not retained for the next four') + assert.deepEqual(samples(chunks[0].pcm), Array.from({ length: 8 }, () => 16383)) + await environment.plugin.stop() + }) + + test(`${label}: historical defect overlaps slow callbacks and paints stale completion`, async () => { + const pending = [] + const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: () => { + const call = deferredAudio() + pending.push(call) + return call.promise + } }) + await environment.emit('play') + environment.send(new Float32Array(8)) + const first = environment.tick() + environment.send(new Float32Array(8)) + const second = environment.tick() + assert.equal(pending.length, 2, 'No backpressure while the first callback is pending') + pending[1].resolve('New result.') + await second + assert.equal(environment.layer.innerHTML, '
New result.
') + pending[0].resolve('Old result.') + await first + assert.equal(environment.layer.innerHTML, '
Old result.
') + await environment.plugin.stop() + }) + + test(`${label}: historical defect rejects the detached interval callback`, async () => { + const failure = new Error('recognizer unavailable') + const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: () => Promise.reject(failure) }) + await environment.emit('play') + environment.send(new Float32Array(8)) + // Capture the real returned Promise; do not create an unhandled rejection in node:test. + await assert.rejects(environment.tick(), error => error === failure) + assert.equal(environment.intervals.size, 1) + assert.equal(environment.errors.length, 0) + assert.equal(environment.layer.innerHTML, '') + await environment.plugin.stop() + }) + + for (const event of ['pause', 'destroy', 'stop']) { + test(`${label}: historical defect accepts callback completion after ${event}`, async () => { + const pending = deferredAudio() + const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: () => pending.promise }) + await environment.emit('play') + environment.send(new Float32Array(8)) + const work = environment.tick() + await (event === 'stop' ? environment.plugin.stop() : environment.emit(event)) + assert.equal(environment.intervals.size, 0) + assert.equal(environment.recorders[0].port.onmessage, null) + assert.equal(environment.contexts[0].closes, event === 'pause' ? 0 : 1) + pending.resolve('Late result.') + await work + assert.equal(environment.layer.innerHTML, '
Late result.
') + assert.equal(environment.timeouts.size, 1, 'Late completion creates a new hide timer') + await environment.plugin.stop() + }) + } + + test(`${label}: historical defect resume replaces element source with captured stream`, async () => { + const environment = asrAudioEnvironment(implementation, {}, { suspended: true, webkit: true, moz: true }) + await environment.emit('play') + assert.equal(environment.contexts[0].resumes, 1) + assert.equal(environment.captures(), 0) + assert.equal(environment.nodes.find(node => node.kind === 'gain').gain.value, 1, 'Initial gain ignores art.volume=0.5') + await environment.emit('video:volumechange') + assert.equal(environment.nodes.find(node => node.kind === 'gain').gain.value, 0.5) + await environment.emit('pause') + assert.equal(environment.recorders[0].disconnects, 1) + assert.equal(environment.nodes.find(node => node.kind === 'element').disconnects, 0) + await environment.emit('play') + assert.equal(environment.contexts.length, 1) + assert.equal(environment.modules.length, 1) + assert.equal(environment.captures(), 1) + assert.equal(environment.nodes.filter(node => node.kind === 'stream').length, 1) + await environment.plugin.stop() + assert.equal(environment.tracks[0].stops, 1) + assert.equal(environment.contexts[0].closes, 1) + }) + + test(`${label}: historical defect cannot resume direct source without captureStream`, async () => { + const environment = asrAudioEnvironment(implementation, {}, { capture: false }) + await environment.emit('play') + await environment.emit('pause') + await environment.emit('play') + assert.equal(environment.recorders.length, 1) + assert.equal(environment.intervals.size, 0) + assert.equal(environment.errors.length, 1) + assert.equal(environment.errors[0][1].message, 'Could not establish audio source') + await environment.plugin.stop() + }) + + test(`${label}: fallback stops captured tracks and disconnects on destroy`, async () => { + const environment = asrAudioEnvironment(implementation, {}, { directError: new Error('already attached') }) + await environment.emit('play') + assert.equal(environment.captures(), 1) + assert.equal(environment.warnings.length, 1) + const source = environment.nodes.find(node => node.kind === 'stream') + assert.equal(source.connections.length, 2) + await environment.emit('destroy') + assert.equal(source.disconnects, 1) + assert.equal(environment.tracks[0].stops, 1) + assert.equal(environment.contexts[0].closes, 1) + await environment.plugin.stop() + assert.equal(environment.tracks[0].stops, 1) + }) + + test(`${label}: play after successful initialization is idempotent`, async () => { + const environment = asrAudioEnvironment(implementation) + await environment.emit('play') + await environment.emit('play') + assert.equal(environment.contexts.length, 1) + assert.equal(environment.modules.length, 1) + assert.equal(environment.recorders.length, 1) + assert.equal(environment.intervals.size, 1) + assert.equal(environment.captures(), 0) + await environment.plugin.stop() + }) + + test(`${label}: historical defect source restart keeps queued audio and stale recognizer result`, async () => { + const calls = [] + const pending = deferredAudio() + const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: (chunk) => { + calls.push(chunk) + return calls.length === 1 ? pending.promise : undefined + } }) + await environment.emit('play') + environment.send(new Float32Array(8).fill(0.5)) + const recognizing = environment.tick() + environment.send(new Float32Array(8).fill(-0.5)) + environment.art.video.src = 'local-new-source.mp4' + await environment.emit('restart') + await environment.tick() + assert.equal(calls.length, 2) + assert.deepEqual(samples(calls[1].pcm), Array.from({ length: 8 }, () => -16384)) + pending.resolve('Previous source.') + await recognizing + assert.equal(environment.layer.innerHTML, '
Previous source.
') + await environment.plugin.stop() + }) + + test(`${label}: explicit stop followed by play uses capture fallback for already attached video`, async () => { + const environment = asrAudioEnvironment(implementation) + await environment.emit('play') + await environment.plugin.stop() + await environment.emit('play') + assert.equal(environment.contexts.length, 2) + assert.equal(environment.contexts[0].closes, 1) + assert.equal(environment.contexts[1].closes, 0) + assert.equal(environment.captures(), 1) + assert.equal(environment.modules.length, 2) + assert.equal(environment.recorders.length, 2) + assert.equal(environment.warnings[0][1].message, 'HTMLMediaElement already connected to a MediaElementSourceNode') + assert.equal(environment.intervals.size, 1) + await environment.plugin.stop() + }) + + test(`${label}: historical defect concurrent play creates two graphs and leaks one timer`, async () => { + const gate = deferredAudio() + const environment = asrAudioEnvironment(implementation, {}, { moduleGate: gate }) + const first = environment.emit('play') + const second = environment.emit('play') + gate.resolve() + await Promise.all([first, second]) + assert.equal(environment.contexts.length, 1) + assert.equal(environment.modules.length, 2) + assert.equal(environment.recorders.length, 2) + assert.equal(environment.intervals.size, 2) + await environment.plugin.stop() + assert.equal(environment.intervals.size, 1, 'Only the most recently assigned interval is cleared') + assert.equal(typeof environment.recorders[0].port.onmessage, 'function') + assert.equal(environment.recorders[0].disconnects, 0) + assert.equal(environment.recorders[1].port.onmessage, null) + }) + + test(`${label}: historical defect pause during worklet loading does not cancel start`, async () => { + const gate = deferredAudio() + const entered = deferredAudio() + const environment = asrAudioEnvironment(implementation, {}, { moduleGate: gate, moduleEntered: entered }) + const starting = environment.emit('play') + await entered.promise + await environment.emit('pause') + assert.equal(environment.intervals.size, 0) + gate.resolve() + await starting + assert.equal(environment.intervals.size, 1) + assert.equal(environment.recorders.length, 1) + await environment.plugin.stop() + }) + + test(`${label}: historical defect destroy during pending worklet reports late initialization error`, async () => { + const gate = deferredAudio() + const entered = deferredAudio() + const environment = asrAudioEnvironment(implementation, {}, { moduleGate: gate, moduleEntered: entered }) + const starting = environment.emit('play') + await entered.promise + await environment.emit('destroy') + assert.equal(environment.contexts[0].closes, 1) + gate.resolve() + await starting + assert.equal(environment.intervals.size, 0) + assert.equal(environment.recorders.length, 0) + assert.equal(environment.revoked.length, 1) + assert.equal(environment.errors.length, 1) + assert.equal(environment.errors[0][1].name, 'TypeError') + assert.match(environment.errors[0][1].message, /createGain/u) + }) + + test(`${label}: historical defect failed worklet load retains context, source and blob URL`, async () => { + const failure = new Error('worklet unavailable') + const environment = asrAudioEnvironment(implementation, {}, { moduleError: failure }) + await environment.emit('play') + assert.equal(environment.errors[0][1], failure) + assert.equal(environment.intervals.size, 0) + assert.equal(environment.contexts[0].closes, 0) + assert.equal(environment.blobs.size, 1) + assert.equal(environment.revoked.length, 0) + const source = environment.nodes.find(node => node.kind === 'element') + assert.equal(source.disconnects, 0) + await environment.plugin.stop() + assert.equal(source.disconnects, 1) + assert.equal(environment.contexts[0].closes, 1) + assert.equal(environment.blobs.size, 1, 'Explicit stop also leaves the failed module URL allocated') + }) +} diff --git a/test/browser/asr-audio.spec.js b/test/browser/asr-audio.spec.js new file mode 100644 index 000000000..8ebde5ec5 --- /dev/null +++ b/test/browser/asr-audio.spec.js @@ -0,0 +1,69 @@ +import fs from 'node:fs' +import process from 'node:process' +import { verifyAsrContract } from '../../refactor/scripts/asr-contract.mjs' +import { hash, readMember } from '../../refactor/scripts/releases.mjs' +import { expect, test } from './fixtures.js' + +let contract +test.beforeAll(async () => { + contract = await verifyAsrContract() +}) + +for (const version of ['2.0.0', '2.1.0']) { + for (const core of ['published', 'candidate']) { + test(`ASR ${version} + ${core}: native Worklet supplies local PCM and stop closes AudioContext`, async ({ page }, testInfo) => { + const release = [contract.baseline.release, ...contract.baseline.previous].find(release => release.version === version) + const code = readMember(contract.archives.get(version), 'package/dist/artplayer-plugin-asr.js') + const media = fs.readFileSync(new URL('./media/audio-tone.m4a', import.meta.url)) + await testInfo.attach('asr-inputs', { contentType: 'application/json', body: JSON.stringify({ version, core, pluginSha256: hash(code), integrity: release.integrity, mediaSha256: hash(media), media: 'local mono AAC 440 Hz tone', networkService: false }) }) + await page.goto(`/test/player.html?core=${core}`) + const capabilities = await page.evaluate(() => ({ AudioContext: typeof window.AudioContext, webkitAudioContext: typeof window.webkitAudioContext, AudioWorkletNode: typeof window.AudioWorkletNode })) + await testInfo.attach('asr-audio-capabilities', { contentType: 'application/json', body: JSON.stringify(capabilities) }) + const missing = capabilities.AudioContext === 'undefined' && capabilities.webkitAudioContext === 'undefined' + if (missing) { + expect(process.platform).toBe('win32') + expect(testInfo.project.name).toBe('webkit') + expect(capabilities.AudioWorkletNode).toBe('undefined') + test.skip(true, 'Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open') + } + expect(capabilities.AudioWorkletNode).toBe('function') + await page.addScriptTag({ content: code.toString() }) + await page.evaluate(() => { + const NativeContext = window.AudioContext || window.webkitAudioContext + window.asrContexts = [] + window.AudioContext = class extends NativeContext { + constructor(options) { + super(options) + window.asrContexts.push(this) + } + } + window.asrChunks = [] + window.art = new window.Artplayer({ + container: '.player', + url: '/test/audio-tone.m4a', + volume: 0.2, + plugins: [window.artplayerPluginAsr({ + onAudioChunk({ pcm, wav }) { + const samples = new Int16Array(pcm) + let max = 0 + for (const sample of samples) max = Math.max(max, Math.abs(sample)) + window.asrChunks.push({ samples: samples.length, max, wavBytes: wav.byteLength, sampleRate: new DataView(wav).getUint32(24, true) }) + return 'Local transcript.' + }, + })], + }) + document.querySelector('#play').onclick = () => window.art.play() + }) + await expect.poll(() => page.evaluate(() => window.art.isReady)).toBe(true) + await page.locator('#play').click() + await expect.poll(() => page.evaluate(() => window.asrChunks.filter(chunk => chunk.max > 0).length)).toBeGreaterThanOrEqual(3) + const chunks = await page.evaluate(() => window.asrChunks) + expect(chunks.every(chunk => chunk.samples === 1600 && chunk.wavBytes === 3244 && chunk.sampleRate === 16000)).toBe(true) + await expect(page.locator('.art-layer-asr')).toContainText('Local transcript.') + await page.evaluate(() => window.art.plugins.artplayerPluginAsr.stop()) + expect(await page.evaluate(() => window.asrContexts.map(context => context.state))).toEqual(['closed']) + await testInfo.attach('asr-native-chunks', { contentType: 'application/json', body: JSON.stringify(chunks) }) + await page.evaluate(() => window.art.destroy()) + }) + } +} diff --git a/test/helpers/asr-audio.js b/test/helpers/asr-audio.js new file mode 100644 index 000000000..02ccd129a --- /dev/null +++ b/test/helpers/asr-audio.js @@ -0,0 +1,183 @@ +import assert from 'node:assert/strict' +import vm from 'node:vm' +import { asrEnvironment } from './asr.js' + +export function deferredAudio() { + let resolve + let reject + const promise = new Promise((yes, no) => { + resolve = yes + reject = no + }) + return { promise, resolve, reject } +} + +export function asrAudioEnvironment(implementation, option = {}, capabilities = {}) { + const environment = asrEnvironment(implementation) + const intervals = new Map() + const contexts = [] + const recorders = [] + const nodes = [] + const modules = [] + const blobs = new Map() + const revoked = [] + const warnings = [] + const errors = [] + const tracks = [{ + stops: 0, + stop() { this.stops++ }, + }] + const stream = { getTracks: () => tracks } + const attachedElements = new WeakSet() + let nextTimer = 0 + let nextUrl = 0 + let captures = 0 + + function node(kind) { + const value = { + kind, + connections: [], + disconnects: 0, + connect(target) { this.connections.push(target) }, + disconnect() { + this.disconnects++ + this.connections = [] + }, + } + nodes.push(value) + return value + } + + class ControlledAudioContext { + constructor(options) { + this.options = options + this.state = capabilities.suspended ? 'suspended' : 'running' + this.destination = node('destination') + this.resumes = 0 + this.closes = 0 + this.audioWorklet = { + addModule: async (url) => { + const blob = blobs.get(url) + assert.ok(blob, 'The actual plugin worklet must have an object URL') + modules.push({ url, blob }) + capabilities.moduleEntered?.resolve() + if (capabilities.moduleGate) + await capabilities.moduleGate.promise + if (capabilities.moduleError) + throw capabilities.moduleError + }, + } + contexts.push(this) + } + + async resume() { + this.resumes++ + this.state = 'running' + } + + async close() { + this.closes++ + this.state = 'closed' + } + + createMediaElementSource(video) { + assert.equal(video, environment.art.video) + if (capabilities.directError) + throw capabilities.directError + if (attachedElements.has(video)) + throw new Error('HTMLMediaElement already connected to a MediaElementSourceNode') + attachedElements.add(video) + return node('element') + } + + createMediaStreamSource(value) { + assert.equal(value, stream) + return node('stream') + } + + createGain() { + return Object.assign(node('gain'), { gain: { value: 0 } }) + } + } + + class ControlledAudioWorkletNode { + constructor(context, name) { + assert.ok(contexts.includes(context)) + assert.equal(context.state, 'running') + assert.equal(name, 'recorder-processor') + Object.assign(this, node('recorder'), { port: { onmessage: null } }) + recorders.push(this) + } + } + + Object.assign(environment.context, { + Blob, + URL: { + createObjectURL(blob) { + const url = `blob:controlled-asr-${++nextUrl}` + blobs.set(url, blob) + return url + }, + revokeObjectURL(url) { + assert.ok(blobs.has(url)) + revoked.push(url) + blobs.delete(url) + }, + }, + AudioWorkletNode: ControlledAudioWorkletNode, + console: { warn: (...args) => warnings.push(args), error: (...args) => errors.push(args) }, + setInterval(callback, delay) { + intervals.set(++nextTimer, { callback, delay }) + return nextTimer + }, + clearInterval(timer) { intervals.delete(timer) }, + }) + environment.context[capabilities.webkit ? 'webkitAudioContext' : 'AudioContext'] = ControlledAudioContext + if (capabilities.capture !== false) { + environment.art.video[capabilities.moz ? 'mozCaptureStream' : 'captureStream'] = () => { + captures++ + return stream + } + } + const plugin = environment.factory(option)(environment.art) + return { + ...environment, + plugin, + intervals, + contexts, + recorders, + nodes, + modules, + blobs, + revoked, + tracks, + warnings, + errors, + captures: () => captures, + send(samples, recorder = recorders.at(-1)) { + assert.equal(typeof recorder?.port.onmessage, 'function') + recorder.port.onmessage({ data: Float32Array.from(samples) }) + }, + tick(timer = [...intervals.keys()][0]) { + assert.ok(intervals.has(timer), 'No live capture interval') + return intervals.get(timer).callback() + }, + } +} + +export async function runAsrWorklet(blob) { + const messages = [] + let Processor + const context = vm.createContext({ + AudioWorkletProcessor: class { + port = { postMessage: value => messages.push(value) } + }, + registerProcessor(name, implementation) { + assert.equal(name, 'recorder-processor') + Processor = implementation + }, + }) + vm.runInContext(await blob.text(), context, { timeout: 1000 }) + assert.equal(typeof Processor, 'function') + return { processor: new Processor(), messages } +}