From 0cf2af48400908fbe4217259e7d9fd37c1e15d1f Mon Sep 17 00:00:00 2001 From: Harvey Zhao Date: Sun, 13 Sep 2026 14:44:23 +0800 Subject: [PATCH] refactor(asr): [PKG-ASR-03] own typed audio capture and playback lifetimes --- docs/compiled/artplayer-plugin-asr.js | 2 +- docs/compiled/artplayer-plugin-asr.legacy.js | 2 +- docs/compiled/artplayer-plugin-asr.mjs | 602 +++++--- package.json | 6 +- packages/artplayer-plugin-asr/ARCHITECTURE.md | 107 +- packages/artplayer-plugin-asr/README.md | 5 + .../dist/artplayer-plugin-asr.js | 2 +- .../dist/artplayer-plugin-asr.legacy.js | 2 +- .../dist/artplayer-plugin-asr.mjs | 602 +++++--- .../artplayer-plugin-asr/src/audio-graph.ts | 144 ++ packages/artplayer-plugin-asr/src/capture.ts | 127 ++ packages/artplayer-plugin-asr/src/encoding.ts | 31 + packages/artplayer-plugin-asr/src/index.js | 299 ---- packages/artplayer-plugin-asr/src/index.ts | 46 + .../artplayer-plugin-asr/src/sample-queue.ts | 59 + .../artplayer-plugin-asr/src/subtitles.ts | 37 + packages/artplayer-plugin-asr/src/types.ts | 30 + packages/artplayer-plugin-asr/src/worklet.ts | 12 + packages/artplayer-plugin-asr/tsconfig.json | 5 + .../baselines/asr-ownership-validation.json | 1361 +++++++++++++++++ .../2026-09-13-PKG-ASR-03-audio-ownership.md | 62 + refactor/plan.md | 5 +- refactor/progress.md | 15 + refactor/risk-table.md | 4 +- refactor/risks.json | 24 +- refactor/tasks.json | 7 +- refactor/third-party.json | 2 +- test/asr-distribution.test.js | 34 + test/asr-encoding.test.js | 183 +++ test/asr-lifecycle.test.js | 434 ++++++ test/asr.test.js | 5 +- test/browser/asr-audio.spec.js | 43 +- test/helpers/asr.js | 2 + 33 files changed, 3492 insertions(+), 809 deletions(-) create mode 100644 packages/artplayer-plugin-asr/src/audio-graph.ts create mode 100644 packages/artplayer-plugin-asr/src/capture.ts create mode 100644 packages/artplayer-plugin-asr/src/encoding.ts delete mode 100644 packages/artplayer-plugin-asr/src/index.js create mode 100644 packages/artplayer-plugin-asr/src/index.ts create mode 100644 packages/artplayer-plugin-asr/src/sample-queue.ts create mode 100644 packages/artplayer-plugin-asr/src/subtitles.ts create mode 100644 packages/artplayer-plugin-asr/src/types.ts create mode 100644 packages/artplayer-plugin-asr/src/worklet.ts create mode 100644 packages/artplayer-plugin-asr/tsconfig.json create mode 100644 refactor/baselines/asr-ownership-validation.json create mode 100644 refactor/changes/2026-09-13-PKG-ASR-03-audio-ownership.md create mode 100644 test/asr-distribution.test.js create mode 100644 test/asr-encoding.test.js create mode 100644 test/asr-lifecycle.test.js diff --git a/docs/compiled/artplayer-plugin-asr.js b/docs/compiled/artplayer-plugin-asr.js index eba061fb8..f218500a2 100644 --- a/docs/compiled/artplayer-plugin-asr.js +++ b/docs/compiled/artplayer-plugin-asr.js @@ -4,4 +4,4 @@ * (c) 2017-2026 Harvey Zhao * Released under the MIT License. */ -!function(t,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"==typeof define&&define.amd?(t.artplayerPluginAsr=e(),define(function(){return t.artplayerPluginAsr})):(t="undefined"!==typeof globalThis?globalThis:t||self).artplayerPluginAsr=e()}(this,function(){"use strict";if("undefined"!==typeof document&&!document.getElementById("artplayer-plugin-asr")){const t=document.createElement("style");t.id="artplayer-plugin-asr",t.textContent=".art-video-player .art-layer-asr{pointer-events:none!important;position:absolute;z-index:150;inset:0;display:flex;flex-direction:column;justify-content:end;padding:0 2%;font-size:1rem;gap:var(--art-subtitle-gap);padding-bottom:var(--art-subtitle-bottom);transition:padding-bottom var(--art-transition-duration) ease;text-shadow:var(--art-subtitle-border) 1px 0 1px,var(--art-subtitle-border) 0 1px 1px,var(--art-subtitle-border) -1px 0 1px,var(--art-subtitle-border) 0 -1px 1px,var(--art-subtitle-border) 1px 1px 1px,var(--art-subtitle-border) -1px -1px 1px,var(--art-subtitle-border) 1px -1px 1px,var(--art-subtitle-border) -1px 1px 1px}.art-video-player.art-control-show .art-layer-asr{padding-bottom:calc(var(--art-control-height) + var(--art-subtitle-bottom))}",document.head.appendChild(t)}return function(t={}){const{length:e=3,interval:n=100,sampleRate:r=16e3,autoHideTimeout:a=1e4,onAudioChunk:o=()=>null}=t;return t=>{let i=!1,l=null,s=null,u=null,c=null,d=[],p=null,f=!1,y=null,m=null,g=null;const h=t.layers.add({name:"asr",html:""});function w(){h.style.display="none"}function b(t){var n;"string"===typeof t&&(clearTimeout(y),y=setTimeout(w,a),h.style.display="",h.innerHTML=(n=t,n.split(/(?<=[\u3001\u3002\uff01\uff1f!?.])\s*/u).map(t=>t.trim()).filter(Boolean).slice(-e)).map(t=>`
${t}
`).join(""))}async function v(){if(!s&&t.video)try{return s=l.createMediaElementSource(t.video),s}catch(e){console.warn("[artplayerPluginAsr] Direct connection failed:",e)}try{if(m||(m=await async function(t){try{return t.captureStream?t.captureStream():t.mozCaptureStream?t.mozCaptureStream():(console.warn("[artplayerPluginAsr] captureStream not supported"),null)}catch(e){return console.warn("[artplayerPluginAsr] Error capturing stream:",e),null}}(t.video)),m)return g=l.createMediaStreamSource(m),g}catch(e){console.warn("[artplayerPluginAsr] MediaStream fallback failed:",e)}return null}async function x(){i&&(i=!1,clearInterval(p),p=null,u&&(u.disconnect(),u.port.onmessage=null,u=null),c&&(c.disconnect(),c=null),d=[])}async function A(){await x(),g&&(g.disconnect(),g=null),s&&(s.disconnect(),s=null),m&&(m.getTracks().forEach(t=>t.stop()),m=null),l&&(await l.close(),l=null),f=!1}return t.on("video:volumechange",()=>{c&&(c.gain.value=t.volume)}),t.on("play",async function(){if(!i)try{await async function(){return l||(l=new(window.AudioContext||window.webkitAudioContext)({sampleRate:r}),"suspended"===l.state&&await l.resume(),l)}();const t=await v();if(!t)throw new Error("Could not establish audio source");if(!f){const t=(()=>{const t=new Blob(["\n class RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n }\n registerProcessor('recorder-processor', RecorderProcessor);\n "],{type:"application/javascript"});return URL.createObjectURL(t)})();await l.audioWorklet.addModule(t),URL.revokeObjectURL(t),f=!0}c=l.createGain(),c.gain.value=1,u=new AudioWorkletNode(l,"recorder-processor"),u.port.onmessage=t=>{d.push(new Float32Array(t.data))},t.connect(u),t.connect(c),c.connect(l.destination);const e=r*(n/1e3);p=setInterval(async()=>{if(0===d.length)return;let t=new Float32Array(0);for(;t.length{for(let n=0;n{throw this.preparation=void 0,t})),this.preparation}get ownsMediaConnection(){return this.direct}idle(){const t=this.gain;this.disconnect(),this.direct&&this.context&&this.source&&(this.gain=t||this.context.createGain(),t||(this.gain.gain.value=1),this.source.connect(this.gain),this.gain.connect(this.context.destination))}connect(t){this.assertOpen();const e=this.context,i=this.source;if(!e||!i)throw new Error("Audio capture is not prepared");const s=this.gain;this.disconnect();const n=this.gain=s||e.createGain();s||(n.gain.value=1);const r=this.recorder=new AudioWorkletNode(e,"recorder-processor");r.port.onmessage=e=>{this.closed||this.recorder!==r||t(e.data)},i.connect(r),i.connect(n),n.connect(e.destination)}volume(t){this.gain&&(this.gain.gain.value=t)}disconnect(){this.recorder&&(this.recorder.port.onmessage=null,this.recorder.disconnect(),this.recorder=void 0),this.gain?.disconnect(),this.gain=void 0,this.source?.disconnect()}async close(){if(this.closed)return;this.closed=!0,this.disconnect(),this.revokeWorklet(),this.stream?.getTracks().forEach(t=>t.stop()),this.stream=void 0;const t=this.context;this.context=void 0,this.source=void 0,t&&"closed"!==t.state&&await t.close()}}class e{constructor(){this.offset=0,this.size=0}get length(){return this.size}push(t){if(0===t.length)return;const e={samples:new Float32Array(t)};this.tail?this.tail.next=e:this.head=e,this.tail=e,this.size+=t.length}take(t){if(!Number.isSafeInteger(t)||t<=0)throw new RangeError("Sample count must be a positive safe integer");if(t>this.size)return null;const e=new Float32Array(t);let i=0;for(;this.head&&it?this.start():void 0):t?this.start():Promise.resolve()}start(){if(this.terminal||this.running)return Promise.resolve();if(this.starting)return this.starting;const e=this.epoch,i=()=>!this.terminal&&this.epoch===e,s=(async()=>{if(await this.closing,!i())return;const{sampleRate:e,interval:s}=this.options,n=Math.floor(e*s/1e3);if(!Number.isSafeInteger(n)||n<1||!Number.isFinite(s)||s<=0)throw new Error("Audio chunk length must be positive and finite");const r=this.graph??(this.graph=new t(this.video,e));await r.prepare(),i()?(r.connect(t=>{if(this.running&&this.graph===r)return this.queue.length+t.length>Math.max(60*e,2*n)?(console.error("[artplayerPluginAsr] Audio callback backlog exceeded its capture limit"),void this.pause()):void this.queue.push(t)}),this.running=!0,this.timer=setInterval(()=>this.tick(n),s)):this.running||this.graph!==r||r.idle()})().catch(async t=>{i()&&(console.error("[artplayerPluginAsr] Initialization failed:",t),await this.stop())}).finally(()=>{this.starting===s&&(this.starting=void 0)});return this.starting=s,s}async tick(t){if(!this.running||this.pending)return;const e=this.queue.take(t);if(!e)return;const i=this.epoch,s=this.pending={};try{const t=await this.options.onAudioChunk(function(t,e){const i=new ArrayBuffer(2*t.length),s=new DataView(i);for(let a=0;a{})),this.closing}}function s(t={}){const{length:e=3,interval:s=100,sampleRate:n=16e3,autoHideTimeout:r=1e4,onAudioChunk:o=()=>null}=t;return t=>{const a=t.layers.add({name:"asr",html:""});if(!a)throw new Error("Could not create ASR subtitle layer");const h=function(t,e,i){let s,n=!1;function r(){clearTimeout(s),s=void 0}function o(){n||(t.style.display="none")}return{append:function(a){n||"string"!==typeof a||(r(),s=setTimeout(o,i),t.style.display="",t.innerHTML=a.split(/(?<=[\u3001\u3002\uff01\uff1f!?.])\s*/u).map(t=>t.trim()).filter(Boolean).slice(-e).map(t=>`
${t}
`).join(""))},hide:o,destroy(){n=!0,r()}}}(a,e,r),c=new i(t.video,{interval:s,sampleRate:n,onAudioChunk:o},h.append),l=()=>c.volume(t.volume),d=()=>c.start(),u=()=>c.pause(),p=()=>c.restart(),g=()=>(h.destroy(),t.off("video:volumechange",l),t.off("play",d),t.off("pause",u),t.off("restart",p),t.off("destroy",g),c.stop(!0));return t.on("video:volumechange",l),t.on("play",d),t.on("pause",u),t.on("restart",p),t.on("destroy",g),{name:"artplayerPluginAsr",stop:()=>c.stop(),hide:h.hide,append:h.append}}}if(Object.defineProperty(s,"default",{value:s}),"undefined"!==typeof document&&!document.getElementById("artplayer-plugin-asr")){const t=document.createElement("style");t.id="artplayer-plugin-asr",t.textContent=".art-video-player .art-layer-asr{pointer-events:none!important;position:absolute;z-index:150;inset:0;display:flex;flex-direction:column;justify-content:end;padding:0 2%;font-size:1rem;gap:var(--art-subtitle-gap);padding-bottom:var(--art-subtitle-bottom);transition:padding-bottom var(--art-transition-duration) ease;text-shadow:var(--art-subtitle-border) 1px 0 1px,var(--art-subtitle-border) 0 1px 1px,var(--art-subtitle-border) -1px 0 1px,var(--art-subtitle-border) 0 -1px 1px,var(--art-subtitle-border) 1px 1px 1px,var(--art-subtitle-border) -1px -1px 1px,var(--art-subtitle-border) 1px -1px 1px,var(--art-subtitle-border) -1px 1px 1px}.art-video-player.art-control-show .art-layer-asr{padding-bottom:calc(var(--art-control-height) + var(--art-subtitle-bottom))}",document.head.appendChild(t)}return s}); diff --git a/docs/compiled/artplayer-plugin-asr.legacy.js b/docs/compiled/artplayer-plugin-asr.legacy.js index 71d3bc90c..3a00665c7 100644 --- a/docs/compiled/artplayer-plugin-asr.legacy.js +++ b/docs/compiled/artplayer-plugin-asr.legacy.js @@ -4,4 +4,4 @@ * (c) 2017-2026 Harvey Zhao * Released under the MIT License. */ -!function(t,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"==typeof define&&define.amd?(t.artplayerPluginAsr=e(),define(function(){return t.artplayerPluginAsr})):(t="undefined"!==typeof globalThis?globalThis:t||self).artplayerPluginAsr=e()}(this,function(){"use strict";var t=(t,e,n)=>new Promise((r,o)=>{var a=t=>{try{l(n.next(t))}catch(e){o(e)}},i=t=>{try{l(n.throw(t))}catch(e){o(e)}},l=t=>t.done?r(t.value):Promise.resolve(t.value).then(a,i);l((n=n.apply(t,e)).next())});if("undefined"!==typeof document&&!document.getElementById("artplayer-plugin-asr")){const t=document.createElement("style");t.id="artplayer-plugin-asr",t.textContent=".art-video-player .art-layer-asr{pointer-events:none!important;position:absolute;z-index:150;inset:0;display:flex;flex-direction:column;justify-content:end;padding:0 2%;font-size:1rem;gap:var(--art-subtitle-gap);padding-bottom:var(--art-subtitle-bottom);transition:padding-bottom var(--art-transition-duration) ease;text-shadow:var(--art-subtitle-border) 1px 0 1px,var(--art-subtitle-border) 0 1px 1px,var(--art-subtitle-border) -1px 0 1px,var(--art-subtitle-border) 0 -1px 1px,var(--art-subtitle-border) 1px 1px 1px,var(--art-subtitle-border) -1px -1px 1px,var(--art-subtitle-border) 1px -1px 1px,var(--art-subtitle-border) -1px 1px 1px}.art-video-player.art-control-show .art-layer-asr{padding-bottom:calc(var(--art-control-height) + var(--art-subtitle-bottom))}",document.head.appendChild(t)}return function(e={}){const{length:n=3,interval:r=100,sampleRate:o=16e3,autoHideTimeout:a=1e4,onAudioChunk:i=()=>null}=e;return e=>{let l=!1,s=null,u=null,c=null,d=null,p=[],f=null,y=!1,m=null,h=null,g=null;const v=e.layers.add({name:"asr",html:""});function b(){v.style.display="none"}function x(t){var e;"string"===typeof t&&(clearTimeout(m),m=setTimeout(b,a),v.style.display="",v.innerHTML=(e=t,e.split(new RegExp("(?<=[\u3001\u3002\uff01\uff1f!?.])\\s*","u")).map(t=>t.trim()).filter(Boolean).slice(-n)).map(t=>`
${t}
`).join(""))}function w(){return t(this,null,function*(){if(!u&&e.video)try{return u=s.createMediaElementSource(e.video),u}catch(n){console.warn("[artplayerPluginAsr] Direct connection failed:",n)}try{if(h||(h=yield function(e){return t(this,null,function*(){try{return e.captureStream?e.captureStream():e.mozCaptureStream?e.mozCaptureStream():(console.warn("[artplayerPluginAsr] captureStream not supported"),null)}catch(n){return console.warn("[artplayerPluginAsr] Error capturing stream:",n),null}})}(e.video)),h)return g=s.createMediaStreamSource(h),g}catch(n){console.warn("[artplayerPluginAsr] MediaStream fallback failed:",n)}return null})}function A(){return t(this,null,function*(){l&&(l=!1,clearInterval(f),f=null,c&&(c.disconnect(),c.port.onmessage=null,c=null),d&&(d.disconnect(),d=null),p=[])})}function U(){return t(this,null,function*(){yield A(),g&&(g.disconnect(),g=null),u&&(u.disconnect(),u=null),h&&(h.getTracks().forEach(t=>t.stop()),h=null),s&&(yield s.close(),s=null),y=!1})}return e.on("video:volumechange",()=>{d&&(d.gain.value=e.volume)}),e.on("play",function(){return t(this,null,function*(){if(!l)try{yield function(){return t(this,null,function*(){return s||(s=new(window.AudioContext||window.webkitAudioContext)({sampleRate:o}),"suspended"===s.state&&(yield s.resume()),s)})}();const e=yield w();if(!e)throw new Error("Could not establish audio source");if(!y){const t=(()=>{const t=new Blob(["\n class RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n }\n registerProcessor('recorder-processor', RecorderProcessor);\n "],{type:"application/javascript"});return URL.createObjectURL(t)})();yield s.audioWorklet.addModule(t),URL.revokeObjectURL(t),y=!0}d=s.createGain(),d.gain.value=1,c=new AudioWorkletNode(s,"recorder-processor"),c.port.onmessage=t=>{p.push(new Float32Array(t.data))},e.connect(c),e.connect(d),d.connect(s.destination);const n=o*(r/1e3);f=setInterval(()=>t(null,null,function*(){if(0===p.length)return;let t=new Float32Array(0);for(;t.length{for(let n=0;nnew Promise((s,n)=>{var r=t=>{try{a(i.next(t))}catch(e){n(e)}},o=t=>{try{a(i.throw(t))}catch(e){n(e)}},a=t=>t.done?s(t.value):Promise.resolve(t.value).then(r,o);a((i=i.apply(t,e)).next())});class e{constructor(t,e){this.video=t,this.sampleRate=e,this.closed=!1,this.direct=!1,this.workletLoaded=!1}assertOpen(){if(this.closed)throw new Error("Audio capture was closed")}revokeWorklet(){void 0!==this.blobUrl&&(URL.revokeObjectURL(this.blobUrl),this.blobUrl=void 0)}initialize(){return t(this,null,function*(){var t;this.assertOpen();const e=window,i=window.AudioContext||e.webkitAudioContext;if(!i)throw new Error("AudioContext is not supported");const s=null!=(t=this.context)?t:this.context=new i({sampleRate:this.sampleRate});if("suspended"===s.state&&(yield s.resume()),this.assertOpen(),!this.workletLoaded){this.blobUrl=URL.createObjectURL(new Blob(["\nclass RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n}\nregisterProcessor('recorder-processor', RecorderProcessor);\n"],{type:"application/javascript"}));try{yield s.audioWorklet.addModule(this.blobUrl),this.workletLoaded=!0}finally{this.revokeWorklet()}}if(this.assertOpen(),!this.source)try{this.source=s.createMediaElementSource(this.video),this.direct=!0}catch(n){console.warn("[artplayerPluginAsr] Direct connection failed:",n);const t=this.video.captureStream||this.video.mozCaptureStream;if(!t)throw new Error("Could not establish audio source");this.stream=t.call(this.video),this.source=s.createMediaStreamSource(this.stream)}})}prepare(){return null!=this.preparation||(this.preparation=this.initialize().catch(t=>{throw this.preparation=void 0,t})),this.preparation}get ownsMediaConnection(){return this.direct}idle(){const t=this.gain;this.disconnect(),this.direct&&this.context&&this.source&&(this.gain=t||this.context.createGain(),t||(this.gain.gain.value=1),this.source.connect(this.gain),this.gain.connect(this.context.destination))}connect(t){this.assertOpen();const e=this.context,i=this.source;if(!e||!i)throw new Error("Audio capture is not prepared");const s=this.gain;this.disconnect();const n=this.gain=s||e.createGain();s||(n.gain.value=1);const r=this.recorder=new AudioWorkletNode(e,"recorder-processor");r.port.onmessage=e=>{this.closed||this.recorder!==r||t(e.data)},i.connect(r),i.connect(n),n.connect(e.destination)}volume(t){this.gain&&(this.gain.gain.value=t)}disconnect(){var t,e;this.recorder&&(this.recorder.port.onmessage=null,this.recorder.disconnect(),this.recorder=void 0),null==(t=this.gain)||t.disconnect(),this.gain=void 0,null==(e=this.source)||e.disconnect()}close(){return t(this,null,function*(){var t;if(this.closed)return;this.closed=!0,this.disconnect(),this.revokeWorklet(),null==(t=this.stream)||t.getTracks().forEach(t=>t.stop()),this.stream=void 0;const e=this.context;this.context=void 0,this.source=void 0,e&&"closed"!==e.state&&(yield e.close())})}}class i{constructor(){this.offset=0,this.size=0}get length(){return this.size}push(t){if(0===t.length)return;const e={samples:new Float32Array(t)};this.tail?this.tail.next=e:this.head=e,this.tail=e,this.size+=t.length}take(t){if(!Number.isSafeInteger(t)||t<=0)throw new RangeError("Sample count must be a positive safe integer");if(t>this.size)return null;const e=new Float32Array(t);let i=0;for(;this.head&&it?this.start():void 0):t?this.start():Promise.resolve()}start(){if(this.terminal||this.running)return Promise.resolve();if(this.starting)return this.starting;const i=this.epoch,s=()=>!this.terminal&&this.epoch===i,n=(()=>t(this,null,function*(){var t;if(yield this.closing,!s())return;const{sampleRate:i,interval:n}=this.options,r=Math.floor(i*n/1e3);if(!Number.isSafeInteger(r)||r<1||!Number.isFinite(n)||n<=0)throw new Error("Audio chunk length must be positive and finite");const o=null!=(t=this.graph)?t:this.graph=new e(this.video,i);yield o.prepare(),s()?(o.connect(t=>{if(this.running&&this.graph===o)return this.queue.length+t.length>Math.max(60*i,2*r)?(console.error("[artplayerPluginAsr] Audio callback backlog exceeded its capture limit"),void this.pause()):void this.queue.push(t)}),this.running=!0,this.timer=setInterval(()=>this.tick(r),n)):this.running||this.graph!==o||o.idle()}))().catch(e=>t(this,null,function*(){s()&&(console.error("[artplayerPluginAsr] Initialization failed:",e),yield this.stop())})).finally(()=>{this.starting===n&&(this.starting=void 0)});return this.starting=n,n}tick(e){return t(this,null,function*(){if(!this.running||this.pending)return;const t=this.queue.take(e);if(!t)return;const i=this.epoch,s=this.pending={};try{const e=yield this.options.onAudioChunk(function(t,e){const i=new ArrayBuffer(2*t.length),s=new DataView(i);for(let a=0;a{})),this.closing}}function n(t={}){const{length:e=3,interval:i=100,sampleRate:n=16e3,autoHideTimeout:r=1e4,onAudioChunk:o=()=>null}=t;return t=>{const a=t.layers.add({name:"asr",html:""});if(!a)throw new Error("Could not create ASR subtitle layer");const h=function(t,e,i){let s,n=!1;function r(){clearTimeout(s),s=void 0}function o(){n||(t.style.display="none")}return{append:function(a){n||"string"!==typeof a||(r(),s=setTimeout(o,i),t.style.display="",t.innerHTML=a.split(new RegExp("(?<=[\u3001\u3002\uff01\uff1f!?.])\\s*","u")).map(t=>t.trim()).filter(Boolean).slice(-e).map(t=>`
${t}
`).join(""))},hide:o,destroy(){n=!0,r()}}}(a,e,r),l=new s(t.video,{interval:i,sampleRate:n,onAudioChunk:o},h.append),c=()=>l.volume(t.volume),d=()=>l.start(),u=()=>l.pause(),p=()=>l.restart(),f=()=>(h.destroy(),t.off("video:volumechange",c),t.off("play",d),t.off("pause",u),t.off("restart",p),t.off("destroy",f),l.stop(!0));return t.on("video:volumechange",c),t.on("play",d),t.on("pause",u),t.on("restart",p),t.on("destroy",f),{name:"artplayerPluginAsr",stop:()=>l.stop(),hide:h.hide,append:h.append}}}if(Object.defineProperty(n,"default",{value:n}),"undefined"!==typeof document&&!document.getElementById("artplayer-plugin-asr")){const t=document.createElement("style");t.id="artplayer-plugin-asr",t.textContent=".art-video-player .art-layer-asr{pointer-events:none!important;position:absolute;z-index:150;inset:0;display:flex;flex-direction:column;justify-content:end;padding:0 2%;font-size:1rem;gap:var(--art-subtitle-gap);padding-bottom:var(--art-subtitle-bottom);transition:padding-bottom var(--art-transition-duration) ease;text-shadow:var(--art-subtitle-border) 1px 0 1px,var(--art-subtitle-border) 0 1px 1px,var(--art-subtitle-border) -1px 0 1px,var(--art-subtitle-border) 0 -1px 1px,var(--art-subtitle-border) 1px 1px 1px,var(--art-subtitle-border) -1px -1px 1px,var(--art-subtitle-border) 1px -1px 1px,var(--art-subtitle-border) -1px 1px 1px}.art-video-player.art-control-show .art-layer-asr{padding-bottom:calc(var(--art-control-height) + var(--art-subtitle-bottom))}",document.head.appendChild(t)}return n}); diff --git a/docs/compiled/artplayer-plugin-asr.mjs b/docs/compiled/artplayer-plugin-asr.mjs index 37fc36858..3da312943 100644 --- a/docs/compiled/artplayer-plugin-asr.mjs +++ b/docs/compiled/artplayer-plugin-asr.mjs @@ -4,248 +4,394 @@ * (c) 2017-2026 Harvey Zhao * Released under the MIT License. */ -const style = ".art-video-player .art-layer-asr {\n pointer-events: none !important;\n position: absolute;\n z-index: 150;\n inset: 0px;\n display: flex;\n flex-direction: column;\n justify-content: end;\n padding: 0 2%;\n font-size: 1rem;\n gap: var(--art-subtitle-gap);\n padding-bottom: var(--art-subtitle-bottom);\n transition: padding-bottom var(--art-transition-duration) ease;\n text-shadow: var(--art-subtitle-border) 1px 0 1px, var(--art-subtitle-border) 0 1px 1px, var(--art-subtitle-border) -1px 0 1px, var(--art-subtitle-border) 0 -1px 1px, var(--art-subtitle-border) 1px 1px 1px, var(--art-subtitle-border) -1px -1px 1px, var(--art-subtitle-border) 1px -1px 1px, var(--art-subtitle-border) -1px 1px 1px;\n}\n.art-video-player.art-control-show .art-layer-asr {\n padding-bottom: calc(var(--art-control-height) + var(--art-subtitle-bottom));\n}\n"; -function artplayerPluginAsr(option = {}) { - const { - length = 3, - interval = 100, - sampleRate = 16e3, - autoHideTimeout = 1e4, - onAudioChunk = () => null - } = option; - return (art) => { - let started = false; - let audioCtx = null; - let sourceNode = null; - let recorderNode = null; - let gainNode = null; - let bufferChunks = []; - let timer = null; - let workletLoaded = false; - let hideTimer = null; - let mediaStream = null; - let mediaStreamSource = null; - const $asr = art.layers.add({ - name: "asr", - html: "" - }); - function splitByPunctuation(text) { - return text.split(/(?<=[、。！？!?.])\s*/u).map((s) => s.trim()).filter(Boolean).slice(-length); - } - function hide() { - $asr.style.display = "none"; - } - function append(subtitle) { - if (typeof subtitle !== "string") - return; - clearTimeout(hideTimer); - hideTimer = setTimeout(hide, autoHideTimeout); - $asr.style.display = ""; - $asr.innerHTML = splitByPunctuation(subtitle).map((line) => `
${line}
`).join(""); - } - const recorderProcessorCode = ` - class RecorderProcessor extends AudioWorkletProcessor { - process(inputs) { - const input = inputs[0]; - if (input && input[0]) { - this.port.postMessage(input[0]); - } - return true; - } - } - registerProcessor('recorder-processor', RecorderProcessor); - `; - const createWorkletBlobUrl = () => { - const blob = new Blob([recorderProcessorCode], { type: "application/javascript" }); - return URL.createObjectURL(blob); - }; - function floatTo16BitPCM(float32Array) { - const len = float32Array.length; - const buffer = new ArrayBuffer(len * 2); - const view = new DataView(buffer); - for (let i = 0; i < len; i++) { - const s = Math.max(-1, Math.min(1, float32Array[i])); - view.setInt16(i * 2, s < 0 ? s * 32768 : s * 32767, true); - } - return buffer; - } - function pcmToWav(pcmBuffer, sampleRate2) { - const pcmLength = pcmBuffer.byteLength; - const wavBuffer = new ArrayBuffer(44 + pcmLength); - const view = new DataView(wavBuffer); - const writeString = (offset, str) => { - for (let i = 0; i < str.length; i++) { - view.setUint8(offset + i, str.charCodeAt(i)); +const recorderProcessorCode = ` +class RecorderProcessor extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]; + if (input && input[0]) { + this.port.postMessage(input[0]); } - }; - writeString(0, "RIFF"); - view.setUint32(4, 36 + pcmLength, true); - writeString(8, "WAVE"); - writeString(12, "fmt "); - view.setUint32(16, 16, true); - view.setUint16(20, 1, true); - view.setUint16(22, 1, true); - view.setUint32(24, sampleRate2, true); - view.setUint32(28, sampleRate2 * 2, true); - view.setUint16(32, 2, true); - view.setUint16(34, 16, true); - writeString(36, "data"); - view.setUint32(40, pcmLength, true); - new Uint8Array(wavBuffer).set(new Uint8Array(pcmBuffer), 44); - return wavBuffer; + return true; } - async function setupAudioContext() { - if (audioCtx) - return audioCtx; - audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate }); - if (audioCtx.state === "suspended") - await audioCtx.resume(); - return audioCtx; +} +registerProcessor('recorder-processor', RecorderProcessor); +`; +class AudioGraph { + constructor(video, sampleRate) { + this.video = video; + this.sampleRate = sampleRate; + this.closed = false; + this.direct = false; + this.workletLoaded = false; + } + assertOpen() { + if (this.closed) + throw new Error("Audio capture was closed"); + } + revokeWorklet() { + if (this.blobUrl !== void 0) { + URL.revokeObjectURL(this.blobUrl); + this.blobUrl = void 0; } - async function setupAudioSource() { - if (!sourceNode && art.video) { - try { - sourceNode = audioCtx.createMediaElementSource(art.video); - return sourceNode; - } catch (err) { - console.warn("[artplayerPluginAsr] Direct connection failed:", err); - } - } + } + async initialize() { + this.assertOpen(); + const platform = window; + const Context = window.AudioContext || platform.webkitAudioContext; + if (!Context) + throw new Error("AudioContext is not supported"); + const context = this.context ?? (this.context = new Context({ sampleRate: this.sampleRate })); + if (context.state === "suspended") + await context.resume(); + this.assertOpen(); + if (!this.workletLoaded) { + this.blobUrl = URL.createObjectURL(new Blob([recorderProcessorCode], { type: "application/javascript" })); try { - if (!mediaStream) { - mediaStream = await captureAudioFromVideo(art.video); - } - if (mediaStream) { - mediaStreamSource = audioCtx.createMediaStreamSource(mediaStream); - return mediaStreamSource; - } - } catch (err) { - console.warn("[artplayerPluginAsr] MediaStream fallback failed:", err); - } - return null; - } - async function captureAudioFromVideo(videoElement) { - try { - if (videoElement.captureStream) - return videoElement.captureStream(); - if (videoElement.mozCaptureStream) - return videoElement.mozCaptureStream(); - console.warn("[artplayerPluginAsr] captureStream not supported"); - return null; - } catch (err) { - console.warn("[artplayerPluginAsr] Error capturing stream:", err); - return null; + await context.audioWorklet.addModule(this.blobUrl); + this.workletLoaded = true; + } finally { + this.revokeWorklet(); } } - async function startCapture() { - if (started) - return; + this.assertOpen(); + if (!this.source) { try { - await setupAudioContext(); - const audioSource = await setupAudioSource(); - if (!audioSource) + this.source = context.createMediaElementSource(this.video); + this.direct = true; + } catch (error) { + console.warn("[artplayerPluginAsr] Direct connection failed:", error); + const capture = this.video.captureStream || this.video.mozCaptureStream; + if (!capture) throw new Error("Could not establish audio source"); - if (!workletLoaded) { - const blobUrl = createWorkletBlobUrl(); - await audioCtx.audioWorklet.addModule(blobUrl); - URL.revokeObjectURL(blobUrl); - workletLoaded = true; - } - gainNode = audioCtx.createGain(); - gainNode.gain.value = 1; - recorderNode = new AudioWorkletNode(audioCtx, "recorder-processor"); - recorderNode.port.onmessage = (event) => { - bufferChunks.push(new Float32Array(event.data)); - }; - audioSource.connect(recorderNode); - audioSource.connect(gainNode); - gainNode.connect(audioCtx.destination); - const CHUNK_SAMPLES = sampleRate * (interval / 1e3); - timer = setInterval(async () => { - if (bufferChunks.length === 0) - return; - let accumulated = new Float32Array(0); - while (accumulated.length < CHUNK_SAMPLES && bufferChunks.length) { - const next = bufferChunks.shift(); - const tmp = new Float32Array(accumulated.length + next.length); - tmp.set(accumulated, 0); - tmp.set(next, accumulated.length); - accumulated = tmp; - } - if (accumulated.length < CHUNK_SAMPLES) - return; - const chunkToSend = accumulated.slice(0, CHUNK_SAMPLES); - const pcm = floatTo16BitPCM(chunkToSend); - const wav = pcmToWav(pcm, sampleRate); - const subtitle = await onAudioChunk({ pcm, wav }); - append(subtitle); - }, interval); - started = true; - } catch (err) { - console.error("[artplayerPluginAsr] Initialization failed:", err); - await stopCapture(); + this.stream = capture.call(this.video); + this.source = context.createMediaStreamSource(this.stream); } } - async function stopCapture() { - if (!started) - return; - started = false; - clearInterval(timer); - timer = null; - if (recorderNode) { - recorderNode.disconnect(); - recorderNode.port.onmessage = null; - recorderNode = null; - } - if (gainNode) { - gainNode.disconnect(); - gainNode = null; - } - bufferChunks = []; + } + prepare() { + this.preparation ?? (this.preparation = this.initialize().catch((error) => { + this.preparation = void 0; + throw error; + })); + return this.preparation; + } + get ownsMediaConnection() { + return this.direct; + } + idle() { + const gain = this.gain; + this.disconnect(); + if (this.direct && this.context && this.source) { + this.gain = gain || this.context.createGain(); + if (!gain) + this.gain.gain.value = 1; + this.source.connect(this.gain); + this.gain.connect(this.context.destination); } - async function destroy() { - await stopCapture(); - if (mediaStreamSource) { - mediaStreamSource.disconnect(); - mediaStreamSource = null; - } - if (sourceNode) { - sourceNode.disconnect(); - sourceNode = null; - } - if (mediaStream) { - mediaStream.getTracks().forEach((track) => track.stop()); - mediaStream = null; - } - if (audioCtx) { - await audioCtx.close(); - audioCtx = null; - } - workletLoaded = false; - } - art.on("video:volumechange", () => { - if (gainNode) { - gainNode.gain.value = art.volume; - } - }); - art.on("play", startCapture); - art.on("pause", stopCapture); - art.on("destroy", destroy); - return { - name: "artplayerPluginAsr", - stop: destroy, - hide, - append + } + connect(receive) { + this.assertOpen(); + const context = this.context; + const source = this.source; + if (!context || !source) + throw new Error("Audio capture is not prepared"); + const previousGain = this.gain; + this.disconnect(); + const gain = this.gain = previousGain || context.createGain(); + if (!previousGain) + gain.gain.value = 1; + const recorder = this.recorder = new AudioWorkletNode(context, "recorder-processor"); + recorder.port.onmessage = (event) => { + if (!this.closed && this.recorder === recorder) + receive(event.data); }; + source.connect(recorder); + source.connect(gain); + gain.connect(context.destination); + } + volume(value) { + if (this.gain) + this.gain.gain.value = value; + } + disconnect() { + if (this.recorder) { + this.recorder.port.onmessage = null; + this.recorder.disconnect(); + this.recorder = void 0; + } + this.gain?.disconnect(); + this.gain = void 0; + this.source?.disconnect(); + } + async close() { + if (this.closed) + return; + this.closed = true; + this.disconnect(); + this.revokeWorklet(); + this.stream?.getTracks().forEach((track) => track.stop()); + this.stream = void 0; + const context = this.context; + this.context = void 0; + this.source = void 0; + if (context && context.state !== "closed") + await context.close(); + } +} +function encodeAudio(samples, sampleRate) { + const pcm = new ArrayBuffer(samples.length * 2); + const pcmView = new DataView(pcm); + for (let index = 0; index < samples.length; index++) { + const value = Math.max(-1, Math.min(1, samples[index])); + pcmView.setInt16(index * 2, value < 0 ? value * 32768 : value * 32767, true); + } + const wav = new ArrayBuffer(44 + pcm.byteLength); + const header = new DataView(wav); + function writeString(offset, value) { + for (let index = 0; index < value.length; index++) + header.setUint8(offset + index, value.charCodeAt(index)); + } + writeString(0, "RIFF"); + header.setUint32(4, 36 + pcm.byteLength, true); + writeString(8, "WAVE"); + writeString(12, "fmt "); + header.setUint32(16, 16, true); + header.setUint16(20, 1, true); + header.setUint16(22, 1, true); + header.setUint32(24, sampleRate, true); + header.setUint32(28, sampleRate * 2, true); + header.setUint16(32, 2, true); + header.setUint16(34, 16, true); + writeString(36, "data"); + header.setUint32(40, pcm.byteLength, true); + new Uint8Array(wav).set(new Uint8Array(pcm), 44); + return { pcm, wav }; +} +class SampleQueue { + constructor() { + this.offset = 0; + this.size = 0; + } + get length() { + return this.size; + } + push(input) { + if (input.length === 0) + return; + const block = { samples: new Float32Array(input) }; + if (this.tail) + this.tail.next = block; + else + this.head = block; + this.tail = block; + this.size += input.length; + } + take(count) { + if (!Number.isSafeInteger(count) || count <= 0) + throw new RangeError("Sample count must be a positive safe integer"); + if (count > this.size) + return null; + const output = new Float32Array(count); + let written = 0; + while (this.head && written < count) { + const samples = this.head.samples; + const amount = Math.min(samples.length - this.offset, count - written); + output.set(samples.subarray(this.offset, this.offset + amount), written); + written += amount; + this.offset += amount; + if (this.offset === samples.length) { + this.head = this.head.next; + this.offset = 0; + } + } + if (!this.head) + this.tail = void 0; + this.size -= count; + return output; + } + clear() { + this.head = void 0; + this.tail = void 0; + this.offset = 0; + this.size = 0; + } +} +class Capture { + constructor(video, options, append) { + this.video = video; + this.options = options; + this.append = append; + this.queue = new SampleQueue(); + this.closing = Promise.resolve(); + this.running = false; + this.terminal = false; + this.epoch = 0; + } + reset() { + this.epoch++; + this.pending = void 0; + this.queue.clear(); + } + pause() { + this.reset(); + this.running = false; + clearInterval(this.timer); + this.timer = void 0; + this.starting = void 0; + this.graph?.idle(); + } + restart() { + const active = this.running || Boolean(this.starting); + this.pause(); + if (this.graph && !this.graph.ownsMediaConnection) + return this.stop().then(() => active ? this.start() : void 0); + return active ? this.start() : Promise.resolve(); + } + start() { + if (this.terminal || this.running) + return Promise.resolve(); + if (this.starting) + return this.starting; + const epoch = this.epoch; + const current = () => !this.terminal && this.epoch === epoch; + const operation = (async () => { + await this.closing; + if (!current()) + return; + const { sampleRate, interval } = this.options; + const count = Math.floor(sampleRate * interval / 1e3); + if (!Number.isSafeInteger(count) || count < 1 || !Number.isFinite(interval) || interval <= 0) + throw new Error("Audio chunk length must be positive and finite"); + const graph = this.graph ?? (this.graph = new AudioGraph(this.video, sampleRate)); + await graph.prepare(); + if (!current()) { + if (!this.running && this.graph === graph) + graph.idle(); + return; + } + graph.connect((samples) => { + if (!this.running || this.graph !== graph) + return; + if (this.queue.length + samples.length > Math.max(sampleRate * 60, count * 2)) { + console.error("[artplayerPluginAsr] Audio callback backlog exceeded its capture limit"); + this.pause(); + return; + } + this.queue.push(samples); + }); + this.running = true; + this.timer = setInterval(() => this.tick(count), interval); + })().catch(async (error) => { + if (current()) { + console.error("[artplayerPluginAsr] Initialization failed:", error); + await this.stop(); + } + }).finally(() => { + if (this.starting === operation) + this.starting = void 0; + }); + this.starting = operation; + return operation; + } + async tick(count) { + if (!this.running || this.pending) + return; + const samples = this.queue.take(count); + if (!samples) + return; + const epoch = this.epoch; + const pending = this.pending = {}; + try { + const text = await this.options.onAudioChunk(encodeAudio(samples, this.options.sampleRate)); + if (this.running && !this.terminal && this.epoch === epoch) + this.append(text); + } catch (error) { + if (this.epoch === epoch && !this.terminal) + console.error("[artplayerPluginAsr] Audio callback failed:", error); + } finally { + if (this.pending === pending) + this.pending = void 0; + } + } + volume(value) { + this.graph?.volume(value); + } + stop(terminal = false) { + this.terminal || (this.terminal = terminal); + this.pause(); + const graph = this.graph; + if (terminal || !graph?.ownsMediaConnection) { + this.graph = void 0; + this.closing = Promise.all([this.closing, graph?.close()]).then(() => void 0); + } + return this.closing; + } +} +const style = ".art-video-player .art-layer-asr {\n pointer-events: none !important;\n position: absolute;\n z-index: 150;\n inset: 0px;\n display: flex;\n flex-direction: column;\n justify-content: end;\n padding: 0 2%;\n font-size: 1rem;\n gap: var(--art-subtitle-gap);\n padding-bottom: var(--art-subtitle-bottom);\n transition: padding-bottom var(--art-transition-duration) ease;\n text-shadow: var(--art-subtitle-border) 1px 0 1px, var(--art-subtitle-border) 0 1px 1px, var(--art-subtitle-border) -1px 0 1px, var(--art-subtitle-border) 0 -1px 1px, var(--art-subtitle-border) 1px 1px 1px, var(--art-subtitle-border) -1px -1px 1px, var(--art-subtitle-border) 1px -1px 1px, var(--art-subtitle-border) -1px 1px 1px;\n}\n.art-video-player.art-control-show .art-layer-asr {\n padding-bottom: calc(var(--art-control-height) + var(--art-subtitle-bottom));\n}\n"; +function createSubtitles(layer, length, autoHideTimeout) { + let timer; + let destroyed = false; + function cancelTimer() { + clearTimeout(timer); + timer = void 0; + } + function hide() { + if (!destroyed) + layer.style.display = "none"; + } + function append(subtitle) { + if (destroyed || typeof subtitle !== "string") + return; + cancelTimer(); + timer = setTimeout(hide, autoHideTimeout); + layer.style.display = ""; + layer.innerHTML = subtitle.split(/(?<=[、。！？!?.])\s*/u).map((line) => line.trim()).filter(Boolean).slice(-length).map((line) => `
${line}
`).join(""); + } + return { + append, + hide, + destroy() { + destroyed = true; + cancelTimer(); + } }; } -if (typeof document !== "undefined") { - if (!document.getElementById("artplayer-plugin-asr")) { - const $style = document.createElement("style"); - $style.id = "artplayer-plugin-asr"; - $style.textContent = style; - document.head.appendChild($style); - } +function artplayerPluginAsr(option = {}) { + const { length = 3, interval = 100, sampleRate = 16e3, autoHideTimeout = 1e4, onAudioChunk = () => null } = option; + return (art) => { + const layer = art.layers.add({ name: "asr", html: "" }); + if (!layer) + throw new Error("Could not create ASR subtitle layer"); + const subtitles = createSubtitles(layer, length, autoHideTimeout); + const capture = new Capture(art.video, { interval, sampleRate, onAudioChunk }, subtitles.append); + const volume = () => capture.volume(art.volume); + const play = () => capture.start(); + const pause = () => capture.pause(); + const restart = () => capture.restart(); + const stop = () => capture.stop(); + const destroy = () => { + subtitles.destroy(); + art.off("video:volumechange", volume); + art.off("play", play); + art.off("pause", pause); + art.off("restart", restart); + art.off("destroy", destroy); + return capture.stop(true); + }; + art.on("video:volumechange", volume); + art.on("play", play); + art.on("pause", pause); + art.on("restart", restart); + art.on("destroy", destroy); + return { name: "artplayerPluginAsr", stop, hide: subtitles.hide, append: subtitles.append }; + }; +} +Object.defineProperty(artplayerPluginAsr, "default", { value: artplayerPluginAsr }); +if (typeof document !== "undefined" && !document.getElementById("artplayer-plugin-asr")) { + const styleElement = document.createElement("style"); + styleElement.id = "artplayer-plugin-asr"; + styleElement.textContent = style; + document.head.appendChild(styleElement); } export { artplayerPluginAsr as default diff --git a/package.json b/package.json index 570e5936b..c9bb0a383 100644 --- a/package.json +++ b/package.json @@ -49,9 +49,9 @@ "test:baseline": "node --test refactor/scripts/*.test.mjs", "ci:check": "yarn check:toolchain --strict && yarn check:commits --report && yarn check:impact --report && yarn check:ci && yarn test:contracts && yarn check:contracts --report && yarn check:plan && yarn lint && yarn check:types && yarn typecheck && yarn test", "ci:build": "yarn build:types && yarn build all && yarn build:i18n && yarn build:ts && yarn build:docs && yarn test:imports", - "test:imports": "node --test test/esm.test.js test/i18n.test.js test/ssr.test.js", + "test:imports": "node --test test/esm.test.js test/i18n.test.js test/ssr.test.js test/asr-distribution.test.js", "typecheck": "node scripts/typecheck.mjs", - "test:unit": "node --test test/asr-audio.test.js test/asr.test.js test/jassub.test.js test/multiple-subtitles-merge.test.js test/multiple-subtitles-lifecycle.test.js test/multiple-subtitles-failures.test.js test/multiple-subtitles.test.js test/multiple-subtitles-vendor.test.js test/vtt-thumbnail-parser.test.js test/vtt-thumbnail-lifecycle.test.js test/vtt-thumbnail.test.js test/auto-thumbnail-frames.test.js test/auto-thumbnail-lifecycle.test.js test/auto-thumbnail.test.js test/thumbnail-emitter.test.js test/thumbnail-runtime.test.js test/thumbnail-vendor.test.js test/thumbnail-lifecycle.test.js test/thumbnail-input.test.js test/thumbnail.test.js test/iframe-navigation.test.js test/iframe-boundaries.test.js test/iframe-lifecycle.test.js test/iframe.test.js test/mediabunny.test.js test/mediabunny-shim.test.js test/mediabunny-coordination.test.js test/mediabunny-video.test.js test/mediabunny-audio.test.js test/mediabunny-hls.test.js test/mediabunny-entry.test.js test/mediabunny-capability.test.js test/mediabunny-load.test.js test/mediabunny-input.test.js test/dpip.test.js test/dpip-lifecycle.test.js test/canvas.test.js test/canvas-lifecycle.test.js test/ambilight.test.js test/ambilight-lifecycle.test.js test/ambilight-proxy.test.js test/vast.test.js test/vast-lifecycle.test.js test/ads.test.js test/ads-lifecycle.test.js test/playback.test.js test/dash-control.test.js test/dash-contract.test.js test/dash-lifecycle.test.js test/dash-events.test.js test/hls-control.test.js test/audio-track.test.js test/public-behavior.test.js test/helpers.test.js test/chapter.test.js test/utils.test.js test/resource-scope.test.js test/instance-lifecycle.test.js test/options.test.js test/media-hosts.test.js test/plugins.test.js test/source.test.js test/playback-properties.test.js test/media-events.test.js test/template-resources.test.js test/core-vendor.test.js test/component-resources.test.js test/setting-model.test.js test/setting-layout.test.js test/setting-resources.test.js test/subtitle.test.js test/display-native.test.js test/display-video-fullscreen.test.js test/display-pip.test.js test/display-mini.test.js test/display-sizing.test.js test/display-orientation.test.js test/hotkey.test.js test/listener-registry.test.js test/global-events.test.js test/pointer-events.test.js test/gesture.test.js test/event-scheduling.test.js test/notice.test.js test/fast-forward.test.js test/auto-playback.test.js test/builtin-layers.test.js test/prompt-components.test.js test/screenshot.test.js test/thumbnails.test.js test/progress.test.js test/environment.test.js test/storage.test.js test/facade-properties.test.js test/dom-boundaries.test.js test/initialization.test.js test/entry.test.js test/accessibility-button.test.js test/accessibility-focus.test.js test/accessibility-slider.test.js", + "test:unit": "node --test test/asr-encoding.test.js test/asr-lifecycle.test.js test/asr-audio.test.js test/asr.test.js test/jassub.test.js test/multiple-subtitles-merge.test.js test/multiple-subtitles-lifecycle.test.js test/multiple-subtitles-failures.test.js test/multiple-subtitles.test.js test/multiple-subtitles-vendor.test.js test/vtt-thumbnail-parser.test.js test/vtt-thumbnail-lifecycle.test.js test/vtt-thumbnail.test.js test/auto-thumbnail-frames.test.js test/auto-thumbnail-lifecycle.test.js test/auto-thumbnail.test.js test/thumbnail-emitter.test.js test/thumbnail-runtime.test.js test/thumbnail-vendor.test.js test/thumbnail-lifecycle.test.js test/thumbnail-input.test.js test/thumbnail.test.js test/iframe-navigation.test.js test/iframe-boundaries.test.js test/iframe-lifecycle.test.js test/iframe.test.js test/mediabunny.test.js test/mediabunny-shim.test.js test/mediabunny-coordination.test.js test/mediabunny-video.test.js test/mediabunny-audio.test.js test/mediabunny-hls.test.js test/mediabunny-entry.test.js test/mediabunny-capability.test.js test/mediabunny-load.test.js test/mediabunny-input.test.js test/dpip.test.js test/dpip-lifecycle.test.js test/canvas.test.js test/canvas-lifecycle.test.js test/ambilight.test.js test/ambilight-lifecycle.test.js test/ambilight-proxy.test.js test/vast.test.js test/vast-lifecycle.test.js test/ads.test.js test/ads-lifecycle.test.js test/playback.test.js test/dash-control.test.js test/dash-contract.test.js test/dash-lifecycle.test.js test/dash-events.test.js test/hls-control.test.js test/audio-track.test.js test/public-behavior.test.js test/helpers.test.js test/chapter.test.js test/utils.test.js test/resource-scope.test.js test/instance-lifecycle.test.js test/options.test.js test/media-hosts.test.js test/plugins.test.js test/source.test.js test/playback-properties.test.js test/media-events.test.js test/template-resources.test.js test/core-vendor.test.js test/component-resources.test.js test/setting-model.test.js test/setting-layout.test.js test/setting-resources.test.js test/subtitle.test.js test/display-native.test.js test/display-video-fullscreen.test.js test/display-pip.test.js test/display-mini.test.js test/display-sizing.test.js test/display-orientation.test.js test/hotkey.test.js test/listener-registry.test.js test/global-events.test.js test/pointer-events.test.js test/gesture.test.js test/event-scheduling.test.js test/notice.test.js test/fast-forward.test.js test/auto-playback.test.js test/builtin-layers.test.js test/prompt-components.test.js test/screenshot.test.js test/thumbnails.test.js test/progress.test.js test/environment.test.js test/storage.test.js test/facade-properties.test.js test/dom-boundaries.test.js test/initialization.test.js test/entry.test.js test/accessibility-button.test.js test/accessibility-focus.test.js test/accessibility-slider.test.js", "test:coverage": "node --test test/coverage.test.js && node scripts/coverage.mjs", "test": "yarn test:node && yarn test:baseline", "test:browser": "playwright test", @@ -94,7 +94,7 @@ "check:ci": "node refactor/scripts/ci-workflow.mjs", "test:ci": "node --test test/ci-summary.test.js refactor/scripts/ci-workflow.test.mjs refactor/scripts/impact.test.mjs test/package-runtime.test.js", "test:package:runtime": "node scripts/package-runtime.mjs --canonical", - "test:asr": "node --test test/asr.test.js test/asr-audio.test.js" + "test:asr": "node --test test/asr.test.js test/asr-audio.test.js test/asr-encoding.test.js test/asr-lifecycle.test.js test/asr-distribution.test.js" }, "browserslist": "last 1 Chrome version", "devDependencies": { diff --git a/packages/artplayer-plugin-asr/ARCHITECTURE.md b/packages/artplayer-plugin-asr/ARCHITECTURE.md index 86696a00f..411f19eec 100644 --- a/packages/artplayer-plugin-asr/ARCHITECTURE.md +++ b/packages/artplayer-plugin-asr/ARCHITECTURE.md @@ -1,33 +1,86 @@ # ASR maintenance -The current implementation is still in `src/index.js`. TypeScript migration and -resource ownership changes are tracked in the repository refactor plan as -PKG-ASR-02 through PKG-ASR-06. Do not interpret the baseline tests as completion. +The player captures first-channel audio and hands PCM/WAV to a caller-provided +recognizer. There is no network request, microphone permission or embedded model. -- The default factory snapshots options. Registration creates the `asr` layer - and subscribes to player play, pause, volumechange and destroy events. -- Play creates an AudioContext and tries a media element source, then a captured - MediaStream source. An inline Blob contains the `recorder-processor` Worklet. -- Worklet messages supply first-channel Float32 samples. The interval callback - encodes little-endian mono PCM16 and a 44-byte-header WAV and invokes the - caller's `onAudioChunk({ pcm, wav })`. A returned string replaces subtitles. -- `append` splits punctuation and sets HTML in `div.art-asr-line`; `hide` changes - display only. `src/style.less` owns layout and existing player CSS variables. -- `stop()` is actually asynchronous and releases audio resources, but retains - playback subscriptions. It is not a permanent disable API. There is no public - `start()` method. Public declarations currently describe stop as returning void. +| File | Responsibility | +| --- | --- | +| src/index.ts | Snapshot options, register player events, return the compatible facade and install the stylesheet | +| src/types.ts | Internal options, result and browser capability types; public declarations remain in types/ | +| src/subtitles.ts | Punctuation/HTML rendering and the owned auto-hide timer | +| src/capture.ts | Initialization deduplication, active generation, FIFO consumption and one pending recognizer call | +| src/audio-graph.ts | AudioContext, element/stream source, Worklet loading, playback routing and recorder ownership | +| src/sample-queue.ts | Copied sample blocks with a linked queue and head offset; preserves partial blocks and tails | +| src/encoding.ts | Pure PCM16LE/WAV encoding, including historical clamping and the 44-byte mono WAV header | +| src/worklet.ts | Inline processor code; first-channel messages and the existing recorder-processor registration name | +| src/style.less | Existing art-layer-asr layout, art-asr-line content and player CSS variables | -This package does not implement recognition or open an ASR network connection. -The demo service and its WebSocket belong to the caller. Do not run that service -to validate audio capture; use local media and a local callback. +Dependencies flow from the facade into subtitles/capture, then from capture into +the graph, queue and pure encoder. Audio modules do not receive the whole player. +The Worklet is loaded from a temporary Blob before binding the media element, so +a failed module load can close its context without taking over video playback. -The initial implementation has suspected chunk loss, partial initialization -cleanup and stale callback issues. Preserve the frozen historical tests while -adding candidate regression tests when fixing these. Keep callback/stop typing -compatible with historical assignments, and preserve public entrypoints, HTML -rendering behavior, layer names and stylesheet identity. +## States and ownership -From the repository root, `yarn test:asr` verifies historical public behavior; -`yarn test:baseline` includes archive provenance. `yarn dev artplayer-plugin-asr` -and `yarn build artplayer-plugin-asr` use repository build tools. Real browser -audio and lifecycle validation remains a separate required migration step. +Play shares one pending initialization. Successful setup attaches one recorder +and starts one interval. Each tick takes exactly one complete block; an incomplete +block stays queued. Only one callback is pending within the active generation. +Rejections are logged and later ticks continue. Pause, source restart, stop and +destroy invalidate old recognizer completions and discard obsolete queued audio. +Each recorder also checks its own identity, rejecting messages from retired ports. + +Pause detaches the recorder and cancels the interval. The direct media source +continues through its playback gain to the destination. A source switch reuses the +direct source and replaces the recorder; a captured-stream fallback is closed and +reacquired because captured tracks can end on source changes. + +`stop()` returns `Promise` and releases ASR work. An already displayed +subtitle retains its original auto-hide deadline; obsolete callbacks cannot add timers. +It retains playback subscriptions and allows explicit append or a later play. +For a directly bound video, the player keeps its original source/context and audio +route until `art.destroy()`. Closing that context at every stop caused Chromium +to lose audio on restart; a media element cannot simply bind a replacement source. +For a captured fallback, stop closes its disposable context and stops its tracks. +Before direct binding, failed/cancelled initialization closes its context and +revokes the Worklet URL. Destroy closes all contexts, disconnects nodes, stops +captured tracks, cancels subtitle timers and removes subscriptions. + +This lifetime follows the [Web Audio close semantics](https://www.w3.org/TR/webaudio-1.0/#dom-audiocontext-close) +and [media-element source behavior](https://www.w3.org/TR/webaudio-1.0/#MediaElementAudioSourceNode). +Captured-stream source changes follow [Media Capture from DOM Elements](https://www.w3.org/TR/mediacapture-fromelement/#html-media-element-media-capture-extensions). + +Queued audio is limited to the larger of 60 seconds or two configured chunks. A caller that +cannot keep up gets an explicit console error and capture pauses, leaving normal +playback routed. This avoids an unbounded backlog; it does not claim lossless +recognition under overload. A later play can restart capture. The caller owns +cancellation of its own network requests; obsolete completions cannot repaint. + +## Compatibility and verification + +The factory still snapshots options and registers synchronously, returning +`name`, `stop`, `hide`, `append`. There is no public start method. Append preserves +HTML, punctuation splitting, replacement, ignored non-string values and the +historical `length=0` behavior. The CJS function has a non-enumerable self `default` +alias for consumers of the 2.0 namespace, while preserving the callable 2.1 form. + +Root public declarations have not yet migrated: they describe stop as void and +the callback as void/Promise. PKG-ASR-04 owns exact asynchronous public types, +old assignment consumers, module forms and installed tarball acceptance. Internal +strict TypeScript does not prove that public type migration complete. + +From the repository root: + +- `yarn test:asr`: frozen public/audio observations, queue/encoding parity and + candidate lifecycle regressions. Historical defects remain separate tests. +- `yarn test:browser test/browser/asr-audio.spec.js`: local AAC, actual Worklet, + published/current core, pause/resume, source switch, stop/restart and destroy. + Set `ARTPLAYER_ASR_ARTIFACT` to an actual main/legacy file to test that artifact. +- `yarn typecheck`: includes the package strict/noUncheckedIndexedAccess project. +- `yarn dev artplayer-plugin-asr` and `yarn build artplayer-plugin-asr`: normal + repository development and three-format production builds. + +Windows Playwright WebKit 26.6 lacks AudioContext and AudioWorkletNode. Its native +ASR cases remain explicit skips and do not establish Safari/device compatibility. +Real fallback track replacement, native volume/mute, CORS, unsupported media and +physical devices remain PKG-ASR-05 acceptance work. Do not use the demo's external +recognizer service as a prerequisite for local audio verification. diff --git a/packages/artplayer-plugin-asr/README.md b/packages/artplayer-plugin-asr/README.md index 545056cf2..e1c4ae10b 100644 --- a/packages/artplayer-plugin-asr/README.md +++ b/packages/artplayer-plugin-asr/README.md @@ -3,6 +3,11 @@ Audio capture and subtitle display plugin for ArtPlayer. Supplies PCM/WAV chunks to a caller-provided recognition service. +`stop()` stops recognition and returns a Promise. It permits later playback to +restart capture; the player's direct Web Audio connection remains alive until +the player is destroyed so that stopping recognition does not silence playback. +Call `art.destroy()` when disposing the player. + ## Maintenance See [ARCHITECTURE.md](ARCHITECTURE.md) for the current audio flow, public behavior, diff --git a/packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.js b/packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.js index eba061fb8..f218500a2 100644 --- a/packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.js +++ b/packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.js @@ -4,4 +4,4 @@ * (c) 2017-2026 Harvey Zhao * Released under the MIT License. */ -!function(t,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"==typeof define&&define.amd?(t.artplayerPluginAsr=e(),define(function(){return t.artplayerPluginAsr})):(t="undefined"!==typeof globalThis?globalThis:t||self).artplayerPluginAsr=e()}(this,function(){"use strict";if("undefined"!==typeof document&&!document.getElementById("artplayer-plugin-asr")){const t=document.createElement("style");t.id="artplayer-plugin-asr",t.textContent=".art-video-player .art-layer-asr{pointer-events:none!important;position:absolute;z-index:150;inset:0;display:flex;flex-direction:column;justify-content:end;padding:0 2%;font-size:1rem;gap:var(--art-subtitle-gap);padding-bottom:var(--art-subtitle-bottom);transition:padding-bottom var(--art-transition-duration) ease;text-shadow:var(--art-subtitle-border) 1px 0 1px,var(--art-subtitle-border) 0 1px 1px,var(--art-subtitle-border) -1px 0 1px,var(--art-subtitle-border) 0 -1px 1px,var(--art-subtitle-border) 1px 1px 1px,var(--art-subtitle-border) -1px -1px 1px,var(--art-subtitle-border) 1px -1px 1px,var(--art-subtitle-border) -1px 1px 1px}.art-video-player.art-control-show .art-layer-asr{padding-bottom:calc(var(--art-control-height) + var(--art-subtitle-bottom))}",document.head.appendChild(t)}return function(t={}){const{length:e=3,interval:n=100,sampleRate:r=16e3,autoHideTimeout:a=1e4,onAudioChunk:o=()=>null}=t;return t=>{let i=!1,l=null,s=null,u=null,c=null,d=[],p=null,f=!1,y=null,m=null,g=null;const h=t.layers.add({name:"asr",html:""});function w(){h.style.display="none"}function b(t){var n;"string"===typeof t&&(clearTimeout(y),y=setTimeout(w,a),h.style.display="",h.innerHTML=(n=t,n.split(/(?<=[\u3001\u3002\uff01\uff1f!?.])\s*/u).map(t=>t.trim()).filter(Boolean).slice(-e)).map(t=>`
${t}
`).join(""))}async function v(){if(!s&&t.video)try{return s=l.createMediaElementSource(t.video),s}catch(e){console.warn("[artplayerPluginAsr] Direct connection failed:",e)}try{if(m||(m=await async function(t){try{return t.captureStream?t.captureStream():t.mozCaptureStream?t.mozCaptureStream():(console.warn("[artplayerPluginAsr] captureStream not supported"),null)}catch(e){return console.warn("[artplayerPluginAsr] Error capturing stream:",e),null}}(t.video)),m)return g=l.createMediaStreamSource(m),g}catch(e){console.warn("[artplayerPluginAsr] MediaStream fallback failed:",e)}return null}async function x(){i&&(i=!1,clearInterval(p),p=null,u&&(u.disconnect(),u.port.onmessage=null,u=null),c&&(c.disconnect(),c=null),d=[])}async function A(){await x(),g&&(g.disconnect(),g=null),s&&(s.disconnect(),s=null),m&&(m.getTracks().forEach(t=>t.stop()),m=null),l&&(await l.close(),l=null),f=!1}return t.on("video:volumechange",()=>{c&&(c.gain.value=t.volume)}),t.on("play",async function(){if(!i)try{await async function(){return l||(l=new(window.AudioContext||window.webkitAudioContext)({sampleRate:r}),"suspended"===l.state&&await l.resume(),l)}();const t=await v();if(!t)throw new Error("Could not establish audio source");if(!f){const t=(()=>{const t=new Blob(["\n class RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n }\n registerProcessor('recorder-processor', RecorderProcessor);\n "],{type:"application/javascript"});return URL.createObjectURL(t)})();await l.audioWorklet.addModule(t),URL.revokeObjectURL(t),f=!0}c=l.createGain(),c.gain.value=1,u=new AudioWorkletNode(l,"recorder-processor"),u.port.onmessage=t=>{d.push(new Float32Array(t.data))},t.connect(u),t.connect(c),c.connect(l.destination);const e=r*(n/1e3);p=setInterval(async()=>{if(0===d.length)return;let t=new Float32Array(0);for(;t.length{for(let n=0;n{throw this.preparation=void 0,t})),this.preparation}get ownsMediaConnection(){return this.direct}idle(){const t=this.gain;this.disconnect(),this.direct&&this.context&&this.source&&(this.gain=t||this.context.createGain(),t||(this.gain.gain.value=1),this.source.connect(this.gain),this.gain.connect(this.context.destination))}connect(t){this.assertOpen();const e=this.context,i=this.source;if(!e||!i)throw new Error("Audio capture is not prepared");const s=this.gain;this.disconnect();const n=this.gain=s||e.createGain();s||(n.gain.value=1);const r=this.recorder=new AudioWorkletNode(e,"recorder-processor");r.port.onmessage=e=>{this.closed||this.recorder!==r||t(e.data)},i.connect(r),i.connect(n),n.connect(e.destination)}volume(t){this.gain&&(this.gain.gain.value=t)}disconnect(){this.recorder&&(this.recorder.port.onmessage=null,this.recorder.disconnect(),this.recorder=void 0),this.gain?.disconnect(),this.gain=void 0,this.source?.disconnect()}async close(){if(this.closed)return;this.closed=!0,this.disconnect(),this.revokeWorklet(),this.stream?.getTracks().forEach(t=>t.stop()),this.stream=void 0;const t=this.context;this.context=void 0,this.source=void 0,t&&"closed"!==t.state&&await t.close()}}class e{constructor(){this.offset=0,this.size=0}get length(){return this.size}push(t){if(0===t.length)return;const e={samples:new Float32Array(t)};this.tail?this.tail.next=e:this.head=e,this.tail=e,this.size+=t.length}take(t){if(!Number.isSafeInteger(t)||t<=0)throw new RangeError("Sample count must be a positive safe integer");if(t>this.size)return null;const e=new Float32Array(t);let i=0;for(;this.head&&it?this.start():void 0):t?this.start():Promise.resolve()}start(){if(this.terminal||this.running)return Promise.resolve();if(this.starting)return this.starting;const e=this.epoch,i=()=>!this.terminal&&this.epoch===e,s=(async()=>{if(await this.closing,!i())return;const{sampleRate:e,interval:s}=this.options,n=Math.floor(e*s/1e3);if(!Number.isSafeInteger(n)||n<1||!Number.isFinite(s)||s<=0)throw new Error("Audio chunk length must be positive and finite");const r=this.graph??(this.graph=new t(this.video,e));await r.prepare(),i()?(r.connect(t=>{if(this.running&&this.graph===r)return this.queue.length+t.length>Math.max(60*e,2*n)?(console.error("[artplayerPluginAsr] Audio callback backlog exceeded its capture limit"),void this.pause()):void this.queue.push(t)}),this.running=!0,this.timer=setInterval(()=>this.tick(n),s)):this.running||this.graph!==r||r.idle()})().catch(async t=>{i()&&(console.error("[artplayerPluginAsr] Initialization failed:",t),await this.stop())}).finally(()=>{this.starting===s&&(this.starting=void 0)});return this.starting=s,s}async tick(t){if(!this.running||this.pending)return;const e=this.queue.take(t);if(!e)return;const i=this.epoch,s=this.pending={};try{const t=await this.options.onAudioChunk(function(t,e){const i=new ArrayBuffer(2*t.length),s=new DataView(i);for(let a=0;a{})),this.closing}}function s(t={}){const{length:e=3,interval:s=100,sampleRate:n=16e3,autoHideTimeout:r=1e4,onAudioChunk:o=()=>null}=t;return t=>{const a=t.layers.add({name:"asr",html:""});if(!a)throw new Error("Could not create ASR subtitle layer");const h=function(t,e,i){let s,n=!1;function r(){clearTimeout(s),s=void 0}function o(){n||(t.style.display="none")}return{append:function(a){n||"string"!==typeof a||(r(),s=setTimeout(o,i),t.style.display="",t.innerHTML=a.split(/(?<=[\u3001\u3002\uff01\uff1f!?.])\s*/u).map(t=>t.trim()).filter(Boolean).slice(-e).map(t=>`
${t}
`).join(""))},hide:o,destroy(){n=!0,r()}}}(a,e,r),c=new i(t.video,{interval:s,sampleRate:n,onAudioChunk:o},h.append),l=()=>c.volume(t.volume),d=()=>c.start(),u=()=>c.pause(),p=()=>c.restart(),g=()=>(h.destroy(),t.off("video:volumechange",l),t.off("play",d),t.off("pause",u),t.off("restart",p),t.off("destroy",g),c.stop(!0));return t.on("video:volumechange",l),t.on("play",d),t.on("pause",u),t.on("restart",p),t.on("destroy",g),{name:"artplayerPluginAsr",stop:()=>c.stop(),hide:h.hide,append:h.append}}}if(Object.defineProperty(s,"default",{value:s}),"undefined"!==typeof document&&!document.getElementById("artplayer-plugin-asr")){const t=document.createElement("style");t.id="artplayer-plugin-asr",t.textContent=".art-video-player .art-layer-asr{pointer-events:none!important;position:absolute;z-index:150;inset:0;display:flex;flex-direction:column;justify-content:end;padding:0 2%;font-size:1rem;gap:var(--art-subtitle-gap);padding-bottom:var(--art-subtitle-bottom);transition:padding-bottom var(--art-transition-duration) ease;text-shadow:var(--art-subtitle-border) 1px 0 1px,var(--art-subtitle-border) 0 1px 1px,var(--art-subtitle-border) -1px 0 1px,var(--art-subtitle-border) 0 -1px 1px,var(--art-subtitle-border) 1px 1px 1px,var(--art-subtitle-border) -1px -1px 1px,var(--art-subtitle-border) 1px -1px 1px,var(--art-subtitle-border) -1px 1px 1px}.art-video-player.art-control-show .art-layer-asr{padding-bottom:calc(var(--art-control-height) + var(--art-subtitle-bottom))}",document.head.appendChild(t)}return s}); diff --git a/packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.legacy.js b/packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.legacy.js index 71d3bc90c..3a00665c7 100644 --- a/packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.legacy.js +++ b/packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.legacy.js @@ -4,4 +4,4 @@ * (c) 2017-2026 Harvey Zhao * Released under the MIT License. */ -!function(t,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"==typeof define&&define.amd?(t.artplayerPluginAsr=e(),define(function(){return t.artplayerPluginAsr})):(t="undefined"!==typeof globalThis?globalThis:t||self).artplayerPluginAsr=e()}(this,function(){"use strict";var t=(t,e,n)=>new Promise((r,o)=>{var a=t=>{try{l(n.next(t))}catch(e){o(e)}},i=t=>{try{l(n.throw(t))}catch(e){o(e)}},l=t=>t.done?r(t.value):Promise.resolve(t.value).then(a,i);l((n=n.apply(t,e)).next())});if("undefined"!==typeof document&&!document.getElementById("artplayer-plugin-asr")){const t=document.createElement("style");t.id="artplayer-plugin-asr",t.textContent=".art-video-player .art-layer-asr{pointer-events:none!important;position:absolute;z-index:150;inset:0;display:flex;flex-direction:column;justify-content:end;padding:0 2%;font-size:1rem;gap:var(--art-subtitle-gap);padding-bottom:var(--art-subtitle-bottom);transition:padding-bottom var(--art-transition-duration) ease;text-shadow:var(--art-subtitle-border) 1px 0 1px,var(--art-subtitle-border) 0 1px 1px,var(--art-subtitle-border) -1px 0 1px,var(--art-subtitle-border) 0 -1px 1px,var(--art-subtitle-border) 1px 1px 1px,var(--art-subtitle-border) -1px -1px 1px,var(--art-subtitle-border) 1px -1px 1px,var(--art-subtitle-border) -1px 1px 1px}.art-video-player.art-control-show .art-layer-asr{padding-bottom:calc(var(--art-control-height) + var(--art-subtitle-bottom))}",document.head.appendChild(t)}return function(e={}){const{length:n=3,interval:r=100,sampleRate:o=16e3,autoHideTimeout:a=1e4,onAudioChunk:i=()=>null}=e;return e=>{let l=!1,s=null,u=null,c=null,d=null,p=[],f=null,y=!1,m=null,h=null,g=null;const v=e.layers.add({name:"asr",html:""});function b(){v.style.display="none"}function x(t){var e;"string"===typeof t&&(clearTimeout(m),m=setTimeout(b,a),v.style.display="",v.innerHTML=(e=t,e.split(new RegExp("(?<=[\u3001\u3002\uff01\uff1f!?.])\\s*","u")).map(t=>t.trim()).filter(Boolean).slice(-n)).map(t=>`
${t}
`).join(""))}function w(){return t(this,null,function*(){if(!u&&e.video)try{return u=s.createMediaElementSource(e.video),u}catch(n){console.warn("[artplayerPluginAsr] Direct connection failed:",n)}try{if(h||(h=yield function(e){return t(this,null,function*(){try{return e.captureStream?e.captureStream():e.mozCaptureStream?e.mozCaptureStream():(console.warn("[artplayerPluginAsr] captureStream not supported"),null)}catch(n){return console.warn("[artplayerPluginAsr] Error capturing stream:",n),null}})}(e.video)),h)return g=s.createMediaStreamSource(h),g}catch(n){console.warn("[artplayerPluginAsr] MediaStream fallback failed:",n)}return null})}function A(){return t(this,null,function*(){l&&(l=!1,clearInterval(f),f=null,c&&(c.disconnect(),c.port.onmessage=null,c=null),d&&(d.disconnect(),d=null),p=[])})}function U(){return t(this,null,function*(){yield A(),g&&(g.disconnect(),g=null),u&&(u.disconnect(),u=null),h&&(h.getTracks().forEach(t=>t.stop()),h=null),s&&(yield s.close(),s=null),y=!1})}return e.on("video:volumechange",()=>{d&&(d.gain.value=e.volume)}),e.on("play",function(){return t(this,null,function*(){if(!l)try{yield function(){return t(this,null,function*(){return s||(s=new(window.AudioContext||window.webkitAudioContext)({sampleRate:o}),"suspended"===s.state&&(yield s.resume()),s)})}();const e=yield w();if(!e)throw new Error("Could not establish audio source");if(!y){const t=(()=>{const t=new Blob(["\n class RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n }\n registerProcessor('recorder-processor', RecorderProcessor);\n "],{type:"application/javascript"});return URL.createObjectURL(t)})();yield s.audioWorklet.addModule(t),URL.revokeObjectURL(t),y=!0}d=s.createGain(),d.gain.value=1,c=new AudioWorkletNode(s,"recorder-processor"),c.port.onmessage=t=>{p.push(new Float32Array(t.data))},e.connect(c),e.connect(d),d.connect(s.destination);const n=o*(r/1e3);f=setInterval(()=>t(null,null,function*(){if(0===p.length)return;let t=new Float32Array(0);for(;t.length{for(let n=0;nnew Promise((s,n)=>{var r=t=>{try{a(i.next(t))}catch(e){n(e)}},o=t=>{try{a(i.throw(t))}catch(e){n(e)}},a=t=>t.done?s(t.value):Promise.resolve(t.value).then(r,o);a((i=i.apply(t,e)).next())});class e{constructor(t,e){this.video=t,this.sampleRate=e,this.closed=!1,this.direct=!1,this.workletLoaded=!1}assertOpen(){if(this.closed)throw new Error("Audio capture was closed")}revokeWorklet(){void 0!==this.blobUrl&&(URL.revokeObjectURL(this.blobUrl),this.blobUrl=void 0)}initialize(){return t(this,null,function*(){var t;this.assertOpen();const e=window,i=window.AudioContext||e.webkitAudioContext;if(!i)throw new Error("AudioContext is not supported");const s=null!=(t=this.context)?t:this.context=new i({sampleRate:this.sampleRate});if("suspended"===s.state&&(yield s.resume()),this.assertOpen(),!this.workletLoaded){this.blobUrl=URL.createObjectURL(new Blob(["\nclass RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n}\nregisterProcessor('recorder-processor', RecorderProcessor);\n"],{type:"application/javascript"}));try{yield s.audioWorklet.addModule(this.blobUrl),this.workletLoaded=!0}finally{this.revokeWorklet()}}if(this.assertOpen(),!this.source)try{this.source=s.createMediaElementSource(this.video),this.direct=!0}catch(n){console.warn("[artplayerPluginAsr] Direct connection failed:",n);const t=this.video.captureStream||this.video.mozCaptureStream;if(!t)throw new Error("Could not establish audio source");this.stream=t.call(this.video),this.source=s.createMediaStreamSource(this.stream)}})}prepare(){return null!=this.preparation||(this.preparation=this.initialize().catch(t=>{throw this.preparation=void 0,t})),this.preparation}get ownsMediaConnection(){return this.direct}idle(){const t=this.gain;this.disconnect(),this.direct&&this.context&&this.source&&(this.gain=t||this.context.createGain(),t||(this.gain.gain.value=1),this.source.connect(this.gain),this.gain.connect(this.context.destination))}connect(t){this.assertOpen();const e=this.context,i=this.source;if(!e||!i)throw new Error("Audio capture is not prepared");const s=this.gain;this.disconnect();const n=this.gain=s||e.createGain();s||(n.gain.value=1);const r=this.recorder=new AudioWorkletNode(e,"recorder-processor");r.port.onmessage=e=>{this.closed||this.recorder!==r||t(e.data)},i.connect(r),i.connect(n),n.connect(e.destination)}volume(t){this.gain&&(this.gain.gain.value=t)}disconnect(){var t,e;this.recorder&&(this.recorder.port.onmessage=null,this.recorder.disconnect(),this.recorder=void 0),null==(t=this.gain)||t.disconnect(),this.gain=void 0,null==(e=this.source)||e.disconnect()}close(){return t(this,null,function*(){var t;if(this.closed)return;this.closed=!0,this.disconnect(),this.revokeWorklet(),null==(t=this.stream)||t.getTracks().forEach(t=>t.stop()),this.stream=void 0;const e=this.context;this.context=void 0,this.source=void 0,e&&"closed"!==e.state&&(yield e.close())})}}class i{constructor(){this.offset=0,this.size=0}get length(){return this.size}push(t){if(0===t.length)return;const e={samples:new Float32Array(t)};this.tail?this.tail.next=e:this.head=e,this.tail=e,this.size+=t.length}take(t){if(!Number.isSafeInteger(t)||t<=0)throw new RangeError("Sample count must be a positive safe integer");if(t>this.size)return null;const e=new Float32Array(t);let i=0;for(;this.head&&it?this.start():void 0):t?this.start():Promise.resolve()}start(){if(this.terminal||this.running)return Promise.resolve();if(this.starting)return this.starting;const i=this.epoch,s=()=>!this.terminal&&this.epoch===i,n=(()=>t(this,null,function*(){var t;if(yield this.closing,!s())return;const{sampleRate:i,interval:n}=this.options,r=Math.floor(i*n/1e3);if(!Number.isSafeInteger(r)||r<1||!Number.isFinite(n)||n<=0)throw new Error("Audio chunk length must be positive and finite");const o=null!=(t=this.graph)?t:this.graph=new e(this.video,i);yield o.prepare(),s()?(o.connect(t=>{if(this.running&&this.graph===o)return this.queue.length+t.length>Math.max(60*i,2*r)?(console.error("[artplayerPluginAsr] Audio callback backlog exceeded its capture limit"),void this.pause()):void this.queue.push(t)}),this.running=!0,this.timer=setInterval(()=>this.tick(r),n)):this.running||this.graph!==o||o.idle()}))().catch(e=>t(this,null,function*(){s()&&(console.error("[artplayerPluginAsr] Initialization failed:",e),yield this.stop())})).finally(()=>{this.starting===n&&(this.starting=void 0)});return this.starting=n,n}tick(e){return t(this,null,function*(){if(!this.running||this.pending)return;const t=this.queue.take(e);if(!t)return;const i=this.epoch,s=this.pending={};try{const e=yield this.options.onAudioChunk(function(t,e){const i=new ArrayBuffer(2*t.length),s=new DataView(i);for(let a=0;a{})),this.closing}}function n(t={}){const{length:e=3,interval:i=100,sampleRate:n=16e3,autoHideTimeout:r=1e4,onAudioChunk:o=()=>null}=t;return t=>{const a=t.layers.add({name:"asr",html:""});if(!a)throw new Error("Could not create ASR subtitle layer");const h=function(t,e,i){let s,n=!1;function r(){clearTimeout(s),s=void 0}function o(){n||(t.style.display="none")}return{append:function(a){n||"string"!==typeof a||(r(),s=setTimeout(o,i),t.style.display="",t.innerHTML=a.split(new RegExp("(?<=[\u3001\u3002\uff01\uff1f!?.])\\s*","u")).map(t=>t.trim()).filter(Boolean).slice(-e).map(t=>`
${t}
`).join(""))},hide:o,destroy(){n=!0,r()}}}(a,e,r),l=new s(t.video,{interval:i,sampleRate:n,onAudioChunk:o},h.append),c=()=>l.volume(t.volume),d=()=>l.start(),u=()=>l.pause(),p=()=>l.restart(),f=()=>(h.destroy(),t.off("video:volumechange",c),t.off("play",d),t.off("pause",u),t.off("restart",p),t.off("destroy",f),l.stop(!0));return t.on("video:volumechange",c),t.on("play",d),t.on("pause",u),t.on("restart",p),t.on("destroy",f),{name:"artplayerPluginAsr",stop:()=>l.stop(),hide:h.hide,append:h.append}}}if(Object.defineProperty(n,"default",{value:n}),"undefined"!==typeof document&&!document.getElementById("artplayer-plugin-asr")){const t=document.createElement("style");t.id="artplayer-plugin-asr",t.textContent=".art-video-player .art-layer-asr{pointer-events:none!important;position:absolute;z-index:150;inset:0;display:flex;flex-direction:column;justify-content:end;padding:0 2%;font-size:1rem;gap:var(--art-subtitle-gap);padding-bottom:var(--art-subtitle-bottom);transition:padding-bottom var(--art-transition-duration) ease;text-shadow:var(--art-subtitle-border) 1px 0 1px,var(--art-subtitle-border) 0 1px 1px,var(--art-subtitle-border) -1px 0 1px,var(--art-subtitle-border) 0 -1px 1px,var(--art-subtitle-border) 1px 1px 1px,var(--art-subtitle-border) -1px -1px 1px,var(--art-subtitle-border) 1px -1px 1px,var(--art-subtitle-border) -1px 1px 1px}.art-video-player.art-control-show .art-layer-asr{padding-bottom:calc(var(--art-control-height) + var(--art-subtitle-bottom))}",document.head.appendChild(t)}return n}); diff --git a/packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.mjs b/packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.mjs index 37fc36858..3da312943 100644 --- a/packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.mjs +++ b/packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.mjs @@ -4,248 +4,394 @@ * (c) 2017-2026 Harvey Zhao * Released under the MIT License. */ -const style = ".art-video-player .art-layer-asr {\n pointer-events: none !important;\n position: absolute;\n z-index: 150;\n inset: 0px;\n display: flex;\n flex-direction: column;\n justify-content: end;\n padding: 0 2%;\n font-size: 1rem;\n gap: var(--art-subtitle-gap);\n padding-bottom: var(--art-subtitle-bottom);\n transition: padding-bottom var(--art-transition-duration) ease;\n text-shadow: var(--art-subtitle-border) 1px 0 1px, var(--art-subtitle-border) 0 1px 1px, var(--art-subtitle-border) -1px 0 1px, var(--art-subtitle-border) 0 -1px 1px, var(--art-subtitle-border) 1px 1px 1px, var(--art-subtitle-border) -1px -1px 1px, var(--art-subtitle-border) 1px -1px 1px, var(--art-subtitle-border) -1px 1px 1px;\n}\n.art-video-player.art-control-show .art-layer-asr {\n padding-bottom: calc(var(--art-control-height) + var(--art-subtitle-bottom));\n}\n"; -function artplayerPluginAsr(option = {}) { - const { - length = 3, - interval = 100, - sampleRate = 16e3, - autoHideTimeout = 1e4, - onAudioChunk = () => null - } = option; - return (art) => { - let started = false; - let audioCtx = null; - let sourceNode = null; - let recorderNode = null; - let gainNode = null; - let bufferChunks = []; - let timer = null; - let workletLoaded = false; - let hideTimer = null; - let mediaStream = null; - let mediaStreamSource = null; - const $asr = art.layers.add({ - name: "asr", - html: "" - }); - function splitByPunctuation(text) { - return text.split(/(?<=[、。！？!?.])\s*/u).map((s) => s.trim()).filter(Boolean).slice(-length); - } - function hide() { - $asr.style.display = "none"; - } - function append(subtitle) { - if (typeof subtitle !== "string") - return; - clearTimeout(hideTimer); - hideTimer = setTimeout(hide, autoHideTimeout); - $asr.style.display = ""; - $asr.innerHTML = splitByPunctuation(subtitle).map((line) => `
${line}
`).join(""); - } - const recorderProcessorCode = ` - class RecorderProcessor extends AudioWorkletProcessor { - process(inputs) { - const input = inputs[0]; - if (input && input[0]) { - this.port.postMessage(input[0]); - } - return true; - } - } - registerProcessor('recorder-processor', RecorderProcessor); - `; - const createWorkletBlobUrl = () => { - const blob = new Blob([recorderProcessorCode], { type: "application/javascript" }); - return URL.createObjectURL(blob); - }; - function floatTo16BitPCM(float32Array) { - const len = float32Array.length; - const buffer = new ArrayBuffer(len * 2); - const view = new DataView(buffer); - for (let i = 0; i < len; i++) { - const s = Math.max(-1, Math.min(1, float32Array[i])); - view.setInt16(i * 2, s < 0 ? s * 32768 : s * 32767, true); - } - return buffer; - } - function pcmToWav(pcmBuffer, sampleRate2) { - const pcmLength = pcmBuffer.byteLength; - const wavBuffer = new ArrayBuffer(44 + pcmLength); - const view = new DataView(wavBuffer); - const writeString = (offset, str) => { - for (let i = 0; i < str.length; i++) { - view.setUint8(offset + i, str.charCodeAt(i)); +const recorderProcessorCode = ` +class RecorderProcessor extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]; + if (input && input[0]) { + this.port.postMessage(input[0]); } - }; - writeString(0, "RIFF"); - view.setUint32(4, 36 + pcmLength, true); - writeString(8, "WAVE"); - writeString(12, "fmt "); - view.setUint32(16, 16, true); - view.setUint16(20, 1, true); - view.setUint16(22, 1, true); - view.setUint32(24, sampleRate2, true); - view.setUint32(28, sampleRate2 * 2, true); - view.setUint16(32, 2, true); - view.setUint16(34, 16, true); - writeString(36, "data"); - view.setUint32(40, pcmLength, true); - new Uint8Array(wavBuffer).set(new Uint8Array(pcmBuffer), 44); - return wavBuffer; + return true; } - async function setupAudioContext() { - if (audioCtx) - return audioCtx; - audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate }); - if (audioCtx.state === "suspended") - await audioCtx.resume(); - return audioCtx; +} +registerProcessor('recorder-processor', RecorderProcessor); +`; +class AudioGraph { + constructor(video, sampleRate) { + this.video = video; + this.sampleRate = sampleRate; + this.closed = false; + this.direct = false; + this.workletLoaded = false; + } + assertOpen() { + if (this.closed) + throw new Error("Audio capture was closed"); + } + revokeWorklet() { + if (this.blobUrl !== void 0) { + URL.revokeObjectURL(this.blobUrl); + this.blobUrl = void 0; } - async function setupAudioSource() { - if (!sourceNode && art.video) { - try { - sourceNode = audioCtx.createMediaElementSource(art.video); - return sourceNode; - } catch (err) { - console.warn("[artplayerPluginAsr] Direct connection failed:", err); - } - } + } + async initialize() { + this.assertOpen(); + const platform = window; + const Context = window.AudioContext || platform.webkitAudioContext; + if (!Context) + throw new Error("AudioContext is not supported"); + const context = this.context ?? (this.context = new Context({ sampleRate: this.sampleRate })); + if (context.state === "suspended") + await context.resume(); + this.assertOpen(); + if (!this.workletLoaded) { + this.blobUrl = URL.createObjectURL(new Blob([recorderProcessorCode], { type: "application/javascript" })); try { - if (!mediaStream) { - mediaStream = await captureAudioFromVideo(art.video); - } - if (mediaStream) { - mediaStreamSource = audioCtx.createMediaStreamSource(mediaStream); - return mediaStreamSource; - } - } catch (err) { - console.warn("[artplayerPluginAsr] MediaStream fallback failed:", err); - } - return null; - } - async function captureAudioFromVideo(videoElement) { - try { - if (videoElement.captureStream) - return videoElement.captureStream(); - if (videoElement.mozCaptureStream) - return videoElement.mozCaptureStream(); - console.warn("[artplayerPluginAsr] captureStream not supported"); - return null; - } catch (err) { - console.warn("[artplayerPluginAsr] Error capturing stream:", err); - return null; + await context.audioWorklet.addModule(this.blobUrl); + this.workletLoaded = true; + } finally { + this.revokeWorklet(); } } - async function startCapture() { - if (started) - return; + this.assertOpen(); + if (!this.source) { try { - await setupAudioContext(); - const audioSource = await setupAudioSource(); - if (!audioSource) + this.source = context.createMediaElementSource(this.video); + this.direct = true; + } catch (error) { + console.warn("[artplayerPluginAsr] Direct connection failed:", error); + const capture = this.video.captureStream || this.video.mozCaptureStream; + if (!capture) throw new Error("Could not establish audio source"); - if (!workletLoaded) { - const blobUrl = createWorkletBlobUrl(); - await audioCtx.audioWorklet.addModule(blobUrl); - URL.revokeObjectURL(blobUrl); - workletLoaded = true; - } - gainNode = audioCtx.createGain(); - gainNode.gain.value = 1; - recorderNode = new AudioWorkletNode(audioCtx, "recorder-processor"); - recorderNode.port.onmessage = (event) => { - bufferChunks.push(new Float32Array(event.data)); - }; - audioSource.connect(recorderNode); - audioSource.connect(gainNode); - gainNode.connect(audioCtx.destination); - const CHUNK_SAMPLES = sampleRate * (interval / 1e3); - timer = setInterval(async () => { - if (bufferChunks.length === 0) - return; - let accumulated = new Float32Array(0); - while (accumulated.length < CHUNK_SAMPLES && bufferChunks.length) { - const next = bufferChunks.shift(); - const tmp = new Float32Array(accumulated.length + next.length); - tmp.set(accumulated, 0); - tmp.set(next, accumulated.length); - accumulated = tmp; - } - if (accumulated.length < CHUNK_SAMPLES) - return; - const chunkToSend = accumulated.slice(0, CHUNK_SAMPLES); - const pcm = floatTo16BitPCM(chunkToSend); - const wav = pcmToWav(pcm, sampleRate); - const subtitle = await onAudioChunk({ pcm, wav }); - append(subtitle); - }, interval); - started = true; - } catch (err) { - console.error("[artplayerPluginAsr] Initialization failed:", err); - await stopCapture(); + this.stream = capture.call(this.video); + this.source = context.createMediaStreamSource(this.stream); } } - async function stopCapture() { - if (!started) - return; - started = false; - clearInterval(timer); - timer = null; - if (recorderNode) { - recorderNode.disconnect(); - recorderNode.port.onmessage = null; - recorderNode = null; - } - if (gainNode) { - gainNode.disconnect(); - gainNode = null; - } - bufferChunks = []; + } + prepare() { + this.preparation ?? (this.preparation = this.initialize().catch((error) => { + this.preparation = void 0; + throw error; + })); + return this.preparation; + } + get ownsMediaConnection() { + return this.direct; + } + idle() { + const gain = this.gain; + this.disconnect(); + if (this.direct && this.context && this.source) { + this.gain = gain || this.context.createGain(); + if (!gain) + this.gain.gain.value = 1; + this.source.connect(this.gain); + this.gain.connect(this.context.destination); } - async function destroy() { - await stopCapture(); - if (mediaStreamSource) { - mediaStreamSource.disconnect(); - mediaStreamSource = null; - } - if (sourceNode) { - sourceNode.disconnect(); - sourceNode = null; - } - if (mediaStream) { - mediaStream.getTracks().forEach((track) => track.stop()); - mediaStream = null; - } - if (audioCtx) { - await audioCtx.close(); - audioCtx = null; - } - workletLoaded = false; - } - art.on("video:volumechange", () => { - if (gainNode) { - gainNode.gain.value = art.volume; - } - }); - art.on("play", startCapture); - art.on("pause", stopCapture); - art.on("destroy", destroy); - return { - name: "artplayerPluginAsr", - stop: destroy, - hide, - append + } + connect(receive) { + this.assertOpen(); + const context = this.context; + const source = this.source; + if (!context || !source) + throw new Error("Audio capture is not prepared"); + const previousGain = this.gain; + this.disconnect(); + const gain = this.gain = previousGain || context.createGain(); + if (!previousGain) + gain.gain.value = 1; + const recorder = this.recorder = new AudioWorkletNode(context, "recorder-processor"); + recorder.port.onmessage = (event) => { + if (!this.closed && this.recorder === recorder) + receive(event.data); }; + source.connect(recorder); + source.connect(gain); + gain.connect(context.destination); + } + volume(value) { + if (this.gain) + this.gain.gain.value = value; + } + disconnect() { + if (this.recorder) { + this.recorder.port.onmessage = null; + this.recorder.disconnect(); + this.recorder = void 0; + } + this.gain?.disconnect(); + this.gain = void 0; + this.source?.disconnect(); + } + async close() { + if (this.closed) + return; + this.closed = true; + this.disconnect(); + this.revokeWorklet(); + this.stream?.getTracks().forEach((track) => track.stop()); + this.stream = void 0; + const context = this.context; + this.context = void 0; + this.source = void 0; + if (context && context.state !== "closed") + await context.close(); + } +} +function encodeAudio(samples, sampleRate) { + const pcm = new ArrayBuffer(samples.length * 2); + const pcmView = new DataView(pcm); + for (let index = 0; index < samples.length; index++) { + const value = Math.max(-1, Math.min(1, samples[index])); + pcmView.setInt16(index * 2, value < 0 ? value * 32768 : value * 32767, true); + } + const wav = new ArrayBuffer(44 + pcm.byteLength); + const header = new DataView(wav); + function writeString(offset, value) { + for (let index = 0; index < value.length; index++) + header.setUint8(offset + index, value.charCodeAt(index)); + } + writeString(0, "RIFF"); + header.setUint32(4, 36 + pcm.byteLength, true); + writeString(8, "WAVE"); + writeString(12, "fmt "); + header.setUint32(16, 16, true); + header.setUint16(20, 1, true); + header.setUint16(22, 1, true); + header.setUint32(24, sampleRate, true); + header.setUint32(28, sampleRate * 2, true); + header.setUint16(32, 2, true); + header.setUint16(34, 16, true); + writeString(36, "data"); + header.setUint32(40, pcm.byteLength, true); + new Uint8Array(wav).set(new Uint8Array(pcm), 44); + return { pcm, wav }; +} +class SampleQueue { + constructor() { + this.offset = 0; + this.size = 0; + } + get length() { + return this.size; + } + push(input) { + if (input.length === 0) + return; + const block = { samples: new Float32Array(input) }; + if (this.tail) + this.tail.next = block; + else + this.head = block; + this.tail = block; + this.size += input.length; + } + take(count) { + if (!Number.isSafeInteger(count) || count <= 0) + throw new RangeError("Sample count must be a positive safe integer"); + if (count > this.size) + return null; + const output = new Float32Array(count); + let written = 0; + while (this.head && written < count) { + const samples = this.head.samples; + const amount = Math.min(samples.length - this.offset, count - written); + output.set(samples.subarray(this.offset, this.offset + amount), written); + written += amount; + this.offset += amount; + if (this.offset === samples.length) { + this.head = this.head.next; + this.offset = 0; + } + } + if (!this.head) + this.tail = void 0; + this.size -= count; + return output; + } + clear() { + this.head = void 0; + this.tail = void 0; + this.offset = 0; + this.size = 0; + } +} +class Capture { + constructor(video, options, append) { + this.video = video; + this.options = options; + this.append = append; + this.queue = new SampleQueue(); + this.closing = Promise.resolve(); + this.running = false; + this.terminal = false; + this.epoch = 0; + } + reset() { + this.epoch++; + this.pending = void 0; + this.queue.clear(); + } + pause() { + this.reset(); + this.running = false; + clearInterval(this.timer); + this.timer = void 0; + this.starting = void 0; + this.graph?.idle(); + } + restart() { + const active = this.running || Boolean(this.starting); + this.pause(); + if (this.graph && !this.graph.ownsMediaConnection) + return this.stop().then(() => active ? this.start() : void 0); + return active ? this.start() : Promise.resolve(); + } + start() { + if (this.terminal || this.running) + return Promise.resolve(); + if (this.starting) + return this.starting; + const epoch = this.epoch; + const current = () => !this.terminal && this.epoch === epoch; + const operation = (async () => { + await this.closing; + if (!current()) + return; + const { sampleRate, interval } = this.options; + const count = Math.floor(sampleRate * interval / 1e3); + if (!Number.isSafeInteger(count) || count < 1 || !Number.isFinite(interval) || interval <= 0) + throw new Error("Audio chunk length must be positive and finite"); + const graph = this.graph ?? (this.graph = new AudioGraph(this.video, sampleRate)); + await graph.prepare(); + if (!current()) { + if (!this.running && this.graph === graph) + graph.idle(); + return; + } + graph.connect((samples) => { + if (!this.running || this.graph !== graph) + return; + if (this.queue.length + samples.length > Math.max(sampleRate * 60, count * 2)) { + console.error("[artplayerPluginAsr] Audio callback backlog exceeded its capture limit"); + this.pause(); + return; + } + this.queue.push(samples); + }); + this.running = true; + this.timer = setInterval(() => this.tick(count), interval); + })().catch(async (error) => { + if (current()) { + console.error("[artplayerPluginAsr] Initialization failed:", error); + await this.stop(); + } + }).finally(() => { + if (this.starting === operation) + this.starting = void 0; + }); + this.starting = operation; + return operation; + } + async tick(count) { + if (!this.running || this.pending) + return; + const samples = this.queue.take(count); + if (!samples) + return; + const epoch = this.epoch; + const pending = this.pending = {}; + try { + const text = await this.options.onAudioChunk(encodeAudio(samples, this.options.sampleRate)); + if (this.running && !this.terminal && this.epoch === epoch) + this.append(text); + } catch (error) { + if (this.epoch === epoch && !this.terminal) + console.error("[artplayerPluginAsr] Audio callback failed:", error); + } finally { + if (this.pending === pending) + this.pending = void 0; + } + } + volume(value) { + this.graph?.volume(value); + } + stop(terminal = false) { + this.terminal || (this.terminal = terminal); + this.pause(); + const graph = this.graph; + if (terminal || !graph?.ownsMediaConnection) { + this.graph = void 0; + this.closing = Promise.all([this.closing, graph?.close()]).then(() => void 0); + } + return this.closing; + } +} +const style = ".art-video-player .art-layer-asr {\n pointer-events: none !important;\n position: absolute;\n z-index: 150;\n inset: 0px;\n display: flex;\n flex-direction: column;\n justify-content: end;\n padding: 0 2%;\n font-size: 1rem;\n gap: var(--art-subtitle-gap);\n padding-bottom: var(--art-subtitle-bottom);\n transition: padding-bottom var(--art-transition-duration) ease;\n text-shadow: var(--art-subtitle-border) 1px 0 1px, var(--art-subtitle-border) 0 1px 1px, var(--art-subtitle-border) -1px 0 1px, var(--art-subtitle-border) 0 -1px 1px, var(--art-subtitle-border) 1px 1px 1px, var(--art-subtitle-border) -1px -1px 1px, var(--art-subtitle-border) 1px -1px 1px, var(--art-subtitle-border) -1px 1px 1px;\n}\n.art-video-player.art-control-show .art-layer-asr {\n padding-bottom: calc(var(--art-control-height) + var(--art-subtitle-bottom));\n}\n"; +function createSubtitles(layer, length, autoHideTimeout) { + let timer; + let destroyed = false; + function cancelTimer() { + clearTimeout(timer); + timer = void 0; + } + function hide() { + if (!destroyed) + layer.style.display = "none"; + } + function append(subtitle) { + if (destroyed || typeof subtitle !== "string") + return; + cancelTimer(); + timer = setTimeout(hide, autoHideTimeout); + layer.style.display = ""; + layer.innerHTML = subtitle.split(/(?<=[、。！？!?.])\s*/u).map((line) => line.trim()).filter(Boolean).slice(-length).map((line) => `
${line}
`).join(""); + } + return { + append, + hide, + destroy() { + destroyed = true; + cancelTimer(); + } }; } -if (typeof document !== "undefined") { - if (!document.getElementById("artplayer-plugin-asr")) { - const $style = document.createElement("style"); - $style.id = "artplayer-plugin-asr"; - $style.textContent = style; - document.head.appendChild($style); - } +function artplayerPluginAsr(option = {}) { + const { length = 3, interval = 100, sampleRate = 16e3, autoHideTimeout = 1e4, onAudioChunk = () => null } = option; + return (art) => { + const layer = art.layers.add({ name: "asr", html: "" }); + if (!layer) + throw new Error("Could not create ASR subtitle layer"); + const subtitles = createSubtitles(layer, length, autoHideTimeout); + const capture = new Capture(art.video, { interval, sampleRate, onAudioChunk }, subtitles.append); + const volume = () => capture.volume(art.volume); + const play = () => capture.start(); + const pause = () => capture.pause(); + const restart = () => capture.restart(); + const stop = () => capture.stop(); + const destroy = () => { + subtitles.destroy(); + art.off("video:volumechange", volume); + art.off("play", play); + art.off("pause", pause); + art.off("restart", restart); + art.off("destroy", destroy); + return capture.stop(true); + }; + art.on("video:volumechange", volume); + art.on("play", play); + art.on("pause", pause); + art.on("restart", restart); + art.on("destroy", destroy); + return { name: "artplayerPluginAsr", stop, hide: subtitles.hide, append: subtitles.append }; + }; +} +Object.defineProperty(artplayerPluginAsr, "default", { value: artplayerPluginAsr }); +if (typeof document !== "undefined" && !document.getElementById("artplayer-plugin-asr")) { + const styleElement = document.createElement("style"); + styleElement.id = "artplayer-plugin-asr"; + styleElement.textContent = style; + document.head.appendChild(styleElement); } export { artplayerPluginAsr as default diff --git a/packages/artplayer-plugin-asr/src/audio-graph.ts b/packages/artplayer-plugin-asr/src/audio-graph.ts new file mode 100644 index 000000000..2a00d7bab --- /dev/null +++ b/packages/artplayer-plugin-asr/src/audio-graph.ts @@ -0,0 +1,144 @@ +import type { CaptureVideo } from './types' +import { recorderProcessorCode } from './worklet' + +export class AudioGraph { + private context: AudioContext | undefined + private source: AudioNode | undefined + private stream: MediaStream | undefined + private recorder: AudioWorkletNode | undefined + private gain: GainNode | undefined + private preparation: Promise | undefined + private blobUrl: string | undefined + private closed = false + private direct = false + private workletLoaded = false + + constructor(private readonly video: CaptureVideo, private readonly sampleRate: number) {} + + private assertOpen() { + if (this.closed) + throw new Error('Audio capture was closed') + } + + private revokeWorklet() { + if (this.blobUrl !== undefined) { + URL.revokeObjectURL(this.blobUrl) + this.blobUrl = undefined + } + } + + private async initialize() { + this.assertOpen() + const platform = window as Window & { webkitAudioContext?: typeof AudioContext } + const Context = window.AudioContext || platform.webkitAudioContext + if (!Context) + throw new Error('AudioContext is not supported') + const context = this.context ??= new Context({ sampleRate: this.sampleRate }) + if (context.state === 'suspended') + await context.resume() + this.assertOpen() + if (!this.workletLoaded) { + this.blobUrl = URL.createObjectURL(new Blob([recorderProcessorCode], { type: 'application/javascript' })) + try { + await context.audioWorklet.addModule(this.blobUrl) + this.workletLoaded = true + } + finally { + this.revokeWorklet() + } + } + this.assertOpen() + if (!this.source) { + try { + this.source = context.createMediaElementSource(this.video) + this.direct = true + } + catch (error) { + console.warn('[artplayerPluginAsr] Direct connection failed:', error) + const capture = this.video.captureStream || this.video.mozCaptureStream + if (!capture) + throw new Error('Could not establish audio source') + this.stream = capture.call(this.video) + this.source = context.createMediaStreamSource(this.stream) + } + } + } + + prepare() { + this.preparation ??= this.initialize().catch((error: unknown) => { + this.preparation = undefined + throw error + }) + return this.preparation + } + + get ownsMediaConnection() { + return this.direct + } + + idle() { + const gain = this.gain + this.disconnect() + // A media element cannot be reattached after its original context is closed. + // Keep its audio route alive until the player releases the element itself. + if (this.direct && this.context && this.source) { + this.gain = gain || this.context.createGain() + if (!gain) + this.gain.gain.value = 1 + this.source.connect(this.gain) + this.gain.connect(this.context.destination) + } + } + + connect(receive: (samples: Float32Array) => void) { + this.assertOpen() + const context = this.context + const source = this.source + if (!context || !source) + throw new Error('Audio capture is not prepared') + const previousGain = this.gain + this.disconnect() + const gain = this.gain = previousGain || context.createGain() + if (!previousGain) + gain.gain.value = 1 + const recorder = this.recorder = new AudioWorkletNode(context, 'recorder-processor') + recorder.port.onmessage = (event: MessageEvent) => { + if (!this.closed && this.recorder === recorder) + receive(event.data) + } + source.connect(recorder) + source.connect(gain) + gain.connect(context.destination) + } + + volume(value: number) { + if (this.gain) + this.gain.gain.value = value + } + + disconnect() { + if (this.recorder) { + this.recorder.port.onmessage = null + this.recorder.disconnect() + this.recorder = undefined + } + this.gain?.disconnect() + this.gain = undefined + this.source?.disconnect() + } + + async close() { + if (this.closed) + return + this.closed = true + this.disconnect() + this.revokeWorklet() + this.stream?.getTracks().forEach(track => track.stop()) + this.stream = undefined + const context = this.context + this.context = undefined + this.source = undefined + if (context && context.state !== 'closed') + await context.close() + } +} diff --git a/packages/artplayer-plugin-asr/src/capture.ts b/packages/artplayer-plugin-asr/src/capture.ts new file mode 100644 index 000000000..e7ac2f3da --- /dev/null +++ b/packages/artplayer-plugin-asr/src/capture.ts @@ -0,0 +1,127 @@ +import type { CaptureOptions, CaptureVideo } from './types' +import { AudioGraph } from './audio-graph' +import { encodeAudio } from './encoding' +import { SampleQueue } from './sample-queue' + +export class Capture { + private graph: AudioGraph | undefined + private readonly queue = new SampleQueue() + private timer: ReturnType | undefined + private starting: Promise | undefined + private closing: Promise = Promise.resolve() + private running = false + private terminal = false + private epoch = 0 + private pending: object | undefined + + constructor(private readonly video: CaptureVideo, private readonly options: CaptureOptions, private readonly append: (text: unknown) => void) {} + + private reset() { + this.epoch++ + this.pending = undefined + this.queue.clear() + } + + pause() { + this.reset() + this.running = false + clearInterval(this.timer) + this.timer = undefined + this.starting = undefined + this.graph?.idle() + } + + restart() { + const active = this.running || Boolean(this.starting) + this.pause() + if (this.graph && !this.graph.ownsMediaConnection) + return this.stop().then(() => active ? this.start() : undefined) + return active ? this.start() : Promise.resolve() + } + + start(): Promise { + if (this.terminal || this.running) + return Promise.resolve() + if (this.starting) + return this.starting + const epoch = this.epoch + const current = () => !this.terminal && this.epoch === epoch + const operation = (async () => { + await this.closing + if (!current()) + return + const { sampleRate, interval } = this.options + const count = Math.floor(sampleRate * interval / 1000) + if (!Number.isSafeInteger(count) || count < 1 || !Number.isFinite(interval) || interval <= 0) + throw new Error('Audio chunk length must be positive and finite') + const graph = this.graph ??= new AudioGraph(this.video, sampleRate) + await graph.prepare() + if (!current()) { + if (!this.running && this.graph === graph) + graph.idle() + return + } + graph.connect((samples) => { + if (!this.running || this.graph !== graph) + return + // Stop overload explicitly instead of retaining an unbounded recognition backlog. + if (this.queue.length + samples.length > Math.max(sampleRate * 60, count * 2)) { + console.error('[artplayerPluginAsr] Audio callback backlog exceeded its capture limit') + this.pause() + return + } + this.queue.push(samples) + }) + this.running = true + this.timer = setInterval(() => this.tick(count), interval) + })().catch(async (error: unknown) => { + if (current()) { + console.error('[artplayerPluginAsr] Initialization failed:', error) + await this.stop() + } + }).finally(() => { + if (this.starting === operation) + this.starting = undefined + }) + this.starting = operation + return operation + } + + private async tick(count: number) { + if (!this.running || this.pending) + return + const samples = this.queue.take(count) + if (!samples) + return + const epoch = this.epoch + const pending = this.pending = {} + try { + const text = await this.options.onAudioChunk(encodeAudio(samples, this.options.sampleRate)) + if (this.running && !this.terminal && this.epoch === epoch) + this.append(text) + } + catch (error) { + if (this.epoch === epoch && !this.terminal) + console.error('[artplayerPluginAsr] Audio callback failed:', error) + } + finally { + if (this.pending === pending) + this.pending = undefined + } + } + + volume(value: number) { + this.graph?.volume(value) + } + + stop(terminal = false) { + this.terminal ||= terminal + this.pause() + const graph = this.graph + if (terminal || !graph?.ownsMediaConnection) { + this.graph = undefined + this.closing = Promise.all([this.closing, graph?.close()]).then(() => undefined) + } + return this.closing + } +} diff --git a/packages/artplayer-plugin-asr/src/encoding.ts b/packages/artplayer-plugin-asr/src/encoding.ts new file mode 100644 index 000000000..b4fb282d8 --- /dev/null +++ b/packages/artplayer-plugin-asr/src/encoding.ts @@ -0,0 +1,31 @@ +export function encodeAudio(samples: Float32Array, sampleRate: number): { pcm: ArrayBuffer, wav: ArrayBuffer } { + const pcm = new ArrayBuffer(samples.length * 2) + const pcmView = new DataView(pcm) + for (let index = 0; index < samples.length; index++) { + const value = Math.max(-1, Math.min(1, samples[index]!)) + pcmView.setInt16(index * 2, value < 0 ? value * 0x8000 : value * 0x7FFF, true) + } + + const wav = new ArrayBuffer(44 + pcm.byteLength) + const header = new DataView(wav) + function writeString(offset: number, value: string): void { + for (let index = 0; index < value.length; index++) + header.setUint8(offset + index, value.charCodeAt(index)) + } + + writeString(0, 'RIFF') + header.setUint32(4, 36 + pcm.byteLength, true) + writeString(8, 'WAVE') + writeString(12, 'fmt ') + header.setUint32(16, 16, true) + header.setUint16(20, 1, true) + header.setUint16(22, 1, true) + header.setUint32(24, sampleRate, true) + header.setUint32(28, sampleRate * 2, true) + header.setUint16(32, 2, true) + header.setUint16(34, 16, true) + writeString(36, 'data') + header.setUint32(40, pcm.byteLength, true) + new Uint8Array(wav).set(new Uint8Array(pcm), 44) + return { pcm, wav } +} diff --git a/packages/artplayer-plugin-asr/src/index.js b/packages/artplayer-plugin-asr/src/index.js deleted file mode 100644 index c4dd6b136..000000000 --- a/packages/artplayer-plugin-asr/src/index.js +++ /dev/null @@ -1,299 +0,0 @@ -import style from './style.less?inline' - -export default function artplayerPluginAsr(option = {}) { - const { - length = 3, - interval = 100, - sampleRate = 16000, - autoHideTimeout = 10000, - onAudioChunk = () => null, - } = option - - return (art) => { - let started = false - let audioCtx = null - let sourceNode = null - let recorderNode = null - let gainNode = null - let bufferChunks = [] - let timer = null - let workletLoaded = false - let hideTimer = null - let mediaStream = null - let mediaStreamSource = null - - const $asr = art.layers.add({ - name: 'asr', - html: '', - }) - - function splitByPunctuation(text) { - return ( - text - - .split(/(?<=[、。！？!?.])\s*/u) - .map(s => s.trim()) - .filter(Boolean) - .slice(-length) - ) - } - - function hide() { - $asr.style.display = 'none' - } - - function append(subtitle) { - if (typeof subtitle !== 'string') - return - clearTimeout(hideTimer) - hideTimer = setTimeout(hide, autoHideTimeout) - $asr.style.display = '' - $asr.innerHTML = splitByPunctuation(subtitle) - .map(line => `
${line}
`) - .join('') - } - - const recorderProcessorCode = ` - class RecorderProcessor extends AudioWorkletProcessor { - process(inputs) { - const input = inputs[0]; - if (input && input[0]) { - this.port.postMessage(input[0]); - } - return true; - } - } - registerProcessor('recorder-processor', RecorderProcessor); - ` - - const createWorkletBlobUrl = () => { - const blob = new Blob([recorderProcessorCode], { type: 'application/javascript' }) - return URL.createObjectURL(blob) - } - - function floatTo16BitPCM(float32Array) { - const len = float32Array.length - const buffer = new ArrayBuffer(len * 2) - const view = new DataView(buffer) - for (let i = 0; i < len; i++) { - const s = Math.max(-1, Math.min(1, float32Array[i])) - view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7FFF, true) - } - return buffer - } - - function pcmToWav(pcmBuffer, sampleRate) { - const pcmLength = pcmBuffer.byteLength - const wavBuffer = new ArrayBuffer(44 + pcmLength) - const view = new DataView(wavBuffer) - - const writeString = (offset, str) => { - for (let i = 0; i < str.length; i++) { - view.setUint8(offset + i, str.charCodeAt(i)) - } - } - - writeString(0, 'RIFF') - view.setUint32(4, 36 + pcmLength, true) - writeString(8, 'WAVE') - writeString(12, 'fmt ') - view.setUint32(16, 16, true) - view.setUint16(20, 1, true) - view.setUint16(22, 1, true) - view.setUint32(24, sampleRate, true) - view.setUint32(28, sampleRate * 2, true) - view.setUint16(32, 2, true) - view.setUint16(34, 16, true) - writeString(36, 'data') - view.setUint32(40, pcmLength, true) - - new Uint8Array(wavBuffer).set(new Uint8Array(pcmBuffer), 44) - - return wavBuffer - } - - async function setupAudioContext() { - if (audioCtx) - return audioCtx - audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate }) - if (audioCtx.state === 'suspended') - await audioCtx.resume() - return audioCtx - } - - async function setupAudioSource() { - if (!sourceNode && art.video) { - try { - sourceNode = audioCtx.createMediaElementSource(art.video) - return sourceNode - } - catch (err) { - console.warn('[artplayerPluginAsr] Direct connection failed:', err) - } - } - - try { - if (!mediaStream) { - mediaStream = await captureAudioFromVideo(art.video) - } - if (mediaStream) { - mediaStreamSource = audioCtx.createMediaStreamSource(mediaStream) - return mediaStreamSource - } - } - catch (err) { - console.warn('[artplayerPluginAsr] MediaStream fallback failed:', err) - } - - return null - } - - async function captureAudioFromVideo(videoElement) { - try { - if (videoElement.captureStream) - return videoElement.captureStream() - if (videoElement.mozCaptureStream) - return videoElement.mozCaptureStream() - console.warn('[artplayerPluginAsr] captureStream not supported') - return null - } - catch (err) { - console.warn('[artplayerPluginAsr] Error capturing stream:', err) - return null - } - } - - async function startCapture() { - if (started) - return - - try { - await setupAudioContext() - const audioSource = await setupAudioSource() - if (!audioSource) - throw new Error('Could not establish audio source') - - if (!workletLoaded) { - const blobUrl = createWorkletBlobUrl() - await audioCtx.audioWorklet.addModule(blobUrl) - URL.revokeObjectURL(blobUrl) - workletLoaded = true - } - - gainNode = audioCtx.createGain() - gainNode.gain.value = 1 - recorderNode = new AudioWorkletNode(audioCtx, 'recorder-processor') - recorderNode.port.onmessage = (event) => { - bufferChunks.push(new Float32Array(event.data)) - } - audioSource.connect(recorderNode) - audioSource.connect(gainNode) - gainNode.connect(audioCtx.destination) - - const CHUNK_SAMPLES = sampleRate * (interval / 1000) - timer = setInterval(async () => { - if (bufferChunks.length === 0) - return - - let accumulated = new Float32Array(0) - while (accumulated.length < CHUNK_SAMPLES && bufferChunks.length) { - const next = bufferChunks.shift() - const tmp = new Float32Array(accumulated.length + next.length) - tmp.set(accumulated, 0) - tmp.set(next, accumulated.length) - accumulated = tmp - } - - if (accumulated.length < CHUNK_SAMPLES) - return - - const chunkToSend = accumulated.slice(0, CHUNK_SAMPLES) - const pcm = floatTo16BitPCM(chunkToSend) - const wav = pcmToWav(pcm, sampleRate) - const subtitle = await onAudioChunk({ pcm, wav }) - append(subtitle) - }, interval) - - started = true - } - catch (err) { - console.error('[artplayerPluginAsr] Initialization failed:', err) - await stopCapture() - } - } - - async function stopCapture() { - if (!started) - return - started = false - - clearInterval(timer) - timer = null - - if (recorderNode) { - recorderNode.disconnect() - recorderNode.port.onmessage = null - recorderNode = null - } - - if (gainNode) { - gainNode.disconnect() - gainNode = null - } - - bufferChunks = [] - } - - async function destroy() { - await stopCapture() - - if (mediaStreamSource) { - mediaStreamSource.disconnect() - mediaStreamSource = null - } - - if (sourceNode) { - sourceNode.disconnect() - sourceNode = null - } - - if (mediaStream) { - mediaStream.getTracks().forEach(track => track.stop()) - mediaStream = null - } - - if (audioCtx) { - await audioCtx.close() - audioCtx = null - } - - workletLoaded = false - } - - art.on('video:volumechange', () => { - if (gainNode) { - gainNode.gain.value = art.volume - } - }) - - art.on('play', startCapture) - art.on('pause', stopCapture) - art.on('destroy', destroy) - - return { - name: 'artplayerPluginAsr', - stop: destroy, - hide, - append, - } - } -} - -if (typeof document !== 'undefined') { - if (!document.getElementById('artplayer-plugin-asr')) { - const $style = document.createElement('style') - $style.id = 'artplayer-plugin-asr' - $style.textContent = style - document.head.appendChild($style) - } -} diff --git a/packages/artplayer-plugin-asr/src/index.ts b/packages/artplayer-plugin-asr/src/index.ts new file mode 100644 index 000000000..c51c3be36 --- /dev/null +++ b/packages/artplayer-plugin-asr/src/index.ts @@ -0,0 +1,46 @@ +import type Artplayer from 'artplayer' +import type { AsrOptions, AsrResult } from './types' +import { Capture } from './capture' +import style from './style.less?inline' +import { createSubtitles } from './subtitles' + +export default function artplayerPluginAsr(option: AsrOptions = {}) { + const { length = 3, interval = 100, sampleRate = 16000, autoHideTimeout = 10000, onAudioChunk = () => null } = option + return (art: Artplayer): AsrResult => { + const layer = art.layers.add({ name: 'asr', html: '' }) + if (!layer) + throw new Error('Could not create ASR subtitle layer') + const subtitles = createSubtitles(layer, length, autoHideTimeout) + const capture = new Capture(art.video, { interval, sampleRate, onAudioChunk }, subtitles.append) + const volume = () => capture.volume(art.volume) + const play = () => capture.start() + const pause = () => capture.pause() + const restart = () => capture.restart() + const stop = () => capture.stop() + const destroy = () => { + subtitles.destroy() + art.off('video:volumechange', volume) + art.off('play', play) + art.off('pause', pause) + art.off('restart', restart) + art.off('destroy', destroy) + return capture.stop(true) + } + art.on('video:volumechange', volume) + art.on('play', play) + art.on('pause', pause) + art.on('restart', restart) + art.on('destroy', destroy) + return { name: 'artplayerPluginAsr', stop, hide: subtitles.hide, append: subtitles.append } + } +} + +// Published 2.0 CJS consumers used require(...).default; 2.1 exported the function. +Object.defineProperty(artplayerPluginAsr, 'default', { value: artplayerPluginAsr }) + +if (typeof document !== 'undefined' && !document.getElementById('artplayer-plugin-asr')) { + const styleElement = document.createElement('style') + styleElement.id = 'artplayer-plugin-asr' + styleElement.textContent = style + document.head.appendChild(styleElement) +} diff --git a/packages/artplayer-plugin-asr/src/sample-queue.ts b/packages/artplayer-plugin-asr/src/sample-queue.ts new file mode 100644 index 000000000..c54b8a2f1 --- /dev/null +++ b/packages/artplayer-plugin-asr/src/sample-queue.ts @@ -0,0 +1,59 @@ +interface SampleBlock { + samples: Float32Array + next?: SampleBlock +} + +export class SampleQueue { + private head?: SampleBlock + private tail?: SampleBlock + private offset = 0 + private size = 0 + + get length(): number { + return this.size + } + + push(input: Float32Array): void { + if (input.length === 0) + return + const block: SampleBlock = { samples: new Float32Array(input) } + if (this.tail) + this.tail.next = block + else + this.head = block + this.tail = block + this.size += input.length + } + + take(count: number): Float32Array | null { + if (!Number.isSafeInteger(count) || count <= 0) + throw new RangeError('Sample count must be a positive safe integer') + if (count > this.size) + return null + + const output = new Float32Array(count) + let written = 0 + while (this.head && written < count) { + const samples = this.head.samples + const amount = Math.min(samples.length - this.offset, count - written) + output.set(samples.subarray(this.offset, this.offset + amount), written) + written += amount + this.offset += amount + if (this.offset === samples.length) { + this.head = this.head.next + this.offset = 0 + } + } + if (!this.head) + this.tail = undefined + this.size -= count + return output + } + + clear(): void { + this.head = undefined + this.tail = undefined + this.offset = 0 + this.size = 0 + } +} diff --git a/packages/artplayer-plugin-asr/src/subtitles.ts b/packages/artplayer-plugin-asr/src/subtitles.ts new file mode 100644 index 000000000..ddeeda732 --- /dev/null +++ b/packages/artplayer-plugin-asr/src/subtitles.ts @@ -0,0 +1,37 @@ +export function createSubtitles(layer: HTMLElement, length: number, autoHideTimeout: number) { + let timer: ReturnType | undefined + let destroyed = false + + function cancelTimer() { + clearTimeout(timer) + timer = undefined + } + + function hide() { + if (!destroyed) + layer.style.display = 'none' + } + + function append(subtitle: unknown) { + if (destroyed || typeof subtitle !== 'string') + return + cancelTimer() + timer = setTimeout(hide, autoHideTimeout) + layer.style.display = '' + layer.innerHTML = subtitle.split(/(?<=[、。！？!?.])\s*/u) + .map(line => line.trim()) + .filter(Boolean) + .slice(-length) + .map(line => `
${line}
`) + .join('') + } + + return { + append, + hide, + destroy() { + destroyed = true + cancelTimer() + }, + } +} diff --git a/packages/artplayer-plugin-asr/src/types.ts b/packages/artplayer-plugin-asr/src/types.ts new file mode 100644 index 000000000..02afd3366 --- /dev/null +++ b/packages/artplayer-plugin-asr/src/types.ts @@ -0,0 +1,30 @@ +export interface AudioChunk { + pcm: ArrayBuffer + wav: ArrayBuffer +} + +export interface AsrOptions { + length?: number + interval?: number + sampleRate?: number + autoHideTimeout?: number + onAudioChunk?: (chunk: AudioChunk) => unknown +} + +export interface AsrResult { + name: 'artplayerPluginAsr' + stop: () => Promise + hide: () => void + append: (subtitle: string) => void +} + +export interface CaptureVideo extends HTMLVideoElement { + captureStream?: () => MediaStream + mozCaptureStream?: () => MediaStream +} + +export interface CaptureOptions { + interval: number + sampleRate: number + onAudioChunk: (chunk: AudioChunk) => unknown +} diff --git a/packages/artplayer-plugin-asr/src/worklet.ts b/packages/artplayer-plugin-asr/src/worklet.ts new file mode 100644 index 000000000..df9325b7a --- /dev/null +++ b/packages/artplayer-plugin-asr/src/worklet.ts @@ -0,0 +1,12 @@ +export const recorderProcessorCode = ` +class RecorderProcessor extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]; + if (input && input[0]) { + this.port.postMessage(input[0]); + } + return true; + } +} +registerProcessor('recorder-processor', RecorderProcessor); +` diff --git a/packages/artplayer-plugin-asr/tsconfig.json b/packages/artplayer-plugin-asr/tsconfig.json new file mode 100644 index 000000000..c9218405d --- /dev/null +++ b/packages/artplayer-plugin-asr/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "allowJs": false }, + "include": ["src/**/*.ts", "types/**/*.d.ts", "../../types/assets.d.ts"] +} diff --git a/refactor/baselines/asr-ownership-validation.json b/refactor/baselines/asr-ownership-validation.json new file mode 100644 index 000000000..2b133c9a8 --- /dev/null +++ b/refactor/baselines/asr-ownership-validation.json @@ -0,0 +1,1361 @@ +{ + "schemaVersion": 1, + "task": "PKG-ASR-03", + "capturedAt": "2026-09-13T06:43:08.711Z", + "baseCommit": "53c295819db004e507b35689a7bdec1955e08396", + "production": { + "language": "TypeScript", + "strict": true, + "noUncheckedIndexedAccess": true, + "modules": 8, + "publicDeclarationChanged": false, + "dependenciesAdded": [], + "versionChanged": false + }, + "unit": { + "historicalPublic": 28, + "historicalAudio": 133, + "provenance": 4, + "pureAudio": 20, + "candidateLifecycle": 25, + "distribution": 2, + "totalFocused": 212, + "repeatedCandidateArtifacts": { + "main": 25, + "legacy": 25 + }, + "candidateRed": { + "retiredPortAfterPauseResume": "Expected new negative PCM but received old positive PCM; fixed recorder identity check" + } + }, + "browsers": [ + { + "mode": "source", + "passed": 12, + "skipped": 6, + "file": "refactor/.cache/asr03-browser-source-final-v3/report.json", + "sha256": "eb3bcf40ad36448e553c98a9b42988b9a417a231056f2067530cde5b4b7b0638", + "rows": [ + { + "title": "ASR 2.0.0 + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "2.0.0", + "core": "published", + "pluginSha256": "30c7b1fc92a262cda8c3aba63afea9c6c038f52c057039d1c6aa957baed155b6", + "integrity": "sha512-nXTf2drMSKnHRbyU9CQ+3o7Jsf1Meam2O9T8LqYakIr18ag/+i71p2x9zdFrLz16Xa6jfQgKeddmU5r3sXp4LA==", + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 4, + "nonzero": 4, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a" + ], + "sha256": "a47709732484a2565af8014af97f2b3c953f5a0d5ac424fbbb00df4210f75382" + }, + "annotations": [] + }, + { + "title": "ASR 2.0.0 + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "2.0.0", + "core": "candidate", + "pluginSha256": "30c7b1fc92a262cda8c3aba63afea9c6c038f52c057039d1c6aa957baed155b6", + "integrity": "sha512-nXTf2drMSKnHRbyU9CQ+3o7Jsf1Meam2O9T8LqYakIr18ag/+i71p2x9zdFrLz16Xa6jfQgKeddmU5r3sXp4LA==", + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 3, + "nonzero": 3, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a" + ], + "sha256": "246313154c06084282eaf7b3df6f83d89974e1997582adc19e83574119a8eba4" + }, + "annotations": [] + }, + { + "title": "ASR 2.1.0 + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "2.1.0", + "core": "published", + "pluginSha256": "7f975b131a9ea207f88a157aafec778bc4906230500db9b30f5bbfbf327b251d", + "integrity": "sha512-15gXff458PAx9LeJxKzZEvmpFhfwKUfSLgb1Mfr7Xb/ZGsnE+6EA7ezTnO5S55W5mFExjv/e95iOIXP4ijuruQ==", + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 4, + "nonzero": 4, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a" + ], + "sha256": "febb5fd96895faffdab64891ca035bed5af27add24d5a7af690b74861bce1b32" + }, + "annotations": [] + }, + { + "title": "ASR 2.1.0 + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "2.1.0", + "core": "candidate", + "pluginSha256": "7f975b131a9ea207f88a157aafec778bc4906230500db9b30f5bbfbf327b251d", + "integrity": "sha512-15gXff458PAx9LeJxKzZEvmpFhfwKUfSLgb1Mfr7Xb/ZGsnE+6EA7ezTnO5S55W5mFExjv/e95iOIXP4ijuruQ==", + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 4, + "nonzero": 4, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a" + ], + "sha256": "246fc4e906cf1ac321801c2b39eecb5f6d03a4fd328ff5c674bbe55b2746d317" + }, + "annotations": [] + }, + { + "title": "ASR candidate + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "candidate", + "core": "published", + "pluginSha256": "f3a6621365e043a8e2370b6eaa39e94cb9925d3e6615d8e97bd702b71d8a111c", + "integrity": null, + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 21, + "nonzero": 21, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a", + "http://127.0.0.1:8084/test/audio-tone.m4a?asr=second" + ], + "sha256": "2c2484a976c515e7e3f6852f38ac829d9300fca6799b7a5807bfa5800e2f2531" + }, + "annotations": [] + }, + { + "title": "ASR candidate + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "candidate", + "core": "candidate", + "pluginSha256": "f3a6621365e043a8e2370b6eaa39e94cb9925d3e6615d8e97bd702b71d8a111c", + "integrity": null, + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 16, + "nonzero": 16, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a", + "http://127.0.0.1:8084/test/audio-tone.m4a?asr=second" + ], + "sha256": "247b90f3f65dead3740a73a31cef3d53695b3b1400288a30b6515db8b767f3d2" + }, + "annotations": [] + }, + { + "title": "ASR 2.0.0 + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "2.0.0", + "core": "published", + "pluginSha256": "30c7b1fc92a262cda8c3aba63afea9c6c038f52c057039d1c6aa957baed155b6", + "integrity": "sha512-nXTf2drMSKnHRbyU9CQ+3o7Jsf1Meam2O9T8LqYakIr18ag/+i71p2x9zdFrLz16Xa6jfQgKeddmU5r3sXp4LA==", + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 5, + "nonzero": 5, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a" + ], + "sha256": "6cf045be45860018f43c5504722c537160fb71f2b5f0df4ecb5c484bf0420b83" + }, + "annotations": [] + }, + { + "title": "ASR 2.0.0 + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "2.0.0", + "core": "candidate", + "pluginSha256": "30c7b1fc92a262cda8c3aba63afea9c6c038f52c057039d1c6aa957baed155b6", + "integrity": "sha512-nXTf2drMSKnHRbyU9CQ+3o7Jsf1Meam2O9T8LqYakIr18ag/+i71p2x9zdFrLz16Xa6jfQgKeddmU5r3sXp4LA==", + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 5, + "nonzero": 4, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a" + ], + "sha256": "bcd50ffe10ce801ca6a0362cf69e9488eb3a55b6d60c98031b8f8fa472c14256" + }, + "annotations": [] + }, + { + "title": "ASR 2.1.0 + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "2.1.0", + "core": "published", + "pluginSha256": "7f975b131a9ea207f88a157aafec778bc4906230500db9b30f5bbfbf327b251d", + "integrity": "sha512-15gXff458PAx9LeJxKzZEvmpFhfwKUfSLgb1Mfr7Xb/ZGsnE+6EA7ezTnO5S55W5mFExjv/e95iOIXP4ijuruQ==", + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 5, + "nonzero": 5, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a" + ], + "sha256": "fcf3d919299c2c165e3035f34f5bd965a6a6f56e4155dad6fe0f68be95ce9a4e" + }, + "annotations": [] + }, + { + "title": "ASR 2.1.0 + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "2.1.0", + "core": "candidate", + "pluginSha256": "7f975b131a9ea207f88a157aafec778bc4906230500db9b30f5bbfbf327b251d", + "integrity": "sha512-15gXff458PAx9LeJxKzZEvmpFhfwKUfSLgb1Mfr7Xb/ZGsnE+6EA7ezTnO5S55W5mFExjv/e95iOIXP4ijuruQ==", + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 4, + "nonzero": 4, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a" + ], + "sha256": "b4f12ce7bbd64f29f20e3ec838b1ea8439fd848e8dfcc33b08c001dc113a0eb3" + }, + "annotations": [] + }, + { + "title": "ASR candidate + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "candidate", + "core": "published", + "pluginSha256": "f3a6621365e043a8e2370b6eaa39e94cb9925d3e6615d8e97bd702b71d8a111c", + "integrity": null, + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 22, + "nonzero": 22, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a", + "http://127.0.0.1:8084/test/audio-tone.m4a?asr=second" + ], + "sha256": "dcebcb83f5964a081b7bd5f0b03ab0712148f0381ef408d516b0ddef841325d4" + }, + "annotations": [] + }, + { + "title": "ASR candidate + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "candidate", + "core": "candidate", + "pluginSha256": "f3a6621365e043a8e2370b6eaa39e94cb9925d3e6615d8e97bd702b71d8a111c", + "integrity": null, + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 18, + "nonzero": 17, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a", + "http://127.0.0.1:8084/test/audio-tone.m4a?asr=second" + ], + "sha256": "136fdf2be0fb5074c359379857019b67aebf2cfd8edce61d7b8a8db2af3e6726" + }, + "annotations": [] + }, + { + "title": "ASR 2.0.0 + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "2.0.0", + "core": "published", + "pluginSha256": "30c7b1fc92a262cda8c3aba63afea9c6c038f52c057039d1c6aa957baed155b6", + "integrity": "sha512-nXTf2drMSKnHRbyU9CQ+3o7Jsf1Meam2O9T8LqYakIr18ag/+i71p2x9zdFrLz16Xa6jfQgKeddmU5r3sXp4LA==", + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 0, + "nonzero": 0, + "sources": [], + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 30, + "column": 14 + } + } + ] + }, + { + "title": "ASR 2.0.0 + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "2.0.0", + "core": "candidate", + "pluginSha256": "30c7b1fc92a262cda8c3aba63afea9c6c038f52c057039d1c6aa957baed155b6", + "integrity": "sha512-nXTf2drMSKnHRbyU9CQ+3o7Jsf1Meam2O9T8LqYakIr18ag/+i71p2x9zdFrLz16Xa6jfQgKeddmU5r3sXp4LA==", + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 0, + "nonzero": 0, + "sources": [], + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 30, + "column": 14 + } + } + ] + }, + { + "title": "ASR 2.1.0 + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "2.1.0", + "core": "published", + "pluginSha256": "7f975b131a9ea207f88a157aafec778bc4906230500db9b30f5bbfbf327b251d", + "integrity": "sha512-15gXff458PAx9LeJxKzZEvmpFhfwKUfSLgb1Mfr7Xb/ZGsnE+6EA7ezTnO5S55W5mFExjv/e95iOIXP4ijuruQ==", + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 0, + "nonzero": 0, + "sources": [], + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 30, + "column": 14 + } + } + ] + }, + { + "title": "ASR 2.1.0 + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "2.1.0", + "core": "candidate", + "pluginSha256": "7f975b131a9ea207f88a157aafec778bc4906230500db9b30f5bbfbf327b251d", + "integrity": "sha512-15gXff458PAx9LeJxKzZEvmpFhfwKUfSLgb1Mfr7Xb/ZGsnE+6EA7ezTnO5S55W5mFExjv/e95iOIXP4ijuruQ==", + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 0, + "nonzero": 0, + "sources": [], + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 30, + "column": 14 + } + } + ] + }, + { + "title": "ASR candidate + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "candidate", + "core": "published", + "pluginSha256": "f3a6621365e043a8e2370b6eaa39e94cb9925d3e6615d8e97bd702b71d8a111c", + "integrity": null, + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 0, + "nonzero": 0, + "sources": [], + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 30, + "column": 14 + } + } + ] + }, + { + "title": "ASR candidate + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "candidate", + "core": "candidate", + "pluginSha256": "f3a6621365e043a8e2370b6eaa39e94cb9925d3e6615d8e97bd702b71d8a111c", + "integrity": null, + "candidate": "workspace source", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 0, + "nonzero": 0, + "sources": [], + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 30, + "column": 14 + } + } + ] + } + ] + }, + { + "mode": "main", + "passed": 4, + "skipped": 2, + "file": "refactor/.cache/asr03-browser-main-final-v3/report.json", + "sha256": "87108e019cd5fe7fd085d6169e428d09314496e5510dd7dd1e327c7502cdd9fe", + "rows": [ + { + "title": "ASR candidate + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "candidate", + "core": "published", + "pluginSha256": "533b60d6348764fa460c2436cc1197b95544051fb4d1ec4d950fec9f88e1c5cb", + "integrity": null, + "candidate": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.js", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 17, + "nonzero": 17, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a", + "http://127.0.0.1:8084/test/audio-tone.m4a?asr=second" + ], + "sha256": "c1bf3b72b7684e234804bc8c75f0c54b4aa0b7e3ed9c17db19320349d4c474e6" + }, + "annotations": [] + }, + { + "title": "ASR candidate + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "candidate", + "core": "candidate", + "pluginSha256": "533b60d6348764fa460c2436cc1197b95544051fb4d1ec4d950fec9f88e1c5cb", + "integrity": null, + "candidate": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.js", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 27, + "nonzero": 27, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a", + "http://127.0.0.1:8084/test/audio-tone.m4a?asr=second" + ], + "sha256": "e53f416557ecec1e46a738de8deedd9dc2b6f2ded3963cd9d1771abe264568cd" + }, + "annotations": [] + }, + { + "title": "ASR candidate + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "candidate", + "core": "published", + "pluginSha256": "533b60d6348764fa460c2436cc1197b95544051fb4d1ec4d950fec9f88e1c5cb", + "integrity": null, + "candidate": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.js", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 22, + "nonzero": 21, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a", + "http://127.0.0.1:8084/test/audio-tone.m4a?asr=second" + ], + "sha256": "d5967d1c8a490517b529938292e1adafe751ddbb6dacae883c2ef453799f8506" + }, + "annotations": [] + }, + { + "title": "ASR candidate + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "candidate", + "core": "candidate", + "pluginSha256": "533b60d6348764fa460c2436cc1197b95544051fb4d1ec4d950fec9f88e1c5cb", + "integrity": null, + "candidate": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.js", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 21, + "nonzero": 20, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a", + "http://127.0.0.1:8084/test/audio-tone.m4a?asr=second" + ], + "sha256": "b6f205718c51da9a64157b299f87fe3278bffb26a36a0daf40eace9811f3963c" + }, + "annotations": [] + }, + { + "title": "ASR candidate + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "candidate", + "core": "published", + "pluginSha256": "533b60d6348764fa460c2436cc1197b95544051fb4d1ec4d950fec9f88e1c5cb", + "integrity": null, + "candidate": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.js", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 0, + "nonzero": 0, + "sources": [], + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 30, + "column": 14 + } + } + ] + }, + { + "title": "ASR candidate + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "candidate", + "core": "candidate", + "pluginSha256": "533b60d6348764fa460c2436cc1197b95544051fb4d1ec4d950fec9f88e1c5cb", + "integrity": null, + "candidate": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.js", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 0, + "nonzero": 0, + "sources": [], + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 30, + "column": 14 + } + } + ] + } + ] + }, + { + "mode": "legacy", + "passed": 4, + "skipped": 2, + "file": "refactor/.cache/asr03-browser-legacy-final-v3/report.json", + "sha256": "affe582473b8748029c93de7993216c80c1a9bb48a6f570f52af630a104d82b8", + "rows": [ + { + "title": "ASR candidate + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "candidate", + "core": "published", + "pluginSha256": "c36ebbea3a4e5f30b2736ffb6a964d45e330b0911f51e32d9cab44397ec16912", + "integrity": null, + "candidate": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.legacy.js", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 18, + "nonzero": 18, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a", + "http://127.0.0.1:8084/test/audio-tone.m4a?asr=second" + ], + "sha256": "27a0fa2126c3972e23f02bf003263c48c0270b0d6765634c23788b44b5010f72" + }, + "annotations": [] + }, + { + "title": "ASR candidate + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "chromium", + "status": "passed", + "browser": "153.0.8010.12", + "inputs": { + "version": "candidate", + "core": "candidate", + "pluginSha256": "c36ebbea3a4e5f30b2736ffb6a964d45e330b0911f51e32d9cab44397ec16912", + "integrity": null, + "candidate": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.legacy.js", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 18, + "nonzero": 18, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a", + "http://127.0.0.1:8084/test/audio-tone.m4a?asr=second" + ], + "sha256": "7d24b81a28bad9d5f12584df64e8c9cb94f1b62f7ea2f8b45ea8d910e2977c7d" + }, + "annotations": [] + }, + { + "title": "ASR candidate + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "candidate", + "core": "published", + "pluginSha256": "c36ebbea3a4e5f30b2736ffb6a964d45e330b0911f51e32d9cab44397ec16912", + "integrity": null, + "candidate": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.legacy.js", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 18, + "nonzero": 17, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a", + "http://127.0.0.1:8084/test/audio-tone.m4a?asr=second" + ], + "sha256": "a4af9fb4921b3b2d7d0b0d3452c5ffa4a59652135d7ad1b2dd2a7f3bab0bf783" + }, + "annotations": [] + }, + { + "title": "ASR candidate + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "firefox", + "status": "passed", + "browser": "155.0", + "inputs": { + "version": "candidate", + "core": "candidate", + "pluginSha256": "c36ebbea3a4e5f30b2736ffb6a964d45e330b0911f51e32d9cab44397ec16912", + "integrity": null, + "candidate": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.legacy.js", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "function", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "function" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 17, + "nonzero": 16, + "sources": [ + "http://127.0.0.1:8084/test/audio-tone.m4a", + "http://127.0.0.1:8084/test/audio-tone.m4a?asr=second" + ], + "sha256": "4728408786921fae730cc16acd442bc95af3349f59905491896e56b3d590667c" + }, + "annotations": [] + }, + { + "title": "ASR candidate + published: native Worklet supplies local PCM with owned stop and destroy", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "candidate", + "core": "published", + "pluginSha256": "c36ebbea3a4e5f30b2736ffb6a964d45e330b0911f51e32d9cab44397ec16912", + "integrity": null, + "candidate": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.legacy.js", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "core": { + "source": { + "kind": "npm-release", + "version": "5.4.0", + "integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==", + "member": "package/dist/artplayer.js" + }, + "sha256": "f1db1fa3406aa6d41baa7c554749429a8e79f99d56a794bde23345eb9c07573f", + "bytes": 134037 + }, + "audio": { + "chunks": 0, + "nonzero": 0, + "sources": [], + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 30, + "column": 14 + } + } + ] + }, + { + "title": "ASR candidate + candidate: native Worklet supplies local PCM with owned stop and destroy", + "project": "webkit", + "status": "skipped", + "browser": "26.6", + "inputs": { + "version": "candidate", + "core": "candidate", + "pluginSha256": "c36ebbea3a4e5f30b2736ffb6a964d45e330b0911f51e32d9cab44397ec16912", + "integrity": null, + "candidate": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.legacy.js", + "mediaSha256": "83e6672b880b5d1c97093b1e26bdc40221c0b279615f40e99dbf9d2ade7b4d56", + "media": "local mono AAC 440 Hz tone", + "networkService": false + }, + "capabilities": { + "AudioContext": "undefined", + "webkitAudioContext": "undefined", + "AudioWorkletNode": "undefined" + }, + "core": { + "source": { + "kind": "workspace-build", + "entry": "packages\\artplayer\\src\\index.ts" + }, + "sha256": "d9aeee649395b78eb30b38dcaefb3c19c731c21b08e356485bac7cda08ead242", + "bytes": 378117 + }, + "audio": { + "chunks": 0, + "nonzero": 0, + "sources": [], + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "annotations": [ + { + "type": "skip", + "description": "Windows WebKit 26.6 has no AudioContext/AudioWorkletNode; native ASR acceptance remains open", + "location": { + "file": "D:\\github\\ArtPlayer\\test\\browser\\asr-audio.spec.js", + "line": 30, + "column": 14 + } + } + ] + } + ] + } + ], + "artifacts": [ + { + "file": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.js", + "sha256": "533b60d6348764fa460c2436cc1197b95544051fb4d1ec4d950fec9f88e1c5cb", + "bytes": 8862, + "docsCopyIdentical": true + }, + { + "file": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.legacy.js", + "sha256": "c36ebbea3a4e5f30b2736ffb6a964d45e330b0911f51e32d9cab44397ec16912", + "bytes": 9357, + "docsCopyIdentical": true + }, + { + "file": "packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.mjs", + "sha256": "751201241a848cdc26235334ce7603b174a90bf4ac793ff3f85182b1783f6996", + "bytes": 13311, + "docsCopyIdentical": true + } + ], + "nativeStopCorrection": { + "before": { + "file": "refactor/.cache/asr03-browser-restart-before/report.json", + "passed": 2, + "failed": 2, + "skipped": 2, + "observation": "Chromium both cores did not resume nonzero PCM after direct context close; Firefox did." + }, + "after": "Direct player-owned audio route remains until terminal destroy; stop/restart reuses one context. Final source/main/legacy browser cases all pass on Chromium and Firefox.", + "semantics": "Internal context ownership changed; Promise stop, stopped recognizer and later play remain public behavior." + }, + "limitation": [ + "Windows WebKit 26.6 native WebAudio unsupported; not Safari/device evidence.", + "Installed public TS/module consumers remain PKG-ASR-04.", + "Native captured fallback topology, volume/mute, CORS and device matrix remain PKG-ASR-05.", + "Queue overload explicitly stops capture after the larger of 60 seconds or two configured chunks; existing subtitles keep their hide deadline; caller cancellation is outside plugin ownership." + ], + "fullCiRun": false, + "releaseReady": false +} diff --git a/refactor/changes/2026-09-13-PKG-ASR-03-audio-ownership.md b/refactor/changes/2026-09-13-PKG-ASR-03-audio-ownership.md new file mode 100644 index 000000000..e54504344 --- /dev/null +++ b/refactor/changes/2026-09-13-PKG-ASR-03-audio-ownership.md @@ -0,0 +1,62 @@ +# PKG-ASR-03：拆分 ASR 并修复音频归属 + +从 53c295819db004e507b35689a7bdec1955e08396 开始，将单个生产 JS 文件拆成 +8 个严格 TypeScript 模块。公共声明不在本任务改写，ASYNC/旧工厂类型与实际安装 +消费者仍由 PKG-ASR-04 验收。无新依赖，版本不变，正常构建生成三格式及 docs 副本。 + +## 结构改动 + +index 只承担工厂、事件和兼容门面；subtitles 管字幕和隐藏定时器；capture 管代际、 +初始化、节拍和回调；audio-graph 管浏览器节点；encoding/sample-queue/worklet +分离纯数据与处理器边界。完整维护地图在包内 ARCHITECTURE.md。 + +## 独立列明的缺陷修正 + +- 链式块队列保留不足块及尾部，复制输入且避免重复 concat/shift。编码与 7 个 + 历史实现逐字节匹配。每个活动代际只允许一个识别回调，拒绝被捕获且后续可继续。 +- 暂停/切源/停止/终止使旧结果失效；旧结果不会解锁新回调。退休 recorder 的 + 消息即使已经被外部保留也被身份检查拒绝。首次候选回归发现此消息竞态后修复。 +- 初始化去重，取消后不晚建录音图；Worklet URL 在成功、失败和终止路径释放。 + Worklet 模块先准备再绑定 video，失败不会先接管并关闭视频的播放通路。 +- stop 清录音/识别任务，保持可再次播放；已有字幕保留原自动隐藏期限，与历史版一致， + 并拒绝旧识别结果新建隐藏任务。直接绑定视频的 context/source/ + 播放 gain 由播放器持有，直到 art.destroy 才关闭；pause/stop 保持播放路由。 + 捕获流 fallback 属于可释放的采集资源，stop 关闭；切源时重新取得流与音轨。 +- destroy 清监听器、节点、流、timer，并禁止通过保留的 append 引用复活。 +- 慢回调超过队列上限时明确报错并暂停采集，播放路由保留。上限取 60 秒与两个配置块 + 长度的较大值，以支持原本合法的长 interval；有超过 60 秒配置块的回归。不会无限缓存， + 也不声称过载时音频无损；调用者网络请求仍由调用者负责取消。 +- 工厂补不可枚举的自引用 default，兼容 2.0 的 require(...).default 及 2.1 的 + require(...) 函数路径；保留键顺序、同步注册、选项快照、字幕 HTML/CSS 行为。 + +## 为什么 stop 不再关闭 direct context + +实际 Chromium 在 close 后经 capture fallback 重启时无法得到非零 PCM；两个核心 +组合失败，Firefox 两组合通过。原生探测报告保留在证据文件中，没有把能力失败 +抹成通过。Web Audio 规定绑定后的媒体声音走节点图，close 会忽略关联媒体输出； +关闭后不能简单重新绑定节点来恢复。用户公开 stop/play 用法需要可靠恢复，因此 +修正内部生命周期，而不删除重启测试或更改公开方法。 + +子代理独立复核了 [Web Audio close](https://www.w3.org/TR/webaudio-1.0/#dom-audiocontext-close)、 +[媒体源节点](https://www.w3.org/TR/webaudio-1.0/#MediaElementAudioSourceNode)及 +[捕获流切源](https://www.w3.org/TR/mediacapture-fromelement/#html-media-element-media-capture-extensions)。 +保留 direct-first，避免把采样语义和兼容范围改成尚未全面支持的 capture-first。 +旧版 close 行为在 ASR-02 历史测试中保留；候选测试要求停止后有播放路由、同一 +context 恢复非零 PCM，最终销毁才关闭。初始化尚未绑定 video 时仍立即完整关闭。 + +## 验证与剩余范围 + +源代码分包 strict/noUncheckedIndexedAccess、定向 ESLint、正常 main/legacy/ESM +构建通过。历史/纯模块/候选共 210 项，真实 ESM 与副本校验另 2 项；main 与 legacy +各重跑 25 项候选生命周期。浏览器源码全组 12 通过/6 Windows WebKit 能力跳过， +main/legacy 候选各 4 通过/2 能力跳过；没有 failed/flaky。 +具体输入哈希、报告、构建与浏览器版本见 [验证记录](../baselines/asr-ownership-validation.json)。 + +浏览器实际验证：两核心、非零 PCM/WAV、本地回调字幕、暂停恢复、同视频切源、 +stop 后恢复及最终 context 关闭。Windows WebKit 无 WebAudio，不能代表 macOS Safari； +真实 fallback 音轨拓扑、音量/静音、跨源媒体、设备仍归 ASR-05。ASR-04 的公共类型和 +实际包安装、ASR-06 的示例及发行文档仍未完成，本记录不表示整包已可发布。 + +仅运行本包、类型、构建和台账检查，未重复整仓全量 CI。两次子代理承担纯音频模块/ +回归与规范审查，主代理负责图与控制器集成、浏览器、文档及提交。每项职责已核对。 +回退本任务提交可恢复旧源码及构建产物，但会同时恢复已复现丢样本和生命周期缺陷。 diff --git a/refactor/plan.md b/refactor/plan.md index 9efe68b5b..c7ee6c3b6 100644 --- a/refactor/plan.md +++ b/refactor/plan.md @@ -4,7 +4,7 @@ 基线：`40fcda6a37d0049d42e49c1e64e70d4fd9ba5f7f`。总任务 225 项，范围 22 个包及工作区/示例。 -状态：todo 84 / doing 16 / blocked 0 / done 125 / deferred 0。风险 L/M/H 表示兼容风险，不表示工期。 +状态：todo 83 / doing 16 / blocked 0 / done 126 / deferred 0。风险 L/M/H 表示兼容风险，不表示工期。 前置依赖是启动条件；验收是完成条件。任务可以继续拆分，但不能复用或悄悄删除旧 ID。 @@ -253,7 +253,7 @@ | --- | --- | --- | --- | --- | --- | --- | | PKG-ASR-01 | artplayer-plugin-asr
核对包契约与历史用法 | BASE-05 | length/interval/sampleRate/onAudioChunk、append/hide/stop 的真实能力 | 源码/声明/README/demo/发布包差异已登记；公开形状和版本范围冻结 | H | done | | PKG-ASR-02 | artplayer-plugin-asr
建立特有行为与错误测试 | PKG-ASR-01, ENG-03, ENG-05 | PCM/WAV、chunk 时序、回调慢/拒绝、重复初始化与停止 | 旧版本行为可重跑，成功/失败/切源/销毁有必要断言 | H | done | -| PKG-ASR-03 | artplayer-plugin-asr
整理内部职责与资源 | PKG-ASR-02, CORE-10, CORE-18 | AudioContext/Worklet/Stream 生命周期和背压分离 | 结构变化和缺陷修复分开记录；原 API/事件/资源生命周期通过 | H | todo | +| PKG-ASR-03 | artplayer-plugin-asr
整理内部职责与资源 | PKG-ASR-02, CORE-10, CORE-18 | AudioContext/Worklet/Stream 生命周期和背压分离 | 结构变化和缺陷修复分开记录；原 API/事件/资源生命周期通过 | H | done | | PKG-ASR-04 | artplayer-plugin-asr
迁移自有源码和公开类型 | PKG-ASR-03, ENG-04, ENG-06, CORE-07 | 主线程/Worklet/音频 buffer 与异步回调类型 | 严格类型检查、旧消费样例通过；声明路径/导出和同步异步兼容 | H | todo | | PKG-ASR-05 | artplayer-plugin-asr
验证新旧核心和组合 | PKG-ASR-04, CORE-22 | 真实 WebAudio 分块、播放暂停/切源/销毁，不引入网络 ASR | 最终核心与原支持范围核心分别通过；设备/SDK 缺证据不能标完成 | H | todo | | PKG-ASR-06 | artplayer-plugin-asr
验证分发并同步文档 | PKG-ASR-05, ENG-07 | asr.js、Worklet 资源、输出格式与声明证据 | tarball 入口/资源、类型、8082 demo 和 README 一致，有回退记录 | H | todo | @@ -528,6 +528,7 @@ - PKG-JASSUB-01: [记录](baselines/jassub-release.json) [记录](baselines/jassub-vendor.json) [记录](baselines/jassub-font-metadata.json) [记录](baselines/jassub-contract.md) [记录](baselines/jassub-contract-validation.json) [记录](changes/2026-09-13-PKG-JASSUB-01-baseline.md) - PKG-ASR-01: [记录](changes/2026-09-13-PKG-ASR-01-contracts.md) [记录](baselines/asr-contract.md) [记录](baselines/asr-release.json) - PKG-ASR-02: [记录](changes/2026-09-13-PKG-ASR-02-audio-baseline.md) [记录](baselines/asr-audio-validation.json) +- PKG-ASR-03: [记录](changes/2026-09-13-PKG-ASR-03-audio-ownership.md) [记录](baselines/asr-ownership-validation.json) - PKG-ADS-01: [记录](baselines/ads-release.json) [记录](baselines/ads-contract.md) [记录](baselines/ads-contract-validation.json) [记录](changes/2026-09-12-PKG-ADS-01-contract.md) [记录](scripts/ads-contract.test.mjs) - PKG-ADS-02: [记录](changes/2026-09-12-PKG-ADS-02-tests.md) [记录](ads-validation.md) [记录](baselines/ads-validation.json) - PKG-ADS-03: [记录](changes/2026-09-12-PKG-ADS-03-lifecycle.md) [记录](ads-validation.md) [记录](baselines/ads-lifecycle-validation.json) diff --git a/refactor/progress.md b/refactor/progress.md index 6d2f2d278..aaa999a03 100644 --- a/refactor/progress.md +++ b/refactor/progress.md @@ -1,5 +1,20 @@ # 进度与证据 +## PKG-ASR-03 TS 拆分与音频生命周期完成 + +生产源码拆为 8 个严格 TS 模块；队列保留半块/尾部，识别回调背压、拒绝恢复、 +初始化去重和代际清理完成。退休 Worklet 消息反例由失败转通过。 +Chromium stop 后不能恢复的原生问题已修复：直接媒体路由由播放器持有到 destroy， +stop 只停 ASR 并允许重新播放；先准备 Worklet 后绑定视频，失败可真正释放未绑定 context。 +定向 212 项通过，main/legacy 各重跑 25 候选项；严格分包 tsc、lint、正常三格式构建通过。 +最终浏览器源码 12 过/6 能力跳过，main/legacy 各 4 过/2 能力跳过；共 20 过、10 明确 +Windows WebKit 无 WebAudio 跳过。非零 PCM、暂停、切源、stop 重启与 destroy 已实测。 +见[变更](changes/2026-09-13-PKG-ASR-03-audio-ownership.md)和[证据](baselines/asr-ownership-validation.json)。 +225 项：126 done、16 doing、83 todo；205 风险，两项 ASR 音频风险闭环。 +下一步 ASR-04 公共异步类型与安装消费者；完整设备/fallback/音量和发行仍未完成。 +没有重复全量 CI、没有推送或发布。子代理承担独立模块/测试和规范审查，主代理集成验收。 + + ## PKG-ASR-02 音频与错误基线完成 子代理交付并由主代理审查的 133 项音频历史测试通过，连同公开契约/来源共 165 项。 diff --git a/refactor/risk-table.md b/refactor/risk-table.md index 51d9d0679..da7acd235 100644 --- a/refactor/risk-table.md +++ b/refactor/risk-table.md @@ -207,5 +207,5 @@ | CORE-DEFAULTS-SSR-01 | resolved / 已复现 | Reading static defaults throws when navigator is absent | CORE-25 | | ENG-DIST-PUBLIC-01 | resolved / 已复现 | Vite copied authored declarations into library distribution | ENG-12 | | ASR-TYPE-01 | open / 源码/产物事实 | ASR declarations omit asynchronous stop and callback text results | PKG-ASR-04 | -| ASR-CHUNK-01 | open / 已复现 | ASR tick discards underfilled chunks and overrun samples in source | PKG-ASR-02, PKG-ASR-03 | -| ASR-LIFE-01 | open / 已复现 | ASR partial initialization, pause resume and stale callback cleanup need controlled evidence | PKG-ASR-02, PKG-ASR-03, PKG-ASR-05 | +| ASR-CHUNK-01 | resolved / 已复现 | ASR tick discards underfilled chunks and overrun samples in source | PKG-ASR-02, PKG-ASR-03 | +| ASR-LIFE-01 | resolved / 已复现 | ASR partial initialization, pause resume and stale callback cleanup need controlled evidence | PKG-ASR-02, PKG-ASR-03, PKG-ASR-05 | diff --git a/refactor/risks.json b/refactor/risks.json index b450d3ec4..e11ca9a0d 100644 --- a/refactor/risks.json +++ b/refactor/risks.json @@ -798,7 +798,7 @@ "EX-03" ], "evidence": [ - "packages/artplayer-plugin-asr/src/index.js", + "packages/artplayer-plugin-asr/src/index.ts", "docs/assets/example/asr.js" ], "compatibleResolution": "Core plugin supplies PCM to caller onAudioChunk. Example fetches an external ASR endpoint and opens its returned WebSocket; not an embedded ASR SDK or a verified service.", @@ -4561,7 +4561,7 @@ "id": "ASR-CHUNK-01", "title": "ASR tick discards underfilled chunks and overrun samples in source", "confirmation": "reproduced", - "status": "open", + "status": "resolved", "owners": [ "PKG-ASR-02", "PKG-ASR-03" @@ -4574,13 +4574,20 @@ "refactor/baselines/asr-audio-validation.json" ], "compatibleResolution": "Reproduce exact sample loss and retain queue remainders with bounded, ordered callback delivery.", - "closureCriteria": "Exact PCM/WAV byte sequence and slow/rejected callback regression tests pass without losing or reordering accepted input." + "closureCriteria": "Exact PCM/WAV byte sequence and slow/rejected callback regression tests pass without losing or reordering accepted input.", + "resolutionEvidence": [ + "refactor/changes/2026-09-13-PKG-ASR-03-audio-ownership.md", + "refactor/baselines/asr-ownership-validation.json", + "test/asr-lifecycle.test.js", + "test/asr-encoding.test.js" + ], + "resolutionRationale": "Copied linked blocks retain underfilled input and tails; exact byte parity and FIFO tests pass. One callback per generation prevents overlap; rejection recovery and obsolete completions are covered. Exceeding the larger of 60 seconds or two configured chunks pauses capture with an explicit error; long configured intervals remain collectable." }, { "id": "ASR-LIFE-01", "title": "ASR partial initialization, pause resume and stale callback cleanup need controlled evidence", "confirmation": "reproduced", - "status": "open", + "status": "resolved", "owners": [ "PKG-ASR-02", "PKG-ASR-03", @@ -4594,7 +4601,14 @@ "refactor/baselines/asr-audio-validation.json" ], "compatibleResolution": "Own pending setup, URLs, source nodes and callbacks across pause, stop, restart and terminal destruction.", - "closureCriteria": "Failure/race regression tests plus native WebAudio playback and cleanup evidence prove resource ownership." + "closureCriteria": "Failure/race regression tests plus native WebAudio playback and cleanup evidence prove resource ownership.", + "resolutionEvidence": [ + "refactor/changes/2026-09-13-PKG-ASR-03-audio-ownership.md", + "refactor/baselines/asr-ownership-validation.json", + "test/asr-lifecycle.test.js", + "test/asr-encoding.test.js" + ], + "resolutionRationale": "Candidate ownership/race regression tests pass, including retired-port delivery and module failure before binding video. Actual Chromium/Firefox source/main/legacy with both cores verify pause, source change, explicit stop/restart and final close. Direct media contexts deliberately remain player-owned until destroy; disposable stream contexts close on stop. General native fallback/volume/device acceptance remains ASR-05 and SDK-11, not claimed by this specific fix." } ] } diff --git a/refactor/tasks.json b/refactor/tasks.json index b468ecf9e..0b70fd2f7 100644 --- a/refactor/tasks.json +++ b/refactor/tasks.json @@ -2660,11 +2660,14 @@ "CORE-10", "CORE-18" ], - "status": "todo", + "status": "done", "risk": "H", "deliverable": "AudioContext/Worklet/Stream 生命周期和背压分离", "acceptance": "结构变化和缺陷修复分开记录；原 API/事件/资源生命周期通过", - "evidence": [] + "evidence": [ + "changes/2026-09-13-PKG-ASR-03-audio-ownership.md", + "baselines/asr-ownership-validation.json" + ] }, { "id": "PKG-ASR-04", diff --git a/refactor/third-party.json b/refactor/third-party.json index b18cb202d..f6c1eaede 100644 --- a/refactor/third-party.json +++ b/refactor/third-party.json @@ -1240,7 +1240,7 @@ "EX-03" ], "sourceFiles": [ - "packages/artplayer-plugin-asr/src/index.js", + "packages/artplayer-plugin-asr/src/index.ts", "docs/assets/example/asr.js" ], "contract": "Core plugin supplies PCM to caller onAudioChunk. Example fetches an external ASR endpoint and opens its returned WebSocket; not an embedded ASR SDK or a verified service.", diff --git a/test/asr-distribution.test.js b/test/asr-distribution.test.js new file mode 100644 index 000000000..07cfb082b --- /dev/null +++ b/test/asr-distribution.test.js @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +// eslint-disable-next-line test/no-import-node-test -- Actual ASR distribution imports use the repository runner. +import test from 'node:test' + +test('ASR built ESM imports without DOM and keeps both factory paths and synchronous registration', async () => { + const module = await import('../packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.mjs') + assert.deepEqual(Object.keys(module), ['default']) + assert.equal(module.default.default, module.default) + assert.deepEqual(Object.keys(module.default), []) + const events = new Map() + const layer = { style: {}, innerHTML: '' } + const result = module.default({ length: 1 })({ + video: {}, + volume: 0.5, + layers: { add: () => layer }, + on: (name, callback) => events.set(name, callback), + off: name => events.delete(name), + }) + assert.equal(result.then, undefined) + assert.deepEqual(Object.keys(result), ['name', 'stop', 'hide', 'append']) + result.append('One. Two') + assert.equal(layer.innerHTML, '
Two
') + await result.stop() + await events.get('destroy')() + assert.equal(events.size, 0) +}) + +test('ASR three generated package artifacts exactly match the demo copies', () => { + for (const suffix of ['js', 'legacy.js', 'mjs']) { + const name = `artplayer-plugin-asr.${suffix}` + assert.deepEqual(fs.readFileSync(`packages/artplayer-plugin-asr/dist/${name}`), fs.readFileSync(`docs/compiled/${name}`)) + } +}) diff --git a/test/asr-encoding.test.js b/test/asr-encoding.test.js new file mode 100644 index 000000000..2408fc8f2 --- /dev/null +++ b/test/asr-encoding.test.js @@ -0,0 +1,183 @@ +import assert from 'node:assert/strict' +import { Buffer } from 'node:buffer' +// eslint-disable-next-line test/no-import-node-test -- Pure ASR audio modules use the repository runner. +import test from 'node:test' +import { asrAudioEnvironment, runAsrWorklet } from './helpers/asr-audio.js' +import { asrHistorical } from './helpers/asr.js' +import { loadModules } from './helpers/load.js' + +const root = 'packages/artplayer-plugin-asr/src' +const { encodeAudio, SampleQueue, recorderProcessorCode } = await loadModules({ + encodeAudio: { file: `${root}/encoding`, name: 'encodeAudio' }, + SampleQueue: { file: `${root}/sample-queue`, name: 'SampleQueue' }, + recorderProcessorCode: { file: `${root}/worklet`, name: 'recorderProcessorCode' }, +}) + +test('ASR encoder preserves signed PCM16 clamping, truncation, NaN and little endian bytes', () => { + const input = Float32Array.of(-Infinity, -2, -1, -0.5, -0, 0, 0.5, 1, 2, Infinity, Number.NaN) + const encoded = encodeAudio(input, 16000) + assert.deepEqual(Object.keys(encoded), ['pcm', 'wav']) + const expected = [0, 128, 0, 128, 0, 128, 0, 192, 0, 0, 0, 0, 255, 63, 255, 127, 255, 127, 255, 127, 0, 0] + assert.deepEqual([...new Uint8Array(encoded.pcm)], expected) + assert.equal(input[0], -Infinity) + assert.equal(Number.isNaN(input.at(-1)), true) +}) + +for (const sampleRate of [8000, 16000, 44100, 48000]) { + test(`ASR encoder writes exact mono WAV metadata at ${sampleRate} Hz`, () => { + const { pcm, wav } = encodeAudio(Float32Array.of(-1, 0, 1), sampleRate) + const data = Buffer.from(wav) + assert.equal(pcm.byteLength, 6) + assert.equal(data.length, 50) + assert.equal(data.toString('ascii', 0, 4), 'RIFF') + assert.equal(data.readUInt32LE(4), 42) + assert.equal(data.toString('ascii', 8, 16), 'WAVEfmt ') + assert.equal(data.readUInt32LE(16), 16) + assert.equal(data.readUInt16LE(20), 1) + assert.equal(data.readUInt16LE(22), 1) + assert.equal(data.readUInt32LE(24), sampleRate) + assert.equal(data.readUInt32LE(28), sampleRate * 2) + assert.equal(data.readUInt16LE(32), 2) + assert.equal(data.readUInt16LE(34), 16) + assert.equal(data.toString('ascii', 36, 40), 'data') + assert.equal(data.readUInt32LE(40), 6) + assert.deepEqual([...data.subarray(44)], [0, 128, 0, 0, 255, 127]) + new Uint8Array(pcm).fill(0) + assert.deepEqual([...data.subarray(44)], [0, 128, 0, 0, 255, 127], 'The WAV payload owns its bytes') + }) +} + +test('ASR encoder empty and offset views retain exact payload boundaries', () => { + const empty = encodeAudio(new Float32Array(0), 16000) + assert.equal(empty.pcm.byteLength, 0) + assert.equal(empty.wav.byteLength, 44) + assert.equal(new DataView(empty.wav).getUint32(4, true), 36) + assert.equal(new DataView(empty.wav).getUint32(40, true), 0) + const backing = Float32Array.of(1, -1, 0.5, 1) + const { pcm } = encodeAudio(backing.subarray(1, 3), 16000) + assert.deepEqual([...new Uint8Array(pcm)], [0, 128, 255, 63]) +}) + +for (const implementation of await asrHistorical()) { + test(`ASR encoder byte parity with ${implementation.name}`, async () => { + const chunks = [] + const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: chunk => chunks.push(chunk) }) + await environment.emit('play') + const input = Float32Array.of(-Infinity, -1, -0.5, Number.NaN, 0, 0.5, 1, Infinity) + environment.send(input) + await environment.tick() + assert.equal(chunks.length, 1) + const candidate = encodeAudio(input, 8000) + assert.deepEqual(Buffer.from(candidate.pcm), Buffer.from(chunks[0].pcm)) + assert.deepEqual(Buffer.from(candidate.wav), Buffer.from(chunks[0].wav)) + await environment.plugin.stop() + }) +} + +test('ASR sample queue retains insufficient chunks and excess tails', () => { + const queue = new SampleQueue() + assert.equal(queue.take(8), null) + queue.push(Float32Array.of(1, 2, 3, 4)) + assert.equal(queue.take(8), null) + assert.equal(queue.length, 4) + queue.push(Float32Array.of(5, 6, 7, 8, 9, 10)) + assert.deepEqual([...queue.take(8)], [1, 2, 3, 4, 5, 6, 7, 8]) + assert.equal(queue.length, 2) + assert.equal(queue.take(3), null) + queue.push(Float32Array.of(11)) + assert.deepEqual([...queue.take(3)], [9, 10, 11]) + assert.equal(queue.length, 0) + assert.equal(queue.take(1), null) +}) + +test('ASR sample queue copies input views and returns independently owned output', () => { + const queue = new SampleQueue() + const input = Float32Array.of(0, 1, 2, 3) + queue.push(input.subarray(1, 3)) + input.fill(9) + const first = queue.take(1) + assert.deepEqual([...first], [1]) + first[0] = 8 + assert.deepEqual([...queue.take(1)], [2]) +}) + +test('ASR sample queue clears partial blocks and can be reused after draining', () => { + const queue = new SampleQueue() + queue.push(Float32Array.of(1, 2, 3)) + assert.deepEqual([...queue.take(1)], [1]) + queue.clear() + queue.clear() + queue.push(new Float32Array(0)) + assert.equal(queue.length, 0) + assert.equal(queue.take(1), null) + queue.push(Float32Array.of(4, 5)) + assert.deepEqual([...queue.take(2)], [4, 5]) + queue.push(Float32Array.of(6)) + assert.deepEqual([...queue.take(1)], [6]) +}) + +test('ASR sample queue rejects invalid count without losing buffered audio', () => { + const queue = new SampleQueue() + queue.push(Float32Array.of(1, 2)) + for (const count of [0, -1, 0.5, Number.NaN, Infinity, -Infinity, Number.MAX_SAFE_INTEGER + 1]) { + assert.throws(() => queue.take(count), { name: 'RangeError', message: 'Sample count must be a positive safe integer' }) + assert.equal(queue.length, 2) + } + assert.equal(queue.take(Number.MAX_SAFE_INTEGER), null) + assert.deepEqual([...queue.take(2)], [1, 2]) +}) + +test('ASR sample queue handles many worklet blocks without dropping boundary samples', () => { + const queue = new SampleQueue() + for (let index = 0; index < 4096; index++) + queue.push(Float32Array.of(index, -index)) + assert.equal(queue.length, 8192) + const first = queue.take(4097) + const second = queue.take(4095) + assert.equal(queue.length, 0) + const output = [...first, ...second] + for (let index = 0; index < 4096; index++) { + assert.equal(output[index * 2], index) + assert.equal(output[index * 2 + 1], -index) + } +}) + +test('ASR sample queue preserves FIFO through deterministic mixed push, take and clear operations', () => { + const queue = new SampleQueue() + let expected = [] + let serial = 0 + for (let index = 1; index <= 1000; index++) { + if (index % 113 === 0) { + queue.clear() + expected = [] + } + else if (index % 3 !== 0) { + const chunk = Float32Array.from({ length: index % 17 }, () => ++serial) + queue.push(chunk) + expected.push(...chunk) + } + else { + const count = index % 23 + 1 + const actual = queue.take(count) + if (expected.length < count) + assert.equal(actual, null) + else + assert.deepEqual([...actual], expected.splice(0, count)) + } + assert.equal(queue.length, expected.length) + } + if (expected.length) + assert.deepEqual([...queue.take(expected.length)], expected) +}) + +test('ASR candidate Worklet keeps registration, first-channel identity and empty-input behavior', async () => { + const { processor, messages } = await runAsrWorklet(new Blob([recorderProcessorCode])) + const first = Float32Array.of(-0.5, 0.5) + const second = Float32Array.of(1, 1) + assert.equal(processor.process([]), true) + assert.equal(processor.process([[]]), true) + assert.equal(messages.length, 0) + assert.equal(processor.process([[first, second]]), true) + assert.equal(messages.length, 1) + assert.equal(messages[0], first) +}) diff --git a/test/asr-lifecycle.test.js b/test/asr-lifecycle.test.js new file mode 100644 index 000000000..ab77568b8 --- /dev/null +++ b/test/asr-lifecycle.test.js @@ -0,0 +1,434 @@ +import assert from 'node:assert/strict' +import { Buffer } from 'node:buffer' +import fs from 'node:fs' +import process from 'node:process' +// eslint-disable-next-line test/no-import-node-test -- Candidate ASR lifecycle uses the repository runner. +import test from 'node:test' +import { asrAudioEnvironment, deferredAudio } from './helpers/asr-audio.js' +import { asrEnvironment } from './helpers/asr.js' +import { compilePackage } from './helpers/load.js' + +const artifact = process.env.ARTPLAYER_ASR_ARTIFACT +const implementation = { + name: artifact || 'candidate-asr', + code: artifact ? fs.readFileSync(artifact, 'utf8') : await compilePackage('artplayer-plugin-asr', 'umd'), +} +const block = (value = 0, length = 8) => new Float32Array(length).fill(value) + +function create(option = {}, capabilities = {}) { + return asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, ...option }, capabilities) +} + +function pcmSamples(chunk) { + const bytes = new DataView(chunk.pcm) + return Array.from({ length: chunk.pcm.byteLength / 2 }, (_, index) => bytes.getInt16(index * 2, true)) +} + +test('ASR candidate lazy factory and synchronous registration keep result keys and Promise stop', async () => { + const environment = asrEnvironment(implementation) + assert.equal(environment.exported.default, environment.exported, 'Both historical CJS factory paths remain callable') + assert.deepEqual(Object.keys(environment.exported), []) + const option = { length: 1, autoHideTimeout: 321 } + const register = environment.factory(option) + assert.equal(typeof register, 'function') + assert.equal(environment.layers.length, 0) + assert.equal(environment.listeners.size, 0) + option.length = 3 + option.autoHideTimeout = 999 + const result = register(environment.art) + assert.equal(result.then, undefined) + assert.deepEqual(Object.keys(result), ['name', 'stop', 'hide', 'append']) + assert.equal(result.name, 'artplayerPluginAsr') + assert.equal(environment.layers.length, 1) + assert.equal(environment.layers[0].name, 'asr') + assert.equal(environment.layers[0].html, '') + result.append('One. Two! Three') + assert.equal(environment.layer.innerHTML, '
Three
') + assert.equal([...environment.timeouts.values()][0].delay, 321, 'Factory snapshots its configuration') + const stopping = result.stop() + assert.equal(typeof stopping.then, 'function') + assert.equal(await stopping, undefined) + assert.equal(environment.timeouts.size, 1, 'A displayed subtitle keeps its existing auto-hide deadline after stop') + ;[...environment.timeouts.values()][0].callback() + assert.equal(environment.layer.style.display, 'none') + await environment.emit('destroy') + assert.equal(environment.timeouts.size, 0) +}) + +test('ASR candidate append keeps punctuation, HTML, ignored non-string values and timer replacement', async () => { + const environment = create({ length: 2, autoHideTimeout: 432 }) + assert.equal(environment.plugin.append('First. Second! Third?'), undefined) + assert.equal(environment.layer.innerHTML, '
Second!
Third?
') + const firstTimer = [...environment.timeouts.keys()][0] + environment.plugin.append(42) + assert.equal([...environment.timeouts.keys()][0], firstTimer) + environment.plugin.append('Raw') + assert.equal(environment.layer.innerHTML, '
Raw
') + assert.equal(environment.timeouts.size, 1) + assert.notEqual([...environment.timeouts.keys()][0], firstTimer) + const timer = [...environment.timeouts.values()][0] + assert.equal(timer.delay, 432) + timer.callback() + assert.equal(environment.layer.style.display, 'none') + environment.plugin.append('Visible.') + assert.equal(environment.layer.style.display, '') + assert.equal(environment.plugin.hide(), undefined) + assert.equal(environment.layer.style.display, 'none') + assert.equal(environment.layer.innerHTML, '
Visible.
') + await environment.plugin.stop() +}) + +test('ASR candidate preserves chunk payload, exact PCM/WAV bytes and callback subtitles', async () => { + let chunk + const environment = create({ onAudioChunk(value) { + chunk = value + return 'One. Two!' + } }) + await environment.emit('play') + environment.send([-Infinity, -1, -0.5, Number.NaN, 0, 0.5, 1, Infinity]) + await environment.tick() + assert.deepEqual(Object.keys(chunk), ['pcm', 'wav']) + assert.deepEqual(pcmSamples(chunk), [-32768, -32768, -16384, 0, 0, 16383, 32767, 32767]) + const wav = Buffer.from(chunk.wav) + assert.equal(wav.byteLength, 60) + assert.equal(wav.toString('ascii', 0, 4), 'RIFF') + assert.equal(wav.readUInt32LE(4), 52) + assert.equal(wav.readUInt16LE(22), 1) + assert.equal(wav.readUInt32LE(24), 8000) + assert.equal(wav.readUInt32LE(28), 16000) + assert.equal(wav.readUInt32LE(40), 16) + assert.deepEqual(wav.subarray(44), Buffer.from(chunk.pcm)) + assert.equal(environment.layer.innerHTML, '
One.
Two!
') + await environment.plugin.stop() +}) + +test('ASR candidate preserves short chunks across ticks and excess tail in FIFO order', async () => { + const chunks = [] + const environment = create({ onAudioChunk: value => chunks.push(value) }) + await environment.emit('play') + environment.send(block(0.5, 4)) + await environment.tick() + assert.equal(chunks.length, 0) + environment.send(block(-0.5, 8)) + await environment.tick() + assert.deepEqual(pcmSamples(chunks[0]), [16383, 16383, 16383, 16383, -16384, -16384, -16384, -16384]) + await environment.tick() + assert.equal(chunks.length, 1) + environment.send(block(1, 4)) + await environment.tick() + assert.deepEqual(pcmSamples(chunks[1]), [-16384, -16384, -16384, -16384, 32767, 32767, 32767, 32767]) + await environment.plugin.stop() +}) + +test('ASR candidate allows one in-flight recognizer and retains queued audio', async () => { + const pending = deferredAudio() + const calls = [] + const environment = create({ onAudioChunk: (chunk) => { + calls.push(chunk) + return calls.length === 1 ? pending.promise : 'Second.' + } }) + await environment.emit('play') + environment.send(block(0.5)) + const first = environment.tick() + environment.send(block(-0.5)) + await environment.tick() + await environment.tick() + assert.equal(calls.length, 1) + pending.resolve('First.') + await first + assert.equal(environment.layer.innerHTML, '
First.
') + await environment.tick() + assert.equal(calls.length, 2) + assert.deepEqual(pcmSamples(calls[1]), Array.from({ length: 8 }, () => -16384)) + assert.equal(environment.layer.innerHTML, '
Second.
') + await environment.plugin.stop() +}) + +for (const mode of ['throw', 'reject']) { + test(`ASR candidate handles callback ${mode} and continues subsequent audio`, async () => { + const failure = new Error('recognition failed') + let count = 0 + const environment = create({ onAudioChunk: () => { + if (++count > 1) + return 'Recovered.' + if (mode === 'throw') + throw failure + return Promise.reject(failure) + } }) + await environment.emit('play') + environment.send(block()) + await assert.doesNotReject(environment.tick()) + assert.equal(environment.errors.length, 1) + assert.equal(environment.errors[0][1], failure) + environment.send(block()) + await environment.tick() + assert.equal(count, 2) + assert.equal(environment.layer.innerHTML, '
Recovered.
') + await environment.plugin.stop() + }) +} + +for (const event of ['pause', 'stop', 'destroy', 'restart']) { + test(`ASR candidate invalidates pending callback after ${event}`, async () => { + const pending = deferredAudio() + const environment = create({ onAudioChunk: () => pending.promise }) + await environment.emit('play') + environment.send(block()) + const recognizing = environment.tick() + await (event === 'stop' ? environment.plugin.stop() : environment.emit(event)) + pending.resolve('Obsolete.') + await recognizing + assert.equal(environment.layer.innerHTML, '') + assert.equal(environment.timeouts.size, 0) + assert.equal(environment.errors.length, 0) + assert.equal(environment.intervals.size, event === 'restart' ? 1 : 0) + await environment.plugin.stop() + }) +} + +test('ASR candidate reset clears old queued audio without dropping the next source', async () => { + const calls = [] + const environment = create({ onAudioChunk: value => calls.push(value) }) + await environment.emit('play') + environment.send(block(0.5)) + environment.art.video.src = 'local-new-source.mp4' + await environment.emit('restart') + await environment.tick() + assert.equal(calls.length, 0) + environment.send(block(-0.5)) + await environment.tick() + assert.deepEqual(pcmSamples(calls[0]), Array.from({ length: 8 }, () => -16384)) + await environment.plugin.stop() +}) + +test('ASR candidate old callback completion cannot unlock a new session pending callback', async () => { + const pending = [deferredAudio(), deferredAudio()] + let calls = 0 + const environment = create({ onAudioChunk: () => pending[calls++].promise }) + await environment.emit('play') + environment.send(block()) + const old = environment.tick() + await environment.emit('pause') + await environment.emit('play') + environment.send(block()) + const current = environment.tick() + pending[0].resolve('Old.') + await old + environment.send(block()) + await environment.tick() + assert.equal(calls, 2) + assert.equal(environment.layer.innerHTML, '') + pending[1].resolve('Current.') + await current + assert.equal(environment.layer.innerHTML, '
Current.
') + await environment.plugin.stop() +}) + +test('ASR candidate concurrent play creates one graph and one interval', async () => { + const moduleGate = deferredAudio() + const moduleEntered = deferredAudio() + const environment = create({}, { moduleGate, moduleEntered }) + const starting = environment.emit('play') + const duplicate = environment.emit('play') + await moduleEntered.promise + assert.equal(environment.modules.length, 1) + moduleGate.resolve() + await Promise.all([starting, duplicate]) + await environment.emit('play') + assert.equal(environment.contexts.length, 1) + assert.equal(environment.recorders.length, 1) + assert.equal(environment.intervals.size, 1) + await environment.plugin.stop() + assert.equal(environment.intervals.size, 0) + assert.equal(environment.recorders[0].port.onmessage, null) + assert.equal(environment.recorders[0].disconnects, 1) +}) + +test('ASR candidate pause during module loading prevents late connection and later resumes prepared graph', async () => { + const moduleGate = deferredAudio() + const moduleEntered = deferredAudio() + const environment = create({}, { moduleGate, moduleEntered, capture: false }) + const starting = environment.emit('play') + await moduleEntered.promise + await environment.emit('pause') + moduleGate.resolve() + await starting + assert.equal(environment.intervals.size, 0) + assert.equal(environment.recorders.length, 0) + assert.equal(environment.contexts[0].closes, 0) + await environment.emit('play') + assert.equal(environment.contexts.length, 1) + assert.equal(environment.modules.length, 1) + assert.equal(environment.recorders.length, 1) + assert.equal(environment.intervals.size, 1) + assert.equal(environment.captures(), 0) + assert.equal(environment.errors.length, 0) + await environment.plugin.stop() +}) + +for (const event of ['stop', 'destroy']) { + test(`ASR candidate ${event} during module loading frees URL and context without resurrection`, async () => { + const moduleGate = deferredAudio() + const moduleEntered = deferredAudio() + const environment = create({}, { moduleGate, moduleEntered }) + const starting = environment.emit('play') + await moduleEntered.promise + assert.equal(environment.blobs.size, 1) + await (event === 'stop' ? environment.plugin.stop() : environment.emit(event)) + assert.equal(environment.blobs.size, 0) + assert.equal(environment.contexts[0].closes, 1) + moduleGate.resolve() + await starting + assert.equal(environment.revoked.length, 1) + assert.equal(environment.intervals.size, 0) + assert.equal(environment.recorders.length, 0) + assert.equal(environment.errors.length, 0) + }) +} + +test('ASR candidate failed module initialization releases context and URL before binding video', async () => { + const failure = new Error('failed worklet') + const environment = create({}, { moduleError: failure }) + await environment.emit('play') + assert.equal(environment.errors.length, 1) + assert.equal(environment.errors[0][1], failure) + assert.equal(environment.contexts[0].closes, 1) + assert.equal(environment.blobs.size, 0) + assert.equal(environment.revoked.length, 1) + assert.equal(environment.nodes.filter(node => node.kind === 'element').length, 0) + assert.equal(environment.intervals.size, 0) + await environment.plugin.stop() + assert.equal(environment.contexts[0].closes, 1) +}) + +test('ASR candidate pause/resume reuses direct source without captureStream', async () => { + const environment = create({}, { capture: false }) + await environment.emit('play') + const source = environment.nodes.find(node => node.kind === 'element') + await environment.emit('pause') + assert.equal(source.connections.length, 1, 'The player-owned audio route remains available while ASR is paused') + assert.equal(source.connections[0].kind, 'gain') + assert.equal(source.connections[0].connections[0], environment.contexts[0].destination) + assert.equal(environment.recorders[0].port.onmessage, null) + await environment.emit('play') + assert.equal(environment.contexts.length, 1) + assert.equal(environment.modules.length, 1) + assert.equal(environment.nodes.filter(node => node.kind === 'element').length, 1) + assert.equal(source.connections.length, 2) + assert.equal(environment.errors.length, 0) + assert.equal(environment.intervals.size, 1) + await environment.plugin.stop() +}) + +test('ASR candidate ignores old worklet delivery after pause/resume reconnect', async () => { + const calls = [] + const environment = create({ onAudioChunk: value => calls.push(value) }) + await environment.emit('play') + const staleMessage = environment.recorders[0].port.onmessage + await environment.emit('pause') + await environment.emit('play') + staleMessage({ data: block(0.5) }) + environment.send(block(-0.5)) + await environment.tick() + assert.equal(calls.length, 1) + assert.deepEqual(pcmSamples(calls[0]), Array.from({ length: 8 }, () => -16384)) + await environment.plugin.stop() +}) + +test('ASR candidate overload explicitly reports and pauses instead of silently discarding samples', async () => { + const environment = create() + await environment.emit('play') + environment.send(block(0, 8000 * 60)) + assert.equal(environment.errors.length, 0) + assert.equal(environment.intervals.size, 1) + environment.send(block(0, 1)) + assert.equal(environment.errors.length, 1) + assert.deepEqual(environment.errors[0], ['[artplayerPluginAsr] Audio callback backlog exceeded its capture limit']) + assert.equal(environment.intervals.size, 0) + assert.equal(environment.recorders[0].port.onmessage, null) + await environment.plugin.stop() + await environment.emit('destroy') + const longChunk = create({ interval: 120000 }) + await longChunk.emit('play') + longChunk.send(block(0, 8000 * 60 + 1)) + assert.equal(longChunk.errors.length, 0, 'A configured chunk longer than 60 seconds must still be collectable') + assert.equal(longChunk.intervals.size, 1) + await longChunk.emit('destroy') +}) + +test('ASR candidate destroy cancels subtitle timers, removes subscriptions and forbids public append resurrection', async () => { + const environment = create() + await environment.emit('play') + environment.plugin.append('Kept.') + assert.equal(environment.timeouts.size, 1) + await environment.emit('destroy') + assert.equal(environment.timeouts.size, 0) + assert.equal([...environment.listeners.values()].flat().length, 0) + const html = environment.layer.innerHTML + const display = environment.layer.style.display + environment.plugin.append('Unexpected.') + environment.plugin.hide() + await environment.emit('play') + await environment.emit('restart') + await environment.emit('video:volumechange') + await environment.plugin.stop() + assert.equal(environment.layer.innerHTML, html) + assert.equal(environment.layer.style.display, display) + assert.equal(environment.timeouts.size, 0) + assert.equal(environment.contexts.length, 1) + assert.equal(environment.contexts[0].closes, 1) + assert.equal(environment.intervals.size, 0) +}) + +test('ASR candidate public stop retains the current hide deadline and allows append and subsequent play', async () => { + const environment = create({}, { capture: false }) + await environment.emit('play') + environment.plugin.append('Before.') + const beforeTimer = [...environment.timeouts.keys()][0] + await environment.plugin.stop() + assert.equal(environment.timeouts.size, 1) + assert.equal([...environment.timeouts.keys()][0], beforeTimer) + assert.equal(environment.contexts[0].closes, 0, 'Direct media ownership lasts until player destruction') + const source = environment.nodes.find(node => node.kind === 'element') + assert.equal(source.connections[0].kind, 'gain') + assert.equal(source.connections[0].connections[0], environment.contexts[0].destination) + environment.plugin.append('After.') + assert.equal(environment.layer.innerHTML, '
After.
') + await environment.emit('play') + assert.equal(environment.contexts.length, 1) + assert.equal(environment.captures(), 0) + assert.equal(environment.intervals.size, 1) + await environment.plugin.stop() + await environment.emit('destroy') + assert.equal(environment.contexts[0].closes, 1) + assert.equal(source.connections.length, 0) +}) + +test('ASR candidate volume route survives stop and resume without resetting gain', async () => { + const environment = create({}, { capture: false }) + await environment.emit('play') + environment.art.volume = 0.25 + await environment.emit('video:volumechange') + await environment.plugin.stop() + const source = environment.nodes.find(node => node.kind === 'element') + assert.equal(source.connections[0].gain.value, 0.25) + await environment.emit('play') + assert.equal(source.connections.find(node => node.kind === 'gain').gain.value, 0.25) + await environment.emit('destroy') + assert.equal(environment.contexts[0].closes, 1) +}) + +test('ASR candidate source restart releases captured tracks and obtains a new fallback source', async () => { + const environment = create({}, { directError: new Error('video belongs to another context') }) + await environment.emit('play') + assert.equal(environment.captures(), 1) + await environment.emit('restart') + assert.equal(environment.captures(), 2) + assert.equal(environment.contexts[0].closes, 1) + assert.equal(environment.contexts[1].closes, 0) + assert.equal(environment.tracks[0].stops, 1) + assert.equal(environment.intervals.size, 1) + await environment.plugin.stop() + assert.equal(environment.contexts[1].closes, 1) + assert.equal(environment.tracks[0].stops, 2) +}) diff --git a/test/asr.test.js b/test/asr.test.js index 0cecc29ad..009ce3537 100644 --- a/test/asr.test.js +++ b/test/asr.test.js @@ -30,7 +30,7 @@ for (const implementation of await asrHistorical()) { assert.equal(env.styles.size, 1) }) } - test(`ASR ${implementation.name}: append replaces punctuation lines, permits markup, and hide retains content`, () => { + test(`ASR ${implementation.name}: append replaces punctuation lines, permits markup, and hide retains content`, async () => { const env = asrEnvironment(implementation) const result = env.factory()(env.art) assert.equal(result.append('One. Two! 三。 Four?'), undefined) @@ -47,6 +47,9 @@ for (const implementation of await asrHistorical()) { result.append('') assert.equal(env.layer.innerHTML, '') assert.equal(env.layer.style.display, '') + const timer = [...env.timeouts.keys()][0] + await result.stop() + assert.equal([...env.timeouts.keys()][0], timer, 'Stopping audio preserves an existing subtitle hide deadline') ;[...env.timeouts.values()][0].callback() assert.equal(env.layer.style.display, 'none') }) diff --git a/test/browser/asr-audio.spec.js b/test/browser/asr-audio.spec.js index 8ebde5ec5..829aa37b9 100644 --- a/test/browser/asr-audio.spec.js +++ b/test/browser/asr-audio.spec.js @@ -2,20 +2,23 @@ import fs from 'node:fs' import process from 'node:process' import { verifyAsrContract } from '../../refactor/scripts/asr-contract.mjs' import { hash, readMember } from '../../refactor/scripts/releases.mjs' +import { compilePackage } from '../helpers/load.js' import { expect, test } from './fixtures.js' let contract +let candidateCode test.beforeAll(async () => { contract = await verifyAsrContract() + candidateCode = process.env.ARTPLAYER_ASR_ARTIFACT ? fs.readFileSync(process.env.ARTPLAYER_ASR_ARTIFACT) : await compilePackage('artplayer-plugin-asr', 'umd') }) -for (const version of ['2.0.0', '2.1.0']) { +for (const version of ['2.0.0', '2.1.0', 'candidate']) { for (const core of ['published', 'candidate']) { - test(`ASR ${version} + ${core}: native Worklet supplies local PCM and stop closes AudioContext`, async ({ page }, testInfo) => { + test(`ASR ${version} + ${core}: native Worklet supplies local PCM with owned stop and destroy`, async ({ page }, testInfo) => { const release = [contract.baseline.release, ...contract.baseline.previous].find(release => release.version === version) - const code = readMember(contract.archives.get(version), 'package/dist/artplayer-plugin-asr.js') + const code = version === 'candidate' ? candidateCode : readMember(contract.archives.get(version), 'package/dist/artplayer-plugin-asr.js') const media = fs.readFileSync(new URL('./media/audio-tone.m4a', import.meta.url)) - await testInfo.attach('asr-inputs', { contentType: 'application/json', body: JSON.stringify({ version, core, pluginSha256: hash(code), integrity: release.integrity, mediaSha256: hash(media), media: 'local mono AAC 440 Hz tone', networkService: false }) }) + await testInfo.attach('asr-inputs', { contentType: 'application/json', body: JSON.stringify({ version, core, pluginSha256: hash(code), integrity: release?.integrity || null, candidate: process.env.ARTPLAYER_ASR_ARTIFACT || 'workspace source', mediaSha256: hash(media), media: 'local mono AAC 440 Hz tone', networkService: false }) }) await page.goto(`/test/player.html?core=${core}`) const capabilities = await page.evaluate(() => ({ AudioContext: typeof window.AudioContext, webkitAudioContext: typeof window.webkitAudioContext, AudioWorkletNode: typeof window.AudioWorkletNode })) await testInfo.attach('asr-audio-capabilities', { contentType: 'application/json', body: JSON.stringify(capabilities) }) @@ -47,7 +50,7 @@ for (const version of ['2.0.0', '2.1.0']) { const samples = new Int16Array(pcm) let max = 0 for (const sample of samples) max = Math.max(max, Math.abs(sample)) - window.asrChunks.push({ samples: samples.length, max, wavBytes: wav.byteLength, sampleRate: new DataView(wav).getUint32(24, true) }) + window.asrChunks.push({ samples: samples.length, max, wavBytes: wav.byteLength, sampleRate: new DataView(wav).getUint32(24, true), source: window.art.video.currentSrc }) return 'Local transcript.' }, })], @@ -60,10 +63,36 @@ for (const version of ['2.0.0', '2.1.0']) { const chunks = await page.evaluate(() => window.asrChunks) expect(chunks.every(chunk => chunk.samples === 1600 && chunk.wavBytes === 3244 && chunk.sampleRate === 16000)).toBe(true) await expect(page.locator('.art-layer-asr')).toContainText('Local transcript.') + if (version === 'candidate') { + await page.evaluate(() => window.art.pause()) + const before = await page.evaluate(() => window.asrChunks.length) + await page.waitForTimeout(250) + expect(await page.evaluate(() => window.asrChunks.length)).toBe(before) + await page.locator('#play').click() + await expect.poll(() => page.evaluate(() => window.asrChunks.length)).toBeGreaterThan(before + 2) + expect(await page.evaluate(() => window.asrContexts.length)).toBe(1) + await page.evaluate(() => { + window.art.url = '/test/audio-tone.m4a?asr=second' + }) + await expect.poll(() => page.evaluate(() => window.art.video.currentSrc.includes('asr=second') && window.art.video.readyState >= 2)).toBe(true) + await page.locator('#play').click() + await expect.poll(() => page.evaluate(() => window.asrChunks.filter(chunk => chunk.source.includes('asr=second') && chunk.max > 0).length)).toBeGreaterThanOrEqual(3) + expect(await page.evaluate(() => window.asrContexts.length)).toBe(1) + } await page.evaluate(() => window.art.plugins.artplayerPluginAsr.stop()) - expect(await page.evaluate(() => window.asrContexts.map(context => context.state))).toEqual(['closed']) - await testInfo.attach('asr-native-chunks', { contentType: 'application/json', body: JSON.stringify(chunks) }) + expect(await page.evaluate(() => window.asrContexts.map(context => context.state))).toEqual([version === 'candidate' ? 'running' : 'closed']) + if (version === 'candidate') { + await page.evaluate(() => window.art.pause()) + const before = await page.evaluate(() => window.asrChunks.filter(chunk => chunk.max > 0).length) + await page.locator('#play').click() + await expect.poll(() => page.evaluate(() => window.asrChunks.filter(chunk => chunk.max > 0).length)).toBeGreaterThan(before + 2) + expect(await page.evaluate(() => window.asrContexts.length)).toBe(1) + await page.evaluate(() => window.art.plugins.artplayerPluginAsr.stop()) + expect(await page.evaluate(() => window.asrContexts.map(context => context.state))).toEqual(['running']) + } + await testInfo.attach('asr-native-chunks', { contentType: 'application/json', body: JSON.stringify(await page.evaluate(() => window.asrChunks)) }) await page.evaluate(() => window.art.destroy()) + await expect.poll(() => page.evaluate(() => window.asrContexts.map(context => context.state))).toEqual(['closed']) }) } } diff --git a/test/helpers/asr.js b/test/helpers/asr.js index 965dc69ae..d0851d7c5 100644 --- a/test/helpers/asr.js +++ b/test/helpers/asr.js @@ -32,6 +32,7 @@ export function asrEnvironment(implementation, { script = false } = {}) { return id }, clearTimeout(timer) { timeouts.delete(timer) }, + clearInterval() {}, document: { getElementById: id => styles.get(id), createElement: () => ({}), @@ -57,6 +58,7 @@ export function asrEnvironment(implementation, { script = false } = {}) { return layer } }, on(name, callback) { listeners.set(name, [...listeners.get(name) || [], callback]) }, + off(name, callback) { listeners.set(name, (listeners.get(name) || []).filter(listener => listener !== callback)) }, } return { context, factory, exported, styles, layer, layers, listeners, timeouts, art, reload: () => vm.runInContext(code, context), emit: name => Promise.all((listeners.get(name) || []).map(callback => callback())) } }