mirror of
https://github.com/zhw2590582/ArtPlayer.git
synced 2026-10-12 15:26:16 -08:00
refactor(asr): [PKG-ASR-03] own typed audio capture and playback lifetimes
This commit is contained in:
Harvey Zhao committed
1 parent
53c295819d
commit
0cf2af4840
33 files changed
+3492
-809
No files matched your search
@@ -4,4 +4,4 @@
|
||||
* (c) 2017-2026 Harvey Zhao
|
||||
* Released under the MIT License.
|
||||
*/
|
||||
!function(t,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"==typeof define&&define.amd?(t.artplayerPluginAsr=e(),define(function(){return t.artplayerPluginAsr})):(t="undefined"!==typeof globalThis?globalThis:t||self).artplayerPluginAsr=e()}(this,function(){"use strict";if("undefined"!==typeof document&&!document.getElementById("artplayer-plugin-asr")){const t=document.createElement("style");t.id="artplayer-plugin-asr",t.textContent=".art-video-player .art-layer-asr{pointer-events:none!important;position:absolute;z-index:150;inset:0;display:flex;flex-direction:column;justify-content:end;padding:0 2%;font-size:1rem;gap:var(--art-subtitle-gap);padding-bottom:var(--art-subtitle-bottom);transition:padding-bottom var(--art-transition-duration) ease;text-shadow:var(--art-subtitle-border) 1px 0 1px,var(--art-subtitle-border) 0 1px 1px,var(--art-subtitle-border) -1px 0 1px,var(--art-subtitle-border) 0 -1px 1px,var(--art-subtitle-border) 1px 1px 1px,var(--art-subtitle-border) -1px -1px 1px,var(--art-subtitle-border) 1px -1px 1px,var(--art-subtitle-border) -1px 1px 1px}.art-video-player.art-control-show .art-layer-asr{padding-bottom:calc(var(--art-control-height) + var(--art-subtitle-bottom))}",document.head.appendChild(t)}return function(t={}){const{length:e=3,interval:n=100,sampleRate:r=16e3,autoHideTimeout:a=1e4,onAudioChunk:o=()=>null}=t;return t=>{let i=!1,l=null,s=null,u=null,c=null,d=[],p=null,f=!1,y=null,m=null,g=null;const h=t.layers.add({name:"asr",html:""});function w(){h.style.display="none"}function b(t){var n;"string"===typeof t&&(clearTimeout(y),y=setTimeout(w,a),h.style.display="",h.innerHTML=(n=t,n.split(/(?<=[\u3001\u3002\uff01\uff1f!?.])\s*/u).map(t=>t.trim()).filter(Boolean).slice(-e)).map(t=>`<div class="art-asr-line">${t}</div>`).join(""))}async function v(){if(!s&&t.video)try{return s=l.createMediaElementSource(t.video),s}catch(e){console.warn("[artplayerPluginAsr] Direct connection failed:",e)}try{if(m||(m=await async function(t){try{return t.captureStream?t.captureStream():t.mozCaptureStream?t.mozCaptureStream():(console.warn("[artplayerPluginAsr] captureStream not supported"),null)}catch(e){return console.warn("[artplayerPluginAsr] Error capturing stream:",e),null}}(t.video)),m)return g=l.createMediaStreamSource(m),g}catch(e){console.warn("[artplayerPluginAsr] MediaStream fallback failed:",e)}return null}async function x(){i&&(i=!1,clearInterval(p),p=null,u&&(u.disconnect(),u.port.onmessage=null,u=null),c&&(c.disconnect(),c=null),d=[])}async function A(){await x(),g&&(g.disconnect(),g=null),s&&(s.disconnect(),s=null),m&&(m.getTracks().forEach(t=>t.stop()),m=null),l&&(await l.close(),l=null),f=!1}return t.on("video:volumechange",()=>{c&&(c.gain.value=t.volume)}),t.on("play",async function(){if(!i)try{await async function(){return l||(l=new(window.AudioContext||window.webkitAudioContext)({sampleRate:r}),"suspended"===l.state&&await l.resume(),l)}();const t=await v();if(!t)throw new Error("Could not establish audio source");if(!f){const t=(()=>{const t=new Blob(["\n class RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n }\n registerProcessor('recorder-processor', RecorderProcessor);\n "],{type:"application/javascript"});return URL.createObjectURL(t)})();await l.audioWorklet.addModule(t),URL.revokeObjectURL(t),f=!0}c=l.createGain(),c.gain.value=1,u=new AudioWorkletNode(l,"recorder-processor"),u.port.onmessage=t=>{d.push(new Float32Array(t.data))},t.connect(u),t.connect(c),c.connect(l.destination);const e=r*(n/1e3);p=setInterval(async()=>{if(0===d.length)return;let t=new Float32Array(0);for(;t.length<e&&d.length;){const e=d.shift(),n=new Float32Array(t.length+e.length);n.set(t,0),n.set(e,t.length),t=n}if(t.length<e)return;const n=function(t){const e=t.length,n=new ArrayBuffer(2*e),r=new DataView(n);for(let a=0;a<e;a++){const e=Math.max(-1,Math.min(1,t[a]));r.setInt16(2*a,e<0?32768*e:32767*e,!0)}return n}(t.slice(0,e)),a=function(t,e){const n=t.byteLength,r=new ArrayBuffer(44+n),a=new DataView(r),o=(t,e)=>{for(let n=0;n<e.length;n++)a.setUint8(t+n,e.charCodeAt(n))};return o(0,"RIFF"),a.setUint32(4,36+n,!0),o(8,"WAVE"),o(12,"fmt "),a.setUint32(16,16,!0),a.setUint16(20,1,!0),a.setUint16(22,1,!0),a.setUint32(24,e,!0),a.setUint32(28,2*e,!0),a.setUint16(32,2,!0),a.setUint16(34,16,!0),o(36,"data"),a.setUint32(40,n,!0),new Uint8Array(r).set(new Uint8Array(t),44),r}(n,r);b(await o({pcm:n,wav:a}))},n),i=!0}catch(t){console.error("[artplayerPluginAsr] Initialization failed:",t),await x()}}),t.on("pause",x),t.on("destroy",A),{name:"artplayerPluginAsr",stop:A,hide:w,append:b}}}});
|
||||
!function(t,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"==typeof define&&define.amd?(t.artplayerPluginAsr=e(),define(function(){return t.artplayerPluginAsr})):(t="undefined"!==typeof globalThis?globalThis:t||self).artplayerPluginAsr=e()}(this,function(){"use strict";class t{constructor(t,e){this.video=t,this.sampleRate=e,this.closed=!1,this.direct=!1,this.workletLoaded=!1}assertOpen(){if(this.closed)throw new Error("Audio capture was closed")}revokeWorklet(){void 0!==this.blobUrl&&(URL.revokeObjectURL(this.blobUrl),this.blobUrl=void 0)}async initialize(){this.assertOpen();const t=window,e=window.AudioContext||t.webkitAudioContext;if(!e)throw new Error("AudioContext is not supported");const i=this.context??(this.context=new e({sampleRate:this.sampleRate}));if("suspended"===i.state&&await i.resume(),this.assertOpen(),!this.workletLoaded){this.blobUrl=URL.createObjectURL(new Blob(["\nclass RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n}\nregisterProcessor('recorder-processor', RecorderProcessor);\n"],{type:"application/javascript"}));try{await i.audioWorklet.addModule(this.blobUrl),this.workletLoaded=!0}finally{this.revokeWorklet()}}if(this.assertOpen(),!this.source)try{this.source=i.createMediaElementSource(this.video),this.direct=!0}catch(s){console.warn("[artplayerPluginAsr] Direct connection failed:",s);const t=this.video.captureStream||this.video.mozCaptureStream;if(!t)throw new Error("Could not establish audio source");this.stream=t.call(this.video),this.source=i.createMediaStreamSource(this.stream)}}prepare(){return this.preparation??(this.preparation=this.initialize().catch(t=>{throw this.preparation=void 0,t})),this.preparation}get ownsMediaConnection(){return this.direct}idle(){const t=this.gain;this.disconnect(),this.direct&&this.context&&this.source&&(this.gain=t||this.context.createGain(),t||(this.gain.gain.value=1),this.source.connect(this.gain),this.gain.connect(this.context.destination))}connect(t){this.assertOpen();const e=this.context,i=this.source;if(!e||!i)throw new Error("Audio capture is not prepared");const s=this.gain;this.disconnect();const n=this.gain=s||e.createGain();s||(n.gain.value=1);const r=this.recorder=new AudioWorkletNode(e,"recorder-processor");r.port.onmessage=e=>{this.closed||this.recorder!==r||t(e.data)},i.connect(r),i.connect(n),n.connect(e.destination)}volume(t){this.gain&&(this.gain.gain.value=t)}disconnect(){this.recorder&&(this.recorder.port.onmessage=null,this.recorder.disconnect(),this.recorder=void 0),this.gain?.disconnect(),this.gain=void 0,this.source?.disconnect()}async close(){if(this.closed)return;this.closed=!0,this.disconnect(),this.revokeWorklet(),this.stream?.getTracks().forEach(t=>t.stop()),this.stream=void 0;const t=this.context;this.context=void 0,this.source=void 0,t&&"closed"!==t.state&&await t.close()}}class e{constructor(){this.offset=0,this.size=0}get length(){return this.size}push(t){if(0===t.length)return;const e={samples:new Float32Array(t)};this.tail?this.tail.next=e:this.head=e,this.tail=e,this.size+=t.length}take(t){if(!Number.isSafeInteger(t)||t<=0)throw new RangeError("Sample count must be a positive safe integer");if(t>this.size)return null;const e=new Float32Array(t);let i=0;for(;this.head&&i<t;){const s=this.head.samples,n=Math.min(s.length-this.offset,t-i);e.set(s.subarray(this.offset,this.offset+n),i),i+=n,this.offset+=n,this.offset===s.length&&(this.head=this.head.next,this.offset=0)}return this.head||(this.tail=void 0),this.size-=t,e}clear(){this.head=void 0,this.tail=void 0,this.offset=0,this.size=0}}class i{constructor(t,i,s){this.video=t,this.options=i,this.append=s,this.queue=new e,this.closing=Promise.resolve(),this.running=!1,this.terminal=!1,this.epoch=0}reset(){this.epoch++,this.pending=void 0,this.queue.clear()}pause(){this.reset(),this.running=!1,clearInterval(this.timer),this.timer=void 0,this.starting=void 0,this.graph?.idle()}restart(){const t=this.running||Boolean(this.starting);return this.pause(),this.graph&&!this.graph.ownsMediaConnection?this.stop().then(()=>t?this.start():void 0):t?this.start():Promise.resolve()}start(){if(this.terminal||this.running)return Promise.resolve();if(this.starting)return this.starting;const e=this.epoch,i=()=>!this.terminal&&this.epoch===e,s=(async()=>{if(await this.closing,!i())return;const{sampleRate:e,interval:s}=this.options,n=Math.floor(e*s/1e3);if(!Number.isSafeInteger(n)||n<1||!Number.isFinite(s)||s<=0)throw new Error("Audio chunk length must be positive and finite");const r=this.graph??(this.graph=new t(this.video,e));await r.prepare(),i()?(r.connect(t=>{if(this.running&&this.graph===r)return this.queue.length+t.length>Math.max(60*e,2*n)?(console.error("[artplayerPluginAsr] Audio callback backlog exceeded its capture limit"),void this.pause()):void this.queLine truncated
|
||||
@@ -4,4 +4,4 @@
|
||||
* (c) 2017-2026 Harvey Zhao
|
||||
* Released under the MIT License.
|
||||
*/
|
||||
!function(t,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"==typeof define&&define.amd?(t.artplayerPluginAsr=e(),define(function(){return t.artplayerPluginAsr})):(t="undefined"!==typeof globalThis?globalThis:t||self).artplayerPluginAsr=e()}(this,function(){"use strict";var t=(t,e,n)=>new Promise((r,o)=>{var a=t=>{try{l(n.next(t))}catch(e){o(e)}},i=t=>{try{l(n.throw(t))}catch(e){o(e)}},l=t=>t.done?r(t.value):Promise.resolve(t.value).then(a,i);l((n=n.apply(t,e)).next())});if("undefined"!==typeof document&&!document.getElementById("artplayer-plugin-asr")){const t=document.createElement("style");t.id="artplayer-plugin-asr",t.textContent=".art-video-player .art-layer-asr{pointer-events:none!important;position:absolute;z-index:150;inset:0;display:flex;flex-direction:column;justify-content:end;padding:0 2%;font-size:1rem;gap:var(--art-subtitle-gap);padding-bottom:var(--art-subtitle-bottom);transition:padding-bottom var(--art-transition-duration) ease;text-shadow:var(--art-subtitle-border) 1px 0 1px,var(--art-subtitle-border) 0 1px 1px,var(--art-subtitle-border) -1px 0 1px,var(--art-subtitle-border) 0 -1px 1px,var(--art-subtitle-border) 1px 1px 1px,var(--art-subtitle-border) -1px -1px 1px,var(--art-subtitle-border) 1px -1px 1px,var(--art-subtitle-border) -1px 1px 1px}.art-video-player.art-control-show .art-layer-asr{padding-bottom:calc(var(--art-control-height) + var(--art-subtitle-bottom))}",document.head.appendChild(t)}return function(e={}){const{length:n=3,interval:r=100,sampleRate:o=16e3,autoHideTimeout:a=1e4,onAudioChunk:i=()=>null}=e;return e=>{let l=!1,s=null,u=null,c=null,d=null,p=[],f=null,y=!1,m=null,h=null,g=null;const v=e.layers.add({name:"asr",html:""});function b(){v.style.display="none"}function x(t){var e;"string"===typeof t&&(clearTimeout(m),m=setTimeout(b,a),v.style.display="",v.innerHTML=(e=t,e.split(new RegExp("(?<=[\u3001\u3002\uff01\uff1f!?.])\\s*","u")).map(t=>t.trim()).filter(Boolean).slice(-n)).map(t=>`<div class="art-asr-line">${t}</div>`).join(""))}function w(){return t(this,null,function*(){if(!u&&e.video)try{return u=s.createMediaElementSource(e.video),u}catch(n){console.warn("[artplayerPluginAsr] Direct connection failed:",n)}try{if(h||(h=yield function(e){return t(this,null,function*(){try{return e.captureStream?e.captureStream():e.mozCaptureStream?e.mozCaptureStream():(console.warn("[artplayerPluginAsr] captureStream not supported"),null)}catch(n){return console.warn("[artplayerPluginAsr] Error capturing stream:",n),null}})}(e.video)),h)return g=s.createMediaStreamSource(h),g}catch(n){console.warn("[artplayerPluginAsr] MediaStream fallback failed:",n)}return null})}function A(){return t(this,null,function*(){l&&(l=!1,clearInterval(f),f=null,c&&(c.disconnect(),c.port.onmessage=null,c=null),d&&(d.disconnect(),d=null),p=[])})}function U(){return t(this,null,function*(){yield A(),g&&(g.disconnect(),g=null),u&&(u.disconnect(),u=null),h&&(h.getTracks().forEach(t=>t.stop()),h=null),s&&(yield s.close(),s=null),y=!1})}return e.on("video:volumechange",()=>{d&&(d.gain.value=e.volume)}),e.on("play",function(){return t(this,null,function*(){if(!l)try{yield function(){return t(this,null,function*(){return s||(s=new(window.AudioContext||window.webkitAudioContext)({sampleRate:o}),"suspended"===s.state&&(yield s.resume()),s)})}();const e=yield w();if(!e)throw new Error("Could not establish audio source");if(!y){const t=(()=>{const t=new Blob(["\n class RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n }\n registerProcessor('recorder-processor', RecorderProcessor);\n "],{type:"application/javascript"});return URL.createObjectURL(t)})();yield s.audioWorklet.addModule(t),URL.revokeObjectURL(t),y=!0}d=s.createGain(),d.gain.value=1,c=new AudioWorkletNode(s,"recorder-processor"),c.port.onmessage=t=>{p.push(new Float32Array(t.data))},e.connect(c),e.connect(d),d.connect(s.destination);const n=o*(r/1e3);f=setInterval(()=>t(null,null,function*(){if(0===p.length)return;let t=new Float32Array(0);for(;t.length<n&&p.length;){const e=p.shift(),n=new Float32Array(t.length+e.length);n.set(t,0),n.set(e,t.length),t=n}if(t.length<n)return;const e=function(t){const e=t.length,n=new ArrayBuffer(2*e),r=new DataView(n);for(let o=0;o<e;o++){const e=Math.max(-1,Math.min(1,t[o]));r.setInt16(2*o,e<0?32768*e:32767*e,!0)}return n}(t.slice(0,n)),r=function(t,e){const n=t.byteLength,r=new ArrayBuffer(44+n),o=new DataView(r),a=(t,e)=>{for(let n=0;n<e.length;n++)o.setUint8(t+n,e.charCodeAt(n))};return a(0,"RIFF"),o.setUint32(4,36+n,!0),a(8,"WAVE"),a(12,"fmt "),o.setUint32(16,16,!0),o.setUint16(20,1,!0),o.setUint16(22,1,!0),o.setUint32(24,e,!0),o.setUint32(28,2*e,!0),o.setUint16(32,2,!0),o.setUintLine truncated
|
||||
!function(t,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"==typeof define&&define.amd?(t.artplayerPluginAsr=e(),define(function(){return t.artplayerPluginAsr})):(t="undefined"!==typeof globalThis?globalThis:t||self).artplayerPluginAsr=e()}(this,function(){"use strict";var t=(t,e,i)=>new Promise((s,n)=>{var r=t=>{try{a(i.next(t))}catch(e){n(e)}},o=t=>{try{a(i.throw(t))}catch(e){n(e)}},a=t=>t.done?s(t.value):Promise.resolve(t.value).then(r,o);a((i=i.apply(t,e)).next())});class e{constructor(t,e){this.video=t,this.sampleRate=e,this.closed=!1,this.direct=!1,this.workletLoaded=!1}assertOpen(){if(this.closed)throw new Error("Audio capture was closed")}revokeWorklet(){void 0!==this.blobUrl&&(URL.revokeObjectURL(this.blobUrl),this.blobUrl=void 0)}initialize(){return t(this,null,function*(){var t;this.assertOpen();const e=window,i=window.AudioContext||e.webkitAudioContext;if(!i)throw new Error("AudioContext is not supported");const s=null!=(t=this.context)?t:this.context=new i({sampleRate:this.sampleRate});if("suspended"===s.state&&(yield s.resume()),this.assertOpen(),!this.workletLoaded){this.blobUrl=URL.createObjectURL(new Blob(["\nclass RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n}\nregisterProcessor('recorder-processor', RecorderProcessor);\n"],{type:"application/javascript"}));try{yield s.audioWorklet.addModule(this.blobUrl),this.workletLoaded=!0}finally{this.revokeWorklet()}}if(this.assertOpen(),!this.source)try{this.source=s.createMediaElementSource(this.video),this.direct=!0}catch(n){console.warn("[artplayerPluginAsr] Direct connection failed:",n);const t=this.video.captureStream||this.video.mozCaptureStream;if(!t)throw new Error("Could not establish audio source");this.stream=t.call(this.video),this.source=s.createMediaStreamSource(this.stream)}})}prepare(){return null!=this.preparation||(this.preparation=this.initialize().catch(t=>{throw this.preparation=void 0,t})),this.preparation}get ownsMediaConnection(){return this.direct}idle(){const t=this.gain;this.disconnect(),this.direct&&this.context&&this.source&&(this.gain=t||this.context.createGain(),t||(this.gain.gain.value=1),this.source.connect(this.gain),this.gain.connect(this.context.destination))}connect(t){this.assertOpen();const e=this.context,i=this.source;if(!e||!i)throw new Error("Audio capture is not prepared");const s=this.gain;this.disconnect();const n=this.gain=s||e.createGain();s||(n.gain.value=1);const r=this.recorder=new AudioWorkletNode(e,"recorder-processor");r.port.onmessage=e=>{this.closed||this.recorder!==r||t(e.data)},i.connect(r),i.connect(n),n.connect(e.destination)}volume(t){this.gain&&(this.gain.gain.value=t)}disconnect(){var t,e;this.recorder&&(this.recorder.port.onmessage=null,this.recorder.disconnect(),this.recorder=void 0),null==(t=this.gain)||t.disconnect(),this.gain=void 0,null==(e=this.source)||e.disconnect()}close(){return t(this,null,function*(){var t;if(this.closed)return;this.closed=!0,this.disconnect(),this.revokeWorklet(),null==(t=this.stream)||t.getTracks().forEach(t=>t.stop()),this.stream=void 0;const e=this.context;this.context=void 0,this.source=void 0,e&&"closed"!==e.state&&(yield e.close())})}}class i{constructor(){this.offset=0,this.size=0}get length(){return this.size}push(t){if(0===t.length)return;const e={samples:new Float32Array(t)};this.tail?this.tail.next=e:this.head=e,this.tail=e,this.size+=t.length}take(t){if(!Number.isSafeInteger(t)||t<=0)throw new RangeError("Sample count must be a positive safe integer");if(t>this.size)return null;const e=new Float32Array(t);let i=0;for(;this.head&&i<t;){const s=this.head.samples,n=Math.min(s.length-this.offset,t-i);e.set(s.subarray(this.offset,this.offset+n),i),i+=n,this.offset+=n,this.offset===s.length&&(this.head=this.head.next,this.offset=0)}return this.head||(this.tail=void 0),this.size-=t,e}clear(){this.head=void 0,this.tail=void 0,this.offset=0,this.size=0}}class s{constructor(t,e,s){this.video=t,this.options=e,this.append=s,this.queue=new i,this.closing=Promise.resolve(),this.running=!1,this.terminal=!1,this.epoch=0}reset(){this.epoch++,this.pending=void 0,this.queue.clear()}pause(){var t;this.reset(),this.running=!1,clearInterval(this.timer),this.timer=void 0,this.starting=void 0,null==(t=this.graph)||t.idle()}restart(){const t=this.running||Boolean(this.starting);return this.pause(),this.graph&&!this.graph.ownsMediaConnection?this.stop().then(()=>t?this.start():void 0):t?this.start():Promise.resolve()}start(){if(this.terminal||this.running)return Promise.resolve();if(this.starting)return this.starting;const i=this.epoch,s=()=>!this.terminal&&this.epoch===i,n=(()=>t(this,null,function*(){var t;if(yield this.closing,!s())return;const{sampleRate:i,interval:n}=this.options,r=Math.floor(i*n/1e3);if(!Number.isSafeInteger(r)||r<1||!Number.isFiLine truncated
|
||||
@@ -4,248 +4,394 @@
|
||||
* (c) 2017-2026 Harvey Zhao
|
||||
* Released under the MIT License.
|
||||
*/
|
||||
const style = ".art-video-player .art-layer-asr {\n pointer-events: none !important;\n position: absolute;\n z-index: 150;\n inset: 0px;\n display: flex;\n flex-direction: column;\n justify-content: end;\n padding: 0 2%;\n font-size: 1rem;\n gap: var(--art-subtitle-gap);\n padding-bottom: var(--art-subtitle-bottom);\n transition: padding-bottom var(--art-transition-duration) ease;\n text-shadow: var(--art-subtitle-border) 1px 0 1px, var(--art-subtitle-border) 0 1px 1px, var(--art-subtitle-border) -1px 0 1px, var(--art-subtitle-border) 0 -1px 1px, var(--art-subtitle-border) 1px 1px 1px, var(--art-subtitle-border) -1px -1px 1px, var(--art-subtitle-border) 1px -1px 1px, var(--art-subtitle-border) -1px 1px 1px;\n}\n.art-video-player.art-control-show .art-layer-asr {\n padding-bottom: calc(var(--art-control-height) + var(--art-subtitle-bottom));\n}\n";
|
||||
function artplayerPluginAsr(option = {}) {
|
||||
const {
|
||||
length = 3,
|
||||
interval = 100,
|
||||
sampleRate = 16e3,
|
||||
autoHideTimeout = 1e4,
|
||||
onAudioChunk = () => null
|
||||
} = option;
|
||||
return (art) => {
|
||||
let started = false;
|
||||
let audioCtx = null;
|
||||
let sourceNode = null;
|
||||
let recorderNode = null;
|
||||
let gainNode = null;
|
||||
let bufferChunks = [];
|
||||
let timer = null;
|
||||
let workletLoaded = false;
|
||||
let hideTimer = null;
|
||||
let mediaStream = null;
|
||||
let mediaStreamSource = null;
|
||||
const $asr = art.layers.add({
|
||||
name: "asr",
|
||||
html: ""
|
||||
});
|
||||
function splitByPunctuation(text) {
|
||||
return text.split(/(?<=[、。！？!?.])\s*/u).map((s) => s.trim()).filter(Boolean).slice(-length);
|
||||
}
|
||||
function hide() {
|
||||
$asr.style.display = "none";
|
||||
}
|
||||
function append(subtitle) {
|
||||
if (typeof subtitle !== "string")
|
||||
return;
|
||||
clearTimeout(hideTimer);
|
||||
hideTimer = setTimeout(hide, autoHideTimeout);
|
||||
$asr.style.display = "";
|
||||
$asr.innerHTML = splitByPunctuation(subtitle).map((line) => `<div class="art-asr-line">${line}</div>`).join("");
|
||||
}
|
||||
const recorderProcessorCode = `
|
||||
class RecorderProcessor extends AudioWorkletProcessor {
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (input && input[0]) {
|
||||
this.port.postMessage(input[0]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('recorder-processor', RecorderProcessor);
|
||||
`;
|
||||
const createWorkletBlobUrl = () => {
|
||||
const blob = new Blob([recorderProcessorCode], { type: "application/javascript" });
|
||||
return URL.createObjectURL(blob);
|
||||
};
|
||||
function floatTo16BitPCM(float32Array) {
|
||||
const len = float32Array.length;
|
||||
const buffer = new ArrayBuffer(len * 2);
|
||||
const view = new DataView(buffer);
|
||||
for (let i = 0; i < len; i++) {
|
||||
const s = Math.max(-1, Math.min(1, float32Array[i]));
|
||||
view.setInt16(i * 2, s < 0 ? s * 32768 : s * 32767, true);
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
function pcmToWav(pcmBuffer, sampleRate2) {
|
||||
const pcmLength = pcmBuffer.byteLength;
|
||||
const wavBuffer = new ArrayBuffer(44 + pcmLength);
|
||||
const view = new DataView(wavBuffer);
|
||||
const writeString = (offset, str) => {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
const recorderProcessorCode = `
|
||||
class RecorderProcessor extends AudioWorkletProcessor {
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (input && input[0]) {
|
||||
this.port.postMessage(input[0]);
|
||||
}
|
||||
};
|
||||
writeString(0, "RIFF");
|
||||
view.setUint32(4, 36 + pcmLength, true);
|
||||
writeString(8, "WAVE");
|
||||
writeString(12, "fmt ");
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, 1, true);
|
||||
view.setUint32(24, sampleRate2, true);
|
||||
view.setUint32(28, sampleRate2 * 2, true);
|
||||
view.setUint16(32, 2, true);
|
||||
view.setUint16(34, 16, true);
|
||||
writeString(36, "data");
|
||||
view.setUint32(40, pcmLength, true);
|
||||
new Uint8Array(wavBuffer).set(new Uint8Array(pcmBuffer), 44);
|
||||
return wavBuffer;
|
||||
return true;
|
||||
}
|
||||
async function setupAudioContext() {
|
||||
if (audioCtx)
|
||||
return audioCtx;
|
||||
audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate });
|
||||
if (audioCtx.state === "suspended")
|
||||
await audioCtx.resume();
|
||||
return audioCtx;
|
||||
}
|
||||
registerProcessor('recorder-processor', RecorderProcessor);
|
||||
`;
|
||||
class AudioGraph {
|
||||
constructor(video, sampleRate) {
|
||||
this.video = video;
|
||||
this.sampleRate = sampleRate;
|
||||
this.closed = false;
|
||||
this.direct = false;
|
||||
this.workletLoaded = false;
|
||||
}
|
||||
assertOpen() {
|
||||
if (this.closed)
|
||||
throw new Error("Audio capture was closed");
|
||||
}
|
||||
revokeWorklet() {
|
||||
if (this.blobUrl !== void 0) {
|
||||
URL.revokeObjectURL(this.blobUrl);
|
||||
this.blobUrl = void 0;
|
||||
}
|
||||
async function setupAudioSource() {
|
||||
if (!sourceNode && art.video) {
|
||||
try {
|
||||
sourceNode = audioCtx.createMediaElementSource(art.video);
|
||||
return sourceNode;
|
||||
} catch (err) {
|
||||
console.warn("[artplayerPluginAsr] Direct connection failed:", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
async initialize() {
|
||||
this.assertOpen();
|
||||
const platform = window;
|
||||
const Context = window.AudioContext || platform.webkitAudioContext;
|
||||
if (!Context)
|
||||
throw new Error("AudioContext is not supported");
|
||||
const context = this.context ?? (this.context = new Context({ sampleRate: this.sampleRate }));
|
||||
if (context.state === "suspended")
|
||||
await context.resume();
|
||||
this.assertOpen();
|
||||
if (!this.workletLoaded) {
|
||||
this.blobUrl = URL.createObjectURL(new Blob([recorderProcessorCode], { type: "application/javascript" }));
|
||||
try {
|
||||
if (!mediaStream) {
|
||||
mediaStream = await captureAudioFromVideo(art.video);
|
||||
}
|
||||
if (mediaStream) {
|
||||
mediaStreamSource = audioCtx.createMediaStreamSource(mediaStream);
|
||||
return mediaStreamSource;
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn("[artplayerPluginAsr] MediaStream fallback failed:", err);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
async function captureAudioFromVideo(videoElement) {
|
||||
try {
|
||||
if (videoElement.captureStream)
|
||||
return videoElement.captureStream();
|
||||
if (videoElement.mozCaptureStream)
|
||||
return videoElement.mozCaptureStream();
|
||||
console.warn("[artplayerPluginAsr] captureStream not supported");
|
||||
return null;
|
||||
} catch (err) {
|
||||
console.warn("[artplayerPluginAsr] Error capturing stream:", err);
|
||||
return null;
|
||||
await context.audioWorklet.addModule(this.blobUrl);
|
||||
this.workletLoaded = true;
|
||||
} finally {
|
||||
this.revokeWorklet();
|
||||
}
|
||||
}
|
||||
async function startCapture() {
|
||||
if (started)
|
||||
return;
|
||||
this.assertOpen();
|
||||
if (!this.source) {
|
||||
try {
|
||||
await setupAudioContext();
|
||||
const audioSource = await setupAudioSource();
|
||||
if (!audioSource)
|
||||
this.source = context.createMediaElementSource(this.video);
|
||||
this.direct = true;
|
||||
} catch (error) {
|
||||
console.warn("[artplayerPluginAsr] Direct connection failed:", error);
|
||||
const capture = this.video.captureStream || this.video.mozCaptureStream;
|
||||
if (!capture)
|
||||
throw new Error("Could not establish audio source");
|
||||
if (!workletLoaded) {
|
||||
const blobUrl = createWorkletBlobUrl();
|
||||
await audioCtx.audioWorklet.addModule(blobUrl);
|
||||
URL.revokeObjectURL(blobUrl);
|
||||
workletLoaded = true;
|
||||
}
|
||||
gainNode = audioCtx.createGain();
|
||||
gainNode.gain.value = 1;
|
||||
recorderNode = new AudioWorkletNode(audioCtx, "recorder-processor");
|
||||
recorderNode.port.onmessage = (event) => {
|
||||
bufferChunks.push(new Float32Array(event.data));
|
||||
};
|
||||
audioSource.connect(recorderNode);
|
||||
audioSource.connect(gainNode);
|
||||
gainNode.connect(audioCtx.destination);
|
||||
const CHUNK_SAMPLES = sampleRate * (interval / 1e3);
|
||||
timer = setInterval(async () => {
|
||||
if (bufferChunks.length === 0)
|
||||
return;
|
||||
let accumulated = new Float32Array(0);
|
||||
while (accumulated.length < CHUNK_SAMPLES && bufferChunks.length) {
|
||||
const next = bufferChunks.shift();
|
||||
const tmp = new Float32Array(accumulated.length + next.length);
|
||||
tmp.set(accumulated, 0);
|
||||
tmp.set(next, accumulated.length);
|
||||
accumulated = tmp;
|
||||
}
|
||||
if (accumulated.length < CHUNK_SAMPLES)
|
||||
return;
|
||||
const chunkToSend = accumulated.slice(0, CHUNK_SAMPLES);
|
||||
const pcm = floatTo16BitPCM(chunkToSend);
|
||||
const wav = pcmToWav(pcm, sampleRate);
|
||||
const subtitle = await onAudioChunk({ pcm, wav });
|
||||
append(subtitle);
|
||||
}, interval);
|
||||
started = true;
|
||||
} catch (err) {
|
||||
console.error("[artplayerPluginAsr] Initialization failed:", err);
|
||||
await stopCapture();
|
||||
this.stream = capture.call(this.video);
|
||||
this.source = context.createMediaStreamSource(this.stream);
|
||||
}
|
||||
}
|
||||
async function stopCapture() {
|
||||
if (!started)
|
||||
return;
|
||||
started = false;
|
||||
clearInterval(timer);
|
||||
timer = null;
|
||||
if (recorderNode) {
|
||||
recorderNode.disconnect();
|
||||
recorderNode.port.onmessage = null;
|
||||
recorderNode = null;
|
||||
}
|
||||
if (gainNode) {
|
||||
gainNode.disconnect();
|
||||
gainNode = null;
|
||||
}
|
||||
bufferChunks = [];
|
||||
}
|
||||
prepare() {
|
||||
this.preparation ?? (this.preparation = this.initialize().catch((error) => {
|
||||
this.preparation = void 0;
|
||||
throw error;
|
||||
}));
|
||||
return this.preparation;
|
||||
}
|
||||
get ownsMediaConnection() {
|
||||
return this.direct;
|
||||
}
|
||||
idle() {
|
||||
const gain = this.gain;
|
||||
this.disconnect();
|
||||
if (this.direct && this.context && this.source) {
|
||||
this.gain = gain || this.context.createGain();
|
||||
if (!gain)
|
||||
this.gain.gain.value = 1;
|
||||
this.source.connect(this.gain);
|
||||
this.gain.connect(this.context.destination);
|
||||
}
|
||||
async function destroy() {
|
||||
await stopCapture();
|
||||
if (mediaStreamSource) {
|
||||
mediaStreamSource.disconnect();
|
||||
mediaStreamSource = null;
|
||||
}
|
||||
if (sourceNode) {
|
||||
sourceNode.disconnect();
|
||||
sourceNode = null;
|
||||
}
|
||||
if (mediaStream) {
|
||||
mediaStream.getTracks().forEach((track) => track.stop());
|
||||
mediaStream = null;
|
||||
}
|
||||
if (audioCtx) {
|
||||
await audioCtx.close();
|
||||
audioCtx = null;
|
||||
}
|
||||
workletLoaded = false;
|
||||
}
|
||||
art.on("video:volumechange", () => {
|
||||
if (gainNode) {
|
||||
gainNode.gain.value = art.volume;
|
||||
}
|
||||
});
|
||||
art.on("play", startCapture);
|
||||
art.on("pause", stopCapture);
|
||||
art.on("destroy", destroy);
|
||||
return {
|
||||
name: "artplayerPluginAsr",
|
||||
stop: destroy,
|
||||
hide,
|
||||
append
|
||||
}
|
||||
connect(receive) {
|
||||
this.assertOpen();
|
||||
const context = this.context;
|
||||
const source = this.source;
|
||||
if (!context || !source)
|
||||
throw new Error("Audio capture is not prepared");
|
||||
const previousGain = this.gain;
|
||||
this.disconnect();
|
||||
const gain = this.gain = previousGain || context.createGain();
|
||||
if (!previousGain)
|
||||
gain.gain.value = 1;
|
||||
const recorder = this.recorder = new AudioWorkletNode(context, "recorder-processor");
|
||||
recorder.port.onmessage = (event) => {
|
||||
if (!this.closed && this.recorder === recorder)
|
||||
receive(event.data);
|
||||
};
|
||||
source.connect(recorder);
|
||||
source.connect(gain);
|
||||
gain.connect(context.destination);
|
||||
}
|
||||
volume(value) {
|
||||
if (this.gain)
|
||||
this.gain.gain.value = value;
|
||||
}
|
||||
disconnect() {
|
||||
if (this.recorder) {
|
||||
this.recorder.port.onmessage = null;
|
||||
this.recorder.disconnect();
|
||||
this.recorder = void 0;
|
||||
}
|
||||
this.gain?.disconnect();
|
||||
this.gain = void 0;
|
||||
this.source?.disconnect();
|
||||
}
|
||||
async close() {
|
||||
if (this.closed)
|
||||
return;
|
||||
this.closed = true;
|
||||
this.disconnect();
|
||||
this.revokeWorklet();
|
||||
this.stream?.getTracks().forEach((track) => track.stop());
|
||||
this.stream = void 0;
|
||||
const context = this.context;
|
||||
this.context = void 0;
|
||||
this.source = void 0;
|
||||
if (context && context.state !== "closed")
|
||||
await context.close();
|
||||
}
|
||||
}
|
||||
function encodeAudio(samples, sampleRate) {
|
||||
const pcm = new ArrayBuffer(samples.length * 2);
|
||||
const pcmView = new DataView(pcm);
|
||||
for (let index = 0; index < samples.length; index++) {
|
||||
const value = Math.max(-1, Math.min(1, samples[index]));
|
||||
pcmView.setInt16(index * 2, value < 0 ? value * 32768 : value * 32767, true);
|
||||
}
|
||||
const wav = new ArrayBuffer(44 + pcm.byteLength);
|
||||
const header = new DataView(wav);
|
||||
function writeString(offset, value) {
|
||||
for (let index = 0; index < value.length; index++)
|
||||
header.setUint8(offset + index, value.charCodeAt(index));
|
||||
}
|
||||
writeString(0, "RIFF");
|
||||
header.setUint32(4, 36 + pcm.byteLength, true);
|
||||
writeString(8, "WAVE");
|
||||
writeString(12, "fmt ");
|
||||
header.setUint32(16, 16, true);
|
||||
header.setUint16(20, 1, true);
|
||||
header.setUint16(22, 1, true);
|
||||
header.setUint32(24, sampleRate, true);
|
||||
header.setUint32(28, sampleRate * 2, true);
|
||||
header.setUint16(32, 2, true);
|
||||
header.setUint16(34, 16, true);
|
||||
writeString(36, "data");
|
||||
header.setUint32(40, pcm.byteLength, true);
|
||||
new Uint8Array(wav).set(new Uint8Array(pcm), 44);
|
||||
return { pcm, wav };
|
||||
}
|
||||
class SampleQueue {
|
||||
constructor() {
|
||||
this.offset = 0;
|
||||
this.size = 0;
|
||||
}
|
||||
get length() {
|
||||
return this.size;
|
||||
}
|
||||
push(input) {
|
||||
if (input.length === 0)
|
||||
return;
|
||||
const block = { samples: new Float32Array(input) };
|
||||
if (this.tail)
|
||||
this.tail.next = block;
|
||||
else
|
||||
this.head = block;
|
||||
this.tail = block;
|
||||
this.size += input.length;
|
||||
}
|
||||
take(count) {
|
||||
if (!Number.isSafeInteger(count) || count <= 0)
|
||||
throw new RangeError("Sample count must be a positive safe integer");
|
||||
if (count > this.size)
|
||||
return null;
|
||||
const output = new Float32Array(count);
|
||||
let written = 0;
|
||||
while (this.head && written < count) {
|
||||
const samples = this.head.samples;
|
||||
const amount = Math.min(samples.length - this.offset, count - written);
|
||||
output.set(samples.subarray(this.offset, this.offset + amount), written);
|
||||
written += amount;
|
||||
this.offset += amount;
|
||||
if (this.offset === samples.length) {
|
||||
this.head = this.head.next;
|
||||
this.offset = 0;
|
||||
}
|
||||
}
|
||||
if (!this.head)
|
||||
this.tail = void 0;
|
||||
this.size -= count;
|
||||
return output;
|
||||
}
|
||||
clear() {
|
||||
this.head = void 0;
|
||||
this.tail = void 0;
|
||||
this.offset = 0;
|
||||
this.size = 0;
|
||||
}
|
||||
}
|
||||
class Capture {
|
||||
constructor(video, options, append) {
|
||||
this.video = video;
|
||||
this.options = options;
|
||||
this.append = append;
|
||||
this.queue = new SampleQueue();
|
||||
this.closing = Promise.resolve();
|
||||
this.running = false;
|
||||
this.terminal = false;
|
||||
this.epoch = 0;
|
||||
}
|
||||
reset() {
|
||||
this.epoch++;
|
||||
this.pending = void 0;
|
||||
this.queue.clear();
|
||||
}
|
||||
pause() {
|
||||
this.reset();
|
||||
this.running = false;
|
||||
clearInterval(this.timer);
|
||||
this.timer = void 0;
|
||||
this.starting = void 0;
|
||||
this.graph?.idle();
|
||||
}
|
||||
restart() {
|
||||
const active = this.running || Boolean(this.starting);
|
||||
this.pause();
|
||||
if (this.graph && !this.graph.ownsMediaConnection)
|
||||
return this.stop().then(() => active ? this.start() : void 0);
|
||||
return active ? this.start() : Promise.resolve();
|
||||
}
|
||||
start() {
|
||||
if (this.terminal || this.running)
|
||||
return Promise.resolve();
|
||||
if (this.starting)
|
||||
return this.starting;
|
||||
const epoch = this.epoch;
|
||||
const current = () => !this.terminal && this.epoch === epoch;
|
||||
const operation = (async () => {
|
||||
await this.closing;
|
||||
if (!current())
|
||||
return;
|
||||
const { sampleRate, interval } = this.options;
|
||||
const count = Math.floor(sampleRate * interval / 1e3);
|
||||
if (!Number.isSafeInteger(count) || count < 1 || !Number.isFinite(interval) || interval <= 0)
|
||||
throw new Error("Audio chunk length must be positive and finite");
|
||||
const graph = this.graph ?? (this.graph = new AudioGraph(this.video, sampleRate));
|
||||
await graph.prepare();
|
||||
if (!current()) {
|
||||
if (!this.running && this.graph === graph)
|
||||
graph.idle();
|
||||
return;
|
||||
}
|
||||
graph.connect((samples) => {
|
||||
if (!this.running || this.graph !== graph)
|
||||
return;
|
||||
if (this.queue.length + samples.length > Math.max(sampleRate * 60, count * 2)) {
|
||||
console.error("[artplayerPluginAsr] Audio callback backlog exceeded its capture limit");
|
||||
this.pause();
|
||||
return;
|
||||
}
|
||||
this.queue.push(samples);
|
||||
});
|
||||
this.running = true;
|
||||
this.timer = setInterval(() => this.tick(count), interval);
|
||||
})().catch(async (error) => {
|
||||
if (current()) {
|
||||
console.error("[artplayerPluginAsr] Initialization failed:", error);
|
||||
await this.stop();
|
||||
}
|
||||
}).finally(() => {
|
||||
if (this.starting === operation)
|
||||
this.starting = void 0;
|
||||
});
|
||||
this.starting = operation;
|
||||
return operation;
|
||||
}
|
||||
async tick(count) {
|
||||
if (!this.running || this.pending)
|
||||
return;
|
||||
const samples = this.queue.take(count);
|
||||
if (!samples)
|
||||
return;
|
||||
const epoch = this.epoch;
|
||||
const pending = this.pending = {};
|
||||
try {
|
||||
const text = await this.options.onAudioChunk(encodeAudio(samples, this.options.sampleRate));
|
||||
if (this.running && !this.terminal && this.epoch === epoch)
|
||||
this.append(text);
|
||||
} catch (error) {
|
||||
if (this.epoch === epoch && !this.terminal)
|
||||
console.error("[artplayerPluginAsr] Audio callback failed:", error);
|
||||
} finally {
|
||||
if (this.pending === pending)
|
||||
this.pending = void 0;
|
||||
}
|
||||
}
|
||||
volume(value) {
|
||||
this.graph?.volume(value);
|
||||
}
|
||||
stop(terminal = false) {
|
||||
this.terminal || (this.terminal = terminal);
|
||||
this.pause();
|
||||
const graph = this.graph;
|
||||
if (terminal || !graph?.ownsMediaConnection) {
|
||||
this.graph = void 0;
|
||||
this.closing = Promise.all([this.closing, graph?.close()]).then(() => void 0);
|
||||
}
|
||||
return this.closing;
|
||||
}
|
||||
}
|
||||
const style = ".art-video-player .art-layer-asr {\n pointer-events: none !important;\n position: absolute;\n z-index: 150;\n inset: 0px;\n display: flex;\n flex-direction: column;\n justify-content: end;\n padding: 0 2%;\n font-size: 1rem;\n gap: var(--art-subtitle-gap);\n padding-bottom: var(--art-subtitle-bottom);\n transition: padding-bottom var(--art-transition-duration) ease;\n text-shadow: var(--art-subtitle-border) 1px 0 1px, var(--art-subtitle-border) 0 1px 1px, var(--art-subtitle-border) -1px 0 1px, var(--art-subtitle-border) 0 -1px 1px, var(--art-subtitle-border) 1px 1px 1px, var(--art-subtitle-border) -1px -1px 1px, var(--art-subtitle-border) 1px -1px 1px, var(--art-subtitle-border) -1px 1px 1px;\n}\n.art-video-player.art-control-show .art-layer-asr {\n padding-bottom: calc(var(--art-control-height) + var(--art-subtitle-bottom));\n}\n";
|
||||
function createSubtitles(layer, length, autoHideTimeout) {
|
||||
let timer;
|
||||
let destroyed = false;
|
||||
function cancelTimer() {
|
||||
clearTimeout(timer);
|
||||
timer = void 0;
|
||||
}
|
||||
function hide() {
|
||||
if (!destroyed)
|
||||
layer.style.display = "none";
|
||||
}
|
||||
function append(subtitle) {
|
||||
if (destroyed || typeof subtitle !== "string")
|
||||
return;
|
||||
cancelTimer();
|
||||
timer = setTimeout(hide, autoHideTimeout);
|
||||
layer.style.display = "";
|
||||
layer.innerHTML = subtitle.split(/(?<=[、。！？!?.])\s*/u).map((line) => line.trim()).filter(Boolean).slice(-length).map((line) => `<div class="art-asr-line">${line}</div>`).join("");
|
||||
}
|
||||
return {
|
||||
append,
|
||||
hide,
|
||||
destroy() {
|
||||
destroyed = true;
|
||||
cancelTimer();
|
||||
}
|
||||
};
|
||||
}
|
||||
if (typeof document !== "undefined") {
|
||||
if (!document.getElementById("artplayer-plugin-asr")) {
|
||||
const $style = document.createElement("style");
|
||||
$style.id = "artplayer-plugin-asr";
|
||||
$style.textContent = style;
|
||||
document.head.appendChild($style);
|
||||
}
|
||||
function artplayerPluginAsr(option = {}) {
|
||||
const { length = 3, interval = 100, sampleRate = 16e3, autoHideTimeout = 1e4, onAudioChunk = () => null } = option;
|
||||
return (art) => {
|
||||
const layer = art.layers.add({ name: "asr", html: "" });
|
||||
if (!layer)
|
||||
throw new Error("Could not create ASR subtitle layer");
|
||||
const subtitles = createSubtitles(layer, length, autoHideTimeout);
|
||||
const capture = new Capture(art.video, { interval, sampleRate, onAudioChunk }, subtitles.append);
|
||||
const volume = () => capture.volume(art.volume);
|
||||
const play = () => capture.start();
|
||||
const pause = () => capture.pause();
|
||||
const restart = () => capture.restart();
|
||||
const stop = () => capture.stop();
|
||||
const destroy = () => {
|
||||
subtitles.destroy();
|
||||
art.off("video:volumechange", volume);
|
||||
art.off("play", play);
|
||||
art.off("pause", pause);
|
||||
art.off("restart", restart);
|
||||
art.off("destroy", destroy);
|
||||
return capture.stop(true);
|
||||
};
|
||||
art.on("video:volumechange", volume);
|
||||
art.on("play", play);
|
||||
art.on("pause", pause);
|
||||
art.on("restart", restart);
|
||||
art.on("destroy", destroy);
|
||||
return { name: "artplayerPluginAsr", stop, hide: subtitles.hide, append: subtitles.append };
|
||||
};
|
||||
}
|
||||
Object.defineProperty(artplayerPluginAsr, "default", { value: artplayerPluginAsr });
|
||||
if (typeof document !== "undefined" && !document.getElementById("artplayer-plugin-asr")) {
|
||||
const styleElement = document.createElement("style");
|
||||
styleElement.id = "artplayer-plugin-asr";
|
||||
styleElement.textContent = style;
|
||||
document.head.appendChild(styleElement);
|
||||
}
|
||||
export {
|
||||
artplayerPluginAsr as default
|
||||
|
||||
+3
-3
@@ -49,9 +49,9 @@
|
||||
"test:baseline": "node --test refactor/scripts/*.test.mjs",
|
||||
"ci:check": "yarn check:toolchain --strict && yarn check:commits --report && yarn check:impact --report && yarn check:ci && yarn test:contracts && yarn check:contracts --report && yarn check:plan && yarn lint && yarn check:types && yarn typecheck && yarn test",
|
||||
"ci:build": "yarn build:types && yarn build all && yarn build:i18n && yarn build:ts && yarn build:docs && yarn test:imports",
|
||||
"test:imports": "node --test test/esm.test.js test/i18n.test.js test/ssr.test.js",
|
||||
"test:imports": "node --test test/esm.test.js test/i18n.test.js test/ssr.test.js test/asr-distribution.test.js",
|
||||
"typecheck": "node scripts/typecheck.mjs",
|
||||
"test:unit": "node --test test/asr-audio.test.js test/asr.test.js test/jassub.test.js test/multiple-subtitles-merge.test.js test/multiple-subtitles-lifecycle.test.js test/multiple-subtitles-failures.test.js test/multiple-subtitles.test.js test/multiple-subtitles-vendor.test.js test/vtt-thumbnail-parser.test.js test/vtt-thumbnail-lifecycle.test.js test/vtt-thumbnail.test.js test/auto-thumbnail-frames.test.js test/auto-thumbnail-lifecycle.test.js test/auto-thumbnail.test.js test/thumbnail-emitter.test.js test/thumbnail-runtime.test.js test/thumbnail-vendor.test.js test/thumbnail-lifecycle.test.js test/thumbnail-input.test.js test/thumbnail.test.js test/iframe-navigation.test.js test/iframe-boundaries.test.js test/iframe-lifecycle.test.js test/iframe.test.js test/mediabunny.test.js test/mediabunny-shim.test.js test/mediabunny-coordination.test.js test/mediabunny-video.test.js test/mediabunny-audio.test.js test/mediabunny-hls.test.js test/mediabunny-entry.test.js test/mediabunny-capability.test.js test/mediabunny-load.test.js test/mediabunny-input.test.js test/dpip.test.js test/dpip-lifecycle.test.js test/canvas.test.js test/canvas-lifecycle.test.js test/ambilight.test.js test/ambilight-lifecycle.test.js test/ambilight-proxy.test.js test/vast.test.js test/vast-lifecycle.test.js test/ads.test.js test/ads-lifecycle.test.js test/playback.test.js test/dash-control.test.js test/dash-contract.test.js test/dash-lifecycle.test.js test/dash-events.test.js test/hls-control.test.js test/audio-track.test.js test/public-behavior.test.js test/helpers.test.js test/chapter.test.js test/utils.test.js test/resource-scope.test.js test/instance-lifecycle.test.js test/options.test.js test/media-hosts.test.js test/plugins.test.js test/source.test.js test/playback-properties.test.js test/media-events.test.js test/template-resources.test.js test/core-vendor.test.js test/component-resources.test.js test/setting-model.test.js test/setting-layout.test.js test/setting-resources.test.js test/subtitle.test.js test/display-native.test.js test/display-video-fullscreen.test.js test/display-pip.test.js test/display-mini.test.js test/display-sizing.test.js test/display-orientation.test.js test/hotkey.test.js test/listener-registry.test.js test/global-events.test.js test/pointer-events.test.js test/gesture.test.js test/event-scheduling.test.js test/notice.test.js test/fast-forward.test.js test/auto-playback.test.js test/builtin-layers.test.js test/prompt-components.test.js test/screenshot.test.js test/thumbnails.test.js test/progress.test.js test/environment.test.js test/storage.test.js test/facade-properties.test.js test/dom-boundaries.test.js test/initialization.test.js test/entry.test.js test/accessibility-button.test.js test/accessibility-focus.test.js test/accessibility-slider.test.js",
|
||||
"test:unit": "node --test test/asr-encoding.test.js test/asr-lifecycle.test.js test/asr-audio.test.js test/asr.test.js test/jassub.test.js test/multiple-subtitles-merge.test.js test/multiple-subtitles-lifecycle.test.js test/multiple-subtitles-failures.test.js test/multiple-subtitles.test.js test/multiple-subtitles-vendor.test.js test/vtt-thumbnail-parser.test.js test/vtt-thumbnail-lifecycle.test.js test/vtt-thumbnail.test.js test/auto-thumbnail-frames.test.js test/auto-thumbnail-lifecycle.test.js test/auto-thumbnail.test.js test/thumbnail-emitter.test.js test/thumbnail-runtime.test.js test/thumbnail-vendor.test.js test/thumbnail-lifecycle.test.js test/thumbnail-input.test.js test/thumbnail.test.js test/iframe-navigation.test.js test/iframe-boundaries.test.js test/iframe-lifecycle.test.js test/iframe.test.js test/mediabunny.test.js test/mediabunny-shim.test.js test/mediabunny-coordination.test.js test/mediabunny-video.test.js test/mediabunny-audio.test.js test/mediabunny-hls.test.js test/mediabunny-entry.test.js test/mediabunny-capability.test.js test/mediabunny-load.test.js test/mediabunny-input.test.js test/dpip.test.js test/dpip-lifecycle.test.js test/canvas.test.js test/canvas-lifecycle.test.js test/ambilight.test.js test/ambilight-lifecycle.test.js test/ambilight-proxy.test.js test/vast.test.js test/vast-lifecycle.test.js test/ads.test.js test/ads-lifecycle.test.js test/playback.test.js test/dash-control.test.js test/dash-contract.test.js test/dash-lifecycle.test.js test/dash-events.test.js test/hls-control.test.js test/audio-track.test.js test/public-behavior.test.js test/helpers.test.js test/chapter.test.js test/utils.test.js test/resource-scope.test.js test/instance-lifecycle.test.js test/options.test.js test/media-hosts.test.js test/plugins.test.js test/source.test.js test/playback-properties.test.js test/media-events.test.js test/template-resources.test.js test/core-vendor.test.js test/component-resources.test.js test/setting-model.test.js test/setting-layout.test.js test/setting-resources.test.js test/subtitle.test.js test/display-native.test.js test/display-video-fullscreen.test.js test/display-pip.test.js test/display-mini.test.js test/display-sizing.test.js test/display-orientation.test.js test/hotkey.test.js test/listener-registry.test.js test/global-events.test.js test/pointer-events.test.js test/gesture.test.js test/event-scheduling.test.js test/notice.test.js test/fast-forward.test.js test/auto-playback.test.js test/builtin-layers.test.js test/prompt-components.test.js test/screenshot.test.js test/thumbnails.test.js test/progress.test.js test/environment.test.js test/storage.test.js test/facade-properties.test.js test/dom-boundaries.test.js test/initialization.test.js test/entry.test.js test/accessibility-button.test.js test/accessibility-focus.test.js test/accessibility-slider.test.js",
|
||||
"test:coverage": "node --test test/coverage.test.js && node scripts/coverage.mjs",
|
||||
"test": "yarn test:node && yarn test:baseline",
|
||||
"test:browser": "playwright test",
|
||||
@@ -94,7 +94,7 @@
|
||||
"check:ci": "node refactor/scripts/ci-workflow.mjs",
|
||||
"test:ci": "node --test test/ci-summary.test.js refactor/scripts/ci-workflow.test.mjs refactor/scripts/impact.test.mjs test/package-runtime.test.js",
|
||||
"test:package:runtime": "node scripts/package-runtime.mjs --canonical",
|
||||
"test:asr": "node --test test/asr.test.js test/asr-audio.test.js"
|
||||
"test:asr": "node --test test/asr.test.js test/asr-audio.test.js test/asr-encoding.test.js test/asr-lifecycle.test.js test/asr-distribution.test.js"
|
||||
},
|
||||
"browserslist": "last 1 Chrome version",
|
||||
"devDependencies": {
|
||||
|
||||
@@ -1,33 +1,86 @@
|
||||
# ASR maintenance
|
||||
|
||||
The current implementation is still in `src/index.js`. TypeScript migration and
|
||||
resource ownership changes are tracked in the repository refactor plan as
|
||||
PKG-ASR-02 through PKG-ASR-06. Do not interpret the baseline tests as completion.
|
||||
The player captures first-channel audio and hands PCM/WAV to a caller-provided
|
||||
recognizer. There is no network request, microphone permission or embedded model.
|
||||
|
||||
- The default factory snapshots options. Registration creates the `asr` layer
|
||||
and subscribes to player play, pause, volumechange and destroy events.
|
||||
- Play creates an AudioContext and tries a media element source, then a captured
|
||||
MediaStream source. An inline Blob contains the `recorder-processor` Worklet.
|
||||
- Worklet messages supply first-channel Float32 samples. The interval callback
|
||||
encodes little-endian mono PCM16 and a 44-byte-header WAV and invokes the
|
||||
caller's `onAudioChunk({ pcm, wav })`. A returned string replaces subtitles.
|
||||
- `append` splits punctuation and sets HTML in `div.art-asr-line`; `hide` changes
|
||||
display only. `src/style.less` owns layout and existing player CSS variables.
|
||||
- `stop()` is actually asynchronous and releases audio resources, but retains
|
||||
playback subscriptions. It is not a permanent disable API. There is no public
|
||||
`start()` method. Public declarations currently describe stop as returning void.
|
||||
| File | Responsibility |
|
||||
| --- | --- |
|
||||
| src/index.ts | Snapshot options, register player events, return the compatible facade and install the stylesheet |
|
||||
| src/types.ts | Internal options, result and browser capability types; public declarations remain in types/ |
|
||||
| src/subtitles.ts | Punctuation/HTML rendering and the owned auto-hide timer |
|
||||
| src/capture.ts | Initialization deduplication, active generation, FIFO consumption and one pending recognizer call |
|
||||
| src/audio-graph.ts | AudioContext, element/stream source, Worklet loading, playback routing and recorder ownership |
|
||||
| src/sample-queue.ts | Copied sample blocks with a linked queue and head offset; preserves partial blocks and tails |
|
||||
| src/encoding.ts | Pure PCM16LE/WAV encoding, including historical clamping and the 44-byte mono WAV header |
|
||||
| src/worklet.ts | Inline processor code; first-channel messages and the existing recorder-processor registration name |
|
||||
| src/style.less | Existing art-layer-asr layout, art-asr-line content and player CSS variables |
|
||||
|
||||
This package does not implement recognition or open an ASR network connection.
|
||||
The demo service and its WebSocket belong to the caller. Do not run that service
|
||||
to validate audio capture; use local media and a local callback.
|
||||
Dependencies flow from the facade into subtitles/capture, then from capture into
|
||||
the graph, queue and pure encoder. Audio modules do not receive the whole player.
|
||||
The Worklet is loaded from a temporary Blob before binding the media element, so
|
||||
a failed module load can close its context without taking over video playback.
|
||||
|
||||
The initial implementation has suspected chunk loss, partial initialization
|
||||
cleanup and stale callback issues. Preserve the frozen historical tests while
|
||||
adding candidate regression tests when fixing these. Keep callback/stop typing
|
||||
compatible with historical assignments, and preserve public entrypoints, HTML
|
||||
rendering behavior, layer names and stylesheet identity.
|
||||
## States and ownership
|
||||
|
||||
From the repository root, `yarn test:asr` verifies historical public behavior;
|
||||
`yarn test:baseline` includes archive provenance. `yarn dev artplayer-plugin-asr`
|
||||
and `yarn build artplayer-plugin-asr` use repository build tools. Real browser
|
||||
audio and lifecycle validation remains a separate required migration step.
|
||||
Play shares one pending initialization. Successful setup attaches one recorder
|
||||
and starts one interval. Each tick takes exactly one complete block; an incomplete
|
||||
block stays queued. Only one callback is pending within the active generation.
|
||||
Rejections are logged and later ticks continue. Pause, source restart, stop and
|
||||
destroy invalidate old recognizer completions and discard obsolete queued audio.
|
||||
Each recorder also checks its own identity, rejecting messages from retired ports.
|
||||
|
||||
Pause detaches the recorder and cancels the interval. The direct media source
|
||||
continues through its playback gain to the destination. A source switch reuses the
|
||||
direct source and replaces the recorder; a captured-stream fallback is closed and
|
||||
reacquired because captured tracks can end on source changes.
|
||||
|
||||
`stop()` returns `Promise<void>` and releases ASR work. An already displayed
|
||||
subtitle retains its original auto-hide deadline; obsolete callbacks cannot add timers.
|
||||
It retains playback subscriptions and allows explicit append or a later play.
|
||||
For a directly bound video, the player keeps its original source/context and audio
|
||||
route until `art.destroy()`. Closing that context at every stop caused Chromium
|
||||
to lose audio on restart; a media element cannot simply bind a replacement source.
|
||||
For a captured fallback, stop closes its disposable context and stops its tracks.
|
||||
Before direct binding, failed/cancelled initialization closes its context and
|
||||
revokes the Worklet URL. Destroy closes all contexts, disconnects nodes, stops
|
||||
captured tracks, cancels subtitle timers and removes subscriptions.
|
||||
|
||||
This lifetime follows the [Web Audio close semantics](https://www.w3.org/TR/webaudio-1.0/#dom-audiocontext-close)
|
||||
and [media-element source behavior](https://www.w3.org/TR/webaudio-1.0/#MediaElementAudioSourceNode).
|
||||
Captured-stream source changes follow [Media Capture from DOM Elements](https://www.w3.org/TR/mediacapture-fromelement/#html-media-element-media-capture-extensions).
|
||||
|
||||
Queued audio is limited to the larger of 60 seconds or two configured chunks. A caller that
|
||||
cannot keep up gets an explicit console error and capture pauses, leaving normal
|
||||
playback routed. This avoids an unbounded backlog; it does not claim lossless
|
||||
recognition under overload. A later play can restart capture. The caller owns
|
||||
cancellation of its own network requests; obsolete completions cannot repaint.
|
||||
|
||||
## Compatibility and verification
|
||||
|
||||
The factory still snapshots options and registers synchronously, returning
|
||||
`name`, `stop`, `hide`, `append`. There is no public start method. Append preserves
|
||||
HTML, punctuation splitting, replacement, ignored non-string values and the
|
||||
historical `length=0` behavior. The CJS function has a non-enumerable self `default`
|
||||
alias for consumers of the 2.0 namespace, while preserving the callable 2.1 form.
|
||||
|
||||
Root public declarations have not yet migrated: they describe stop as void and
|
||||
the callback as void/Promise<void>. PKG-ASR-04 owns exact asynchronous public types,
|
||||
old assignment consumers, module forms and installed tarball acceptance. Internal
|
||||
strict TypeScript does not prove that public type migration complete.
|
||||
|
||||
From the repository root:
|
||||
|
||||
- `yarn test:asr`: frozen public/audio observations, queue/encoding parity and
|
||||
candidate lifecycle regressions. Historical defects remain separate tests.
|
||||
- `yarn test:browser test/browser/asr-audio.spec.js`: local AAC, actual Worklet,
|
||||
published/current core, pause/resume, source switch, stop/restart and destroy.
|
||||
Set `ARTPLAYER_ASR_ARTIFACT` to an actual main/legacy file to test that artifact.
|
||||
- `yarn typecheck`: includes the package strict/noUncheckedIndexedAccess project.
|
||||
- `yarn dev artplayer-plugin-asr` and `yarn build artplayer-plugin-asr`: normal
|
||||
repository development and three-format production builds.
|
||||
|
||||
Windows Playwright WebKit 26.6 lacks AudioContext and AudioWorkletNode. Its native
|
||||
ASR cases remain explicit skips and do not establish Safari/device compatibility.
|
||||
Real fallback track replacement, native volume/mute, CORS, unsupported media and
|
||||
physical devices remain PKG-ASR-05 acceptance work. Do not use the demo's external
|
||||
recognizer service as a prerequisite for local audio verification.
|
||||
@@ -3,6 +3,11 @@
|
||||
Audio capture and subtitle display plugin for ArtPlayer. Supplies PCM/WAV chunks
|
||||
to a caller-provided recognition service.
|
||||
|
||||
`stop()` stops recognition and returns a Promise. It permits later playback to
|
||||
restart capture; the player's direct Web Audio connection remains alive until
|
||||
the player is destroyed so that stopping recognition does not silence playback.
|
||||
Call `art.destroy()` when disposing the player.
|
||||
|
||||
## Maintenance
|
||||
|
||||
See [ARCHITECTURE.md](ARCHITECTURE.md) for the current audio flow, public behavior,
|
||||
|
||||
@@ -4,4 +4,4 @@
|
||||
* (c) 2017-2026 Harvey Zhao
|
||||
* Released under the MIT License.
|
||||
*/
|
||||
!function(t,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"==typeof define&&define.amd?(t.artplayerPluginAsr=e(),define(function(){return t.artplayerPluginAsr})):(t="undefined"!==typeof globalThis?globalThis:t||self).artplayerPluginAsr=e()}(this,function(){"use strict";if("undefined"!==typeof document&&!document.getElementById("artplayer-plugin-asr")){const t=document.createElement("style");t.id="artplayer-plugin-asr",t.textContent=".art-video-player .art-layer-asr{pointer-events:none!important;position:absolute;z-index:150;inset:0;display:flex;flex-direction:column;justify-content:end;padding:0 2%;font-size:1rem;gap:var(--art-subtitle-gap);padding-bottom:var(--art-subtitle-bottom);transition:padding-bottom var(--art-transition-duration) ease;text-shadow:var(--art-subtitle-border) 1px 0 1px,var(--art-subtitle-border) 0 1px 1px,var(--art-subtitle-border) -1px 0 1px,var(--art-subtitle-border) 0 -1px 1px,var(--art-subtitle-border) 1px 1px 1px,var(--art-subtitle-border) -1px -1px 1px,var(--art-subtitle-border) 1px -1px 1px,var(--art-subtitle-border) -1px 1px 1px}.art-video-player.art-control-show .art-layer-asr{padding-bottom:calc(var(--art-control-height) + var(--art-subtitle-bottom))}",document.head.appendChild(t)}return function(t={}){const{length:e=3,interval:n=100,sampleRate:r=16e3,autoHideTimeout:a=1e4,onAudioChunk:o=()=>null}=t;return t=>{let i=!1,l=null,s=null,u=null,c=null,d=[],p=null,f=!1,y=null,m=null,g=null;const h=t.layers.add({name:"asr",html:""});function w(){h.style.display="none"}function b(t){var n;"string"===typeof t&&(clearTimeout(y),y=setTimeout(w,a),h.style.display="",h.innerHTML=(n=t,n.split(/(?<=[\u3001\u3002\uff01\uff1f!?.])\s*/u).map(t=>t.trim()).filter(Boolean).slice(-e)).map(t=>`<div class="art-asr-line">${t}</div>`).join(""))}async function v(){if(!s&&t.video)try{return s=l.createMediaElementSource(t.video),s}catch(e){console.warn("[artplayerPluginAsr] Direct connection failed:",e)}try{if(m||(m=await async function(t){try{return t.captureStream?t.captureStream():t.mozCaptureStream?t.mozCaptureStream():(console.warn("[artplayerPluginAsr] captureStream not supported"),null)}catch(e){return console.warn("[artplayerPluginAsr] Error capturing stream:",e),null}}(t.video)),m)return g=l.createMediaStreamSource(m),g}catch(e){console.warn("[artplayerPluginAsr] MediaStream fallback failed:",e)}return null}async function x(){i&&(i=!1,clearInterval(p),p=null,u&&(u.disconnect(),u.port.onmessage=null,u=null),c&&(c.disconnect(),c=null),d=[])}async function A(){await x(),g&&(g.disconnect(),g=null),s&&(s.disconnect(),s=null),m&&(m.getTracks().forEach(t=>t.stop()),m=null),l&&(await l.close(),l=null),f=!1}return t.on("video:volumechange",()=>{c&&(c.gain.value=t.volume)}),t.on("play",async function(){if(!i)try{await async function(){return l||(l=new(window.AudioContext||window.webkitAudioContext)({sampleRate:r}),"suspended"===l.state&&await l.resume(),l)}();const t=await v();if(!t)throw new Error("Could not establish audio source");if(!f){const t=(()=>{const t=new Blob(["\n class RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n }\n registerProcessor('recorder-processor', RecorderProcessor);\n "],{type:"application/javascript"});return URL.createObjectURL(t)})();await l.audioWorklet.addModule(t),URL.revokeObjectURL(t),f=!0}c=l.createGain(),c.gain.value=1,u=new AudioWorkletNode(l,"recorder-processor"),u.port.onmessage=t=>{d.push(new Float32Array(t.data))},t.connect(u),t.connect(c),c.connect(l.destination);const e=r*(n/1e3);p=setInterval(async()=>{if(0===d.length)return;let t=new Float32Array(0);for(;t.length<e&&d.length;){const e=d.shift(),n=new Float32Array(t.length+e.length);n.set(t,0),n.set(e,t.length),t=n}if(t.length<e)return;const n=function(t){const e=t.length,n=new ArrayBuffer(2*e),r=new DataView(n);for(let a=0;a<e;a++){const e=Math.max(-1,Math.min(1,t[a]));r.setInt16(2*a,e<0?32768*e:32767*e,!0)}return n}(t.slice(0,e)),a=function(t,e){const n=t.byteLength,r=new ArrayBuffer(44+n),a=new DataView(r),o=(t,e)=>{for(let n=0;n<e.length;n++)a.setUint8(t+n,e.charCodeAt(n))};return o(0,"RIFF"),a.setUint32(4,36+n,!0),o(8,"WAVE"),o(12,"fmt "),a.setUint32(16,16,!0),a.setUint16(20,1,!0),a.setUint16(22,1,!0),a.setUint32(24,e,!0),a.setUint32(28,2*e,!0),a.setUint16(32,2,!0),a.setUint16(34,16,!0),o(36,"data"),a.setUint32(40,n,!0),new Uint8Array(r).set(new Uint8Array(t),44),r}(n,r);b(await o({pcm:n,wav:a}))},n),i=!0}catch(t){console.error("[artplayerPluginAsr] Initialization failed:",t),await x()}}),t.on("pause",x),t.on("destroy",A),{name:"artplayerPluginAsr",stop:A,hide:w,append:b}}}});
|
||||
!function(t,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"==typeof define&&define.amd?(t.artplayerPluginAsr=e(),define(function(){return t.artplayerPluginAsr})):(t="undefined"!==typeof globalThis?globalThis:t||self).artplayerPluginAsr=e()}(this,function(){"use strict";class t{constructor(t,e){this.video=t,this.sampleRate=e,this.closed=!1,this.direct=!1,this.workletLoaded=!1}assertOpen(){if(this.closed)throw new Error("Audio capture was closed")}revokeWorklet(){void 0!==this.blobUrl&&(URL.revokeObjectURL(this.blobUrl),this.blobUrl=void 0)}async initialize(){this.assertOpen();const t=window,e=window.AudioContext||t.webkitAudioContext;if(!e)throw new Error("AudioContext is not supported");const i=this.context??(this.context=new e({sampleRate:this.sampleRate}));if("suspended"===i.state&&await i.resume(),this.assertOpen(),!this.workletLoaded){this.blobUrl=URL.createObjectURL(new Blob(["\nclass RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n}\nregisterProcessor('recorder-processor', RecorderProcessor);\n"],{type:"application/javascript"}));try{await i.audioWorklet.addModule(this.blobUrl),this.workletLoaded=!0}finally{this.revokeWorklet()}}if(this.assertOpen(),!this.source)try{this.source=i.createMediaElementSource(this.video),this.direct=!0}catch(s){console.warn("[artplayerPluginAsr] Direct connection failed:",s);const t=this.video.captureStream||this.video.mozCaptureStream;if(!t)throw new Error("Could not establish audio source");this.stream=t.call(this.video),this.source=i.createMediaStreamSource(this.stream)}}prepare(){return this.preparation??(this.preparation=this.initialize().catch(t=>{throw this.preparation=void 0,t})),this.preparation}get ownsMediaConnection(){return this.direct}idle(){const t=this.gain;this.disconnect(),this.direct&&this.context&&this.source&&(this.gain=t||this.context.createGain(),t||(this.gain.gain.value=1),this.source.connect(this.gain),this.gain.connect(this.context.destination))}connect(t){this.assertOpen();const e=this.context,i=this.source;if(!e||!i)throw new Error("Audio capture is not prepared");const s=this.gain;this.disconnect();const n=this.gain=s||e.createGain();s||(n.gain.value=1);const r=this.recorder=new AudioWorkletNode(e,"recorder-processor");r.port.onmessage=e=>{this.closed||this.recorder!==r||t(e.data)},i.connect(r),i.connect(n),n.connect(e.destination)}volume(t){this.gain&&(this.gain.gain.value=t)}disconnect(){this.recorder&&(this.recorder.port.onmessage=null,this.recorder.disconnect(),this.recorder=void 0),this.gain?.disconnect(),this.gain=void 0,this.source?.disconnect()}async close(){if(this.closed)return;this.closed=!0,this.disconnect(),this.revokeWorklet(),this.stream?.getTracks().forEach(t=>t.stop()),this.stream=void 0;const t=this.context;this.context=void 0,this.source=void 0,t&&"closed"!==t.state&&await t.close()}}class e{constructor(){this.offset=0,this.size=0}get length(){return this.size}push(t){if(0===t.length)return;const e={samples:new Float32Array(t)};this.tail?this.tail.next=e:this.head=e,this.tail=e,this.size+=t.length}take(t){if(!Number.isSafeInteger(t)||t<=0)throw new RangeError("Sample count must be a positive safe integer");if(t>this.size)return null;const e=new Float32Array(t);let i=0;for(;this.head&&i<t;){const s=this.head.samples,n=Math.min(s.length-this.offset,t-i);e.set(s.subarray(this.offset,this.offset+n),i),i+=n,this.offset+=n,this.offset===s.length&&(this.head=this.head.next,this.offset=0)}return this.head||(this.tail=void 0),this.size-=t,e}clear(){this.head=void 0,this.tail=void 0,this.offset=0,this.size=0}}class i{constructor(t,i,s){this.video=t,this.options=i,this.append=s,this.queue=new e,this.closing=Promise.resolve(),this.running=!1,this.terminal=!1,this.epoch=0}reset(){this.epoch++,this.pending=void 0,this.queue.clear()}pause(){this.reset(),this.running=!1,clearInterval(this.timer),this.timer=void 0,this.starting=void 0,this.graph?.idle()}restart(){const t=this.running||Boolean(this.starting);return this.pause(),this.graph&&!this.graph.ownsMediaConnection?this.stop().then(()=>t?this.start():void 0):t?this.start():Promise.resolve()}start(){if(this.terminal||this.running)return Promise.resolve();if(this.starting)return this.starting;const e=this.epoch,i=()=>!this.terminal&&this.epoch===e,s=(async()=>{if(await this.closing,!i())return;const{sampleRate:e,interval:s}=this.options,n=Math.floor(e*s/1e3);if(!Number.isSafeInteger(n)||n<1||!Number.isFinite(s)||s<=0)throw new Error("Audio chunk length must be positive and finite");const r=this.graph??(this.graph=new t(this.video,e));await r.prepare(),i()?(r.connect(t=>{if(this.running&&this.graph===r)return this.queue.length+t.length>Math.max(60*e,2*n)?(console.error("[artplayerPluginAsr] Audio callback backlog exceeded its capture limit"),void this.pause()):void this.queLine truncated
|
||||
@@ -4,4 +4,4 @@
|
||||
* (c) 2017-2026 Harvey Zhao
|
||||
* Released under the MIT License.
|
||||
*/
|
||||
!function(t,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"==typeof define&&define.amd?(t.artplayerPluginAsr=e(),define(function(){return t.artplayerPluginAsr})):(t="undefined"!==typeof globalThis?globalThis:t||self).artplayerPluginAsr=e()}(this,function(){"use strict";var t=(t,e,n)=>new Promise((r,o)=>{var a=t=>{try{l(n.next(t))}catch(e){o(e)}},i=t=>{try{l(n.throw(t))}catch(e){o(e)}},l=t=>t.done?r(t.value):Promise.resolve(t.value).then(a,i);l((n=n.apply(t,e)).next())});if("undefined"!==typeof document&&!document.getElementById("artplayer-plugin-asr")){const t=document.createElement("style");t.id="artplayer-plugin-asr",t.textContent=".art-video-player .art-layer-asr{pointer-events:none!important;position:absolute;z-index:150;inset:0;display:flex;flex-direction:column;justify-content:end;padding:0 2%;font-size:1rem;gap:var(--art-subtitle-gap);padding-bottom:var(--art-subtitle-bottom);transition:padding-bottom var(--art-transition-duration) ease;text-shadow:var(--art-subtitle-border) 1px 0 1px,var(--art-subtitle-border) 0 1px 1px,var(--art-subtitle-border) -1px 0 1px,var(--art-subtitle-border) 0 -1px 1px,var(--art-subtitle-border) 1px 1px 1px,var(--art-subtitle-border) -1px -1px 1px,var(--art-subtitle-border) 1px -1px 1px,var(--art-subtitle-border) -1px 1px 1px}.art-video-player.art-control-show .art-layer-asr{padding-bottom:calc(var(--art-control-height) + var(--art-subtitle-bottom))}",document.head.appendChild(t)}return function(e={}){const{length:n=3,interval:r=100,sampleRate:o=16e3,autoHideTimeout:a=1e4,onAudioChunk:i=()=>null}=e;return e=>{let l=!1,s=null,u=null,c=null,d=null,p=[],f=null,y=!1,m=null,h=null,g=null;const v=e.layers.add({name:"asr",html:""});function b(){v.style.display="none"}function x(t){var e;"string"===typeof t&&(clearTimeout(m),m=setTimeout(b,a),v.style.display="",v.innerHTML=(e=t,e.split(new RegExp("(?<=[\u3001\u3002\uff01\uff1f!?.])\\s*","u")).map(t=>t.trim()).filter(Boolean).slice(-n)).map(t=>`<div class="art-asr-line">${t}</div>`).join(""))}function w(){return t(this,null,function*(){if(!u&&e.video)try{return u=s.createMediaElementSource(e.video),u}catch(n){console.warn("[artplayerPluginAsr] Direct connection failed:",n)}try{if(h||(h=yield function(e){return t(this,null,function*(){try{return e.captureStream?e.captureStream():e.mozCaptureStream?e.mozCaptureStream():(console.warn("[artplayerPluginAsr] captureStream not supported"),null)}catch(n){return console.warn("[artplayerPluginAsr] Error capturing stream:",n),null}})}(e.video)),h)return g=s.createMediaStreamSource(h),g}catch(n){console.warn("[artplayerPluginAsr] MediaStream fallback failed:",n)}return null})}function A(){return t(this,null,function*(){l&&(l=!1,clearInterval(f),f=null,c&&(c.disconnect(),c.port.onmessage=null,c=null),d&&(d.disconnect(),d=null),p=[])})}function U(){return t(this,null,function*(){yield A(),g&&(g.disconnect(),g=null),u&&(u.disconnect(),u=null),h&&(h.getTracks().forEach(t=>t.stop()),h=null),s&&(yield s.close(),s=null),y=!1})}return e.on("video:volumechange",()=>{d&&(d.gain.value=e.volume)}),e.on("play",function(){return t(this,null,function*(){if(!l)try{yield function(){return t(this,null,function*(){return s||(s=new(window.AudioContext||window.webkitAudioContext)({sampleRate:o}),"suspended"===s.state&&(yield s.resume()),s)})}();const e=yield w();if(!e)throw new Error("Could not establish audio source");if(!y){const t=(()=>{const t=new Blob(["\n class RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n }\n registerProcessor('recorder-processor', RecorderProcessor);\n "],{type:"application/javascript"});return URL.createObjectURL(t)})();yield s.audioWorklet.addModule(t),URL.revokeObjectURL(t),y=!0}d=s.createGain(),d.gain.value=1,c=new AudioWorkletNode(s,"recorder-processor"),c.port.onmessage=t=>{p.push(new Float32Array(t.data))},e.connect(c),e.connect(d),d.connect(s.destination);const n=o*(r/1e3);f=setInterval(()=>t(null,null,function*(){if(0===p.length)return;let t=new Float32Array(0);for(;t.length<n&&p.length;){const e=p.shift(),n=new Float32Array(t.length+e.length);n.set(t,0),n.set(e,t.length),t=n}if(t.length<n)return;const e=function(t){const e=t.length,n=new ArrayBuffer(2*e),r=new DataView(n);for(let o=0;o<e;o++){const e=Math.max(-1,Math.min(1,t[o]));r.setInt16(2*o,e<0?32768*e:32767*e,!0)}return n}(t.slice(0,n)),r=function(t,e){const n=t.byteLength,r=new ArrayBuffer(44+n),o=new DataView(r),a=(t,e)=>{for(let n=0;n<e.length;n++)o.setUint8(t+n,e.charCodeAt(n))};return a(0,"RIFF"),o.setUint32(4,36+n,!0),a(8,"WAVE"),a(12,"fmt "),o.setUint32(16,16,!0),o.setUint16(20,1,!0),o.setUint16(22,1,!0),o.setUint32(24,e,!0),o.setUint32(28,2*e,!0),o.setUint16(32,2,!0),o.setUintLine truncated
|
||||
!function(t,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"==typeof define&&define.amd?(t.artplayerPluginAsr=e(),define(function(){return t.artplayerPluginAsr})):(t="undefined"!==typeof globalThis?globalThis:t||self).artplayerPluginAsr=e()}(this,function(){"use strict";var t=(t,e,i)=>new Promise((s,n)=>{var r=t=>{try{a(i.next(t))}catch(e){n(e)}},o=t=>{try{a(i.throw(t))}catch(e){n(e)}},a=t=>t.done?s(t.value):Promise.resolve(t.value).then(r,o);a((i=i.apply(t,e)).next())});class e{constructor(t,e){this.video=t,this.sampleRate=e,this.closed=!1,this.direct=!1,this.workletLoaded=!1}assertOpen(){if(this.closed)throw new Error("Audio capture was closed")}revokeWorklet(){void 0!==this.blobUrl&&(URL.revokeObjectURL(this.blobUrl),this.blobUrl=void 0)}initialize(){return t(this,null,function*(){var t;this.assertOpen();const e=window,i=window.AudioContext||e.webkitAudioContext;if(!i)throw new Error("AudioContext is not supported");const s=null!=(t=this.context)?t:this.context=new i({sampleRate:this.sampleRate});if("suspended"===s.state&&(yield s.resume()),this.assertOpen(),!this.workletLoaded){this.blobUrl=URL.createObjectURL(new Blob(["\nclass RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n}\nregisterProcessor('recorder-processor', RecorderProcessor);\n"],{type:"application/javascript"}));try{yield s.audioWorklet.addModule(this.blobUrl),this.workletLoaded=!0}finally{this.revokeWorklet()}}if(this.assertOpen(),!this.source)try{this.source=s.createMediaElementSource(this.video),this.direct=!0}catch(n){console.warn("[artplayerPluginAsr] Direct connection failed:",n);const t=this.video.captureStream||this.video.mozCaptureStream;if(!t)throw new Error("Could not establish audio source");this.stream=t.call(this.video),this.source=s.createMediaStreamSource(this.stream)}})}prepare(){return null!=this.preparation||(this.preparation=this.initialize().catch(t=>{throw this.preparation=void 0,t})),this.preparation}get ownsMediaConnection(){return this.direct}idle(){const t=this.gain;this.disconnect(),this.direct&&this.context&&this.source&&(this.gain=t||this.context.createGain(),t||(this.gain.gain.value=1),this.source.connect(this.gain),this.gain.connect(this.context.destination))}connect(t){this.assertOpen();const e=this.context,i=this.source;if(!e||!i)throw new Error("Audio capture is not prepared");const s=this.gain;this.disconnect();const n=this.gain=s||e.createGain();s||(n.gain.value=1);const r=this.recorder=new AudioWorkletNode(e,"recorder-processor");r.port.onmessage=e=>{this.closed||this.recorder!==r||t(e.data)},i.connect(r),i.connect(n),n.connect(e.destination)}volume(t){this.gain&&(this.gain.gain.value=t)}disconnect(){var t,e;this.recorder&&(this.recorder.port.onmessage=null,this.recorder.disconnect(),this.recorder=void 0),null==(t=this.gain)||t.disconnect(),this.gain=void 0,null==(e=this.source)||e.disconnect()}close(){return t(this,null,function*(){var t;if(this.closed)return;this.closed=!0,this.disconnect(),this.revokeWorklet(),null==(t=this.stream)||t.getTracks().forEach(t=>t.stop()),this.stream=void 0;const e=this.context;this.context=void 0,this.source=void 0,e&&"closed"!==e.state&&(yield e.close())})}}class i{constructor(){this.offset=0,this.size=0}get length(){return this.size}push(t){if(0===t.length)return;const e={samples:new Float32Array(t)};this.tail?this.tail.next=e:this.head=e,this.tail=e,this.size+=t.length}take(t){if(!Number.isSafeInteger(t)||t<=0)throw new RangeError("Sample count must be a positive safe integer");if(t>this.size)return null;const e=new Float32Array(t);let i=0;for(;this.head&&i<t;){const s=this.head.samples,n=Math.min(s.length-this.offset,t-i);e.set(s.subarray(this.offset,this.offset+n),i),i+=n,this.offset+=n,this.offset===s.length&&(this.head=this.head.next,this.offset=0)}return this.head||(this.tail=void 0),this.size-=t,e}clear(){this.head=void 0,this.tail=void 0,this.offset=0,this.size=0}}class s{constructor(t,e,s){this.video=t,this.options=e,this.append=s,this.queue=new i,this.closing=Promise.resolve(),this.running=!1,this.terminal=!1,this.epoch=0}reset(){this.epoch++,this.pending=void 0,this.queue.clear()}pause(){var t;this.reset(),this.running=!1,clearInterval(this.timer),this.timer=void 0,this.starting=void 0,null==(t=this.graph)||t.idle()}restart(){const t=this.running||Boolean(this.starting);return this.pause(),this.graph&&!this.graph.ownsMediaConnection?this.stop().then(()=>t?this.start():void 0):t?this.start():Promise.resolve()}start(){if(this.terminal||this.running)return Promise.resolve();if(this.starting)return this.starting;const i=this.epoch,s=()=>!this.terminal&&this.epoch===i,n=(()=>t(this,null,function*(){var t;if(yield this.closing,!s())return;const{sampleRate:i,interval:n}=this.options,r=Math.floor(i*n/1e3);if(!Number.isSafeInteger(r)||r<1||!Number.isFiLine truncated
|
||||
+374
-228
@@ -4,248 +4,394 @@
|
||||
* (c) 2017-2026 Harvey Zhao
|
||||
* Released under the MIT License.
|
||||
*/
|
||||
const style = ".art-video-player .art-layer-asr {\n pointer-events: none !important;\n position: absolute;\n z-index: 150;\n inset: 0px;\n display: flex;\n flex-direction: column;\n justify-content: end;\n padding: 0 2%;\n font-size: 1rem;\n gap: var(--art-subtitle-gap);\n padding-bottom: var(--art-subtitle-bottom);\n transition: padding-bottom var(--art-transition-duration) ease;\n text-shadow: var(--art-subtitle-border) 1px 0 1px, var(--art-subtitle-border) 0 1px 1px, var(--art-subtitle-border) -1px 0 1px, var(--art-subtitle-border) 0 -1px 1px, var(--art-subtitle-border) 1px 1px 1px, var(--art-subtitle-border) -1px -1px 1px, var(--art-subtitle-border) 1px -1px 1px, var(--art-subtitle-border) -1px 1px 1px;\n}\n.art-video-player.art-control-show .art-layer-asr {\n padding-bottom: calc(var(--art-control-height) + var(--art-subtitle-bottom));\n}\n";
|
||||
function artplayerPluginAsr(option = {}) {
|
||||
const {
|
||||
length = 3,
|
||||
interval = 100,
|
||||
sampleRate = 16e3,
|
||||
autoHideTimeout = 1e4,
|
||||
onAudioChunk = () => null
|
||||
} = option;
|
||||
return (art) => {
|
||||
let started = false;
|
||||
let audioCtx = null;
|
||||
let sourceNode = null;
|
||||
let recorderNode = null;
|
||||
let gainNode = null;
|
||||
let bufferChunks = [];
|
||||
let timer = null;
|
||||
let workletLoaded = false;
|
||||
let hideTimer = null;
|
||||
let mediaStream = null;
|
||||
let mediaStreamSource = null;
|
||||
const $asr = art.layers.add({
|
||||
name: "asr",
|
||||
html: ""
|
||||
});
|
||||
function splitByPunctuation(text) {
|
||||
return text.split(/(?<=[、。！？!?.])\s*/u).map((s) => s.trim()).filter(Boolean).slice(-length);
|
||||
}
|
||||
function hide() {
|
||||
$asr.style.display = "none";
|
||||
}
|
||||
function append(subtitle) {
|
||||
if (typeof subtitle !== "string")
|
||||
return;
|
||||
clearTimeout(hideTimer);
|
||||
hideTimer = setTimeout(hide, autoHideTimeout);
|
||||
$asr.style.display = "";
|
||||
$asr.innerHTML = splitByPunctuation(subtitle).map((line) => `<div class="art-asr-line">${line}</div>`).join("");
|
||||
}
|
||||
const recorderProcessorCode = `
|
||||
class RecorderProcessor extends AudioWorkletProcessor {
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (input && input[0]) {
|
||||
this.port.postMessage(input[0]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('recorder-processor', RecorderProcessor);
|
||||
`;
|
||||
const createWorkletBlobUrl = () => {
|
||||
const blob = new Blob([recorderProcessorCode], { type: "application/javascript" });
|
||||
return URL.createObjectURL(blob);
|
||||
};
|
||||
function floatTo16BitPCM(float32Array) {
|
||||
const len = float32Array.length;
|
||||
const buffer = new ArrayBuffer(len * 2);
|
||||
const view = new DataView(buffer);
|
||||
for (let i = 0; i < len; i++) {
|
||||
const s = Math.max(-1, Math.min(1, float32Array[i]));
|
||||
view.setInt16(i * 2, s < 0 ? s * 32768 : s * 32767, true);
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
function pcmToWav(pcmBuffer, sampleRate2) {
|
||||
const pcmLength = pcmBuffer.byteLength;
|
||||
const wavBuffer = new ArrayBuffer(44 + pcmLength);
|
||||
const view = new DataView(wavBuffer);
|
||||
const writeString = (offset, str) => {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
const recorderProcessorCode = `
|
||||
class RecorderProcessor extends AudioWorkletProcessor {
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (input && input[0]) {
|
||||
this.port.postMessage(input[0]);
|
||||
}
|
||||
};
|
||||
writeString(0, "RIFF");
|
||||
view.setUint32(4, 36 + pcmLength, true);
|
||||
writeString(8, "WAVE");
|
||||
writeString(12, "fmt ");
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, 1, true);
|
||||
view.setUint32(24, sampleRate2, true);
|
||||
view.setUint32(28, sampleRate2 * 2, true);
|
||||
view.setUint16(32, 2, true);
|
||||
view.setUint16(34, 16, true);
|
||||
writeString(36, "data");
|
||||
view.setUint32(40, pcmLength, true);
|
||||
new Uint8Array(wavBuffer).set(new Uint8Array(pcmBuffer), 44);
|
||||
return wavBuffer;
|
||||
return true;
|
||||
}
|
||||
async function setupAudioContext() {
|
||||
if (audioCtx)
|
||||
return audioCtx;
|
||||
audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate });
|
||||
if (audioCtx.state === "suspended")
|
||||
await audioCtx.resume();
|
||||
return audioCtx;
|
||||
}
|
||||
registerProcessor('recorder-processor', RecorderProcessor);
|
||||
`;
|
||||
class AudioGraph {
|
||||
constructor(video, sampleRate) {
|
||||
this.video = video;
|
||||
this.sampleRate = sampleRate;
|
||||
this.closed = false;
|
||||
this.direct = false;
|
||||
this.workletLoaded = false;
|
||||
}
|
||||
assertOpen() {
|
||||
if (this.closed)
|
||||
throw new Error("Audio capture was closed");
|
||||
}
|
||||
revokeWorklet() {
|
||||
if (this.blobUrl !== void 0) {
|
||||
URL.revokeObjectURL(this.blobUrl);
|
||||
this.blobUrl = void 0;
|
||||
}
|
||||
async function setupAudioSource() {
|
||||
if (!sourceNode && art.video) {
|
||||
try {
|
||||
sourceNode = audioCtx.createMediaElementSource(art.video);
|
||||
return sourceNode;
|
||||
} catch (err) {
|
||||
console.warn("[artplayerPluginAsr] Direct connection failed:", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
async initialize() {
|
||||
this.assertOpen();
|
||||
const platform = window;
|
||||
const Context = window.AudioContext || platform.webkitAudioContext;
|
||||
if (!Context)
|
||||
throw new Error("AudioContext is not supported");
|
||||
const context = this.context ?? (this.context = new Context({ sampleRate: this.sampleRate }));
|
||||
if (context.state === "suspended")
|
||||
await context.resume();
|
||||
this.assertOpen();
|
||||
if (!this.workletLoaded) {
|
||||
this.blobUrl = URL.createObjectURL(new Blob([recorderProcessorCode], { type: "application/javascript" }));
|
||||
try {
|
||||
if (!mediaStream) {
|
||||
mediaStream = await captureAudioFromVideo(art.video);
|
||||
}
|
||||
if (mediaStream) {
|
||||
mediaStreamSource = audioCtx.createMediaStreamSource(mediaStream);
|
||||
return mediaStreamSource;
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn("[artplayerPluginAsr] MediaStream fallback failed:", err);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
async function captureAudioFromVideo(videoElement) {
|
||||
try {
|
||||
if (videoElement.captureStream)
|
||||
return videoElement.captureStream();
|
||||
if (videoElement.mozCaptureStream)
|
||||
return videoElement.mozCaptureStream();
|
||||
console.warn("[artplayerPluginAsr] captureStream not supported");
|
||||
return null;
|
||||
} catch (err) {
|
||||
console.warn("[artplayerPluginAsr] Error capturing stream:", err);
|
||||
return null;
|
||||
await context.audioWorklet.addModule(this.blobUrl);
|
||||
this.workletLoaded = true;
|
||||
} finally {
|
||||
this.revokeWorklet();
|
||||
}
|
||||
}
|
||||
async function startCapture() {
|
||||
if (started)
|
||||
return;
|
||||
this.assertOpen();
|
||||
if (!this.source) {
|
||||
try {
|
||||
await setupAudioContext();
|
||||
const audioSource = await setupAudioSource();
|
||||
if (!audioSource)
|
||||
this.source = context.createMediaElementSource(this.video);
|
||||
this.direct = true;
|
||||
} catch (error) {
|
||||
console.warn("[artplayerPluginAsr] Direct connection failed:", error);
|
||||
const capture = this.video.captureStream || this.video.mozCaptureStream;
|
||||
if (!capture)
|
||||
throw new Error("Could not establish audio source");
|
||||
if (!workletLoaded) {
|
||||
const blobUrl = createWorkletBlobUrl();
|
||||
await audioCtx.audioWorklet.addModule(blobUrl);
|
||||
URL.revokeObjectURL(blobUrl);
|
||||
workletLoaded = true;
|
||||
}
|
||||
gainNode = audioCtx.createGain();
|
||||
gainNode.gain.value = 1;
|
||||
recorderNode = new AudioWorkletNode(audioCtx, "recorder-processor");
|
||||
recorderNode.port.onmessage = (event) => {
|
||||
bufferChunks.push(new Float32Array(event.data));
|
||||
};
|
||||
audioSource.connect(recorderNode);
|
||||
audioSource.connect(gainNode);
|
||||
gainNode.connect(audioCtx.destination);
|
||||
const CHUNK_SAMPLES = sampleRate * (interval / 1e3);
|
||||
timer = setInterval(async () => {
|
||||
if (bufferChunks.length === 0)
|
||||
return;
|
||||
let accumulated = new Float32Array(0);
|
||||
while (accumulated.length < CHUNK_SAMPLES && bufferChunks.length) {
|
||||
const next = bufferChunks.shift();
|
||||
const tmp = new Float32Array(accumulated.length + next.length);
|
||||
tmp.set(accumulated, 0);
|
||||
tmp.set(next, accumulated.length);
|
||||
accumulated = tmp;
|
||||
}
|
||||
if (accumulated.length < CHUNK_SAMPLES)
|
||||
return;
|
||||
const chunkToSend = accumulated.slice(0, CHUNK_SAMPLES);
|
||||
const pcm = floatTo16BitPCM(chunkToSend);
|
||||
const wav = pcmToWav(pcm, sampleRate);
|
||||
const subtitle = await onAudioChunk({ pcm, wav });
|
||||
append(subtitle);
|
||||
}, interval);
|
||||
started = true;
|
||||
} catch (err) {
|
||||
console.error("[artplayerPluginAsr] Initialization failed:", err);
|
||||
await stopCapture();
|
||||
this.stream = capture.call(this.video);
|
||||
this.source = context.createMediaStreamSource(this.stream);
|
||||
}
|
||||
}
|
||||
async function stopCapture() {
|
||||
if (!started)
|
||||
return;
|
||||
started = false;
|
||||
clearInterval(timer);
|
||||
timer = null;
|
||||
if (recorderNode) {
|
||||
recorderNode.disconnect();
|
||||
recorderNode.port.onmessage = null;
|
||||
recorderNode = null;
|
||||
}
|
||||
if (gainNode) {
|
||||
gainNode.disconnect();
|
||||
gainNode = null;
|
||||
}
|
||||
bufferChunks = [];
|
||||
}
|
||||
prepare() {
|
||||
this.preparation ?? (this.preparation = this.initialize().catch((error) => {
|
||||
this.preparation = void 0;
|
||||
throw error;
|
||||
}));
|
||||
return this.preparation;
|
||||
}
|
||||
get ownsMediaConnection() {
|
||||
return this.direct;
|
||||
}
|
||||
idle() {
|
||||
const gain = this.gain;
|
||||
this.disconnect();
|
||||
if (this.direct && this.context && this.source) {
|
||||
this.gain = gain || this.context.createGain();
|
||||
if (!gain)
|
||||
this.gain.gain.value = 1;
|
||||
this.source.connect(this.gain);
|
||||
this.gain.connect(this.context.destination);
|
||||
}
|
||||
async function destroy() {
|
||||
await stopCapture();
|
||||
if (mediaStreamSource) {
|
||||
mediaStreamSource.disconnect();
|
||||
mediaStreamSource = null;
|
||||
}
|
||||
if (sourceNode) {
|
||||
sourceNode.disconnect();
|
||||
sourceNode = null;
|
||||
}
|
||||
if (mediaStream) {
|
||||
mediaStream.getTracks().forEach((track) => track.stop());
|
||||
mediaStream = null;
|
||||
}
|
||||
if (audioCtx) {
|
||||
await audioCtx.close();
|
||||
audioCtx = null;
|
||||
}
|
||||
workletLoaded = false;
|
||||
}
|
||||
art.on("video:volumechange", () => {
|
||||
if (gainNode) {
|
||||
gainNode.gain.value = art.volume;
|
||||
}
|
||||
});
|
||||
art.on("play", startCapture);
|
||||
art.on("pause", stopCapture);
|
||||
art.on("destroy", destroy);
|
||||
return {
|
||||
name: "artplayerPluginAsr",
|
||||
stop: destroy,
|
||||
hide,
|
||||
append
|
||||
}
|
||||
connect(receive) {
|
||||
this.assertOpen();
|
||||
const context = this.context;
|
||||
const source = this.source;
|
||||
if (!context || !source)
|
||||
throw new Error("Audio capture is not prepared");
|
||||
const previousGain = this.gain;
|
||||
this.disconnect();
|
||||
const gain = this.gain = previousGain || context.createGain();
|
||||
if (!previousGain)
|
||||
gain.gain.value = 1;
|
||||
const recorder = this.recorder = new AudioWorkletNode(context, "recorder-processor");
|
||||
recorder.port.onmessage = (event) => {
|
||||
if (!this.closed && this.recorder === recorder)
|
||||
receive(event.data);
|
||||
};
|
||||
source.connect(recorder);
|
||||
source.connect(gain);
|
||||
gain.connect(context.destination);
|
||||
}
|
||||
volume(value) {
|
||||
if (this.gain)
|
||||
this.gain.gain.value = value;
|
||||
}
|
||||
disconnect() {
|
||||
if (this.recorder) {
|
||||
this.recorder.port.onmessage = null;
|
||||
this.recorder.disconnect();
|
||||
this.recorder = void 0;
|
||||
}
|
||||
this.gain?.disconnect();
|
||||
this.gain = void 0;
|
||||
this.source?.disconnect();
|
||||
}
|
||||
async close() {
|
||||
if (this.closed)
|
||||
return;
|
||||
this.closed = true;
|
||||
this.disconnect();
|
||||
this.revokeWorklet();
|
||||
this.stream?.getTracks().forEach((track) => track.stop());
|
||||
this.stream = void 0;
|
||||
const context = this.context;
|
||||
this.context = void 0;
|
||||
this.source = void 0;
|
||||
if (context && context.state !== "closed")
|
||||
await context.close();
|
||||
}
|
||||
}
|
||||
function encodeAudio(samples, sampleRate) {
|
||||
const pcm = new ArrayBuffer(samples.length * 2);
|
||||
const pcmView = new DataView(pcm);
|
||||
for (let index = 0; index < samples.length; index++) {
|
||||
const value = Math.max(-1, Math.min(1, samples[index]));
|
||||
pcmView.setInt16(index * 2, value < 0 ? value * 32768 : value * 32767, true);
|
||||
}
|
||||
const wav = new ArrayBuffer(44 + pcm.byteLength);
|
||||
const header = new DataView(wav);
|
||||
function writeString(offset, value) {
|
||||
for (let index = 0; index < value.length; index++)
|
||||
header.setUint8(offset + index, value.charCodeAt(index));
|
||||
}
|
||||
writeString(0, "RIFF");
|
||||
header.setUint32(4, 36 + pcm.byteLength, true);
|
||||
writeString(8, "WAVE");
|
||||
writeString(12, "fmt ");
|
||||
header.setUint32(16, 16, true);
|
||||
header.setUint16(20, 1, true);
|
||||
header.setUint16(22, 1, true);
|
||||
header.setUint32(24, sampleRate, true);
|
||||
header.setUint32(28, sampleRate * 2, true);
|
||||
header.setUint16(32, 2, true);
|
||||
header.setUint16(34, 16, true);
|
||||
writeString(36, "data");
|
||||
header.setUint32(40, pcm.byteLength, true);
|
||||
new Uint8Array(wav).set(new Uint8Array(pcm), 44);
|
||||
return { pcm, wav };
|
||||
}
|
||||
class SampleQueue {
|
||||
constructor() {
|
||||
this.offset = 0;
|
||||
this.size = 0;
|
||||
}
|
||||
get length() {
|
||||
return this.size;
|
||||
}
|
||||
push(input) {
|
||||
if (input.length === 0)
|
||||
return;
|
||||
const block = { samples: new Float32Array(input) };
|
||||
if (this.tail)
|
||||
this.tail.next = block;
|
||||
else
|
||||
this.head = block;
|
||||
this.tail = block;
|
||||
this.size += input.length;
|
||||
}
|
||||
take(count) {
|
||||
if (!Number.isSafeInteger(count) || count <= 0)
|
||||
throw new RangeError("Sample count must be a positive safe integer");
|
||||
if (count > this.size)
|
||||
return null;
|
||||
const output = new Float32Array(count);
|
||||
let written = 0;
|
||||
while (this.head && written < count) {
|
||||
const samples = this.head.samples;
|
||||
const amount = Math.min(samples.length - this.offset, count - written);
|
||||
output.set(samples.subarray(this.offset, this.offset + amount), written);
|
||||
written += amount;
|
||||
this.offset += amount;
|
||||
if (this.offset === samples.length) {
|
||||
this.head = this.head.next;
|
||||
this.offset = 0;
|
||||
}
|
||||
}
|
||||
if (!this.head)
|
||||
this.tail = void 0;
|
||||
this.size -= count;
|
||||
return output;
|
||||
}
|
||||
clear() {
|
||||
this.head = void 0;
|
||||
this.tail = void 0;
|
||||
this.offset = 0;
|
||||
this.size = 0;
|
||||
}
|
||||
}
|
||||
class Capture {
|
||||
constructor(video, options, append) {
|
||||
this.video = video;
|
||||
this.options = options;
|
||||
this.append = append;
|
||||
this.queue = new SampleQueue();
|
||||
this.closing = Promise.resolve();
|
||||
this.running = false;
|
||||
this.terminal = false;
|
||||
this.epoch = 0;
|
||||
}
|
||||
reset() {
|
||||
this.epoch++;
|
||||
this.pending = void 0;
|
||||
this.queue.clear();
|
||||
}
|
||||
pause() {
|
||||
this.reset();
|
||||
this.running = false;
|
||||
clearInterval(this.timer);
|
||||
this.timer = void 0;
|
||||
this.starting = void 0;
|
||||
this.graph?.idle();
|
||||
}
|
||||
restart() {
|
||||
const active = this.running || Boolean(this.starting);
|
||||
this.pause();
|
||||
if (this.graph && !this.graph.ownsMediaConnection)
|
||||
return this.stop().then(() => active ? this.start() : void 0);
|
||||
return active ? this.start() : Promise.resolve();
|
||||
}
|
||||
start() {
|
||||
if (this.terminal || this.running)
|
||||
return Promise.resolve();
|
||||
if (this.starting)
|
||||
return this.starting;
|
||||
const epoch = this.epoch;
|
||||
const current = () => !this.terminal && this.epoch === epoch;
|
||||
const operation = (async () => {
|
||||
await this.closing;
|
||||
if (!current())
|
||||
return;
|
||||
const { sampleRate, interval } = this.options;
|
||||
const count = Math.floor(sampleRate * interval / 1e3);
|
||||
if (!Number.isSafeInteger(count) || count < 1 || !Number.isFinite(interval) || interval <= 0)
|
||||
throw new Error("Audio chunk length must be positive and finite");
|
||||
const graph = this.graph ?? (this.graph = new AudioGraph(this.video, sampleRate));
|
||||
await graph.prepare();
|
||||
if (!current()) {
|
||||
if (!this.running && this.graph === graph)
|
||||
graph.idle();
|
||||
return;
|
||||
}
|
||||
graph.connect((samples) => {
|
||||
if (!this.running || this.graph !== graph)
|
||||
return;
|
||||
if (this.queue.length + samples.length > Math.max(sampleRate * 60, count * 2)) {
|
||||
console.error("[artplayerPluginAsr] Audio callback backlog exceeded its capture limit");
|
||||
this.pause();
|
||||
return;
|
||||
}
|
||||
this.queue.push(samples);
|
||||
});
|
||||
this.running = true;
|
||||
this.timer = setInterval(() => this.tick(count), interval);
|
||||
})().catch(async (error) => {
|
||||
if (current()) {
|
||||
console.error("[artplayerPluginAsr] Initialization failed:", error);
|
||||
await this.stop();
|
||||
}
|
||||
}).finally(() => {
|
||||
if (this.starting === operation)
|
||||
this.starting = void 0;
|
||||
});
|
||||
this.starting = operation;
|
||||
return operation;
|
||||
}
|
||||
async tick(count) {
|
||||
if (!this.running || this.pending)
|
||||
return;
|
||||
const samples = this.queue.take(count);
|
||||
if (!samples)
|
||||
return;
|
||||
const epoch = this.epoch;
|
||||
const pending = this.pending = {};
|
||||
try {
|
||||
const text = await this.options.onAudioChunk(encodeAudio(samples, this.options.sampleRate));
|
||||
if (this.running && !this.terminal && this.epoch === epoch)
|
||||
this.append(text);
|
||||
} catch (error) {
|
||||
if (this.epoch === epoch && !this.terminal)
|
||||
console.error("[artplayerPluginAsr] Audio callback failed:", error);
|
||||
} finally {
|
||||
if (this.pending === pending)
|
||||
this.pending = void 0;
|
||||
}
|
||||
}
|
||||
volume(value) {
|
||||
this.graph?.volume(value);
|
||||
}
|
||||
stop(terminal = false) {
|
||||
this.terminal || (this.terminal = terminal);
|
||||
this.pause();
|
||||
const graph = this.graph;
|
||||
if (terminal || !graph?.ownsMediaConnection) {
|
||||
this.graph = void 0;
|
||||
this.closing = Promise.all([this.closing, graph?.close()]).then(() => void 0);
|
||||
}
|
||||
return this.closing;
|
||||
}
|
||||
}
|
||||
const style = ".art-video-player .art-layer-asr {\n pointer-events: none !important;\n position: absolute;\n z-index: 150;\n inset: 0px;\n display: flex;\n flex-direction: column;\n justify-content: end;\n padding: 0 2%;\n font-size: 1rem;\n gap: var(--art-subtitle-gap);\n padding-bottom: var(--art-subtitle-bottom);\n transition: padding-bottom var(--art-transition-duration) ease;\n text-shadow: var(--art-subtitle-border) 1px 0 1px, var(--art-subtitle-border) 0 1px 1px, var(--art-subtitle-border) -1px 0 1px, var(--art-subtitle-border) 0 -1px 1px, var(--art-subtitle-border) 1px 1px 1px, var(--art-subtitle-border) -1px -1px 1px, var(--art-subtitle-border) 1px -1px 1px, var(--art-subtitle-border) -1px 1px 1px;\n}\n.art-video-player.art-control-show .art-layer-asr {\n padding-bottom: calc(var(--art-control-height) + var(--art-subtitle-bottom));\n}\n";
|
||||
function createSubtitles(layer, length, autoHideTimeout) {
|
||||
let timer;
|
||||
let destroyed = false;
|
||||
function cancelTimer() {
|
||||
clearTimeout(timer);
|
||||
timer = void 0;
|
||||
}
|
||||
function hide() {
|
||||
if (!destroyed)
|
||||
layer.style.display = "none";
|
||||
}
|
||||
function append(subtitle) {
|
||||
if (destroyed || typeof subtitle !== "string")
|
||||
return;
|
||||
cancelTimer();
|
||||
timer = setTimeout(hide, autoHideTimeout);
|
||||
layer.style.display = "";
|
||||
layer.innerHTML = subtitle.split(/(?<=[、。！？!?.])\s*/u).map((line) => line.trim()).filter(Boolean).slice(-length).map((line) => `<div class="art-asr-line">${line}</div>`).join("");
|
||||
}
|
||||
return {
|
||||
append,
|
||||
hide,
|
||||
destroy() {
|
||||
destroyed = true;
|
||||
cancelTimer();
|
||||
}
|
||||
};
|
||||
}
|
||||
if (typeof document !== "undefined") {
|
||||
if (!document.getElementById("artplayer-plugin-asr")) {
|
||||
const $style = document.createElement("style");
|
||||
$style.id = "artplayer-plugin-asr";
|
||||
$style.textContent = style;
|
||||
document.head.appendChild($style);
|
||||
}
|
||||
function artplayerPluginAsr(option = {}) {
|
||||
const { length = 3, interval = 100, sampleRate = 16e3, autoHideTimeout = 1e4, onAudioChunk = () => null } = option;
|
||||
return (art) => {
|
||||
const layer = art.layers.add({ name: "asr", html: "" });
|
||||
if (!layer)
|
||||
throw new Error("Could not create ASR subtitle layer");
|
||||
const subtitles = createSubtitles(layer, length, autoHideTimeout);
|
||||
const capture = new Capture(art.video, { interval, sampleRate, onAudioChunk }, subtitles.append);
|
||||
const volume = () => capture.volume(art.volume);
|
||||
const play = () => capture.start();
|
||||
const pause = () => capture.pause();
|
||||
const restart = () => capture.restart();
|
||||
const stop = () => capture.stop();
|
||||
const destroy = () => {
|
||||
subtitles.destroy();
|
||||
art.off("video:volumechange", volume);
|
||||
art.off("play", play);
|
||||
art.off("pause", pause);
|
||||
art.off("restart", restart);
|
||||
art.off("destroy", destroy);
|
||||
return capture.stop(true);
|
||||
};
|
||||
art.on("video:volumechange", volume);
|
||||
art.on("play", play);
|
||||
art.on("pause", pause);
|
||||
art.on("restart", restart);
|
||||
art.on("destroy", destroy);
|
||||
return { name: "artplayerPluginAsr", stop, hide: subtitles.hide, append: subtitles.append };
|
||||
};
|
||||
}
|
||||
Object.defineProperty(artplayerPluginAsr, "default", { value: artplayerPluginAsr });
|
||||
if (typeof document !== "undefined" && !document.getElementById("artplayer-plugin-asr")) {
|
||||
const styleElement = document.createElement("style");
|
||||
styleElement.id = "artplayer-plugin-asr";
|
||||
styleElement.textContent = style;
|
||||
document.head.appendChild(styleElement);
|
||||
}
|
||||
export {
|
||||
artplayerPluginAsr as default
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import type { CaptureVideo } from './types'
|
||||
import { recorderProcessorCode } from './worklet'
|
||||
|
||||
export class AudioGraph {
|
||||
private context: AudioContext | undefined
|
||||
private source: AudioNode | undefined
|
||||
private stream: MediaStream | undefined
|
||||
private recorder: AudioWorkletNode | undefined
|
||||
private gain: GainNode | undefined
|
||||
private preparation: Promise<void> | undefined
|
||||
private blobUrl: string | undefined
|
||||
private closed = false
|
||||
private direct = false
|
||||
private workletLoaded = false
|
||||
|
||||
constructor(private readonly video: CaptureVideo, private readonly sampleRate: number) {}
|
||||
|
||||
private assertOpen() {
|
||||
if (this.closed)
|
||||
throw new Error('Audio capture was closed')
|
||||
}
|
||||
|
||||
private revokeWorklet() {
|
||||
if (this.blobUrl !== undefined) {
|
||||
URL.revokeObjectURL(this.blobUrl)
|
||||
this.blobUrl = undefined
|
||||
}
|
||||
}
|
||||
|
||||
private async initialize() {
|
||||
this.assertOpen()
|
||||
const platform = window as Window & { webkitAudioContext?: typeof AudioContext }
|
||||
const Context = window.AudioContext || platform.webkitAudioContext
|
||||
if (!Context)
|
||||
throw new Error('AudioContext is not supported')
|
||||
const context = this.context ??= new Context({ sampleRate: this.sampleRate })
|
||||
if (context.state === 'suspended')
|
||||
await context.resume()
|
||||
this.assertOpen()
|
||||
if (!this.workletLoaded) {
|
||||
this.blobUrl = URL.createObjectURL(new Blob([recorderProcessorCode], { type: 'application/javascript' }))
|
||||
try {
|
||||
await context.audioWorklet.addModule(this.blobUrl)
|
||||
this.workletLoaded = true
|
||||
}
|
||||
finally {
|
||||
this.revokeWorklet()
|
||||
}
|
||||
}
|
||||
this.assertOpen()
|
||||
if (!this.source) {
|
||||
try {
|
||||
this.source = context.createMediaElementSource(this.video)
|
||||
this.direct = true
|
||||
}
|
||||
catch (error) {
|
||||
console.warn('[artplayerPluginAsr] Direct connection failed:', error)
|
||||
const capture = this.video.captureStream || this.video.mozCaptureStream
|
||||
if (!capture)
|
||||
throw new Error('Could not establish audio source')
|
||||
this.stream = capture.call(this.video)
|
||||
this.source = context.createMediaStreamSource(this.stream)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
prepare() {
|
||||
this.preparation ??= this.initialize().catch((error: unknown) => {
|
||||
this.preparation = undefined
|
||||
throw error
|
||||
})
|
||||
return this.preparation
|
||||
}
|
||||
|
||||
get ownsMediaConnection() {
|
||||
return this.direct
|
||||
}
|
||||
|
||||
idle() {
|
||||
const gain = this.gain
|
||||
this.disconnect()
|
||||
// A media element cannot be reattached after its original context is closed.
|
||||
// Keep its audio route alive until the player releases the element itself.
|
||||
if (this.direct && this.context && this.source) {
|
||||
this.gain = gain || this.context.createGain()
|
||||
if (!gain)
|
||||
this.gain.gain.value = 1
|
||||
this.source.connect(this.gain)
|
||||
this.gain.connect(this.context.destination)
|
||||
}
|
||||
}
|
||||
|
||||
connect(receive: (samples: Float32Array) => void) {
|
||||
this.assertOpen()
|
||||
const context = this.context
|
||||
const source = this.source
|
||||
if (!context || !source)
|
||||
throw new Error('Audio capture is not prepared')
|
||||
const previousGain = this.gain
|
||||
this.disconnect()
|
||||
const gain = this.gain = previousGain || context.createGain()
|
||||
if (!previousGain)
|
||||
gain.gain.value = 1
|
||||
const recorder = this.recorder = new AudioWorkletNode(context, 'recorder-processor')
|
||||
recorder.port.onmessage = (event: MessageEvent<Float32Array>) => {
|
||||
if (!this.closed && this.recorder === recorder)
|
||||
receive(event.data)
|
||||
}
|
||||
source.connect(recorder)
|
||||
source.connect(gain)
|
||||
gain.connect(context.destination)
|
||||
}
|
||||
|
||||
volume(value: number) {
|
||||
if (this.gain)
|
||||
this.gain.gain.value = value
|
||||
}
|
||||
|
||||
disconnect() {
|
||||
if (this.recorder) {
|
||||
this.recorder.port.onmessage = null
|
||||
this.recorder.disconnect()
|
||||
this.recorder = undefined
|
||||
}
|
||||
this.gain?.disconnect()
|
||||
this.gain = undefined
|
||||
this.source?.disconnect()
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this.closed)
|
||||
return
|
||||
this.closed = true
|
||||
this.disconnect()
|
||||
this.revokeWorklet()
|
||||
this.stream?.getTracks().forEach(track => track.stop())
|
||||
this.stream = undefined
|
||||
const context = this.context
|
||||
this.context = undefined
|
||||
this.source = undefined
|
||||
if (context && context.state !== 'closed')
|
||||
await context.close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import type { CaptureOptions, CaptureVideo } from './types'
|
||||
import { AudioGraph } from './audio-graph'
|
||||
import { encodeAudio } from './encoding'
|
||||
import { SampleQueue } from './sample-queue'
|
||||
|
||||
export class Capture {
|
||||
private graph: AudioGraph | undefined
|
||||
private readonly queue = new SampleQueue()
|
||||
private timer: ReturnType<typeof setInterval> | undefined
|
||||
private starting: Promise<void> | undefined
|
||||
private closing: Promise<void> = Promise.resolve()
|
||||
private running = false
|
||||
private terminal = false
|
||||
private epoch = 0
|
||||
private pending: object | undefined
|
||||
|
||||
constructor(private readonly video: CaptureVideo, private readonly options: CaptureOptions, private readonly append: (text: unknown) => void) {}
|
||||
|
||||
private reset() {
|
||||
this.epoch++
|
||||
this.pending = undefined
|
||||
this.queue.clear()
|
||||
}
|
||||
|
||||
pause() {
|
||||
this.reset()
|
||||
this.running = false
|
||||
clearInterval(this.timer)
|
||||
this.timer = undefined
|
||||
this.starting = undefined
|
||||
this.graph?.idle()
|
||||
}
|
||||
|
||||
restart() {
|
||||
const active = this.running || Boolean(this.starting)
|
||||
this.pause()
|
||||
if (this.graph && !this.graph.ownsMediaConnection)
|
||||
return this.stop().then(() => active ? this.start() : undefined)
|
||||
return active ? this.start() : Promise.resolve()
|
||||
}
|
||||
|
||||
start(): Promise<void> {
|
||||
if (this.terminal || this.running)
|
||||
return Promise.resolve()
|
||||
if (this.starting)
|
||||
return this.starting
|
||||
const epoch = this.epoch
|
||||
const current = () => !this.terminal && this.epoch === epoch
|
||||
const operation = (async () => {
|
||||
await this.closing
|
||||
if (!current())
|
||||
return
|
||||
const { sampleRate, interval } = this.options
|
||||
const count = Math.floor(sampleRate * interval / 1000)
|
||||
if (!Number.isSafeInteger(count) || count < 1 || !Number.isFinite(interval) || interval <= 0)
|
||||
throw new Error('Audio chunk length must be positive and finite')
|
||||
const graph = this.graph ??= new AudioGraph(this.video, sampleRate)
|
||||
await graph.prepare()
|
||||
if (!current()) {
|
||||
if (!this.running && this.graph === graph)
|
||||
graph.idle()
|
||||
return
|
||||
}
|
||||
graph.connect((samples) => {
|
||||
if (!this.running || this.graph !== graph)
|
||||
return
|
||||
// Stop overload explicitly instead of retaining an unbounded recognition backlog.
|
||||
if (this.queue.length + samples.length > Math.max(sampleRate * 60, count * 2)) {
|
||||
console.error('[artplayerPluginAsr] Audio callback backlog exceeded its capture limit')
|
||||
this.pause()
|
||||
return
|
||||
}
|
||||
this.queue.push(samples)
|
||||
})
|
||||
this.running = true
|
||||
this.timer = setInterval(() => this.tick(count), interval)
|
||||
})().catch(async (error: unknown) => {
|
||||
if (current()) {
|
||||
console.error('[artplayerPluginAsr] Initialization failed:', error)
|
||||
await this.stop()
|
||||
}
|
||||
}).finally(() => {
|
||||
if (this.starting === operation)
|
||||
this.starting = undefined
|
||||
})
|
||||
this.starting = operation
|
||||
return operation
|
||||
}
|
||||
|
||||
private async tick(count: number) {
|
||||
if (!this.running || this.pending)
|
||||
return
|
||||
const samples = this.queue.take(count)
|
||||
if (!samples)
|
||||
return
|
||||
const epoch = this.epoch
|
||||
const pending = this.pending = {}
|
||||
try {
|
||||
const text = await this.options.onAudioChunk(encodeAudio(samples, this.options.sampleRate))
|
||||
if (this.running && !this.terminal && this.epoch === epoch)
|
||||
this.append(text)
|
||||
}
|
||||
catch (error) {
|
||||
if (this.epoch === epoch && !this.terminal)
|
||||
console.error('[artplayerPluginAsr] Audio callback failed:', error)
|
||||
}
|
||||
finally {
|
||||
if (this.pending === pending)
|
||||
this.pending = undefined
|
||||
}
|
||||
}
|
||||
|
||||
volume(value: number) {
|
||||
this.graph?.volume(value)
|
||||
}
|
||||
|
||||
stop(terminal = false) {
|
||||
this.terminal ||= terminal
|
||||
this.pause()
|
||||
const graph = this.graph
|
||||
if (terminal || !graph?.ownsMediaConnection) {
|
||||
this.graph = undefined
|
||||
this.closing = Promise.all([this.closing, graph?.close()]).then(() => undefined)
|
||||
}
|
||||
return this.closing
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
export function encodeAudio(samples: Float32Array, sampleRate: number): { pcm: ArrayBuffer, wav: ArrayBuffer } {
|
||||
const pcm = new ArrayBuffer(samples.length * 2)
|
||||
const pcmView = new DataView(pcm)
|
||||
for (let index = 0; index < samples.length; index++) {
|
||||
const value = Math.max(-1, Math.min(1, samples[index]!))
|
||||
pcmView.setInt16(index * 2, value < 0 ? value * 0x8000 : value * 0x7FFF, true)
|
||||
}
|
||||
|
||||
const wav = new ArrayBuffer(44 + pcm.byteLength)
|
||||
const header = new DataView(wav)
|
||||
function writeString(offset: number, value: string): void {
|
||||
for (let index = 0; index < value.length; index++)
|
||||
header.setUint8(offset + index, value.charCodeAt(index))
|
||||
}
|
||||
|
||||
writeString(0, 'RIFF')
|
||||
header.setUint32(4, 36 + pcm.byteLength, true)
|
||||
writeString(8, 'WAVE')
|
||||
writeString(12, 'fmt ')
|
||||
header.setUint32(16, 16, true)
|
||||
header.setUint16(20, 1, true)
|
||||
header.setUint16(22, 1, true)
|
||||
header.setUint32(24, sampleRate, true)
|
||||
header.setUint32(28, sampleRate * 2, true)
|
||||
header.setUint16(32, 2, true)
|
||||
header.setUint16(34, 16, true)
|
||||
writeString(36, 'data')
|
||||
header.setUint32(40, pcm.byteLength, true)
|
||||
new Uint8Array(wav).set(new Uint8Array(pcm), 44)
|
||||
return { pcm, wav }
|
||||
}
|
||||
@@ -1,299 +0,0 @@
|
||||
import style from './style.less?inline'
|
||||
|
||||
export default function artplayerPluginAsr(option = {}) {
|
||||
const {
|
||||
length = 3,
|
||||
interval = 100,
|
||||
sampleRate = 16000,
|
||||
autoHideTimeout = 10000,
|
||||
onAudioChunk = () => null,
|
||||
} = option
|
||||
|
||||
return (art) => {
|
||||
let started = false
|
||||
let audioCtx = null
|
||||
let sourceNode = null
|
||||
let recorderNode = null
|
||||
let gainNode = null
|
||||
let bufferChunks = []
|
||||
let timer = null
|
||||
let workletLoaded = false
|
||||
let hideTimer = null
|
||||
let mediaStream = null
|
||||
let mediaStreamSource = null
|
||||
|
||||
const $asr = art.layers.add({
|
||||
name: 'asr',
|
||||
html: '',
|
||||
})
|
||||
|
||||
function splitByPunctuation(text) {
|
||||
return (
|
||||
text
|
||||
|
||||
.split(/(?<=[、。！？!?.])\s*/u)
|
||||
.map(s => s.trim())
|
||||
.filter(Boolean)
|
||||
.slice(-length)
|
||||
)
|
||||
}
|
||||
|
||||
function hide() {
|
||||
$asr.style.display = 'none'
|
||||
}
|
||||
|
||||
function append(subtitle) {
|
||||
if (typeof subtitle !== 'string')
|
||||
return
|
||||
clearTimeout(hideTimer)
|
||||
hideTimer = setTimeout(hide, autoHideTimeout)
|
||||
$asr.style.display = ''
|
||||
$asr.innerHTML = splitByPunctuation(subtitle)
|
||||
.map(line => `<div class="art-asr-line">${line}</div>`)
|
||||
.join('')
|
||||
}
|
||||
|
||||
const recorderProcessorCode = `
|
||||
class RecorderProcessor extends AudioWorkletProcessor {
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (input && input[0]) {
|
||||
this.port.postMessage(input[0]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('recorder-processor', RecorderProcessor);
|
||||
`
|
||||
|
||||
const createWorkletBlobUrl = () => {
|
||||
const blob = new Blob([recorderProcessorCode], { type: 'application/javascript' })
|
||||
return URL.createObjectURL(blob)
|
||||
}
|
||||
|
||||
function floatTo16BitPCM(float32Array) {
|
||||
const len = float32Array.length
|
||||
const buffer = new ArrayBuffer(len * 2)
|
||||
const view = new DataView(buffer)
|
||||
for (let i = 0; i < len; i++) {
|
||||
const s = Math.max(-1, Math.min(1, float32Array[i]))
|
||||
view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7FFF, true)
|
||||
}
|
||||
return buffer
|
||||
}
|
||||
|
||||
function pcmToWav(pcmBuffer, sampleRate) {
|
||||
const pcmLength = pcmBuffer.byteLength
|
||||
const wavBuffer = new ArrayBuffer(44 + pcmLength)
|
||||
const view = new DataView(wavBuffer)
|
||||
|
||||
const writeString = (offset, str) => {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i))
|
||||
}
|
||||
}
|
||||
|
||||
writeString(0, 'RIFF')
|
||||
view.setUint32(4, 36 + pcmLength, true)
|
||||
writeString(8, 'WAVE')
|
||||
writeString(12, 'fmt ')
|
||||
view.setUint32(16, 16, true)
|
||||
view.setUint16(20, 1, true)
|
||||
view.setUint16(22, 1, true)
|
||||
view.setUint32(24, sampleRate, true)
|
||||
view.setUint32(28, sampleRate * 2, true)
|
||||
view.setUint16(32, 2, true)
|
||||
view.setUint16(34, 16, true)
|
||||
writeString(36, 'data')
|
||||
view.setUint32(40, pcmLength, true)
|
||||
|
||||
new Uint8Array(wavBuffer).set(new Uint8Array(pcmBuffer), 44)
|
||||
|
||||
return wavBuffer
|
||||
}
|
||||
|
||||
async function setupAudioContext() {
|
||||
if (audioCtx)
|
||||
return audioCtx
|
||||
audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate })
|
||||
if (audioCtx.state === 'suspended')
|
||||
await audioCtx.resume()
|
||||
return audioCtx
|
||||
}
|
||||
|
||||
async function setupAudioSource() {
|
||||
if (!sourceNode && art.video) {
|
||||
try {
|
||||
sourceNode = audioCtx.createMediaElementSource(art.video)
|
||||
return sourceNode
|
||||
}
|
||||
catch (err) {
|
||||
console.warn('[artplayerPluginAsr] Direct connection failed:', err)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (!mediaStream) {
|
||||
mediaStream = await captureAudioFromVideo(art.video)
|
||||
}
|
||||
if (mediaStream) {
|
||||
mediaStreamSource = audioCtx.createMediaStreamSource(mediaStream)
|
||||
return mediaStreamSource
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
console.warn('[artplayerPluginAsr] MediaStream fallback failed:', err)
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
async function captureAudioFromVideo(videoElement) {
|
||||
try {
|
||||
if (videoElement.captureStream)
|
||||
return videoElement.captureStream()
|
||||
if (videoElement.mozCaptureStream)
|
||||
return videoElement.mozCaptureStream()
|
||||
console.warn('[artplayerPluginAsr] captureStream not supported')
|
||||
return null
|
||||
}
|
||||
catch (err) {
|
||||
console.warn('[artplayerPluginAsr] Error capturing stream:', err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async function startCapture() {
|
||||
if (started)
|
||||
return
|
||||
|
||||
try {
|
||||
await setupAudioContext()
|
||||
const audioSource = await setupAudioSource()
|
||||
if (!audioSource)
|
||||
throw new Error('Could not establish audio source')
|
||||
|
||||
if (!workletLoaded) {
|
||||
const blobUrl = createWorkletBlobUrl()
|
||||
await audioCtx.audioWorklet.addModule(blobUrl)
|
||||
URL.revokeObjectURL(blobUrl)
|
||||
workletLoaded = true
|
||||
}
|
||||
|
||||
gainNode = audioCtx.createGain()
|
||||
gainNode.gain.value = 1
|
||||
recorderNode = new AudioWorkletNode(audioCtx, 'recorder-processor')
|
||||
recorderNode.port.onmessage = (event) => {
|
||||
bufferChunks.push(new Float32Array(event.data))
|
||||
}
|
||||
audioSource.connect(recorderNode)
|
||||
audioSource.connect(gainNode)
|
||||
gainNode.connect(audioCtx.destination)
|
||||
|
||||
const CHUNK_SAMPLES = sampleRate * (interval / 1000)
|
||||
timer = setInterval(async () => {
|
||||
if (bufferChunks.length === 0)
|
||||
return
|
||||
|
||||
let accumulated = new Float32Array(0)
|
||||
while (accumulated.length < CHUNK_SAMPLES && bufferChunks.length) {
|
||||
const next = bufferChunks.shift()
|
||||
const tmp = new Float32Array(accumulated.length + next.length)
|
||||
tmp.set(accumulated, 0)
|
||||
tmp.set(next, accumulated.length)
|
||||
accumulated = tmp
|
||||
}
|
||||
|
||||
if (accumulated.length < CHUNK_SAMPLES)
|
||||
return
|
||||
|
||||
const chunkToSend = accumulated.slice(0, CHUNK_SAMPLES)
|
||||
const pcm = floatTo16BitPCM(chunkToSend)
|
||||
const wav = pcmToWav(pcm, sampleRate)
|
||||
const subtitle = await onAudioChunk({ pcm, wav })
|
||||
append(subtitle)
|
||||
}, interval)
|
||||
|
||||
started = true
|
||||
}
|
||||
catch (err) {
|
||||
console.error('[artplayerPluginAsr] Initialization failed:', err)
|
||||
await stopCapture()
|
||||
}
|
||||
}
|
||||
|
||||
async function stopCapture() {
|
||||
if (!started)
|
||||
return
|
||||
started = false
|
||||
|
||||
clearInterval(timer)
|
||||
timer = null
|
||||
|
||||
if (recorderNode) {
|
||||
recorderNode.disconnect()
|
||||
recorderNode.port.onmessage = null
|
||||
recorderNode = null
|
||||
}
|
||||
|
||||
if (gainNode) {
|
||||
gainNode.disconnect()
|
||||
gainNode = null
|
||||
}
|
||||
|
||||
bufferChunks = []
|
||||
}
|
||||
|
||||
async function destroy() {
|
||||
await stopCapture()
|
||||
|
||||
if (mediaStreamSource) {
|
||||
mediaStreamSource.disconnect()
|
||||
mediaStreamSource = null
|
||||
}
|
||||
|
||||
if (sourceNode) {
|
||||
sourceNode.disconnect()
|
||||
sourceNode = null
|
||||
}
|
||||
|
||||
if (mediaStream) {
|
||||
mediaStream.getTracks().forEach(track => track.stop())
|
||||
mediaStream = null
|
||||
}
|
||||
|
||||
if (audioCtx) {
|
||||
await audioCtx.close()
|
||||
audioCtx = null
|
||||
}
|
||||
|
||||
workletLoaded = false
|
||||
}
|
||||
|
||||
art.on('video:volumechange', () => {
|
||||
if (gainNode) {
|
||||
gainNode.gain.value = art.volume
|
||||
}
|
||||
})
|
||||
|
||||
art.on('play', startCapture)
|
||||
art.on('pause', stopCapture)
|
||||
art.on('destroy', destroy)
|
||||
|
||||
return {
|
||||
name: 'artplayerPluginAsr',
|
||||
stop: destroy,
|
||||
hide,
|
||||
append,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof document !== 'undefined') {
|
||||
if (!document.getElementById('artplayer-plugin-asr')) {
|
||||
const $style = document.createElement('style')
|
||||
$style.id = 'artplayer-plugin-asr'
|
||||
$style.textContent = style
|
||||
document.head.appendChild($style)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import type Artplayer from 'artplayer'
|
||||
import type { AsrOptions, AsrResult } from './types'
|
||||
import { Capture } from './capture'
|
||||
import style from './style.less?inline'
|
||||
import { createSubtitles } from './subtitles'
|
||||
|
||||
export default function artplayerPluginAsr(option: AsrOptions = {}) {
|
||||
const { length = 3, interval = 100, sampleRate = 16000, autoHideTimeout = 10000, onAudioChunk = () => null } = option
|
||||
return (art: Artplayer): AsrResult => {
|
||||
const layer = art.layers.add({ name: 'asr', html: '' })
|
||||
if (!layer)
|
||||
throw new Error('Could not create ASR subtitle layer')
|
||||
const subtitles = createSubtitles(layer, length, autoHideTimeout)
|
||||
const capture = new Capture(art.video, { interval, sampleRate, onAudioChunk }, subtitles.append)
|
||||
const volume = () => capture.volume(art.volume)
|
||||
const play = () => capture.start()
|
||||
const pause = () => capture.pause()
|
||||
const restart = () => capture.restart()
|
||||
const stop = () => capture.stop()
|
||||
const destroy = () => {
|
||||
subtitles.destroy()
|
||||
art.off('video:volumechange', volume)
|
||||
art.off('play', play)
|
||||
art.off('pause', pause)
|
||||
art.off('restart', restart)
|
||||
art.off('destroy', destroy)
|
||||
return capture.stop(true)
|
||||
}
|
||||
art.on('video:volumechange', volume)
|
||||
art.on('play', play)
|
||||
art.on('pause', pause)
|
||||
art.on('restart', restart)
|
||||
art.on('destroy', destroy)
|
||||
return { name: 'artplayerPluginAsr', stop, hide: subtitles.hide, append: subtitles.append }
|
||||
}
|
||||
}
|
||||
|
||||
// Published 2.0 CJS consumers used require(...).default; 2.1 exported the function.
|
||||
Object.defineProperty(artplayerPluginAsr, 'default', { value: artplayerPluginAsr })
|
||||
|
||||
if (typeof document !== 'undefined' && !document.getElementById('artplayer-plugin-asr')) {
|
||||
const styleElement = document.createElement('style')
|
||||
styleElement.id = 'artplayer-plugin-asr'
|
||||
styleElement.textContent = style
|
||||
document.head.appendChild(styleElement)
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
interface SampleBlock {
|
||||
samples: Float32Array
|
||||
next?: SampleBlock
|
||||
}
|
||||
|
||||
export class SampleQueue {
|
||||
private head?: SampleBlock
|
||||
private tail?: SampleBlock
|
||||
private offset = 0
|
||||
private size = 0
|
||||
|
||||
get length(): number {
|
||||
return this.size
|
||||
}
|
||||
|
||||
push(input: Float32Array): void {
|
||||
if (input.length === 0)
|
||||
return
|
||||
const block: SampleBlock = { samples: new Float32Array(input) }
|
||||
if (this.tail)
|
||||
this.tail.next = block
|
||||
else
|
||||
this.head = block
|
||||
this.tail = block
|
||||
this.size += input.length
|
||||
}
|
||||
|
||||
take(count: number): Float32Array | null {
|
||||
if (!Number.isSafeInteger(count) || count <= 0)
|
||||
throw new RangeError('Sample count must be a positive safe integer')
|
||||
if (count > this.size)
|
||||
return null
|
||||
|
||||
const output = new Float32Array(count)
|
||||
let written = 0
|
||||
while (this.head && written < count) {
|
||||
const samples = this.head.samples
|
||||
const amount = Math.min(samples.length - this.offset, count - written)
|
||||
output.set(samples.subarray(this.offset, this.offset + amount), written)
|
||||
written += amount
|
||||
this.offset += amount
|
||||
if (this.offset === samples.length) {
|
||||
this.head = this.head.next
|
||||
this.offset = 0
|
||||
}
|
||||
}
|
||||
if (!this.head)
|
||||
this.tail = undefined
|
||||
this.size -= count
|
||||
return output
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.head = undefined
|
||||
this.tail = undefined
|
||||
this.offset = 0
|
||||
this.size = 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
export function createSubtitles(layer: HTMLElement, length: number, autoHideTimeout: number) {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
let destroyed = false
|
||||
|
||||
function cancelTimer() {
|
||||
clearTimeout(timer)
|
||||
timer = undefined
|
||||
}
|
||||
|
||||
function hide() {
|
||||
if (!destroyed)
|
||||
layer.style.display = 'none'
|
||||
}
|
||||
|
||||
function append(subtitle: unknown) {
|
||||
if (destroyed || typeof subtitle !== 'string')
|
||||
return
|
||||
cancelTimer()
|
||||
timer = setTimeout(hide, autoHideTimeout)
|
||||
layer.style.display = ''
|
||||
layer.innerHTML = subtitle.split(/(?<=[、。！？!?.])\s*/u)
|
||||
.map(line => line.trim())
|
||||
.filter(Boolean)
|
||||
.slice(-length)
|
||||
.map(line => `<div class="art-asr-line">${line}</div>`)
|
||||
.join('')
|
||||
}
|
||||
|
||||
return {
|
||||
append,
|
||||
hide,
|
||||
destroy() {
|
||||
destroyed = true
|
||||
cancelTimer()
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
export interface AudioChunk {
|
||||
pcm: ArrayBuffer
|
||||
wav: ArrayBuffer
|
||||
}
|
||||
|
||||
export interface AsrOptions {
|
||||
length?: number
|
||||
interval?: number
|
||||
sampleRate?: number
|
||||
autoHideTimeout?: number
|
||||
onAudioChunk?: (chunk: AudioChunk) => unknown
|
||||
}
|
||||
|
||||
export interface AsrResult {
|
||||
name: 'artplayerPluginAsr'
|
||||
stop: () => Promise<void>
|
||||
hide: () => void
|
||||
append: (subtitle: string) => void
|
||||
}
|
||||
|
||||
export interface CaptureVideo extends HTMLVideoElement {
|
||||
captureStream?: () => MediaStream
|
||||
mozCaptureStream?: () => MediaStream
|
||||
}
|
||||
|
||||
export interface CaptureOptions {
|
||||
interval: number
|
||||
sampleRate: number
|
||||
onAudioChunk: (chunk: AudioChunk) => unknown
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
export const recorderProcessorCode = `
|
||||
class RecorderProcessor extends AudioWorkletProcessor {
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (input && input[0]) {
|
||||
this.port.postMessage(input[0]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('recorder-processor', RecorderProcessor);
|
||||
`
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": { "allowJs": false },
|
||||
"include": ["src/**/*.ts", "types/**/*.d.ts", "../../types/assets.d.ts"]
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,62 @@
|
||||
# PKG-ASR-03：拆分 ASR 并修复音频归属
|
||||
|
||||
从 53c295819db004e507b35689a7bdec1955e08396 开始，将单个生产 JS 文件拆成
|
||||
8 个严格 TypeScript 模块。公共声明不在本任务改写，ASYNC/旧工厂类型与实际安装
|
||||
消费者仍由 PKG-ASR-04 验收。无新依赖，版本不变，正常构建生成三格式及 docs 副本。
|
||||
|
||||
## 结构改动
|
||||
|
||||
index 只承担工厂、事件和兼容门面；subtitles 管字幕和隐藏定时器；capture 管代际、
|
||||
初始化、节拍和回调；audio-graph 管浏览器节点；encoding/sample-queue/worklet
|
||||
分离纯数据与处理器边界。完整维护地图在包内 ARCHITECTURE.md。
|
||||
|
||||
## 独立列明的缺陷修正
|
||||
|
||||
- 链式块队列保留不足块及尾部，复制输入且避免重复 concat/shift。编码与 7 个
|
||||
历史实现逐字节匹配。每个活动代际只允许一个识别回调，拒绝被捕获且后续可继续。
|
||||
- 暂停/切源/停止/终止使旧结果失效；旧结果不会解锁新回调。退休 recorder 的
|
||||
消息即使已经被外部保留也被身份检查拒绝。首次候选回归发现此消息竞态后修复。
|
||||
- 初始化去重，取消后不晚建录音图；Worklet URL 在成功、失败和终止路径释放。
|
||||
Worklet 模块先准备再绑定 video，失败不会先接管并关闭视频的播放通路。
|
||||
- stop 清录音/识别任务，保持可再次播放；已有字幕保留原自动隐藏期限，与历史版一致，
|
||||
并拒绝旧识别结果新建隐藏任务。直接绑定视频的 context/source/
|
||||
播放 gain 由播放器持有，直到 art.destroy 才关闭；pause/stop 保持播放路由。
|
||||
捕获流 fallback 属于可释放的采集资源，stop 关闭；切源时重新取得流与音轨。
|
||||
- destroy 清监听器、节点、流、timer，并禁止通过保留的 append 引用复活。
|
||||
- 慢回调超过队列上限时明确报错并暂停采集，播放路由保留。上限取 60 秒与两个配置块
|
||||
长度的较大值，以支持原本合法的长 interval；有超过 60 秒配置块的回归。不会无限缓存，
|
||||
也不声称过载时音频无损；调用者网络请求仍由调用者负责取消。
|
||||
- 工厂补不可枚举的自引用 default，兼容 2.0 的 require(...).default 及 2.1 的
|
||||
require(...) 函数路径；保留键顺序、同步注册、选项快照、字幕 HTML/CSS 行为。
|
||||
|
||||
## 为什么 stop 不再关闭 direct context
|
||||
|
||||
实际 Chromium 在 close 后经 capture fallback 重启时无法得到非零 PCM；两个核心
|
||||
组合失败，Firefox 两组合通过。原生探测报告保留在证据文件中，没有把能力失败
|
||||
抹成通过。Web Audio 规定绑定后的媒体声音走节点图，close 会忽略关联媒体输出；
|
||||
关闭后不能简单重新绑定节点来恢复。用户公开 stop/play 用法需要可靠恢复，因此
|
||||
修正内部生命周期，而不删除重启测试或更改公开方法。
|
||||
|
||||
子代理独立复核了 [Web Audio close](https://www.w3.org/TR/webaudio-1.0/#dom-audiocontext-close)、
|
||||
[媒体源节点](https://www.w3.org/TR/webaudio-1.0/#MediaElementAudioSourceNode)及
|
||||
[捕获流切源](https://www.w3.org/TR/mediacapture-fromelement/#html-media-element-media-capture-extensions)。
|
||||
保留 direct-first，避免把采样语义和兼容范围改成尚未全面支持的 capture-first。
|
||||
旧版 close 行为在 ASR-02 历史测试中保留；候选测试要求停止后有播放路由、同一
|
||||
context 恢复非零 PCM，最终销毁才关闭。初始化尚未绑定 video 时仍立即完整关闭。
|
||||
|
||||
## 验证与剩余范围
|
||||
|
||||
源代码分包 strict/noUncheckedIndexedAccess、定向 ESLint、正常 main/legacy/ESM
|
||||
构建通过。历史/纯模块/候选共 210 项，真实 ESM 与副本校验另 2 项；main 与 legacy
|
||||
各重跑 25 项候选生命周期。浏览器源码全组 12 通过/6 Windows WebKit 能力跳过，
|
||||
main/legacy 候选各 4 通过/2 能力跳过；没有 failed/flaky。
|
||||
具体输入哈希、报告、构建与浏览器版本见 [验证记录](../baselines/asr-ownership-validation.json)。
|
||||
|
||||
浏览器实际验证：两核心、非零 PCM/WAV、本地回调字幕、暂停恢复、同视频切源、
|
||||
stop 后恢复及最终 context 关闭。Windows WebKit 无 WebAudio，不能代表 macOS Safari；
|
||||
真实 fallback 音轨拓扑、音量/静音、跨源媒体、设备仍归 ASR-05。ASR-04 的公共类型和
|
||||
实际包安装、ASR-06 的示例及发行文档仍未完成，本记录不表示整包已可发布。
|
||||
|
||||
仅运行本包、类型、构建和台账检查，未重复整仓全量 CI。两次子代理承担纯音频模块/
|
||||
回归与规范审查，主代理负责图与控制器集成、浏览器、文档及提交。每项职责已核对。
|
||||
回退本任务提交可恢复旧源码及构建产物，但会同时恢复已复现丢样本和生命周期缺陷。
|
||||
+3
-2
@@ -4,7 +4,7 @@
|
||||
|
||||
基线：`40fcda6a37d0049d42e49c1e64e70d4fd9ba5f7f`。总任务 225 项，范围 22 个包及工作区/示例。
|
||||
|
||||
状态：todo 84 / doing 16 / blocked 0 / done 125 / deferred 0。风险 L/M/H 表示兼容风险，不表示工期。
|
||||
状态：todo 83 / doing 16 / blocked 0 / done 126 / deferred 0。风险 L/M/H 表示兼容风险，不表示工期。
|
||||
|
||||
前置依赖是启动条件；验收是完成条件。任务可以继续拆分，但不能复用或悄悄删除旧 ID。
|
||||
|
||||
@@ -253,7 +253,7 @@
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| PKG-ASR-01 | artplayer-plugin-asr<br>核对包契约与历史用法 | BASE-05 | length/interval/sampleRate/onAudioChunk、append/hide/stop 的真实能力 | 源码/声明/README/demo/发布包差异已登记；公开形状和版本范围冻结 | H | done |
|
||||
| PKG-ASR-02 | artplayer-plugin-asr<br>建立特有行为与错误测试 | PKG-ASR-01, ENG-03, ENG-05 | PCM/WAV、chunk 时序、回调慢/拒绝、重复初始化与停止 | 旧版本行为可重跑，成功/失败/切源/销毁有必要断言 | H | done |
|
||||
| PKG-ASR-03 | artplayer-plugin-asr<br>整理内部职责与资源 | PKG-ASR-02, CORE-10, CORE-18 | AudioContext/Worklet/Stream 生命周期和背压分离 | 结构变化和缺陷修复分开记录；原 API/事件/资源生命周期通过 | H | todo |
|
||||
| PKG-ASR-03 | artplayer-plugin-asr<br>整理内部职责与资源 | PKG-ASR-02, CORE-10, CORE-18 | AudioContext/Worklet/Stream 生命周期和背压分离 | 结构变化和缺陷修复分开记录；原 API/事件/资源生命周期通过 | H | done |
|
||||
| PKG-ASR-04 | artplayer-plugin-asr<br>迁移自有源码和公开类型 | PKG-ASR-03, ENG-04, ENG-06, CORE-07 | 主线程/Worklet/音频 buffer 与异步回调类型 | 严格类型检查、旧消费样例通过；声明路径/导出和同步异步兼容 | H | todo |
|
||||
| PKG-ASR-05 | artplayer-plugin-asr<br>验证新旧核心和组合 | PKG-ASR-04, CORE-22 | 真实 WebAudio 分块、播放暂停/切源/销毁，不引入网络 ASR | 最终核心与原支持范围核心分别通过；设备/SDK 缺证据不能标完成 | H | todo |
|
||||
| PKG-ASR-06 | artplayer-plugin-asr<br>验证分发并同步文档 | PKG-ASR-05, ENG-07 | asr.js、Worklet 资源、输出格式与声明证据 | tarball 入口/资源、类型、8082 demo 和 README 一致，有回退记录 | H | todo |
|
||||
@@ -528,6 +528,7 @@
|
||||
- PKG-JASSUB-01: [记录](baselines/jassub-release.json) [记录](baselines/jassub-vendor.json) [记录](baselines/jassub-font-metadata.json) [记录](baselines/jassub-contract.md) [记录](baselines/jassub-contract-validation.json) [记录](changes/2026-09-13-PKG-JASSUB-01-baseline.md)
|
||||
- PKG-ASR-01: [记录](changes/2026-09-13-PKG-ASR-01-contracts.md) [记录](baselines/asr-contract.md) [记录](baselines/asr-release.json)
|
||||
- PKG-ASR-02: [记录](changes/2026-09-13-PKG-ASR-02-audio-baseline.md) [记录](baselines/asr-audio-validation.json)
|
||||
- PKG-ASR-03: [记录](changes/2026-09-13-PKG-ASR-03-audio-ownership.md) [记录](baselines/asr-ownership-validation.json)
|
||||
- PKG-ADS-01: [记录](baselines/ads-release.json) [记录](baselines/ads-contract.md) [记录](baselines/ads-contract-validation.json) [记录](changes/2026-09-12-PKG-ADS-01-contract.md) [记录](scripts/ads-contract.test.mjs)
|
||||
- PKG-ADS-02: [记录](changes/2026-09-12-PKG-ADS-02-tests.md) [记录](ads-validation.md) [记录](baselines/ads-validation.json)
|
||||
- PKG-ADS-03: [记录](changes/2026-09-12-PKG-ADS-03-lifecycle.md) [记录](ads-validation.md) [记录](baselines/ads-lifecycle-validation.json)
|
||||
|
||||
@@ -1,5 +1,20 @@
|
||||
# 进度与证据
|
||||
|
||||
## PKG-ASR-03 TS 拆分与音频生命周期完成
|
||||
|
||||
生产源码拆为 8 个严格 TS 模块；队列保留半块/尾部，识别回调背压、拒绝恢复、
|
||||
初始化去重和代际清理完成。退休 Worklet 消息反例由失败转通过。
|
||||
Chromium stop 后不能恢复的原生问题已修复：直接媒体路由由播放器持有到 destroy，
|
||||
stop 只停 ASR 并允许重新播放；先准备 Worklet 后绑定视频，失败可真正释放未绑定 context。
|
||||
定向 212 项通过，main/legacy 各重跑 25 候选项；严格分包 tsc、lint、正常三格式构建通过。
|
||||
最终浏览器源码 12 过/6 能力跳过，main/legacy 各 4 过/2 能力跳过；共 20 过、10 明确
|
||||
Windows WebKit 无 WebAudio 跳过。非零 PCM、暂停、切源、stop 重启与 destroy 已实测。
|
||||
见[变更](changes/2026-09-13-PKG-ASR-03-audio-ownership.md)和[证据](baselines/asr-ownership-validation.json)。
|
||||
225 项：126 done、16 doing、83 todo；205 风险，两项 ASR 音频风险闭环。
|
||||
下一步 ASR-04 公共异步类型与安装消费者；完整设备/fallback/音量和发行仍未完成。
|
||||
没有重复全量 CI、没有推送或发布。子代理承担独立模块/测试和规范审查，主代理集成验收。
|
||||
|
||||
|
||||
## PKG-ASR-02 音频与错误基线完成
|
||||
|
||||
子代理交付并由主代理审查的 133 项音频历史测试通过，连同公开契约/来源共 165 项。
|
||||
|
||||
@@ -207,5 +207,5 @@
|
||||
| CORE-DEFAULTS-SSR-01 | resolved / 已复现 | Reading static defaults throws when navigator is absent | CORE-25 |
|
||||
| ENG-DIST-PUBLIC-01 | resolved / 已复现 | Vite copied authored declarations into library distribution | ENG-12 |
|
||||
| ASR-TYPE-01 | open / 源码/产物事实 | ASR declarations omit asynchronous stop and callback text results | PKG-ASR-04 |
|
||||
| ASR-CHUNK-01 | open / 已复现 | ASR tick discards underfilled chunks and overrun samples in source | PKG-ASR-02, PKG-ASR-03 |
|
||||
| ASR-LIFE-01 | open / 已复现 | ASR partial initialization, pause resume and stale callback cleanup need controlled evidence | PKG-ASR-02, PKG-ASR-03, PKG-ASR-05 |
|
||||
| ASR-CHUNK-01 | resolved / 已复现 | ASR tick discards underfilled chunks and overrun samples in source | PKG-ASR-02, PKG-ASR-03 |
|
||||
| ASR-LIFE-01 | resolved / 已复现 | ASR partial initialization, pause resume and stale callback cleanup need controlled evidence | PKG-ASR-02, PKG-ASR-03, PKG-ASR-05 |
|
||||
+19
-5
@@ -798,7 +798,7 @@
|
||||
"EX-03"
|
||||
],
|
||||
"evidence": [
|
||||
"packages/artplayer-plugin-asr/src/index.js",
|
||||
"packages/artplayer-plugin-asr/src/index.ts",
|
||||
"docs/assets/example/asr.js"
|
||||
],
|
||||
"compatibleResolution": "Core plugin supplies PCM to caller onAudioChunk. Example fetches an external ASR endpoint and opens its returned WebSocket; not an embedded ASR SDK or a verified service.",
|
||||
@@ -4561,7 +4561,7 @@
|
||||
"id": "ASR-CHUNK-01",
|
||||
"title": "ASR tick discards underfilled chunks and overrun samples in source",
|
||||
"confirmation": "reproduced",
|
||||
"status": "open",
|
||||
"status": "resolved",
|
||||
"owners": [
|
||||
"PKG-ASR-02",
|
||||
"PKG-ASR-03"
|
||||
@@ -4574,13 +4574,20 @@
|
||||
"refactor/baselines/asr-audio-validation.json"
|
||||
],
|
||||
"compatibleResolution": "Reproduce exact sample loss and retain queue remainders with bounded, ordered callback delivery.",
|
||||
"closureCriteria": "Exact PCM/WAV byte sequence and slow/rejected callback regression tests pass without losing or reordering accepted input."
|
||||
"closureCriteria": "Exact PCM/WAV byte sequence and slow/rejected callback regression tests pass without losing or reordering accepted input.",
|
||||
"resolutionEvidence": [
|
||||
"refactor/changes/2026-09-13-PKG-ASR-03-audio-ownership.md",
|
||||
"refactor/baselines/asr-ownership-validation.json",
|
||||
"test/asr-lifecycle.test.js",
|
||||
"test/asr-encoding.test.js"
|
||||
],
|
||||
"resolutionRationale": "Copied linked blocks retain underfilled input and tails; exact byte parity and FIFO tests pass. One callback per generation prevents overlap; rejection recovery and obsolete completions are covered. Exceeding the larger of 60 seconds or two configured chunks pauses capture with an explicit error; long configured intervals remain collectable."
|
||||
},
|
||||
{
|
||||
"id": "ASR-LIFE-01",
|
||||
"title": "ASR partial initialization, pause resume and stale callback cleanup need controlled evidence",
|
||||
"confirmation": "reproduced",
|
||||
"status": "open",
|
||||
"status": "resolved",
|
||||
"owners": [
|
||||
"PKG-ASR-02",
|
||||
"PKG-ASR-03",
|
||||
@@ -4594,7 +4601,14 @@
|
||||
"refactor/baselines/asr-audio-validation.json"
|
||||
],
|
||||
"compatibleResolution": "Own pending setup, URLs, source nodes and callbacks across pause, stop, restart and terminal destruction.",
|
||||
"closureCriteria": "Failure/race regression tests plus native WebAudio playback and cleanup evidence prove resource ownership."
|
||||
"closureCriteria": "Failure/race regression tests plus native WebAudio playback and cleanup evidence prove resource ownership.",
|
||||
"resolutionEvidence": [
|
||||
"refactor/changes/2026-09-13-PKG-ASR-03-audio-ownership.md",
|
||||
"refactor/baselines/asr-ownership-validation.json",
|
||||
"test/asr-lifecycle.test.js",
|
||||
"test/asr-encoding.test.js"
|
||||
],
|
||||
"resolutionRationale": "Candidate ownership/race regression tests pass, including retired-port delivery and module failure before binding video. Actual Chromium/Firefox source/main/legacy with both cores verify pause, source change, explicit stop/restart and final close. Direct media contexts deliberately remain player-owned until destroy; disposable stream contexts close on stop. General native fallback/volume/device acceptance remains ASR-05 and SDK-11, not claimed by this specific fix."
|
||||
}
|
||||
]
|
||||
}
|
||||
+5
-2
@@ -2660,11 +2660,14 @@
|
||||
"CORE-10",
|
||||
"CORE-18"
|
||||
],
|
||||
"status": "todo",
|
||||
"status": "done",
|
||||
"risk": "H",
|
||||
"deliverable": "AudioContext/Worklet/Stream 生命周期和背压分离",
|
||||
"acceptance": "结构变化和缺陷修复分开记录；原 API/事件/资源生命周期通过",
|
||||
"evidence": []
|
||||
"evidence": [
|
||||
"changes/2026-09-13-PKG-ASR-03-audio-ownership.md",
|
||||
"baselines/asr-ownership-validation.json"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "PKG-ASR-04",
|
||||
|
||||
@@ -1240,7 +1240,7 @@
|
||||
"EX-03"
|
||||
],
|
||||
"sourceFiles": [
|
||||
"packages/artplayer-plugin-asr/src/index.js",
|
||||
"packages/artplayer-plugin-asr/src/index.ts",
|
||||
"docs/assets/example/asr.js"
|
||||
],
|
||||
"contract": "Core plugin supplies PCM to caller onAudioChunk. Example fetches an external ASR endpoint and opens its returned WebSocket; not an embedded ASR SDK or a verified service.",
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import fs from 'node:fs'
|
||||
// eslint-disable-next-line test/no-import-node-test -- Actual ASR distribution imports use the repository runner.
|
||||
import test from 'node:test'
|
||||
|
||||
test('ASR built ESM imports without DOM and keeps both factory paths and synchronous registration', async () => {
|
||||
const module = await import('../packages/artplayer-plugin-asr/dist/artplayer-plugin-asr.mjs')
|
||||
assert.deepEqual(Object.keys(module), ['default'])
|
||||
assert.equal(module.default.default, module.default)
|
||||
assert.deepEqual(Object.keys(module.default), [])
|
||||
const events = new Map()
|
||||
const layer = { style: {}, innerHTML: '' }
|
||||
const result = module.default({ length: 1 })({
|
||||
video: {},
|
||||
volume: 0.5,
|
||||
layers: { add: () => layer },
|
||||
on: (name, callback) => events.set(name, callback),
|
||||
off: name => events.delete(name),
|
||||
})
|
||||
assert.equal(result.then, undefined)
|
||||
assert.deepEqual(Object.keys(result), ['name', 'stop', 'hide', 'append'])
|
||||
result.append('One. <b>Two</b>')
|
||||
assert.equal(layer.innerHTML, '<div class="art-asr-line"><b>Two</b></div>')
|
||||
await result.stop()
|
||||
await events.get('destroy')()
|
||||
assert.equal(events.size, 0)
|
||||
})
|
||||
|
||||
test('ASR three generated package artifacts exactly match the demo copies', () => {
|
||||
for (const suffix of ['js', 'legacy.js', 'mjs']) {
|
||||
const name = `artplayer-plugin-asr.${suffix}`
|
||||
assert.deepEqual(fs.readFileSync(`packages/artplayer-plugin-asr/dist/${name}`), fs.readFileSync(`docs/compiled/${name}`))
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,183 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { Buffer } from 'node:buffer'
|
||||
// eslint-disable-next-line test/no-import-node-test -- Pure ASR audio modules use the repository runner.
|
||||
import test from 'node:test'
|
||||
import { asrAudioEnvironment, runAsrWorklet } from './helpers/asr-audio.js'
|
||||
import { asrHistorical } from './helpers/asr.js'
|
||||
import { loadModules } from './helpers/load.js'
|
||||
|
||||
const root = 'packages/artplayer-plugin-asr/src'
|
||||
const { encodeAudio, SampleQueue, recorderProcessorCode } = await loadModules({
|
||||
encodeAudio: { file: `${root}/encoding`, name: 'encodeAudio' },
|
||||
SampleQueue: { file: `${root}/sample-queue`, name: 'SampleQueue' },
|
||||
recorderProcessorCode: { file: `${root}/worklet`, name: 'recorderProcessorCode' },
|
||||
})
|
||||
|
||||
test('ASR encoder preserves signed PCM16 clamping, truncation, NaN and little endian bytes', () => {
|
||||
const input = Float32Array.of(-Infinity, -2, -1, -0.5, -0, 0, 0.5, 1, 2, Infinity, Number.NaN)
|
||||
const encoded = encodeAudio(input, 16000)
|
||||
assert.deepEqual(Object.keys(encoded), ['pcm', 'wav'])
|
||||
const expected = [0, 128, 0, 128, 0, 128, 0, 192, 0, 0, 0, 0, 255, 63, 255, 127, 255, 127, 255, 127, 0, 0]
|
||||
assert.deepEqual([...new Uint8Array(encoded.pcm)], expected)
|
||||
assert.equal(input[0], -Infinity)
|
||||
assert.equal(Number.isNaN(input.at(-1)), true)
|
||||
})
|
||||
|
||||
for (const sampleRate of [8000, 16000, 44100, 48000]) {
|
||||
test(`ASR encoder writes exact mono WAV metadata at ${sampleRate} Hz`, () => {
|
||||
const { pcm, wav } = encodeAudio(Float32Array.of(-1, 0, 1), sampleRate)
|
||||
const data = Buffer.from(wav)
|
||||
assert.equal(pcm.byteLength, 6)
|
||||
assert.equal(data.length, 50)
|
||||
assert.equal(data.toString('ascii', 0, 4), 'RIFF')
|
||||
assert.equal(data.readUInt32LE(4), 42)
|
||||
assert.equal(data.toString('ascii', 8, 16), 'WAVEfmt ')
|
||||
assert.equal(data.readUInt32LE(16), 16)
|
||||
assert.equal(data.readUInt16LE(20), 1)
|
||||
assert.equal(data.readUInt16LE(22), 1)
|
||||
assert.equal(data.readUInt32LE(24), sampleRate)
|
||||
assert.equal(data.readUInt32LE(28), sampleRate * 2)
|
||||
assert.equal(data.readUInt16LE(32), 2)
|
||||
assert.equal(data.readUInt16LE(34), 16)
|
||||
assert.equal(data.toString('ascii', 36, 40), 'data')
|
||||
assert.equal(data.readUInt32LE(40), 6)
|
||||
assert.deepEqual([...data.subarray(44)], [0, 128, 0, 0, 255, 127])
|
||||
new Uint8Array(pcm).fill(0)
|
||||
assert.deepEqual([...data.subarray(44)], [0, 128, 0, 0, 255, 127], 'The WAV payload owns its bytes')
|
||||
})
|
||||
}
|
||||
|
||||
test('ASR encoder empty and offset views retain exact payload boundaries', () => {
|
||||
const empty = encodeAudio(new Float32Array(0), 16000)
|
||||
assert.equal(empty.pcm.byteLength, 0)
|
||||
assert.equal(empty.wav.byteLength, 44)
|
||||
assert.equal(new DataView(empty.wav).getUint32(4, true), 36)
|
||||
assert.equal(new DataView(empty.wav).getUint32(40, true), 0)
|
||||
const backing = Float32Array.of(1, -1, 0.5, 1)
|
||||
const { pcm } = encodeAudio(backing.subarray(1, 3), 16000)
|
||||
assert.deepEqual([...new Uint8Array(pcm)], [0, 128, 255, 63])
|
||||
})
|
||||
|
||||
for (const implementation of await asrHistorical()) {
|
||||
test(`ASR encoder byte parity with ${implementation.name}`, async () => {
|
||||
const chunks = []
|
||||
const environment = asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, onAudioChunk: chunk => chunks.push(chunk) })
|
||||
await environment.emit('play')
|
||||
const input = Float32Array.of(-Infinity, -1, -0.5, Number.NaN, 0, 0.5, 1, Infinity)
|
||||
environment.send(input)
|
||||
await environment.tick()
|
||||
assert.equal(chunks.length, 1)
|
||||
const candidate = encodeAudio(input, 8000)
|
||||
assert.deepEqual(Buffer.from(candidate.pcm), Buffer.from(chunks[0].pcm))
|
||||
assert.deepEqual(Buffer.from(candidate.wav), Buffer.from(chunks[0].wav))
|
||||
await environment.plugin.stop()
|
||||
})
|
||||
}
|
||||
|
||||
test('ASR sample queue retains insufficient chunks and excess tails', () => {
|
||||
const queue = new SampleQueue()
|
||||
assert.equal(queue.take(8), null)
|
||||
queue.push(Float32Array.of(1, 2, 3, 4))
|
||||
assert.equal(queue.take(8), null)
|
||||
assert.equal(queue.length, 4)
|
||||
queue.push(Float32Array.of(5, 6, 7, 8, 9, 10))
|
||||
assert.deepEqual([...queue.take(8)], [1, 2, 3, 4, 5, 6, 7, 8])
|
||||
assert.equal(queue.length, 2)
|
||||
assert.equal(queue.take(3), null)
|
||||
queue.push(Float32Array.of(11))
|
||||
assert.deepEqual([...queue.take(3)], [9, 10, 11])
|
||||
assert.equal(queue.length, 0)
|
||||
assert.equal(queue.take(1), null)
|
||||
})
|
||||
|
||||
test('ASR sample queue copies input views and returns independently owned output', () => {
|
||||
const queue = new SampleQueue()
|
||||
const input = Float32Array.of(0, 1, 2, 3)
|
||||
queue.push(input.subarray(1, 3))
|
||||
input.fill(9)
|
||||
const first = queue.take(1)
|
||||
assert.deepEqual([...first], [1])
|
||||
first[0] = 8
|
||||
assert.deepEqual([...queue.take(1)], [2])
|
||||
})
|
||||
|
||||
test('ASR sample queue clears partial blocks and can be reused after draining', () => {
|
||||
const queue = new SampleQueue()
|
||||
queue.push(Float32Array.of(1, 2, 3))
|
||||
assert.deepEqual([...queue.take(1)], [1])
|
||||
queue.clear()
|
||||
queue.clear()
|
||||
queue.push(new Float32Array(0))
|
||||
assert.equal(queue.length, 0)
|
||||
assert.equal(queue.take(1), null)
|
||||
queue.push(Float32Array.of(4, 5))
|
||||
assert.deepEqual([...queue.take(2)], [4, 5])
|
||||
queue.push(Float32Array.of(6))
|
||||
assert.deepEqual([...queue.take(1)], [6])
|
||||
})
|
||||
|
||||
test('ASR sample queue rejects invalid count without losing buffered audio', () => {
|
||||
const queue = new SampleQueue()
|
||||
queue.push(Float32Array.of(1, 2))
|
||||
for (const count of [0, -1, 0.5, Number.NaN, Infinity, -Infinity, Number.MAX_SAFE_INTEGER + 1]) {
|
||||
assert.throws(() => queue.take(count), { name: 'RangeError', message: 'Sample count must be a positive safe integer' })
|
||||
assert.equal(queue.length, 2)
|
||||
}
|
||||
assert.equal(queue.take(Number.MAX_SAFE_INTEGER), null)
|
||||
assert.deepEqual([...queue.take(2)], [1, 2])
|
||||
})
|
||||
|
||||
test('ASR sample queue handles many worklet blocks without dropping boundary samples', () => {
|
||||
const queue = new SampleQueue()
|
||||
for (let index = 0; index < 4096; index++)
|
||||
queue.push(Float32Array.of(index, -index))
|
||||
assert.equal(queue.length, 8192)
|
||||
const first = queue.take(4097)
|
||||
const second = queue.take(4095)
|
||||
assert.equal(queue.length, 0)
|
||||
const output = [...first, ...second]
|
||||
for (let index = 0; index < 4096; index++) {
|
||||
assert.equal(output[index * 2], index)
|
||||
assert.equal(output[index * 2 + 1], -index)
|
||||
}
|
||||
})
|
||||
|
||||
test('ASR sample queue preserves FIFO through deterministic mixed push, take and clear operations', () => {
|
||||
const queue = new SampleQueue()
|
||||
let expected = []
|
||||
let serial = 0
|
||||
for (let index = 1; index <= 1000; index++) {
|
||||
if (index % 113 === 0) {
|
||||
queue.clear()
|
||||
expected = []
|
||||
}
|
||||
else if (index % 3 !== 0) {
|
||||
const chunk = Float32Array.from({ length: index % 17 }, () => ++serial)
|
||||
queue.push(chunk)
|
||||
expected.push(...chunk)
|
||||
}
|
||||
else {
|
||||
const count = index % 23 + 1
|
||||
const actual = queue.take(count)
|
||||
if (expected.length < count)
|
||||
assert.equal(actual, null)
|
||||
else
|
||||
assert.deepEqual([...actual], expected.splice(0, count))
|
||||
}
|
||||
assert.equal(queue.length, expected.length)
|
||||
}
|
||||
if (expected.length)
|
||||
assert.deepEqual([...queue.take(expected.length)], expected)
|
||||
})
|
||||
|
||||
test('ASR candidate Worklet keeps registration, first-channel identity and empty-input behavior', async () => {
|
||||
const { processor, messages } = await runAsrWorklet(new Blob([recorderProcessorCode]))
|
||||
const first = Float32Array.of(-0.5, 0.5)
|
||||
const second = Float32Array.of(1, 1)
|
||||
assert.equal(processor.process([]), true)
|
||||
assert.equal(processor.process([[]]), true)
|
||||
assert.equal(messages.length, 0)
|
||||
assert.equal(processor.process([[first, second]]), true)
|
||||
assert.equal(messages.length, 1)
|
||||
assert.equal(messages[0], first)
|
||||
})
|
||||
@@ -0,0 +1,434 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { Buffer } from 'node:buffer'
|
||||
import fs from 'node:fs'
|
||||
import process from 'node:process'
|
||||
// eslint-disable-next-line test/no-import-node-test -- Candidate ASR lifecycle uses the repository runner.
|
||||
import test from 'node:test'
|
||||
import { asrAudioEnvironment, deferredAudio } from './helpers/asr-audio.js'
|
||||
import { asrEnvironment } from './helpers/asr.js'
|
||||
import { compilePackage } from './helpers/load.js'
|
||||
|
||||
const artifact = process.env.ARTPLAYER_ASR_ARTIFACT
|
||||
const implementation = {
|
||||
name: artifact || 'candidate-asr',
|
||||
code: artifact ? fs.readFileSync(artifact, 'utf8') : await compilePackage('artplayer-plugin-asr', 'umd'),
|
||||
}
|
||||
const block = (value = 0, length = 8) => new Float32Array(length).fill(value)
|
||||
|
||||
function create(option = {}, capabilities = {}) {
|
||||
return asrAudioEnvironment(implementation, { sampleRate: 8000, interval: 1, ...option }, capabilities)
|
||||
}
|
||||
|
||||
function pcmSamples(chunk) {
|
||||
const bytes = new DataView(chunk.pcm)
|
||||
return Array.from({ length: chunk.pcm.byteLength / 2 }, (_, index) => bytes.getInt16(index * 2, true))
|
||||
}
|
||||
|
||||
test('ASR candidate lazy factory and synchronous registration keep result keys and Promise stop', async () => {
|
||||
const environment = asrEnvironment(implementation)
|
||||
assert.equal(environment.exported.default, environment.exported, 'Both historical CJS factory paths remain callable')
|
||||
assert.deepEqual(Object.keys(environment.exported), [])
|
||||
const option = { length: 1, autoHideTimeout: 321 }
|
||||
const register = environment.factory(option)
|
||||
assert.equal(typeof register, 'function')
|
||||
assert.equal(environment.layers.length, 0)
|
||||
assert.equal(environment.listeners.size, 0)
|
||||
option.length = 3
|
||||
option.autoHideTimeout = 999
|
||||
const result = register(environment.art)
|
||||
assert.equal(result.then, undefined)
|
||||
assert.deepEqual(Object.keys(result), ['name', 'stop', 'hide', 'append'])
|
||||
assert.equal(result.name, 'artplayerPluginAsr')
|
||||
assert.equal(environment.layers.length, 1)
|
||||
assert.equal(environment.layers[0].name, 'asr')
|
||||
assert.equal(environment.layers[0].html, '')
|
||||
result.append('One. Two! <b>Three</b>')
|
||||
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line"><b>Three</b></div>')
|
||||
assert.equal([...environment.timeouts.values()][0].delay, 321, 'Factory snapshots its configuration')
|
||||
const stopping = result.stop()
|
||||
assert.equal(typeof stopping.then, 'function')
|
||||
assert.equal(await stopping, undefined)
|
||||
assert.equal(environment.timeouts.size, 1, 'A displayed subtitle keeps its existing auto-hide deadline after stop')
|
||||
;[...environment.timeouts.values()][0].callback()
|
||||
assert.equal(environment.layer.style.display, 'none')
|
||||
await environment.emit('destroy')
|
||||
assert.equal(environment.timeouts.size, 0)
|
||||
})
|
||||
|
||||
test('ASR candidate append keeps punctuation, HTML, ignored non-string values and timer replacement', async () => {
|
||||
const environment = create({ length: 2, autoHideTimeout: 432 })
|
||||
assert.equal(environment.plugin.append('First. Second! Third?'), undefined)
|
||||
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">Second!</div><div class="art-asr-line">Third?</div>')
|
||||
const firstTimer = [...environment.timeouts.keys()][0]
|
||||
environment.plugin.append(42)
|
||||
assert.equal([...environment.timeouts.keys()][0], firstTimer)
|
||||
environment.plugin.append('<b>Raw</b>')
|
||||
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line"><b>Raw</b></div>')
|
||||
assert.equal(environment.timeouts.size, 1)
|
||||
assert.notEqual([...environment.timeouts.keys()][0], firstTimer)
|
||||
const timer = [...environment.timeouts.values()][0]
|
||||
assert.equal(timer.delay, 432)
|
||||
timer.callback()
|
||||
assert.equal(environment.layer.style.display, 'none')
|
||||
environment.plugin.append('Visible.')
|
||||
assert.equal(environment.layer.style.display, '')
|
||||
assert.equal(environment.plugin.hide(), undefined)
|
||||
assert.equal(environment.layer.style.display, 'none')
|
||||
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">Visible.</div>')
|
||||
await environment.plugin.stop()
|
||||
})
|
||||
|
||||
test('ASR candidate preserves chunk payload, exact PCM/WAV bytes and callback subtitles', async () => {
|
||||
let chunk
|
||||
const environment = create({ onAudioChunk(value) {
|
||||
chunk = value
|
||||
return 'One. Two!'
|
||||
} })
|
||||
await environment.emit('play')
|
||||
environment.send([-Infinity, -1, -0.5, Number.NaN, 0, 0.5, 1, Infinity])
|
||||
await environment.tick()
|
||||
assert.deepEqual(Object.keys(chunk), ['pcm', 'wav'])
|
||||
assert.deepEqual(pcmSamples(chunk), [-32768, -32768, -16384, 0, 0, 16383, 32767, 32767])
|
||||
const wav = Buffer.from(chunk.wav)
|
||||
assert.equal(wav.byteLength, 60)
|
||||
assert.equal(wav.toString('ascii', 0, 4), 'RIFF')
|
||||
assert.equal(wav.readUInt32LE(4), 52)
|
||||
assert.equal(wav.readUInt16LE(22), 1)
|
||||
assert.equal(wav.readUInt32LE(24), 8000)
|
||||
assert.equal(wav.readUInt32LE(28), 16000)
|
||||
assert.equal(wav.readUInt32LE(40), 16)
|
||||
assert.deepEqual(wav.subarray(44), Buffer.from(chunk.pcm))
|
||||
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">One.</div><div class="art-asr-line">Two!</div>')
|
||||
await environment.plugin.stop()
|
||||
})
|
||||
|
||||
test('ASR candidate preserves short chunks across ticks and excess tail in FIFO order', async () => {
|
||||
const chunks = []
|
||||
const environment = create({ onAudioChunk: value => chunks.push(value) })
|
||||
await environment.emit('play')
|
||||
environment.send(block(0.5, 4))
|
||||
await environment.tick()
|
||||
assert.equal(chunks.length, 0)
|
||||
environment.send(block(-0.5, 8))
|
||||
await environment.tick()
|
||||
assert.deepEqual(pcmSamples(chunks[0]), [16383, 16383, 16383, 16383, -16384, -16384, -16384, -16384])
|
||||
await environment.tick()
|
||||
assert.equal(chunks.length, 1)
|
||||
environment.send(block(1, 4))
|
||||
await environment.tick()
|
||||
assert.deepEqual(pcmSamples(chunks[1]), [-16384, -16384, -16384, -16384, 32767, 32767, 32767, 32767])
|
||||
await environment.plugin.stop()
|
||||
})
|
||||
|
||||
test('ASR candidate allows one in-flight recognizer and retains queued audio', async () => {
|
||||
const pending = deferredAudio()
|
||||
const calls = []
|
||||
const environment = create({ onAudioChunk: (chunk) => {
|
||||
calls.push(chunk)
|
||||
return calls.length === 1 ? pending.promise : 'Second.'
|
||||
} })
|
||||
await environment.emit('play')
|
||||
environment.send(block(0.5))
|
||||
const first = environment.tick()
|
||||
environment.send(block(-0.5))
|
||||
await environment.tick()
|
||||
await environment.tick()
|
||||
assert.equal(calls.length, 1)
|
||||
pending.resolve('First.')
|
||||
await first
|
||||
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">First.</div>')
|
||||
await environment.tick()
|
||||
assert.equal(calls.length, 2)
|
||||
assert.deepEqual(pcmSamples(calls[1]), Array.from({ length: 8 }, () => -16384))
|
||||
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">Second.</div>')
|
||||
await environment.plugin.stop()
|
||||
})
|
||||
|
||||
for (const mode of ['throw', 'reject']) {
|
||||
test(`ASR candidate handles callback ${mode} and continues subsequent audio`, async () => {
|
||||
const failure = new Error('recognition failed')
|
||||
let count = 0
|
||||
const environment = create({ onAudioChunk: () => {
|
||||
if (++count > 1)
|
||||
return 'Recovered.'
|
||||
if (mode === 'throw')
|
||||
throw failure
|
||||
return Promise.reject(failure)
|
||||
} })
|
||||
await environment.emit('play')
|
||||
environment.send(block())
|
||||
await assert.doesNotReject(environment.tick())
|
||||
assert.equal(environment.errors.length, 1)
|
||||
assert.equal(environment.errors[0][1], failure)
|
||||
environment.send(block())
|
||||
await environment.tick()
|
||||
assert.equal(count, 2)
|
||||
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">Recovered.</div>')
|
||||
await environment.plugin.stop()
|
||||
})
|
||||
}
|
||||
|
||||
for (const event of ['pause', 'stop', 'destroy', 'restart']) {
|
||||
test(`ASR candidate invalidates pending callback after ${event}`, async () => {
|
||||
const pending = deferredAudio()
|
||||
const environment = create({ onAudioChunk: () => pending.promise })
|
||||
await environment.emit('play')
|
||||
environment.send(block())
|
||||
const recognizing = environment.tick()
|
||||
await (event === 'stop' ? environment.plugin.stop() : environment.emit(event))
|
||||
pending.resolve('Obsolete.')
|
||||
await recognizing
|
||||
assert.equal(environment.layer.innerHTML, '')
|
||||
assert.equal(environment.timeouts.size, 0)
|
||||
assert.equal(environment.errors.length, 0)
|
||||
assert.equal(environment.intervals.size, event === 'restart' ? 1 : 0)
|
||||
await environment.plugin.stop()
|
||||
})
|
||||
}
|
||||
|
||||
test('ASR candidate reset clears old queued audio without dropping the next source', async () => {
|
||||
const calls = []
|
||||
const environment = create({ onAudioChunk: value => calls.push(value) })
|
||||
await environment.emit('play')
|
||||
environment.send(block(0.5))
|
||||
environment.art.video.src = 'local-new-source.mp4'
|
||||
await environment.emit('restart')
|
||||
await environment.tick()
|
||||
assert.equal(calls.length, 0)
|
||||
environment.send(block(-0.5))
|
||||
await environment.tick()
|
||||
assert.deepEqual(pcmSamples(calls[0]), Array.from({ length: 8 }, () => -16384))
|
||||
await environment.plugin.stop()
|
||||
})
|
||||
|
||||
test('ASR candidate old callback completion cannot unlock a new session pending callback', async () => {
|
||||
const pending = [deferredAudio(), deferredAudio()]
|
||||
let calls = 0
|
||||
const environment = create({ onAudioChunk: () => pending[calls++].promise })
|
||||
await environment.emit('play')
|
||||
environment.send(block())
|
||||
const old = environment.tick()
|
||||
await environment.emit('pause')
|
||||
await environment.emit('play')
|
||||
environment.send(block())
|
||||
const current = environment.tick()
|
||||
pending[0].resolve('Old.')
|
||||
await old
|
||||
environment.send(block())
|
||||
await environment.tick()
|
||||
assert.equal(calls, 2)
|
||||
assert.equal(environment.layer.innerHTML, '')
|
||||
pending[1].resolve('Current.')
|
||||
await current
|
||||
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">Current.</div>')
|
||||
await environment.plugin.stop()
|
||||
})
|
||||
|
||||
test('ASR candidate concurrent play creates one graph and one interval', async () => {
|
||||
const moduleGate = deferredAudio()
|
||||
const moduleEntered = deferredAudio()
|
||||
const environment = create({}, { moduleGate, moduleEntered })
|
||||
const starting = environment.emit('play')
|
||||
const duplicate = environment.emit('play')
|
||||
await moduleEntered.promise
|
||||
assert.equal(environment.modules.length, 1)
|
||||
moduleGate.resolve()
|
||||
await Promise.all([starting, duplicate])
|
||||
await environment.emit('play')
|
||||
assert.equal(environment.contexts.length, 1)
|
||||
assert.equal(environment.recorders.length, 1)
|
||||
assert.equal(environment.intervals.size, 1)
|
||||
await environment.plugin.stop()
|
||||
assert.equal(environment.intervals.size, 0)
|
||||
assert.equal(environment.recorders[0].port.onmessage, null)
|
||||
assert.equal(environment.recorders[0].disconnects, 1)
|
||||
})
|
||||
|
||||
test('ASR candidate pause during module loading prevents late connection and later resumes prepared graph', async () => {
|
||||
const moduleGate = deferredAudio()
|
||||
const moduleEntered = deferredAudio()
|
||||
const environment = create({}, { moduleGate, moduleEntered, capture: false })
|
||||
const starting = environment.emit('play')
|
||||
await moduleEntered.promise
|
||||
await environment.emit('pause')
|
||||
moduleGate.resolve()
|
||||
await starting
|
||||
assert.equal(environment.intervals.size, 0)
|
||||
assert.equal(environment.recorders.length, 0)
|
||||
assert.equal(environment.contexts[0].closes, 0)
|
||||
await environment.emit('play')
|
||||
assert.equal(environment.contexts.length, 1)
|
||||
assert.equal(environment.modules.length, 1)
|
||||
assert.equal(environment.recorders.length, 1)
|
||||
assert.equal(environment.intervals.size, 1)
|
||||
assert.equal(environment.captures(), 0)
|
||||
assert.equal(environment.errors.length, 0)
|
||||
await environment.plugin.stop()
|
||||
})
|
||||
|
||||
for (const event of ['stop', 'destroy']) {
|
||||
test(`ASR candidate ${event} during module loading frees URL and context without resurrection`, async () => {
|
||||
const moduleGate = deferredAudio()
|
||||
const moduleEntered = deferredAudio()
|
||||
const environment = create({}, { moduleGate, moduleEntered })
|
||||
const starting = environment.emit('play')
|
||||
await moduleEntered.promise
|
||||
assert.equal(environment.blobs.size, 1)
|
||||
await (event === 'stop' ? environment.plugin.stop() : environment.emit(event))
|
||||
assert.equal(environment.blobs.size, 0)
|
||||
assert.equal(environment.contexts[0].closes, 1)
|
||||
moduleGate.resolve()
|
||||
await starting
|
||||
assert.equal(environment.revoked.length, 1)
|
||||
assert.equal(environment.intervals.size, 0)
|
||||
assert.equal(environment.recorders.length, 0)
|
||||
assert.equal(environment.errors.length, 0)
|
||||
})
|
||||
}
|
||||
|
||||
test('ASR candidate failed module initialization releases context and URL before binding video', async () => {
|
||||
const failure = new Error('failed worklet')
|
||||
const environment = create({}, { moduleError: failure })
|
||||
await environment.emit('play')
|
||||
assert.equal(environment.errors.length, 1)
|
||||
assert.equal(environment.errors[0][1], failure)
|
||||
assert.equal(environment.contexts[0].closes, 1)
|
||||
assert.equal(environment.blobs.size, 0)
|
||||
assert.equal(environment.revoked.length, 1)
|
||||
assert.equal(environment.nodes.filter(node => node.kind === 'element').length, 0)
|
||||
assert.equal(environment.intervals.size, 0)
|
||||
await environment.plugin.stop()
|
||||
assert.equal(environment.contexts[0].closes, 1)
|
||||
})
|
||||
|
||||
test('ASR candidate pause/resume reuses direct source without captureStream', async () => {
|
||||
const environment = create({}, { capture: false })
|
||||
await environment.emit('play')
|
||||
const source = environment.nodes.find(node => node.kind === 'element')
|
||||
await environment.emit('pause')
|
||||
assert.equal(source.connections.length, 1, 'The player-owned audio route remains available while ASR is paused')
|
||||
assert.equal(source.connections[0].kind, 'gain')
|
||||
assert.equal(source.connections[0].connections[0], environment.contexts[0].destination)
|
||||
assert.equal(environment.recorders[0].port.onmessage, null)
|
||||
await environment.emit('play')
|
||||
assert.equal(environment.contexts.length, 1)
|
||||
assert.equal(environment.modules.length, 1)
|
||||
assert.equal(environment.nodes.filter(node => node.kind === 'element').length, 1)
|
||||
assert.equal(source.connections.length, 2)
|
||||
assert.equal(environment.errors.length, 0)
|
||||
assert.equal(environment.intervals.size, 1)
|
||||
await environment.plugin.stop()
|
||||
})
|
||||
|
||||
test('ASR candidate ignores old worklet delivery after pause/resume reconnect', async () => {
|
||||
const calls = []
|
||||
const environment = create({ onAudioChunk: value => calls.push(value) })
|
||||
await environment.emit('play')
|
||||
const staleMessage = environment.recorders[0].port.onmessage
|
||||
await environment.emit('pause')
|
||||
await environment.emit('play')
|
||||
staleMessage({ data: block(0.5) })
|
||||
environment.send(block(-0.5))
|
||||
await environment.tick()
|
||||
assert.equal(calls.length, 1)
|
||||
assert.deepEqual(pcmSamples(calls[0]), Array.from({ length: 8 }, () => -16384))
|
||||
await environment.plugin.stop()
|
||||
})
|
||||
|
||||
test('ASR candidate overload explicitly reports and pauses instead of silently discarding samples', async () => {
|
||||
const environment = create()
|
||||
await environment.emit('play')
|
||||
environment.send(block(0, 8000 * 60))
|
||||
assert.equal(environment.errors.length, 0)
|
||||
assert.equal(environment.intervals.size, 1)
|
||||
environment.send(block(0, 1))
|
||||
assert.equal(environment.errors.length, 1)
|
||||
assert.deepEqual(environment.errors[0], ['[artplayerPluginAsr] Audio callback backlog exceeded its capture limit'])
|
||||
assert.equal(environment.intervals.size, 0)
|
||||
assert.equal(environment.recorders[0].port.onmessage, null)
|
||||
await environment.plugin.stop()
|
||||
await environment.emit('destroy')
|
||||
const longChunk = create({ interval: 120000 })
|
||||
await longChunk.emit('play')
|
||||
longChunk.send(block(0, 8000 * 60 + 1))
|
||||
assert.equal(longChunk.errors.length, 0, 'A configured chunk longer than 60 seconds must still be collectable')
|
||||
assert.equal(longChunk.intervals.size, 1)
|
||||
await longChunk.emit('destroy')
|
||||
})
|
||||
|
||||
test('ASR candidate destroy cancels subtitle timers, removes subscriptions and forbids public append resurrection', async () => {
|
||||
const environment = create()
|
||||
await environment.emit('play')
|
||||
environment.plugin.append('Kept.')
|
||||
assert.equal(environment.timeouts.size, 1)
|
||||
await environment.emit('destroy')
|
||||
assert.equal(environment.timeouts.size, 0)
|
||||
assert.equal([...environment.listeners.values()].flat().length, 0)
|
||||
const html = environment.layer.innerHTML
|
||||
const display = environment.layer.style.display
|
||||
environment.plugin.append('Unexpected.')
|
||||
environment.plugin.hide()
|
||||
await environment.emit('play')
|
||||
await environment.emit('restart')
|
||||
await environment.emit('video:volumechange')
|
||||
await environment.plugin.stop()
|
||||
assert.equal(environment.layer.innerHTML, html)
|
||||
assert.equal(environment.layer.style.display, display)
|
||||
assert.equal(environment.timeouts.size, 0)
|
||||
assert.equal(environment.contexts.length, 1)
|
||||
assert.equal(environment.contexts[0].closes, 1)
|
||||
assert.equal(environment.intervals.size, 0)
|
||||
})
|
||||
|
||||
test('ASR candidate public stop retains the current hide deadline and allows append and subsequent play', async () => {
|
||||
const environment = create({}, { capture: false })
|
||||
await environment.emit('play')
|
||||
environment.plugin.append('Before.')
|
||||
const beforeTimer = [...environment.timeouts.keys()][0]
|
||||
await environment.plugin.stop()
|
||||
assert.equal(environment.timeouts.size, 1)
|
||||
assert.equal([...environment.timeouts.keys()][0], beforeTimer)
|
||||
assert.equal(environment.contexts[0].closes, 0, 'Direct media ownership lasts until player destruction')
|
||||
const source = environment.nodes.find(node => node.kind === 'element')
|
||||
assert.equal(source.connections[0].kind, 'gain')
|
||||
assert.equal(source.connections[0].connections[0], environment.contexts[0].destination)
|
||||
environment.plugin.append('After.')
|
||||
assert.equal(environment.layer.innerHTML, '<div class="art-asr-line">After.</div>')
|
||||
await environment.emit('play')
|
||||
assert.equal(environment.contexts.length, 1)
|
||||
assert.equal(environment.captures(), 0)
|
||||
assert.equal(environment.intervals.size, 1)
|
||||
await environment.plugin.stop()
|
||||
await environment.emit('destroy')
|
||||
assert.equal(environment.contexts[0].closes, 1)
|
||||
assert.equal(source.connections.length, 0)
|
||||
})
|
||||
|
||||
test('ASR candidate volume route survives stop and resume without resetting gain', async () => {
|
||||
const environment = create({}, { capture: false })
|
||||
await environment.emit('play')
|
||||
environment.art.volume = 0.25
|
||||
await environment.emit('video:volumechange')
|
||||
await environment.plugin.stop()
|
||||
const source = environment.nodes.find(node => node.kind === 'element')
|
||||
assert.equal(source.connections[0].gain.value, 0.25)
|
||||
await environment.emit('play')
|
||||
assert.equal(source.connections.find(node => node.kind === 'gain').gain.value, 0.25)
|
||||
await environment.emit('destroy')
|
||||
assert.equal(environment.contexts[0].closes, 1)
|
||||
})
|
||||
|
||||
test('ASR candidate source restart releases captured tracks and obtains a new fallback source', async () => {
|
||||
const environment = create({}, { directError: new Error('video belongs to another context') })
|
||||
await environment.emit('play')
|
||||
assert.equal(environment.captures(), 1)
|
||||
await environment.emit('restart')
|
||||
assert.equal(environment.captures(), 2)
|
||||
assert.equal(environment.contexts[0].closes, 1)
|
||||
assert.equal(environment.contexts[1].closes, 0)
|
||||
assert.equal(environment.tracks[0].stops, 1)
|
||||
assert.equal(environment.intervals.size, 1)
|
||||
await environment.plugin.stop()
|
||||
assert.equal(environment.contexts[1].closes, 1)
|
||||
assert.equal(environment.tracks[0].stops, 2)
|
||||
})
|
||||
+4
-1
@@ -30,7 +30,7 @@ for (const implementation of await asrHistorical()) {
|
||||
assert.equal(env.styles.size, 1)
|
||||
})
|
||||
}
|
||||
test(`ASR ${implementation.name}: append replaces punctuation lines, permits markup, and hide retains content`, () => {
|
||||
test(`ASR ${implementation.name}: append replaces punctuation lines, permits markup, and hide retains content`, async () => {
|
||||
const env = asrEnvironment(implementation)
|
||||
const result = env.factory()(env.art)
|
||||
assert.equal(result.append('One. Two! 三。 Four?'), undefined)
|
||||
@@ -47,6 +47,9 @@ for (const implementation of await asrHistorical()) {
|
||||
result.append('')
|
||||
assert.equal(env.layer.innerHTML, '')
|
||||
assert.equal(env.layer.style.display, '')
|
||||
const timer = [...env.timeouts.keys()][0]
|
||||
await result.stop()
|
||||
assert.equal([...env.timeouts.keys()][0], timer, 'Stopping audio preserves an existing subtitle hide deadline')
|
||||
;[...env.timeouts.values()][0].callback()
|
||||
assert.equal(env.layer.style.display, 'none')
|
||||
})
|
||||
|
||||
@@ -2,20 +2,23 @@ import fs from 'node:fs'
|
||||
import process from 'node:process'
|
||||
import { verifyAsrContract } from '../../refactor/scripts/asr-contract.mjs'
|
||||
import { hash, readMember } from '../../refactor/scripts/releases.mjs'
|
||||
import { compilePackage } from '../helpers/load.js'
|
||||
import { expect, test } from './fixtures.js'
|
||||
|
||||
let contract
|
||||
let candidateCode
|
||||
test.beforeAll(async () => {
|
||||
contract = await verifyAsrContract()
|
||||
candidateCode = process.env.ARTPLAYER_ASR_ARTIFACT ? fs.readFileSync(process.env.ARTPLAYER_ASR_ARTIFACT) : await compilePackage('artplayer-plugin-asr', 'umd')
|
||||
})
|
||||
|
||||
for (const version of ['2.0.0', '2.1.0']) {
|
||||
for (const version of ['2.0.0', '2.1.0', 'candidate']) {
|
||||
for (const core of ['published', 'candidate']) {
|
||||
test(`ASR ${version} + ${core}: native Worklet supplies local PCM and stop closes AudioContext`, async ({ page }, testInfo) => {
|
||||
test(`ASR ${version} + ${core}: native Worklet supplies local PCM with owned stop and destroy`, async ({ page }, testInfo) => {
|
||||
const release = [contract.baseline.release, ...contract.baseline.previous].find(release => release.version === version)
|
||||
const code = readMember(contract.archives.get(version), 'package/dist/artplayer-plugin-asr.js')
|
||||
const code = version === 'candidate' ? candidateCode : readMember(contract.archives.get(version), 'package/dist/artplayer-plugin-asr.js')
|
||||
const media = fs.readFileSync(new URL('./media/audio-tone.m4a', import.meta.url))
|
||||
await testInfo.attach('asr-inputs', { contentType: 'application/json', body: JSON.stringify({ version, core, pluginSha256: hash(code), integrity: release.integrity, mediaSha256: hash(media), media: 'local mono AAC 440 Hz tone', networkService: false }) })
|
||||
await testInfo.attach('asr-inputs', { contentType: 'application/json', body: JSON.stringify({ version, core, pluginSha256: hash(code), integrity: release?.integrity || null, candidate: process.env.ARTPLAYER_ASR_ARTIFACT || 'workspace source', mediaSha256: hash(media), media: 'local mono AAC 440 Hz tone', networkService: false }) })
|
||||
await page.goto(`/test/player.html?core=${core}`)
|
||||
const capabilities = await page.evaluate(() => ({ AudioContext: typeof window.AudioContext, webkitAudioContext: typeof window.webkitAudioContext, AudioWorkletNode: typeof window.AudioWorkletNode }))
|
||||
await testInfo.attach('asr-audio-capabilities', { contentType: 'application/json', body: JSON.stringify(capabilities) })
|
||||
@@ -47,7 +50,7 @@ for (const version of ['2.0.0', '2.1.0']) {
|
||||
const samples = new Int16Array(pcm)
|
||||
let max = 0
|
||||
for (const sample of samples) max = Math.max(max, Math.abs(sample))
|
||||
window.asrChunks.push({ samples: samples.length, max, wavBytes: wav.byteLength, sampleRate: new DataView(wav).getUint32(24, true) })
|
||||
window.asrChunks.push({ samples: samples.length, max, wavBytes: wav.byteLength, sampleRate: new DataView(wav).getUint32(24, true), source: window.art.video.currentSrc })
|
||||
return 'Local transcript.'
|
||||
},
|
||||
})],
|
||||
@@ -60,10 +63,36 @@ for (const version of ['2.0.0', '2.1.0']) {
|
||||
const chunks = await page.evaluate(() => window.asrChunks)
|
||||
expect(chunks.every(chunk => chunk.samples === 1600 && chunk.wavBytes === 3244 && chunk.sampleRate === 16000)).toBe(true)
|
||||
await expect(page.locator('.art-layer-asr')).toContainText('Local transcript.')
|
||||
if (version === 'candidate') {
|
||||
await page.evaluate(() => window.art.pause())
|
||||
const before = await page.evaluate(() => window.asrChunks.length)
|
||||
await page.waitForTimeout(250)
|
||||
expect(await page.evaluate(() => window.asrChunks.length)).toBe(before)
|
||||
await page.locator('#play').click()
|
||||
await expect.poll(() => page.evaluate(() => window.asrChunks.length)).toBeGreaterThan(before + 2)
|
||||
expect(await page.evaluate(() => window.asrContexts.length)).toBe(1)
|
||||
await page.evaluate(() => {
|
||||
window.art.url = '/test/audio-tone.m4a?asr=second'
|
||||
})
|
||||
await expect.poll(() => page.evaluate(() => window.art.video.currentSrc.includes('asr=second') && window.art.video.readyState >= 2)).toBe(true)
|
||||
await page.locator('#play').click()
|
||||
await expect.poll(() => page.evaluate(() => window.asrChunks.filter(chunk => chunk.source.includes('asr=second') && chunk.max > 0).length)).toBeGreaterThanOrEqual(3)
|
||||
expect(await page.evaluate(() => window.asrContexts.length)).toBe(1)
|
||||
}
|
||||
await page.evaluate(() => window.art.plugins.artplayerPluginAsr.stop())
|
||||
expect(await page.evaluate(() => window.asrContexts.map(context => context.state))).toEqual(['closed'])
|
||||
await testInfo.attach('asr-native-chunks', { contentType: 'application/json', body: JSON.stringify(chunks) })
|
||||
expect(await page.evaluate(() => window.asrContexts.map(context => context.state))).toEqual([version === 'candidate' ? 'running' : 'closed'])
|
||||
if (version === 'candidate') {
|
||||
await page.evaluate(() => window.art.pause())
|
||||
const before = await page.evaluate(() => window.asrChunks.filter(chunk => chunk.max > 0).length)
|
||||
await page.locator('#play').click()
|
||||
await expect.poll(() => page.evaluate(() => window.asrChunks.filter(chunk => chunk.max > 0).length)).toBeGreaterThan(before + 2)
|
||||
expect(await page.evaluate(() => window.asrContexts.length)).toBe(1)
|
||||
await page.evaluate(() => window.art.plugins.artplayerPluginAsr.stop())
|
||||
expect(await page.evaluate(() => window.asrContexts.map(context => context.state))).toEqual(['running'])
|
||||
}
|
||||
await testInfo.attach('asr-native-chunks', { contentType: 'application/json', body: JSON.stringify(await page.evaluate(() => window.asrChunks)) })
|
||||
await page.evaluate(() => window.art.destroy())
|
||||
await expect.poll(() => page.evaluate(() => window.asrContexts.map(context => context.state))).toEqual(['closed'])
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -32,6 +32,7 @@ export function asrEnvironment(implementation, { script = false } = {}) {
|
||||
return id
|
||||
},
|
||||
clearTimeout(timer) { timeouts.delete(timer) },
|
||||
clearInterval() {},
|
||||
document: {
|
||||
getElementById: id => styles.get(id),
|
||||
createElement: () => ({}),
|
||||
@@ -57,6 +58,7 @@ export function asrEnvironment(implementation, { script = false } = {}) {
|
||||
return layer
|
||||
} },
|
||||
on(name, callback) { listeners.set(name, [...listeners.get(name) || [], callback]) },
|
||||
off(name, callback) { listeners.set(name, (listeners.get(name) || []).filter(listener => listener !== callback)) },
|
||||
}
|
||||
return { context, factory, exported, styles, layer, layers, listeners, timeouts, art, reload: () => vm.runInContext(code, context), emit: name => Promise.all((listeners.get(name) || []).map(callback => callback())) }
|
||||
}
|
||||
Reference in new issue
Block a user