diff --git a/docs/assets/example/asr.js b/docs/assets/example/asr.js index 08f88ca5c..05835f654 100644 --- a/docs/assets/example/asr.js +++ b/docs/assets/example/asr.js @@ -10,8 +10,8 @@ var art = new Artplayer({ interval: 40, sampleRate: 16000, autoHideTimeout: 10000, - // Use your AI tool to convert buffer into subtitles - onAudioChunk: (buffer) => startAsr(buffer), + // Use your AI tool to convert pcm/wav into subtitles + onAudioChunk: ({ pcm, wav }) => startAsr(pcm), }), ] }); diff --git a/docs/assets/ts/artplayer-plugin-asr.d.ts b/docs/assets/ts/artplayer-plugin-asr.d.ts index 8e63eeb66..c55e63144 100644 --- a/docs/assets/ts/artplayer-plugin-asr.d.ts +++ b/docs/assets/ts/artplayer-plugin-asr.d.ts @@ -3,19 +3,24 @@ import type Artplayer from 'artplayer'; export = artplayerPluginAsr; export as namespace artplayerPluginAsr; +interface AudioChunk { + pcm: ArrayBuffer; + wav: ArrayBuffer; +} + interface AsrPluginOption { length?: number; interval?: number; sampleRate?: number; autoHideTimeout?: number; - onAudioChunk?: (buffer: ArrayBuffer) => void | Promise; + onAudioChunk?: (chunk: AudioChunk) => void | Promise; } interface AsrPluginInstance { name: 'artplayerPluginAsr'; stop: () => void; hide: () => void; - append: () => void; + append: (subtitle: string) => void; } declare function artplayerPluginAsr(option?: AsrPluginOption): (art: Artplayer) => AsrPluginInstance; diff --git a/docs/uncompiled/artplayer-plugin-asr/index.js b/docs/uncompiled/artplayer-plugin-asr/index.js index 6239cb59c..1959d6311 100644 --- a/docs/uncompiled/artplayer-plugin-asr/index.js +++ b/docs/uncompiled/artplayer-plugin-asr/index.js @@ -228,6 +228,29 @@ function artplayerPluginAsr(option = {}) { } return buffer; } + function pcmToWav(pcmBuffer, sampleRate) { + const pcmLength = pcmBuffer.byteLength; + const wavBuffer = new ArrayBuffer(44 + pcmLength); + const view = new DataView(wavBuffer); + const writeString = (offset, str)=>{ + for(let i = 0; i < str.length; i++)view.setUint8(offset + i, str.charCodeAt(i)); + }; + writeString(0, 'RIFF'); + view.setUint32(4, 36 + pcmLength, true); + writeString(8, 'WAVE'); + writeString(12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + writeString(36, 'data'); + view.setUint32(40, pcmLength, true); + new Uint8Array(wavBuffer).set(new Uint8Array(pcmBuffer), 44); + return wavBuffer; + } async function setupAudioContext() { if (audioCtx) return audioCtx; audioCtx = new (window.AudioContext || window.webkitAudioContext)({ @@ -241,7 +264,7 @@ function artplayerPluginAsr(option = {}) { sourceNode = audioCtx.createMediaElementSource(art.video); return sourceNode; } catch (err) { - console.warn('[artplayerPluginAsr] Direct connection failed, trying fallback...', err); + console.warn('[artplayerPluginAsr] Direct connection failed:', err); } try { if (!mediaStream) mediaStream = await captureAudioFromVideo(art.video); @@ -299,8 +322,12 @@ function artplayerPluginAsr(option = {}) { } if (accumulated.length < CHUNK_SAMPLES) return; const chunkToSend = accumulated.slice(0, CHUNK_SAMPLES); - const buffer = floatTo16BitPCM(chunkToSend); - const subtitle = await onAudioChunk(buffer); + const pcm = floatTo16BitPCM(chunkToSend); + const wav = pcmToWav(pcm, sampleRate); + const subtitle = await onAudioChunk({ + pcm, + wav + }); append(subtitle); }, interval); started = true; diff --git a/docs/uncompiled/artplayer-plugin-asr/index.js.map b/docs/uncompiled/artplayer-plugin-asr/index.js.map index e9be9b6d5..9981b6d5c 100644 --- a/docs/uncompiled/artplayer-plugin-asr/index.js.map +++ b/docs/uncompiled/artplayer-plugin-asr/index.js.map @@ -1 +1 @@ -{"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;6CAEwB;AAFxB;;AAEe,SAAS,mBAAmB,SAAS,CAAC,CAAC;IAClD,MAAM,EACF,SAAS,CAAC,EACV,WAAW,GAAG,EACd,aAAa,KAAK,EAClB,kBAAkB,KAAK,EACvB,eAAe,IAAM,IAAI,EAC5B,GAAG;IAEJ,OAAO,CAAC;QACJ,IAAI,UAAU;QACd,IAAI,WAAW;QACf,IAAI,aAAa;QACjB,IAAI,eAAe;QACnB,IAAI,WAAW;QACf,IAAI,eAAe,EAAE;QACrB,IAAI,QAAQ;QACZ,IAAI,gBAAgB;QACpB,IAAI,YAAY;QAChB,IAAI,cAAc;QAClB,IAAI,oBAAoB;QAExB,MAAM,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC;YACxB,MAAM;YACN,MAAM;QACV;QAEA,SAAS,mBAAmB,IAAI;YAC5B,OACI,IACI,6CAA6C;aAC5C,KAAK,CAAC,uBACN,GAAG,CAAC,CAAC,IAAM,EAAE,IAAI,IACjB,MAAM,CAAC,SACP,KAAK,CAAC,CAAC;QAEpB;QAEA,SAAS;YACL,KAAK,KAAK,CAAC,OAAO,GAAG;QACzB;QAEA,SAAS,OAAO,QAAQ;YACpB,IAAI,OAAO,aAAa,UAAU;YAClC,aAAa;YACb,YAAY,WAAW,MAAM;YAC7B,KAAK,KAAK,CAAC,OAAO,GAAG;YACrB,KAAK,SAAS,GAAG,mBAAmB,UAC/B,GAAG,CAAC,CAAC,OAAS,CAAC,0BAA0B,EAAE,KAAK,MAAM,CAAC,EACvD,IAAI,CAAC;QACd;QAEA,MAAM,wBAAwB,CAAC;;;;;;;;;;;QAW/B,CAAC;QAED,MAAM,uBAAuB;YACzB,MAAM,OAAO,IAAI,KAAK;gBAAC;aAAsB,EAAE;gBAAE,MAAM;YAAyB;YAChF,OAAO,IAAI,eAAe,CAAC;QAC/B;QAEA,SAAS,gBAAgB,YAAY;YACjC,MAAM,MAAM,aAAa,MAAM;YAC/B,MAAM,SAAS,IAAI,YAAY,MAAM;YACrC,MAAM,OAAO,IAAI,SAAS;YAC1B,IAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IAAK;gBAC1B,IAAI,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,GAAG,YAAY,CAAC,EAAE;gBAChD,KAAK,QAAQ,CAAC,IAAI,GAAG,IAAI,IAAI,IAAI,SAAS,IAAI,QAAQ;YAC1D;YACA,OAAO;QACX;QAEA,eAAe;YACX,IAAI,UAAU,OAAO;YACrB,WAAW,IAAK,CAAA,OAAO,YAAY,IAAI,OAAO,kBAAkB,AAAD,EAAG;gBAAE;YAAW;YAC/E,IAAI,SAAS,KAAK,KAAK,aAAa,MAAM,SAAS,MAAM;YACzD,OAAO;QACX;QAEA,eAAe;YACX,IAAI,CAAC,cAAc,IAAI,KAAK,EACxB,IAAI;gBACA,aAAa,SAAS,wBAAwB,CAAC,IAAI,KAAK;gBACxD,OAAO;YACX,EAAE,OAAO,KAAK;gBACV,QAAQ,IAAI,CAAC,qEAAqE;YACtF;YAGJ,IAAI;gBACA,IAAI,CAAC,aACD,cAAc,MAAM,sBAAsB,IAAI,KAAK;gBAEvD,IAAI,aAAa;oBACb,oBAAoB,SAAS,uBAAuB,CAAC;oBACrD,OAAO;gBACX;YACJ,EAAE,OAAO,KAAK;gBACV,QAAQ,IAAI,CAAC,qDAAqD;YACtE;YAEA,OAAO;QACX;QAEA,eAAe,sBAAsB,YAAY;YAC7C,IAAI;gBACA,IAAI,aAAa,aAAa,EAAE,OAAO,aAAa,aAAa;gBACjE,IAAI,aAAa,gBAAgB,EAAE,OAAO,aAAa,gBAAgB;gBACvE,QAAQ,IAAI,CAAC;gBACb,OAAO;YACX,EAAE,OAAO,KAAK;gBACV,QAAQ,IAAI,CAAC,gDAAgD;gBAC7D,OAAO;YACX;QACJ;QAEA,eAAe;YACX,IAAI,SAAS;YAEb,IAAI;gBACA,MAAM;gBACN,MAAM,cAAc,MAAM;gBAC1B,IAAI,CAAC,aAAa,MAAM,IAAI,MAAM;gBAElC,IAAI,CAAC,eAAe;oBAChB,MAAM,UAAU;oBAChB,MAAM,SAAS,YAAY,CAAC,SAAS,CAAC;oBACtC,IAAI,eAAe,CAAC;oBACpB,gBAAgB;gBACpB;gBAEA,WAAW,SAAS,UAAU;gBAC9B,SAAS,IAAI,CAAC,KAAK,GAAG;gBACtB,eAAe,IAAI,iBAAiB,UAAU;gBAC9C,aAAa,IAAI,CAAC,SAAS,GAAG,CAAC;oBAC3B,aAAa,IAAI,CAAC,IAAI,aAAa,MAAM,IAAI;gBACjD;gBACA,YAAY,OAAO,CAAC;gBACpB,YAAY,OAAO,CAAC;gBACpB,SAAS,OAAO,CAAC,SAAS,WAAW;gBAErC,MAAM,gBAAgB,aAAc,CAAA,WAAW,IAAG;gBAClD,QAAQ,YAAY;oBAChB,IAAI,aAAa,MAAM,KAAK,GAAG;oBAE/B,IAAI,cAAc,IAAI,aAAa;oBACnC,MAAO,YAAY,MAAM,GAAG,iBAAiB,aAAa,MAAM,CAAE;wBAC9D,MAAM,OAAO,aAAa,KAAK;wBAC/B,MAAM,MAAM,IAAI,aAAa,YAAY,MAAM,GAAG,KAAK,MAAM;wBAC7D,IAAI,GAAG,CAAC,aAAa;wBACrB,IAAI,GAAG,CAAC,MAAM,YAAY,MAAM;wBAChC,cAAc;oBAClB;oBAEA,IAAI,YAAY,MAAM,GAAG,eAAe;oBAExC,MAAM,cAAc,YAAY,KAAK,CAAC,GAAG;oBACzC,MAAM,SAAS,gBAAgB;oBAC/B,MAAM,WAAW,MAAM,aAAa;oBACpC,OAAO;gBACX,GAAG;gBAEH,UAAU;YACd,EAAE,OAAO,KAAK;gBACV,QAAQ,KAAK,CAAC,+CAA+C;gBAC7D,MAAM;YACV;QACJ;QAEA,eAAe;YACX,IAAI,CAAC,SAAS;YACd,UAAU;YAEV,cAAc;YACd,QAAQ;YAER,IAAI,cAAc;gBACd,aAAa,UAAU;gBACvB,aAAa,IAAI,CAAC,SAAS,GAAG;gBAC9B,eAAe;YACnB;YAEA,IAAI,UAAU;gBACV,SAAS,UAAU;gBACnB,WAAW;YACf;YAEA,eAAe,EAAE;QACrB;QAEA,eAAe;YACX,MAAM;YAEN,IAAI,mBAAmB;gBACnB,kBAAkB,UAAU;gBAC5B,oBAAoB;YACxB;YAEA,IAAI,YAAY;gBACZ,WAAW,UAAU;gBACrB,aAAa;YACjB;YAEA,IAAI,aAAa;gBACb,YAAY,SAAS,GAAG,OAAO,CAAC,CAAC,QAAU,MAAM,IAAI;gBACrD,cAAc;YAClB;YAEA,IAAI,UAAU;gBACV,MAAM,SAAS,KAAK;gBACpB,WAAW;YACf;YAEA,gBAAgB;QACpB;QAEA,IAAI,EAAE,CAAC,QAAQ;QACf,IAAI,EAAE,CAAC,SAAS;QAChB,IAAI,EAAE,CAAC,WAAW;QAElB,OAAO;YACH,MAAM;YACN,MAAM;YACN;YACA;QACJ;IACJ;AACJ;AAEA,IAAI,OAAO,aAAa,aACpB;IAAA,IAAI,CAAC,SAAS,cAAc,CAAC,yBAAyB;QAClD,MAAM,SAAS,SAAS,aAAa,CAAC;QACtC,OAAO,EAAE,GAAG;QACZ,OAAO,WAAW,GAAG,CAAA,GAAA,yBAAK,AAAD;QACzB,SAAS,IAAI,CAAC,WAAW,CAAC;IAC9B;AAAA;AAGJ,IAAI,OAAO,WAAW,aAClB,MAAM,CAAC,qBAAqB,GAAG;;;AC1PnC,OAAO,OAAO,GAAG;;;ACAjB,QAAQ,cAAc,GAAG,SAAU,CAAC;IAClC,OAAO,KAAK,EAAE,UAAU,GAAG,IAAI;QAAC,SAAS;IAAC;AAC5C;AAEA,QAAQ,iBAAiB,GAAG,SAAU,CAAC;IACrC,OAAO,cAAc,CAAC,GAAG,cAAc;QAAC,OAAO;IAAI;AACrD;AAEA,QAAQ,SAAS,GAAG,SAAU,MAAM,EAAE,IAAI;IACxC,OAAO,IAAI,CAAC,QAAQ,OAAO,CAAC,SAAU,GAAG;QACvC,IACE,QAAQ,aACR,QAAQ,gBACR,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,MAE3C;QAGF,OAAO,cAAc,CAAC,MAAM,KAAK;YAC/B,YAAY;YACZ,KAAK;gBACH,OAAO,MAAM,CAAC,IAAI;YACpB;QACF;IACF;IAEA,OAAO;AACT;AAEA,QAAQ,MAAM,GAAG,SAAU,IAAI,EAAE,QAAQ,EAAE,GAAG;IAC5C,OAAO,cAAc,CAAC,MAAM,UAAU;QACpC,YAAY;QACZ,KAAK;IACP;AACF","sources":["packages/artplayer-plugin-asr/src/index.js","node_modules/@parcel/runtime-js/lib/bundles/runtime-54f3e2ccbee34ab4.js","node_modules/@parcel/transformer-js/src/esmodule-helpers.js"],"sourcesContent":["import style from 'bundle-text:./style.less';\n\nexport default function artplayerPluginAsr(option = {}) {\n const {\n length = 3,\n interval = 100,\n sampleRate = 16000,\n autoHideTimeout = 10000,\n onAudioChunk = () => null,\n } = option;\n\n return (art) => {\n let started = false;\n let audioCtx = null;\n let sourceNode = null;\n let recorderNode = null;\n let gainNode = null;\n let bufferChunks = [];\n let timer = null;\n let workletLoaded = false;\n let hideTimer = null;\n let mediaStream = null;\n let mediaStreamSource = null;\n\n const $asr = art.layers.add({\n name: 'asr',\n html: '',\n });\n\n function splitByPunctuation(text) {\n return (\n text\n // eslint-disable-next-line no-useless-escape\n .split(/(?<=[、。！？!?\\.])\\s*/u)\n .map((s) => s.trim())\n .filter(Boolean)\n .slice(-length)\n );\n }\n\n function hide() {\n $asr.style.display = 'none';\n }\n\n function append(subtitle) {\n if (typeof subtitle !== 'string') return;\n clearTimeout(hideTimer);\n hideTimer = setTimeout(hide, autoHideTimeout);\n $asr.style.display = '';\n $asr.innerHTML = splitByPunctuation(subtitle)\n .map((line) => `
${line}
`)\n .join('');\n }\n\n const recorderProcessorCode = `\n class RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n }\n registerProcessor('recorder-processor', RecorderProcessor);\n `;\n\n const createWorkletBlobUrl = () => {\n const blob = new Blob([recorderProcessorCode], { type: 'application/javascript' });\n return URL.createObjectURL(blob);\n };\n\n function floatTo16BitPCM(float32Array) {\n const len = float32Array.length;\n const buffer = new ArrayBuffer(len * 2);\n const view = new DataView(buffer);\n for (let i = 0; i < len; i++) {\n let s = Math.max(-1, Math.min(1, float32Array[i]));\n view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true);\n }\n return buffer;\n }\n\n async function setupAudioContext() {\n if (audioCtx) return audioCtx;\n audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate });\n if (audioCtx.state === 'suspended') await audioCtx.resume();\n return audioCtx;\n }\n\n async function setupAudioSource() {\n if (!sourceNode && art.video) {\n try {\n sourceNode = audioCtx.createMediaElementSource(art.video);\n return sourceNode;\n } catch (err) {\n console.warn('[artplayerPluginAsr] Direct connection failed, trying fallback...', err);\n }\n }\n\n try {\n if (!mediaStream) {\n mediaStream = await captureAudioFromVideo(art.video);\n }\n if (mediaStream) {\n mediaStreamSource = audioCtx.createMediaStreamSource(mediaStream);\n return mediaStreamSource;\n }\n } catch (err) {\n console.warn('[artplayerPluginAsr] MediaStream fallback failed:', err);\n }\n\n return null;\n }\n\n async function captureAudioFromVideo(videoElement) {\n try {\n if (videoElement.captureStream) return videoElement.captureStream();\n if (videoElement.mozCaptureStream) return videoElement.mozCaptureStream();\n console.warn('[artplayerPluginAsr] captureStream not supported');\n return null;\n } catch (err) {\n console.warn('[artplayerPluginAsr] Error capturing stream:', err);\n return null;\n }\n }\n\n async function startCapture() {\n if (started) return;\n\n try {\n await setupAudioContext();\n const audioSource = await setupAudioSource();\n if (!audioSource) throw new Error('Could not establish audio source');\n\n if (!workletLoaded) {\n const blobUrl = createWorkletBlobUrl();\n await audioCtx.audioWorklet.addModule(blobUrl);\n URL.revokeObjectURL(blobUrl);\n workletLoaded = true;\n }\n\n gainNode = audioCtx.createGain();\n gainNode.gain.value = 1;\n recorderNode = new AudioWorkletNode(audioCtx, 'recorder-processor');\n recorderNode.port.onmessage = (event) => {\n bufferChunks.push(new Float32Array(event.data));\n };\n audioSource.connect(recorderNode);\n audioSource.connect(gainNode);\n gainNode.connect(audioCtx.destination);\n\n const CHUNK_SAMPLES = sampleRate * (interval / 1000);\n timer = setInterval(async () => {\n if (bufferChunks.length === 0) return;\n\n let accumulated = new Float32Array(0);\n while (accumulated.length < CHUNK_SAMPLES && bufferChunks.length) {\n const next = bufferChunks.shift();\n const tmp = new Float32Array(accumulated.length + next.length);\n tmp.set(accumulated, 0);\n tmp.set(next, accumulated.length);\n accumulated = tmp;\n }\n\n if (accumulated.length < CHUNK_SAMPLES) return;\n\n const chunkToSend = accumulated.slice(0, CHUNK_SAMPLES);\n const buffer = floatTo16BitPCM(chunkToSend);\n const subtitle = await onAudioChunk(buffer);\n append(subtitle);\n }, interval);\n\n started = true;\n } catch (err) {\n console.error('[artplayerPluginAsr] Initialization failed:', err);\n await stopCapture();\n }\n }\n\n async function stopCapture() {\n if (!started) return;\n started = false;\n\n clearInterval(timer);\n timer = null;\n\n if (recorderNode) {\n recorderNode.disconnect();\n recorderNode.port.onmessage = null;\n recorderNode = null;\n }\n\n if (gainNode) {\n gainNode.disconnect();\n gainNode = null;\n }\n\n bufferChunks = [];\n }\n\n async function destroy() {\n await stopCapture();\n\n if (mediaStreamSource) {\n mediaStreamSource.disconnect();\n mediaStreamSource = null;\n }\n\n if (sourceNode) {\n sourceNode.disconnect();\n sourceNode = null;\n }\n\n if (mediaStream) {\n mediaStream.getTracks().forEach((track) => track.stop());\n mediaStream = null;\n }\n\n if (audioCtx) {\n await audioCtx.close();\n audioCtx = null;\n }\n\n workletLoaded = false;\n }\n\n art.on('play', startCapture);\n art.on('pause', stopCapture);\n art.on('destroy', destroy);\n\n return {\n name: 'artplayerPluginAsr',\n stop: destroy,\n hide,\n append,\n };\n };\n}\n\nif (typeof document !== 'undefined') {\n if (!document.getElementById('artplayer-plugin-asr')) {\n const $style = document.createElement('style');\n $style.id = 'artplayer-plugin-asr';\n $style.textContent = style;\n document.head.appendChild($style);\n }\n}\n\nif (typeof window !== 'undefined') {\n window['artplayerPluginAsr'] = artplayerPluginAsr;\n}\n","module.exports = \"c5e0457df51aebc9\";","exports.interopDefault = function (a) {\n return a && a.__esModule ? a : {default: a};\n};\n\nexports.defineInteropFlag = function (a) {\n Object.defineProperty(a, '__esModule', {value: true});\n};\n\nexports.exportAll = function (source, dest) {\n Object.keys(source).forEach(function (key) {\n if (\n key === 'default' ||\n key === '__esModule' ||\n Object.prototype.hasOwnProperty.call(dest, key)\n ) {\n return;\n }\n\n Object.defineProperty(dest, key, {\n enumerable: true,\n get: function () {\n return source[key];\n },\n });\n });\n\n return dest;\n};\n\nexports.export = function (dest, destName, get) {\n Object.defineProperty(dest, destName, {\n enumerable: true,\n get: get,\n });\n};\n"],"names":[],"version":3,"file":"index.js.map","sourceRoot":"../../../"} \ No newline at end of file +{"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;6CAEwB;AAFxB;;AAEe,SAAS,mBAAmB,SAAS,CAAC,CAAC;IAClD,MAAM,EACF,SAAS,CAAC,EACV,WAAW,GAAG,EACd,aAAa,KAAK,EAClB,kBAAkB,KAAK,EACvB,eAAe,IAAM,IAAI,EAC5B,GAAG;IAEJ,OAAO,CAAC;QACJ,IAAI,UAAU;QACd,IAAI,WAAW;QACf,IAAI,aAAa;QACjB,IAAI,eAAe;QACnB,IAAI,WAAW;QACf,IAAI,eAAe,EAAE;QACrB,IAAI,QAAQ;QACZ,IAAI,gBAAgB;QACpB,IAAI,YAAY;QAChB,IAAI,cAAc;QAClB,IAAI,oBAAoB;QAExB,MAAM,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC;YACxB,MAAM;YACN,MAAM;QACV;QAEA,SAAS,mBAAmB,IAAI;YAC5B,OACI,IACI,6CAA6C;aAC5C,KAAK,CAAC,uBACN,GAAG,CAAC,CAAC,IAAM,EAAE,IAAI,IACjB,MAAM,CAAC,SACP,KAAK,CAAC,CAAC;QAEpB;QAEA,SAAS;YACL,KAAK,KAAK,CAAC,OAAO,GAAG;QACzB;QAEA,SAAS,OAAO,QAAQ;YACpB,IAAI,OAAO,aAAa,UAAU;YAClC,aAAa;YACb,YAAY,WAAW,MAAM;YAC7B,KAAK,KAAK,CAAC,OAAO,GAAG;YACrB,KAAK,SAAS,GAAG,mBAAmB,UAC/B,GAAG,CAAC,CAAC,OAAS,CAAC,0BAA0B,EAAE,KAAK,MAAM,CAAC,EACvD,IAAI,CAAC;QACd;QAEA,MAAM,wBAAwB,CAAC;;;;;;;;;;;QAW/B,CAAC;QAED,MAAM,uBAAuB;YACzB,MAAM,OAAO,IAAI,KAAK;gBAAC;aAAsB,EAAE;gBAAE,MAAM;YAAyB;YAChF,OAAO,IAAI,eAAe,CAAC;QAC/B;QAEA,SAAS,gBAAgB,YAAY;YACjC,MAAM,MAAM,aAAa,MAAM;YAC/B,MAAM,SAAS,IAAI,YAAY,MAAM;YACrC,MAAM,OAAO,IAAI,SAAS;YAC1B,IAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IAAK;gBAC1B,IAAI,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,GAAG,YAAY,CAAC,EAAE;gBAChD,KAAK,QAAQ,CAAC,IAAI,GAAG,IAAI,IAAI,IAAI,SAAS,IAAI,QAAQ;YAC1D;YACA,OAAO;QACX;QAEA,SAAS,SAAS,SAAS,EAAE,UAAU;YACnC,MAAM,YAAY,UAAU,UAAU;YACtC,MAAM,YAAY,IAAI,YAAY,KAAK;YACvC,MAAM,OAAO,IAAI,SAAS;YAE1B,MAAM,cAAc,CAAC,QAAQ;gBACzB,IAAK,IAAI,IAAI,GAAG,IAAI,IAAI,MAAM,EAAE,IAC5B,KAAK,QAAQ,CAAC,SAAS,GAAG,IAAI,UAAU,CAAC;YAEjD;YAEA,YAAY,GAAG;YACf,KAAK,SAAS,CAAC,GAAG,KAAK,WAAW;YAClC,YAAY,GAAG;YACf,YAAY,IAAI;YAChB,KAAK,SAAS,CAAC,IAAI,IAAI;YACvB,KAAK,SAAS,CAAC,IAAI,GAAG;YACtB,KAAK,SAAS,CAAC,IAAI,GAAG;YACtB,KAAK,SAAS,CAAC,IAAI,YAAY;YAC/B,KAAK,SAAS,CAAC,IAAI,aAAa,GAAG;YACnC,KAAK,SAAS,CAAC,IAAI,GAAG;YACtB,KAAK,SAAS,CAAC,IAAI,IAAI;YACvB,YAAY,IAAI;YAChB,KAAK,SAAS,CAAC,IAAI,WAAW;YAE9B,IAAI,WAAW,WAAW,GAAG,CAAC,IAAI,WAAW,YAAY;YAEzD,OAAO;QACX;QAEA,eAAe;YACX,IAAI,UAAU,OAAO;YACrB,WAAW,IAAK,CAAA,OAAO,YAAY,IAAI,OAAO,kBAAkB,AAAD,EAAG;gBAAE;YAAW;YAC/E,IAAI,SAAS,KAAK,KAAK,aAAa,MAAM,SAAS,MAAM;YACzD,OAAO;QACX;QAEA,eAAe;YACX,IAAI,CAAC,cAAc,IAAI,KAAK,EACxB,IAAI;gBACA,aAAa,SAAS,wBAAwB,CAAC,IAAI,KAAK;gBACxD,OAAO;YACX,EAAE,OAAO,KAAK;gBACV,QAAQ,IAAI,CAAC,kDAAkD;YACnE;YAGJ,IAAI;gBACA,IAAI,CAAC,aACD,cAAc,MAAM,sBAAsB,IAAI,KAAK;gBAEvD,IAAI,aAAa;oBACb,oBAAoB,SAAS,uBAAuB,CAAC;oBACrD,OAAO;gBACX;YACJ,EAAE,OAAO,KAAK;gBACV,QAAQ,IAAI,CAAC,qDAAqD;YACtE;YAEA,OAAO;QACX;QAEA,eAAe,sBAAsB,YAAY;YAC7C,IAAI;gBACA,IAAI,aAAa,aAAa,EAAE,OAAO,aAAa,aAAa;gBACjE,IAAI,aAAa,gBAAgB,EAAE,OAAO,aAAa,gBAAgB;gBACvE,QAAQ,IAAI,CAAC;gBACb,OAAO;YACX,EAAE,OAAO,KAAK;gBACV,QAAQ,IAAI,CAAC,gDAAgD;gBAC7D,OAAO;YACX;QACJ;QAEA,eAAe;YACX,IAAI,SAAS;YAEb,IAAI;gBACA,MAAM;gBACN,MAAM,cAAc,MAAM;gBAC1B,IAAI,CAAC,aAAa,MAAM,IAAI,MAAM;gBAElC,IAAI,CAAC,eAAe;oBAChB,MAAM,UAAU;oBAChB,MAAM,SAAS,YAAY,CAAC,SAAS,CAAC;oBACtC,IAAI,eAAe,CAAC;oBACpB,gBAAgB;gBACpB;gBAEA,WAAW,SAAS,UAAU;gBAC9B,SAAS,IAAI,CAAC,KAAK,GAAG;gBACtB,eAAe,IAAI,iBAAiB,UAAU;gBAC9C,aAAa,IAAI,CAAC,SAAS,GAAG,CAAC;oBAC3B,aAAa,IAAI,CAAC,IAAI,aAAa,MAAM,IAAI;gBACjD;gBACA,YAAY,OAAO,CAAC;gBACpB,YAAY,OAAO,CAAC;gBACpB,SAAS,OAAO,CAAC,SAAS,WAAW;gBAErC,MAAM,gBAAgB,aAAc,CAAA,WAAW,IAAG;gBAClD,QAAQ,YAAY;oBAChB,IAAI,aAAa,MAAM,KAAK,GAAG;oBAE/B,IAAI,cAAc,IAAI,aAAa;oBACnC,MAAO,YAAY,MAAM,GAAG,iBAAiB,aAAa,MAAM,CAAE;wBAC9D,MAAM,OAAO,aAAa,KAAK;wBAC/B,MAAM,MAAM,IAAI,aAAa,YAAY,MAAM,GAAG,KAAK,MAAM;wBAC7D,IAAI,GAAG,CAAC,aAAa;wBACrB,IAAI,GAAG,CAAC,MAAM,YAAY,MAAM;wBAChC,cAAc;oBAClB;oBAEA,IAAI,YAAY,MAAM,GAAG,eAAe;oBAExC,MAAM,cAAc,YAAY,KAAK,CAAC,GAAG;oBACzC,MAAM,MAAM,gBAAgB;oBAC5B,MAAM,MAAM,SAAS,KAAK;oBAC1B,MAAM,WAAW,MAAM,aAAa;wBAAE;wBAAK;oBAAI;oBAC/C,OAAO;gBACX,GAAG;gBAEH,UAAU;YACd,EAAE,OAAO,KAAK;gBACV,QAAQ,KAAK,CAAC,+CAA+C;gBAC7D,MAAM;YACV;QACJ;QAEA,eAAe;YACX,IAAI,CAAC,SAAS;YACd,UAAU;YAEV,cAAc;YACd,QAAQ;YAER,IAAI,cAAc;gBACd,aAAa,UAAU;gBACvB,aAAa,IAAI,CAAC,SAAS,GAAG;gBAC9B,eAAe;YACnB;YAEA,IAAI,UAAU;gBACV,SAAS,UAAU;gBACnB,WAAW;YACf;YAEA,eAAe,EAAE;QACrB;QAEA,eAAe;YACX,MAAM;YAEN,IAAI,mBAAmB;gBACnB,kBAAkB,UAAU;gBAC5B,oBAAoB;YACxB;YAEA,IAAI,YAAY;gBACZ,WAAW,UAAU;gBACrB,aAAa;YACjB;YAEA,IAAI,aAAa;gBACb,YAAY,SAAS,GAAG,OAAO,CAAC,CAAC,QAAU,MAAM,IAAI;gBACrD,cAAc;YAClB;YAEA,IAAI,UAAU;gBACV,MAAM,SAAS,KAAK;gBACpB,WAAW;YACf;YAEA,gBAAgB;QACpB;QAEA,IAAI,EAAE,CAAC,QAAQ;QACf,IAAI,EAAE,CAAC,SAAS;QAChB,IAAI,EAAE,CAAC,WAAW;QAElB,OAAO;YACH,MAAM;YACN,MAAM;YACN;YACA;QACJ;IACJ;AACJ;AAEA,IAAI,OAAO,aAAa,aACpB;IAAA,IAAI,CAAC,SAAS,cAAc,CAAC,yBAAyB;QAClD,MAAM,SAAS,SAAS,aAAa,CAAC;QACtC,OAAO,EAAE,GAAG;QACZ,OAAO,WAAW,GAAG,CAAA,GAAA,yBAAK,AAAD;QACzB,SAAS,IAAI,CAAC,WAAW,CAAC;IAC9B;AAAA;AAGJ,IAAI,OAAO,WAAW,aAClB,MAAM,CAAC,qBAAqB,GAAG;;;ACzRnC,OAAO,OAAO,GAAG;;;ACAjB,QAAQ,cAAc,GAAG,SAAU,CAAC;IAClC,OAAO,KAAK,EAAE,UAAU,GAAG,IAAI;QAAC,SAAS;IAAC;AAC5C;AAEA,QAAQ,iBAAiB,GAAG,SAAU,CAAC;IACrC,OAAO,cAAc,CAAC,GAAG,cAAc;QAAC,OAAO;IAAI;AACrD;AAEA,QAAQ,SAAS,GAAG,SAAU,MAAM,EAAE,IAAI;IACxC,OAAO,IAAI,CAAC,QAAQ,OAAO,CAAC,SAAU,GAAG;QACvC,IACE,QAAQ,aACR,QAAQ,gBACR,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,MAE3C;QAGF,OAAO,cAAc,CAAC,MAAM,KAAK;YAC/B,YAAY;YACZ,KAAK;gBACH,OAAO,MAAM,CAAC,IAAI;YACpB;QACF;IACF;IAEA,OAAO;AACT;AAEA,QAAQ,MAAM,GAAG,SAAU,IAAI,EAAE,QAAQ,EAAE,GAAG;IAC5C,OAAO,cAAc,CAAC,MAAM,UAAU;QACpC,YAAY;QACZ,KAAK;IACP;AACF","sources":["packages/artplayer-plugin-asr/src/index.js","node_modules/@parcel/runtime-js/lib/bundles/runtime-54f3e2ccbee34ab4.js","node_modules/@parcel/transformer-js/src/esmodule-helpers.js"],"sourcesContent":["import style from 'bundle-text:./style.less';\n\nexport default function artplayerPluginAsr(option = {}) {\n const {\n length = 3,\n interval = 100,\n sampleRate = 16000,\n autoHideTimeout = 10000,\n onAudioChunk = () => null,\n } = option;\n\n return (art) => {\n let started = false;\n let audioCtx = null;\n let sourceNode = null;\n let recorderNode = null;\n let gainNode = null;\n let bufferChunks = [];\n let timer = null;\n let workletLoaded = false;\n let hideTimer = null;\n let mediaStream = null;\n let mediaStreamSource = null;\n\n const $asr = art.layers.add({\n name: 'asr',\n html: '',\n });\n\n function splitByPunctuation(text) {\n return (\n text\n // eslint-disable-next-line no-useless-escape\n .split(/(?<=[、。！？!?\\.])\\s*/u)\n .map((s) => s.trim())\n .filter(Boolean)\n .slice(-length)\n );\n }\n\n function hide() {\n $asr.style.display = 'none';\n }\n\n function append(subtitle) {\n if (typeof subtitle !== 'string') return;\n clearTimeout(hideTimer);\n hideTimer = setTimeout(hide, autoHideTimeout);\n $asr.style.display = '';\n $asr.innerHTML = splitByPunctuation(subtitle)\n .map((line) => `
${line}
`)\n .join('');\n }\n\n const recorderProcessorCode = `\n class RecorderProcessor extends AudioWorkletProcessor {\n process(inputs) {\n const input = inputs[0];\n if (input && input[0]) {\n this.port.postMessage(input[0]);\n }\n return true;\n }\n }\n registerProcessor('recorder-processor', RecorderProcessor);\n `;\n\n const createWorkletBlobUrl = () => {\n const blob = new Blob([recorderProcessorCode], { type: 'application/javascript' });\n return URL.createObjectURL(blob);\n };\n\n function floatTo16BitPCM(float32Array) {\n const len = float32Array.length;\n const buffer = new ArrayBuffer(len * 2);\n const view = new DataView(buffer);\n for (let i = 0; i < len; i++) {\n let s = Math.max(-1, Math.min(1, float32Array[i]));\n view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true);\n }\n return buffer;\n }\n\n function pcmToWav(pcmBuffer, sampleRate) {\n const pcmLength = pcmBuffer.byteLength;\n const wavBuffer = new ArrayBuffer(44 + pcmLength);\n const view = new DataView(wavBuffer);\n\n const writeString = (offset, str) => {\n for (let i = 0; i < str.length; i++) {\n view.setUint8(offset + i, str.charCodeAt(i));\n }\n };\n\n writeString(0, 'RIFF');\n view.setUint32(4, 36 + pcmLength, true);\n writeString(8, 'WAVE');\n writeString(12, 'fmt ');\n view.setUint32(16, 16, true);\n view.setUint16(20, 1, true);\n view.setUint16(22, 1, true);\n view.setUint32(24, sampleRate, true);\n view.setUint32(28, sampleRate * 2, true);\n view.setUint16(32, 2, true);\n view.setUint16(34, 16, true);\n writeString(36, 'data');\n view.setUint32(40, pcmLength, true);\n\n new Uint8Array(wavBuffer).set(new Uint8Array(pcmBuffer), 44);\n\n return wavBuffer;\n }\n\n async function setupAudioContext() {\n if (audioCtx) return audioCtx;\n audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate });\n if (audioCtx.state === 'suspended') await audioCtx.resume();\n return audioCtx;\n }\n\n async function setupAudioSource() {\n if (!sourceNode && art.video) {\n try {\n sourceNode = audioCtx.createMediaElementSource(art.video);\n return sourceNode;\n } catch (err) {\n console.warn('[artplayerPluginAsr] Direct connection failed:', err);\n }\n }\n\n try {\n if (!mediaStream) {\n mediaStream = await captureAudioFromVideo(art.video);\n }\n if (mediaStream) {\n mediaStreamSource = audioCtx.createMediaStreamSource(mediaStream);\n return mediaStreamSource;\n }\n } catch (err) {\n console.warn('[artplayerPluginAsr] MediaStream fallback failed:', err);\n }\n\n return null;\n }\n\n async function captureAudioFromVideo(videoElement) {\n try {\n if (videoElement.captureStream) return videoElement.captureStream();\n if (videoElement.mozCaptureStream) return videoElement.mozCaptureStream();\n console.warn('[artplayerPluginAsr] captureStream not supported');\n return null;\n } catch (err) {\n console.warn('[artplayerPluginAsr] Error capturing stream:', err);\n return null;\n }\n }\n\n async function startCapture() {\n if (started) return;\n\n try {\n await setupAudioContext();\n const audioSource = await setupAudioSource();\n if (!audioSource) throw new Error('Could not establish audio source');\n\n if (!workletLoaded) {\n const blobUrl = createWorkletBlobUrl();\n await audioCtx.audioWorklet.addModule(blobUrl);\n URL.revokeObjectURL(blobUrl);\n workletLoaded = true;\n }\n\n gainNode = audioCtx.createGain();\n gainNode.gain.value = 1;\n recorderNode = new AudioWorkletNode(audioCtx, 'recorder-processor');\n recorderNode.port.onmessage = (event) => {\n bufferChunks.push(new Float32Array(event.data));\n };\n audioSource.connect(recorderNode);\n audioSource.connect(gainNode);\n gainNode.connect(audioCtx.destination);\n\n const CHUNK_SAMPLES = sampleRate * (interval / 1000);\n timer = setInterval(async () => {\n if (bufferChunks.length === 0) return;\n\n let accumulated = new Float32Array(0);\n while (accumulated.length < CHUNK_SAMPLES && bufferChunks.length) {\n const next = bufferChunks.shift();\n const tmp = new Float32Array(accumulated.length + next.length);\n tmp.set(accumulated, 0);\n tmp.set(next, accumulated.length);\n accumulated = tmp;\n }\n\n if (accumulated.length < CHUNK_SAMPLES) return;\n\n const chunkToSend = accumulated.slice(0, CHUNK_SAMPLES);\n const pcm = floatTo16BitPCM(chunkToSend);\n const wav = pcmToWav(pcm, sampleRate);\n const subtitle = await onAudioChunk({ pcm, wav });\n append(subtitle);\n }, interval);\n\n started = true;\n } catch (err) {\n console.error('[artplayerPluginAsr] Initialization failed:', err);\n await stopCapture();\n }\n }\n\n async function stopCapture() {\n if (!started) return;\n started = false;\n\n clearInterval(timer);\n timer = null;\n\n if (recorderNode) {\n recorderNode.disconnect();\n recorderNode.port.onmessage = null;\n recorderNode = null;\n }\n\n if (gainNode) {\n gainNode.disconnect();\n gainNode = null;\n }\n\n bufferChunks = [];\n }\n\n async function destroy() {\n await stopCapture();\n\n if (mediaStreamSource) {\n mediaStreamSource.disconnect();\n mediaStreamSource = null;\n }\n\n if (sourceNode) {\n sourceNode.disconnect();\n sourceNode = null;\n }\n\n if (mediaStream) {\n mediaStream.getTracks().forEach((track) => track.stop());\n mediaStream = null;\n }\n\n if (audioCtx) {\n await audioCtx.close();\n audioCtx = null;\n }\n\n workletLoaded = false;\n }\n\n art.on('play', startCapture);\n art.on('pause', stopCapture);\n art.on('destroy', destroy);\n\n return {\n name: 'artplayerPluginAsr',\n stop: destroy,\n hide,\n append,\n };\n };\n}\n\nif (typeof document !== 'undefined') {\n if (!document.getElementById('artplayer-plugin-asr')) {\n const $style = document.createElement('style');\n $style.id = 'artplayer-plugin-asr';\n $style.textContent = style;\n document.head.appendChild($style);\n }\n}\n\nif (typeof window !== 'undefined') {\n window['artplayerPluginAsr'] = artplayerPluginAsr;\n}\n","module.exports = \"c5e0457df51aebc9\";","exports.interopDefault = function (a) {\n return a && a.__esModule ? a : {default: a};\n};\n\nexports.defineInteropFlag = function (a) {\n Object.defineProperty(a, '__esModule', {value: true});\n};\n\nexports.exportAll = function (source, dest) {\n Object.keys(source).forEach(function (key) {\n if (\n key === 'default' ||\n key === '__esModule' ||\n Object.prototype.hasOwnProperty.call(dest, key)\n ) {\n return;\n }\n\n Object.defineProperty(dest, key, {\n enumerable: true,\n get: function () {\n return source[key];\n },\n });\n });\n\n return dest;\n};\n\nexports.export = function (dest, destName, get) {\n Object.defineProperty(dest, destName, {\n enumerable: true,\n get: get,\n });\n};\n"],"names":[],"version":3,"file":"index.js.map","sourceRoot":"../../../"} \ No newline at end of file diff --git a/packages/artplayer-plugin-asr/src/index.js b/packages/artplayer-plugin-asr/src/index.js index 222d20c23..d09407fb2 100644 --- a/packages/artplayer-plugin-asr/src/index.js +++ b/packages/artplayer-plugin-asr/src/index.js @@ -81,6 +81,36 @@ export default function artplayerPluginAsr(option = {}) { return buffer; } + function pcmToWav(pcmBuffer, sampleRate) { + const pcmLength = pcmBuffer.byteLength; + const wavBuffer = new ArrayBuffer(44 + pcmLength); + const view = new DataView(wavBuffer); + + const writeString = (offset, str) => { + for (let i = 0; i < str.length; i++) { + view.setUint8(offset + i, str.charCodeAt(i)); + } + }; + + writeString(0, 'RIFF'); + view.setUint32(4, 36 + pcmLength, true); + writeString(8, 'WAVE'); + writeString(12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + writeString(36, 'data'); + view.setUint32(40, pcmLength, true); + + new Uint8Array(wavBuffer).set(new Uint8Array(pcmBuffer), 44); + + return wavBuffer; + } + async function setupAudioContext() { if (audioCtx) return audioCtx; audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate }); @@ -94,7 +124,7 @@ export default function artplayerPluginAsr(option = {}) { sourceNode = audioCtx.createMediaElementSource(art.video); return sourceNode; } catch (err) { - console.warn('[artplayerPluginAsr] Direct connection failed, trying fallback...', err); + console.warn('[artplayerPluginAsr] Direct connection failed:', err); } } @@ -166,8 +196,9 @@ export default function artplayerPluginAsr(option = {}) { if (accumulated.length < CHUNK_SAMPLES) return; const chunkToSend = accumulated.slice(0, CHUNK_SAMPLES); - const buffer = floatTo16BitPCM(chunkToSend); - const subtitle = await onAudioChunk(buffer); + const pcm = floatTo16BitPCM(chunkToSend); + const wav = pcmToWav(pcm, sampleRate); + const subtitle = await onAudioChunk({ pcm, wav }); append(subtitle); }, interval); diff --git a/packages/artplayer-plugin-asr/types/artplayer-plugin-asr.d.ts b/packages/artplayer-plugin-asr/types/artplayer-plugin-asr.d.ts index 8e63eeb66..c55e63144 100644 --- a/packages/artplayer-plugin-asr/types/artplayer-plugin-asr.d.ts +++ b/packages/artplayer-plugin-asr/types/artplayer-plugin-asr.d.ts @@ -3,19 +3,24 @@ import type Artplayer from 'artplayer'; export = artplayerPluginAsr; export as namespace artplayerPluginAsr; +interface AudioChunk { + pcm: ArrayBuffer; + wav: ArrayBuffer; +} + interface AsrPluginOption { length?: number; interval?: number; sampleRate?: number; autoHideTimeout?: number; - onAudioChunk?: (buffer: ArrayBuffer) => void | Promise; + onAudioChunk?: (chunk: AudioChunk) => void | Promise; } interface AsrPluginInstance { name: 'artplayerPluginAsr'; stop: () => void; hide: () => void; - append: () => void; + append: (subtitle: string) => void; } declare function artplayerPluginAsr(option?: AsrPluginOption): (art: Artplayer) => AsrPluginInstance;