mirror of
https://github.com/zhw2590582/ArtPlayer.git
synced 2026-10-27 01:21:20 -08:00
refactor(asr): [PKG-ASR-03] own typed audio capture and playback lifetimes
This commit is contained in:
Harvey Zhao committed
1 parent
53c295819d
commit
0cf2af4840
33 files changed
+3492
-809
No files matched your search
@@ -0,0 +1,144 @@
|
||||
import type { CaptureVideo } from './types'
|
||||
import { recorderProcessorCode } from './worklet'
|
||||
|
||||
export class AudioGraph {
|
||||
private context: AudioContext | undefined
|
||||
private source: AudioNode | undefined
|
||||
private stream: MediaStream | undefined
|
||||
private recorder: AudioWorkletNode | undefined
|
||||
private gain: GainNode | undefined
|
||||
private preparation: Promise<void> | undefined
|
||||
private blobUrl: string | undefined
|
||||
private closed = false
|
||||
private direct = false
|
||||
private workletLoaded = false
|
||||
|
||||
constructor(private readonly video: CaptureVideo, private readonly sampleRate: number) {}
|
||||
|
||||
private assertOpen() {
|
||||
if (this.closed)
|
||||
throw new Error('Audio capture was closed')
|
||||
}
|
||||
|
||||
private revokeWorklet() {
|
||||
if (this.blobUrl !== undefined) {
|
||||
URL.revokeObjectURL(this.blobUrl)
|
||||
this.blobUrl = undefined
|
||||
}
|
||||
}
|
||||
|
||||
private async initialize() {
|
||||
this.assertOpen()
|
||||
const platform = window as Window & { webkitAudioContext?: typeof AudioContext }
|
||||
const Context = window.AudioContext || platform.webkitAudioContext
|
||||
if (!Context)
|
||||
throw new Error('AudioContext is not supported')
|
||||
const context = this.context ??= new Context({ sampleRate: this.sampleRate })
|
||||
if (context.state === 'suspended')
|
||||
await context.resume()
|
||||
this.assertOpen()
|
||||
if (!this.workletLoaded) {
|
||||
this.blobUrl = URL.createObjectURL(new Blob([recorderProcessorCode], { type: 'application/javascript' }))
|
||||
try {
|
||||
await context.audioWorklet.addModule(this.blobUrl)
|
||||
this.workletLoaded = true
|
||||
}
|
||||
finally {
|
||||
this.revokeWorklet()
|
||||
}
|
||||
}
|
||||
this.assertOpen()
|
||||
if (!this.source) {
|
||||
try {
|
||||
this.source = context.createMediaElementSource(this.video)
|
||||
this.direct = true
|
||||
}
|
||||
catch (error) {
|
||||
console.warn('[artplayerPluginAsr] Direct connection failed:', error)
|
||||
const capture = this.video.captureStream || this.video.mozCaptureStream
|
||||
if (!capture)
|
||||
throw new Error('Could not establish audio source')
|
||||
this.stream = capture.call(this.video)
|
||||
this.source = context.createMediaStreamSource(this.stream)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
prepare() {
|
||||
this.preparation ??= this.initialize().catch((error: unknown) => {
|
||||
this.preparation = undefined
|
||||
throw error
|
||||
})
|
||||
return this.preparation
|
||||
}
|
||||
|
||||
get ownsMediaConnection() {
|
||||
return this.direct
|
||||
}
|
||||
|
||||
idle() {
|
||||
const gain = this.gain
|
||||
this.disconnect()
|
||||
// A media element cannot be reattached after its original context is closed.
|
||||
// Keep its audio route alive until the player releases the element itself.
|
||||
if (this.direct && this.context && this.source) {
|
||||
this.gain = gain || this.context.createGain()
|
||||
if (!gain)
|
||||
this.gain.gain.value = 1
|
||||
this.source.connect(this.gain)
|
||||
this.gain.connect(this.context.destination)
|
||||
}
|
||||
}
|
||||
|
||||
connect(receive: (samples: Float32Array) => void) {
|
||||
this.assertOpen()
|
||||
const context = this.context
|
||||
const source = this.source
|
||||
if (!context || !source)
|
||||
throw new Error('Audio capture is not prepared')
|
||||
const previousGain = this.gain
|
||||
this.disconnect()
|
||||
const gain = this.gain = previousGain || context.createGain()
|
||||
if (!previousGain)
|
||||
gain.gain.value = 1
|
||||
const recorder = this.recorder = new AudioWorkletNode(context, 'recorder-processor')
|
||||
recorder.port.onmessage = (event: MessageEvent<Float32Array>) => {
|
||||
if (!this.closed && this.recorder === recorder)
|
||||
receive(event.data)
|
||||
}
|
||||
source.connect(recorder)
|
||||
source.connect(gain)
|
||||
gain.connect(context.destination)
|
||||
}
|
||||
|
||||
volume(value: number) {
|
||||
if (this.gain)
|
||||
this.gain.gain.value = value
|
||||
}
|
||||
|
||||
disconnect() {
|
||||
if (this.recorder) {
|
||||
this.recorder.port.onmessage = null
|
||||
this.recorder.disconnect()
|
||||
this.recorder = undefined
|
||||
}
|
||||
this.gain?.disconnect()
|
||||
this.gain = undefined
|
||||
this.source?.disconnect()
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this.closed)
|
||||
return
|
||||
this.closed = true
|
||||
this.disconnect()
|
||||
this.revokeWorklet()
|
||||
this.stream?.getTracks().forEach(track => track.stop())
|
||||
this.stream = undefined
|
||||
const context = this.context
|
||||
this.context = undefined
|
||||
this.source = undefined
|
||||
if (context && context.state !== 'closed')
|
||||
await context.close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import type { CaptureOptions, CaptureVideo } from './types'
|
||||
import { AudioGraph } from './audio-graph'
|
||||
import { encodeAudio } from './encoding'
|
||||
import { SampleQueue } from './sample-queue'
|
||||
|
||||
export class Capture {
|
||||
private graph: AudioGraph | undefined
|
||||
private readonly queue = new SampleQueue()
|
||||
private timer: ReturnType<typeof setInterval> | undefined
|
||||
private starting: Promise<void> | undefined
|
||||
private closing: Promise<void> = Promise.resolve()
|
||||
private running = false
|
||||
private terminal = false
|
||||
private epoch = 0
|
||||
private pending: object | undefined
|
||||
|
||||
constructor(private readonly video: CaptureVideo, private readonly options: CaptureOptions, private readonly append: (text: unknown) => void) {}
|
||||
|
||||
private reset() {
|
||||
this.epoch++
|
||||
this.pending = undefined
|
||||
this.queue.clear()
|
||||
}
|
||||
|
||||
pause() {
|
||||
this.reset()
|
||||
this.running = false
|
||||
clearInterval(this.timer)
|
||||
this.timer = undefined
|
||||
this.starting = undefined
|
||||
this.graph?.idle()
|
||||
}
|
||||
|
||||
restart() {
|
||||
const active = this.running || Boolean(this.starting)
|
||||
this.pause()
|
||||
if (this.graph && !this.graph.ownsMediaConnection)
|
||||
return this.stop().then(() => active ? this.start() : undefined)
|
||||
return active ? this.start() : Promise.resolve()
|
||||
}
|
||||
|
||||
start(): Promise<void> {
|
||||
if (this.terminal || this.running)
|
||||
return Promise.resolve()
|
||||
if (this.starting)
|
||||
return this.starting
|
||||
const epoch = this.epoch
|
||||
const current = () => !this.terminal && this.epoch === epoch
|
||||
const operation = (async () => {
|
||||
await this.closing
|
||||
if (!current())
|
||||
return
|
||||
const { sampleRate, interval } = this.options
|
||||
const count = Math.floor(sampleRate * interval / 1000)
|
||||
if (!Number.isSafeInteger(count) || count < 1 || !Number.isFinite(interval) || interval <= 0)
|
||||
throw new Error('Audio chunk length must be positive and finite')
|
||||
const graph = this.graph ??= new AudioGraph(this.video, sampleRate)
|
||||
await graph.prepare()
|
||||
if (!current()) {
|
||||
if (!this.running && this.graph === graph)
|
||||
graph.idle()
|
||||
return
|
||||
}
|
||||
graph.connect((samples) => {
|
||||
if (!this.running || this.graph !== graph)
|
||||
return
|
||||
// Stop overload explicitly instead of retaining an unbounded recognition backlog.
|
||||
if (this.queue.length + samples.length > Math.max(sampleRate * 60, count * 2)) {
|
||||
console.error('[artplayerPluginAsr] Audio callback backlog exceeded its capture limit')
|
||||
this.pause()
|
||||
return
|
||||
}
|
||||
this.queue.push(samples)
|
||||
})
|
||||
this.running = true
|
||||
this.timer = setInterval(() => this.tick(count), interval)
|
||||
})().catch(async (error: unknown) => {
|
||||
if (current()) {
|
||||
console.error('[artplayerPluginAsr] Initialization failed:', error)
|
||||
await this.stop()
|
||||
}
|
||||
}).finally(() => {
|
||||
if (this.starting === operation)
|
||||
this.starting = undefined
|
||||
})
|
||||
this.starting = operation
|
||||
return operation
|
||||
}
|
||||
|
||||
private async tick(count: number) {
|
||||
if (!this.running || this.pending)
|
||||
return
|
||||
const samples = this.queue.take(count)
|
||||
if (!samples)
|
||||
return
|
||||
const epoch = this.epoch
|
||||
const pending = this.pending = {}
|
||||
try {
|
||||
const text = await this.options.onAudioChunk(encodeAudio(samples, this.options.sampleRate))
|
||||
if (this.running && !this.terminal && this.epoch === epoch)
|
||||
this.append(text)
|
||||
}
|
||||
catch (error) {
|
||||
if (this.epoch === epoch && !this.terminal)
|
||||
console.error('[artplayerPluginAsr] Audio callback failed:', error)
|
||||
}
|
||||
finally {
|
||||
if (this.pending === pending)
|
||||
this.pending = undefined
|
||||
}
|
||||
}
|
||||
|
||||
volume(value: number) {
|
||||
this.graph?.volume(value)
|
||||
}
|
||||
|
||||
stop(terminal = false) {
|
||||
this.terminal ||= terminal
|
||||
this.pause()
|
||||
const graph = this.graph
|
||||
if (terminal || !graph?.ownsMediaConnection) {
|
||||
this.graph = undefined
|
||||
this.closing = Promise.all([this.closing, graph?.close()]).then(() => undefined)
|
||||
}
|
||||
return this.closing
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
export function encodeAudio(samples: Float32Array, sampleRate: number): { pcm: ArrayBuffer, wav: ArrayBuffer } {
|
||||
const pcm = new ArrayBuffer(samples.length * 2)
|
||||
const pcmView = new DataView(pcm)
|
||||
for (let index = 0; index < samples.length; index++) {
|
||||
const value = Math.max(-1, Math.min(1, samples[index]!))
|
||||
pcmView.setInt16(index * 2, value < 0 ? value * 0x8000 : value * 0x7FFF, true)
|
||||
}
|
||||
|
||||
const wav = new ArrayBuffer(44 + pcm.byteLength)
|
||||
const header = new DataView(wav)
|
||||
function writeString(offset: number, value: string): void {
|
||||
for (let index = 0; index < value.length; index++)
|
||||
header.setUint8(offset + index, value.charCodeAt(index))
|
||||
}
|
||||
|
||||
writeString(0, 'RIFF')
|
||||
header.setUint32(4, 36 + pcm.byteLength, true)
|
||||
writeString(8, 'WAVE')
|
||||
writeString(12, 'fmt ')
|
||||
header.setUint32(16, 16, true)
|
||||
header.setUint16(20, 1, true)
|
||||
header.setUint16(22, 1, true)
|
||||
header.setUint32(24, sampleRate, true)
|
||||
header.setUint32(28, sampleRate * 2, true)
|
||||
header.setUint16(32, 2, true)
|
||||
header.setUint16(34, 16, true)
|
||||
writeString(36, 'data')
|
||||
header.setUint32(40, pcm.byteLength, true)
|
||||
new Uint8Array(wav).set(new Uint8Array(pcm), 44)
|
||||
return { pcm, wav }
|
||||
}
|
||||
@@ -1,299 +0,0 @@
|
||||
import style from './style.less?inline'
|
||||
|
||||
export default function artplayerPluginAsr(option = {}) {
|
||||
const {
|
||||
length = 3,
|
||||
interval = 100,
|
||||
sampleRate = 16000,
|
||||
autoHideTimeout = 10000,
|
||||
onAudioChunk = () => null,
|
||||
} = option
|
||||
|
||||
return (art) => {
|
||||
let started = false
|
||||
let audioCtx = null
|
||||
let sourceNode = null
|
||||
let recorderNode = null
|
||||
let gainNode = null
|
||||
let bufferChunks = []
|
||||
let timer = null
|
||||
let workletLoaded = false
|
||||
let hideTimer = null
|
||||
let mediaStream = null
|
||||
let mediaStreamSource = null
|
||||
|
||||
const $asr = art.layers.add({
|
||||
name: 'asr',
|
||||
html: '',
|
||||
})
|
||||
|
||||
function splitByPunctuation(text) {
|
||||
return (
|
||||
text
|
||||
|
||||
.split(/(?<=[、。！？!?.])\s*/u)
|
||||
.map(s => s.trim())
|
||||
.filter(Boolean)
|
||||
.slice(-length)
|
||||
)
|
||||
}
|
||||
|
||||
function hide() {
|
||||
$asr.style.display = 'none'
|
||||
}
|
||||
|
||||
function append(subtitle) {
|
||||
if (typeof subtitle !== 'string')
|
||||
return
|
||||
clearTimeout(hideTimer)
|
||||
hideTimer = setTimeout(hide, autoHideTimeout)
|
||||
$asr.style.display = ''
|
||||
$asr.innerHTML = splitByPunctuation(subtitle)
|
||||
.map(line => `<div class="art-asr-line">${line}</div>`)
|
||||
.join('')
|
||||
}
|
||||
|
||||
const recorderProcessorCode = `
|
||||
class RecorderProcessor extends AudioWorkletProcessor {
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (input && input[0]) {
|
||||
this.port.postMessage(input[0]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('recorder-processor', RecorderProcessor);
|
||||
`
|
||||
|
||||
const createWorkletBlobUrl = () => {
|
||||
const blob = new Blob([recorderProcessorCode], { type: 'application/javascript' })
|
||||
return URL.createObjectURL(blob)
|
||||
}
|
||||
|
||||
function floatTo16BitPCM(float32Array) {
|
||||
const len = float32Array.length
|
||||
const buffer = new ArrayBuffer(len * 2)
|
||||
const view = new DataView(buffer)
|
||||
for (let i = 0; i < len; i++) {
|
||||
const s = Math.max(-1, Math.min(1, float32Array[i]))
|
||||
view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7FFF, true)
|
||||
}
|
||||
return buffer
|
||||
}
|
||||
|
||||
function pcmToWav(pcmBuffer, sampleRate) {
|
||||
const pcmLength = pcmBuffer.byteLength
|
||||
const wavBuffer = new ArrayBuffer(44 + pcmLength)
|
||||
const view = new DataView(wavBuffer)
|
||||
|
||||
const writeString = (offset, str) => {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i))
|
||||
}
|
||||
}
|
||||
|
||||
writeString(0, 'RIFF')
|
||||
view.setUint32(4, 36 + pcmLength, true)
|
||||
writeString(8, 'WAVE')
|
||||
writeString(12, 'fmt ')
|
||||
view.setUint32(16, 16, true)
|
||||
view.setUint16(20, 1, true)
|
||||
view.setUint16(22, 1, true)
|
||||
view.setUint32(24, sampleRate, true)
|
||||
view.setUint32(28, sampleRate * 2, true)
|
||||
view.setUint16(32, 2, true)
|
||||
view.setUint16(34, 16, true)
|
||||
writeString(36, 'data')
|
||||
view.setUint32(40, pcmLength, true)
|
||||
|
||||
new Uint8Array(wavBuffer).set(new Uint8Array(pcmBuffer), 44)
|
||||
|
||||
return wavBuffer
|
||||
}
|
||||
|
||||
async function setupAudioContext() {
|
||||
if (audioCtx)
|
||||
return audioCtx
|
||||
audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate })
|
||||
if (audioCtx.state === 'suspended')
|
||||
await audioCtx.resume()
|
||||
return audioCtx
|
||||
}
|
||||
|
||||
async function setupAudioSource() {
|
||||
if (!sourceNode && art.video) {
|
||||
try {
|
||||
sourceNode = audioCtx.createMediaElementSource(art.video)
|
||||
return sourceNode
|
||||
}
|
||||
catch (err) {
|
||||
console.warn('[artplayerPluginAsr] Direct connection failed:', err)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (!mediaStream) {
|
||||
mediaStream = await captureAudioFromVideo(art.video)
|
||||
}
|
||||
if (mediaStream) {
|
||||
mediaStreamSource = audioCtx.createMediaStreamSource(mediaStream)
|
||||
return mediaStreamSource
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
console.warn('[artplayerPluginAsr] MediaStream fallback failed:', err)
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
async function captureAudioFromVideo(videoElement) {
|
||||
try {
|
||||
if (videoElement.captureStream)
|
||||
return videoElement.captureStream()
|
||||
if (videoElement.mozCaptureStream)
|
||||
return videoElement.mozCaptureStream()
|
||||
console.warn('[artplayerPluginAsr] captureStream not supported')
|
||||
return null
|
||||
}
|
||||
catch (err) {
|
||||
console.warn('[artplayerPluginAsr] Error capturing stream:', err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async function startCapture() {
|
||||
if (started)
|
||||
return
|
||||
|
||||
try {
|
||||
await setupAudioContext()
|
||||
const audioSource = await setupAudioSource()
|
||||
if (!audioSource)
|
||||
throw new Error('Could not establish audio source')
|
||||
|
||||
if (!workletLoaded) {
|
||||
const blobUrl = createWorkletBlobUrl()
|
||||
await audioCtx.audioWorklet.addModule(blobUrl)
|
||||
URL.revokeObjectURL(blobUrl)
|
||||
workletLoaded = true
|
||||
}
|
||||
|
||||
gainNode = audioCtx.createGain()
|
||||
gainNode.gain.value = 1
|
||||
recorderNode = new AudioWorkletNode(audioCtx, 'recorder-processor')
|
||||
recorderNode.port.onmessage = (event) => {
|
||||
bufferChunks.push(new Float32Array(event.data))
|
||||
}
|
||||
audioSource.connect(recorderNode)
|
||||
audioSource.connect(gainNode)
|
||||
gainNode.connect(audioCtx.destination)
|
||||
|
||||
const CHUNK_SAMPLES = sampleRate * (interval / 1000)
|
||||
timer = setInterval(async () => {
|
||||
if (bufferChunks.length === 0)
|
||||
return
|
||||
|
||||
let accumulated = new Float32Array(0)
|
||||
while (accumulated.length < CHUNK_SAMPLES && bufferChunks.length) {
|
||||
const next = bufferChunks.shift()
|
||||
const tmp = new Float32Array(accumulated.length + next.length)
|
||||
tmp.set(accumulated, 0)
|
||||
tmp.set(next, accumulated.length)
|
||||
accumulated = tmp
|
||||
}
|
||||
|
||||
if (accumulated.length < CHUNK_SAMPLES)
|
||||
return
|
||||
|
||||
const chunkToSend = accumulated.slice(0, CHUNK_SAMPLES)
|
||||
const pcm = floatTo16BitPCM(chunkToSend)
|
||||
const wav = pcmToWav(pcm, sampleRate)
|
||||
const subtitle = await onAudioChunk({ pcm, wav })
|
||||
append(subtitle)
|
||||
}, interval)
|
||||
|
||||
started = true
|
||||
}
|
||||
catch (err) {
|
||||
console.error('[artplayerPluginAsr] Initialization failed:', err)
|
||||
await stopCapture()
|
||||
}
|
||||
}
|
||||
|
||||
async function stopCapture() {
|
||||
if (!started)
|
||||
return
|
||||
started = false
|
||||
|
||||
clearInterval(timer)
|
||||
timer = null
|
||||
|
||||
if (recorderNode) {
|
||||
recorderNode.disconnect()
|
||||
recorderNode.port.onmessage = null
|
||||
recorderNode = null
|
||||
}
|
||||
|
||||
if (gainNode) {
|
||||
gainNode.disconnect()
|
||||
gainNode = null
|
||||
}
|
||||
|
||||
bufferChunks = []
|
||||
}
|
||||
|
||||
async function destroy() {
|
||||
await stopCapture()
|
||||
|
||||
if (mediaStreamSource) {
|
||||
mediaStreamSource.disconnect()
|
||||
mediaStreamSource = null
|
||||
}
|
||||
|
||||
if (sourceNode) {
|
||||
sourceNode.disconnect()
|
||||
sourceNode = null
|
||||
}
|
||||
|
||||
if (mediaStream) {
|
||||
mediaStream.getTracks().forEach(track => track.stop())
|
||||
mediaStream = null
|
||||
}
|
||||
|
||||
if (audioCtx) {
|
||||
await audioCtx.close()
|
||||
audioCtx = null
|
||||
}
|
||||
|
||||
workletLoaded = false
|
||||
}
|
||||
|
||||
art.on('video:volumechange', () => {
|
||||
if (gainNode) {
|
||||
gainNode.gain.value = art.volume
|
||||
}
|
||||
})
|
||||
|
||||
art.on('play', startCapture)
|
||||
art.on('pause', stopCapture)
|
||||
art.on('destroy', destroy)
|
||||
|
||||
return {
|
||||
name: 'artplayerPluginAsr',
|
||||
stop: destroy,
|
||||
hide,
|
||||
append,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof document !== 'undefined') {
|
||||
if (!document.getElementById('artplayer-plugin-asr')) {
|
||||
const $style = document.createElement('style')
|
||||
$style.id = 'artplayer-plugin-asr'
|
||||
$style.textContent = style
|
||||
document.head.appendChild($style)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import type Artplayer from 'artplayer'
|
||||
import type { AsrOptions, AsrResult } from './types'
|
||||
import { Capture } from './capture'
|
||||
import style from './style.less?inline'
|
||||
import { createSubtitles } from './subtitles'
|
||||
|
||||
export default function artplayerPluginAsr(option: AsrOptions = {}) {
|
||||
const { length = 3, interval = 100, sampleRate = 16000, autoHideTimeout = 10000, onAudioChunk = () => null } = option
|
||||
return (art: Artplayer): AsrResult => {
|
||||
const layer = art.layers.add({ name: 'asr', html: '' })
|
||||
if (!layer)
|
||||
throw new Error('Could not create ASR subtitle layer')
|
||||
const subtitles = createSubtitles(layer, length, autoHideTimeout)
|
||||
const capture = new Capture(art.video, { interval, sampleRate, onAudioChunk }, subtitles.append)
|
||||
const volume = () => capture.volume(art.volume)
|
||||
const play = () => capture.start()
|
||||
const pause = () => capture.pause()
|
||||
const restart = () => capture.restart()
|
||||
const stop = () => capture.stop()
|
||||
const destroy = () => {
|
||||
subtitles.destroy()
|
||||
art.off('video:volumechange', volume)
|
||||
art.off('play', play)
|
||||
art.off('pause', pause)
|
||||
art.off('restart', restart)
|
||||
art.off('destroy', destroy)
|
||||
return capture.stop(true)
|
||||
}
|
||||
art.on('video:volumechange', volume)
|
||||
art.on('play', play)
|
||||
art.on('pause', pause)
|
||||
art.on('restart', restart)
|
||||
art.on('destroy', destroy)
|
||||
return { name: 'artplayerPluginAsr', stop, hide: subtitles.hide, append: subtitles.append }
|
||||
}
|
||||
}
|
||||
|
||||
// Published 2.0 CJS consumers used require(...).default; 2.1 exported the function.
|
||||
Object.defineProperty(artplayerPluginAsr, 'default', { value: artplayerPluginAsr })
|
||||
|
||||
if (typeof document !== 'undefined' && !document.getElementById('artplayer-plugin-asr')) {
|
||||
const styleElement = document.createElement('style')
|
||||
styleElement.id = 'artplayer-plugin-asr'
|
||||
styleElement.textContent = style
|
||||
document.head.appendChild(styleElement)
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
interface SampleBlock {
|
||||
samples: Float32Array
|
||||
next?: SampleBlock
|
||||
}
|
||||
|
||||
export class SampleQueue {
|
||||
private head?: SampleBlock
|
||||
private tail?: SampleBlock
|
||||
private offset = 0
|
||||
private size = 0
|
||||
|
||||
get length(): number {
|
||||
return this.size
|
||||
}
|
||||
|
||||
push(input: Float32Array): void {
|
||||
if (input.length === 0)
|
||||
return
|
||||
const block: SampleBlock = { samples: new Float32Array(input) }
|
||||
if (this.tail)
|
||||
this.tail.next = block
|
||||
else
|
||||
this.head = block
|
||||
this.tail = block
|
||||
this.size += input.length
|
||||
}
|
||||
|
||||
take(count: number): Float32Array | null {
|
||||
if (!Number.isSafeInteger(count) || count <= 0)
|
||||
throw new RangeError('Sample count must be a positive safe integer')
|
||||
if (count > this.size)
|
||||
return null
|
||||
|
||||
const output = new Float32Array(count)
|
||||
let written = 0
|
||||
while (this.head && written < count) {
|
||||
const samples = this.head.samples
|
||||
const amount = Math.min(samples.length - this.offset, count - written)
|
||||
output.set(samples.subarray(this.offset, this.offset + amount), written)
|
||||
written += amount
|
||||
this.offset += amount
|
||||
if (this.offset === samples.length) {
|
||||
this.head = this.head.next
|
||||
this.offset = 0
|
||||
}
|
||||
}
|
||||
if (!this.head)
|
||||
this.tail = undefined
|
||||
this.size -= count
|
||||
return output
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.head = undefined
|
||||
this.tail = undefined
|
||||
this.offset = 0
|
||||
this.size = 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
export function createSubtitles(layer: HTMLElement, length: number, autoHideTimeout: number) {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
let destroyed = false
|
||||
|
||||
function cancelTimer() {
|
||||
clearTimeout(timer)
|
||||
timer = undefined
|
||||
}
|
||||
|
||||
function hide() {
|
||||
if (!destroyed)
|
||||
layer.style.display = 'none'
|
||||
}
|
||||
|
||||
function append(subtitle: unknown) {
|
||||
if (destroyed || typeof subtitle !== 'string')
|
||||
return
|
||||
cancelTimer()
|
||||
timer = setTimeout(hide, autoHideTimeout)
|
||||
layer.style.display = ''
|
||||
layer.innerHTML = subtitle.split(/(?<=[、。！？!?.])\s*/u)
|
||||
.map(line => line.trim())
|
||||
.filter(Boolean)
|
||||
.slice(-length)
|
||||
.map(line => `<div class="art-asr-line">${line}</div>`)
|
||||
.join('')
|
||||
}
|
||||
|
||||
return {
|
||||
append,
|
||||
hide,
|
||||
destroy() {
|
||||
destroyed = true
|
||||
cancelTimer()
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
export interface AudioChunk {
|
||||
pcm: ArrayBuffer
|
||||
wav: ArrayBuffer
|
||||
}
|
||||
|
||||
export interface AsrOptions {
|
||||
length?: number
|
||||
interval?: number
|
||||
sampleRate?: number
|
||||
autoHideTimeout?: number
|
||||
onAudioChunk?: (chunk: AudioChunk) => unknown
|
||||
}
|
||||
|
||||
export interface AsrResult {
|
||||
name: 'artplayerPluginAsr'
|
||||
stop: () => Promise<void>
|
||||
hide: () => void
|
||||
append: (subtitle: string) => void
|
||||
}
|
||||
|
||||
export interface CaptureVideo extends HTMLVideoElement {
|
||||
captureStream?: () => MediaStream
|
||||
mozCaptureStream?: () => MediaStream
|
||||
}
|
||||
|
||||
export interface CaptureOptions {
|
||||
interval: number
|
||||
sampleRate: number
|
||||
onAudioChunk: (chunk: AudioChunk) => unknown
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
export const recorderProcessorCode = `
|
||||
class RecorderProcessor extends AudioWorkletProcessor {
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (input && input[0]) {
|
||||
this.port.postMessage(input[0]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('recorder-processor', RecorderProcessor);
|
||||
`
|
||||
Reference in new issue
Block a user