From 3586ec8aab2071e323a98babd3faae0069d020de Mon Sep 17 00:00:00 2001 From: MartinDelophy <845046459@qq.com> Date: Mon, 13 Jul 2026 17:08:15 +0800 Subject: [PATCH] Merge pull request #1 from MartinDelophy/codex/vocal-worker-discord Move vocal separation to a worker and add Discord community links --- README.md | 5 + README.zh-CN.md | 5 + src/hooks/useEditorLifecycle.js | 2 + src/lib/vocalSeparation.js | 193 ++++++++----------------- src/workers/vocal-separation.worker.js | 161 +++++++++++++++++++++ vite.config.mjs | 6 + 6 files changed, 239 insertions(+), 133 deletions(-) create mode 100644 src/workers/vocal-separation.worker.js diff --git a/README.md b/README.md index 9298a5f..ef5b8ec 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Language: **English** | [中文](README.zh-CN.md) [![GitHub stars](https://img.shields.io/github/stars/MartinDelophy/ai-video-editor?style=flat-square)](https://github.com/MartinDelophy/ai-video-editor/stargazers) [![GitHub forks](https://img.shields.io/github/forks/MartinDelophy/ai-video-editor?style=flat-square)](https://github.com/MartinDelophy/ai-video-editor/forks) [![MIT License](https://img.shields.io/github/license/MartinDelophy/ai-video-editor?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1526147673290313768?label=Discord&logo=discord&style=flat-square)](https://discord.gg/uq2uvUTBr) A local-first, browser-based AI video editor with ONNX voiceover generation, Whisper automatic captions, JoyVASA + LivePortrait talking avatars, multi-track timeline editing, and MP4/WebM export. Timeline Studio runs its core AI workflows directly in the browser and is inspired by modern editors such as CapCut. @@ -218,6 +219,10 @@ Released under the [MIT License](LICENSE). You may use, modify, and distribute t ## Community +Join the Discord community for questions, feedback, and project updates: + +[Join the Timeline Studio Discord](https://discord.gg/uq2uvUTBr) + The current star count is updated live in the GitHub Stars badge at the top of this README. [View current stargazers](https://github.com/MartinDelophy/ai-video-editor/stargazers) · [Star Timeline Studio](https://github.com/MartinDelophy/ai-video-editor) diff --git a/README.zh-CN.md b/README.zh-CN.md index f37165d..20a2a2d 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -6,6 +6,7 @@ [![GitHub Stars](https://img.shields.io/github/stars/MartinDelophy/ai-video-editor?style=flat-square)](https://github.com/MartinDelophy/ai-video-editor/stargazers) [![GitHub Forks](https://img.shields.io/github/forks/MartinDelophy/ai-video-editor?style=flat-square)](https://github.com/MartinDelophy/ai-video-editor/forks) [![MIT License](https://img.shields.io/github/license/MartinDelophy/ai-video-editor?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1526147673290313768?label=Discord&logo=discord&style=flat-square)](https://discord.gg/uq2uvUTBr) 一个本地优先的浏览器 AI 视频编辑器，支持 ONNX AI 配音、Whisper 自动字幕、JoyVASA + LivePortrait 数字人、多轨时间线以及 MP4/WebM 导出。核心 AI 流程直接在浏览器运行，交互参考 CapCut/剪映等现代时间线编辑器。 @@ -203,6 +204,10 @@ npx netlify-cli deploy --prod --dir=dist ## 社区 +欢迎加入 Discord 社区，交流使用问题、产品建议和项目进展： + +[加入 Timeline Studio Discord 社区](https://discord.gg/uq2uvUTBr) + README 顶部的 GitHub Stars 徽章会实时显示当前 Star 数量。 [查看当前 Stargazers](https://github.com/MartinDelophy/ai-video-editor/stargazers) · [为 Timeline Studio 点 Star](https://github.com/MartinDelophy/ai-video-editor) diff --git a/src/hooks/useEditorLifecycle.js b/src/hooks/useEditorLifecycle.js index cba910a..7adca46 100644 --- a/src/hooks/useEditorLifecycle.js +++ b/src/hooks/useEditorLifecycle.js @@ -3,6 +3,7 @@ import { useEffect } from "react"; import { RATIO_OPTIONS } from "../config/editor.js"; import { decodeWaveform } from "../lib/media.js"; import { disposeVisionWorker } from "../lib/vision.js"; +import { disposeVocalSeparationWorker } from "../lib/vocalSeparation.js"; import { getNearestRatioIdForSize, revokeVisionObjectUrls, @@ -158,6 +159,7 @@ export function useEditorLifecycle(d) { d.visionObjectUrlsRef.current.forEach((urls) => revokeVisionObjectUrls(urls)); d.visionObjectUrlsRef.current.clear(); disposeVisionWorker(); + disposeVocalSeparationWorker(); d.voiceRecorderStreamRef.current?.getTracks().forEach((track) => track.stop()); window.clearInterval(d.voiceRecorderTimerRef.current); }, []); diff --git a/src/lib/vocalSeparation.js b/src/lib/vocalSeparation.js index d4a348e..2e80028 100644 --- a/src/lib/vocalSeparation.js +++ b/src/lib/vocalSeparation.js @@ -1,144 +1,71 @@ -const MODEL_URL = "https://huggingface.co/haixin/timeline-studio-vocal-remover/resolve/main/model.json"; -const CHUNK_SIZE = 31744; -const PADDING = 3072; -const FFT_SIZE = 6144; -const HOP_SIZE = 1024; +let worker; +let nextRequestId = 0; +const requests = new Map(); -let modelPromise; -let runtimePromise; -let activeBackend = "webgl"; +function rejectRequests(error) { + requests.forEach(({ reject }) => reject(error)); + requests.clear(); +} -function loadScript(src) { +function resetWorker(error) { + worker?.terminate(); + worker = null; + if (error) rejectRequests(error); +} + +function getWorker() { + if (typeof Worker === "undefined") throw new Error("VOCAL_WORKER_UNAVAILABLE"); + if (worker) return worker; + + worker = new Worker(new URL("../workers/vocal-separation.worker.js", import.meta.url)); + worker.addEventListener("message", ({ data }) => { + const request = requests.get(data?.requestId); + if (!request) return; + if (data.type === "progress") { + request.onProgress(data.progress, data.phase); + return; + } + requests.delete(data.requestId); + if (data.type === "result") { + request.resolve({ + vocals: new Blob([data.vocalsBuffer], { type: "audio/wav" }), + accompaniment: new Blob([data.accompanimentBuffer], { type: "audio/wav" }), + backend: data.backend, + }); + return; + } + request.reject(new Error(data.error || "VOCAL_MODEL_FAILED")); + }); + worker.addEventListener("error", (event) => { + resetWorker(new Error(event.message || "VOCAL_WORKER_FAILED")); + }); + return worker; +} + +function runWorker(left, right, sampleRate, onProgress) { + const requestId = `vocal-${++nextRequestId}`; + const activeWorker = getWorker(); return new Promise((resolve, reject) => { - const existing = document.querySelector(`script[src="${src}"]`); - if (existing) { if (window.tf) resolve(); else existing.addEventListener("load", resolve, { once: true }); return; } - const script = document.createElement("script"); script.src = src; script.crossOrigin = "anonymous"; - script.onload = resolve; script.onerror = () => reject(new Error("VOCAL_RUNTIME_DOWNLOAD_FAILED")); document.head.appendChild(script); + requests.set(requestId, { resolve, reject, onProgress }); + activeWorker.postMessage( + { type: "separate", requestId, leftBuffer: left.buffer, rightBuffer: right.buffer, sampleRate }, + [left.buffer, right.buffer], + ); }); } -async function getRuntime() { - if (!runtimePromise) runtimePromise = (async () => { - await loadScript("/vendor/vocal-remover/tf.min.js?v=4.22.0"); - if (navigator.gpu) await loadScript("/vendor/vocal-remover/tf-backend-webgpu.js?v=4.22.0"); - return window.tf; - })(); - return runtimePromise; -} - -async function getModel(tf) { - if (!modelPromise) { - modelPromise = (async () => { - const requestedBackend = navigator.gpu && tf.findBackend("webgpu") ? "webgpu" : "webgl"; - let ready = false; - try { ready = await tf.setBackend(requestedBackend); } catch { ready = false; } - if (!ready && requestedBackend === "webgpu") ready = await tf.setBackend("webgl"); - if (!ready) throw new Error("VOCAL_BACKEND_UNAVAILABLE"); - activeBackend = tf.getBackend(); - await tf.ready(); - return tf.loadGraphModel(MODEL_URL); - })(); - } - return modelPromise; -} - -function stft(tf, input) { - return tf.tidy(() => { - const spectrum = tf.signal.stft(input, FFT_SIZE, HOP_SIZE, FFT_SIZE, (length) => tf.signal.hannWindow(length)); - const real = tf.real(spectrum).slice([0, 0], [32, 3072]).transpose(); - const imag = tf.imag(spectrum).slice([0, 0], [32, 3072]).transpose(); - const output = tf.stack([real, imag], 0); - return tf.where(tf.isNaN(output), tf.zerosLike(output), output); - }); -} - -async function inverseChannel(tf, spectrogram) { - const frames = spectrogram.shape[1]; - const output = new Float32Array(FFT_SIZE + HOP_SIZE * (frames - 1)); - const window = await tf.signal.hannWindow(FFT_SIZE).data(); - for (let index = 0; index < frames; index += 1) { - const frame = tf.tidy(() => { - const values = spectrogram.slice([0, index, 0], [-1, 1, 2]).squeeze([1]); - return tf.spectral.irfft(tf.complex(values.slice([0, 0], [-1, 1]).squeeze(), values.slice([0, 1], [-1, 1]).squeeze())); - }); - const samples = await frame.data(); - frame.dispose(); - const offset = index * HOP_SIZE; - for (let cursor = 0; cursor < samples.length; cursor += 1) output[offset + cursor] += samples[cursor] * window[cursor]; - } - return output; -} - -async function istft(tf, tensor) { - const padded = tf.pad(tensor, [[0, 0], [0, 0], [0, 1], [0, 0]]); - const shaped = padded.reshape([2, 2, 3073, 32]).transpose([0, 2, 3, 1]); - const left = shaped.slice([0, 0, 0, 0], [1, -1, -1, -1]).squeeze([0]); - const right = shaped.slice([1, 0, 0, 0], [1, -1, -1, -1]).squeeze([0]); - const result = await Promise.all([inverseChannel(tf, left), inverseChannel(tf, right)]); - tf.dispose([padded, shaped, left, right]); - return result; -} - -function wavBlob(channels, sampleRate) { - const frames = channels[0].length; - const buffer = new ArrayBuffer(44 + frames * 4); - const view = new DataView(buffer); - const text = (offset, value) => [...value].forEach((char, index) => view.setUint8(offset + index, char.charCodeAt(0))); - text(0, "RIFF"); view.setUint32(4, buffer.byteLength - 8, true); text(8, "WAVE"); text(12, "fmt "); - view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, 2, true); - view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * 4, true); view.setUint16(32, 4, true); view.setUint16(34, 16, true); - text(36, "data"); view.setUint32(40, frames * 4, true); - let offset = 44; - for (let frame = 0; frame < frames; frame += 1) for (let channel = 0; channel < 2; channel += 1) { - const sample = Math.max(-1, Math.min(1, channels[channel][frame] || 0)); - view.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); offset += 2; - } - return new Blob([buffer], { type: "audio/wav" }); -} - export async function separateVocals(blob, onProgress = () => {}) { - const tf = await getRuntime(); - onProgress(4, { key: "vocalSeparationLoadingModel" }); - const model = await getModel(tf); - onProgress(7, { - key: activeBackend === "webgpu" ? "vocalSeparationUsingWebGpu" : "vocalSeparationUsingWebGl", - }); const context = new AudioContext({ sampleRate: 44100 }); try { const decoded = await context.decodeAudioData((await blob.arrayBuffer()).slice(0)); - const left = decoded.getChannelData(0); - const right = decoded.numberOfChannels > 1 ? decoded.getChannelData(1) : left; - const accompaniment = [[], []]; - const vocals = [[], []]; - const chunks = Math.ceil(left.length / CHUNK_SIZE); - for (let index = 0; index < chunks; index += 1) { - const start = index * CHUNK_SIZE; - const valid = Math.min(CHUNK_SIZE, left.length - start); - const paddedLeft = new Float32Array(CHUNK_SIZE + PADDING * 2); paddedLeft.set(left.subarray(start, start + valid), PADDING); - const paddedRight = new Float32Array(CHUNK_SIZE + PADDING * 2); paddedRight.set(right.subarray(start, start + valid), PADDING); - const input = tf.tidy(() => { - const l = stft(tf, tf.tensor1d(paddedLeft)); const r = stft(tf, tf.tensor1d(paddedRight)); - return tf.stack([l, r], 3).transpose([0, 3, 1, 2]).reshape([1, 4, 3072, 32]); - }); - const musicTensor = model.predict(input); - const vocalTensor = tf.sub(input, musicTensor); - const [music, voice] = await Promise.all([istft(tf, musicTensor), istft(tf, vocalTensor)]); - for (let channel = 0; channel < 2; channel += 1) { - accompaniment[channel].push(...music[channel].slice(PADDING, PADDING + valid)); - vocals[channel].push(...voice[channel].slice(PADDING, PADDING + valid)); - } - tf.dispose([input, musicTensor, vocalTensor]); - onProgress(8 + Math.round(((index + 1) / chunks) * 88), { - key: "vocalSeparationProcessingChunk", - current: index + 1, - total: chunks, - }); - await tf.nextFrame(); - } - return { - vocals: wavBlob(vocals, decoded.sampleRate), - accompaniment: wavBlob(accompaniment, decoded.sampleRate), - backend: activeBackend, - }; - } finally { await context.close().catch(() => {}); } + const left = decoded.getChannelData(0).slice(); + const right = decoded.numberOfChannels > 1 ? decoded.getChannelData(1).slice() : left.slice(); + return await runWorker(left, right, decoded.sampleRate, onProgress); + } finally { + await context.close().catch(() => {}); + } +} + +export function disposeVocalSeparationWorker() { + resetWorker(new Error("VOCAL_WORKER_DISPOSED")); } diff --git a/src/workers/vocal-separation.worker.js b/src/workers/vocal-separation.worker.js new file mode 100644 index 0000000..4ef3266 --- /dev/null +++ b/src/workers/vocal-separation.worker.js @@ -0,0 +1,161 @@ +const MODEL_URL = "https://huggingface.co/haixin/timeline-studio-vocal-remover/resolve/main/model.json"; +const CHUNK_SIZE = 31744; +const PADDING = 3072; +const FFT_SIZE = 6144; +const HOP_SIZE = 1024; + +let modelPromise; +let runtimePromise; +let activeBackend = "webgl"; + +function postProgress(requestId, progress, phase) { + self.postMessage({ type: "progress", requestId, progress, phase }); +} + +async function getRuntime() { + runtimePromise ??= (async () => { + try { + importScripts("/vendor/vocal-remover/tf.min.js?v=4.22.0"); + if (self.navigator.gpu) importScripts("/vendor/vocal-remover/tf-backend-webgpu.js?v=4.22.0"); + } catch { + throw new Error("VOCAL_RUNTIME_DOWNLOAD_FAILED"); + } + return self.tf; + })(); + return runtimePromise; +} + +async function getModel(runtime) { + modelPromise ??= (async () => { + const requestedBackend = self.navigator.gpu && runtime.findBackend("webgpu") ? "webgpu" : "webgl"; + let ready; + try { ready = await runtime.setBackend(requestedBackend); } catch { ready = false; } + if (!ready && requestedBackend === "webgpu") ready = await runtime.setBackend("webgl"); + if (!ready) throw new Error("VOCAL_BACKEND_UNAVAILABLE"); + activeBackend = runtime.getBackend(); + await runtime.ready(); + return runtime.loadGraphModel(MODEL_URL); + })(); + return modelPromise; +} + +function stft(runtime, input) { + return runtime.tidy(() => { + const spectrum = runtime.signal.stft(input, FFT_SIZE, HOP_SIZE, FFT_SIZE, (length) => runtime.signal.hannWindow(length)); + const real = runtime.real(spectrum).slice([0, 0], [32, 3072]).transpose(); + const imag = runtime.imag(spectrum).slice([0, 0], [32, 3072]).transpose(); + const output = runtime.stack([real, imag], 0); + return runtime.where(runtime.isNaN(output), runtime.zerosLike(output), output); + }); +} + +function createHannWindow() { + const window = new Float32Array(FFT_SIZE); + for (let index = 0; index < FFT_SIZE; index += 1) { + window[index] = 0.5 - 0.5 * Math.cos((2 * Math.PI * index) / FFT_SIZE); + } + return window; +} + +const hannWindow = createHannWindow(); + +async function inverseChannel(runtime, spectrogram) { + const frames = spectrogram.shape[1]; + const timeFrames = runtime.tidy(() => { + const frameMajor = spectrogram.transpose([1, 0, 2]); + const real = frameMajor.slice([0, 0, 0], [-1, -1, 1]).squeeze([2]); + const imag = frameMajor.slice([0, 0, 1], [-1, -1, 1]).squeeze([2]); + return runtime.spectral.irfft(runtime.complex(real, imag)); + }); + const samples = await timeFrames.data(); + timeFrames.dispose(); + + const output = new Float32Array(FFT_SIZE + HOP_SIZE * (frames - 1)); + for (let frame = 0; frame < frames; frame += 1) { + const sourceOffset = frame * FFT_SIZE; + const outputOffset = frame * HOP_SIZE; + for (let cursor = 0; cursor < FFT_SIZE; cursor += 1) { + output[outputOffset + cursor] += samples[sourceOffset + cursor] * hannWindow[cursor]; + } + } + return output; +} + +async function istft(runtime, tensor) { + const padded = runtime.pad(tensor, [[0, 0], [0, 0], [0, 1], [0, 0]]); + const shaped = padded.reshape([2, 2, 3073, 32]).transpose([0, 2, 3, 1]); + const left = shaped.slice([0, 0, 0, 0], [1, -1, -1, -1]).squeeze([0]); + const right = shaped.slice([1, 0, 0, 0], [1, -1, -1, -1]).squeeze([0]); + const result = await Promise.all([inverseChannel(runtime, left), inverseChannel(runtime, right)]); + runtime.dispose([padded, shaped, left, right]); + return result; +} + +function wavBuffer(channels, sampleRate) { + const frames = channels[0].length; + const buffer = new ArrayBuffer(44 + frames * 4); + const view = new DataView(buffer); + const text = (offset, value) => [...value].forEach((char, index) => view.setUint8(offset + index, char.charCodeAt(0))); + text(0, "RIFF"); view.setUint32(4, buffer.byteLength - 8, true); text(8, "WAVE"); text(12, "fmt "); + view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, 2, true); + view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * 4, true); view.setUint16(32, 4, true); view.setUint16(34, 16, true); + text(36, "data"); view.setUint32(40, frames * 4, true); + let offset = 44; + for (let frame = 0; frame < frames; frame += 1) for (let channel = 0; channel < 2; channel += 1) { + const sample = Math.max(-1, Math.min(1, channels[channel][frame] || 0)); + view.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); offset += 2; + } + return buffer; +} + +async function separate(requestId, left, right, sampleRate) { + const runtime = await getRuntime(); + postProgress(requestId, 4, { key: "vocalSeparationLoadingModel" }); + const model = await getModel(runtime); + postProgress(requestId, 7, { key: activeBackend === "webgpu" ? "vocalSeparationUsingWebGpu" : "vocalSeparationUsingWebGl" }); + const accompaniment = [[], []]; + const vocals = [[], []]; + const chunks = Math.ceil(left.length / CHUNK_SIZE); + for (let index = 0; index < chunks; index += 1) { + const start = index * CHUNK_SIZE; + const valid = Math.min(CHUNK_SIZE, left.length - start); + const paddedLeft = new Float32Array(CHUNK_SIZE + PADDING * 2); paddedLeft.set(left.subarray(start, start + valid), PADDING); + const paddedRight = new Float32Array(CHUNK_SIZE + PADDING * 2); paddedRight.set(right.subarray(start, start + valid), PADDING); + const input = runtime.tidy(() => { + const l = stft(runtime, runtime.tensor1d(paddedLeft)); + const r = stft(runtime, runtime.tensor1d(paddedRight)); + return runtime.stack([l, r], 3).transpose([0, 3, 1, 2]).reshape([1, 4, 3072, 32]); + }); + const musicTensor = model.predict(input); + const vocalTensor = runtime.sub(input, musicTensor); + const [music, voice] = await Promise.all([istft(runtime, musicTensor), istft(runtime, vocalTensor)]); + for (let channel = 0; channel < 2; channel += 1) { + accompaniment[channel].push(...music[channel].slice(PADDING, PADDING + valid)); + vocals[channel].push(...voice[channel].slice(PADDING, PADDING + valid)); + } + runtime.dispose([input, musicTensor, vocalTensor]); + postProgress(requestId, 8 + Math.round(((index + 1) / chunks) * 88), { + key: "vocalSeparationProcessingChunk", current: index + 1, total: chunks, + }); + } + const vocalsBuffer = wavBuffer(vocals, sampleRate); + const accompanimentBuffer = wavBuffer(accompaniment, sampleRate); + self.postMessage( + { type: "result", requestId, vocalsBuffer, accompanimentBuffer, backend: activeBackend }, + [vocalsBuffer, accompanimentBuffer], + ); +} + +self.addEventListener("message", async ({ data }) => { + if (data?.type !== "separate") return; + try { + await separate( + data.requestId, + new Float32Array(data.leftBuffer), + new Float32Array(data.rightBuffer), + data.sampleRate, + ); + } catch (error) { + self.postMessage({ type: "error", requestId: data.requestId, error: error?.message || "VOCAL_MODEL_FAILED" }); + } +}); diff --git a/vite.config.mjs b/vite.config.mjs index 5eae3ce..e5efc64 100644 --- a/vite.config.mjs +++ b/vite.config.mjs @@ -14,4 +14,10 @@ export default defineConfig({ }, }, plugins: [react()], + test: { + coverage: { + include: ["src/**/*.{js,jsx,ts,tsx}"], + exclude: ["src/**/*.test.*", "src/**/__fixtures__/**"], + }, + }, });