Merge pull request #1 from MartinDelophy/codex/vocal-worker-discord

Move vocal separation to a worker and add Discord community links
This commit is contained in:
MartinDelophy
2026-07-13 17:08:15 +08:00
committed by GitHub
parent fe799d322f
commit 3586ec8aab
6 changed files with 239 additions and 133 deletions
+5
View File
@@ -6,6 +6,7 @@ Language: **English** | [中文](README.zh-CN.md)
[![GitHub stars](https://img.shields.io/github/stars/MartinDelophy/ai-video-editor?style=flat-square)](https://github.com/MartinDelophy/ai-video-editor/stargazers)
[![GitHub forks](https://img.shields.io/github/forks/MartinDelophy/ai-video-editor?style=flat-square)](https://github.com/MartinDelophy/ai-video-editor/forks)
[![MIT License](https://img.shields.io/github/license/MartinDelophy/ai-video-editor?style=flat-square)](LICENSE)
[![Discord](https://img.shields.io/discord/1526147673290313768?label=Discord&logo=discord&style=flat-square)](https://discord.gg/uq2uvUTBr)
A local-first, browser-based AI video editor with ONNX voiceover generation, Whisper automatic captions, JoyVASA + LivePortrait talking avatars, multi-track timeline editing, and MP4/WebM export. Timeline Studio runs its core AI workflows directly in the browser and is inspired by modern editors such as CapCut.
@@ -218,6 +219,10 @@ Released under the [MIT License](LICENSE). You may use, modify, and distribute t
## Community
Join the Discord community for questions, feedback, and project updates:
[Join the Timeline Studio Discord](https://discord.gg/uq2uvUTBr)
The current star count is updated live in the GitHub Stars badge at the top of this README.
[View current stargazers](https://github.com/MartinDelophy/ai-video-editor/stargazers) · [Star Timeline Studio](https://github.com/MartinDelophy/ai-video-editor)
+5
View File
@@ -6,6 +6,7 @@
[![GitHub Stars](https://img.shields.io/github/stars/MartinDelophy/ai-video-editor?style=flat-square)](https://github.com/MartinDelophy/ai-video-editor/stargazers)
[![GitHub Forks](https://img.shields.io/github/forks/MartinDelophy/ai-video-editor?style=flat-square)](https://github.com/MartinDelophy/ai-video-editor/forks)
[![MIT License](https://img.shields.io/github/license/MartinDelophy/ai-video-editor?style=flat-square)](LICENSE)
[![Discord](https://img.shields.io/discord/1526147673290313768?label=Discord&logo=discord&style=flat-square)](https://discord.gg/uq2uvUTBr)
一个本地优先的浏览器 AI 视频编辑器,支持 ONNX AI 配音、Whisper 自动字幕、JoyVASA + LivePortrait 数字人、多轨时间线以及 MP4/WebM 导出。核心 AI 流程直接在浏览器运行,交互参考 CapCut/剪映等现代时间线编辑器。
@@ -203,6 +204,10 @@ npx netlify-cli deploy --prod --dir=dist
## 社区
欢迎加入 Discord 社区,交流使用问题、产品建议和项目进展:
[加入 Timeline Studio Discord 社区](https://discord.gg/uq2uvUTBr)
README 顶部的 GitHub Stars 徽章会实时显示当前 Star 数量。
[查看当前 Stargazers](https://github.com/MartinDelophy/ai-video-editor/stargazers) · [为 Timeline Studio 点 Star](https://github.com/MartinDelophy/ai-video-editor)
+2
View File
@@ -3,6 +3,7 @@ import { useEffect } from "react";
import { RATIO_OPTIONS } from "../config/editor.js";
import { decodeWaveform } from "../lib/media.js";
import { disposeVisionWorker } from "../lib/vision.js";
import { disposeVocalSeparationWorker } from "../lib/vocalSeparation.js";
import {
getNearestRatioIdForSize,
revokeVisionObjectUrls,
@@ -158,6 +159,7 @@ export function useEditorLifecycle(d) {
d.visionObjectUrlsRef.current.forEach((urls) => revokeVisionObjectUrls(urls));
d.visionObjectUrlsRef.current.clear();
disposeVisionWorker();
disposeVocalSeparationWorker();
d.voiceRecorderStreamRef.current?.getTracks().forEach((track) => track.stop());
window.clearInterval(d.voiceRecorderTimerRef.current);
}, []);
+60 -133
View File
@@ -1,144 +1,71 @@
const MODEL_URL = "https://huggingface.co/haixin/timeline-studio-vocal-remover/resolve/main/model.json";
const CHUNK_SIZE = 31744;
const PADDING = 3072;
const FFT_SIZE = 6144;
const HOP_SIZE = 1024;
let worker;
let nextRequestId = 0;
const requests = new Map();
let modelPromise;
let runtimePromise;
let activeBackend = "webgl";
function rejectRequests(error) {
requests.forEach(({ reject }) => reject(error));
requests.clear();
}
function loadScript(src) {
function resetWorker(error) {
worker?.terminate();
worker = null;
if (error) rejectRequests(error);
}
function getWorker() {
if (typeof Worker === "undefined") throw new Error("VOCAL_WORKER_UNAVAILABLE");
if (worker) return worker;
worker = new Worker(new URL("../workers/vocal-separation.worker.js", import.meta.url));
worker.addEventListener("message", ({ data }) => {
const request = requests.get(data?.requestId);
if (!request) return;
if (data.type === "progress") {
request.onProgress(data.progress, data.phase);
return;
}
requests.delete(data.requestId);
if (data.type === "result") {
request.resolve({
vocals: new Blob([data.vocalsBuffer], { type: "audio/wav" }),
accompaniment: new Blob([data.accompanimentBuffer], { type: "audio/wav" }),
backend: data.backend,
});
return;
}
request.reject(new Error(data.error || "VOCAL_MODEL_FAILED"));
});
worker.addEventListener("error", (event) => {
resetWorker(new Error(event.message || "VOCAL_WORKER_FAILED"));
});
return worker;
}
function runWorker(left, right, sampleRate, onProgress) {
const requestId = `vocal-${++nextRequestId}`;
const activeWorker = getWorker();
return new Promise((resolve, reject) => {
const existing = document.querySelector(`script[src="${src}"]`);
if (existing) { if (window.tf) resolve(); else existing.addEventListener("load", resolve, { once: true }); return; }
const script = document.createElement("script"); script.src = src; script.crossOrigin = "anonymous";
script.onload = resolve; script.onerror = () => reject(new Error("VOCAL_RUNTIME_DOWNLOAD_FAILED")); document.head.appendChild(script);
requests.set(requestId, { resolve, reject, onProgress });
activeWorker.postMessage(
{ type: "separate", requestId, leftBuffer: left.buffer, rightBuffer: right.buffer, sampleRate },
[left.buffer, right.buffer],
);
});
}
async function getRuntime() {
if (!runtimePromise) runtimePromise = (async () => {
await loadScript("/vendor/vocal-remover/tf.min.js?v=4.22.0");
if (navigator.gpu) await loadScript("/vendor/vocal-remover/tf-backend-webgpu.js?v=4.22.0");
return window.tf;
})();
return runtimePromise;
}
async function getModel(tf) {
if (!modelPromise) {
modelPromise = (async () => {
const requestedBackend = navigator.gpu && tf.findBackend("webgpu") ? "webgpu" : "webgl";
let ready = false;
try { ready = await tf.setBackend(requestedBackend); } catch { ready = false; }
if (!ready && requestedBackend === "webgpu") ready = await tf.setBackend("webgl");
if (!ready) throw new Error("VOCAL_BACKEND_UNAVAILABLE");
activeBackend = tf.getBackend();
await tf.ready();
return tf.loadGraphModel(MODEL_URL);
})();
}
return modelPromise;
}
function stft(tf, input) {
return tf.tidy(() => {
const spectrum = tf.signal.stft(input, FFT_SIZE, HOP_SIZE, FFT_SIZE, (length) => tf.signal.hannWindow(length));
const real = tf.real(spectrum).slice([0, 0], [32, 3072]).transpose();
const imag = tf.imag(spectrum).slice([0, 0], [32, 3072]).transpose();
const output = tf.stack([real, imag], 0);
return tf.where(tf.isNaN(output), tf.zerosLike(output), output);
});
}
async function inverseChannel(tf, spectrogram) {
const frames = spectrogram.shape[1];
const output = new Float32Array(FFT_SIZE + HOP_SIZE * (frames - 1));
const window = await tf.signal.hannWindow(FFT_SIZE).data();
for (let index = 0; index < frames; index += 1) {
const frame = tf.tidy(() => {
const values = spectrogram.slice([0, index, 0], [-1, 1, 2]).squeeze([1]);
return tf.spectral.irfft(tf.complex(values.slice([0, 0], [-1, 1]).squeeze(), values.slice([0, 1], [-1, 1]).squeeze()));
});
const samples = await frame.data();
frame.dispose();
const offset = index * HOP_SIZE;
for (let cursor = 0; cursor < samples.length; cursor += 1) output[offset + cursor] += samples[cursor] * window[cursor];
}
return output;
}
async function istft(tf, tensor) {
const padded = tf.pad(tensor, [[0, 0], [0, 0], [0, 1], [0, 0]]);
const shaped = padded.reshape([2, 2, 3073, 32]).transpose([0, 2, 3, 1]);
const left = shaped.slice([0, 0, 0, 0], [1, -1, -1, -1]).squeeze([0]);
const right = shaped.slice([1, 0, 0, 0], [1, -1, -1, -1]).squeeze([0]);
const result = await Promise.all([inverseChannel(tf, left), inverseChannel(tf, right)]);
tf.dispose([padded, shaped, left, right]);
return result;
}
function wavBlob(channels, sampleRate) {
const frames = channels[0].length;
const buffer = new ArrayBuffer(44 + frames * 4);
const view = new DataView(buffer);
const text = (offset, value) => [...value].forEach((char, index) => view.setUint8(offset + index, char.charCodeAt(0)));
text(0, "RIFF"); view.setUint32(4, buffer.byteLength - 8, true); text(8, "WAVE"); text(12, "fmt ");
view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, 2, true);
view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * 4, true); view.setUint16(32, 4, true); view.setUint16(34, 16, true);
text(36, "data"); view.setUint32(40, frames * 4, true);
let offset = 44;
for (let frame = 0; frame < frames; frame += 1) for (let channel = 0; channel < 2; channel += 1) {
const sample = Math.max(-1, Math.min(1, channels[channel][frame] || 0));
view.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); offset += 2;
}
return new Blob([buffer], { type: "audio/wav" });
}
export async function separateVocals(blob, onProgress = () => {}) {
const tf = await getRuntime();
onProgress(4, { key: "vocalSeparationLoadingModel" });
const model = await getModel(tf);
onProgress(7, {
key: activeBackend === "webgpu" ? "vocalSeparationUsingWebGpu" : "vocalSeparationUsingWebGl",
});
const context = new AudioContext({ sampleRate: 44100 });
try {
const decoded = await context.decodeAudioData((await blob.arrayBuffer()).slice(0));
const left = decoded.getChannelData(0);
const right = decoded.numberOfChannels > 1 ? decoded.getChannelData(1) : left;
const accompaniment = [[], []];
const vocals = [[], []];
const chunks = Math.ceil(left.length / CHUNK_SIZE);
for (let index = 0; index < chunks; index += 1) {
const start = index * CHUNK_SIZE;
const valid = Math.min(CHUNK_SIZE, left.length - start);
const paddedLeft = new Float32Array(CHUNK_SIZE + PADDING * 2); paddedLeft.set(left.subarray(start, start + valid), PADDING);
const paddedRight = new Float32Array(CHUNK_SIZE + PADDING * 2); paddedRight.set(right.subarray(start, start + valid), PADDING);
const input = tf.tidy(() => {
const l = stft(tf, tf.tensor1d(paddedLeft)); const r = stft(tf, tf.tensor1d(paddedRight));
return tf.stack([l, r], 3).transpose([0, 3, 1, 2]).reshape([1, 4, 3072, 32]);
});
const musicTensor = model.predict(input);
const vocalTensor = tf.sub(input, musicTensor);
const [music, voice] = await Promise.all([istft(tf, musicTensor), istft(tf, vocalTensor)]);
for (let channel = 0; channel < 2; channel += 1) {
accompaniment[channel].push(...music[channel].slice(PADDING, PADDING + valid));
vocals[channel].push(...voice[channel].slice(PADDING, PADDING + valid));
}
tf.dispose([input, musicTensor, vocalTensor]);
onProgress(8 + Math.round(((index + 1) / chunks) * 88), {
key: "vocalSeparationProcessingChunk",
current: index + 1,
total: chunks,
});
await tf.nextFrame();
}
return {
vocals: wavBlob(vocals, decoded.sampleRate),
accompaniment: wavBlob(accompaniment, decoded.sampleRate),
backend: activeBackend,
};
} finally { await context.close().catch(() => {}); }
const left = decoded.getChannelData(0).slice();
const right = decoded.numberOfChannels > 1 ? decoded.getChannelData(1).slice() : left.slice();
return await runWorker(left, right, decoded.sampleRate, onProgress);
} finally {
await context.close().catch(() => {});
}
}
export function disposeVocalSeparationWorker() {
resetWorker(new Error("VOCAL_WORKER_DISPOSED"));
}
+161
View File
@@ -0,0 +1,161 @@
const MODEL_URL = "https://huggingface.co/haixin/timeline-studio-vocal-remover/resolve/main/model.json";
const CHUNK_SIZE = 31744;
const PADDING = 3072;
const FFT_SIZE = 6144;
const HOP_SIZE = 1024;
let modelPromise;
let runtimePromise;
let activeBackend = "webgl";
function postProgress(requestId, progress, phase) {
self.postMessage({ type: "progress", requestId, progress, phase });
}
async function getRuntime() {
runtimePromise ??= (async () => {
try {
importScripts("/vendor/vocal-remover/tf.min.js?v=4.22.0");
if (self.navigator.gpu) importScripts("/vendor/vocal-remover/tf-backend-webgpu.js?v=4.22.0");
} catch {
throw new Error("VOCAL_RUNTIME_DOWNLOAD_FAILED");
}
return self.tf;
})();
return runtimePromise;
}
async function getModel(runtime) {
modelPromise ??= (async () => {
const requestedBackend = self.navigator.gpu && runtime.findBackend("webgpu") ? "webgpu" : "webgl";
let ready;
try { ready = await runtime.setBackend(requestedBackend); } catch { ready = false; }
if (!ready && requestedBackend === "webgpu") ready = await runtime.setBackend("webgl");
if (!ready) throw new Error("VOCAL_BACKEND_UNAVAILABLE");
activeBackend = runtime.getBackend();
await runtime.ready();
return runtime.loadGraphModel(MODEL_URL);
})();
return modelPromise;
}
function stft(runtime, input) {
return runtime.tidy(() => {
const spectrum = runtime.signal.stft(input, FFT_SIZE, HOP_SIZE, FFT_SIZE, (length) => runtime.signal.hannWindow(length));
const real = runtime.real(spectrum).slice([0, 0], [32, 3072]).transpose();
const imag = runtime.imag(spectrum).slice([0, 0], [32, 3072]).transpose();
const output = runtime.stack([real, imag], 0);
return runtime.where(runtime.isNaN(output), runtime.zerosLike(output), output);
});
}
function createHannWindow() {
const window = new Float32Array(FFT_SIZE);
for (let index = 0; index < FFT_SIZE; index += 1) {
window[index] = 0.5 - 0.5 * Math.cos((2 * Math.PI * index) / FFT_SIZE);
}
return window;
}
const hannWindow = createHannWindow();
async function inverseChannel(runtime, spectrogram) {
const frames = spectrogram.shape[1];
const timeFrames = runtime.tidy(() => {
const frameMajor = spectrogram.transpose([1, 0, 2]);
const real = frameMajor.slice([0, 0, 0], [-1, -1, 1]).squeeze([2]);
const imag = frameMajor.slice([0, 0, 1], [-1, -1, 1]).squeeze([2]);
return runtime.spectral.irfft(runtime.complex(real, imag));
});
const samples = await timeFrames.data();
timeFrames.dispose();
const output = new Float32Array(FFT_SIZE + HOP_SIZE * (frames - 1));
for (let frame = 0; frame < frames; frame += 1) {
const sourceOffset = frame * FFT_SIZE;
const outputOffset = frame * HOP_SIZE;
for (let cursor = 0; cursor < FFT_SIZE; cursor += 1) {
output[outputOffset + cursor] += samples[sourceOffset + cursor] * hannWindow[cursor];
}
}
return output;
}
async function istft(runtime, tensor) {
const padded = runtime.pad(tensor, [[0, 0], [0, 0], [0, 1], [0, 0]]);
const shaped = padded.reshape([2, 2, 3073, 32]).transpose([0, 2, 3, 1]);
const left = shaped.slice([0, 0, 0, 0], [1, -1, -1, -1]).squeeze([0]);
const right = shaped.slice([1, 0, 0, 0], [1, -1, -1, -1]).squeeze([0]);
const result = await Promise.all([inverseChannel(runtime, left), inverseChannel(runtime, right)]);
runtime.dispose([padded, shaped, left, right]);
return result;
}
function wavBuffer(channels, sampleRate) {
const frames = channels[0].length;
const buffer = new ArrayBuffer(44 + frames * 4);
const view = new DataView(buffer);
const text = (offset, value) => [...value].forEach((char, index) => view.setUint8(offset + index, char.charCodeAt(0)));
text(0, "RIFF"); view.setUint32(4, buffer.byteLength - 8, true); text(8, "WAVE"); text(12, "fmt ");
view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, 2, true);
view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * 4, true); view.setUint16(32, 4, true); view.setUint16(34, 16, true);
text(36, "data"); view.setUint32(40, frames * 4, true);
let offset = 44;
for (let frame = 0; frame < frames; frame += 1) for (let channel = 0; channel < 2; channel += 1) {
const sample = Math.max(-1, Math.min(1, channels[channel][frame] || 0));
view.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); offset += 2;
}
return buffer;
}
async function separate(requestId, left, right, sampleRate) {
const runtime = await getRuntime();
postProgress(requestId, 4, { key: "vocalSeparationLoadingModel" });
const model = await getModel(runtime);
postProgress(requestId, 7, { key: activeBackend === "webgpu" ? "vocalSeparationUsingWebGpu" : "vocalSeparationUsingWebGl" });
const accompaniment = [[], []];
const vocals = [[], []];
const chunks = Math.ceil(left.length / CHUNK_SIZE);
for (let index = 0; index < chunks; index += 1) {
const start = index * CHUNK_SIZE;
const valid = Math.min(CHUNK_SIZE, left.length - start);
const paddedLeft = new Float32Array(CHUNK_SIZE + PADDING * 2); paddedLeft.set(left.subarray(start, start + valid), PADDING);
const paddedRight = new Float32Array(CHUNK_SIZE + PADDING * 2); paddedRight.set(right.subarray(start, start + valid), PADDING);
const input = runtime.tidy(() => {
const l = stft(runtime, runtime.tensor1d(paddedLeft));
const r = stft(runtime, runtime.tensor1d(paddedRight));
return runtime.stack([l, r], 3).transpose([0, 3, 1, 2]).reshape([1, 4, 3072, 32]);
});
const musicTensor = model.predict(input);
const vocalTensor = runtime.sub(input, musicTensor);
const [music, voice] = await Promise.all([istft(runtime, musicTensor), istft(runtime, vocalTensor)]);
for (let channel = 0; channel < 2; channel += 1) {
accompaniment[channel].push(...music[channel].slice(PADDING, PADDING + valid));
vocals[channel].push(...voice[channel].slice(PADDING, PADDING + valid));
}
runtime.dispose([input, musicTensor, vocalTensor]);
postProgress(requestId, 8 + Math.round(((index + 1) / chunks) * 88), {
key: "vocalSeparationProcessingChunk", current: index + 1, total: chunks,
});
}
const vocalsBuffer = wavBuffer(vocals, sampleRate);
const accompanimentBuffer = wavBuffer(accompaniment, sampleRate);
self.postMessage(
{ type: "result", requestId, vocalsBuffer, accompanimentBuffer, backend: activeBackend },
[vocalsBuffer, accompanimentBuffer],
);
}
self.addEventListener("message", async ({ data }) => {
if (data?.type !== "separate") return;
try {
await separate(
data.requestId,
new Float32Array(data.leftBuffer),
new Float32Array(data.rightBuffer),
data.sampleRate,
);
} catch (error) {
self.postMessage({ type: "error", requestId: data.requestId, error: error?.message || "VOCAL_MODEL_FAILED" });
}
});
+6
View File
@@ -14,4 +14,10 @@ export default defineConfig({
},
},
plugins: [react()],
test: {
coverage: {
include: ["src/**/*.{js,jsx,ts,tsx}"],
exclude: ["src/**/*.test.*", "src/**/__fixtures__/**"],
},
},
});