Release v0.3.3 (#18)

* Add auto edit and polish timeline interactions

* Release v0.3.3

---------

Co-authored-by: haixin.yang <haixin.yang@weimob.com>
This commit is contained in:
Martin Delophy
2026-07-17 17:52:43 +08:00
committed by GitHub
co-authored by haixin.yang
parent 0be8627f95
commit 86aef694ea
37 changed files with 1532 additions and 231 deletions
+72
View File
@@ -0,0 +1,72 @@
import { expect, test } from "@playwright/test";
const videos = [process.env.AUTO_EDIT_VIDEO_A, process.env.AUTO_EDIT_VIDEO_B].filter(Boolean);
test.describe("Auto Edit desktop video smoke", () => {
test.skip(videos.length !== 2, "Set AUTO_EDIT_VIDEO_A and AUTO_EDIT_VIDEO_B to run the local smoke test");
for (const [index, videoPath] of videos.entries()) {
test(`imports and samples desktop video ${index + 1}`, async ({ page }) => {
if (process.env.AUTO_EDIT_FULL && index === 0) test.setTimeout(12 * 60 * 1000);
if (process.env.AUTO_EDIT_MOCK_MODEL) await page.addInitScript(() => {
window.LanguageModel = {
availability: async () => "available",
create: async () => ({
prompt: async (messages) => {
const prompt = messages?.[0]?.content?.find?.((item) => item.type === "text")?.value || "";
const firstTime = Number(prompt.match(/timestamps?:\s*([\d.]+)/i)?.[1] || prompt.match(/at\s+([\d.]+)\s+seconds/i)?.[1] || 0);
return JSON.stringify({ captions: [{ start: firstTime, end: firstTime + 2.8, text: `Visual description for clip at ${firstTime.toFixed(1)} seconds.` }] });
},
destroy() {},
}),
};
});
await page.goto("/");
const languageIntro = page.locator(".language-intro");
if (await languageIntro.isVisible()) {
await languageIntro.locator(".language-grid button").filter({ hasText: "简体中文" }).click();
await expect(languageIntro).toBeHidden();
}
const mediaInput = page.locator('input[type="file"][accept*="video/mp4"]');
if (process.env.AUTO_EDIT_MULTI) {
await mediaInput.setInputFiles(videos[0]);
await expect(page.locator(".asset-row-button")).toHaveCount(1);
await mediaInput.setInputFiles(videos[1]);
await expect(page.locator(".asset-row-button")).toHaveCount(2);
} else {
await mediaInput.setInputFiles(videoPath);
}
const preview = page.locator(".preview-video");
await expect(preview).toBeVisible();
await expect.poll(async () => Number(await preview.evaluate((video) => video.duration))).toBeGreaterThan(0);
const result = await preview.evaluate(async (video) => {
const { extractAutoEditFrames, probeBuiltInAI } = await import("/src/lib/autoEdit.js");
const duration = video.duration;
const frames = await extractAutoEditFrames([{ id: "desktop-smoke", type: "video", src: video.src, duration, sourceDuration: duration }]);
return { duration, frameCount: frames.length, times: frames.map((frame) => Number(frame.time.toFixed(2))), support: await probeBuiltInAI("zh") };
});
console.log(`AUTO_EDIT_RESULT_${index + 1}=${JSON.stringify(result)}`);
expect(result.frameCount).toBeGreaterThanOrEqual(2);
expect(result.times[0]).toBe(0);
expect(result.times.at(-1)).toBeCloseTo(result.duration, 1);
if (process.env.AUTO_EDIT_FULL && index === 0) {
await page.getByRole("button", { name: "智能", exact: true }).click();
await page.getByRole("button", { name: "检测浏览器支持", exact: true }).click();
await expect(page.locator(".auto-edit-availability")).toHaveText(/可用|模型待下载|下载中/);
await expect(page.locator(".toast")).toBeHidden({ timeout: 10_000 });
await page.locator(".auto-edit-generate").click();
await expect(page.getByRole("dialog", { name: "画面分析与字幕检查" })).toBeVisible();
await page.waitForTimeout(3_000);
console.log(`AUTO_EDIT_UI=${JSON.stringify(await page.locator(".auto-edit-panel").innerText())}`);
if (await page.locator(".toast").isVisible()) console.log(`AUTO_EDIT_TOAST=${JSON.stringify(await page.locator(".toast").innerText())}`);
await expect(page.getByText("字幕草稿已准备好", { exact: true })).toBeVisible({ timeout: 10 * 60 * 1000 });
if (process.env.AUTO_EDIT_MULTI) expect(await page.locator(".auto-edit-clip-result").count()).toBe(2);
if (process.env.AUTO_EDIT_MOCK_MODEL) await page.screenshot({ path: "test-results/auto-edit-review.png", fullPage: true });
await page.getByRole("button", { name: "应用到字幕轨", exact: true }).click();
await expect(page.getByText("画面字幕已写入时间轴", { exact: true })).toBeVisible();
console.log(`AUTO_EDIT_CAPTIONS=${await page.locator(".caption-segment").count()}`);
}
});
}
});
+131
View File
@@ -0,0 +1,131 @@
import { expect, test } from "@playwright/test";
test("offline export preserves stickers, captions, voice audio, dimensions and duration", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
const result = await page.evaluate(async () => {
const { exportOfflineVideo } = await import("/src/lib/offlineVideoExport.js");
const makeImage = (draw) => {
const canvas = document.createElement("canvas");
canvas.width = 320; canvas.height = 180;
draw(canvas.getContext("2d"), canvas);
return canvas.toDataURL("image/png");
};
const background = makeImage((context, canvas) => {
context.fillStyle = "#c20d18"; context.fillRect(0, 0, canvas.width, canvas.height);
});
const sticker = makeImage((context) => {
context.clearRect(0, 0, 320, 180);
context.fillStyle = "#00ffff"; context.fillRect(120, 50, 80, 80);
});
const sampleRate = 48_000;
const frames = sampleRate;
const wav = new ArrayBuffer(44 + frames * 2);
const view = new DataView(wav);
const text = (offset, value) => [...value].forEach((char, index) => view.setUint8(offset + index, char.charCodeAt(0)));
text(0, "RIFF"); view.setUint32(4, 36 + frames * 2, true); text(8, "WAVEfmt ");
view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, 1, true);
view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * 2, true);
view.setUint16(32, 2, true); view.setUint16(34, 16, true); text(36, "data"); view.setUint32(40, frames * 2, true);
for (let index = 0; index < frames; index += 1) view.setInt16(44 + index * 2, Math.sin(index / sampleRate * Math.PI * 2 * 440) * 8000, true);
const voice = new Blob([wav], { type: "audio/wav" });
const exported = await exportOfflineVideo({
imageSrc: background, visualType: "image",
visualSegments: [{ id: "visual", src: background, type: "image", duration: 1 }],
voiceAudioSegments: [{ id: "voice", blob: voice, start: 0, duration: 1, volume: 1 }],
sourceAudioBlob: null, sourceAudioSegments: [], musicBlob: null,
text: "", captionSegments: [{ id: "caption", text: "E2E CAPTION", start: 0, end: 1 }],
duration: 1, ratio: { width: 16, height: 9 }, fitMode: "cover", filter: "none",
captionsEnabled: true, captionPosition: "bottom", captionPlacement: "bottom", captionSize: 12,
captionStyle: { textColor: "#ffffff", backgroundColor: "#000000", backgroundOpacity: 0.8, fontWeight: 700 },
captionReferenceSize: { width: 320, height: 180 },
sticker: null,
stickerSegments: [{ id: "sticker", src: sticker, start: 0, duration: 1, x: 50, y: 50, scale: 2, opacity: 1 }],
exportSettings: { codec: "vp9", width: 320, height: 180, frameRate: 30, videoBitsPerSecond: 3_000_000 },
});
const audioContext = new AudioContext();
const decodedAudio = await audioContext.decodeAudioData((await exported.blob.arrayBuffer()).slice(0));
await audioContext.close();
const url = URL.createObjectURL(exported.blob);
const video = document.createElement("video");
video.muted = true; video.src = url;
await new Promise((resolve, reject) => { video.onloadedmetadata = resolve; video.onerror = reject; });
video.currentTime = 0.5;
await new Promise((resolve, reject) => { video.onseeked = resolve; video.onerror = reject; });
const sample = document.createElement("canvas"); sample.width = 320; sample.height = 180;
const context = sample.getContext("2d"); context.drawImage(video, 0, 0);
const center = [...context.getImageData(160, 90, 1, 1).data];
const lower = [...context.getImageData(160, 156, 1, 1).data];
const metadata = {
width: video.videoWidth, height: video.videoHeight,
duration: decodedAudio.duration, hasAudio: decodedAudio.length > 0,
mediaDuration: video.duration, center, lower, size: exported.blob.size,
diagnostics: exported.diagnostics,
};
URL.revokeObjectURL(url);
return metadata;
});
expect(result.width).toBe(320);
expect(result.height).toBe(180);
expect(result.duration).toBeGreaterThanOrEqual(0.95);
expect(result.mediaDuration).toBeGreaterThanOrEqual(0.95);
expect(result.hasAudio).toBe(true);
expect(result.size).toBeGreaterThan(10_000);
expect(result.center[1]).toBeGreaterThan(result.center[0]);
expect(result.center[2]).toBeGreaterThan(result.center[0]);
expect(result.lower[0] + result.lower[1] + result.lower[2]).toBeGreaterThan(40);
expect(result.diagnostics.frameCount).toBe(30);
});
test("offline export decodes video sequentially instead of seeking every output frame", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
const result = await page.evaluate(async () => {
const { exportOfflineVideo } = await import("/src/lib/offlineVideoExport.js");
const sourceCanvas = document.createElement("canvas"); sourceCanvas.width = 160; sourceCanvas.height = 90;
const sourceContext = sourceCanvas.getContext("2d");
sourceContext.fillStyle = "#000"; sourceContext.fillRect(0, 0, 160, 90);
sourceContext.fillStyle = "#fff"; sourceContext.beginPath(); sourceContext.arc(80, 45, 25, 0, Math.PI * 2); sourceContext.fill();
const sourceImage = sourceCanvas.toDataURL("image/png");
const sourceExport = await exportOfflineVideo({
imageSrc: sourceImage, visualType: "image", visualSegments: [{ id: "image", src: sourceImage, type: "image", duration: 0.6 }],
voiceAudioSegments: [], sourceAudioBlob: null, sourceAudioSegments: [], musicBlob: null, text: "", captionSegments: [],
duration: 0.6, ratio: { width: 16, height: 9 }, fitMode: "cover", filter: "none", captionsEnabled: false,
captionSize: 12, captionStyle: {}, captionReferenceSize: { width: 160, height: 90 }, sticker: null, stickerSegments: [],
exportSettings: { codec: "vp9", width: 160, height: 90, frameRate: 24, videoBitsPerSecond: 1_000_000 },
});
const sourceBlob = sourceExport.blob;
const sourceUrl = URL.createObjectURL(sourceBlob);
const exported = await exportOfflineVideo({
imageSrc: sourceUrl, visualType: "video",
visualSegments: [{ id: "video", src: sourceUrl, blob: sourceBlob, type: "video", duration: 0.6, sourceDuration: 0.6, sourceStart: 0, playbackRate: 1 }],
voiceAudioSegments: [], sourceAudioBlob: null, sourceAudioSegments: [], musicBlob: null,
text: "", captionSegments: [], duration: 0.6, ratio: { width: 16, height: 9 },
fitMode: "cover", filter: "none", captionsEnabled: false, captionSize: 12,
captionStyle: {}, captionReferenceSize: { width: 160, height: 90 }, sticker: null, stickerSegments: [],
exportSettings: { codec: "vp9", width: 160, height: 90, frameRate: 24, videoBitsPerSecond: 1_000_000 },
});
const exportedUrl = URL.createObjectURL(exported.blob);
const video = document.createElement("video"); video.muted = true; video.src = exportedUrl;
await new Promise((resolve, reject) => { video.onloadedmetadata = resolve; video.onerror = reject; });
video.currentTime = 0.3;
await new Promise((resolve, reject) => { video.onseeked = resolve; video.onerror = reject; });
const checkCanvas = document.createElement("canvas"); checkCanvas.width = 160; checkCanvas.height = 90;
const checkContext = checkCanvas.getContext("2d"); checkContext.drawImage(video, 0, 0);
const pixels = checkContext.getImageData(0, 0, 160, 90).data;
let minX = 160; let maxX = 0; let minY = 90; let maxY = 0;
for (let y = 0; y < 90; y += 1) for (let x = 0; x < 160; x += 1) {
const offset = (y * 160 + x) * 4;
if (pixels[offset] > 180 && pixels[offset + 1] > 180 && pixels[offset + 2] > 180) {
minX = Math.min(minX, x); maxX = Math.max(maxX, x); minY = Math.min(minY, y); maxY = Math.max(maxY, y);
}
}
URL.revokeObjectURL(sourceUrl); URL.revokeObjectURL(exportedUrl);
return { size: exported.blob.size, diagnostics: exported.diagnostics, shapeRatio: (maxX - minX + 1) / (maxY - minY + 1) };
});
expect(result.size).toBeGreaterThan(500);
expect(result.diagnostics.videoDecodeModes).toEqual(["sequential-webcodecs"]);
expect(result.diagnostics.frameCount).toBe(15);
expect(result.shapeRatio).toBeGreaterThan(0.9);
expect(result.shapeRatio).toBeLessThan(1.1);
});
+50 -2
View File
@@ -1,12 +1,12 @@
{
"name": "web-player",
"version": "0.3.2",
"version": "0.3.3",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "web-player",
"version": "0.3.2",
"version": "0.3.3",
"license": "MIT",
"dependencies": {
"@diffusionstudio/vits-web": "^1.0.3",
@@ -14,9 +14,11 @@
"@ffmpeg/ffmpeg": "^0.12.15",
"@ffmpeg/util": "^0.12.2",
"@huggingface/transformers": "^3.8.1",
"@mediabunny/aac-encoder": "^1.50.8",
"@phosphor-icons/react": "^2.1.10",
"fflate": "^0.8.3",
"kokoro-js": "^1.2.1",
"mediabunny": "^1.50.8",
"onnxruntime-web": "^1.27.0",
"pinyin-pro": "^3.27.0",
"react": "19.2.0",
@@ -1617,6 +1619,19 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@mediabunny/aac-encoder": {
"version": "1.50.8",
"resolved": "https://registry.npmmirror.com/@mediabunny/aac-encoder/-/aac-encoder-1.50.8.tgz",
"integrity": "sha512-A5Se/LZd6RmYq/h36lBMSEsHvsyW8d0toR7FrAwpsFYbK+DVQYf90KiBT1Aw/mzLXx8/ypIOORJnd1sVZqOvJQ==",
"license": "MPL-2.0",
"funding": {
"type": "individual",
"url": "https://github.com/sponsors/Vanilagy"
},
"peerDependencies": {
"mediabunny": "^1.0.0"
}
},
"node_modules/@phosphor-icons/react": {
"version": "2.1.10",
"resolved": "https://registry.npmmirror.com/@phosphor-icons/react/-/react-2.1.10.tgz",
@@ -2130,6 +2145,21 @@
"dev": true,
"license": "MIT"
},
"node_modules/@types/dom-mediacapture-transform": {
"version": "0.1.12",
"resolved": "https://registry.npmmirror.com/@types/dom-mediacapture-transform/-/dom-mediacapture-transform-0.1.12.tgz",
"integrity": "sha512-d7/QsLRwF864A5mgIM/YrfiglHoYn7zgCcAoJgW404r+2DwnNr7EBbLnCWpmOMgH8y0te73L1AV6H1bmauaWFw==",
"license": "MIT",
"dependencies": {
"@types/dom-webcodecs": "*"
}
},
"node_modules/@types/dom-webcodecs": {
"version": "0.1.13",
"resolved": "https://registry.npmmirror.com/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz",
"integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==",
"license": "MIT"
},
"node_modules/@types/esrecurse": {
"version": "4.3.1",
"resolved": "https://registry.npmmirror.com/@types/esrecurse/-/esrecurse-4.3.1.tgz",
@@ -3794,6 +3824,24 @@
"node": ">=10"
}
},
"node_modules/mediabunny": {
"version": "1.50.8",
"resolved": "https://registry.npmmirror.com/mediabunny/-/mediabunny-1.50.8.tgz",
"integrity": "sha512-LgykLyQzhdpo0V2yw3UXmOpj+b4JAGdpHBwsPE6kjSt8Za0d1VllD+FV7EGHBcdV4+oHUAo+yrqbVAWxNSDCPQ==",
"license": "MPL-2.0",
"workspaces": [
".",
"packages/*"
],
"dependencies": {
"@types/dom-mediacapture-transform": "^0.1.11",
"@types/dom-webcodecs": "0.1.13"
},
"funding": {
"type": "individual",
"url": "https://github.com/sponsors/Vanilagy"
}
},
"node_modules/minimatch": {
"version": "10.2.5",
"resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-10.2.5.tgz",
+3 -1
View File
@@ -1,6 +1,6 @@
{
"name": "web-player",
"version": "0.3.2",
"version": "0.3.3",
"private": true,
"description": "A local-first browser AI video editor for voiceovers, captions, talking avatars, and multi-track timeline export.",
"license": "MIT",
@@ -33,9 +33,11 @@
"@ffmpeg/ffmpeg": "^0.12.15",
"@ffmpeg/util": "^0.12.2",
"@huggingface/transformers": "^3.8.1",
"@mediabunny/aac-encoder": "^1.50.8",
"@phosphor-icons/react": "^2.1.10",
"fflate": "^0.8.3",
"kokoro-js": "^1.2.1",
"mediabunny": "^1.50.8",
"onnxruntime-web": "^1.27.0",
"pinyin-pro": "^3.27.0",
"react": "19.2.0",
+4
View File
@@ -8,6 +8,10 @@ export default defineConfig({
use: {
baseURL: "http://127.0.0.1:5173",
headless: true,
channel: process.env.PLAYWRIGHT_CHANNEL || undefined,
launchOptions: process.env.PLAYWRIGHT_BUILT_IN_AI ? {
ignoreDefaultArgs: ["--disable-background-networking", "--disable-component-update"],
} : undefined,
},
webServer: {
command: "npm run dev",
+8 -1
View File
@@ -66,7 +66,11 @@ async function cacheFirst(request) {
const response = await fetch(request);
if (response.ok || response.type === "opaque") {
cache.put(request, response.clone()).catch(() => {});
// Cache persistence is an optimization. A full browser quota must never
// block the live response consumed by an inference worker.
cache.put(request, response.clone()).catch((error) => {
if (error?.name !== "QuotaExceededError") console.warn("Model cache write failed.", error);
});
}
return response;
}
@@ -112,6 +116,9 @@ self.addEventListener("activate", (event) => {
)
.then(() => self.clients.claim()),
);
// Older Transformers.js builds created a second copy of Hugging Face model
// assets here. The service worker is now the sole cache owner.
event.waitUntil(caches.delete("transformers-cache").catch(() => false));
});
self.addEventListener("fetch", (event) => {
+5 -14
View File
@@ -58,19 +58,7 @@ import { getImageThumbnailCount, getVisualSegmentsTotal } from "./lib/timeline.j
import { removeVisualPropertyKeyframe, updateVisualSegmentPlaybackRate, upsertVisualKeyframe, upsertVisualPropertyKeyframe } from "./lib/visualEffects.js";
import { getLinkedSourceAudioEnd, getLinkedSourceAudioSegments } from "./lib/sourceAudioSync.js";
import { getTimelineInitialContentZoom } from "./lib/timelineScale.js";
function getExportDimensions(ratio, longEdge) {
const sourceLongEdge = Math.max(ratio.width, ratio.height);
const scale = longEdge / sourceLongEdge;
const even = (value) => Math.max(2, Math.round(value / 2) * 2);
return { width: even(ratio.width * scale), height: even(ratio.height * scale) };
}
function getExportBitrate(resolution, quality, frameRate) {
const base = { 720: 5, 1080: 10, 1440: 18, 2160: 38 }[resolution] || 10;
const qualityScale = { standard: 0.65, high: 1, ultra: 1.45 }[quality] || 1;
return Math.round(base * qualityScale * (frameRate / 30) * 1_000_000);
}
import { getExportBitrate, getExportDimensions } from "./lib/exportSettings.js";
export function App() {
const [uiLanguage, setUiLanguage] = useState(() => getStoredLanguage());
@@ -670,7 +658,10 @@ export function App() {
ratioId={ratioId}
showRatioMenu={showRatioMenu}
setShowRatioMenu={setShowRatioMenu}
setRatioId={setRatioId}
setRatioId={(nextRatioId) => {
setRatioId(nextRatioId);
setFitModeFromUser("contain");
}}
notify={notify}
isPlaying={isPlaying}
handlePlayToggle={handlePlayToggle}
+5 -1
View File
@@ -13,6 +13,7 @@ import {
import { formatTime } from "../lib/timeline.js";
import { getVisualMaskInsets, getVisualMaskSvgDataUrl, resolveVisualTransform } from "../lib/visualEffects.js";
import { resolveVisualClipAnimation } from "../lib/visualClipAnimations.js";
import { getStickerBaseSize } from "../lib/stickerGeometry.js";
import { CaptionOverlay } from "./CaptionOverlay.jsx";
import { IconButton } from "./ui.jsx";
@@ -69,7 +70,7 @@ export function PreviewStage({
getDraggedAsset,
applyAssetToTrack,
}) {
const visibleStickers = stickers.length ? stickers : selectedSticker?.src || selectedSticker?.text ? [selectedSticker] : [];
const visibleStickers = stickers;
const hasStickerOverlay = visibleStickers.some((sticker) => sticker?.src || sticker?.text);
const hasPreviewContent = Boolean(previewVisualSrc || hasStickerOverlay);
const renderedVisualSrc = previewVisualRenderSrc || previewVisualSrc;
@@ -88,6 +89,7 @@ export function PreviewStage({
const frameWidth = Math.max(1, previewFrameSize.width || 1);
const frameHeight = Math.max(1, previewFrameSize.height || 1);
const frameMinDimension = Math.min(frameWidth, frameHeight);
const stickerBaseSize = getStickerBaseSize({ width: frameWidth, height: frameHeight });
const circleSize = Number.isFinite(visualMask.size) ? visualMask.size : 72;
const maskWidth = visualMask.type === "circle" ? (circleSize * frameMinDimension) / frameWidth : Number.isFinite(visualMask.width) ? visualMask.width : 80;
const maskHeight = visualMask.type === "circle" ? (circleSize * frameMinDimension) / frameHeight : Number.isFinite(visualMask.height) ? visualMask.height : 80;
@@ -337,6 +339,8 @@ export function PreviewStage({
className={`sticker-overlay sticker-transform-box ${isEditable ? "is-editable" : ""}`}
onPointerDown={(event) => startStickerDrag(event, sticker)}
style={{
width: `${stickerBaseSize}px`,
height: `${stickerBaseSize}px`,
left: `${Number.isFinite(sticker.x) ? sticker.x : 82}%`,
top: `${Number.isFinite(sticker.y) ? sticker.y : 20}%`,
transform: `translate(-50%, -50%) scale(${Number.isFinite(sticker.scale) ? sticker.scale : 1}) rotate(${Number.isFinite(sticker.rotation) ? sticker.rotation : 0}deg)`,
+9 -8
View File
@@ -258,8 +258,9 @@ export function Timeline({
...audioLanes.map((_, index) => ["audio", `${t("voiceTrack")} ${index + 1}`, `audio-${index}`]),
["music", t("musicTrack")],
];
const isRowVisible = (track, rowId = track) =>
trackVisibility[rowId] ?? trackVisibility[track] ?? true;
// Visibility is track-scoped even when overlapping clips are packed into
// multiple visual rows. Preview, playback and export all read the track key.
const isRowVisible = (track) => trackVisibility[track] ?? true;
const [rulerViewport, setRulerViewport] = useState({
scrollLeft: 0,
viewportWidth: 0,
@@ -597,7 +598,7 @@ export function Timeline({
<div
key={`sticker-lane-${laneIndex}`}
className={`sticker-track ${selectedTrack === "sticker" ? "is-selected" : ""} ${
!isRowVisible("sticker", `sticker-${laneIndex}`) ? "is-track-disabled" : ""
!isRowVisible("sticker") ? "is-track-disabled" : ""
} ${
assetDropTargetTrack === "sticker" ? "is-drop-target" : ""
} ${assetDropPulseTrack === "sticker" ? "is-drop-landing" : ""}`}
@@ -738,7 +739,7 @@ export function Timeline({
{timelineTrackLabels.map(([track, label, rowId = track]) => (
<div
className={`${selectedTrack === track ? "is-selected" : ""} ${
!isRowVisible(track, rowId) ? "is-track-disabled" : ""
!isRowVisible(track) ? "is-track-disabled" : ""
}`}
key={rowId}
onContextMenu={(event) => showTrackContextMenu(event, track)}
@@ -748,10 +749,10 @@ export function Timeline({
aria-label={`${label} ${t("visible")}`}
onClick={(event) => {
event.stopPropagation();
toggleTrackVisibility(rowId);
toggleTrackVisibility(track);
}}
>
{isRowVisible(track, rowId) ? <Eye size={15} /> : <EyeSlash size={15} />}
{isRowVisible(track) ? <Eye size={15} /> : <EyeSlash size={15} />}
</button>
<button
type="button"
@@ -991,7 +992,7 @@ export function Timeline({
{captionLanes.map((lane, laneIndex) => (
<div
className={`caption-track ${selectedTrack === "caption" ? "is-selected" : ""} ${
!isRowVisible("caption", `caption-${laneIndex}`) ? "is-track-disabled" : ""
!isRowVisible("caption") ? "is-track-disabled" : ""
} ${
activeTimelineClipDrag?.track === "caption" ? "is-reordering" : ""
}`}
@@ -1134,7 +1135,7 @@ export function Timeline({
{audioLanes.map((lane, laneIndex) => (
<button
className={`audio-track ${selectedTrack === "audio" ? "is-selected" : ""} ${
!isRowVisible("audio", `audio-${laneIndex}`) ? "is-track-disabled" : ""
!isRowVisible("audio") ? "is-track-disabled" : ""
} ${
laneIndex === 0 && assetDropTargetTrack === "audio" ? "is-drop-target" : ""
} ${laneIndex === 0 && assetDropPulseTrack === "audio" ? "is-drop-landing" : ""}`}
+63 -3
View File
@@ -7,10 +7,13 @@ import {
ListBullets,
PersonSimpleRun,
Scissors,
Sparkle,
Trash,
Waveform,
X,
} from "@phosphor-icons/react";
import { useEffect, useRef, useState } from "react";
import { createPortal } from "react-dom";
import { formatTime, getSegmentStartTime } from "../lib/timeline.js";
import { LIVE_PORTRAIT_WEB_MODEL } from "../config/livePortrait.js";
@@ -19,11 +22,63 @@ import { getCaptionVoiceSegment } from "../lib/captionVoice.js";
import { normalizeVisualKeyframes } from "../lib/visualEffects.js";
import { HistoryPanel, MyVoicesPanel, SmartVisionPanel, VisualEffectsPanel, VoiceSynthesisPanel } from "./panels.jsx";
function AutoEditReviewDialog({ t, autoEdit }) {
const { review, job } = autoEdit || {};
if (!review?.open || typeof document === "undefined") return null;
const complete = !job.running && review.captions.length > 0;
return createPortal(
<div className="auto-edit-review-backdrop" role="presentation">
<section className="auto-edit-review-dialog" role="dialog" aria-modal="true" aria-label={t("autoEditReviewTitle")}>
<header className="auto-edit-review-header">
<div className="auto-edit-review-mark"><Sparkle size={19} weight="fill" /></div>
<div><span>{t("smartAutoEdit")}</span><h2>{t("autoEditReviewTitle")}</h2></div>
<div className={`auto-edit-review-status ${complete ? "is-complete" : review.error ? "is-error" : ""}`}><i />{review.error ? t("autoEditReviewFailed") : complete ? t("autoEditReviewReady") : job.phase}</div>
<button type="button" className="auto-edit-review-close" aria-label={t("close")} onClick={autoEdit.closeReview}><X size={18} /></button>
</header>
<div className="auto-edit-review-progress"><span style={{ width: `${job.progress || 0}%` }} /></div>
<div className="auto-edit-review-body">
<section className="auto-edit-review-section">
<div className="auto-edit-review-section-title"><div><span>01</span><strong>{t("autoEditCandidateTitle")}</strong></div><em>{review.candidates.length} {t("autoEditFramesUnit")}</em></div>
<p>{t("autoEditCandidateHint")}</p>
{review.candidates.length ? <div className="auto-edit-candidate-grid">{review.candidates.map((candidate, index) => (
<article className="auto-edit-candidate-card" key={candidate.id}>
<div><img src={candidate.url} alt={`${t("autoEditCandidateFrame")} ${index + 1}`} /><span>#{String(index + 1).padStart(2, "0")}</span><em>{candidate.aspectRatio}</em><time>{formatTime(candidate.time)}</time></div>
<footer><span>{t("autoEditVisualChange")}</span><strong>{Math.round(candidate.difference * 100)}%</strong><i><b style={{ width: `${Math.min(100, Math.max(5, candidate.difference * 100))}%` }} /></i></footer>
</article>
))}</div> : <div className="auto-edit-review-loading"><i /><span>{t("autoEditFindingScenes")}</span></div>}
</section>
<section className="auto-edit-review-section auto-edit-model-results">
<div className="auto-edit-review-section-title"><div><span>02</span><strong>{t("autoEditModelResultTitle")}</strong></div><em>{review.captions.length} {t("captionSegmentsUnit")}</em></div>
<p>{t("autoEditModelResultHint")}</p>
{review.error ? <div className="auto-edit-review-error"><strong>{t("autoEditReviewFailed")}</strong><span>{review.error}</span></div> : review.segments.length ? <div className="auto-edit-clip-results">{review.segments.map((segment) => {
const segmentCaptions = review.captions.filter((caption) => caption.visualSegmentId === segment.id);
const preview = review.candidates.find((candidate) => candidate.segmentId === segment.id);
return <article className={`auto-edit-clip-result is-${segment.status}`} key={segment.id}>
<header>{preview ? <img src={preview.url} alt="" /> : null}<div><strong>{segment.name || `${t("autoEditClip")} ${(segment.index ?? 0) + 1}`}</strong><span>{review.candidates.filter((candidate) => candidate.segmentId === segment.id).length} {t("autoEditFramesUnit")}</span></div><em>{t(`autoEditSegmentStatus_${segment.status}`)}</em></header>
{segment.error ? <p className="auto-edit-clip-error">{segment.error}</p> : segmentCaptions.length ? <><div className="auto-edit-result-list">{segmentCaptions.map((caption, index) => (
<article key={caption.id}><span>{String(index + 1).padStart(2, "0")}</span><div><p>{caption.text}</p><time>{formatTime(caption.start)} {formatTime(caption.end)}</time></div></article>
))}</div>{segment.status === "running" ? <div className="auto-edit-clip-pending"><i /><span>{t("autoEditWindowProgress").replace("{current}", segment.windowIndex || 0).replace("{total}", segment.totalWindows || 0)}</span></div> : null}</> : <div className="auto-edit-clip-pending">{segment.status === "running" ? <i /> : null}<span>{segment.status === "running" && segment.totalWindows ? t("autoEditWindowProgress").replace("{current}", segment.windowIndex || 0).replace("{total}", segment.totalWindows) : t(`autoEditSegmentHint_${segment.status}`)}</span></div>}
</article>;
})}</div> : <div className="auto-edit-review-loading"><i /><span>{job.running ? job.phase : t("autoEditWaitingForModel")}</span></div>}
</section>
</div>
<footer className="auto-edit-review-actions">
<div><strong>{complete ? t("autoEditReviewSummaryReady") : t("autoEditReviewSummaryRunning")}</strong><span>{t("autoEditReviewSummaryHint")}</span></div>
<button type="button" className="panel-secondary" onClick={autoEdit.closeReview}>{job.running ? t("cancel") : t("close")}</button>
<button type="button" className="auto-edit-apply" disabled={!complete} onClick={autoEdit.applyCaptions}><Sparkle size={16} weight="fill" />{t("autoEditApplyCaptions")}</button>
</footer>
</section>
</div>, document.body,
);
}
function AutoEditPanel({ t, hasVisual, language, autoEdit }) {
const availability = autoEdit?.support?.availability || "unknown";
const languageFallback = autoEdit?.support?.language && autoEdit.support.language !== language;
const ready = availability === "available" || availability === "downloadable" || availability === "downloading";
return (
return (<>
<div className="auto-edit-panel">
<section className="auto-edit-intro"><Scissors size={28} weight="duotone" /><div><strong>{t("autoEditCreateTitle")}</strong><span>{t("autoEditCreateDesc")}</span></div></section>
<section className="auto-edit-status-card">
@@ -34,9 +89,14 @@ function AutoEditPanel({ t, hasVisual, language, autoEdit }) {
</section>
<div className="auto-edit-flow"><span>1</span><p><strong>{t("autoEditStepScenes")}</strong><small>{t("autoEditStepScenesHint")}</small></p><span>2</span><p><strong>{t("autoEditStepCaptions")}</strong><small>{t("autoEditStepCaptionsHint")}</small></p><span>3</span><p><strong>{t("autoEditStepTimeline")}</strong><small>{t("autoEditStepTimelineHint")}</small></p></div>
{autoEdit?.job?.running ? <div className="auto-edit-progress"><div><span>{autoEdit.job.phase}</span><strong>{autoEdit.job.progress}%</strong></div><progress max="100" value={autoEdit.job.progress} /><button className="panel-secondary" type="button" onClick={autoEdit.cancel}>{t("cancel")}</button></div> : null}
<button className="primary-action" type="button" disabled={!hasVisual || !ready || autoEdit?.job?.running} onClick={autoEdit?.run}>{hasVisual ? t("autoEditGenerate") : t("autoEditNeedsVisual")}</button>
<button className="auto-edit-generate" type="button" disabled={!hasVisual || !ready || autoEdit?.job?.running} onClick={autoEdit?.run}>
<span className="auto-edit-generate-icon"><Sparkle size={17} weight="fill" /></span>
<span><strong>{hasVisual ? t("autoEditGenerate") : t("autoEditNeedsVisual")}</strong><small>{hasVisual ? t("autoEditGenerateHint") : t("autoEditNeedsVisualHint")}</small></span>
<span className="auto-edit-generate-arrow"></span>
</button>
</div>
);
<AutoEditReviewDialog t={t} autoEdit={autoEdit} />
</>);
}
function CaptionContextPanel({
+62 -10
View File
@@ -1,11 +1,18 @@
import { useCallback, useRef, useState } from "react";
import { extractAutoEditFrames, generateFrameCaptions, probeBuiltInAI } from "../lib/autoEdit.js";
import { useCallback, useEffect, useRef, useState } from "react";
import { createFrameCaptionSession, extractAutoEditFrames, generateFrameCaptions, probeBuiltInAI } from "../lib/autoEdit.js";
import { getVisualSegmentsTotal } from "../lib/timeline.js";
export function useAutoEdit({ language, visualSegments, commitCaptionSegments, setCaptionsEnabled, setSelectedSegmentId, setSelectedTrack, notify, t }) {
const [support, setSupport] = useState({ availability: "unknown", reason: "", language: "en" });
const [job, setJob] = useState({ running: false, progress: 0, phase: "" });
const [review, setReview] = useState({ open: false, candidates: [], captions: [], segments: [], error: "" });
const abortRef = useRef(null);
const candidateUrlsRef = useRef([]);
const clearCandidateUrls = useCallback(() => {
candidateUrlsRef.current.forEach((url) => URL.revokeObjectURL(url));
candidateUrlsRef.current = [];
}, []);
useEffect(() => clearCandidateUrls, [clearCandidateUrls]);
const checkSupport = useCallback(async () => {
setSupport((value) => ({ ...value, availability: "checking" }));
const result = await probeBuiltInAI(language);
@@ -17,21 +24,66 @@ export function useAutoEdit({ language, visualSegments, commitCaptionSegments, s
const environment = support.availability === "unknown" ? await checkSupport() : support;
if (environment.availability === "unavailable") return void notify(t("autoEditUnavailable"));
abortRef.current = new AbortController();
clearCandidateUrls();
setReview({ open: true, candidates: [], captions: [], segments: [], error: "" });
let session = null;
setJob({ running: true, progress: 2, phase: t("autoEditFindingScenes") });
// Chrome requires LanguageModel.create() to happen during the button's
// transient user activation when the model still needs downloading.
const sessionPromise = createFrameCaptionSession({
language,
signal: abortRef.current.signal,
onDownloadProgress: (loaded) => setJob({ running: true, progress: Math.max(4, Math.round(loaded * 55)), phase: t("autoEditDownloadingModel") }),
});
try {
const frames = await extractAutoEditFrames(visualSegments, (progress) => setJob({ running: true, progress, phase: t("autoEditFindingScenes") }), abortRef.current.signal);
const candidates = frames.map((frame, index) => {
const url = URL.createObjectURL(frame.blob);
candidateUrlsRef.current.push(url);
return { id: `${frame.segmentId}-${index}`, segmentId: frame.segmentId, segmentIndex: frame.segmentIndex, segmentName: frame.segmentName, url, time: frame.time, difference: frame.difference, aspectRatio: frame.aspectRatio };
});
const segments = candidates.reduce((items, candidate) => items.some((item) => item.id === candidate.segmentId) ? items : [...items, { id: candidate.segmentId, index: candidate.segmentIndex, name: candidate.segmentName, status: "waiting", error: "" }], []);
setReview((value) => ({ ...value, candidates, segments }));
setJob({ running: true, progress: 60, phase: t("autoEditWritingCaptions") });
const captions = await generateFrameCaptions({ frames, duration: getVisualSegmentsTotal(visualSegments), language, onDownloadProgress: (loaded) => setJob({ running: true, progress: 60 + Math.round(loaded * 20), phase: t("autoEditDownloadingModel") }) });
if (!captions.length) throw new Error("No captions generated");
commitCaptionSegments(captions);
setCaptionsEnabled(true); setSelectedTrack("caption"); setSelectedSegmentId(captions[0].id);
session = await sessionPromise;
const captions = await generateFrameCaptions({
frames, duration: getVisualSegmentsTotal(visualSegments), language, session,
onPartial: (partial) => {
const modelProgress = partial.allWindows ? partial.completedWindows / partial.allWindows : 0;
setJob({ running: true, progress: Math.min(96, 60 + Math.round(modelProgress * 36)), phase: t("autoEditWritingCaptions") });
setReview((value) => ({
...value,
captions: partial.captions.length ? [...value.captions.filter((caption) => caption.visualSegmentId !== partial.segmentId), ...partial.captions].sort((a, b) => a.start - b.start) : value.captions,
segments: value.segments.map((segment) => segment.id === partial.segmentId ? { ...segment, status: partial.status, error: partial.error || "", windowIndex: partial.windowIndex || 0, totalWindows: partial.totalWindows || 0 } : segment),
}));
},
});
if (!captions.length) {
setJob({ running: false, progress: 100, phase: t("autoEditNoResults") });
return;
}
setReview((value) => ({ ...value, captions }));
setJob({ running: false, progress: 100, phase: t("autoEditDone") });
notify(t("autoEditDone"));
} catch (error) {
if (error?.name !== "AbortError") notify(`${t("autoEditFailed")}: ${error?.message || error}`);
if (error?.name !== "AbortError") setReview((value) => ({ ...value, error: error?.message || String(error) }));
setJob({ running: false, progress: 0, phase: "" });
} finally {
session?.destroy?.();
}
}, [checkSupport, commitCaptionSegments, job.running, language, notify, setCaptionsEnabled, setSelectedSegmentId, setSelectedTrack, support, t, visualSegments]);
}, [checkSupport, clearCandidateUrls, job.running, language, notify, support, t, visualSegments]);
const cancel = () => { abortRef.current?.abort(); setJob({ running: false, progress: 0, phase: "" }); };
return { support, job, checkSupport, run, cancel };
const closeReview = () => {
if (job.running) abortRef.current?.abort();
setJob({ running: false, progress: 0, phase: "" });
setReview({ open: false, candidates: [], captions: [], segments: [], error: "" });
clearCandidateUrls();
};
const applyCaptions = () => {
if (!review.captions.length) return;
commitCaptionSegments(review.captions);
setCaptionsEnabled(true); setSelectedTrack("caption"); setSelectedSegmentId(review.captions[0].id);
notify(t("autoEditDone"));
closeReview();
};
return { support, job, review, checkSupport, run, cancel, closeReview, applyCaptions };
}
+1 -1
View File
@@ -20,7 +20,7 @@ export function useCaptionState() {
const [script, setScript] = useState(DEFAULT_SCRIPT);
const [captionPosition, setCaptionPosition] = useState("bottom");
const [captionPlacement, setCaptionPlacement] = useState({ x: 50, y: 78 });
const [captionSize, setCaptionSize] = useState(12);
const [captionSize, setCaptionSize] = useState(14);
const [captionStyle, setCaptionStyle] = useState(DEFAULT_CAPTION_STYLE);
const [captionsEnabled, setCaptionsEnabled] = useState(true);
const [captionSegments, setCaptionSegments] = useState(() => createCaptionSegments(DEFAULT_SCRIPT));
+2 -1
View File
@@ -31,6 +31,7 @@ export function usePreviewModel(d) {
}), [d.previewFrameSize.height, d.previewFrameSize.width, d.ratio.height, d.ratio.width]);
const previewSmartCropRect = useMemo(() => {
if (
d.fitMode !== "cover" ||
!previewVisionOptions.smartCrop ||
!previewVisionAnalysis?.subject?.box ||
!previewVisionAnalysis?.sourceSize
@@ -41,7 +42,7 @@ export function usePreviewModel(d) {
previewVisionAnalysis.subject.box,
{ padding: 0.14 },
);
}, [previewVisionAnalysis, previewVisionFrameSize, previewVisionOptions.smartCrop]);
}, [d.fitMode, previewVisionAnalysis, previewVisionFrameSize, previewVisionOptions.smartCrop]);
const previewVisionOverlayBoxes = useMemo(() => {
if (!previewVisionOptions.showDetections || !previewVisionAnalysis?.sourceSize) return [];
return (previewVisionAnalysis.detections ?? []).map((detection) => {
+1 -1
View File
@@ -68,7 +68,7 @@ export function useProjectFiles(deps) {
deps.setSelectedVoiceId(data.selectedVoiceId || VOICES[0].id); deps.setSpeed(Number(data.speed) || 1);
deps.setVolume(Number(data.volume) || 1); deps.setRatioId(RATIO_OPTIONS.some((option) => option.id === data.ratioId) ? data.ratioId : "16:9");
deps.setFitMode(data.fitMode || "contain"); deps.setCaptionPosition(data.captionPosition || "bottom");
deps.setCaptionPlacement(data.captionPlacement || { x: 50, y: 78 }); deps.setCaptionSize(Number(data.captionSize) || 12);
deps.setCaptionPlacement(data.captionPlacement || { x: 50, y: 78 }); deps.setCaptionSize(Number(data.captionSize) || 14);
deps.setCaptionStyle(data.captionStyle || deps.captionStyle); deps.setCaptionsEnabled(data.captionsEnabled !== false);
deps.setTrackVisibility(data.trackVisibility || deps.trackVisibility); deps.setTimelineZoom(Number(data.timelineZoom) || 1);
deps.setSelectedFilterId(data.selectedFilterId || "none"); deps.setSelectedTransitionId(data.selectedTransitionId || "none");
+4 -12
View File
@@ -133,18 +133,10 @@ export function useTimelineModel(d) {
return d.currentTime >= start && d.currentTime < end;
})
: [];
const previewSticker = d.trackVisibility.sticker && currentStickerSegment
? currentStickerSegment
: d.stickerSegments.length
? STICKERS[0]
: selectedSticker;
const previewStickers = currentStickerSegments.length
? currentStickerSegments
: d.stickerSegments.length
? []
: previewSticker?.src || previewSticker?.text
? [previewSticker]
: [];
// The preview is timeline-driven. A library selection is only a source for
// creating a clip; it must not survive after the final sticker clip is deleted.
const previewSticker = d.trackVisibility.sticker ? currentStickerSegment : null;
const previewStickers = currentStickerSegments;
const currentVisualSegmentIndex = getVisualSegmentIndexAtTime(d.visualSegments, d.currentTime);
const currentVisualSegment = currentVisualSegmentIndex >= 0
? d.visualSegments[currentVisualSegmentIndex] ?? null
+13 -3
View File
@@ -1,5 +1,6 @@
import { useCallback } from "react";
import { downloadBlob, exportBrowserVideo, getSupportedRecordingFormat, transcodeWebmToMp4 } from "../lib/media.js";
import { exportOfflineVideo } from "../lib/offlineVideoExport.js";
import { estimateDuration } from "../lib/timeline.js";
import { getVisionKey } from "../lib/vision.js";
@@ -16,7 +17,7 @@ export function useVideoExport(d) {
};
const finish = async (phase) => { d.setExportPhase(phase); d.setExportProgress(100); await new Promise((resolve) => setTimeout(resolve, 450)); };
try {
const video = await exportBrowserVideo({
const exportOptions = {
imageSrc: d.imageSrc, visualType: d.visualType,
visualSegments: d.renderedVisualSegments.map((segment) => {
const record = d.visionRecords[getVisionKey(segment)];
@@ -37,10 +38,19 @@ export function useVideoExport(d) {
captionSize: d.captionSize, captionStyle: d.captionStyle,
captionReferenceSize: d.previewFrameSize.width > 0 && d.previewFrameSize.height > 0 ? d.previewFrameSize
: { width: (360 * d.ratio.width) / d.ratio.height, height: 360 },
sticker: d.stickerSegments.length ? null : d.selectedSticker,
// Stickers are timeline clips; a selected library item is not export content.
sticker: null,
stickerSegments: d.trackVisibility.sticker ? d.stickerSegments : [],
transitionId: "none", exportSettings: d.exportSettings, onProgress: progress,
});
};
let video;
try {
video = await exportOfflineVideo(exportOptions);
} catch (offlineError) {
console.warn("Offline WebCodecs export unavailable; using compatibility recorder", offlineError);
progress({ progress: 5, phase: "切换兼容导出模式" });
video = await exportBrowserVideo(exportOptions);
}
const name = `ai-voiceover-${d.ratio.id.replace(":", "x")}`;
if (d.exportSettings.codec !== "h264") {
progress({ progress: 99, phase: `保存 ${video.label} 文件` });
+35
View File
@@ -269,10 +269,40 @@ const AUTO_EDIT_COPY = {
en: { smartAutoEditHint: "Local captions", autoEditCreateTitle: "Create timed captions from visuals", autoEditCreateDesc: "Detect scene changes, sample representative frames, and use Chrome's built-in model to write timed captions.", autoEditBrowserModel: "Chrome built-in model", autoEditPrivacyHint: "Frames are processed on this device and are not sent to the project server. Chrome may download the model on first use.", autoEditLanguageFallback: "The current UI language is not officially supported by Chrome Prompt API output. Captions will be generated in English and remain editable.", autoEditCheckSupport: "Check browser support", autoEditStepScenes: "Detect scenes", autoEditStepScenesHint: "Select key visuals by frame difference", autoEditStepCaptions: "Understand visuals", autoEditStepCaptionsHint: "Generate copy with the local multimodal model", autoEditStepTimeline: "Write timeline", autoEditStepTimelineHint: "Keep caption start and end times", autoEditGenerate: "Generate visual captions", autoEditNeedsVisual: "Add an image or video first", autoEditFindingScenes: "Detecting scene changes", autoEditWritingCaptions: "Writing captions", autoEditDownloadingModel: "Downloading browser model", autoEditDone: "Visual captions added to the timeline", autoEditUnavailable: "Chrome built-in AI is unavailable on this browser or device", autoEditFailed: "Auto Edit failed", autoEditStatus_unknown: "Not checked", autoEditStatus_checking: "Checking", autoEditStatus_available: "Available", autoEditStatus_downloadable: "Model download needed", autoEditStatus_downloading: "Downloading", autoEditStatus_unavailable: "Unavailable" },
};
const AUTO_EDIT_BUTTON_COPY = {
zh: { autoEditGenerateHint: "检测画面变化并创建时间轴字幕", autoEditNeedsVisualHint: "导入素材后即可开始" },
en: { autoEditGenerateHint: "Detect visual changes and create timed captions", autoEditNeedsVisualHint: "Import media to get started" },
};
const AUTO_EDIT_REVIEW_COPY = {
zh: { autoEditReviewTitle: "画面分析与字幕检查", autoEditReviewReady: "分析完成", autoEditReviewFailed: "分析失败", autoEditCandidateTitle: "候选关键帧", autoEditCandidateHint: "根据连续帧的视觉变化筛选,以下画面将作为模型输入。", autoEditFramesUnit: "帧", autoEditCandidateFrame: "候选帧", autoEditVisualChange: "画面变化", autoEditModelResultTitle: "模型解析结果", autoEditModelResultHint: "模型根据候选画面生成的字幕内容与时间范围。", autoEditWaitingForModel: "等待模型解析候选画面", autoEditReviewSummaryReady: "字幕草稿已准备好", autoEditReviewSummaryRunning: "正在构建字幕草稿", autoEditReviewSummaryHint: "确认后才会写入现有字幕轨。", autoEditApplyCaptions: "应用到字幕轨" },
en: { autoEditReviewTitle: "Visual analysis & caption review", autoEditReviewReady: "Analysis complete", autoEditReviewFailed: "Analysis failed", autoEditCandidateTitle: "Candidate keyframes", autoEditCandidateHint: "Selected from visual changes across consecutive frames. These frames will be sent to the model.", autoEditFramesUnit: "frames", autoEditCandidateFrame: "Candidate frame", autoEditVisualChange: "Visual change", autoEditModelResultTitle: "Model output", autoEditModelResultHint: "Caption text and time ranges generated from the candidate visuals.", autoEditWaitingForModel: "Waiting for the model to analyze candidates", autoEditReviewSummaryReady: "Caption draft is ready", autoEditReviewSummaryRunning: "Building caption draft", autoEditReviewSummaryHint: "Nothing is written to the caption track until you confirm.", autoEditApplyCaptions: "Apply to caption track" },
};
const AUTO_EDIT_FLOW_COPY = {
zh: { autoEditCandidateHint: "通过轻量级块匹配光流追踪连续画面变化,以下帧将作为模型输入。", autoEditVisualChange: "光流变化", autoEditStepScenesHint: "在 Worker 中通过光流筛选关键画面" },
en: { autoEditCandidateHint: "Hybrid scene-cut and motion analysis selects representative frames across each clip. These candidates will be sent to the model.", autoEditVisualChange: "Scene score", autoEditStepScenesHint: "Find representative visuals in a background Worker" },
};
const AUTO_EDIT_SEGMENT_COPY = {
zh: { autoEditClip: "片段", autoEditSegmentStatus_waiting: "等待中", autoEditSegmentStatus_running: "解析中", autoEditSegmentStatus_complete: "已完成", autoEditSegmentStatus_empty: "无结果", autoEditSegmentStatus_error: "失败", autoEditSegmentHint_waiting: "排队等待模型处理", autoEditSegmentHint_running: "正在分析这个片段的候选画面", autoEditSegmentHint_complete: "片段解析完成", autoEditSegmentHint_empty: "模型未为这个片段生成字幕", autoEditSegmentHint_error: "这个片段解析失败", autoEditWindowProgress: "正在处理窗口 {current} / {total}" },
en: { autoEditClip: "Clip", autoEditSegmentStatus_waiting: "Waiting", autoEditSegmentStatus_running: "Analyzing", autoEditSegmentStatus_complete: "Complete", autoEditSegmentStatus_empty: "No result", autoEditSegmentStatus_error: "Failed", autoEditSegmentHint_waiting: "Queued for model processing", autoEditSegmentHint_running: "Analyzing this clip's candidate visuals", autoEditSegmentHint_complete: "Clip analysis complete", autoEditSegmentHint_empty: "The model returned no caption for this clip", autoEditSegmentHint_error: "This clip could not be analyzed", autoEditWindowProgress: "Processing window {current} / {total}" },
};
const AUTO_EDIT_RESULT_COPY = {
zh: { autoEditNoResults: "所有片段均已处理,但没有可用字幕" },
en: { autoEditNoResults: "All clips were processed, but no usable captions were generated" },
};
export const UI_COPY = {
zh: {
...SMART_WORKSPACE_COPY.zh,
...AUTO_EDIT_COPY.zh,
...AUTO_EDIT_BUTTON_COPY.zh,
...AUTO_EDIT_REVIEW_COPY.zh,
...AUTO_EDIT_FLOW_COPY.zh,
...AUTO_EDIT_SEGMENT_COPY.zh,
...AUTO_EDIT_RESULT_COPY.zh,
...VISUAL_EDITOR_COPY.zh,
...TRANSITION_EDITOR_COPY.zh,
...VISUAL_ANIMATION_COPY.zh,
@@ -647,6 +677,11 @@ export const UI_COPY = {
en: {
...SMART_WORKSPACE_COPY.en,
...AUTO_EDIT_COPY.en,
...AUTO_EDIT_BUTTON_COPY.en,
...AUTO_EDIT_REVIEW_COPY.en,
...AUTO_EDIT_FLOW_COPY.en,
...AUTO_EDIT_SEGMENT_COPY.en,
...AUTO_EDIT_RESULT_COPY.en,
...VISUAL_EDITOR_COPY.en,
...TRANSITION_EDITOR_COPY.en,
...VISUAL_ANIMATION_COPY.en,
-16
View File
@@ -1,16 +0,0 @@
import { describe, expect, it } from "vitest";
import { APP_LANGUAGES, createTranslator, translateOptionName } from "./i18n.js";
describe("transition editor translations", () => {
it("provides editor labels and localized transition names for every UI language", () => {
APP_LANGUAGES.forEach(({ id }) => {
const t = createTranslator(id);
expect(t("transitionSettings")).not.toBe("transitionSettings");
expect(t("close")).not.toBe("close");
expect(t("duration")).not.toBe("duration");
expect(t("secondsShort")).not.toBe("secondsShort");
if (id !== "zh") expect(translateOptionName(id, "淡入淡出")).not.toBe("淡入淡出");
});
});
});
-17
View File
@@ -1,17 +0,0 @@
import { describe, expect, it } from "vitest";
import { APP_LANGUAGES, createTranslator } from "./i18n.js";
describe("visual animation translations", () => {
it("provides native animation copy for every supported UI language", () => {
for (const language of APP_LANGUAGES) {
const t = createTranslator(language.id);
expect(t("visualTabAnimation")).not.toBe("visualTabAnimation");
expect(t("visualAnimationHoverHint")).not.toBe("visualAnimationHoverHint");
expect(t("visualAnimationDuration")).not.toBe("visualAnimationDuration");
expect(t("stickerProperties")).not.toBe("stickerProperties");
expect(t("stickerOpacity")).not.toBe("stickerOpacity");
expect(t("deleteSticker")).not.toBe("deleteSticker");
expect(t("newCaptionDefault")).not.toBe("newCaptionDefault");
}
});
});
+8 -2
View File
@@ -562,7 +562,8 @@ async function getTranscriber(onProgress) {
transcriberState = {
modelId,
promise: (async () => {
const { pipeline } = await import("@huggingface/transformers");
const { env, pipeline } = await import("@huggingface/transformers");
env.useBrowserCache = false;
const reportModelLoadProgress = createModelLoadProgressCallback(onProgress);
return pipeline("automatic-speech-recognition", modelId, {
dtype: "q8",
@@ -573,7 +574,12 @@ async function getTranscriber(onProgress) {
};
}
return transcriberState.promise;
try {
return await transcriberState.promise;
} catch (error) {
transcriberState = null;
throw error;
}
}
function rejectWorkerRequests(error) {
+239 -59
View File
@@ -23,17 +23,59 @@ export async function probeBuiltInAI(language = "en") {
}
}
export function selectChangedFrames(frames, { threshold = 0.12, maxFrames = 12 } = {}) {
function median(values) {
if (!values.length) return 0;
const sorted = [...values].sort((a, b) => a - b);
const middle = Math.floor(sorted.length / 2);
return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
}
export function getAdaptiveSceneThreshold(frames, floor = 0.1) {
const scores = frames.slice(1).map((frame) => Number(frame.difference) || 0);
const center = median(scores);
const deviation = median(scores.map((score) => Math.abs(score - center)));
return Math.max(floor, center + Math.max(0.035, deviation * 2.5));
}
export function selectChangedFrames(frames, { threshold, maxFrames = Infinity, minTimeGap = 1.2 } = {}) {
if (!frames.length) return [];
const selected = [frames[0]];
for (let index = 1; index < frames.length && selected.length < maxFrames; index += 1) {
const frame = frames[index];
const previous = selected.at(-1);
if (frame.segmentId !== previous.segmentId || frame.difference >= threshold) selected.push(frame);
if (maxFrames <= 1) return [frames[0]];
const selected = frames.filter((frame, index) => index === 0 || frame.segmentId !== frames[index - 1].segmentId).slice(0, maxFrames);
const effectiveThreshold = Number.isFinite(threshold) ? threshold : getAdaptiveSceneThreshold(frames);
const ranked = frames.slice(1).map((frame, index) => {
const previousScore = frames[index]?.difference || 0;
const localPeak = frame.difference >= previousScore && frame.difference >= (frames[index + 2]?.difference || 0);
const quality = frame.quality ?? 1;
return { frame, score: frame.difference * (.7 + quality * .3), localPeak };
}).filter(({ frame, score }) => score >= effectiveThreshold && (frame.quality ?? 1) >= .28).sort((a, b) => b.score - a.score);
for (const { frame } of ranked) {
if (selected.length >= maxFrames) break;
if (selected.every((item) => item.segmentId !== frame.segmentId || Math.abs(item.time - frame.time) >= minTimeGap)) selected.push(frame);
}
const last = frames.at(-1);
if (selected.length < maxFrames && last && selected.at(-1)?.time !== last.time) selected.push(last);
return selected;
// Explicit finite budgets (used only by callers/tests) may request coverage fill.
if (Number.isFinite(maxFrames) && selected.length < maxFrames) {
const fallback = frames.slice(1, -1).filter((frame) => (frame.quality ?? 1) >= .28).sort((a, b) => {
const distance = (frame) => Math.min(...selected.map((item) => Math.abs(item.time - frame.time)));
return distance(b) - distance(a);
});
for (const frame of fallback) {
if (selected.length >= maxFrames) break;
if (selected.every((item) => item.segmentId !== frame.segmentId || Math.abs(item.time - frame.time) >= minTimeGap)) selected.push(frame);
}
}
return selected.sort((a, b) => a.time - b.time);
}
export function selectCandidatesBySegment(frames, maxPerSegment) {
const groups = [];
frames.forEach((frame) => {
let group = groups.find((item) => item.segmentId === frame.segmentId);
if (!group) { group = { segmentId: frame.segmentId, frames: [] }; groups.push(group); }
group.frames.push(frame);
});
return groups.flatMap((group) => {
return selectChangedFrames(group.frames, { maxFrames: Number.isFinite(maxPerSegment) ? maxPerSegment : Infinity, minTimeGap: 1.2 });
});
}
export function normalizeGeneratedCaptions(value, duration) {
@@ -41,7 +83,7 @@ export function normalizeGeneratedCaptions(value, duration) {
if (!Array.isArray(items)) return [];
return items
.map((item) => {
const start = Math.max(0, Math.min(duration, Number(item.start) || 0));
const start = Math.max(0, Math.min(Math.max(0, duration - 0.2), Number(item.start) || 0));
const end = Math.max(start + 0.2, Math.min(duration, Number(item.end) || start + 2));
return { id: makeId("caption"), text: String(item.text || "").trim(), start, end, hidden: false };
})
@@ -49,6 +91,29 @@ export function normalizeGeneratedCaptions(value, duration) {
.sort((a, b) => a.start - b.start);
}
export function normalizeClipCaptionTimings(captions, clipStart, clipEnd, preferredMinimum = 1.2) {
if (!captions.length) return [];
const safeStart = Math.max(0, Number(clipStart) || 0);
const safeEnd = Math.max(safeStart + 0.2, Number(clipEnd) || safeStart + 0.2);
const minimum = Math.min(preferredMinimum, (safeEnd - safeStart) / captions.length);
const normalized = captions
.map((caption) => ({
...caption,
start: Math.max(safeStart, Math.min(safeEnd - 0.2, Number(caption.start) || safeStart)),
end: Math.max(safeStart + 0.2, Math.min(safeEnd, Number(caption.end) || safeEnd)),
}))
.sort((a, b) => a.start - b.start);
for (let index = normalized.length - 1; index >= 0; index -= 1) {
const caption = normalized[index];
if (index < normalized.length - 1) caption.end = Math.min(caption.end, normalized[index + 1].start);
if (caption.end - caption.start < minimum) caption.start = Math.max(safeStart, caption.end - minimum);
}
for (let index = 0; index < normalized.length - 1; index += 1) {
normalized[index].end = Math.min(normalized[index].end, normalized[index + 1].start);
}
return normalized.filter((caption) => caption.end - caption.start >= 0.19);
}
function waitForMedia(element, event) {
return new Promise((resolve, reject) => {
const done = () => { cleanup(); resolve(); };
@@ -63,58 +128,99 @@ async function canvasBlob(canvas) {
return new Promise((resolve, reject) => canvas.toBlob((blob) => blob ? resolve(blob) : reject(new Error("Frame encoding failed")), "image/jpeg", 0.82));
}
function pixelDifference(current, previous) {
if (!previous) return 1;
let sum = 0;
for (let index = 0; index < current.length; index += 4) {
sum += Math.abs(current[index] - previous[index]);
sum += Math.abs(current[index + 1] - previous[index + 1]);
sum += Math.abs(current[index + 2] - previous[index + 2]);
}
return sum / ((current.length / 4) * 3 * 255);
function createFlowWorkerClient() {
const worker = new Worker(new URL("../workers/auto-edit.worker.js", import.meta.url), { type: "module" });
const pending = new Map();
let requestId = 0;
worker.onmessage = (event) => {
const request = pending.get(event.data?.id);
if (!request) return;
pending.delete(event.data.id);
request.resolve(event.data);
};
worker.onerror = (error) => {
pending.forEach(({ reject }) => reject(error));
pending.clear();
};
return {
analyze(pixels, width, height, segmentId) {
const id = ++requestId;
const promise = new Promise((resolve, reject) => pending.set(id, { resolve, reject }));
worker.postMessage({ type: "analyze", id, pixels: pixels.buffer, width, height, segmentId }, [pixels.buffer]);
return promise;
},
terminate() { worker.terminate(); pending.forEach(({ reject }) => reject(new DOMException("Aborted", "AbortError"))); pending.clear(); },
};
}
function drawContainedFrame(context, source, sourceWidth, sourceHeight) {
const width = context.canvas.width;
const height = context.canvas.height;
const scale = Math.min(width / sourceWidth, height / sourceHeight);
const drawWidth = sourceWidth * scale;
const drawHeight = sourceHeight * scale;
context.fillStyle = "#000";
context.fillRect(0, 0, width, height);
context.drawImage(source, (width - drawWidth) / 2, (height - drawHeight) / 2, drawWidth, drawHeight);
}
export function getAspectRatioLabel(width, height) {
const ratio = width / Math.max(1, height);
const presets = [[9, 16], [1, 1], [4, 3], [3, 2], [16, 9], [21, 9]];
const [w, h] = presets.reduce((best, value) => Math.abs(value[0] / value[1] - ratio) < Math.abs(best[0] / best[1] - ratio) ? value : best);
return `${w}:${h}`;
}
export async function extractAutoEditFrames(segments, onProgress = () => {}, signal) {
const frames = [];
const flowWorker = createFlowWorkerClient();
let timelineStart = 0;
for (let segmentIndex = 0; segmentIndex < segments.length; segmentIndex += 1) {
if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
const segment = segments[segmentIndex];
const duration = Math.max(0.2, Number(segment.duration) || 0.2);
const canvas = document.createElement("canvas");
canvas.width = 224; canvas.height = 126;
const context = canvas.getContext("2d", { willReadFrequently: true });
let previousPixels = null;
if (segment.type === "video") {
const video = document.createElement("video");
video.muted = true; video.preload = "auto"; video.src = segment.src;
if (video.readyState < 1) await waitForMedia(video, "loadedmetadata");
const sourceStart = Number(segment.sourceStart) || 0;
const sourceDuration = Math.max(0.2, Number(segment.sourceDuration) || video.duration || duration);
const sampleCount = Math.min(30, Math.max(3, Math.ceil(duration * 1.5)));
for (let sample = 0; sample < sampleCount; sample += 1) {
const ratio = sampleCount === 1 ? 0 : sample / (sampleCount - 1);
video.currentTime = Math.min(Math.max(0, video.duration - 0.05), sourceStart + ratio * sourceDuration);
await waitForMedia(video, "seeked");
context.drawImage(video, 0, 0, canvas.width, canvas.height);
const pixels = context.getImageData(0, 0, canvas.width, canvas.height).data;
frames.push({ segmentId: segment.id, time: timelineStart + ratio * duration, difference: pixelDifference(pixels, previousPixels), blob: await canvasBlob(canvas) });
previousPixels = new Uint8ClampedArray(pixels);
try {
for (let segmentIndex = 0; segmentIndex < segments.length; segmentIndex += 1) {
if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
const segment = segments[segmentIndex];
const duration = Math.max(0.2, Number(segment.duration) || 0.2);
const canvas = document.createElement("canvas");
canvas.width = 224; canvas.height = 224;
const context = canvas.getContext("2d", { willReadFrequently: true });
if (segment.type === "video") {
const video = document.createElement("video");
video.muted = true; video.preload = "auto"; video.src = segment.src;
if (video.readyState < 1) await waitForMedia(video, "loadedmetadata");
const sourceStart = Number(segment.sourceStart) || 0;
const sourceDuration = Math.max(0.2, Number(segment.sourceDuration) || video.duration || duration);
// Keep a stable temporal resolution instead of a fixed total-frame cap.
// Long videos therefore receive proportionally more analysis samples.
const samplesPerSecond = duration <= 120 ? 1.5 : duration <= 600 ? 1 : 0.75;
const sampleCount = Math.max(3, Math.ceil(duration * samplesPerSecond) + 1);
for (let sample = 0; sample < sampleCount; sample += 1) {
if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
const ratio = sampleCount === 1 ? 0 : sample / (sampleCount - 1);
video.currentTime = Math.min(Math.max(0, video.duration - 0.05), sourceStart + ratio * sourceDuration);
await waitForMedia(video, "seeked");
drawContainedFrame(context, video, video.videoWidth, video.videoHeight);
const pixels = context.getImageData(0, 0, canvas.width, canvas.height).data;
const blob = await canvasBlob(canvas);
const metrics = await flowWorker.analyze(pixels, canvas.width, canvas.height, segment.id);
frames.push({ segmentId: segment.id, segmentIndex, segmentName: segment.name || `Clip ${segmentIndex + 1}`, segmentStart: timelineStart, segmentEnd: timelineStart + duration, time: timelineStart + ratio * duration, ...metrics, blob, aspectRatio: getAspectRatioLabel(video.videoWidth, video.videoHeight) });
}
video.removeAttribute("src"); video.load();
} else if (segment.src) {
const image = new Image(); image.src = segment.src;
if (!image.complete) await waitForMedia(image, "load");
drawContainedFrame(context, image, image.naturalWidth, image.naturalHeight);
frames.push({ segmentId: segment.id, segmentIndex, segmentName: segment.name || `Clip ${segmentIndex + 1}`, segmentStart: timelineStart, segmentEnd: timelineStart + duration, time: timelineStart, difference: 1, blob: await canvasBlob(canvas), aspectRatio: getAspectRatioLabel(image.naturalWidth, image.naturalHeight) });
}
video.removeAttribute("src"); video.load();
} else if (segment.src) {
const image = new Image(); image.src = segment.src;
if (!image.complete) await waitForMedia(image, "load");
context.drawImage(image, 0, 0, canvas.width, canvas.height);
frames.push({ segmentId: segment.id, time: timelineStart, difference: 1, blob: await canvasBlob(canvas) });
timelineStart += duration;
onProgress(Math.round(((segmentIndex + 1) / segments.length) * 55));
}
timelineStart += duration;
onProgress(Math.round(((segmentIndex + 1) / segments.length) * 55));
return selectCandidatesBySegment(frames);
} finally {
flowWorker.terminate();
}
return selectChangedFrames(frames);
}
export async function generateFrameCaptions({ frames, duration, language, onDownloadProgress }) {
export function createFrameCaptionSession({ language, onDownloadProgress, signal }) {
const modelLanguage = getAutoEditLanguage(language);
const options = {
expectedInputs: [{ type: "text", languages: ["en"] }, { type: "image" }],
@@ -122,15 +228,89 @@ export async function generateFrameCaptions({ frames, duration, language, onDown
monitor(monitor) {
monitor.addEventListener("downloadprogress", (event) => onDownloadProgress?.(event.loaded));
},
signal,
};
const session = await window.LanguageModel.create(options);
return window.LanguageModel.create(options);
}
async function generateCaptionGroup(session, frames, duration, modelLanguage) {
const content = [{ type: "text", value: `Describe every provided candidate frame with one concise on-screen caption. Output in ${modelLanguage}. Return exactly ${frames.length} captions in the same order as the images. Use only visible evidence; do not invent names or facts. Frame timestamps: ${frames.map((frame) => frame.time.toFixed(2)).join(", ")} seconds.` }];
frames.forEach((frame) => content.push({ type: "image", value: frame.blob }));
const schema = { type: "object", properties: { captions: { type: "array", minItems: frames.length, maxItems: frames.length, items: { type: "object", properties: { text: { type: "string", minLength: 1 } }, required: ["text"], additionalProperties: false } } }, required: ["captions"], additionalProperties: false };
const response = await session.prompt([{ role: "user", content }], { responseConstraint: schema });
const returned = JSON.parse(response)?.captions;
const descriptions = Array.isArray(returned) ? returned.map((item) => String(item?.text || "").trim()) : [];
const singleSchema = { type: "object", properties: { text: { type: "string", minLength: 1 } }, required: ["text"], additionalProperties: false };
for (let index = 0; index < frames.length; index += 1) {
if (descriptions[index]) continue;
const frame = frames[index];
const singleResponse = await session.prompt([{ role: "user", content: [
{ type: "text", value: `Write one concise ${modelLanguage} on-screen caption describing only what is visibly happening in this video frame at ${frame.time.toFixed(2)} seconds. Return a meaningful non-empty caption. Do not mention the timestamp.` },
{ type: "image", value: frame.blob },
] }], { responseConstraint: singleSchema });
descriptions[index] = String(JSON.parse(singleResponse)?.text || "").trim();
}
return frames.map((frame, index) => ({
id: makeId("caption"),
text: descriptions[index],
start: frame.time,
end: Math.min(duration, Math.max(frame.time + 1.2, frames[index + 1]?.time ?? frame.segmentEnd ?? duration)),
hidden: false,
})).filter((caption) => caption.text);
}
function createSlidingWindows(frames, size = 6, overlap = 2) {
if (!frames.length) return [];
const stride = Math.max(1, size - overlap);
const windows = [];
for (let start = 0; start < frames.length; start += stride) {
windows.push({ frames: frames.slice(start, start + size), start, commitEnd: frames[start + stride]?.time ?? Infinity });
if (start + size >= frames.length) break;
}
return windows;
}
export async function generateFrameCaptions({ frames, duration, language, session: providedSession, onDownloadProgress, onPartial }) {
const modelLanguage = getAutoEditLanguage(language);
const session = providedSession || await createFrameCaptionSession({ language, onDownloadProgress });
const groups = [];
frames.forEach((frame) => {
let group = groups.find((item) => item.segmentId === frame.segmentId);
if (!group) { group = { segmentId: frame.segmentId, segmentIndex: frame.segmentIndex, segmentName: frame.segmentName, frames: [] }; groups.push(group); }
group.frames.push(frame);
});
const allCaptions = [];
const windowsBySegment = new Map(groups.map((group) => [group.segmentId, createSlidingWindows(group.frames)]));
const totalWindows = [...windowsBySegment.values()].reduce((sum, windows) => sum + windows.length, 0);
let completedWindows = 0;
try {
const content = [{ type: "text", value: `Create concise on-screen captions for these chronological video frames. Output in ${modelLanguage}. Use only visible evidence, do not invent names or facts. Frame timestamps: ${frames.map((frame) => frame.time.toFixed(2)).join(", ")} seconds. Keep each caption on screen for 1.5 to 4 seconds and within 0 to ${duration.toFixed(2)} seconds.` }];
frames.forEach((frame) => content.push({ type: "image", value: frame.blob }));
const schema = { type: "object", properties: { captions: { type: "array", items: { type: "object", properties: { start: { type: "number" }, end: { type: "number" }, text: { type: "string" } }, required: ["start", "end", "text"], additionalProperties: false } } }, required: ["captions"], additionalProperties: false };
const response = await session.prompt([{ role: "user", content }], { responseConstraint: schema });
return normalizeGeneratedCaptions(JSON.parse(response), duration);
for (const group of groups) {
const windows = windowsBySegment.get(group.segmentId) || [];
const clipStart = group.frames[0]?.segmentStart ?? 0;
const clipEnd = group.frames[0]?.segmentEnd ?? duration;
let groupCaptions = [];
onPartial?.({ segmentId: group.segmentId, status: "running", captions: [], windowIndex: 0, totalWindows: windows.length, completedWindows, allWindows: totalWindows });
try {
for (let windowIndex = 0; windowIndex < windows.length; windowIndex += 1) {
const window = windows[windowIndex];
const generated = await generateCaptionGroup(session, window.frames, duration, modelLanguage);
// Overlap gives the model context; each window commits only its new time region.
const committed = generated.filter((caption) => (caption.start + caption.end) / 2 < window.commitEnd || windowIndex === windows.length - 1);
groupCaptions.push(...committed);
completedWindows += 1;
const partialCaptions = normalizeClipCaptionTimings(groupCaptions, clipStart, clipEnd).map((caption) => ({ ...caption, visualSegmentId: group.segmentId, visualSegmentIndex: group.segmentIndex, visualSegmentName: group.segmentName }));
onPartial?.({ segmentId: group.segmentId, status: "running", captions: partialCaptions, windowIndex: windowIndex + 1, totalWindows: windows.length, completedWindows, allWindows: totalWindows });
}
const timedCaptions = normalizeClipCaptionTimings(groupCaptions, clipStart, clipEnd);
const captions = timedCaptions.map((caption) => ({ ...caption, visualSegmentId: group.segmentId, visualSegmentIndex: group.segmentIndex, visualSegmentName: group.segmentName }));
allCaptions.push(...captions);
onPartial?.({ segmentId: group.segmentId, status: captions.length ? "complete" : "empty", captions, windowIndex: windows.length, totalWindows: windows.length, completedWindows, allWindows: totalWindows });
} catch (error) {
onPartial?.({ segmentId: group.segmentId, status: "error", captions: [], error: error?.message || String(error) });
}
}
return allCaptions.sort((a, b) => a.start - b.start);
} finally {
session.destroy?.();
if (!providedSession) session.destroy?.();
}
}
+69 -2
View File
@@ -1,14 +1,81 @@
import { describe, expect, it } from "vitest";
import { getAutoEditLanguage, normalizeGeneratedCaptions, selectChangedFrames } from "./autoEdit.js";
import { describe, expect, it, vi } from "vitest";
import { generateFrameCaptions, getAdaptiveSceneThreshold, getAspectRatioLabel, getAutoEditLanguage, normalizeClipCaptionTimings, normalizeGeneratedCaptions, selectCandidatesBySegment, selectChangedFrames } from "./autoEdit.js";
describe("auto edit", () => {
it("keeps scene changes and clip boundaries", () => {
const frames = [{ segmentId: "a", time: 0, difference: 1 }, { segmentId: "a", time: 1, difference: .03 }, { segmentId: "a", time: 2, difference: .3 }, { segmentId: "b", time: 3, difference: .01 }];
expect(selectChangedFrames(frames).map((frame) => frame.time)).toEqual([0, 2, 3]);
});
it("keeps candidates for every visual segment", () => {
const frames = [
{ segmentId: "a", time: 0, difference: 1 }, { segmentId: "a", time: 2, difference: .4 },
{ segmentId: "b", time: 4, difference: 1 }, { segmentId: "b", time: 6, difference: .5 },
];
expect(selectCandidatesBySegment(frames, 1).map((frame) => frame.segmentId)).toEqual(["a", "b"]);
});
it("derives the scene threshold from the clip's own motion distribution", () => {
const quiet = [{ difference: 1 }, ...Array.from({ length: 8 }, () => ({ difference: .04 })), { difference: .35 }];
expect(getAdaptiveSceneThreshold(quiet)).toBeLessThan(.35);
});
it("returns every separated scene peak without a fixed candidate cap", () => {
const frames = Array.from({ length: 31 }, (_, index) => ({
segmentId: "a", time: index * 2, difference: index > 0 && index % 3 === 0 ? .75 : .03, quality: .9,
}));
expect(selectCandidatesBySegment(frames)).toHaveLength(11);
});
it("does not force a low-quality end card into the candidates", () => {
const frames = [
{ segmentId: "a", time: 0, difference: 1, quality: .9 },
{ segmentId: "a", time: 2, difference: .65, quality: .9 },
{ segmentId: "a", time: 5, difference: .5, quality: .8 },
{ segmentId: "a", time: 10, difference: .9, quality: .12 },
];
expect(selectChangedFrames(frames, { maxFrames: 3 }).map((frame) => frame.time)).toEqual([0, 2, 5]);
});
it("falls back to an officially supported output language", () => expect(getAutoEditLanguage("zh")).toBe("en"));
it("labels common source aspect ratios", () => {
expect(getAspectRatioLabel(1080, 1920)).toBe("9:16");
expect(getAspectRatioLabel(1920, 1080)).toBe("16:9");
});
it("clamps and sorts generated captions", () => {
const result = normalizeGeneratedCaptions({ captions: [{ start: 4, end: 9, text: "B" }, { start: -2, end: 1, text: "A" }] }, 5);
expect(result.map(({ text, start, end }) => ({ text, start, end }))).toEqual([{ text: "A", start: 0, end: 1 }, { text: "B", start: 4, end: 5 }]);
});
it("removes end-of-clip overlap and keeps the last caption readable", () => {
const result = normalizeClipCaptionTimings([
{ id: "a", start: 0, end: 1.37 },
{ id: "b", start: 1.37, end: 15.06 },
{ id: "c", start: 14.86, end: 15.06 },
], 0, 15.06);
expect(result[2].start).toBeCloseTo(13.86, 5);
expect(result[2].end).toBeCloseTo(15.06, 5);
expect(result[1].end).toBeCloseTo(13.86, 5);
expect(result[0].end).toBeLessThanOrEqual(result[1].start);
});
it("falls back to per-frame descriptions when the batch is empty", async () => {
const onPartial = vi.fn();
const session = {
prompt: vi.fn()
.mockResolvedValueOnce('{"captions":[]}')
.mockResolvedValueOnce('{"text":"A lantern-lit interior"}')
.mockResolvedValueOnce('{"text":"A person enters the room"}'),
};
const result = await generateFrameCaptions({
frames: [{ segmentId: "clip-a", time: 0, blob: {} }, { segmentId: "clip-a", time: 3, blob: {} }], duration: 6, language: "en", session, onPartial,
});
expect(result.map(({ text, start, end }) => ({ text, start, end }))).toEqual([
{ text: "A lantern-lit interior", start: 0, end: 3 },
{ text: "A person enters the room", start: 3, end: 6 },
]);
expect(session.prompt).toHaveBeenCalledTimes(3);
expect(onPartial.mock.calls.map(([value]) => value.status)).toEqual(["running", "running", "complete"]);
expect(onPartial.mock.calls[1][0]).toMatchObject({ windowIndex: 1, totalWindows: 1 });
});
it("returns one model result for every candidate frame", async () => {
const session = { prompt: vi.fn().mockResolvedValue('{"captions":[{"text":"A"},{"text":"B"},{"text":"C"},{"text":"D"}]}') };
const frames = Array.from({ length: 4 }, (_, index) => ({ segmentId: "clip-a", segmentStart: 0, segmentEnd: 8, time: index * 2, blob: {} }));
const result = await generateFrameCaptions({ frames, duration: 8, language: "en", session });
expect(result.map(({ text }) => text)).toEqual(["A", "B", "C", "D"]);
expect(result).toHaveLength(frames.length);
});
});
+8 -7
View File
@@ -9,6 +9,7 @@ const CAPTION_MAX_WIDTH = 680;
const CAPTION_RADIUS = 7;
const CAPTION_SHADOW_BLUR = 6;
const CAPTION_SHADOW_OFFSET_Y = 1;
export const CAPTION_DESIGN_SHORT_EDGE = 360;
export const CAPTION_FONT_FAMILY =
'Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", "PingFang SC", "Microsoft YaHei", sans-serif';
@@ -29,24 +30,24 @@ function clamp(value, minimum, maximum) {
return Math.max(minimum, Math.min(maximum, value));
}
export function getCaptionScale(referenceFrame, renderFrame) {
const reference = normalizeFrameSize(referenceFrame);
export function getCaptionScale(_referenceFrame, renderFrame) {
const render = normalizeFrameSize(renderFrame);
if (!reference.height || !render.height) {
const renderShortEdge = Math.min(render.width, render.height);
if (!renderShortEdge) {
return 1;
}
return render.height / reference.height;
return renderShortEdge / CAPTION_DESIGN_SHORT_EDGE;
}
export function resolveCaptionMetrics({
captionSize = 12,
captionSize = 14,
captionStyle = {},
referenceFrame,
renderFrame,
} = {}) {
const frame = normalizeFrameSize(renderFrame);
const scale = getCaptionScale(referenceFrame, frame);
const fontSize = Math.max(1, toPositiveNumber(captionSize, 12) * scale);
const fontSize = Math.max(1, toPositiveNumber(captionSize, 14) * scale);
const paddingX = toPositiveNumber(captionStyle.paddingX, CAPTION_PADDING_X) * scale;
const paddingY = toPositiveNumber(captionStyle.paddingY, CAPTION_PADDING_Y) * scale;
const minWidth = frame.width * CAPTION_MIN_WIDTH_RATIO;
@@ -172,7 +173,7 @@ function getFallbackTextWidth(text, fontSize) {
export function getCaptionTextLayout({
context = getCaptionMeasurementContext(),
text = "",
captionSize = 12,
captionSize = 14,
captionStyle = {},
referenceFrame,
renderFrame,
+28
View File
@@ -0,0 +1,28 @@
import { describe, expect, it } from "vitest";
import { getCaptionScale, resolveCaptionMetrics } from "./captionLayout.js";
describe("caption design scaling", () => {
it("uses a readable 14px default without changing explicit project sizes", () => {
expect(resolveCaptionMetrics({ renderFrame: { width: 360, height: 640 } }).fontSize).toBe(14);
expect(resolveCaptionMetrics({ captionSize: 12, renderFrame: { width: 360, height: 640 } }).fontSize).toBe(12);
});
it("keeps 12px captions independent from the editor preview window size", () => {
const smallPreview = resolveCaptionMetrics({ captionSize: 12, renderFrame: { width: 276, height: 491 } });
const largePreview = resolveCaptionMetrics({ captionSize: 12, renderFrame: { width: 360, height: 640 } });
expect(smallPreview.fontSize).toBeCloseTo(9.2, 6);
expect(largePreview.fontSize).toBe(12);
});
it("maps the 360px design short edge consistently to vertical 1080p and 4K", () => {
expect(getCaptionScale(null, { width: 1080, height: 1920 })).toBe(3);
expect(resolveCaptionMetrics({ captionSize: 12, renderFrame: { width: 1080, height: 1920 } }).fontSize).toBe(36);
expect(resolveCaptionMetrics({ captionSize: 12, renderFrame: { width: 2160, height: 3840 } }).fontSize).toBe(72);
});
it("uses the same scale for landscape and portrait outputs with the same short edge", () => {
expect(getCaptionScale(null, { width: 3840, height: 2160 })).toBe(6);
expect(getCaptionScale(null, { width: 2160, height: 3840 })).toBe(6);
});
});
+12
View File
@@ -0,0 +1,12 @@
export function getExportDimensions(ratio, shortEdge) {
const sourceShortEdge = Math.min(ratio.width, ratio.height);
const scale = shortEdge / sourceShortEdge;
const even = (value) => Math.max(2, Math.round(value / 2) * 2);
return { width: even(ratio.width * scale), height: even(ratio.height * scale) };
}
export function getExportBitrate(resolution, quality, frameRate) {
const base = { 720: 5, 1080: 10, 1440: 18, 2160: 38 }[resolution] || 10;
const qualityScale = { standard: 0.65, high: 1, ultra: 1.45 }[quality] || 1;
return Math.round(base * qualityScale * (frameRate / 30) * 1_000_000);
}
+17
View File
@@ -0,0 +1,17 @@
import { describe, expect, it } from "vitest";
import { getExportDimensions } from "./exportSettings.js";
describe("export dimensions", () => {
it("uses the selected resolution as the short edge for landscape video", () => {
expect(getExportDimensions({ width: 16, height: 9 }, 2160)).toEqual({ width: 3840, height: 2160 });
});
it("produces full vertical 4K instead of 1216x2160", () => {
expect(getExportDimensions({ width: 9, height: 16 }, 2160)).toEqual({ width: 2160, height: 3840 });
});
it("keeps square exports square", () => {
expect(getExportDimensions({ width: 1, height: 1 }, 1080)).toEqual({ width: 1080, height: 1080 });
});
});
+20 -42
View File
@@ -16,7 +16,7 @@ import {
positionCaptionLayout,
} from "./captionLayout.js";
import { resolveVisionAnalysisAtTime } from "./vision.js";
import { getCaptionAvoidancePlacement, getSmartCropRect } from "./visualGeometry.js";
import { getCaptionAvoidancePlacement, getSmartCropRect, getVisualFitRect } from "./visualGeometry.js";
import {
getVisualMaskFeatherPixels,
getVisualMaskGeometry,
@@ -25,6 +25,7 @@ import {
resolveVisualTransform,
} from "./visualEffects.js";
import { resolveVisualClipAnimation } from "./visualClipAnimations.js";
import { getStickerRenderGeometry } from "./stickerGeometry.js";
export function getAudioRecordingFormat() {
if (typeof MediaRecorder === "undefined") {
@@ -63,7 +64,7 @@ export function getVideoTrackSampleCount(duration, maxFrames = VIDEO_TRACK_FRAME
return Math.max(1, Math.min(maxFrames, Math.ceil(safeDuration / targetStep)));
}
function seekVideoFrame(video, time) {
export function seekVideoFrame(video, time) {
const safeTime = Math.max(0, Math.min(time, Math.max(0, (video.duration || time) - 0.04)));
if (video.readyState >= 2 && Math.abs(video.currentTime - safeTime) < 0.015) {
return new Promise((resolve) => window.requestAnimationFrame(resolve));
@@ -263,7 +264,7 @@ export function downloadBlob(blob, filename) {
window.setTimeout(() => URL.revokeObjectURL(url), 800);
}
function loadImage(src) {
export function loadImage(src) {
return new Promise((resolve, reject) => {
const image = new Image();
image.crossOrigin = "anonymous";
@@ -273,7 +274,7 @@ function loadImage(src) {
});
}
function createTemporalMaskCache(urls, maxEntries = 8) {
export function createTemporalMaskCache(urls, maxEntries = 8) {
const orderedUrls = Array.from(new Set(urls.filter(Boolean)));
const urlIndexes = new Map(orderedUrls.map((url, index) => [url, index]));
const entries = new Map();
@@ -363,7 +364,7 @@ function createTemporalMaskCache(urls, maxEntries = 8) {
};
}
function loadVideo(src) {
export function loadVideo(src) {
return new Promise((resolve, reject) => {
const video = document.createElement("video");
video.crossOrigin = "anonymous";
@@ -376,10 +377,10 @@ function loadVideo(src) {
});
}
function getVisualDimensions(visual) {
export function getVisualDimensions(visual) {
return {
width: visual.videoWidth || visual.naturalWidth || 1,
height: visual.videoHeight || visual.naturalHeight || 1,
width: visual.videoWidth || visual.naturalWidth || visual.displayWidth || visual.width || 1,
height: visual.videoHeight || visual.naturalHeight || visual.displayHeight || visual.height || 1,
};
}
@@ -473,7 +474,7 @@ function drawVisualUsingLayout(context, visual, layout, isMask = false) {
function drawFittedVisual(context, visual, canvas, fitMode, filter, vision = null) {
const { width, height } = canvas;
const visualSize = getVisualDimensions(visual);
const smartCropEnabled = Boolean(vision?.options?.smartCrop && vision?.subject?.box);
const smartCropEnabled = Boolean(fitMode === "cover" && vision?.options?.smartCrop && vision?.subject?.box);
const smartCropRect = smartCropEnabled
? getSmartCropRect(visualSize, canvas, vision.subject.box, { padding: 0.14 })
: null;
@@ -488,27 +489,12 @@ function drawFittedVisual(context, visual, canvas, fitMode, filter, vision = nul
outputSize: { width, height },
};
} else {
const imageRatio = visualSize.width / visualSize.height;
const canvasRatio = width / height;
const cover = fitMode === "cover";
let drawWidth;
let drawHeight;
if (cover ? imageRatio > canvasRatio : imageRatio < canvasRatio) {
drawHeight = height;
drawWidth = height * imageRatio;
} else {
drawWidth = width;
drawHeight = width / imageRatio;
}
const x = (width - drawWidth) / 2;
const y = (height - drawHeight) / 2;
const fitRect = getVisualFitRect(visualSize, canvas, fitMode);
layout = {
sourceSize: visualSize,
smartCropRect: null,
drawRect: { x, y, width: drawWidth, height: drawHeight },
fitMode,
drawRect: { x: fitRect.x, y: fitRect.y, width: fitRect.width, height: fitRect.height },
fitMode: fitRect.fitMode,
outputSize: { width, height },
};
}
@@ -536,7 +522,7 @@ function drawFittedVisual(context, visual, canvas, fitMode, filter, vision = nul
return layout;
}
function drawPreviewFrame(context, visual, canvas, options) {
export function drawPreviewFrame(context, visual, canvas, options) {
const {
subtitle,
fitMode = "contain",
@@ -544,7 +530,7 @@ function drawPreviewFrame(context, visual, canvas, options) {
captionsEnabled = true,
captionPosition = "bottom",
captionPlacement = null,
captionSize = 12,
captionSize = 14,
captionStyle = {},
captionReferenceSize = null,
sticker = null,
@@ -682,20 +668,12 @@ function drawPreviewFrame(context, visual, canvas, options) {
visibleStickers.forEach((activeSticker, index) => {
const activeStickerImage = stickerImages[index] ?? (activeSticker === sticker ? stickerImage : null);
if (activeSticker?.src && activeStickerImage) {
const stickerRatio =
(activeStickerImage.naturalWidth || activeStickerImage.width || 1) /
(activeStickerImage.naturalHeight || activeStickerImage.height || 1);
const stickerScale = Math.max(0.2, Math.min(3, Number(activeSticker.scale) || 1));
const maxStickerSize = Math.min(width, height) * 0.22 * stickerScale;
const stickerWidth = stickerRatio >= 1 ? maxStickerSize : maxStickerSize * stickerRatio;
const stickerHeight = stickerRatio >= 1 ? maxStickerSize / stickerRatio : maxStickerSize;
const centerX = (Number.isFinite(activeSticker.x) ? activeSticker.x : 82) / 100 * width;
const centerY = (Number.isFinite(activeSticker.y) ? activeSticker.y : 20) / 100 * height;
const geometry = getStickerRenderGeometry(activeSticker, activeStickerImage, canvas);
context.save();
context.globalAlpha = Math.max(0, Math.min(1, Number.isFinite(activeSticker.opacity) ? activeSticker.opacity : 1));
context.translate(centerX, centerY);
context.rotate(((Number(activeSticker.rotation) || 0) * Math.PI) / 180);
context.drawImage(activeStickerImage, -stickerWidth / 2, -stickerHeight / 2, stickerWidth, stickerHeight);
context.globalAlpha = geometry.opacity;
context.translate(geometry.centerX, geometry.centerY);
context.rotate((geometry.rotation * Math.PI) / 180);
context.drawImage(activeStickerImage, -geometry.width / 2, -geometry.height / 2, geometry.width, geometry.height);
context.restore();
} else if (activeSticker?.text) {
context.fillStyle = "rgba(53, 240, 221, 0.92)";
+290
View File
@@ -0,0 +1,290 @@
import {
AudioBufferSource,
ALL_FORMATS,
BlobSource,
BufferTarget,
CanvasSink,
CanvasSource,
Input,
Mp4OutputFormat,
Output,
WebMOutputFormat,
} from "mediabunny";
import { registerAacEncoder } from "@mediabunny/aac-encoder";
import {
createTemporalMaskCache,
drawPreviewFrame,
loadImage,
loadVideo,
seekVideoFrame,
} from "./media.js";
import {
createCaptionSegments,
getSegmentIndexAtTime,
getVisualSegmentIndexAtTime,
getVisualSegmentTimeline,
} from "./timeline.js";
import { resolveVisionAnalysisAtTime } from "./vision.js";
import { getVisualSourceTime } from "./visualEffects.js";
const clamp = (value, min, max) => Math.max(min, Math.min(max, value));
let aacFallbackRegistered = false;
export function createOfflineFramePlan(duration, frameRate) {
const fps = clamp(Math.round(Number(frameRate) || 30), 24, 60);
const safeDuration = Math.max(1 / fps, Number(duration) || 0);
const frameCount = Math.max(1, Math.ceil(safeDuration * fps));
return Array.from({ length: frameCount }, (_, index) => ({
index,
timestamp: index / fps,
duration: 1 / fps,
keyFrame: index % (fps * 2) === 0,
}));
}
export function getOfflineExportCodec(settings = {}) {
if (settings.codec === "vp8") return { video: "vp8", audio: "opus", extension: "webm", mimeType: "video/webm" };
if (settings.codec === "vp9") return { video: "vp9", audio: "opus", extension: "webm", mimeType: "video/webm" };
return { video: "avc", audio: "aac", extension: "mp4", mimeType: "video/mp4" };
}
export function getOfflineStickersAtTime(stickerSegments = [], sticker = null, time = 0) {
if (!stickerSegments.length) return sticker ? [sticker] : [];
return stickerSegments.filter((item) => time >= item.start && time < item.start + item.duration);
}
async function decodeAudioInputs(inputs) {
if (!inputs.length) return [];
const AudioContextClass = window.AudioContext || window.webkitAudioContext;
if (!AudioContextClass) throw new Error("当前浏览器不支持音频解码。");
const context = new AudioContextClass();
const cache = new Map();
try {
return await Promise.all(inputs.map(async (input) => {
if (!cache.has(input.blob)) {
cache.set(input.blob, input.blob.arrayBuffer().then((data) => context.decodeAudioData(data.slice(0))));
}
return { ...input, decoded: await cache.get(input.blob) };
}));
} finally {
await context.close().catch(() => {});
}
}
export async function mixOfflineAudio({
duration,
voiceAudioSegments = [],
sourceAudioBlob = null,
sourceAudioSegments = [],
sourceAudioVolume = 1,
sourceAudioStart = 0,
musicBlob = null,
musicVolume = 0.35,
musicStart = 0,
}) {
const inputs = [
...voiceAudioSegments.filter((item) => item.blob).map((item) => ({
blob: item.blob, start: Math.max(0, item.start || 0), volume: item.volume ?? 1,
sourceOffset: 0, sourceDuration: 0, playbackRate: 1,
fadeIn: Math.max(0, item.fadeIn || 0), fadeOut: Math.max(0, item.fadeOut || 0),
})),
...(sourceAudioBlob && sourceAudioSegments.length ? sourceAudioSegments.map((item) => ({
blob: sourceAudioBlob, start: Math.max(0, item.start || 0), volume: sourceAudioVolume,
sourceOffset: Math.max(0, item.sourceStart || 0), sourceDuration: Math.max(0, item.sourceDuration || 0),
playbackRate: clamp(Number(item.playbackRate) || 1, 0.25, 4), fadeIn: 0, fadeOut: 0,
})) : sourceAudioBlob ? [{ blob: sourceAudioBlob, start: Math.max(0, sourceAudioStart), volume: sourceAudioVolume, sourceOffset: 0, sourceDuration: 0, playbackRate: 1, fadeIn: 0, fadeOut: 0 }] : []),
...(musicBlob ? [{ blob: musicBlob, start: Math.max(0, musicStart), volume: musicVolume, sourceOffset: 0, sourceDuration: 0, playbackRate: 1, fadeIn: 0, fadeOut: 0 }] : []),
];
if (!inputs.length) return null;
const decoded = await decodeAudioInputs(inputs);
const sampleRate = 48_000;
const OfflineContextClass = window.OfflineAudioContext || window.webkitOfflineAudioContext;
if (!OfflineContextClass) throw new Error("当前浏览器不支持离线音频混合。");
const context = new OfflineContextClass(2, Math.ceil(Math.max(0.01, duration) * sampleRate), sampleRate);
decoded.forEach((input) => {
const source = context.createBufferSource();
const gain = context.createGain();
source.buffer = input.decoded;
source.playbackRate.value = input.playbackRate;
const offset = Math.min(input.decoded.duration, input.sourceOffset);
const available = Math.max(0, input.decoded.duration - offset);
const sourceDuration = Math.min(available, input.sourceDuration || available);
const outputDuration = sourceDuration / input.playbackRate;
gain.gain.setValueAtTime(input.volume, input.start);
if (input.fadeIn > 0) {
gain.gain.setValueAtTime(0, input.start);
gain.gain.linearRampToValueAtTime(input.volume, input.start + Math.min(input.fadeIn, outputDuration));
}
if (input.fadeOut > 0) {
const fadeStart = input.start + Math.max(0, outputDuration - input.fadeOut);
gain.gain.setValueAtTime(input.volume, fadeStart);
gain.gain.linearRampToValueAtTime(0, input.start + outputDuration);
}
source.connect(gain).connect(context.destination);
source.start(input.start, offset, sourceDuration);
});
return context.startRendering();
}
async function prepareComposition(options) {
const segments = options.visualSegments.some((segment) => segment.src)
? options.visualSegments.filter((segment) => segment.src)
: [{ id: "offline-visual", src: options.imageSrc, type: options.visualType, duration: options.duration }];
const timeline = getVisualSegmentTimeline(segments);
const items = await Promise.all(segments.map(async (segment, index) => {
const visual = segment.type === "video" ? await loadVideo(segment.src) : await loadImage(segment.src);
const cutoutVisual = segment.type === "image" && segment.vision?.options?.removeBackground && segment.vision?.cutoutUrl
? await loadImage(segment.vision.cutoutUrl).catch(() => null) : null;
const maskUrls = segment.type === "video" && segment.vision?.options?.removeBackground
? [...new Set((segment.vision.samples || []).map((sample) => sample.cutoutUrl).filter(Boolean))] : [];
let sequentialFrames = null;
if (segment.type === "video" && options.framePlan?.length) {
try {
const blob = segment.blob instanceof Blob ? segment.blob : await fetch(segment.src).then((response) => {
if (!response.ok) throw new Error(`Unable to read video source (${response.status})`);
return response.blob();
});
const input = new Input({ source: new BlobSource(blob), formats: ALL_FORMATS });
const track = await input.getPrimaryVideoTrack();
if (!track || !(await track.canDecode())) throw new Error("VideoDecoder does not support this source codec");
const sink = new CanvasSink(track, { poolSize: 3, decoderOptions: { optimizeForLatency: true } });
const range = timeline[index];
const timestamps = options.framePlan
.filter((frame) => frame.timestamp >= range.start && frame.timestamp < range.end)
.map((frame) => getVisualSourceTime(segment, frame.timestamp - range.start));
const iterator = sink.canvasesAtTimestamps(timestamps)[Symbol.asyncIterator]();
sequentialFrames = {
async next() {
const result = await iterator.next();
return result.done ? null : result.value?.canvas || null;
},
};
} catch (error) {
console.warn("Sequential WebCodecs video decode unavailable; using precise seek fallback", error);
}
}
return {
segment, visual, cutoutVisual,
temporalMaskCache: maskUrls.length ? createTemporalMaskCache(maskUrls) : null,
sequentialFrames,
decodeMode: segment.type === "video" ? (sequentialFrames ? "sequential-webcodecs" : "precise-seek") : "static-image",
};
}));
const stickerSources = [...new Set([
...options.stickerSegments.map((item) => item.src).filter(Boolean),
...(options.sticker?.src ? [options.sticker.src] : []),
])];
const stickerImages = new Map((await Promise.all(stickerSources.map(async (src) => [src, await loadImage(src).catch(() => null)]))).filter(([, image]) => image));
return { segments, timeline, items, stickerImages };
}
async function renderCompositionAt(context, canvas, prepared, options, time) {
const resolvedIndex = getVisualSegmentIndexAtTime(prepared.segments, time);
const index = resolvedIndex >= 0 ? resolvedIndex : Math.max(0, prepared.items.length - 1);
const item = prepared.items[index] || prepared.items[0];
const range = prepared.timeline[index] || prepared.timeline[0];
const localTime = Math.max(0, time - (range?.start || 0));
let sourceTime = getVisualSourceTime(item.segment, localTime);
let frameVisual = item.cutoutVisual || item.visual;
if (item.segment.type === "video" && item.sequentialFrames) {
frameVisual = await item.sequentialFrames.next() || item.visual;
} else if (item.segment.type === "video") {
sourceTime = Math.min(Math.max(0, (item.visual.duration || 0) - 0.04), sourceTime);
await seekVideoFrame(item.visual, sourceTime);
}
const vision = resolveVisionAnalysisAtTime(item.segment.vision || null, sourceTime);
if (vision?.cutoutUrl) await item.temporalMaskCache?.prepare(vision.cutoutUrl);
const frameVision = vision ? {
...vision,
options: item.segment.vision?.options || vision.options,
maskVisual: vision.cutoutUrl ? item.temporalMaskCache?.get(vision.cutoutUrl) : null,
} : null;
const junction = item.segment.transition;
const transitionDuration = junction?.id && junction.id !== "none"
? Math.min(Math.max(0.1, Number(junction.duration) || 0.5), Math.max(0, (range?.end || 0) - (range?.start || 0))) : 0;
const transitionProgress = transitionDuration && time >= range.end - transitionDuration
? clamp((time - (range.end - transitionDuration)) / transitionDuration, 0, 1) : 0;
const next = transitionProgress > 0 ? prepared.items[index + 1] : null;
if (next?.segment.type === "video") await seekVideoFrame(next.visual, getVisualSourceTime(next.segment, transitionProgress * transitionDuration));
const captionSegments = options.captionSegments?.length ? options.captionSegments : createCaptionSegments(options.text);
const captionIndex = getSegmentIndexAtTime(captionSegments, time, 0);
const caption = captionIndex >= 0 && !captionSegments[captionIndex]?.hidden ? captionSegments[captionIndex].text : "";
const stickers = getOfflineStickersAtTime(options.stickerSegments, options.sticker, time);
drawPreviewFrame(context, frameVisual, canvas, {
subtitle: caption, fitMode: options.fitMode, filter: options.filter,
captionsEnabled: options.captionsEnabled, captionPosition: options.captionPosition,
captionPlacement: options.captionPlacement, captionSize: options.captionSize,
captionStyle: options.captionStyle, captionReferenceSize: options.captionReferenceSize,
stickers, stickerImages: stickers.map((sticker) => prepared.stickerImages.get(sticker.src)),
transitionId: next ? junction.id : "none",
transitionNext: next ? { visual: next.cutoutVisual || next.visual } : null,
transitionProgress, vision: frameVision, visualEffects: item.segment, visualTime: localTime,
});
}
export async function exportOfflineVideo(options) {
if (typeof VideoEncoder === "undefined") throw new Error("当前浏览器不支持 WebCodecs 离线编码。");
const settings = options.exportSettings || {};
const width = Math.max(2, Math.round(Number(settings.width) || options.ratio.width));
const height = Math.max(2, Math.round(Number(settings.height) || options.ratio.height));
const codec = getOfflineExportCodec(settings);
if (codec.audio === "aac" && !aacFallbackRegistered) {
registerAacEncoder();
aacFallbackRegistered = true;
}
const canvas = document.createElement("canvas");
canvas.width = width; canvas.height = height;
const context = canvas.getContext("2d", { alpha: false, desynchronized: false });
if (!context) throw new Error("无法创建离线导出画布。");
options.onProgress?.({ progress: 4, phase: "准备离线渲染" });
const frames = createOfflineFramePlan(options.duration, settings.frameRate);
const preparedPromise = prepareComposition({ ...options, framePlan: frames });
const audioPromise = mixOfflineAudio({ ...options, duration: frames.length * frames[0].duration });
const [prepared, audioBuffer] = await Promise.all([preparedPromise, audioPromise]);
const target = new BufferTarget();
const output = new Output({ format: codec.extension === "mp4" ? new Mp4OutputFormat({ fastStart: "in-memory" }) : new WebMOutputFormat(), target });
let encoderConfig = null;
const videoSource = new CanvasSource(canvas, {
codec: codec.video,
bitrate: Math.max(2_000_000, Number(settings.videoBitsPerSecond) || 12_000_000),
keyFrameInterval: 2,
latencyMode: "quality",
onEncoderConfig: (config) => { encoderConfig = config; },
});
output.addVideoTrack(videoSource, { frameRate: frames.length / Math.max(options.duration, 1 / frames[0].duration) });
let audioSource = null;
if (audioBuffer) {
audioSource = new AudioBufferSource({ codec: codec.audio, bitrate: codec.audio === "aac" ? 256_000 : 192_000 });
output.addAudioTrack(audioSource);
}
await output.start();
if (audioSource) await audioSource.add(audioBuffer);
try {
for (const frame of frames) {
await renderCompositionAt(context, canvas, prepared, options, frame.timestamp);
await videoSource.add(frame.timestamp, frame.duration, { keyFrame: frame.keyFrame });
if (frame.index % Math.max(1, Math.round(frames.length / 100)) === 0) {
options.onProgress?.({ progress: 10 + Math.round((frame.index / frames.length) * 84), phase: `离线渲染 ${frame.index + 1} / ${frames.length}` });
}
}
await output.finalize();
} catch (error) {
await output.cancel().catch(() => {});
throw error;
} finally {
prepared.items.forEach((item) => {
item.temporalMaskCache?.dispose();
if (item.segment.type === "video") { item.visual.removeAttribute("src"); item.visual.load(); }
});
}
options.onProgress?.({ progress: 98, phase: "验证导出文件" });
return {
blob: new Blob([target.buffer], { type: codec.mimeType }), extension: codec.extension,
label: codec.extension === "mp4" ? "MP4" : "WebM", mimeType: codec.mimeType,
nativeMp4: codec.extension === "mp4", diagnostics: {
width, height, frameCount: frames.length, frameRate: settings.frameRate, encoderConfig,
videoDecodeModes: prepared.items.map((item) => item.decodeMode),
},
};
}
+45
View File
@@ -0,0 +1,45 @@
import { describe, expect, it } from "vitest";
import {
createOfflineFramePlan,
getOfflineExportCodec,
getOfflineStickersAtTime,
} from "./offlineVideoExport.js";
import { getVisualDimensions } from "./media.js";
describe("offline video export", () => {
it("creates deterministic frame timestamps without wall-clock drift", () => {
const frames = createOfflineFramePlan(1, 30);
expect(frames).toHaveLength(30);
expect(frames[0]).toMatchObject({ index: 0, timestamp: 0, duration: 1 / 30, keyFrame: true });
expect(frames[29].timestamp).toBeCloseTo(29 / 30, 8);
});
it("uses stable two-second keyframe intervals", () => {
const frames = createOfflineFramePlan(4.1, 30);
expect(frames.filter((frame) => frame.keyFrame).map((frame) => frame.index)).toEqual([0, 60, 120]);
});
it("maps codecs directly to their final container without a lossy intermediate", () => {
expect(getOfflineExportCodec({ codec: "h264" })).toMatchObject({ video: "avc", audio: "aac", extension: "mp4" });
expect(getOfflineExportCodec({ codec: "vp9" })).toMatchObject({ video: "vp9", audio: "opus", extension: "webm" });
});
it("resolves overlapping sticker clips at exact timeline timestamps", () => {
const stickers = [
{ id: "a", start: 0, duration: 2 },
{ id: "b", start: 1, duration: 2 },
];
expect(getOfflineStickersAtTime(stickers, null, 1.5).map((item) => item.id)).toEqual(["a", "b"]);
expect(getOfflineStickersAtTime(stickers, null, 2).map((item) => item.id)).toEqual(["b"]);
});
it("renders no sticker after the final timeline sticker is deleted", () => {
expect(getOfflineStickersAtTime([], null, 1)).toEqual([]);
});
it("preserves decoded canvas dimensions instead of treating frames as square", () => {
expect(getVisualDimensions({ width: 1920, height: 1080 })).toEqual({ width: 1920, height: 1080 });
expect(getVisualDimensions({ videoWidth: 1280, videoHeight: 720, width: 1, height: 1 })).toEqual({ width: 1280, height: 720 });
});
});
+21
View File
@@ -0,0 +1,21 @@
export const STICKER_FRAME_RATIO = 0.22;
export function getStickerBaseSize(frame) {
return Math.max(1, Math.min(frame.width, frame.height) * STICKER_FRAME_RATIO);
}
export function getStickerRenderGeometry(sticker, image, frame) {
const imageWidth = image?.naturalWidth || image?.videoWidth || image?.width || 1;
const imageHeight = image?.naturalHeight || image?.videoHeight || image?.height || 1;
const ratio = imageWidth / imageHeight;
const scale = Math.max(0.2, Math.min(3, Number(sticker?.scale) || 1));
const boxSize = getStickerBaseSize(frame) * scale;
return {
width: ratio >= 1 ? boxSize : boxSize * ratio,
height: ratio >= 1 ? boxSize / ratio : boxSize,
centerX: (Number.isFinite(sticker?.x) ? sticker.x : 82) / 100 * frame.width,
centerY: (Number.isFinite(sticker?.y) ? sticker.y : 20) / 100 * frame.height,
rotation: Number(sticker?.rotation) || 0,
opacity: Math.max(0, Math.min(1, Number.isFinite(sticker?.opacity) ? sticker.opacity : 1)),
};
}
+24
View File
@@ -0,0 +1,24 @@
import { describe, expect, it } from "vitest";
import { getStickerBaseSize, getStickerRenderGeometry } from "./stickerGeometry.js";
describe("sticker geometry", () => {
it("uses the same short-edge ratio for portrait preview and 4K export", () => {
const preview = { width: 276, height: 491 };
const output = { width: 2160, height: 3840 };
expect(getStickerBaseSize(preview) / preview.width).toBeCloseTo(0.22, 8);
expect(getStickerBaseSize(output) / output.width).toBeCloseTo(0.22, 8);
});
it("preserves the sticker image aspect ratio while scaling", () => {
const geometry = getStickerRenderGeometry(
{ x: 70, y: 20, scale: 1.5, rotation: 12, opacity: 0.8 },
{ naturalWidth: 200, naturalHeight: 100 },
{ width: 2160, height: 3840 },
);
expect(geometry.width / geometry.height).toBeCloseTo(2, 8);
expect(geometry.width / 2160).toBeCloseTo(0.33, 8);
expect(geometry.centerX).toBe(1512);
expect(geometry.centerY).toBe(768);
});
});
+23 -11
View File
@@ -418,18 +418,9 @@ export function mapNormalizedBoxToFrame(box, sourceSize, frameSize, layout = {})
: null;
}
const fitMode = layout?.fitMode === "cover" ? "cover" : "contain";
const sourceAspectRatio = source.width / source.height;
const frameAspectRatio = frame.width / frame.height;
const useWidth =
fitMode === "cover"
? sourceAspectRatio < frameAspectRatio
: sourceAspectRatio > frameAspectRatio;
const drawWidth = useWidth ? frame.width : frame.height * sourceAspectRatio;
const drawHeight = useWidth ? frame.width / sourceAspectRatio : frame.height;
const position = getLayoutPosition(layout);
const drawX = (frame.width - drawWidth) * position.x;
const drawY = (frame.height - drawHeight) * position.y;
const fitRect = getVisualFitRect(source, frame, layout?.fitMode, position);
const { fitMode, x: drawX, y: drawY, width: drawWidth, height: drawHeight } = fitRect;
const raw = {
xMin: drawX + normalizedBox.xMin * drawWidth,
yMin: drawY + normalizedBox.yMin * drawHeight,
@@ -482,6 +473,27 @@ export function mapNormalizedBoxToFrame(box, sourceSize, frameSize, layout = {})
};
}
export function getVisualFitRect(sourceSize, frameSize, requestedFitMode = "contain", position = { x: 0.5, y: 0.5 }) {
const source = getSize(sourceSize);
const frame = getSize(frameSize);
if (!source.valid || !frame.valid) return { x: 0, y: 0, width: 0, height: 0, fitMode: "contain" };
const fitMode = requestedFitMode === "cover" ? "cover" : "contain";
const sourceAspectRatio = source.width / source.height;
const frameAspectRatio = frame.width / frame.height;
const useWidth = fitMode === "cover"
? sourceAspectRatio < frameAspectRatio
: sourceAspectRatio > frameAspectRatio;
const width = useWidth ? frame.width : frame.height * sourceAspectRatio;
const height = useWidth ? frame.width / sourceAspectRatio : frame.height;
return {
x: (frame.width - width) * (Number.isFinite(position?.x) ? position.x : 0.5),
y: (frame.height - height) * (Number.isFinite(position?.y) ? position.y : 0.5),
width,
height,
fitMode,
};
}
function normalizeFrameRect(rect, frameSize) {
if (rect?.normalized) {
return normalizeBoundingBox(rect.normalized);
+16
View File
@@ -2,11 +2,27 @@ import { describe, expect, it } from "vitest";
import {
getSmartCropRect,
getVisualFitRect,
normalizeBoundingBox,
selectPrimarySubject,
} from "./visualGeometry.js";
describe("visual geometry", () => {
it("contains a 9:16 source inside 16:9 with centered black side space", () => {
const rect = getVisualFitRect({ width: 1080, height: 1920 }, { width: 1920, height: 1080 }, "contain");
expect(rect.height).toBe(1080);
expect(rect.width).toBeCloseTo(607.5, 6);
expect(rect.x).toBeCloseTo(656.25, 6);
expect(rect.y).toBe(0);
});
it("only crops a portrait source when cover is explicitly requested", () => {
const rect = getVisualFitRect({ width: 1080, height: 1920 }, { width: 1920, height: 1080 }, "cover");
expect(rect.width).toBe(1920);
expect(rect.height).toBeCloseTo(3413.333333, 5);
expect(rect.y).toBeLessThan(0);
});
it("normalizes pixel coordinates", () => {
const box = normalizeBoundingBox(
{ xMin: 100, yMin: 50, xMax: 500, yMax: 450 },
+136 -14
View File
@@ -794,12 +794,12 @@ button:disabled {
.drop-zone {
display: grid;
grid-template-columns: 42px minmax(0, 1fr);
grid-template-columns: 32px minmax(0, 1fr);
align-items: center;
gap: 6px 14px;
min-height: 116px;
margin-top: 18px;
padding: 16px 18px;
gap: 2px 10px;
min-height: 78px;
margin-top: 12px;
padding: 10px 12px;
border: 1px dashed rgba(173, 184, 196, 0.24);
border-radius: 8px;
color: #99a5b1;
@@ -819,14 +819,20 @@ button:disabled {
}
.drop-zone span {
min-width: 0;
overflow: hidden;
color: #7c8793;
font-size: 12px;
line-height: 1.45;
font-size: 11px;
line-height: 1.35;
text-overflow: ellipsis;
white-space: nowrap;
}
.drop-zone svg {
grid-row: 1 / span 2;
justify-self: center;
width: 32px;
height: 32px;
color: #8c96a2;
}
@@ -948,7 +954,7 @@ button:disabled {
.asset-list {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
grid-template-columns: repeat(2, minmax(0, 1fr));
align-content: start;
gap: 10px;
margin-top: 14px;
@@ -1177,7 +1183,99 @@ button:disabled {
.auto-edit-progress { display: grid; gap: 8px; }
.auto-edit-progress > div { display: flex; justify-content: space-between; color: #9eacb4; font-size: 11px; }
.auto-edit-progress progress { width: 100%; accent-color: #35ead9; }
.auto-edit-panel .primary-action { width: 100%; }
.auto-edit-generate { position: relative; display: grid; grid-template-columns: 34px minmax(0,1fr) 22px; align-items: center; gap: 10px; width: 100%; min-height: 58px; overflow: hidden; border: 1px solid rgba(53,234,217,.72); border-radius: 11px; padding: 9px 12px; color: #061817; text-align: left; background: linear-gradient(120deg,#73b7ff 0%,#38e9d9 54%,#69f1c7 100%); box-shadow: 0 10px 28px rgba(38,217,200,.18), inset 0 1px 0 rgba(255,255,255,.35); cursor: pointer; transition: transform 160ms ease, box-shadow 160ms ease, filter 160ms ease; }
.auto-edit-generate::after { content:""; position:absolute; inset:-60% auto -60% -35%; width:34%; transform:rotate(18deg); background:linear-gradient(90deg,transparent,rgba(255,255,255,.28),transparent); transition:left 360ms ease; }
.auto-edit-generate:hover:not(:disabled) { transform: translateY(-1px); filter: brightness(1.04); box-shadow: 0 14px 34px rgba(38,217,200,.27), inset 0 1px 0 rgba(255,255,255,.42); }
.auto-edit-generate:hover:not(:disabled)::after { left:115%; }
.auto-edit-generate:active:not(:disabled) { transform: translateY(0); }
.auto-edit-generate:disabled { border-color: rgba(255,255,255,.1); color:#7d898e; background:rgba(255,255,255,.055); box-shadow:none; cursor:not-allowed; }
.auto-edit-generate-icon { display:grid; place-items:center; width:34px; height:34px; border-radius:9px; color:#07302d; background:rgba(255,255,255,.34); box-shadow:inset 0 0 0 1px rgba(255,255,255,.28); }
.auto-edit-generate:disabled .auto-edit-generate-icon { color:#68757b; background:rgba(255,255,255,.055); }
.auto-edit-generate > span:nth-child(2) { display:grid; gap:2px; min-width:0; }
.auto-edit-generate strong { overflow:hidden; font-size:12px; font-weight:800; text-overflow:ellipsis; white-space:nowrap; }
.auto-edit-generate small { overflow:hidden; color:rgba(5,38,35,.68); font-size:9px; font-weight:650; text-overflow:ellipsis; white-space:nowrap; }
.auto-edit-generate:disabled small { color:#66747a; }
.auto-edit-generate-arrow { font-size:18px; font-weight:700; transition:transform 160ms ease; }
.auto-edit-generate:hover:not(:disabled) .auto-edit-generate-arrow { transform:translateX(3px); }
.auto-edit-review-backdrop { position:fixed; inset:0; z-index:120; display:grid; place-items:center; padding:28px; background:rgba(3,7,11,.78); backdrop-filter:blur(9px); animation:auto-edit-backdrop-in 160ms ease-out; }
.auto-edit-review-dialog { display:grid; grid-template-rows:auto 3px minmax(0,1fr) auto; width:min(960px,calc(100vw - 56px)); height:min(720px,calc(100vh - 56px)); overflow:hidden; border:1px solid rgba(117,177,199,.25); border-radius:18px; color:#dce8eb; background:linear-gradient(145deg,rgba(18,25,33,.99),rgba(9,14,20,.99)); box-shadow:0 38px 120px rgba(0,0,0,.68),inset 0 1px 0 rgba(255,255,255,.05); animation:auto-edit-dialog-in 180ms ease-out; }
.auto-edit-review-header { display:grid; grid-template-columns:42px minmax(0,1fr) auto 34px; align-items:center; gap:12px; min-height:76px; padding:13px 18px; border-bottom:1px solid rgba(255,255,255,.07); }
.auto-edit-review-mark { display:grid; place-items:center; width:40px; height:40px; border:1px solid rgba(53,234,217,.4); border-radius:12px; color:#041817; background:linear-gradient(145deg,#71b8ff,#39ead8); box-shadow:0 8px 24px rgba(53,234,217,.16); }
.auto-edit-review-header > div:nth-child(2) { display:grid; gap:2px; }
.auto-edit-review-header > div:nth-child(2) > span { color:#5fddd1; font-size:9px; font-weight:750; letter-spacing:.15em; text-transform:uppercase; }
.auto-edit-review-header h2 { margin:0; color:#f2f8f9; font-size:17px; letter-spacing:-.02em; }
.auto-edit-review-status { display:flex; align-items:center; gap:7px; border:1px solid rgba(255,255,255,.08); border-radius:999px; padding:6px 10px; color:#91a0a8; font-size:10px; background:rgba(255,255,255,.035); }
.auto-edit-review-status i { width:6px; height:6px; border-radius:50%; background:#e5bd68; box-shadow:0 0 9px rgba(229,189,104,.7); animation:auto-edit-pulse 1.1s ease-in-out infinite; }
.auto-edit-review-status.is-complete { color:#78e6d9; border-color:rgba(53,234,217,.24); background:rgba(53,234,217,.07); }
.auto-edit-review-status.is-complete i { background:#35ead9; box-shadow:0 0 9px rgba(53,234,217,.8); animation:none; }
.auto-edit-review-status.is-error { color:#ffaaaa; border-color:rgba(255,104,117,.25); background:rgba(255,83,99,.07); }
.auto-edit-review-status.is-error i { background:#ff6875; animation:none; }
.auto-edit-review-close { display:grid; place-items:center; width:32px; height:32px; border:0; border-radius:8px; color:#80909a; background:transparent; cursor:pointer; }
.auto-edit-review-close:hover { color:#f3f8f9; background:rgba(255,255,255,.07); }
.auto-edit-review-progress { overflow:hidden; background:rgba(255,255,255,.04); }
.auto-edit-review-progress span { display:block; height:100%; border-radius:inherit; background:linear-gradient(90deg,#68aaff,#35ead9); box-shadow:0 0 16px rgba(53,234,217,.55); transition:width 220ms ease; }
.auto-edit-review-body { display:grid; grid-template-columns:minmax(0,1.16fr) minmax(320px,.84fr); gap:1px; min-height:0; overflow:hidden; background:rgba(255,255,255,.07); }
.auto-edit-review-section { min-width:0; overflow:auto; padding:19px; background:#0d131a; scrollbar-width:thin; scrollbar-color:#33434e transparent; }
.auto-edit-review-section-title { display:flex; align-items:center; justify-content:space-between; gap:12px; }
.auto-edit-review-section-title > div { display:flex; align-items:center; gap:9px; }
.auto-edit-review-section-title > div > span { display:grid; place-items:center; width:26px; height:26px; border:1px solid rgba(53,234,217,.24); border-radius:8px; color:#35ead9; font-size:9px; font-weight:800; background:rgba(53,234,217,.06); }
.auto-edit-review-section-title strong { color:#e8f1f3; font-size:13px; }
.auto-edit-review-section-title em { color:#6d7d87; font-size:9px; font-style:normal; }
.auto-edit-review-section > p { margin:7px 0 16px 35px; color:#6d7d87; font-size:10px; line-height:1.5; }
.auto-edit-candidate-grid { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:11px; }
.auto-edit-candidate-card { overflow:hidden; border:1px solid rgba(255,255,255,.08); border-radius:11px; background:rgba(255,255,255,.025); transition:border-color 150ms ease,transform 150ms ease; }
.auto-edit-candidate-card:hover { transform:translateY(-2px); border-color:rgba(53,234,217,.34); }
.auto-edit-candidate-card > div { position:relative; aspect-ratio:1/1; overflow:hidden; background:#000; }
.auto-edit-candidate-card img { width:100%; height:100%; object-fit:contain; }
.auto-edit-candidate-card > div span,.auto-edit-candidate-card > div time { position:absolute; top:8px; border:1px solid rgba(255,255,255,.12); border-radius:6px; padding:4px 6px; color:#eaf4f4; font-size:8px; background:rgba(5,10,15,.72); backdrop-filter:blur(5px); }
.auto-edit-candidate-card > div span { left:8px; color:#42e7d6; }
.auto-edit-candidate-card > div time { right:8px; font-variant-numeric:tabular-nums; }
.auto-edit-candidate-card > div em { position:absolute; right:8px; bottom:8px; border:1px solid rgba(255,255,255,.12); border-radius:6px; padding:4px 6px; color:#aebcc2; font-size:8px; font-style:normal; background:rgba(5,10,15,.72); backdrop-filter:blur(5px); }
.auto-edit-candidate-card footer { display:grid; grid-template-columns:auto auto; gap:6px; align-items:center; padding:8px 9px; color:#72818b; font-size:8px; }
.auto-edit-candidate-card footer strong { justify-self:end; color:#9cabb2; font-variant-numeric:tabular-nums; }
.auto-edit-candidate-card footer i { grid-column:1/-1; height:3px; overflow:hidden; border-radius:9px; background:rgba(255,255,255,.06); }
.auto-edit-candidate-card footer b { display:block; height:100%; border-radius:inherit; background:linear-gradient(90deg,#6aa9ff,#35ead9); }
.auto-edit-model-results { background:#0b1117; }
.auto-edit-result-list { display:grid; gap:8px; }
.auto-edit-result-list article { display:grid; grid-template-columns:28px minmax(0,1fr); gap:9px; padding:10px; border:1px solid rgba(255,255,255,.07); border-radius:9px; background:rgba(255,255,255,.025); }
.auto-edit-result-list article > span { display:grid; place-items:center; width:27px; height:27px; border-radius:7px; color:#76e5d9; font-size:9px; font-weight:750; background:rgba(53,234,217,.08); }
.auto-edit-result-list article div { display:grid; gap:5px; min-width:0; }
.auto-edit-result-list p { margin:0; color:#dce7ea; font-size:11px; line-height:1.45; }
.auto-edit-result-list time { color:#667984; font-size:8px; font-variant-numeric:tabular-nums; }
.auto-edit-clip-results { display:grid; gap:10px; }
.auto-edit-clip-result { overflow:hidden; border:1px solid rgba(255,255,255,.075); border-radius:11px; background:rgba(255,255,255,.02); }
.auto-edit-clip-result.is-running { border-color:rgba(229,189,104,.28); box-shadow:inset 3px 0 0 rgba(229,189,104,.65); }
.auto-edit-clip-result.is-complete { border-color:rgba(53,234,217,.2); box-shadow:inset 3px 0 0 rgba(53,234,217,.65); }
.auto-edit-clip-result.is-error { border-color:rgba(255,104,117,.22); box-shadow:inset 3px 0 0 rgba(255,104,117,.62); }
.auto-edit-clip-result > header { display:grid; grid-template-columns:52px minmax(0,1fr) auto; align-items:center; gap:9px; min-height:50px; padding:7px 9px; border-bottom:1px solid rgba(255,255,255,.055); background:rgba(255,255,255,.018); }
.auto-edit-clip-result > header img { width:52px; height:36px; border-radius:6px; object-fit:contain; background:#000; }
.auto-edit-clip-result > header div { display:grid; gap:3px; min-width:0; }
.auto-edit-clip-result > header strong { overflow:hidden; color:#dce7e9; font-size:10px; text-overflow:ellipsis; white-space:nowrap; }
.auto-edit-clip-result > header span { color:#667680; font-size:8px; }
.auto-edit-clip-result > header em { border:1px solid rgba(255,255,255,.08); border-radius:999px; padding:4px 7px; color:#7d8c95; font-size:8px; font-style:normal; white-space:nowrap; }
.auto-edit-clip-result.is-running > header em { color:#d9bd7d; border-color:rgba(229,189,104,.2); background:rgba(229,189,104,.06); }
.auto-edit-clip-result.is-complete > header em { color:#69dbcf; border-color:rgba(53,234,217,.2); background:rgba(53,234,217,.06); }
.auto-edit-clip-result .auto-edit-result-list { gap:6px; padding:8px; }
.auto-edit-clip-result .auto-edit-result-list article { border:0; padding:7px; background:rgba(255,255,255,.022); }
.auto-edit-clip-pending { display:flex; align-items:center; justify-content:center; gap:8px; min-height:62px; color:#61717b; font-size:9px; }
.auto-edit-clip-pending i { width:13px; height:13px; border:2px solid rgba(229,189,104,.15); border-top-color:#e5bd68; border-radius:50%; animation:auto-edit-spin .8s linear infinite; }
.auto-edit-clip-error { margin:0; padding:10px; color:#d18c94; font-size:9px; line-height:1.45; }
.auto-edit-review-loading { display:grid; place-items:center; align-content:center; gap:12px; min-height:220px; border:1px dashed rgba(255,255,255,.08); border-radius:11px; color:#667781; font-size:10px; background:rgba(255,255,255,.012); }
.auto-edit-review-loading i { width:25px; height:25px; border:2px solid rgba(53,234,217,.13); border-top-color:#35ead9; border-radius:50%; animation:auto-edit-spin .85s linear infinite; }
.auto-edit-review-error { display:grid; gap:5px; border:1px solid rgba(255,104,117,.2); border-radius:10px; padding:12px; color:#c4868d; font-size:10px; background:rgba(255,83,99,.055); }
.auto-edit-review-error strong { color:#ffadb4; font-size:11px; }
.auto-edit-review-actions { display:grid; grid-template-columns:minmax(0,1fr) auto auto; align-items:center; gap:10px; min-height:72px; padding:12px 18px; border-top:1px solid rgba(255,255,255,.07); background:rgba(8,13,18,.96); }
.auto-edit-review-actions > div { display:grid; gap:3px; }
.auto-edit-review-actions > div strong { color:#dfe9eb; font-size:11px; }
.auto-edit-review-actions > div span { color:#677883; font-size:9px; }
.auto-edit-review-actions .panel-secondary { min-width:78px; }
.auto-edit-apply { display:flex; align-items:center; justify-content:center; gap:7px; min-width:148px; min-height:38px; border:1px solid rgba(53,234,217,.7); border-radius:9px; color:#061817; font-size:11px; font-weight:800; background:linear-gradient(125deg,#69b6ff,#39ead7); box-shadow:0 8px 20px rgba(53,234,217,.15); cursor:pointer; }
.auto-edit-apply:disabled { border-color:rgba(255,255,255,.08); color:#5f6e75; background:rgba(255,255,255,.045); box-shadow:none; cursor:not-allowed; }
@keyframes auto-edit-spin { to { transform:rotate(360deg); } }
@keyframes auto-edit-pulse { 50% { opacity:.35; transform:scale(.75); } }
@keyframes auto-edit-backdrop-in { from { opacity:0; } }
@keyframes auto-edit-dialog-in { from { opacity:0; transform:translateY(10px) scale(.985); } }
@media (max-width:760px) { .auto-edit-review-body { grid-template-columns:1fr; overflow:auto; } .auto-edit-review-section { overflow:visible; } .auto-edit-review-dialog { height:calc(100vh - 24px); width:calc(100vw - 24px); } .auto-edit-review-actions { grid-template-columns:1fr 1fr; } .auto-edit-review-actions > div { grid-column:1/-1; } }
.smart-vision-heading {
display: flex;
@@ -2460,9 +2558,9 @@ button:disabled {
.preview-stage {
display: grid;
grid-template-rows: 1fr 94px;
grid-template-rows: minmax(0, 1fr) 70px;
gap: 0;
padding: 10px;
padding: 4px;
overflow: hidden;
background:
linear-gradient(180deg, rgba(255, 255, 255, 0.02), transparent 42%),
@@ -2474,7 +2572,7 @@ button:disabled {
display: grid;
place-items: center;
min-height: 0;
padding: 20px 8px 12px;
padding: 3px 4px 2px;
background: #171b22;
}
@@ -2545,6 +2643,9 @@ button:disabled {
place-items: center;
width: 100%;
height: 100%;
min-width: 0;
min-height: 0;
overflow: hidden;
}
.visual-media-layer > img,
@@ -2555,9 +2656,15 @@ button:disabled {
.visual-media-layer > img,
.visual-media-layer > video {
position: absolute;
inset: 0;
display: block;
width: 100%;
height: 100%;
min-width: 0;
min-height: 0;
max-width: 100%;
max-height: 100%;
object-fit: contain;
object-position: center center;
background: #080a0e;
@@ -2772,7 +2879,7 @@ button:disabled {
}
.transport {
padding: 12px 12px 4px;
padding: 7px 10px 2px;
}
.scrubber,
@@ -2787,7 +2894,7 @@ button:disabled {
justify-content: space-between;
gap: 10px;
min-width: 0;
margin-top: 10px;
margin-top: 6px;
flex-wrap: nowrap;
}
@@ -4868,6 +4975,21 @@ button:disabled {
box-shadow: 0 18px 42px rgba(0, 0, 0, 0.35);
}
@media (min-width: 1181px) and (max-width: 1600px) {
.editor-grid {
grid-template-columns: 74px minmax(270px, 0.66fr) minmax(540px, 1.62fr) minmax(280px, 0.76fr);
}
.editor-grid.is-compact-rail {
grid-template-columns: 50px minmax(270px, 0.66fr) minmax(540px, 1.62fr) minmax(280px, 0.76fr);
}
.preview-stage {
padding-right: 4px;
padding-left: 4px;
}
}
@media (max-width: 1180px) {
body {
overflow: auto;
+13 -3
View File
@@ -125,7 +125,11 @@ function getPreferredInferenceDevice() {
}
async function createTranscriber(requestId, device) {
const { pipeline } = await import("@huggingface/transformers");
const { env, pipeline } = await import("@huggingface/transformers");
// The app's service worker is the single cache owner for large model files.
// A second Transformers Cache Storage copy can exhaust the site quota after
// other local AI workflows have already downloaded their models.
env.useBrowserCache = false;
return pipeline("automatic-speech-recognition", AUTOMATIC_CAPTION_MODEL_ID, {
dtype: "q8",
device,
@@ -157,8 +161,14 @@ async function getTranscriber(requestId) {
})(),
};
}
return transcriberState.promise;
try {
return await transcriberState.promise;
} catch (error) {
// Never retain a rejected initialization promise: the next click must be
// able to retry after a transient download, quota, or runtime failure.
transcriberState = null;
throw error;
}
}
async function detectWhisperLanguage(transcriber, audio, preferredLanguage, requestId) {
+94
View File
@@ -0,0 +1,94 @@
let previousPixels = null;
let previousSegmentId = "";
function luminance(pixels, width, x, y) {
const index = (y * width + x) * 4;
return pixels[index] * 0.299 + pixels[index + 1] * 0.587 + pixels[index + 2] * 0.114;
}
function opticalFlowDifference(current, previous, width, height) {
if (!previous) return { difference: 1, motion: 0, sceneChange: 1, quality: frameQuality(current, width, height) };
const block = 8;
const search = 4;
let motion = 0;
let residual = 0;
let samples = 0;
for (let y = search; y < height - block - search; y += block) {
for (let x = search; x < width - block - search; x += block) {
let bestError = Infinity;
let bestDistance = 0;
let texture = 0;
const center = luminance(current, width, x + block / 2, y + block / 2);
for (let by = 0; by < block; by += 2) for (let bx = 0; bx < block; bx += 2) texture += Math.abs(luminance(current, width, x + bx, y + by) - center);
for (let dy = -search; dy <= search; dy += 2) {
for (let dx = -search; dx <= search; dx += 2) {
let error = 0;
for (let by = 0; by < block; by += 2) {
for (let bx = 0; bx < block; bx += 2) {
error += Math.abs(luminance(current, width, x + bx, y + by) - luminance(previous, width, x + bx + dx, y + by + dy));
}
}
// Prefer zero displacement when several matches are effectively equal.
const distance = Math.hypot(dx, dy);
if (error + distance * 1.5 < bestError) { bestError = error + distance * 1.5; bestDistance = distance; }
}
}
motion += texture > 180 ? bestDistance / Math.hypot(search, search) : 0;
residual += bestError / ((block / 2) ** 2 * 255);
samples += 1;
}
}
const motionScore = samples ? Math.min(1, motion / samples) : 0;
const residualScore = samples ? Math.min(1, residual / samples) : 1;
const sceneChange = histogramDistance(current, previous, width, height);
const quality = frameQuality(current, width, height);
return { difference: Math.min(1, sceneChange * 0.62 + residualScore * 0.25 + motionScore * 0.13), motion: motionScore, sceneChange, quality };
}
function histogramDistance(current, previous, width, height) {
const a = new Uint32Array(32);
const b = new Uint32Array(32);
for (let y = 0; y < height; y += 4) for (let x = 0; x < width; x += 4) {
const index = (y * width + x) * 4;
const currentLuma = current[index] * .299 + current[index + 1] * .587 + current[index + 2] * .114;
const previousLuma = previous[index] * .299 + previous[index + 1] * .587 + previous[index + 2] * .114;
// Ignore letterbox pixels so aspect-ratio padding cannot become a scene signal.
if (currentLuma > 5) a[Math.min(31, currentLuma >> 3)] += 1;
if (previousLuma > 5) b[Math.min(31, previousLuma >> 3)] += 1;
}
const totalA = a.reduce((sum, value) => sum + value, 0) || 1;
const totalB = b.reduce((sum, value) => sum + value, 0) || 1;
let distance = 0;
for (let index = 0; index < a.length; index += 1) distance += Math.abs(a[index] / totalA - b[index] / totalB);
return Math.min(1, distance / 2);
}
function frameQuality(pixels, width, height) {
let visible = 0;
let contrast = 0;
let count = 0;
for (let y = 0; y < height; y += 4) for (let x = 0; x < width; x += 4) {
const value = luminance(pixels, width, x, y);
if (value > 8) visible += 1;
if (x + 4 < width) contrast += Math.abs(value - luminance(pixels, width, x + 4, y));
count += 1;
}
const coverage = visible / Math.max(1, count);
return Math.min(1, coverage * .75 + Math.min(1, contrast / Math.max(1, count) / 24) * .25);
}
self.onmessage = (event) => {
const { type, id, pixels, width, height, segmentId } = event.data || {};
if (type === "reset") {
previousPixels = null;
previousSegmentId = "";
return;
}
if (type !== "analyze") return;
const current = new Uint8ClampedArray(pixels);
const previous = segmentId === previousSegmentId ? previousPixels : null;
const metrics = opticalFlowDifference(current, previous, width, height);
previousPixels = current;
previousSegmentId = segmentId;
self.postMessage({ id, ...metrics });
};
+1
View File
@@ -67,6 +67,7 @@ function createModelLoadProgressCallback(requestId, { start, end, label }) {
function getTransformers() {
transformersPromise ??= import("@huggingface/transformers").then((transformers) => {
transformers.env.useBrowserCache = false;
if (transformers.env?.backends?.onnx?.wasm) {
transformers.env.backends.onnx.wasm.numThreads = 1;
}