diff --git a/e2e/auto-edit-desktop.spec.js b/e2e/auto-edit-desktop.spec.js new file mode 100644 index 0000000..22bac5d --- /dev/null +++ b/e2e/auto-edit-desktop.spec.js @@ -0,0 +1,72 @@ +import { expect, test } from "@playwright/test"; + +const videos = [process.env.AUTO_EDIT_VIDEO_A, process.env.AUTO_EDIT_VIDEO_B].filter(Boolean); + +test.describe("Auto Edit desktop video smoke", () => { + test.skip(videos.length !== 2, "Set AUTO_EDIT_VIDEO_A and AUTO_EDIT_VIDEO_B to run the local smoke test"); + + for (const [index, videoPath] of videos.entries()) { + test(`imports and samples desktop video ${index + 1}`, async ({ page }) => { + if (process.env.AUTO_EDIT_FULL && index === 0) test.setTimeout(12 * 60 * 1000); + if (process.env.AUTO_EDIT_MOCK_MODEL) await page.addInitScript(() => { + window.LanguageModel = { + availability: async () => "available", + create: async () => ({ + prompt: async (messages) => { + const prompt = messages?.[0]?.content?.find?.((item) => item.type === "text")?.value || ""; + const firstTime = Number(prompt.match(/timestamps?:\s*([\d.]+)/i)?.[1] || prompt.match(/at\s+([\d.]+)\s+seconds/i)?.[1] || 0); + return JSON.stringify({ captions: [{ start: firstTime, end: firstTime + 2.8, text: `Visual description for clip at ${firstTime.toFixed(1)} seconds.` }] }); + }, + destroy() {}, + }), + }; + }); + await page.goto("/"); + const languageIntro = page.locator(".language-intro"); + if (await languageIntro.isVisible()) { + await languageIntro.locator(".language-grid button").filter({ hasText: "简体中文" }).click(); + await expect(languageIntro).toBeHidden(); + } + const mediaInput = page.locator('input[type="file"][accept*="video/mp4"]'); + if (process.env.AUTO_EDIT_MULTI) { + await mediaInput.setInputFiles(videos[0]); + await expect(page.locator(".asset-row-button")).toHaveCount(1); + await mediaInput.setInputFiles(videos[1]); + await expect(page.locator(".asset-row-button")).toHaveCount(2); + } else { + await mediaInput.setInputFiles(videoPath); + } + const preview = page.locator(".preview-video"); + await expect(preview).toBeVisible(); + await expect.poll(async () => Number(await preview.evaluate((video) => video.duration))).toBeGreaterThan(0); + const result = await preview.evaluate(async (video) => { + const { extractAutoEditFrames, probeBuiltInAI } = await import("/src/lib/autoEdit.js"); + const duration = video.duration; + const frames = await extractAutoEditFrames([{ id: "desktop-smoke", type: "video", src: video.src, duration, sourceDuration: duration }]); + return { duration, frameCount: frames.length, times: frames.map((frame) => Number(frame.time.toFixed(2))), support: await probeBuiltInAI("zh") }; + }); + console.log(`AUTO_EDIT_RESULT_${index + 1}=${JSON.stringify(result)}`); + expect(result.frameCount).toBeGreaterThanOrEqual(2); + expect(result.times[0]).toBe(0); + expect(result.times.at(-1)).toBeCloseTo(result.duration, 1); + + if (process.env.AUTO_EDIT_FULL && index === 0) { + await page.getByRole("button", { name: "智能", exact: true }).click(); + await page.getByRole("button", { name: "检测浏览器支持", exact: true }).click(); + await expect(page.locator(".auto-edit-availability")).toHaveText(/可用|模型待下载|下载中/); + await expect(page.locator(".toast")).toBeHidden({ timeout: 10_000 }); + await page.locator(".auto-edit-generate").click(); + await expect(page.getByRole("dialog", { name: "画面分析与字幕检查" })).toBeVisible(); + await page.waitForTimeout(3_000); + console.log(`AUTO_EDIT_UI=${JSON.stringify(await page.locator(".auto-edit-panel").innerText())}`); + if (await page.locator(".toast").isVisible()) console.log(`AUTO_EDIT_TOAST=${JSON.stringify(await page.locator(".toast").innerText())}`); + await expect(page.getByText("字幕草稿已准备好", { exact: true })).toBeVisible({ timeout: 10 * 60 * 1000 }); + if (process.env.AUTO_EDIT_MULTI) expect(await page.locator(".auto-edit-clip-result").count()).toBe(2); + if (process.env.AUTO_EDIT_MOCK_MODEL) await page.screenshot({ path: "test-results/auto-edit-review.png", fullPage: true }); + await page.getByRole("button", { name: "应用到字幕轨", exact: true }).click(); + await expect(page.getByText("画面字幕已写入时间轴", { exact: true })).toBeVisible(); + console.log(`AUTO_EDIT_CAPTIONS=${await page.locator(".caption-segment").count()}`); + } + }); + } +}); diff --git a/e2e/offline-export.spec.js b/e2e/offline-export.spec.js new file mode 100644 index 0000000..a1eefe4 --- /dev/null +++ b/e2e/offline-export.spec.js @@ -0,0 +1,131 @@ +import { expect, test } from "@playwright/test"; + +test("offline export preserves stickers, captions, voice audio, dimensions and duration", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + const result = await page.evaluate(async () => { + const { exportOfflineVideo } = await import("/src/lib/offlineVideoExport.js"); + const makeImage = (draw) => { + const canvas = document.createElement("canvas"); + canvas.width = 320; canvas.height = 180; + draw(canvas.getContext("2d"), canvas); + return canvas.toDataURL("image/png"); + }; + const background = makeImage((context, canvas) => { + context.fillStyle = "#c20d18"; context.fillRect(0, 0, canvas.width, canvas.height); + }); + const sticker = makeImage((context) => { + context.clearRect(0, 0, 320, 180); + context.fillStyle = "#00ffff"; context.fillRect(120, 50, 80, 80); + }); + const sampleRate = 48_000; + const frames = sampleRate; + const wav = new ArrayBuffer(44 + frames * 2); + const view = new DataView(wav); + const text = (offset, value) => [...value].forEach((char, index) => view.setUint8(offset + index, char.charCodeAt(0))); + text(0, "RIFF"); view.setUint32(4, 36 + frames * 2, true); text(8, "WAVEfmt "); + view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, 1, true); + view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * 2, true); + view.setUint16(32, 2, true); view.setUint16(34, 16, true); text(36, "data"); view.setUint32(40, frames * 2, true); + for (let index = 0; index < frames; index += 1) view.setInt16(44 + index * 2, Math.sin(index / sampleRate * Math.PI * 2 * 440) * 8000, true); + const voice = new Blob([wav], { type: "audio/wav" }); + const exported = await exportOfflineVideo({ + imageSrc: background, visualType: "image", + visualSegments: [{ id: "visual", src: background, type: "image", duration: 1 }], + voiceAudioSegments: [{ id: "voice", blob: voice, start: 0, duration: 1, volume: 1 }], + sourceAudioBlob: null, sourceAudioSegments: [], musicBlob: null, + text: "", captionSegments: [{ id: "caption", text: "E2E CAPTION", start: 0, end: 1 }], + duration: 1, ratio: { width: 16, height: 9 }, fitMode: "cover", filter: "none", + captionsEnabled: true, captionPosition: "bottom", captionPlacement: "bottom", captionSize: 12, + captionStyle: { textColor: "#ffffff", backgroundColor: "#000000", backgroundOpacity: 0.8, fontWeight: 700 }, + captionReferenceSize: { width: 320, height: 180 }, + sticker: null, + stickerSegments: [{ id: "sticker", src: sticker, start: 0, duration: 1, x: 50, y: 50, scale: 2, opacity: 1 }], + exportSettings: { codec: "vp9", width: 320, height: 180, frameRate: 30, videoBitsPerSecond: 3_000_000 }, + }); + const audioContext = new AudioContext(); + const decodedAudio = await audioContext.decodeAudioData((await exported.blob.arrayBuffer()).slice(0)); + await audioContext.close(); + const url = URL.createObjectURL(exported.blob); + const video = document.createElement("video"); + video.muted = true; video.src = url; + await new Promise((resolve, reject) => { video.onloadedmetadata = resolve; video.onerror = reject; }); + video.currentTime = 0.5; + await new Promise((resolve, reject) => { video.onseeked = resolve; video.onerror = reject; }); + const sample = document.createElement("canvas"); sample.width = 320; sample.height = 180; + const context = sample.getContext("2d"); context.drawImage(video, 0, 0); + const center = [...context.getImageData(160, 90, 1, 1).data]; + const lower = [...context.getImageData(160, 156, 1, 1).data]; + const metadata = { + width: video.videoWidth, height: video.videoHeight, + duration: decodedAudio.duration, hasAudio: decodedAudio.length > 0, + mediaDuration: video.duration, center, lower, size: exported.blob.size, + diagnostics: exported.diagnostics, + }; + URL.revokeObjectURL(url); + return metadata; + }); + expect(result.width).toBe(320); + expect(result.height).toBe(180); + expect(result.duration).toBeGreaterThanOrEqual(0.95); + expect(result.mediaDuration).toBeGreaterThanOrEqual(0.95); + expect(result.hasAudio).toBe(true); + expect(result.size).toBeGreaterThan(10_000); + expect(result.center[1]).toBeGreaterThan(result.center[0]); + expect(result.center[2]).toBeGreaterThan(result.center[0]); + expect(result.lower[0] + result.lower[1] + result.lower[2]).toBeGreaterThan(40); + expect(result.diagnostics.frameCount).toBe(30); +}); + +test("offline export decodes video sequentially instead of seeking every output frame", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + const result = await page.evaluate(async () => { + const { exportOfflineVideo } = await import("/src/lib/offlineVideoExport.js"); + const sourceCanvas = document.createElement("canvas"); sourceCanvas.width = 160; sourceCanvas.height = 90; + const sourceContext = sourceCanvas.getContext("2d"); + sourceContext.fillStyle = "#000"; sourceContext.fillRect(0, 0, 160, 90); + sourceContext.fillStyle = "#fff"; sourceContext.beginPath(); sourceContext.arc(80, 45, 25, 0, Math.PI * 2); sourceContext.fill(); + const sourceImage = sourceCanvas.toDataURL("image/png"); + const sourceExport = await exportOfflineVideo({ + imageSrc: sourceImage, visualType: "image", visualSegments: [{ id: "image", src: sourceImage, type: "image", duration: 0.6 }], + voiceAudioSegments: [], sourceAudioBlob: null, sourceAudioSegments: [], musicBlob: null, text: "", captionSegments: [], + duration: 0.6, ratio: { width: 16, height: 9 }, fitMode: "cover", filter: "none", captionsEnabled: false, + captionSize: 12, captionStyle: {}, captionReferenceSize: { width: 160, height: 90 }, sticker: null, stickerSegments: [], + exportSettings: { codec: "vp9", width: 160, height: 90, frameRate: 24, videoBitsPerSecond: 1_000_000 }, + }); + const sourceBlob = sourceExport.blob; + const sourceUrl = URL.createObjectURL(sourceBlob); + const exported = await exportOfflineVideo({ + imageSrc: sourceUrl, visualType: "video", + visualSegments: [{ id: "video", src: sourceUrl, blob: sourceBlob, type: "video", duration: 0.6, sourceDuration: 0.6, sourceStart: 0, playbackRate: 1 }], + voiceAudioSegments: [], sourceAudioBlob: null, sourceAudioSegments: [], musicBlob: null, + text: "", captionSegments: [], duration: 0.6, ratio: { width: 16, height: 9 }, + fitMode: "cover", filter: "none", captionsEnabled: false, captionSize: 12, + captionStyle: {}, captionReferenceSize: { width: 160, height: 90 }, sticker: null, stickerSegments: [], + exportSettings: { codec: "vp9", width: 160, height: 90, frameRate: 24, videoBitsPerSecond: 1_000_000 }, + }); + const exportedUrl = URL.createObjectURL(exported.blob); + const video = document.createElement("video"); video.muted = true; video.src = exportedUrl; + await new Promise((resolve, reject) => { video.onloadedmetadata = resolve; video.onerror = reject; }); + video.currentTime = 0.3; + await new Promise((resolve, reject) => { video.onseeked = resolve; video.onerror = reject; }); + const checkCanvas = document.createElement("canvas"); checkCanvas.width = 160; checkCanvas.height = 90; + const checkContext = checkCanvas.getContext("2d"); checkContext.drawImage(video, 0, 0); + const pixels = checkContext.getImageData(0, 0, 160, 90).data; + let minX = 160; let maxX = 0; let minY = 90; let maxY = 0; + for (let y = 0; y < 90; y += 1) for (let x = 0; x < 160; x += 1) { + const offset = (y * 160 + x) * 4; + if (pixels[offset] > 180 && pixels[offset + 1] > 180 && pixels[offset + 2] > 180) { + minX = Math.min(minX, x); maxX = Math.max(maxX, x); minY = Math.min(minY, y); maxY = Math.max(maxY, y); + } + } + URL.revokeObjectURL(sourceUrl); URL.revokeObjectURL(exportedUrl); + return { size: exported.blob.size, diagnostics: exported.diagnostics, shapeRatio: (maxX - minX + 1) / (maxY - minY + 1) }; + }); + expect(result.size).toBeGreaterThan(500); + expect(result.diagnostics.videoDecodeModes).toEqual(["sequential-webcodecs"]); + expect(result.diagnostics.frameCount).toBe(15); + expect(result.shapeRatio).toBeGreaterThan(0.9); + expect(result.shapeRatio).toBeLessThan(1.1); +}); diff --git a/package-lock.json b/package-lock.json index 98530cb..32e2586 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "web-player", - "version": "0.3.2", + "version": "0.3.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "web-player", - "version": "0.3.2", + "version": "0.3.3", "license": "MIT", "dependencies": { "@diffusionstudio/vits-web": "^1.0.3", @@ -14,9 +14,11 @@ "@ffmpeg/ffmpeg": "^0.12.15", "@ffmpeg/util": "^0.12.2", "@huggingface/transformers": "^3.8.1", + "@mediabunny/aac-encoder": "^1.50.8", "@phosphor-icons/react": "^2.1.10", "fflate": "^0.8.3", "kokoro-js": "^1.2.1", + "mediabunny": "^1.50.8", "onnxruntime-web": "^1.27.0", "pinyin-pro": "^3.27.0", "react": "19.2.0", @@ -1617,6 +1619,19 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@mediabunny/aac-encoder": { + "version": "1.50.8", + "resolved": "https://registry.npmmirror.com/@mediabunny/aac-encoder/-/aac-encoder-1.50.8.tgz", + "integrity": "sha512-A5Se/LZd6RmYq/h36lBMSEsHvsyW8d0toR7FrAwpsFYbK+DVQYf90KiBT1Aw/mzLXx8/ypIOORJnd1sVZqOvJQ==", + "license": "MPL-2.0", + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/Vanilagy" + }, + "peerDependencies": { + "mediabunny": "^1.0.0" + } + }, "node_modules/@phosphor-icons/react": { "version": "2.1.10", "resolved": "https://registry.npmmirror.com/@phosphor-icons/react/-/react-2.1.10.tgz", @@ -2130,6 +2145,21 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/dom-mediacapture-transform": { + "version": "0.1.12", + "resolved": "https://registry.npmmirror.com/@types/dom-mediacapture-transform/-/dom-mediacapture-transform-0.1.12.tgz", + "integrity": "sha512-d7/QsLRwF864A5mgIM/YrfiglHoYn7zgCcAoJgW404r+2DwnNr7EBbLnCWpmOMgH8y0te73L1AV6H1bmauaWFw==", + "license": "MIT", + "dependencies": { + "@types/dom-webcodecs": "*" + } + }, + "node_modules/@types/dom-webcodecs": { + "version": "0.1.13", + "resolved": "https://registry.npmmirror.com/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz", + "integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==", + "license": "MIT" + }, "node_modules/@types/esrecurse": { "version": "4.3.1", "resolved": "https://registry.npmmirror.com/@types/esrecurse/-/esrecurse-4.3.1.tgz", @@ -3794,6 +3824,24 @@ "node": ">=10" } }, + "node_modules/mediabunny": { + "version": "1.50.8", + "resolved": "https://registry.npmmirror.com/mediabunny/-/mediabunny-1.50.8.tgz", + "integrity": "sha512-LgykLyQzhdpo0V2yw3UXmOpj+b4JAGdpHBwsPE6kjSt8Za0d1VllD+FV7EGHBcdV4+oHUAo+yrqbVAWxNSDCPQ==", + "license": "MPL-2.0", + "workspaces": [ + ".", + "packages/*" + ], + "dependencies": { + "@types/dom-mediacapture-transform": "^0.1.11", + "@types/dom-webcodecs": "0.1.13" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/Vanilagy" + } + }, "node_modules/minimatch": { "version": "10.2.5", "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-10.2.5.tgz", diff --git a/package.json b/package.json index be67f49..a228862 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "web-player", - "version": "0.3.2", + "version": "0.3.3", "private": true, "description": "A local-first browser AI video editor for voiceovers, captions, talking avatars, and multi-track timeline export.", "license": "MIT", @@ -33,9 +33,11 @@ "@ffmpeg/ffmpeg": "^0.12.15", "@ffmpeg/util": "^0.12.2", "@huggingface/transformers": "^3.8.1", + "@mediabunny/aac-encoder": "^1.50.8", "@phosphor-icons/react": "^2.1.10", "fflate": "^0.8.3", "kokoro-js": "^1.2.1", + "mediabunny": "^1.50.8", "onnxruntime-web": "^1.27.0", "pinyin-pro": "^3.27.0", "react": "19.2.0", diff --git a/playwright.config.js b/playwright.config.js index 9eaec2c..9799977 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -8,6 +8,10 @@ export default defineConfig({ use: { baseURL: "http://127.0.0.1:5173", headless: true, + channel: process.env.PLAYWRIGHT_CHANNEL || undefined, + launchOptions: process.env.PLAYWRIGHT_BUILT_IN_AI ? { + ignoreDefaultArgs: ["--disable-background-networking", "--disable-component-update"], + } : undefined, }, webServer: { command: "npm run dev", diff --git a/public/model-cache-sw.js b/public/model-cache-sw.js index 927ba94..903bf29 100644 --- a/public/model-cache-sw.js +++ b/public/model-cache-sw.js @@ -66,7 +66,11 @@ async function cacheFirst(request) { const response = await fetch(request); if (response.ok || response.type === "opaque") { - cache.put(request, response.clone()).catch(() => {}); + // Cache persistence is an optimization. A full browser quota must never + // block the live response consumed by an inference worker. + cache.put(request, response.clone()).catch((error) => { + if (error?.name !== "QuotaExceededError") console.warn("Model cache write failed.", error); + }); } return response; } @@ -112,6 +116,9 @@ self.addEventListener("activate", (event) => { ) .then(() => self.clients.claim()), ); + // Older Transformers.js builds created a second copy of Hugging Face model + // assets here. The service worker is now the sole cache owner. + event.waitUntil(caches.delete("transformers-cache").catch(() => false)); }); self.addEventListener("fetch", (event) => { diff --git a/src/App.jsx b/src/App.jsx index 4629a2d..b0637b7 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -58,19 +58,7 @@ import { getImageThumbnailCount, getVisualSegmentsTotal } from "./lib/timeline.j import { removeVisualPropertyKeyframe, updateVisualSegmentPlaybackRate, upsertVisualKeyframe, upsertVisualPropertyKeyframe } from "./lib/visualEffects.js"; import { getLinkedSourceAudioEnd, getLinkedSourceAudioSegments } from "./lib/sourceAudioSync.js"; import { getTimelineInitialContentZoom } from "./lib/timelineScale.js"; - -function getExportDimensions(ratio, longEdge) { - const sourceLongEdge = Math.max(ratio.width, ratio.height); - const scale = longEdge / sourceLongEdge; - const even = (value) => Math.max(2, Math.round(value / 2) * 2); - return { width: even(ratio.width * scale), height: even(ratio.height * scale) }; -} - -function getExportBitrate(resolution, quality, frameRate) { - const base = { 720: 5, 1080: 10, 1440: 18, 2160: 38 }[resolution] || 10; - const qualityScale = { standard: 0.65, high: 1, ultra: 1.45 }[quality] || 1; - return Math.round(base * qualityScale * (frameRate / 30) * 1_000_000); -} +import { getExportBitrate, getExportDimensions } from "./lib/exportSettings.js"; export function App() { const [uiLanguage, setUiLanguage] = useState(() => getStoredLanguage()); @@ -670,7 +658,10 @@ export function App() { ratioId={ratioId} showRatioMenu={showRatioMenu} setShowRatioMenu={setShowRatioMenu} - setRatioId={setRatioId} + setRatioId={(nextRatioId) => { + setRatioId(nextRatioId); + setFitModeFromUser("contain"); + }} notify={notify} isPlaying={isPlaying} handlePlayToggle={handlePlayToggle} diff --git a/src/components/PreviewStage.jsx b/src/components/PreviewStage.jsx index 0512765..e255498 100644 --- a/src/components/PreviewStage.jsx +++ b/src/components/PreviewStage.jsx @@ -13,6 +13,7 @@ import { import { formatTime } from "../lib/timeline.js"; import { getVisualMaskInsets, getVisualMaskSvgDataUrl, resolveVisualTransform } from "../lib/visualEffects.js"; import { resolveVisualClipAnimation } from "../lib/visualClipAnimations.js"; +import { getStickerBaseSize } from "../lib/stickerGeometry.js"; import { CaptionOverlay } from "./CaptionOverlay.jsx"; import { IconButton } from "./ui.jsx"; @@ -69,7 +70,7 @@ export function PreviewStage({ getDraggedAsset, applyAssetToTrack, }) { - const visibleStickers = stickers.length ? stickers : selectedSticker?.src || selectedSticker?.text ? [selectedSticker] : []; + const visibleStickers = stickers; const hasStickerOverlay = visibleStickers.some((sticker) => sticker?.src || sticker?.text); const hasPreviewContent = Boolean(previewVisualSrc || hasStickerOverlay); const renderedVisualSrc = previewVisualRenderSrc || previewVisualSrc; @@ -88,6 +89,7 @@ export function PreviewStage({ const frameWidth = Math.max(1, previewFrameSize.width || 1); const frameHeight = Math.max(1, previewFrameSize.height || 1); const frameMinDimension = Math.min(frameWidth, frameHeight); + const stickerBaseSize = getStickerBaseSize({ width: frameWidth, height: frameHeight }); const circleSize = Number.isFinite(visualMask.size) ? visualMask.size : 72; const maskWidth = visualMask.type === "circle" ? (circleSize * frameMinDimension) / frameWidth : Number.isFinite(visualMask.width) ? visualMask.width : 80; const maskHeight = visualMask.type === "circle" ? (circleSize * frameMinDimension) / frameHeight : Number.isFinite(visualMask.height) ? visualMask.height : 80; @@ -337,6 +339,8 @@ export function PreviewStage({ className={`sticker-overlay sticker-transform-box ${isEditable ? "is-editable" : ""}`} onPointerDown={(event) => startStickerDrag(event, sticker)} style={{ + width: `${stickerBaseSize}px`, + height: `${stickerBaseSize}px`, left: `${Number.isFinite(sticker.x) ? sticker.x : 82}%`, top: `${Number.isFinite(sticker.y) ? sticker.y : 20}%`, transform: `translate(-50%, -50%) scale(${Number.isFinite(sticker.scale) ? sticker.scale : 1}) rotate(${Number.isFinite(sticker.rotation) ? sticker.rotation : 0}deg)`, diff --git a/src/components/Timeline.jsx b/src/components/Timeline.jsx index 7250c6a..a5b39a4 100644 --- a/src/components/Timeline.jsx +++ b/src/components/Timeline.jsx @@ -258,8 +258,9 @@ export function Timeline({ ...audioLanes.map((_, index) => ["audio", `${t("voiceTrack")} ${index + 1}`, `audio-${index}`]), ["music", t("musicTrack")], ]; - const isRowVisible = (track, rowId = track) => - trackVisibility[rowId] ?? trackVisibility[track] ?? true; + // Visibility is track-scoped even when overlapping clips are packed into + // multiple visual rows. Preview, playback and export all read the track key. + const isRowVisible = (track) => trackVisibility[track] ?? true; const [rulerViewport, setRulerViewport] = useState({ scrollLeft: 0, viewportWidth: 0, @@ -597,7 +598,7 @@ export function Timeline({
(
showTrackContextMenu(event, track)} @@ -748,10 +749,10 @@ export function Timeline({ aria-label={`${label} ${t("visible")}`} onClick={(event) => { event.stopPropagation(); - toggleTrackVisibility(rowId); + toggleTrackVisibility(track); }} > - {isRowVisible(track, rowId) ? : } + {isRowVisible(track) ? : } + + +
+
+
+
01{t("autoEditCandidateTitle")}
{review.candidates.length} {t("autoEditFramesUnit")}
+

{t("autoEditCandidateHint")}

+ {review.candidates.length ?
{review.candidates.map((candidate, index) => ( +
+
{`${t("autoEditCandidateFrame")}#{String(index + 1).padStart(2, "0")}{candidate.aspectRatio}
+
{t("autoEditVisualChange")}{Math.round(candidate.difference * 100)}%
+
+ ))}
:
{t("autoEditFindingScenes")}
} +
+ +
+
02{t("autoEditModelResultTitle")}
{review.captions.length} {t("captionSegmentsUnit")}
+

{t("autoEditModelResultHint")}

+ {review.error ?
{t("autoEditReviewFailed")}{review.error}
: review.segments.length ?
{review.segments.map((segment) => { + const segmentCaptions = review.captions.filter((caption) => caption.visualSegmentId === segment.id); + const preview = review.candidates.find((candidate) => candidate.segmentId === segment.id); + return
+
{preview ? : null}
{segment.name || `${t("autoEditClip")} ${(segment.index ?? 0) + 1}`}{review.candidates.filter((candidate) => candidate.segmentId === segment.id).length} {t("autoEditFramesUnit")}
{t(`autoEditSegmentStatus_${segment.status}`)}
+ {segment.error ?

{segment.error}

: segmentCaptions.length ? <>
{segmentCaptions.map((caption, index) => ( +
{String(index + 1).padStart(2, "0")}

{caption.text}

+ ))}
{segment.status === "running" ?
{t("autoEditWindowProgress").replace("{current}", segment.windowIndex || 0).replace("{total}", segment.totalWindows || 0)}
: null} :
{segment.status === "running" ? : null}{segment.status === "running" && segment.totalWindows ? t("autoEditWindowProgress").replace("{current}", segment.windowIndex || 0).replace("{total}", segment.totalWindows) : t(`autoEditSegmentHint_${segment.status}`)}
} +
; + })}
:
{job.running ? job.phase : t("autoEditWaitingForModel")}
} +
+
+
+
{complete ? t("autoEditReviewSummaryReady") : t("autoEditReviewSummaryRunning")}{t("autoEditReviewSummaryHint")}
+ + +
+ +
, document.body, + ); +} + function AutoEditPanel({ t, hasVisual, language, autoEdit }) { const availability = autoEdit?.support?.availability || "unknown"; const languageFallback = autoEdit?.support?.language && autoEdit.support.language !== language; const ready = availability === "available" || availability === "downloadable" || availability === "downloading"; - return ( + return (<>
{t("autoEditCreateTitle")}{t("autoEditCreateDesc")}
@@ -34,9 +89,14 @@ function AutoEditPanel({ t, hasVisual, language, autoEdit }) {
1

{t("autoEditStepScenes")}{t("autoEditStepScenesHint")}

2

{t("autoEditStepCaptions")}{t("autoEditStepCaptionsHint")}

3

{t("autoEditStepTimeline")}{t("autoEditStepTimelineHint")}

{autoEdit?.job?.running ?
{autoEdit.job.phase}{autoEdit.job.progress}%
: null} - +
- ); + + ); } function CaptionContextPanel({ diff --git a/src/hooks/useAutoEdit.js b/src/hooks/useAutoEdit.js index 206f4af..3c73f4d 100644 --- a/src/hooks/useAutoEdit.js +++ b/src/hooks/useAutoEdit.js @@ -1,11 +1,18 @@ -import { useCallback, useRef, useState } from "react"; -import { extractAutoEditFrames, generateFrameCaptions, probeBuiltInAI } from "../lib/autoEdit.js"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { createFrameCaptionSession, extractAutoEditFrames, generateFrameCaptions, probeBuiltInAI } from "../lib/autoEdit.js"; import { getVisualSegmentsTotal } from "../lib/timeline.js"; export function useAutoEdit({ language, visualSegments, commitCaptionSegments, setCaptionsEnabled, setSelectedSegmentId, setSelectedTrack, notify, t }) { const [support, setSupport] = useState({ availability: "unknown", reason: "", language: "en" }); const [job, setJob] = useState({ running: false, progress: 0, phase: "" }); + const [review, setReview] = useState({ open: false, candidates: [], captions: [], segments: [], error: "" }); const abortRef = useRef(null); + const candidateUrlsRef = useRef([]); + const clearCandidateUrls = useCallback(() => { + candidateUrlsRef.current.forEach((url) => URL.revokeObjectURL(url)); + candidateUrlsRef.current = []; + }, []); + useEffect(() => clearCandidateUrls, [clearCandidateUrls]); const checkSupport = useCallback(async () => { setSupport((value) => ({ ...value, availability: "checking" })); const result = await probeBuiltInAI(language); @@ -17,21 +24,66 @@ export function useAutoEdit({ language, visualSegments, commitCaptionSegments, s const environment = support.availability === "unknown" ? await checkSupport() : support; if (environment.availability === "unavailable") return void notify(t("autoEditUnavailable")); abortRef.current = new AbortController(); + clearCandidateUrls(); + setReview({ open: true, candidates: [], captions: [], segments: [], error: "" }); + let session = null; setJob({ running: true, progress: 2, phase: t("autoEditFindingScenes") }); + // Chrome requires LanguageModel.create() to happen during the button's + // transient user activation when the model still needs downloading. + const sessionPromise = createFrameCaptionSession({ + language, + signal: abortRef.current.signal, + onDownloadProgress: (loaded) => setJob({ running: true, progress: Math.max(4, Math.round(loaded * 55)), phase: t("autoEditDownloadingModel") }), + }); try { const frames = await extractAutoEditFrames(visualSegments, (progress) => setJob({ running: true, progress, phase: t("autoEditFindingScenes") }), abortRef.current.signal); + const candidates = frames.map((frame, index) => { + const url = URL.createObjectURL(frame.blob); + candidateUrlsRef.current.push(url); + return { id: `${frame.segmentId}-${index}`, segmentId: frame.segmentId, segmentIndex: frame.segmentIndex, segmentName: frame.segmentName, url, time: frame.time, difference: frame.difference, aspectRatio: frame.aspectRatio }; + }); + const segments = candidates.reduce((items, candidate) => items.some((item) => item.id === candidate.segmentId) ? items : [...items, { id: candidate.segmentId, index: candidate.segmentIndex, name: candidate.segmentName, status: "waiting", error: "" }], []); + setReview((value) => ({ ...value, candidates, segments })); setJob({ running: true, progress: 60, phase: t("autoEditWritingCaptions") }); - const captions = await generateFrameCaptions({ frames, duration: getVisualSegmentsTotal(visualSegments), language, onDownloadProgress: (loaded) => setJob({ running: true, progress: 60 + Math.round(loaded * 20), phase: t("autoEditDownloadingModel") }) }); - if (!captions.length) throw new Error("No captions generated"); - commitCaptionSegments(captions); - setCaptionsEnabled(true); setSelectedTrack("caption"); setSelectedSegmentId(captions[0].id); + session = await sessionPromise; + const captions = await generateFrameCaptions({ + frames, duration: getVisualSegmentsTotal(visualSegments), language, session, + onPartial: (partial) => { + const modelProgress = partial.allWindows ? partial.completedWindows / partial.allWindows : 0; + setJob({ running: true, progress: Math.min(96, 60 + Math.round(modelProgress * 36)), phase: t("autoEditWritingCaptions") }); + setReview((value) => ({ + ...value, + captions: partial.captions.length ? [...value.captions.filter((caption) => caption.visualSegmentId !== partial.segmentId), ...partial.captions].sort((a, b) => a.start - b.start) : value.captions, + segments: value.segments.map((segment) => segment.id === partial.segmentId ? { ...segment, status: partial.status, error: partial.error || "", windowIndex: partial.windowIndex || 0, totalWindows: partial.totalWindows || 0 } : segment), + })); + }, + }); + if (!captions.length) { + setJob({ running: false, progress: 100, phase: t("autoEditNoResults") }); + return; + } + setReview((value) => ({ ...value, captions })); setJob({ running: false, progress: 100, phase: t("autoEditDone") }); - notify(t("autoEditDone")); } catch (error) { - if (error?.name !== "AbortError") notify(`${t("autoEditFailed")}: ${error?.message || error}`); + if (error?.name !== "AbortError") setReview((value) => ({ ...value, error: error?.message || String(error) })); setJob({ running: false, progress: 0, phase: "" }); + } finally { + session?.destroy?.(); } - }, [checkSupport, commitCaptionSegments, job.running, language, notify, setCaptionsEnabled, setSelectedSegmentId, setSelectedTrack, support, t, visualSegments]); + }, [checkSupport, clearCandidateUrls, job.running, language, notify, support, t, visualSegments]); const cancel = () => { abortRef.current?.abort(); setJob({ running: false, progress: 0, phase: "" }); }; - return { support, job, checkSupport, run, cancel }; + const closeReview = () => { + if (job.running) abortRef.current?.abort(); + setJob({ running: false, progress: 0, phase: "" }); + setReview({ open: false, candidates: [], captions: [], segments: [], error: "" }); + clearCandidateUrls(); + }; + const applyCaptions = () => { + if (!review.captions.length) return; + commitCaptionSegments(review.captions); + setCaptionsEnabled(true); setSelectedTrack("caption"); setSelectedSegmentId(review.captions[0].id); + notify(t("autoEditDone")); + closeReview(); + }; + return { support, job, review, checkSupport, run, cancel, closeReview, applyCaptions }; } diff --git a/src/hooks/useCaptionState.js b/src/hooks/useCaptionState.js index 96198db..9e4f70b 100644 --- a/src/hooks/useCaptionState.js +++ b/src/hooks/useCaptionState.js @@ -20,7 +20,7 @@ export function useCaptionState() { const [script, setScript] = useState(DEFAULT_SCRIPT); const [captionPosition, setCaptionPosition] = useState("bottom"); const [captionPlacement, setCaptionPlacement] = useState({ x: 50, y: 78 }); - const [captionSize, setCaptionSize] = useState(12); + const [captionSize, setCaptionSize] = useState(14); const [captionStyle, setCaptionStyle] = useState(DEFAULT_CAPTION_STYLE); const [captionsEnabled, setCaptionsEnabled] = useState(true); const [captionSegments, setCaptionSegments] = useState(() => createCaptionSegments(DEFAULT_SCRIPT)); diff --git a/src/hooks/usePreviewModel.js b/src/hooks/usePreviewModel.js index dbee17b..ab7f090 100644 --- a/src/hooks/usePreviewModel.js +++ b/src/hooks/usePreviewModel.js @@ -31,6 +31,7 @@ export function usePreviewModel(d) { }), [d.previewFrameSize.height, d.previewFrameSize.width, d.ratio.height, d.ratio.width]); const previewSmartCropRect = useMemo(() => { if ( + d.fitMode !== "cover" || !previewVisionOptions.smartCrop || !previewVisionAnalysis?.subject?.box || !previewVisionAnalysis?.sourceSize @@ -41,7 +42,7 @@ export function usePreviewModel(d) { previewVisionAnalysis.subject.box, { padding: 0.14 }, ); - }, [previewVisionAnalysis, previewVisionFrameSize, previewVisionOptions.smartCrop]); + }, [d.fitMode, previewVisionAnalysis, previewVisionFrameSize, previewVisionOptions.smartCrop]); const previewVisionOverlayBoxes = useMemo(() => { if (!previewVisionOptions.showDetections || !previewVisionAnalysis?.sourceSize) return []; return (previewVisionAnalysis.detections ?? []).map((detection) => { diff --git a/src/hooks/useProjectFiles.js b/src/hooks/useProjectFiles.js index c94dd2b..42af1e4 100644 --- a/src/hooks/useProjectFiles.js +++ b/src/hooks/useProjectFiles.js @@ -68,7 +68,7 @@ export function useProjectFiles(deps) { deps.setSelectedVoiceId(data.selectedVoiceId || VOICES[0].id); deps.setSpeed(Number(data.speed) || 1); deps.setVolume(Number(data.volume) || 1); deps.setRatioId(RATIO_OPTIONS.some((option) => option.id === data.ratioId) ? data.ratioId : "16:9"); deps.setFitMode(data.fitMode || "contain"); deps.setCaptionPosition(data.captionPosition || "bottom"); - deps.setCaptionPlacement(data.captionPlacement || { x: 50, y: 78 }); deps.setCaptionSize(Number(data.captionSize) || 12); + deps.setCaptionPlacement(data.captionPlacement || { x: 50, y: 78 }); deps.setCaptionSize(Number(data.captionSize) || 14); deps.setCaptionStyle(data.captionStyle || deps.captionStyle); deps.setCaptionsEnabled(data.captionsEnabled !== false); deps.setTrackVisibility(data.trackVisibility || deps.trackVisibility); deps.setTimelineZoom(Number(data.timelineZoom) || 1); deps.setSelectedFilterId(data.selectedFilterId || "none"); deps.setSelectedTransitionId(data.selectedTransitionId || "none"); diff --git a/src/hooks/useTimelineModel.js b/src/hooks/useTimelineModel.js index 403c22b..575538d 100644 --- a/src/hooks/useTimelineModel.js +++ b/src/hooks/useTimelineModel.js @@ -133,18 +133,10 @@ export function useTimelineModel(d) { return d.currentTime >= start && d.currentTime < end; }) : []; - const previewSticker = d.trackVisibility.sticker && currentStickerSegment - ? currentStickerSegment - : d.stickerSegments.length - ? STICKERS[0] - : selectedSticker; - const previewStickers = currentStickerSegments.length - ? currentStickerSegments - : d.stickerSegments.length - ? [] - : previewSticker?.src || previewSticker?.text - ? [previewSticker] - : []; + // The preview is timeline-driven. A library selection is only a source for + // creating a clip; it must not survive after the final sticker clip is deleted. + const previewSticker = d.trackVisibility.sticker ? currentStickerSegment : null; + const previewStickers = currentStickerSegments; const currentVisualSegmentIndex = getVisualSegmentIndexAtTime(d.visualSegments, d.currentTime); const currentVisualSegment = currentVisualSegmentIndex >= 0 ? d.visualSegments[currentVisualSegmentIndex] ?? null diff --git a/src/hooks/useVideoExport.js b/src/hooks/useVideoExport.js index bcef836..d87e539 100644 --- a/src/hooks/useVideoExport.js +++ b/src/hooks/useVideoExport.js @@ -1,5 +1,6 @@ import { useCallback } from "react"; import { downloadBlob, exportBrowserVideo, getSupportedRecordingFormat, transcodeWebmToMp4 } from "../lib/media.js"; +import { exportOfflineVideo } from "../lib/offlineVideoExport.js"; import { estimateDuration } from "../lib/timeline.js"; import { getVisionKey } from "../lib/vision.js"; @@ -16,7 +17,7 @@ export function useVideoExport(d) { }; const finish = async (phase) => { d.setExportPhase(phase); d.setExportProgress(100); await new Promise((resolve) => setTimeout(resolve, 450)); }; try { - const video = await exportBrowserVideo({ + const exportOptions = { imageSrc: d.imageSrc, visualType: d.visualType, visualSegments: d.renderedVisualSegments.map((segment) => { const record = d.visionRecords[getVisionKey(segment)]; @@ -37,10 +38,19 @@ export function useVideoExport(d) { captionSize: d.captionSize, captionStyle: d.captionStyle, captionReferenceSize: d.previewFrameSize.width > 0 && d.previewFrameSize.height > 0 ? d.previewFrameSize : { width: (360 * d.ratio.width) / d.ratio.height, height: 360 }, - sticker: d.stickerSegments.length ? null : d.selectedSticker, + // Stickers are timeline clips; a selected library item is not export content. + sticker: null, stickerSegments: d.trackVisibility.sticker ? d.stickerSegments : [], transitionId: "none", exportSettings: d.exportSettings, onProgress: progress, - }); + }; + let video; + try { + video = await exportOfflineVideo(exportOptions); + } catch (offlineError) { + console.warn("Offline WebCodecs export unavailable; using compatibility recorder", offlineError); + progress({ progress: 5, phase: "切换兼容导出模式" }); + video = await exportBrowserVideo(exportOptions); + } const name = `ai-voiceover-${d.ratio.id.replace(":", "x")}`; if (d.exportSettings.codec !== "h264") { progress({ progress: 99, phase: `保存 ${video.label} 文件` }); diff --git a/src/i18n.js b/src/i18n.js index 60270df..f6733ac 100644 --- a/src/i18n.js +++ b/src/i18n.js @@ -269,10 +269,40 @@ const AUTO_EDIT_COPY = { en: { smartAutoEditHint: "Local captions", autoEditCreateTitle: "Create timed captions from visuals", autoEditCreateDesc: "Detect scene changes, sample representative frames, and use Chrome's built-in model to write timed captions.", autoEditBrowserModel: "Chrome built-in model", autoEditPrivacyHint: "Frames are processed on this device and are not sent to the project server. Chrome may download the model on first use.", autoEditLanguageFallback: "The current UI language is not officially supported by Chrome Prompt API output. Captions will be generated in English and remain editable.", autoEditCheckSupport: "Check browser support", autoEditStepScenes: "Detect scenes", autoEditStepScenesHint: "Select key visuals by frame difference", autoEditStepCaptions: "Understand visuals", autoEditStepCaptionsHint: "Generate copy with the local multimodal model", autoEditStepTimeline: "Write timeline", autoEditStepTimelineHint: "Keep caption start and end times", autoEditGenerate: "Generate visual captions", autoEditNeedsVisual: "Add an image or video first", autoEditFindingScenes: "Detecting scene changes", autoEditWritingCaptions: "Writing captions", autoEditDownloadingModel: "Downloading browser model", autoEditDone: "Visual captions added to the timeline", autoEditUnavailable: "Chrome built-in AI is unavailable on this browser or device", autoEditFailed: "Auto Edit failed", autoEditStatus_unknown: "Not checked", autoEditStatus_checking: "Checking", autoEditStatus_available: "Available", autoEditStatus_downloadable: "Model download needed", autoEditStatus_downloading: "Downloading", autoEditStatus_unavailable: "Unavailable" }, }; +const AUTO_EDIT_BUTTON_COPY = { + zh: { autoEditGenerateHint: "检测画面变化并创建时间轴字幕", autoEditNeedsVisualHint: "导入素材后即可开始" }, + en: { autoEditGenerateHint: "Detect visual changes and create timed captions", autoEditNeedsVisualHint: "Import media to get started" }, +}; + +const AUTO_EDIT_REVIEW_COPY = { + zh: { autoEditReviewTitle: "画面分析与字幕检查", autoEditReviewReady: "分析完成", autoEditReviewFailed: "分析失败", autoEditCandidateTitle: "候选关键帧", autoEditCandidateHint: "根据连续帧的视觉变化筛选，以下画面将作为模型输入。", autoEditFramesUnit: "帧", autoEditCandidateFrame: "候选帧", autoEditVisualChange: "画面变化", autoEditModelResultTitle: "模型解析结果", autoEditModelResultHint: "模型根据候选画面生成的字幕内容与时间范围。", autoEditWaitingForModel: "等待模型解析候选画面", autoEditReviewSummaryReady: "字幕草稿已准备好", autoEditReviewSummaryRunning: "正在构建字幕草稿", autoEditReviewSummaryHint: "确认后才会写入现有字幕轨。", autoEditApplyCaptions: "应用到字幕轨" }, + en: { autoEditReviewTitle: "Visual analysis & caption review", autoEditReviewReady: "Analysis complete", autoEditReviewFailed: "Analysis failed", autoEditCandidateTitle: "Candidate keyframes", autoEditCandidateHint: "Selected from visual changes across consecutive frames. These frames will be sent to the model.", autoEditFramesUnit: "frames", autoEditCandidateFrame: "Candidate frame", autoEditVisualChange: "Visual change", autoEditModelResultTitle: "Model output", autoEditModelResultHint: "Caption text and time ranges generated from the candidate visuals.", autoEditWaitingForModel: "Waiting for the model to analyze candidates", autoEditReviewSummaryReady: "Caption draft is ready", autoEditReviewSummaryRunning: "Building caption draft", autoEditReviewSummaryHint: "Nothing is written to the caption track until you confirm.", autoEditApplyCaptions: "Apply to caption track" }, +}; + +const AUTO_EDIT_FLOW_COPY = { + zh: { autoEditCandidateHint: "通过轻量级块匹配光流追踪连续画面变化，以下帧将作为模型输入。", autoEditVisualChange: "光流变化", autoEditStepScenesHint: "在 Worker 中通过光流筛选关键画面" }, + en: { autoEditCandidateHint: "Hybrid scene-cut and motion analysis selects representative frames across each clip. These candidates will be sent to the model.", autoEditVisualChange: "Scene score", autoEditStepScenesHint: "Find representative visuals in a background Worker" }, +}; + +const AUTO_EDIT_SEGMENT_COPY = { + zh: { autoEditClip: "片段", autoEditSegmentStatus_waiting: "等待中", autoEditSegmentStatus_running: "解析中", autoEditSegmentStatus_complete: "已完成", autoEditSegmentStatus_empty: "无结果", autoEditSegmentStatus_error: "失败", autoEditSegmentHint_waiting: "排队等待模型处理", autoEditSegmentHint_running: "正在分析这个片段的候选画面", autoEditSegmentHint_complete: "片段解析完成", autoEditSegmentHint_empty: "模型未为这个片段生成字幕", autoEditSegmentHint_error: "这个片段解析失败", autoEditWindowProgress: "正在处理窗口 {current} / {total}" }, + en: { autoEditClip: "Clip", autoEditSegmentStatus_waiting: "Waiting", autoEditSegmentStatus_running: "Analyzing", autoEditSegmentStatus_complete: "Complete", autoEditSegmentStatus_empty: "No result", autoEditSegmentStatus_error: "Failed", autoEditSegmentHint_waiting: "Queued for model processing", autoEditSegmentHint_running: "Analyzing this clip's candidate visuals", autoEditSegmentHint_complete: "Clip analysis complete", autoEditSegmentHint_empty: "The model returned no caption for this clip", autoEditSegmentHint_error: "This clip could not be analyzed", autoEditWindowProgress: "Processing window {current} / {total}" }, +}; + +const AUTO_EDIT_RESULT_COPY = { + zh: { autoEditNoResults: "所有片段均已处理，但没有可用字幕" }, + en: { autoEditNoResults: "All clips were processed, but no usable captions were generated" }, +}; + export const UI_COPY = { zh: { ...SMART_WORKSPACE_COPY.zh, ...AUTO_EDIT_COPY.zh, + ...AUTO_EDIT_BUTTON_COPY.zh, + ...AUTO_EDIT_REVIEW_COPY.zh, + ...AUTO_EDIT_FLOW_COPY.zh, + ...AUTO_EDIT_SEGMENT_COPY.zh, + ...AUTO_EDIT_RESULT_COPY.zh, ...VISUAL_EDITOR_COPY.zh, ...TRANSITION_EDITOR_COPY.zh, ...VISUAL_ANIMATION_COPY.zh, @@ -647,6 +677,11 @@ export const UI_COPY = { en: { ...SMART_WORKSPACE_COPY.en, ...AUTO_EDIT_COPY.en, + ...AUTO_EDIT_BUTTON_COPY.en, + ...AUTO_EDIT_REVIEW_COPY.en, + ...AUTO_EDIT_FLOW_COPY.en, + ...AUTO_EDIT_SEGMENT_COPY.en, + ...AUTO_EDIT_RESULT_COPY.en, ...VISUAL_EDITOR_COPY.en, ...TRANSITION_EDITOR_COPY.en, ...VISUAL_ANIMATION_COPY.en, diff --git a/src/i18nTransition.test.js b/src/i18nTransition.test.js deleted file mode 100644 index 253428c..0000000 --- a/src/i18nTransition.test.js +++ /dev/null @@ -1,16 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { APP_LANGUAGES, createTranslator, translateOptionName } from "./i18n.js"; - -describe("transition editor translations", () => { - it("provides editor labels and localized transition names for every UI language", () => { - APP_LANGUAGES.forEach(({ id }) => { - const t = createTranslator(id); - expect(t("transitionSettings")).not.toBe("transitionSettings"); - expect(t("close")).not.toBe("close"); - expect(t("duration")).not.toBe("duration"); - expect(t("secondsShort")).not.toBe("secondsShort"); - if (id !== "zh") expect(translateOptionName(id, "淡入淡出")).not.toBe("淡入淡出"); - }); - }); -}); diff --git a/src/i18nVisualAnimation.test.js b/src/i18nVisualAnimation.test.js deleted file mode 100644 index a5f9bc1..0000000 --- a/src/i18nVisualAnimation.test.js +++ /dev/null @@ -1,17 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { APP_LANGUAGES, createTranslator } from "./i18n.js"; - -describe("visual animation translations", () => { - it("provides native animation copy for every supported UI language", () => { - for (const language of APP_LANGUAGES) { - const t = createTranslator(language.id); - expect(t("visualTabAnimation")).not.toBe("visualTabAnimation"); - expect(t("visualAnimationHoverHint")).not.toBe("visualAnimationHoverHint"); - expect(t("visualAnimationDuration")).not.toBe("visualAnimationDuration"); - expect(t("stickerProperties")).not.toBe("stickerProperties"); - expect(t("stickerOpacity")).not.toBe("stickerOpacity"); - expect(t("deleteSticker")).not.toBe("deleteSticker"); - expect(t("newCaptionDefault")).not.toBe("newCaptionDefault"); - } - }); -}); diff --git a/src/lib/asr.js b/src/lib/asr.js index be4b0d8..b6b40b4 100644 --- a/src/lib/asr.js +++ b/src/lib/asr.js @@ -562,7 +562,8 @@ async function getTranscriber(onProgress) { transcriberState = { modelId, promise: (async () => { - const { pipeline } = await import("@huggingface/transformers"); + const { env, pipeline } = await import("@huggingface/transformers"); + env.useBrowserCache = false; const reportModelLoadProgress = createModelLoadProgressCallback(onProgress); return pipeline("automatic-speech-recognition", modelId, { dtype: "q8", @@ -573,7 +574,12 @@ async function getTranscriber(onProgress) { }; } - return transcriberState.promise; + try { + return await transcriberState.promise; + } catch (error) { + transcriberState = null; + throw error; + } } function rejectWorkerRequests(error) { diff --git a/src/lib/autoEdit.js b/src/lib/autoEdit.js index 071a55e..d24e53b 100644 --- a/src/lib/autoEdit.js +++ b/src/lib/autoEdit.js @@ -23,17 +23,59 @@ export async function probeBuiltInAI(language = "en") { } } -export function selectChangedFrames(frames, { threshold = 0.12, maxFrames = 12 } = {}) { +function median(values) { + if (!values.length) return 0; + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; +} + +export function getAdaptiveSceneThreshold(frames, floor = 0.1) { + const scores = frames.slice(1).map((frame) => Number(frame.difference) || 0); + const center = median(scores); + const deviation = median(scores.map((score) => Math.abs(score - center))); + return Math.max(floor, center + Math.max(0.035, deviation * 2.5)); +} + +export function selectChangedFrames(frames, { threshold, maxFrames = Infinity, minTimeGap = 1.2 } = {}) { if (!frames.length) return []; - const selected = [frames[0]]; - for (let index = 1; index < frames.length && selected.length < maxFrames; index += 1) { - const frame = frames[index]; - const previous = selected.at(-1); - if (frame.segmentId !== previous.segmentId || frame.difference >= threshold) selected.push(frame); + if (maxFrames <= 1) return [frames[0]]; + const selected = frames.filter((frame, index) => index === 0 || frame.segmentId !== frames[index - 1].segmentId).slice(0, maxFrames); + const effectiveThreshold = Number.isFinite(threshold) ? threshold : getAdaptiveSceneThreshold(frames); + const ranked = frames.slice(1).map((frame, index) => { + const previousScore = frames[index]?.difference || 0; + const localPeak = frame.difference >= previousScore && frame.difference >= (frames[index + 2]?.difference || 0); + const quality = frame.quality ?? 1; + return { frame, score: frame.difference * (.7 + quality * .3), localPeak }; + }).filter(({ frame, score }) => score >= effectiveThreshold && (frame.quality ?? 1) >= .28).sort((a, b) => b.score - a.score); + for (const { frame } of ranked) { + if (selected.length >= maxFrames) break; + if (selected.every((item) => item.segmentId !== frame.segmentId || Math.abs(item.time - frame.time) >= minTimeGap)) selected.push(frame); } - const last = frames.at(-1); - if (selected.length < maxFrames && last && selected.at(-1)?.time !== last.time) selected.push(last); - return selected; + // Explicit finite budgets (used only by callers/tests) may request coverage fill. + if (Number.isFinite(maxFrames) && selected.length < maxFrames) { + const fallback = frames.slice(1, -1).filter((frame) => (frame.quality ?? 1) >= .28).sort((a, b) => { + const distance = (frame) => Math.min(...selected.map((item) => Math.abs(item.time - frame.time))); + return distance(b) - distance(a); + }); + for (const frame of fallback) { + if (selected.length >= maxFrames) break; + if (selected.every((item) => item.segmentId !== frame.segmentId || Math.abs(item.time - frame.time) >= minTimeGap)) selected.push(frame); + } + } + return selected.sort((a, b) => a.time - b.time); +} + +export function selectCandidatesBySegment(frames, maxPerSegment) { + const groups = []; + frames.forEach((frame) => { + let group = groups.find((item) => item.segmentId === frame.segmentId); + if (!group) { group = { segmentId: frame.segmentId, frames: [] }; groups.push(group); } + group.frames.push(frame); + }); + return groups.flatMap((group) => { + return selectChangedFrames(group.frames, { maxFrames: Number.isFinite(maxPerSegment) ? maxPerSegment : Infinity, minTimeGap: 1.2 }); + }); } export function normalizeGeneratedCaptions(value, duration) { @@ -41,7 +83,7 @@ export function normalizeGeneratedCaptions(value, duration) { if (!Array.isArray(items)) return []; return items .map((item) => { - const start = Math.max(0, Math.min(duration, Number(item.start) || 0)); + const start = Math.max(0, Math.min(Math.max(0, duration - 0.2), Number(item.start) || 0)); const end = Math.max(start + 0.2, Math.min(duration, Number(item.end) || start + 2)); return { id: makeId("caption"), text: String(item.text || "").trim(), start, end, hidden: false }; }) @@ -49,6 +91,29 @@ export function normalizeGeneratedCaptions(value, duration) { .sort((a, b) => a.start - b.start); } +export function normalizeClipCaptionTimings(captions, clipStart, clipEnd, preferredMinimum = 1.2) { + if (!captions.length) return []; + const safeStart = Math.max(0, Number(clipStart) || 0); + const safeEnd = Math.max(safeStart + 0.2, Number(clipEnd) || safeStart + 0.2); + const minimum = Math.min(preferredMinimum, (safeEnd - safeStart) / captions.length); + const normalized = captions + .map((caption) => ({ + ...caption, + start: Math.max(safeStart, Math.min(safeEnd - 0.2, Number(caption.start) || safeStart)), + end: Math.max(safeStart + 0.2, Math.min(safeEnd, Number(caption.end) || safeEnd)), + })) + .sort((a, b) => a.start - b.start); + for (let index = normalized.length - 1; index >= 0; index -= 1) { + const caption = normalized[index]; + if (index < normalized.length - 1) caption.end = Math.min(caption.end, normalized[index + 1].start); + if (caption.end - caption.start < minimum) caption.start = Math.max(safeStart, caption.end - minimum); + } + for (let index = 0; index < normalized.length - 1; index += 1) { + normalized[index].end = Math.min(normalized[index].end, normalized[index + 1].start); + } + return normalized.filter((caption) => caption.end - caption.start >= 0.19); +} + function waitForMedia(element, event) { return new Promise((resolve, reject) => { const done = () => { cleanup(); resolve(); }; @@ -63,58 +128,99 @@ async function canvasBlob(canvas) { return new Promise((resolve, reject) => canvas.toBlob((blob) => blob ? resolve(blob) : reject(new Error("Frame encoding failed")), "image/jpeg", 0.82)); } -function pixelDifference(current, previous) { - if (!previous) return 1; - let sum = 0; - for (let index = 0; index < current.length; index += 4) { - sum += Math.abs(current[index] - previous[index]); - sum += Math.abs(current[index + 1] - previous[index + 1]); - sum += Math.abs(current[index + 2] - previous[index + 2]); - } - return sum / ((current.length / 4) * 3 * 255); +function createFlowWorkerClient() { + const worker = new Worker(new URL("../workers/auto-edit.worker.js", import.meta.url), { type: "module" }); + const pending = new Map(); + let requestId = 0; + worker.onmessage = (event) => { + const request = pending.get(event.data?.id); + if (!request) return; + pending.delete(event.data.id); + request.resolve(event.data); + }; + worker.onerror = (error) => { + pending.forEach(({ reject }) => reject(error)); + pending.clear(); + }; + return { + analyze(pixels, width, height, segmentId) { + const id = ++requestId; + const promise = new Promise((resolve, reject) => pending.set(id, { resolve, reject })); + worker.postMessage({ type: "analyze", id, pixels: pixels.buffer, width, height, segmentId }, [pixels.buffer]); + return promise; + }, + terminate() { worker.terminate(); pending.forEach(({ reject }) => reject(new DOMException("Aborted", "AbortError"))); pending.clear(); }, + }; +} + +function drawContainedFrame(context, source, sourceWidth, sourceHeight) { + const width = context.canvas.width; + const height = context.canvas.height; + const scale = Math.min(width / sourceWidth, height / sourceHeight); + const drawWidth = sourceWidth * scale; + const drawHeight = sourceHeight * scale; + context.fillStyle = "#000"; + context.fillRect(0, 0, width, height); + context.drawImage(source, (width - drawWidth) / 2, (height - drawHeight) / 2, drawWidth, drawHeight); +} + +export function getAspectRatioLabel(width, height) { + const ratio = width / Math.max(1, height); + const presets = [[9, 16], [1, 1], [4, 3], [3, 2], [16, 9], [21, 9]]; + const [w, h] = presets.reduce((best, value) => Math.abs(value[0] / value[1] - ratio) < Math.abs(best[0] / best[1] - ratio) ? value : best); + return `${w}:${h}`; } export async function extractAutoEditFrames(segments, onProgress = () => {}, signal) { const frames = []; + const flowWorker = createFlowWorkerClient(); let timelineStart = 0; - for (let segmentIndex = 0; segmentIndex < segments.length; segmentIndex += 1) { - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - const segment = segments[segmentIndex]; - const duration = Math.max(0.2, Number(segment.duration) || 0.2); - const canvas = document.createElement("canvas"); - canvas.width = 224; canvas.height = 126; - const context = canvas.getContext("2d", { willReadFrequently: true }); - let previousPixels = null; - if (segment.type === "video") { - const video = document.createElement("video"); - video.muted = true; video.preload = "auto"; video.src = segment.src; - if (video.readyState < 1) await waitForMedia(video, "loadedmetadata"); - const sourceStart = Number(segment.sourceStart) || 0; - const sourceDuration = Math.max(0.2, Number(segment.sourceDuration) || video.duration || duration); - const sampleCount = Math.min(30, Math.max(3, Math.ceil(duration * 1.5))); - for (let sample = 0; sample < sampleCount; sample += 1) { - const ratio = sampleCount === 1 ? 0 : sample / (sampleCount - 1); - video.currentTime = Math.min(Math.max(0, video.duration - 0.05), sourceStart + ratio * sourceDuration); - await waitForMedia(video, "seeked"); - context.drawImage(video, 0, 0, canvas.width, canvas.height); - const pixels = context.getImageData(0, 0, canvas.width, canvas.height).data; - frames.push({ segmentId: segment.id, time: timelineStart + ratio * duration, difference: pixelDifference(pixels, previousPixels), blob: await canvasBlob(canvas) }); - previousPixels = new Uint8ClampedArray(pixels); + try { + for (let segmentIndex = 0; segmentIndex < segments.length; segmentIndex += 1) { + if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); + const segment = segments[segmentIndex]; + const duration = Math.max(0.2, Number(segment.duration) || 0.2); + const canvas = document.createElement("canvas"); + canvas.width = 224; canvas.height = 224; + const context = canvas.getContext("2d", { willReadFrequently: true }); + if (segment.type === "video") { + const video = document.createElement("video"); + video.muted = true; video.preload = "auto"; video.src = segment.src; + if (video.readyState < 1) await waitForMedia(video, "loadedmetadata"); + const sourceStart = Number(segment.sourceStart) || 0; + const sourceDuration = Math.max(0.2, Number(segment.sourceDuration) || video.duration || duration); + // Keep a stable temporal resolution instead of a fixed total-frame cap. + // Long videos therefore receive proportionally more analysis samples. + const samplesPerSecond = duration <= 120 ? 1.5 : duration <= 600 ? 1 : 0.75; + const sampleCount = Math.max(3, Math.ceil(duration * samplesPerSecond) + 1); + for (let sample = 0; sample < sampleCount; sample += 1) { + if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); + const ratio = sampleCount === 1 ? 0 : sample / (sampleCount - 1); + video.currentTime = Math.min(Math.max(0, video.duration - 0.05), sourceStart + ratio * sourceDuration); + await waitForMedia(video, "seeked"); + drawContainedFrame(context, video, video.videoWidth, video.videoHeight); + const pixels = context.getImageData(0, 0, canvas.width, canvas.height).data; + const blob = await canvasBlob(canvas); + const metrics = await flowWorker.analyze(pixels, canvas.width, canvas.height, segment.id); + frames.push({ segmentId: segment.id, segmentIndex, segmentName: segment.name || `Clip ${segmentIndex + 1}`, segmentStart: timelineStart, segmentEnd: timelineStart + duration, time: timelineStart + ratio * duration, ...metrics, blob, aspectRatio: getAspectRatioLabel(video.videoWidth, video.videoHeight) }); + } + video.removeAttribute("src"); video.load(); + } else if (segment.src) { + const image = new Image(); image.src = segment.src; + if (!image.complete) await waitForMedia(image, "load"); + drawContainedFrame(context, image, image.naturalWidth, image.naturalHeight); + frames.push({ segmentId: segment.id, segmentIndex, segmentName: segment.name || `Clip ${segmentIndex + 1}`, segmentStart: timelineStart, segmentEnd: timelineStart + duration, time: timelineStart, difference: 1, blob: await canvasBlob(canvas), aspectRatio: getAspectRatioLabel(image.naturalWidth, image.naturalHeight) }); } - video.removeAttribute("src"); video.load(); - } else if (segment.src) { - const image = new Image(); image.src = segment.src; - if (!image.complete) await waitForMedia(image, "load"); - context.drawImage(image, 0, 0, canvas.width, canvas.height); - frames.push({ segmentId: segment.id, time: timelineStart, difference: 1, blob: await canvasBlob(canvas) }); + timelineStart += duration; + onProgress(Math.round(((segmentIndex + 1) / segments.length) * 55)); } - timelineStart += duration; - onProgress(Math.round(((segmentIndex + 1) / segments.length) * 55)); + return selectCandidatesBySegment(frames); + } finally { + flowWorker.terminate(); } - return selectChangedFrames(frames); } -export async function generateFrameCaptions({ frames, duration, language, onDownloadProgress }) { +export function createFrameCaptionSession({ language, onDownloadProgress, signal }) { const modelLanguage = getAutoEditLanguage(language); const options = { expectedInputs: [{ type: "text", languages: ["en"] }, { type: "image" }], @@ -122,15 +228,89 @@ export async function generateFrameCaptions({ frames, duration, language, onDown monitor(monitor) { monitor.addEventListener("downloadprogress", (event) => onDownloadProgress?.(event.loaded)); }, + signal, }; - const session = await window.LanguageModel.create(options); + return window.LanguageModel.create(options); +} + +async function generateCaptionGroup(session, frames, duration, modelLanguage) { + const content = [{ type: "text", value: `Describe every provided candidate frame with one concise on-screen caption. Output in ${modelLanguage}. Return exactly ${frames.length} captions in the same order as the images. Use only visible evidence; do not invent names or facts. Frame timestamps: ${frames.map((frame) => frame.time.toFixed(2)).join(", ")} seconds.` }]; + frames.forEach((frame) => content.push({ type: "image", value: frame.blob })); + const schema = { type: "object", properties: { captions: { type: "array", minItems: frames.length, maxItems: frames.length, items: { type: "object", properties: { text: { type: "string", minLength: 1 } }, required: ["text"], additionalProperties: false } } }, required: ["captions"], additionalProperties: false }; + const response = await session.prompt([{ role: "user", content }], { responseConstraint: schema }); + const returned = JSON.parse(response)?.captions; + const descriptions = Array.isArray(returned) ? returned.map((item) => String(item?.text || "").trim()) : []; + const singleSchema = { type: "object", properties: { text: { type: "string", minLength: 1 } }, required: ["text"], additionalProperties: false }; + for (let index = 0; index < frames.length; index += 1) { + if (descriptions[index]) continue; + const frame = frames[index]; + const singleResponse = await session.prompt([{ role: "user", content: [ + { type: "text", value: `Write one concise ${modelLanguage} on-screen caption describing only what is visibly happening in this video frame at ${frame.time.toFixed(2)} seconds. Return a meaningful non-empty caption. Do not mention the timestamp.` }, + { type: "image", value: frame.blob }, + ] }], { responseConstraint: singleSchema }); + descriptions[index] = String(JSON.parse(singleResponse)?.text || "").trim(); + } + return frames.map((frame, index) => ({ + id: makeId("caption"), + text: descriptions[index], + start: frame.time, + end: Math.min(duration, Math.max(frame.time + 1.2, frames[index + 1]?.time ?? frame.segmentEnd ?? duration)), + hidden: false, + })).filter((caption) => caption.text); +} + +function createSlidingWindows(frames, size = 6, overlap = 2) { + if (!frames.length) return []; + const stride = Math.max(1, size - overlap); + const windows = []; + for (let start = 0; start < frames.length; start += stride) { + windows.push({ frames: frames.slice(start, start + size), start, commitEnd: frames[start + stride]?.time ?? Infinity }); + if (start + size >= frames.length) break; + } + return windows; +} + +export async function generateFrameCaptions({ frames, duration, language, session: providedSession, onDownloadProgress, onPartial }) { + const modelLanguage = getAutoEditLanguage(language); + const session = providedSession || await createFrameCaptionSession({ language, onDownloadProgress }); + const groups = []; + frames.forEach((frame) => { + let group = groups.find((item) => item.segmentId === frame.segmentId); + if (!group) { group = { segmentId: frame.segmentId, segmentIndex: frame.segmentIndex, segmentName: frame.segmentName, frames: [] }; groups.push(group); } + group.frames.push(frame); + }); + const allCaptions = []; + const windowsBySegment = new Map(groups.map((group) => [group.segmentId, createSlidingWindows(group.frames)])); + const totalWindows = [...windowsBySegment.values()].reduce((sum, windows) => sum + windows.length, 0); + let completedWindows = 0; try { - const content = [{ type: "text", value: `Create concise on-screen captions for these chronological video frames. Output in ${modelLanguage}. Use only visible evidence, do not invent names or facts. Frame timestamps: ${frames.map((frame) => frame.time.toFixed(2)).join(", ")} seconds. Keep each caption on screen for 1.5 to 4 seconds and within 0 to ${duration.toFixed(2)} seconds.` }]; - frames.forEach((frame) => content.push({ type: "image", value: frame.blob })); - const schema = { type: "object", properties: { captions: { type: "array", items: { type: "object", properties: { start: { type: "number" }, end: { type: "number" }, text: { type: "string" } }, required: ["start", "end", "text"], additionalProperties: false } } }, required: ["captions"], additionalProperties: false }; - const response = await session.prompt([{ role: "user", content }], { responseConstraint: schema }); - return normalizeGeneratedCaptions(JSON.parse(response), duration); + for (const group of groups) { + const windows = windowsBySegment.get(group.segmentId) || []; + const clipStart = group.frames[0]?.segmentStart ?? 0; + const clipEnd = group.frames[0]?.segmentEnd ?? duration; + let groupCaptions = []; + onPartial?.({ segmentId: group.segmentId, status: "running", captions: [], windowIndex: 0, totalWindows: windows.length, completedWindows, allWindows: totalWindows }); + try { + for (let windowIndex = 0; windowIndex < windows.length; windowIndex += 1) { + const window = windows[windowIndex]; + const generated = await generateCaptionGroup(session, window.frames, duration, modelLanguage); + // Overlap gives the model context; each window commits only its new time region. + const committed = generated.filter((caption) => (caption.start + caption.end) / 2 < window.commitEnd || windowIndex === windows.length - 1); + groupCaptions.push(...committed); + completedWindows += 1; + const partialCaptions = normalizeClipCaptionTimings(groupCaptions, clipStart, clipEnd).map((caption) => ({ ...caption, visualSegmentId: group.segmentId, visualSegmentIndex: group.segmentIndex, visualSegmentName: group.segmentName })); + onPartial?.({ segmentId: group.segmentId, status: "running", captions: partialCaptions, windowIndex: windowIndex + 1, totalWindows: windows.length, completedWindows, allWindows: totalWindows }); + } + const timedCaptions = normalizeClipCaptionTimings(groupCaptions, clipStart, clipEnd); + const captions = timedCaptions.map((caption) => ({ ...caption, visualSegmentId: group.segmentId, visualSegmentIndex: group.segmentIndex, visualSegmentName: group.segmentName })); + allCaptions.push(...captions); + onPartial?.({ segmentId: group.segmentId, status: captions.length ? "complete" : "empty", captions, windowIndex: windows.length, totalWindows: windows.length, completedWindows, allWindows: totalWindows }); + } catch (error) { + onPartial?.({ segmentId: group.segmentId, status: "error", captions: [], error: error?.message || String(error) }); + } + } + return allCaptions.sort((a, b) => a.start - b.start); } finally { - session.destroy?.(); + if (!providedSession) session.destroy?.(); } } diff --git a/src/lib/autoEdit.test.js b/src/lib/autoEdit.test.js index 303f7c5..6cd73e1 100644 --- a/src/lib/autoEdit.test.js +++ b/src/lib/autoEdit.test.js @@ -1,14 +1,81 @@ -import { describe, expect, it } from "vitest"; -import { getAutoEditLanguage, normalizeGeneratedCaptions, selectChangedFrames } from "./autoEdit.js"; +import { describe, expect, it, vi } from "vitest"; +import { generateFrameCaptions, getAdaptiveSceneThreshold, getAspectRatioLabel, getAutoEditLanguage, normalizeClipCaptionTimings, normalizeGeneratedCaptions, selectCandidatesBySegment, selectChangedFrames } from "./autoEdit.js"; describe("auto edit", () => { it("keeps scene changes and clip boundaries", () => { const frames = [{ segmentId: "a", time: 0, difference: 1 }, { segmentId: "a", time: 1, difference: .03 }, { segmentId: "a", time: 2, difference: .3 }, { segmentId: "b", time: 3, difference: .01 }]; expect(selectChangedFrames(frames).map((frame) => frame.time)).toEqual([0, 2, 3]); }); + it("keeps candidates for every visual segment", () => { + const frames = [ + { segmentId: "a", time: 0, difference: 1 }, { segmentId: "a", time: 2, difference: .4 }, + { segmentId: "b", time: 4, difference: 1 }, { segmentId: "b", time: 6, difference: .5 }, + ]; + expect(selectCandidatesBySegment(frames, 1).map((frame) => frame.segmentId)).toEqual(["a", "b"]); + }); + it("derives the scene threshold from the clip's own motion distribution", () => { + const quiet = [{ difference: 1 }, ...Array.from({ length: 8 }, () => ({ difference: .04 })), { difference: .35 }]; + expect(getAdaptiveSceneThreshold(quiet)).toBeLessThan(.35); + }); + it("returns every separated scene peak without a fixed candidate cap", () => { + const frames = Array.from({ length: 31 }, (_, index) => ({ + segmentId: "a", time: index * 2, difference: index > 0 && index % 3 === 0 ? .75 : .03, quality: .9, + })); + expect(selectCandidatesBySegment(frames)).toHaveLength(11); + }); + it("does not force a low-quality end card into the candidates", () => { + const frames = [ + { segmentId: "a", time: 0, difference: 1, quality: .9 }, + { segmentId: "a", time: 2, difference: .65, quality: .9 }, + { segmentId: "a", time: 5, difference: .5, quality: .8 }, + { segmentId: "a", time: 10, difference: .9, quality: .12 }, + ]; + expect(selectChangedFrames(frames, { maxFrames: 3 }).map((frame) => frame.time)).toEqual([0, 2, 5]); + }); it("falls back to an officially supported output language", () => expect(getAutoEditLanguage("zh")).toBe("en")); + it("labels common source aspect ratios", () => { + expect(getAspectRatioLabel(1080, 1920)).toBe("9:16"); + expect(getAspectRatioLabel(1920, 1080)).toBe("16:9"); + }); it("clamps and sorts generated captions", () => { const result = normalizeGeneratedCaptions({ captions: [{ start: 4, end: 9, text: "B" }, { start: -2, end: 1, text: "A" }] }, 5); expect(result.map(({ text, start, end }) => ({ text, start, end }))).toEqual([{ text: "A", start: 0, end: 1 }, { text: "B", start: 4, end: 5 }]); }); + it("removes end-of-clip overlap and keeps the last caption readable", () => { + const result = normalizeClipCaptionTimings([ + { id: "a", start: 0, end: 1.37 }, + { id: "b", start: 1.37, end: 15.06 }, + { id: "c", start: 14.86, end: 15.06 }, + ], 0, 15.06); + expect(result[2].start).toBeCloseTo(13.86, 5); + expect(result[2].end).toBeCloseTo(15.06, 5); + expect(result[1].end).toBeCloseTo(13.86, 5); + expect(result[0].end).toBeLessThanOrEqual(result[1].start); + }); + it("falls back to per-frame descriptions when the batch is empty", async () => { + const onPartial = vi.fn(); + const session = { + prompt: vi.fn() + .mockResolvedValueOnce('{"captions":[]}') + .mockResolvedValueOnce('{"text":"A lantern-lit interior"}') + .mockResolvedValueOnce('{"text":"A person enters the room"}'), + }; + const result = await generateFrameCaptions({ + frames: [{ segmentId: "clip-a", time: 0, blob: {} }, { segmentId: "clip-a", time: 3, blob: {} }], duration: 6, language: "en", session, onPartial, + }); + expect(result.map(({ text, start, end }) => ({ text, start, end }))).toEqual([ + { text: "A lantern-lit interior", start: 0, end: 3 }, + { text: "A person enters the room", start: 3, end: 6 }, + ]); + expect(session.prompt).toHaveBeenCalledTimes(3); + expect(onPartial.mock.calls.map(([value]) => value.status)).toEqual(["running", "running", "complete"]); + expect(onPartial.mock.calls[1][0]).toMatchObject({ windowIndex: 1, totalWindows: 1 }); + }); + it("returns one model result for every candidate frame", async () => { + const session = { prompt: vi.fn().mockResolvedValue('{"captions":[{"text":"A"},{"text":"B"},{"text":"C"},{"text":"D"}]}') }; + const frames = Array.from({ length: 4 }, (_, index) => ({ segmentId: "clip-a", segmentStart: 0, segmentEnd: 8, time: index * 2, blob: {} })); + const result = await generateFrameCaptions({ frames, duration: 8, language: "en", session }); + expect(result.map(({ text }) => text)).toEqual(["A", "B", "C", "D"]); + expect(result).toHaveLength(frames.length); + }); }); diff --git a/src/lib/captionLayout.js b/src/lib/captionLayout.js index f82da8b..8fba6be 100644 --- a/src/lib/captionLayout.js +++ b/src/lib/captionLayout.js @@ -9,6 +9,7 @@ const CAPTION_MAX_WIDTH = 680; const CAPTION_RADIUS = 7; const CAPTION_SHADOW_BLUR = 6; const CAPTION_SHADOW_OFFSET_Y = 1; +export const CAPTION_DESIGN_SHORT_EDGE = 360; export const CAPTION_FONT_FAMILY = 'Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", "PingFang SC", "Microsoft YaHei", sans-serif'; @@ -29,24 +30,24 @@ function clamp(value, minimum, maximum) { return Math.max(minimum, Math.min(maximum, value)); } -export function getCaptionScale(referenceFrame, renderFrame) { - const reference = normalizeFrameSize(referenceFrame); +export function getCaptionScale(_referenceFrame, renderFrame) { const render = normalizeFrameSize(renderFrame); - if (!reference.height || !render.height) { + const renderShortEdge = Math.min(render.width, render.height); + if (!renderShortEdge) { return 1; } - return render.height / reference.height; + return renderShortEdge / CAPTION_DESIGN_SHORT_EDGE; } export function resolveCaptionMetrics({ - captionSize = 12, + captionSize = 14, captionStyle = {}, referenceFrame, renderFrame, } = {}) { const frame = normalizeFrameSize(renderFrame); const scale = getCaptionScale(referenceFrame, frame); - const fontSize = Math.max(1, toPositiveNumber(captionSize, 12) * scale); + const fontSize = Math.max(1, toPositiveNumber(captionSize, 14) * scale); const paddingX = toPositiveNumber(captionStyle.paddingX, CAPTION_PADDING_X) * scale; const paddingY = toPositiveNumber(captionStyle.paddingY, CAPTION_PADDING_Y) * scale; const minWidth = frame.width * CAPTION_MIN_WIDTH_RATIO; @@ -172,7 +173,7 @@ function getFallbackTextWidth(text, fontSize) { export function getCaptionTextLayout({ context = getCaptionMeasurementContext(), text = "", - captionSize = 12, + captionSize = 14, captionStyle = {}, referenceFrame, renderFrame, diff --git a/src/lib/captionLayout.test.js b/src/lib/captionLayout.test.js new file mode 100644 index 0000000..1cbc67e --- /dev/null +++ b/src/lib/captionLayout.test.js @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; + +import { getCaptionScale, resolveCaptionMetrics } from "./captionLayout.js"; + +describe("caption design scaling", () => { + it("uses a readable 14px default without changing explicit project sizes", () => { + expect(resolveCaptionMetrics({ renderFrame: { width: 360, height: 640 } }).fontSize).toBe(14); + expect(resolveCaptionMetrics({ captionSize: 12, renderFrame: { width: 360, height: 640 } }).fontSize).toBe(12); + }); + + it("keeps 12px captions independent from the editor preview window size", () => { + const smallPreview = resolveCaptionMetrics({ captionSize: 12, renderFrame: { width: 276, height: 491 } }); + const largePreview = resolveCaptionMetrics({ captionSize: 12, renderFrame: { width: 360, height: 640 } }); + expect(smallPreview.fontSize).toBeCloseTo(9.2, 6); + expect(largePreview.fontSize).toBe(12); + }); + + it("maps the 360px design short edge consistently to vertical 1080p and 4K", () => { + expect(getCaptionScale(null, { width: 1080, height: 1920 })).toBe(3); + expect(resolveCaptionMetrics({ captionSize: 12, renderFrame: { width: 1080, height: 1920 } }).fontSize).toBe(36); + expect(resolveCaptionMetrics({ captionSize: 12, renderFrame: { width: 2160, height: 3840 } }).fontSize).toBe(72); + }); + + it("uses the same scale for landscape and portrait outputs with the same short edge", () => { + expect(getCaptionScale(null, { width: 3840, height: 2160 })).toBe(6); + expect(getCaptionScale(null, { width: 2160, height: 3840 })).toBe(6); + }); +}); diff --git a/src/lib/exportSettings.js b/src/lib/exportSettings.js new file mode 100644 index 0000000..b7f6fad --- /dev/null +++ b/src/lib/exportSettings.js @@ -0,0 +1,12 @@ +export function getExportDimensions(ratio, shortEdge) { + const sourceShortEdge = Math.min(ratio.width, ratio.height); + const scale = shortEdge / sourceShortEdge; + const even = (value) => Math.max(2, Math.round(value / 2) * 2); + return { width: even(ratio.width * scale), height: even(ratio.height * scale) }; +} + +export function getExportBitrate(resolution, quality, frameRate) { + const base = { 720: 5, 1080: 10, 1440: 18, 2160: 38 }[resolution] || 10; + const qualityScale = { standard: 0.65, high: 1, ultra: 1.45 }[quality] || 1; + return Math.round(base * qualityScale * (frameRate / 30) * 1_000_000); +} diff --git a/src/lib/exportSettings.test.js b/src/lib/exportSettings.test.js new file mode 100644 index 0000000..f02c999 --- /dev/null +++ b/src/lib/exportSettings.test.js @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vitest"; + +import { getExportDimensions } from "./exportSettings.js"; + +describe("export dimensions", () => { + it("uses the selected resolution as the short edge for landscape video", () => { + expect(getExportDimensions({ width: 16, height: 9 }, 2160)).toEqual({ width: 3840, height: 2160 }); + }); + + it("produces full vertical 4K instead of 1216x2160", () => { + expect(getExportDimensions({ width: 9, height: 16 }, 2160)).toEqual({ width: 2160, height: 3840 }); + }); + + it("keeps square exports square", () => { + expect(getExportDimensions({ width: 1, height: 1 }, 1080)).toEqual({ width: 1080, height: 1080 }); + }); +}); diff --git a/src/lib/media.js b/src/lib/media.js index 7cb4a2b..ef03da3 100644 --- a/src/lib/media.js +++ b/src/lib/media.js @@ -16,7 +16,7 @@ import { positionCaptionLayout, } from "./captionLayout.js"; import { resolveVisionAnalysisAtTime } from "./vision.js"; -import { getCaptionAvoidancePlacement, getSmartCropRect } from "./visualGeometry.js"; +import { getCaptionAvoidancePlacement, getSmartCropRect, getVisualFitRect } from "./visualGeometry.js"; import { getVisualMaskFeatherPixels, getVisualMaskGeometry, @@ -25,6 +25,7 @@ import { resolveVisualTransform, } from "./visualEffects.js"; import { resolveVisualClipAnimation } from "./visualClipAnimations.js"; +import { getStickerRenderGeometry } from "./stickerGeometry.js"; export function getAudioRecordingFormat() { if (typeof MediaRecorder === "undefined") { @@ -63,7 +64,7 @@ export function getVideoTrackSampleCount(duration, maxFrames = VIDEO_TRACK_FRAME return Math.max(1, Math.min(maxFrames, Math.ceil(safeDuration / targetStep))); } -function seekVideoFrame(video, time) { +export function seekVideoFrame(video, time) { const safeTime = Math.max(0, Math.min(time, Math.max(0, (video.duration || time) - 0.04))); if (video.readyState >= 2 && Math.abs(video.currentTime - safeTime) < 0.015) { return new Promise((resolve) => window.requestAnimationFrame(resolve)); @@ -263,7 +264,7 @@ export function downloadBlob(blob, filename) { window.setTimeout(() => URL.revokeObjectURL(url), 800); } -function loadImage(src) { +export function loadImage(src) { return new Promise((resolve, reject) => { const image = new Image(); image.crossOrigin = "anonymous"; @@ -273,7 +274,7 @@ function loadImage(src) { }); } -function createTemporalMaskCache(urls, maxEntries = 8) { +export function createTemporalMaskCache(urls, maxEntries = 8) { const orderedUrls = Array.from(new Set(urls.filter(Boolean))); const urlIndexes = new Map(orderedUrls.map((url, index) => [url, index])); const entries = new Map(); @@ -363,7 +364,7 @@ function createTemporalMaskCache(urls, maxEntries = 8) { }; } -function loadVideo(src) { +export function loadVideo(src) { return new Promise((resolve, reject) => { const video = document.createElement("video"); video.crossOrigin = "anonymous"; @@ -376,10 +377,10 @@ function loadVideo(src) { }); } -function getVisualDimensions(visual) { +export function getVisualDimensions(visual) { return { - width: visual.videoWidth || visual.naturalWidth || 1, - height: visual.videoHeight || visual.naturalHeight || 1, + width: visual.videoWidth || visual.naturalWidth || visual.displayWidth || visual.width || 1, + height: visual.videoHeight || visual.naturalHeight || visual.displayHeight || visual.height || 1, }; } @@ -473,7 +474,7 @@ function drawVisualUsingLayout(context, visual, layout, isMask = false) { function drawFittedVisual(context, visual, canvas, fitMode, filter, vision = null) { const { width, height } = canvas; const visualSize = getVisualDimensions(visual); - const smartCropEnabled = Boolean(vision?.options?.smartCrop && vision?.subject?.box); + const smartCropEnabled = Boolean(fitMode === "cover" && vision?.options?.smartCrop && vision?.subject?.box); const smartCropRect = smartCropEnabled ? getSmartCropRect(visualSize, canvas, vision.subject.box, { padding: 0.14 }) : null; @@ -488,27 +489,12 @@ function drawFittedVisual(context, visual, canvas, fitMode, filter, vision = nul outputSize: { width, height }, }; } else { - const imageRatio = visualSize.width / visualSize.height; - const canvasRatio = width / height; - const cover = fitMode === "cover"; - let drawWidth; - let drawHeight; - - if (cover ? imageRatio > canvasRatio : imageRatio < canvasRatio) { - drawHeight = height; - drawWidth = height * imageRatio; - } else { - drawWidth = width; - drawHeight = width / imageRatio; - } - - const x = (width - drawWidth) / 2; - const y = (height - drawHeight) / 2; + const fitRect = getVisualFitRect(visualSize, canvas, fitMode); layout = { sourceSize: visualSize, smartCropRect: null, - drawRect: { x, y, width: drawWidth, height: drawHeight }, - fitMode, + drawRect: { x: fitRect.x, y: fitRect.y, width: fitRect.width, height: fitRect.height }, + fitMode: fitRect.fitMode, outputSize: { width, height }, }; } @@ -536,7 +522,7 @@ function drawFittedVisual(context, visual, canvas, fitMode, filter, vision = nul return layout; } -function drawPreviewFrame(context, visual, canvas, options) { +export function drawPreviewFrame(context, visual, canvas, options) { const { subtitle, fitMode = "contain", @@ -544,7 +530,7 @@ function drawPreviewFrame(context, visual, canvas, options) { captionsEnabled = true, captionPosition = "bottom", captionPlacement = null, - captionSize = 12, + captionSize = 14, captionStyle = {}, captionReferenceSize = null, sticker = null, @@ -682,20 +668,12 @@ function drawPreviewFrame(context, visual, canvas, options) { visibleStickers.forEach((activeSticker, index) => { const activeStickerImage = stickerImages[index] ?? (activeSticker === sticker ? stickerImage : null); if (activeSticker?.src && activeStickerImage) { - const stickerRatio = - (activeStickerImage.naturalWidth || activeStickerImage.width || 1) / - (activeStickerImage.naturalHeight || activeStickerImage.height || 1); - const stickerScale = Math.max(0.2, Math.min(3, Number(activeSticker.scale) || 1)); - const maxStickerSize = Math.min(width, height) * 0.22 * stickerScale; - const stickerWidth = stickerRatio >= 1 ? maxStickerSize : maxStickerSize * stickerRatio; - const stickerHeight = stickerRatio >= 1 ? maxStickerSize / stickerRatio : maxStickerSize; - const centerX = (Number.isFinite(activeSticker.x) ? activeSticker.x : 82) / 100 * width; - const centerY = (Number.isFinite(activeSticker.y) ? activeSticker.y : 20) / 100 * height; + const geometry = getStickerRenderGeometry(activeSticker, activeStickerImage, canvas); context.save(); - context.globalAlpha = Math.max(0, Math.min(1, Number.isFinite(activeSticker.opacity) ? activeSticker.opacity : 1)); - context.translate(centerX, centerY); - context.rotate(((Number(activeSticker.rotation) || 0) * Math.PI) / 180); - context.drawImage(activeStickerImage, -stickerWidth / 2, -stickerHeight / 2, stickerWidth, stickerHeight); + context.globalAlpha = geometry.opacity; + context.translate(geometry.centerX, geometry.centerY); + context.rotate((geometry.rotation * Math.PI) / 180); + context.drawImage(activeStickerImage, -geometry.width / 2, -geometry.height / 2, geometry.width, geometry.height); context.restore(); } else if (activeSticker?.text) { context.fillStyle = "rgba(53, 240, 221, 0.92)"; diff --git a/src/lib/offlineVideoExport.js b/src/lib/offlineVideoExport.js new file mode 100644 index 0000000..9c7ec54 --- /dev/null +++ b/src/lib/offlineVideoExport.js @@ -0,0 +1,290 @@ +import { + AudioBufferSource, + ALL_FORMATS, + BlobSource, + BufferTarget, + CanvasSink, + CanvasSource, + Input, + Mp4OutputFormat, + Output, + WebMOutputFormat, +} from "mediabunny"; +import { registerAacEncoder } from "@mediabunny/aac-encoder"; + +import { + createTemporalMaskCache, + drawPreviewFrame, + loadImage, + loadVideo, + seekVideoFrame, +} from "./media.js"; +import { + createCaptionSegments, + getSegmentIndexAtTime, + getVisualSegmentIndexAtTime, + getVisualSegmentTimeline, +} from "./timeline.js"; +import { resolveVisionAnalysisAtTime } from "./vision.js"; +import { getVisualSourceTime } from "./visualEffects.js"; + +const clamp = (value, min, max) => Math.max(min, Math.min(max, value)); +let aacFallbackRegistered = false; + +export function createOfflineFramePlan(duration, frameRate) { + const fps = clamp(Math.round(Number(frameRate) || 30), 24, 60); + const safeDuration = Math.max(1 / fps, Number(duration) || 0); + const frameCount = Math.max(1, Math.ceil(safeDuration * fps)); + return Array.from({ length: frameCount }, (_, index) => ({ + index, + timestamp: index / fps, + duration: 1 / fps, + keyFrame: index % (fps * 2) === 0, + })); +} + +export function getOfflineExportCodec(settings = {}) { + if (settings.codec === "vp8") return { video: "vp8", audio: "opus", extension: "webm", mimeType: "video/webm" }; + if (settings.codec === "vp9") return { video: "vp9", audio: "opus", extension: "webm", mimeType: "video/webm" }; + return { video: "avc", audio: "aac", extension: "mp4", mimeType: "video/mp4" }; +} + +export function getOfflineStickersAtTime(stickerSegments = [], sticker = null, time = 0) { + if (!stickerSegments.length) return sticker ? [sticker] : []; + return stickerSegments.filter((item) => time >= item.start && time < item.start + item.duration); +} + +async function decodeAudioInputs(inputs) { + if (!inputs.length) return []; + const AudioContextClass = window.AudioContext || window.webkitAudioContext; + if (!AudioContextClass) throw new Error("当前浏览器不支持音频解码。"); + const context = new AudioContextClass(); + const cache = new Map(); + try { + return await Promise.all(inputs.map(async (input) => { + if (!cache.has(input.blob)) { + cache.set(input.blob, input.blob.arrayBuffer().then((data) => context.decodeAudioData(data.slice(0)))); + } + return { ...input, decoded: await cache.get(input.blob) }; + })); + } finally { + await context.close().catch(() => {}); + } +} + +export async function mixOfflineAudio({ + duration, + voiceAudioSegments = [], + sourceAudioBlob = null, + sourceAudioSegments = [], + sourceAudioVolume = 1, + sourceAudioStart = 0, + musicBlob = null, + musicVolume = 0.35, + musicStart = 0, +}) { + const inputs = [ + ...voiceAudioSegments.filter((item) => item.blob).map((item) => ({ + blob: item.blob, start: Math.max(0, item.start || 0), volume: item.volume ?? 1, + sourceOffset: 0, sourceDuration: 0, playbackRate: 1, + fadeIn: Math.max(0, item.fadeIn || 0), fadeOut: Math.max(0, item.fadeOut || 0), + })), + ...(sourceAudioBlob && sourceAudioSegments.length ? sourceAudioSegments.map((item) => ({ + blob: sourceAudioBlob, start: Math.max(0, item.start || 0), volume: sourceAudioVolume, + sourceOffset: Math.max(0, item.sourceStart || 0), sourceDuration: Math.max(0, item.sourceDuration || 0), + playbackRate: clamp(Number(item.playbackRate) || 1, 0.25, 4), fadeIn: 0, fadeOut: 0, + })) : sourceAudioBlob ? [{ blob: sourceAudioBlob, start: Math.max(0, sourceAudioStart), volume: sourceAudioVolume, sourceOffset: 0, sourceDuration: 0, playbackRate: 1, fadeIn: 0, fadeOut: 0 }] : []), + ...(musicBlob ? [{ blob: musicBlob, start: Math.max(0, musicStart), volume: musicVolume, sourceOffset: 0, sourceDuration: 0, playbackRate: 1, fadeIn: 0, fadeOut: 0 }] : []), + ]; + if (!inputs.length) return null; + const decoded = await decodeAudioInputs(inputs); + const sampleRate = 48_000; + const OfflineContextClass = window.OfflineAudioContext || window.webkitOfflineAudioContext; + if (!OfflineContextClass) throw new Error("当前浏览器不支持离线音频混合。"); + const context = new OfflineContextClass(2, Math.ceil(Math.max(0.01, duration) * sampleRate), sampleRate); + decoded.forEach((input) => { + const source = context.createBufferSource(); + const gain = context.createGain(); + source.buffer = input.decoded; + source.playbackRate.value = input.playbackRate; + const offset = Math.min(input.decoded.duration, input.sourceOffset); + const available = Math.max(0, input.decoded.duration - offset); + const sourceDuration = Math.min(available, input.sourceDuration || available); + const outputDuration = sourceDuration / input.playbackRate; + gain.gain.setValueAtTime(input.volume, input.start); + if (input.fadeIn > 0) { + gain.gain.setValueAtTime(0, input.start); + gain.gain.linearRampToValueAtTime(input.volume, input.start + Math.min(input.fadeIn, outputDuration)); + } + if (input.fadeOut > 0) { + const fadeStart = input.start + Math.max(0, outputDuration - input.fadeOut); + gain.gain.setValueAtTime(input.volume, fadeStart); + gain.gain.linearRampToValueAtTime(0, input.start + outputDuration); + } + source.connect(gain).connect(context.destination); + source.start(input.start, offset, sourceDuration); + }); + return context.startRendering(); +} + +async function prepareComposition(options) { + const segments = options.visualSegments.some((segment) => segment.src) + ? options.visualSegments.filter((segment) => segment.src) + : [{ id: "offline-visual", src: options.imageSrc, type: options.visualType, duration: options.duration }]; + const timeline = getVisualSegmentTimeline(segments); + const items = await Promise.all(segments.map(async (segment, index) => { + const visual = segment.type === "video" ? await loadVideo(segment.src) : await loadImage(segment.src); + const cutoutVisual = segment.type === "image" && segment.vision?.options?.removeBackground && segment.vision?.cutoutUrl + ? await loadImage(segment.vision.cutoutUrl).catch(() => null) : null; + const maskUrls = segment.type === "video" && segment.vision?.options?.removeBackground + ? [...new Set((segment.vision.samples || []).map((sample) => sample.cutoutUrl).filter(Boolean))] : []; + let sequentialFrames = null; + if (segment.type === "video" && options.framePlan?.length) { + try { + const blob = segment.blob instanceof Blob ? segment.blob : await fetch(segment.src).then((response) => { + if (!response.ok) throw new Error(`Unable to read video source (${response.status})`); + return response.blob(); + }); + const input = new Input({ source: new BlobSource(blob), formats: ALL_FORMATS }); + const track = await input.getPrimaryVideoTrack(); + if (!track || !(await track.canDecode())) throw new Error("VideoDecoder does not support this source codec"); + const sink = new CanvasSink(track, { poolSize: 3, decoderOptions: { optimizeForLatency: true } }); + const range = timeline[index]; + const timestamps = options.framePlan + .filter((frame) => frame.timestamp >= range.start && frame.timestamp < range.end) + .map((frame) => getVisualSourceTime(segment, frame.timestamp - range.start)); + const iterator = sink.canvasesAtTimestamps(timestamps)[Symbol.asyncIterator](); + sequentialFrames = { + async next() { + const result = await iterator.next(); + return result.done ? null : result.value?.canvas || null; + }, + }; + } catch (error) { + console.warn("Sequential WebCodecs video decode unavailable; using precise seek fallback", error); + } + } + return { + segment, visual, cutoutVisual, + temporalMaskCache: maskUrls.length ? createTemporalMaskCache(maskUrls) : null, + sequentialFrames, + decodeMode: segment.type === "video" ? (sequentialFrames ? "sequential-webcodecs" : "precise-seek") : "static-image", + }; + })); + const stickerSources = [...new Set([ + ...options.stickerSegments.map((item) => item.src).filter(Boolean), + ...(options.sticker?.src ? [options.sticker.src] : []), + ])]; + const stickerImages = new Map((await Promise.all(stickerSources.map(async (src) => [src, await loadImage(src).catch(() => null)]))).filter(([, image]) => image)); + return { segments, timeline, items, stickerImages }; +} + +async function renderCompositionAt(context, canvas, prepared, options, time) { + const resolvedIndex = getVisualSegmentIndexAtTime(prepared.segments, time); + const index = resolvedIndex >= 0 ? resolvedIndex : Math.max(0, prepared.items.length - 1); + const item = prepared.items[index] || prepared.items[0]; + const range = prepared.timeline[index] || prepared.timeline[0]; + const localTime = Math.max(0, time - (range?.start || 0)); + let sourceTime = getVisualSourceTime(item.segment, localTime); + let frameVisual = item.cutoutVisual || item.visual; + if (item.segment.type === "video" && item.sequentialFrames) { + frameVisual = await item.sequentialFrames.next() || item.visual; + } else if (item.segment.type === "video") { + sourceTime = Math.min(Math.max(0, (item.visual.duration || 0) - 0.04), sourceTime); + await seekVideoFrame(item.visual, sourceTime); + } + const vision = resolveVisionAnalysisAtTime(item.segment.vision || null, sourceTime); + if (vision?.cutoutUrl) await item.temporalMaskCache?.prepare(vision.cutoutUrl); + const frameVision = vision ? { + ...vision, + options: item.segment.vision?.options || vision.options, + maskVisual: vision.cutoutUrl ? item.temporalMaskCache?.get(vision.cutoutUrl) : null, + } : null; + const junction = item.segment.transition; + const transitionDuration = junction?.id && junction.id !== "none" + ? Math.min(Math.max(0.1, Number(junction.duration) || 0.5), Math.max(0, (range?.end || 0) - (range?.start || 0))) : 0; + const transitionProgress = transitionDuration && time >= range.end - transitionDuration + ? clamp((time - (range.end - transitionDuration)) / transitionDuration, 0, 1) : 0; + const next = transitionProgress > 0 ? prepared.items[index + 1] : null; + if (next?.segment.type === "video") await seekVideoFrame(next.visual, getVisualSourceTime(next.segment, transitionProgress * transitionDuration)); + const captionSegments = options.captionSegments?.length ? options.captionSegments : createCaptionSegments(options.text); + const captionIndex = getSegmentIndexAtTime(captionSegments, time, 0); + const caption = captionIndex >= 0 && !captionSegments[captionIndex]?.hidden ? captionSegments[captionIndex].text : ""; + const stickers = getOfflineStickersAtTime(options.stickerSegments, options.sticker, time); + drawPreviewFrame(context, frameVisual, canvas, { + subtitle: caption, fitMode: options.fitMode, filter: options.filter, + captionsEnabled: options.captionsEnabled, captionPosition: options.captionPosition, + captionPlacement: options.captionPlacement, captionSize: options.captionSize, + captionStyle: options.captionStyle, captionReferenceSize: options.captionReferenceSize, + stickers, stickerImages: stickers.map((sticker) => prepared.stickerImages.get(sticker.src)), + transitionId: next ? junction.id : "none", + transitionNext: next ? { visual: next.cutoutVisual || next.visual } : null, + transitionProgress, vision: frameVision, visualEffects: item.segment, visualTime: localTime, + }); +} + +export async function exportOfflineVideo(options) { + if (typeof VideoEncoder === "undefined") throw new Error("当前浏览器不支持 WebCodecs 离线编码。"); + const settings = options.exportSettings || {}; + const width = Math.max(2, Math.round(Number(settings.width) || options.ratio.width)); + const height = Math.max(2, Math.round(Number(settings.height) || options.ratio.height)); + const codec = getOfflineExportCodec(settings); + if (codec.audio === "aac" && !aacFallbackRegistered) { + registerAacEncoder(); + aacFallbackRegistered = true; + } + const canvas = document.createElement("canvas"); + canvas.width = width; canvas.height = height; + const context = canvas.getContext("2d", { alpha: false, desynchronized: false }); + if (!context) throw new Error("无法创建离线导出画布。"); + options.onProgress?.({ progress: 4, phase: "准备离线渲染" }); + const frames = createOfflineFramePlan(options.duration, settings.frameRate); + const preparedPromise = prepareComposition({ ...options, framePlan: frames }); + const audioPromise = mixOfflineAudio({ ...options, duration: frames.length * frames[0].duration }); + const [prepared, audioBuffer] = await Promise.all([preparedPromise, audioPromise]); + const target = new BufferTarget(); + const output = new Output({ format: codec.extension === "mp4" ? new Mp4OutputFormat({ fastStart: "in-memory" }) : new WebMOutputFormat(), target }); + let encoderConfig = null; + const videoSource = new CanvasSource(canvas, { + codec: codec.video, + bitrate: Math.max(2_000_000, Number(settings.videoBitsPerSecond) || 12_000_000), + keyFrameInterval: 2, + latencyMode: "quality", + onEncoderConfig: (config) => { encoderConfig = config; }, + }); + output.addVideoTrack(videoSource, { frameRate: frames.length / Math.max(options.duration, 1 / frames[0].duration) }); + let audioSource = null; + if (audioBuffer) { + audioSource = new AudioBufferSource({ codec: codec.audio, bitrate: codec.audio === "aac" ? 256_000 : 192_000 }); + output.addAudioTrack(audioSource); + } + await output.start(); + if (audioSource) await audioSource.add(audioBuffer); + try { + for (const frame of frames) { + await renderCompositionAt(context, canvas, prepared, options, frame.timestamp); + await videoSource.add(frame.timestamp, frame.duration, { keyFrame: frame.keyFrame }); + if (frame.index % Math.max(1, Math.round(frames.length / 100)) === 0) { + options.onProgress?.({ progress: 10 + Math.round((frame.index / frames.length) * 84), phase: `离线渲染 ${frame.index + 1} / ${frames.length}` }); + } + } + await output.finalize(); + } catch (error) { + await output.cancel().catch(() => {}); + throw error; + } finally { + prepared.items.forEach((item) => { + item.temporalMaskCache?.dispose(); + if (item.segment.type === "video") { item.visual.removeAttribute("src"); item.visual.load(); } + }); + } + options.onProgress?.({ progress: 98, phase: "验证导出文件" }); + return { + blob: new Blob([target.buffer], { type: codec.mimeType }), extension: codec.extension, + label: codec.extension === "mp4" ? "MP4" : "WebM", mimeType: codec.mimeType, + nativeMp4: codec.extension === "mp4", diagnostics: { + width, height, frameCount: frames.length, frameRate: settings.frameRate, encoderConfig, + videoDecodeModes: prepared.items.map((item) => item.decodeMode), + }, + }; +} diff --git a/src/lib/offlineVideoExport.test.js b/src/lib/offlineVideoExport.test.js new file mode 100644 index 0000000..2e19158 --- /dev/null +++ b/src/lib/offlineVideoExport.test.js @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; + +import { + createOfflineFramePlan, + getOfflineExportCodec, + getOfflineStickersAtTime, +} from "./offlineVideoExport.js"; +import { getVisualDimensions } from "./media.js"; + +describe("offline video export", () => { + it("creates deterministic frame timestamps without wall-clock drift", () => { + const frames = createOfflineFramePlan(1, 30); + expect(frames).toHaveLength(30); + expect(frames[0]).toMatchObject({ index: 0, timestamp: 0, duration: 1 / 30, keyFrame: true }); + expect(frames[29].timestamp).toBeCloseTo(29 / 30, 8); + }); + + it("uses stable two-second keyframe intervals", () => { + const frames = createOfflineFramePlan(4.1, 30); + expect(frames.filter((frame) => frame.keyFrame).map((frame) => frame.index)).toEqual([0, 60, 120]); + }); + + it("maps codecs directly to their final container without a lossy intermediate", () => { + expect(getOfflineExportCodec({ codec: "h264" })).toMatchObject({ video: "avc", audio: "aac", extension: "mp4" }); + expect(getOfflineExportCodec({ codec: "vp9" })).toMatchObject({ video: "vp9", audio: "opus", extension: "webm" }); + }); + + it("resolves overlapping sticker clips at exact timeline timestamps", () => { + const stickers = [ + { id: "a", start: 0, duration: 2 }, + { id: "b", start: 1, duration: 2 }, + ]; + expect(getOfflineStickersAtTime(stickers, null, 1.5).map((item) => item.id)).toEqual(["a", "b"]); + expect(getOfflineStickersAtTime(stickers, null, 2).map((item) => item.id)).toEqual(["b"]); + }); + + it("renders no sticker after the final timeline sticker is deleted", () => { + expect(getOfflineStickersAtTime([], null, 1)).toEqual([]); + }); + + it("preserves decoded canvas dimensions instead of treating frames as square", () => { + expect(getVisualDimensions({ width: 1920, height: 1080 })).toEqual({ width: 1920, height: 1080 }); + expect(getVisualDimensions({ videoWidth: 1280, videoHeight: 720, width: 1, height: 1 })).toEqual({ width: 1280, height: 720 }); + }); +}); diff --git a/src/lib/stickerGeometry.js b/src/lib/stickerGeometry.js new file mode 100644 index 0000000..1f7beac --- /dev/null +++ b/src/lib/stickerGeometry.js @@ -0,0 +1,21 @@ +export const STICKER_FRAME_RATIO = 0.22; + +export function getStickerBaseSize(frame) { + return Math.max(1, Math.min(frame.width, frame.height) * STICKER_FRAME_RATIO); +} + +export function getStickerRenderGeometry(sticker, image, frame) { + const imageWidth = image?.naturalWidth || image?.videoWidth || image?.width || 1; + const imageHeight = image?.naturalHeight || image?.videoHeight || image?.height || 1; + const ratio = imageWidth / imageHeight; + const scale = Math.max(0.2, Math.min(3, Number(sticker?.scale) || 1)); + const boxSize = getStickerBaseSize(frame) * scale; + return { + width: ratio >= 1 ? boxSize : boxSize * ratio, + height: ratio >= 1 ? boxSize / ratio : boxSize, + centerX: (Number.isFinite(sticker?.x) ? sticker.x : 82) / 100 * frame.width, + centerY: (Number.isFinite(sticker?.y) ? sticker.y : 20) / 100 * frame.height, + rotation: Number(sticker?.rotation) || 0, + opacity: Math.max(0, Math.min(1, Number.isFinite(sticker?.opacity) ? sticker.opacity : 1)), + }; +} diff --git a/src/lib/stickerGeometry.test.js b/src/lib/stickerGeometry.test.js new file mode 100644 index 0000000..a8838b1 --- /dev/null +++ b/src/lib/stickerGeometry.test.js @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; + +import { getStickerBaseSize, getStickerRenderGeometry } from "./stickerGeometry.js"; + +describe("sticker geometry", () => { + it("uses the same short-edge ratio for portrait preview and 4K export", () => { + const preview = { width: 276, height: 491 }; + const output = { width: 2160, height: 3840 }; + expect(getStickerBaseSize(preview) / preview.width).toBeCloseTo(0.22, 8); + expect(getStickerBaseSize(output) / output.width).toBeCloseTo(0.22, 8); + }); + + it("preserves the sticker image aspect ratio while scaling", () => { + const geometry = getStickerRenderGeometry( + { x: 70, y: 20, scale: 1.5, rotation: 12, opacity: 0.8 }, + { naturalWidth: 200, naturalHeight: 100 }, + { width: 2160, height: 3840 }, + ); + expect(geometry.width / geometry.height).toBeCloseTo(2, 8); + expect(geometry.width / 2160).toBeCloseTo(0.33, 8); + expect(geometry.centerX).toBe(1512); + expect(geometry.centerY).toBe(768); + }); +}); diff --git a/src/lib/visualGeometry.js b/src/lib/visualGeometry.js index d42f878..5f5e093 100644 --- a/src/lib/visualGeometry.js +++ b/src/lib/visualGeometry.js @@ -418,18 +418,9 @@ export function mapNormalizedBoxToFrame(box, sourceSize, frameSize, layout = {}) : null; } - const fitMode = layout?.fitMode === "cover" ? "cover" : "contain"; - const sourceAspectRatio = source.width / source.height; - const frameAspectRatio = frame.width / frame.height; - const useWidth = - fitMode === "cover" - ? sourceAspectRatio < frameAspectRatio - : sourceAspectRatio > frameAspectRatio; - const drawWidth = useWidth ? frame.width : frame.height * sourceAspectRatio; - const drawHeight = useWidth ? frame.width / sourceAspectRatio : frame.height; const position = getLayoutPosition(layout); - const drawX = (frame.width - drawWidth) * position.x; - const drawY = (frame.height - drawHeight) * position.y; + const fitRect = getVisualFitRect(source, frame, layout?.fitMode, position); + const { fitMode, x: drawX, y: drawY, width: drawWidth, height: drawHeight } = fitRect; const raw = { xMin: drawX + normalizedBox.xMin * drawWidth, yMin: drawY + normalizedBox.yMin * drawHeight, @@ -482,6 +473,27 @@ export function mapNormalizedBoxToFrame(box, sourceSize, frameSize, layout = {}) }; } +export function getVisualFitRect(sourceSize, frameSize, requestedFitMode = "contain", position = { x: 0.5, y: 0.5 }) { + const source = getSize(sourceSize); + const frame = getSize(frameSize); + if (!source.valid || !frame.valid) return { x: 0, y: 0, width: 0, height: 0, fitMode: "contain" }; + const fitMode = requestedFitMode === "cover" ? "cover" : "contain"; + const sourceAspectRatio = source.width / source.height; + const frameAspectRatio = frame.width / frame.height; + const useWidth = fitMode === "cover" + ? sourceAspectRatio < frameAspectRatio + : sourceAspectRatio > frameAspectRatio; + const width = useWidth ? frame.width : frame.height * sourceAspectRatio; + const height = useWidth ? frame.width / sourceAspectRatio : frame.height; + return { + x: (frame.width - width) * (Number.isFinite(position?.x) ? position.x : 0.5), + y: (frame.height - height) * (Number.isFinite(position?.y) ? position.y : 0.5), + width, + height, + fitMode, + }; +} + function normalizeFrameRect(rect, frameSize) { if (rect?.normalized) { return normalizeBoundingBox(rect.normalized); diff --git a/src/lib/visualGeometry.test.js b/src/lib/visualGeometry.test.js index 6158c87..ae44065 100644 --- a/src/lib/visualGeometry.test.js +++ b/src/lib/visualGeometry.test.js @@ -2,11 +2,27 @@ import { describe, expect, it } from "vitest"; import { getSmartCropRect, + getVisualFitRect, normalizeBoundingBox, selectPrimarySubject, } from "./visualGeometry.js"; describe("visual geometry", () => { + it("contains a 9:16 source inside 16:9 with centered black side space", () => { + const rect = getVisualFitRect({ width: 1080, height: 1920 }, { width: 1920, height: 1080 }, "contain"); + expect(rect.height).toBe(1080); + expect(rect.width).toBeCloseTo(607.5, 6); + expect(rect.x).toBeCloseTo(656.25, 6); + expect(rect.y).toBe(0); + }); + + it("only crops a portrait source when cover is explicitly requested", () => { + const rect = getVisualFitRect({ width: 1080, height: 1920 }, { width: 1920, height: 1080 }, "cover"); + expect(rect.width).toBe(1920); + expect(rect.height).toBeCloseTo(3413.333333, 5); + expect(rect.y).toBeLessThan(0); + }); + it("normalizes pixel coordinates", () => { const box = normalizeBoundingBox( { xMin: 100, yMin: 50, xMax: 500, yMax: 450 }, diff --git a/src/styles.css b/src/styles.css index b94260e..af10c4f 100644 --- a/src/styles.css +++ b/src/styles.css @@ -794,12 +794,12 @@ button:disabled { .drop-zone { display: grid; - grid-template-columns: 42px minmax(0, 1fr); + grid-template-columns: 32px minmax(0, 1fr); align-items: center; - gap: 6px 14px; - min-height: 116px; - margin-top: 18px; - padding: 16px 18px; + gap: 2px 10px; + min-height: 78px; + margin-top: 12px; + padding: 10px 12px; border: 1px dashed rgba(173, 184, 196, 0.24); border-radius: 8px; color: #99a5b1; @@ -819,14 +819,20 @@ button:disabled { } .drop-zone span { + min-width: 0; + overflow: hidden; color: #7c8793; - font-size: 12px; - line-height: 1.45; + font-size: 11px; + line-height: 1.35; + text-overflow: ellipsis; + white-space: nowrap; } .drop-zone svg { grid-row: 1 / span 2; justify-self: center; + width: 32px; + height: 32px; color: #8c96a2; } @@ -948,7 +954,7 @@ button:disabled { .asset-list { display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); + grid-template-columns: repeat(2, minmax(0, 1fr)); align-content: start; gap: 10px; margin-top: 14px; @@ -1177,7 +1183,99 @@ button:disabled { .auto-edit-progress { display: grid; gap: 8px; } .auto-edit-progress > div { display: flex; justify-content: space-between; color: #9eacb4; font-size: 11px; } .auto-edit-progress progress { width: 100%; accent-color: #35ead9; } -.auto-edit-panel .primary-action { width: 100%; } +.auto-edit-generate { position: relative; display: grid; grid-template-columns: 34px minmax(0,1fr) 22px; align-items: center; gap: 10px; width: 100%; min-height: 58px; overflow: hidden; border: 1px solid rgba(53,234,217,.72); border-radius: 11px; padding: 9px 12px; color: #061817; text-align: left; background: linear-gradient(120deg,#73b7ff 0%,#38e9d9 54%,#69f1c7 100%); box-shadow: 0 10px 28px rgba(38,217,200,.18), inset 0 1px 0 rgba(255,255,255,.35); cursor: pointer; transition: transform 160ms ease, box-shadow 160ms ease, filter 160ms ease; } +.auto-edit-generate::after { content:""; position:absolute; inset:-60% auto -60% -35%; width:34%; transform:rotate(18deg); background:linear-gradient(90deg,transparent,rgba(255,255,255,.28),transparent); transition:left 360ms ease; } +.auto-edit-generate:hover:not(:disabled) { transform: translateY(-1px); filter: brightness(1.04); box-shadow: 0 14px 34px rgba(38,217,200,.27), inset 0 1px 0 rgba(255,255,255,.42); } +.auto-edit-generate:hover:not(:disabled)::after { left:115%; } +.auto-edit-generate:active:not(:disabled) { transform: translateY(0); } +.auto-edit-generate:disabled { border-color: rgba(255,255,255,.1); color:#7d898e; background:rgba(255,255,255,.055); box-shadow:none; cursor:not-allowed; } +.auto-edit-generate-icon { display:grid; place-items:center; width:34px; height:34px; border-radius:9px; color:#07302d; background:rgba(255,255,255,.34); box-shadow:inset 0 0 0 1px rgba(255,255,255,.28); } +.auto-edit-generate:disabled .auto-edit-generate-icon { color:#68757b; background:rgba(255,255,255,.055); } +.auto-edit-generate > span:nth-child(2) { display:grid; gap:2px; min-width:0; } +.auto-edit-generate strong { overflow:hidden; font-size:12px; font-weight:800; text-overflow:ellipsis; white-space:nowrap; } +.auto-edit-generate small { overflow:hidden; color:rgba(5,38,35,.68); font-size:9px; font-weight:650; text-overflow:ellipsis; white-space:nowrap; } +.auto-edit-generate:disabled small { color:#66747a; } +.auto-edit-generate-arrow { font-size:18px; font-weight:700; transition:transform 160ms ease; } +.auto-edit-generate:hover:not(:disabled) .auto-edit-generate-arrow { transform:translateX(3px); } +.auto-edit-review-backdrop { position:fixed; inset:0; z-index:120; display:grid; place-items:center; padding:28px; background:rgba(3,7,11,.78); backdrop-filter:blur(9px); animation:auto-edit-backdrop-in 160ms ease-out; } +.auto-edit-review-dialog { display:grid; grid-template-rows:auto 3px minmax(0,1fr) auto; width:min(960px,calc(100vw - 56px)); height:min(720px,calc(100vh - 56px)); overflow:hidden; border:1px solid rgba(117,177,199,.25); border-radius:18px; color:#dce8eb; background:linear-gradient(145deg,rgba(18,25,33,.99),rgba(9,14,20,.99)); box-shadow:0 38px 120px rgba(0,0,0,.68),inset 0 1px 0 rgba(255,255,255,.05); animation:auto-edit-dialog-in 180ms ease-out; } +.auto-edit-review-header { display:grid; grid-template-columns:42px minmax(0,1fr) auto 34px; align-items:center; gap:12px; min-height:76px; padding:13px 18px; border-bottom:1px solid rgba(255,255,255,.07); } +.auto-edit-review-mark { display:grid; place-items:center; width:40px; height:40px; border:1px solid rgba(53,234,217,.4); border-radius:12px; color:#041817; background:linear-gradient(145deg,#71b8ff,#39ead8); box-shadow:0 8px 24px rgba(53,234,217,.16); } +.auto-edit-review-header > div:nth-child(2) { display:grid; gap:2px; } +.auto-edit-review-header > div:nth-child(2) > span { color:#5fddd1; font-size:9px; font-weight:750; letter-spacing:.15em; text-transform:uppercase; } +.auto-edit-review-header h2 { margin:0; color:#f2f8f9; font-size:17px; letter-spacing:-.02em; } +.auto-edit-review-status { display:flex; align-items:center; gap:7px; border:1px solid rgba(255,255,255,.08); border-radius:999px; padding:6px 10px; color:#91a0a8; font-size:10px; background:rgba(255,255,255,.035); } +.auto-edit-review-status i { width:6px; height:6px; border-radius:50%; background:#e5bd68; box-shadow:0 0 9px rgba(229,189,104,.7); animation:auto-edit-pulse 1.1s ease-in-out infinite; } +.auto-edit-review-status.is-complete { color:#78e6d9; border-color:rgba(53,234,217,.24); background:rgba(53,234,217,.07); } +.auto-edit-review-status.is-complete i { background:#35ead9; box-shadow:0 0 9px rgba(53,234,217,.8); animation:none; } +.auto-edit-review-status.is-error { color:#ffaaaa; border-color:rgba(255,104,117,.25); background:rgba(255,83,99,.07); } +.auto-edit-review-status.is-error i { background:#ff6875; animation:none; } +.auto-edit-review-close { display:grid; place-items:center; width:32px; height:32px; border:0; border-radius:8px; color:#80909a; background:transparent; cursor:pointer; } +.auto-edit-review-close:hover { color:#f3f8f9; background:rgba(255,255,255,.07); } +.auto-edit-review-progress { overflow:hidden; background:rgba(255,255,255,.04); } +.auto-edit-review-progress span { display:block; height:100%; border-radius:inherit; background:linear-gradient(90deg,#68aaff,#35ead9); box-shadow:0 0 16px rgba(53,234,217,.55); transition:width 220ms ease; } +.auto-edit-review-body { display:grid; grid-template-columns:minmax(0,1.16fr) minmax(320px,.84fr); gap:1px; min-height:0; overflow:hidden; background:rgba(255,255,255,.07); } +.auto-edit-review-section { min-width:0; overflow:auto; padding:19px; background:#0d131a; scrollbar-width:thin; scrollbar-color:#33434e transparent; } +.auto-edit-review-section-title { display:flex; align-items:center; justify-content:space-between; gap:12px; } +.auto-edit-review-section-title > div { display:flex; align-items:center; gap:9px; } +.auto-edit-review-section-title > div > span { display:grid; place-items:center; width:26px; height:26px; border:1px solid rgba(53,234,217,.24); border-radius:8px; color:#35ead9; font-size:9px; font-weight:800; background:rgba(53,234,217,.06); } +.auto-edit-review-section-title strong { color:#e8f1f3; font-size:13px; } +.auto-edit-review-section-title em { color:#6d7d87; font-size:9px; font-style:normal; } +.auto-edit-review-section > p { margin:7px 0 16px 35px; color:#6d7d87; font-size:10px; line-height:1.5; } +.auto-edit-candidate-grid { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:11px; } +.auto-edit-candidate-card { overflow:hidden; border:1px solid rgba(255,255,255,.08); border-radius:11px; background:rgba(255,255,255,.025); transition:border-color 150ms ease,transform 150ms ease; } +.auto-edit-candidate-card:hover { transform:translateY(-2px); border-color:rgba(53,234,217,.34); } +.auto-edit-candidate-card > div { position:relative; aspect-ratio:1/1; overflow:hidden; background:#000; } +.auto-edit-candidate-card img { width:100%; height:100%; object-fit:contain; } +.auto-edit-candidate-card > div span,.auto-edit-candidate-card > div time { position:absolute; top:8px; border:1px solid rgba(255,255,255,.12); border-radius:6px; padding:4px 6px; color:#eaf4f4; font-size:8px; background:rgba(5,10,15,.72); backdrop-filter:blur(5px); } +.auto-edit-candidate-card > div span { left:8px; color:#42e7d6; } +.auto-edit-candidate-card > div time { right:8px; font-variant-numeric:tabular-nums; } +.auto-edit-candidate-card > div em { position:absolute; right:8px; bottom:8px; border:1px solid rgba(255,255,255,.12); border-radius:6px; padding:4px 6px; color:#aebcc2; font-size:8px; font-style:normal; background:rgba(5,10,15,.72); backdrop-filter:blur(5px); } +.auto-edit-candidate-card footer { display:grid; grid-template-columns:auto auto; gap:6px; align-items:center; padding:8px 9px; color:#72818b; font-size:8px; } +.auto-edit-candidate-card footer strong { justify-self:end; color:#9cabb2; font-variant-numeric:tabular-nums; } +.auto-edit-candidate-card footer i { grid-column:1/-1; height:3px; overflow:hidden; border-radius:9px; background:rgba(255,255,255,.06); } +.auto-edit-candidate-card footer b { display:block; height:100%; border-radius:inherit; background:linear-gradient(90deg,#6aa9ff,#35ead9); } +.auto-edit-model-results { background:#0b1117; } +.auto-edit-result-list { display:grid; gap:8px; } +.auto-edit-result-list article { display:grid; grid-template-columns:28px minmax(0,1fr); gap:9px; padding:10px; border:1px solid rgba(255,255,255,.07); border-radius:9px; background:rgba(255,255,255,.025); } +.auto-edit-result-list article > span { display:grid; place-items:center; width:27px; height:27px; border-radius:7px; color:#76e5d9; font-size:9px; font-weight:750; background:rgba(53,234,217,.08); } +.auto-edit-result-list article div { display:grid; gap:5px; min-width:0; } +.auto-edit-result-list p { margin:0; color:#dce7ea; font-size:11px; line-height:1.45; } +.auto-edit-result-list time { color:#667984; font-size:8px; font-variant-numeric:tabular-nums; } +.auto-edit-clip-results { display:grid; gap:10px; } +.auto-edit-clip-result { overflow:hidden; border:1px solid rgba(255,255,255,.075); border-radius:11px; background:rgba(255,255,255,.02); } +.auto-edit-clip-result.is-running { border-color:rgba(229,189,104,.28); box-shadow:inset 3px 0 0 rgba(229,189,104,.65); } +.auto-edit-clip-result.is-complete { border-color:rgba(53,234,217,.2); box-shadow:inset 3px 0 0 rgba(53,234,217,.65); } +.auto-edit-clip-result.is-error { border-color:rgba(255,104,117,.22); box-shadow:inset 3px 0 0 rgba(255,104,117,.62); } +.auto-edit-clip-result > header { display:grid; grid-template-columns:52px minmax(0,1fr) auto; align-items:center; gap:9px; min-height:50px; padding:7px 9px; border-bottom:1px solid rgba(255,255,255,.055); background:rgba(255,255,255,.018); } +.auto-edit-clip-result > header img { width:52px; height:36px; border-radius:6px; object-fit:contain; background:#000; } +.auto-edit-clip-result > header div { display:grid; gap:3px; min-width:0; } +.auto-edit-clip-result > header strong { overflow:hidden; color:#dce7e9; font-size:10px; text-overflow:ellipsis; white-space:nowrap; } +.auto-edit-clip-result > header span { color:#667680; font-size:8px; } +.auto-edit-clip-result > header em { border:1px solid rgba(255,255,255,.08); border-radius:999px; padding:4px 7px; color:#7d8c95; font-size:8px; font-style:normal; white-space:nowrap; } +.auto-edit-clip-result.is-running > header em { color:#d9bd7d; border-color:rgba(229,189,104,.2); background:rgba(229,189,104,.06); } +.auto-edit-clip-result.is-complete > header em { color:#69dbcf; border-color:rgba(53,234,217,.2); background:rgba(53,234,217,.06); } +.auto-edit-clip-result .auto-edit-result-list { gap:6px; padding:8px; } +.auto-edit-clip-result .auto-edit-result-list article { border:0; padding:7px; background:rgba(255,255,255,.022); } +.auto-edit-clip-pending { display:flex; align-items:center; justify-content:center; gap:8px; min-height:62px; color:#61717b; font-size:9px; } +.auto-edit-clip-pending i { width:13px; height:13px; border:2px solid rgba(229,189,104,.15); border-top-color:#e5bd68; border-radius:50%; animation:auto-edit-spin .8s linear infinite; } +.auto-edit-clip-error { margin:0; padding:10px; color:#d18c94; font-size:9px; line-height:1.45; } +.auto-edit-review-loading { display:grid; place-items:center; align-content:center; gap:12px; min-height:220px; border:1px dashed rgba(255,255,255,.08); border-radius:11px; color:#667781; font-size:10px; background:rgba(255,255,255,.012); } +.auto-edit-review-loading i { width:25px; height:25px; border:2px solid rgba(53,234,217,.13); border-top-color:#35ead9; border-radius:50%; animation:auto-edit-spin .85s linear infinite; } +.auto-edit-review-error { display:grid; gap:5px; border:1px solid rgba(255,104,117,.2); border-radius:10px; padding:12px; color:#c4868d; font-size:10px; background:rgba(255,83,99,.055); } +.auto-edit-review-error strong { color:#ffadb4; font-size:11px; } +.auto-edit-review-actions { display:grid; grid-template-columns:minmax(0,1fr) auto auto; align-items:center; gap:10px; min-height:72px; padding:12px 18px; border-top:1px solid rgba(255,255,255,.07); background:rgba(8,13,18,.96); } +.auto-edit-review-actions > div { display:grid; gap:3px; } +.auto-edit-review-actions > div strong { color:#dfe9eb; font-size:11px; } +.auto-edit-review-actions > div span { color:#677883; font-size:9px; } +.auto-edit-review-actions .panel-secondary { min-width:78px; } +.auto-edit-apply { display:flex; align-items:center; justify-content:center; gap:7px; min-width:148px; min-height:38px; border:1px solid rgba(53,234,217,.7); border-radius:9px; color:#061817; font-size:11px; font-weight:800; background:linear-gradient(125deg,#69b6ff,#39ead7); box-shadow:0 8px 20px rgba(53,234,217,.15); cursor:pointer; } +.auto-edit-apply:disabled { border-color:rgba(255,255,255,.08); color:#5f6e75; background:rgba(255,255,255,.045); box-shadow:none; cursor:not-allowed; } +@keyframes auto-edit-spin { to { transform:rotate(360deg); } } +@keyframes auto-edit-pulse { 50% { opacity:.35; transform:scale(.75); } } +@keyframes auto-edit-backdrop-in { from { opacity:0; } } +@keyframes auto-edit-dialog-in { from { opacity:0; transform:translateY(10px) scale(.985); } } +@media (max-width:760px) { .auto-edit-review-body { grid-template-columns:1fr; overflow:auto; } .auto-edit-review-section { overflow:visible; } .auto-edit-review-dialog { height:calc(100vh - 24px); width:calc(100vw - 24px); } .auto-edit-review-actions { grid-template-columns:1fr 1fr; } .auto-edit-review-actions > div { grid-column:1/-1; } } .smart-vision-heading { display: flex; @@ -2460,9 +2558,9 @@ button:disabled { .preview-stage { display: grid; - grid-template-rows: 1fr 94px; + grid-template-rows: minmax(0, 1fr) 70px; gap: 0; - padding: 10px; + padding: 4px; overflow: hidden; background: linear-gradient(180deg, rgba(255, 255, 255, 0.02), transparent 42%), @@ -2474,7 +2572,7 @@ button:disabled { display: grid; place-items: center; min-height: 0; - padding: 20px 8px 12px; + padding: 3px 4px 2px; background: #171b22; } @@ -2545,6 +2643,9 @@ button:disabled { place-items: center; width: 100%; height: 100%; + min-width: 0; + min-height: 0; + overflow: hidden; } .visual-media-layer > img, @@ -2555,9 +2656,15 @@ button:disabled { .visual-media-layer > img, .visual-media-layer > video { + position: absolute; + inset: 0; display: block; width: 100%; height: 100%; + min-width: 0; + min-height: 0; + max-width: 100%; + max-height: 100%; object-fit: contain; object-position: center center; background: #080a0e; @@ -2772,7 +2879,7 @@ button:disabled { } .transport { - padding: 12px 12px 4px; + padding: 7px 10px 2px; } .scrubber, @@ -2787,7 +2894,7 @@ button:disabled { justify-content: space-between; gap: 10px; min-width: 0; - margin-top: 10px; + margin-top: 6px; flex-wrap: nowrap; } @@ -4868,6 +4975,21 @@ button:disabled { box-shadow: 0 18px 42px rgba(0, 0, 0, 0.35); } +@media (min-width: 1181px) and (max-width: 1600px) { + .editor-grid { + grid-template-columns: 74px minmax(270px, 0.66fr) minmax(540px, 1.62fr) minmax(280px, 0.76fr); + } + + .editor-grid.is-compact-rail { + grid-template-columns: 50px minmax(270px, 0.66fr) minmax(540px, 1.62fr) minmax(280px, 0.76fr); + } + + .preview-stage { + padding-right: 4px; + padding-left: 4px; + } +} + @media (max-width: 1180px) { body { overflow: auto; diff --git a/src/workers/asr.worker.js b/src/workers/asr.worker.js index 3596228..1bbf494 100644 --- a/src/workers/asr.worker.js +++ b/src/workers/asr.worker.js @@ -125,7 +125,11 @@ function getPreferredInferenceDevice() { } async function createTranscriber(requestId, device) { - const { pipeline } = await import("@huggingface/transformers"); + const { env, pipeline } = await import("@huggingface/transformers"); + // The app's service worker is the single cache owner for large model files. + // A second Transformers Cache Storage copy can exhaust the site quota after + // other local AI workflows have already downloaded their models. + env.useBrowserCache = false; return pipeline("automatic-speech-recognition", AUTOMATIC_CAPTION_MODEL_ID, { dtype: "q8", device, @@ -157,8 +161,14 @@ async function getTranscriber(requestId) { })(), }; } - - return transcriberState.promise; + try { + return await transcriberState.promise; + } catch (error) { + // Never retain a rejected initialization promise: the next click must be + // able to retry after a transient download, quota, or runtime failure. + transcriberState = null; + throw error; + } } async function detectWhisperLanguage(transcriber, audio, preferredLanguage, requestId) { diff --git a/src/workers/auto-edit.worker.js b/src/workers/auto-edit.worker.js new file mode 100644 index 0000000..aff28a5 --- /dev/null +++ b/src/workers/auto-edit.worker.js @@ -0,0 +1,94 @@ +let previousPixels = null; +let previousSegmentId = ""; + +function luminance(pixels, width, x, y) { + const index = (y * width + x) * 4; + return pixels[index] * 0.299 + pixels[index + 1] * 0.587 + pixels[index + 2] * 0.114; +} + +function opticalFlowDifference(current, previous, width, height) { + if (!previous) return { difference: 1, motion: 0, sceneChange: 1, quality: frameQuality(current, width, height) }; + const block = 8; + const search = 4; + let motion = 0; + let residual = 0; + let samples = 0; + for (let y = search; y < height - block - search; y += block) { + for (let x = search; x < width - block - search; x += block) { + let bestError = Infinity; + let bestDistance = 0; + let texture = 0; + const center = luminance(current, width, x + block / 2, y + block / 2); + for (let by = 0; by < block; by += 2) for (let bx = 0; bx < block; bx += 2) texture += Math.abs(luminance(current, width, x + bx, y + by) - center); + for (let dy = -search; dy <= search; dy += 2) { + for (let dx = -search; dx <= search; dx += 2) { + let error = 0; + for (let by = 0; by < block; by += 2) { + for (let bx = 0; bx < block; bx += 2) { + error += Math.abs(luminance(current, width, x + bx, y + by) - luminance(previous, width, x + bx + dx, y + by + dy)); + } + } + // Prefer zero displacement when several matches are effectively equal. + const distance = Math.hypot(dx, dy); + if (error + distance * 1.5 < bestError) { bestError = error + distance * 1.5; bestDistance = distance; } + } + } + motion += texture > 180 ? bestDistance / Math.hypot(search, search) : 0; + residual += bestError / ((block / 2) ** 2 * 255); + samples += 1; + } + } + const motionScore = samples ? Math.min(1, motion / samples) : 0; + const residualScore = samples ? Math.min(1, residual / samples) : 1; + const sceneChange = histogramDistance(current, previous, width, height); + const quality = frameQuality(current, width, height); + return { difference: Math.min(1, sceneChange * 0.62 + residualScore * 0.25 + motionScore * 0.13), motion: motionScore, sceneChange, quality }; +} + +function histogramDistance(current, previous, width, height) { + const a = new Uint32Array(32); + const b = new Uint32Array(32); + for (let y = 0; y < height; y += 4) for (let x = 0; x < width; x += 4) { + const index = (y * width + x) * 4; + const currentLuma = current[index] * .299 + current[index + 1] * .587 + current[index + 2] * .114; + const previousLuma = previous[index] * .299 + previous[index + 1] * .587 + previous[index + 2] * .114; + // Ignore letterbox pixels so aspect-ratio padding cannot become a scene signal. + if (currentLuma > 5) a[Math.min(31, currentLuma >> 3)] += 1; + if (previousLuma > 5) b[Math.min(31, previousLuma >> 3)] += 1; + } + const totalA = a.reduce((sum, value) => sum + value, 0) || 1; + const totalB = b.reduce((sum, value) => sum + value, 0) || 1; + let distance = 0; + for (let index = 0; index < a.length; index += 1) distance += Math.abs(a[index] / totalA - b[index] / totalB); + return Math.min(1, distance / 2); +} + +function frameQuality(pixels, width, height) { + let visible = 0; + let contrast = 0; + let count = 0; + for (let y = 0; y < height; y += 4) for (let x = 0; x < width; x += 4) { + const value = luminance(pixels, width, x, y); + if (value > 8) visible += 1; + if (x + 4 < width) contrast += Math.abs(value - luminance(pixels, width, x + 4, y)); + count += 1; + } + const coverage = visible / Math.max(1, count); + return Math.min(1, coverage * .75 + Math.min(1, contrast / Math.max(1, count) / 24) * .25); +} + +self.onmessage = (event) => { + const { type, id, pixels, width, height, segmentId } = event.data || {}; + if (type === "reset") { + previousPixels = null; + previousSegmentId = ""; + return; + } + if (type !== "analyze") return; + const current = new Uint8ClampedArray(pixels); + const previous = segmentId === previousSegmentId ? previousPixels : null; + const metrics = opticalFlowDifference(current, previous, width, height); + previousPixels = current; + previousSegmentId = segmentId; + self.postMessage({ id, ...metrics }); +}; diff --git a/src/workers/vision.worker.js b/src/workers/vision.worker.js index 089d0f0..896ea30 100644 --- a/src/workers/vision.worker.js +++ b/src/workers/vision.worker.js @@ -67,6 +67,7 @@ function createModelLoadProgressCallback(requestId, { start, end, label }) { function getTransformers() { transformersPromise ??= import("@huggingface/transformers").then((transformers) => { + transformers.env.useBrowserCache = false; if (transformers.env?.backends?.onnx?.wasm) { transformers.env.backends.onnx.wasm.numThreads = 1; }