From 58ce75d55f32ea6ed3f390e90dfbcb9a0eaac080 Mon Sep 17 00:00:00 2001 From: "haixin.yang" Date: Tue, 21 Jul 2026 18:04:16 +0800 Subject: [PATCH] Improve voice generation and caption audio linking --- e2e/caption-audio-link.spec.js | 118 ++++++++++++++++++++++++ e2e/mobile-timeline-pinch.spec.js | 124 ++++++++++++++++++++++++++ src/App.jsx | 16 +++- src/components/Timeline.jsx | 84 ++++++++++++++--- src/components/VoicePanel.jsx | 43 ++++++++- src/components/panels.jsx | 15 +++- src/hooks/useVoiceGeneration.js | 41 +++++---- src/i18n.js | 39 +++++--- src/lib/captionEditingActions.js | 99 ++++++++++++++++++++ src/lib/captionEditingActions.test.js | 51 ++++++++++- src/lib/mobileClipActions.js | 4 +- src/lib/mobileClipActions.test.js | 11 ++- src/lib/piperVoiceRuntime.js | 3 + src/styles.css | 98 ++++++++++++++++++-- 14 files changed, 688 insertions(+), 58 deletions(-) create mode 100644 e2e/caption-audio-link.spec.js diff --git a/e2e/caption-audio-link.spec.js b/e2e/caption-audio-link.spec.js new file mode 100644 index 0000000..87622fa --- /dev/null +++ b/e2e/caption-audio-link.spec.js @@ -0,0 +1,118 @@ +import { expect, test } from "@playwright/test"; + +const ONE_PIXEL_PNG = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZQmcAAAAASUVORK5CYII=", + "base64", +); + +function createSilentWav(durationSeconds = 2) { + const sampleRate = 8_000; + const sampleCount = Math.round(sampleRate * durationSeconds); + const buffer = Buffer.alloc(44 + sampleCount * 2); + buffer.write("RIFF", 0); buffer.writeUInt32LE(buffer.length - 8, 4); buffer.write("WAVEfmt ", 8); + buffer.writeUInt32LE(16, 16); buffer.writeUInt16LE(1, 20); buffer.writeUInt16LE(1, 22); + buffer.writeUInt32LE(sampleRate, 24); buffer.writeUInt32LE(sampleRate * 2, 28); + buffer.writeUInt16LE(2, 32); buffer.writeUInt16LE(16, 34); buffer.write("data", 36); + buffer.writeUInt32LE(sampleCount * 2, 40); + return buffer; +} + +async function dragHorizontally(page, locator, deltaX) { + const box = await locator.boundingBox(); + if (!box) throw new Error("Timeline clip geometry is unavailable"); + const x = box.x + box.width / 2; + const y = box.y + box.height / 2; + await page.mouse.move(x, y); + await page.mouse.down(); + await page.mouse.move(x + deltaX, y, { steps: 8 }); + await page.mouse.up(); +} + +test("caption voiceover link can align, unlink, and relink without deleting either clip", async ({ page }) => { + const pageErrors = []; + page.on("pageerror", (error) => pageErrors.push(error.message)); + await page.setViewportSize({ width: 412, height: 915 }); + await page.addInitScript(() => { + localStorage.clear(); + localStorage.setItem("ai-voiceover-ui-language", "en"); + }); + await page.goto("/"); + + const input = page.locator('input[type="file"][multiple]'); + await input.setInputFiles({ name: "visual.png", mimeType: "image/png", buffer: ONE_PIXEL_PNG }); + await page.keyboard.press("Escape"); + await page.getByText("Media", { exact: true }).last().click(); + await input.setInputFiles({ name: "linked-voice.wav", mimeType: "audio/wav", buffer: createSilentWav(2) }); + await page.getByRole("button", { name: "Add to voiceover" }).click(); + await expect(page.locator(".audio-clip:not(.is-source):not(.is-music)")).toHaveCount(1); + + await page.setViewportSize({ width: 1440, height: 900 }); + await page.getByRole("button", { name: "Captions", exact: true }).click(); + await page.getByRole("button", { name: "Add caption", exact: true }).click(); + const caption = page.locator(".caption-segment"); + const voice = page.locator(".audio-clip:not(.is-source):not(.is-music)"); + await expect(caption).toHaveCount(1); + + const linkPanel = page.getByTestId("caption-audio-link"); + await expect(linkPanel).toContainText("No linked voiceover"); + await linkPanel.getByRole("button", { name: "Link audio", exact: true }).click(); + await expect(linkPanel).toContainText("Linked voiceover"); + + await caption.click({ button: "right" }); + const contextMenu = page.getByRole("menu", { name: "Timeline context menu" }); + await expect(contextMenu.getByRole("menuitem", { name: "Unlink", exact: true })).toBeVisible(); + await expect(contextMenu.getByRole("menuitem", { name: "Align to audio", exact: true })).toBeVisible(); + await page.keyboard.press("Escape"); + await voice.click({ button: "right" }); + await expect(contextMenu.getByRole("menuitem", { name: "Unlink", exact: true })).toBeVisible(); + await expect(contextMenu.getByRole("menuitem", { name: "Align to audio", exact: true })).toBeVisible(); + await page.keyboard.press("Escape"); + + const captionLeftBeforeLinkedMove = await caption.evaluate((element) => element.style.getPropertyValue("--caption-left")); + await dragHorizontally(page, voice, 90); + const captionLeftAfterLinkedMove = await caption.evaluate((element) => element.style.getPropertyValue("--caption-left")); + expect(captionLeftAfterLinkedMove).not.toBe(captionLeftBeforeLinkedMove); + + await caption.click(); + await expect(linkPanel).toContainText("Linked voiceover"); + await linkPanel.getByRole("button", { name: "Unlink", exact: true }).click(); + await expect(linkPanel).toContainText("No linked voiceover"); + await expect(voice).toHaveCount(1); + await expect(caption).toHaveCount(1); + + const captionLeftBeforeUnlinkedMove = await caption.evaluate((element) => element.style.getPropertyValue("--caption-left")); + await dragHorizontally(page, voice, 70); + const captionLeftAfterUnlinkedMove = await caption.evaluate((element) => element.style.getPropertyValue("--caption-left")); + expect(captionLeftAfterUnlinkedMove).toBe(captionLeftBeforeUnlinkedMove); + + await caption.click(); + await linkPanel.getByRole("button", { name: "Link audio", exact: true }).click(); + await linkPanel.getByRole("button", { name: "Align to audio", exact: true }).click(); + const [captionBox, voiceBox] = await Promise.all([caption.boundingBox(), voice.boundingBox()]); + expect(Math.abs(captionBox.x - voiceBox.x)).toBeLessThan(2); + expect(Math.abs(captionBox.width - voiceBox.width)).toBeLessThan(2); + + await page.setViewportSize({ width: 412, height: 915 }); + await caption.click(); + const mobileActions = page.locator(".timeline-mobile-clip-actions"); + await expect(mobileActions.locator("button")).toHaveText([ + "Back", "Edit", "Split", "Copy", "Unlink", "Align to audio", "Delete", + ]); + const back = mobileActions.getByRole("button", { name: "Back", exact: true }); + const backLeftBeforeScroll = (await back.boundingBox()).x; + const actionScroller = mobileActions.locator(".timeline-mobile-clip-action-scroller"); + const actionsBox = await actionScroller.boundingBox(); + await page.mouse.move(actionsBox.x + actionsBox.width - 20, actionsBox.y + actionsBox.height / 2); + await page.mouse.wheel(500, 0); + await expect.poll(() => actionScroller.evaluate((element) => element.scrollLeft)).toBeGreaterThan(0); + expect(Math.abs((await back.boundingBox()).x - backLeftBeforeScroll)).toBeLessThan(1); + await expect(mobileActions.getByRole("button", { name: "Unlink", exact: true })).toBeVisible(); + await expect(mobileActions.getByRole("button", { name: "Align to audio", exact: true })).toBeVisible(); + await expect(mobileActions.getByRole("button", { name: "Delete", exact: true })).toBeVisible(); + await page.mouse.wheel(-500, 0); + await expect.poll(() => actionScroller.evaluate((element) => element.scrollLeft)).toBe(0); + await mobileActions.getByRole("button", { name: "Edit", exact: true }).click(); + await expect(page.getByTestId("caption-audio-link")).toBeVisible(); + await expect(page.getByTestId("caption-audio-link").getByRole("button", { name: "Align to audio", exact: true })).toBeVisible(); + expect(pageErrors).toEqual([]); +}); diff --git a/e2e/mobile-timeline-pinch.spec.js b/e2e/mobile-timeline-pinch.spec.js index 6fcaf5e..42ce3ec 100644 --- a/e2e/mobile-timeline-pinch.spec.js +++ b/e2e/mobile-timeline-pinch.spec.js @@ -123,6 +123,130 @@ test("desktop timeline keeps its existing draggable playhead behavior", async ({ await expect(page.locator(".mobile-fixed-playhead")).toBeHidden(); }); +test("desktop trackpad zoom keeps ruler and track geometry synchronized before commit", async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 900 }); + await page.addInitScript(() => localStorage.setItem("ai-voiceover-ui-language", "zh")); + await page.goto("/"); + await page.locator('input[type="file"][multiple]').setInputFiles({ + name: "desktop-wheel-fixture.png", + mimeType: "image/png", + buffer: ONE_PIXEL_PNG, + }); + await page.keyboard.press("Escape"); + + const clip = page.locator(".image-clip"); + await expect(clip).toBeVisible(); + const box = await clip.boundingBox(); + if (!box) throw new Error("Timeline clip is unavailable"); + + const readGeometry = () => page.evaluate(() => { + const track = document.querySelector(".track-scroll")?.getBoundingClientRect(); + const ruler = document.querySelector(".timeline-ruler-canvas")?.getBoundingClientRect(); + const trackPlayhead = document.querySelector(".playhead")?.getBoundingClientRect(); + const rulerPlayhead = document.querySelector(".playhead-ruler")?.getBoundingClientRect(); + return { + trackWidth: track?.width || 0, + rulerWidth: ruler?.width || 0, + trackPlayheadX: trackPlayhead?.left || 0, + rulerPlayheadX: rulerPlayhead?.left || 0, + }; + }); + + const before = await readGeometry(); + await clip.dispatchEvent("wheel", { + deltaY: -120, + deltaX: 0, + deltaMode: 0, + ctrlKey: true, + clientX: box.x + box.width / 2, + clientY: box.y + box.height / 2, + bubbles: true, + cancelable: true, + }); + await page.waitForTimeout(32); + const during = await readGeometry(); + await page.waitForTimeout(260); + const committed = await readGeometry(); + + expect(during.trackWidth).toBeGreaterThan(before.trackWidth); + expect(during.rulerWidth).toBeCloseTo(during.trackWidth, 0); + expect(during.rulerPlayheadX).toBeCloseTo(during.trackPlayheadX, 0); + expect(committed.rulerWidth).toBeCloseTo(committed.trackWidth, 0); + expect(committed.rulerPlayheadX).toBeCloseTo(committed.trackPlayheadX, 0); +}); + +test("desktop horizontal scrolling keeps the ruler playhead aligned on every scroll event", async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 900 }); + await page.addInitScript(() => localStorage.setItem("ai-voiceover-ui-language", "zh")); + await page.goto("/"); + await page.locator('input[type="file"][multiple]').setInputFiles({ + name: "desktop-scroll-fixture.png", + mimeType: "image/png", + buffer: ONE_PIXEL_PNG, + }); + await page.keyboard.press("Escape"); + await expect(page.locator(".image-clip")).toBeVisible(); + + await page.getByRole("button", { name: "放大时间线" }).click(); + await page.getByRole("button", { name: "放大时间线" }).click(); + await page.getByRole("button", { name: "放大时间线" }).click(); + await page.waitForTimeout(180); + + const offsets = await page.locator(".tracks").evaluate((element) => { + const samples = []; + for (const ratio of [0.2, 0.5, 0.8, 0.35]) { + element.scrollLeft = (element.scrollWidth - element.clientWidth) * ratio; + element.dispatchEvent(new Event("scroll", { bubbles: true })); + const trackPlayhead = document.querySelector(".playhead")?.getBoundingClientRect(); + const rulerPlayhead = document.querySelector(".playhead-ruler")?.getBoundingClientRect(); + samples.push(Math.abs((trackPlayhead?.left || 0) - (rulerPlayhead?.left || 0))); + } + return samples; + }); + + for (const offset of offsets) expect(offset).toBeLessThanOrEqual(1); +}); + +test("desktop fast trackpad wheel scrolling advances ruler and tracks in the same frame", async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 900 }); + await page.addInitScript(() => localStorage.setItem("ai-voiceover-ui-language", "zh")); + await page.goto("/"); + await page.locator('input[type="file"][multiple]').setInputFiles({ + name: "desktop-fast-scroll-fixture.png", + mimeType: "image/png", + buffer: ONE_PIXEL_PNG, + }); + await page.keyboard.press("Escape"); + const clip = page.locator(".image-clip"); + await expect(clip).toBeVisible(); + await page.getByRole("button", { name: "放大时间线" }).click(); + await page.getByRole("button", { name: "放大时间线" }).click(); + await page.getByRole("button", { name: "放大时间线" }).click(); + await page.waitForTimeout(180); + + const box = await clip.boundingBox(); + if (!box) throw new Error("Timeline clip is unavailable"); + const offsets = []; + for (const deltaX of [42, 88, 136, -54, 172, -96]) { + await clip.dispatchEvent("wheel", { + deltaX, + deltaY: 2, + deltaMode: 0, + clientX: box.x + Math.min(40, box.width / 2), + clientY: box.y + box.height / 2, + bubbles: true, + cancelable: true, + }); + offsets.push(await page.evaluate(() => { + const trackPlayhead = document.querySelector(".playhead")?.getBoundingClientRect(); + const rulerPlayhead = document.querySelector(".playhead-ruler")?.getBoundingClientRect(); + return Math.abs((trackPlayhead?.left || 0) - (rulerPlayhead?.left || 0)); + })); + } + + for (const offset of offsets) expect(offset).toBeLessThanOrEqual(1); +}); + test("mobile trackpad wheel zoom uses pixels and does not jump when committed", async ({ page }) => { await page.setViewportSize({ width: 412, height: 915 }); await page.addInitScript(() => localStorage.setItem("ai-voiceover-ui-language", "zh")); diff --git a/src/App.jsx b/src/App.jsx index c6d6717..b3044ce 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -382,15 +382,16 @@ export function App() { }); const { - commitCaptionSegments, deleteCaptionSegment, handleCaptionPositionChange, + alignAudioCaptions, alignCaptionToAudio, commitCaptionSegments, deleteCaptionSegment, handleCaptionPositionChange, + linkAudioToCaption, linkCaptionAudio, startCaptionDrag, toggleCaptionSegmentHidden, - updateCaptionSegmentText, updateScript, + unlinkAudioCaptions, unlinkCaptionAudio, updateCaptionSegmentText, updateScript, } = createCaptionEditingActions({ audioSegments, captionSegments, currentCaptionSegment, focusedSegmentIndex, notify, previewCanvasRef, previewVisionKey, previewVisionRecord, script, selectedSegmentId, setCaptionPlacement, setCaptionPosition, setCaptionSegments, setScript, setSelectedSegmentId, setSelectedTrack, - setVisionRecords, trackLocks, + setVisionRecords, t, trackLocks, }); const importCaptionSegments = (importedSegments, mode, skipped = 0) => { const nextSegments = mode === "append" @@ -1021,6 +1022,9 @@ export function App() { currentSegmentIndex={currentSegmentIndex} captionTargetDuration={captionTargetDuration} updateCaptionSegmentText={updateCaptionSegmentText} + alignCaptionToAudio={alignCaptionToAudio} + linkCaptionAudio={linkCaptionAudio} + unlinkCaptionAudio={unlinkCaptionAudio} toggleCaptionSegmentHidden={toggleCaptionSegmentHidden} deleteCaptionSegment={deleteCaptionSegment} importCaptionSegments={importCaptionSegments} @@ -1108,6 +1112,12 @@ export function App() { openMobileTools={() => changeMobilePanel("tools")} openMobileFilePicker={() => fileInputRef.current?.click()} requestCaptionVoiceFocus={() => setCaptionVoiceFocusRequest((request) => request + 1)} + alignCaptionToAudio={alignCaptionToAudio} + linkCaptionAudio={linkCaptionAudio} + unlinkCaptionAudio={unlinkCaptionAudio} + alignAudioCaptions={alignAudioCaptions} + linkAudioToCaption={linkAudioToCaption} + unlinkAudioCaptions={unlinkAudioCaptions} trackVisibility={trackVisibility} toggleTrackVisibility={toggleTrackVisibility} trackLocks={trackLocks} diff --git a/src/components/Timeline.jsx b/src/components/Timeline.jsx index 28e9d3b..02dc3b7 100644 --- a/src/components/Timeline.jsx +++ b/src/components/Timeline.jsx @@ -140,6 +140,12 @@ export function Timeline({ openMobileTools, openMobileFilePicker, requestCaptionVoiceFocus, + alignCaptionToAudio, + linkCaptionAudio, + unlinkCaptionAudio, + alignAudioCaptions, + linkAudioToCaption, + unlinkAudioCaptions, trackVisibility, toggleTrackVisibility, trackLocks, @@ -272,11 +278,37 @@ export function Timeline({ const selectedMobileVisualSegment = selectedMobileClipTrack === "image" ? displayedVisualSegments.find((segment) => segment.id === selectedVisualSegmentId) ?? null : null; + const selectedMobileCaptionSegment = selectedMobileClipTrack === "caption" + ? displayedCaptionSegments.find((segment) => segment.id === selectedSegmentId) ?? null + : null; + const selectedMobileAudioHasLinkedCaption = selectedMobileClipTrack === "audio" && selectedMobileAudioSegment + ? displayedCaptionSegments.some((caption) => caption.audioSegmentId === selectedMobileAudioSegment.id) + : false; + const selectedMobileHasLinkedCaption = selectedMobileClipTrack === "caption" + ? Boolean(selectedMobileCaptionSegment?.audioSegmentId) + : selectedMobileAudioHasLinkedCaption; const canExtractSelectedMobileSourceAudio = selectedMobileVisualSegment?.type === "video" && !Number.isFinite(selectedMobileVisualSegment.sourceAudioOffset); const mobileClipActionIds = getMobileClipActionIds(selectedMobileClipTrack, { canExtractSourceAudio: canExtractSelectedMobileSourceAudio, + hasLinkedCaption: selectedMobileHasLinkedCaption, }); + const toggleSelectedMobileCaptionAudioLink = () => { + if (selectedMobileClipTrack === "caption" && selectedMobileCaptionSegment) { + return selectedMobileCaptionSegment.audioSegmentId + ? unlinkCaptionAudio?.(selectedMobileCaptionSegment.id) + : linkCaptionAudio?.(selectedMobileCaptionSegment.id); + } + if (selectedMobileClipTrack === "audio" && selectedMobileAudioSegment) { + return selectedMobileAudioHasLinkedCaption + ? unlinkAudioCaptions?.(selectedMobileAudioSegment.id) + : linkAudioToCaption?.(selectedMobileAudioSegment.id); + } + }; + const alignSelectedMobileCaptionAudio = () => { + if (selectedMobileClipTrack === "caption" && selectedMobileCaptionSegment) alignCaptionToAudio?.(selectedMobileCaptionSegment.id); + if (selectedMobileClipTrack === "audio" && selectedMobileAudioSegment) alignAudioCaptions?.(selectedMobileAudioSegment.id); + }; const closeMobileClipActions = () => { setMobileClipActionsVisible(false); setMobileClipActionTrack(""); @@ -421,6 +453,12 @@ export function Timeline({ const contextAudioSegment = contextMenu?.track === "audio" && contextMenu.segmentId ? audioSegments.find((segment) => segment.id === contextMenu.segmentId) : null; + const contextCaptionSegment = contextMenu?.track === "caption" && contextMenu.segmentId + ? displayedCaptionSegments.find((segment) => segment.id === contextMenu.segmentId) + : null; + const contextAudioHasLinkedCaption = contextAudioSegment + ? displayedCaptionSegments.some((caption) => caption.audioSegmentId === contextAudioSegment.id) + : false; const contextMusicSegment = contextMenu?.track === "music" && contextMenu.segmentId ? (musicSegments.length ? musicSegments : [{ id: "music-audio", start: musicStartPercent / 100 * timelineDuration, duration: musicDuration, peaks: musicPeaks }]) .find((segment) => segment.id === contextMenu.segmentId) @@ -642,6 +680,7 @@ export function Timeline({ window.cancelAnimationFrame(wheelZoomFrameRef.current); } trackScrollRef.current?.classList.remove("is-wheel-zooming"); + rulerCanvasRef.current?.classList.remove("is-wheel-zooming"); wheelZoomActiveRef.current = false; window.clearTimeout(commitZoomTimerRef.current); }, @@ -760,7 +799,11 @@ export function Timeline({ wheelZoomActiveRef.current = true; timelineZoomRef.current = nextZoom; anchor.trackElement.classList.add("is-wheel-zooming"); + rulerCanvasRef.current?.classList.add("is-wheel-zooming"); anchor.trackElement.style.width = anchor.isMobile ? `${nextTrackWidth}px` : `${nextTrackWidthPercent}%`; + if (rulerCanvasRef.current) { + rulerCanvasRef.current.style.width = anchor.isMobile ? `${nextTrackWidth}px` : `${nextTrackWidthPercent}%`; + } anchor.trackElement.style.setProperty("--timeline-zoom", String(nextZoom)); if (anchor.isMobile) { const nextTrackRect = anchor.trackElement.getBoundingClientRect(); @@ -786,6 +829,7 @@ export function Timeline({ setTimelineZoom(nextZoom); window.requestAnimationFrame(() => { anchor.trackElement.classList.remove("is-wheel-zooming"); + rulerCanvasRef.current?.classList.remove("is-wheel-zooming"); rulerViewportSyncRef.current?.(); }); }, TIMELINE_WHEEL_ZOOM_COMMIT_DELAY); @@ -795,6 +839,22 @@ export function Timeline({ event.target instanceof Element && event.target.closest(TIMELINE_WHEEL_ZOOM_CONTENT_SELECTOR), ); const hasZoomModifier = event.ctrlKey || event.metaKey; + const trackElement = trackScrollRef.current; + const scrollElement = trackElement?.parentElement; + + // Keep desktop trackpad momentum on the main thread so the independently + // rendered ruler and the scrolling clips advance in the same frame. Native + // compositor scrolling can otherwise move the track layer one or more + // frames ahead of the sticky ruler during a fast two-finger swipe. + if (!hasZoomModifier && Math.abs(event.deltaX) > Math.abs(event.deltaY)) { + if (!scrollElement) return; + const deltaModeMultiplier = + event.deltaMode === 1 ? 16 : event.deltaMode === 2 ? scrollElement.clientWidth : 1; + event.preventDefault(); + scrollElement.scrollLeft += event.deltaX * deltaModeMultiplier; + rulerViewportSyncRef.current?.(); + return; + } if (!hasZoomModifier && !isOverTimelineContent) { if (!event.shiftKey && Math.abs(event.deltaY) >= Math.abs(event.deltaX)) { @@ -807,12 +867,6 @@ export function Timeline({ return; } - if (!hasZoomModifier && Math.abs(event.deltaX) > Math.abs(event.deltaY)) { - return; - } - - const trackElement = trackScrollRef.current; - const scrollElement = trackElement?.parentElement; if (!trackElement || !scrollElement) { return; } @@ -1398,7 +1452,7 @@ export function Timeline({
{rulerTicks.map((tick) => ( @@ -2004,18 +2058,22 @@ export function Timeline({ ) : null} {mobileClipActionsVisible && selectedMobileClipTrack && typeof document !== "undefined" ? createPortal(( ), document.body) : null} {contextMenu ? ( @@ -2025,10 +2083,14 @@ export function Timeline({ {contextMenu.kind === "clip" ? ( <> {contextMenu.track === "caption" ? ( - + {contextCaptionSegment ? <> + + {contextCaptionSegment.audioSegmentId ? : null} + : null} ) : null} {contextMenu.track === "image" && builtInImageCaptionAvailable && contextImageSegment && contextImageSegment.type !== "video" ? ( ) : null} {contextMenu.track === "audio" && contextAudioSegment ? <> + + {contextAudioHasLinkedCaption ? : null} : null} diff --git a/src/components/VoicePanel.jsx b/src/components/VoicePanel.jsx index 315aa04..52aa1cc 100644 --- a/src/components/VoicePanel.jsx +++ b/src/components/VoicePanel.jsx @@ -4,6 +4,8 @@ import { Eye, EyeSlash, ImageSquare, + Link, + LinkBreak, ListBullets, PersonSimpleRun, Plus, @@ -21,6 +23,7 @@ import { formatTime, getSegmentStartTime } from "../lib/timeline.js"; import { LIVE_PORTRAIT_WEB_MODEL } from "../config/livePortrait.js"; import { probeLivePortraitWebEnvironment } from "../lib/livePortraitWeb.js"; import { getCaptionVoiceSegment } from "../lib/captionVoice.js"; +import { findCaptionAudioLinkTarget } from "../lib/captionEditingActions.js"; import { normalizeVisualKeyframes } from "../lib/visualEffects.js"; import { MAX_SRT_FILE_BYTES, parseSrt } from "../lib/subtitles.js"; import { HistoryPanel, MyVoicesPanel, SmartVisionPanel, VisualEffectsPanel, VoiceSynthesisPanel } from "./panels.jsx"; @@ -118,6 +121,10 @@ function CaptionContextPanel({ automaticCaptionProgress, importCaptionSegments, addCaptionSegment, + alignCaptionToAudio, + linkCaptionAudio, + unlinkCaptionAudio, + audioSegments, }) { const srtInputRef = useRef(null); const focusNewCaptionRef = useRef(false); @@ -129,6 +136,8 @@ function CaptionContextPanel({ const selectedStart = captionSegments.length ? getSegmentStartTime(captionSegments, selectedIndex, captionTargetDuration) : 0; + const linkedAudioSegment = audioSegments.find((segment) => segment.id === selectedCaptionSegment?.audioSegmentId); + const relinkTarget = findCaptionAudioLinkTarget(selectedCaptionSegment, audioSegments); useEffect(() => { if (!focusNewCaptionRef.current || !selectedCaptionSegment) return; @@ -197,6 +206,28 @@ function CaptionContextPanel({
)} + {selectedCaptionSegment ? ( +
+
+ {linkedAudioSegment ? : } +
+ {linkedAudioSegment ? t("captionLinkedAudio") : t("captionAudioNotLinked")} + {linkedAudioSegment + ? `${linkedAudioSegment.name || t("audioClip")} · ${formatTime(linkedAudioSegment.duration)}` + : relinkTarget ? t("captionAudioRelinkHint") : t("captionAudioUnavailable")} +
+
+
+ {linkedAudioSegment ? <> + + + : ( + + )} +
+
+ ) : null} +
{status === "generating" ? ( -
- +
+