Improve voice generation and caption audio linking
This commit is contained in:
haixin.yang
@@ -0,0 +1,118 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
const ONE_PIXEL_PNG = Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZQmcAAAAASUVORK5CYII=",
|
||||
"base64",
|
||||
);
|
||||
|
||||
function createSilentWav(durationSeconds = 2) {
|
||||
const sampleRate = 8_000;
|
||||
const sampleCount = Math.round(sampleRate * durationSeconds);
|
||||
const buffer = Buffer.alloc(44 + sampleCount * 2);
|
||||
buffer.write("RIFF", 0); buffer.writeUInt32LE(buffer.length - 8, 4); buffer.write("WAVEfmt ", 8);
|
||||
buffer.writeUInt32LE(16, 16); buffer.writeUInt16LE(1, 20); buffer.writeUInt16LE(1, 22);
|
||||
buffer.writeUInt32LE(sampleRate, 24); buffer.writeUInt32LE(sampleRate * 2, 28);
|
||||
buffer.writeUInt16LE(2, 32); buffer.writeUInt16LE(16, 34); buffer.write("data", 36);
|
||||
buffer.writeUInt32LE(sampleCount * 2, 40);
|
||||
return buffer;
|
||||
}
|
||||
|
||||
async function dragHorizontally(page, locator, deltaX) {
|
||||
const box = await locator.boundingBox();
|
||||
if (!box) throw new Error("Timeline clip geometry is unavailable");
|
||||
const x = box.x + box.width / 2;
|
||||
const y = box.y + box.height / 2;
|
||||
await page.mouse.move(x, y);
|
||||
await page.mouse.down();
|
||||
await page.mouse.move(x + deltaX, y, { steps: 8 });
|
||||
await page.mouse.up();
|
||||
}
|
||||
|
||||
test("caption voiceover link can align, unlink, and relink without deleting either clip", async ({ page }) => {
|
||||
const pageErrors = [];
|
||||
page.on("pageerror", (error) => pageErrors.push(error.message));
|
||||
await page.setViewportSize({ width: 412, height: 915 });
|
||||
await page.addInitScript(() => {
|
||||
localStorage.clear();
|
||||
localStorage.setItem("ai-voiceover-ui-language", "en");
|
||||
});
|
||||
await page.goto("/");
|
||||
|
||||
const input = page.locator('input[type="file"][multiple]');
|
||||
await input.setInputFiles({ name: "visual.png", mimeType: "image/png", buffer: ONE_PIXEL_PNG });
|
||||
await page.keyboard.press("Escape");
|
||||
await page.getByText("Media", { exact: true }).last().click();
|
||||
await input.setInputFiles({ name: "linked-voice.wav", mimeType: "audio/wav", buffer: createSilentWav(2) });
|
||||
await page.getByRole("button", { name: "Add to voiceover" }).click();
|
||||
await expect(page.locator(".audio-clip:not(.is-source):not(.is-music)")).toHaveCount(1);
|
||||
|
||||
await page.setViewportSize({ width: 1440, height: 900 });
|
||||
await page.getByRole("button", { name: "Captions", exact: true }).click();
|
||||
await page.getByRole("button", { name: "Add caption", exact: true }).click();
|
||||
const caption = page.locator(".caption-segment");
|
||||
const voice = page.locator(".audio-clip:not(.is-source):not(.is-music)");
|
||||
await expect(caption).toHaveCount(1);
|
||||
|
||||
const linkPanel = page.getByTestId("caption-audio-link");
|
||||
await expect(linkPanel).toContainText("No linked voiceover");
|
||||
await linkPanel.getByRole("button", { name: "Link audio", exact: true }).click();
|
||||
await expect(linkPanel).toContainText("Linked voiceover");
|
||||
|
||||
await caption.click({ button: "right" });
|
||||
const contextMenu = page.getByRole("menu", { name: "Timeline context menu" });
|
||||
await expect(contextMenu.getByRole("menuitem", { name: "Unlink", exact: true })).toBeVisible();
|
||||
await expect(contextMenu.getByRole("menuitem", { name: "Align to audio", exact: true })).toBeVisible();
|
||||
await page.keyboard.press("Escape");
|
||||
await voice.click({ button: "right" });
|
||||
await expect(contextMenu.getByRole("menuitem", { name: "Unlink", exact: true })).toBeVisible();
|
||||
await expect(contextMenu.getByRole("menuitem", { name: "Align to audio", exact: true })).toBeVisible();
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
const captionLeftBeforeLinkedMove = await caption.evaluate((element) => element.style.getPropertyValue("--caption-left"));
|
||||
await dragHorizontally(page, voice, 90);
|
||||
const captionLeftAfterLinkedMove = await caption.evaluate((element) => element.style.getPropertyValue("--caption-left"));
|
||||
expect(captionLeftAfterLinkedMove).not.toBe(captionLeftBeforeLinkedMove);
|
||||
|
||||
await caption.click();
|
||||
await expect(linkPanel).toContainText("Linked voiceover");
|
||||
await linkPanel.getByRole("button", { name: "Unlink", exact: true }).click();
|
||||
await expect(linkPanel).toContainText("No linked voiceover");
|
||||
await expect(voice).toHaveCount(1);
|
||||
await expect(caption).toHaveCount(1);
|
||||
|
||||
const captionLeftBeforeUnlinkedMove = await caption.evaluate((element) => element.style.getPropertyValue("--caption-left"));
|
||||
await dragHorizontally(page, voice, 70);
|
||||
const captionLeftAfterUnlinkedMove = await caption.evaluate((element) => element.style.getPropertyValue("--caption-left"));
|
||||
expect(captionLeftAfterUnlinkedMove).toBe(captionLeftBeforeUnlinkedMove);
|
||||
|
||||
await caption.click();
|
||||
await linkPanel.getByRole("button", { name: "Link audio", exact: true }).click();
|
||||
await linkPanel.getByRole("button", { name: "Align to audio", exact: true }).click();
|
||||
const [captionBox, voiceBox] = await Promise.all([caption.boundingBox(), voice.boundingBox()]);
|
||||
expect(Math.abs(captionBox.x - voiceBox.x)).toBeLessThan(2);
|
||||
expect(Math.abs(captionBox.width - voiceBox.width)).toBeLessThan(2);
|
||||
|
||||
await page.setViewportSize({ width: 412, height: 915 });
|
||||
await caption.click();
|
||||
const mobileActions = page.locator(".timeline-mobile-clip-actions");
|
||||
await expect(mobileActions.locator("button")).toHaveText([
|
||||
"Back", "Edit", "Split", "Copy", "Unlink", "Align to audio", "Delete",
|
||||
]);
|
||||
const back = mobileActions.getByRole("button", { name: "Back", exact: true });
|
||||
const backLeftBeforeScroll = (await back.boundingBox()).x;
|
||||
const actionScroller = mobileActions.locator(".timeline-mobile-clip-action-scroller");
|
||||
const actionsBox = await actionScroller.boundingBox();
|
||||
await page.mouse.move(actionsBox.x + actionsBox.width - 20, actionsBox.y + actionsBox.height / 2);
|
||||
await page.mouse.wheel(500, 0);
|
||||
await expect.poll(() => actionScroller.evaluate((element) => element.scrollLeft)).toBeGreaterThan(0);
|
||||
expect(Math.abs((await back.boundingBox()).x - backLeftBeforeScroll)).toBeLessThan(1);
|
||||
await expect(mobileActions.getByRole("button", { name: "Unlink", exact: true })).toBeVisible();
|
||||
await expect(mobileActions.getByRole("button", { name: "Align to audio", exact: true })).toBeVisible();
|
||||
await expect(mobileActions.getByRole("button", { name: "Delete", exact: true })).toBeVisible();
|
||||
await page.mouse.wheel(-500, 0);
|
||||
await expect.poll(() => actionScroller.evaluate((element) => element.scrollLeft)).toBe(0);
|
||||
await mobileActions.getByRole("button", { name: "Edit", exact: true }).click();
|
||||
await expect(page.getByTestId("caption-audio-link")).toBeVisible();
|
||||
await expect(page.getByTestId("caption-audio-link").getByRole("button", { name: "Align to audio", exact: true })).toBeVisible();
|
||||
expect(pageErrors).toEqual([]);
|
||||
});
|
||||
@@ -123,6 +123,130 @@ test("desktop timeline keeps its existing draggable playhead behavior", async ({
|
||||
await expect(page.locator(".mobile-fixed-playhead")).toBeHidden();
|
||||
});
|
||||
|
||||
test("desktop trackpad zoom keeps ruler and track geometry synchronized before commit", async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1280, height: 900 });
|
||||
await page.addInitScript(() => localStorage.setItem("ai-voiceover-ui-language", "zh"));
|
||||
await page.goto("/");
|
||||
await page.locator('input[type="file"][multiple]').setInputFiles({
|
||||
name: "desktop-wheel-fixture.png",
|
||||
mimeType: "image/png",
|
||||
buffer: ONE_PIXEL_PNG,
|
||||
});
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
const clip = page.locator(".image-clip");
|
||||
await expect(clip).toBeVisible();
|
||||
const box = await clip.boundingBox();
|
||||
if (!box) throw new Error("Timeline clip is unavailable");
|
||||
|
||||
const readGeometry = () => page.evaluate(() => {
|
||||
const track = document.querySelector(".track-scroll")?.getBoundingClientRect();
|
||||
const ruler = document.querySelector(".timeline-ruler-canvas")?.getBoundingClientRect();
|
||||
const trackPlayhead = document.querySelector(".playhead")?.getBoundingClientRect();
|
||||
const rulerPlayhead = document.querySelector(".playhead-ruler")?.getBoundingClientRect();
|
||||
return {
|
||||
trackWidth: track?.width || 0,
|
||||
rulerWidth: ruler?.width || 0,
|
||||
trackPlayheadX: trackPlayhead?.left || 0,
|
||||
rulerPlayheadX: rulerPlayhead?.left || 0,
|
||||
};
|
||||
});
|
||||
|
||||
const before = await readGeometry();
|
||||
await clip.dispatchEvent("wheel", {
|
||||
deltaY: -120,
|
||||
deltaX: 0,
|
||||
deltaMode: 0,
|
||||
ctrlKey: true,
|
||||
clientX: box.x + box.width / 2,
|
||||
clientY: box.y + box.height / 2,
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
});
|
||||
await page.waitForTimeout(32);
|
||||
const during = await readGeometry();
|
||||
await page.waitForTimeout(260);
|
||||
const committed = await readGeometry();
|
||||
|
||||
expect(during.trackWidth).toBeGreaterThan(before.trackWidth);
|
||||
expect(during.rulerWidth).toBeCloseTo(during.trackWidth, 0);
|
||||
expect(during.rulerPlayheadX).toBeCloseTo(during.trackPlayheadX, 0);
|
||||
expect(committed.rulerWidth).toBeCloseTo(committed.trackWidth, 0);
|
||||
expect(committed.rulerPlayheadX).toBeCloseTo(committed.trackPlayheadX, 0);
|
||||
});
|
||||
|
||||
test("desktop horizontal scrolling keeps the ruler playhead aligned on every scroll event", async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1280, height: 900 });
|
||||
await page.addInitScript(() => localStorage.setItem("ai-voiceover-ui-language", "zh"));
|
||||
await page.goto("/");
|
||||
await page.locator('input[type="file"][multiple]').setInputFiles({
|
||||
name: "desktop-scroll-fixture.png",
|
||||
mimeType: "image/png",
|
||||
buffer: ONE_PIXEL_PNG,
|
||||
});
|
||||
await page.keyboard.press("Escape");
|
||||
await expect(page.locator(".image-clip")).toBeVisible();
|
||||
|
||||
await page.getByRole("button", { name: "放大时间线" }).click();
|
||||
await page.getByRole("button", { name: "放大时间线" }).click();
|
||||
await page.getByRole("button", { name: "放大时间线" }).click();
|
||||
await page.waitForTimeout(180);
|
||||
|
||||
const offsets = await page.locator(".tracks").evaluate((element) => {
|
||||
const samples = [];
|
||||
for (const ratio of [0.2, 0.5, 0.8, 0.35]) {
|
||||
element.scrollLeft = (element.scrollWidth - element.clientWidth) * ratio;
|
||||
element.dispatchEvent(new Event("scroll", { bubbles: true }));
|
||||
const trackPlayhead = document.querySelector(".playhead")?.getBoundingClientRect();
|
||||
const rulerPlayhead = document.querySelector(".playhead-ruler")?.getBoundingClientRect();
|
||||
samples.push(Math.abs((trackPlayhead?.left || 0) - (rulerPlayhead?.left || 0)));
|
||||
}
|
||||
return samples;
|
||||
});
|
||||
|
||||
for (const offset of offsets) expect(offset).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
test("desktop fast trackpad wheel scrolling advances ruler and tracks in the same frame", async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1280, height: 900 });
|
||||
await page.addInitScript(() => localStorage.setItem("ai-voiceover-ui-language", "zh"));
|
||||
await page.goto("/");
|
||||
await page.locator('input[type="file"][multiple]').setInputFiles({
|
||||
name: "desktop-fast-scroll-fixture.png",
|
||||
mimeType: "image/png",
|
||||
buffer: ONE_PIXEL_PNG,
|
||||
});
|
||||
await page.keyboard.press("Escape");
|
||||
const clip = page.locator(".image-clip");
|
||||
await expect(clip).toBeVisible();
|
||||
await page.getByRole("button", { name: "放大时间线" }).click();
|
||||
await page.getByRole("button", { name: "放大时间线" }).click();
|
||||
await page.getByRole("button", { name: "放大时间线" }).click();
|
||||
await page.waitForTimeout(180);
|
||||
|
||||
const box = await clip.boundingBox();
|
||||
if (!box) throw new Error("Timeline clip is unavailable");
|
||||
const offsets = [];
|
||||
for (const deltaX of [42, 88, 136, -54, 172, -96]) {
|
||||
await clip.dispatchEvent("wheel", {
|
||||
deltaX,
|
||||
deltaY: 2,
|
||||
deltaMode: 0,
|
||||
clientX: box.x + Math.min(40, box.width / 2),
|
||||
clientY: box.y + box.height / 2,
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
});
|
||||
offsets.push(await page.evaluate(() => {
|
||||
const trackPlayhead = document.querySelector(".playhead")?.getBoundingClientRect();
|
||||
const rulerPlayhead = document.querySelector(".playhead-ruler")?.getBoundingClientRect();
|
||||
return Math.abs((trackPlayhead?.left || 0) - (rulerPlayhead?.left || 0));
|
||||
}));
|
||||
}
|
||||
|
||||
for (const offset of offsets) expect(offset).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
test("mobile trackpad wheel zoom uses pixels and does not jump when committed", async ({ page }) => {
|
||||
await page.setViewportSize({ width: 412, height: 915 });
|
||||
await page.addInitScript(() => localStorage.setItem("ai-voiceover-ui-language", "zh"));
|
||||
|
||||
+13
-3
@@ -382,15 +382,16 @@ export function App() {
|
||||
});
|
||||
|
||||
const {
|
||||
commitCaptionSegments, deleteCaptionSegment, handleCaptionPositionChange,
|
||||
alignAudioCaptions, alignCaptionToAudio, commitCaptionSegments, deleteCaptionSegment, handleCaptionPositionChange,
|
||||
linkAudioToCaption, linkCaptionAudio,
|
||||
startCaptionDrag, toggleCaptionSegmentHidden,
|
||||
updateCaptionSegmentText, updateScript,
|
||||
unlinkAudioCaptions, unlinkCaptionAudio, updateCaptionSegmentText, updateScript,
|
||||
} = createCaptionEditingActions({
|
||||
audioSegments, captionSegments, currentCaptionSegment, focusedSegmentIndex,
|
||||
notify, previewCanvasRef, previewVisionKey, previewVisionRecord, script,
|
||||
selectedSegmentId, setCaptionPlacement, setCaptionPosition, setCaptionSegments,
|
||||
setScript, setSelectedSegmentId, setSelectedTrack,
|
||||
setVisionRecords, trackLocks,
|
||||
setVisionRecords, t, trackLocks,
|
||||
});
|
||||
const importCaptionSegments = (importedSegments, mode, skipped = 0) => {
|
||||
const nextSegments = mode === "append"
|
||||
@@ -1021,6 +1022,9 @@ export function App() {
|
||||
currentSegmentIndex={currentSegmentIndex}
|
||||
captionTargetDuration={captionTargetDuration}
|
||||
updateCaptionSegmentText={updateCaptionSegmentText}
|
||||
alignCaptionToAudio={alignCaptionToAudio}
|
||||
linkCaptionAudio={linkCaptionAudio}
|
||||
unlinkCaptionAudio={unlinkCaptionAudio}
|
||||
toggleCaptionSegmentHidden={toggleCaptionSegmentHidden}
|
||||
deleteCaptionSegment={deleteCaptionSegment}
|
||||
importCaptionSegments={importCaptionSegments}
|
||||
@@ -1108,6 +1112,12 @@ export function App() {
|
||||
openMobileTools={() => changeMobilePanel("tools")}
|
||||
openMobileFilePicker={() => fileInputRef.current?.click()}
|
||||
requestCaptionVoiceFocus={() => setCaptionVoiceFocusRequest((request) => request + 1)}
|
||||
alignCaptionToAudio={alignCaptionToAudio}
|
||||
linkCaptionAudio={linkCaptionAudio}
|
||||
unlinkCaptionAudio={unlinkCaptionAudio}
|
||||
alignAudioCaptions={alignAudioCaptions}
|
||||
linkAudioToCaption={linkAudioToCaption}
|
||||
unlinkAudioCaptions={unlinkAudioCaptions}
|
||||
trackVisibility={trackVisibility}
|
||||
toggleTrackVisibility={toggleTrackVisibility}
|
||||
trackLocks={trackLocks}
|
||||
|
||||
+74
-10
@@ -140,6 +140,12 @@ export function Timeline({
|
||||
openMobileTools,
|
||||
openMobileFilePicker,
|
||||
requestCaptionVoiceFocus,
|
||||
alignCaptionToAudio,
|
||||
linkCaptionAudio,
|
||||
unlinkCaptionAudio,
|
||||
alignAudioCaptions,
|
||||
linkAudioToCaption,
|
||||
unlinkAudioCaptions,
|
||||
trackVisibility,
|
||||
toggleTrackVisibility,
|
||||
trackLocks,
|
||||
@@ -272,11 +278,37 @@ export function Timeline({
|
||||
const selectedMobileVisualSegment = selectedMobileClipTrack === "image"
|
||||
? displayedVisualSegments.find((segment) => segment.id === selectedVisualSegmentId) ?? null
|
||||
: null;
|
||||
const selectedMobileCaptionSegment = selectedMobileClipTrack === "caption"
|
||||
? displayedCaptionSegments.find((segment) => segment.id === selectedSegmentId) ?? null
|
||||
: null;
|
||||
const selectedMobileAudioHasLinkedCaption = selectedMobileClipTrack === "audio" && selectedMobileAudioSegment
|
||||
? displayedCaptionSegments.some((caption) => caption.audioSegmentId === selectedMobileAudioSegment.id)
|
||||
: false;
|
||||
const selectedMobileHasLinkedCaption = selectedMobileClipTrack === "caption"
|
||||
? Boolean(selectedMobileCaptionSegment?.audioSegmentId)
|
||||
: selectedMobileAudioHasLinkedCaption;
|
||||
const canExtractSelectedMobileSourceAudio = selectedMobileVisualSegment?.type === "video"
|
||||
&& !Number.isFinite(selectedMobileVisualSegment.sourceAudioOffset);
|
||||
const mobileClipActionIds = getMobileClipActionIds(selectedMobileClipTrack, {
|
||||
canExtractSourceAudio: canExtractSelectedMobileSourceAudio,
|
||||
hasLinkedCaption: selectedMobileHasLinkedCaption,
|
||||
});
|
||||
const toggleSelectedMobileCaptionAudioLink = () => {
|
||||
if (selectedMobileClipTrack === "caption" && selectedMobileCaptionSegment) {
|
||||
return selectedMobileCaptionSegment.audioSegmentId
|
||||
? unlinkCaptionAudio?.(selectedMobileCaptionSegment.id)
|
||||
: linkCaptionAudio?.(selectedMobileCaptionSegment.id);
|
||||
}
|
||||
if (selectedMobileClipTrack === "audio" && selectedMobileAudioSegment) {
|
||||
return selectedMobileAudioHasLinkedCaption
|
||||
? unlinkAudioCaptions?.(selectedMobileAudioSegment.id)
|
||||
: linkAudioToCaption?.(selectedMobileAudioSegment.id);
|
||||
}
|
||||
};
|
||||
const alignSelectedMobileCaptionAudio = () => {
|
||||
if (selectedMobileClipTrack === "caption" && selectedMobileCaptionSegment) alignCaptionToAudio?.(selectedMobileCaptionSegment.id);
|
||||
if (selectedMobileClipTrack === "audio" && selectedMobileAudioSegment) alignAudioCaptions?.(selectedMobileAudioSegment.id);
|
||||
};
|
||||
const closeMobileClipActions = () => {
|
||||
setMobileClipActionsVisible(false);
|
||||
setMobileClipActionTrack("");
|
||||
@@ -421,6 +453,12 @@ export function Timeline({
|
||||
const contextAudioSegment = contextMenu?.track === "audio" && contextMenu.segmentId
|
||||
? audioSegments.find((segment) => segment.id === contextMenu.segmentId)
|
||||
: null;
|
||||
const contextCaptionSegment = contextMenu?.track === "caption" && contextMenu.segmentId
|
||||
? displayedCaptionSegments.find((segment) => segment.id === contextMenu.segmentId)
|
||||
: null;
|
||||
const contextAudioHasLinkedCaption = contextAudioSegment
|
||||
? displayedCaptionSegments.some((caption) => caption.audioSegmentId === contextAudioSegment.id)
|
||||
: false;
|
||||
const contextMusicSegment = contextMenu?.track === "music" && contextMenu.segmentId
|
||||
? (musicSegments.length ? musicSegments : [{ id: "music-audio", start: musicStartPercent / 100 * timelineDuration, duration: musicDuration, peaks: musicPeaks }])
|
||||
.find((segment) => segment.id === contextMenu.segmentId)
|
||||
@@ -642,6 +680,7 @@ export function Timeline({
|
||||
window.cancelAnimationFrame(wheelZoomFrameRef.current);
|
||||
}
|
||||
trackScrollRef.current?.classList.remove("is-wheel-zooming");
|
||||
rulerCanvasRef.current?.classList.remove("is-wheel-zooming");
|
||||
wheelZoomActiveRef.current = false;
|
||||
window.clearTimeout(commitZoomTimerRef.current);
|
||||
},
|
||||
@@ -760,7 +799,11 @@ export function Timeline({
|
||||
wheelZoomActiveRef.current = true;
|
||||
timelineZoomRef.current = nextZoom;
|
||||
anchor.trackElement.classList.add("is-wheel-zooming");
|
||||
rulerCanvasRef.current?.classList.add("is-wheel-zooming");
|
||||
anchor.trackElement.style.width = anchor.isMobile ? `${nextTrackWidth}px` : `${nextTrackWidthPercent}%`;
|
||||
if (rulerCanvasRef.current) {
|
||||
rulerCanvasRef.current.style.width = anchor.isMobile ? `${nextTrackWidth}px` : `${nextTrackWidthPercent}%`;
|
||||
}
|
||||
anchor.trackElement.style.setProperty("--timeline-zoom", String(nextZoom));
|
||||
if (anchor.isMobile) {
|
||||
const nextTrackRect = anchor.trackElement.getBoundingClientRect();
|
||||
@@ -786,6 +829,7 @@ export function Timeline({
|
||||
setTimelineZoom(nextZoom);
|
||||
window.requestAnimationFrame(() => {
|
||||
anchor.trackElement.classList.remove("is-wheel-zooming");
|
||||
rulerCanvasRef.current?.classList.remove("is-wheel-zooming");
|
||||
rulerViewportSyncRef.current?.();
|
||||
});
|
||||
}, TIMELINE_WHEEL_ZOOM_COMMIT_DELAY);
|
||||
@@ -795,6 +839,22 @@ export function Timeline({
|
||||
event.target instanceof Element && event.target.closest(TIMELINE_WHEEL_ZOOM_CONTENT_SELECTOR),
|
||||
);
|
||||
const hasZoomModifier = event.ctrlKey || event.metaKey;
|
||||
const trackElement = trackScrollRef.current;
|
||||
const scrollElement = trackElement?.parentElement;
|
||||
|
||||
// Keep desktop trackpad momentum on the main thread so the independently
|
||||
// rendered ruler and the scrolling clips advance in the same frame. Native
|
||||
// compositor scrolling can otherwise move the track layer one or more
|
||||
// frames ahead of the sticky ruler during a fast two-finger swipe.
|
||||
if (!hasZoomModifier && Math.abs(event.deltaX) > Math.abs(event.deltaY)) {
|
||||
if (!scrollElement) return;
|
||||
const deltaModeMultiplier =
|
||||
event.deltaMode === 1 ? 16 : event.deltaMode === 2 ? scrollElement.clientWidth : 1;
|
||||
event.preventDefault();
|
||||
scrollElement.scrollLeft += event.deltaX * deltaModeMultiplier;
|
||||
rulerViewportSyncRef.current?.();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!hasZoomModifier && !isOverTimelineContent) {
|
||||
if (!event.shiftKey && Math.abs(event.deltaY) >= Math.abs(event.deltaX)) {
|
||||
@@ -807,12 +867,6 @@ export function Timeline({
|
||||
return;
|
||||
}
|
||||
|
||||
if (!hasZoomModifier && Math.abs(event.deltaX) > Math.abs(event.deltaY)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const trackElement = trackScrollRef.current;
|
||||
const scrollElement = trackElement?.parentElement;
|
||||
if (!trackElement || !scrollElement) {
|
||||
return;
|
||||
}
|
||||
@@ -1398,7 +1452,7 @@ export function Timeline({
|
||||
<div
|
||||
ref={rulerCanvasRef}
|
||||
className="timeline-ruler-canvas"
|
||||
style={{ width: localTrackWidth, transform: `translateX(${-rulerViewport.scrollLeft}px)` }}
|
||||
style={{ width: localTrackWidth }}
|
||||
>
|
||||
<div className="ruler" onPointerDown={handleTimelineSurfacePointerDown}>
|
||||
{rulerTicks.map((tick) => (
|
||||
@@ -2004,18 +2058,22 @@ export function Timeline({
|
||||
) : null}
|
||||
{mobileClipActionsVisible && selectedMobileClipTrack && typeof document !== "undefined" ? createPortal((
|
||||
<nav className={`timeline-mobile-clip-actions ${mobileClipActionIds.length > 5 ? "is-scroll-actions" : ""}`} aria-label={t("clipActions")}>
|
||||
{mobileClipActionIds.map((actionId) => {
|
||||
if (actionId === "dismiss") return <button className="is-back" type="button" key={actionId} onClick={() => { closeMobileClipActions(); clearClipSelections(); }}><ArrowLeft size={21} /><span>{t("mobileClipDismiss")}</span></button>;
|
||||
<button className="is-back" type="button" onClick={() => { closeMobileClipActions(); clearClipSelections(); }}><ArrowLeft size={21} /><span>{t("mobileClipDismiss")}</span></button>
|
||||
<div className="timeline-mobile-clip-action-scroller">
|
||||
{mobileClipActionIds.filter((actionId) => actionId !== "dismiss").map((actionId) => {
|
||||
if (actionId === "edit") return <button type="button" key={actionId} onClick={openSelectedClipInspector}><SlidersHorizontal size={20} /><span>{t("mobileClipEdit")}</span></button>;
|
||||
if (actionId === "properties") return <button type="button" key={actionId} onClick={openSelectedClipInspector}><SlidersHorizontal size={20} /><span>{t("properties")}</span></button>;
|
||||
if (actionId === "audio") return <button type="button" key={actionId} onClick={openSelectedClipInspector}><SlidersHorizontal size={20} /><span>{t("mobileClipAudio")}</span></button>;
|
||||
if (actionId === "split") return <button type="button" key={actionId} onClick={() => runMobileClipAction(handleCutTrack)}><Scissors size={20} /><span>{t("mobileClipSplit")}</span></button>;
|
||||
if (actionId === "copy") return <button type="button" key={actionId} onClick={() => runMobileClipAction(handleDuplicateTrack)}><CopySimple size={20} /><span>{t("mobileClipCopy")}</span></button>;
|
||||
if (actionId === "captions") return <button type="button" key={actionId} disabled={audioProcessingBusy || !selectedMobileAudioSegment} onClick={generateSelectedMobileAudioCaptions}><ClosedCaptioning size={20} /><span>{t("mobileClipCaptions")}</span></button>;
|
||||
if (actionId === "caption-link") return <button type="button" key={actionId} onClick={toggleSelectedMobileCaptionAudioLink}>{selectedMobileHasLinkedCaption ? <LinkBreak size={20} /> : <LinkSimple size={20} />}<span>{t(selectedMobileHasLinkedCaption ? "captionUnlinkAudio" : "captionLinkAudio")}</span></button>;
|
||||
if (actionId === "caption-align") return <button type="button" key={actionId} onClick={alignSelectedMobileCaptionAudio}><ArrowsInLineHorizontal size={20} /><span>{t("captionAlignToAudio")}</span></button>;
|
||||
if (actionId === "separate") return <button type="button" key={actionId} disabled={audioProcessingBusy || !selectedMobileAudioSegment} onClick={separateSelectedMobileAudio}><Waveform size={20} /><span>{t("mobileClipSeparate")}</span></button>;
|
||||
if (actionId === "extract-source-audio") return <button type="button" key={actionId} disabled={Boolean(sourceAudioExtractionPendingId) || !selectedMobileVisualSegment} onClick={() => void runSourceAudioExtraction(selectedMobileVisualSegment)}>{sourceAudioExtractionPendingId === selectedMobileVisualSegment?.id ? <CircleNotch className="spin" size={20} /> : <Waveform size={20} />}<span>{t(sourceAudioExtractionPendingId === selectedMobileVisualSegment?.id ? "separatingSourceAudio" : "separateSourceAudio", sourceAudioExtractionPendingId === selectedMobileVisualSegment?.id ? "正在分离音频…" : "分离音频")}</span></button>;
|
||||
return <button className="is-danger" type="button" key={actionId} onClick={() => runMobileClipAction(handleDeleteTrack)}><Trash size={20} /><span>{t("mobileClipDelete")}</span></button>;
|
||||
})}
|
||||
</div>
|
||||
</nav>
|
||||
), document.body) : null}
|
||||
{contextMenu ? (
|
||||
@@ -2025,10 +2083,14 @@ export function Timeline({
|
||||
{contextMenu.kind === "clip" ? (
|
||||
<>
|
||||
{contextMenu.track === "caption" ? (
|
||||
<button type="button" role="menuitem" onClick={() => runContextAction(() => {
|
||||
<><button type="button" role="menuitem" onClick={() => runContextAction(() => {
|
||||
openTrackPanel("caption");
|
||||
requestCaptionVoiceFocus?.();
|
||||
})}><Waveform size={16} />{t("aiVoice")}</button>
|
||||
{contextCaptionSegment ? <>
|
||||
<button type="button" role="menuitem" onClick={() => runContextAction(() => contextCaptionSegment.audioSegmentId ? unlinkCaptionAudio?.(contextCaptionSegment.id) : linkCaptionAudio?.(contextCaptionSegment.id))}>{contextCaptionSegment.audioSegmentId ? <LinkBreak size={16} /> : <LinkSimple size={16} />}{t(contextCaptionSegment.audioSegmentId ? "captionUnlinkAudio" : "captionLinkAudio")}</button>
|
||||
{contextCaptionSegment.audioSegmentId ? <button type="button" role="menuitem" onClick={() => runContextAction(() => alignCaptionToAudio?.(contextCaptionSegment.id))}><ArrowsInLineHorizontal size={16} />{t("captionAlignToAudio")}</button> : null}
|
||||
</> : null}</>
|
||||
) : null}
|
||||
{contextMenu.track === "image" && builtInImageCaptionAvailable && contextImageSegment && contextImageSegment.type !== "video" ? (
|
||||
<button className={imageCaptionPendingId === contextImageSegment.id ? "is-loading" : ""} type="button" role="menuitem" disabled={Boolean(imageCaptionPendingId)} onClick={() => runImageCaptionAction(contextImageSegment)}>
|
||||
@@ -2047,6 +2109,8 @@ export function Timeline({
|
||||
<button type="button" role="menuitem" disabled={Boolean(trackLocks.overlay)} onClick={() => runContextAction(() => setVisualOverlaySegments((items) => items.map((item) => item.id === contextOverlaySegment.id ? { ...item, muted: !item.muted } : item)))}>{contextOverlaySegment.muted ? <SpeakerHigh size={16} /> : <SpeakerSlash size={16} />}{t(contextOverlaySegment.muted ? "unmuteClip" : "muteClip", contextOverlaySegment.muted ? "取消静音" : "静音")}</button>
|
||||
) : null}
|
||||
{contextMenu.track === "audio" && contextAudioSegment ? <>
|
||||
<button type="button" role="menuitem" onClick={() => runContextAction(() => contextAudioHasLinkedCaption ? unlinkAudioCaptions?.(contextAudioSegment.id) : linkAudioToCaption?.(contextAudioSegment.id))}>{contextAudioHasLinkedCaption ? <LinkBreak size={16} /> : <LinkSimple size={16} />}{t(contextAudioHasLinkedCaption ? "captionUnlinkAudio" : "captionLinkAudio")}</button>
|
||||
{contextAudioHasLinkedCaption ? <button type="button" role="menuitem" onClick={() => runContextAction(() => alignAudioCaptions?.(contextAudioSegment.id))}><ArrowsInLineHorizontal size={16} />{t("captionAlignToAudio")}</button> : null}
|
||||
<button type="button" role="menuitem" disabled={audioProcessingBusy} onClick={() => runContextAction(() => separateAudioClipVocals?.({ blob: contextAudioSegment.blob, name: contextAudioSegment.name, start: contextAudioSegment.start, sourceStart: contextAudioSegment.sourceStart || 0, duration: contextAudioSegment.duration, segmentId: contextAudioSegment.id, track: "audio" }))}><Waveform size={16} />{t("separateVocalsFromClip")}</button>
|
||||
<button type="button" role="menuitem" disabled={audioProcessingBusy} onClick={() => runContextAction(() => generateCaptionsFromAudioClip?.({ blob: contextAudioSegment.blob, start: contextAudioSegment.start, sourceStart: contextAudioSegment.sourceStart || 0, duration: contextAudioSegment.duration, append: true }))}><ClosedCaptioning size={16} />{t("generateCaptionsFromClip")}</button>
|
||||
</> : null}
|
||||
|
||||
@@ -4,6 +4,8 @@ import {
|
||||
Eye,
|
||||
EyeSlash,
|
||||
ImageSquare,
|
||||
Link,
|
||||
LinkBreak,
|
||||
ListBullets,
|
||||
PersonSimpleRun,
|
||||
Plus,
|
||||
@@ -21,6 +23,7 @@ import { formatTime, getSegmentStartTime } from "../lib/timeline.js";
|
||||
import { LIVE_PORTRAIT_WEB_MODEL } from "../config/livePortrait.js";
|
||||
import { probeLivePortraitWebEnvironment } from "../lib/livePortraitWeb.js";
|
||||
import { getCaptionVoiceSegment } from "../lib/captionVoice.js";
|
||||
import { findCaptionAudioLinkTarget } from "../lib/captionEditingActions.js";
|
||||
import { normalizeVisualKeyframes } from "../lib/visualEffects.js";
|
||||
import { MAX_SRT_FILE_BYTES, parseSrt } from "../lib/subtitles.js";
|
||||
import { HistoryPanel, MyVoicesPanel, SmartVisionPanel, VisualEffectsPanel, VoiceSynthesisPanel } from "./panels.jsx";
|
||||
@@ -118,6 +121,10 @@ function CaptionContextPanel({
|
||||
automaticCaptionProgress,
|
||||
importCaptionSegments,
|
||||
addCaptionSegment,
|
||||
alignCaptionToAudio,
|
||||
linkCaptionAudio,
|
||||
unlinkCaptionAudio,
|
||||
audioSegments,
|
||||
}) {
|
||||
const srtInputRef = useRef(null);
|
||||
const focusNewCaptionRef = useRef(false);
|
||||
@@ -129,6 +136,8 @@ function CaptionContextPanel({
|
||||
const selectedStart = captionSegments.length
|
||||
? getSegmentStartTime(captionSegments, selectedIndex, captionTargetDuration)
|
||||
: 0;
|
||||
const linkedAudioSegment = audioSegments.find((segment) => segment.id === selectedCaptionSegment?.audioSegmentId);
|
||||
const relinkTarget = findCaptionAudioLinkTarget(selectedCaptionSegment, audioSegments);
|
||||
|
||||
useEffect(() => {
|
||||
if (!focusNewCaptionRef.current || !selectedCaptionSegment) return;
|
||||
@@ -197,6 +206,28 @@ function CaptionContextPanel({
|
||||
</div>
|
||||
)}
|
||||
|
||||
{selectedCaptionSegment ? (
|
||||
<section className={`caption-audio-link ${linkedAudioSegment ? "is-linked" : ""}`} data-testid="caption-audio-link">
|
||||
<div>
|
||||
<span>{linkedAudioSegment ? <Link size={16} /> : <LinkBreak size={16} />}</span>
|
||||
<div>
|
||||
<strong>{linkedAudioSegment ? t("captionLinkedAudio") : t("captionAudioNotLinked")}</strong>
|
||||
<em>{linkedAudioSegment
|
||||
? `${linkedAudioSegment.name || t("audioClip")} · ${formatTime(linkedAudioSegment.duration)}`
|
||||
: relinkTarget ? t("captionAudioRelinkHint") : t("captionAudioUnavailable")}</em>
|
||||
</div>
|
||||
</div>
|
||||
<div className="caption-audio-link-actions">
|
||||
{linkedAudioSegment ? <>
|
||||
<button type="button" onClick={() => alignCaptionToAudio(selectedCaptionSegment.id)}>{t("captionAlignToAudio")}</button>
|
||||
<button type="button" className="is-unlink" onClick={() => unlinkCaptionAudio(selectedCaptionSegment.id)}>{t("captionUnlinkAudio")}</button>
|
||||
</> : (
|
||||
<button type="button" disabled={!relinkTarget} onClick={() => linkCaptionAudio(selectedCaptionSegment.id)}>{t("captionLinkAudio")}</button>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
) : null}
|
||||
|
||||
<div className="caption-context-actions">
|
||||
<button
|
||||
className="panel-secondary"
|
||||
@@ -550,6 +581,9 @@ export function VoicePanel({
|
||||
updateVisualOverlaySegment,
|
||||
deleteVisualOverlay,
|
||||
applyVisualOverlayPreset,
|
||||
alignCaptionToAudio,
|
||||
linkCaptionAudio,
|
||||
unlinkCaptionAudio,
|
||||
}) {
|
||||
const [captionPanelTab, setCaptionPanelTab] = useState("caption");
|
||||
const panelRef = useRef(null);
|
||||
@@ -565,6 +599,7 @@ export function VoicePanel({
|
||||
const isStickerContext = selectedTrack === "sticker" && Boolean(selectedStickerSegment);
|
||||
const isOverlayContext = selectedTrack === "overlay" && Boolean(selectedVisualOverlay);
|
||||
const selectedCaptionAudioSegment = getCaptionVoiceSegment(audioSegments, selectedCaptionSegment);
|
||||
const localizedStatusText = statusText?.startsWith?.("tts") ? t(statusText) : statusText;
|
||||
const title = isSmartAutoContext ? t("smartAutoEdit") : isSmartFrameContext ? t("smartFrame") : isAvatarContext ? t("avatarTitle") : isOverlayContext ? t("pictureInPicture", "画中画") : isStickerContext ? t("stickerProperties") : isVisualContext ? t("visualPanelTitle") : isCaptionContext ? t("caption") : isAudioClipContext ? t("audioClipProperties") : t("aiVoice");
|
||||
const panelStatusText = isSmartAutoContext ? t(`autoEditStatus_${autoEdit?.support?.availability || "unknown"}`) : isSmartFrameContext ? (hasVisual ? t("smartVisualReady") : t("smartWaitingVisual")) : isCaptionContext
|
||||
? captionSegments.length
|
||||
@@ -584,7 +619,7 @@ export function VoicePanel({
|
||||
? `${formatTime(audioPropertySegment.duration)} · ${audioPropertySegment.start.toFixed(1)}s`
|
||||
: statusText === "模型待命"
|
||||
? t("modelReady")
|
||||
: statusText;
|
||||
: localizedStatusText;
|
||||
|
||||
useEffect(() => {
|
||||
if (!captionVoiceFocusRequest || !isCaptionContext) return;
|
||||
@@ -702,6 +737,10 @@ export function VoicePanel({
|
||||
deleteCaptionSegment={deleteCaptionSegment}
|
||||
importCaptionSegments={importCaptionSegments}
|
||||
addCaptionSegment={addCaptionSegment}
|
||||
alignCaptionToAudio={alignCaptionToAudio}
|
||||
linkCaptionAudio={linkCaptionAudio}
|
||||
unlinkCaptionAudio={unlinkCaptionAudio}
|
||||
audioSegments={audioSegments}
|
||||
seekTo={seekTo}
|
||||
sourceAudioBlob={sourceAudioBlob}
|
||||
generateCaptionsFromSourceAudio={generateCaptionsFromSourceAudio}
|
||||
@@ -733,6 +772,7 @@ export function VoicePanel({
|
||||
volume={volume}
|
||||
setVolume={setVolume}
|
||||
status={status}
|
||||
statusText={localizedStatusText}
|
||||
progressPercent={progressPercent}
|
||||
audioBlob={selectedCaptionAudioSegment?.blob ?? null}
|
||||
audioUrl={selectedCaptionAudioSegment?.url ?? ""}
|
||||
@@ -766,6 +806,7 @@ export function VoicePanel({
|
||||
volume={volume}
|
||||
setVolume={setVolume}
|
||||
status={status}
|
||||
statusText={localizedStatusText}
|
||||
progressPercent={progressPercent}
|
||||
audioBlob={audioBlob}
|
||||
audioUrl={audioUrl}
|
||||
|
||||
@@ -1361,6 +1361,7 @@ export function VoiceSynthesisPanel({
|
||||
volume,
|
||||
setVolume,
|
||||
status,
|
||||
statusText,
|
||||
progressPercent,
|
||||
audioBlob,
|
||||
audioUrl,
|
||||
@@ -1513,14 +1514,22 @@ export function VoiceSynthesisPanel({
|
||||
</div>
|
||||
|
||||
{status === "generating" ? (
|
||||
<div className="progress-track" aria-label={t("generationProgress")}>
|
||||
<span style={{ width: `${progressPercent}%` }} />
|
||||
<div className="voice-generation-loading" role="status" aria-live="polite">
|
||||
<i className="voice-generation-spinner" aria-hidden="true" />
|
||||
<div>
|
||||
<strong>{statusText || t("generating")}</strong>
|
||||
<span>{t("ttsFirstRunHint")}</span>
|
||||
</div>
|
||||
<em>{Math.round(progressPercent)}%</em>
|
||||
<div className="progress-track" aria-label={t("generationProgress")}>
|
||||
<span style={{ width: `${progressPercent}%` }} />
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="voice-actions">
|
||||
<button className="generate-button" type="button" disabled={status === "generating" || !script.trim()} onClick={generateVoiceover}>
|
||||
<Waveform size={18} weight="bold" />
|
||||
{status === "generating" ? <i className="generate-button-spinner" aria-hidden="true" /> : <Waveform size={18} weight="bold" />}
|
||||
{status === "generating" ? t("generating") : audioBlob ? t("regenerateVoice") : t("generateVoice")}
|
||||
</button>
|
||||
<button className="secondary-download" type="button" disabled={!audioBlob} onClick={() => audioBlob && downloadBlob(audioBlob, "ai-voiceover.wav")}>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useCallback } from "react";
|
||||
import { MODEL_ID } from "../config/editor.js";
|
||||
import { predictPiperVoice } from "../lib/piperVoiceRuntime.js";
|
||||
import { isBuiltInPinyinVoice, predictPiperVoice } from "../lib/piperVoiceRuntime.js";
|
||||
import { predictMmsVoice } from "../lib/mmsVoiceRuntime.js";
|
||||
import { clearPiperCacheIfStorageTight, isPiperSymbolError, isStorageQuotaError, prepareTextForVoice, TtsInputError } from "../lib/ttsText.js";
|
||||
|
||||
@@ -14,18 +14,23 @@ export function useVoiceGeneration(d) {
|
||||
const message = error instanceof TtsInputError ? d.t(error.code) : error instanceof Error ? error.message : d.t("ttsErrorVoiceMismatch");
|
||||
d.setStatus("error"); d.setStatusText(message); d.setProgress(0); d.notify(message); return;
|
||||
}
|
||||
d.setVoiceTab("synthesis"); d.setStatus("generating"); d.setStatusText(d.t("ttsStatusPreparingModel")); d.setProgress(6);
|
||||
d.setVoiceTab("synthesis"); d.setStatus("generating"); d.setStatusText("ttsStatusPreparingModel"); d.setProgress(6);
|
||||
if (prepared.warningKey) d.notify(d.t(prepared.warningKey));
|
||||
try {
|
||||
let blob;
|
||||
if (d.selectedVoice.engine === "piper") {
|
||||
const tts = await import("@diffusionstudio/vits-web");
|
||||
if (await clearPiperCacheIfStorageTight(tts)) d.notify(d.t("ttsNoticePiperCacheCleared"));
|
||||
// Xiao Ya and Chaowen use our dedicated ONNX runtime. Avoid loading the
|
||||
// separate vits-web bundle on their first run; it is only needed by the
|
||||
// remaining Piper catalog voices.
|
||||
const builtInPinyinVoice = isBuiltInPinyinVoice(d.selectedVoice.id);
|
||||
const tts = builtInPinyinVoice ? null : await import("@diffusionstudio/vits-web");
|
||||
if (tts && await clearPiperCacheIfStorageTight(tts)) d.notify(d.t("ttsNoticePiperCacheCleared"));
|
||||
d.setStatusText(d.selectedVoice.language === "中文"
|
||||
? d.t("ttsStatusLoadingChineseModel")
|
||||
: d.t("ttsStatusPreparingModel"));
|
||||
? "ttsStatusLoadingChineseModel"
|
||||
: "ttsStatusPreparingModel");
|
||||
const progress = (event) => {
|
||||
if (event?.backend) d.setStatusText(d.t(event.backend === "webgpu" ? "ttsStatusGeneratingWebGpu" : "ttsStatusGeneratingWasm"));
|
||||
if (event?.phase === "initializing") d.setStatusText("ttsStatusInitializingModel");
|
||||
if (event?.phase === "generating" || event?.backend) d.setStatusText(event.backend === "webgpu" ? "ttsStatusGeneratingWebGpu" : "ttsStatusGeneratingWasm");
|
||||
if (event?.total) {
|
||||
d.setProgress((current) => Math.max(
|
||||
current,
|
||||
@@ -37,19 +42,19 @@ export function useVoiceGeneration(d) {
|
||||
try { blob = await predictPiperVoice(tts, input, progress); }
|
||||
catch (error) {
|
||||
if (!isStorageQuotaError(error)) throw error;
|
||||
d.setStatusText(d.t("ttsStatusClearingCache")); await tts.flush?.(); blob = await predictPiperVoice(tts, input, progress);
|
||||
d.setStatusText("ttsStatusClearingCache"); await tts?.flush?.(); blob = await predictPiperVoice(tts, input, progress);
|
||||
}
|
||||
} else if (d.selectedVoice.engine === "mms") {
|
||||
d.setStatusText(d.t("ttsStatusPreparingModel"));
|
||||
d.setStatusText("ttsStatusPreparingModel");
|
||||
blob = await predictMmsVoice({ text: prepared.text, voiceId: d.selectedVoice.id }, (event) => {
|
||||
if (event?.backend) d.setStatusText(d.t("ttsStatusGeneratingWasm"));
|
||||
if (event?.backend) d.setStatusText("ttsStatusGeneratingWasm");
|
||||
if (Number.isFinite(event?.progress)) d.setProgress((current) => Math.max(current, Math.min(86, Math.round(event.progress))));
|
||||
});
|
||||
} else if (d.selectedVoice.engine === "supertonic") {
|
||||
d.setStatusText(d.t("ttsStatusPreparingModel"));
|
||||
d.setStatusText("ttsStatusPreparingModel");
|
||||
const { predictSupertonicVoice } = await import("../lib/supertonicVoiceRuntime.js");
|
||||
blob = await predictSupertonicVoice({ text: prepared.text, speed: d.speed }, (event) => {
|
||||
if (event?.backend) d.setStatusText(d.t("ttsStatusGeneratingWasm"));
|
||||
if (event?.backend) d.setStatusText("ttsStatusGeneratingWasm");
|
||||
if (Number.isFinite(event?.progress)) d.setProgress((current) => Math.max(current, Math.min(86, Math.round(event.progress))));
|
||||
});
|
||||
} else {
|
||||
@@ -61,7 +66,7 @@ export function useVoiceGeneration(d) {
|
||||
import("kokoro-js"),
|
||||
]);
|
||||
transformersEnv.useBrowserCache = false;
|
||||
d.setStatusText(d.t("ttsStatusLoadingKokoro"));
|
||||
d.setStatusText("ttsStatusLoadingKokoro");
|
||||
const progressCallback = (event) => { if (event?.progress) d.setProgress((current) => Math.max(current, Math.min(86, Math.max(10, Math.round(event.progress))))); };
|
||||
let tts;
|
||||
const webGpuAdapter = globalThis.navigator?.gpu
|
||||
@@ -69,22 +74,22 @@ export function useVoiceGeneration(d) {
|
||||
: null;
|
||||
if (webGpuAdapter) {
|
||||
try {
|
||||
d.setStatusText(d.t("ttsStatusLoadingWebGpu"));
|
||||
d.setStatusText("ttsStatusLoadingWebGpu");
|
||||
tts = await KokoroTTS.from_pretrained(MODEL_ID, { dtype: "fp32", device: "webgpu", progress_callback: progressCallback });
|
||||
d.setStatusText(d.t("ttsStatusGeneratingWebGpu"));
|
||||
d.setStatusText("ttsStatusGeneratingWebGpu");
|
||||
blob = (await tts.generate(prepared.text, { voice: d.selectedVoice.id, speed: d.speed })).toBlob();
|
||||
} catch (error) {
|
||||
console.warn("Kokoro WebGPU failed; retrying with WASM.", error);
|
||||
d.setStatusText(d.t("ttsStatusFallingBackWasm"));
|
||||
d.setStatusText("ttsStatusFallingBackWasm");
|
||||
}
|
||||
}
|
||||
if (!blob) {
|
||||
tts = await KokoroTTS.from_pretrained(MODEL_ID, { dtype: "q8", device: "wasm", progress_callback: progressCallback });
|
||||
d.setStatusText(d.t("ttsStatusGeneratingWasm"));
|
||||
d.setStatusText("ttsStatusGeneratingWasm");
|
||||
blob = (await tts.generate(prepared.text, { voice: d.selectedVoice.id, speed: d.speed })).toBlob();
|
||||
}
|
||||
}
|
||||
d.setStatusText(d.t("ttsStatusDecodingWaveform"));
|
||||
d.setStatusText("ttsStatusDecodingWaveform");
|
||||
await d.commitAudio(blob, `${d.selectedVoice.name} · ${d.t("ttsGenerated")}`, {
|
||||
captionSegment,
|
||||
script: rawText,
|
||||
|
||||
+28
-11
@@ -300,16 +300,31 @@ Object.entries(TIMELINE_AUDIO_MENU_COPY).forEach(([language, copy]) => {
|
||||
});
|
||||
|
||||
const TTS_BACKEND_COPY = {
|
||||
zh: { ttsStatusLoadingWebGpu: "正在加载 WebGPU 配音模型", ttsStatusGeneratingWebGpu: "正在使用 WebGPU 生成配音", ttsStatusFallingBackWasm: "WebGPU 不可用，正在切换到 WASM", ttsStatusGeneratingWasm: "正在使用 WASM 生成配音" },
|
||||
en: { ttsStatusLoadingWebGpu: "Loading WebGPU voice model", ttsStatusGeneratingWebGpu: "Generating voice with WebGPU", ttsStatusFallingBackWasm: "WebGPU unavailable; switching to WASM", ttsStatusGeneratingWasm: "Generating voice with WASM" },
|
||||
ja: { ttsStatusLoadingWebGpu: "WebGPU音声モデルを読み込み中", ttsStatusGeneratingWebGpu: "WebGPUで音声を生成中", ttsStatusFallingBackWasm: "WebGPUを利用できないためWASMに切り替え中", ttsStatusGeneratingWasm: "WASMで音声を生成中" },
|
||||
ko: { ttsStatusLoadingWebGpu: "WebGPU 음성 모델 로드 중", ttsStatusGeneratingWebGpu: "WebGPU로 음성 생성 중", ttsStatusFallingBackWasm: "WebGPU를 사용할 수 없어 WASM으로 전환 중", ttsStatusGeneratingWasm: "WASM으로 음성 생성 중" },
|
||||
es: { ttsStatusLoadingWebGpu: "Cargando modelo de voz WebGPU", ttsStatusGeneratingWebGpu: "Generando voz con WebGPU", ttsStatusFallingBackWasm: "WebGPU no disponible; cambiando a WASM", ttsStatusGeneratingWasm: "Generando voz con WASM" },
|
||||
fr: { ttsStatusLoadingWebGpu: "Chargement du modèle vocal WebGPU", ttsStatusGeneratingWebGpu: "Génération vocale avec WebGPU", ttsStatusFallingBackWasm: "WebGPU indisponible ; passage à WASM", ttsStatusGeneratingWasm: "Génération vocale avec WASM" },
|
||||
de: { ttsStatusLoadingWebGpu: "WebGPU-Sprachmodell wird geladen", ttsStatusGeneratingWebGpu: "Stimme wird mit WebGPU erzeugt", ttsStatusFallingBackWasm: "WebGPU nicht verfügbar; Wechsel zu WASM", ttsStatusGeneratingWasm: "Stimme wird mit WASM erzeugt" },
|
||||
pt: { ttsStatusLoadingWebGpu: "Carregando modelo de voz WebGPU", ttsStatusGeneratingWebGpu: "Gerando voz com WebGPU", ttsStatusFallingBackWasm: "WebGPU indisponível; mudando para WASM", ttsStatusGeneratingWasm: "Gerando voz com WASM" },
|
||||
th: { ttsStatusLoadingWebGpu: "กำลังโหลดโมเดลเสียง WebGPU", ttsStatusGeneratingWebGpu: "กำลังสร้างเสียงด้วย WebGPU", ttsStatusFallingBackWasm: "ใช้ WebGPU ไม่ได้ กำลังเปลี่ยนเป็น WASM", ttsStatusGeneratingWasm: "กำลังสร้างเสียงด้วย WASM" },
|
||||
vi: { ttsStatusLoadingWebGpu: "Đang tải mô hình giọng nói WebGPU", ttsStatusGeneratingWebGpu: "Đang tạo giọng nói bằng WebGPU", ttsStatusFallingBackWasm: "Không dùng được WebGPU; đang chuyển sang WASM", ttsStatusGeneratingWasm: "Đang tạo giọng nói bằng WASM" },
|
||||
zh: { ttsStatusLoadingWebGpu: "正在加载 WebGPU 配音模型", ttsStatusGeneratingWebGpu: "正在使用 WebGPU 生成配音", ttsStatusFallingBackWasm: "WebGPU 不可用，正在切换到 WASM", ttsStatusGeneratingWasm: "正在使用 WASM 生成配音", ttsStatusInitializingModel: "正在初始化语音模型", ttsFirstRunHint: "首次生成需下载本地模型，完成后会缓存并明显加快" },
|
||||
en: { ttsStatusLoadingWebGpu: "Loading WebGPU voice model", ttsStatusGeneratingWebGpu: "Generating voice with WebGPU", ttsStatusFallingBackWasm: "WebGPU unavailable; switching to WASM", ttsStatusGeneratingWasm: "Generating voice with WASM", ttsStatusInitializingModel: "Initializing voice model", ttsFirstRunHint: "The first run downloads the local model; later runs will be much faster" },
|
||||
ja: { ttsStatusLoadingWebGpu: "WebGPU音声モデルを読み込み中", ttsStatusGeneratingWebGpu: "WebGPUで音声を生成中", ttsStatusFallingBackWasm: "WebGPUを利用できないためWASMに切り替え中", ttsStatusGeneratingWasm: "WASMで音声を生成中", ttsStatusInitializingModel: "音声モデルを初期化中", ttsFirstRunHint: "初回はローカルモデルをダウンロードします。次回以降は大幅に速くなります" },
|
||||
ko: { ttsStatusLoadingWebGpu: "WebGPU 음성 모델 로드 중", ttsStatusGeneratingWebGpu: "WebGPU로 음성 생성 중", ttsStatusFallingBackWasm: "WebGPU를 사용할 수 없어 WASM으로 전환 중", ttsStatusGeneratingWasm: "WASM으로 음성 생성 중", ttsStatusInitializingModel: "음성 모델 초기화 중", ttsFirstRunHint: "첫 실행 시 로컬 모델을 다운로드하며 이후 실행은 훨씬 빨라집니다" },
|
||||
es: { ttsStatusLoadingWebGpu: "Cargando modelo de voz WebGPU", ttsStatusGeneratingWebGpu: "Generando voz con WebGPU", ttsStatusFallingBackWasm: "WebGPU no disponible; cambiando a WASM", ttsStatusGeneratingWasm: "Generando voz con WASM", ttsStatusInitializingModel: "Inicializando el modelo de voz", ttsFirstRunHint: "La primera vez se descarga el modelo local; las siguientes serán mucho más rápidas" },
|
||||
fr: { ttsStatusLoadingWebGpu: "Chargement du modèle vocal WebGPU", ttsStatusGeneratingWebGpu: "Génération vocale avec WebGPU", ttsStatusFallingBackWasm: "WebGPU indisponible ; passage à WASM", ttsStatusGeneratingWasm: "Génération vocale avec WASM", ttsStatusInitializingModel: "Initialisation du modèle vocal", ttsFirstRunHint: "La première utilisation télécharge le modèle local ; les suivantes seront bien plus rapides" },
|
||||
de: { ttsStatusLoadingWebGpu: "WebGPU-Sprachmodell wird geladen", ttsStatusGeneratingWebGpu: "Stimme wird mit WebGPU erzeugt", ttsStatusFallingBackWasm: "WebGPU nicht verfügbar; Wechsel zu WASM", ttsStatusGeneratingWasm: "Stimme wird mit WASM erzeugt", ttsStatusInitializingModel: "Sprachmodell wird initialisiert", ttsFirstRunHint: "Beim ersten Mal wird das lokale Modell geladen; danach geht es deutlich schneller" },
|
||||
pt: { ttsStatusLoadingWebGpu: "Carregando modelo de voz WebGPU", ttsStatusGeneratingWebGpu: "Gerando voz com WebGPU", ttsStatusFallingBackWasm: "WebGPU indisponível; mudando para WASM", ttsStatusGeneratingWasm: "Gerando voz com WASM", ttsStatusInitializingModel: "Inicializando o modelo de voz", ttsFirstRunHint: "O primeiro uso baixa o modelo local; os próximos serão muito mais rápidos" },
|
||||
th: { ttsStatusLoadingWebGpu: "กำลังโหลดโมเดลเสียง WebGPU", ttsStatusGeneratingWebGpu: "กำลังสร้างเสียงด้วย WebGPU", ttsStatusFallingBackWasm: "ใช้ WebGPU ไม่ได้ กำลังเปลี่ยนเป็น WASM", ttsStatusGeneratingWasm: "กำลังสร้างเสียงด้วย WASM", ttsStatusInitializingModel: "กำลังเริ่มต้นโมเดลเสียง", ttsFirstRunHint: "ครั้งแรกจะดาวน์โหลดโมเดลลงเครื่อง และครั้งถัดไปจะเร็วขึ้นมาก" },
|
||||
vi: { ttsStatusLoadingWebGpu: "Đang tải mô hình giọng nói WebGPU", ttsStatusGeneratingWebGpu: "Đang tạo giọng nói bằng WebGPU", ttsStatusFallingBackWasm: "Không dùng được WebGPU; đang chuyển sang WASM", ttsStatusGeneratingWasm: "Đang tạo giọng nói bằng WASM", ttsStatusInitializingModel: "Đang khởi tạo mô hình giọng nói", ttsFirstRunHint: "Lần đầu sẽ tải mô hình cục bộ; những lần sau sẽ nhanh hơn nhiều" },
|
||||
ru: { ttsStatusLoadingWebGpu: "Загрузка голосовой модели WebGPU", ttsStatusGeneratingWebGpu: "Создание озвучки с WebGPU", ttsStatusFallingBackWasm: "WebGPU недоступен; переключение на WASM", ttsStatusGeneratingWasm: "Создание озвучки с WASM", ttsStatusInitializingModel: "Инициализация голосовой модели", ttsFirstRunHint: "При первом запуске загружается локальная модель; последующие будут намного быстрее" },
|
||||
};
|
||||
|
||||
const CAPTION_AUDIO_LINK_COPY = {
|
||||
zh: { captionLinkedAudio: "关联配音", captionAudioNotLinked: "未关联配音", captionAudioRelinkHint: "可重新关联之前的配音片段", captionAlignToAudio: "对齐到音频", captionUnlinkAudio: "取消关联", captionLinkAudio: "关联音频", captionAudioLinked: "字幕已关联配音", captionAudioUnlinked: "字幕与配音已取消关联", captionAlignedToAudio: "字幕时间已对齐到配音", captionAudioUnavailable: "没有可关联的配音片段" },
|
||||
en: { captionLinkedAudio: "Linked voiceover", captionAudioNotLinked: "No linked voiceover", captionAudioRelinkHint: "The previous voiceover can be linked again", captionAlignToAudio: "Align to audio", captionUnlinkAudio: "Unlink", captionLinkAudio: "Link audio", captionAudioLinked: "Caption linked to voiceover", captionAudioUnlinked: "Caption unlinked from voiceover", captionAlignedToAudio: "Caption timing aligned to voiceover", captionAudioUnavailable: "No voiceover clip is available to link" },
|
||||
ja: { captionLinkedAudio: "リンク済み音声", captionAudioNotLinked: "音声未リンク", captionAudioRelinkHint: "以前の音声に再リンクできます", captionAlignToAudio: "音声に合わせる", captionUnlinkAudio: "リンク解除", captionLinkAudio: "音声をリンク", captionAudioLinked: "字幕を音声にリンクしました", captionAudioUnlinked: "字幕と音声のリンクを解除しました", captionAlignedToAudio: "字幕時間を音声に合わせました", captionAudioUnavailable: "リンクできる音声クリップがありません" },
|
||||
ko: { captionLinkedAudio: "연결된 음성", captionAudioNotLinked: "연결된 음성 없음", captionAudioRelinkHint: "이전 음성에 다시 연결할 수 있습니다", captionAlignToAudio: "오디오에 맞추기", captionUnlinkAudio: "연결 해제", captionLinkAudio: "오디오 연결", captionAudioLinked: "자막을 음성에 연결했습니다", captionAudioUnlinked: "자막과 음성 연결을 해제했습니다", captionAlignedToAudio: "자막 시간을 음성에 맞췄습니다", captionAudioUnavailable: "연결할 음성 클립이 없습니다" },
|
||||
es: { captionLinkedAudio: "Voz vinculada", captionAudioNotLinked: "Sin voz vinculada", captionAudioRelinkHint: "Puedes volver a vincular la voz anterior", captionAlignToAudio: "Alinear con audio", captionUnlinkAudio: "Desvincular", captionLinkAudio: "Vincular audio", captionAudioLinked: "Subtítulo vinculado a la voz", captionAudioUnlinked: "Subtítulo desvinculado de la voz", captionAlignedToAudio: "Tiempo del subtítulo alineado con la voz", captionAudioUnavailable: "No hay clips de voz disponibles" },
|
||||
fr: { captionLinkedAudio: "Voix liée", captionAudioNotLinked: "Aucune voix liée", captionAudioRelinkHint: "La voix précédente peut être reliée", captionAlignToAudio: "Aligner sur l’audio", captionUnlinkAudio: "Dissocier", captionLinkAudio: "Lier l’audio", captionAudioLinked: "Sous-titre lié à la voix", captionAudioUnlinked: "Sous-titre dissocié de la voix", captionAlignedToAudio: "Sous-titre aligné sur la voix", captionAudioUnavailable: "Aucun clip vocal disponible" },
|
||||
de: { captionLinkedAudio: "Verknüpfte Stimme", captionAudioNotLinked: "Keine Stimme verknüpft", captionAudioRelinkHint: "Die vorherige Stimme kann erneut verknüpft werden", captionAlignToAudio: "An Audio ausrichten", captionUnlinkAudio: "Trennen", captionLinkAudio: "Audio verknüpfen", captionAudioLinked: "Untertitel mit Stimme verknüpft", captionAudioUnlinked: "Untertitel von Stimme getrennt", captionAlignedToAudio: "Untertitel an Stimme ausgerichtet", captionAudioUnavailable: "Kein Sprachclip zum Verknüpfen verfügbar" },
|
||||
pt: { captionLinkedAudio: "Voz vinculada", captionAudioNotLinked: "Sem voz vinculada", captionAudioRelinkHint: "A voz anterior pode ser vinculada novamente", captionAlignToAudio: "Alinhar ao áudio", captionUnlinkAudio: "Desvincular", captionLinkAudio: "Vincular áudio", captionAudioLinked: "Legenda vinculada à voz", captionAudioUnlinked: "Legenda desvinculada da voz", captionAlignedToAudio: "Tempo da legenda alinhado à voz", captionAudioUnavailable: "Nenhum clipe de voz disponível" },
|
||||
th: { captionLinkedAudio: "เสียงพากย์ที่เชื่อมโยง", captionAudioNotLinked: "ยังไม่ได้เชื่อมโยงเสียงพากย์", captionAudioRelinkHint: "สามารถเชื่อมโยงเสียงพากย์เดิมอีกครั้ง", captionAlignToAudio: "จัดให้ตรงกับเสียง", captionUnlinkAudio: "ยกเลิกการเชื่อมโยง", captionLinkAudio: "เชื่อมโยงเสียง", captionAudioLinked: "เชื่อมโยงคำบรรยายกับเสียงพากย์แล้ว", captionAudioUnlinked: "ยกเลิกการเชื่อมโยงคำบรรยายกับเสียงแล้ว", captionAlignedToAudio: "ปรับเวลาคำบรรยายให้ตรงกับเสียงแล้ว", captionAudioUnavailable: "ไม่มีคลิปเสียงพากย์ให้เชื่อมโยง" },
|
||||
vi: { captionLinkedAudio: "Lồng tiếng đã liên kết", captionAudioNotLinked: "Chưa liên kết lồng tiếng", captionAudioRelinkHint: "Có thể liên kết lại đoạn lồng tiếng trước", captionAlignToAudio: "Căn theo âm thanh", captionUnlinkAudio: "Hủy liên kết", captionLinkAudio: "Liên kết âm thanh", captionAudioLinked: "Đã liên kết phụ đề với lồng tiếng", captionAudioUnlinked: "Đã hủy liên kết phụ đề và lồng tiếng", captionAlignedToAudio: "Đã căn thời gian phụ đề theo lồng tiếng", captionAudioUnavailable: "Không có đoạn lồng tiếng để liên kết" },
|
||||
ru: { captionLinkedAudio: "Связанная озвучка", captionAudioNotLinked: "Озвучка не связана", captionAudioRelinkHint: "Можно снова связать прежнюю озвучку", captionAlignToAudio: "Выровнять по аудио", captionUnlinkAudio: "Отвязать", captionLinkAudio: "Связать аудио", captionAudioLinked: "Субтитр связан с озвучкой", captionAudioUnlinked: "Субтитр отвязан от озвучки", captionAlignedToAudio: "Время субтитра выровнено по озвучке", captionAudioUnavailable: "Нет доступного клипа озвучки" },
|
||||
};
|
||||
|
||||
const VISUAL_ANIMATION_COPY = {
|
||||
@@ -1876,10 +1891,12 @@ export function createTranslator(languageId) {
|
||||
const assetPreviewCopy = ASSET_PREVIEW_COPY[copyLanguage] ?? ASSET_PREVIEW_COPY.en;
|
||||
const assetDropCopy = ASSET_DROP_COPY[copyLanguage] ?? ASSET_DROP_COPY.en;
|
||||
const autoCaptionStatusCopy = AUTO_CAPTION_STATUS_COPY[copyLanguage] ?? AUTO_CAPTION_STATUS_COPY.en;
|
||||
const ttsBackendCopy = TTS_BACKEND_COPY[languageId] ?? TTS_BACKEND_COPY[copyLanguage] ?? TTS_BACKEND_COPY.en;
|
||||
const captionAudioLinkCopy = CAPTION_AUDIO_LINK_COPY[languageId] ?? CAPTION_AUDIO_LINK_COPY[copyLanguage] ?? CAPTION_AUDIO_LINK_COPY.en;
|
||||
const mobileDrawerCopy = MOBILE_DRAWER_COPY[languageId] ?? MOBILE_DRAWER_COPY.en;
|
||||
const mobileClipActionCopy = MOBILE_CLIP_ACTION_COPY[languageId] ?? MOBILE_CLIP_ACTION_COPY.en;
|
||||
const mobileStickerCopy = MOBILE_STICKER_COPY[languageId] ?? MOBILE_STICKER_COPY.en;
|
||||
return (key, fallbackText) => mobileStickerCopy[key] ?? MOBILE_STICKER_COPY.en[key] ?? mobileClipActionCopy[key] ?? MOBILE_CLIP_ACTION_COPY.en[key] ?? mobileDrawerCopy[key] ?? MOBILE_DRAWER_COPY.en[key] ?? srtImportCopy[key] ?? exportCopy[key] ?? EXPORT_RENDER_COPY.en[key] ?? assetPreviewCopy[key] ?? ASSET_PREVIEW_COPY.en[key] ?? assetDropCopy[key] ?? ASSET_DROP_COPY.en[key] ?? autoCaptionStatusCopy[key] ?? AUTO_CAPTION_STATUS_COPY.en[key] ?? copy[key] ?? fallback[key] ?? UI_COPY.zh[key] ?? fallbackText ?? key;
|
||||
return (key, fallbackText) => captionAudioLinkCopy[key] ?? CAPTION_AUDIO_LINK_COPY.en[key] ?? ttsBackendCopy[key] ?? TTS_BACKEND_COPY.en[key] ?? mobileStickerCopy[key] ?? MOBILE_STICKER_COPY.en[key] ?? mobileClipActionCopy[key] ?? MOBILE_CLIP_ACTION_COPY.en[key] ?? mobileDrawerCopy[key] ?? MOBILE_DRAWER_COPY.en[key] ?? srtImportCopy[key] ?? exportCopy[key] ?? EXPORT_RENDER_COPY.en[key] ?? assetPreviewCopy[key] ?? ASSET_PREVIEW_COPY.en[key] ?? assetDropCopy[key] ?? ASSET_DROP_COPY.en[key] ?? autoCaptionStatusCopy[key] ?? AUTO_CAPTION_STATUS_COPY.en[key] ?? copy[key] ?? fallback[key] ?? UI_COPY.zh[key] ?? fallbackText ?? key;
|
||||
}
|
||||
|
||||
export function translateOptionName(languageId, name) {
|
||||
|
||||
@@ -23,7 +23,24 @@ export function snapCaptionPlacement(x, y, thresholdX, thresholdY) {
|
||||
};
|
||||
}
|
||||
|
||||
export function findCaptionAudioLinkTarget(caption, audioSegments) {
|
||||
if (!caption || !audioSegments?.length) return null;
|
||||
const remembered = audioSegments.find((segment) => segment.id === caption.detachedAudioSegmentId);
|
||||
if (remembered) return remembered;
|
||||
const captionStart = Number(caption.start) || 0;
|
||||
const captionEnd = Number(caption.end) || captionStart;
|
||||
const captionCenter = (captionStart + captionEnd) / 2;
|
||||
return [...audioSegments].sort((a, b) => {
|
||||
const overlap = (segment) => Math.max(0, Math.min(captionEnd, segment.start + segment.duration) - Math.max(captionStart, segment.start));
|
||||
const overlapDelta = overlap(b) - overlap(a);
|
||||
if (overlapDelta) return overlapDelta;
|
||||
const center = (segment) => segment.start + segment.duration / 2;
|
||||
return Math.abs(center(a) - captionCenter) - Math.abs(center(b) - captionCenter);
|
||||
})[0] ?? null;
|
||||
}
|
||||
|
||||
export function createCaptionEditingActions(d) {
|
||||
const t = typeof d.t === "function" ? d.t : (key) => key;
|
||||
function updateScript(nextScript) {
|
||||
d.setScript(nextScript);
|
||||
}
|
||||
@@ -55,6 +72,82 @@ export function createCaptionEditingActions(d) {
|
||||
d.notify("字幕显示状态已更新");
|
||||
}
|
||||
|
||||
function unlinkCaptionAudio(segmentId) {
|
||||
if (d.trackLocks.caption) return void d.notify(t("captionTrackLocked"));
|
||||
d.setCaptionSegments((items) => items.map((segment) => segment.id === segmentId && segment.audioSegmentId
|
||||
? { ...segment, detachedAudioSegmentId: segment.audioSegmentId, audioSegmentId: "" }
|
||||
: segment));
|
||||
d.notify(t("captionAudioUnlinked"));
|
||||
}
|
||||
|
||||
function linkCaptionAudio(segmentId) {
|
||||
if (d.trackLocks.caption) return void d.notify(t("captionTrackLocked"));
|
||||
let linked = false;
|
||||
d.setCaptionSegments((items) => {
|
||||
const caption = items.find((segment) => segment.id === segmentId);
|
||||
const target = findCaptionAudioLinkTarget(caption, d.audioSegments);
|
||||
if (!caption || !target) return items;
|
||||
linked = true;
|
||||
return items.map((segment) => segment.id === segmentId
|
||||
? { ...segment, audioSegmentId: target.id, detachedAudioSegmentId: "" }
|
||||
: segment);
|
||||
});
|
||||
d.notify(t(linked ? "captionAudioLinked" : "captionAudioUnavailable"));
|
||||
}
|
||||
|
||||
function alignCaptionToAudio(segmentId) {
|
||||
if (d.trackLocks.caption) return void d.notify(t("captionTrackLocked"));
|
||||
let aligned = false;
|
||||
d.setCaptionSegments((items) => items.map((segment) => {
|
||||
if (segment.id !== segmentId) return segment;
|
||||
const target = d.audioSegments.find((audio) => audio.id === segment.audioSegmentId);
|
||||
if (!target) return segment;
|
||||
aligned = true;
|
||||
return { ...segment, start: target.start, end: target.start + target.duration };
|
||||
}));
|
||||
d.notify(t(aligned ? "captionAlignedToAudio" : "captionAudioUnavailable"));
|
||||
}
|
||||
|
||||
function linkAudioToCaption(audioId) {
|
||||
if (d.trackLocks.caption) return void d.notify(t("captionTrackLocked"));
|
||||
const audio = d.audioSegments.find((segment) => segment.id === audioId);
|
||||
if (!audio) return void d.notify(t("captionAudioUnavailable"));
|
||||
let linked = false;
|
||||
d.setCaptionSegments((items) => {
|
||||
const remembered = items.find((caption) => caption.detachedAudioSegmentId === audioId);
|
||||
const candidates = items.filter((caption) => !caption.audioSegmentId);
|
||||
const caption = remembered ?? findCaptionAudioLinkTarget(
|
||||
{ start: audio.start, end: audio.start + audio.duration },
|
||||
candidates.map((item) => ({ id: item.id, start: item.start, duration: item.end - item.start })),
|
||||
);
|
||||
const targetId = remembered?.id ?? caption?.id;
|
||||
if (!targetId) return items;
|
||||
linked = true;
|
||||
return items.map((item) => item.id === targetId
|
||||
? { ...item, audioSegmentId: audioId, detachedAudioSegmentId: "" }
|
||||
: item);
|
||||
});
|
||||
d.notify(t(linked ? "captionAudioLinked" : "captionAudioUnavailable"));
|
||||
}
|
||||
|
||||
function unlinkAudioCaptions(audioId) {
|
||||
if (d.trackLocks.caption) return void d.notify(t("captionTrackLocked"));
|
||||
d.setCaptionSegments((items) => items.map((caption) => caption.audioSegmentId === audioId
|
||||
? { ...caption, detachedAudioSegmentId: audioId, audioSegmentId: "" }
|
||||
: caption));
|
||||
d.notify(t("captionAudioUnlinked"));
|
||||
}
|
||||
|
||||
function alignAudioCaptions(audioId) {
|
||||
if (d.trackLocks.caption) return void d.notify(t("captionTrackLocked"));
|
||||
const audio = d.audioSegments.find((segment) => segment.id === audioId);
|
||||
if (!audio) return void d.notify(t("captionAudioUnavailable"));
|
||||
d.setCaptionSegments((items) => items.map((caption) => caption.audioSegmentId === audioId
|
||||
? { ...caption, start: audio.start, end: audio.start + audio.duration }
|
||||
: caption));
|
||||
d.notify(t("captionAlignedToAudio"));
|
||||
}
|
||||
|
||||
function disableSmartCaptionAvoidance() {
|
||||
if (!d.previewVisionKey || !d.previewVisionRecord?.options?.avoidCaptions) return false;
|
||||
d.setVisionRecords((records) => {
|
||||
@@ -171,10 +264,16 @@ export function createCaptionEditingActions(d) {
|
||||
|
||||
return {
|
||||
commitCaptionSegments,
|
||||
alignCaptionToAudio,
|
||||
alignAudioCaptions,
|
||||
deleteCaptionSegment,
|
||||
handleCaptionPositionChange,
|
||||
linkCaptionAudio,
|
||||
linkAudioToCaption,
|
||||
startCaptionDrag,
|
||||
toggleCaptionSegmentHidden,
|
||||
unlinkCaptionAudio,
|
||||
unlinkAudioCaptions,
|
||||
updateCaptionSegmentText,
|
||||
updateScript,
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { snapCaptionPlacement } from "./captionEditingActions.js";
|
||||
import { createCaptionEditingActions, findCaptionAudioLinkTarget, snapCaptionPlacement } from "./captionEditingActions.js";
|
||||
|
||||
describe("caption canvas snapping", () => {
|
||||
it("snaps captions to the horizontal center and bottom safe position", () => {
|
||||
@@ -15,3 +15,50 @@ describe("caption canvas snapping", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("caption voiceover links", () => {
|
||||
const createHarness = () => {
|
||||
let captions = [{ id: "caption-1", text: "Hello", start: 3, end: 4, audioSegmentId: "voice-1" }];
|
||||
const audioSegments = [{ id: "voice-1", name: "Heart", start: 1, duration: 2 }];
|
||||
const setCaptionSegments = vi.fn((update) => { captions = typeof update === "function" ? update(captions) : update; });
|
||||
const d = {
|
||||
audioSegments, captionSegments: captions, trackLocks: { caption: false }, setCaptionSegments,
|
||||
notify: vi.fn(), t: (key) => key,
|
||||
};
|
||||
return { actions: createCaptionEditingActions(d), audioSegments, getCaptions: () => captions };
|
||||
};
|
||||
|
||||
it("unlinks without moving either clip and relinks the remembered voiceover", () => {
|
||||
const { actions, getCaptions } = createHarness();
|
||||
actions.unlinkCaptionAudio("caption-1");
|
||||
expect(getCaptions()[0]).toMatchObject({ start: 3, end: 4, audioSegmentId: "", detachedAudioSegmentId: "voice-1" });
|
||||
actions.linkCaptionAudio("caption-1");
|
||||
expect(getCaptions()[0]).toMatchObject({ start: 3, end: 4, audioSegmentId: "voice-1" });
|
||||
});
|
||||
|
||||
it("aligns a linked caption to the voiceover range only on demand", () => {
|
||||
const { actions, getCaptions } = createHarness();
|
||||
actions.alignCaptionToAudio("caption-1");
|
||||
expect(getCaptions()[0]).toMatchObject({ start: 1, end: 3, audioSegmentId: "voice-1" });
|
||||
});
|
||||
|
||||
it("prefers a remembered link before overlap-based matching", () => {
|
||||
const target = findCaptionAudioLinkTarget(
|
||||
{ start: 9, end: 10, detachedAudioSegmentId: "voice-1" },
|
||||
[{ id: "voice-1", start: 0, duration: 1 }, { id: "voice-2", start: 9, duration: 1 }],
|
||||
);
|
||||
expect(target.id).toBe("voice-1");
|
||||
});
|
||||
|
||||
it("does not crash when a future caller omits the translator", () => {
|
||||
let captions = [{ id: "caption-1", start: 0, end: 1, audioSegmentId: "voice-1" }];
|
||||
const actions = createCaptionEditingActions({
|
||||
audioSegments: [{ id: "voice-1", start: 0, duration: 1 }],
|
||||
captionSegments: captions,
|
||||
trackLocks: { caption: false },
|
||||
setCaptionSegments: (update) => { captions = update(captions); },
|
||||
notify: vi.fn(),
|
||||
});
|
||||
expect(() => actions.unlinkCaptionAudio("caption-1")).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,7 +5,9 @@ const SOURCE_AUDIO_CLIP_ACTIONS = ["dismiss", "audio", "split", "captions", "del
|
||||
const STICKER_CLIP_ACTIONS = ["dismiss", "properties", "copy", "delete"];
|
||||
|
||||
export function getMobileClipActionIds(track, options = {}) {
|
||||
if (track === "audio") return AUDIO_CLIP_ACTIONS;
|
||||
const associationActions = ["caption-link", ...(options.hasLinkedCaption ? ["caption-align"] : [])];
|
||||
if (track === "caption") return ["dismiss", "edit", "split", "copy", ...associationActions, "delete"];
|
||||
if (track === "audio") return [...AUDIO_CLIP_ACTIONS.slice(0, -1), ...associationActions, "delete"];
|
||||
if (track === "music") return MUSIC_CLIP_ACTIONS;
|
||||
if (track === "source") return SOURCE_AUDIO_CLIP_ACTIONS;
|
||||
if (track === "sticker") return STICKER_CLIP_ACTIONS;
|
||||
|
||||
@@ -9,10 +9,19 @@ import {
|
||||
|
||||
describe("mobile clip actions", () => {
|
||||
it("shows audio-specific actions for voiceover and music clips", () => {
|
||||
expect(getMobileClipActionIds("audio")).toEqual(["dismiss", "edit", "split", "captions", "separate", "delete"]);
|
||||
expect(getMobileClipActionIds("audio")).toEqual(["dismiss", "edit", "split", "captions", "separate", "caption-link", "delete"]);
|
||||
expect(getMobileClipActionIds("music")).toEqual(["dismiss", "audio", "split", "captions", "separate", "delete"]);
|
||||
});
|
||||
|
||||
it("adds caption link controls for caption and linked voiceover clips", () => {
|
||||
expect(getMobileClipActionIds("caption", { hasLinkedCaption: true })).toEqual([
|
||||
"dismiss", "edit", "split", "copy", "caption-link", "caption-align", "delete",
|
||||
]);
|
||||
expect(getMobileClipActionIds("audio", { hasLinkedCaption: true })).toEqual([
|
||||
"dismiss", "edit", "split", "captions", "separate", "caption-link", "caption-align", "delete",
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not offer vocal separation for linked source-audio pieces", () => {
|
||||
expect(getMobileClipActionIds("source")).toEqual(["dismiss", "audio", "split", "captions", "delete"]);
|
||||
});
|
||||
|
||||
@@ -12,6 +12,7 @@ const PINYIN_VOICES = {
|
||||
"zh_CN-xiao_ya-medium": "zh/zh_CN/xiao_ya/medium",
|
||||
"zh_CN-chaowen-medium": "zh/zh_CN/chaowen/medium",
|
||||
};
|
||||
export const isBuiltInPinyinVoice = (voiceId) => voiceId in PINYIN_VOICES;
|
||||
const PIPER_BASE = "https://huggingface.co/rhasspy/piper-voices/resolve/main";
|
||||
const INITIALS = ["zh", "ch", "sh", "b", "p", "m", "f", "d", "t", "n", "l", "g", "k", "h", "j", "q", "x", "r", "z", "c", "s", "y", "w"];
|
||||
const GROUP_END = new Set(["1", "2", "3", "4", "5", "。", ".", "？", "?", "！", "!", "—", "…", "、", "，", ",", "：", ":", "；", ";", " "]);
|
||||
@@ -147,6 +148,7 @@ async function loadPinyinVoice(voiceId, onProgress) {
|
||||
await writePinyinCache(`${voiceName}-${voiceId}.onnx.json`, configBytes);
|
||||
}
|
||||
const model = await fetchArrayBufferWithProgress(`${voiceBase}/${voiceId}.onnx`, onProgress, voiceName);
|
||||
onProgress?.({ phase: "initializing", loaded: model.byteLength, total: model.byteLength });
|
||||
if (globalThis.navigator?.gpu) {
|
||||
try {
|
||||
const session = await ort.InferenceSession.create(model, {
|
||||
@@ -190,6 +192,7 @@ async function predictPinyinVoice(input, onProgress) {
|
||||
}
|
||||
let runtime = await pinyinRuntimePromises.get(input.voiceId);
|
||||
const feeds = createPinyinFeeds(input, runtime.config);
|
||||
onProgress?.({ phase: "generating", backend: runtime.backend });
|
||||
let result;
|
||||
try {
|
||||
result = await runtime.session.run(feeds);
|
||||
|
||||
+90
-8
@@ -3618,6 +3618,25 @@ button:disabled {
|
||||
min-height: 150px;
|
||||
}
|
||||
|
||||
.caption-audio-link {
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
padding: 11px;
|
||||
border: 1px solid rgba(173, 184, 196, 0.16);
|
||||
border-radius: 9px;
|
||||
background: rgba(255, 255, 255, 0.028);
|
||||
}
|
||||
.caption-audio-link.is-linked { border-color: rgba(53, 234, 217, 0.24); background: rgba(53, 234, 217, 0.045); }
|
||||
.caption-audio-link > div:first-child { display: flex; align-items: center; gap: 9px; min-width: 0; }
|
||||
.caption-audio-link > div:first-child > span { display: grid; place-items: center; width: 28px; height: 28px; flex: 0 0 auto; border-radius: 7px; color: #5eece0; background: rgba(53, 234, 217, 0.1); }
|
||||
.caption-audio-link > div:first-child > div { display: grid; min-width: 0; gap: 2px; }
|
||||
.caption-audio-link strong { color: #d9e5e9; font-size: 11px; }
|
||||
.caption-audio-link em { overflow: hidden; color: #82919b; font-size: 10px; font-style: normal; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.caption-audio-link-actions { display: flex; gap: 7px; }
|
||||
.caption-audio-link-actions button { min-height: 30px; flex: 1; padding: 6px 8px; border: 1px solid rgba(53, 234, 217, 0.2); border-radius: 7px; color: #a8eee8; font-size: 10px; background: rgba(53, 234, 217, 0.08); }
|
||||
.caption-audio-link-actions button.is-unlink { border-color: rgba(255, 255, 255, 0.1); color: #a7b1b8; background: rgba(255, 255, 255, 0.045); }
|
||||
.caption-audio-link-actions button:disabled { opacity: 0.42; }
|
||||
|
||||
.caption-context-empty {
|
||||
display: grid;
|
||||
place-items: center;
|
||||
@@ -3939,6 +3958,40 @@ button:disabled {
|
||||
transition: width 180ms ease;
|
||||
}
|
||||
|
||||
.voice-generation-loading {
|
||||
display: grid;
|
||||
grid-template-columns: 18px minmax(0, 1fr) auto;
|
||||
align-items: center;
|
||||
gap: 9px 10px;
|
||||
margin-top: 13px;
|
||||
padding: 11px 12px;
|
||||
border: 1px solid rgba(50, 234, 216, 0.22);
|
||||
border-radius: 10px;
|
||||
background: rgba(50, 234, 216, 0.055);
|
||||
}
|
||||
|
||||
.voice-generation-loading > div:not(.progress-track) { display: grid; min-width: 0; gap: 2px; }
|
||||
.voice-generation-loading strong { overflow: hidden; color: #dffbf8; font-size: 12px; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.voice-generation-loading div > span { color: #83959f; font-size: 10px; line-height: 1.35; }
|
||||
.voice-generation-loading em { color: #62e8db; font-size: 11px; font-style: normal; font-variant-numeric: tabular-nums; }
|
||||
.voice-generation-loading .progress-track { grid-column: 1 / -1; width: 100%; margin-top: 0; }
|
||||
.voice-generation-spinner,
|
||||
.generate-button-spinner {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
border: 2px solid rgba(7, 22, 22, 0.25);
|
||||
border-top-color: currentColor;
|
||||
border-radius: 50%;
|
||||
animation: voice-loading-spin 750ms linear infinite;
|
||||
}
|
||||
.voice-generation-spinner { color: #42ead9; border-color: rgba(66, 234, 217, 0.2); border-top-color: currentColor; }
|
||||
.generate-button-spinner { width: 15px; height: 15px; }
|
||||
@keyframes voice-loading-spin { to { transform: rotate(360deg); } }
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.voice-generation-spinner,
|
||||
.generate-button-spinner { animation-duration: 1.8s; }
|
||||
}
|
||||
|
||||
.voice-actions {
|
||||
gap: 10px;
|
||||
margin-top: 14px;
|
||||
@@ -4335,7 +4388,8 @@ button:disabled {
|
||||
transition: width 140ms cubic-bezier(0.16, 1, 0.3, 1);
|
||||
}
|
||||
|
||||
.track-scroll.is-wheel-zooming {
|
||||
.track-scroll.is-wheel-zooming,
|
||||
.timeline-ruler-canvas.is-wheel-zooming {
|
||||
transition: none;
|
||||
}
|
||||
|
||||
@@ -5996,8 +6050,7 @@ button:disabled {
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
z-index: 110;
|
||||
display: grid;
|
||||
grid-template-columns: repeat(5, minmax(0, 1fr));
|
||||
display: flex;
|
||||
min-height: calc(64px + env(safe-area-inset-bottom));
|
||||
padding: 5px 5px env(safe-area-inset-bottom);
|
||||
border-top: 1px solid rgba(255, 255, 255, .09);
|
||||
@@ -6020,17 +6073,46 @@ button:disabled {
|
||||
}
|
||||
.timeline-mobile-clip-actions button:active { color: #35ead9; background: rgba(53, 234, 217, .1); }
|
||||
.timeline-mobile-clip-actions button.is-back {
|
||||
position: relative;
|
||||
z-index: 3;
|
||||
flex: 0 0 72px;
|
||||
min-width: 72px;
|
||||
border-right: 1px solid rgba(255, 255, 255, .08);
|
||||
border-radius: 0;
|
||||
color: #e6eff2;
|
||||
background: rgba(13, 18, 24, .995);
|
||||
}
|
||||
.timeline-mobile-clip-actions button.is-back::after {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
right: -15px;
|
||||
bottom: 0;
|
||||
width: 15px;
|
||||
pointer-events: none;
|
||||
content: "";
|
||||
background: linear-gradient(90deg, rgba(0, 0, 0, .34), transparent);
|
||||
}
|
||||
.timeline-mobile-clip-actions button.is-danger { color: #ff8d92; }
|
||||
.timeline-mobile-clip-actions.is-scroll-actions {
|
||||
grid-template-columns: repeat(6, minmax(60px, 1fr));
|
||||
overflow-x: auto;
|
||||
scrollbar-width: none;
|
||||
.timeline-mobile-clip-action-scroller {
|
||||
display: grid;
|
||||
grid-auto-columns: minmax(0, 1fr);
|
||||
grid-auto-flow: column;
|
||||
min-width: 0;
|
||||
flex: 1;
|
||||
}
|
||||
.timeline-mobile-clip-actions.is-scroll-actions::-webkit-scrollbar { display: none; }
|
||||
.timeline-mobile-clip-actions.is-scroll-actions {
|
||||
overflow: hidden;
|
||||
}
|
||||
.timeline-mobile-clip-actions.is-scroll-actions .timeline-mobile-clip-action-scroller {
|
||||
display: flex;
|
||||
overflow-x: auto;
|
||||
overflow-y: hidden;
|
||||
overscroll-behavior-x: contain;
|
||||
scrollbar-width: none;
|
||||
touch-action: pan-x;
|
||||
}
|
||||
.timeline-mobile-clip-actions.is-scroll-actions .timeline-mobile-clip-action-scroller button { flex: 0 0 68px; min-width: 68px; }
|
||||
.timeline-mobile-clip-actions.is-scroll-actions .timeline-mobile-clip-action-scroller::-webkit-scrollbar { display: none; }
|
||||
.timeline-mobile-clip-actions span {
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
|
||||
Reference in New Issue
Block a user