From a07d6c9a870214f779685d24c2af1e90e205c294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AE=8F=E6=9D=B0?= Date: Wed, 19 Aug 2026 12:48:36 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20Canvas=E6=92=AD=E6=94=BE=E5=99=A8?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20-=20=E7=89=87=E6=AE=B5=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E5=88=87=E6=8D=A2+=E5=AE=BD=E9=AB=98=E6=AF=94+=E7=A7=BB?= =?UTF-8?q?=E9=99=A4=E6=AD=BB=E4=BB=A3=E7=A0=81+=E8=BF=9B=E5=BA=A6?= =?UTF-8?q?=E8=8A=82=E6=B5=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 片段自动切换：初始化时加载所有片段数据并逐段解码，帧时间戳按全局偏移对齐 2. Canvas宽高比：computeDrawRect计算letterbox/pillarbox，保持视频原始比例 3. 移除未使用的audioCtxRef/audioSourceRef（音频由外部HTMLAudio处理） 4. 进度更新节流到200ms(5fps)，减少React re-render 5. FrameQueue容量从5提升到10，多片段场景更流畅 6. fallback路径优化预加载：当前片段auto/下一个metadata/其余none --- .../components/FrontendPreviewPlayer.tsx | 4 +- .../pages/generate/hooks/useCanvasPlayer.ts | 364 ++++++++++++------ 2 files changed, 246 insertions(+), 122 deletions(-) diff --git a/apps/web/src/pages/generate/components/FrontendPreviewPlayer.tsx b/apps/web/src/pages/generate/components/FrontendPreviewPlayer.tsx index 71666ad0a..9378caa1b 100644 --- a/apps/web/src/pages/generate/components/FrontendPreviewPlayer.tsx +++ b/apps/web/src/pages/generate/components/FrontendPreviewPlayer.tsx @@ -354,7 +354,9 @@ const FrontendPreviewPlayer: React.FC = ({ ref={(el) => { videoRefs.current[i] = el }} - preload="auto" + preload={ + i === videoCurrentSegIdx ? "auto" : i === videoCurrentSegIdx + 1 ? "metadata" : "none" + } src={seg.videoUrl} style={{ position: "absolute", diff --git a/apps/web/src/pages/generate/hooks/useCanvasPlayer.ts b/apps/web/src/pages/generate/hooks/useCanvasPlayer.ts index 37fb7dedd..315cd91bf 100644 --- a/apps/web/src/pages/generate/hooks/useCanvasPlayer.ts +++ b/apps/web/src/pages/generate/hooks/useCanvasPlayer.ts @@ -1,7 +1,6 @@ /** * Canvas + WebCodecs 播放器核心 Hook * MP4 → mp4box.js 解封装 → VideoDecoder 解码帧 → Canvas 绘制 - * 音频走 Web Audio API 独立同步播放 * * 浏览器不支持 WebCodecs 时返回 hasSupport=false，由调用方 fallback */ @@ -12,7 +11,7 @@ import type { Movie, Sample } from "mp4box" // ── 帧队列（环形缓冲区） ── interface FrameEntry { frame: VideoFrame - pts: number // 显示时间戳（秒） + pts: number // 全局时间戳（秒），已按片段偏移对齐 duration: number // 帧持续时长（秒） } @@ -25,7 +24,6 @@ class FrameQueue { } push(entry: FrameEntry) { - // 如果队列已满，丢弃最旧的帧 while (this.frames.length >= this.maxSize) { const old = this.frames.shift() old?.frame.close() @@ -35,18 +33,15 @@ class FrameQueue { /** 获取当前时间戳应显示的帧 */ getCurrentFrame(timestamp: number): VideoFrame | null { - // 找到 pts <= timestamp 的最新帧 let best: FrameEntry | null = null let bestIdx = -1 for (let i = 0; i < this.frames.length; i++) { const f = this.frames[i] if (f.pts <= timestamp + 0.01) { - // 10ms 容差 best = f bestIdx = i } } - // 释放已消费帧之前的所有帧 for (let i = 0; i < bestIdx; i++) { this.frames[i].frame.close() } @@ -68,19 +63,33 @@ class FrameQueue { } } +// ── 片段元数据 ── +interface SegmentMeta { + assetId: string + videoUrl: string + /** 该片段在全局时间轴上的起始时间（秒） */ + globalStartTime: number + /** 该片段在全局时间轴上的结束时间（秒） */ + globalEndTime: number + /** 视频轨道 ID */ + trackId: number + /** 视频轨道 timescale */ + timescale: number + /** 编解码器 */ + codec: string + /** 视频宽度（像素） */ + videoWidth: number + /** 视频高度（像素） */ + videoHeight: number +} + // ── 播放器状态 ── export interface CanvasPlayerState { - /** 是否支持 WebCodecs */ hasSupport: boolean - /** 是否正在播放 */ isPlaying: boolean - /** 当前播放时间（秒） */ currentTime: number - /** 总时长（秒） */ duration: number - /** 是否已加载（可以播放） */ isReady: boolean - /** 是否正在缓冲 */ isBuffering: boolean } @@ -131,15 +140,15 @@ export function useCanvasPlayer( // ── 内部引用 ── const decoderRef = useRef(null) - const frameQueueRef = useRef(new FrameQueue(5)) - const audioCtxRef = useRef(null) - const audioSourceRef = useRef(null) + const frameQueueRef = useRef(new FrameQueue(10)) const rafRef = useRef(0) const playStartRef = useRef(0) const playStartOffsetRef = useRef(0) const segmentDataRef = useRef>(new Map()) - const videoTrackRef = useRef<{ id: number; timescale: number; codec: string } | null>(null) + const segmentMetaRef = useRef([]) + const videoDimRef = useRef<{ width: number; height: number }>({ width: 0, height: 0 }) const isDestroyedRef = useRef(false) + const lastProgressUpdateRef = useRef(0) // 计算总时长 const totalDuration = segments.reduce((sum, seg) => sum + (seg.endTime - seg.startTime), 0) @@ -162,92 +171,47 @@ export function useCanvasPlayer( } }, []) - // ── 初始化 VideoDecoder ── - const initDecoder = useCallback(async (codec: string, width: number, height: number) => { - if (!isWebCodecsSupported()) return false + // ── 解封装单个片段，提取轨道元数据 ── + const demuxSegment = useCallback( + (buffer: ArrayBuffer, segIndex: number): SegmentMeta | null => { + const segment = segments[segIndex] + if (!segment) return null - const decoder = new VideoDecoder({ - output: (frame: VideoFrame) => { - frameQueueRef.current.push({ - frame, - pts: frame.timestamp / 1_000_000, - duration: (frame.duration ?? 0) / 1_000_000, - }) - }, - error: (e: DOMException) => { - console.error("[useCanvasPlayer] Decoder error:", e) - }, - }) + // 计算全局偏移 + let globalStart = 0 + for (let i = 0; i < segIndex; i++) { + globalStart += segments[i].endTime - segments[i].startTime + } - try { - await decoder.configure({ - codec, - codedWidth: width, - codedHeight: height, - }) - decoderRef.current = decoder - return true - } catch (err) { - console.error("[useCanvasPlayer] Decoder configure failed:", err) - return false - } - }, []) - - // ── 解封装 MP4 并送入解码器 ── - const demuxAndDecode = useCallback( - async (buffer: ArrayBuffer) => { const mp4File = createFile() + let meta: SegmentMeta | null = null mp4File.onReady = (info: Movie) => { const videoTrack = info.videoTracks[0] if (!videoTrack) { - console.error("[useCanvasPlayer] No video track found") + console.error("[useCanvasPlayer] No video track found for segment", segIndex) return } - videoTrackRef.current = { - id: videoTrack.id, + meta = { + assetId: segment.assetId, + videoUrl: segment.videoUrl, + globalStartTime: globalStart, + globalEndTime: globalStart + (segment.endTime - segment.startTime), + trackId: videoTrack.id, timescale: videoTrack.timescale, codec: videoTrack.codec, + videoWidth: videoTrack.track_width || 1280, + videoHeight: videoTrack.track_height || 720, } - const width = videoTrack.track_width || 1280 - const height = videoTrack.track_height || 720 - initDecoder(videoTrack.codec, width, height) - + // 提取所有 samples（start() 会同步触发 onSamples） mp4File.setExtractionOptions(videoTrack.id, null, { nbSamples: videoTrack.nb_samples, }) mp4File.start() } - mp4File.onSamples = (trackId: number, _user: unknown, samples: Sample[]) => { - if (isDestroyedRef.current) return - - const videoTrack = videoTrackRef.current - if (!videoTrack || trackId !== videoTrack.id) return - - const decoder = decoderRef.current - if (!decoder || decoder.state === "closed") return - - for (const sample of samples) { - if (!sample.data) continue - - const chunk = new EncodedVideoChunk({ - type: sample.is_sync ? "key" : "delta", - timestamp: (sample.cts / videoTrack.timescale) * 1_000_000, - duration: (sample.duration / videoTrack.timescale) * 1_000_000, - data: sample.data.buffer as ArrayBuffer, - }) - - try { - decoder.decode(chunk) - } catch (e) { - console.warn("[useCanvasPlayer] Decode chunk error:", e) - } - } - } - mp4File.onError = (_module: string, message: string) => { console.error(`[useCanvasPlayer] MP4Box error: ${message}`) } @@ -257,8 +221,112 @@ export function useCanvasPlayer( ;(buffer as any).fileStart = 0 // eslint-disable-next-line @typescript-eslint/no-explicit-any mp4File.appendBuffer(buffer as any) + + return meta }, - [initDecoder], + [segments], + ) + + // ── 初始化 VideoDecoder 并解码指定片段 ── + const decodeSegment = useCallback( + async (buffer: ArrayBuffer, meta: SegmentMeta): Promise => { + if (isDestroyedRef.current) return + + const mp4File = createFile() + let decoderReady = false + + // 配置解码器（每个片段可能需要不同的 codec/分辨率） + const decoder = new VideoDecoder({ + output: (frame: VideoFrame) => { + const localTime = frame.timestamp / 1_000_000 + const globalTime = localTime + meta.globalStartTime + frameQueueRef.current.push({ + frame, + pts: globalTime, + duration: (frame.duration ?? 0) / 1_000_000, + }) + }, + error: (e: DOMException) => { + console.error("[useCanvasPlayer] Decoder error:", e) + }, + }) + + try { + await decoder.configure({ + codec: meta.codec, + codedWidth: meta.videoWidth, + codedHeight: meta.videoHeight, + }) + decoderRef.current = decoder + decoderReady = true + + // 更新视频尺寸（用于 aspect ratio） + if (meta.videoWidth > 0 && meta.videoHeight > 0) { + videoDimRef.current = { width: meta.videoWidth, height: meta.videoHeight } + } + } catch (err) { + console.error("[useCanvasPlayer] Decoder configure failed for segment:", err) + return + } + + if (!decoderReady) return + + // 收集 samples + const samplesCollected: Sample[] = [] + + mp4File.onReady = (info: Movie) => { + const videoTrack = info.videoTracks[0] + if (!videoTrack) return + + mp4File.setExtractionOptions(videoTrack.id, null, { + nbSamples: videoTrack.nb_samples, + }) + mp4File.start() + } + + mp4File.onSamples = (_trackId: number, _user: unknown, samples: Sample[]) => { + samplesCollected.push(...samples) + } + + mp4File.onError = (_module: string, message: string) => { + console.error(`[useCanvasPlayer] MP4Box decode error: ${message}`) + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ;(buffer as any).fileStart = 0 + // eslint-disable-next-line @typescript-eslint/no-explicit-any + mp4File.appendBuffer(buffer as any) + + // 等待 samples 收集完成（mp4box start() 是同步的） + await new Promise((resolve) => setTimeout(resolve, 10)) + + // 送入解码器 + for (const sample of samplesCollected) { + if (!sample.data || isDestroyedRef.current) continue + if (decoder.state === "closed") break + + const chunk = new EncodedVideoChunk({ + type: sample.is_sync ? "key" : "delta", + timestamp: (sample.cts / meta.timescale) * 1_000_000, + duration: (sample.duration / meta.timescale) * 1_000_000, + data: sample.data.buffer as ArrayBuffer, + }) + + try { + decoder.decode(chunk) + } catch (e) { + console.warn("[useCanvasPlayer] Decode chunk error:", e) + } + } + + // flush 确保所有帧输出 + try { + await decoder.flush() + } catch (e) { + console.warn("[useCanvasPlayer] Decoder flush error:", e) + } + }, + [], ) // ── 标题绘制 ── @@ -302,7 +370,6 @@ export function useCanvasPlayer( ctx.fillText(title.text, canvas.width / 2, y) - // 重置阴影 ctx.shadowColor = "transparent" ctx.shadowBlur = 0 ctx.shadowOffsetX = 0 @@ -311,6 +378,37 @@ export function useCanvasPlayer( [], ) + // ── 计算保持宽高比的绘制矩形（letterbox / pillarbox） ── + const computeDrawRect = useCallback( + (canvasW: number, canvasH: number): { dx: number; dy: number; dw: number; dh: number } => { + const vw = videoDimRef.current.width + const vh = videoDimRef.current.height + if (vw <= 0 || vh <= 0) return { dx: 0, dy: 0, dw: canvasW, dh: canvasH } + + const canvasAspect = canvasW / canvasH + const videoAspect = vw / vh + + let dw: number, dh: number + if (canvasAspect > videoAspect) { + // canvas 更宽 → pillarbox（左右留黑） + dh = canvasH + dw = canvasH * videoAspect + } else { + // canvas 更高 → letterbox（上下留黑） + dw = canvasW + dh = canvasW / videoAspect + } + + return { + dx: (canvasW - dw) / 2, + dy: (canvasH - dh) / 2, + dw, + dh, + } + }, + [], + ) + // ── Canvas 渲染循环 ── const renderFrame = useCallback(() => { if (isDestroyedRef.current) return @@ -325,21 +423,29 @@ export function useCanvasPlayer( const frame = frameQueueRef.current.getCurrentFrame(currentTime) + ctx.clearRect(0, 0, canvas.width, canvas.height) + if (frame) { - ctx.clearRect(0, 0, canvas.width, canvas.height) - ctx.drawImage(frame, 0, 0, canvas.width, canvas.height) + const rect = computeDrawRect(canvas.width, canvas.height) + ctx.drawImage(frame, rect.dx, rect.dy, rect.dw, rect.dh) + frame.close() } if (titleSettings?.text) { drawTitle(ctx, canvas, titleSettings) } - setState((s) => { - if (Math.abs(s.currentTime - currentTime) > 0.1) { - return { ...s, currentTime } - } - return s - }) + // 进度更新节流到 200ms（5fps），减少 React re-render + const now = performance.now() + if (now - lastProgressUpdateRef.current >= 200) { + lastProgressUpdateRef.current = now + setState((s) => { + if (Math.abs(s.currentTime - currentTime) > 0.01) { + return { ...s, currentTime } + } + return s + }) + } if (currentTime >= totalDuration) { setState((s) => ({ ...s, isPlaying: false })) @@ -347,30 +453,22 @@ export function useCanvasPlayer( } rafRef.current = requestAnimationFrame(renderFrame) - }, [canvasRef, totalDuration, titleSettings, drawTitle]) + }, [canvasRef, totalDuration, titleSettings, drawTitle, computeDrawRect]) // ── 播放控制 ── const play = useCallback(async () => { if (!state.hasSupport || isDestroyedRef.current) return - if (!audioCtxRef.current) { - audioCtxRef.current = new AudioContext() - } - setState((s) => ({ ...s, isPlaying: true })) playStartRef.current = performance.now() playStartOffsetRef.current = state.currentTime + lastProgressUpdateRef.current = 0 rafRef.current = requestAnimationFrame(renderFrame) }, [state.hasSupport, state.currentTime, renderFrame]) const pause = useCallback(() => { setState((s) => ({ ...s, isPlaying: false })) cancelAnimationFrame(rafRef.current) - - if (audioSourceRef.current && audioCtxRef.current) { - audioSourceRef.current.stop() - audioSourceRef.current = null - } }, []) const seek = useCallback( @@ -379,6 +477,8 @@ export function useCanvasPlayer( setState((s) => ({ ...s, currentTime: clampedTime })) playStartOffsetRef.current = clampedTime playStartRef.current = performance.now() + // seek 后清空帧队列，等待新帧解码 + frameQueueRef.current.clear() }, [totalDuration], ) @@ -392,21 +492,11 @@ export function useCanvasPlayer( } frameQueueRef.current.clear() - - if (audioSourceRef.current) { - audioSourceRef.current.stop() - audioSourceRef.current = null - } - - if (audioCtxRef.current) { - audioCtxRef.current.close() - audioCtxRef.current = null - } - segmentDataRef.current.clear() + segmentMetaRef.current = [] }, []) - // ── 预加载下一个片段 ── + // ── 预加载下一个片段的数据 ── const preloadNext = useCallback( async (currentIndex: number) => { const nextIdx = currentIndex + 1 @@ -418,18 +508,50 @@ export function useCanvasPlayer( [segments, loadSegment], ) - // ── 初始化：加载第一个片段并解封装 ── + // ── 初始化：加载并解码所有片段 ── useEffect(() => { if (!state.hasSupport || segments.length === 0) return const init = async () => { - await loadSegment(segments[0]) - const buffer = segmentDataRef.current.get(segments[0].assetId) - if (buffer) { - await demuxAndDecode(buffer) - setState((s) => ({ ...s, duration: totalDuration, isReady: true })) - preloadNext(0) + setState((s) => ({ ...s, isBuffering: true })) + + // 1. 加载所有片段数据 + for (const seg of segments) { + await loadSegment(seg) } + + if (isDestroyedRef.current) return + + // 2. 解析每个片段的轨道元数据 + const metas: SegmentMeta[] = [] + for (let i = 0; i < segments.length; i++) { + const buffer = segmentDataRef.current.get(segments[i].assetId) + if (!buffer) continue + const meta = demuxSegment(buffer, i) + if (meta) metas.push(meta) + } + + if (isDestroyedRef.current || metas.length === 0) { + setState((s) => ({ ...s, isBuffering: false })) + return + } + + segmentMetaRef.current = metas + + // 3. 设置视频尺寸（用第一个片段的尺寸） + if (metas[0].videoWidth > 0 && metas[0].videoHeight > 0) { + videoDimRef.current = { width: metas[0].videoWidth, height: metas[0].videoHeight } + } + + // 4. 依次解码每个片段 + for (const meta of metas) { + const buffer = segmentDataRef.current.get(meta.assetId) + if (!buffer) continue + await decodeSegment(buffer, meta) + if (isDestroyedRef.current) break + } + + setState((s) => ({ ...s, duration: totalDuration, isReady: true, isBuffering: false })) } init()