Compare commits

..

1 Commits

Author SHA1 Message Date
张宏杰 5ef5d0d75b fix: setExtractionOptions 改为 nbSamples: Infinity,提取全部样本 2026-08-19 19:53:34 +08:00
7 changed files with 101 additions and 270 deletions
+4
View File
@@ -3,9 +3,13 @@ export type {
CreatePreviewRequest,
CreatePreviewResponse,
PreviewTaskResponse,
ConfirmGenerationRequest,
ConfirmGenerationResponse,
ConfirmGenerationTaskItem,
} from "./types"
export { createPreview, getPreviewStatus } from "./preview"
export { confirmGeneration } from "./confirm"
export { generateCover } from "./cover"
export type { GenerateCoverRequest, GenerateCoverResponse } from "./cover"
+2 -25
View File
@@ -57,31 +57,8 @@ export interface TaskListResponse {
export interface CreateGenerationTaskRequest {
template_id: string
asset_ids: string[]
title_ids?: string[]
voice_ids?: string[]
/** 输出视频宽度 */
output_width?: number
/** 输出视频高度 */
output_height?: number
/** 自定义封面图片 URL */
cover_url?: string
/** 自定义视频标题 */
custom_title?: string
/** 视频时长(秒) */
duration?: number
/** 视频宽高比,如 "9:16" */
video_ratio?: string
/** 标题烧录配置 */
title_config?: {
text?: string
font?: string
font_size?: number
font_color?: string
position?: string
bold?: boolean
stroke?: boolean
shadow?: boolean
}
title_ids: string[]
voice_ids: string[]
}
/** 创建生成任务响应(对齐后端 GenerationTaskResponse */
@@ -171,6 +171,7 @@ const GeneratePage: React.FC = () => {
autoSubtitles,
bgm,
generateCount,
previewTaskId: "",
})
/* ================================================================
@@ -19,6 +19,8 @@ export interface UseGenerateVideoProps {
autoSubtitles: boolean
bgm: boolean
generateCount: number
/** 预览任务的 task_id(用于新确认生成 API */
previewTaskId: string
}
/** 生成阶段 */
@@ -10,25 +10,14 @@ import type { Movie, Sample } from "mp4box"
// ── MP4 Box 解析辅助函数 ──
/**
* 递归搜索 box 树,找到 hvcC 或 avcC box 并返回其配置数据(不含 8 字节 box header
*
* MP4 box 嵌套结构:moov → trak → mdia → minf → stbl → stsd → hev1 → hvcC
* 普通容器 box 从 offset+8 开始递归;VisualSampleEntry (avc1/avc3/hvc1/hev1) 前 78 字节
* 是固定字段(ISO 14496-12),子 box 从 offset+8+78 开始
*/
function findCodecConfigRecursive(
buffer: ArrayBuffer,
start: number,
end: number,
): ArrayBuffer | undefined {
/** 在指定范围内查找 avcC / hvcC box,返回其数据 */
function findCodecConfig(buffer: ArrayBuffer, start: number, end: number): ArrayBuffer | undefined {
const view = new DataView(buffer)
let offset = start
while (offset < end - 8) {
const size = view.getUint32(offset)
if (size < 8 || offset + size > end) break
if (size < 8) break
const type = String.fromCharCode(
view.getUint8(offset + 4),
view.getUint8(offset + 5),
@@ -36,45 +25,37 @@ function findCodecConfigRecursive(
view.getUint8(offset + 7),
)
// 找到目标 codec 配置 box,返回内容(不含 8 字节 header
if (type === "avcC" || type === "hvcC") {
console.log("[findCodecConfig] Found", type, "at offset", offset, "size", size)
return buffer.slice(offset + 8, offset + size)
}
const visualSampleEntryTypes = ["avc1", "avc3", "hvc1", "hev1"]
const isVisualSampleEntry = visualSampleEntryTypes.includes(type)
if (isVisualSampleEntry) {
// VisualSampleEntry: 前 78 字节是固定字段,子 box 在 78 字节之后
// 先尝试 offset+8+78,如果没找到再尝试 offset+8(兼容不同 MP4 结构)
console.log(
"[findCodecConfig] VisualSampleEntry:",
type,
"at",
offset,
"size",
size,
"trying offset+8+78",
)
const childResult1 = findCodecConfigRecursive(buffer, offset + 8 + 78, offset + size)
if (childResult1) return childResult1
console.log(
"[findCodecConfig] VisualSampleEntry:",
type,
"at",
offset,
"trying offset+8 (fallback)",
)
const childResult2 = findCodecConfigRecursive(buffer, offset + 8, offset + size)
if (childResult2) return childResult2
} else {
// 普通容器 box,从 offset+8 开始递归
const childResult = findCodecConfigRecursive(buffer, offset + 8, offset + size)
if (childResult) return childResult
// 容器 boxfullbox 多 4 字节
const containerBoxes = ["trak", "mdia", "minf", "stbl"]
if (containerBoxes.includes(type)) {
// fullbox: size(4) + type(4) + version(1) + flags(3) = 12 bytes header
const contentStart = offset + 12
const result = findCodecConfig(buffer, contentStart, offset + size)
if (result) return result
} else if (type === "stsd") {
// SampleDescriptionBox 是 fullbox: 8 header + 4 version/flags + 4 entry_count
const entryCount = view.getUint32(offset + 12)
let entryOffset = offset + 16
for (let i = 0; i < entryCount && entryOffset < offset + size; i++) {
const entrySize = view.getUint32(entryOffset)
// 视觉样本条目: 8 header + 6 reserved + 2 data_ref_index + remaining
// 子 box 从 entryOffset + 16 + 62 开始 (skip reserved + data_ref_index + predefined)
// 实际结构: 8(header) + 6(reserved) + 2(data_ref_index) + 16(predefined+reserved) + 2(width) + 2(height) + ...
// 子 box 从 entryOffset + 8 + 6 + 2 + 16 + 2 + 2 + 2 + 2 + 4 + 2 + 2 + 2 + 2 = entryOffset + 78
// 更简单的做法:扫描 entry 内的子 box
const entryEnd = entryOffset + entrySize
const subBoxStart = entryOffset + 8 + 70 // VisualSampleEntry 固定字段共 70 字节
const result = findCodecConfig(buffer, subBoxStart, entryEnd)
if (result) return result
entryOffset += entrySize
}
} else if (type === "avcC" || type === "hvcC") {
// 找到目标 box,返回完整 box(含 header
// 返回完整 box(含 size + type header),WebCodecs HEVC decoder 需要
return buffer.slice(offset, offset + size)
}
if (size === 0) break
offset += size
}
return undefined
@@ -217,10 +198,6 @@ export function useCanvasPlayer(
// ── 内部引用 ──
const decoderRef = useRef<VideoDecoder | null>(null)
const frameQueueRef = useRef(new FrameQueue(10))
/** 已解码的片段索引集合,用于按需解码 */
const decodedSegmentsRef = useRef(new Set<number>())
/** 上一帧引用,用于在绘制新帧前释放上一帧防止内存泄漏 */
const lastDrawnFrameRef = useRef<VideoFrame | null>(null)
const rafRef = useRef<number>(0)
const playStartRef = useRef<number>(0)
const playStartOffsetRef = useRef<number>(0)
@@ -229,6 +206,7 @@ export function useCanvasPlayer(
const videoDimRef = useRef<{ width: number; height: number }>({ width: 0, height: 0 })
const isDestroyedRef = useRef(false)
const lastProgressUpdateRef = useRef<number>(0)
const descriptionCache = useRef<Map<string, ArrayBuffer>>(new Map())
// 计算总时长
const totalDuration = segments.reduce((sum, seg) => sum + (seg.endTime - seg.startTime), 0)
@@ -268,15 +246,7 @@ export function useCanvasPlayer(
view.getUint8(offset + 7),
)
if (type === "moov") {
const result = findCodecConfigRecursive(buffer, offset + 8, offset + size)
console.log("[useCanvasPlayer] extractCodecDescription:", {
moovOffset: offset,
moovSize: size,
searchRange: [offset + 8, offset + size],
found: !!result,
resultByteLength: result?.byteLength,
})
return result
return findCodecConfig(buffer, offset + 8, offset + size)
}
if (size === 0) break
offset += size
@@ -342,7 +312,15 @@ export function useCanvasPlayer(
}
// 提取编解码器配置数据(HEVC 必需,H.264 也需要)
const description = extractCodecDescription(buffer)
let description = extractCodecDescription(buffer)
// 如果当前分片没有 description,尝试从缓存获取
if (!description) {
for (const cached of descriptionCache.current.values()) {
description = cached
break
}
}
// ✅ 如果 description 缺失,无法解码 HEVC
if (!description) {
@@ -355,6 +333,9 @@ export function useCanvasPlayer(
return
}
// 缓存 description 供后续分片使用
descriptionCache.current.set(segment.assetId, description)
meta = {
assetId: segment.assetId,
videoUrl: segment.videoUrl,
@@ -535,52 +516,6 @@ export function useCanvasPlayer(
[],
)
/**
* 按需解码当前播放位置 ±1 个片段。
* 在渲染循环中定期调用,避免一次性解码所有片段导致环形缓冲区溢出丢帧。
*/
const decodeAroundPosition = useCallback(
async (currentTime: number) => {
const metas = segmentMetaRef.current
if (!metas || metas.length === 0) return
// 找到当前时间对应的片段索引
let targetIdx = -1
let acc = 0
for (let i = 0; i < metas.length; i++) {
const dur = metas[i].globalEndTime - metas[i].globalStartTime
if (currentTime < acc + dur) {
targetIdx = i
break
}
acc += dur
}
if (targetIdx === -1) targetIdx = metas.length - 1
// 解码当前 ±1 片段
for (
let i = Math.max(0, targetIdx - 1);
i <= Math.min(metas.length - 1, targetIdx + 1);
i++
) {
if (decodedSegmentsRef.current.has(i)) continue
const meta = metas[i]
const buffer = segmentDataRef.current.get(meta.assetId)
if (!buffer) continue
// 先标记为解码中,防止下一帧重复发起解码
decodedSegmentsRef.current.add(i)
try {
await decodeSegment(buffer, meta)
} catch (e) {
// 解码失败则移除标记,允许后续重试
decodedSegmentsRef.current.delete(i)
console.warn(`[useCanvasPlayer] 按需解码片段 ${i} 失败:`, e)
}
}
},
[decodeSegment],
)
// ── 标题绘制 ──
const drawTitle = useCallback(
(
@@ -684,19 +619,11 @@ export function useCanvasPlayer(
const frame = frameQueueRef.current.getCurrentFrame(currentTime)
// 释放上一帧,防止 VideoFrame 内存泄漏
if (lastDrawnFrameRef.current) {
lastDrawnFrameRef.current.close()
lastDrawnFrameRef.current = null
}
ctx.clearRect(0, 0, canvas.width, canvas.height)
if (frame) {
const rect = computeDrawRect(canvas.width, canvas.height)
ctx.drawImage(frame, rect.dx, rect.dy, rect.dw, rect.dh)
// 保持引用,下一帧绘制时再释放
lastDrawnFrameRef.current = frame
}
if (titleSettings?.text) {
@@ -713,8 +640,6 @@ export function useCanvasPlayer(
}
return s
})
// 按需解码当前 ±1 片段
decodeAroundPosition(currentTime)
}
if (currentTime >= totalDuration) {
@@ -723,7 +648,7 @@ export function useCanvasPlayer(
}
rafRef.current = requestAnimationFrame(renderFrame)
}, [canvasRef, totalDuration, titleSettings, drawTitle, computeDrawRect, decodeAroundPosition])
}, [canvasRef, totalDuration, titleSettings, drawTitle, computeDrawRect])
// ── 播放控制 ──
const play = useCallback(async () => {
@@ -742,22 +667,15 @@ export function useCanvasPlayer(
}, [])
const seek = useCallback(
async (time: number) => {
(time: number) => {
const clampedTime = Math.max(0, Math.min(time, totalDuration))
setState((s) => ({ ...s, currentTime: clampedTime }))
playStartOffsetRef.current = clampedTime
playStartRef.current = performance.now()
// seek 后清空帧队列 + 释放上一帧
// seek 后清空帧队列,等待新帧解码
frameQueueRef.current.clear()
if (lastDrawnFrameRef.current) {
lastDrawnFrameRef.current.close()
lastDrawnFrameRef.current = null
}
// 清空已解码标记,重新解码 seek 目标区域
decodedSegmentsRef.current.clear()
await decodeAroundPosition(clampedTime)
},
[totalDuration, decodeAroundPosition],
[totalDuration],
)
const destroy = useCallback(() => {
@@ -768,16 +686,10 @@ export function useCanvasPlayer(
decoderRef.current.close()
}
// 释放上一帧引用
if (lastDrawnFrameRef.current) {
lastDrawnFrameRef.current.close()
lastDrawnFrameRef.current = null
}
frameQueueRef.current.clear()
segmentDataRef.current.clear()
segmentMetaRef.current = []
decodedSegmentsRef.current.clear()
descriptionCache.current.clear()
}, [])
// ── 预加载下一个片段的数据 ──
@@ -794,16 +706,7 @@ export function useCanvasPlayer(
// ── 初始化:加载并解码所有片段 ──
useEffect(() => {
if (!state.hasSupport || segments.length === 0) {
console.log("[useCanvasPlayer] Skip init:", {
hasSupport: state.hasSupport,
segmentCount: segments.length,
})
return
}
let cancelled = false
console.log("[useCanvasPlayer] Init start, segments:", segments.length)
if (!state.hasSupport || segments.length === 0) return
const init = async () => {
setState((s) => ({ ...s, isBuffering: true }))
@@ -811,37 +714,20 @@ export function useCanvasPlayer(
// 1. 加载所有片段数据
for (const seg of segments) {
await loadSegment(seg)
if (cancelled) {
console.log("[useCanvasPlayer] Cancelled during loadSegment")
return
}
}
// 验证 buffer 是否都已存入
const bufferCheck = segments.map((s) => ({
assetId: s.assetId,
hasBuffer: segmentDataRef.current.has(s.assetId),
}))
console.log("[useCanvasPlayer] Buffers loaded:", bufferCheck)
if (isDestroyedRef.current) return
// 2. 解析每个片段的轨道元数据(await 等待 onSamples 回调完成)
const metas: SegmentMeta[] = []
for (let i = 0; i < segments.length; i++) {
const buffer = segmentDataRef.current.get(segments[i].assetId)
if (!buffer) {
console.warn("[useCanvasPlayer] No buffer for segment", i, segments[i].assetId)
continue
}
if (!buffer) continue
const meta = await demuxSegment(buffer, i)
if (cancelled) {
console.log("[useCanvasPlayer] Cancelled during demuxSegment")
return
}
if (meta) metas.push(meta)
}
if (cancelled || metas.length === 0) {
console.warn("[useCanvasPlayer] Init failed:", { cancelled, metasCount: metas.length })
if (isDestroyedRef.current || metas.length === 0) {
setState((s) => ({ ...s, isBuffering: false }))
return
}
@@ -853,36 +739,20 @@ export function useCanvasPlayer(
videoDimRef.current = { width: metas[0].videoWidth, height: metas[0].videoHeight }
}
// 4. 按需解码:初始只解码前 3 个片段,后续通过 decodeAroundPosition 动态加载
// 避免一次性全量解码导致 frameQueue 环形缓冲区旧帧被丢弃引发黑屏
decodedSegmentsRef.current.clear()
const initialDecodeCount = Math.min(metas.length, 3)
for (let i = 0; i < initialDecodeCount; i++) {
if (cancelled) break
const meta = metas[i]
// 4. 依次解码每个片段
for (const meta of metas) {
const buffer = segmentDataRef.current.get(meta.assetId)
if (!buffer) continue
// 先标记为解码中,防止重复解码
decodedSegmentsRef.current.add(i)
try {
await decodeSegment(buffer, meta)
} catch (e) {
// 解码失败则移除标记,允许后续重试
decodedSegmentsRef.current.delete(i)
console.warn(`[useCanvasPlayer] 初始化解码片段 ${i} 失败:`, e)
}
await decodeSegment(buffer, meta)
if (isDestroyedRef.current) break
}
if (!cancelled) {
console.log("[useCanvasPlayer] Init complete, isReady = true")
setState((s) => ({ ...s, duration: totalDuration, isReady: true, isBuffering: false }))
}
setState((s) => ({ ...s, duration: totalDuration, isReady: true, isBuffering: false }))
}
init()
return () => {
cancelled = true
destroy()
}
// eslint-disable-next-line react-hooks/exhaustive-deps
@@ -5,7 +5,7 @@
import { useState, useCallback } from "react"
import { message } from "antd"
import type { GeneratedVideo } from "@/api/template-editor"
import { createGenerationTask } from "@/api/tasks/tasks"
import { confirmGeneration, createPreview } from "@/api/generation"
import type { UseGenerateVideoProps } from "./generate-video/types"
import { getGenerationPhase } from "./generate-video/phase"
import { useGenerationPolling } from "./generate-video/useGenerationPolling"
@@ -55,7 +55,7 @@ export function useGenerateVideo(props: UseGenerateVideoProps) {
clearTimer()
try {
// 解析分辨率
// 解析分辨率videoRatio 可能是 "9:16"(宽高比)或 "1080x1920"(分辨率)
const ratio = props.videoRatio || "9:16"
let outputWidth: number
let outputHeight: number
@@ -87,22 +87,20 @@ export function useGenerateVideo(props: UseGenerateVideoProps) {
outputHeight = 1920
}
const assetIds =
props.materialMode === "auto" ? props.smartSelectedIds : props.selectedMaterials
// 直接创建正式生成任务
await createGenerationTask({
template_id: selectedTemplate,
asset_ids: assetIds,
output_width: outputWidth,
output_height: outputHeight,
cover_url: props.coverSettings?.upload_url || "",
custom_title: props.titleSettings?.title || "",
duration: props.duration || undefined,
video_ratio: props.videoRatio,
...(props.titleSettings?.title
? {
title_config: {
// 获取或创建后端任务 ID
// 预览改为前端播放后,不再有预览任务,需要在此处创建
let taskId = props.previewTaskId
if (!taskId) {
const assetIds =
props.materialMode === "auto" ? props.smartSelectedIds : props.selectedMaterials
const previewResp = await createPreview({
template_id: selectedTemplate,
asset_ids: assetIds,
duration: props.duration || undefined,
video_ratio: props.videoRatio,
voice_ids: undefined,
title_config: props.titleSettings?.title
? {
text: props.titleSettings.title,
font: props.titleSettings.font,
font_size: props.titleSettings.size,
@@ -111,9 +109,17 @@ export function useGenerateVideo(props: UseGenerateVideoProps) {
bold: props.titleSettings.bold,
stroke: props.titleSettings.stroke,
shadow: props.titleSettings.shadow,
},
}
: {}),
}
: undefined,
})
taskId = previewResp.task_id
}
await confirmGeneration(taskId, {
output_width: outputWidth,
output_height: outputHeight,
cover_url: props.coverSettings.upload_url || "",
custom_title: props.titleSettings.title || "",
})
startPolling()
@@ -114,22 +114,9 @@ export function useStep6Cover({
const anyErr = err as any
const statusCode = anyErr?.response?.status
// 400 错误:精确判断是否为"预览缺失",避免误判其他 400 错误
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const errCode = anyErr?.response?.data?.code as string | undefined
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const errMsg = (anyErr?.response?.data?.message ||
anyErr?.response?.data?.detail ||
"") as string
const isPreviewMissing =
statusCode === 400 &&
(errCode?.includes("PREVIEW") ||
/预览.*(?:缺失|不存在|未找到)|(?:missing|not found|does not exist).*preview/i.test(
errMsg,
))
if (isPreviewMissing) {
console.log("[Step6] 检测到预览缺失,尝试自动创建预览渲染任务...")
// 400 错误:后端缺少预览视频,自动创建后重试
if (statusCode === 400) {
console.log("[Step6] 后端返回 400,尝试自动创建预览渲染任务...")
message.info("正在准备预览视频,请稍候...")
try {
const previewResp = await createPreview({
@@ -137,39 +124,23 @@ export function useStep6Cover({
asset_ids: assetIds,
duration: duration || 30,
})
// 轮询等待预览渲染完成,双重超时保护
const maxPolls = 60 // 最多轮询 60 次(每 2 秒,共 120 秒)
let pollCount = 0
// 轮询等待预览渲染完成
await new Promise<void>((resolve, reject) => {
// 总超时保护:120 秒后强制 reject
const timeoutId = setTimeout(() => {
clearInterval(poll)
reject(new Error("预览生成超时,请稍后重试"))
}, 120_000)
const poll = setInterval(async () => {
pollCount++
try {
const status = await getPreviewStatus(previewResp.task_id)
if (status.status === "completed") {
clearTimeout(timeoutId)
clearInterval(poll)
resolve()
} else if (status.status === "failed") {
clearTimeout(timeoutId)
clearInterval(poll)
reject(new Error(status.error_message || "预览渲染失败"))
}
if (pollCount >= maxPolls) {
clearTimeout(timeoutId)
clearInterval(poll)
reject(new Error("预览生成超时,请稍后重试"))
}
} catch (e) {
clearTimeout(timeoutId)
clearInterval(poll)
reject(e)
}
}, 2000)
}, 3000)
})
message.success("预览视频就绪,重新生成封面...")
// 重试封面生成