From fe2fb8f29336a29907e32c3f21faf43aec71bc55 Mon Sep 17 00:00:00 2001 From: saas-frontend Date: Sat, 5 Sep 2026 16:01:16 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix(#1714):=20=E7=B4=A0=E6=9D=90=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E5=8E=BB=E9=87=8D=E9=98=B2=E9=87=8D=20+=20complete?= =?UTF-8?q?=E5=B9=82=E7=AD=89=E4=BC=A0=E9=80=92=20+=20=E8=BD=AE=E8=AF=A2?= =?UTF-8?q?=E6=94=B6=E6=95=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 入队按文件指纹(name+size+lastModified)去重:同一文件已在队列/上传中/处理中/已完成时跳过;失败项重新激活复用clientUploadId - 上传前计算SHA-256(大文件抽样头尾8MB+大小),prepare/complete携带file_hash+client_upload_id幂等token,打开后端去重闸门;配音/封面/克隆等一次性上传链路自动补算 - complete/prepare单独放宽超时(60s/30s,全局axios仅10s);complete失败不盲目重传,复用handle只幂等重发complete;prepare/transfer失败才全量重跑 - 直传进行中禁用上传按钮并拦截拖拽/选择,防重复提交;重试按钮仅error状态可点并区分complete安全重试 - 素材列表处理中快轮询(3s)加10分钟上限,超时停止轮询并标记「处理超时」,避免孤儿任务永久转圈 - 单测:指纹/去重/幂等token/hash纯函数 + 同文件多次选择不重复入队 + complete超时重试只重发complete --- apps/web/src/api/assets/upload.ts | 38 ++- apps/web/src/api/assets/uploadDedup.ts | 125 ++++++++++ apps/web/src/pages/assets/AssetLibrary.tsx | 4 + apps/web/src/pages/assets/assets.css | 22 ++ .../src/pages/assets/components/AssetCard.tsx | 17 +- .../assets/components/AssetGridSection.tsx | 4 + .../assets/components/AssetUploadZone.tsx | 21 +- .../assets/components/UploadQueuePanel.tsx | 5 +- .../src/pages/assets/hooks/useAssetUpload.ts | 219 +++++++++++++++--- .../src/pages/assets/hooks/useAssetsData.ts | 49 +++- apps/web/src/test/api/uploadDedup.test.ts | 84 +++++++ .../test/pages/assets/useAssetUpload.test.tsx | 163 +++++++++++-- 12 files changed, 685 insertions(+), 66 deletions(-) create mode 100644 apps/web/src/api/assets/uploadDedup.ts create mode 100644 apps/web/src/test/api/uploadDedup.test.ts diff --git a/apps/web/src/api/assets/upload.ts b/apps/web/src/api/assets/upload.ts index e396b5d77..ab450cd69 100644 --- a/apps/web/src/api/assets/upload.ts +++ b/apps/web/src/api/assets/upload.ts @@ -4,6 +4,7 @@ import apiClient from "../client" import { getOrCreateDefaultProject } from "../projects" import type { DirectUploadPrepareResult, DirectUploadCompleteResult } from "./types" +import { computeFileHash, makeClientUploadId } from "./uploadDedup" /** 预签名直传准备 */ export const prepareDirectUpload = async (data: { @@ -12,8 +13,13 @@ export const prepareDirectUpload = async (data: { filename: string content_type: string file_size: number + /** 前端算好的文件内容哈希(SHA-256 hex),打开后端 file_hash 去重闸门 */ + file_hash?: string + /** 前端生成的上传幂等 token,同一次逻辑上传(含重试)保持不变 */ + client_upload_id?: string }): Promise => { - const response = await apiClient.post("/upload/direct/prepare", data) + // prepare 单独放宽到 30s(全局 axios 实例只有 10s,staging 抖动时易超时) + const response = await apiClient.post("/upload/direct/prepare", data, { timeout: 30_000 }) return response.data } @@ -22,8 +28,14 @@ export const completeDirectUpload = async (data: { project_id: string library_id: string storage_key: string + /** 前端算好的文件内容哈希(与 prepare 一致),后端按 hash 幂等去重 */ + file_hash?: string + /** 前端上传幂等 token(与 prepare 一致),同一次上传重发 complete 不重复建记录 */ + client_upload_id?: string }): Promise => { - const response = await apiClient.post("/upload/direct/complete", data) + // complete 内含 OSS 存在性检查 + 建库 + 派单,放宽到 60s; + // 超时不代表失败(记录可能已建成),调用方禁止超时后盲目重传整个文件 + const response = await apiClient.post("/upload/direct/complete", data, { timeout: 60_000 }) return response.data } @@ -109,6 +121,10 @@ export interface DirectUploadHandle { export const prepareDirectUploadHandle = async (data: { file: File library_id: string + /** 前端算好的文件内容哈希(SHA-256 hex),prepare/complete 均携带 */ + fileHash?: string + /** 本次逻辑上传的幂等 token,prepare/complete 一致、重试复用 */ + clientUploadId?: string }): Promise => { const project = await getOrCreateDefaultProject() @@ -118,6 +134,8 @@ export const prepareDirectUploadHandle = async (data: { filename: data.file.name, content_type: data.file.type || "application/octet-stream", file_size: data.file.size, + file_hash: data.fileHash, + client_upload_id: data.clientUploadId, }) return { @@ -128,6 +146,8 @@ export const prepareDirectUploadHandle = async (data: { project_id: project.id, library_id: data.library_id, storage_key: prepared.storage_key, + file_hash: data.fileHash, + client_upload_id: data.clientUploadId, }), } } @@ -137,8 +157,20 @@ export const uploadAssetDirect = async (data: { file: File library_id: string onProgress?: (percent: number) => void + /** 文件内容哈希;未传时自动补算(配音/封面/克隆等非队列链路统一受益) */ + fileHash?: string + /** 幂等 token;未传时自动生成 */ + clientUploadId?: string }): Promise => { - const handle = await prepareDirectUploadHandle({ file: data.file, library_id: data.library_id }) + // 自动补算哈希与幂等 token:确保 file_hash 去重闸门对所有上传链路生效 + const fileHash = data.fileHash ?? (await computeFileHash(data.file)) + const clientUploadId = data.clientUploadId ?? makeClientUploadId() + const handle = await prepareDirectUploadHandle({ + file: data.file, + library_id: data.library_id, + fileHash, + clientUploadId, + }) await handle.transfer(data.onProgress) return handle.complete() } diff --git a/apps/web/src/api/assets/uploadDedup.ts b/apps/web/src/api/assets/uploadDedup.ts new file mode 100644 index 000000000..396333dae --- /dev/null +++ b/apps/web/src/api/assets/uploadDedup.ts @@ -0,0 +1,125 @@ +/** + * 上传去重 / 幂等工具(Issue #1714) + * + * 背景:同一文件被反复入队、complete 超时后盲目重传,导致后端创建大量重复 + * PROCESSING 素材记录。本模块提供两类纯函数: + * + * 1. 文件指纹: + * - makeFileFingerprint():文件名+大小+lastModified,入队去重用(同步、零开销) + * - computeFileHash():SHA-256 内容哈希(小文件全量、大文件抽样头尾), + * prepare/complete 时发给后端打开 file_hash 去重闸门 + * 2. 队列去重:findDuplicateInQueue() 判断文件是否已在队列中 + * 3. 幂等 token:makeClientUploadId() 生成上传幂等 ID(每次"一次逻辑上传"一个, + * 重试复用同一 ID,重新入队才生成新 ID) + */ + +/** 大文件抽样阈值:超过此大小只哈希头尾片段,避免上传前长时间卡 UI */ +export const HASH_FULL_READ_LIMIT = 256 * 1024 * 1024 // 256MB +/** 抽样读取的头尾片段大小(各 8MB) */ +export const HASH_SAMPLE_CHUNK = 8 * 1024 * 1024 + +/** 计算指纹时,文件在队列中已存在的状态(已失败的可以重试,不算重复) */ +export type DedupExcludeStatus = "error" | "done" + +/** + * 文件入队指纹:同库 + 文件名 + 大小 + 修改时间。 + * 同一文件(File 对象由 重选或拖拽重复触发时三个字段均一致)稳定复现; + * 不同文件极小概率碰撞时可由后端 file_hash 内容去重兜底。 + */ +export function makeFileFingerprint(file: Pick): string { + return `${file.name}::${file.size}::${file.lastModified}` +} + +/** + * 在现有队列项中查找同一文件的在途记录。 + * 已失败(error)的项允许重试路径复用、已完成(done)的可跳过; + * 处于 preparing/uploading/ingesting 的在途项一律视为重复,禁止重复入队。 + * + * 返回命中的队列项 id(tempId),未命中返回 null。 + */ +export function findDuplicateInQueue( + queue: T[], + fileKey: string, + excludeStatuses: DedupExcludeStatus[] = [], +): T | null { + const exclude = new Set(excludeStatuses) + return queue.find((it) => it.fileKey === fileKey && !exclude.has(it.status)) ?? null +} + +/** 生成上传幂等 token:一次"逻辑上传"一个,重试复用、重新入队换新 */ +export function makeClientUploadId(): string { + const rand = + typeof crypto !== "undefined" && "randomUUID" in crypto + ? crypto.randomUUID() + : `${Date.now()}-${Math.random().toString(36).slice(2, 10)}-${Math.random() + .toString(36) + .slice(2, 10)}` + return `up_${Date.now().toString(36)}_${rand.replace(/-/g, "").slice(0, 16)}` +} + +/** 读取 Blob/File 片段为 ArrayBuffer:优先 Blob.arrayBuffer(),老环境/测试环境回退 FileReader */ +function readAsArrayBuffer(blob: Blob): Promise { + if (typeof blob.arrayBuffer === "function") { + return blob.arrayBuffer() + } + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(reader.result as ArrayBuffer) + reader.onerror = () => reject(reader.error ?? new Error("FileReader read failed")) + reader.readAsArrayBuffer(blob) + }) +} + +function toHex(buffer: ArrayBuffer): string { + const bytes = new Uint8Array(buffer) + let hex = "" + for (let i = 0; i < bytes.length; i += 1) { + hex += bytes[i].toString(16).padStart(2, "0") + } + return hex +} + +/** + * 计算文件内容 SHA-256(hex,64 字符,与后端 file_hash 字段长度一致)。 + * - ≤256MB:全量哈希,内容一致必然一致 + * - >256MB:哈希「头部 8MB + 尾部 8MB + 文件大小」,视频素材体积大、 + * 头部含 moov 元数据、尾部含 mdat 结尾,抽样碰撞概率可忽略, + * 且避免上传前对 2GB 文件全量读取造成长时间卡顿 + * + * 运行环境不支持 crypto.subtle(非安全上下文/老浏览器)时返回空字符串, + * 调用方据此降级为不传 hash(后端仍有幂等 token + 同文件名兜底去重)。 + */ +export async function computeFileHash(file: File): Promise { + try { + const subtle = + typeof globalThis !== "undefined" && + globalThis.crypto && + typeof globalThis.crypto.subtle?.digest === "function" + ? globalThis.crypto.subtle + : null + if (!subtle) return "" + + if (file.size <= HASH_FULL_READ_LIMIT) { + const data = await readAsArrayBuffer(file.slice(0, file.size)) + return toHex(await subtle.digest("SHA-256", data)) + } + + // 大文件:头 8MB + 尾 8MB + 大小,拼成一段后哈希 + const head = await readAsArrayBuffer(file.slice(0, HASH_SAMPLE_CHUNK)) + const tail = + file.size > HASH_SAMPLE_CHUNK + ? await readAsArrayBuffer(file.slice(Math.max(0, file.size - HASH_SAMPLE_CHUNK), file.size)) + : new ArrayBuffer(0) + const merged = new Uint8Array(head.byteLength + tail.byteLength + 8) + merged.set(new Uint8Array(head), 0) + merged.set(new Uint8Array(tail), head.byteLength) + const sizeView = new DataView(merged.buffer, head.byteLength + tail.byteLength, 8) + // 文件大小以 64 位大端写入(JS 安全整数远小于 2^53,高 4 字节为 0) + sizeView.setUint32(0, 0) + sizeView.setUint32(4, file.size >>> 0) + return toHex(await subtle.digest("SHA-256", merged.buffer)) + } catch (err) { + console.warn("[uploadDedup] 计算文件哈希失败,降级为不传 file_hash:", err) + return "" + } +} diff --git a/apps/web/src/pages/assets/AssetLibrary.tsx b/apps/web/src/pages/assets/AssetLibrary.tsx index 17e379623..5e6ace7e7 100644 --- a/apps/web/src/pages/assets/AssetLibrary.tsx +++ b/apps/web/src/pages/assets/AssetLibrary.tsx @@ -42,6 +42,7 @@ const AssetLibrary: React.FC = () => { assetsError, assetsErrorObj, refetchAssets, + stalledAssetIds, searchText, setSearchText, filterType, @@ -77,6 +78,7 @@ const AssetLibrary: React.FC = () => { removeUpload, clearFinished, uploading, + transferActive, activeCount, pendingCount, } = useAssetUpload({ effectiveLibId }) @@ -168,6 +170,7 @@ const AssetLibrary: React.FC = () => { {/* 上传区域 */} { selectedIds={selectedIds} diagnosingId={diagnosingId} uploadProgressMap={uploadProgressMap} + stalledAssetIds={stalledAssetIds} onRetry={refetchAssets} onToggleSelect={toggleSelect} onDiagnose={handleDiagnose} diff --git a/apps/web/src/pages/assets/assets.css b/apps/web/src/pages/assets/assets.css index f237ae738..1c143fb79 100644 --- a/apps/web/src/pages/assets/assets.css +++ b/apps/web/src/pages/assets/assets.css @@ -1041,3 +1041,25 @@ background: #fef2f2; color: #dc2626; } + +/* 上传入口禁用态(直传进行中,防重复提交,Issue #1714) */ +.xx-asset-upload-btn:disabled { + opacity: 0.6; + cursor: not-allowed; +} +.xx-asset-upload-btn:disabled:hover { + opacity: 0.6; +} +.xx-asset-upload-btn:disabled:active { + transform: none; +} + +/* 处理超时遮罩:创建超过 10 分钟仍在处理中(疑似后端卡住),停止转圈并警示 */ +.xx-asset-thumb-stalled { + background: rgba(217, 119, 6, 0.28); + color: #fde68a; + backdrop-filter: blur(2px); +} +.xx-asset-thumb-stalled :first-child { + font-size: var(--font-size-2xl); +} diff --git a/apps/web/src/pages/assets/components/AssetCard.tsx b/apps/web/src/pages/assets/components/AssetCard.tsx index b0b0ea21c..e88a3f6fd 100644 --- a/apps/web/src/pages/assets/components/AssetCard.tsx +++ b/apps/web/src/pages/assets/components/AssetCard.tsx @@ -22,6 +22,8 @@ export interface AssetCardProps { diagnosing?: boolean /** 上传中实时进度(仅 uploading 态有值;ingesting 后由后端状态接管) */ uploadProgress?: { progress: number; uploading: boolean } + /** 处理超过 10 分钟仍未就绪(疑似后端卡住):停止转圈并提示处理超时 */ + stalled?: boolean onToggle: () => void onDiagnose: () => void onPlay: () => void @@ -33,6 +35,7 @@ const AssetCard: React.FC = ({ selected, diagnosing, uploadProgress, + stalled, onToggle, onDiagnose, onPlay, @@ -67,11 +70,15 @@ const AssetCard: React.FC = ({ )} - {/* 转码/处理中遮罩 */} + {/* 转码/处理中遮罩(卡死超过 10 分钟时停止转圈,提示超时) */} {asset.loading && !isUploading && ( -
- - 转码处理中 +
+ {stalled ? : } + {stalled ? "处理超时,可重试上传" : "转码处理中"}
)} @@ -129,7 +136,7 @@ const AssetCard: React.FC = ({

- + {asset.duration && {asset.duration}}
diff --git a/apps/web/src/pages/assets/components/AssetGridSection.tsx b/apps/web/src/pages/assets/components/AssetGridSection.tsx index c88307734..2af6d87ff 100644 --- a/apps/web/src/pages/assets/components/AssetGridSection.tsx +++ b/apps/web/src/pages/assets/components/AssetGridSection.tsx @@ -19,6 +19,8 @@ export interface AssetGridSectionProps { selectedIds: Set diagnosingId: string | null uploadProgressMap?: UploadProgressMap + /** 创建超过 10 分钟仍在处理中的素材 id(疑似后端卡住),卡片提示处理超时 */ + stalledAssetIds?: Set onRetry?: () => void onToggleSelect: (id: string) => void onDiagnose: (asset: AssetItem) => void @@ -34,6 +36,7 @@ export const AssetGridSection: React.FC = ({ selectedIds, diagnosingId, uploadProgressMap, + stalledAssetIds, onRetry, onToggleSelect, onDiagnose, @@ -76,6 +79,7 @@ export const AssetGridSection: React.FC = ({ selected={selectedIds.has(asset.id)} diagnosing={diagnosingId === asset.id} uploadProgress={uploadProgressMap?.get(asset.id)} + stalled={stalledAssetIds?.has(asset.id)} onToggle={() => onToggleSelect(asset.id)} onDiagnose={() => onDiagnose(asset)} onPlay={() => onPlay(asset)} diff --git a/apps/web/src/pages/assets/components/AssetUploadZone.tsx b/apps/web/src/pages/assets/components/AssetUploadZone.tsx index bb9abdac8..c06d3f515 100644 --- a/apps/web/src/pages/assets/components/AssetUploadZone.tsx +++ b/apps/web/src/pages/assets/components/AssetUploadZone.tsx @@ -4,10 +4,13 @@ * - 拖拽文件到内容区任意位置同样触发上传(不再占用大面积虚线框) */ import React, { useRef, useState } from "react" +import { message } from "antd" import { PlusOutlined, CloudUploadOutlined } from "@ant-design/icons" export interface AssetUploadZoneProps { uploading: boolean + /** 有文件正在本地指纹/prepare/直传(非服务端转码),此时禁用入口防重复提交 */ + transferActive: boolean activeCount: number pendingCount: number onUpload: (files: File[]) => void @@ -15,6 +18,7 @@ export interface AssetUploadZoneProps { export const AssetUploadZone: React.FC = ({ uploading, + transferActive, activeCount, pendingCount, onUpload, @@ -27,6 +31,11 @@ export const AssetUploadZone: React.FC = ({ const pickFiles = (list: FileList | null) => { if (!list || list.length === 0) return + // 直传进行中拦截重复触发:相同文件仍由入队去重兜底,这里先给明确反馈 + if (transferActive) { + message.warning("文件正在上传中,请等待当前上传完成后再添加") + return + } onUpload(Array.from(list)) } @@ -58,10 +67,18 @@ export const AssetUploadZone: React.FC = ({ {uploading ? ( diff --git a/apps/web/src/pages/assets/components/UploadQueuePanel.tsx b/apps/web/src/pages/assets/components/UploadQueuePanel.tsx index 1dead1405..a88f4f9eb 100644 --- a/apps/web/src/pages/assets/components/UploadQueuePanel.tsx +++ b/apps/web/src/pages/assets/components/UploadQueuePanel.tsx @@ -80,6 +80,7 @@ const UploadQueuePanel: React.FC = ({ ) : null}
{it.duplicated ? "素材已存在,已跳过" : STATUS_TEXT[it.status]} + {it.status === "preparing" && it.hint ? `(${it.hint})` : ""} {it.status === "uploading" ? ` ${it.progress}%` : ""} {it.status === "error" && it.error ? `:${it.error}` : ""}
@@ -89,7 +90,9 @@ const UploadQueuePanel: React.FC = ({