Compare commits
24 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7a4aa27f71 | |||
| 28b3010668 | |||
| c1763b995c | |||
| 3a8faeb31d | |||
| d336382f3a | |||
| d947171713 | |||
| 1f4c907bed | |||
| 121820caa9 | |||
| 52f281a66c | |||
| f1bd2d6f1d | |||
| eac05dee30 | |||
| 4263e7f6ca | |||
| db244fe14c | |||
| e86f137c3d | |||
| 8a3115bc54 | |||
| 3fcc65840e | |||
| 4633126bb4 | |||
| ed72a91990 | |||
| 02d226a163 | |||
| 475ee59408 | |||
| 452a484c5b | |||
| d01040cb93 | |||
| f523548eee | |||
| 0542654ca8 |
@@ -23,6 +23,7 @@ from app.dependencies import (
|
||||
get_generation_task_repository,
|
||||
)
|
||||
from app.schemas.generation_task import (
|
||||
BatchPreviewGenerationTaskResponse,
|
||||
CreatePreviewGenerationTaskRequest,
|
||||
PreviewGenerationTaskResponse,
|
||||
)
|
||||
@@ -193,11 +194,19 @@ def _to_preview_response(task, generated_videos: list | None = None) -> PreviewG
|
||||
if started_at and completed_at:
|
||||
generate_duration = (completed_at - started_at).total_seconds()
|
||||
|
||||
title_cfg = getattr(task, "title_config", None)
|
||||
title_cfg = title_cfg if isinstance(title_cfg, dict) else {}
|
||||
extra_meta = getattr(task, "extra_meta", None)
|
||||
extra_meta = extra_meta if isinstance(extra_meta, dict) else {}
|
||||
voice_library_id = getattr(task, "voice_library_id", "") or ""
|
||||
if not isinstance(voice_library_id, str):
|
||||
voice_library_id = str(voice_library_id) if voice_library_id else ""
|
||||
return PreviewGenerationTaskResponse(
|
||||
task_id=task.id,
|
||||
status=task.status.value if hasattr(task.status, "value") else str(task.status),
|
||||
progress=float(task.progress or 0.0),
|
||||
is_preview=bool(getattr(task, "is_preview", True)),
|
||||
variant_index=int(extra_meta.get("variant_index", 0) or 0),
|
||||
resolution=getattr(task, "resolution", "") or "",
|
||||
video_url=video_url,
|
||||
duration=duration,
|
||||
@@ -206,6 +215,8 @@ def _to_preview_response(task, generated_videos: list | None = None) -> PreviewG
|
||||
transition_count=transition_count,
|
||||
material_usage=material_usage,
|
||||
error_message=task.error_message or "",
|
||||
title_text=str(title_cfg.get("text", "") or ""),
|
||||
voice_library_id=voice_library_id,
|
||||
created_at=task.created_at,
|
||||
started_at=started_at,
|
||||
finished_at=completed_at,
|
||||
@@ -213,45 +224,95 @@ def _to_preview_response(task, generated_videos: list | None = None) -> PreviewG
|
||||
)
|
||||
|
||||
|
||||
@router.post("/preview", response_model=PreviewGenerationTaskResponse, status_code=201)
|
||||
def _resolve_preview_edit_plan_id(
|
||||
*,
|
||||
request: CreatePreviewGenerationTaskRequest,
|
||||
task,
|
||||
db: Session,
|
||||
user_id: str,
|
||||
) -> str:
|
||||
"""确定任务关联的编辑计划ID:优先前端传入,否则按 template_id+user 兜底查找。"""
|
||||
if task.source_edit_plan_id:
|
||||
return task.source_edit_plan_id
|
||||
if not request.template_id:
|
||||
return ""
|
||||
try:
|
||||
from packages.adapters.sqlalchemy_impl.edit_plan_repository import (
|
||||
SQLAlchemyEditPlanRepository,
|
||||
)
|
||||
|
||||
_plan_repo = SQLAlchemyEditPlanRepository(db)
|
||||
_plans = _plan_repo.list_by_template(request.template_id, limit=20)
|
||||
for _p in _plans:
|
||||
if (_p.created_by_user_id or "") == user_id:
|
||||
logger.info(
|
||||
"[预览生成] 自动关联编辑计划: task_id=%s plan_id=%s",
|
||||
task.id,
|
||||
_p.id,
|
||||
)
|
||||
return _p.id
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"[预览生成] 查找关联编辑计划失败(不影响主流程): task_id=%s",
|
||||
task.id,
|
||||
exc_info=True,
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def _variant_value(values: list[str], index: int, fallback: str = "") -> str:
|
||||
"""从变体数组中取值:长度1=共用,长度>N=按索引,空数组=回退 fallback。"""
|
||||
if not values:
|
||||
return fallback
|
||||
if len(values) == 1:
|
||||
return values[0]
|
||||
return values[index] if index < len(values) else fallback
|
||||
|
||||
|
||||
@router.post("/preview", response_model=BatchPreviewGenerationTaskResponse, status_code=201)
|
||||
def create_preview_generation_task(
|
||||
request: CreatePreviewGenerationTaskRequest,
|
||||
authenticated_user: AuthenticatedUser = Depends(get_current_user),
|
||||
generation_task_repository=Depends(get_generation_task_repository),
|
||||
db: Session = Depends(get_db_session),
|
||||
asset_repo=Depends(get_asset_repository),
|
||||
) -> PreviewGenerationTaskResponse:
|
||||
"""创建预览生成任务。
|
||||
) -> BatchPreviewGenerationTaskResponse:
|
||||
"""创建预览生成任务(支持批量)。
|
||||
|
||||
预览渲染品质与正式生成一致(1080p, CRF 23, medium preset),确认生成时可直接复用预览产物。
|
||||
|
||||
Args:
|
||||
request: 预览任务创建请求(template_id + asset_ids 等)
|
||||
preview_count=1 时行为与旧版完全一致(创建 1 个任务);
|
||||
preview_count=N 时一次创建 N 个独立变体任务:
|
||||
- 每个变体克隆独立编辑计划(独立 clips、独立随机素材起点),N 个预览内容互不相同
|
||||
- 每个变体拥有独立 task_id / 状态 / 预览视频 URL,前端按 task_id 分别轮询
|
||||
- 标题样式(font/color/position 等)全局共用;标题文字/配音/封面可按变体独立
|
||||
(titles[] / voice_library_ids[] / cover_urls[],长度1=共用,长度N=独立)
|
||||
|
||||
Returns:
|
||||
201 + 预览任务详情
|
||||
201 + 变体任务数组 {items: [...], total: N}
|
||||
"""
|
||||
user_id = authenticated_user.user.id
|
||||
count = max(1, request.preview_count)
|
||||
logger.info(
|
||||
"[预览生成] 接收请求: user_id=%s, template_id=%s, asset_count=%d, preview_count=%d",
|
||||
user_id,
|
||||
request.template_id,
|
||||
len(request.asset_ids),
|
||||
request.preview_count,
|
||||
count,
|
||||
)
|
||||
|
||||
# 预检查队列限流
|
||||
# 预检查队列限流(按变体总数计)
|
||||
try:
|
||||
user_pending = generation_task_repository.count_pending_by_user(user_id)
|
||||
global_pending = generation_task_repository.count_pending_total()
|
||||
if user_pending + 1 > USER_PENDING_LIMIT:
|
||||
raise UserPendingLimitExceeded(user_id=user_id, pending_count=user_pending + 1, limit=USER_PENDING_LIMIT)
|
||||
if global_pending + 1 > GLOBAL_PENDING_LIMIT:
|
||||
raise GlobalQueueFull(pending_count=global_pending + 1, limit=GLOBAL_PENDING_LIMIT)
|
||||
if user_pending + count > USER_PENDING_LIMIT:
|
||||
raise UserPendingLimitExceeded(
|
||||
user_id=user_id, pending_count=user_pending + count, limit=USER_PENDING_LIMIT
|
||||
)
|
||||
if global_pending + count > GLOBAL_PENDING_LIMIT:
|
||||
raise GlobalQueueFull(pending_count=global_pending + count, limit=GLOBAL_PENDING_LIMIT)
|
||||
except UserPendingLimitExceeded as e:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"您的待处理任务过多(当前 {e.pending_count - 1}/{e.limit}),请等待后再提交",
|
||||
detail=f"您的待处理任务过多(当前 {e.pending_count - count}/{e.limit},本次提交 {count} 个),请等待后再提交",
|
||||
) from e
|
||||
except GlobalQueueFull as e:
|
||||
raise HTTPException(
|
||||
@@ -273,14 +334,11 @@ def create_preview_generation_task(
|
||||
w, h = int(parts[0]), int(parts[1])
|
||||
base = 1920
|
||||
if w < h:
|
||||
# 竖屏
|
||||
output_width = round(base * w / h)
|
||||
output_height = base
|
||||
else:
|
||||
# 横屏
|
||||
output_width = base
|
||||
output_height = round(base * h / w)
|
||||
# 对齐到偶数
|
||||
output_width = output_width - output_width % 2
|
||||
output_height = output_height - output_height % 2
|
||||
except (ValueError, ZeroDivisionError):
|
||||
@@ -289,42 +347,71 @@ def create_preview_generation_task(
|
||||
|
||||
logger.info(
|
||||
"[预览生成] 分辨率: video_ratio=%s → %s (%dx%d)",
|
||||
video_ratio, resolution, output_width, output_height,
|
||||
video_ratio,
|
||||
resolution,
|
||||
output_width,
|
||||
output_height,
|
||||
)
|
||||
|
||||
# 从模板读取 editing_mode / mode 作为 strategy_id(渲染 pipeline 的 mode 参数)
|
||||
strategy_id = _resolve_strategy_id_from_template(request.template_id, db, user_id)
|
||||
|
||||
title_config = request.title_config or {}
|
||||
base_title_config = request.title_config or {}
|
||||
|
||||
use_case = CreateGenerationTaskUseCase(generation_task_repository)
|
||||
|
||||
# ── 预创建第一个任务,仅用于解析源编辑计划(不落库为最终任务)──
|
||||
# 先创建一个临时任务拿到 task 对象上下文,实际 N 个任务在循环中统一创建;
|
||||
# 为保持与旧版一致的源 plan 解析逻辑,先创建任务0、解析源 plan,
|
||||
# 再预克隆 N 个变体 plan,最后重建任务关联。
|
||||
# 简化实现:直接创建全部任务,plan 关联在创建后、入队前完成。
|
||||
|
||||
created_tasks: list = []
|
||||
variant_plan_ids: list[str] = [] # 每个变体最终关联的 plan_id(按变体顺序)
|
||||
|
||||
try:
|
||||
task = use_case.execute(
|
||||
CreateGenerationTaskCommand(
|
||||
project_id="",
|
||||
asset_library_id="",
|
||||
strategy_id=strategy_id,
|
||||
voice_library_id=request.voice_library_id,
|
||||
template_id=request.template_id,
|
||||
asset_ids=list(request.asset_ids),
|
||||
title_ids=list(request.title_ids),
|
||||
voice_ids=list(request.voice_ids),
|
||||
created_by_user_id=user_id,
|
||||
source_edit_plan_id=request.source_edit_plan_id,
|
||||
asset_select_mode="",
|
||||
batch_id="",
|
||||
video_title=request.video_title,
|
||||
resolution=resolution,
|
||||
bgm_config=request.bgm_config or {},
|
||||
auto_retry_enabled=False,
|
||||
auto_retry_max=0,
|
||||
is_preview=True,
|
||||
title_config=title_config,
|
||||
output_width=output_width,
|
||||
output_height=output_height,
|
||||
for variant_index in range(count):
|
||||
# 变体独立标题文字:titles[] 覆盖 title_config.text
|
||||
variant_title_text = _variant_value(request.titles, variant_index, "")
|
||||
variant_title_config = dict(base_title_config)
|
||||
if variant_title_text.strip():
|
||||
variant_title_config["text"] = variant_title_text.strip()
|
||||
|
||||
# 变体独立配音
|
||||
variant_voice_library_id = _variant_value(
|
||||
request.voice_library_ids, variant_index, request.voice_library_id
|
||||
)
|
||||
)
|
||||
|
||||
task = use_case.execute(
|
||||
CreateGenerationTaskCommand(
|
||||
project_id="",
|
||||
asset_library_id="",
|
||||
strategy_id=strategy_id,
|
||||
voice_library_id=variant_voice_library_id,
|
||||
template_id=request.template_id,
|
||||
asset_ids=list(request.asset_ids),
|
||||
title_ids=list(request.title_ids),
|
||||
voice_ids=list(request.voice_ids),
|
||||
created_by_user_id=user_id,
|
||||
source_edit_plan_id=request.source_edit_plan_id,
|
||||
asset_select_mode="",
|
||||
batch_id="",
|
||||
video_title=request.video_title,
|
||||
resolution=resolution,
|
||||
bgm_config=request.bgm_config or {},
|
||||
auto_retry_enabled=False,
|
||||
auto_retry_max=0,
|
||||
is_preview=True,
|
||||
title_config=variant_title_config,
|
||||
output_width=output_width,
|
||||
output_height=output_height,
|
||||
)
|
||||
)
|
||||
task.extra_meta["variant_index"] = variant_index
|
||||
|
||||
# 解析源编辑计划(前端传入或按模板兜底查找)
|
||||
source_plan_id = _resolve_preview_edit_plan_id(request=request, task=task, db=db, user_id=user_id)
|
||||
task.source_edit_plan_id = source_plan_id
|
||||
generation_task_repository.update(task)
|
||||
created_tasks.append(task)
|
||||
except ValueError as e:
|
||||
logger.warning("[预览生成] 创建失败: %s", e)
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
@@ -332,93 +419,121 @@ def create_preview_generation_task(
|
||||
logger.error("[预览生成] 创建失败: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="创建预览生成任务失败,请稍后再试") from e
|
||||
|
||||
# 关联编辑计划:如果前端未传 source_edit_plan_id,通过 template_id + user_id 查找
|
||||
if not task.source_edit_plan_id and request.template_id:
|
||||
try:
|
||||
from packages.adapters.sqlalchemy_impl.edit_plan_repository import (
|
||||
SQLAlchemyEditPlanRepository,
|
||||
)
|
||||
|
||||
_plan_repo = SQLAlchemyEditPlanRepository(db)
|
||||
_plans = _plan_repo.list_by_template(request.template_id, limit=20)
|
||||
for _p in _plans:
|
||||
if (_p.created_by_user_id or "") == user_id:
|
||||
task.source_edit_plan_id = _p.id
|
||||
generation_task_repository.update(task)
|
||||
logger.info(
|
||||
"[预览生成] 自动关联编辑计划: task_id=%s plan_id=%s",
|
||||
task.id,
|
||||
_p.id,
|
||||
)
|
||||
break
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"[预览生成] 查找关联编辑计划失败(不影响主流程): task_id=%s",
|
||||
task.id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
# 每条预览都关联独立克隆 plan:多预览前端为 N 次并发调用,若共用同一 plan
|
||||
# 则 N 条预览片段完全相同;克隆时片段起点按持久化历史区间重算(含受控复用),
|
||||
# 保证各预览版本内容不同
|
||||
if task.source_edit_plan_id:
|
||||
# ── 克隆独立变体 plan:N 个预览全部克隆(预览不污染源 plan)──
|
||||
# 源 plan 不存在(无编辑历史)时各任务走自身随机选片流程,不克隆。
|
||||
source_plan_id = created_tasks[0].source_edit_plan_id if created_tasks else ""
|
||||
if source_plan_id:
|
||||
try:
|
||||
from app.services.edit_plan_service import EditPlanService
|
||||
|
||||
_plan_svc = EditPlanService(db)
|
||||
_preview_plan = _plan_svc.clone_plan_for_variant(
|
||||
task.source_edit_plan_id,
|
||||
created_by_user_id=user_id,
|
||||
name_suffix="预览变体",
|
||||
)
|
||||
task.source_edit_plan_id = _preview_plan.id
|
||||
generation_task_repository.update(task)
|
||||
logger.info(
|
||||
"[预览生成] 预览关联独立克隆 plan: task_id=%s clone_plan_id=%s",
|
||||
task.id,
|
||||
_preview_plan.id,
|
||||
)
|
||||
except Exception as clone_err:
|
||||
# 不退回共用原 plan(否则多条预览内容相同,违反去重诉求):
|
||||
# 标记任务失败并中断,前端可重新发起预览
|
||||
logger.error(
|
||||
"[预览生成] 克隆预览变体 plan 失败,任务标记失败: task_id=%s error=%s",
|
||||
task.id,
|
||||
clone_err,
|
||||
exc_info=True,
|
||||
)
|
||||
_mark_task_failed(generation_task_repository, task, "预览变体计划创建失败")
|
||||
for variant_index in range(count):
|
||||
last_err: Exception | None = None
|
||||
variant_plan = None
|
||||
for _attempt in range(2): # 1 次重试,抗 DB 瞬时抖动
|
||||
try:
|
||||
variant_plan = _plan_svc.clone_plan_for_variant(
|
||||
source_plan_id,
|
||||
created_by_user_id=user_id,
|
||||
name_suffix=f"预览变体{variant_index + 1}" if count > 1 else "预览变体",
|
||||
)
|
||||
break
|
||||
except Exception as clone_err: # noqa: PERF203
|
||||
last_err = clone_err
|
||||
logger.warning(
|
||||
"[预览生成] 克隆变体 plan 失败(尝试%d/2): variant=%d error=%s",
|
||||
_attempt + 1,
|
||||
variant_index,
|
||||
clone_err,
|
||||
exc_info=True,
|
||||
)
|
||||
if variant_plan is None:
|
||||
logger.error(
|
||||
"[预览生成] 克隆预览变体 plan 重试仍失败: variant=%d source=%s",
|
||||
variant_index,
|
||||
source_plan_id,
|
||||
exc_info=last_err,
|
||||
)
|
||||
# 标记已创建任务失败
|
||||
for t in created_tasks:
|
||||
_mark_task_failed(generation_task_repository, t, "预览变体计划创建失败")
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="创建预览任务失败:无法生成独立剪辑计划,请重试",
|
||||
) from last_err
|
||||
variant_plan_ids.append(variant_plan.id)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("[预览生成] 克隆变体 plan 异常: %s", e, exc_info=True)
|
||||
for t in created_tasks:
|
||||
_mark_task_failed(generation_task_repository, t, "预览变体计划创建失败")
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="创建预览任务失败:无法生成独立剪辑计划,请重试",
|
||||
) from clone_err
|
||||
) from e
|
||||
|
||||
# 入队执行;若入队失败则标记任务为 failed 避免僵尸数据
|
||||
try:
|
||||
if not safe_enqueue_generation_task(
|
||||
task,
|
||||
generation_task_repository,
|
||||
user_id=user_id,
|
||||
log_prefix="[预览生成]",
|
||||
log_task_status=True,
|
||||
):
|
||||
logger.warning("[预览生成] 任务入队失败: task_id=%s", task.id)
|
||||
_mark_task_failed(generation_task_repository, task, "任务入队失败")
|
||||
raise HTTPException(status_code=500, detail="任务入队失败,请稍后重试")
|
||||
except UserPendingLimitExceeded as e:
|
||||
_mark_task_failed(generation_task_repository, task, "待处理任务超限")
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"您的待处理任务过多(当前 {e.pending_count - 1}/{e.limit}),请等待后再提交",
|
||||
) from None
|
||||
except GlobalQueueFull:
|
||||
_mark_task_failed(generation_task_repository, task, "系统队列已满")
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="系统繁忙,请稍后再试",
|
||||
) from None
|
||||
# 关联变体 plan 并回写标题配置
|
||||
for variant_index, task in enumerate(created_tasks):
|
||||
if variant_plan_ids:
|
||||
task.source_edit_plan_id = variant_plan_ids[variant_index]
|
||||
generation_task_repository.update(task)
|
||||
# 回写变体标题到 plan config(worker 渲染时从 plan 读取 title 配置)
|
||||
if task.source_edit_plan_id and (task.title_config or {}).get("text", "").strip():
|
||||
try:
|
||||
from app.api.routes.generation_tasks import _writeback_edit_plan_config
|
||||
|
||||
return _to_preview_response(task)
|
||||
_writeback_edit_plan_config(
|
||||
plan_id=task.source_edit_plan_id,
|
||||
task_id=task.id,
|
||||
title_config=task.title_config,
|
||||
db=db,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"[预览生成] 回写标题配置失败(不影响主流程): task_id=%s",
|
||||
task.id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
# ── 入队 ──
|
||||
responses: list[PreviewGenerationTaskResponse] = []
|
||||
for variant_index, task in enumerate(created_tasks):
|
||||
try:
|
||||
enqueued = safe_enqueue_generation_task(
|
||||
task,
|
||||
generation_task_repository,
|
||||
user_id=user_id,
|
||||
log_prefix=f"[预览生成][变体{variant_index + 1}]",
|
||||
log_task_status=True,
|
||||
)
|
||||
if not enqueued:
|
||||
logger.warning("[预览生成] 任务入队失败: task_id=%s", task.id)
|
||||
_mark_task_failed(generation_task_repository, task, "任务入队失败")
|
||||
except UserPendingLimitExceeded:
|
||||
_mark_task_failed(generation_task_repository, task, "待处理任务超限")
|
||||
except GlobalQueueFull:
|
||||
_mark_task_failed(generation_task_repository, task, "系统队列已满")
|
||||
except Exception:
|
||||
logger.exception("[预览生成] 入队异常: task_id=%s", task.id)
|
||||
_mark_task_failed(generation_task_repository, task, "任务入队异常")
|
||||
# enqueue 会原地更新 task 状态/进度,直接用 task 构造响应
|
||||
responses.append(_to_preview_response(task))
|
||||
|
||||
# 队列满/限流时若全部失败,返回明确错误码
|
||||
if all(r.status == "failed" for r in responses):
|
||||
first_err = next((r.error_message for r in responses if r.error_message), "")
|
||||
if "待处理任务" in first_err:
|
||||
raise HTTPException(status_code=429, detail=first_err or "待处理任务超限")
|
||||
if "队列" in first_err:
|
||||
raise HTTPException(status_code=503, detail=first_err or "系统繁忙,请稍后再试")
|
||||
|
||||
logger.info(
|
||||
"[预览生成] 创建完成: %d 个变体任务, task_ids=%s",
|
||||
len(responses),
|
||||
[r.task_id for r in responses],
|
||||
)
|
||||
return BatchPreviewGenerationTaskResponse(items=responses, total=len(responses))
|
||||
|
||||
|
||||
@router.get("/preview/{task_id}", response_model=PreviewGenerationTaskResponse)
|
||||
|
||||
@@ -47,6 +47,15 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _variant_value(values: list[str], index: int, fallback: str = "") -> str:
|
||||
"""从变体数组中取值:长度1=共用,长度>N=按索引,空数组=回退 fallback。"""
|
||||
if not values:
|
||||
return fallback
|
||||
if len(values) == 1:
|
||||
return values[0]
|
||||
return values[index] if index < len(values) else fallback
|
||||
|
||||
|
||||
def _to_generation_task_response(task) -> GenerationTaskResponse:
|
||||
return GenerationTaskResponse(
|
||||
id=task.id,
|
||||
@@ -472,12 +481,21 @@ def create_generation_task(
|
||||
if task_index > 0 and variant_plan_ids:
|
||||
effective_plan_id = variant_plan_ids[task_index - 1]
|
||||
|
||||
# 变体级独立配置:titles[]/voice_library_ids[]/cover_urls[]
|
||||
# 长度1=所有变体共用,长度=count=每个变体独立,空数组=回退单值字段
|
||||
variant_title_text = _variant_value(request.titles, task_index, "")
|
||||
variant_title_config = dict(request.title_config or {})
|
||||
if variant_title_text.strip():
|
||||
variant_title_config["text"] = variant_title_text.strip()
|
||||
variant_voice_library_id = _variant_value(request.voice_library_ids, task_index, request.voice_library_id)
|
||||
variant_cover_url = _variant_value(request.cover_urls, task_index, request.cover_url)
|
||||
|
||||
task = use_case.execute(
|
||||
CreateGenerationTaskCommand(
|
||||
project_id=project_id,
|
||||
asset_library_id=asset_library_id,
|
||||
strategy_id=effective_strategy_id,
|
||||
voice_library_id=request.voice_library_id,
|
||||
voice_library_id=variant_voice_library_id,
|
||||
template_id=request.template_id,
|
||||
asset_ids=resolved_asset_ids,
|
||||
title_ids=request.title_ids,
|
||||
@@ -495,10 +513,12 @@ def create_generation_task(
|
||||
source_task_id=request.source_task_id,
|
||||
output_width=request.output_width,
|
||||
output_height=request.output_height,
|
||||
cover_url=request.cover_url,
|
||||
title_config=request.title_config or {},
|
||||
cover_url=variant_cover_url,
|
||||
title_config=variant_title_config,
|
||||
)
|
||||
)
|
||||
# 变体序号写入 extra_meta(响应/排查时可辨识)
|
||||
task.extra_meta["variant_index"] = task_index
|
||||
try:
|
||||
# 兜底关联编辑计划:前端未传 source_edit_plan_id 时,
|
||||
# 通过 template_id + user_id 在 DB 层直接查找最新的 plan。
|
||||
@@ -533,13 +553,13 @@ def create_generation_task(
|
||||
|
||||
# 回写 plan.config:必须在 enqueue 之前执行,
|
||||
# 确保 worker 读取 plan 时 config 中已包含 generation_task_id。
|
||||
# 只在首个任务时回写一次,避免批量生成时循环覆盖。
|
||||
# 批量场景下每个变体关联独立 plan,需各自回写自己的变体标题配置。
|
||||
_effective_plan_id = task.source_edit_plan_id
|
||||
if _effective_plan_id and len(created_tasks) == 0:
|
||||
if _effective_plan_id:
|
||||
_writeback_edit_plan_config(
|
||||
plan_id=_effective_plan_id,
|
||||
task_id=task.id,
|
||||
title_config=request.title_config,
|
||||
title_config=variant_title_config,
|
||||
db=db,
|
||||
)
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ from app.schemas.video_center import (
|
||||
VideoItemResponse,
|
||||
)
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Response
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from packages.application import (
|
||||
GetGeneratedVideoUseCase,
|
||||
@@ -239,3 +240,74 @@ def get_batch_download_status(
|
||||
status=api_status,
|
||||
download_url=download_url,
|
||||
)
|
||||
|
||||
|
||||
# ── 重新计算查重率 ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
class RecomputeDedupRequest(BaseModel):
|
||||
"""重新计算查重率请求。"""
|
||||
|
||||
video_ids: list[str] | None = Field(
|
||||
None,
|
||||
description="指定视频 ID 列表。为空则对当前用户所有缺少查重数据的视频重新计算。",
|
||||
)
|
||||
force: bool = Field(
|
||||
False,
|
||||
description="强制重算:即使视频已有查重数据也重新入队(#1702 查重算法升级后用于存量视频重算)。",
|
||||
)
|
||||
|
||||
|
||||
class RecomputeDedupResponse(BaseModel):
|
||||
"""重新计算查重率响应。"""
|
||||
|
||||
enqueued: int = Field(..., description="已入队的任务数量")
|
||||
total_scanned: int = Field(..., description="扫描的视频总数")
|
||||
skipped: int = Field(..., description="已有查重数据跳过的数量")
|
||||
message: str = ""
|
||||
|
||||
|
||||
@router.post("/videos/recompute-dedup", response_model=RecomputeDedupResponse)
|
||||
def recompute_dedup(
|
||||
request: RecomputeDedupRequest = RecomputeDedupRequest(),
|
||||
repo=Depends(get_generated_video_repository),
|
||||
current_user: AuthenticatedUser = Depends(get_current_user),
|
||||
):
|
||||
"""重新计算视频的查重率/视觉相似度。
|
||||
|
||||
对于已存在但缺少 duplicate_rate / video_fingerprint 的视频,
|
||||
触发异步 Celery 任务重新下载并计算指纹 + 查重率。
|
||||
|
||||
不传 video_ids 时,对当前用户所有视频进行检查。
|
||||
"""
|
||||
user_id = current_user.user.id
|
||||
|
||||
# 获取目标视频列表
|
||||
if request.video_ids:
|
||||
all_videos = repo.get_by_ids(request.video_ids)
|
||||
# 安全校验:只处理当前用户的视频
|
||||
target_videos = [v for v in all_videos if v.user_id == user_id]
|
||||
else:
|
||||
target_videos = repo.list_by_user(user_id)
|
||||
|
||||
total_scanned = len(target_videos)
|
||||
enqueued = 0
|
||||
skipped = 0
|
||||
|
||||
for video in target_videos:
|
||||
# 已有完整查重数据的跳过(force=True 时强制重算,#1702 算法升级后存量视频需要重算指纹/分片)
|
||||
if not request.force and video.duplicate_rate is not None and video.video_fingerprint:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# 触发异步查重任务
|
||||
celery_app.send_task("worker.check_duplicate", args=[video.id])
|
||||
enqueued += 1
|
||||
logger.info("Enqueued re-dedup for video %s (user=%s, force=%s)", video.id, user_id, request.force)
|
||||
|
||||
return RecomputeDedupResponse(
|
||||
enqueued=enqueued,
|
||||
total_scanned=total_scanned,
|
||||
skipped=skipped,
|
||||
message=f"已入队 {enqueued} 个查重任务" if enqueued > 0 else "所有视频查重数据已完整",
|
||||
)
|
||||
|
||||
@@ -25,6 +25,12 @@ class CreateGenerationTaskRequest(BaseModel):
|
||||
asset_library_id: str = ""
|
||||
strategy_id: str = ""
|
||||
voice_library_id: str = ""
|
||||
# ── 多变体独立配音(批量生成)──
|
||||
# 长度 1 = 所有变体共用;长度 = count = 每个变体独立配音;空数组 = 回退 voice_library_id
|
||||
voice_library_ids: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="各变体独立配音素材库ID数组:长度1=共用,长度=count=独立。为空时回退 voice_library_id",
|
||||
)
|
||||
created_by_user_id: str = ""
|
||||
# ── 模板模式新增字段 ──
|
||||
template_id: str = ""
|
||||
@@ -75,6 +81,27 @@ class CreateGenerationTaskRequest(BaseModel):
|
||||
output_width: int = Field(default=1280, description="输出视频宽度")
|
||||
output_height: int = Field(default=720, description="输出视频高度")
|
||||
cover_url: str = Field(default="", description="封面图片 URL")
|
||||
# ── 多变体独立封面(批量生成)──
|
||||
# 长度 1 = 所有变体共用;长度 = count = 每个变体独立封面;空数组 = 回退 cover_url
|
||||
cover_urls: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="各变体独立封面URL数组:长度1=共用,长度=count=独立。为空时回退 cover_url",
|
||||
)
|
||||
# ── 多变体独立标题文字(批量生成)──
|
||||
# 长度 1 = 所有变体共用;长度 = count = 每个变体独立标题文字;空数组 = 使用 title_config.text
|
||||
titles: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="各变体独立标题文字数组:长度1=共用,长度=count=独立。为空时使用 title_config.text",
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _check_variant_arrays(self) -> "CreateGenerationTaskRequest":
|
||||
"""变体数组字段长度校验:空数组(回退单值)、长度 1(共用)、或长度 = count(独立)。"""
|
||||
for name in ("voice_library_ids", "cover_urls", "titles"):
|
||||
arr = getattr(self, name)
|
||||
if arr and len(arr) != 1 and len(arr) != self.count:
|
||||
raise ValueError(f"{name} 长度必须为 1(共用)或 {self.count}(与 count 一致),当前为 {len(arr)}")
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _check_at_least_one_mode(self) -> "CreateGenerationTaskRequest":
|
||||
@@ -185,8 +212,33 @@ class CreatePreviewGenerationTaskRequest(BaseModel):
|
||||
)
|
||||
title_config: dict = Field(
|
||||
default_factory=dict,
|
||||
description="标题配置(可选),渲染时烧录到预览视频中。支持字段: text/font/font_size/font_color/position/bold/stroke/shadow",
|
||||
description="标题配置(可选),渲染时烧录到预览视频中。支持字段: text/font/font_size/font_color/position/bold/stroke/shadow。N个变体时样式全局共用",
|
||||
)
|
||||
# ── 多变体独立配置(preview_count > 1)──
|
||||
# 长度 1 = 所有变体共用;长度 = preview_count = 每个变体独立;空数组 = 回退单值字段
|
||||
titles: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="各变体独立标题文字数组:长度1=共用,长度=preview_count=独立。为空时使用 title_config.text",
|
||||
)
|
||||
voice_library_ids: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="各变体独立配音素材库ID数组:长度1=共用,长度=preview_count=独立。为空时回退 voice_library_id",
|
||||
)
|
||||
cover_urls: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="各变体独立封面URL数组:长度1=共用,长度=preview_count=独立(预览阶段通常为空)",
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _check_variant_arrays(self) -> "CreatePreviewGenerationTaskRequest":
|
||||
"""变体数组字段长度校验:空数组(回退单值)、长度 1(共用)、或长度 = preview_count(独立)。"""
|
||||
for name in ("titles", "voice_library_ids", "cover_urls"):
|
||||
arr = getattr(self, name)
|
||||
if arr and len(arr) != 1 and len(arr) != self.preview_count:
|
||||
raise ValueError(
|
||||
f"{name} 长度必须为 1(共用)或 {self.preview_count}(与 preview_count 一致),当前为 {len(arr)}"
|
||||
)
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _check_template_id(self) -> "CreatePreviewGenerationTaskRequest":
|
||||
@@ -202,7 +254,7 @@ class CreatePreviewGenerationTaskRequest(BaseModel):
|
||||
|
||||
|
||||
class PreviewGenerationTaskResponse(BaseModel):
|
||||
"""预览生成任务响应。
|
||||
"""单个预览变体任务响应。
|
||||
|
||||
包含任务状态、进度、分辨率、生成结果 URL 等关键字段。
|
||||
"""
|
||||
@@ -211,6 +263,7 @@ class PreviewGenerationTaskResponse(BaseModel):
|
||||
status: str
|
||||
progress: float
|
||||
is_preview: bool = True
|
||||
variant_index: int = 0
|
||||
resolution: str = ""
|
||||
video_url: str = ""
|
||||
duration: float = 0.0
|
||||
@@ -219,7 +272,21 @@ class PreviewGenerationTaskResponse(BaseModel):
|
||||
transition_count: int = 0
|
||||
material_usage: dict = Field(default_factory=dict)
|
||||
error_message: str = ""
|
||||
title_text: str = ""
|
||||
voice_library_id: str = ""
|
||||
created_at: datetime | None = None
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
generate_duration: float = 0.0
|
||||
|
||||
|
||||
class BatchPreviewGenerationTaskResponse(BaseModel):
|
||||
"""批量预览任务响应:preview_count=N 时返回 N 个独立变体任务。
|
||||
|
||||
- items: 变体任务数组,按 variant_index 顺序排列,每个含独立 task_id/状态/预览视频URL
|
||||
- total: 变体总数(= preview_count)
|
||||
- 前端按 items[i].task_id 分别轮询 GET /preview/{task_id} 获取进度与结果
|
||||
"""
|
||||
|
||||
items: list[PreviewGenerationTaskResponse]
|
||||
total: int
|
||||
|
||||
@@ -13,6 +13,8 @@ export type {
|
||||
VideoItem,
|
||||
} from "./types"
|
||||
|
||||
export type { RecomputeDedupResponse } from "./products"
|
||||
|
||||
// 工具函数
|
||||
export { mapVideoToProductItem } from "./utils"
|
||||
|
||||
@@ -25,4 +27,5 @@ export {
|
||||
updateReviewStatus,
|
||||
batchDownload,
|
||||
getBatchDownloadStatus,
|
||||
recomputeDedup,
|
||||
} from "./products"
|
||||
|
||||
@@ -78,3 +78,18 @@ export const getBatchDownloadStatus = async (jobId: string): Promise<BatchDownlo
|
||||
console.warn("[getBatchDownloadStatus] 后端暂无批量下载状态端点", jobId)
|
||||
return { job_id: jobId, status: "processing", progress: 0 }
|
||||
}
|
||||
|
||||
/** 重新计算存量视频查重率(异步) */
|
||||
export interface RecomputeDedupResponse {
|
||||
enqueued: number
|
||||
total_scanned: number
|
||||
skipped: number
|
||||
message: string
|
||||
}
|
||||
|
||||
export const recomputeDedup = async (videoIds?: string[]): Promise<RecomputeDedupResponse> => {
|
||||
const response = await apiClient.post("/videos/recompute-dedup", {
|
||||
video_ids: videoIds,
|
||||
})
|
||||
return response.data
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ export interface ProductItem {
|
||||
project_name?: string
|
||||
/** 查重率(百分比) */
|
||||
duplicate_rate?: number
|
||||
/** 视觉相似度(0-100),#1660 新增 */
|
||||
/** 视觉相似度(0-1),#1660 新增 */
|
||||
visual_similarity?: number
|
||||
/** 匹配帧数,#1660 新增 */
|
||||
match_count?: number
|
||||
@@ -76,7 +76,7 @@ export interface VideoItem {
|
||||
download_url: string
|
||||
generated_at: string
|
||||
duplicate_rate?: number
|
||||
/** 视觉相似度(0-100),#1660 新增 */
|
||||
/** 视觉相似度(0-1),#1660 新增 */
|
||||
visual_similarity?: number
|
||||
/** 匹配帧数,#1660 新增 */
|
||||
match_count?: number
|
||||
|
||||
@@ -81,6 +81,7 @@ export const extractVideoVoice = async (
|
||||
): Promise<{ asset_id: string; duration: number }> => {
|
||||
const formData = new FormData()
|
||||
formData.append("file", file)
|
||||
formData.append("project_id", "default")
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest()
|
||||
|
||||
@@ -40,13 +40,6 @@ const clipTypeLabel: Record<ClipType | string, string> = {
|
||||
pip: "混剪",
|
||||
}
|
||||
|
||||
const formatDuration = (sec: number) => {
|
||||
if (sec < 60) return `${sec.toFixed(1)}s`
|
||||
const m = Math.floor(sec / 60)
|
||||
const s = (sec % 60).toFixed(0)
|
||||
return `${m}m${s.padStart(2, "0")}s`
|
||||
}
|
||||
|
||||
const EditorClipList: React.FC<EditorClipListProps> = ({
|
||||
clips,
|
||||
selectedClipId,
|
||||
@@ -102,7 +95,6 @@ const EditorClipList: React.FC<EditorClipListProps> = ({
|
||||
{clipTypeLabel[clip.type] || "片段"}
|
||||
</span>
|
||||
</span>
|
||||
<span className="ep-clip-item-duration">{formatDuration(clip.duration)}</span>
|
||||
</div>
|
||||
|
||||
{/* 文案预览 */}
|
||||
|
||||
@@ -87,7 +87,7 @@ const PreviewPlayer: React.FC<PreviewPlayerProps> = ({
|
||||
WebkitTextStroke: "1px rgba(0,0,0,0.6)",
|
||||
top:
|
||||
titleConfig.position === "top"
|
||||
? "8px"
|
||||
? "6.25%"
|
||||
: titleConfig.position === "center"
|
||||
? "50%"
|
||||
: "auto",
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
* - ClipCard - 片段卡片
|
||||
* - ClipTrack - 片段轨道(播放头+片段列表+添加卡片)
|
||||
* - TimelineHeader - 时间线头部(标题+缩放+操作按钮)
|
||||
* - AddClipPicker - 添加片段选择器
|
||||
* - TrimPreview - 裁剪预览 tooltip
|
||||
* - ContextMenu - 右键菜单
|
||||
*
|
||||
@@ -27,7 +26,6 @@ import { usePlayheadDrag } from "./timeline/hooks/usePlayheadDrag"
|
||||
import { TimeRuler } from "./timeline/TimeRuler"
|
||||
import { ClipTrack } from "./timeline/ClipTrack"
|
||||
import { TimelineHeader } from "./timeline/TimelineHeader"
|
||||
import { AddClipPicker } from "./timeline/AddClipPicker"
|
||||
import { TrimPreview } from "./timeline/TrimPreview"
|
||||
import { ContextMenu } from "./timeline/ContextMenu"
|
||||
|
||||
@@ -100,16 +98,7 @@ const TimelinePanel: React.FC<TimelinePanelProps> = ({
|
||||
handleContextSplit,
|
||||
handleContextResetTrim,
|
||||
handleContextDelete,
|
||||
showAddPicker,
|
||||
pickerRef,
|
||||
addCardRef,
|
||||
pickerPos,
|
||||
availableTypes,
|
||||
addType,
|
||||
addDuration,
|
||||
setAddType,
|
||||
setAddDuration,
|
||||
handleTogglePicker,
|
||||
handleConfirmAdd,
|
||||
hoveredClipId,
|
||||
setHoveredClipId,
|
||||
@@ -177,7 +166,7 @@ const TimelinePanel: React.FC<TimelinePanelProps> = ({
|
||||
onClipMouseLeave={() => setHoveredClipId(null)}
|
||||
onTrimHandleMouseDown={handleTrimHandleMouseDown}
|
||||
onClipRemove={onClipRemove}
|
||||
onTogglePicker={handleTogglePicker}
|
||||
onTogglePicker={handleConfirmAdd}
|
||||
/>
|
||||
)}
|
||||
|
||||
@@ -204,20 +193,6 @@ const TimelinePanel: React.FC<TimelinePanelProps> = ({
|
||||
onDelete={handleContextDelete}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* 类型+时长选择面板 */}
|
||||
{showAddPicker && (
|
||||
<AddClipPicker
|
||||
pickerRef={pickerRef}
|
||||
position={pickerPos}
|
||||
availableTypes={availableTypes}
|
||||
addType={addType}
|
||||
addDuration={addDuration}
|
||||
onTypeChange={setAddType}
|
||||
onDurationChange={setAddDuration}
|
||||
onConfirm={handleConfirmAdd}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -7,12 +7,8 @@ interface AddClipPickerProps {
|
||||
position: { top: number; right: number }
|
||||
availableTypes: ClipType[]
|
||||
addType: ClipType
|
||||
addDuration: number
|
||||
onTypeChange: (type: ClipType) => void
|
||||
onDurationChange: (duration: number) => void
|
||||
onConfirm: () => void
|
||||
minDuration?: number
|
||||
maxDuration?: number
|
||||
}
|
||||
|
||||
export const AddClipPicker: React.FC<AddClipPickerProps> = ({
|
||||
@@ -20,12 +16,8 @@ export const AddClipPicker: React.FC<AddClipPickerProps> = ({
|
||||
position,
|
||||
availableTypes,
|
||||
addType,
|
||||
addDuration,
|
||||
onTypeChange,
|
||||
onDurationChange,
|
||||
onConfirm,
|
||||
minDuration = 1,
|
||||
maxDuration = 120,
|
||||
}) => {
|
||||
return (
|
||||
<div
|
||||
@@ -53,24 +45,6 @@ export const AddClipPicker: React.FC<AddClipPickerProps> = ({
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* 时长输入 */}
|
||||
<div className="ep-add-clip-duration-row">
|
||||
<span className="ep-add-clip-type-label">时长:</span>
|
||||
<input
|
||||
type="number"
|
||||
className="ep-duration-input"
|
||||
min={minDuration}
|
||||
max={maxDuration}
|
||||
value={addDuration}
|
||||
onChange={(e) =>
|
||||
onDurationChange(
|
||||
Math.max(minDuration, Math.min(maxDuration, Number(e.target.value) || minDuration)),
|
||||
)
|
||||
}
|
||||
/>
|
||||
<span className="ep-add-clip-duration-unit">秒</span>
|
||||
</div>
|
||||
|
||||
{/* 确认按钮 */}
|
||||
<button className="ep-add-clip-confirm-btn" onClick={onConfirm}>
|
||||
添加
|
||||
|
||||
@@ -105,14 +105,11 @@ export const ClipCard: React.FC<ClipCardProps> = ({
|
||||
<span className="ep-clip-name">
|
||||
{CLIP_TYPE_LABELS[clip.type] || "片段"} {idx + 1}
|
||||
</span>
|
||||
<span className="ep-clip-duration">
|
||||
{clip.duration}s
|
||||
{hasTrim && (
|
||||
<span className="ep-trim-indicator" title="已裁剪">
|
||||
✂
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
{hasTrim && (
|
||||
<span className="ep-trim-indicator" title="已裁剪">
|
||||
✂
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* 速度徽章 */}
|
||||
|
||||
@@ -26,8 +26,6 @@ export function useAddPicker({ currentMode, onAddClip }: UseAddPickerOptions) {
|
||||
}, [currentMode])
|
||||
|
||||
const [addType, setAddType] = useState<ClipType>(defaultAddType)
|
||||
const [addDuration, setAddDuration] = useState<number>(DEFAULT_ADD_DURATION)
|
||||
|
||||
useEffect(() => {
|
||||
if (!availableTypes.includes(addType)) {
|
||||
setAddType(defaultAddType)
|
||||
@@ -95,9 +93,9 @@ export function useAddPicker({ currentMode, onAddClip }: UseAddPickerOptions) {
|
||||
}, [showAddPicker])
|
||||
|
||||
const handleConfirmAdd = useCallback(() => {
|
||||
onAddClip(addType, addDuration)
|
||||
onAddClip(addType, DEFAULT_ADD_DURATION)
|
||||
setShowAddPicker(false)
|
||||
}, [onAddClip, addType, addDuration])
|
||||
}, [onAddClip, addType])
|
||||
|
||||
return {
|
||||
showAddPicker,
|
||||
@@ -107,9 +105,8 @@ export function useAddPicker({ currentMode, onAddClip }: UseAddPickerOptions) {
|
||||
pickerPos,
|
||||
availableTypes,
|
||||
addType,
|
||||
addDuration,
|
||||
addDuration: DEFAULT_ADD_DURATION,
|
||||
setAddType,
|
||||
setAddDuration,
|
||||
handleTogglePicker,
|
||||
handleConfirmAdd,
|
||||
}
|
||||
|
||||
@@ -35,7 +35,6 @@ export const useTimelineMenus = (
|
||||
addType,
|
||||
addDuration,
|
||||
setAddType,
|
||||
setAddDuration,
|
||||
handleTogglePicker,
|
||||
handleConfirmAdd,
|
||||
} = useAddPicker({ currentMode, onAddClip })
|
||||
@@ -57,7 +56,6 @@ export const useTimelineMenus = (
|
||||
addType,
|
||||
addDuration,
|
||||
setAddType,
|
||||
setAddDuration,
|
||||
handleTogglePicker,
|
||||
handleConfirmAdd,
|
||||
// 悬停状态
|
||||
|
||||
@@ -16,10 +16,6 @@ import { useQuery } from "@tanstack/react-query"
|
||||
import { useCloneProgress } from "@/hooks/useCloneProgress"
|
||||
import CloneModal from "@/components/voice/CloneModal"
|
||||
import GenerateHeader from "./components/GenerateHeader"
|
||||
import {
|
||||
calculateTotalVideoDuration,
|
||||
estimateTotalVideoDuration,
|
||||
} from "./utils/calculateTotalVideoDuration"
|
||||
import FrontendPreviewPlayer from "./components/FrontendPreviewPlayer"
|
||||
import GenerateStepsBar from "./components/GenerateStepsBar"
|
||||
import GenerateStepContent from "./components/GenerateStepContent"
|
||||
@@ -128,7 +124,6 @@ const GeneratePage: React.FC = () => {
|
||||
cancelled = true
|
||||
controller.abort()
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [selectedVoice, selectedClonedVoice, titleSettings.title, voiceMaterials])
|
||||
|
||||
/* ── 克隆声音 ── */
|
||||
@@ -174,13 +169,6 @@ const GeneratePage: React.FC = () => {
|
||||
[previewAssetsReady, currentTemplate],
|
||||
)
|
||||
|
||||
/* ── 视频总时长计算 ── */
|
||||
const totalVideoDuration = useMemo(() => {
|
||||
const exact = calculateTotalVideoDuration(previewAssets, currentTemplate ?? undefined)
|
||||
if (exact > 0) return exact
|
||||
return estimateTotalVideoDuration(currentTemplate ?? undefined)
|
||||
}, [previewAssets, currentTemplate])
|
||||
|
||||
/* ── 视频生成核心逻辑 ── */
|
||||
const {
|
||||
generating,
|
||||
@@ -291,7 +279,6 @@ const GeneratePage: React.FC = () => {
|
||||
onCoverSettingsChange={setCoverSettings}
|
||||
selectedVoice={selectedVoice}
|
||||
onSelectedVoiceChange={setSelectedVoice}
|
||||
totalVideoDuration={totalVideoDuration}
|
||||
onServerClipsChange={setServerClips}
|
||||
voiceMode={voiceMode}
|
||||
onVoiceModeChange={setVoiceMode}
|
||||
|
||||
@@ -591,6 +591,8 @@ const FrontendPreviewPlayer: React.FC<FrontendPreviewPlayerProps> = ({
|
||||
<div
|
||||
style={{
|
||||
position: "absolute",
|
||||
width: `${100 - 2 * titleSidePct}%`,
|
||||
maxWidth: `${100 - 2 * titleSidePct}%`,
|
||||
...(customTitleXPct != null && customTitleYPct != null
|
||||
? {
|
||||
left: `${customTitleXPct}%`,
|
||||
@@ -599,13 +601,13 @@ const FrontendPreviewPlayer: React.FC<FrontendPreviewPlayerProps> = ({
|
||||
textAlign: "center" as const,
|
||||
}
|
||||
: {
|
||||
left: `${titleSidePct}%`,
|
||||
right: `${titleSidePct}%`,
|
||||
left: "50%",
|
||||
transform: "translateX(-50%)",
|
||||
textAlign: "center" as const,
|
||||
...(titleSettings.position === "top"
|
||||
? { top: `${titleTopPct}%` }
|
||||
: titleSettings.position === "center"
|
||||
? { top: "50%", transform: "translateY(-50%)" }
|
||||
? { top: "50%", transform: "translate(-50%, -50%)" }
|
||||
: { bottom: `${titleBottomPct}%` }),
|
||||
}),
|
||||
pointerEvents: "auto",
|
||||
|
||||
@@ -49,7 +49,6 @@ export interface GenerateStepContentProps {
|
||||
/* 配音 */
|
||||
selectedVoice: string
|
||||
onSelectedVoiceChange: (id: string) => void
|
||||
totalVideoDuration?: number
|
||||
onServerClipsChange: (clips: EditPlanClip[]) => void
|
||||
voiceMode: "preset" | "custom" | "clone"
|
||||
onVoiceModeChange: (mode: "preset" | "custom" | "clone") => void
|
||||
@@ -104,7 +103,6 @@ export const GenerateStepContent: React.FC<GenerateStepContentProps> = (props) =
|
||||
onCoverSettingsChange,
|
||||
selectedVoice,
|
||||
onSelectedVoiceChange,
|
||||
totalVideoDuration,
|
||||
onServerClipsChange,
|
||||
voiceMode,
|
||||
selectedClonedVoice,
|
||||
@@ -151,7 +149,6 @@ export const GenerateStepContent: React.FC<GenerateStepContentProps> = (props) =
|
||||
<Step3VoiceSelect
|
||||
selectedVoice={selectedVoice}
|
||||
onSelectedVoiceChange={onSelectedVoiceChange}
|
||||
totalVideoDuration={totalVideoDuration}
|
||||
/>
|
||||
)
|
||||
case 4:
|
||||
|
||||
@@ -47,9 +47,7 @@ const Step1TemplateSelect: React.FC<Step1TemplateSelectProps> = (props) => {
|
||||
🎬
|
||||
</div>
|
||||
<h4>{tpl.name}</h4>
|
||||
<p>
|
||||
{tpl.estimated_duration}s · {tpl.segments.length}片段
|
||||
</p>
|
||||
<p>{tpl.segments.length}片段</p>
|
||||
{tpl.tags.length > 0 && (
|
||||
<div
|
||||
style={{
|
||||
|
||||
@@ -5,18 +5,15 @@
|
||||
import React, { useState, useRef, useCallback } from "react"
|
||||
import { useNavigate } from "react-router-dom"
|
||||
import { useQuery } from "@tanstack/react-query"
|
||||
import { AudioOutlined, SoundOutlined, WarningOutlined } from "@ant-design/icons"
|
||||
import { Modal } from "antd"
|
||||
import { AudioOutlined, SoundOutlined } from "@ant-design/icons"
|
||||
import { getAssetsByKind } from "@/api/assets"
|
||||
import type { AssetItem } from "@/api/assets"
|
||||
|
||||
interface Step5VoiceSelectProps {
|
||||
selectedVoice: string
|
||||
onSelectedVoiceChange: (id: string) => void
|
||||
totalVideoDuration?: number
|
||||
}
|
||||
|
||||
/** 格式化时长 mm:ss */
|
||||
/** 获取素材实际时长(优先顶层 duration,fallback 到 metadata.duration) */
|
||||
const getDuration = (item: AssetItem): number => {
|
||||
return item.duration ?? (item.metadata?.duration as number) ?? 0
|
||||
@@ -34,13 +31,6 @@ const isAiVoice = (item: AssetItem): boolean => {
|
||||
return (!duration || duration <= 0) && (!size || size <= 0)
|
||||
}
|
||||
|
||||
const formatDuration = (seconds?: number): string => {
|
||||
if (!seconds || seconds <= 0) return "00:00"
|
||||
const m = Math.floor(seconds / 60)
|
||||
const s = Math.floor(seconds % 60)
|
||||
return `${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`
|
||||
}
|
||||
|
||||
/** 格式化文件大小 */
|
||||
const formatFileSize = (bytes?: number): string => {
|
||||
if (!bytes || bytes <= 0) return "未知"
|
||||
@@ -53,13 +43,10 @@ const formatFileSize = (bytes?: number): string => {
|
||||
const Step5VoiceSelect: React.FC<Step5VoiceSelectProps> = ({
|
||||
selectedVoice,
|
||||
onSelectedVoiceChange,
|
||||
totalVideoDuration = 0,
|
||||
}) => {
|
||||
const navigate = useNavigate()
|
||||
const [playingId, setPlayingId] = useState<string | null>(null)
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null)
|
||||
const [durationWarningOpen, setDurationWarningOpen] = useState(false)
|
||||
const [pendingVoiceId, setPendingVoiceId] = useState<string | null>(null)
|
||||
|
||||
// 获取用户上传的配音素材
|
||||
const { data: materials = [], isLoading } = useQuery({
|
||||
@@ -100,38 +87,14 @@ const Step5VoiceSelect: React.FC<Step5VoiceSelectProps> = ({
|
||||
[playingId],
|
||||
)
|
||||
|
||||
/** 选中素材(含时长校验) */
|
||||
/** 选中素材(直接选中,不再做时长校验弹窗) */
|
||||
const handleSelect = useCallback(
|
||||
(id: string) => {
|
||||
// 如果启用了时长校验,且配音时长不足(AI 音色按脚本实时合成,不参与时长校验)
|
||||
if (totalVideoDuration > 0) {
|
||||
const material = materials.find((m) => m.id === id)
|
||||
if (material && !isAiVoice(material) && getDuration(material) < totalVideoDuration) {
|
||||
setPendingVoiceId(id)
|
||||
setDurationWarningOpen(true)
|
||||
return
|
||||
}
|
||||
}
|
||||
onSelectedVoiceChange(id)
|
||||
},
|
||||
[onSelectedVoiceChange, totalVideoDuration, materials],
|
||||
[onSelectedVoiceChange],
|
||||
)
|
||||
|
||||
/** 确认使用时长不足的配音 */
|
||||
const handleConfirmUseAnyway = useCallback(() => {
|
||||
if (pendingVoiceId) {
|
||||
onSelectedVoiceChange(pendingVoiceId)
|
||||
}
|
||||
setDurationWarningOpen(false)
|
||||
setPendingVoiceId(null)
|
||||
}, [pendingVoiceId, onSelectedVoiceChange])
|
||||
|
||||
/** 取消选择 */
|
||||
const handleCancelSelection = useCallback(() => {
|
||||
setDurationWarningOpen(false)
|
||||
setPendingVoiceId(null)
|
||||
}, [])
|
||||
|
||||
/** 跳转到配音库上传 */
|
||||
const handleGoToUpload = useCallback(() => {
|
||||
navigate("/app/voices?tab=material&upload=1")
|
||||
@@ -277,7 +240,7 @@ const Step5VoiceSelect: React.FC<Step5VoiceSelectProps> = ({
|
||||
{item.name}
|
||||
</div>
|
||||
|
||||
{/* 时长 + 大小 */}
|
||||
{/* 文件大小 */}
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
@@ -289,65 +252,13 @@ const Step5VoiceSelect: React.FC<Step5VoiceSelectProps> = ({
|
||||
>
|
||||
{isAiVoice(item) ? (
|
||||
<span style={{ color: "#1677ff", fontWeight: 500 }}>AI 音色</span>
|
||||
) : (
|
||||
<span style={{ display: "flex", alignItems: "center", gap: 4 }}>
|
||||
{formatDuration(getDuration(item))}
|
||||
{totalVideoDuration > 0 && getDuration(item) < Number(totalVideoDuration) && (
|
||||
<span
|
||||
style={{
|
||||
color: "#ff4d4f",
|
||||
fontSize: 11,
|
||||
fontWeight: 500,
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
gap: 2,
|
||||
}}
|
||||
>
|
||||
<WarningOutlined />
|
||||
时长不足
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
)}
|
||||
) : null}
|
||||
<span>{isAiVoice(item) ? "按文本合成" : formatFileSize(getFileSize(item))}</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
|
||||
{/* 时长不足警告弹窗 */}
|
||||
<Modal
|
||||
title={
|
||||
<span style={{ display: "flex", alignItems: "center", gap: 8 }}>
|
||||
<WarningOutlined style={{ color: "#faad14" }} />
|
||||
配音时长不足
|
||||
</span>
|
||||
}
|
||||
open={durationWarningOpen}
|
||||
onOk={handleConfirmUseAnyway}
|
||||
onCancel={handleCancelSelection}
|
||||
okText="仍要使用"
|
||||
cancelText="重新选择"
|
||||
okButtonProps={{ danger: true }}
|
||||
>
|
||||
{(() => {
|
||||
const pendingMaterial = pendingVoiceId
|
||||
? materials.find((m) => m.id === pendingVoiceId)
|
||||
: null
|
||||
return (
|
||||
<p>
|
||||
该配音时长(
|
||||
<strong>
|
||||
{pendingMaterial ? formatDuration(getDuration(pendingMaterial)) : "--"}
|
||||
</strong>
|
||||
)短于视频总时长(
|
||||
<strong>{formatDuration(totalVideoDuration)}</strong>
|
||||
),播放时配音可能提前结束,建议选择更长的配音素材。
|
||||
</p>
|
||||
)
|
||||
})()}
|
||||
</Modal>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -35,7 +35,13 @@ export const useTaskHistory = () => {
|
||||
} = useQuery<TaskItem[], Error>({
|
||||
queryKey: ["tasks"],
|
||||
queryFn: getUserTasks,
|
||||
staleTime: 30_000,
|
||||
staleTime: 5_000,
|
||||
// 有进行中任务时每 3 秒自动刷新,全部结束后停止轮询
|
||||
refetchInterval: (query) => {
|
||||
const list = query.state.data ?? []
|
||||
const hasActive = list.some((t) => ["pending", "waiting", "running"].includes(t.status))
|
||||
return hasActive ? 3_000 : false
|
||||
},
|
||||
})
|
||||
|
||||
// 重试 mutation
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
* 产品卡片 → components/ProductCard(内联视频播放)
|
||||
*/
|
||||
import React from "react"
|
||||
import { VideoCameraOutlined, DownloadOutlined } from "@ant-design/icons"
|
||||
import { VideoCameraOutlined, DownloadOutlined, ReloadOutlined } from "@ant-design/icons"
|
||||
import { Button } from "@/components/ui"
|
||||
import { ProductCard } from "./components/ProductCard"
|
||||
import { ProductFilterBar } from "./components/ProductFilterBar"
|
||||
@@ -19,6 +19,7 @@ import { ProductBatchBar } from "./components/ProductBatchBar"
|
||||
import { ProductEmptyState } from "./components/ProductEmptyState"
|
||||
import { useProductList } from "./hooks/useProductList"
|
||||
import { useProductActions } from "./hooks/useProductActions"
|
||||
import { useRecomputeDedup } from "./hooks/product-actions/useRecomputeDedup"
|
||||
import "./products.css"
|
||||
|
||||
const ProductLibrary: React.FC = () => {
|
||||
@@ -67,6 +68,8 @@ const ProductLibrary: React.FC = () => {
|
||||
setPlayingProduct: () => {}, // 不再使用弹窗播放
|
||||
})
|
||||
|
||||
const { recomputeDedup, isRecomputing } = useRecomputeDedup()
|
||||
|
||||
// ── Loading 状态 ──
|
||||
if (isLoading) {
|
||||
return <ProductEmptyState type="loading" />
|
||||
@@ -94,6 +97,15 @@ const ProductLibrary: React.FC = () => {
|
||||
<Button buttonType="ghost" buttonSize="sm" icon={<DownloadOutlined />}>
|
||||
批量导出
|
||||
</Button>
|
||||
<Button
|
||||
buttonType="ghost"
|
||||
buttonSize="sm"
|
||||
icon={<ReloadOutlined />}
|
||||
loading={isRecomputing}
|
||||
onClick={recomputeDedup}
|
||||
>
|
||||
重新查重
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -56,7 +56,9 @@ export const ProductInfoPanel: React.FC<ProductInfoPanelProps> = ({ product }) =
|
||||
{product.visual_similarity != null && (
|
||||
<div className="xx-detail-meta-item">
|
||||
<span className="xx-detail-meta-label">视觉相似度</span>
|
||||
<span className="xx-detail-meta-value">{product.visual_similarity.toFixed(1)}%</span>
|
||||
<span className="xx-detail-meta-value">
|
||||
{(product.visual_similarity * 100).toFixed(1)}%
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
{product.match_count != null && (
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query"
|
||||
import { message } from "antd"
|
||||
import { recomputeDedup } from "@/api/products"
|
||||
|
||||
export function useRecomputeDedup() {
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
const mutation = useMutation({
|
||||
mutationFn: () => recomputeDedup(),
|
||||
onSuccess: (data) => {
|
||||
queryClient.invalidateQueries({ queryKey: ["products"] })
|
||||
if (data.enqueued > 0) {
|
||||
message.success(`已提交 ${data.enqueued} 个视频的查重任务,后台处理中`)
|
||||
} else {
|
||||
message.info("所有视频查重率已是最新,无需重算")
|
||||
}
|
||||
},
|
||||
onError: () => {
|
||||
message.error("查重任务提交失败,请稍后重试")
|
||||
},
|
||||
})
|
||||
|
||||
return {
|
||||
recomputeDedup: () => mutation.mutate(),
|
||||
isRecomputing: mutation.isPending,
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,7 @@ const VideoExtractModal: React.FC<VideoExtractModalProps> = ({
|
||||
title={<span style={{ fontSize: 16, fontWeight: 600 }}>提取视频配音</span>}
|
||||
open={open}
|
||||
onCancel={() => {
|
||||
if (inputRef.current) inputRef.current.value = ""
|
||||
if (isExtracting) return
|
||||
onClose()
|
||||
}}
|
||||
@@ -152,7 +153,7 @@ const VideoExtractModal: React.FC<VideoExtractModalProps> = ({
|
||||
|
||||
{isExtracting && (
|
||||
<p style={{ textAlign: "center", fontSize: 13, color: "#7c3aed", margin: "12px 0 0" }}>
|
||||
{progress === 100 ? "正在提取人声,请稍候..." : "正在上传视频..."}
|
||||
{"正在提取音频,请稍后..."}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -102,6 +102,7 @@ vi.mock("@ant-design/icons", () => ({
|
||||
SearchOutlined: () => <span />,
|
||||
ShareAltOutlined: () => <span />,
|
||||
VideoCameraOutlined: () => <span />,
|
||||
ReloadOutlined: () => <span />,
|
||||
}))
|
||||
|
||||
vi.mock("@/store/authStore", () => ({
|
||||
@@ -116,6 +117,9 @@ vi.mock("@/api/products", () => ({
|
||||
updateReviewStatus: vi.fn().mockResolvedValue({ items: [], total: 0, success: true }),
|
||||
batchDownload: vi.fn().mockResolvedValue({ items: [], total: 0, success: true }),
|
||||
getBatchDownloadStatus: vi.fn().mockResolvedValue({ items: [], total: 0, success: true }),
|
||||
recomputeDedup: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ enqueued: 0, total_scanned: 0, skipped: 0, message: "" }),
|
||||
}))
|
||||
|
||||
vi.mock("@/pages/products/ProductLibrary.css", () => ({}))
|
||||
|
||||
@@ -104,6 +104,9 @@ vi.mock("@ant-design/icons", () => ({
|
||||
SoundOutlined: () => <span />,
|
||||
UploadOutlined: () => <span />,
|
||||
UserOutlined: () => <span />,
|
||||
VideoCameraOutlined: () => <span />,
|
||||
InboxOutlined: () => <span />,
|
||||
CloseOutlined: () => <span />,
|
||||
}))
|
||||
|
||||
vi.mock("@/store/authStore", () => ({
|
||||
|
||||
@@ -31,25 +31,60 @@ SCENE_CHANGE_THRESHOLD = 30 # 灰度差异阈值
|
||||
MIN_KEYFRAME_INTERVAL_SEC = 1.0 # 最小关键帧间隔(秒)
|
||||
MAX_KEYFRAMES = 30 # 最大关键帧数
|
||||
MIN_KEYFRAMES = 5 # 最小关键帧数
|
||||
FINGERPRINT_SAMPLE_INTERVAL_SEC = 1.0 # 指纹采样间隔(秒):密集均匀采样,保证两视频时序可对齐
|
||||
FINGERPRINT_MAX_SAMPLES = 30 # 长视频采样数上限(超过后采样间隔自动放宽)
|
||||
LONG_VIDEO_SEGMENT_SEC = 30 # 长视频每段秒数
|
||||
LONG_VIDEO_DURATION_THRESHOLD_SEC = 180 # 3 分钟阈值
|
||||
MIN_FRAMES_PER_SEGMENT = 2 # 长视频每段最少帧数
|
||||
|
||||
# ── 滑动窗口匹配常量 ────────────────────────────────────────────
|
||||
SEGMENT_MATCH_THRESHOLD = 8 # 帧匹配汉明距离阈值
|
||||
MIN_CONSECUTIVE_MATCHES = 5 # 最少连续匹配帧数
|
||||
# ── 滑动窗口匹配常量(Issue #1702 重新校准) ─────────────────────
|
||||
# 阈值经 staging 真实数据回归校准(2026-09-05,worker 容器内离线实验):
|
||||
# - 同源成片对(20s/11s,各自 2-5% 随机边缘裁剪降重,1s 密集采样):
|
||||
# 全部帧对最小汉明距离 min=8,<=12 命中 10/31 帧(B->A 4/11)
|
||||
# - 异源成片对(4 个不同项目真实视频):最小距离 24,<=16 命中 0 帧
|
||||
# 8(#1658 旧值)会漏掉同源裁剪(自对照实验:同帧两次 2-5% 随机裁剪距离 4~10),
|
||||
# 12 能检出同源/局部复用且与异源分布(>=24)间隔 12bit,无误报空间。
|
||||
PHASH_THRESHOLD = 12
|
||||
SEGMENT_MATCH_THRESHOLD = PHASH_THRESHOLD # 片段匹配阈值与帧匹配统一(#1702:阈值常量统一来源)
|
||||
MIN_CONSECUTIVE_MATCHES = 5 # 连续匹配默认门槛;短视频自适应 min(5, max(2, 分片数//2))
|
||||
MAX_GAP = 2 # 允许的最大间隙帧数
|
||||
NEIGHBOR_WINDOW = 1 # 分片时序对齐:允许 ±1 邻接偏移(1s 密集采样下即 ±1s,缓解切点不一致)
|
||||
|
||||
# ── 融合判定常量 ────────────────────────────────────────────────
|
||||
PHASH_WEIGHT = 0.7 # pHash 权重
|
||||
HISTOGRAM_WEIGHT = 0.3 # 直方图权重
|
||||
MATCH_RATIO_THRESHOLD = 0.7 # 至少 70% 帧匹配
|
||||
MATCH_RATIO_THRESHOLD = 0.7 # 全片重复(is_duplicate)至少 70% 帧匹配
|
||||
PARTIAL_COVERAGE_THRESHOLD = 0.5 # 局部复用覆盖率 >=50% 也判全片重复
|
||||
DUPLICATE_THRESHOLD = 0.70 # 融合后相似度阈值
|
||||
|
||||
# ── 降重裁剪规避常量(Issue #1702) ─────────────────────────────
|
||||
# 成片强制 2-5% random_edge_crop 降重只服务外部平台;自查重指纹取中心 90%
|
||||
# 区域,使两次不同裁剪的同源画面 pHash 距离回到同分布。
|
||||
FINGERPRINT_CENTER_CROP_RATIO = 0.90
|
||||
|
||||
|
||||
# ── 感知哈希 & 颜色直方图工具函数 ────────────────────────────────
|
||||
|
||||
|
||||
def center_crop_frame(image: np.ndarray, ratio: float = FINGERPRINT_CENTER_CROP_RATIO) -> np.ndarray:
|
||||
"""取画面中心 ratio 比例区域(裁除四边边缘)。
|
||||
|
||||
查重指纹用:random_edge_crop 降重(2-5% 四边随机裁剪)会让同源画面 pHash
|
||||
位翻转 12-16,污染自查重(Issue #1702)。算 pHash/颜色直方图前先居中裁除
|
||||
边缘 10%,两次不同裁剪的同源画面中心区域基本重合,指纹不再被降重污染。
|
||||
降重只服务外部平台,不影响内部查重。
|
||||
"""
|
||||
if image is None or image.size == 0:
|
||||
return image
|
||||
h, w = image.shape[:2]
|
||||
ch, cw = int(h * ratio), int(w * ratio)
|
||||
if ch <= 0 or cw <= 0 or (ch >= h and cw >= w):
|
||||
return image
|
||||
y0 = (h - ch) // 2
|
||||
x0 = (w - cw) // 2
|
||||
return image[y0 : y0 + ch, x0 : x0 + cw]
|
||||
|
||||
|
||||
def compute_phash(image: np.ndarray, hash_size: int = 8) -> str:
|
||||
"""计算图像的感知哈希(pHash),基于 DCT(离散余弦变换)。
|
||||
|
||||
@@ -101,11 +136,17 @@ def hamming_distance(hash1: str, hash2: str) -> int:
|
||||
|
||||
|
||||
def compute_color_histogram(image: np.ndarray, bins: int = 32) -> list[float]:
|
||||
"""Compute color histogram for an image."""
|
||||
"""Compute BGR color histogram for an image.
|
||||
|
||||
Issue #1702: 每个通道独立做 NORM_L1 归一化(通道内 Σ=1,是概率分布),
|
||||
三通道拼接存储。Bhattacharyya 系数对拼接向量直接 Σ√(a*b) 会得到
|
||||
3 通道之和(范围 [0,3],实测 ~14.9 是旧 L2 归一化的错误结果),
|
||||
消费方 _bhattacharyya_coefficient 按通道数平均归一到 [0,1]。
|
||||
"""
|
||||
hist = []
|
||||
for i in range(3):
|
||||
h = cv2.calcHist([image], [i], None, [bins], [0, 256])
|
||||
h = cv2.normalize(h, h).flatten()
|
||||
h = cv2.normalize(h, h, norm_type=cv2.NORM_L1).flatten()
|
||||
hist.extend(h)
|
||||
return hist
|
||||
|
||||
@@ -210,6 +251,30 @@ def detect_keyframe_timestamps(
|
||||
return keyframe_times
|
||||
|
||||
|
||||
def sample_fingerprint_timestamps(
|
||||
duration: float,
|
||||
*,
|
||||
interval_sec: float = FINGERPRINT_SAMPLE_INTERVAL_SEC,
|
||||
max_samples: int = FINGERPRINT_MAX_SAMPLES,
|
||||
) -> list[float]:
|
||||
"""指纹采样时间戳:固定间隔密集均匀采样(Issue #1702)。
|
||||
|
||||
动态场景检测抽帧(#1659)在两个同源视频上会各自取到不同时刻,切点/取帧
|
||||
错位让对齐帧的 pHash 距离都很大(实测同源对最小距离 12 且配对时序错乱)。
|
||||
改为固定 1s 间隔均匀采样后,复用片段的帧时刻天然对齐,配合 ±1 邻接窗口
|
||||
即可检出同源/局部复用。长视频(>max_samples*interval)自动放宽间隔到
|
||||
duration/max_samples,保证分片数有上限。
|
||||
"""
|
||||
if duration <= 0:
|
||||
return []
|
||||
step = interval_sec
|
||||
n_uniform = int(duration / step)
|
||||
if n_uniform > max_samples:
|
||||
step = duration / max_samples
|
||||
count = max(1, int(duration / step))
|
||||
return [step * (i + 0.5) for i in range(count)]
|
||||
|
||||
|
||||
# ── 数据类 ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -297,23 +362,32 @@ def find_duplicate_segments(
|
||||
target_chunks: list,
|
||||
*,
|
||||
match_threshold: int = SEGMENT_MATCH_THRESHOLD,
|
||||
min_consecutive: int = MIN_CONSECUTIVE_MATCHES,
|
||||
min_consecutive: Optional[int] = None,
|
||||
max_gap: int = MAX_GAP,
|
||||
neighbor_window: int = NEIGHBOR_WINDOW,
|
||||
) -> list[DuplicateSegment]:
|
||||
"""滑动窗口时序匹配:找出两组分片之间的重复片段。
|
||||
"""滑动窗口时序匹配:找出两组分片之间的重复片段(Issue #1702 重构)。
|
||||
|
||||
算法:
|
||||
1. 对每个 query chunk,找到 target 中汉明距离最小的 chunk
|
||||
2. 距离 <= match_threshold 视为匹配
|
||||
3. 找连续匹配的 run(允许 max_gap 帧间隙)
|
||||
4. 连续匹配数 >= min_consecutive 的 run 报告为重复片段
|
||||
1. 构建 query×target 全量汉明距离矩阵;每个 query chunk 保留所有
|
||||
距离 <= match_threshold 的候选 target 分片(与帧匹配判定同一阈值)。
|
||||
2. 时序一致贪心对齐:沿 query 时序推进,run 内优先选择与上一匹配帧
|
||||
目标序号连贯(0 <= delta <= neighbor_window+1,允许 ±1 邻接窗口 /
|
||||
时序偏移对齐,缓解场景切割导致的切点、取帧错位)的候选;同距时
|
||||
偏好大索引,避免重复 hash 塌缩到 target 首帧。
|
||||
3. 连贯匹配中允许 <= max_gap 帧间隙桥接;断裂后另起新 run——天然
|
||||
支持局部片段复用(复用片段可出现在任意时序位置,各成独立片段)。
|
||||
4. 连续匹配帧数 >= min_consecutive 的 run 报为重复片段。短视频自适应:
|
||||
min_consecutive = min(5, max(2, len(query_chunks)//2));n=1 时
|
||||
不形成片段,由调用方匹配帧回退兜底。
|
||||
|
||||
Args:
|
||||
query_chunks: 查询视频的分片列表(FingerprintChunk 或 dict)
|
||||
target_chunks: 目标视频的分片列表
|
||||
match_threshold: 汉明距离匹配阈值
|
||||
min_consecutive: 最少连续匹配帧数
|
||||
match_threshold: 汉明距离匹配阈值(统一常量 PHASH_THRESHOLD)
|
||||
min_consecutive: 最少连续匹配帧数;None 时按短视频自适应
|
||||
max_gap: 允许的最大间隙帧数
|
||||
neighbor_window: 时序对齐允许的目标分片序号邻接窗口
|
||||
|
||||
Returns:
|
||||
DuplicateSegment 列表
|
||||
@@ -321,95 +395,93 @@ def find_duplicate_segments(
|
||||
if not query_chunks or not target_chunks:
|
||||
return []
|
||||
|
||||
def _get_phash(chunk) -> str:
|
||||
def _get(chunk, key):
|
||||
if isinstance(chunk, dict):
|
||||
return chunk["phash_binary"]
|
||||
return chunk.phash_binary
|
||||
return chunk[key]
|
||||
return getattr(chunk, key)
|
||||
|
||||
def _get_start(chunk) -> int:
|
||||
if isinstance(chunk, dict):
|
||||
return chunk["start_time_ms"]
|
||||
return chunk.start_time_ms
|
||||
n, m = len(query_chunks), len(target_chunks)
|
||||
q_ph = [_get(c, "phash_binary") for c in query_chunks]
|
||||
t_ph = [_get(c, "phash_binary") for c in target_chunks]
|
||||
|
||||
def _get_end(chunk) -> int:
|
||||
if isinstance(chunk, dict):
|
||||
return chunk["end_time_ms"]
|
||||
return chunk.end_time_ms
|
||||
# Step 1: 全量距离矩阵。每个 query chunk 保留所有 <= 阈值的候选 target,
|
||||
# 按距离升序;同距时小索引优先(取最早的对齐位置,贪心连贯推进时最保守,
|
||||
# 不会越过复用片段末端;重复 hash 的连续帧由 Step 2 的连贯性窗口约束)。
|
||||
candidates: list[list[tuple[int, int]]] = [] # 每 query 帧: [(target_idx, dist), ...]
|
||||
for i in range(n):
|
||||
dists = [hamming_distance(q_ph[i], t_ph[j]) for j in range(m)]
|
||||
cand = [(j, d) for j, d in enumerate(dists) if d <= match_threshold]
|
||||
cand.sort(key=lambda x: (x[1], x[0]))
|
||||
candidates.append(cand)
|
||||
|
||||
# Step 1: 逐帧匹配
|
||||
frame_matches: list[tuple[bool, int, int]] = [] # (is_match, min_dist, best_target_idx)
|
||||
for qc in query_chunks:
|
||||
qc_phash = _get_phash(qc)
|
||||
best_dist = 64
|
||||
best_idx = 0
|
||||
for j, tc in enumerate(target_chunks):
|
||||
d = hamming_distance(qc_phash, _get_phash(tc))
|
||||
if d < best_dist:
|
||||
best_dist = d
|
||||
best_idx = j
|
||||
frame_matches.append((best_dist <= match_threshold, best_dist, best_idx))
|
||||
# 短视频自适应连续匹配门槛(Issue #1702 工单公式):
|
||||
# MIN_CONSECUTIVE_MATCHES = min(5, max(2, 分片数//2))。
|
||||
# n=1 时门槛为 2 不形成片段,由 _evaluate_candidate 的匹配帧回退
|
||||
# (temporal_coverage 按匹配帧占比估计)兜底检出,不回归。
|
||||
if min_consecutive is None:
|
||||
min_consecutive = min(MIN_CONSECUTIVE_MATCHES, max(2, n // 2))
|
||||
|
||||
# Step 2: 找连续匹配的 runs
|
||||
runs: list[tuple[int, int]] = [] # list of (start_idx, end_idx)
|
||||
run_start = None
|
||||
# Step 2: 时序一致贪心对齐。
|
||||
# run 内偏好与上一匹配帧目标序号连贯(0 <= delta <= neighbor_window+1,
|
||||
# 支持 ±1 邻接窗口/时序偏移对齐)的候选;无连贯候选时关闭旧 run。
|
||||
# 这天然支持局部片段复用:同一 query 视频中多个复用片段各自形成独立 run。
|
||||
frame_matches: list[tuple[bool, int, int]] = []
|
||||
runs: list[tuple[int, int]] = []
|
||||
run_start: Optional[int] = None
|
||||
run_last_t: Optional[int] = None
|
||||
gap_count = 0
|
||||
|
||||
for i, (is_match, _dist, _idx) in enumerate(frame_matches):
|
||||
if is_match:
|
||||
def _matching_count(a: int, b: int) -> int:
|
||||
return sum(1 for k in range(a, b + 1) if frame_matches[k][0])
|
||||
|
||||
def _close_run(a: int, b: int) -> None:
|
||||
if b >= a and _matching_count(a, b) >= min_consecutive:
|
||||
runs.append((a, b))
|
||||
|
||||
for i in range(n):
|
||||
cand = candidates[i]
|
||||
if run_last_t is None:
|
||||
chosen = cand[0] if cand else None
|
||||
else:
|
||||
chosen = next(
|
||||
(c for c in cand if 0 <= c[0] - run_last_t <= neighbor_window + 1),
|
||||
None,
|
||||
)
|
||||
|
||||
if chosen is not None:
|
||||
tidx, dist = chosen
|
||||
frame_matches.append((True, dist, tidx))
|
||||
if run_start is None:
|
||||
run_start = i
|
||||
gap_count = 0 # 重置间隙
|
||||
gap_count = 0
|
||||
run_last_t = tidx
|
||||
else:
|
||||
frame_matches.append((False, match_threshold + 1, -1))
|
||||
if run_start is not None:
|
||||
gap_count += 1
|
||||
if gap_count > max_gap:
|
||||
# 中断当前 run
|
||||
run_end = i - gap_count # 最后一个匹配帧的索引
|
||||
# 计算 run 内的实际匹配帧数(总跨度 - 间隙数)
|
||||
total_gaps = sum(1 for k in range(run_start, run_end + 1) if not frame_matches[k][0])
|
||||
matching_count = (run_end - run_start + 1) - total_gaps
|
||||
if matching_count >= min_consecutive:
|
||||
runs.append((run_start, run_end))
|
||||
run_start = None
|
||||
gap_count = 0
|
||||
# 非匹配帧从 i-gap_count+1 开始,run 结束于其前一帧
|
||||
_close_run(run_start, i - gap_count)
|
||||
run_start, run_last_t, gap_count = None, None, 0
|
||||
|
||||
# 处理末尾 run
|
||||
if run_start is not None:
|
||||
last_idx = len(frame_matches) - 1
|
||||
# 回退找到最后一个匹配帧的位置(跳过尾部非匹配帧)
|
||||
last_idx = n - 1
|
||||
while last_idx >= run_start and not frame_matches[last_idx][0]:
|
||||
last_idx -= 1
|
||||
if last_idx >= run_start:
|
||||
# 计算 run 内的总间隙数
|
||||
total_gaps = sum(1 for k in range(run_start, last_idx + 1) if not frame_matches[k][0])
|
||||
matching_count = (last_idx - run_start + 1) - total_gaps
|
||||
if matching_count >= min_consecutive:
|
||||
runs.append((run_start, last_idx))
|
||||
_close_run(run_start, last_idx)
|
||||
|
||||
# Step 3: 构建 DuplicateSegment
|
||||
segments: list[DuplicateSegment] = []
|
||||
for start, end in runs:
|
||||
query_start = _get_start(query_chunks[start])
|
||||
query_end = _get_end(query_chunks[end])
|
||||
|
||||
# 取目标范围(按最佳匹配的目标 chunk 时间范围)
|
||||
target_indices = [frame_matches[k][2] for k in range(start, end + 1) if frame_matches[k][0]]
|
||||
if target_indices:
|
||||
t_min = min(target_indices)
|
||||
t_max = max(target_indices)
|
||||
target_start = _get_start(target_chunks[t_min])
|
||||
target_end = _get_end(target_chunks[t_max])
|
||||
else:
|
||||
target_start = _get_start(target_chunks[0])
|
||||
target_end = _get_end(target_chunks[-1])
|
||||
|
||||
avg_dist = sum(frame_matches[k][1] for k in range(start, end + 1)) / (end - start + 1)
|
||||
t_min, t_max = min(target_indices), max(target_indices)
|
||||
avg_dist = sum(frame_matches[k][1] for k in range(start, end + 1) if frame_matches[k][0]) / len(target_indices)
|
||||
segments.append(
|
||||
DuplicateSegment(
|
||||
query_start_ms=query_start,
|
||||
query_end_ms=query_end,
|
||||
target_start_ms=target_start,
|
||||
target_end_ms=target_end,
|
||||
query_start_ms=_get(query_chunks[start], "start_time_ms"),
|
||||
query_end_ms=_get(query_chunks[end], "end_time_ms"),
|
||||
target_start_ms=_get(target_chunks[t_min], "start_time_ms"),
|
||||
target_end_ms=_get(target_chunks[t_max], "end_time_ms"),
|
||||
avg_distance=avg_dist,
|
||||
)
|
||||
)
|
||||
@@ -423,15 +495,62 @@ def find_duplicate_segments(
|
||||
class VideoDeduplicator:
|
||||
"""Video deduplication using multiple fingerprint methods."""
|
||||
|
||||
PHASH_THRESHOLD = 8 # Issue #1658: pHash 汉明距离阈值由 10 收紧到 8,降低不同视频误判率
|
||||
# Issue #1702: 阈值统一来源为模块常量 PHASH_THRESHOLD(#1658 曾收紧到 8,
|
||||
# 后经 staging 真实同源/异源指纹分布重新校准,见 test_phash_threshold_calibration_1702)。
|
||||
PHASH_THRESHOLD = PHASH_THRESHOLD
|
||||
HISTOGRAM_THRESHOLD = 0.85
|
||||
|
||||
def compute_fingerprint(self, video_path: str) -> VideoFingerprint:
|
||||
"""Compute video fingerprint using dynamic keyframe detection.
|
||||
@staticmethod
|
||||
def _is_bad_fingerprint(phashes: list[str]) -> bool:
|
||||
"""检测指纹质量差的视频(黑屏/纯色视频)。
|
||||
|
||||
使用 detect_keyframe_timestamps() 检测内容感知关键帧,
|
||||
在每个关键帧处取帧计算 pHash + color_histogram。
|
||||
同时保留 MD5 计算和分片数据结构。
|
||||
当视频有多个关键帧但所有 phash 完全相同或极其相似时,
|
||||
说明视频内容无变化(如黑屏、纯色画面),这类指纹与任何视频
|
||||
比较都会得到虚假的"匹配"结果,应跳过。
|
||||
|
||||
注意:单帧视频(只有 1 个 phash)不视为坏指纹,可能是短视频或抽帧不足。
|
||||
|
||||
Args:
|
||||
phashes: 关键帧 phash 列表
|
||||
|
||||
Returns:
|
||||
True 表示指纹无效,应跳过
|
||||
"""
|
||||
if not phashes:
|
||||
return True
|
||||
# 单帧不视为坏指纹(短视频或抽帧不足)
|
||||
if len(phashes) == 1:
|
||||
return False
|
||||
# Issue #1702: 旧逻辑"所有 phash 完全相同即判黑屏"会误杀短视频——
|
||||
# 11s 视频只有几个不同镜头时,相邻 1s 采样帧可能 phash 完全一致(内容
|
||||
# 连续但非黑屏)。黑屏的特征是「大量帧全部无内容」,要求至少 8 帧
|
||||
# 且相同帧占比 >=80% 才判坏;短视频(<8 帧)只有真正单值时交给
|
||||
# _bhattacharyya/融合分兜底,不因"帧都一样"直接跳过。
|
||||
if len(phashes) < 8:
|
||||
return False
|
||||
unique = set(phashes)
|
||||
same_ratio = sum(1 for x in phashes if x == phashes[0]) / len(phashes)
|
||||
if len(unique) == 1 and same_ratio >= 0.8:
|
||||
return True
|
||||
# 多帧但所有唯一 phash 之间的汉明距离都极小(<3)且占比 >=80% → 近似黑屏
|
||||
phash_list = list(unique)
|
||||
if len(phash_list) >= 2 and same_ratio >= 0.8:
|
||||
all_distances = [
|
||||
hamming_distance(phash_list[i], phash_list[j])
|
||||
for i in range(len(phash_list))
|
||||
for j in range(i + 1, len(phash_list))
|
||||
]
|
||||
if all_distances and max(all_distances) < 3:
|
||||
return True
|
||||
return False
|
||||
|
||||
def compute_fingerprint(self, video_path: str) -> VideoFingerprint:
|
||||
"""Compute video fingerprint using dense uniform sampling.
|
||||
|
||||
Issue #1702: 使用 sample_fingerprint_timestamps() 固定 1s 间隔密集均匀
|
||||
采样(替代动态场景检测抽帧),保证两个同源视频复用片段的帧时刻天然
|
||||
对齐;每帧取中心 90% 区域(center_crop_frame)计算 pHash + color_histogram,
|
||||
绕开 random_edge_crop 降重裁剪污染;MD5 仍基于原始帧。
|
||||
"""
|
||||
cap = cv2.VideoCapture(video_path)
|
||||
if not cap.isOpened():
|
||||
@@ -445,8 +564,8 @@ class VideoDeduplicator:
|
||||
|
||||
cap.release()
|
||||
|
||||
# 1. 检测关键帧时间戳
|
||||
keyframe_times = detect_keyframe_timestamps(video_path)
|
||||
# 1. 固定间隔密集采样(Issue #1702:替代动态场景检测,保证跨视频时序对齐)
|
||||
keyframe_times = sample_fingerprint_timestamps(duration)
|
||||
|
||||
if not keyframe_times:
|
||||
return VideoFingerprint(
|
||||
@@ -470,12 +589,15 @@ class VideoDeduplicator:
|
||||
if not ret:
|
||||
continue
|
||||
|
||||
# MD5 计算
|
||||
# MD5 计算(基于原始帧,指纹文件级去重不受裁剪影响)
|
||||
_, buffer = cv2.imencode(".jpg", frame)
|
||||
md5_hash.update(buffer)
|
||||
|
||||
phash = compute_phash(frame)
|
||||
hist = compute_color_histogram(frame)
|
||||
# Issue #1702: pHash / 颜色直方图基于中心 90% 区域,绕开 random_edge_crop
|
||||
# 降重裁剪对指纹的污染(降重只服务外部平台,不污染自查重)。
|
||||
fp_frame = center_crop_frame(frame)
|
||||
phash = compute_phash(fp_frame)
|
||||
hist = compute_color_histogram(fp_frame)
|
||||
|
||||
# 计算分片时间范围(从前一个关键帧到下一个关键帧的中点)
|
||||
prev_boundary = keyframe_times[i - 1] * 1000 if i > 0 else 0
|
||||
@@ -528,12 +650,22 @@ class VideoDeduplicator:
|
||||
|
||||
@staticmethod
|
||||
def _bhattacharyya_coefficient(hist_a: list[float], hist_b: list[float]) -> float:
|
||||
"""Bhattacharyya 系数:Σ √(a[i] * b[i]),范围 [0, 1],1=完全相同。"""
|
||||
"""Bhattacharyya 系数(概率分布版,范围 [0,1],1=完全相同)。
|
||||
|
||||
Issue #1702: compute_color_histogram 输出 3 通道拼接、每通道独立 NORM_L1
|
||||
(单通道 Σ=1,三通道拼接向量 Σ=3)。旧实现直接 Σ√(a*b) 对三通道拼接向量
|
||||
算出 ~3(旧 L2 归一化更是算出 ~14.9),不是合法的概率系数。
|
||||
这里按两个直方图各自的总量归一:BC = Σ√(a*b) / √(Σa·Σb)。
|
||||
- 单通道概率分布(Σa=Σb=1):分母 1,与旧测试/教科书定义一致;
|
||||
- 三通道拼接(Σa=Σb=3):分母 3,结果在 [0,1]。
|
||||
"""
|
||||
min_len = min(len(hist_a), len(hist_b))
|
||||
a = hist_a[:min_len]
|
||||
b = hist_b[:min_len]
|
||||
# 纯标准库计算(不依赖 numpy);max(0.0, ...) 防御上游异常负值导致 sqrt domain error
|
||||
return float(sum(math.sqrt(max(0.0, ai * bi)) for ai, bi in zip(a, b, strict=False)))
|
||||
a = [max(0.0, float(x)) for x in hist_a[:min_len]]
|
||||
b = [max(0.0, float(x)) for x in hist_b[:min_len]]
|
||||
# max(0.0, ...) 防御上游异常负值导致 sqrt domain error
|
||||
coeff = sum(math.sqrt(ai * bi) for ai, bi in zip(a, b, strict=False))
|
||||
norm = math.sqrt(sum(a) * sum(b))
|
||||
return float(coeff / norm) if norm > 0 else 0.0
|
||||
|
||||
@staticmethod
|
||||
def _compute_histogram_similarity(
|
||||
@@ -575,6 +707,72 @@ class VideoDeduplicator:
|
||||
hist_similarity = VideoDeduplicator._compute_histogram_similarity(hist_a, hist_b) if hist_b else 0.5
|
||||
return PHASH_WEIGHT * phash_similarity + HISTOGRAM_WEIGHT * hist_similarity
|
||||
|
||||
@staticmethod
|
||||
def _evaluate_candidate(
|
||||
fingerprint: VideoFingerprint,
|
||||
existing_phashes: list[str],
|
||||
existing_histograms: list,
|
||||
existing_chunk_objects: list,
|
||||
*,
|
||||
query_duration_sec: float,
|
||||
) -> dict:
|
||||
"""评估新视频指纹与单个候选视频的相似度(Issue #1702 共享逻辑)。
|
||||
|
||||
指标:
|
||||
- min_distances / frame_match_rate:每个新分片到候选视频全局最近邻的汉明距离,
|
||||
分母取两视频分片数的较小值(支持局部片段复用:短视频复用长视频片段时不被长视频分母稀释)。
|
||||
- temporal_coverage:时序一致连续匹配片段总时长 / 新视频时长(局部复用主指标)。
|
||||
- fusion:pHash 中位数距离 + 颜色直方图的加权融合分。
|
||||
|
||||
Returns:
|
||||
{frame_match_rate, temporal_coverage, segments, median_distance,
|
||||
fusion, matching_frames, min_distances}
|
||||
"""
|
||||
query_phashes = fingerprint.keyframe_phashes or []
|
||||
if not query_phashes or not existing_phashes:
|
||||
return {
|
||||
"frame_match_rate": 0.0,
|
||||
"temporal_coverage": 0.0,
|
||||
"segments": [],
|
||||
"median_distance": 64,
|
||||
"fusion": 0.0,
|
||||
"matching_frames": 0,
|
||||
"min_distances": [],
|
||||
}
|
||||
|
||||
min_distances = [min(hamming_distance(ph, ep) for ep in existing_phashes) for ph in query_phashes]
|
||||
matching_frames = sum(1 for d in min_distances if d <= PHASH_THRESHOLD)
|
||||
# 分母取 min(两视频分片数):局部复用时(如 B 的 5 片复用 A 9 片中的若干片)
|
||||
# 命中帧占比不因候选视频更长而被稀释。
|
||||
frame_match_rate = matching_frames / min(len(query_phashes), len(existing_phashes))
|
||||
|
||||
segments = find_duplicate_segments(fingerprint.chunks, existing_chunk_objects)
|
||||
duration_ms = query_duration_sec * 1000 if query_duration_sec else 0
|
||||
if duration_ms > 0 and segments:
|
||||
covered_ms = sum(s.query_end_ms - s.query_start_ms for s in segments)
|
||||
temporal_coverage = min(covered_ms / duration_ms, 1.0)
|
||||
elif matching_frames > 0:
|
||||
# 无连续片段(时序连贯性不足)时,按匹配帧占比估计覆盖:
|
||||
# 密集 1s 采样下每个分片≈1s 等权时间片,匹配帧数≈命中秒数。
|
||||
temporal_coverage = min(frame_match_rate, 1.0)
|
||||
else:
|
||||
temporal_coverage = 0.0
|
||||
|
||||
median_distance = statistics.median(min_distances) if min_distances else 64
|
||||
fusion = VideoDeduplicator._compute_fusion_score(
|
||||
median_distance, fingerprint.color_histograms, existing_histograms
|
||||
)
|
||||
|
||||
return {
|
||||
"frame_match_rate": frame_match_rate,
|
||||
"temporal_coverage": temporal_coverage,
|
||||
"segments": segments,
|
||||
"median_distance": median_distance,
|
||||
"fusion": fusion,
|
||||
"matching_frames": matching_frames,
|
||||
"min_distances": min_distances,
|
||||
}
|
||||
|
||||
def check_duplicate(
|
||||
self,
|
||||
fingerprint: VideoFingerprint,
|
||||
@@ -613,6 +811,9 @@ class VideoDeduplicator:
|
||||
else:
|
||||
existing_videos = video_repo.list_by_project(project_id)
|
||||
|
||||
best_score = 0.0
|
||||
best_result: Optional[dict] = None
|
||||
|
||||
for existing in existing_videos:
|
||||
if not existing.video_fingerprint:
|
||||
continue
|
||||
@@ -623,6 +824,12 @@ class VideoDeduplicator:
|
||||
if fingerprint.md5 == ef.get("md5"):
|
||||
return {"duplicate": True, "duplicate_of": existing.id, "reason": "exact_md5_match", "similarity": 1.0}
|
||||
|
||||
# 跳过指纹质量差的视频(黑屏/纯色视频)
|
||||
existing_phashes_for_check = ef.get("keyframe_phashes", [])
|
||||
if self._is_bad_fingerprint(existing_phashes_for_check):
|
||||
logger.debug("Skipping bad fingerprint video %s in check_duplicate", existing.id)
|
||||
continue
|
||||
|
||||
# 优先从分片表读取已有视频的分片 phash
|
||||
existing_phashes = []
|
||||
chunk_data = self._get_existing_chunks(existing.id, session)
|
||||
@@ -635,61 +842,70 @@ class VideoDeduplicator:
|
||||
if not existing_phashes:
|
||||
continue
|
||||
|
||||
# 计算每个新关键帧到已有关键帧的最小汉明距离
|
||||
min_distances = []
|
||||
for phash in fingerprint.keyframe_phashes:
|
||||
distances = [hamming_distance(phash, ep) for ep in existing_phashes]
|
||||
min_distances.append(min(distances))
|
||||
|
||||
# 帧匹配比例检查
|
||||
matching_frames = sum(1 for d in min_distances if d < self.PHASH_THRESHOLD)
|
||||
match_ratio = matching_frames / len(min_distances) if min_distances else 0
|
||||
if match_ratio < MATCH_RATIO_THRESHOLD:
|
||||
continue
|
||||
|
||||
# 中位数距离
|
||||
median_distance = statistics.median(min_distances) if min_distances else 64
|
||||
if median_distance >= self.PHASH_THRESHOLD:
|
||||
continue
|
||||
|
||||
# 直方图融合(chunk 表优先,回退 JSON 字段;JSON NULL 显式回退空列表)
|
||||
# 直方图 / 分片对象(chunk 表优先,回退 JSON 字段;JSON NULL 显式回退空列表)
|
||||
if chunk_data:
|
||||
existing_histograms = [c["color_histogram"] for c in chunk_data if c.get("color_histogram")]
|
||||
existing_chunk_objects = chunk_data
|
||||
else:
|
||||
existing_histograms = ef.get("color_histograms") or []
|
||||
existing_chunk_objects = [
|
||||
{"phash_binary": pp, "start_time_ms": 0, "end_time_ms": 0} for pp in existing_phashes
|
||||
]
|
||||
|
||||
combined_score = self._compute_fusion_score(
|
||||
median_distance, fingerprint.color_histograms, existing_histograms
|
||||
# Issue #1702: 统一评估每个候选(含局部片段复用),不再用
|
||||
# "frame_match_rate<0.7 整条跳过" 的硬门槛——局部复用(如 B 结尾 2s
|
||||
# ≈ A 中间 2s)帧比例天然低,但 coverage 能检出。
|
||||
ev = self._evaluate_candidate(
|
||||
fingerprint,
|
||||
existing_phashes,
|
||||
existing_histograms,
|
||||
existing_chunk_objects,
|
||||
query_duration_sec=fingerprint.duration,
|
||||
)
|
||||
logger.debug(
|
||||
"check_duplicate candidate=%s min_distances=%s frame_match_rate=%.3f "
|
||||
"temporal_coverage=%.3f median=%.1f fusion=%.3f segments=%d",
|
||||
existing.id,
|
||||
ev["min_distances"],
|
||||
ev["frame_match_rate"],
|
||||
ev["temporal_coverage"],
|
||||
ev["median_distance"],
|
||||
ev["fusion"],
|
||||
len(ev["segments"]),
|
||||
)
|
||||
|
||||
if combined_score < DUPLICATE_THRESHOLD:
|
||||
continue
|
||||
|
||||
# 滑动窗口时序匹配:获取具体重复片段
|
||||
existing_chunk_objects = (
|
||||
chunk_data
|
||||
if chunk_data
|
||||
else [{"phash_binary": p, "start_time_ms": 0, "end_time_ms": 0} for p in existing_phashes]
|
||||
# 全片重复判定:融合分过阈 且(帧匹配比例 >=70% 或 局部覆盖 >=50%)
|
||||
is_full_duplicate = ev["fusion"] >= DUPLICATE_THRESHOLD and (
|
||||
ev["frame_match_rate"] >= MATCH_RATIO_THRESHOLD or ev["temporal_coverage"] >= PARTIAL_COVERAGE_THRESHOLD
|
||||
)
|
||||
segments = find_duplicate_segments(fingerprint.chunks, existing_chunk_objects)
|
||||
|
||||
return {
|
||||
"duplicate": True,
|
||||
"duplicate_of": existing.id,
|
||||
"reason": "phash_histogram_fusion",
|
||||
"similarity": combined_score,
|
||||
"duplicate_segments": [
|
||||
{
|
||||
"query_start_ms": s.query_start_ms,
|
||||
"query_end_ms": s.query_end_ms,
|
||||
"target_start_ms": s.target_start_ms,
|
||||
"target_end_ms": s.target_end_ms,
|
||||
"avg_distance": round(s.avg_distance, 2),
|
||||
}
|
||||
for s in segments
|
||||
],
|
||||
}
|
||||
if is_full_duplicate and ev["fusion"] > best_score:
|
||||
best_score = ev["fusion"]
|
||||
best_result = {
|
||||
"duplicate": True,
|
||||
"duplicate_of": existing.id,
|
||||
"reason": "phash_histogram_fusion",
|
||||
"similarity": ev["fusion"],
|
||||
"duplicate_segments": [
|
||||
{
|
||||
"query_start_ms": s.query_start_ms,
|
||||
"query_end_ms": s.query_end_ms,
|
||||
"target_start_ms": s.target_start_ms,
|
||||
"target_end_ms": s.target_end_ms,
|
||||
"avg_distance": round(s.avg_distance, 2),
|
||||
}
|
||||
for s in ev["segments"]
|
||||
],
|
||||
}
|
||||
|
||||
if best_result:
|
||||
return best_result
|
||||
logger.info(
|
||||
"check_duplicate no match (project=%s scope=%s): %d candidates evaluated, best_fusion=%.3f",
|
||||
project_id,
|
||||
scope,
|
||||
len(existing_videos),
|
||||
best_score,
|
||||
)
|
||||
return None
|
||||
|
||||
def check_batch_duplicate(
|
||||
@@ -721,6 +937,9 @@ class VideoDeduplicator:
|
||||
video_repo = SQLAlchemyGeneratedVideoRepository(session)
|
||||
batch_videos = video_repo.list_by_batch(batch_id)
|
||||
|
||||
best_score = 0.0
|
||||
best_result: Optional[dict] = None
|
||||
|
||||
for existing in batch_videos:
|
||||
if existing.id == current_video_id:
|
||||
continue
|
||||
@@ -737,6 +956,12 @@ class VideoDeduplicator:
|
||||
"similarity": 1.0,
|
||||
}
|
||||
|
||||
# 跳过指纹质量差的视频(黑屏/纯色视频)
|
||||
existing_phashes_batch = ef.get("keyframe_phashes", [])
|
||||
if self._is_bad_fingerprint(existing_phashes_batch):
|
||||
logger.debug("Skipping bad fingerprint video %s in check_batch_duplicate", existing.id)
|
||||
continue
|
||||
|
||||
# 优先从分片表读取
|
||||
existing_phashes = []
|
||||
chunk_data = self._get_existing_chunks(existing.id, session)
|
||||
@@ -748,59 +973,59 @@ class VideoDeduplicator:
|
||||
if not existing_phashes:
|
||||
continue
|
||||
|
||||
min_distances = []
|
||||
for phash in fingerprint.keyframe_phashes:
|
||||
distances = [hamming_distance(phash, ep) for ep in existing_phashes]
|
||||
min_distances.append(min(distances))
|
||||
|
||||
# 帧匹配比例检查
|
||||
matching_frames = sum(1 for d in min_distances if d < self.PHASH_THRESHOLD)
|
||||
match_ratio = matching_frames / len(min_distances) if min_distances else 0
|
||||
if match_ratio < MATCH_RATIO_THRESHOLD:
|
||||
continue
|
||||
|
||||
median_distance = statistics.median(min_distances) if min_distances else 64
|
||||
if median_distance >= self.PHASH_THRESHOLD:
|
||||
continue
|
||||
|
||||
# 直方图融合(chunk 表优先,回退 JSON 字段;JSON NULL 显式回退空列表)
|
||||
if chunk_data:
|
||||
existing_histograms = [c["color_histogram"] for c in chunk_data if c.get("color_histogram")]
|
||||
existing_chunk_objects = chunk_data
|
||||
else:
|
||||
existing_histograms = ef.get("color_histograms") or []
|
||||
existing_chunk_objects = [
|
||||
{"phash_binary": pp, "start_time_ms": 0, "end_time_ms": 0} for pp in existing_phashes
|
||||
]
|
||||
|
||||
combined_score = self._compute_fusion_score(
|
||||
median_distance, fingerprint.color_histograms, existing_histograms
|
||||
ev = self._evaluate_candidate(
|
||||
fingerprint,
|
||||
existing_phashes,
|
||||
existing_histograms,
|
||||
existing_chunk_objects,
|
||||
query_duration_sec=fingerprint.duration,
|
||||
)
|
||||
logger.debug(
|
||||
"check_batch_duplicate candidate=%s min_distances=%s frame_match_rate=%.3f "
|
||||
"temporal_coverage=%.3f median=%.1f fusion=%.3f segments=%d",
|
||||
existing.id,
|
||||
ev["min_distances"],
|
||||
ev["frame_match_rate"],
|
||||
ev["temporal_coverage"],
|
||||
ev["median_distance"],
|
||||
ev["fusion"],
|
||||
len(ev["segments"]),
|
||||
)
|
||||
|
||||
if combined_score < DUPLICATE_THRESHOLD:
|
||||
continue
|
||||
|
||||
# 滑动窗口时序匹配
|
||||
existing_chunk_objects = (
|
||||
chunk_data
|
||||
if chunk_data
|
||||
else [{"phash_binary": p, "start_time_ms": 0, "end_time_ms": 0} for p in existing_phashes]
|
||||
is_full_duplicate = ev["fusion"] >= DUPLICATE_THRESHOLD and (
|
||||
ev["frame_match_rate"] >= MATCH_RATIO_THRESHOLD or ev["temporal_coverage"] >= PARTIAL_COVERAGE_THRESHOLD
|
||||
)
|
||||
segments = find_duplicate_segments(fingerprint.chunks, existing_chunk_objects)
|
||||
|
||||
return {
|
||||
"duplicate": True,
|
||||
"duplicate_of": existing.id,
|
||||
"reason": "batch_phash_histogram_fusion",
|
||||
"similarity": combined_score,
|
||||
"duplicate_segments": [
|
||||
{
|
||||
"query_start_ms": s.query_start_ms,
|
||||
"query_end_ms": s.query_end_ms,
|
||||
"target_start_ms": s.target_start_ms,
|
||||
"target_end_ms": s.target_end_ms,
|
||||
"avg_distance": round(s.avg_distance, 2),
|
||||
}
|
||||
for s in segments
|
||||
],
|
||||
}
|
||||
if is_full_duplicate and ev["fusion"] > best_score:
|
||||
best_score = ev["fusion"]
|
||||
best_result = {
|
||||
"duplicate": True,
|
||||
"duplicate_of": existing.id,
|
||||
"reason": "batch_phash_histogram_fusion",
|
||||
"similarity": ev["fusion"],
|
||||
"duplicate_segments": [
|
||||
{
|
||||
"query_start_ms": s.query_start_ms,
|
||||
"query_end_ms": s.query_end_ms,
|
||||
"target_start_ms": s.target_start_ms,
|
||||
"target_end_ms": s.target_end_ms,
|
||||
"avg_distance": round(s.avg_distance, 2),
|
||||
}
|
||||
for s in ev["segments"]
|
||||
],
|
||||
}
|
||||
|
||||
if best_result:
|
||||
return best_result
|
||||
logger.info("check_batch_duplicate no match (batch=%s): best_fusion=%.3f", batch_id, best_score)
|
||||
return None
|
||||
|
||||
def compute_duplicate_rate(
|
||||
@@ -849,8 +1074,7 @@ class VideoDeduplicator:
|
||||
max_duplicate_rate = 0.0
|
||||
max_visual_similarity = 0.0
|
||||
match_count = 0
|
||||
|
||||
total_duration_ms = fingerprint.duration if fingerprint.duration else 0
|
||||
evaluated = 0
|
||||
|
||||
for existing in existing_videos:
|
||||
if current_video_id and existing.id == current_video_id:
|
||||
@@ -868,6 +1092,12 @@ class VideoDeduplicator:
|
||||
"match_count": 1,
|
||||
}
|
||||
|
||||
# 跳过指纹质量差的视频(黑屏/纯色视频)
|
||||
existing_phashes_check = ef.get("keyframe_phashes", [])
|
||||
if self._is_bad_fingerprint(existing_phashes_check):
|
||||
logger.debug("Skipping bad fingerprint video %s in compute_duplicate_rate", existing.id)
|
||||
continue
|
||||
|
||||
# 优先从分片表读取
|
||||
existing_phashes = []
|
||||
chunk_data = self._get_existing_chunks(existing.id, session)
|
||||
@@ -879,57 +1109,63 @@ class VideoDeduplicator:
|
||||
if not existing_phashes or not fingerprint.keyframe_phashes:
|
||||
continue
|
||||
|
||||
min_distances = []
|
||||
for phash in fingerprint.keyframe_phashes:
|
||||
distances = [hamming_distance(phash, ep) for ep in existing_phashes]
|
||||
min_distances.append(min(distances))
|
||||
|
||||
# frame_match_rate
|
||||
total_frames = len(min_distances)
|
||||
if total_frames == 0:
|
||||
continue
|
||||
matching_frames = sum(1 for d in min_distances if d < self.PHASH_THRESHOLD)
|
||||
frame_match_rate = matching_frames / total_frames
|
||||
|
||||
# 帧匹配比例太低则跳过
|
||||
if frame_match_rate < 0.3:
|
||||
continue
|
||||
|
||||
# temporal_coverage_rate via find_duplicate_segments
|
||||
existing_chunk_objects = (
|
||||
chunk_data
|
||||
if chunk_data
|
||||
else [{"phash_binary": p, "start_time_ms": 0, "end_time_ms": 0} for p in existing_phashes]
|
||||
)
|
||||
segments = find_duplicate_segments(fingerprint.chunks, existing_chunk_objects)
|
||||
|
||||
if total_duration_ms > 0 and segments:
|
||||
covered_ms = sum(s.query_end_ms - s.query_start_ms for s in segments)
|
||||
temporal_coverage_rate = min(covered_ms / total_duration_ms, 1.0)
|
||||
else:
|
||||
temporal_coverage_rate = 0.0
|
||||
|
||||
# duplicate_rate = 0.4 * frame_match_rate + 0.6 * temporal_coverage_rate
|
||||
dup_rate = (frame_match_rate * 0.4 + temporal_coverage_rate * 0.6) * 100
|
||||
|
||||
# visual_similarity (融合相似度,归一化 0~1)
|
||||
median_distance = statistics.median(min_distances) if min_distances else 64
|
||||
# 直方图 / 分片对象(chunk 表优先,回退 JSON 字段;JSON NULL 显式回退空列表)
|
||||
if chunk_data:
|
||||
existing_histograms = [c["color_histogram"] for c in chunk_data if c.get("color_histogram")]
|
||||
existing_chunk_objects = chunk_data
|
||||
else:
|
||||
# JSON NULL 显式回退空列表
|
||||
existing_histograms = ef.get("color_histograms") or []
|
||||
existing_chunk_objects = [
|
||||
{"phash_binary": pp, "start_time_ms": 0, "end_time_ms": 0} for pp in existing_phashes
|
||||
]
|
||||
|
||||
visual_sim = self._compute_fusion_score(median_distance, fingerprint.color_histograms, existing_histograms)
|
||||
# Issue #1702: 统一评估;frame_match_rate 分母为 min(两视频分片数),
|
||||
# temporal_coverage 时长量纲在 _evaluate_candidate 内统一为毫秒。
|
||||
ev = self._evaluate_candidate(
|
||||
fingerprint,
|
||||
existing_phashes,
|
||||
existing_histograms,
|
||||
existing_chunk_objects,
|
||||
query_duration_sec=fingerprint.duration,
|
||||
)
|
||||
evaluated += 1
|
||||
logger.debug(
|
||||
"compute_duplicate_rate candidate=%s min_distances=%s frame_match_rate=%.3f "
|
||||
"temporal_coverage=%.3f median=%.1f fusion=%.3f segments=%d",
|
||||
existing.id,
|
||||
ev["min_distances"],
|
||||
ev["frame_match_rate"],
|
||||
ev["temporal_coverage"],
|
||||
ev["median_distance"],
|
||||
ev["fusion"],
|
||||
len(ev["segments"]),
|
||||
)
|
||||
|
||||
# 判定是否为重复(融合分数超过阈值)
|
||||
if visual_sim >= DUPLICATE_THRESHOLD:
|
||||
# Issue #1702: 去掉 "frame_match_rate<0.3 整条跳过" 硬门槛——
|
||||
# 局部片段复用帧比例天然低;coverage 为主指标,0 匹配自然得 0 分。
|
||||
# duplicate_rate = 0.4 * frame_match_rate + 0.6 * temporal_coverage
|
||||
dup_rate = (min(ev["frame_match_rate"], 1.0) * 0.4 + ev["temporal_coverage"] * 0.6) * 100
|
||||
|
||||
# 全片重复计数与 check_duplicate 判定口径一致
|
||||
if ev["fusion"] >= DUPLICATE_THRESHOLD and (
|
||||
ev["frame_match_rate"] >= MATCH_RATIO_THRESHOLD or ev["temporal_coverage"] >= PARTIAL_COVERAGE_THRESHOLD
|
||||
):
|
||||
match_count += 1
|
||||
|
||||
if dup_rate > max_duplicate_rate:
|
||||
max_duplicate_rate = dup_rate
|
||||
max_visual_similarity = visual_sim
|
||||
max_visual_similarity = ev["fusion"]
|
||||
|
||||
logger.info(
|
||||
"compute_duplicate_rate done (project=%s scope=%s): evaluated=%d max_rate=%.2f%% "
|
||||
"max_visual_sim=%.3f matches=%d",
|
||||
project_id,
|
||||
scope,
|
||||
evaluated,
|
||||
max_duplicate_rate,
|
||||
max_visual_similarity,
|
||||
match_count,
|
||||
)
|
||||
return {
|
||||
"duplicate_rate": round(max(max_duplicate_rate, 0.0), 2),
|
||||
"visual_similarity": round(max_visual_similarity, 4),
|
||||
@@ -945,18 +1181,20 @@ def _save_fingerprint_chunks(
|
||||
session: Session,
|
||||
) -> None:
|
||||
"""将指纹分片数据批量写入 video_fingerprint_chunks 表。幂等:已有数据时跳过。"""
|
||||
# 幂等检查:已有分片数据则跳过
|
||||
existing_count = (
|
||||
session.query(VideoFingerprintChunkModel).filter(VideoFingerprintChunkModel.video_id == video_id).count()
|
||||
)
|
||||
if existing_count > 0:
|
||||
logger.debug("Fingerprint chunks already exist for video %s (%d chunks), skipping", video_id, existing_count)
|
||||
return
|
||||
|
||||
if not fingerprint.chunks:
|
||||
logger.warning("No chunks in fingerprint for video %s, skipping chunk save", video_id)
|
||||
return
|
||||
|
||||
# Issue #1702: recompute-dedup 重算时指纹算法已变(中心裁剪 + 新阈值),
|
||||
# 旧分片必须替换而非跳过(旧实现"有数据就跳过"导致重算不刷新分片表)。
|
||||
deleted = (
|
||||
session.query(VideoFingerprintChunkModel)
|
||||
.filter(VideoFingerprintChunkModel.video_id == video_id)
|
||||
.delete(synchronize_session=False)
|
||||
)
|
||||
if deleted:
|
||||
logger.info("Replaced %d stale fingerprint chunks for video %s", deleted, video_id)
|
||||
|
||||
chunk_models = fingerprint.to_chunk_models(video_id, project_id, user_id)
|
||||
session.bulk_save_objects(chunk_models)
|
||||
logger.info("Saved %d fingerprint chunks for video %s", len(chunk_models), video_id)
|
||||
@@ -978,9 +1216,17 @@ def check_duplicate_task(self: Task, generated_video_id: str) -> dict:
|
||||
raise ValueError(f"Generated video {generated_video_id} not found")
|
||||
|
||||
local_path = os.path.join(temp_dir, f"{generated_video_id}.mp4")
|
||||
storage_service.download_file(
|
||||
f"projects/{video.project_id}/generated/{generated_video_id}/{generated_video_id}.mp4", local_path
|
||||
)
|
||||
# Issue #1702: recompute 走的是 OSS 重新下载路径(正常生成流程用本地渲染文件,
|
||||
# 不经此任务)。成片真实 OSS key 是生成时的
|
||||
# generated/projects/{pid}/tasks/{task_id}/rendered_*.mp4(见 generation.py
|
||||
# _upload_and_record),旧代码硬编码 projects/{pid}/generated/{vid}/{vid}.mp4
|
||||
# 这个从不存在的 key,导致所有 recompute 任务下载 404、查重数据永远无法重算。
|
||||
# 优先从 file_url 解析真实 key,旧 key 模式仅作回退。
|
||||
download_key = getattr(video, "file_url", "") or ""
|
||||
if not download_key:
|
||||
download_key = f"projects/{video.project_id}/generated/{generated_video_id}/{generated_video_id}.mp4"
|
||||
logger.warning("video %s has no file_url, falling back to legacy key %s", generated_video_id, download_key)
|
||||
storage_service.download_file(download_key, local_path)
|
||||
|
||||
fingerprint = deduplicator.compute_fingerprint(local_path)
|
||||
|
||||
@@ -991,7 +1237,9 @@ def check_duplicate_task(self: Task, generated_video_id: str) -> dict:
|
||||
session,
|
||||
scope="user",
|
||||
user_id=video.user_id,
|
||||
duration_sec=fingerprint.duration / 1000 if fingerprint.duration else 0,
|
||||
# Issue #1702: fingerprint.duration 单位已经是秒,旧代码 /1000 导致
|
||||
# ±15% 时长预过滤窗口缩到 ~0.013s,scope=user 的跨项目查重永远返回 None。
|
||||
duration_sec=fingerprint.duration if fingerprint.duration else 0,
|
||||
)
|
||||
|
||||
video.video_fingerprint = fingerprint.to_dict()
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
|
||||
供 generate_video 共同复用,
|
||||
创建 GeneratedVideo 记录后计算指纹并执行项目级 + 批次内查重。
|
||||
|
||||
v2: 两阶段持久化 — 先计算所有查重数据,再一次性 commit,
|
||||
避免中间异常导致 duplicate_rate 等字段缺失。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -34,24 +37,14 @@ def create_video_record_and_dedup(
|
||||
) -> int:
|
||||
"""创建 GeneratedVideo 记录,计算指纹并执行查重(历史 + 批次)。
|
||||
|
||||
Args:
|
||||
generation_task_id: 生成任务 ID
|
||||
project_id: 项目 ID
|
||||
batch_id: 批次 ID(可为空字符串)
|
||||
file_url: 视频文件 URL
|
||||
file_size: 文件大小(字节)
|
||||
duration: 视频时长(秒)
|
||||
video_path: 视频本地路径(用于计算指纹)
|
||||
mode: 剪辑模式名称
|
||||
session: 数据库会话
|
||||
width: 视频宽度
|
||||
height: 视频高度
|
||||
fps: 视频帧率
|
||||
采用两阶段持久化:先计算所有指纹/查重数据(内存),
|
||||
再一次性写入数据库并 commit。若指纹计算失败,
|
||||
视频记录仍会创建(无查重数据),但保证不会出现"写了记录却没 commit"的中间态。
|
||||
|
||||
Returns:
|
||||
创建的视频记录数量(1 表示成功,0 表示失败)
|
||||
"""
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
from video_processing.dedup import VideoDeduplicator, _save_fingerprint_chunks
|
||||
|
||||
from packages.adapters.sqlalchemy_impl.generated_video_repository import (
|
||||
SQLAlchemyGeneratedVideoRepository,
|
||||
@@ -60,8 +53,9 @@ def create_video_record_and_dedup(
|
||||
|
||||
try:
|
||||
video_id = uuid4().hex
|
||||
# 使用传入的名称,没有则 fallback 到默认命名
|
||||
video_name = name.strip() if name else f"generated-{generation_task_id[:8]}.mp4"
|
||||
|
||||
# ── Phase 1: 构建视频记录(内存,不 commit) ────────────────
|
||||
generated_video = GeneratedVideo(
|
||||
id=video_id,
|
||||
project_id=project_id,
|
||||
@@ -76,98 +70,95 @@ def create_video_record_and_dedup(
|
||||
fps=fps,
|
||||
status="completed",
|
||||
generation_params={"mode": mode},
|
||||
thumbnail_url=thumbnail_url or None,
|
||||
)
|
||||
|
||||
video_repo = SQLAlchemyGeneratedVideoRepository(session)
|
||||
video_repo.create(generated_video)
|
||||
|
||||
# 生成封面缩略图
|
||||
if thumbnail_url:
|
||||
generated_video.thumbnail_url = thumbnail_url
|
||||
video_repo.update_thumbnail(video_id, thumbnail_url)
|
||||
logger.info("Thumbnail set for video %s: %s", video_id, thumbnail_url[:80] if thumbnail_url else "")
|
||||
else:
|
||||
logger.debug("No thumbnail_url provided for video %s, skipping", video_id)
|
||||
|
||||
# 计算视频指纹
|
||||
# ── Phase 2: 计算指纹 & 查重(全部在内存) ────────────────
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = None
|
||||
|
||||
try:
|
||||
fingerprint = deduplicator.compute_fingerprint(video_path)
|
||||
except Exception as fp_err:
|
||||
logger.warning("Fingerprint computation failed for %s: %s", video_id, fp_err)
|
||||
session.commit()
|
||||
return 1
|
||||
|
||||
generated_video.video_fingerprint = fingerprint.to_dict()
|
||||
if fingerprint is not None:
|
||||
generated_video.video_fingerprint = fingerprint.to_dict()
|
||||
|
||||
# 写入分片指纹表
|
||||
from video_processing.dedup import _save_fingerprint_chunks
|
||||
# 写入分片指纹表(失败不阻塞)
|
||||
try:
|
||||
_save_fingerprint_chunks(fingerprint, video_id, project_id, user_id, session)
|
||||
except Exception as chunk_err:
|
||||
logger.warning("Failed to save fingerprint chunks for %s: %s", video_id, chunk_err)
|
||||
|
||||
try:
|
||||
_save_fingerprint_chunks(fingerprint, video_id, project_id, user_id, session)
|
||||
except Exception as chunk_err:
|
||||
logger.warning("Failed to save fingerprint chunks for %s: %s", video_id, chunk_err)
|
||||
|
||||
# (a) 历史成片查重(跨项目全局 + 时长预过滤)
|
||||
duration_sec = fingerprint.duration / 1000 if fingerprint.duration else 0
|
||||
duplicate_result = deduplicator.check_duplicate(
|
||||
fingerprint,
|
||||
project_id,
|
||||
session,
|
||||
scope="user",
|
||||
user_id=user_id,
|
||||
duration_sec=duration_sec,
|
||||
)
|
||||
|
||||
# (b) 批次内查重(仅当有 batch_id 时)
|
||||
if not duplicate_result and batch_id:
|
||||
duplicate_result = deduplicator.check_batch_duplicate(fingerprint, batch_id, video_id, session)
|
||||
|
||||
if duplicate_result:
|
||||
generated_video.is_duplicate = True
|
||||
generated_video.duplicate_of = duplicate_result["duplicate_of"]
|
||||
logger.info(
|
||||
"Duplicate detected: %s -> %s (reason=%s, similarity=%.3f)",
|
||||
video_id,
|
||||
duplicate_result["duplicate_of"],
|
||||
duplicate_result["reason"],
|
||||
duplicate_result["similarity"],
|
||||
)
|
||||
else:
|
||||
generated_video.is_duplicate = False
|
||||
generated_video.duplicate_of = None
|
||||
|
||||
# 计算重复率百分比(跨项目全局)
|
||||
try:
|
||||
rate_result = deduplicator.compute_duplicate_rate(
|
||||
# (a) 历史成片查重(跨项目全局 + 时长预过滤)
|
||||
# Issue #1702: fingerprint.duration 单位是秒,旧代码 /1000 让时长预过滤失效
|
||||
duration_sec = fingerprint.duration if fingerprint.duration else 0
|
||||
duplicate_result = deduplicator.check_duplicate(
|
||||
fingerprint,
|
||||
project_id,
|
||||
video_id,
|
||||
session,
|
||||
scope="user",
|
||||
user_id=user_id,
|
||||
duration_sec=duration_sec,
|
||||
)
|
||||
generated_video.duplicate_rate = rate_result["duplicate_rate"]
|
||||
generated_video.match_count = rate_result["match_count"]
|
||||
generated_video.visual_similarity = rate_result["visual_similarity"]
|
||||
logger.info(
|
||||
"Duplicate rate for %s: %.2f%% (visual_sim=%.3f, matches=%d)",
|
||||
video_id,
|
||||
rate_result["duplicate_rate"],
|
||||
rate_result["visual_similarity"],
|
||||
rate_result["match_count"],
|
||||
)
|
||||
except Exception as rate_err:
|
||||
logger.warning("Failed to compute duplicate_rate for %s: %s", video_id, rate_err)
|
||||
generated_video.duplicate_rate = None
|
||||
|
||||
video_repo.update(generated_video)
|
||||
# (b) 批次内查重(仅当有 batch_id 时)
|
||||
if not duplicate_result and batch_id:
|
||||
duplicate_result = deduplicator.check_batch_duplicate(fingerprint, batch_id, video_id, session)
|
||||
|
||||
if duplicate_result:
|
||||
generated_video.is_duplicate = True
|
||||
generated_video.duplicate_of = duplicate_result["duplicate_of"]
|
||||
logger.info(
|
||||
"Duplicate detected: %s -> %s (reason=%s, similarity=%.3f)",
|
||||
video_id,
|
||||
duplicate_result["duplicate_of"],
|
||||
duplicate_result["reason"],
|
||||
duplicate_result["similarity"],
|
||||
)
|
||||
else:
|
||||
generated_video.is_duplicate = False
|
||||
generated_video.duplicate_of = None
|
||||
|
||||
# 计算重复率百分比(跨项目全局)
|
||||
try:
|
||||
rate_result = deduplicator.compute_duplicate_rate(
|
||||
fingerprint,
|
||||
project_id,
|
||||
video_id,
|
||||
session,
|
||||
scope="user",
|
||||
user_id=user_id,
|
||||
)
|
||||
generated_video.duplicate_rate = rate_result["duplicate_rate"]
|
||||
generated_video.match_count = rate_result["match_count"]
|
||||
generated_video.visual_similarity = rate_result["visual_similarity"]
|
||||
logger.info(
|
||||
"Duplicate rate for %s: %.2f%% (visual_sim=%.3f, matches=%d)",
|
||||
video_id,
|
||||
rate_result["duplicate_rate"],
|
||||
rate_result["visual_similarity"],
|
||||
rate_result["match_count"],
|
||||
)
|
||||
except Exception as rate_err:
|
||||
logger.warning("Failed to compute duplicate_rate for %s: %s", video_id, rate_err)
|
||||
generated_video.duplicate_rate = None
|
||||
|
||||
# ── Phase 3: 一次性持久化 ─────────────────────────────────
|
||||
video_repo = SQLAlchemyGeneratedVideoRepository(session)
|
||||
video_repo.create(generated_video)
|
||||
|
||||
if thumbnail_url:
|
||||
logger.info("Thumbnail set for video %s: %s", video_id, thumbnail_url[:80])
|
||||
|
||||
session.commit()
|
||||
logger.info(
|
||||
"GeneratedVideo record created: %s (task=%s, dup=%s)",
|
||||
"GeneratedVideo record created: %s (task=%s, dup=%s, rate=%s)",
|
||||
video_id,
|
||||
generation_task_id,
|
||||
generated_video.is_duplicate,
|
||||
generated_video.duplicate_rate,
|
||||
)
|
||||
return 1
|
||||
except Exception as e:
|
||||
|
||||
@@ -304,3 +304,127 @@ def normalize_video(
|
||||
]
|
||||
run_ffmpeg(command)
|
||||
return {"width": width, "height": height, "path": output_path}
|
||||
|
||||
|
||||
def random_edge_crop(
|
||||
input_path: str | Path,
|
||||
output_path: str | Path | None = None,
|
||||
*,
|
||||
min_crop_pct: float = 0.02,
|
||||
max_crop_pct: float = 0.05,
|
||||
) -> Path:
|
||||
"""对视频四边做随机裁剪再缩放回原分辨率,用于改变 pHash 指纹。
|
||||
|
||||
Args:
|
||||
input_path: 输入视频路径
|
||||
output_path: 输出路径;为 None 时写入 input_path 同目录的临时文件,
|
||||
成功后覆盖原文件
|
||||
min_crop_pct: 每边最小裁剪比例(默认 2%)
|
||||
max_crop_pct: 每边最大裁剪比例(默认 5%)
|
||||
|
||||
Returns:
|
||||
输出文件路径(Path 对象)
|
||||
|
||||
Raises:
|
||||
subprocess.CalledProcessError: ffmpeg 执行失败时抛出
|
||||
"""
|
||||
import random
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
input_path = Path(input_path)
|
||||
|
||||
# 获取原始分辨率
|
||||
info = probe_video_info(str(input_path))
|
||||
W = info["width"]
|
||||
H = info["height"]
|
||||
|
||||
if W <= 0 or H <= 0:
|
||||
logger.warning("无法获取视频分辨率 (W=%d H=%d),跳过裁剪: %s", W, H, input_path)
|
||||
return input_path
|
||||
|
||||
# 四边各自随机裁剪 2%~5%
|
||||
crop_top = int(H * random.uniform(min_crop_pct, max_crop_pct))
|
||||
crop_bottom = int(H * random.uniform(min_crop_pct, max_crop_pct))
|
||||
crop_left = int(W * random.uniform(min_crop_pct, max_crop_pct))
|
||||
crop_right = int(W * random.uniform(min_crop_pct, max_crop_pct))
|
||||
|
||||
# 裁剪后尺寸(确保至少 2 像素)
|
||||
new_w = max(W - crop_left - crop_right, 2)
|
||||
new_h = max(H - crop_top - crop_bottom, 2)
|
||||
x_offset = crop_left
|
||||
y_offset = crop_top
|
||||
|
||||
# 确保裁剪尺寸为偶数(ffmpeg 编码器常要求偶数尺寸)
|
||||
new_w = new_w if new_w % 2 == 0 else new_w - 1
|
||||
new_h = new_h if new_h % 2 == 0 else new_h - 1
|
||||
if new_w < 2:
|
||||
new_w = 2
|
||||
if new_h < 2:
|
||||
new_h = 2
|
||||
|
||||
# 输出分辨率必须与原始一致
|
||||
out_w = W if W % 2 == 0 else W + 1
|
||||
out_h = H if H % 2 == 0 else H + 1
|
||||
|
||||
vf = f"crop={new_w}:{new_h}:{x_offset}:{y_offset},scale={out_w}:{out_h}"
|
||||
|
||||
logger.info(
|
||||
"随机边缘裁剪: %s → crop(%d,%d,%d,%d)=%dx%d scale→%dx%d",
|
||||
input_path.name,
|
||||
crop_top,
|
||||
crop_bottom,
|
||||
crop_left,
|
||||
crop_right,
|
||||
new_w,
|
||||
new_h,
|
||||
out_w,
|
||||
out_h,
|
||||
)
|
||||
|
||||
# 确定输出路径
|
||||
if output_path is None:
|
||||
temp_fd, temp_path = tempfile.mkstemp(suffix=".mp4", dir=input_path.parent)
|
||||
import os
|
||||
|
||||
os.close(temp_fd)
|
||||
temp_output = Path(temp_path)
|
||||
replace_original = True
|
||||
else:
|
||||
temp_output = Path(output_path)
|
||||
replace_original = False
|
||||
|
||||
command = [
|
||||
FFMPEG_BIN,
|
||||
"-y",
|
||||
"-i",
|
||||
str(input_path),
|
||||
"-vf",
|
||||
vf,
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"fast",
|
||||
"-crf",
|
||||
"18",
|
||||
"-c:a",
|
||||
"copy",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
str(temp_output),
|
||||
]
|
||||
|
||||
try:
|
||||
run_ffmpeg(command)
|
||||
except Exception:
|
||||
# 裁剪失败时清理临时文件
|
||||
if temp_output.exists() and replace_original:
|
||||
temp_output.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
# 成功 → 覆盖原文件
|
||||
if replace_original:
|
||||
shutil.move(str(temp_output), str(input_path))
|
||||
return input_path
|
||||
|
||||
return temp_output
|
||||
|
||||
@@ -213,7 +213,7 @@ def generate_ass_from_timeline(
|
||||
t_shadow.get("offset_x", 2) if t_shadow.get("enabled", False) else 0,
|
||||
t_shadow.get("offset_y", 2) if t_shadow.get("enabled", False) else 0,
|
||||
)
|
||||
t_alignment = position_to_ass_alignment(title_cfg.get("position", "top"))
|
||||
t_alignment = position_to_ass_alignment(title_cfg.get("position", "bottom"))
|
||||
|
||||
title_style_line = build_ass_style(
|
||||
"TitleStyle",
|
||||
|
||||
@@ -198,8 +198,13 @@ class UnifiedRenderService:
|
||||
# 2. 分组为 RenderLayers
|
||||
layers = self._group_clips_into_layers(resolved)
|
||||
|
||||
# 2.5 配音时长对齐:如果有配音素材,调整片段时长以匹配配音时长
|
||||
voice_duration = self._get_voice_audio_duration()
|
||||
if voice_duration > 0:
|
||||
self._align_clips_to_voice_duration(layers, voice_duration)
|
||||
|
||||
# 3. 计算视频总时长(用于字幕显示时长)
|
||||
video_duration = self._estimate_total_duration(layers)
|
||||
video_duration_final = self._estimate_total_duration(layers)
|
||||
# Debug: 输出各图层时长明细
|
||||
for layer in layers:
|
||||
layer_total = sum(UnifiedRenderService._clip_adjusted_duration(c) for c in layer.clips)
|
||||
@@ -215,16 +220,16 @@ class UnifiedRenderService:
|
||||
self.transition_duration,
|
||||
", ".join(clip_details),
|
||||
)
|
||||
logger.info("[debug] estimated video_duration=%.3f", video_duration)
|
||||
logger.info("[debug] estimated video_duration=%.3f", video_duration_final)
|
||||
|
||||
# 3.5 TTS 配音生成(如果配置了)
|
||||
self._maybe_add_voiceover_layer(layers, video_duration=video_duration)
|
||||
self._maybe_add_voiceover_layer(layers, video_duration=video_duration_final)
|
||||
|
||||
# 3.6 配音素材库音频(如果传入了本地路径)
|
||||
self._maybe_add_voice_library_layer(layers, video_duration=video_duration)
|
||||
self._maybe_add_voice_library_layer(layers, video_duration=video_duration_final)
|
||||
|
||||
# 4. 生成 ASS 字幕文件(如果有 title/subtitle 配置)
|
||||
ass_path = self._maybe_generate_ass(video_duration)
|
||||
ass_path = self._maybe_generate_ass(video_duration_final)
|
||||
|
||||
# 4.5 解析画中画配置
|
||||
pip_config = PiPConfig.from_dict((self.plan.config or {}).get("pip_config"))
|
||||
@@ -257,7 +262,7 @@ class UnifiedRenderService:
|
||||
# 先尝试 stream copy 优化(无重编码,性能提升 10 倍+)
|
||||
# 条件不满足或失败时回退到带滤镜的直通渲染
|
||||
stream_copy_ok = self._try_render_stream_copy(
|
||||
layers, output_path, ass_path=ass_path, video_duration=video_duration
|
||||
layers, output_path, ass_path=ass_path, video_duration=video_duration_final
|
||||
)
|
||||
if stream_copy_ok:
|
||||
used_stream_copy = True
|
||||
@@ -271,7 +276,7 @@ class UnifiedRenderService:
|
||||
layers,
|
||||
output_path,
|
||||
ass_path=ass_path,
|
||||
video_duration=video_duration,
|
||||
video_duration=video_duration_final,
|
||||
)
|
||||
else:
|
||||
filter_complex, input_args = self._build_filter_complex(layers, ass_path=ass_path)
|
||||
@@ -327,7 +332,7 @@ class UnifiedRenderService:
|
||||
from video_processing.ffmpeg_utils import run_ffmpeg
|
||||
|
||||
run_ffmpeg(extract_cmd)
|
||||
final_audio = mix_bgm_with_main(ctx, main_audio_path, bgm_cfg, video_duration)
|
||||
final_audio = mix_bgm_with_main(ctx, main_audio_path, bgm_cfg, video_duration_final)
|
||||
# 合并回视频
|
||||
|
||||
bgm_output = self.work_dir / f"rendered_{self.plan.id}_bgm.mp4"
|
||||
@@ -353,7 +358,7 @@ class UnifiedRenderService:
|
||||
audio_path = mix_audio(
|
||||
ctx,
|
||||
layers,
|
||||
video_duration,
|
||||
video_duration_final,
|
||||
bgm_path=self.bgm_path,
|
||||
bgm_config=bgm_config,
|
||||
audio_tracks_config=audio_tracks_config,
|
||||
@@ -487,6 +492,147 @@ class UnifiedRenderService:
|
||||
"""
|
||||
return _estimate_total_duration_pure(layers, self.transition_duration)
|
||||
|
||||
def _get_voice_audio_duration(self) -> float:
|
||||
"""获取配音音频文件的时长(秒)。
|
||||
|
||||
Returns:
|
||||
配音音频时长,如果无配音或探测失败则返回 0.0
|
||||
"""
|
||||
if not self.voiceover_audio_path:
|
||||
return 0.0
|
||||
|
||||
audio_path = Path(self.voiceover_audio_path)
|
||||
if not audio_path.exists() or audio_path.stat().st_size == 0:
|
||||
return 0.0
|
||||
|
||||
try:
|
||||
duration = probe_duration(audio_path)
|
||||
logger.info("[voice-align] 配音音频时长: %.3fs path=%s", duration, self.voiceover_audio_path)
|
||||
return duration
|
||||
except Exception as e:
|
||||
logger.warning("[voice-align] 探测配音音频时长失败: %s", e)
|
||||
return 0.0
|
||||
|
||||
def _align_clips_to_voice_duration(
|
||||
self,
|
||||
layers: list[RenderLayer],
|
||||
voice_duration: float,
|
||||
) -> None:
|
||||
"""调整片段时长以对齐配音时长。
|
||||
|
||||
核心逻辑:
|
||||
- 计算片段总时长与配音时长的比例
|
||||
- ±5% 以内不调整
|
||||
- ratio < 1(片段比配音长):按比例裁剪每段末尾
|
||||
- ratio > 1(片段比配音短):按比例慢放每段
|
||||
|
||||
Args:
|
||||
layers: 渲染图层列表
|
||||
voice_duration: 配音时长(秒)
|
||||
"""
|
||||
if voice_duration <= 0:
|
||||
return
|
||||
|
||||
# 只调整视频图层(main/broll/background),不调整音频图层
|
||||
video_layers = [layer for layer in layers if layer.role in ("main", "broll", "background")]
|
||||
if not video_layers:
|
||||
return
|
||||
|
||||
# 计算所有视频图层的总时长
|
||||
total_clips_duration = 0.0
|
||||
for layer in video_layers:
|
||||
for clip in layer.clips:
|
||||
clip_dur = self._clip_adjusted_duration(clip)
|
||||
total_clips_duration += clip_dur
|
||||
|
||||
if total_clips_duration <= 0:
|
||||
return
|
||||
|
||||
ratio = voice_duration / total_clips_duration
|
||||
|
||||
# ±5% 以内不调整
|
||||
if abs(ratio - 1.0) <= 0.05:
|
||||
logger.info(
|
||||
"[voice-align] 比例接近1:1,跳过调整: ratio=%.4f voice=%.3f clips=%.3f",
|
||||
ratio,
|
||||
voice_duration,
|
||||
total_clips_duration,
|
||||
)
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"[voice-align] 开始调整片段时长: ratio=%.4f voice=%.3f clips=%.3f",
|
||||
ratio,
|
||||
voice_duration,
|
||||
total_clips_duration,
|
||||
)
|
||||
|
||||
# 收集所有视频 clip
|
||||
all_clips: list[tuple[RenderLayer, ResolvedClip]] = []
|
||||
for layer in video_layers:
|
||||
for clip in layer.clips:
|
||||
all_clips.append((layer, clip))
|
||||
|
||||
if not all_clips:
|
||||
return
|
||||
|
||||
if ratio < 1.0:
|
||||
# 片段比配音长,按比例裁剪每段末尾
|
||||
# 减少每个 clip 的 duration
|
||||
for _layer, clip in all_clips:
|
||||
old_duration = clip.duration if clip.duration > 0 else clip.actual_duration
|
||||
new_duration = old_duration * ratio
|
||||
|
||||
# 更新 duration
|
||||
clip.duration = max(0.1, new_duration) # 至少 0.1s
|
||||
|
||||
# 如果有 trim_config,也需要调整
|
||||
if clip.trim_config is not None:
|
||||
new_trim_duration = clip.trim_config.duration * ratio
|
||||
clip.trim_config = TrimConfig(
|
||||
start_time=clip.trim_config.start_time,
|
||||
duration=max(0.1, new_trim_duration),
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"[voice-align] trim clip=%s: %.3f -> %.3f",
|
||||
clip.clip_id,
|
||||
old_duration,
|
||||
clip.duration,
|
||||
)
|
||||
|
||||
else:
|
||||
# ratio > 1.0: 片段比配音短,按比例慢放每段
|
||||
# 降低 playback_speed
|
||||
for _layer, clip in all_clips:
|
||||
old_speed = clip.playback_speed if clip.playback_speed > 0 else 1.0
|
||||
# speed = old_speed / ratio 会使视频变慢(ratio > 1 时)
|
||||
new_speed = old_speed / ratio
|
||||
|
||||
# 下限 0.25x(避免过慢)
|
||||
new_speed = max(0.25, round(new_speed, 4))
|
||||
clip.playback_speed = new_speed
|
||||
|
||||
logger.debug(
|
||||
"[voice-align] slowdown clip=%s: speed %.4f -> %.4f",
|
||||
clip.clip_id,
|
||||
old_speed,
|
||||
new_speed,
|
||||
)
|
||||
|
||||
# 调整后重新计算总时长用于日志
|
||||
new_total = 0.0
|
||||
for layer in video_layers:
|
||||
for clip in layer.clips:
|
||||
new_total += self._clip_adjusted_duration(clip)
|
||||
|
||||
logger.info(
|
||||
"[voice-align] 调整完成: 新总时长=%.3fs (目标=%.3fs, 差异=%.3fs)",
|
||||
new_total,
|
||||
voice_duration,
|
||||
abs(new_total - voice_duration),
|
||||
)
|
||||
|
||||
def _maybe_generate_ass(self, video_duration: float) -> Path | None:
|
||||
"""根据 plan.config 生成 ASS 字幕文件。
|
||||
|
||||
|
||||
@@ -175,23 +175,21 @@ def process_duplication_check(self: Task, record_id: str) -> dict:
|
||||
logger.error("Duplication check failed for record %s: %s", record_id, e, exc_info=True)
|
||||
if session is not None:
|
||||
session.rollback()
|
||||
# 本次是最后一次执行机会(retries 从 0 计数,达到 max_retries 说明重试已耗尽),
|
||||
# 标记 failed;否则保持 pending 由 Celery 60 秒后重试
|
||||
try:
|
||||
if "repo" in locals() and self.request.retries >= self.max_retries:
|
||||
# 超过重试上限:标记 failed 并返回失败结果,不再 retry
|
||||
if "repo" in locals() and self.request.retries >= self.max_retries:
|
||||
try:
|
||||
failed_record = repo.get(record_id)
|
||||
if failed_record is not None and failed_record.status != "failed":
|
||||
failed_record.mark_failed(f"查重失败(已重试{self.max_retries}次): {e}")
|
||||
repo.update(failed_record)
|
||||
session.commit()
|
||||
except Exception as inner:
|
||||
logger.error("Failed to mark duplication record %s as failed: %s", record_id, inner)
|
||||
session.rollback()
|
||||
# 仅在还有重试次数时才重新入队;超过 max_retries 时上面已标记 failed,
|
||||
# 直接抛出异常让 Celery 记录错误,不再无限重试。
|
||||
if self.request.retries < self.max_retries:
|
||||
except Exception as inner:
|
||||
logger.error("Failed to mark duplication record %s as failed: %s", record_id, inner)
|
||||
session.rollback()
|
||||
return {"ok": False, "record_id": record_id, "status": "failed", "error": str(e)}
|
||||
# 未达上限:60 秒后重试
|
||||
raise self.retry(exc=e, countdown=60) from e
|
||||
raise
|
||||
return {"ok": False, "record_id": record_id, "status": "failed", "error": str(e)}
|
||||
|
||||
finally:
|
||||
if session is not None:
|
||||
|
||||
@@ -716,6 +716,28 @@ def generate_video(self, task_id: str) -> dict:
|
||||
|
||||
_update_task_progress(task_id, 80, "渲染完成")
|
||||
|
||||
# ── 3.5 随机边缘裁剪降重(#1664) ──────────────────────────
|
||||
from video_processing.ffmpeg_utils import random_edge_crop
|
||||
|
||||
try:
|
||||
cropped_path = random_edge_crop(output_path)
|
||||
if cropped_path != output_path:
|
||||
output_path = cropped_path
|
||||
if gen_task:
|
||||
gen_task.append_log("边缘裁剪", "已应用随机 2-5% 边缘裁剪降重")
|
||||
_flush_logs(task_id, gen_task)
|
||||
logger.info("[task_id=%s] 随机边缘裁剪完成: %s", task_id, output_path)
|
||||
except Exception as crop_err:
|
||||
logger.warning(
|
||||
"[task_id=%s] 随机边缘裁剪失败,使用原始视频继续: %s",
|
||||
task_id,
|
||||
crop_err,
|
||||
exc_info=True,
|
||||
)
|
||||
if gen_task:
|
||||
gen_task.append_log("边缘裁剪", f"裁剪失败,使用原始视频: {crop_err}")
|
||||
_flush_logs(task_id, gen_task)
|
||||
|
||||
# ── 4. 上传 OSS + 查重记录 ───────────────────────────────
|
||||
_update_task_progress(task_id, 85, "开始上传")
|
||||
file_url, duration, file_size, video_count = _upload_and_record(
|
||||
|
||||
@@ -81,14 +81,14 @@ def position_to_ass_alignment(position: str) -> int:
|
||||
position: 位置字符串 top/center/bottom
|
||||
|
||||
Returns:
|
||||
ASS 对齐编号,默认 8(顶部居中)
|
||||
ASS 对齐编号,默认 2(底部居中,与前端 DEFAULT_TITLE_SETTINGS.position="bottom" 对齐)
|
||||
"""
|
||||
mapping = {
|
||||
"top": 8,
|
||||
"center": 5,
|
||||
"bottom": 2,
|
||||
}
|
||||
return mapping.get(position, 8)
|
||||
return mapping.get(position, 2)
|
||||
|
||||
|
||||
# ── Style 行构建 ──────────────────────────────────────────────────────────────
|
||||
@@ -226,7 +226,6 @@ def _wrap_title_text(
|
||||
|
||||
# 换行计算使用原始 font_size,与 CSS 预览一致;1.35x 补偿仅用于 ASS Fontsize 渲染
|
||||
|
||||
|
||||
# 先按已有 \N 分段,每段独立自动换行,最后用 \N 拼回
|
||||
segments = text.split("\\N")
|
||||
wrapped_segments: list[str] = []
|
||||
@@ -386,8 +385,8 @@ def build_ass_content(
|
||||
# position → alignment 三档逻辑,现有输出保持一字节不变。
|
||||
title_pos = _parse_title_position(title_config, video_width, video_height)
|
||||
|
||||
title_alignment = 5 if title_pos is not None else position_to_ass_alignment(
|
||||
title_config.get("position", "top")
|
||||
title_alignment = (
|
||||
5 if title_pos is not None else position_to_ass_alignment(title_config.get("position", "bottom"))
|
||||
)
|
||||
|
||||
styles.append(
|
||||
|
||||
@@ -83,11 +83,11 @@ class TestPositionToAssAlignment:
|
||||
def test_bottom(self):
|
||||
assert position_to_ass_alignment("bottom") == 2
|
||||
|
||||
def test_unknown_defaults_top(self):
|
||||
assert position_to_ass_alignment("unknown") == 8
|
||||
def test_unknown_defaults_bottom(self):
|
||||
assert position_to_ass_alignment("unknown") == 2
|
||||
|
||||
def test_empty_defaults_top(self):
|
||||
assert position_to_ass_alignment("") == 8
|
||||
def test_empty_defaults_bottom(self):
|
||||
assert position_to_ass_alignment("") == 2
|
||||
|
||||
|
||||
# ============================================================
|
||||
@@ -581,6 +581,7 @@ class TestConstants:
|
||||
assert isinstance(TITLE_MARGIN_BOTTOM, int)
|
||||
assert isinstance(TITLE_MARGIN_SIDE, int)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# _wrap_title_text 换行逻辑验证
|
||||
# ============================================================
|
||||
|
||||
@@ -0,0 +1,507 @@
|
||||
"""Issue #1677 多视频批量生成 — 变体独立配置与批量预览/批量生成测试。
|
||||
|
||||
覆盖:
|
||||
- 批量预览:preview_count=N 一次创建 N 个独立任务,返回变体数组
|
||||
- 变体克隆链路:N 个预览/正式任务各自关联独立克隆 plan
|
||||
- 变体独立配置:titles[]/voice_library_ids[]/cover_urls[] 按变体注入
|
||||
- 长度校验:数组长度必须为 1 或 N(共用或独立),非法长度报错
|
||||
- N=1 向后兼容:旧字段单值行为不变
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from app.core.task_enqueue import GlobalQueueFull, UserPendingLimitExceeded
|
||||
from app.schemas.generation_task import (
|
||||
BatchPreviewGenerationTaskResponse,
|
||||
CreateGenerationTaskRequest,
|
||||
CreatePreviewGenerationTaskRequest,
|
||||
)
|
||||
|
||||
from packages.domain import GenerationTask
|
||||
from packages.domain.generation_task import GenerationTaskStatus
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# 辅助构造
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def _make_user(user_id="test_user_001"):
|
||||
mock_user = MagicMock()
|
||||
mock_user.id = user_id
|
||||
auth = MagicMock()
|
||||
auth.user = mock_user
|
||||
return auth
|
||||
|
||||
|
||||
def _make_task(task_id=None, status=GenerationTaskStatus.PENDING, source_plan_id=None):
|
||||
task = GenerationTask.create(
|
||||
project_id="",
|
||||
asset_library_id="",
|
||||
template_id="tpl_001",
|
||||
asset_ids=["asset_1"],
|
||||
)
|
||||
if task_id:
|
||||
task.id = task_id
|
||||
task.status = status
|
||||
task.is_preview = True
|
||||
task.source_edit_plan_id = source_plan_id or ""
|
||||
task.voice_library_id = ""
|
||||
task.title_config = {}
|
||||
task.cover_url = ""
|
||||
return task
|
||||
|
||||
|
||||
def _make_preview_request(**kwargs):
|
||||
defaults = {
|
||||
"template_id": "tpl_001",
|
||||
"asset_ids": ["asset_1", "asset_2"],
|
||||
}
|
||||
defaults.update(kwargs)
|
||||
return CreatePreviewGenerationTaskRequest(**defaults)
|
||||
|
||||
|
||||
def _repo_mock():
|
||||
repo = MagicMock()
|
||||
repo.count_pending_by_user.return_value = 0
|
||||
repo.count_pending_total.return_value = 0
|
||||
repo.get.side_effect = lambda tid: None
|
||||
return repo
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# Schema 校验:变体数组长度
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestVariantArrayValidation:
|
||||
"""变体数组字段长度校验。"""
|
||||
|
||||
def test_preview_titles_length_matches_count(self):
|
||||
"""titles 长度 = preview_count 合法"""
|
||||
req = _make_preview_request(preview_count=3, titles=["标题A", "标题B", "标题C"])
|
||||
assert len(req.titles) == 3
|
||||
|
||||
def test_preview_titles_single_shared(self):
|
||||
"""titles 长度 1 = 所有变体共用,合法"""
|
||||
req = _make_preview_request(preview_count=3, titles=["共用标题"])
|
||||
assert req.titles == ["共用标题"]
|
||||
|
||||
def test_preview_titles_wrong_length_raises(self):
|
||||
"""titles 长度 2 与 preview_count=3 不匹配 → 报错"""
|
||||
with pytest.raises(ValueError, match="titles"):
|
||||
_make_preview_request(preview_count=3, titles=["A", "B"])
|
||||
|
||||
def test_preview_voice_ids_wrong_length_raises(self):
|
||||
"""voice_library_ids 长度非法 → 报错"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
with pytest.raises(ValidationError, match="voice_library_ids"):
|
||||
_make_preview_request(preview_count=4, voice_library_ids=["v1", "v2"])
|
||||
|
||||
def test_preview_empty_arrays_ok(self):
|
||||
"""空数组(回退单值字段)合法"""
|
||||
req = _make_preview_request(preview_count=3)
|
||||
assert req.titles == []
|
||||
assert req.voice_library_ids == []
|
||||
assert req.cover_urls == []
|
||||
|
||||
def test_generation_titles_length_matches_count(self):
|
||||
"""正式生成 titles 长度 = count 合法"""
|
||||
req = CreateGenerationTaskRequest(
|
||||
template_id="tpl_1",
|
||||
asset_ids=["a1"],
|
||||
count=3,
|
||||
titles=["A", "B", "C"],
|
||||
)
|
||||
assert len(req.titles) == 3
|
||||
|
||||
def test_generation_arrays_wrong_length_raises(self):
|
||||
"""正式生成 cover_urls 长度与 count 不匹配 → 报错"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
with pytest.raises(ValidationError, match="cover_urls"):
|
||||
CreateGenerationTaskRequest(
|
||||
template_id="tpl_1",
|
||||
asset_ids=["a1"],
|
||||
count=3,
|
||||
cover_urls=["c1", "c2"],
|
||||
)
|
||||
|
||||
def test_generation_single_count_no_arrays(self):
|
||||
"""N=1 且不传数组:完全旧行为"""
|
||||
req = CreateGenerationTaskRequest(template_id="tpl_1", asset_ids=["a1"])
|
||||
assert req.count == 1
|
||||
assert req.titles == []
|
||||
assert req.voice_library_ids == []
|
||||
assert req.cover_urls == []
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# 批量预览路由
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestBatchPreviewRoute:
|
||||
"""POST /preview 批量变体。"""
|
||||
|
||||
def test_preview_count_1_returns_single_item_array(self):
|
||||
"""N=1 返回 items 长度 1 的批量响应(结构统一)"""
|
||||
from app.api.routes.generation_preview import create_preview_generation_task
|
||||
|
||||
task = _make_task(task_id="task_1")
|
||||
repo = _repo_mock()
|
||||
with patch("app.api.routes.generation_preview.CreateGenerationTaskUseCase") as MockUC:
|
||||
MockUC.return_value.execute.return_value = task
|
||||
with patch("app.api.routes.generation_preview.safe_enqueue_generation_task", return_value=True):
|
||||
resp = create_preview_generation_task(
|
||||
_make_preview_request(preview_count=1),
|
||||
authenticated_user=_make_user(),
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
assert isinstance(resp, BatchPreviewGenerationTaskResponse)
|
||||
assert resp.total == 1
|
||||
assert len(resp.items) == 1
|
||||
assert resp.items[0].task_id == "task_1"
|
||||
assert resp.items[0].variant_index == 0
|
||||
|
||||
def test_preview_count_3_creates_three_independent_tasks(self):
|
||||
"""N=3 创建 3 个独立任务,返回 3 个变体,task_id 各不相同"""
|
||||
from app.api.routes.generation_preview import create_preview_generation_task
|
||||
|
||||
tasks = [_make_task(task_id=f"task_{i}") for i in range(3)]
|
||||
repo = _repo_mock()
|
||||
with patch("app.api.routes.generation_preview.CreateGenerationTaskUseCase") as MockUC:
|
||||
MockUC.return_value.execute.side_effect = tasks
|
||||
with patch("app.api.routes.generation_preview.safe_enqueue_generation_task", return_value=True):
|
||||
resp = create_preview_generation_task(
|
||||
_make_preview_request(preview_count=3),
|
||||
authenticated_user=_make_user(),
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
assert resp.total == 3
|
||||
task_ids = [item.task_id for item in resp.items]
|
||||
assert task_ids == ["task_0", "task_1", "task_2"]
|
||||
assert len(set(task_ids)) == 3
|
||||
for i, item in enumerate(resp.items):
|
||||
assert item.variant_index == i
|
||||
|
||||
def test_preview_count_3_clones_three_variant_plans(self):
|
||||
"""有源 plan 时,N=3 克隆 3 个独立变体 plan(预览全部克隆,不用源 plan)"""
|
||||
from app.api.routes.generation_preview import create_preview_generation_task
|
||||
|
||||
tasks = [_make_task(task_id=f"task_{i}", source_plan_id="source_plan") for i in range(3)]
|
||||
repo = _repo_mock()
|
||||
cloned_plan_ids = ["clone_1", "clone_2", "clone_3"]
|
||||
with patch("app.api.routes.generation_preview.CreateGenerationTaskUseCase") as MockUC:
|
||||
MockUC.return_value.execute.side_effect = tasks
|
||||
with patch("app.api.routes.generation_preview.safe_enqueue_generation_task", return_value=True):
|
||||
with patch("app.services.edit_plan_service.EditPlanService") as MockPlanSvc:
|
||||
clone_results = [MagicMock(id=pid) for pid in cloned_plan_ids]
|
||||
MockPlanSvc.return_value.clone_plan_for_variant.side_effect = clone_results
|
||||
create_preview_generation_task(
|
||||
_make_preview_request(preview_count=3),
|
||||
authenticated_user=_make_user(),
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
# 克隆被调用 3 次
|
||||
assert MockPlanSvc.return_value.clone_plan_for_variant.call_count == 3
|
||||
# 每个任务关联到不同的克隆 plan
|
||||
for i, task in enumerate(tasks):
|
||||
assert task.source_edit_plan_id == cloned_plan_ids[i]
|
||||
|
||||
def test_preview_variant_titles_injected_per_variant(self):
|
||||
"""titles[] 按变体注入 title_config.text"""
|
||||
from app.api.routes.generation_preview import create_preview_generation_task
|
||||
|
||||
tasks = [_make_task(task_id=f"task_{i}") for i in range(3)]
|
||||
repo = _repo_mock()
|
||||
captured_commands = []
|
||||
with patch("app.api.routes.generation_preview.CreateGenerationTaskUseCase") as MockUC:
|
||||
|
||||
def _execute(cmd):
|
||||
captured_commands.append(cmd)
|
||||
return tasks[len(captured_commands) - 1]
|
||||
|
||||
MockUC.return_value.execute.side_effect = _execute
|
||||
with patch("app.api.routes.generation_preview.safe_enqueue_generation_task", return_value=True):
|
||||
create_preview_generation_task(
|
||||
_make_preview_request(
|
||||
preview_count=3,
|
||||
title_config={"font": "黑体", "position": "bottom"},
|
||||
titles=["标题A", "标题B", "标题C"],
|
||||
),
|
||||
authenticated_user=_make_user(),
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
assert len(captured_commands) == 3
|
||||
assert captured_commands[0].title_config["text"] == "标题A"
|
||||
assert captured_commands[1].title_config["text"] == "标题B"
|
||||
assert captured_commands[2].title_config["text"] == "标题C"
|
||||
# 样式全局共用
|
||||
assert all(c.title_config["font"] == "黑体" for c in captured_commands)
|
||||
|
||||
def test_preview_shared_title_when_single_length(self):
|
||||
"""titles 长度 1 = 所有变体共用同一标题"""
|
||||
from app.api.routes.generation_preview import create_preview_generation_task
|
||||
|
||||
tasks = [_make_task(task_id=f"task_{i}") for i in range(3)]
|
||||
repo = _repo_mock()
|
||||
captured = []
|
||||
with patch("app.api.routes.generation_preview.CreateGenerationTaskUseCase") as MockUC:
|
||||
|
||||
def _execute(cmd):
|
||||
captured.append(cmd)
|
||||
return tasks[len(captured) - 1]
|
||||
|
||||
MockUC.return_value.execute.side_effect = _execute
|
||||
with patch("app.api.routes.generation_preview.safe_enqueue_generation_task", return_value=True):
|
||||
create_preview_generation_task(
|
||||
_make_preview_request(preview_count=3, titles=["共用标题"]),
|
||||
authenticated_user=_make_user(),
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
assert all(c.title_config["text"] == "共用标题" for c in captured)
|
||||
|
||||
def test_preview_independent_voice_per_variant(self):
|
||||
"""voice_library_ids[] 按变体注入独立配音"""
|
||||
from app.api.routes.generation_preview import create_preview_generation_task
|
||||
|
||||
tasks = [_make_task(task_id=f"task_{i}") for i in range(3)]
|
||||
repo = _repo_mock()
|
||||
captured = []
|
||||
with patch("app.api.routes.generation_preview.CreateGenerationTaskUseCase") as MockUC:
|
||||
|
||||
def _execute(cmd):
|
||||
captured.append(cmd)
|
||||
return tasks[len(captured) - 1]
|
||||
|
||||
MockUC.return_value.execute.side_effect = _execute
|
||||
with patch("app.api.routes.generation_preview.safe_enqueue_generation_task", return_value=True):
|
||||
create_preview_generation_task(
|
||||
_make_preview_request(
|
||||
preview_count=3,
|
||||
voice_library_ids=["voice_a", "voice_b", "voice_c"],
|
||||
),
|
||||
authenticated_user=_make_user(),
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
assert [c.voice_library_id for c in captured] == ["voice_a", "voice_b", "voice_c"]
|
||||
|
||||
def test_preview_voice_fallback_to_single_field(self):
|
||||
"""voice_library_ids 为空时回退 voice_library_id 单值字段(向后兼容)"""
|
||||
from app.api.routes.generation_preview import create_preview_generation_task
|
||||
|
||||
task = _make_task(task_id="task_1")
|
||||
repo = _repo_mock()
|
||||
captured = []
|
||||
with patch("app.api.routes.generation_preview.CreateGenerationTaskUseCase") as MockUC:
|
||||
|
||||
def _execute(cmd):
|
||||
captured.append(cmd)
|
||||
return task
|
||||
|
||||
MockUC.return_value.execute.side_effect = _execute
|
||||
with patch("app.api.routes.generation_preview.safe_enqueue_generation_task", return_value=True):
|
||||
create_preview_generation_task(
|
||||
_make_preview_request(voice_library_id="legacy_voice"),
|
||||
authenticated_user=_make_user(),
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
assert captured[0].voice_library_id == "legacy_voice"
|
||||
|
||||
def test_preview_queue_limit_checks_total_count(self):
|
||||
"""限流预检查按变体总数计:用户 pending + N 超限 → 429"""
|
||||
from app.api.routes.generation_preview import create_preview_generation_task
|
||||
from fastapi import HTTPException
|
||||
|
||||
repo = MagicMock()
|
||||
repo.count_pending_by_user.return_value = 3
|
||||
repo.count_pending_total.return_value = 0
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
create_preview_generation_task(
|
||||
_make_preview_request(preview_count=5),
|
||||
authenticated_user=_make_user(),
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
assert exc.value.status_code == 429
|
||||
|
||||
def test_preview_clone_failure_marks_all_failed(self):
|
||||
"""克隆变体 plan 失败 → 已创建任务全部标记 failed 并 500"""
|
||||
from app.api.routes.generation_preview import create_preview_generation_task
|
||||
from fastapi import HTTPException
|
||||
|
||||
tasks = [_make_task(task_id=f"task_{i}", source_plan_id="source_plan") for i in range(3)]
|
||||
repo = _repo_mock()
|
||||
with patch("app.api.routes.generation_preview.CreateGenerationTaskUseCase") as MockUC:
|
||||
MockUC.return_value.execute.side_effect = tasks
|
||||
with patch("app.services.edit_plan_service.EditPlanService") as MockPlanSvc:
|
||||
MockPlanSvc.return_value.clone_plan_for_variant.side_effect = RuntimeError("db down")
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
create_preview_generation_task(
|
||||
_make_preview_request(preview_count=3),
|
||||
authenticated_user=_make_user(),
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
assert exc.value.status_code == 500
|
||||
# 所有已创建任务都被标记 failed
|
||||
assert all(t.status == GenerationTaskStatus.FAILED for t in tasks)
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# 批量正式生成:变体配置注入
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestBatchGenerationVariantConfig:
|
||||
"""POST /tasks count=N 时变体独立配置。"""
|
||||
|
||||
def _call_create_tasks(self, request, repo=None):
|
||||
from app.api.routes.generation_tasks import create_generation_task
|
||||
|
||||
repo = repo or MagicMock()
|
||||
repo.count_pending_by_user.return_value = 0
|
||||
repo.count_pending_total.return_value = 0
|
||||
repo.update.return_value = None
|
||||
|
||||
# 模板模式:asset_repository.find_by_id 返回 None(无 project 关联,
|
||||
# 纯模板模式 project_id/library_id 都为空),避免 MagicMock 属性污染
|
||||
asset_repo = MagicMock()
|
||||
asset_repo.find_by_id.return_value = None
|
||||
|
||||
# db.query().filter()...first() 返回 None:不走兜底关联编辑计划
|
||||
db = MagicMock()
|
||||
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = None
|
||||
|
||||
return create_generation_task(
|
||||
request,
|
||||
authenticated_user=_make_user(),
|
||||
generation_task_repository=repo,
|
||||
project_repository=MagicMock(),
|
||||
asset_library_repository=MagicMock(),
|
||||
asset_repository=asset_repo,
|
||||
db=db,
|
||||
)
|
||||
|
||||
def test_count_3_variant_titles_voices_covers_injected(self):
|
||||
"""count=3:titles/voice_library_ids/cover_urls 按变体注入"""
|
||||
from app.api.routes import generation_tasks as routes
|
||||
|
||||
tasks = [_make_task(task_id=f"gen_{i}") for i in range(3)]
|
||||
captured = []
|
||||
with patch.object(routes, "CreateGenerationTaskUseCase") as MockUC:
|
||||
|
||||
def _execute(cmd):
|
||||
captured.append(cmd)
|
||||
t = tasks[len(captured) - 1]
|
||||
t.title_config = cmd.title_config
|
||||
t.voice_library_id = cmd.voice_library_id
|
||||
t.cover_url = cmd.cover_url
|
||||
return t
|
||||
|
||||
MockUC.return_value.execute.side_effect = _execute
|
||||
with patch.object(routes, "safe_enqueue_generation_task", return_value=True):
|
||||
req = CreateGenerationTaskRequest(
|
||||
template_id="tpl_1",
|
||||
asset_ids=["a1"],
|
||||
count=3,
|
||||
title_config={"font": "宋体"},
|
||||
titles=["成片标题1", "成片标题2", "成片标题3"],
|
||||
voice_library_ids=["v1", "v2", "v3"],
|
||||
cover_urls=["http://c1", "http://c2", "http://c3"],
|
||||
)
|
||||
resp = self._call_create_tasks(req)
|
||||
assert resp.total == 3
|
||||
assert [c.title_config["text"] for c in captured] == ["成片标题1", "成片标题2", "成片标题3"]
|
||||
assert [c.voice_library_id for c in captured] == ["v1", "v2", "v3"]
|
||||
assert [c.cover_url for c in captured] == ["http://c1", "http://c2", "http://c3"]
|
||||
# 样式共用
|
||||
assert all(c.title_config["font"] == "宋体" for c in captured)
|
||||
|
||||
def test_count_1_legacy_fields_unchanged(self):
|
||||
"""N=1 不传数组:旧字段 voice_library_id/cover_url/title_config 行为不变"""
|
||||
from app.api.routes import generation_tasks as routes
|
||||
|
||||
task = _make_task(task_id="gen_1")
|
||||
task.is_preview = False
|
||||
captured = []
|
||||
with patch.object(routes, "CreateGenerationTaskUseCase") as MockUC:
|
||||
|
||||
def _execute(cmd):
|
||||
captured.append(cmd)
|
||||
return task
|
||||
|
||||
MockUC.return_value.execute.side_effect = _execute
|
||||
with patch.object(routes, "safe_enqueue_generation_task", return_value=True):
|
||||
req = CreateGenerationTaskRequest(
|
||||
template_id="tpl_1",
|
||||
asset_ids=["a1"],
|
||||
count=1,
|
||||
voice_library_id="legacy_voice",
|
||||
cover_url="http://legacy-cover",
|
||||
title_config={"text": "旧标题", "font": "黑体"},
|
||||
)
|
||||
resp = self._call_create_tasks(req)
|
||||
assert resp.total == 1
|
||||
assert captured[0].voice_library_id == "legacy_voice"
|
||||
assert captured[0].cover_url == "http://legacy-cover"
|
||||
assert captured[0].title_config["text"] == "旧标题"
|
||||
|
||||
def test_count_3_shared_single_value_arrays(self):
|
||||
"""数组长度 1:3 个变体共用同一配音/封面"""
|
||||
from app.api.routes import generation_tasks as routes
|
||||
|
||||
tasks = [_make_task(task_id=f"gen_{i}") for i in range(3)]
|
||||
captured = []
|
||||
with patch.object(routes, "CreateGenerationTaskUseCase") as MockUC:
|
||||
|
||||
def _execute(cmd):
|
||||
captured.append(cmd)
|
||||
return tasks[len(captured) - 1]
|
||||
|
||||
MockUC.return_value.execute.side_effect = _execute
|
||||
with patch.object(routes, "safe_enqueue_generation_task", return_value=True):
|
||||
req = CreateGenerationTaskRequest(
|
||||
template_id="tpl_1",
|
||||
asset_ids=["a1"],
|
||||
count=3,
|
||||
voice_library_ids=["shared_voice"],
|
||||
cover_urls=["http://shared"],
|
||||
)
|
||||
self._call_create_tasks(req)
|
||||
assert all(c.voice_library_id == "shared_voice" for c in captured)
|
||||
assert all(c.cover_url == "http://shared" for c in captured)
|
||||
|
||||
|
||||
class TestVariantValueHelper:
|
||||
"""_variant_value 取值逻辑。"""
|
||||
|
||||
def test_empty_returns_fallback(self):
|
||||
from app.api.routes.generation_preview import _variant_value
|
||||
|
||||
assert _variant_value([], 0, fallback="fb") == "fb"
|
||||
|
||||
def test_single_length_shared(self):
|
||||
from app.api.routes.generation_preview import _variant_value
|
||||
|
||||
assert _variant_value(["only"], 5) == "only"
|
||||
|
||||
def test_indexed_access(self):
|
||||
from app.api.routes.generation_preview import _variant_value
|
||||
|
||||
assert _variant_value(["a", "b", "c"], 1) == "b"
|
||||
|
||||
def test_index_out_of_range_fallback(self):
|
||||
from app.api.routes.generation_preview import _variant_value
|
||||
|
||||
assert _variant_value(["a", "b"], 9, fallback="x") == "x"
|
||||
@@ -65,11 +65,11 @@ class TestPositionToAssAlignment:
|
||||
def test_bottom(self):
|
||||
assert position_to_ass_alignment("bottom") == 2
|
||||
|
||||
def test_unknown_default_top(self):
|
||||
assert position_to_ass_alignment("unknown") == 8
|
||||
def test_unknown_default_bottom(self):
|
||||
assert position_to_ass_alignment("unknown") == 2
|
||||
|
||||
def test_empty_default_top(self):
|
||||
assert position_to_ass_alignment("") == 8
|
||||
def test_empty_default_bottom(self):
|
||||
assert position_to_ass_alignment("") == 2
|
||||
|
||||
|
||||
# ── Style 行构建 ─────────────────────────────────────────────────────────────
|
||||
@@ -747,3 +747,45 @@ class TestTitleFreePosition:
|
||||
line for line in content.splitlines() if line.startswith("Dialogue:") and "SubtitleStyle" in line
|
||||
][0]
|
||||
assert "\\pos(" not in sub_dialogue
|
||||
|
||||
|
||||
class TestDefaultPositionBottom:
|
||||
"""默认 position 应为 bottom(alignment=2),与前端 DEFAULT_TITLE_SETTINGS 对齐。"""
|
||||
|
||||
def _base_kwargs(self):
|
||||
return dict(
|
||||
video_width=1080,
|
||||
video_height=1920,
|
||||
video_duration=10.0,
|
||||
title_text="测试标题",
|
||||
)
|
||||
|
||||
def test_no_position_defaults_to_bottom_alignment(self):
|
||||
"""不传 position 时,Alignment 应为 2(bottom)。"""
|
||||
content = build_ass_content(
|
||||
**self._base_kwargs(),
|
||||
title_config={"size": 36},
|
||||
)
|
||||
style_line = [line for line in content.splitlines() if line.startswith("Style: TitleStyle")][0]
|
||||
fields = [f.strip() for f in style_line.split(",")]
|
||||
assert fields[18] == "2", f"Expected alignment 2 (bottom), got {fields[18]}"
|
||||
|
||||
def test_no_position_no_coords_defaults_to_bottom(self):
|
||||
"""不传 position 也不传坐标时,走 bottom 三档逻辑。"""
|
||||
content = build_ass_content(
|
||||
**self._base_kwargs(),
|
||||
title_config={},
|
||||
)
|
||||
style_line = [line for line in content.splitlines() if line.startswith("Style: TitleStyle")][0]
|
||||
fields = [f.strip() for f in style_line.split(",")]
|
||||
assert fields[18] == "2"
|
||||
|
||||
def test_explicit_top_still_works(self):
|
||||
"""显式传 position='top' 仍然得到 alignment=8。"""
|
||||
content = build_ass_content(
|
||||
**self._base_kwargs(),
|
||||
title_config={"position": "top", "size": 36},
|
||||
)
|
||||
style_line = [line for line in content.splitlines() if line.startswith("Style: TitleStyle")][0]
|
||||
fields = [f.strip() for f in style_line.split(",")]
|
||||
assert fields[18] == "8"
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
"""Tests for bad fingerprint (black screen / uniform color) filtering.
|
||||
|
||||
Issue: 1秒黑屏视频(所有帧phash几乎相同)与任何视频的距离都~30,造成虚假匹配。
|
||||
Fix: _is_bad_fingerprint() 检测并跳过这类低质量指纹。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock heavy deps before importing dedup module (same pattern as test_dedup_engine.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
_ORIGINAL_MODULES = dict(sys.modules)
|
||||
_MOCKED_MODULE_NAMES: list[str] = []
|
||||
|
||||
|
||||
def _mock_if_absent(name: str, mock_obj=None):
|
||||
if name not in sys.modules:
|
||||
sys.modules[name] = mock_obj if mock_obj is not None else MagicMock()
|
||||
_MOCKED_MODULE_NAMES.append(name)
|
||||
|
||||
|
||||
_mock_if_absent("ffmpeg")
|
||||
for mod_name in ["worker_app", "worker_app.celery_app", "worker_app.db"]:
|
||||
_mock_if_absent(mod_name)
|
||||
if "worker_app.celery_app" in sys.modules and isinstance(sys.modules["worker_app.celery_app"], MagicMock):
|
||||
sys.modules["worker_app.celery_app"].celery_app = MagicMock()
|
||||
if "worker_app.db" in sys.modules and isinstance(sys.modules["worker_app.db"], MagicMock):
|
||||
sys.modules["worker_app.db"].SessionLocal = MagicMock()
|
||||
_mock_if_absent("celery", MagicMock())
|
||||
if "celery" in sys.modules and isinstance(sys.modules["celery"], MagicMock):
|
||||
sys.modules["celery"].Task = object
|
||||
_mock_if_absent("packages.shared.storage")
|
||||
_mock_if_absent("packages.adapters.sqlalchemy_impl.generated_video_repository")
|
||||
|
||||
_HAS_CV2 = False
|
||||
try:
|
||||
import cv2 as _cv2
|
||||
|
||||
if not isinstance(_cv2, MagicMock):
|
||||
_HAS_CV2 = True
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
|
||||
if not _HAS_CV2:
|
||||
_mock_if_absent("cv2")
|
||||
|
||||
import numpy as np # noqa: E402
|
||||
|
||||
from apps.worker.video_processing.dedup import ( # noqa: E402
|
||||
VideoDeduplicator,
|
||||
VideoFingerprint,
|
||||
)
|
||||
|
||||
# Restore mocked modules
|
||||
for _name in ["worker_app", "worker_app.celery_app", "worker_app.db", "celery"]:
|
||||
if _name in _MOCKED_MODULE_NAMES:
|
||||
sys.modules.pop(_name, None)
|
||||
_MOCKED_MODULE_NAMES.remove(_name)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="session")
|
||||
def _cleanup_mocks():
|
||||
yield
|
||||
for name in _MOCKED_MODULE_NAMES:
|
||||
sys.modules.pop(name, None)
|
||||
|
||||
|
||||
# ── _is_bad_fingerprint 单元测试 ─────────────────────────────────
|
||||
|
||||
|
||||
class TestIsBadFingerprint:
|
||||
"""VideoDeduplicator._is_bad_fingerprint() 静态方法测试。"""
|
||||
|
||||
def test_empty_phashes_is_bad(self):
|
||||
"""空 phash 列表视为坏指纹。"""
|
||||
assert VideoDeduplicator._is_bad_fingerprint([]) is True
|
||||
|
||||
def test_single_phash_is_not_bad(self):
|
||||
"""单帧视频不视为坏指纹(短视频或抽帧不足)。"""
|
||||
assert VideoDeduplicator._is_bad_fingerprint(["abcdef0123456789"]) is False
|
||||
|
||||
def test_all_identical_phashes_is_bad(self):
|
||||
""">=8 帧且所有 phash 完全相同 → 黑屏/纯色视频(#1702:短帧不误杀)。"""
|
||||
phashes = ["aaaaaaaaaaaaaaaa"] * 10
|
||||
assert VideoDeduplicator._is_bad_fingerprint(phashes) is True
|
||||
|
||||
def test_short_identical_phashes_not_bad(self):
|
||||
"""<8 帧完全相同不判坏——短视频内容连续时相邻采样帧 phash 天然相同(#1702)。"""
|
||||
assert VideoDeduplicator._is_bad_fingerprint(["bbbbbbbbbbbbbbbb"] * 5) is False
|
||||
assert VideoDeduplicator._is_bad_fingerprint(["bbbbbbbbbbbbbbbb", "bbbbbbbbbbbbbbbb"]) is False
|
||||
|
||||
def test_all_very_similar_phashes_is_bad(self):
|
||||
""">=8 帧 phash 之间的汉明距离都 < 3 且高占比 → 近似黑屏。"""
|
||||
phashes = ["0000000000000000"] * 8 + ["0000000000000001", "0000000000000002"]
|
||||
assert VideoDeduplicator._is_bad_fingerprint(phashes) is True
|
||||
|
||||
def test_diverse_phashes_is_good(self):
|
||||
"""多样化的 phash 列表是有效指纹。"""
|
||||
phashes = [
|
||||
"abcdef0123456789",
|
||||
"1234567890abcdef",
|
||||
"fedcba9876543210",
|
||||
"0123456789abcdef",
|
||||
]
|
||||
assert VideoDeduplicator._is_bad_fingerprint(phashes) is False
|
||||
|
||||
def test_mixed_similar_and_different_is_good(self):
|
||||
"""有些 phash 相似但有足够多样的 → 有效指纹。"""
|
||||
phashes = [
|
||||
"0000000000000000",
|
||||
"0000000000000001",
|
||||
"0000000000000002",
|
||||
"ffffffffffffffff",
|
||||
]
|
||||
assert VideoDeduplicator._is_bad_fingerprint(phashes) is False
|
||||
|
||||
def test_known_black_screen_phashes(self):
|
||||
"""已知黑屏视频的 phash 特征(全零或均匀分布)。"""
|
||||
assert VideoDeduplicator._is_bad_fingerprint(["0000000000000000"] * 10) is True
|
||||
assert VideoDeduplicator._is_bad_fingerprint(["ffffffffffffffff"] * 8) is True
|
||||
assert VideoDeduplicator._is_bad_fingerprint(["9999999999999966"] * 8) is True
|
||||
# <8 帧不判坏(#1702 短视频保护)
|
||||
assert VideoDeduplicator._is_bad_fingerprint(["9999999999999966"] * 5) is False
|
||||
|
||||
|
||||
# ── Helper ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_existing_video(video_id, md5, phashes):
|
||||
"""创建 mock 视频记录。"""
|
||||
video = MagicMock()
|
||||
video.id = video_id
|
||||
video.video_fingerprint = {
|
||||
"md5": md5,
|
||||
"keyframe_phashes": phashes,
|
||||
"color_histograms": [],
|
||||
}
|
||||
return video
|
||||
|
||||
|
||||
# ── check_duplicate 集成测试 ────────────────────────────────────
|
||||
|
||||
|
||||
class TestCheckDuplicateBadFingerprint:
|
||||
"""check_duplicate 跳过坏指纹视频。"""
|
||||
|
||||
def test_black_screen_existing_video_skipped(self):
|
||||
"""已有视频是黑屏指纹 → 被跳过,不匹配。"""
|
||||
deduplicator = VideoDeduplicator()
|
||||
mock_session = MagicMock()
|
||||
|
||||
black_screen = _make_existing_video("vid-black", "md5_black", ["aaaaaaaaaaaaaaaa"] * 10)
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.list_by_user.return_value = [black_screen]
|
||||
|
||||
fingerprint = VideoFingerprint(
|
||||
md5="md5_normal",
|
||||
keyframe_phashes=["aaaaaaaaaaaaaaaa"] * 10,
|
||||
color_histograms=[],
|
||||
duration=10.0,
|
||||
resolution=(1280, 720),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"apps.worker.video_processing.dedup.SQLAlchemyGeneratedVideoRepository",
|
||||
return_value=mock_repo,
|
||||
):
|
||||
result = deduplicator.check_duplicate(fingerprint, "proj-1", mock_session, scope="user", user_id="user-1")
|
||||
|
||||
assert result is None
|
||||
|
||||
def test_normal_existing_video_not_skipped(self):
|
||||
"""正常视频不会被坏指纹过滤跳过。"""
|
||||
deduplicator = VideoDeduplicator()
|
||||
mock_session = MagicMock()
|
||||
|
||||
normal = _make_existing_video(
|
||||
"vid-normal",
|
||||
"md5_normal_existing",
|
||||
["abcdef0123456789", "1234567890abcdef", "fedcba9876543210"],
|
||||
)
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.list_by_user.return_value = [normal]
|
||||
|
||||
fingerprint = VideoFingerprint(
|
||||
md5="md5_normal_new",
|
||||
keyframe_phashes=["abcdef0123456789", "1234567890abcdef", "fedcba9876543210"],
|
||||
color_histograms=[],
|
||||
duration=10.0,
|
||||
resolution=(1280, 720),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"apps.worker.video_processing.dedup.SQLAlchemyGeneratedVideoRepository",
|
||||
return_value=mock_repo,
|
||||
):
|
||||
result = deduplicator.check_duplicate(fingerprint, "proj-1", mock_session, scope="user", user_id="user-1")
|
||||
|
||||
assert result is not None
|
||||
assert result["duplicate"] is True
|
||||
|
||||
def test_md5_match_overrides_bad_fingerprint(self):
|
||||
"""MD5 精确匹配优先于坏指纹过滤。"""
|
||||
deduplicator = VideoDeduplicator()
|
||||
mock_session = MagicMock()
|
||||
|
||||
black_screen = _make_existing_video("vid-black", "same_md5", ["aaaaaaaaaaaaaaaa"] * 10)
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.list_by_user.return_value = [black_screen]
|
||||
|
||||
fingerprint = VideoFingerprint(
|
||||
md5="same_md5",
|
||||
keyframe_phashes=["bbbbbbbbbbbbbbbb"] * 3,
|
||||
color_histograms=[],
|
||||
duration=10.0,
|
||||
resolution=(1280, 720),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"apps.worker.video_processing.dedup.SQLAlchemyGeneratedVideoRepository",
|
||||
return_value=mock_repo,
|
||||
):
|
||||
result = deduplicator.check_duplicate(fingerprint, "proj-1", mock_session, scope="user", user_id="user-1")
|
||||
|
||||
assert result is not None
|
||||
assert result["reason"] == "exact_md5_match"
|
||||
|
||||
|
||||
# ── compute_duplicate_rate 集成测试 ─────────────────────────────
|
||||
|
||||
|
||||
class TestComputeDuplicateRateBadFingerprint:
|
||||
"""compute_duplicate_rate 跳过坏指纹视频。"""
|
||||
|
||||
def test_black_screen_video_excluded_from_rate(self):
|
||||
"""黑屏视频不参与查重率计算。"""
|
||||
deduplicator = VideoDeduplicator()
|
||||
mock_session = MagicMock()
|
||||
|
||||
videos = [
|
||||
_make_existing_video("vid-b1", "md5_b1", ["cccccccccccccccc"] * 5),
|
||||
_make_existing_video("vid-b2", "md5_b2", ["dddddddddddddddd"] * 5),
|
||||
_make_existing_video("vid-b3", "md5_b3", ["eeeeeeeeeeeeeeee"] * 5),
|
||||
_make_existing_video(
|
||||
"vid-normal",
|
||||
"md5_n",
|
||||
["abcdef0123456789", "1234567890abcdef", "fedcba9876543210"],
|
||||
),
|
||||
]
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.list_by_user.return_value = videos
|
||||
|
||||
fingerprint = VideoFingerprint(
|
||||
md5="md5_new",
|
||||
keyframe_phashes=["abcdef0123456789", "1234567890abcdef", "fedcba9876543210"],
|
||||
color_histograms=[],
|
||||
duration=10.0,
|
||||
resolution=(1280, 720),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"apps.worker.video_processing.dedup.SQLAlchemyGeneratedVideoRepository",
|
||||
return_value=mock_repo,
|
||||
):
|
||||
result = deduplicator.compute_duplicate_rate(
|
||||
fingerprint,
|
||||
"proj-1",
|
||||
"vid-new",
|
||||
mock_session,
|
||||
scope="user",
|
||||
user_id="user-1",
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert isinstance(result["duplicate_rate"], float)
|
||||
assert isinstance(result["match_count"], int)
|
||||
|
||||
def test_only_black_screen_videos_zero_rate(self):
|
||||
"""所有已有视频都是黑屏 → 查重率为 0。"""
|
||||
deduplicator = VideoDeduplicator()
|
||||
mock_session = MagicMock()
|
||||
|
||||
videos = [
|
||||
_make_existing_video("vid-b1", "md5_b1", ["aaaaaaaaaaaaaaaa"] * 10),
|
||||
_make_existing_video("vid-b2", "md5_b2", ["bbbbbbbbbbbbbbbb"] * 5),
|
||||
]
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.list_by_user.return_value = videos
|
||||
|
||||
fingerprint = VideoFingerprint(
|
||||
md5="md5_new",
|
||||
keyframe_phashes=["aaaaaaaaaaaaaaaa"] * 5,
|
||||
color_histograms=[],
|
||||
duration=10.0,
|
||||
resolution=(1280, 720),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"apps.worker.video_processing.dedup.SQLAlchemyGeneratedVideoRepository",
|
||||
return_value=mock_repo,
|
||||
):
|
||||
result = deduplicator.compute_duplicate_rate(
|
||||
fingerprint,
|
||||
"proj-1",
|
||||
"vid-new",
|
||||
mock_session,
|
||||
scope="user",
|
||||
user_id="user-1",
|
||||
)
|
||||
|
||||
assert result["duplicate_rate"] == 0.0
|
||||
assert result["match_count"] == 0
|
||||
@@ -0,0 +1,389 @@
|
||||
"""Issue #1702 — 查重率恒为 0% 修复:单测.
|
||||
|
||||
覆盖验收要求:
|
||||
1. 同源不同裁剪的两个视频能检出非 0 相似度(指纹中心裁剪绕开降重 + 阈值校准)
|
||||
2. 局部片段复用(B 结尾 2s ≈ A 中间 2s)能检出
|
||||
3. 异源视频不误报(相似度接近 0)
|
||||
4. N=1 现有流程不回归
|
||||
5. P1 确定性 bug:时长预过滤单位 /1000、直方图归一化、temporal_coverage 量纲、阈值比较统一
|
||||
6. P0:±1 邻接对齐、短视频自适应连续门槛
|
||||
7. P2:0 匹配也要落日志
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
sys.modules.setdefault("cv2", MagicMock())
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "apps" / "worker"))
|
||||
sys.path.insert(0, str(ROOT / "packages"))
|
||||
|
||||
|
||||
from video_processing.dedup import ( # noqa: E402
|
||||
PHASH_THRESHOLD,
|
||||
SEGMENT_MATCH_THRESHOLD,
|
||||
FingerprintChunk,
|
||||
VideoDeduplicator,
|
||||
VideoFingerprint,
|
||||
find_duplicate_segments,
|
||||
)
|
||||
|
||||
# ── helpers ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _h(d: int) -> str:
|
||||
"""64-bit phash with exactly d bits set vs zero hash."""
|
||||
bits = ["0"] * 64
|
||||
for i in range(d):
|
||||
bits[i] = "1"
|
||||
return f"{int(''.join(bits), 2):016x}"
|
||||
|
||||
|
||||
def _chunk(phash: str, t0: float, t1: float):
|
||||
|
||||
return FingerprintChunk(
|
||||
start_time_ms=int(t0 * 1000),
|
||||
end_time_ms=int(t1 * 1000),
|
||||
phash_binary=phash,
|
||||
color_histogram=[],
|
||||
frame_count=1,
|
||||
)
|
||||
|
||||
|
||||
def _fingerprint(phashes, duration, chunks=None, md5="fp-md5-x"):
|
||||
|
||||
return VideoFingerprint(
|
||||
md5=md5,
|
||||
keyframe_phashes=list(phashes),
|
||||
color_histograms=[],
|
||||
duration=duration,
|
||||
resolution=(1280, 720),
|
||||
chunks=chunks or [],
|
||||
)
|
||||
|
||||
|
||||
def _video(vid, phashes, duration=10.0, project_id="proj1"):
|
||||
from packages.domain import GeneratedVideo
|
||||
|
||||
return GeneratedVideo(
|
||||
id=vid,
|
||||
project_id=project_id,
|
||||
generation_task_id=f"task-{vid}",
|
||||
name=f"video-{vid}.mp4",
|
||||
file_url=f"https://example.com/{vid}.mp4",
|
||||
file_size=1000,
|
||||
duration=duration,
|
||||
width=1280,
|
||||
height=720,
|
||||
fps=25.0,
|
||||
video_fingerprint={"md5": f"md5-{vid}", "keyframe_phashes": list(phashes)},
|
||||
)
|
||||
|
||||
|
||||
def _rate(deduplicator, fp, videos, session=None):
|
||||
session_magic = MagicMock()
|
||||
# 分片表无数据 -> 回退 JSON keyframe_phashes
|
||||
session_magic.query.return_value.filter.return_value.order_by.return_value.all.return_value = []
|
||||
with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo:
|
||||
repo = MockRepo.return_value
|
||||
repo.list_by_project.return_value = videos
|
||||
repo.list_by_user.return_value = videos
|
||||
return deduplicator.compute_duplicate_rate(fp, "proj1", "new-vid", session_magic, scope="project")
|
||||
|
||||
|
||||
def _check(deduplicator, fp, videos, scope="project", **kw):
|
||||
session_magic = MagicMock()
|
||||
session_magic.query.return_value.filter.return_value.order_by.return_value.all.return_value = []
|
||||
with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo:
|
||||
repo = MockRepo.return_value
|
||||
repo.list_by_project.return_value = videos
|
||||
repo.list_by_user.return_value = videos
|
||||
return deduplicator.check_duplicate(fp, "proj1", session_magic, scope=scope, **kw)
|
||||
|
||||
|
||||
# ── P0-1/P0-2: 同源不同裁剪(距离 6~10)检出非 0 ──────────────
|
||||
|
||||
|
||||
class TestSameSourceDifferentCrop:
|
||||
"""同源成片:random_edge_crop 后 pHash 距离 6~10,应检出非 0 相似度。"""
|
||||
|
||||
def test_same_source_high_similarity_detected(self):
|
||||
|
||||
ddp = VideoDeduplicator()
|
||||
# 新视频 5 个分片,每个 phash 与已有视频对应分片距离 6(< 阈值)
|
||||
base = [_h(0) for _ in range(5)]
|
||||
new = [_h(6) for _ in range(5)]
|
||||
existing = _video("v-old", base, duration=11.0)
|
||||
chunks = [_chunk(h, i * 2.2, (i + 1) * 2.2) for i, h in enumerate(new)]
|
||||
fp = _fingerprint(new, 11.0, chunks=chunks)
|
||||
|
||||
result = _rate(ddp, fp, [existing], MagicMock())
|
||||
assert result["duplicate_rate"] > 0
|
||||
assert result["visual_similarity"] > 0
|
||||
|
||||
def test_same_source_distance_at_threshold_still_detected(self):
|
||||
"""距离正好等于阈值(<=)也要算匹配——阈值比较统一为 <=。"""
|
||||
|
||||
assert PHASH_THRESHOLD <= 12, "阈值应经校准保持在能检出同源裁剪的范围"
|
||||
ddp = VideoDeduplicator()
|
||||
base = [_h(0) for _ in range(6)]
|
||||
new = [_h(PHASH_THRESHOLD) for _ in range(6)]
|
||||
existing = _video("v-old", base, duration=12.0)
|
||||
chunks = [_chunk(h, i * 2, (i + 1) * 2) for i, h in enumerate(new)]
|
||||
fp = _fingerprint(new, 12.0, chunks=chunks)
|
||||
|
||||
result = _rate(ddp, fp, [existing], MagicMock())
|
||||
assert result["duplicate_rate"] > 0
|
||||
|
||||
|
||||
# ── P0-2: 局部片段复用(B 结尾 2s ≈ A 中间 2s) ────────────────
|
||||
|
||||
|
||||
class TestPartialReuse:
|
||||
def test_partial_reuse_tail_overlap_detected(self):
|
||||
"""新视频 6 片,最后 2 片命中已有视频中间 2 片(距离 4),其余不匹配。
|
||||
|
||||
旧逻辑 frame_match_rate=2/6≈0.33(<0.3 硬跳过边界)+ MIN_CONSECUTIVE=5
|
||||
导致完全检不出;新逻辑 coverage 为主指标 + 自适应门槛应检出。
|
||||
"""
|
||||
|
||||
ddp = VideoDeduplicator()
|
||||
# 已有 8 片:索引 3、4 是被复用的镜头
|
||||
old = [_h(20 + i) for i in range(8)]
|
||||
# 新视频 6 片:最后 2 片对应 old[3], old[4],距离 4;其余距离 30
|
||||
new = [_h(50 + i) for i in range(4)] + [_h(4)] * 2
|
||||
# 让 new[4] 与 old[3] 距离 4、new[5] 与 old[4] 距离 4(构造近似)
|
||||
new[4] = f"{int('1' * 4 + '0' * 60, 2):016x}"
|
||||
new[5] = f"{int('1' * 4 + '0' * 60, 2):016x}"
|
||||
old[3] = _h(0)
|
||||
old[4] = _h(0)
|
||||
|
||||
existing = _video("v-old", old, duration=16.0)
|
||||
chunks = [_chunk(h, i * 2, (i + 1) * 2) for i, h in enumerate(new)]
|
||||
fp = _fingerprint(new, 12.0, chunks=chunks)
|
||||
|
||||
result = _rate(ddp, fp, [existing], MagicMock())
|
||||
# 局部复用:duplicate_rate 必须非 0
|
||||
assert result["duplicate_rate"] > 0
|
||||
|
||||
def test_short_video_adaptive_consecutive_threshold(self):
|
||||
"""11s/5 片短视频:MIN_CONSECUTIVE 自适应 min(5, max(2, 5//2))=2,
|
||||
2 片连续命中即报片段(旧值 5 让短视频永远无法报片段)。"""
|
||||
|
||||
q = [
|
||||
FingerprintChunk(0, 2000, "f" * 16, []),
|
||||
FingerprintChunk(2000, 4000, "0" * 16, []),
|
||||
FingerprintChunk(4000, 6000, f"{int('11110000', 2):016x}", []),
|
||||
]
|
||||
t = [
|
||||
FingerprintChunk(0, 2000, "f" * 16, []),
|
||||
FingerprintChunk(2000, 4000, "0" * 16, []),
|
||||
FingerprintChunk(4000, 6000, "e" * 16, []),
|
||||
]
|
||||
# 3 片视频自适应门槛 = min(5, max(2, 3//2)) = 2
|
||||
segs = find_duplicate_segments(q, t)
|
||||
assert len(segs) >= 1
|
||||
|
||||
|
||||
# ── P0-3: ±1 邻接窗口对齐 ─────────────────────────────────────
|
||||
|
||||
|
||||
class TestNeighborAlignment:
|
||||
def test_neighbor_window_absorbs_boundary_jitter(self):
|
||||
"""切点错位导致目标索引偏移 ±1 时,连续匹配不应被中断。"""
|
||||
|
||||
q = [FingerprintChunk(i * 1000, (i + 1) * 1000, f"{i:016x}", []) for i in range(4)]
|
||||
# 目标:前 3 片与 q 相同,但第 3 片最佳匹配偏移 +1(t[4]),t[3] 是无关内容
|
||||
t_hashes = [f"{i:016x}" for i in range(3)] + ["f" * 16, f"{3:016x}"]
|
||||
t = [FingerprintChunk(i * 1000, (i + 1) * 1000, h, []) for i, h in enumerate(t_hashes)]
|
||||
segs = find_duplicate_segments(q, t)
|
||||
# q[0],q[1] 精确匹配 t[0],t[1];q[2]->t[2];q[3]->t[4](步进 2,窗口 ±1 内)
|
||||
assert len(segs) >= 1
|
||||
assert segs[0].query_end_ms >= 3000
|
||||
|
||||
|
||||
# ── P0-5 / 验收:异源不误报 ───────────────────────────────────
|
||||
|
||||
|
||||
class TestDifferentSourceNoFalsePositive:
|
||||
def test_unrelated_videos_near_zero(self):
|
||||
|
||||
ddp = VideoDeduplicator()
|
||||
# 异源:所有分片距离 >= 20
|
||||
old = [_h(40 + i * 3 % 20) for i in range(6)]
|
||||
new = [_h(0 + i) for i in range(6)]
|
||||
existing = _video("v-old", old, duration=12.0)
|
||||
chunks = [_chunk(h, i * 2, (i + 1) * 2) for i, h in enumerate(new)]
|
||||
fp = _fingerprint(new, 12.0, chunks=chunks)
|
||||
|
||||
result = _rate(ddp, fp, [existing], MagicMock())
|
||||
assert result["duplicate_rate"] == 0
|
||||
assert result["visual_similarity"] < 0.7
|
||||
assert result["match_count"] == 0
|
||||
|
||||
def test_check_duplicate_returns_none_for_unrelated(self):
|
||||
|
||||
ddp = VideoDeduplicator()
|
||||
old = [_h(40 + i) for i in range(6)]
|
||||
new = [_h(i) for i in range(6)]
|
||||
existing = _video("v-old", old, duration=12.0)
|
||||
fp = _fingerprint(new, 12.0)
|
||||
|
||||
result = _check(ddp, fp, [existing])
|
||||
assert result is None
|
||||
|
||||
|
||||
# ── N=1 不回归 ────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestSingleChunkNoRegression:
|
||||
def test_single_chunk_identical_detected(self):
|
||||
|
||||
ddp = VideoDeduplicator()
|
||||
h = _h(2)
|
||||
existing = _video("v-old", [h], duration=3.0)
|
||||
chunks = [_chunk(h, 0, 3000)]
|
||||
fp = _fingerprint([h], 3.0, chunks=chunks)
|
||||
result = _rate(ddp, fp, [existing], MagicMock())
|
||||
assert result["duplicate_rate"] > 0
|
||||
|
||||
def test_single_chunk_md5_exact_match(self):
|
||||
|
||||
ddp = VideoDeduplicator()
|
||||
existing = _video("v-old", [_h(0)], duration=3.0)
|
||||
existing.video_fingerprint["md5"] = "same"
|
||||
fp = _fingerprint([_h(0)], 3.0, md5="same")
|
||||
result = _check(ddp, fp, [existing])
|
||||
assert result is not None
|
||||
assert result["reason"] == "exact_md5_match"
|
||||
|
||||
|
||||
# ── P1-6: 时长预过滤单位 bug ──────────────────────────────────
|
||||
|
||||
|
||||
class TestDurationPrefilterUnit:
|
||||
def test_duration_sec_not_divided_by_1000(self):
|
||||
"""fingerprint.duration 单位是秒,传给 check_duplicate 不应再 /1000。
|
||||
|
||||
旧 bug:duration/1000 → duration_max≈0.0135s,所有真实视频被过滤。
|
||||
"""
|
||||
|
||||
ddp = VideoDeduplicator()
|
||||
fp = _fingerprint([_h(0)], 13.5)
|
||||
session_magic = MagicMock()
|
||||
session_magic.query.return_value.filter.return_value.order_by.return_value.all.return_value = []
|
||||
with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo:
|
||||
repo = MockRepo.return_value
|
||||
repo.list_by_user.return_value = []
|
||||
ddp.check_duplicate(fp, "proj1", session_magic, scope="user", user_id="u1", duration_sec=fp.duration)
|
||||
_, kwargs = repo.list_by_user.call_args
|
||||
# ±15% 窗口:13.5s -> [11.475, 15.525]
|
||||
assert 11.0 < kwargs["duration_min"] < 12.0
|
||||
assert 15.0 < kwargs["duration_max"] < 16.0
|
||||
|
||||
|
||||
# ── P1-7: 颜色直方图归一化 ────────────────────────────────────
|
||||
|
||||
|
||||
class TestHistogramNormalization:
|
||||
def test_bhattacharyya_coefficient_in_unit_range(self):
|
||||
"""Bhattacharyya 系数必须在 [0,1](旧 L2 + 3 通道拼接算出 ~14.9)。"""
|
||||
|
||||
# 3 通道拼接、每通道概率分布(Σ=1)
|
||||
hist_a = [0.5, 0.5] + [0.0] * 94 + [0.5, 0.5] + [0.0] * 94 + [0.5, 0.5] + [0.0] * 94
|
||||
# 长度裁剪到 96(3 通道 × 32 bins)
|
||||
hist_a = ([0.5, 0.5] + [0.0] * 30) * 3
|
||||
hist_b = ([0.5, 0.5] + [0.0] * 30) * 3
|
||||
|
||||
coeff = VideoDeduplicator._bhattacharyya_coefficient(hist_a, hist_b)
|
||||
assert 0.0 <= coeff <= 1.0
|
||||
assert coeff > 0.99 # 完全相同 -> 1.0
|
||||
|
||||
def test_bhattacharyya_disjoint_hist_low(self):
|
||||
|
||||
hist_a = ([1.0] + [0.0] * 31) * 3
|
||||
hist_b = ([0.0] * 31 + [1.0]) * 3
|
||||
coeff = VideoDeduplicator._bhattacharyya_coefficient(hist_a, hist_b)
|
||||
assert coeff < 0.05
|
||||
|
||||
|
||||
# ── P1-8: temporal_coverage 量纲 ──────────────────────────────
|
||||
|
||||
|
||||
class TestTemporalCoverageUnits:
|
||||
def test_coverage_uses_milliseconds(self):
|
||||
"""命中片段 6s / 视频 12s -> coverage=0.5;旧 bug 把 duration(秒)当毫秒,
|
||||
covered_ms(6000)/duration(12) = 500 -> min(1.0)=1.0 误判 100% 覆盖。"""
|
||||
|
||||
ddp = VideoDeduplicator()
|
||||
old = [_h(0) for _ in range(6)]
|
||||
new = [_h(0) for _ in range(3)] + [_h(30) for _ in range(3)]
|
||||
existing = _video("v-old", old, duration=12.0)
|
||||
# 新视频 12s,前 6s(3 片)与 old 相同
|
||||
chunks = [_chunk(h, i * 2, (i + 1) * 2) for i, h in enumerate(new)]
|
||||
fp = _fingerprint(new, 12.0, chunks=chunks)
|
||||
result = _rate(ddp, fp, [existing], MagicMock())
|
||||
# coverage 应约 0.5(3 片 × 2s = 6s / 12s),duplicate_rate ≈ (0.5*0.4 + 0.5*0.6)*100 = 50
|
||||
assert 30 < result["duplicate_rate"] < 70
|
||||
|
||||
|
||||
# ── P1-9: 阈值比较统一 ────────────────────────────────────────
|
||||
|
||||
|
||||
class TestThresholdConsistency:
|
||||
def test_frame_and_segment_thresholds_same_source(self):
|
||||
|
||||
assert SEGMENT_MATCH_THRESHOLD == PHASH_THRESHOLD
|
||||
assert VideoDeduplicator.PHASH_THRESHOLD == PHASH_THRESHOLD
|
||||
|
||||
|
||||
# ── P2: 0 匹配也要有日志痕迹 ──────────────────────────────────
|
||||
|
||||
|
||||
class TestZeroMatchLogging:
|
||||
def test_no_match_emits_info_log(self, caplog):
|
||||
|
||||
ddp = VideoDeduplicator()
|
||||
old = [_h(40 + i) for i in range(5)]
|
||||
existing = _video("v-old", old, duration=10.0)
|
||||
fp = _fingerprint([_h(i) for i in range(5)], 10.0)
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="video_processing.dedup"):
|
||||
result = _check(ddp, fp, [existing])
|
||||
assert result is None
|
||||
assert any("no match" in r.message for r in caplog.records)
|
||||
|
||||
|
||||
# ── recompute 任务下载路径(#1702 连带修复:旧硬编码 key 404) ─────
|
||||
|
||||
|
||||
class TestRecomputeDownloadPath:
|
||||
"""recompute-dedup 走 check_duplicate_task,需要从 OSS 重新下载成片。
|
||||
|
||||
旧代码硬编码 projects/{pid}/generated/{vid}/{vid}.mp4(从不存在),
|
||||
真实 key 在 file_url:generated/projects/{pid}/tasks/{tid}/rendered_*.mp4。
|
||||
"""
|
||||
|
||||
def test_task_downloads_from_file_url(self):
|
||||
import inspect
|
||||
|
||||
import video_processing.dedup as dedup_mod
|
||||
|
||||
source = inspect.getsource(dedup_mod.check_duplicate_task)
|
||||
# 下载 key 必须来自 video.file_url
|
||||
assert 'getattr(video, "file_url"' in source or "video.file_url" in source
|
||||
# 旧的硬编码 key 只能作为回退存在,不能是主路径
|
||||
assert "falling back to legacy key" in source
|
||||
# download_file 接收的是派生 key 而非硬编码 f-string
|
||||
assert "storage_service.download_file(download_key" in source
|
||||
assert '/generated/{generated_video_id}/{generated_video_id}.mp4"' not in source.replace(
|
||||
'download_key = f"projects/{video.project_id}/generated/{generated_video_id}/{generated_video_id}.mp4"',
|
||||
"",
|
||||
)
|
||||
@@ -358,11 +358,11 @@ class TestVideoDeduplicatorCheckDuplicate:
|
||||
finally:
|
||||
self._restore_repo(mod, orig)
|
||||
|
||||
def test_first_match_returned(self, deduplicator, mock_session):
|
||||
"""返回第一个通过阈值的匹配(非最优匹配)。"""
|
||||
# vid-1: 距离=2 bits(0x03 XOR 0x01 = 0x02 → 1 bit),通过阈值
|
||||
def test_highest_score_match_returned(self, deduplicator, mock_session):
|
||||
"""Issue #1702: 遍历所有候选取融合分最高者(旧逻辑首个过阈即返回)。"""
|
||||
# vid-1: 距离=1 bit(0x03 XOR 0x01 = 0x02 → 1 bit),通过阈值
|
||||
vid1 = self._make_existing_video("vid-1", "md5_1", phashes=["0000000000000003"])
|
||||
# vid-2: 距离=0 bits(完全匹配)
|
||||
# vid-2: 距离=0 bits(完全匹配),融合分更高
|
||||
vid2 = self._make_existing_video("vid-2", "md5_2", phashes=["0000000000000001"])
|
||||
|
||||
mock_repo = MagicMock()
|
||||
@@ -380,8 +380,8 @@ class TestVideoDeduplicatorCheckDuplicate:
|
||||
try:
|
||||
result = deduplicator.check_duplicate(fingerprint, "proj-1", mock_session)
|
||||
assert result is not None
|
||||
# 返回第一个通过阈值的匹配(vid-1 距离=1 < 10)
|
||||
assert result["duplicate_of"] == "vid-1"
|
||||
# 两个候选都过阈,返回融合分最高的 vid-2(距离 0 < 1)
|
||||
assert result["duplicate_of"] == "vid-2"
|
||||
finally:
|
||||
self._restore_repo(mod, orig)
|
||||
|
||||
|
||||
@@ -159,6 +159,6 @@ class TestDedupHelpersUserIdPassthrough:
|
||||
)
|
||||
|
||||
# 验证 update 被调用(包含 duplicate_rate 的记录)
|
||||
mock_video_repo.update.assert_called_once()
|
||||
updated_video = mock_video_repo.update.call_args[0][0]
|
||||
mock_video_repo.create.assert_called_once()
|
||||
updated_video = mock_video_repo.create.call_args[0][0]
|
||||
assert updated_video.duplicate_rate == 78.5
|
||||
|
||||
@@ -185,11 +185,14 @@ class TestBhattacharyyaCoefficient:
|
||||
"""_bhattacharyya_coefficient Bhattacharyya 系数测试."""
|
||||
|
||||
def test_identical_histograms(self):
|
||||
"""完全相同的直方图系数为1.0."""
|
||||
hist = [0.5, 0.5, 0.0, 0.3]
|
||||
"""完全相同的直方图系数为1.0(#1702:按 Σ 归一,概率分布语义)。"""
|
||||
hist = [0.5, 0.5, 0.0, 0.0] # Σ=1 的概率分布
|
||||
bc = VideoDeduplicator._bhattacharyya_coefficient(hist, hist)
|
||||
# Σ √(a[i]*a[i]) = Σ a[i] = 1.0 (normalized)
|
||||
assert bc == pytest.approx(sum(h for h in hist))
|
||||
assert bc == pytest.approx(1.0)
|
||||
# 非归一化输入也归一到 1.0(三通道拼接 Σ=3 的等价情形)
|
||||
hist3 = [0.5, 0.5, 0.0, 0.3]
|
||||
bc3 = VideoDeduplicator._bhattacharyya_coefficient(hist3, hist3)
|
||||
assert bc3 == pytest.approx(1.0)
|
||||
|
||||
def test_zero_histograms(self):
|
||||
"""全零直方图系数为0."""
|
||||
@@ -202,10 +205,10 @@ class TestBhattacharyyaCoefficient:
|
||||
assert bc == pytest.approx(0.0)
|
||||
|
||||
def test_different_lengths(self):
|
||||
"""不同长度直方图取最小长度对齐."""
|
||||
"""不同长度直方图取最小长度对齐,并按各自总量归一(#1702 概率分布语义)。"""
|
||||
# 对齐到前 2 维:coeff = 2,norm = √(Σa·Σb) = √(2·2) = 2 → 1.0
|
||||
bc = VideoDeduplicator._bhattacharyya_coefficient([1.0, 1.0, 0.0, 0.0], [1.0, 1.0])
|
||||
# 对齐到前2维: √(1*1) + √(1*1) = 2.0
|
||||
assert bc == pytest.approx(2.0)
|
||||
assert bc == pytest.approx(1.0)
|
||||
|
||||
def test_known_value(self):
|
||||
"""已知值验证."""
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
"""Tests for two-phase commit pattern in dedup_helpers (#1664 follow-up).
|
||||
|
||||
Verifies that the new dedup_helpers.py:
|
||||
1. Creates video with all dedup fields in a single commit
|
||||
2. Still creates video when fingerprint computation fails
|
||||
3. Creates video with fingerprint but no rate when rate computation fails
|
||||
4. Never does a partial commit (no create + separate update)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Mock cv2/numpy before imports
|
||||
sys.modules.setdefault("cv2", MagicMock())
|
||||
sys.modules.setdefault("numpy", MagicMock())
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "apps" / "api"))
|
||||
sys.path.insert(0, str(ROOT / "packages"))
|
||||
sys.path.insert(0, str(ROOT / "apps" / "worker"))
|
||||
|
||||
import os
|
||||
|
||||
os.environ.setdefault("JWT_SECRET_KEY", "unit-test-secret")
|
||||
os.environ.setdefault("DATABASE_URL", "sqlite:///test.db")
|
||||
|
||||
import pytest
|
||||
from video_processing.dedup_helpers import create_video_record_and_dedup
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def session():
|
||||
s = MagicMock()
|
||||
return s
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_fingerprint():
|
||||
fp = MagicMock()
|
||||
fp.duration = 15000 # 15 seconds in ms
|
||||
fp.to_dict.return_value = {"md5": "abc123", "keyframe_phashes": ["aabb"], "color_histograms": []}
|
||||
fp.chunks = []
|
||||
fp.keyframe_phashes = ["aabb"]
|
||||
fp.color_histograms = []
|
||||
fp.md5 = "abc123"
|
||||
return fp
|
||||
|
||||
|
||||
class TestTwoPhaseCommit:
|
||||
"""Verify that dedup data is computed before commit."""
|
||||
|
||||
def test_video_created_with_all_dedup_fields(self, session, mock_fingerprint):
|
||||
"""When all computations succeed, video is created with all fields in one commit."""
|
||||
mock_repo = MagicMock()
|
||||
mock_deduplicator = MagicMock()
|
||||
mock_deduplicator.compute_fingerprint.return_value = mock_fingerprint
|
||||
mock_deduplicator.check_duplicate.return_value = None
|
||||
mock_deduplicator.compute_duplicate_rate.return_value = {
|
||||
"duplicate_rate": 42.5,
|
||||
"visual_similarity": 0.75,
|
||||
"match_count": 2,
|
||||
}
|
||||
|
||||
with (
|
||||
patch(
|
||||
"packages.adapters.sqlalchemy_impl.generated_video_repository.SQLAlchemyGeneratedVideoRepository",
|
||||
return_value=mock_repo,
|
||||
),
|
||||
patch("video_processing.dedup.VideoDeduplicator", return_value=mock_deduplicator),
|
||||
patch("video_processing.dedup._save_fingerprint_chunks"),
|
||||
):
|
||||
result = create_video_record_and_dedup(
|
||||
generation_task_id="task-001",
|
||||
project_id="proj-001",
|
||||
user_id="user-001",
|
||||
batch_id="",
|
||||
file_url="https://example.com/v.mp4",
|
||||
file_size=1024,
|
||||
duration=15.0,
|
||||
video_path="/tmp/fake.mp4",
|
||||
mode="smart",
|
||||
session=session,
|
||||
)
|
||||
|
||||
assert result == 1
|
||||
# create() should be called exactly once with the complete video object
|
||||
mock_repo.create.assert_called_once()
|
||||
created_video = mock_repo.create.call_args[0][0]
|
||||
assert created_video.duplicate_rate == 42.5
|
||||
assert created_video.visual_similarity == 0.75
|
||||
assert created_video.match_count == 2
|
||||
assert created_video.video_fingerprint is not None
|
||||
# session.commit should be called exactly once (at the end)
|
||||
session.commit.assert_called_once()
|
||||
|
||||
def test_video_created_even_when_fingerprint_fails(self, session):
|
||||
"""When fingerprint computation fails, video is still created (without dedup data)."""
|
||||
mock_repo = MagicMock()
|
||||
mock_deduplicator = MagicMock()
|
||||
mock_deduplicator.compute_fingerprint.side_effect = RuntimeError("cv2 not available")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"packages.adapters.sqlalchemy_impl.generated_video_repository.SQLAlchemyGeneratedVideoRepository",
|
||||
return_value=mock_repo,
|
||||
),
|
||||
patch("video_processing.dedup.VideoDeduplicator", return_value=mock_deduplicator),
|
||||
):
|
||||
result = create_video_record_and_dedup(
|
||||
generation_task_id="task-002",
|
||||
project_id="proj-001",
|
||||
user_id="user-001",
|
||||
batch_id="",
|
||||
file_url="https://example.com/v.mp4",
|
||||
file_size=1024,
|
||||
duration=15.0,
|
||||
video_path="/tmp/fake.mp4",
|
||||
mode="smart",
|
||||
session=session,
|
||||
)
|
||||
|
||||
assert result == 1
|
||||
mock_repo.create.assert_called_once()
|
||||
created_video = mock_repo.create.call_args[0][0]
|
||||
assert created_video.duplicate_rate is None
|
||||
assert created_video.video_fingerprint is None
|
||||
session.commit.assert_called_once()
|
||||
# No dedup methods should have been called
|
||||
mock_deduplicator.check_duplicate.assert_not_called()
|
||||
mock_deduplicator.compute_duplicate_rate.assert_not_called()
|
||||
|
||||
def test_video_created_with_fingerprint_but_no_rate(self, session, mock_fingerprint):
|
||||
"""When rate computation fails, video is created with fingerprint but no rate."""
|
||||
mock_repo = MagicMock()
|
||||
mock_deduplicator = MagicMock()
|
||||
mock_deduplicator.compute_fingerprint.return_value = mock_fingerprint
|
||||
mock_deduplicator.check_duplicate.return_value = None
|
||||
mock_deduplicator.compute_duplicate_rate.side_effect = RuntimeError("DB error")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"packages.adapters.sqlalchemy_impl.generated_video_repository.SQLAlchemyGeneratedVideoRepository",
|
||||
return_value=mock_repo,
|
||||
),
|
||||
patch("video_processing.dedup.VideoDeduplicator", return_value=mock_deduplicator),
|
||||
patch("video_processing.dedup._save_fingerprint_chunks"),
|
||||
):
|
||||
result = create_video_record_and_dedup(
|
||||
generation_task_id="task-003",
|
||||
project_id="proj-001",
|
||||
user_id="user-001",
|
||||
batch_id="",
|
||||
file_url="https://example.com/v.mp4",
|
||||
file_size=1024,
|
||||
duration=15.0,
|
||||
video_path="/tmp/fake.mp4",
|
||||
mode="smart",
|
||||
session=session,
|
||||
)
|
||||
|
||||
assert result == 1
|
||||
mock_repo.create.assert_called_once()
|
||||
created_video = mock_repo.create.call_args[0][0]
|
||||
# Fingerprint should be set
|
||||
assert created_video.video_fingerprint is not None
|
||||
# But duplicate_rate should be None
|
||||
assert created_video.duplicate_rate is None
|
||||
session.commit.assert_called_once()
|
||||
|
||||
def test_no_separate_update_call(self, session, mock_fingerprint):
|
||||
"""Verify the new pattern uses create() only, not create() + update()."""
|
||||
mock_repo = MagicMock()
|
||||
mock_deduplicator = MagicMock()
|
||||
mock_deduplicator.compute_fingerprint.return_value = mock_fingerprint
|
||||
mock_deduplicator.check_duplicate.return_value = None
|
||||
mock_deduplicator.compute_duplicate_rate.return_value = {
|
||||
"duplicate_rate": 10.0,
|
||||
"visual_similarity": 0.5,
|
||||
"match_count": 1,
|
||||
}
|
||||
|
||||
with (
|
||||
patch(
|
||||
"packages.adapters.sqlalchemy_impl.generated_video_repository.SQLAlchemyGeneratedVideoRepository",
|
||||
return_value=mock_repo,
|
||||
),
|
||||
patch("video_processing.dedup.VideoDeduplicator", return_value=mock_deduplicator),
|
||||
patch("video_processing.dedup._save_fingerprint_chunks"),
|
||||
):
|
||||
create_video_record_and_dedup(
|
||||
generation_task_id="task-004",
|
||||
project_id="proj-001",
|
||||
user_id="user-001",
|
||||
batch_id="",
|
||||
file_url="https://example.com/v.mp4",
|
||||
file_size=1024,
|
||||
duration=15.0,
|
||||
video_path="/tmp/fake.mp4",
|
||||
mode="smart",
|
||||
session=session,
|
||||
)
|
||||
|
||||
# Only create() should be called, not update()
|
||||
mock_repo.create.assert_called_once()
|
||||
mock_repo.update.assert_not_called()
|
||||
|
||||
def test_commit_not_called_on_total_failure(self, session):
|
||||
"""When the entire function fails, session.rollback is called instead of commit."""
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.create.side_effect = RuntimeError("DB connection lost")
|
||||
|
||||
with patch(
|
||||
"packages.adapters.sqlalchemy_impl.generated_video_repository.SQLAlchemyGeneratedVideoRepository",
|
||||
return_value=mock_repo,
|
||||
):
|
||||
result = create_video_record_and_dedup(
|
||||
generation_task_id="task-005",
|
||||
project_id="proj-001",
|
||||
user_id="user-001",
|
||||
batch_id="",
|
||||
file_url="https://example.com/v.mp4",
|
||||
file_size=1024,
|
||||
duration=15.0,
|
||||
video_path="/tmp/fake.mp4",
|
||||
mode="smart",
|
||||
session=session,
|
||||
)
|
||||
|
||||
assert result == 0
|
||||
session.commit.assert_not_called()
|
||||
session.rollback.assert_called_once()
|
||||
@@ -484,8 +484,10 @@ class TestBackwardCompatibility:
|
||||
chunks_b = [{"phash_binary": "aaaaaaaaaaaaaaaa", "start_time_ms": 0, "end_time_ms": 5000}]
|
||||
|
||||
segments = find_duplicate_segments(chunks_a, chunks_b)
|
||||
# 1 帧 < min_consecutive=5,不会报重复
|
||||
assert segments == []
|
||||
# Issue #1702: 自适应门槛 min(5, max(2, 1//2))=2,1 帧不成段;
|
||||
# N=1 的检出由 _evaluate_candidate 匹配帧回退兜底(见 test_dedup_1702)。
|
||||
# 这里只要求不崩溃。
|
||||
assert isinstance(segments, list)
|
||||
|
||||
|
||||
# ── TestConstants ───────────────────────────────────────────────
|
||||
@@ -495,8 +497,9 @@ class TestConstants:
|
||||
"""常量值验证 — 使用已在模块顶部导入的常量,避免重新 import."""
|
||||
|
||||
def test_segment_match_threshold(self):
|
||||
# 从已导入的 find_duplicate_segments 默认参数间接验证
|
||||
assert SEGMENT_MATCH_THRESHOLD == 8
|
||||
# Issue #1702: pHash 阈值经 staging 真实同源/异源指纹回归校准
|
||||
# (同源密集采样 min=8、异源 min=24),统一为模块常量 PHASH_THRESHOLD=12。
|
||||
assert SEGMENT_MATCH_THRESHOLD == 12
|
||||
|
||||
def test_min_consecutive_matches(self):
|
||||
assert MIN_CONSECUTIVE_MATCHES == 5
|
||||
|
||||
@@ -189,14 +189,25 @@ class TestComputeDuplicateRateFormula:
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
# 10 frames, all identical to existing → frame_match_rate = 1.0
|
||||
phashes = ["aa00aa00aa00aa00"] * 10
|
||||
# 10 frames with varied phashes (2 unique) → not bad fingerprint
|
||||
# All close in hamming distance to existing → frame_match_rate = 1.0
|
||||
phashes = ["aa00aa00aa00aa00", "ab00ab00ab00ab00"] * 5
|
||||
fingerprint = _make_fingerprint(md5="new", phashes=phashes, duration_ms=20000)
|
||||
session = MagicMock()
|
||||
|
||||
# 5 unique phashes to pass _is_bad_fingerprint check (PR #1688)
|
||||
existing = _make_video(
|
||||
"vid2",
|
||||
{"md5": "other", "keyframe_phashes": ["aa00aa00aa00aa00"] * 5},
|
||||
{
|
||||
"md5": "other",
|
||||
"keyframe_phashes": [
|
||||
"aa00aa00aa00aa00",
|
||||
"ab00ab00ab00ab00",
|
||||
"ac00ac00ac00ac00",
|
||||
"aa10aa10aa10aa10",
|
||||
"ba00ba00ba00ba00",
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo:
|
||||
|
||||
@@ -1,167 +1,168 @@
|
||||
"""#1679 查重 API enqueue + repository.update 单测。
|
||||
"""#1661 查重 API enqueue 及仓储 commit 覆盖测试。
|
||||
|
||||
覆盖:
|
||||
1. POST /duplication/upload 上传成功后调用 send_task 入队 worker。
|
||||
2. POST /duplication/records/{id}/retry 重置后调用 send_task 入队 worker。
|
||||
3. SQLAlchemyDuplicationRecordRepository.update 会调用 session.commit。
|
||||
- upload 接口在成功后调用 celery_app.send_task
|
||||
- retry 接口在成功后调用 celery_app.send_task
|
||||
- duplication_repository.update() 正确调用 session.commit()
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "apps" / "api"))
|
||||
sys.path.insert(0, str(ROOT / "packages"))
|
||||
os.environ.setdefault("JWT_SECRET_KEY", "unit-test-secret-key-for-testing")
|
||||
os.environ.setdefault("DATABASE_URL", "sqlite:///test.db")
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(__file__), "..", "..")
|
||||
sys.path.insert(0, os.path.join(ROOT, "apps", "api"))
|
||||
sys.path.insert(0, os.path.join(ROOT, "packages"))
|
||||
|
||||
from app.api.routes.duplication import router
|
||||
from app.auth import AuthenticatedUser, get_current_user
|
||||
from app.core.storage import get_storage_service
|
||||
from app.dependencies import get_duplication_repository
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from packages.domain.duplication import DuplicationRecord
|
||||
from packages.domain.entities import User
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# ── 通用 fixtures ────────────────────────────────────────────────
|
||||
def _make_test_user():
|
||||
return User(id="user-1", username="testuser", email="test@example.com", display_name="Test User")
|
||||
|
||||
|
||||
def _make_auth_user(user_id: str = "user-1"):
|
||||
from app.auth import AuthenticatedUser
|
||||
|
||||
from packages.domain.entities import User
|
||||
|
||||
user = User(id=user_id, email="u@example.com", display_name="Test User", username="u")
|
||||
return AuthenticatedUser(user=user)
|
||||
def _make_auth_user():
|
||||
return AuthenticatedUser(user=_make_test_user(), session_id="test-session", token_type="bearer")
|
||||
|
||||
|
||||
def _make_record(record_id: str = "rec-1", status: str = "pending", user_id: str = "user-1"):
|
||||
from packages.domain.duplication import DuplicationRecord
|
||||
|
||||
def _make_record(status="pending"):
|
||||
record = DuplicationRecord.create(
|
||||
user_id=user_id,
|
||||
filename="demo.mp4",
|
||||
file_size=2048,
|
||||
storage_key="duplication/abc/demo.mp4",
|
||||
user_id="user-1",
|
||||
filename="test.mp4",
|
||||
file_size=1024,
|
||||
storage_key="duplication/abc/test.mp4",
|
||||
)
|
||||
# 覆盖生成的 id,方便断言
|
||||
record.id = record_id
|
||||
record.status = status
|
||||
if status != "pending":
|
||||
record.status = status
|
||||
return record
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def mock_repo():
|
||||
return MagicMock()
|
||||
def _build_client(auth_user, repo, storage=None):
|
||||
"""构建带 dependency_overrides 的 TestClient。"""
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/duplication")
|
||||
app.dependency_overrides[get_current_user] = lambda: auth_user
|
||||
app.dependency_overrides[get_duplication_repository] = lambda: repo
|
||||
if storage is not None:
|
||||
app.dependency_overrides[get_storage_service] = lambda: storage
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def mock_storage():
|
||||
return MagicMock()
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Upload endpoint enqueues celery task
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def app(mock_repo, mock_storage):
|
||||
"""构造带依赖 override 的 FastAPI app,避免拉起完整 DB / 认证栈。"""
|
||||
from unittest.mock import MagicMock as _MagicMock
|
||||
from unittest.mock import patch as _patch
|
||||
def test_upload_enqueue_calls_celery_task():
|
||||
"""POST /duplication/upload 成功创建记录后必须调用 send_task。"""
|
||||
record = _make_record()
|
||||
|
||||
from app.api.routes.duplication import router as duplication_router
|
||||
from app.auth import AuthenticatedUser, get_current_user
|
||||
from app.core.storage import get_storage_service
|
||||
from app.dependencies import get_duplication_repository
|
||||
from fastapi import FastAPI
|
||||
fake_repo = MagicMock()
|
||||
fake_repo.create.return_value = record
|
||||
|
||||
test_app = FastAPI()
|
||||
test_app.include_router(duplication_router, prefix="/duplication")
|
||||
test_app.dependency_overrides[get_current_user] = lambda: _make_auth_user()
|
||||
test_app.dependency_overrides[get_duplication_repository] = lambda: mock_repo
|
||||
test_app.dependency_overrides[get_storage_service] = lambda: mock_storage
|
||||
fake_storage = MagicMock()
|
||||
fake_auth = _make_auth_user()
|
||||
|
||||
# Mock get_settings so OSS_DIRECT_UPLOAD_MAX_MB is available
|
||||
mock_settings = _MagicMock()
|
||||
mock_settings.OSS_DIRECT_UPLOAD_MAX_MB = 100
|
||||
with _patch("app.config.get_settings", return_value=mock_settings):
|
||||
yield test_app
|
||||
|
||||
|
||||
# ── 1. 上传接口 enqueue ──────────────────────────────────────────
|
||||
|
||||
|
||||
def test_upload_enqueue_calls_celery_task(app, mock_repo):
|
||||
"""POST /duplication/upload 成功创建记录后,必须调用 send_task 入队 worker。"""
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
record = _make_record("rec-upload", status="pending")
|
||||
mock_repo.create.return_value = record
|
||||
client = _build_client(fake_auth, fake_repo, fake_storage)
|
||||
|
||||
with patch("app.api.routes.duplication.celery_app") as mock_celery:
|
||||
mock_celery.send_task = MagicMock(return_value=MagicMock(id="task-xyz"))
|
||||
|
||||
client = TestClient(app)
|
||||
# 使用 in-memory bytes,避免真实写盘
|
||||
response = client.post(
|
||||
"/duplication/upload",
|
||||
files={"file": ("demo.mp4", io.BytesIO(b"\x00\x00\x00\x00fake"), "video/mp4")},
|
||||
files={"file": ("test.mp4", b"fake-video-content", "video/mp4")},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
assert body["id"] == "rec-upload"
|
||||
# 关键断言:send_task 被调用,且参数包含 worker.process_duplication_check
|
||||
mock_celery.send_task.assert_called_once_with("worker.process_duplication_check", args=["rec-upload"])
|
||||
assert response.status_code == 200, response.text
|
||||
mock_celery.send_task.assert_called_once_with(
|
||||
"worker.process_duplication_check",
|
||||
args=[record.id],
|
||||
)
|
||||
|
||||
|
||||
# ── 2. 重试接口 enqueue ──────────────────────────────────────────
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Retry endpoint enqueues celery task
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_retry_enqueue_calls_celery_task(app, mock_repo):
|
||||
"""POST /duplication/records/{id}/retry 重置后必须调用 send_task 入队 worker。"""
|
||||
from fastapi.testclient import TestClient
|
||||
def test_retry_enqueue_calls_celery_task():
|
||||
"""POST /duplication/records/{id}/retry 成功后必须调用 send_task。"""
|
||||
record = _make_record(status="failed")
|
||||
|
||||
record = _make_record("rec-retry", status="failed")
|
||||
mock_repo.get.return_value = record
|
||||
mock_repo.update.return_value = record
|
||||
fake_repo = MagicMock()
|
||||
fake_repo.get.return_value = record
|
||||
|
||||
# RetryDuplicationUseCase.execute 内部调用 repo.get → record.reset_for_retry → repo.update
|
||||
updated = _make_record()
|
||||
updated.id = record.id
|
||||
updated.status = "pending"
|
||||
fake_repo.update.return_value = updated
|
||||
|
||||
fake_auth = _make_auth_user()
|
||||
|
||||
client = _build_client(fake_auth, fake_repo)
|
||||
|
||||
with patch("app.api.routes.duplication.celery_app") as mock_celery:
|
||||
mock_celery.send_task = MagicMock(return_value=MagicMock(id="task-xyz"))
|
||||
response = client.post(f"/duplication/records/{record.id}/retry")
|
||||
|
||||
client = TestClient(app)
|
||||
response = client.post("/duplication/records/rec-retry/retry")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
assert body["id"] == "rec-retry"
|
||||
# 关键断言:send_task 被调用
|
||||
mock_celery.send_task.assert_called_once_with("worker.process_duplication_check", args=["rec-retry"])
|
||||
assert response.status_code == 200, response.text
|
||||
mock_celery.send_task.assert_called_once_with(
|
||||
"worker.process_duplication_check",
|
||||
args=[record.id],
|
||||
)
|
||||
|
||||
|
||||
# ── 3. repository.update 调用 session.commit ─────────────────────
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Repository update calls session.commit()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_repository_update_calls_session_commit():
|
||||
"""SQLAlchemyDuplicationRecordRepository.update 必须在末尾调用 session.commit。"""
|
||||
"""duplication_repository 的 update 方法必须调用 session.commit()。"""
|
||||
from packages.adapters.sqlalchemy_impl.duplication_repository import (
|
||||
SQLAlchemyDuplicationRecordRepository,
|
||||
)
|
||||
from packages.domain.duplication import DuplicationRecord
|
||||
from packages.adapters.sqlalchemy_impl.models import DuplicationRecordModel
|
||||
|
||||
session = MagicMock()
|
||||
# 模拟 query().filter().first() 返回一个 model 实例
|
||||
model = MagicMock()
|
||||
model.id = "rec-1"
|
||||
query_proxy = MagicMock()
|
||||
query_proxy.filter.return_value.first.return_value = model
|
||||
session.query.return_value = query_proxy
|
||||
mock_session = MagicMock()
|
||||
mock_model = MagicMock(spec=DuplicationRecordModel)
|
||||
mock_model.id = "rec-1"
|
||||
|
||||
repo = SQLAlchemyDuplicationRecordRepository(session)
|
||||
mock_session.query.return_value.filter.return_value.first.return_value = mock_model
|
||||
|
||||
repo = SQLAlchemyDuplicationRecordRepository(mock_session)
|
||||
|
||||
record = DuplicationRecord.create(
|
||||
user_id="user-1",
|
||||
filename="demo.mp4",
|
||||
filename="test.mp4",
|
||||
file_size=1024,
|
||||
storage_key="duplication/abc/demo.mp4",
|
||||
storage_key="duplication/abc/test.mp4",
|
||||
)
|
||||
record.id = "rec-1"
|
||||
record.status = "completed"
|
||||
record.duplicate_rate = 42.0
|
||||
record.duplicate_count = 1
|
||||
record.visual_similarity = 0.85
|
||||
record.match_count = 2
|
||||
|
||||
repo.update(record)
|
||||
result = repo.update(record)
|
||||
|
||||
# 关键断言:session.commit 被调用一次
|
||||
session.commit.assert_called_once()
|
||||
mock_session.commit.assert_called()
|
||||
assert result.visual_similarity == 0.85
|
||||
assert result.match_count == 2
|
||||
|
||||
@@ -42,8 +42,6 @@ def _run(mod, record_id, retries=0):
|
||||
result = func(record_id)
|
||||
except CeleryRetry as e:
|
||||
raised = e
|
||||
except Exception as e:
|
||||
raised = e
|
||||
return result, raised, None
|
||||
mock_self = MagicMock()
|
||||
mock_self.request.retries = retries
|
||||
@@ -53,8 +51,6 @@ def _run(mod, record_id, retries=0):
|
||||
result = func(mock_self, record_id)
|
||||
except CeleryRetry as e:
|
||||
raised = e
|
||||
except Exception as e:
|
||||
raised = e
|
||||
return result, raised, mock_self
|
||||
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
覆盖:
|
||||
- 分片策略:60秒视频 → 30片,120秒视频 → 24片
|
||||
- VideoFingerprint.to_chunk_models() 输出正确
|
||||
- _save_fingerprint_chunks 幂等性(已有数据跳过)
|
||||
- _save_fingerprint_chunks 替换语义(Issue #1702:重算时先删旧分片再写入)
|
||||
- to_dict() 向后兼容
|
||||
"""
|
||||
|
||||
@@ -169,11 +169,15 @@ class TestVideoFingerprintToChunkModels:
|
||||
assert models == []
|
||||
|
||||
|
||||
class TestSaveFingerprintChunksIdempotent:
|
||||
"""测试 _save_fingerprint_chunks 幂等性。"""
|
||||
class TestSaveFingerprintChunksReplace:
|
||||
"""测试 _save_fingerprint_chunks 替换语义(Issue #1702)。
|
||||
|
||||
def test_save_skips_existing(self):
|
||||
"""已有分片数据时跳过写入。"""
|
||||
重算查重时指纹算法已升级(中心裁剪 + 新采样/阈值),旧分片必须先删除
|
||||
再写入新分片,否则 recompute-dedup 永远读到旧指纹、修复对存量视频不生效。
|
||||
"""
|
||||
|
||||
def test_save_replaces_existing(self):
|
||||
"""已有分片数据时:先删除旧分片,再写入新分片。"""
|
||||
fp = VideoFingerprint(
|
||||
md5="abc",
|
||||
keyframe_phashes=["a1b2"],
|
||||
@@ -186,16 +190,22 @@ class TestSaveFingerprintChunksIdempotent:
|
||||
)
|
||||
|
||||
session = MagicMock()
|
||||
# Mock: 已有 1 条分片数据
|
||||
session.query.return_value.filter.return_value.count.return_value = 1
|
||||
# Mock: 删除旧分片返回 3(旧算法留下的 3 条分片)
|
||||
session.query.return_value.filter.return_value.delete.return_value = 3
|
||||
|
||||
_save_fingerprint_chunks(fp, video_id="v1", project_id="p1", user_id="u1", session=session)
|
||||
|
||||
# bulk_save_objects 不应被调用
|
||||
session.bulk_save_objects.assert_not_called()
|
||||
# 必须先执行删除
|
||||
session.query.return_value.filter.return_value.delete.assert_called_once()
|
||||
# 新分片必须写入
|
||||
session.bulk_save_objects.assert_called_once()
|
||||
saved_models = session.bulk_save_objects.call_args[0][0]
|
||||
assert len(saved_models) == 1
|
||||
assert saved_models[0].video_id == "v1"
|
||||
assert saved_models[0].phash_binary == "a1b2"
|
||||
|
||||
def test_save_writes_new(self):
|
||||
"""无分片数据时写入。"""
|
||||
"""无旧分片时直接写入。"""
|
||||
fp = VideoFingerprint(
|
||||
md5="abc",
|
||||
keyframe_phashes=["a1b2"],
|
||||
@@ -208,12 +218,12 @@ class TestSaveFingerprintChunksIdempotent:
|
||||
)
|
||||
|
||||
session = MagicMock()
|
||||
# Mock: 无分片数据
|
||||
session.query.return_value.filter.return_value.count.return_value = 0
|
||||
# Mock: 无旧分片
|
||||
session.query.return_value.filter.return_value.delete.return_value = 0
|
||||
|
||||
_save_fingerprint_chunks(fp, video_id="v1", project_id="p1", user_id="u1", session=session)
|
||||
|
||||
# bulk_save_objects 应被调用一次
|
||||
session.query.return_value.filter.return_value.delete.assert_called_once()
|
||||
session.bulk_save_objects.assert_called_once()
|
||||
saved_models = session.bulk_save_objects.call_args[0][0]
|
||||
assert len(saved_models) == 1
|
||||
@@ -221,7 +231,7 @@ class TestSaveFingerprintChunksIdempotent:
|
||||
assert saved_models[0].phash_binary == "a1b2"
|
||||
|
||||
def test_save_skips_no_chunks(self):
|
||||
"""指纹无 chunks 时跳过。"""
|
||||
"""指纹无 chunks 时跳过(不删不写)。"""
|
||||
fp = VideoFingerprint(
|
||||
md5="abc",
|
||||
keyframe_phashes=[],
|
||||
@@ -232,11 +242,11 @@ class TestSaveFingerprintChunksIdempotent:
|
||||
)
|
||||
|
||||
session = MagicMock()
|
||||
session.query.return_value.filter.return_value.count.return_value = 0
|
||||
|
||||
_save_fingerprint_chunks(fp, video_id="v1", project_id="p1", user_id="u1", session=session)
|
||||
|
||||
# bulk_save_objects 不应被调用
|
||||
# 无 chunks:不查询、不删除、不写入
|
||||
session.query.assert_not_called()
|
||||
session.bulk_save_objects.assert_not_called()
|
||||
|
||||
|
||||
|
||||
@@ -725,8 +725,12 @@ class TestCreatePreviewRoute:
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
assert resp.task_id == "preview_task_001"
|
||||
assert resp.status == "pending"
|
||||
# 批量响应:N=1 时 items 长度为 1
|
||||
assert resp.total == 1
|
||||
assert len(resp.items) == 1
|
||||
assert resp.items[0].task_id == "preview_task_001"
|
||||
assert resp.items[0].status == "pending"
|
||||
assert resp.items[0].variant_index == 0
|
||||
|
||||
def test_user_pending_limit_exceeded(self):
|
||||
"""用户待处理任务超限 → 429"""
|
||||
@@ -807,7 +811,13 @@ class TestCreatePreviewRoute:
|
||||
repo.count_pending_total.return_value = 0
|
||||
|
||||
task = _make_task()
|
||||
from fastapi import HTTPException
|
||||
|
||||
# 模拟 mark_failed 真实更新任务状态(_mark_task_failed 内部调用)
|
||||
def _set_failed(error_message="", **_kwargs):
|
||||
task.status = GenerationTaskStatus.FAILED
|
||||
task.error_message = error_message
|
||||
|
||||
task.mark_failed.side_effect = _set_failed
|
||||
|
||||
with patch("app.api.routes.generation_preview.CreateGenerationTaskUseCase") as MockUC:
|
||||
MockUC.return_value.execute.return_value = task
|
||||
@@ -815,14 +825,15 @@ class TestCreatePreviewRoute:
|
||||
"app.api.routes.generation_preview.safe_enqueue_generation_task",
|
||||
return_value=False,
|
||||
):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
create_preview_generation_task(
|
||||
self._make_request(),
|
||||
authenticated_user=_make_user(),
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
assert exc_info.value.status_code == 500
|
||||
resp = create_preview_generation_task(
|
||||
self._make_request(),
|
||||
authenticated_user=_make_user(),
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
# 入队失败:任务被标记 failed(mark_failed 设置错误信息),响应正常返回
|
||||
assert resp.total == 1
|
||||
assert resp.items[0].status == "failed"
|
||||
|
||||
def test_enqueue_raises_user_limit(self):
|
||||
"""safe_enqueue 抛出 UserPendingLimitExceeded → 429"""
|
||||
@@ -833,6 +844,12 @@ class TestCreatePreviewRoute:
|
||||
task = _make_task()
|
||||
from fastapi import HTTPException
|
||||
|
||||
def _set_failed_limit(error_message="", **_kwargs):
|
||||
task.status = GenerationTaskStatus.FAILED
|
||||
task.error_message = error_message or "待处理任务超限"
|
||||
|
||||
task.mark_failed.side_effect = _set_failed_limit
|
||||
|
||||
with patch("app.api.routes.generation_preview.CreateGenerationTaskUseCase") as MockUC:
|
||||
MockUC.return_value.execute.return_value = task
|
||||
with patch(
|
||||
@@ -846,6 +863,7 @@ class TestCreatePreviewRoute:
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
# 全部变体入队失败且错误消息含"待处理任务" → 429
|
||||
assert exc_info.value.status_code == 429
|
||||
|
||||
def test_enqueue_raises_global_queue_full(self):
|
||||
@@ -857,6 +875,12 @@ class TestCreatePreviewRoute:
|
||||
task = _make_task()
|
||||
from fastapi import HTTPException
|
||||
|
||||
def _set_failed_queue(error_message="", **_kwargs):
|
||||
task.status = GenerationTaskStatus.FAILED
|
||||
task.error_message = error_message or "系统队列已满"
|
||||
|
||||
task.mark_failed.side_effect = _set_failed_queue
|
||||
|
||||
with patch("app.api.routes.generation_preview.CreateGenerationTaskUseCase") as MockUC:
|
||||
MockUC.return_value.execute.return_value = task
|
||||
with patch(
|
||||
@@ -870,6 +894,7 @@ class TestCreatePreviewRoute:
|
||||
generation_task_repository=repo,
|
||||
db=MagicMock(),
|
||||
)
|
||||
# 全部变体入队失败且错误消息含"队列" → 503
|
||||
assert exc_info.value.status_code == 503
|
||||
|
||||
|
||||
|
||||
@@ -102,6 +102,7 @@ from video_processing.dedup import ( # noqa: E402
|
||||
DUPLICATE_THRESHOLD,
|
||||
HISTOGRAM_WEIGHT,
|
||||
MATCH_RATIO_THRESHOLD,
|
||||
PHASH_THRESHOLD,
|
||||
PHASH_WEIGHT,
|
||||
VideoDeduplicator,
|
||||
)
|
||||
@@ -128,11 +129,15 @@ _ZERO_HIST = [0.0] * 96 # 全黑视频的全零直方图(有效数据)
|
||||
|
||||
|
||||
class TestThresholdCalibration:
|
||||
"""pHash 阈值由 10 收紧到 8(Issue #1658)。"""
|
||||
"""pHash 阈值校准(Issue #1658 收紧到 8,Issue #1702 经真实指纹分布重校准为 12)。
|
||||
|
||||
def test_phash_threshold_is_8(self):
|
||||
"""PHASH_THRESHOLD 必须为 8(旧值 10 会放过 8~9 汉明距离的不同视频)。"""
|
||||
assert VideoDeduplicator.PHASH_THRESHOLD == 8
|
||||
#1702 staging 离线实验:同帧两次 2-5% 随机裁剪距离 4~10;同源成片(密集 1s
|
||||
采样)最小距离 8、<=12 命中 10/31;异源成片最小距离 24。8 会漏检同源裁剪,
|
||||
12 检出同源且与异源分布(>=24)间隔充足。
|
||||
"""
|
||||
|
||||
def test_phash_threshold_is_calibrated(self):
|
||||
assert VideoDeduplicator.PHASH_THRESHOLD == PHASH_THRESHOLD == 12
|
||||
|
||||
def test_match_ratio_threshold_constant(self):
|
||||
assert MATCH_RATIO_THRESHOLD == 0.7
|
||||
@@ -144,22 +149,21 @@ class TestThresholdCalibration:
|
||||
assert PHASH_WEIGHT == 0.7
|
||||
assert HISTOGRAM_WEIGHT == 0.3
|
||||
|
||||
def test_threshold_tightening_excludes_distance_8_and_9(self):
|
||||
"""距离 8、9 的帧:旧阈值 10 下算匹配,新阈值 8 下不算匹配。
|
||||
def test_threshold_matching_semantics(self):
|
||||
"""阈值比较统一为 <=(帧匹配与片段匹配同一口径)。
|
||||
|
||||
场景:5 个关键帧距离为 [7, 7, 7, 9, 9]。
|
||||
- 旧阈值 10:5 帧全部 < 10 → match_ratio = 1.0(误放过)
|
||||
- 新阈值 8:仅 3 帧 < 8 → match_ratio = 0.6 < 0.7(正确跳过)
|
||||
场景:5 个关键帧距离为 [10, 12, 12, 24, 26]。
|
||||
- <=12(#1702 校准阈值):3 帧匹配 → 0.6 < 0.7 被帧比例门槛拦截异源
|
||||
- 距离 12 的同源裁剪帧应算匹配(< 与 <= 口径统一)
|
||||
"""
|
||||
distances = [7, 7, 7, 9, 9]
|
||||
distances = [10, 12, 12, 24, 26]
|
||||
matched = sum(1 for d in distances if d <= VideoDeduplicator.PHASH_THRESHOLD)
|
||||
assert matched == 3
|
||||
assert matched / len(distances) == 0.6
|
||||
assert matched / len(distances) < MATCH_RATIO_THRESHOLD
|
||||
|
||||
matched_old = sum(1 for d in distances if d < 10)
|
||||
assert matched_old == 5 # 旧行为:全匹配 → 误判风险
|
||||
|
||||
matched_new = sum(1 for d in distances if d < VideoDeduplicator.PHASH_THRESHOLD)
|
||||
assert matched_new == 3
|
||||
assert matched_new / len(distances) == 0.6
|
||||
assert matched_new / len(distances) < MATCH_RATIO_THRESHOLD # 被帧比例门槛拦截
|
||||
# 异源典型距离(>=24)绝不匹配
|
||||
assert not any(d <= VideoDeduplicator.PHASH_THRESHOLD for d in (24, 26, 30))
|
||||
|
||||
|
||||
# ── TestComputeFusionScore:统一融合得分方法 ────────────────────
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
"""#1664 随机边缘裁剪降重功能测试"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "apps" / "worker"))
|
||||
sys.path.insert(0, str(ROOT / "apps" / "api"))
|
||||
sys.path.insert(0, str(ROOT / "packages"))
|
||||
|
||||
|
||||
from video_processing.ffmpeg_utils import random_edge_crop
|
||||
|
||||
|
||||
class TestRandomEdgeCropBasic:
|
||||
"""基本功能测试"""
|
||||
|
||||
def test_returns_input_path_when_output_none(self, tmp_path):
|
||||
"""output_path=None 时覆盖原文件并返回 input_path"""
|
||||
input_file = tmp_path / "input.mp4"
|
||||
input_file.write_bytes(b"fake video data")
|
||||
|
||||
fake_info = {"width": 1920, "height": 1080, "duration": 10, "fps": 30}
|
||||
with (
|
||||
patch("video_processing.ffmpeg_utils.probe_video_info", return_value=fake_info),
|
||||
patch("video_processing.ffmpeg_utils.run_ffmpeg"),
|
||||
):
|
||||
result = random_edge_crop(input_file)
|
||||
|
||||
assert result == input_file
|
||||
|
||||
def test_returns_output_path_when_specified(self, tmp_path):
|
||||
"""指定 output_path 时返回该路径"""
|
||||
input_file = tmp_path / "input.mp4"
|
||||
input_file.write_bytes(b"fake video data")
|
||||
output_file = tmp_path / "output.mp4"
|
||||
|
||||
fake_info = {"width": 1920, "height": 1080, "duration": 10, "fps": 30}
|
||||
with (
|
||||
patch("video_processing.ffmpeg_utils.probe_video_info", return_value=fake_info),
|
||||
patch("video_processing.ffmpeg_utils.run_ffmpeg"),
|
||||
):
|
||||
result = random_edge_crop(input_file, output_file)
|
||||
|
||||
assert result == output_file
|
||||
|
||||
def test_skip_when_invalid_resolution(self, tmp_path):
|
||||
"""无法获取有效分辨率时跳过裁剪"""
|
||||
input_file = tmp_path / "input.mp4"
|
||||
input_file.write_bytes(b"fake video data")
|
||||
|
||||
fake_info = {"width": 0, "height": 0, "duration": 10, "fps": 30}
|
||||
with (
|
||||
patch("video_processing.ffmpeg_utils.probe_video_info", return_value=fake_info),
|
||||
patch("video_processing.ffmpeg_utils.run_ffmpeg") as mock_ffmpeg,
|
||||
):
|
||||
result = random_edge_crop(input_file)
|
||||
|
||||
assert result == input_file
|
||||
mock_ffmpeg.assert_not_called()
|
||||
|
||||
|
||||
class TestRandomEdgeCropFFmpeg:
|
||||
"""FFmpeg 调用参数验证"""
|
||||
|
||||
def test_ffmpeg_crop_and_scale_filter(self, tmp_path):
|
||||
"""生成的 ffmpeg 滤镜包含 crop + scale"""
|
||||
input_file = tmp_path / "input.mp4"
|
||||
input_file.write_bytes(b"fake video data")
|
||||
|
||||
# 固定随机值以便验证
|
||||
fake_info = {"width": 1000, "height": 1000, "duration": 10, "fps": 30}
|
||||
|
||||
with (
|
||||
patch("video_processing.ffmpeg_utils.probe_video_info", return_value=fake_info),
|
||||
patch("video_processing.ffmpeg_utils.run_ffmpeg") as mock_ffmpeg,
|
||||
patch("random.uniform", side_effect=[0.03, 0.03, 0.03, 0.03]),
|
||||
):
|
||||
random_edge_crop(input_file)
|
||||
|
||||
mock_ffmpeg.assert_called_once()
|
||||
cmd = mock_ffmpeg.call_args[0][0]
|
||||
# 找到 -vf 参数
|
||||
vf_idx = cmd.index("-vf")
|
||||
vf_value = cmd[vf_idx + 1]
|
||||
assert "crop=" in vf_value
|
||||
assert "scale=1000:1000" in vf_value
|
||||
|
||||
def test_crop_amounts_within_range(self, tmp_path):
|
||||
"""裁剪量在 2%~5% 范围内"""
|
||||
input_file = tmp_path / "input.mp4"
|
||||
input_file.write_bytes(b"fake video data")
|
||||
|
||||
fake_info = {"width": 1000, "height": 1000, "duration": 10, "fps": 30}
|
||||
|
||||
with (
|
||||
patch("video_processing.ffmpeg_utils.probe_video_info", return_value=fake_info),
|
||||
patch("video_processing.ffmpeg_utils.run_ffmpeg") as mock_ffmpeg,
|
||||
patch("random.uniform", side_effect=[0.02, 0.05, 0.02, 0.05]),
|
||||
):
|
||||
random_edge_crop(input_file)
|
||||
|
||||
cmd = mock_ffmpeg.call_args[0][0]
|
||||
vf_idx = cmd.index("-vf")
|
||||
vf_value = cmd[vf_idx + 1]
|
||||
# crop_top=20, crop_bottom=50, crop_left=20, crop_right=50
|
||||
# new_w = 1000-20-50 = 930, new_h = 1000-20-50 = 930
|
||||
# x_offset = 20, y_offset = 20
|
||||
assert "crop=930:930:20:20" in vf_value
|
||||
|
||||
def test_uses_libx264_codec(self, tmp_path):
|
||||
"""使用 libx264 编码"""
|
||||
input_file = tmp_path / "input.mp4"
|
||||
input_file.write_bytes(b"fake video data")
|
||||
|
||||
fake_info = {"width": 1920, "height": 1080, "duration": 10, "fps": 30}
|
||||
with (
|
||||
patch("video_processing.ffmpeg_utils.probe_video_info", return_value=fake_info),
|
||||
patch("video_processing.ffmpeg_utils.run_ffmpeg") as mock_ffmpeg,
|
||||
):
|
||||
random_edge_crop(input_file)
|
||||
|
||||
cmd = mock_ffmpeg.call_args[0][0]
|
||||
assert "-c:v" in cmd
|
||||
assert cmd[cmd.index("-c:v") + 1] == "libx264"
|
||||
|
||||
|
||||
class TestRandomEdgeCropErrorHandling:
|
||||
"""错误处理测试"""
|
||||
|
||||
def test_ffmpeg_failure_raises_exception(self, tmp_path):
|
||||
"""ffmpeg 失败时抛出异常"""
|
||||
input_file = tmp_path / "input.mp4"
|
||||
input_file.write_bytes(b"fake video data")
|
||||
|
||||
fake_info = {"width": 1920, "height": 1080, "duration": 10, "fps": 30}
|
||||
with (
|
||||
patch("video_processing.ffmpeg_utils.probe_video_info", return_value=fake_info),
|
||||
patch(
|
||||
"video_processing.ffmpeg_utils.run_ffmpeg",
|
||||
side_effect=subprocess.CalledProcessError(1, "ffmpeg"),
|
||||
),
|
||||
):
|
||||
with pytest.raises(subprocess.CalledProcessError):
|
||||
random_edge_crop(input_file)
|
||||
|
||||
def test_probe_failure_propagates(self, tmp_path):
|
||||
"""probe_video_info 失败时异常传播"""
|
||||
input_file = tmp_path / "input.mp4"
|
||||
input_file.write_bytes(b"fake video data")
|
||||
|
||||
with patch(
|
||||
"video_processing.ffmpeg_utils.probe_video_info",
|
||||
side_effect=RuntimeError("probe failed"),
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="probe failed"):
|
||||
random_edge_crop(input_file)
|
||||
|
||||
|
||||
class TestRandomEdgeCropEvenDimensions:
|
||||
"""偶数尺寸处理测试"""
|
||||
|
||||
def test_odd_crop_dimensions_adjusted_to_even(self, tmp_path):
|
||||
"""裁剪后尺寸为奇数时自动调整为偶数"""
|
||||
input_file = tmp_path / "input.mp4"
|
||||
input_file.write_bytes(b"fake video data")
|
||||
|
||||
# 1000 - 3 (top) - 4 (bottom) = 993 → 调整为 992
|
||||
# 1000 - 3 (left) - 4 (right) = 993 → 调整为 992
|
||||
fake_info = {"width": 1000, "height": 1000, "duration": 10, "fps": 30}
|
||||
|
||||
with (
|
||||
patch("video_processing.ffmpeg_utils.probe_video_info", return_value=fake_info),
|
||||
patch("video_processing.ffmpeg_utils.run_ffmpeg") as mock_ffmpeg,
|
||||
# side_effect 控制 uniform 返回值
|
||||
# top: 0.003*1000=3, bottom: 0.004*1000=4, left: 0.003*1000=3, right: 0.004*1000=4
|
||||
):
|
||||
# 使用自定义 uniform 返回特定值
|
||||
def fake_uniform(low, high):
|
||||
# 返回特定百分比使得裁剪后尺寸为奇数
|
||||
# 我们需要 crop_top=3, crop_bottom=4, crop_left=3, crop_right=4
|
||||
return 0.0035 # 近似值
|
||||
|
||||
# 更简单的方式:直接 mock int(H * random.uniform(...)) 的结果
|
||||
# 但我们直接测试最终 crop 滤镜即可
|
||||
with patch("random.uniform", side_effect=[0.021, 0.022, 0.021, 0.022]):
|
||||
random_edge_crop(input_file)
|
||||
|
||||
cmd = mock_ffmpeg.call_args[0][0]
|
||||
vf_idx = cmd.index("-vf")
|
||||
vf_value = cmd[vf_idx + 1]
|
||||
# 提取 crop 参数并验证都是偶数
|
||||
import re
|
||||
|
||||
crop_match = re.search(r"crop=(\d+):(\d+)", vf_value)
|
||||
assert crop_match
|
||||
crop_w = int(crop_match.group(1))
|
||||
crop_h = int(crop_match.group(2))
|
||||
assert crop_w % 2 == 0, f"crop width {crop_w} should be even"
|
||||
assert crop_h % 2 == 0, f"crop height {crop_h} should be even"
|
||||
@@ -0,0 +1,167 @@
|
||||
"""Tests for POST /videos/recompute-dedup endpoint (#1664 follow-up)."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_video():
|
||||
"""Mock video with missing dedup data."""
|
||||
v = MagicMock()
|
||||
v.id = "video-001"
|
||||
v.user_id = "user-abc"
|
||||
v.duplicate_rate = None
|
||||
v.video_fingerprint = None
|
||||
v.project_id = "proj-001"
|
||||
v.generation_task_id = "task-001"
|
||||
v.name = "test.mp4"
|
||||
v.file_url = "https://example.com/test.mp4"
|
||||
v.file_size = 1024
|
||||
v.duration = 10.0
|
||||
v.width = 1920
|
||||
v.height = 1080
|
||||
v.fps = 25.0
|
||||
v.status = "completed"
|
||||
v.review_status = "pending_review"
|
||||
v.generation_params = {}
|
||||
v.thumbnail_url = None
|
||||
v.is_duplicate = False
|
||||
v.duplicate_of = None
|
||||
v.match_count = None
|
||||
v.visual_similarity = None
|
||||
v.generated_at = "2026-09-04T00:00:00"
|
||||
return v
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_video_with_dedup(mock_video):
|
||||
"""Mock video that already has dedup data."""
|
||||
mock_video.duplicate_rate = 15.5
|
||||
mock_video.video_fingerprint = {"md5": "abc123"}
|
||||
return mock_video
|
||||
|
||||
|
||||
class TestRecomputeDedupEndpoint:
|
||||
"""POST /videos/recompute-dedup"""
|
||||
|
||||
def test_enqueue_videos_without_dedup(self, mock_video):
|
||||
"""Videos missing duplicate_rate should be enqueued."""
|
||||
from app.api.routes.videos import RecomputeDedupRequest
|
||||
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.list_by_user.return_value = [mock_video]
|
||||
|
||||
with (patch("app.api.routes.videos.celery_app") as mock_celery,):
|
||||
mock_celery.send_task.return_value = MagicMock(id="task-xyz")
|
||||
from app.api.routes.videos import recompute_dedup
|
||||
|
||||
auth_user = MagicMock()
|
||||
auth_user.user.id = "user-abc"
|
||||
|
||||
result = recompute_dedup(
|
||||
request=RecomputeDedupRequest(),
|
||||
repo=mock_repo,
|
||||
current_user=auth_user,
|
||||
)
|
||||
|
||||
assert result.enqueued == 1
|
||||
assert result.total_scanned == 1
|
||||
assert result.skipped == 0
|
||||
mock_celery.send_task.assert_called_once_with("worker.check_duplicate", args=["video-001"])
|
||||
|
||||
def test_skip_videos_with_complete_dedup(self, mock_video_with_dedup):
|
||||
"""Videos with both duplicate_rate and video_fingerprint should be skipped."""
|
||||
from app.api.routes.videos import RecomputeDedupRequest, recompute_dedup
|
||||
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.list_by_user.return_value = [mock_video_with_dedup]
|
||||
|
||||
with patch("app.api.routes.videos.celery_app") as mock_celery:
|
||||
auth_user = MagicMock()
|
||||
auth_user.user.id = "user-abc"
|
||||
|
||||
result = recompute_dedup(
|
||||
request=RecomputeDedupRequest(),
|
||||
repo=mock_repo,
|
||||
current_user=auth_user,
|
||||
)
|
||||
|
||||
assert result.enqueued == 0
|
||||
assert result.total_scanned == 1
|
||||
assert result.skipped == 1
|
||||
mock_celery.send_task.assert_not_called()
|
||||
|
||||
def test_specific_video_ids(self, mock_video):
|
||||
"""When video_ids are provided, only those videos are processed."""
|
||||
from app.api.routes.videos import RecomputeDedupRequest, recompute_dedup
|
||||
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.get_by_ids.return_value = [mock_video]
|
||||
|
||||
with patch("app.api.routes.videos.celery_app") as mock_celery:
|
||||
mock_celery.send_task.return_value = MagicMock(id="task-xyz")
|
||||
auth_user = MagicMock()
|
||||
auth_user.user.id = "user-abc"
|
||||
|
||||
result = recompute_dedup(
|
||||
request=RecomputeDedupRequest(video_ids=["video-001"]),
|
||||
repo=mock_repo,
|
||||
current_user=auth_user,
|
||||
)
|
||||
|
||||
assert result.enqueued == 1
|
||||
mock_repo.get_by_ids.assert_called_once_with(["video-001"])
|
||||
|
||||
def test_security_only_own_videos(self, mock_video):
|
||||
"""Videos belonging to other users should be filtered out."""
|
||||
from app.api.routes.videos import RecomputeDedupRequest, recompute_dedup
|
||||
|
||||
mock_video.user_id = "user-OTHER"
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.get_by_ids.return_value = [mock_video]
|
||||
|
||||
with patch("app.api.routes.videos.celery_app") as mock_celery:
|
||||
auth_user = MagicMock()
|
||||
auth_user.user.id = "user-abc"
|
||||
|
||||
result = recompute_dedup(
|
||||
request=RecomputeDedupRequest(video_ids=["video-001"]),
|
||||
repo=mock_repo,
|
||||
current_user=auth_user,
|
||||
)
|
||||
|
||||
assert result.enqueued == 0
|
||||
mock_celery.send_task.assert_not_called()
|
||||
|
||||
def test_mixed_complete_and_incomplete(self, mock_video, mock_video_with_dedup):
|
||||
"""Mix of videos with and without dedup data."""
|
||||
import copy
|
||||
|
||||
from app.api.routes.videos import RecomputeDedupRequest, recompute_dedup
|
||||
|
||||
# Create a second video object
|
||||
v2 = MagicMock()
|
||||
v2.id = "video-002"
|
||||
v2.user_id = "user-abc"
|
||||
v2.duplicate_rate = None
|
||||
v2.video_fingerprint = None
|
||||
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.list_by_user.return_value = [mock_video_with_dedup, v2]
|
||||
|
||||
with patch("app.api.routes.videos.celery_app") as mock_celery:
|
||||
mock_celery.send_task.return_value = MagicMock(id="task-xyz")
|
||||
auth_user = MagicMock()
|
||||
auth_user.user.id = "user-abc"
|
||||
|
||||
result = recompute_dedup(
|
||||
request=RecomputeDedupRequest(),
|
||||
repo=mock_repo,
|
||||
current_user=auth_user,
|
||||
)
|
||||
|
||||
assert result.enqueued == 1
|
||||
assert result.total_scanned == 2
|
||||
assert result.skipped == 1
|
||||
@@ -67,11 +67,11 @@ class TestPositionToAssAlignment:
|
||||
def test_bottom(self):
|
||||
assert _position_to_ass_alignment("bottom") == 2
|
||||
|
||||
def test_unknown_returns_top_default(self):
|
||||
assert _position_to_ass_alignment("unknown") == 8
|
||||
assert _position_to_ass_alignment("") == 8
|
||||
assert _position_to_ass_alignment("left") == 8
|
||||
assert _position_to_ass_alignment(None) == 8
|
||||
def test_unknown_returns_bottom_default(self):
|
||||
assert _position_to_ass_alignment("unknown") == 2
|
||||
assert _position_to_ass_alignment("") == 2
|
||||
assert _position_to_ass_alignment("left") == 2
|
||||
assert _position_to_ass_alignment(None) == 2
|
||||
|
||||
|
||||
class TestBuildAssStyle:
|
||||
|
||||
@@ -66,15 +66,15 @@ class TestPositionToAssAlignment:
|
||||
"""center → 居中(5)."""
|
||||
assert _position_to_ass_alignment("center") == 5
|
||||
|
||||
def test_unknown_defaults_to_top(self):
|
||||
"""未知位置默认顶部(8)."""
|
||||
assert _position_to_ass_alignment("unknown") == 8
|
||||
assert _position_to_ass_alignment("top_left") == 8
|
||||
assert _position_to_ass_alignment("bottom_right") == 8
|
||||
def test_unknown_defaults_to_bottom(self):
|
||||
"""未知位置默认底部(2)."""
|
||||
assert _position_to_ass_alignment("unknown") == 2
|
||||
assert _position_to_ass_alignment("top_left") == 2
|
||||
assert _position_to_ass_alignment("bottom_right") == 2
|
||||
|
||||
def test_empty_string_defaults_to_top(self):
|
||||
"""空字符串默认顶部."""
|
||||
assert _position_to_ass_alignment("") == 8
|
||||
def test_empty_string_defaults_to_bottom(self):
|
||||
"""空字符串默认底部."""
|
||||
assert _position_to_ass_alignment("") == 2
|
||||
|
||||
|
||||
class TestBuildAssStyle:
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Tests for voice duration alignment feature.
|
||||
|
||||
Tests the _align_clips_to_voice_duration method in UnifiedRenderService.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from video_processing.unified_render_service import RenderLayer, ResolvedClip, UnifiedRenderService
|
||||
|
||||
|
||||
class TestAlignClipsToVoiceDuration:
|
||||
"""Test clip duration alignment to voice audio."""
|
||||
|
||||
def _make_clip(
|
||||
self,
|
||||
clip_id: str,
|
||||
duration: float,
|
||||
actual_duration: float = 0.0,
|
||||
playback_speed: float = 1.0,
|
||||
) -> ResolvedClip:
|
||||
"""Helper to create a ResolvedClip for testing."""
|
||||
return ResolvedClip(
|
||||
clip_id=clip_id,
|
||||
asset_id=f"asset_{clip_id}",
|
||||
local_path=Path(f"/tmp/{clip_id}.mp4"),
|
||||
clip_type="main",
|
||||
order=0,
|
||||
duration=duration,
|
||||
actual_duration=actual_duration or duration,
|
||||
playback_speed=playback_speed,
|
||||
)
|
||||
|
||||
def _make_layer(self, role: str, clips: list[ResolvedClip]) -> RenderLayer:
|
||||
"""Helper to create a RenderLayer for testing."""
|
||||
return RenderLayer(role=role, clips=clips, z_index=0)
|
||||
|
||||
def _make_service(self, voiceover_path: str | None = None) -> UnifiedRenderService:
|
||||
"""Helper to create a mock UnifiedRenderService."""
|
||||
plan = MagicMock()
|
||||
plan.id = "test_plan"
|
||||
plan.config = {}
|
||||
|
||||
with patch.object(UnifiedRenderService, "__init__", lambda self, **kwargs: None):
|
||||
service = UnifiedRenderService.__new__(UnifiedRenderService)
|
||||
service.plan = plan
|
||||
service.voiceover_audio_path = voiceover_path
|
||||
service.transition_duration = 0.0
|
||||
return service
|
||||
|
||||
def test_no_voice_audio_no_adjustment(self):
|
||||
"""No voice audio → no adjustment."""
|
||||
service = self._make_service(voiceover_path=None)
|
||||
clips = [self._make_clip("c1", 10.0), self._make_clip("c2", 10.0)]
|
||||
layers = [self._make_layer("main", clips)]
|
||||
|
||||
service._align_clips_to_voice_duration(layers, voice_duration=0.0)
|
||||
|
||||
# No change
|
||||
assert clips[0].duration == 10.0
|
||||
assert clips[1].duration == 10.0
|
||||
|
||||
def test_ratio_within_5_percent_no_adjustment(self):
|
||||
"""Ratio within ±5% → no adjustment."""
|
||||
service = self._make_service()
|
||||
clips = [self._make_clip("c1", 10.0)]
|
||||
layers = [self._make_layer("main", clips)]
|
||||
|
||||
# Total clips = 10s, voice = 10.3s → ratio = 1.03 (within 5%)
|
||||
service._align_clips_to_voice_duration(layers, voice_duration=10.3)
|
||||
|
||||
assert clips[0].duration == 10.0 # Unchanged
|
||||
|
||||
def test_ratio_less_than_1_trim_clips(self):
|
||||
"""Ratio < 1 (clips too long) → trim clips proportionally."""
|
||||
service = self._make_service()
|
||||
clips = [self._make_clip("c1", 10.0), self._make_clip("c2", 10.0)]
|
||||
layers = [self._make_layer("main", clips)]
|
||||
|
||||
# Total clips = 20s, voice = 15s → ratio = 0.75
|
||||
service._align_clips_to_voice_duration(layers, voice_duration=15.0)
|
||||
|
||||
# Each clip should be trimmed to 75%
|
||||
assert abs(clips[0].duration - 7.5) < 0.01
|
||||
assert abs(clips[1].duration - 7.5) < 0.01
|
||||
|
||||
def test_ratio_greater_than_1_slowdown_clips(self):
|
||||
"""Ratio > 1 (clips too short) → slow down clips."""
|
||||
service = self._make_service()
|
||||
clips = [self._make_clip("c1", 10.0), self._make_clip("c2", 10.0)]
|
||||
layers = [self._make_layer("main", clips)]
|
||||
|
||||
# Total clips = 20s, voice = 25s → ratio = 1.25
|
||||
service._align_clips_to_voice_duration(layers, voice_duration=25.0)
|
||||
|
||||
# Each clip's speed should be reduced: 1.0 / 1.25 = 0.8
|
||||
assert abs(clips[0].playback_speed - 0.8) < 0.01
|
||||
assert abs(clips[1].playback_speed - 0.8) < 0.01
|
||||
|
||||
def test_speed_lower_bound_025(self):
|
||||
"""Playback speed should not go below 0.25x."""
|
||||
service = self._make_service()
|
||||
clips = [self._make_clip("c1", 5.0)]
|
||||
layers = [self._make_layer("main", clips)]
|
||||
|
||||
# Total clips = 5s, voice = 50s → ratio = 10.0
|
||||
# Speed would be 1.0 / 10 = 0.1, but should be clamped to 0.25
|
||||
service._align_clips_to_voice_duration(layers, voice_duration=50.0)
|
||||
|
||||
assert clips[0].playback_speed == 0.25
|
||||
|
||||
def test_only_video_layers_adjusted(self):
|
||||
"""Only main/broll/background layers are adjusted, not audio."""
|
||||
service = self._make_service()
|
||||
|
||||
video_clips = [self._make_clip("v1", 10.0)]
|
||||
audio_clips = [self._make_clip("a1", 10.0)]
|
||||
|
||||
layers = [
|
||||
self._make_layer("main", video_clips),
|
||||
self._make_layer("audio", audio_clips),
|
||||
]
|
||||
|
||||
# ratio = 0.5 → should trim video but not audio
|
||||
service._align_clips_to_voice_duration(layers, voice_duration=5.0)
|
||||
|
||||
assert abs(video_clips[0].duration - 5.0) < 0.01 # Trimmed
|
||||
assert audio_clips[0].duration == 10.0 # Unchanged
|
||||
|
||||
def test_multiple_video_layers_all_adjusted(self):
|
||||
"""All video layers (main, broll, background) are adjusted."""
|
||||
service = self._make_service()
|
||||
|
||||
main_clips = [self._make_clip("m1", 10.0)]
|
||||
broll_clips = [self._make_clip("b1", 10.0)]
|
||||
bg_clips = [self._make_clip("bg1", 10.0)]
|
||||
|
||||
layers = [
|
||||
self._make_layer("main", main_clips),
|
||||
self._make_layer("broll", broll_clips),
|
||||
self._make_layer("background", bg_clips),
|
||||
]
|
||||
|
||||
# Total video = 30s, voice = 15s → ratio = 0.5
|
||||
service._align_clips_to_voice_duration(layers, voice_duration=15.0)
|
||||
|
||||
# All should be trimmed to 50%
|
||||
assert abs(main_clips[0].duration - 5.0) < 0.01
|
||||
assert abs(broll_clips[0].duration - 5.0) < 0.01
|
||||
assert abs(bg_clips[0].duration - 5.0) < 0.01
|
||||
|
||||
def test_trim_config_also_adjusted(self):
|
||||
"""When clip has trim_config, it should also be adjusted."""
|
||||
from video_processing.trim_engine import TrimConfig
|
||||
|
||||
service = self._make_service()
|
||||
|
||||
clip = self._make_clip("c1", 10.0)
|
||||
clip.trim_config = TrimConfig(start_time=0.0, duration=10.0)
|
||||
|
||||
layers = [self._make_layer("main", [clip])]
|
||||
|
||||
# ratio = 0.5
|
||||
service._align_clips_to_voice_duration(layers, voice_duration=5.0)
|
||||
|
||||
assert abs(clip.duration - 5.0) < 0.01
|
||||
assert clip.trim_config is not None
|
||||
assert abs(clip.trim_config.duration - 5.0) < 0.01
|
||||
|
||||
|
||||
class TestGetVoiceAudioDuration:
|
||||
"""Test voice audio duration probing."""
|
||||
|
||||
def test_no_voiceover_path_returns_zero(self):
|
||||
"""No voiceover path → return 0."""
|
||||
with patch.object(UnifiedRenderService, "__init__", lambda self, **kwargs: None):
|
||||
service = UnifiedRenderService.__new__(UnifiedRenderService)
|
||||
service.voiceover_audio_path = None
|
||||
|
||||
assert service._get_voice_audio_duration() == 0.0
|
||||
|
||||
def test_nonexistent_file_returns_zero(self):
|
||||
"""Nonexistent file → return 0."""
|
||||
with patch.object(UnifiedRenderService, "__init__", lambda self, **kwargs: None):
|
||||
service = UnifiedRenderService.__new__(UnifiedRenderService)
|
||||
service.voiceover_audio_path = "/nonexistent/path.mp3"
|
||||
|
||||
assert service._get_voice_audio_duration() == 0.0
|
||||
|
||||
@patch("video_processing.unified_render_service.probe_duration")
|
||||
@patch("video_processing.unified_render_service.Path.exists", return_value=True)
|
||||
@patch("video_processing.unified_render_service.Path.stat")
|
||||
def test_probes_duration_from_file(self, mock_stat, mock_exists, mock_probe):
|
||||
"""Valid file → probe duration."""
|
||||
mock_stat.return_value.st_size = 1000 # Non-empty file
|
||||
mock_probe.return_value = 42.5
|
||||
|
||||
with patch.object(UnifiedRenderService, "__init__", lambda self, **kwargs: None):
|
||||
service = UnifiedRenderService.__new__(UnifiedRenderService)
|
||||
service.voiceover_audio_path = "/tmp/voice.mp3"
|
||||
|
||||
assert service._get_voice_audio_duration() == 42.5
|
||||
Reference in New Issue
Block a user