fix(#1714): 同名兜底去重误杀新视频 + ingest 链路孤儿清理/启动恢复 #1734

Merged
auto-approve-bot merged 2 commits from fix/fallback-dedup-false-positive-1714 into develop 2026-09-06 13:41:28 +08:00
10 changed files with 963 additions and 50 deletions
+29 -43
View File
@@ -108,9 +108,8 @@ def _infer_mime_type_from_storage_key(storage_key: str) -> str:
return "video/mp4" # default
# 兜底去重:无 file_hash / client_upload_id 时,同库同名近期活动记录视为重复
# 兜底去重:无 file_hash / client_upload_id 且大小已知时,同库同名同大小近期活动记录视为重复
FALLBACK_DEDUP_WINDOW_MINUTES = 30
ACTIVE_ASSET_STATUSES = (AssetStatus.UPLOADING, AssetStatus.PROCESSING)
def _find_duplicate_asset(
@@ -126,8 +125,12 @@ def _find_duplicate_asset(
1. client_upload_id(客户端幂等 token,同一次上传的重试保持一致)
2. file_hash(内容哈希,不同上传只要内容相同即去重)
3. 兜底:同库 + 同文件名(+同大小)且 30 分钟内仍处 uploading/processing
的记录——旧客户端不传 hash/token 时,防止 complete 超时重试反复建占位。
3. 兜底(严格模式,宁可漏判不可误杀):file_hash 与 client_upload_id
均缺失、且 file_size > 0 时,同库 + 同文件名 + **同大小** 且 30 分钟内
仍处 uploading/processing 的记录才判重。
- file_hash 非空时跳过兜底(hash 已代表内容;同名但内容全新的视频
如 iPhone 的 IMG_xxxx.MOV 绝不能被同名占位误杀)
- file_size=0(未知)时不允许仅凭同名 + processing 判重,直接放行
全部为鸭子类型调用:旧仓储无对应方法时静默跳过,不破坏既有实现。
"""
@@ -156,53 +159,35 @@ def _find_duplicate_asset(
existing.id,
)
return existing
if filename:
# 同名兜底去重(最后防线,严格模式):
# - 仅当 file_hash / client_upload_id 均缺失时启用(hash 能代表内容时不靠同名猜)
# - file_size 必须 > 0 且与记录大小严格一致;大小未知(0)直接放行
# - 只命中近期 UPLOADING/PROCESSING 活动记录(READY 历史素材不拦)
if filename and not file_hash and not client_upload_id and file_size and file_size > 0:
find_recent = getattr(asset_repository, "find_recent_active_by_library_and_name", None)
if callable(find_recent):
existing = find_recent(
library_id=library_id,
name=filename,
within_minutes=FALLBACK_DEDUP_WINDOW_MINUTES,
file_size=file_size or 0,
file_size=file_size,
)
# 兜底去重:按状态区分处理
# - READY/ERROR:稳定素材,总命中(避免重复创建)
# - PROCESSING/UPLOADING:预建或 complete 占位,仅当 hash 一致才命中
# - 占位无 hash(旧客户端 complete 建的)→ 命中
# - 占位有 hash 且与当前请求 hash 一致 → 命中
# - 占位有 hash 且与当前请求 hash 不同 → 跳过(内容不同)
if existing is not None:
status = getattr(existing, "status", None)
existing_hash = getattr(existing, "file_hash", "") or ""
if status in (AssetStatus.READY, AssetStatus.ERROR):
logger.info(
"素材幂等兜底命中(近期同名稳定记录): library=%s name=%s asset=%s status=%s",
library_id,
filename,
getattr(existing, "id", "?"),
status,
)
return existing
elif status in ACTIVE_ASSET_STATUSES:
if existing_hash and file_hash and existing_hash != file_hash:
logger.debug(
"素材兜底去重跳过(占位 hash 不同): library=%s name=%s asset=%s hash=%s req_hash=%s",
library_id,
filename,
getattr(existing, "id", "?"),
existing_hash,
file_hash,
)
existing = None
else:
logger.info(
"素材幂等兜底命中(近期同名活动记录): library=%s name=%s asset=%s status=%s",
library_id,
filename,
getattr(existing, "id", "?"),
status,
)
return existing
logger.info(
"素材幂等兜底命中(近期同名同大小活动记录): library=%s name=%s asset=%s status=%s size=%s",
library_id,
filename,
getattr(existing, "id", "?"),
getattr(existing, "status", None),
file_size,
)
return existing
elif filename and not file_hash and not client_upload_id and not file_size:
logger.debug(
"同名兜底去重跳过(file_size 未知,宁可放行不可误杀): library=%s name=%s",
library_id,
filename,
)
return None
@@ -487,6 +472,7 @@ async def complete_direct_upload(
user_id=authenticated_user.user.id,
file_hash=request.file_hash,
client_upload_id=request.client_upload_id,
file_size=request.file_size,
)
job = _submit_ingest_job(
+13
View File
@@ -17,6 +17,12 @@ apply_queue_settings(celery_app)
# 长渲染任务预取 1,避免任务被预取占住导致调度不均
celery_app.conf.worker_prefetch_multiplier = GENERATION_WORKER_PREFETCH_MULTIPLIER
celery_app.conf.task_acks_late = True # worker 崩溃时未完成任务重回队列,由执行前守卫丢弃作废消息
# worker 进程被 OOM/容器硬杀时拒绝 ack,消息留在队列由其他 worker 接手
celery_app.conf.task_reject_on_worker_lost = True
# Redis broker 消息可见性超时(#1714):acks_late 下,消息被预取后 visibility_timeout
# 内未 ack 才会重投。长任务(ingest HEVC 转码 20-30 分钟、生成硬超时 11 分钟)
# 必须远大于最长执行时间,否则正常任务会在执行中被误重投;4 小时覆盖最长转码 + 余量。
celery_app.conf.broker_transport_options = {"visibility_timeout": 4 * 60 * 60}
celery_app.conf.imports = (
"worker_app.tasks.health",
@@ -48,4 +54,11 @@ celery_app.conf.beat_schedule = {
"schedule": 300.0, # 每 5 分钟(秒)
"options": {"expires": 240},
},
# 上传/转码链路孤儿巡检:worker 重启丢 prefetch 消息后,卡 pending/processing
# 的 ingest_job + asset 占位超时标终态(#1714)。转码任务较长,10 分钟一轮
"cleanup-stale-ingest-jobs": {
"task": "worker.cleanup_stale_ingest_jobs",
"schedule": 600.0, # 每 10 分钟(秒)
"options": {"expires": 540},
},
}
+29
View File
@@ -257,3 +257,32 @@ def _on_worker_ready(sender, **kwargs): # pragma: no cover
result = cleanup_all_stale_tasks()
total = result["generation_tasks"] + result["jobs"]
logger.info("Worker 启动清理完成,共清理 %d 个孤儿任务", total)
@worker_ready.connect
def _recover_stuck_ingest_jobs_on_ready(sender, **kwargs): # pragma: no cover
"""Worker 启动完成后恢复卡死在 processing 的 ingest_job#1714)。
容器重启/进程 OOM 导致 transcode 队列 unacked 消息未重投时,processing
ingest_job 会永久卡死。启动时扫描 processing 超 10 分钟的 jobCAS 重置
pending 并重新派单;Redis 锁保证同容器 generation/transcode 双 worker
只有一个执行恢复。旧消息若后来重投,ingest_asset 执行前守卫会丢弃。
"""
try:
from packages.application.ingest_orphan_cleanup import (
make_redis_recovery_lock,
recover_stuck_ingest_jobs_on_startup,
)
session = SessionLocal()
try:
recovered = recover_stuck_ingest_jobs_on_startup(
session,
lock_acquire=make_redis_recovery_lock(),
stuck_minutes=10,
)
finally:
session.close()
logger.info("Worker 启动 ingest 恢复完成,共重新派单 %d 个卡死任务", recovered)
except Exception as e: # noqa: BLE001 — 启动恢复失败不能阻断 worker 起服
logger.error("启动 ingest 恢复扫描失败(beat 巡检仍会兜底标 failed: %s", e, exc_info=True)
+56
View File
@@ -16,6 +16,12 @@ from worker_app.tasks._startup import (
cleanup_stale_pending_tasks,
)
from packages.application.ingest_orphan_cleanup import (
ASSET_ORPHAN_TIMEOUT_MINUTES,
INGEST_PENDING_TIMEOUT_MINUTES,
INGEST_PROCESSING_TIMEOUT_MINUTES,
)
logger = logging.getLogger(__name__)
@@ -69,3 +75,53 @@ def scheduled_cleanup_stale_running(timeout_minutes: int = ORPHAN_TASK_TIMEOUT_M
timeout_minutes,
)
return {"generation_tasks": gen_count, "jobs": job_count}
@shared_task(name="worker.cleanup_stale_ingest_jobs")
def scheduled_cleanup_stale_ingest_jobs(
processing_timeout_minutes: int = INGEST_PROCESSING_TIMEOUT_MINUTES,
pending_timeout_minutes: int = INGEST_PENDING_TIMEOUT_MINUTES,
orphan_asset_timeout_minutes: int = ASSET_ORPHAN_TIMEOUT_MINUTES,
) -> dict:
"""Celery Beat 调度:清理上传/转码链路(IngestJob + Asset)孤儿记录。
每 10 分钟执行一次。worker 容器重启/进程 OOM 时,已 prefetch 的 transcode
celery 消息会丢失(队列里也不存在),ingest_job 永久卡 pending/processing、
asset 永久卡 processing/uploading,没有兜底永远不会恢复(#1714)。
- ingest_job processing > processing_timeout_minutes / pending > pending_timeout_minutes
→ 标 failed;关联 asset 占位(processing/uploading)联动标 error
- 无 ingest_job 关联、created_at > orphan_asset_timeout_minutes 的占位 asset
→ 标 error
- 作废 celery 消息 revoke + 物理清除(防重投,执行前守卫是第二道防线)
"""
from worker_app.db import SessionLocal
from packages.application.ingest_orphan_cleanup import (
cleanup_orphan_processing_assets,
cleanup_stale_ingest_jobs,
revoke_stale_ingest_messages,
)
session = SessionLocal()
try:
job_items, asset_ids = cleanup_stale_ingest_jobs(
session,
processing_timeout_minutes=processing_timeout_minutes,
pending_timeout_minutes=pending_timeout_minutes,
)
orphan_asset_ids = cleanup_orphan_processing_assets(session, timeout_minutes=orphan_asset_timeout_minutes)
finally:
session.close()
purged = revoke_stale_ingest_messages(job_items) if job_items else 0
total_jobs = len(job_items)
total_assets = len(set(asset_ids) | set(orphan_asset_ids))
if total_jobs or total_assets:
logger.warning(
"[Beat] 清理 ingest 链路孤儿: stale_jobs=%d, assets→error=%d, 队列清除消息=%d",
total_jobs,
total_assets,
purged,
)
return {"stale_jobs": total_jobs, "assets_to_error": total_assets, "purged_messages": purged}
@@ -488,20 +488,25 @@ class SQLAlchemyAssetRepository:
用于旧客户端未传 file_hash/client_upload_id 时,防止 complete 超时重试
反复创建 PROCESSING 占位记录。只命中"活动中"的近期记录,READY 历史素材不拦。
严格模式(#1714 误杀修复):file_size 必须 > 0 且与记录大小严格一致;
file_size=0(大小未知)时直接返回 None——宁可漏判(极端情况下多建一条
占位)也不可仅凭同名 + processing 误杀内容全新的视频。
"""
from datetime import datetime, timedelta, timezone
if not name:
return None
if not file_size or file_size <= 0:
return None
cutoff = datetime.now(timezone.utc) - timedelta(minutes=within_minutes)
query = self.session.query(AssetModel).filter(
AssetModel.asset_library_id == library_id,
AssetModel.name == name,
AssetModel.status.in_([AssetStatus.UPLOADING.value, AssetStatus.PROCESSING.value]),
AssetModel.created_at >= cutoff,
AssetModel.file_size == file_size,
)
if file_size and file_size > 0:
query = query.filter(AssetModel.file_size == file_size)
model = query.order_by(AssetModel.created_at.desc()).first()
if model is None:
return None
@@ -0,0 +1,310 @@
"""上传/转码链路(IngestJob + Asset)孤儿清理核心逻辑。
#1714generation 链路有 cleanup_stale_running/pending 兜底,但上传链路
ingest_jobs + assets)没有。worker 容器重启/进程 OOM 时,已 prefetch 的
celery 消息会丢失(transcode 队列 worker_prefetch_multiplier=1,消息预取后
宕机即丢失,Redis 队列里也不再存在),导致:
- ingest_jobs.status 永久卡 pending/processing
- assets.status 永久卡 processing/uploadingcomplete 阶段预建的占位)
本模块提供纯核心(session 注入,便于单测):超时阈值内无更新的记录
批量标终态(job→failed、asset→error),并返回 (job_id, celery_task_id)
列表供调用方 revoke + purge 残留队列消息。
"""
from __future__ import annotations
import logging
from datetime import datetime, timedelta, timezone
from typing import Any, Callable
logger = logging.getLogger(__name__)
# ingest_job PROCESSING 超时阈值:ingest 任务包含下载 + ffprobe + HEVC 转码
# 1GB 视频约 10-20 分钟)+ 回传 OSS,正常任务可能跑 20-30 分钟;
# 60 分钟阈值覆盖大文件转码 + 抖动,绝不误杀正常任务。
INGEST_PROCESSING_TIMEOUT_MINUTES = 60
# ingest_job PENDING 超时阈值:transcode 队列 concurrency=1,队列积压时
# 正常排队可能较久;90 分钟覆盖 worker 短暂停消费 + 排队。
INGEST_PENDING_TIMEOUT_MINUTES = 90
# Asset 占位超时阈值:无关联 ingest_job 的孤儿占位(complete 预建后派单失败等),
# 阈值放宽到 120 分钟,避免与 ingest_job 生命周期错杀。
ASSET_ORPHAN_TIMEOUT_MINUTES = 120
_TERMINAL_JOB_STATUSES = ("failed", "completed")
_TERMINAL_ASSET_STATUSES = ("ready", "error", "deleted")
def _now() -> datetime:
return datetime.now(timezone.utc)
def cleanup_stale_ingest_jobs(
session: Any,
*,
processing_timeout_minutes: int = INGEST_PROCESSING_TIMEOUT_MINUTES,
pending_timeout_minutes: int = INGEST_PENDING_TIMEOUT_MINUTES,
commit: bool = True,
) -> tuple[list[tuple[str, str]], list[str]]:
"""清理超时卡 pending/processing 的 ingest_jobs,并联动关联 asset。
Args:
session: SQLAlchemy session(或提供 query/commit 的鸭子类型)
processing_timeout_minutes: processing 状态超时阈值
pending_timeout_minutes: pending 状态超时阈值
commit: 是否提交事务
Returns:
(job_items, asset_ids)
- job_items: [(job_id, celery_task_id), ...] 供 revoke/purge
- asset_ids: 被联动标记为 error 的 asset id 列表
"""
from packages.adapters.sqlalchemy_impl.models import AssetModel, IngestJobModel
now = _now()
processing_cutoff = now - timedelta(minutes=processing_timeout_minutes)
pending_cutoff = now - timedelta(minutes=pending_timeout_minutes)
stale_jobs = (
session.query(IngestJobModel)
.filter(
IngestJobModel.status.in_(["pending", "processing"]),
(
(IngestJobModel.status == "processing") & (IngestJobModel.updated_at < processing_cutoff)
| (IngestJobModel.status == "pending") & (IngestJobModel.created_at < pending_cutoff)
),
)
.all()
)
job_items: list[tuple[str, str]] = []
asset_ids: list[str] = []
stale_asset_models: list[Any] = []
for job_model in stale_jobs:
ref_time = job_model.updated_at or job_model.created_at
if ref_time.tzinfo is None: # SQLite 读回 naive datetime 的防御
ref_time = ref_time.replace(tzinfo=timezone.utc)
stale_minutes = int((now - ref_time).total_seconds() // 60)
job_model.status = "failed"
job_model.error_message = (
f"转码任务执行中断(超过超时阈值未更新,疑似 worker 重启/进程退出,已卡死 {stale_minutes} 分钟)"
)
job_model.updated_at = now
job_items.append((job_model.id, getattr(job_model, "celery_task_id", "") or ""))
if job_model.asset_id:
asset_ids.append(job_model.asset_id)
if asset_ids:
stale_asset_models = (
session.query(AssetModel)
.filter(
AssetModel.id.in_(asset_ids),
AssetModel.status.in_(["processing", "uploading"]),
)
.all()
)
for asset_model in stale_asset_models:
asset_model.status = "error"
asset_model.updated_at = now
if commit and (job_items or stale_asset_models):
session.commit()
if job_items:
logger.warning(
"[ingest-cleanup] 清理 %d 个超时 ingest_jobprocessing>%dm / pending>%dm),联动 %d 个 asset 标 error",
len(job_items),
processing_timeout_minutes,
pending_timeout_minutes,
len(stale_asset_models),
)
return job_items, [a.id for a in stale_asset_models]
def cleanup_orphan_processing_assets(
session: Any,
*,
timeout_minutes: int = ASSET_ORPHAN_TIMEOUT_MINUTES,
commit: bool = True,
) -> list[str]:
"""清理无 ingest_job 关联、超时卡 processing/uploading 的孤儿 asset 占位。
complete 阶段预建 asset 后若派单失败(或 direct 上传 complete 后
未触发 ingest),占位会永久卡住。这类 asset 没有对应 ingest_job
只能按 created_at 超时兜底标 error。
"""
from packages.adapters.sqlalchemy_impl.models import AssetModel, IngestJobModel
cutoff = _now() - timedelta(minutes=timeout_minutes)
orphan_assets = (
session.query(AssetModel)
.outerjoin(IngestJobModel, IngestJobModel.asset_id == AssetModel.id)
.filter(
AssetModel.status.in_(["processing", "uploading"]),
AssetModel.created_at < cutoff,
IngestJobModel.id.is_(None),
)
.all()
)
for asset_model in orphan_assets:
asset_model.status = "error"
asset_model.updated_at = _now()
if commit and orphan_assets:
session.commit()
logger.warning("[ingest-cleanup] 清理 %d 个无 job 关联的超时孤儿 asset 占位", len(orphan_assets))
return [a.id for a in orphan_assets]
def revoke_stale_ingest_messages(
job_items: list[tuple[str, str]],
*,
celery_app_factory: Callable[[], Any] | None = None,
broker_url_factory: Callable[[], str] | None = None,
) -> int:
"""revoke + 物理清理 ingest 作废消息(transcode/celery 队列)。
消息可能已在 worker 宕机时丢失(队列里查不到),那也无害;
若消息还在(极端重复投递),物理清除防止重投执行。
失败不阻断清理(ingest_asset 的执行前状态守卫是第二道防线)。
"""
biz_ids = [jid for jid, _ in job_items if jid]
celery_ids = [cid for _, cid in job_items if cid]
if not biz_ids and not celery_ids:
return 0
try:
from packages.shared.celery_orphan_guard import revoke_and_purge
app = celery_app_factory() if celery_app_factory else None
broker_url = broker_url_factory() if broker_url_factory else ""
if app is None or not broker_url:
from worker_app.celery_app import celery_app as _app
from worker_app.core.config import get_settings
app = _app
broker_url = get_settings().broker_url
return revoke_and_purge(
app,
broker_url,
business_task_ids=biz_ids,
celery_task_ids=celery_ids,
queue_names=("transcode", "celery"),
)
except Exception as e: # noqa: BLE001
logger.error("撤销作废 ingest 队列消息失败(执行前守卫仍会兜底): %s", e, exc_info=True)
return 0
# ── worker 启动恢复(#1714)──────────────────────────────────────────────
#
# task_acks_late=True 下,worker 崩溃/容器重启时未 ack 的消息理论上会在
# visibility_timeout 到期后重新投递;但 prefork 进程异常、部署窗口跨
# visibility 配置边界等场景仍可能留下卡在 processing 的 ingest_job
# staging 实证:03:16 派单、03:45 置 processing 后 worker 重启,
# unacked 消息未重投,任务永久卡死)。启动时做一次显式恢复扫描兜底。
#
# 恢复策略:processing 超过 stuck_minutes(默认 10 分钟,部署中跨进程
# 交接的正常窗口 < 10 分钟,不会误抢别的 worker 正在执行的任务)的 job,
# CAS 重置为 pending 并重新 send_task;旧消息若后来重投,ingest_asset
# 的执行前守卫会把状态不匹配的旧 celery 消息丢弃。
def recover_stuck_ingest_jobs_on_startup(
session: Any,
*,
send_task: Callable[..., Any] | None = None,
update_celery_task_id: Callable[[str, str], None] | None = None,
lock_acquire: Callable[[], bool] | None = None,
stuck_minutes: int = 10,
commit: bool = True,
) -> int:
"""worker 启动时把卡在 processing 超时的 ingest_job 重新派单。
Args:
session: SQLAlchemy session
send_task: celery send_task 可调用(注入便于测试);不传则用 worker celery_app
update_celery_task_id: 回写新 celery task id 的回调(job_id, new_task_id
lock_acquire: 分布式锁获取回调(多 worker 进程同时启动时只允许一个恢复);
返回 False 表示未抢到锁,本次跳过
stuck_minutes: processing 超过该分钟数视为卡死
Returns:
重新派单的 job 数
"""
if lock_acquire is not None and not lock_acquire():
logger.info("[ingest-recover] 未抢到恢复锁,跳过(另一进程正在恢复)")
return 0
from packages.adapters.sqlalchemy_impl.models import IngestJobModel
cutoff = _now() - timedelta(minutes=stuck_minutes)
stuck_jobs = (
session.query(IngestJobModel)
.filter(IngestJobModel.status == "processing", IngestJobModel.updated_at < cutoff)
.order_by(IngestJobModel.updated_at.asc())
.all()
)
if not stuck_jobs:
logger.info("[ingest-recover] 无卡死 processing ingest_job 需要恢复")
return 0
if send_task is None:
from worker_app.celery_app import celery_app as _app
send_task = _app.send_task
recovered = 0
for job_model in stuck_jobs:
# CAS:只有仍是 processing 才重置(并发/旧消息已回写终态时不碰)
updated = (
session.query(IngestJobModel)
.filter(IngestJobModel.id == job_model.id, IngestJobModel.status == "processing")
.update({"status": "pending", "error_message": "", "updated_at": _now()})
)
if not updated:
continue
try:
result = send_task("worker.ingest_asset", args=[job_model.id])
new_task_id = getattr(result, "id", "") or ""
except Exception as e: # noqa: BLE001
logger.error("[ingest-recover] 重新派单失败 job_id=%s: %s", job_model.id, e)
continue
if new_task_id:
job_model.celery_task_id = new_task_id
if update_celery_task_id is not None:
update_celery_task_id(job_model.id, new_task_id)
logger.warning(
"[ingest-recover] 卡死 ingest_job %s 已重置 pending 并重新派单 (new celery task=%s)",
job_model.id,
new_task_id,
)
recovered += 1
if commit and recovered:
session.commit()
logger.warning("[ingest-recover] 启动恢复完成,共重新派单 %d 个卡死 ingest_job", recovered)
return recovered
def make_redis_recovery_lock(lock_key: str = "ingest:recover:startup", ttl_seconds: int = 300):
"""构造基于 Redis SET NX 的恢复锁工厂(多 worker 进程互斥)。
返回一个无参 callable,调用时尝试抢锁:抢到返回 True,未抢到返回 False。
Redis 不可用时不阻断启动恢复(返回 True,恢复逻辑自身有 CAS 幂等保护)。
"""
def _acquire() -> bool:
try:
import redis as redis_lib
from worker_app.core.config import get_settings
client = redis_lib.Redis.from_url(get_settings().broker_url)
return bool(client.set(lock_key, "1", nx=True, ex=ttl_seconds))
except Exception as e: # noqa: BLE001
logger.warning("[ingest-recover] Redis 锁不可用,降级为无锁执行(CAS 兜底): %s", e)
return True
return _acquire
@@ -0,0 +1,93 @@
"""#1714 find_recent_active_by_library_and_name 严格模式测试。
file_size=0(未知)时必须返回 None(宁可漏判不可误杀);
大小严格匹配;只命中近期 UPLOADING/PROCESSING 记录。
"""
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "api"))
from sqlalchemy import create_engine # noqa: E402
from sqlalchemy.orm import sessionmaker # noqa: E402
from packages.adapters.sqlalchemy_impl.asset_repository import SQLAlchemyAssetRepository # noqa: E402
from packages.adapters.sqlalchemy_impl.models import Base # noqa: E402
from packages.domain import Asset, AssetStatus # noqa: E402
def _repository():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
session = sessionmaker(bind=engine)()
return SQLAlchemyAssetRepository(session)
def _mk_asset(name="IMG_2285.MOV", file_size=5_000_000, status=AssetStatus.PROCESSING, minutes_ago=5):
asset = Asset.create(
project_id="proj-1",
library_id="lib-1",
name=name,
storage_key=f"uploads/x/{name}",
mime_type="video/quicktime",
file_size=file_size,
)
asset.status = status
asset.created_at = datetime.now(timezone.utc) - timedelta(minutes=minutes_ago)
return asset
def test_returns_none_when_file_size_zero():
"""file_size=0(大小未知)直接返回 None——不许仅凭同名 + processing 判重。"""
repo = _repository()
repo.create(_mk_asset(file_size=0))
result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="IMG_2285.MOV", file_size=0)
assert result is None
def test_matches_when_name_size_strict_equal():
"""同名 + 同大小 + processing 近期记录 → 命中。"""
repo = _repository()
repo.create(_mk_asset(file_size=5_000_000))
result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="IMG_2285.MOV", file_size=5_000_000)
assert result is not None
assert result.name == "IMG_2285.MOV"
def test_no_match_when_same_name_but_different_size():
"""同名但大小不同 → 不命中(内容全新的视频不能误杀)。"""
repo = _repository()
repo.create(_mk_asset(file_size=5_000_000))
result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="IMG_2285.MOV", file_size=9_999_999)
assert result is None
def test_no_match_ready_history_even_with_same_size():
"""READY 历史同名素材不命中(允许再次上传同名文件)。"""
repo = _repository()
repo.create(_mk_asset(file_size=5_000_000, status=AssetStatus.READY))
result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="IMG_2285.MOV", file_size=5_000_000)
assert result is None
def test_no_match_when_window_expired():
"""超过 30 分钟窗口的活动记录不命中。"""
repo = _repository()
repo.create(_mk_asset(file_size=5_000_000, minutes_ago=45))
result = repo.find_recent_active_by_library_and_name(
library_id="lib-1", name="IMG_2285.MOV", within_minutes=30, file_size=5_000_000
)
assert result is None
def test_returns_none_when_name_empty():
repo = _repository()
result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="", file_size=100)
assert result is None
@@ -0,0 +1,75 @@
"""#1714 beat 任务 scheduled_cleanup_stale_ingest_jobs 薄封装测试。
mock SessionLocal 和清理核心,验证 beat 任务正确串联
cleanup_stale_ingest_jobs → cleanup_orphan_processing_assets → revoke 消息。
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
from unittest.mock import MagicMock, patch
os.environ.setdefault("JWT_SECRET_KEY", "unit-test-secret")
os.environ.setdefault("DATABASE_URL", "sqlite:///test_beat.db")
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "worker"))
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "api"))
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
import worker_app.tasks.cleanup as cleanup # noqa: E402
def test_beat_cleanup_calls_core_and_revokes():
"""beat 任务串联三个核心步骤,返回汇总计数。"""
fake_session = MagicMock()
with (
patch("worker_app.db.SessionLocal", return_value=fake_session) as m_db,
patch(
"packages.application.ingest_orphan_cleanup.cleanup_stale_ingest_jobs",
return_value=([("job-1", "cel-1"), ("job-2", "")], ["a-1"]),
) as m_jobs,
patch(
"packages.application.ingest_orphan_cleanup.cleanup_orphan_processing_assets",
return_value=["a-2"],
) as m_assets,
patch(
"packages.shared.celery_orphan_guard.revoke_and_purge",
return_value=1,
) as m_revoke,
):
result = cleanup.scheduled_cleanup_stale_ingest_jobs()
m_db.assert_called_once()
m_jobs.assert_called_once()
assert m_jobs.call_args.kwargs["processing_timeout_minutes"] == 60
m_assets.assert_called_once()
m_revoke.assert_called_once()
# 队列名只传 transcode/celery(不传 generation
assert m_revoke.call_args.kwargs["queue_names"] == ("transcode", "celery")
fake_session.close.assert_called_once()
assert result == {"stale_jobs": 2, "assets_to_error": 2, "purged_messages": 1}
def test_beat_cleanup_no_op_when_nothing_stale():
"""无孤儿时不调 revoke,返回全 0。"""
fake_session = MagicMock()
with (
patch("worker_app.db.SessionLocal", return_value=fake_session),
patch(
"packages.application.ingest_orphan_cleanup.cleanup_stale_ingest_jobs",
return_value=([], []),
),
patch(
"packages.application.ingest_orphan_cleanup.cleanup_orphan_processing_assets",
return_value=[],
),
patch("packages.shared.celery_orphan_guard.revoke_and_purge") as m_revoke,
):
result = cleanup.scheduled_cleanup_stale_ingest_jobs()
m_revoke.assert_not_called()
assert result == {"stale_jobs": 0, "assets_to_error": 0, "purged_messages": 0}
@@ -0,0 +1,262 @@
"""#1714 上传/转码链路(IngestJob + Asset)孤儿清理测试。
场景:worker 容器重启/进程 OOM 时,已 prefetch 的 transcode celery 消息丢失,
ingest_job 永久卡 pending/processing、asset 永久卡 processing/uploading。
"""
from __future__ import annotations
import os
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest
os.environ.setdefault("JWT_SECRET_KEY", "unit-test-secret")
os.environ.setdefault("DATABASE_URL", "sqlite:///test_ingest_orphan.db")
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "worker"))
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "api"))
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from sqlalchemy import create_engine # noqa: E402
from sqlalchemy.orm import sessionmaker # noqa: E402
from packages.adapters.sqlalchemy_impl.models import AssetModel, Base, IngestJobModel # noqa: E402
from packages.application.ingest_orphan_cleanup import ( # noqa: E402
cleanup_orphan_processing_assets,
cleanup_stale_ingest_jobs,
)
@pytest.fixture()
def session():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(bind=engine)
Session = sessionmaker(bind=engine)
db = Session()
yield db
db.close()
def _mk_job(session, *, status="processing", celery_task_id="cel-1", asset_id="a-1", minutes_ago=90):
now = datetime.now(timezone.utc)
job = IngestJobModel(
id=f"job-{minutes_ago}-{status}-{celery_task_id}",
project_id="p-1",
library_id="lib-1",
storage_key="uploads/x/IMG_2285.MOV",
status=status,
asset_id=asset_id,
celery_task_id=celery_task_id,
created_at=now - timedelta(minutes=minutes_ago),
updated_at=now - timedelta(minutes=minutes_ago),
)
session.add(job)
session.commit()
return job
def _mk_asset(session, *, id="a-1", status="processing", minutes_ago=90, file_size=0):
now = datetime.now(timezone.utc)
asset = AssetModel(
id=id,
project_id="p-1",
asset_library_id="lib-1",
name="IMG_2285.MOV",
file_type="video",
file_size=file_size,
file_url="https://example.com/x.mov",
storage_key="uploads/x/IMG_2285.MOV",
status=status,
uploaded_by_user_id="u-1",
created_at=now - timedelta(minutes=minutes_ago),
updated_at=now - timedelta(minutes=minutes_ago),
)
session.add(asset)
session.commit()
return asset
class TestCleanupStaleIngestJobs:
def test_stale_processing_job_marked_failed_and_asset_to_error(self, session):
"""processing 超 60 分钟 → job failed,关联 processing asset → error。"""
_mk_asset(session, id="a-1", status="processing")
_mk_job(session, status="processing", celery_task_id="cel-dead", asset_id="a-1", minutes_ago=90)
items, asset_ids = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90)
assert len(items) == 1
assert items[0] == ("job-90-processing-cel-dead", "cel-dead")
assert asset_ids == ["a-1"]
db_job = session.query(IngestJobModel).one()
assert db_job.status == "failed"
assert "中断" in db_job.error_message
db_asset = session.query(AssetModel).one()
assert db_asset.status == "error"
def test_stale_pending_job_marked_failed(self, session):
"""pending 超 90 分钟(从未被消费)→ job failed。"""
_mk_asset(session, id="a-2", status="uploading")
_mk_job(session, status="pending", celery_task_id="", asset_id="a-2", minutes_ago=120)
items, asset_ids = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90)
assert len(items) == 1
assert items[0][1] == "" # 无 celery task id
assert session.query(IngestJobModel).one().status == "failed"
assert session.query(AssetModel).one().status == "error"
def test_recent_processing_job_not_touched(self, session):
"""processing 仅 10 分钟(正常转码中)→ 不误杀。"""
_mk_asset(session, id="a-3", status="processing", minutes_ago=10)
_mk_job(session, status="processing", celery_task_id="cel-live", asset_id="a-3", minutes_ago=10)
items, asset_ids = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90)
assert items == []
assert asset_ids == []
assert session.query(IngestJobModel).one().status == "processing"
assert session.query(AssetModel).one().status == "processing"
def test_recent_pending_job_not_touched(self, session):
"""pending 仅 30 分钟(队列积压排队中)→ 不误杀。"""
_mk_job(session, status="pending", asset_id="", minutes_ago=30)
items, _ = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90)
assert items == []
assert session.query(IngestJobModel).one().status == "pending"
def test_terminal_job_not_touched(self, session):
"""已 completed/failed 的 job 不动。"""
_mk_job(session, status="completed", celery_task_id="", asset_id="", minutes_ago=999)
_mk_job(session, status="failed", celery_task_id="", asset_id="", minutes_ago=999)
items, _ = cleanup_stale_ingest_jobs(session)
assert items == []
statuses = sorted(j.status for j in session.query(IngestJobModel).all())
assert statuses == ["completed", "failed"]
def test_ready_asset_not_demoted(self, session):
"""关联 asset 已是 ready(转码其实成功了,仅 job 回写失败)→ 不降级为 error。"""
_mk_asset(session, id="a-4", status="ready")
_mk_job(session, status="processing", celery_task_id="cel-x", asset_id="a-4", minutes_ago=90)
_, asset_ids = cleanup_stale_ingest_jobs(session)
assert asset_ids == [] # ready 不动
assert session.query(AssetModel).one().status == "ready"
class TestCleanupOrphanProcessingAssets:
def test_orphan_asset_without_job_marked_error(self, session):
"""无 ingest_job 关联、created 超 120 分钟的 processing 占位 → error。"""
_mk_asset(session, id="orphan-1", status="processing", minutes_ago=150)
ids = cleanup_orphan_processing_assets(session, timeout_minutes=120)
assert ids == ["orphan-1"]
assert session.query(AssetModel).one().status == "error"
def test_asset_with_active_job_not_touched(self, session):
"""有 processing job 关联的 asset 不由本函数处理(归 cleanup_stale_ingest_jobs)。"""
_mk_asset(session, id="a-5", status="processing", minutes_ago=150)
_mk_job(session, status="processing", asset_id="a-5", minutes_ago=150)
ids = cleanup_orphan_processing_assets(session, timeout_minutes=120)
assert ids == []
assert session.query(AssetModel).one().status == "processing"
def test_recent_orphan_asset_not_touched(self, session):
"""无 job 但才创建 30 分钟 → 可能 complete 刚建、job 派单中,不动。"""
_mk_asset(session, id="orphan-2", status="processing", minutes_ago=30)
ids = cleanup_orphan_processing_assets(session, timeout_minutes=120)
assert ids == []
assert session.query(AssetModel).one().status == "processing"
class TestRecoverStuckIngestJobsOnStartup:
def test_stuck_processing_job_requeued(self, session):
"""processing 超 10 分钟 → 重置 pending 并重新 send_task,回写新 celery id。"""
from types import SimpleNamespace
from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup
job = _mk_job(session, status="processing", celery_task_id="old-cel-1", asset_id="a-1", minutes_ago=30)
sent = []
def fake_send_task(name, args=None, **kw):
sent.append((name, args))
return SimpleNamespace(id="new-cel-9")
updated_ids = []
recovered = recover_stuck_ingest_jobs_on_startup(
session,
send_task=fake_send_task,
update_celery_task_id=lambda jid, cid: updated_ids.append((jid, cid)),
stuck_minutes=10,
)
assert recovered == 1
assert sent == [("worker.ingest_asset", [job.id])]
refreshed = session.query(IngestJobModel).filter_by(id=job.id).one()
assert refreshed.status == "pending"
assert refreshed.celery_task_id == "new-cel-9"
assert updated_ids == [(job.id, "new-cel-9")]
def test_recent_processing_job_not_touched(self, session):
"""processing 仅 5 分钟(正常转码中/部署交接窗口)→ 不抢。"""
from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup
_mk_job(session, status="processing", celery_task_id="live", asset_id="", minutes_ago=5)
sent = []
recovered = recover_stuck_ingest_jobs_on_startup(
session,
send_task=lambda *a, **k: sent.append(a),
stuck_minutes=10,
)
assert recovered == 0
assert sent == []
assert session.query(IngestJobModel).one().status == "processing"
def test_lock_not_acquired_skips(self, session):
"""未抢到分布式锁(另一 worker 正在恢复)→ 跳过。"""
from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup
_mk_job(session, status="processing", celery_task_id="x", asset_id="", minutes_ago=30)
recovered = recover_stuck_ingest_jobs_on_startup(
session,
send_task=lambda *a, **k: None,
lock_acquire=lambda: False,
stuck_minutes=10,
)
assert recovered == 0
assert session.query(IngestJobModel).one().status == "processing"
def test_pending_and_terminal_not_requeued(self, session):
"""pending/已终态 job 不在恢复范围。"""
from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup
_mk_job(session, status="pending", celery_task_id="", asset_id="", minutes_ago=60)
_mk_job(session, status="failed", celery_task_id="", asset_id="", minutes_ago=60)
recovered = recover_stuck_ingest_jobs_on_startup(
session,
send_task=lambda *a, **k: None,
stuck_minutes=10,
)
assert recovered == 0
statuses = sorted(j.status for j in session.query(IngestJobModel).all())
assert statuses == ["failed", "pending"]
@@ -73,6 +73,9 @@ class StubAssetRepository:
def find_recent_active_by_library_and_name(
self, library_id: str, name: str, within_minutes: int = 30, file_size: int = 0
) -> Asset | None:
# 严格模式(#1714):大小未知(0)直接不命中,宁可漏判不可误杀
if not file_size or file_size <= 0:
return None
cutoff = datetime.now(timezone.utc) - timedelta(minutes=within_minutes)
candidates = [
a
@@ -81,7 +84,7 @@ class StubAssetRepository:
and a.name == name
and a.status in (AssetStatus.UPLOADING, AssetStatus.PROCESSING)
and a.created_at >= cutoff
and (not file_size or a.file_size == file_size)
and a.file_size == file_size
]
return max(candidates, key=lambda a: a.created_at) if candidates else None
@@ -234,14 +237,21 @@ class TestDirectCompleteIdempotency:
不应再建第二条。
"""
client, asset_repo, ingest_repo, _ = _client()
# 第一次 complete(旧客户端无 token/hash
r1 = client.post("/api/v1/direct/complete", json=COMPLETE_BODY)
# 第一次 complete(旧客户端无 token/hash,但 file_size 可知
r1 = client.post(
"/api/v1/direct/complete",
json={**COMPLETE_BODY, "file_size": 5_000_000},
)
assert r1.json()["duplicated"] is False
# 重试:重新 prepare 产生新 storage_key(仅 uuid 目录不同,文件名一致——
# 前端重试传的是同一个 File),且近期
# 前端重试传的是同一个 File),且近期;同大小才允许兜底命中
r2 = client.post(
"/api/v1/direct/complete",
json={**COMPLETE_BODY, "storage_key": "uploads/retry/IMG_2282.MOV", "file_size": 0},
json={
**COMPLETE_BODY,
"storage_key": "uploads/retry/IMG_2282.MOV",
"file_size": 5_000_000,
},
)
assert r2.status_code == 200
assert r2.json()["duplicated"] is True
@@ -249,6 +259,80 @@ class TestDirectCompleteIdempotency:
assert len(asset_repo.created) == 1
assert ingest_repo.created_count == 1
def test_fallback_dedup_skipped_when_file_size_unknown(self):
"""file_size=0(未知)时不允许仅凭同名 + processing 判重,直接放行(#1714)。
根因场景:complete 没传 file_size30 分钟内同名占位(如 iPhone 的
IMG_2285.MOV)会把内容/大小全新的视频误判为重复跳过。
"""
client, asset_repo, _ingest_repo, _ = _client()
r1 = client.post(
"/api/v1/direct/complete",
json={**COMPLETE_BODY, "file_size": 0},
)
assert r1.json()["duplicated"] is False
# 第二个全新视频:同名(IMG_2285.MOV)、无 hash/token、file_size 仍未知
r2 = client.post(
"/api/v1/direct/complete",
json={**COMPLETE_BODY, "storage_key": "uploads/retry2/IMG_2282.MOV", "file_size": 0},
)
assert r2.status_code == 200
assert r2.json()["duplicated"] is False # 不能误杀
assert len(asset_repo.created) == 2 # 两条记录,放行新上传
def test_fallback_dedup_skipped_when_same_name_but_different_size(self):
"""同名但 file_size 不同 → 不判重,正常建记录(#1714)。"""
client, asset_repo, _ingest_repo, _ = _client()
r1 = client.post(
"/api/v1/direct/complete",
json={**COMPLETE_BODY, "file_size": 5_000_000},
)
assert r1.json()["duplicated"] is False
r2 = client.post(
"/api/v1/direct/complete",
json={
**COMPLETE_BODY,
"storage_key": "uploads/retry3/IMG_2282.MOV",
"file_size": 9_999_999, # 同名但大小完全不同的新视频
},
)
assert r2.status_code == 200
assert r2.json()["duplicated"] is False
assert len(asset_repo.created) == 2
def test_fallback_dedup_skipped_when_hash_present_even_if_name_size_match(self):
"""file_hash 非空且 hash 未命中时,不允许退回同名兜底(#1714)。
hash 已能代表内容:同名同大小但 hash 不同是真实的新内容,必须放行。
"""
client, asset_repo, _ingest_repo, _ = _client()
# 第一次:某 hash 的视频
r1 = client.post(
"/api/v1/direct/complete",
json={
**COMPLETE_BODY,
"file_hash": "a" * 64,
"client_upload_id": "tok-1",
"file_size": 5_000_000,
},
)
assert r1.json()["duplicated"] is False
# 第二次:同名同大小但 hash 不同(新视频内容不同);
# 注意 client_upload_id 也必须不同,否则会先被 token 命中
r2 = client.post(
"/api/v1/direct/complete",
json={
**COMPLETE_BODY,
"storage_key": "uploads/retry4/IMG_2282.MOV",
"file_hash": "b" * 64,
"client_upload_id": "tok-2",
"file_size": 5_000_000,
},
)
assert r2.status_code == 200
assert r2.json()["duplicated"] is False
assert len(asset_repo.created) == 2
def test_fallback_dedup_ignores_ready_history(self):
"""READY 历史同名素材不触发兜底(允许用户再次上传同名文件)。"""
ready = Asset(