diff --git a/apps/api/app/api/routes/upload.py b/apps/api/app/api/routes/upload.py index 279888a78..08aed4e7b 100644 --- a/apps/api/app/api/routes/upload.py +++ b/apps/api/app/api/routes/upload.py @@ -108,9 +108,8 @@ def _infer_mime_type_from_storage_key(storage_key: str) -> str: return "video/mp4" # default -# 兜底去重:无 file_hash / client_upload_id 时,同库同名近期活动记录视为重复 +# 兜底去重:无 file_hash / client_upload_id 且大小已知时,同库同名同大小近期活动记录视为重复 FALLBACK_DEDUP_WINDOW_MINUTES = 30 -ACTIVE_ASSET_STATUSES = (AssetStatus.UPLOADING, AssetStatus.PROCESSING) def _find_duplicate_asset( @@ -126,8 +125,12 @@ def _find_duplicate_asset( 1. client_upload_id(客户端幂等 token,同一次上传的重试保持一致) 2. file_hash(内容哈希,不同上传只要内容相同即去重) - 3. 兜底:同库 + 同文件名(+同大小)且 30 分钟内仍处 uploading/processing - 的记录——旧客户端不传 hash/token 时,防止 complete 超时重试反复建占位。 + 3. 兜底(严格模式,宁可漏判不可误杀):file_hash 与 client_upload_id + 均缺失、且 file_size > 0 时,同库 + 同文件名 + **同大小** 且 30 分钟内 + 仍处 uploading/processing 的记录才判重。 + - file_hash 非空时跳过兜底(hash 已代表内容;同名但内容全新的视频 + 如 iPhone 的 IMG_xxxx.MOV 绝不能被同名占位误杀) + - file_size=0(未知)时不允许仅凭同名 + processing 判重,直接放行 全部为鸭子类型调用:旧仓储无对应方法时静默跳过,不破坏既有实现。 """ @@ -156,53 +159,35 @@ def _find_duplicate_asset( existing.id, ) return existing - if filename: + # 同名兜底去重(最后防线,严格模式): + # - 仅当 file_hash / client_upload_id 均缺失时启用(hash 能代表内容时不靠同名猜) + # - file_size 必须 > 0 且与记录大小严格一致;大小未知(0)直接放行 + # - 只命中近期 UPLOADING/PROCESSING 活动记录(READY 历史素材不拦) + if filename and not file_hash and not client_upload_id and file_size and file_size > 0: find_recent = getattr(asset_repository, "find_recent_active_by_library_and_name", None) if callable(find_recent): existing = find_recent( library_id=library_id, name=filename, within_minutes=FALLBACK_DEDUP_WINDOW_MINUTES, - file_size=file_size or 0, + file_size=file_size, ) - # 兜底去重:按状态区分处理 - # - READY/ERROR:稳定素材,总命中(避免重复创建) - # - PROCESSING/UPLOADING:预建或 complete 占位,仅当 hash 一致才命中 - # - 占位无 hash(旧客户端 complete 建的)→ 命中 - # - 占位有 hash 且与当前请求 hash 一致 → 命中 - # - 占位有 hash 且与当前请求 hash 不同 → 跳过(内容不同) if existing is not None: - status = getattr(existing, "status", None) - existing_hash = getattr(existing, "file_hash", "") or "" - if status in (AssetStatus.READY, AssetStatus.ERROR): - logger.info( - "素材幂等兜底命中(近期同名稳定记录): library=%s name=%s asset=%s status=%s", - library_id, - filename, - getattr(existing, "id", "?"), - status, - ) - return existing - elif status in ACTIVE_ASSET_STATUSES: - if existing_hash and file_hash and existing_hash != file_hash: - logger.debug( - "素材兜底去重跳过(占位 hash 不同): library=%s name=%s asset=%s hash=%s req_hash=%s", - library_id, - filename, - getattr(existing, "id", "?"), - existing_hash, - file_hash, - ) - existing = None - else: - logger.info( - "素材幂等兜底命中(近期同名活动记录): library=%s name=%s asset=%s status=%s", - library_id, - filename, - getattr(existing, "id", "?"), - status, - ) - return existing + logger.info( + "素材幂等兜底命中(近期同名同大小活动记录): library=%s name=%s asset=%s status=%s size=%s", + library_id, + filename, + getattr(existing, "id", "?"), + getattr(existing, "status", None), + file_size, + ) + return existing + elif filename and not file_hash and not client_upload_id and not file_size: + logger.debug( + "同名兜底去重跳过(file_size 未知,宁可放行不可误杀): library=%s name=%s", + library_id, + filename, + ) return None @@ -487,6 +472,7 @@ async def complete_direct_upload( user_id=authenticated_user.user.id, file_hash=request.file_hash, client_upload_id=request.client_upload_id, + file_size=request.file_size, ) job = _submit_ingest_job( diff --git a/apps/worker/worker_app/celery_app.py b/apps/worker/worker_app/celery_app.py index 75477f583..34d40ad09 100755 --- a/apps/worker/worker_app/celery_app.py +++ b/apps/worker/worker_app/celery_app.py @@ -17,6 +17,12 @@ apply_queue_settings(celery_app) # 长渲染任务预取 1,避免任务被预取占住导致调度不均 celery_app.conf.worker_prefetch_multiplier = GENERATION_WORKER_PREFETCH_MULTIPLIER celery_app.conf.task_acks_late = True # worker 崩溃时未完成任务重回队列,由执行前守卫丢弃作废消息 +# worker 进程被 OOM/容器硬杀时拒绝 ack,消息留在队列由其他 worker 接手 +celery_app.conf.task_reject_on_worker_lost = True +# Redis broker 消息可见性超时(#1714):acks_late 下,消息被预取后 visibility_timeout +# 内未 ack 才会重投。长任务(ingest HEVC 转码 20-30 分钟、生成硬超时 11 分钟) +# 必须远大于最长执行时间,否则正常任务会在执行中被误重投;4 小时覆盖最长转码 + 余量。 +celery_app.conf.broker_transport_options = {"visibility_timeout": 4 * 60 * 60} celery_app.conf.imports = ( "worker_app.tasks.health", @@ -48,4 +54,11 @@ celery_app.conf.beat_schedule = { "schedule": 300.0, # 每 5 分钟(秒) "options": {"expires": 240}, }, + # 上传/转码链路孤儿巡检:worker 重启丢 prefetch 消息后,卡 pending/processing + # 的 ingest_job + asset 占位超时标终态(#1714)。转码任务较长,10 分钟一轮 + "cleanup-stale-ingest-jobs": { + "task": "worker.cleanup_stale_ingest_jobs", + "schedule": 600.0, # 每 10 分钟(秒) + "options": {"expires": 540}, + }, } diff --git a/apps/worker/worker_app/tasks/_startup.py b/apps/worker/worker_app/tasks/_startup.py index 2d3f80b7c..afe329704 100644 --- a/apps/worker/worker_app/tasks/_startup.py +++ b/apps/worker/worker_app/tasks/_startup.py @@ -257,3 +257,32 @@ def _on_worker_ready(sender, **kwargs): # pragma: no cover result = cleanup_all_stale_tasks() total = result["generation_tasks"] + result["jobs"] logger.info("Worker 启动清理完成,共清理 %d 个孤儿任务", total) + + +@worker_ready.connect +def _recover_stuck_ingest_jobs_on_ready(sender, **kwargs): # pragma: no cover + """Worker 启动完成后恢复卡死在 processing 的 ingest_job(#1714)。 + + 容器重启/进程 OOM 导致 transcode 队列 unacked 消息未重投时,processing + ingest_job 会永久卡死。启动时扫描 processing 超 10 分钟的 job,CAS 重置 + pending 并重新派单;Redis 锁保证同容器 generation/transcode 双 worker + 只有一个执行恢复。旧消息若后来重投,ingest_asset 执行前守卫会丢弃。 + """ + try: + from packages.application.ingest_orphan_cleanup import ( + make_redis_recovery_lock, + recover_stuck_ingest_jobs_on_startup, + ) + + session = SessionLocal() + try: + recovered = recover_stuck_ingest_jobs_on_startup( + session, + lock_acquire=make_redis_recovery_lock(), + stuck_minutes=10, + ) + finally: + session.close() + logger.info("Worker 启动 ingest 恢复完成,共重新派单 %d 个卡死任务", recovered) + except Exception as e: # noqa: BLE001 — 启动恢复失败不能阻断 worker 起服 + logger.error("启动 ingest 恢复扫描失败(beat 巡检仍会兜底标 failed): %s", e, exc_info=True) diff --git a/apps/worker/worker_app/tasks/cleanup.py b/apps/worker/worker_app/tasks/cleanup.py index 07a7de8dc..ff29840dc 100644 --- a/apps/worker/worker_app/tasks/cleanup.py +++ b/apps/worker/worker_app/tasks/cleanup.py @@ -16,6 +16,12 @@ from worker_app.tasks._startup import ( cleanup_stale_pending_tasks, ) +from packages.application.ingest_orphan_cleanup import ( + ASSET_ORPHAN_TIMEOUT_MINUTES, + INGEST_PENDING_TIMEOUT_MINUTES, + INGEST_PROCESSING_TIMEOUT_MINUTES, +) + logger = logging.getLogger(__name__) @@ -69,3 +75,53 @@ def scheduled_cleanup_stale_running(timeout_minutes: int = ORPHAN_TASK_TIMEOUT_M timeout_minutes, ) return {"generation_tasks": gen_count, "jobs": job_count} + + +@shared_task(name="worker.cleanup_stale_ingest_jobs") +def scheduled_cleanup_stale_ingest_jobs( + processing_timeout_minutes: int = INGEST_PROCESSING_TIMEOUT_MINUTES, + pending_timeout_minutes: int = INGEST_PENDING_TIMEOUT_MINUTES, + orphan_asset_timeout_minutes: int = ASSET_ORPHAN_TIMEOUT_MINUTES, +) -> dict: + """Celery Beat 调度:清理上传/转码链路(IngestJob + Asset)孤儿记录。 + + 每 10 分钟执行一次。worker 容器重启/进程 OOM 时,已 prefetch 的 transcode + celery 消息会丢失(队列里也不存在),ingest_job 永久卡 pending/processing、 + asset 永久卡 processing/uploading,没有兜底永远不会恢复(#1714)。 + + - ingest_job processing > processing_timeout_minutes / pending > pending_timeout_minutes + → 标 failed;关联 asset 占位(processing/uploading)联动标 error + - 无 ingest_job 关联、created_at > orphan_asset_timeout_minutes 的占位 asset + → 标 error + - 作废 celery 消息 revoke + 物理清除(防重投,执行前守卫是第二道防线) + """ + from worker_app.db import SessionLocal + + from packages.application.ingest_orphan_cleanup import ( + cleanup_orphan_processing_assets, + cleanup_stale_ingest_jobs, + revoke_stale_ingest_messages, + ) + + session = SessionLocal() + try: + job_items, asset_ids = cleanup_stale_ingest_jobs( + session, + processing_timeout_minutes=processing_timeout_minutes, + pending_timeout_minutes=pending_timeout_minutes, + ) + orphan_asset_ids = cleanup_orphan_processing_assets(session, timeout_minutes=orphan_asset_timeout_minutes) + finally: + session.close() + + purged = revoke_stale_ingest_messages(job_items) if job_items else 0 + total_jobs = len(job_items) + total_assets = len(set(asset_ids) | set(orphan_asset_ids)) + if total_jobs or total_assets: + logger.warning( + "[Beat] 清理 ingest 链路孤儿: stale_jobs=%d, assets→error=%d, 队列清除消息=%d", + total_jobs, + total_assets, + purged, + ) + return {"stale_jobs": total_jobs, "assets_to_error": total_assets, "purged_messages": purged} diff --git a/packages/adapters/sqlalchemy_impl/asset_repository.py b/packages/adapters/sqlalchemy_impl/asset_repository.py index ba334d498..fad219727 100755 --- a/packages/adapters/sqlalchemy_impl/asset_repository.py +++ b/packages/adapters/sqlalchemy_impl/asset_repository.py @@ -488,20 +488,25 @@ class SQLAlchemyAssetRepository: 用于旧客户端未传 file_hash/client_upload_id 时,防止 complete 超时重试 反复创建 PROCESSING 占位记录。只命中"活动中"的近期记录,READY 历史素材不拦。 + + 严格模式(#1714 误杀修复):file_size 必须 > 0 且与记录大小严格一致; + file_size=0(大小未知)时直接返回 None——宁可漏判(极端情况下多建一条 + 占位)也不可仅凭同名 + processing 误杀内容全新的视频。 """ from datetime import datetime, timedelta, timezone if not name: return None + if not file_size or file_size <= 0: + return None cutoff = datetime.now(timezone.utc) - timedelta(minutes=within_minutes) query = self.session.query(AssetModel).filter( AssetModel.asset_library_id == library_id, AssetModel.name == name, AssetModel.status.in_([AssetStatus.UPLOADING.value, AssetStatus.PROCESSING.value]), AssetModel.created_at >= cutoff, + AssetModel.file_size == file_size, ) - if file_size and file_size > 0: - query = query.filter(AssetModel.file_size == file_size) model = query.order_by(AssetModel.created_at.desc()).first() if model is None: return None diff --git a/packages/application/ingest_orphan_cleanup.py b/packages/application/ingest_orphan_cleanup.py new file mode 100644 index 000000000..22fbb09a4 --- /dev/null +++ b/packages/application/ingest_orphan_cleanup.py @@ -0,0 +1,310 @@ +"""上传/转码链路(IngestJob + Asset)孤儿清理核心逻辑。 + +#1714:generation 链路有 cleanup_stale_running/pending 兜底,但上传链路 +(ingest_jobs + assets)没有。worker 容器重启/进程 OOM 时,已 prefetch 的 +celery 消息会丢失(transcode 队列 worker_prefetch_multiplier=1,消息预取后 +宕机即丢失,Redis 队列里也不再存在),导致: + +- ingest_jobs.status 永久卡 pending/processing +- assets.status 永久卡 processing/uploading(complete 阶段预建的占位) + +本模块提供纯核心(session 注入,便于单测):超时阈值内无更新的记录 +批量标终态(job→failed、asset→error),并返回 (job_id, celery_task_id) +列表供调用方 revoke + purge 残留队列消息。 +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timedelta, timezone +from typing import Any, Callable + +logger = logging.getLogger(__name__) + +# ingest_job PROCESSING 超时阈值:ingest 任务包含下载 + ffprobe + HEVC 转码 +# (1GB 视频约 10-20 分钟)+ 回传 OSS,正常任务可能跑 20-30 分钟; +# 60 分钟阈值覆盖大文件转码 + 抖动,绝不误杀正常任务。 +INGEST_PROCESSING_TIMEOUT_MINUTES = 60 + +# ingest_job PENDING 超时阈值:transcode 队列 concurrency=1,队列积压时 +# 正常排队可能较久;90 分钟覆盖 worker 短暂停消费 + 排队。 +INGEST_PENDING_TIMEOUT_MINUTES = 90 + +# Asset 占位超时阈值:无关联 ingest_job 的孤儿占位(complete 预建后派单失败等), +# 阈值放宽到 120 分钟,避免与 ingest_job 生命周期错杀。 +ASSET_ORPHAN_TIMEOUT_MINUTES = 120 + +_TERMINAL_JOB_STATUSES = ("failed", "completed") +_TERMINAL_ASSET_STATUSES = ("ready", "error", "deleted") + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def cleanup_stale_ingest_jobs( + session: Any, + *, + processing_timeout_minutes: int = INGEST_PROCESSING_TIMEOUT_MINUTES, + pending_timeout_minutes: int = INGEST_PENDING_TIMEOUT_MINUTES, + commit: bool = True, +) -> tuple[list[tuple[str, str]], list[str]]: + """清理超时卡 pending/processing 的 ingest_jobs,并联动关联 asset。 + + Args: + session: SQLAlchemy session(或提供 query/commit 的鸭子类型) + processing_timeout_minutes: processing 状态超时阈值 + pending_timeout_minutes: pending 状态超时阈值 + commit: 是否提交事务 + + Returns: + (job_items, asset_ids) + - job_items: [(job_id, celery_task_id), ...] 供 revoke/purge + - asset_ids: 被联动标记为 error 的 asset id 列表 + """ + from packages.adapters.sqlalchemy_impl.models import AssetModel, IngestJobModel + + now = _now() + processing_cutoff = now - timedelta(minutes=processing_timeout_minutes) + pending_cutoff = now - timedelta(minutes=pending_timeout_minutes) + + stale_jobs = ( + session.query(IngestJobModel) + .filter( + IngestJobModel.status.in_(["pending", "processing"]), + ( + (IngestJobModel.status == "processing") & (IngestJobModel.updated_at < processing_cutoff) + | (IngestJobModel.status == "pending") & (IngestJobModel.created_at < pending_cutoff) + ), + ) + .all() + ) + + job_items: list[tuple[str, str]] = [] + asset_ids: list[str] = [] + stale_asset_models: list[Any] = [] + for job_model in stale_jobs: + ref_time = job_model.updated_at or job_model.created_at + if ref_time.tzinfo is None: # SQLite 读回 naive datetime 的防御 + ref_time = ref_time.replace(tzinfo=timezone.utc) + stale_minutes = int((now - ref_time).total_seconds() // 60) + job_model.status = "failed" + job_model.error_message = ( + f"转码任务执行中断(超过超时阈值未更新,疑似 worker 重启/进程退出,已卡死 {stale_minutes} 分钟)" + ) + job_model.updated_at = now + job_items.append((job_model.id, getattr(job_model, "celery_task_id", "") or "")) + if job_model.asset_id: + asset_ids.append(job_model.asset_id) + + if asset_ids: + stale_asset_models = ( + session.query(AssetModel) + .filter( + AssetModel.id.in_(asset_ids), + AssetModel.status.in_(["processing", "uploading"]), + ) + .all() + ) + for asset_model in stale_asset_models: + asset_model.status = "error" + asset_model.updated_at = now + + if commit and (job_items or stale_asset_models): + session.commit() + + if job_items: + logger.warning( + "[ingest-cleanup] 清理 %d 个超时 ingest_job(processing>%dm / pending>%dm),联动 %d 个 asset 标 error", + len(job_items), + processing_timeout_minutes, + pending_timeout_minutes, + len(stale_asset_models), + ) + return job_items, [a.id for a in stale_asset_models] + + +def cleanup_orphan_processing_assets( + session: Any, + *, + timeout_minutes: int = ASSET_ORPHAN_TIMEOUT_MINUTES, + commit: bool = True, +) -> list[str]: + """清理无 ingest_job 关联、超时卡 processing/uploading 的孤儿 asset 占位。 + + complete 阶段预建 asset 后若派单失败(或 direct 上传 complete 后 + 未触发 ingest),占位会永久卡住。这类 asset 没有对应 ingest_job, + 只能按 created_at 超时兜底标 error。 + """ + from packages.adapters.sqlalchemy_impl.models import AssetModel, IngestJobModel + + cutoff = _now() - timedelta(minutes=timeout_minutes) + orphan_assets = ( + session.query(AssetModel) + .outerjoin(IngestJobModel, IngestJobModel.asset_id == AssetModel.id) + .filter( + AssetModel.status.in_(["processing", "uploading"]), + AssetModel.created_at < cutoff, + IngestJobModel.id.is_(None), + ) + .all() + ) + for asset_model in orphan_assets: + asset_model.status = "error" + asset_model.updated_at = _now() + if commit and orphan_assets: + session.commit() + logger.warning("[ingest-cleanup] 清理 %d 个无 job 关联的超时孤儿 asset 占位", len(orphan_assets)) + return [a.id for a in orphan_assets] + + +def revoke_stale_ingest_messages( + job_items: list[tuple[str, str]], + *, + celery_app_factory: Callable[[], Any] | None = None, + broker_url_factory: Callable[[], str] | None = None, +) -> int: + """revoke + 物理清理 ingest 作废消息(transcode/celery 队列)。 + + 消息可能已在 worker 宕机时丢失(队列里查不到),那也无害; + 若消息还在(极端重复投递),物理清除防止重投执行。 + 失败不阻断清理(ingest_asset 的执行前状态守卫是第二道防线)。 + """ + biz_ids = [jid for jid, _ in job_items if jid] + celery_ids = [cid for _, cid in job_items if cid] + if not biz_ids and not celery_ids: + return 0 + try: + from packages.shared.celery_orphan_guard import revoke_and_purge + + app = celery_app_factory() if celery_app_factory else None + broker_url = broker_url_factory() if broker_url_factory else "" + if app is None or not broker_url: + from worker_app.celery_app import celery_app as _app + from worker_app.core.config import get_settings + + app = _app + broker_url = get_settings().broker_url + return revoke_and_purge( + app, + broker_url, + business_task_ids=biz_ids, + celery_task_ids=celery_ids, + queue_names=("transcode", "celery"), + ) + except Exception as e: # noqa: BLE001 + logger.error("撤销作废 ingest 队列消息失败(执行前守卫仍会兜底): %s", e, exc_info=True) + return 0 + + +# ── worker 启动恢复(#1714)────────────────────────────────────────────── +# +# task_acks_late=True 下,worker 崩溃/容器重启时未 ack 的消息理论上会在 +# visibility_timeout 到期后重新投递;但 prefork 进程异常、部署窗口跨 +# visibility 配置边界等场景仍可能留下卡在 processing 的 ingest_job +# (staging 实证:03:16 派单、03:45 置 processing 后 worker 重启, +# unacked 消息未重投,任务永久卡死)。启动时做一次显式恢复扫描兜底。 +# +# 恢复策略:processing 超过 stuck_minutes(默认 10 分钟,部署中跨进程 +# 交接的正常窗口 < 10 分钟,不会误抢别的 worker 正在执行的任务)的 job, +# CAS 重置为 pending 并重新 send_task;旧消息若后来重投,ingest_asset +# 的执行前守卫会把状态不匹配的旧 celery 消息丢弃。 + + +def recover_stuck_ingest_jobs_on_startup( + session: Any, + *, + send_task: Callable[..., Any] | None = None, + update_celery_task_id: Callable[[str, str], None] | None = None, + lock_acquire: Callable[[], bool] | None = None, + stuck_minutes: int = 10, + commit: bool = True, +) -> int: + """worker 启动时把卡在 processing 超时的 ingest_job 重新派单。 + + Args: + session: SQLAlchemy session + send_task: celery send_task 可调用(注入便于测试);不传则用 worker celery_app + update_celery_task_id: 回写新 celery task id 的回调(job_id, new_task_id) + lock_acquire: 分布式锁获取回调(多 worker 进程同时启动时只允许一个恢复); + 返回 False 表示未抢到锁,本次跳过 + stuck_minutes: processing 超过该分钟数视为卡死 + + Returns: + 重新派单的 job 数 + """ + if lock_acquire is not None and not lock_acquire(): + logger.info("[ingest-recover] 未抢到恢复锁,跳过(另一进程正在恢复)") + return 0 + + from packages.adapters.sqlalchemy_impl.models import IngestJobModel + + cutoff = _now() - timedelta(minutes=stuck_minutes) + stuck_jobs = ( + session.query(IngestJobModel) + .filter(IngestJobModel.status == "processing", IngestJobModel.updated_at < cutoff) + .order_by(IngestJobModel.updated_at.asc()) + .all() + ) + + if not stuck_jobs: + logger.info("[ingest-recover] 无卡死 processing ingest_job 需要恢复") + return 0 + + if send_task is None: + from worker_app.celery_app import celery_app as _app + + send_task = _app.send_task + + recovered = 0 + for job_model in stuck_jobs: + # CAS:只有仍是 processing 才重置(并发/旧消息已回写终态时不碰) + updated = ( + session.query(IngestJobModel) + .filter(IngestJobModel.id == job_model.id, IngestJobModel.status == "processing") + .update({"status": "pending", "error_message": "", "updated_at": _now()}) + ) + if not updated: + continue + try: + result = send_task("worker.ingest_asset", args=[job_model.id]) + new_task_id = getattr(result, "id", "") or "" + except Exception as e: # noqa: BLE001 + logger.error("[ingest-recover] 重新派单失败 job_id=%s: %s", job_model.id, e) + continue + if new_task_id: + job_model.celery_task_id = new_task_id + if update_celery_task_id is not None: + update_celery_task_id(job_model.id, new_task_id) + logger.warning( + "[ingest-recover] 卡死 ingest_job %s 已重置 pending 并重新派单 (new celery task=%s)", + job_model.id, + new_task_id, + ) + recovered += 1 + + if commit and recovered: + session.commit() + logger.warning("[ingest-recover] 启动恢复完成,共重新派单 %d 个卡死 ingest_job", recovered) + return recovered + + +def make_redis_recovery_lock(lock_key: str = "ingest:recover:startup", ttl_seconds: int = 300): + """构造基于 Redis SET NX 的恢复锁工厂(多 worker 进程互斥)。 + + 返回一个无参 callable,调用时尝试抢锁:抢到返回 True,未抢到返回 False。 + Redis 不可用时不阻断启动恢复(返回 True,恢复逻辑自身有 CAS 幂等保护)。 + """ + + def _acquire() -> bool: + try: + import redis as redis_lib + from worker_app.core.config import get_settings + + client = redis_lib.Redis.from_url(get_settings().broker_url) + return bool(client.set(lock_key, "1", nx=True, ex=ttl_seconds)) + except Exception as e: # noqa: BLE001 + logger.warning("[ingest-recover] Redis 锁不可用,降级为无锁执行(CAS 兜底): %s", e) + return True + + return _acquire diff --git a/tests/unit/test_asset_repo_fallback_dedup_1714.py b/tests/unit/test_asset_repo_fallback_dedup_1714.py new file mode 100644 index 000000000..d68890773 --- /dev/null +++ b/tests/unit/test_asset_repo_fallback_dedup_1714.py @@ -0,0 +1,93 @@ +"""#1714 find_recent_active_by_library_and_name 严格模式测试。 + +file_size=0(未知)时必须返回 None(宁可漏判不可误杀); +大小严格匹配;只命中近期 UPLOADING/PROCESSING 记录。 +""" + +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "api")) + +from sqlalchemy import create_engine # noqa: E402 +from sqlalchemy.orm import sessionmaker # noqa: E402 + +from packages.adapters.sqlalchemy_impl.asset_repository import SQLAlchemyAssetRepository # noqa: E402 +from packages.adapters.sqlalchemy_impl.models import Base # noqa: E402 +from packages.domain import Asset, AssetStatus # noqa: E402 + + +def _repository(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine)() + return SQLAlchemyAssetRepository(session) + + +def _mk_asset(name="IMG_2285.MOV", file_size=5_000_000, status=AssetStatus.PROCESSING, minutes_ago=5): + asset = Asset.create( + project_id="proj-1", + library_id="lib-1", + name=name, + storage_key=f"uploads/x/{name}", + mime_type="video/quicktime", + file_size=file_size, + ) + asset.status = status + asset.created_at = datetime.now(timezone.utc) - timedelta(minutes=minutes_ago) + return asset + + +def test_returns_none_when_file_size_zero(): + """file_size=0(大小未知)直接返回 None——不许仅凭同名 + processing 判重。""" + repo = _repository() + repo.create(_mk_asset(file_size=0)) + + result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="IMG_2285.MOV", file_size=0) + assert result is None + + +def test_matches_when_name_size_strict_equal(): + """同名 + 同大小 + processing 近期记录 → 命中。""" + repo = _repository() + repo.create(_mk_asset(file_size=5_000_000)) + + result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="IMG_2285.MOV", file_size=5_000_000) + assert result is not None + assert result.name == "IMG_2285.MOV" + + +def test_no_match_when_same_name_but_different_size(): + """同名但大小不同 → 不命中(内容全新的视频不能误杀)。""" + repo = _repository() + repo.create(_mk_asset(file_size=5_000_000)) + + result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="IMG_2285.MOV", file_size=9_999_999) + assert result is None + + +def test_no_match_ready_history_even_with_same_size(): + """READY 历史同名素材不命中(允许再次上传同名文件)。""" + repo = _repository() + repo.create(_mk_asset(file_size=5_000_000, status=AssetStatus.READY)) + + result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="IMG_2285.MOV", file_size=5_000_000) + assert result is None + + +def test_no_match_when_window_expired(): + """超过 30 分钟窗口的活动记录不命中。""" + repo = _repository() + repo.create(_mk_asset(file_size=5_000_000, minutes_ago=45)) + + result = repo.find_recent_active_by_library_and_name( + library_id="lib-1", name="IMG_2285.MOV", within_minutes=30, file_size=5_000_000 + ) + assert result is None + + +def test_returns_none_when_name_empty(): + repo = _repository() + result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="", file_size=100) + assert result is None diff --git a/tests/unit/test_cleanup_ingest_beat_1714.py b/tests/unit/test_cleanup_ingest_beat_1714.py new file mode 100644 index 000000000..f7c5bf11d --- /dev/null +++ b/tests/unit/test_cleanup_ingest_beat_1714.py @@ -0,0 +1,75 @@ +"""#1714 beat 任务 scheduled_cleanup_stale_ingest_jobs 薄封装测试。 + +mock SessionLocal 和清理核心,验证 beat 任务正确串联 +cleanup_stale_ingest_jobs → cleanup_orphan_processing_assets → revoke 消息。 +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +os.environ.setdefault("JWT_SECRET_KEY", "unit-test-secret") +os.environ.setdefault("DATABASE_URL", "sqlite:///test_beat.db") + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "worker")) +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "api")) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +import worker_app.tasks.cleanup as cleanup # noqa: E402 + + +def test_beat_cleanup_calls_core_and_revokes(): + """beat 任务串联三个核心步骤,返回汇总计数。""" + fake_session = MagicMock() + + with ( + patch("worker_app.db.SessionLocal", return_value=fake_session) as m_db, + patch( + "packages.application.ingest_orphan_cleanup.cleanup_stale_ingest_jobs", + return_value=([("job-1", "cel-1"), ("job-2", "")], ["a-1"]), + ) as m_jobs, + patch( + "packages.application.ingest_orphan_cleanup.cleanup_orphan_processing_assets", + return_value=["a-2"], + ) as m_assets, + patch( + "packages.shared.celery_orphan_guard.revoke_and_purge", + return_value=1, + ) as m_revoke, + ): + result = cleanup.scheduled_cleanup_stale_ingest_jobs() + + m_db.assert_called_once() + m_jobs.assert_called_once() + assert m_jobs.call_args.kwargs["processing_timeout_minutes"] == 60 + m_assets.assert_called_once() + m_revoke.assert_called_once() + # 队列名只传 transcode/celery(不传 generation) + assert m_revoke.call_args.kwargs["queue_names"] == ("transcode", "celery") + fake_session.close.assert_called_once() + assert result == {"stale_jobs": 2, "assets_to_error": 2, "purged_messages": 1} + + +def test_beat_cleanup_no_op_when_nothing_stale(): + """无孤儿时不调 revoke,返回全 0。""" + fake_session = MagicMock() + + with ( + patch("worker_app.db.SessionLocal", return_value=fake_session), + patch( + "packages.application.ingest_orphan_cleanup.cleanup_stale_ingest_jobs", + return_value=([], []), + ), + patch( + "packages.application.ingest_orphan_cleanup.cleanup_orphan_processing_assets", + return_value=[], + ), + patch("packages.shared.celery_orphan_guard.revoke_and_purge") as m_revoke, + ): + result = cleanup.scheduled_cleanup_stale_ingest_jobs() + + m_revoke.assert_not_called() + assert result == {"stale_jobs": 0, "assets_to_error": 0, "purged_messages": 0} diff --git a/tests/unit/test_ingest_orphan_cleanup_1714.py b/tests/unit/test_ingest_orphan_cleanup_1714.py new file mode 100644 index 000000000..cceccf8d4 --- /dev/null +++ b/tests/unit/test_ingest_orphan_cleanup_1714.py @@ -0,0 +1,262 @@ +"""#1714 上传/转码链路(IngestJob + Asset)孤儿清理测试。 + +场景:worker 容器重启/进程 OOM 时,已 prefetch 的 transcode celery 消息丢失, +ingest_job 永久卡 pending/processing、asset 永久卡 processing/uploading。 +""" + +from __future__ import annotations + +import os +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +os.environ.setdefault("JWT_SECRET_KEY", "unit-test-secret") +os.environ.setdefault("DATABASE_URL", "sqlite:///test_ingest_orphan.db") + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "worker")) +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "api")) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from sqlalchemy import create_engine # noqa: E402 +from sqlalchemy.orm import sessionmaker # noqa: E402 + +from packages.adapters.sqlalchemy_impl.models import AssetModel, Base, IngestJobModel # noqa: E402 +from packages.application.ingest_orphan_cleanup import ( # noqa: E402 + cleanup_orphan_processing_assets, + cleanup_stale_ingest_jobs, +) + + +@pytest.fixture() +def session(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(bind=engine) + Session = sessionmaker(bind=engine) + db = Session() + yield db + db.close() + + +def _mk_job(session, *, status="processing", celery_task_id="cel-1", asset_id="a-1", minutes_ago=90): + now = datetime.now(timezone.utc) + job = IngestJobModel( + id=f"job-{minutes_ago}-{status}-{celery_task_id}", + project_id="p-1", + library_id="lib-1", + storage_key="uploads/x/IMG_2285.MOV", + status=status, + asset_id=asset_id, + celery_task_id=celery_task_id, + created_at=now - timedelta(minutes=minutes_ago), + updated_at=now - timedelta(minutes=minutes_ago), + ) + session.add(job) + session.commit() + return job + + +def _mk_asset(session, *, id="a-1", status="processing", minutes_ago=90, file_size=0): + now = datetime.now(timezone.utc) + asset = AssetModel( + id=id, + project_id="p-1", + asset_library_id="lib-1", + name="IMG_2285.MOV", + file_type="video", + file_size=file_size, + file_url="https://example.com/x.mov", + storage_key="uploads/x/IMG_2285.MOV", + status=status, + uploaded_by_user_id="u-1", + created_at=now - timedelta(minutes=minutes_ago), + updated_at=now - timedelta(minutes=minutes_ago), + ) + session.add(asset) + session.commit() + return asset + + +class TestCleanupStaleIngestJobs: + def test_stale_processing_job_marked_failed_and_asset_to_error(self, session): + """processing 超 60 分钟 → job failed,关联 processing asset → error。""" + _mk_asset(session, id="a-1", status="processing") + _mk_job(session, status="processing", celery_task_id="cel-dead", asset_id="a-1", minutes_ago=90) + + items, asset_ids = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90) + + assert len(items) == 1 + assert items[0] == ("job-90-processing-cel-dead", "cel-dead") + assert asset_ids == ["a-1"] + db_job = session.query(IngestJobModel).one() + assert db_job.status == "failed" + assert "中断" in db_job.error_message + db_asset = session.query(AssetModel).one() + assert db_asset.status == "error" + + def test_stale_pending_job_marked_failed(self, session): + """pending 超 90 分钟(从未被消费)→ job failed。""" + _mk_asset(session, id="a-2", status="uploading") + _mk_job(session, status="pending", celery_task_id="", asset_id="a-2", minutes_ago=120) + + items, asset_ids = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90) + + assert len(items) == 1 + assert items[0][1] == "" # 无 celery task id + assert session.query(IngestJobModel).one().status == "failed" + assert session.query(AssetModel).one().status == "error" + + def test_recent_processing_job_not_touched(self, session): + """processing 仅 10 分钟(正常转码中)→ 不误杀。""" + _mk_asset(session, id="a-3", status="processing", minutes_ago=10) + _mk_job(session, status="processing", celery_task_id="cel-live", asset_id="a-3", minutes_ago=10) + + items, asset_ids = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90) + + assert items == [] + assert asset_ids == [] + assert session.query(IngestJobModel).one().status == "processing" + assert session.query(AssetModel).one().status == "processing" + + def test_recent_pending_job_not_touched(self, session): + """pending 仅 30 分钟(队列积压排队中)→ 不误杀。""" + _mk_job(session, status="pending", asset_id="", minutes_ago=30) + + items, _ = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90) + + assert items == [] + assert session.query(IngestJobModel).one().status == "pending" + + def test_terminal_job_not_touched(self, session): + """已 completed/failed 的 job 不动。""" + _mk_job(session, status="completed", celery_task_id="", asset_id="", minutes_ago=999) + _mk_job(session, status="failed", celery_task_id="", asset_id="", minutes_ago=999) + + items, _ = cleanup_stale_ingest_jobs(session) + + assert items == [] + statuses = sorted(j.status for j in session.query(IngestJobModel).all()) + assert statuses == ["completed", "failed"] + + def test_ready_asset_not_demoted(self, session): + """关联 asset 已是 ready(转码其实成功了,仅 job 回写失败)→ 不降级为 error。""" + _mk_asset(session, id="a-4", status="ready") + _mk_job(session, status="processing", celery_task_id="cel-x", asset_id="a-4", minutes_ago=90) + + _, asset_ids = cleanup_stale_ingest_jobs(session) + + assert asset_ids == [] # ready 不动 + assert session.query(AssetModel).one().status == "ready" + + +class TestCleanupOrphanProcessingAssets: + def test_orphan_asset_without_job_marked_error(self, session): + """无 ingest_job 关联、created 超 120 分钟的 processing 占位 → error。""" + _mk_asset(session, id="orphan-1", status="processing", minutes_ago=150) + + ids = cleanup_orphan_processing_assets(session, timeout_minutes=120) + + assert ids == ["orphan-1"] + assert session.query(AssetModel).one().status == "error" + + def test_asset_with_active_job_not_touched(self, session): + """有 processing job 关联的 asset 不由本函数处理(归 cleanup_stale_ingest_jobs)。""" + _mk_asset(session, id="a-5", status="processing", minutes_ago=150) + _mk_job(session, status="processing", asset_id="a-5", minutes_ago=150) + + ids = cleanup_orphan_processing_assets(session, timeout_minutes=120) + + assert ids == [] + assert session.query(AssetModel).one().status == "processing" + + def test_recent_orphan_asset_not_touched(self, session): + """无 job 但才创建 30 分钟 → 可能 complete 刚建、job 派单中,不动。""" + _mk_asset(session, id="orphan-2", status="processing", minutes_ago=30) + + ids = cleanup_orphan_processing_assets(session, timeout_minutes=120) + + assert ids == [] + assert session.query(AssetModel).one().status == "processing" + + +class TestRecoverStuckIngestJobsOnStartup: + def test_stuck_processing_job_requeued(self, session): + """processing 超 10 分钟 → 重置 pending 并重新 send_task,回写新 celery id。""" + from types import SimpleNamespace + + from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup + + job = _mk_job(session, status="processing", celery_task_id="old-cel-1", asset_id="a-1", minutes_ago=30) + + sent = [] + + def fake_send_task(name, args=None, **kw): + sent.append((name, args)) + return SimpleNamespace(id="new-cel-9") + + updated_ids = [] + recovered = recover_stuck_ingest_jobs_on_startup( + session, + send_task=fake_send_task, + update_celery_task_id=lambda jid, cid: updated_ids.append((jid, cid)), + stuck_minutes=10, + ) + + assert recovered == 1 + assert sent == [("worker.ingest_asset", [job.id])] + refreshed = session.query(IngestJobModel).filter_by(id=job.id).one() + assert refreshed.status == "pending" + assert refreshed.celery_task_id == "new-cel-9" + assert updated_ids == [(job.id, "new-cel-9")] + + def test_recent_processing_job_not_touched(self, session): + """processing 仅 5 分钟(正常转码中/部署交接窗口)→ 不抢。""" + from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup + + _mk_job(session, status="processing", celery_task_id="live", asset_id="", minutes_ago=5) + + sent = [] + recovered = recover_stuck_ingest_jobs_on_startup( + session, + send_task=lambda *a, **k: sent.append(a), + stuck_minutes=10, + ) + + assert recovered == 0 + assert sent == [] + assert session.query(IngestJobModel).one().status == "processing" + + def test_lock_not_acquired_skips(self, session): + """未抢到分布式锁(另一 worker 正在恢复)→ 跳过。""" + from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup + + _mk_job(session, status="processing", celery_task_id="x", asset_id="", minutes_ago=30) + + recovered = recover_stuck_ingest_jobs_on_startup( + session, + send_task=lambda *a, **k: None, + lock_acquire=lambda: False, + stuck_minutes=10, + ) + + assert recovered == 0 + assert session.query(IngestJobModel).one().status == "processing" + + def test_pending_and_terminal_not_requeued(self, session): + """pending/已终态 job 不在恢复范围。""" + from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup + + _mk_job(session, status="pending", celery_task_id="", asset_id="", minutes_ago=60) + _mk_job(session, status="failed", celery_task_id="", asset_id="", minutes_ago=60) + + recovered = recover_stuck_ingest_jobs_on_startup( + session, + send_task=lambda *a, **k: None, + stuck_minutes=10, + ) + + assert recovered == 0 + statuses = sorted(j.status for j in session.query(IngestJobModel).all()) + assert statuses == ["failed", "pending"] diff --git a/tests/unit/test_upload_complete_idempotency_1714.py b/tests/unit/test_upload_complete_idempotency_1714.py index 1effb6e78..c0436b065 100644 --- a/tests/unit/test_upload_complete_idempotency_1714.py +++ b/tests/unit/test_upload_complete_idempotency_1714.py @@ -73,6 +73,9 @@ class StubAssetRepository: def find_recent_active_by_library_and_name( self, library_id: str, name: str, within_minutes: int = 30, file_size: int = 0 ) -> Asset | None: + # 严格模式(#1714):大小未知(0)直接不命中,宁可漏判不可误杀 + if not file_size or file_size <= 0: + return None cutoff = datetime.now(timezone.utc) - timedelta(minutes=within_minutes) candidates = [ a @@ -81,7 +84,7 @@ class StubAssetRepository: and a.name == name and a.status in (AssetStatus.UPLOADING, AssetStatus.PROCESSING) and a.created_at >= cutoff - and (not file_size or a.file_size == file_size) + and a.file_size == file_size ] return max(candidates, key=lambda a: a.created_at) if candidates else None @@ -234,14 +237,21 @@ class TestDirectCompleteIdempotency: 不应再建第二条。 """ client, asset_repo, ingest_repo, _ = _client() - # 第一次 complete(旧客户端无 token/hash) - r1 = client.post("/api/v1/direct/complete", json=COMPLETE_BODY) + # 第一次 complete(旧客户端无 token/hash,但 file_size 可知) + r1 = client.post( + "/api/v1/direct/complete", + json={**COMPLETE_BODY, "file_size": 5_000_000}, + ) assert r1.json()["duplicated"] is False # 重试:重新 prepare 产生新 storage_key(仅 uuid 目录不同,文件名一致—— - # 前端重试传的是同一个 File),且近期 + # 前端重试传的是同一个 File),且近期;同大小才允许兜底命中 r2 = client.post( "/api/v1/direct/complete", - json={**COMPLETE_BODY, "storage_key": "uploads/retry/IMG_2282.MOV", "file_size": 0}, + json={ + **COMPLETE_BODY, + "storage_key": "uploads/retry/IMG_2282.MOV", + "file_size": 5_000_000, + }, ) assert r2.status_code == 200 assert r2.json()["duplicated"] is True @@ -249,6 +259,80 @@ class TestDirectCompleteIdempotency: assert len(asset_repo.created) == 1 assert ingest_repo.created_count == 1 + def test_fallback_dedup_skipped_when_file_size_unknown(self): + """file_size=0(未知)时不允许仅凭同名 + processing 判重,直接放行(#1714)。 + + 根因场景:complete 没传 file_size,30 分钟内同名占位(如 iPhone 的 + IMG_2285.MOV)会把内容/大小全新的视频误判为重复跳过。 + """ + client, asset_repo, _ingest_repo, _ = _client() + r1 = client.post( + "/api/v1/direct/complete", + json={**COMPLETE_BODY, "file_size": 0}, + ) + assert r1.json()["duplicated"] is False + # 第二个全新视频:同名(IMG_2285.MOV)、无 hash/token、file_size 仍未知 + r2 = client.post( + "/api/v1/direct/complete", + json={**COMPLETE_BODY, "storage_key": "uploads/retry2/IMG_2282.MOV", "file_size": 0}, + ) + assert r2.status_code == 200 + assert r2.json()["duplicated"] is False # 不能误杀 + assert len(asset_repo.created) == 2 # 两条记录,放行新上传 + + def test_fallback_dedup_skipped_when_same_name_but_different_size(self): + """同名但 file_size 不同 → 不判重,正常建记录(#1714)。""" + client, asset_repo, _ingest_repo, _ = _client() + r1 = client.post( + "/api/v1/direct/complete", + json={**COMPLETE_BODY, "file_size": 5_000_000}, + ) + assert r1.json()["duplicated"] is False + r2 = client.post( + "/api/v1/direct/complete", + json={ + **COMPLETE_BODY, + "storage_key": "uploads/retry3/IMG_2282.MOV", + "file_size": 9_999_999, # 同名但大小完全不同的新视频 + }, + ) + assert r2.status_code == 200 + assert r2.json()["duplicated"] is False + assert len(asset_repo.created) == 2 + + def test_fallback_dedup_skipped_when_hash_present_even_if_name_size_match(self): + """file_hash 非空且 hash 未命中时,不允许退回同名兜底(#1714)。 + + hash 已能代表内容:同名同大小但 hash 不同是真实的新内容,必须放行。 + """ + client, asset_repo, _ingest_repo, _ = _client() + # 第一次:某 hash 的视频 + r1 = client.post( + "/api/v1/direct/complete", + json={ + **COMPLETE_BODY, + "file_hash": "a" * 64, + "client_upload_id": "tok-1", + "file_size": 5_000_000, + }, + ) + assert r1.json()["duplicated"] is False + # 第二次:同名同大小但 hash 不同(新视频内容不同); + # 注意 client_upload_id 也必须不同,否则会先被 token 命中 + r2 = client.post( + "/api/v1/direct/complete", + json={ + **COMPLETE_BODY, + "storage_key": "uploads/retry4/IMG_2282.MOV", + "file_hash": "b" * 64, + "client_upload_id": "tok-2", + "file_size": 5_000_000, + }, + ) + assert r2.status_code == 200 + assert r2.json()["duplicated"] is False + assert len(asset_repo.created) == 2 + def test_fallback_dedup_ignores_ready_history(self): """READY 历史同名素材不触发兜底(允许用户再次上传同名文件)。""" ready = Asset(