fix(#1714): 同名兜底去重误杀新视频 + ingest 链路孤儿清理/启动恢复 #1734
@@ -108,9 +108,8 @@ def _infer_mime_type_from_storage_key(storage_key: str) -> str:
|
||||
return "video/mp4" # default
|
||||
|
||||
|
||||
# 兜底去重:无 file_hash / client_upload_id 时,同库同名近期活动记录视为重复
|
||||
# 兜底去重:无 file_hash / client_upload_id 且大小已知时,同库同名同大小近期活动记录视为重复
|
||||
FALLBACK_DEDUP_WINDOW_MINUTES = 30
|
||||
ACTIVE_ASSET_STATUSES = (AssetStatus.UPLOADING, AssetStatus.PROCESSING)
|
||||
|
||||
|
||||
def _find_duplicate_asset(
|
||||
@@ -126,8 +125,12 @@ def _find_duplicate_asset(
|
||||
|
||||
1. client_upload_id(客户端幂等 token,同一次上传的重试保持一致)
|
||||
2. file_hash(内容哈希,不同上传只要内容相同即去重)
|
||||
3. 兜底:同库 + 同文件名(+同大小)且 30 分钟内仍处 uploading/processing
|
||||
的记录——旧客户端不传 hash/token 时,防止 complete 超时重试反复建占位。
|
||||
3. 兜底(严格模式,宁可漏判不可误杀):file_hash 与 client_upload_id
|
||||
均缺失、且 file_size > 0 时,同库 + 同文件名 + **同大小** 且 30 分钟内
|
||||
仍处 uploading/processing 的记录才判重。
|
||||
- file_hash 非空时跳过兜底(hash 已代表内容;同名但内容全新的视频
|
||||
如 iPhone 的 IMG_xxxx.MOV 绝不能被同名占位误杀)
|
||||
- file_size=0(未知)时不允许仅凭同名 + processing 判重,直接放行
|
||||
|
||||
全部为鸭子类型调用:旧仓储无对应方法时静默跳过,不破坏既有实现。
|
||||
"""
|
||||
@@ -156,53 +159,35 @@ def _find_duplicate_asset(
|
||||
existing.id,
|
||||
)
|
||||
return existing
|
||||
if filename:
|
||||
# 同名兜底去重(最后防线,严格模式):
|
||||
# - 仅当 file_hash / client_upload_id 均缺失时启用(hash 能代表内容时不靠同名猜)
|
||||
# - file_size 必须 > 0 且与记录大小严格一致;大小未知(0)直接放行
|
||||
# - 只命中近期 UPLOADING/PROCESSING 活动记录(READY 历史素材不拦)
|
||||
if filename and not file_hash and not client_upload_id and file_size and file_size > 0:
|
||||
find_recent = getattr(asset_repository, "find_recent_active_by_library_and_name", None)
|
||||
if callable(find_recent):
|
||||
existing = find_recent(
|
||||
library_id=library_id,
|
||||
name=filename,
|
||||
within_minutes=FALLBACK_DEDUP_WINDOW_MINUTES,
|
||||
file_size=file_size or 0,
|
||||
file_size=file_size,
|
||||
)
|
||||
# 兜底去重:按状态区分处理
|
||||
# - READY/ERROR:稳定素材,总命中(避免重复创建)
|
||||
# - PROCESSING/UPLOADING:预建或 complete 占位,仅当 hash 一致才命中
|
||||
# - 占位无 hash(旧客户端 complete 建的)→ 命中
|
||||
# - 占位有 hash 且与当前请求 hash 一致 → 命中
|
||||
# - 占位有 hash 且与当前请求 hash 不同 → 跳过(内容不同)
|
||||
if existing is not None:
|
||||
status = getattr(existing, "status", None)
|
||||
existing_hash = getattr(existing, "file_hash", "") or ""
|
||||
if status in (AssetStatus.READY, AssetStatus.ERROR):
|
||||
logger.info(
|
||||
"素材幂等兜底命中(近期同名稳定记录): library=%s name=%s asset=%s status=%s",
|
||||
library_id,
|
||||
filename,
|
||||
getattr(existing, "id", "?"),
|
||||
status,
|
||||
)
|
||||
return existing
|
||||
elif status in ACTIVE_ASSET_STATUSES:
|
||||
if existing_hash and file_hash and existing_hash != file_hash:
|
||||
logger.debug(
|
||||
"素材兜底去重跳过(占位 hash 不同): library=%s name=%s asset=%s hash=%s req_hash=%s",
|
||||
library_id,
|
||||
filename,
|
||||
getattr(existing, "id", "?"),
|
||||
existing_hash,
|
||||
file_hash,
|
||||
)
|
||||
existing = None
|
||||
else:
|
||||
logger.info(
|
||||
"素材幂等兜底命中(近期同名活动记录): library=%s name=%s asset=%s status=%s",
|
||||
library_id,
|
||||
filename,
|
||||
getattr(existing, "id", "?"),
|
||||
status,
|
||||
)
|
||||
return existing
|
||||
logger.info(
|
||||
"素材幂等兜底命中(近期同名同大小活动记录): library=%s name=%s asset=%s status=%s size=%s",
|
||||
library_id,
|
||||
filename,
|
||||
getattr(existing, "id", "?"),
|
||||
getattr(existing, "status", None),
|
||||
file_size,
|
||||
)
|
||||
return existing
|
||||
elif filename and not file_hash and not client_upload_id and not file_size:
|
||||
logger.debug(
|
||||
"同名兜底去重跳过(file_size 未知,宁可放行不可误杀): library=%s name=%s",
|
||||
library_id,
|
||||
filename,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -487,6 +472,7 @@ async def complete_direct_upload(
|
||||
user_id=authenticated_user.user.id,
|
||||
file_hash=request.file_hash,
|
||||
client_upload_id=request.client_upload_id,
|
||||
file_size=request.file_size,
|
||||
)
|
||||
|
||||
job = _submit_ingest_job(
|
||||
|
||||
@@ -17,6 +17,12 @@ apply_queue_settings(celery_app)
|
||||
# 长渲染任务预取 1,避免任务被预取占住导致调度不均
|
||||
celery_app.conf.worker_prefetch_multiplier = GENERATION_WORKER_PREFETCH_MULTIPLIER
|
||||
celery_app.conf.task_acks_late = True # worker 崩溃时未完成任务重回队列,由执行前守卫丢弃作废消息
|
||||
# worker 进程被 OOM/容器硬杀时拒绝 ack,消息留在队列由其他 worker 接手
|
||||
celery_app.conf.task_reject_on_worker_lost = True
|
||||
# Redis broker 消息可见性超时(#1714):acks_late 下,消息被预取后 visibility_timeout
|
||||
# 内未 ack 才会重投。长任务(ingest HEVC 转码 20-30 分钟、生成硬超时 11 分钟)
|
||||
# 必须远大于最长执行时间,否则正常任务会在执行中被误重投;4 小时覆盖最长转码 + 余量。
|
||||
celery_app.conf.broker_transport_options = {"visibility_timeout": 4 * 60 * 60}
|
||||
|
||||
celery_app.conf.imports = (
|
||||
"worker_app.tasks.health",
|
||||
@@ -48,4 +54,11 @@ celery_app.conf.beat_schedule = {
|
||||
"schedule": 300.0, # 每 5 分钟(秒)
|
||||
"options": {"expires": 240},
|
||||
},
|
||||
# 上传/转码链路孤儿巡检:worker 重启丢 prefetch 消息后,卡 pending/processing
|
||||
# 的 ingest_job + asset 占位超时标终态(#1714)。转码任务较长,10 分钟一轮
|
||||
"cleanup-stale-ingest-jobs": {
|
||||
"task": "worker.cleanup_stale_ingest_jobs",
|
||||
"schedule": 600.0, # 每 10 分钟(秒)
|
||||
"options": {"expires": 540},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -257,3 +257,32 @@ def _on_worker_ready(sender, **kwargs): # pragma: no cover
|
||||
result = cleanup_all_stale_tasks()
|
||||
total = result["generation_tasks"] + result["jobs"]
|
||||
logger.info("Worker 启动清理完成,共清理 %d 个孤儿任务", total)
|
||||
|
||||
|
||||
@worker_ready.connect
|
||||
def _recover_stuck_ingest_jobs_on_ready(sender, **kwargs): # pragma: no cover
|
||||
"""Worker 启动完成后恢复卡死在 processing 的 ingest_job(#1714)。
|
||||
|
||||
容器重启/进程 OOM 导致 transcode 队列 unacked 消息未重投时,processing
|
||||
ingest_job 会永久卡死。启动时扫描 processing 超 10 分钟的 job,CAS 重置
|
||||
pending 并重新派单;Redis 锁保证同容器 generation/transcode 双 worker
|
||||
只有一个执行恢复。旧消息若后来重投,ingest_asset 执行前守卫会丢弃。
|
||||
"""
|
||||
try:
|
||||
from packages.application.ingest_orphan_cleanup import (
|
||||
make_redis_recovery_lock,
|
||||
recover_stuck_ingest_jobs_on_startup,
|
||||
)
|
||||
|
||||
session = SessionLocal()
|
||||
try:
|
||||
recovered = recover_stuck_ingest_jobs_on_startup(
|
||||
session,
|
||||
lock_acquire=make_redis_recovery_lock(),
|
||||
stuck_minutes=10,
|
||||
)
|
||||
finally:
|
||||
session.close()
|
||||
logger.info("Worker 启动 ingest 恢复完成,共重新派单 %d 个卡死任务", recovered)
|
||||
except Exception as e: # noqa: BLE001 — 启动恢复失败不能阻断 worker 起服
|
||||
logger.error("启动 ingest 恢复扫描失败(beat 巡检仍会兜底标 failed): %s", e, exc_info=True)
|
||||
|
||||
@@ -16,6 +16,12 @@ from worker_app.tasks._startup import (
|
||||
cleanup_stale_pending_tasks,
|
||||
)
|
||||
|
||||
from packages.application.ingest_orphan_cleanup import (
|
||||
ASSET_ORPHAN_TIMEOUT_MINUTES,
|
||||
INGEST_PENDING_TIMEOUT_MINUTES,
|
||||
INGEST_PROCESSING_TIMEOUT_MINUTES,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -69,3 +75,53 @@ def scheduled_cleanup_stale_running(timeout_minutes: int = ORPHAN_TASK_TIMEOUT_M
|
||||
timeout_minutes,
|
||||
)
|
||||
return {"generation_tasks": gen_count, "jobs": job_count}
|
||||
|
||||
|
||||
@shared_task(name="worker.cleanup_stale_ingest_jobs")
|
||||
def scheduled_cleanup_stale_ingest_jobs(
|
||||
processing_timeout_minutes: int = INGEST_PROCESSING_TIMEOUT_MINUTES,
|
||||
pending_timeout_minutes: int = INGEST_PENDING_TIMEOUT_MINUTES,
|
||||
orphan_asset_timeout_minutes: int = ASSET_ORPHAN_TIMEOUT_MINUTES,
|
||||
) -> dict:
|
||||
"""Celery Beat 调度:清理上传/转码链路(IngestJob + Asset)孤儿记录。
|
||||
|
||||
每 10 分钟执行一次。worker 容器重启/进程 OOM 时,已 prefetch 的 transcode
|
||||
celery 消息会丢失(队列里也不存在),ingest_job 永久卡 pending/processing、
|
||||
asset 永久卡 processing/uploading,没有兜底永远不会恢复(#1714)。
|
||||
|
||||
- ingest_job processing > processing_timeout_minutes / pending > pending_timeout_minutes
|
||||
→ 标 failed;关联 asset 占位(processing/uploading)联动标 error
|
||||
- 无 ingest_job 关联、created_at > orphan_asset_timeout_minutes 的占位 asset
|
||||
→ 标 error
|
||||
- 作废 celery 消息 revoke + 物理清除(防重投,执行前守卫是第二道防线)
|
||||
"""
|
||||
from worker_app.db import SessionLocal
|
||||
|
||||
from packages.application.ingest_orphan_cleanup import (
|
||||
cleanup_orphan_processing_assets,
|
||||
cleanup_stale_ingest_jobs,
|
||||
revoke_stale_ingest_messages,
|
||||
)
|
||||
|
||||
session = SessionLocal()
|
||||
try:
|
||||
job_items, asset_ids = cleanup_stale_ingest_jobs(
|
||||
session,
|
||||
processing_timeout_minutes=processing_timeout_minutes,
|
||||
pending_timeout_minutes=pending_timeout_minutes,
|
||||
)
|
||||
orphan_asset_ids = cleanup_orphan_processing_assets(session, timeout_minutes=orphan_asset_timeout_minutes)
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
purged = revoke_stale_ingest_messages(job_items) if job_items else 0
|
||||
total_jobs = len(job_items)
|
||||
total_assets = len(set(asset_ids) | set(orphan_asset_ids))
|
||||
if total_jobs or total_assets:
|
||||
logger.warning(
|
||||
"[Beat] 清理 ingest 链路孤儿: stale_jobs=%d, assets→error=%d, 队列清除消息=%d",
|
||||
total_jobs,
|
||||
total_assets,
|
||||
purged,
|
||||
)
|
||||
return {"stale_jobs": total_jobs, "assets_to_error": total_assets, "purged_messages": purged}
|
||||
|
||||
@@ -488,20 +488,25 @@ class SQLAlchemyAssetRepository:
|
||||
|
||||
用于旧客户端未传 file_hash/client_upload_id 时,防止 complete 超时重试
|
||||
反复创建 PROCESSING 占位记录。只命中"活动中"的近期记录,READY 历史素材不拦。
|
||||
|
||||
严格模式(#1714 误杀修复):file_size 必须 > 0 且与记录大小严格一致;
|
||||
file_size=0(大小未知)时直接返回 None——宁可漏判(极端情况下多建一条
|
||||
占位)也不可仅凭同名 + processing 误杀内容全新的视频。
|
||||
"""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
if not name:
|
||||
return None
|
||||
if not file_size or file_size <= 0:
|
||||
return None
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(minutes=within_minutes)
|
||||
query = self.session.query(AssetModel).filter(
|
||||
AssetModel.asset_library_id == library_id,
|
||||
AssetModel.name == name,
|
||||
AssetModel.status.in_([AssetStatus.UPLOADING.value, AssetStatus.PROCESSING.value]),
|
||||
AssetModel.created_at >= cutoff,
|
||||
AssetModel.file_size == file_size,
|
||||
)
|
||||
if file_size and file_size > 0:
|
||||
query = query.filter(AssetModel.file_size == file_size)
|
||||
model = query.order_by(AssetModel.created_at.desc()).first()
|
||||
if model is None:
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
"""上传/转码链路(IngestJob + Asset)孤儿清理核心逻辑。
|
||||
|
||||
#1714:generation 链路有 cleanup_stale_running/pending 兜底,但上传链路
|
||||
(ingest_jobs + assets)没有。worker 容器重启/进程 OOM 时,已 prefetch 的
|
||||
celery 消息会丢失(transcode 队列 worker_prefetch_multiplier=1,消息预取后
|
||||
宕机即丢失,Redis 队列里也不再存在),导致:
|
||||
|
||||
- ingest_jobs.status 永久卡 pending/processing
|
||||
- assets.status 永久卡 processing/uploading(complete 阶段预建的占位)
|
||||
|
||||
本模块提供纯核心(session 注入,便于单测):超时阈值内无更新的记录
|
||||
批量标终态(job→failed、asset→error),并返回 (job_id, celery_task_id)
|
||||
列表供调用方 revoke + purge 残留队列消息。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Callable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ingest_job PROCESSING 超时阈值:ingest 任务包含下载 + ffprobe + HEVC 转码
|
||||
# (1GB 视频约 10-20 分钟)+ 回传 OSS,正常任务可能跑 20-30 分钟;
|
||||
# 60 分钟阈值覆盖大文件转码 + 抖动,绝不误杀正常任务。
|
||||
INGEST_PROCESSING_TIMEOUT_MINUTES = 60
|
||||
|
||||
# ingest_job PENDING 超时阈值:transcode 队列 concurrency=1,队列积压时
|
||||
# 正常排队可能较久;90 分钟覆盖 worker 短暂停消费 + 排队。
|
||||
INGEST_PENDING_TIMEOUT_MINUTES = 90
|
||||
|
||||
# Asset 占位超时阈值:无关联 ingest_job 的孤儿占位(complete 预建后派单失败等),
|
||||
# 阈值放宽到 120 分钟,避免与 ingest_job 生命周期错杀。
|
||||
ASSET_ORPHAN_TIMEOUT_MINUTES = 120
|
||||
|
||||
_TERMINAL_JOB_STATUSES = ("failed", "completed")
|
||||
_TERMINAL_ASSET_STATUSES = ("ready", "error", "deleted")
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def cleanup_stale_ingest_jobs(
|
||||
session: Any,
|
||||
*,
|
||||
processing_timeout_minutes: int = INGEST_PROCESSING_TIMEOUT_MINUTES,
|
||||
pending_timeout_minutes: int = INGEST_PENDING_TIMEOUT_MINUTES,
|
||||
commit: bool = True,
|
||||
) -> tuple[list[tuple[str, str]], list[str]]:
|
||||
"""清理超时卡 pending/processing 的 ingest_jobs,并联动关联 asset。
|
||||
|
||||
Args:
|
||||
session: SQLAlchemy session(或提供 query/commit 的鸭子类型)
|
||||
processing_timeout_minutes: processing 状态超时阈值
|
||||
pending_timeout_minutes: pending 状态超时阈值
|
||||
commit: 是否提交事务
|
||||
|
||||
Returns:
|
||||
(job_items, asset_ids)
|
||||
- job_items: [(job_id, celery_task_id), ...] 供 revoke/purge
|
||||
- asset_ids: 被联动标记为 error 的 asset id 列表
|
||||
"""
|
||||
from packages.adapters.sqlalchemy_impl.models import AssetModel, IngestJobModel
|
||||
|
||||
now = _now()
|
||||
processing_cutoff = now - timedelta(minutes=processing_timeout_minutes)
|
||||
pending_cutoff = now - timedelta(minutes=pending_timeout_minutes)
|
||||
|
||||
stale_jobs = (
|
||||
session.query(IngestJobModel)
|
||||
.filter(
|
||||
IngestJobModel.status.in_(["pending", "processing"]),
|
||||
(
|
||||
(IngestJobModel.status == "processing") & (IngestJobModel.updated_at < processing_cutoff)
|
||||
| (IngestJobModel.status == "pending") & (IngestJobModel.created_at < pending_cutoff)
|
||||
),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
job_items: list[tuple[str, str]] = []
|
||||
asset_ids: list[str] = []
|
||||
stale_asset_models: list[Any] = []
|
||||
for job_model in stale_jobs:
|
||||
ref_time = job_model.updated_at or job_model.created_at
|
||||
if ref_time.tzinfo is None: # SQLite 读回 naive datetime 的防御
|
||||
ref_time = ref_time.replace(tzinfo=timezone.utc)
|
||||
stale_minutes = int((now - ref_time).total_seconds() // 60)
|
||||
job_model.status = "failed"
|
||||
job_model.error_message = (
|
||||
f"转码任务执行中断(超过超时阈值未更新,疑似 worker 重启/进程退出,已卡死 {stale_minutes} 分钟)"
|
||||
)
|
||||
job_model.updated_at = now
|
||||
job_items.append((job_model.id, getattr(job_model, "celery_task_id", "") or ""))
|
||||
if job_model.asset_id:
|
||||
asset_ids.append(job_model.asset_id)
|
||||
|
||||
if asset_ids:
|
||||
stale_asset_models = (
|
||||
session.query(AssetModel)
|
||||
.filter(
|
||||
AssetModel.id.in_(asset_ids),
|
||||
AssetModel.status.in_(["processing", "uploading"]),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for asset_model in stale_asset_models:
|
||||
asset_model.status = "error"
|
||||
asset_model.updated_at = now
|
||||
|
||||
if commit and (job_items or stale_asset_models):
|
||||
session.commit()
|
||||
|
||||
if job_items:
|
||||
logger.warning(
|
||||
"[ingest-cleanup] 清理 %d 个超时 ingest_job(processing>%dm / pending>%dm),联动 %d 个 asset 标 error",
|
||||
len(job_items),
|
||||
processing_timeout_minutes,
|
||||
pending_timeout_minutes,
|
||||
len(stale_asset_models),
|
||||
)
|
||||
return job_items, [a.id for a in stale_asset_models]
|
||||
|
||||
|
||||
def cleanup_orphan_processing_assets(
|
||||
session: Any,
|
||||
*,
|
||||
timeout_minutes: int = ASSET_ORPHAN_TIMEOUT_MINUTES,
|
||||
commit: bool = True,
|
||||
) -> list[str]:
|
||||
"""清理无 ingest_job 关联、超时卡 processing/uploading 的孤儿 asset 占位。
|
||||
|
||||
complete 阶段预建 asset 后若派单失败(或 direct 上传 complete 后
|
||||
未触发 ingest),占位会永久卡住。这类 asset 没有对应 ingest_job,
|
||||
只能按 created_at 超时兜底标 error。
|
||||
"""
|
||||
from packages.adapters.sqlalchemy_impl.models import AssetModel, IngestJobModel
|
||||
|
||||
cutoff = _now() - timedelta(minutes=timeout_minutes)
|
||||
orphan_assets = (
|
||||
session.query(AssetModel)
|
||||
.outerjoin(IngestJobModel, IngestJobModel.asset_id == AssetModel.id)
|
||||
.filter(
|
||||
AssetModel.status.in_(["processing", "uploading"]),
|
||||
AssetModel.created_at < cutoff,
|
||||
IngestJobModel.id.is_(None),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for asset_model in orphan_assets:
|
||||
asset_model.status = "error"
|
||||
asset_model.updated_at = _now()
|
||||
if commit and orphan_assets:
|
||||
session.commit()
|
||||
logger.warning("[ingest-cleanup] 清理 %d 个无 job 关联的超时孤儿 asset 占位", len(orphan_assets))
|
||||
return [a.id for a in orphan_assets]
|
||||
|
||||
|
||||
def revoke_stale_ingest_messages(
|
||||
job_items: list[tuple[str, str]],
|
||||
*,
|
||||
celery_app_factory: Callable[[], Any] | None = None,
|
||||
broker_url_factory: Callable[[], str] | None = None,
|
||||
) -> int:
|
||||
"""revoke + 物理清理 ingest 作废消息(transcode/celery 队列)。
|
||||
|
||||
消息可能已在 worker 宕机时丢失(队列里查不到),那也无害;
|
||||
若消息还在(极端重复投递),物理清除防止重投执行。
|
||||
失败不阻断清理(ingest_asset 的执行前状态守卫是第二道防线)。
|
||||
"""
|
||||
biz_ids = [jid for jid, _ in job_items if jid]
|
||||
celery_ids = [cid for _, cid in job_items if cid]
|
||||
if not biz_ids and not celery_ids:
|
||||
return 0
|
||||
try:
|
||||
from packages.shared.celery_orphan_guard import revoke_and_purge
|
||||
|
||||
app = celery_app_factory() if celery_app_factory else None
|
||||
broker_url = broker_url_factory() if broker_url_factory else ""
|
||||
if app is None or not broker_url:
|
||||
from worker_app.celery_app import celery_app as _app
|
||||
from worker_app.core.config import get_settings
|
||||
|
||||
app = _app
|
||||
broker_url = get_settings().broker_url
|
||||
return revoke_and_purge(
|
||||
app,
|
||||
broker_url,
|
||||
business_task_ids=biz_ids,
|
||||
celery_task_ids=celery_ids,
|
||||
queue_names=("transcode", "celery"),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error("撤销作废 ingest 队列消息失败(执行前守卫仍会兜底): %s", e, exc_info=True)
|
||||
return 0
|
||||
|
||||
|
||||
# ── worker 启动恢复(#1714)──────────────────────────────────────────────
|
||||
#
|
||||
# task_acks_late=True 下,worker 崩溃/容器重启时未 ack 的消息理论上会在
|
||||
# visibility_timeout 到期后重新投递;但 prefork 进程异常、部署窗口跨
|
||||
# visibility 配置边界等场景仍可能留下卡在 processing 的 ingest_job
|
||||
# (staging 实证:03:16 派单、03:45 置 processing 后 worker 重启,
|
||||
# unacked 消息未重投,任务永久卡死)。启动时做一次显式恢复扫描兜底。
|
||||
#
|
||||
# 恢复策略:processing 超过 stuck_minutes(默认 10 分钟,部署中跨进程
|
||||
# 交接的正常窗口 < 10 分钟,不会误抢别的 worker 正在执行的任务)的 job,
|
||||
# CAS 重置为 pending 并重新 send_task;旧消息若后来重投,ingest_asset
|
||||
# 的执行前守卫会把状态不匹配的旧 celery 消息丢弃。
|
||||
|
||||
|
||||
def recover_stuck_ingest_jobs_on_startup(
|
||||
session: Any,
|
||||
*,
|
||||
send_task: Callable[..., Any] | None = None,
|
||||
update_celery_task_id: Callable[[str, str], None] | None = None,
|
||||
lock_acquire: Callable[[], bool] | None = None,
|
||||
stuck_minutes: int = 10,
|
||||
commit: bool = True,
|
||||
) -> int:
|
||||
"""worker 启动时把卡在 processing 超时的 ingest_job 重新派单。
|
||||
|
||||
Args:
|
||||
session: SQLAlchemy session
|
||||
send_task: celery send_task 可调用(注入便于测试);不传则用 worker celery_app
|
||||
update_celery_task_id: 回写新 celery task id 的回调(job_id, new_task_id)
|
||||
lock_acquire: 分布式锁获取回调(多 worker 进程同时启动时只允许一个恢复);
|
||||
返回 False 表示未抢到锁,本次跳过
|
||||
stuck_minutes: processing 超过该分钟数视为卡死
|
||||
|
||||
Returns:
|
||||
重新派单的 job 数
|
||||
"""
|
||||
if lock_acquire is not None and not lock_acquire():
|
||||
logger.info("[ingest-recover] 未抢到恢复锁,跳过(另一进程正在恢复)")
|
||||
return 0
|
||||
|
||||
from packages.adapters.sqlalchemy_impl.models import IngestJobModel
|
||||
|
||||
cutoff = _now() - timedelta(minutes=stuck_minutes)
|
||||
stuck_jobs = (
|
||||
session.query(IngestJobModel)
|
||||
.filter(IngestJobModel.status == "processing", IngestJobModel.updated_at < cutoff)
|
||||
.order_by(IngestJobModel.updated_at.asc())
|
||||
.all()
|
||||
)
|
||||
|
||||
if not stuck_jobs:
|
||||
logger.info("[ingest-recover] 无卡死 processing ingest_job 需要恢复")
|
||||
return 0
|
||||
|
||||
if send_task is None:
|
||||
from worker_app.celery_app import celery_app as _app
|
||||
|
||||
send_task = _app.send_task
|
||||
|
||||
recovered = 0
|
||||
for job_model in stuck_jobs:
|
||||
# CAS:只有仍是 processing 才重置(并发/旧消息已回写终态时不碰)
|
||||
updated = (
|
||||
session.query(IngestJobModel)
|
||||
.filter(IngestJobModel.id == job_model.id, IngestJobModel.status == "processing")
|
||||
.update({"status": "pending", "error_message": "", "updated_at": _now()})
|
||||
)
|
||||
if not updated:
|
||||
continue
|
||||
try:
|
||||
result = send_task("worker.ingest_asset", args=[job_model.id])
|
||||
new_task_id = getattr(result, "id", "") or ""
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error("[ingest-recover] 重新派单失败 job_id=%s: %s", job_model.id, e)
|
||||
continue
|
||||
if new_task_id:
|
||||
job_model.celery_task_id = new_task_id
|
||||
if update_celery_task_id is not None:
|
||||
update_celery_task_id(job_model.id, new_task_id)
|
||||
logger.warning(
|
||||
"[ingest-recover] 卡死 ingest_job %s 已重置 pending 并重新派单 (new celery task=%s)",
|
||||
job_model.id,
|
||||
new_task_id,
|
||||
)
|
||||
recovered += 1
|
||||
|
||||
if commit and recovered:
|
||||
session.commit()
|
||||
logger.warning("[ingest-recover] 启动恢复完成,共重新派单 %d 个卡死 ingest_job", recovered)
|
||||
return recovered
|
||||
|
||||
|
||||
def make_redis_recovery_lock(lock_key: str = "ingest:recover:startup", ttl_seconds: int = 300):
|
||||
"""构造基于 Redis SET NX 的恢复锁工厂(多 worker 进程互斥)。
|
||||
|
||||
返回一个无参 callable,调用时尝试抢锁:抢到返回 True,未抢到返回 False。
|
||||
Redis 不可用时不阻断启动恢复(返回 True,恢复逻辑自身有 CAS 幂等保护)。
|
||||
"""
|
||||
|
||||
def _acquire() -> bool:
|
||||
try:
|
||||
import redis as redis_lib
|
||||
from worker_app.core.config import get_settings
|
||||
|
||||
client = redis_lib.Redis.from_url(get_settings().broker_url)
|
||||
return bool(client.set(lock_key, "1", nx=True, ex=ttl_seconds))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("[ingest-recover] Redis 锁不可用,降级为无锁执行(CAS 兜底): %s", e)
|
||||
return True
|
||||
|
||||
return _acquire
|
||||
@@ -0,0 +1,93 @@
|
||||
"""#1714 find_recent_active_by_library_and_name 严格模式测试。
|
||||
|
||||
file_size=0(未知)时必须返回 None(宁可漏判不可误杀);
|
||||
大小严格匹配;只命中近期 UPLOADING/PROCESSING 记录。
|
||||
"""
|
||||
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "api"))
|
||||
|
||||
from sqlalchemy import create_engine # noqa: E402
|
||||
from sqlalchemy.orm import sessionmaker # noqa: E402
|
||||
|
||||
from packages.adapters.sqlalchemy_impl.asset_repository import SQLAlchemyAssetRepository # noqa: E402
|
||||
from packages.adapters.sqlalchemy_impl.models import Base # noqa: E402
|
||||
from packages.domain import Asset, AssetStatus # noqa: E402
|
||||
|
||||
|
||||
def _repository():
|
||||
engine = create_engine("sqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
session = sessionmaker(bind=engine)()
|
||||
return SQLAlchemyAssetRepository(session)
|
||||
|
||||
|
||||
def _mk_asset(name="IMG_2285.MOV", file_size=5_000_000, status=AssetStatus.PROCESSING, minutes_ago=5):
|
||||
asset = Asset.create(
|
||||
project_id="proj-1",
|
||||
library_id="lib-1",
|
||||
name=name,
|
||||
storage_key=f"uploads/x/{name}",
|
||||
mime_type="video/quicktime",
|
||||
file_size=file_size,
|
||||
)
|
||||
asset.status = status
|
||||
asset.created_at = datetime.now(timezone.utc) - timedelta(minutes=minutes_ago)
|
||||
return asset
|
||||
|
||||
|
||||
def test_returns_none_when_file_size_zero():
|
||||
"""file_size=0(大小未知)直接返回 None——不许仅凭同名 + processing 判重。"""
|
||||
repo = _repository()
|
||||
repo.create(_mk_asset(file_size=0))
|
||||
|
||||
result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="IMG_2285.MOV", file_size=0)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_matches_when_name_size_strict_equal():
|
||||
"""同名 + 同大小 + processing 近期记录 → 命中。"""
|
||||
repo = _repository()
|
||||
repo.create(_mk_asset(file_size=5_000_000))
|
||||
|
||||
result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="IMG_2285.MOV", file_size=5_000_000)
|
||||
assert result is not None
|
||||
assert result.name == "IMG_2285.MOV"
|
||||
|
||||
|
||||
def test_no_match_when_same_name_but_different_size():
|
||||
"""同名但大小不同 → 不命中(内容全新的视频不能误杀)。"""
|
||||
repo = _repository()
|
||||
repo.create(_mk_asset(file_size=5_000_000))
|
||||
|
||||
result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="IMG_2285.MOV", file_size=9_999_999)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_no_match_ready_history_even_with_same_size():
|
||||
"""READY 历史同名素材不命中(允许再次上传同名文件)。"""
|
||||
repo = _repository()
|
||||
repo.create(_mk_asset(file_size=5_000_000, status=AssetStatus.READY))
|
||||
|
||||
result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="IMG_2285.MOV", file_size=5_000_000)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_no_match_when_window_expired():
|
||||
"""超过 30 分钟窗口的活动记录不命中。"""
|
||||
repo = _repository()
|
||||
repo.create(_mk_asset(file_size=5_000_000, minutes_ago=45))
|
||||
|
||||
result = repo.find_recent_active_by_library_and_name(
|
||||
library_id="lib-1", name="IMG_2285.MOV", within_minutes=30, file_size=5_000_000
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_returns_none_when_name_empty():
|
||||
repo = _repository()
|
||||
result = repo.find_recent_active_by_library_and_name(library_id="lib-1", name="", file_size=100)
|
||||
assert result is None
|
||||
@@ -0,0 +1,75 @@
|
||||
"""#1714 beat 任务 scheduled_cleanup_stale_ingest_jobs 薄封装测试。
|
||||
|
||||
mock SessionLocal 和清理核心,验证 beat 任务正确串联
|
||||
cleanup_stale_ingest_jobs → cleanup_orphan_processing_assets → revoke 消息。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
os.environ.setdefault("JWT_SECRET_KEY", "unit-test-secret")
|
||||
os.environ.setdefault("DATABASE_URL", "sqlite:///test_beat.db")
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "worker"))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "api"))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
import worker_app.tasks.cleanup as cleanup # noqa: E402
|
||||
|
||||
|
||||
def test_beat_cleanup_calls_core_and_revokes():
|
||||
"""beat 任务串联三个核心步骤,返回汇总计数。"""
|
||||
fake_session = MagicMock()
|
||||
|
||||
with (
|
||||
patch("worker_app.db.SessionLocal", return_value=fake_session) as m_db,
|
||||
patch(
|
||||
"packages.application.ingest_orphan_cleanup.cleanup_stale_ingest_jobs",
|
||||
return_value=([("job-1", "cel-1"), ("job-2", "")], ["a-1"]),
|
||||
) as m_jobs,
|
||||
patch(
|
||||
"packages.application.ingest_orphan_cleanup.cleanup_orphan_processing_assets",
|
||||
return_value=["a-2"],
|
||||
) as m_assets,
|
||||
patch(
|
||||
"packages.shared.celery_orphan_guard.revoke_and_purge",
|
||||
return_value=1,
|
||||
) as m_revoke,
|
||||
):
|
||||
result = cleanup.scheduled_cleanup_stale_ingest_jobs()
|
||||
|
||||
m_db.assert_called_once()
|
||||
m_jobs.assert_called_once()
|
||||
assert m_jobs.call_args.kwargs["processing_timeout_minutes"] == 60
|
||||
m_assets.assert_called_once()
|
||||
m_revoke.assert_called_once()
|
||||
# 队列名只传 transcode/celery(不传 generation)
|
||||
assert m_revoke.call_args.kwargs["queue_names"] == ("transcode", "celery")
|
||||
fake_session.close.assert_called_once()
|
||||
assert result == {"stale_jobs": 2, "assets_to_error": 2, "purged_messages": 1}
|
||||
|
||||
|
||||
def test_beat_cleanup_no_op_when_nothing_stale():
|
||||
"""无孤儿时不调 revoke,返回全 0。"""
|
||||
fake_session = MagicMock()
|
||||
|
||||
with (
|
||||
patch("worker_app.db.SessionLocal", return_value=fake_session),
|
||||
patch(
|
||||
"packages.application.ingest_orphan_cleanup.cleanup_stale_ingest_jobs",
|
||||
return_value=([], []),
|
||||
),
|
||||
patch(
|
||||
"packages.application.ingest_orphan_cleanup.cleanup_orphan_processing_assets",
|
||||
return_value=[],
|
||||
),
|
||||
patch("packages.shared.celery_orphan_guard.revoke_and_purge") as m_revoke,
|
||||
):
|
||||
result = cleanup.scheduled_cleanup_stale_ingest_jobs()
|
||||
|
||||
m_revoke.assert_not_called()
|
||||
assert result == {"stale_jobs": 0, "assets_to_error": 0, "purged_messages": 0}
|
||||
@@ -0,0 +1,262 @@
|
||||
"""#1714 上传/转码链路(IngestJob + Asset)孤儿清理测试。
|
||||
|
||||
场景:worker 容器重启/进程 OOM 时,已 prefetch 的 transcode celery 消息丢失,
|
||||
ingest_job 永久卡 pending/processing、asset 永久卡 processing/uploading。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("JWT_SECRET_KEY", "unit-test-secret")
|
||||
os.environ.setdefault("DATABASE_URL", "sqlite:///test_ingest_orphan.db")
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "worker"))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "api"))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from sqlalchemy import create_engine # noqa: E402
|
||||
from sqlalchemy.orm import sessionmaker # noqa: E402
|
||||
|
||||
from packages.adapters.sqlalchemy_impl.models import AssetModel, Base, IngestJobModel # noqa: E402
|
||||
from packages.application.ingest_orphan_cleanup import ( # noqa: E402
|
||||
cleanup_orphan_processing_assets,
|
||||
cleanup_stale_ingest_jobs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def session():
|
||||
engine = create_engine("sqlite:///:memory:")
|
||||
Base.metadata.create_all(bind=engine)
|
||||
Session = sessionmaker(bind=engine)
|
||||
db = Session()
|
||||
yield db
|
||||
db.close()
|
||||
|
||||
|
||||
def _mk_job(session, *, status="processing", celery_task_id="cel-1", asset_id="a-1", minutes_ago=90):
|
||||
now = datetime.now(timezone.utc)
|
||||
job = IngestJobModel(
|
||||
id=f"job-{minutes_ago}-{status}-{celery_task_id}",
|
||||
project_id="p-1",
|
||||
library_id="lib-1",
|
||||
storage_key="uploads/x/IMG_2285.MOV",
|
||||
status=status,
|
||||
asset_id=asset_id,
|
||||
celery_task_id=celery_task_id,
|
||||
created_at=now - timedelta(minutes=minutes_ago),
|
||||
updated_at=now - timedelta(minutes=minutes_ago),
|
||||
)
|
||||
session.add(job)
|
||||
session.commit()
|
||||
return job
|
||||
|
||||
|
||||
def _mk_asset(session, *, id="a-1", status="processing", minutes_ago=90, file_size=0):
|
||||
now = datetime.now(timezone.utc)
|
||||
asset = AssetModel(
|
||||
id=id,
|
||||
project_id="p-1",
|
||||
asset_library_id="lib-1",
|
||||
name="IMG_2285.MOV",
|
||||
file_type="video",
|
||||
file_size=file_size,
|
||||
file_url="https://example.com/x.mov",
|
||||
storage_key="uploads/x/IMG_2285.MOV",
|
||||
status=status,
|
||||
uploaded_by_user_id="u-1",
|
||||
created_at=now - timedelta(minutes=minutes_ago),
|
||||
updated_at=now - timedelta(minutes=minutes_ago),
|
||||
)
|
||||
session.add(asset)
|
||||
session.commit()
|
||||
return asset
|
||||
|
||||
|
||||
class TestCleanupStaleIngestJobs:
|
||||
def test_stale_processing_job_marked_failed_and_asset_to_error(self, session):
|
||||
"""processing 超 60 分钟 → job failed,关联 processing asset → error。"""
|
||||
_mk_asset(session, id="a-1", status="processing")
|
||||
_mk_job(session, status="processing", celery_task_id="cel-dead", asset_id="a-1", minutes_ago=90)
|
||||
|
||||
items, asset_ids = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0] == ("job-90-processing-cel-dead", "cel-dead")
|
||||
assert asset_ids == ["a-1"]
|
||||
db_job = session.query(IngestJobModel).one()
|
||||
assert db_job.status == "failed"
|
||||
assert "中断" in db_job.error_message
|
||||
db_asset = session.query(AssetModel).one()
|
||||
assert db_asset.status == "error"
|
||||
|
||||
def test_stale_pending_job_marked_failed(self, session):
|
||||
"""pending 超 90 分钟(从未被消费)→ job failed。"""
|
||||
_mk_asset(session, id="a-2", status="uploading")
|
||||
_mk_job(session, status="pending", celery_task_id="", asset_id="a-2", minutes_ago=120)
|
||||
|
||||
items, asset_ids = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0][1] == "" # 无 celery task id
|
||||
assert session.query(IngestJobModel).one().status == "failed"
|
||||
assert session.query(AssetModel).one().status == "error"
|
||||
|
||||
def test_recent_processing_job_not_touched(self, session):
|
||||
"""processing 仅 10 分钟(正常转码中)→ 不误杀。"""
|
||||
_mk_asset(session, id="a-3", status="processing", minutes_ago=10)
|
||||
_mk_job(session, status="processing", celery_task_id="cel-live", asset_id="a-3", minutes_ago=10)
|
||||
|
||||
items, asset_ids = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90)
|
||||
|
||||
assert items == []
|
||||
assert asset_ids == []
|
||||
assert session.query(IngestJobModel).one().status == "processing"
|
||||
assert session.query(AssetModel).one().status == "processing"
|
||||
|
||||
def test_recent_pending_job_not_touched(self, session):
|
||||
"""pending 仅 30 分钟(队列积压排队中)→ 不误杀。"""
|
||||
_mk_job(session, status="pending", asset_id="", minutes_ago=30)
|
||||
|
||||
items, _ = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90)
|
||||
|
||||
assert items == []
|
||||
assert session.query(IngestJobModel).one().status == "pending"
|
||||
|
||||
def test_terminal_job_not_touched(self, session):
|
||||
"""已 completed/failed 的 job 不动。"""
|
||||
_mk_job(session, status="completed", celery_task_id="", asset_id="", minutes_ago=999)
|
||||
_mk_job(session, status="failed", celery_task_id="", asset_id="", minutes_ago=999)
|
||||
|
||||
items, _ = cleanup_stale_ingest_jobs(session)
|
||||
|
||||
assert items == []
|
||||
statuses = sorted(j.status for j in session.query(IngestJobModel).all())
|
||||
assert statuses == ["completed", "failed"]
|
||||
|
||||
def test_ready_asset_not_demoted(self, session):
|
||||
"""关联 asset 已是 ready(转码其实成功了,仅 job 回写失败)→ 不降级为 error。"""
|
||||
_mk_asset(session, id="a-4", status="ready")
|
||||
_mk_job(session, status="processing", celery_task_id="cel-x", asset_id="a-4", minutes_ago=90)
|
||||
|
||||
_, asset_ids = cleanup_stale_ingest_jobs(session)
|
||||
|
||||
assert asset_ids == [] # ready 不动
|
||||
assert session.query(AssetModel).one().status == "ready"
|
||||
|
||||
|
||||
class TestCleanupOrphanProcessingAssets:
|
||||
def test_orphan_asset_without_job_marked_error(self, session):
|
||||
"""无 ingest_job 关联、created 超 120 分钟的 processing 占位 → error。"""
|
||||
_mk_asset(session, id="orphan-1", status="processing", minutes_ago=150)
|
||||
|
||||
ids = cleanup_orphan_processing_assets(session, timeout_minutes=120)
|
||||
|
||||
assert ids == ["orphan-1"]
|
||||
assert session.query(AssetModel).one().status == "error"
|
||||
|
||||
def test_asset_with_active_job_not_touched(self, session):
|
||||
"""有 processing job 关联的 asset 不由本函数处理(归 cleanup_stale_ingest_jobs)。"""
|
||||
_mk_asset(session, id="a-5", status="processing", minutes_ago=150)
|
||||
_mk_job(session, status="processing", asset_id="a-5", minutes_ago=150)
|
||||
|
||||
ids = cleanup_orphan_processing_assets(session, timeout_minutes=120)
|
||||
|
||||
assert ids == []
|
||||
assert session.query(AssetModel).one().status == "processing"
|
||||
|
||||
def test_recent_orphan_asset_not_touched(self, session):
|
||||
"""无 job 但才创建 30 分钟 → 可能 complete 刚建、job 派单中,不动。"""
|
||||
_mk_asset(session, id="orphan-2", status="processing", minutes_ago=30)
|
||||
|
||||
ids = cleanup_orphan_processing_assets(session, timeout_minutes=120)
|
||||
|
||||
assert ids == []
|
||||
assert session.query(AssetModel).one().status == "processing"
|
||||
|
||||
|
||||
class TestRecoverStuckIngestJobsOnStartup:
|
||||
def test_stuck_processing_job_requeued(self, session):
|
||||
"""processing 超 10 分钟 → 重置 pending 并重新 send_task,回写新 celery id。"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup
|
||||
|
||||
job = _mk_job(session, status="processing", celery_task_id="old-cel-1", asset_id="a-1", minutes_ago=30)
|
||||
|
||||
sent = []
|
||||
|
||||
def fake_send_task(name, args=None, **kw):
|
||||
sent.append((name, args))
|
||||
return SimpleNamespace(id="new-cel-9")
|
||||
|
||||
updated_ids = []
|
||||
recovered = recover_stuck_ingest_jobs_on_startup(
|
||||
session,
|
||||
send_task=fake_send_task,
|
||||
update_celery_task_id=lambda jid, cid: updated_ids.append((jid, cid)),
|
||||
stuck_minutes=10,
|
||||
)
|
||||
|
||||
assert recovered == 1
|
||||
assert sent == [("worker.ingest_asset", [job.id])]
|
||||
refreshed = session.query(IngestJobModel).filter_by(id=job.id).one()
|
||||
assert refreshed.status == "pending"
|
||||
assert refreshed.celery_task_id == "new-cel-9"
|
||||
assert updated_ids == [(job.id, "new-cel-9")]
|
||||
|
||||
def test_recent_processing_job_not_touched(self, session):
|
||||
"""processing 仅 5 分钟(正常转码中/部署交接窗口)→ 不抢。"""
|
||||
from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup
|
||||
|
||||
_mk_job(session, status="processing", celery_task_id="live", asset_id="", minutes_ago=5)
|
||||
|
||||
sent = []
|
||||
recovered = recover_stuck_ingest_jobs_on_startup(
|
||||
session,
|
||||
send_task=lambda *a, **k: sent.append(a),
|
||||
stuck_minutes=10,
|
||||
)
|
||||
|
||||
assert recovered == 0
|
||||
assert sent == []
|
||||
assert session.query(IngestJobModel).one().status == "processing"
|
||||
|
||||
def test_lock_not_acquired_skips(self, session):
|
||||
"""未抢到分布式锁(另一 worker 正在恢复)→ 跳过。"""
|
||||
from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup
|
||||
|
||||
_mk_job(session, status="processing", celery_task_id="x", asset_id="", minutes_ago=30)
|
||||
|
||||
recovered = recover_stuck_ingest_jobs_on_startup(
|
||||
session,
|
||||
send_task=lambda *a, **k: None,
|
||||
lock_acquire=lambda: False,
|
||||
stuck_minutes=10,
|
||||
)
|
||||
|
||||
assert recovered == 0
|
||||
assert session.query(IngestJobModel).one().status == "processing"
|
||||
|
||||
def test_pending_and_terminal_not_requeued(self, session):
|
||||
"""pending/已终态 job 不在恢复范围。"""
|
||||
from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup
|
||||
|
||||
_mk_job(session, status="pending", celery_task_id="", asset_id="", minutes_ago=60)
|
||||
_mk_job(session, status="failed", celery_task_id="", asset_id="", minutes_ago=60)
|
||||
|
||||
recovered = recover_stuck_ingest_jobs_on_startup(
|
||||
session,
|
||||
send_task=lambda *a, **k: None,
|
||||
stuck_minutes=10,
|
||||
)
|
||||
|
||||
assert recovered == 0
|
||||
statuses = sorted(j.status for j in session.query(IngestJobModel).all())
|
||||
assert statuses == ["failed", "pending"]
|
||||
@@ -73,6 +73,9 @@ class StubAssetRepository:
|
||||
def find_recent_active_by_library_and_name(
|
||||
self, library_id: str, name: str, within_minutes: int = 30, file_size: int = 0
|
||||
) -> Asset | None:
|
||||
# 严格模式(#1714):大小未知(0)直接不命中,宁可漏判不可误杀
|
||||
if not file_size or file_size <= 0:
|
||||
return None
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(minutes=within_minutes)
|
||||
candidates = [
|
||||
a
|
||||
@@ -81,7 +84,7 @@ class StubAssetRepository:
|
||||
and a.name == name
|
||||
and a.status in (AssetStatus.UPLOADING, AssetStatus.PROCESSING)
|
||||
and a.created_at >= cutoff
|
||||
and (not file_size or a.file_size == file_size)
|
||||
and a.file_size == file_size
|
||||
]
|
||||
return max(candidates, key=lambda a: a.created_at) if candidates else None
|
||||
|
||||
@@ -234,14 +237,21 @@ class TestDirectCompleteIdempotency:
|
||||
不应再建第二条。
|
||||
"""
|
||||
client, asset_repo, ingest_repo, _ = _client()
|
||||
# 第一次 complete(旧客户端无 token/hash)
|
||||
r1 = client.post("/api/v1/direct/complete", json=COMPLETE_BODY)
|
||||
# 第一次 complete(旧客户端无 token/hash,但 file_size 可知)
|
||||
r1 = client.post(
|
||||
"/api/v1/direct/complete",
|
||||
json={**COMPLETE_BODY, "file_size": 5_000_000},
|
||||
)
|
||||
assert r1.json()["duplicated"] is False
|
||||
# 重试:重新 prepare 产生新 storage_key(仅 uuid 目录不同,文件名一致——
|
||||
# 前端重试传的是同一个 File),且近期
|
||||
# 前端重试传的是同一个 File),且近期;同大小才允许兜底命中
|
||||
r2 = client.post(
|
||||
"/api/v1/direct/complete",
|
||||
json={**COMPLETE_BODY, "storage_key": "uploads/retry/IMG_2282.MOV", "file_size": 0},
|
||||
json={
|
||||
**COMPLETE_BODY,
|
||||
"storage_key": "uploads/retry/IMG_2282.MOV",
|
||||
"file_size": 5_000_000,
|
||||
},
|
||||
)
|
||||
assert r2.status_code == 200
|
||||
assert r2.json()["duplicated"] is True
|
||||
@@ -249,6 +259,80 @@ class TestDirectCompleteIdempotency:
|
||||
assert len(asset_repo.created) == 1
|
||||
assert ingest_repo.created_count == 1
|
||||
|
||||
def test_fallback_dedup_skipped_when_file_size_unknown(self):
|
||||
"""file_size=0(未知)时不允许仅凭同名 + processing 判重,直接放行(#1714)。
|
||||
|
||||
根因场景:complete 没传 file_size,30 分钟内同名占位(如 iPhone 的
|
||||
IMG_2285.MOV)会把内容/大小全新的视频误判为重复跳过。
|
||||
"""
|
||||
client, asset_repo, _ingest_repo, _ = _client()
|
||||
r1 = client.post(
|
||||
"/api/v1/direct/complete",
|
||||
json={**COMPLETE_BODY, "file_size": 0},
|
||||
)
|
||||
assert r1.json()["duplicated"] is False
|
||||
# 第二个全新视频:同名(IMG_2285.MOV)、无 hash/token、file_size 仍未知
|
||||
r2 = client.post(
|
||||
"/api/v1/direct/complete",
|
||||
json={**COMPLETE_BODY, "storage_key": "uploads/retry2/IMG_2282.MOV", "file_size": 0},
|
||||
)
|
||||
assert r2.status_code == 200
|
||||
assert r2.json()["duplicated"] is False # 不能误杀
|
||||
assert len(asset_repo.created) == 2 # 两条记录,放行新上传
|
||||
|
||||
def test_fallback_dedup_skipped_when_same_name_but_different_size(self):
|
||||
"""同名但 file_size 不同 → 不判重,正常建记录(#1714)。"""
|
||||
client, asset_repo, _ingest_repo, _ = _client()
|
||||
r1 = client.post(
|
||||
"/api/v1/direct/complete",
|
||||
json={**COMPLETE_BODY, "file_size": 5_000_000},
|
||||
)
|
||||
assert r1.json()["duplicated"] is False
|
||||
r2 = client.post(
|
||||
"/api/v1/direct/complete",
|
||||
json={
|
||||
**COMPLETE_BODY,
|
||||
"storage_key": "uploads/retry3/IMG_2282.MOV",
|
||||
"file_size": 9_999_999, # 同名但大小完全不同的新视频
|
||||
},
|
||||
)
|
||||
assert r2.status_code == 200
|
||||
assert r2.json()["duplicated"] is False
|
||||
assert len(asset_repo.created) == 2
|
||||
|
||||
def test_fallback_dedup_skipped_when_hash_present_even_if_name_size_match(self):
|
||||
"""file_hash 非空且 hash 未命中时,不允许退回同名兜底(#1714)。
|
||||
|
||||
hash 已能代表内容:同名同大小但 hash 不同是真实的新内容,必须放行。
|
||||
"""
|
||||
client, asset_repo, _ingest_repo, _ = _client()
|
||||
# 第一次:某 hash 的视频
|
||||
r1 = client.post(
|
||||
"/api/v1/direct/complete",
|
||||
json={
|
||||
**COMPLETE_BODY,
|
||||
"file_hash": "a" * 64,
|
||||
"client_upload_id": "tok-1",
|
||||
"file_size": 5_000_000,
|
||||
},
|
||||
)
|
||||
assert r1.json()["duplicated"] is False
|
||||
# 第二次:同名同大小但 hash 不同(新视频内容不同);
|
||||
# 注意 client_upload_id 也必须不同,否则会先被 token 命中
|
||||
r2 = client.post(
|
||||
"/api/v1/direct/complete",
|
||||
json={
|
||||
**COMPLETE_BODY,
|
||||
"storage_key": "uploads/retry4/IMG_2282.MOV",
|
||||
"file_hash": "b" * 64,
|
||||
"client_upload_id": "tok-2",
|
||||
"file_size": 5_000_000,
|
||||
},
|
||||
)
|
||||
assert r2.status_code == 200
|
||||
assert r2.json()["duplicated"] is False
|
||||
assert len(asset_repo.created) == 2
|
||||
|
||||
def test_fallback_dedup_ignores_ready_history(self):
|
||||
"""READY 历史同名素材不触发兜底(允许用户再次上传同名文件)。"""
|
||||
ready = Asset(
|
||||
|
||||
Reference in New Issue
Block a user