fix(P0): 修复取消链路断裂,前端取消后 GPU 仍继续推理 (#2009)
CI/CD Pipeline / Check push changed paths (pull_request) Has been skipped
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (pull_request) Successful in 2s
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 3s
CI/CD Pipeline / Build Staging API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Validate - Style (pull_request) Has been skipped
CI/CD Pipeline / Validate - Security (pull_request) Has been skipped
CI/CD Pipeline / Unit Tests (pull_request) Has been skipped
CI/CD Pipeline / Validate - Python (mypy + alembic) (pull_request) Has been skipped
CI/CD Pipeline / Frontend Lint (pull_request) Has been skipped
CI/CD Pipeline / Integration Tests (pull_request) Has been skipped
CI/CD Pipeline / Frontend Unit Tests (pull_request) Has been skipped
CI/CD Pipeline / PR Build Worker Image (pull_request) Successful in 42s
CI/CD Pipeline / Retag skipped Staging API Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging Web Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging Worker Image (pull_request) Has been skipped
CI/CD Pipeline / PR Build API Image (pull_request) Successful in 48s
CI/CD Pipeline / PR Build Web Image (pull_request) Successful in 2m23s
CI/CD Pipeline / Build Production API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Has been skipped
CI/CD Pipeline / Deploy Production (pull_request) Has been skipped
CI/CD Pipeline / Staging E2E Tests (pull_request) Has been skipped
CI/CD Pipeline / Staging API Integration Tests (pull_request) Has been skipped
CI/CD Pipeline / ACR Image Cleanup (pull_request) Has been skipped
CI/CD Pipeline / Production Browser E2E (pull_request) Has been skipped
CI/CD Pipeline / CI Gate (pull_request) Successful in 3s
CI/CD Pipeline / Canary Release to Production (pull_request) Has been skipped
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 2m17s
PR Automation / Auto Approve on CI Green (pull_request) Successful in 3m24s
PR Automation / Auto Merge on CI Green + Approved (pull_request) Has been skipped
AI Code Review / AI Code Review (pull_request) Successful in 6m51s
CI/CD Pipeline / Check if frontend-only change (push) Has been skipped
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (push) Successful in 1s
CI/CD Pipeline / Check push changed paths (push) Successful in 2s
CI/CD Pipeline / Frontend Lint (push) Has been skipped
CI/CD Pipeline / PR Build API Image (push) Has been skipped
CI/CD Pipeline / PR Build Web Image (push) Has been skipped
CI/CD Pipeline / PR Build Worker Image (push) Has been skipped
CI/CD Pipeline / Integration Tests (push) Successful in 4m7s
CI/CD Pipeline / Build Staging API Image (push) Successful in 4m14s
CI/CD Pipeline / Validate - Python (mypy + alembic) (push) Successful in 4m28s
CI/CD Pipeline / Build Staging Worker Image (push) Successful in 4m47s
CI/CD Pipeline / Build Staging Web Image (push) Successful in 4m49s
CI/CD Pipeline / Retag skipped Staging API Image (push) Has been skipped
CI/CD Pipeline / Retag skipped Staging Web Image (push) Has been skipped
CI/CD Pipeline / Retag skipped Staging Worker Image (push) Has been skipped
CI/CD Pipeline / Validate - Style (push) Successful in 5m7s
CI/CD Pipeline / Frontend Unit Tests (push) Successful in 6m58s
CI/CD Pipeline / Unit Tests (push) Successful in 12m19s
CI/CD Pipeline / Validate - Security (push) Successful in 13m1s
CI/CD Pipeline / Build Production Web Image (push) Has been skipped
CI/CD Pipeline / Build Production API Image (push) Has been skipped
CI/CD Pipeline / Build Production Worker Image (push) Has been skipped
CI/CD Pipeline / CI Gate (push) Has been skipped
CI/CD Pipeline / Deploy Production (push) Has been skipped
CI/CD Pipeline / Production Browser E2E (push) Has been skipped
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (push) Failing after 10m9s
CI/CD Pipeline / Staging E2E Tests (push) Has been skipped
CI/CD Pipeline / Staging API Integration Tests (push) Has been skipped
CI/CD Pipeline / ACR Image Cleanup (push) Has been skipped
CI/CD Pipeline / Canary Release to Production (push) Has been skipped

Co-authored-by: xiaoxia <dev@xiaoxiajianji.com>
Co-committed-by: xiaoxia <dev@xiaoxiajianji.com>
This commit was merged in pull request #2009.
This commit is contained in:
2026-09-22 01:24:07 +08:00
committed by auto-approve-bot
parent 9927413370
commit f032152eaa
13 changed files with 457 additions and 35 deletions
+50 -6
View File
@@ -173,10 +173,10 @@ def test_timed_out_task_is_redispatched(svc):
def test_register_worker_creates_then_updates(svc):
w = svc.register_worker("w-1", hostname="pc1", gpu_name="RTX2060", free_vram_mb=3500)
w, _cancel = svc.register_worker("w-1", hostname="pc1", gpu_name="RTX2060", free_vram_mb=3500)
assert w.worker_id == "w-1"
assert w.gpu_name == "RTX2060"
w2 = svc.register_worker("w-1", hostname="pc1", gpu_name="RTX2060", free_vram_mb=2000)
w2, _cancel2 = svc.register_worker("w-1", hostname="pc1", gpu_name="RTX2060", free_vram_mb=2000)
assert w2.free_vram_mb == 2000 # 更新
assert w2.created_at == w.created_at # 没新建
@@ -195,7 +195,7 @@ def test_register_with_task_id_refreshes_task_heartbeat(svc):
{"last_heartbeat_at": old_hb - timedelta(seconds=300)}
)
svc.db.commit()
svc.register_worker("w-1", task_id=t.id)
_w, _c = svc.register_worker("w-1", task_id=t.id)
svc.db.refresh(t)
assert t.last_heartbeat_at > old_hb
assert t.status == "processing" # 心跳不改变状态
@@ -213,7 +213,7 @@ def test_register_task_heartbeat_ignores_finished_or_foreign_task(svc):
svc.poll_task("w-1")
done = svc.report_result(t.id, "w-1", success=True, duration_seconds=10.0)
hb_when_done = done.last_heartbeat_at
svc.register_worker("w-1", task_id=t.id)
_w, _c = svc.register_worker("w-1", task_id=t.id)
svc.db.refresh(t)
assert t.status == "done"
assert t.last_heartbeat_at == hb_when_done # 没被改写
@@ -232,17 +232,61 @@ def test_register_task_heartbeat_ignores_finished_or_foreign_task(svc):
{"last_heartbeat_at": owner_hb - timedelta(seconds=600)}
)
svc.db.commit()
svc.register_worker("w-1", task_id=t2.id) # 旧 worker 迟到心跳
_w2, _c2 = svc.register_worker("w-1", task_id=t2.id) # 旧 worker 迟到心跳
svc.db.refresh(t2)
assert t2.worker_id == "w-2"
assert t2.status == "processing"
assert t2.last_heartbeat_at == owner_hb
# 场景 3:不存在的 task_id 不报错
svc.register_worker("w-1", task_id="nonexistent-id")
_wn, _cn = svc.register_worker("w-1", task_id="nonexistent-id")
assert svc.db.get(GpuLipsyncTaskModel, "nonexistent-id") is None
def test_register_task_heartbeat_detects_cancelled(svc):
"""取消链路:任务已 cancelled 时,register 心跳必须返回 cancel_task=True."""
t = svc.create_task(video_url="v", audio_url="a")
svc.poll_task("w-1")
svc.db.refresh(t)
# 用户取消:直接把任务置为 cancelled
t.status = "cancelled"
t.finished_at = datetime.now(UTC)
svc.db.commit()
_w, cancel_task = svc.register_worker("w-1", task_id=t.id)
assert cancel_task is True
svc.db.refresh(t)
assert t.status == "cancelled" # 心跳不改写已取消状态
def test_report_result_cancelled_stays_cancelled(svc):
"""Worker 终止取消任务后上报失败,report_result 必须保持 cancelled 不回退 pending."""
t = svc.create_task(video_url="v", audio_url="a")
svc.poll_task("w-1")
svc.db.refresh(t)
t.status = "cancelled"
svc.db.commit()
result = svc.report_result(t.id, "w-1", success=False, error_msg="推理被终止")
assert result.status == "cancelled"
assert result.finished_at is not None
assert "推理被终止" in (result.error_msg or "")
def test_wait_for_result_returns_when_cancelled(svc):
"""wait_for_result 将 cancelled 视为终态,立即返回,Celery 不回退 MediaKit."""
t = svc.create_task(video_url="v", audio_url="a")
svc.poll_task("w-1")
svc.db.refresh(t)
t.status = "cancelled"
t.finished_at = datetime.now(UTC)
svc.db.commit()
result = svc.wait_for_result(t.id, timeout_seconds=5, poll_interval=0.1)
assert result is not None
assert result.status == "cancelled"
def test_default_gpu_task_timeout_is_900(svc):
"""#1970 默认超时 300→900,覆盖 RTX2060 长视频推理."""
assert svc.settings.gpu_task_timeout_seconds == 900