fix(P0): 修复取消链路断裂，前端取消后 GPU 仍继续推理 (#2009)

Co-authored-by: xiaoxia <dev@xiaoxiajianji.com>
Co-committed-by: xiaoxia <dev@xiaoxiajianji.com>

tests/unit/test_1970_gpu_worker_heartbeat.py

@@ -68,7 +68,9 @@ def test_register_payload_includes_task_id_only_when_provided(worker, monkeypatc
 
     class _Resp:
         status_code = 200
-        text = ""
+
+        def json(self):
+            return {"worker_id": captured[-1]["worker_id"], "cancel_task": False}
 
     def _fake_post(url, json=None, headers=None, timeout=None):
         captured.append(json)
@@ -77,7 +79,9 @@ def test_register_payload_includes_task_id_only_when_provided(worker, monkeypatc
     monkeypatch.setattr(worker.requests, "post", _fake_post)
     monkeypatch.setattr(worker, "_check_musetalk_health", lambda: (True, {}))
 
-    assert worker._register("task-abc") is True
+    _ok, _cancel = worker._register("task-abc")
+    assert _ok is True
+    assert _cancel is False
     assert captured[-1]["task_id"] == "task-abc"
     assert captured[-1]["worker_id"]
 
@@ -93,7 +97,7 @@ def test_task_heartbeat_thread_sends_and_stops(worker, monkeypatch):
 
     def _fake_register(task_id=None):
         calls.append(task_id)
-        return True
+        return True, False
 
     monkeypatch.setattr(worker, "_register", _fake_register)
     hb = worker.TaskHeartbeat("task-hb1", interval=5)
@@ -105,6 +109,50 @@ def test_task_heartbeat_thread_sends_and_stops(worker, monkeypatch):
     assert calls and all(c == "task-hb1" for c in calls)
 
 
+def test_task_heartbeat_cancel_calls_musetalk_cancel(worker, monkeypatch):
+    """心跳响应 cancel_task=True → 调 _cancel_musetalk 并设置 cancelled 标志。"""
+    cancel_calls = []
+
+    monkeypatch.setattr(worker, "_register", lambda *a, **k: (True, True))
+    monkeypatch.setattr(worker, "_cancel_musetalk", lambda: cancel_calls.append(1))
+
+    hb = worker.TaskHeartbeat("task-cancel-1", interval=5)
+    hb.start()
+    hb.join(timeout=2)  # 检测到取消后线程自行 return
+    assert not hb.is_alive()
+    assert hb.cancelled is True
+    assert cancel_calls == [1]
+
+
+def test_handle_task_reports_cancelled_after_musetalk_abort(worker, monkeypatch):
+    """推理被 /cancel 终止后，hb.cancelled=True → 上报失败而非重试。"""
+    reports = []
+
+    monkeypatch.setattr(worker, "_register", lambda *a, **k: (True, False))
+    monkeypatch.setattr(worker, "_download", lambda url, path: True)
+    monkeypatch.setattr(worker, "_probe_duration", lambda path: 12.0)
+    # 模拟推理被终止（/inference 返回错误）
+    monkeypatch.setattr(worker, "_call_musetalk", lambda v, a, o: (False, 0.0, "推理被取消", False))
+    monkeypatch.setattr(
+        worker,
+        "_report_result",
+        lambda task_id, success, duration=0.0, error_msg="": reports.append(error_msg) or True,
+    )
+
+    # 让 TaskHeartbeat 在主线程检查时报告已取消
+    orig_hb_init = worker.TaskHeartbeat
+
+    def _hb(task_id, interval):
+        h = orig_hb_init(task_id, interval)
+        h.cancelled = True
+        return h
+
+    monkeypatch.setattr(worker, "TaskHeartbeat", _hb)
+
+    worker._handle_task({"task_id": "t-canceled", "video_url": "u", "audio_url": "u"})
+    assert reports == ["用户取消任务"]
+
+
 # ── 短视频前置拦截 ─────────────────────────────────────────────────
 
 
@@ -115,7 +163,7 @@ def test_handle_task_short_video_reports_failed_without_inference(worker, monkey
     audio.write_bytes(b"fake-audio")
     reports = []
 
-    monkeypatch.setattr(worker, "_register", lambda *a, **k: True)
+    monkeypatch.setattr(worker, "_register", lambda *a, **k: (True, False))
     monkeypatch.setattr(worker, "_download", lambda url, path: True)
     # ffprobe 读出 1.2s → 低于 3s 阈值
     monkeypatch.setattr(worker, "_probe_duration", lambda path: 1.2)
@@ -148,7 +196,7 @@ def test_handle_task_short_video_reports_failed_without_inference(worker, monkey
 def test_handle_task_probe_failure_does_not_block(worker, monkeypatch):
     """ffprobe 不可用（duration=0.0）时不能误杀，应继续推理."""
     reports = []
-    monkeypatch.setattr(worker, "_register", lambda *a, **k: True)
+    monkeypatch.setattr(worker, "_register", lambda *a, **k: (True, False))
     monkeypatch.setattr(worker, "_download", lambda url, path: True)
     monkeypatch.setattr(worker, "_probe_duration", lambda path: 0.0)
     monkeypatch.setattr(
@@ -214,7 +262,7 @@ def test_handle_task_retries_once_for_transient_then_succeeds(worker, monkeypatc
             return False, 0.0, "MuseTalk HTTP 503: busy", True
         return True, 6.5, "", False
 
-    monkeypatch.setattr(worker, "_register", lambda *a, **k: True)
+    monkeypatch.setattr(worker, "_register", lambda *a, **k: (True, False))
     monkeypatch.setattr(worker, "_download", lambda url, path: True)
     monkeypatch.setattr(worker, "_probe_duration", lambda path: 12.0)
     monkeypatch.setattr(worker, "_call_musetalk", _fake_call)
@@ -239,7 +287,7 @@ def test_handle_task_no_retry_for_deterministic_failure(worker, monkeypatch):
         return False, 0.0, "MuseTalk HTTP 400: bad input", False
 
     reports = []
-    monkeypatch.setattr(worker, "_register", lambda *a, **k: True)
+    monkeypatch.setattr(worker, "_register", lambda *a, **k: (True, False))
     monkeypatch.setattr(worker, "_download", lambda url, path: True)
     monkeypatch.setattr(worker, "_probe_duration", lambda path: 12.0)
     monkeypatch.setattr(worker, "_call_musetalk", _fake_call)

tests/unit/test_gpu_lipsync_routes.py

@@ -0,0 +1,89 @@
+"""GPU Worker 路由单测 — #2009 取消链路.
+
+直接调用路由函数（不经 HTTP 栈），显式注入 svc / _token 以跳过 Depends。
+CI 增量映射: gpu_lipsync.py (route) → test_gpu_lipsync_routes.py
+"""
+
+from __future__ import annotations
+
+import json
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+def _payload(**overrides):
+    from app.schemas.gpu_lipsync import GpuWorkerRegisterRequest
+
+    data = {
+        "worker_id": "w-1",
+        "hostname": "gpu-host",
+        "gpu_name": "RTX3060",
+        "free_vram_mb": 10000,
+        "capabilities": json.dumps({"musetalk": True}),
+    }
+    data.update(overrides)
+    return GpuWorkerRegisterRequest(**data)
+
+
+def test_register_returns_cancel_task_true_when_cancelled():
+    """心跳接口在任务已取消时必须把 cancel_task=True 透传给 Worker."""
+    fake_worker = MagicMock()
+    fake_worker.worker_id = "w-1"
+    fake_worker.hostname = "gpu-host"
+    fake_worker.gpu_name = "RTX3060"
+    fake_worker.free_vram_mb = 10000
+    fake_worker.capabilities = "musetalk"
+    fake_svc = MagicMock()
+    fake_svc.register_worker.return_value = (fake_worker, True)
+
+    from app.api.routes.gpu_lipsync import register_worker as route
+
+    resp = route(_payload(task_id="task-cancelled"), svc=fake_svc, _token="t")
+
+    assert resp.cancel_task is True
+    assert resp.ok is True
+    fake_svc.register_worker.assert_called_once()
+    kwargs = fake_svc.register_worker.call_args.kwargs
+    assert kwargs["task_id"] == "task-cancelled"
+
+
+def test_register_returns_cancel_task_false_normal():
+    """正常心跳 cancel_task=False."""
+    fake_worker = MagicMock()
+    fake_worker.worker_id = "w-1"
+    fake_worker.hostname = "gpu-host"
+    fake_worker.gpu_name = "RTX3060"
+    fake_worker.free_vram_mb = 10000
+    fake_worker.capabilities = "musetalk"
+    fake_svc = MagicMock()
+    fake_svc.register_worker.return_value = (fake_worker, False)
+
+    from app.api.routes.gpu_lipsync import register_worker as route
+
+    resp = route(_payload(), svc=fake_svc, _token="t")
+
+    assert resp.cancel_task is False
+
+
+def test_cancel_route_accepts_processing_status():
+    """cancel 路由允许 processing 状态（GPU 推理中），不再 400。"""
+    fake_job = MagicMock()
+    fake_job.status = "cancelled"
+
+    svc = MagicMock()
+    svc.cancel_job.return_value = fake_job
+
+    current_user = MagicMock()
+    current_user.user.id = "u1"
+
+    from app.api.routes.lipsync import cancel_lipsync_job as route
+
+    result = route("job-1", current_user, svc)
+
+    svc.cancel_job.assert_called_once_with("job-1", "u1")
+    assert result.status == "cancelled"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-q"])

tests/unit/test_gpu_lipsync_service.py

@@ -173,10 +173,10 @@ def test_timed_out_task_is_redispatched(svc):
 
 
 def test_register_worker_creates_then_updates(svc):
-    w = svc.register_worker("w-1", hostname="pc1", gpu_name="RTX2060", free_vram_mb=3500)
+    w, _cancel = svc.register_worker("w-1", hostname="pc1", gpu_name="RTX2060", free_vram_mb=3500)
     assert w.worker_id == "w-1"
     assert w.gpu_name == "RTX2060"
-    w2 = svc.register_worker("w-1", hostname="pc1", gpu_name="RTX2060", free_vram_mb=2000)
+    w2, _cancel2 = svc.register_worker("w-1", hostname="pc1", gpu_name="RTX2060", free_vram_mb=2000)
     assert w2.free_vram_mb == 2000  # 更新
     assert w2.created_at == w.created_at  # 没新建
 
@@ -195,7 +195,7 @@ def test_register_with_task_id_refreshes_task_heartbeat(svc):
         {"last_heartbeat_at": old_hb - timedelta(seconds=300)}
     )
     svc.db.commit()
-    svc.register_worker("w-1", task_id=t.id)
+    _w, _c = svc.register_worker("w-1", task_id=t.id)
     svc.db.refresh(t)
     assert t.last_heartbeat_at > old_hb
     assert t.status == "processing"  # 心跳不改变状态
@@ -213,7 +213,7 @@ def test_register_task_heartbeat_ignores_finished_or_foreign_task(svc):
     svc.poll_task("w-1")
     done = svc.report_result(t.id, "w-1", success=True, duration_seconds=10.0)
     hb_when_done = done.last_heartbeat_at
-    svc.register_worker("w-1", task_id=t.id)
+    _w, _c = svc.register_worker("w-1", task_id=t.id)
     svc.db.refresh(t)
     assert t.status == "done"
     assert t.last_heartbeat_at == hb_when_done  # 没被改写
@@ -232,17 +232,61 @@ def test_register_task_heartbeat_ignores_finished_or_foreign_task(svc):
         {"last_heartbeat_at": owner_hb - timedelta(seconds=600)}
     )
     svc.db.commit()
-    svc.register_worker("w-1", task_id=t2.id)  # 旧 worker 迟到心跳
+    _w2, _c2 = svc.register_worker("w-1", task_id=t2.id)  # 旧 worker 迟到心跳
     svc.db.refresh(t2)
     assert t2.worker_id == "w-2"
     assert t2.status == "processing"
     assert t2.last_heartbeat_at == owner_hb
 
     # 场景 3：不存在的 task_id 不报错
-    svc.register_worker("w-1", task_id="nonexistent-id")
+    _wn, _cn = svc.register_worker("w-1", task_id="nonexistent-id")
     assert svc.db.get(GpuLipsyncTaskModel, "nonexistent-id") is None
 
 
+def test_register_task_heartbeat_detects_cancelled(svc):
+    """取消链路：任务已 cancelled 时，register 心跳必须返回 cancel_task=True."""
+    t = svc.create_task(video_url="v", audio_url="a")
+    svc.poll_task("w-1")
+    svc.db.refresh(t)
+    # 用户取消：直接把任务置为 cancelled
+    t.status = "cancelled"
+    t.finished_at = datetime.now(UTC)
+    svc.db.commit()
+
+    _w, cancel_task = svc.register_worker("w-1", task_id=t.id)
+    assert cancel_task is True
+    svc.db.refresh(t)
+    assert t.status == "cancelled"  # 心跳不改写已取消状态
+
+
+def test_report_result_cancelled_stays_cancelled(svc):
+    """Worker 终止取消任务后上报失败，report_result 必须保持 cancelled 不回退 pending."""
+    t = svc.create_task(video_url="v", audio_url="a")
+    svc.poll_task("w-1")
+    svc.db.refresh(t)
+    t.status = "cancelled"
+    svc.db.commit()
+
+    result = svc.report_result(t.id, "w-1", success=False, error_msg="推理被终止")
+    assert result.status == "cancelled"
+    assert result.finished_at is not None
+    assert "推理被终止" in (result.error_msg or "")
+
+
+def test_wait_for_result_returns_when_cancelled(svc):
+    """wait_for_result 将 cancelled 视为终态，立即返回，Celery 不回退 MediaKit."""
+    t = svc.create_task(video_url="v", audio_url="a")
+    svc.poll_task("w-1")
+    svc.db.refresh(t)
+    t.status = "cancelled"
+    t.finished_at = datetime.now(UTC)
+    svc.db.commit()
+
+    result = svc.wait_for_result(t.id, timeout_seconds=5, poll_interval=0.1)
+    assert result is not None
+    assert result.status == "cancelled"
+
+
 def test_default_gpu_task_timeout_is_900(svc):
     """#1970 默认超时 300→900，覆盖 RTX2060 长视频推理."""
     assert svc.settings.gpu_task_timeout_seconds == 900

tests/unit/test_lipsync_gpu_async_task.py

@@ -248,3 +248,41 @@ class TestSignMediaUrl:
         with patch.object(task_mod, "get_shared_storage_service", side_effect=RuntimeError("x")):
             url = "https://own-bucket.oss-cn-beijing.aliyuncs.com/a.wav"
             assert task_mod._sign_media_url(url) == url
+
+
+def test_cancelled_gpu_task_does_not_fallback_mediakit(monkeypatch):
+    """GPU 任务被用户取消 → Celery 任务直接标记 cancelled，不回退 MediaKit。"""
+    job = MagicMock()
+    job.id = "job-1"
+    job.status = "processing"
+    job.mediakit_task_id = "gpu:gpu-task-1"
+
+    gpu_task = MagicMock()
+    gpu_task.status = "cancelled"
+    gpu_task.error_msg = "用户取消"
+
+    fake_gpu_svc = MagicMock()
+    fake_gpu_svc.wait_for_result.return_value = gpu_task
+    gpu_service_cls = MagicMock(return_value=fake_gpu_svc)
+
+    fake_db = MagicMock()
+    fake_db.query.return_value.filter_by.return_value.first.return_value = job
+
+    # 直接替换 sys.modules 里的 gpu_lipsync_service 模块（全量跑时它可能已被
+    # 其他测试换成 MagicMock），保证任务函数内 from...import 一定拿到我们的类；
+    # 并替换 _get_db_session 绕开 worker_app / app.db 两条 import 分支。
+    import sys
+    from types import SimpleNamespace
+
+    fake_mod = SimpleNamespace(GpuLipsyncService=gpu_service_cls)
+    monkeypatch.setitem(sys.modules, "app.services.gpu_lipsync_service", fake_mod)
+    monkeypatch.setattr(task_mod, "_get_db_session", lambda: fake_db)
+    monkeypatch.setattr(task_mod, "logger", MagicMock())
+
+    task_mod.lipsync_gpu_process_async.run("job-1", "u1", "gpu-task-1")
+
+    assert job.status == "cancelled"
+    assert not str(job.mediakit_task_id).startswith("mk-")
+    fake_db.commit.assert_called()
+    fake_gpu_svc.wait_for_result.assert_called_once()
+    gpu_service_cls.assert_called_once_with(fake_db)

tests/unit/test_lipsync_gpu_integration.py

@@ -323,3 +323,126 @@ class TestGpuServiceHelpers:
         svc = GpuLipsyncService(db=fake_db)
         fake_db.query.return_value.filter.return_value.first.return_value = None
         assert svc.has_available_worker() is False
+
+
+# ── cancel_job 取消链路 (#2009) ─────────────────────────────────────
+
+
+def _build_sqlite_session():
+    from sqlalchemy import create_engine
+    from sqlalchemy.orm import sessionmaker
+
+    from packages.adapters.sqlalchemy_impl import models as _  # noqa: F401
+    from packages.adapters.sqlalchemy_impl.models import Base
+
+    engine = create_engine("sqlite:///:memory:", future=True)
+    Base.metadata.create_all(engine)
+    Session = sessionmaker(bind=engine, future=True)
+    return Session()
+
+
+def _make_real_job(db, *, status="processing", mediakit_task_id="gpu:gpu-task-1"):
+    import uuid
+
+    from packages.adapters.sqlalchemy_impl.models import LipsyncJobModel
+
+    job = LipsyncJobModel(
+        id=str(uuid.uuid4()),
+        user_id="u1",
+        project_id="p1",
+        video_url="videos/v.mp4",
+        audio_url="audios/a.wav",
+        enable_video_loop=True,
+        mediakit_task_id=mediakit_task_id,
+        status=status,
+    )
+    db.add(job)
+    db.commit()
+    return job
+
+
+def test_cancel_processing_gpu_job_marks_gpu_task_cancelled():
+    """processing 的 GPU job 取消时，关联 GpuLipsyncTask 必须同步置 cancelled."""
+    import uuid
+
+    from packages.adapters.sqlalchemy_impl.models import GpuLipsyncTaskModel
+
+    db = _build_sqlite_session()
+    gpu_task_id = str(uuid.uuid4())
+    gpu_task = GpuLipsyncTaskModel(
+        id=gpu_task_id,
+        video_url="v",
+        audio_url="a",
+        status="processing",
+        worker_id="w-1",
+        attempt=1,
+    )
+    db.add(gpu_task)
+    db.commit()
+
+    job = _make_real_job(db, mediakit_task_id=f"gpu:{gpu_task_id}")
+    from app.services.lipsync_service import LipsyncService
+
+    svc = LipsyncService(db=db, client=MagicMock())
+    result = svc.cancel_job(job.id, "u1")
+
+    assert result.status == "cancelled"
+    db.refresh(gpu_task)
+    assert gpu_task.status == "cancelled"
+    assert gpu_task.error_msg == "用户取消"
+    assert gpu_task.finished_at is not None
+
+
+def test_cancel_processing_gpu_job_skips_non_processing_gpu_task():
+    """GPU task 已不在 processing（如已 done）时，取消 job 不应改它，也不报错."""
+    import uuid
+
+    from packages.adapters.sqlalchemy_impl.models import GpuLipsyncTaskModel
+
+    db = _build_sqlite_session()
+    gpu_task_id = str(uuid.uuid4())
+    gpu_task = GpuLipsyncTaskModel(
+        id=gpu_task_id, video_url="v", audio_url="a", status="done", worker_id="w-1", attempt=1
+    )
+    db.add(gpu_task)
+    db.commit()
+
+    job = _make_real_job(db, mediakit_task_id=f"gpu:{gpu_task_id}")
+    from app.services.lipsync_service import LipsyncService
+
+    svc = LipsyncService(db=db, client=MagicMock())
+    result = svc.cancel_job(job.id, "u1")
+
+    assert result.status == "cancelled"
+    db.refresh(gpu_task)
+    assert gpu_task.status == "done"  # 没被动
+
+
+def test_cancel_processing_non_gpu_job_does_not_touch_gpu_table():
+    """mediakit_task_id 不是 gpu: 前缀（普通 MediaKit 任务）时，不查 GPU task."""
+    db = _build_sqlite_session()
+    job = _make_real_job(db, mediakit_task_id="mk-task-99")
+    from app.services.lipsync_service import LipsyncService
+
+    svc = LipsyncService(db=db, client=MagicMock())
+    result = svc.cancel_job(job.id, "u1")
+    assert result.status == "cancelled"
+
+
+def test_cancel_completed_job_unchanged():
+    """completed 状态不可取消，cancel_job 原样返回."""
+    db = _build_sqlite_session()
+    job = _make_real_job(db, status="completed", mediakit_task_id="gpu:x")
+    from app.services.lipsync_service import LipsyncService
+
+    svc = LipsyncService(db=db, client=MagicMock())
+    result = svc.cancel_job(job.id, "u1")
+    assert result.status == "completed"
+
+
+def test_cancel_job_not_found_returns_none():
+    db = _build_sqlite_session()
+    from app.services.lipsync_service import LipsyncService
+
+    svc = LipsyncService(db=db, client=MagicMock())
+    assert svc.cancel_job("nonexistent", "u1") is None