fix(P0): 修复取消链路断裂，前端取消后 GPU 仍继续推理 (#2009)

Co-authored-by: xiaoxia <dev@xiaoxiajianji.com>
Co-committed-by: xiaoxia <dev@xiaoxiajianji.com>

apps/api/app/services/gpu_lipsync_service.py

@@ -50,13 +50,16 @@ class GpuLipsyncService:
         free_vram_mb: int = 0,
         capabilities: str = "musetalk",
         task_id: Optional[str] = None,
-    ) -> GpuWorkerModel:
+    ) -> tuple[GpuWorkerModel, bool]:
         """Worker 注册/心跳。
 
         task_id 非空时（Worker 推理期间的任务级心跳），同步把对应 processing
         任务的 last_heartbeat_at 续到当前时间，使长推理不会被
         ``_recover_timed_out_tasks`` 误回退。任务已结束 / 不属于该 worker
         （如已被超时回收重新派发）时忽略，不报错。
+
+        返回 ``(worker, cancel_task)``：当心跳任务已被用户取消时
+        ``cancel_task=True``，Worker 应尽快终止推理并释放 GPU。
         """
         now = datetime.now(UTC)
         worker = self.db.query(GpuWorkerModel).filter(GpuWorkerModel.worker_id == worker_id).one_or_none()
@@ -77,10 +80,11 @@ class GpuLipsyncService:
             worker.free_vram_mb = free_vram_mb
             worker.capabilities = capabilities or worker.capabilities
             worker.last_heartbeat_at = now
+        cancel_task = False
         if task_id:
-            self._touch_task_heartbeat(task_id, worker_id, now)
+            cancel_task = self._touch_task_heartbeat(task_id, worker_id, now)
         self.db.commit()
-        return worker
+        return worker, cancel_task
 
     # ── 轮询拉任务（Worker 调用） ──────────────────────────────────
 
@@ -166,6 +170,11 @@ class GpuLipsyncService:
             task.result_duration = duration_seconds or 0.0
             task.error_msg = ""
             task.finished_at = now
+        elif task.status == "cancelled":
+            # 用户已取消的任务，Worker 终止后上报失败，保持 cancelled 状态不回退
+            task.finished_at = now
+            task.error_msg = (error_msg or "用户取消")[:2000]
+            logger.info("GPU 任务 %s 已被用户取消，保持 cancelled 状态", task_id)
         else:
             # 失败：若仍可重试（已尝试次数 < MAX_ATTEMPTS）→ 回退 pending；否则 → failed
             if task.attempt < MAX_ATTEMPTS:
@@ -250,15 +259,21 @@ class GpuLipsyncService:
     def _result_key(self, task_id: str) -> str:
         return f"{self.RESULT_PREFIX}{task_id}.mp4"
 
-    def _touch_task_heartbeat(self, task_id: str, worker_id: str, now: datetime) -> None:
+    def _touch_task_heartbeat(self, task_id: str, worker_id: str, now: datetime) -> bool:
         """Worker 推理期间的任务级心跳：只刷新属于该 worker 且仍在 processing 的任务。
 
         任务不存在 / 已被超时回收重新派发 / 已完成 → 静默忽略（此时旧 worker 的
         结果上报会被结果接口按最终态处理）。
+
+        返回 ``cancel_task``：任务已被用户取消时为 True，Worker 应终止推理。
         """
         task = self.db.get(GpuLipsyncTaskModel, task_id)
         if task is None:
-            return
+            return False
+        # 任务已被用户取消 → 通知 Worker 终止推理
+        if task.status == "cancelled":
+            logger.info("任务心跳检测到已取消 task=%s worker=%s，通知 Worker 终止", task_id, worker_id)
+            return True
         if task.status != "processing" or task.worker_id != worker_id:
             logger.info(
                 "忽略过期任务心跳 task=%s worker=%s（status=%s owner=%s）",
@@ -267,10 +282,11 @@ class GpuLipsyncService:
                 task.status,
                 task.worker_id,
             )
-            return
+            return False
         task.last_heartbeat_at = now
         task.updated_at = now
         self.db.flush()
+        return False
 
     def _touch_worker(self, worker_id: str, now: datetime) -> None:
         if not worker_id:
@@ -371,9 +387,7 @@ class GpuLipsyncService:
             task = self.db.get(GpuLipsyncTaskModel, task_id)
             if task is None:
                 return None
-            if task.status == "done":
-                return task
-            if task.status == "failed":
+            if task.status in ("done", "failed", "cancelled"):
                 return task
             # pending/processing 继续等
             if time.monotonic() >= deadline:

apps/api/app/services/lipsync_service.py

@@ -783,12 +783,32 @@ class LipsyncService:
     # ── 取消任务 ──────────────────────────────────────────────────────────
 
     def cancel_job(self, job_id: str, user_id: str) -> Optional[LipsyncJobModel]:
-        """取消任务（仅 pending/tts_processing/submitted 状态可取消）."""
+        """取消任务（pending/tts_processing/submitted/processing 状态可取消）.
+
+        当 job 走 GPU 路径（mediakit_task_id 以 "gpu:" 开头）且状态为 processing 时，
+        同步将关联的 GpuLipsyncTask 标记为 cancelled，以便 Worker 心跳时检测到取消信号。
+        """
         job = self.get_job(job_id, user_id)
         if job is None:
             return None
 
-        if job.status in ("pending", "tts_processing", "submitted"):
+        if job.status in ("pending", "tts_processing", "submitted", "processing"):
+            # GPU 路径：同步标记关联的 GPU 任务为 cancelled
+            if job.status == "processing" and job.mediakit_task_id and job.mediakit_task_id.startswith("gpu:"):
+                gpu_task_id = job.mediakit_task_id[4:]  # 去掉 "gpu:" 前缀
+                try:
+                    from packages.adapters.sqlalchemy_impl.models import GpuLipsyncTaskModel
+
+                    gpu_task = self.db.get(GpuLipsyncTaskModel, gpu_task_id)
+                    if gpu_task and gpu_task.status == "processing":
+                        gpu_task.status = "cancelled"
+                        gpu_task.error_msg = "用户取消"
+                        gpu_task.updated_at = datetime.now(UTC)
+                        gpu_task.finished_at = datetime.now(UTC)
+                        logger.info("GPU 任务 %s 已被用户取消（通过 job_id=%s）", gpu_task_id, job_id)
+                except Exception as exc:
+                    logger.warning("标记 GPU 任务取消失败（不影响 job 取消）: %s", exc)
+
             job.status = "cancelled"
             job.updated_at = datetime.now(UTC)
             self.db.commit()