fix(gpu): #1978 MuseTalk v1.5 推理修复与Worker稳定性补丁

musetalk_server.py:
- v3 API兼容性：支持新的请求格式（audio_url/video_url字段）
- v4 正确pipeline：使用vae.get_latents_for_unet()生成8ch输入
- v4.4 VRAM优化：batch_size=2 + CPU offload + PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128
- RTX2060 6GB可稳定运行，22s视频推理约204s（0.11x实时）

gpu_worker.py:
- 修复健康检查响应格式适配musetalk_server
- REQUEST_TIMEOUT提升到900s，与服务端GPU_TASK_TIMEOUT_SECONDS对齐
- TASK_MAX_RETRY降到1，避免与服务端MAX_ATTEMPTS=3相乘放大重试
- 增加TASK_HEARTBEAT_INTERVAL=30s，推理期间独立线程心跳防超时
- 增加MIN_VIDEO_DURATION_SECONDS=3s前置拦截，避免1s视频division by zero

deploy/gpu_worker/gpu_worker.py

@@ -9,9 +9,12 @@
   WORKER_ID           本机唯一 ID（默认 hostname+网卡MAC 后4位）
   MUSE_TALK_URL       本地 MuseTalk 地址，默认 http://127.0.0.1:7861
   POLL_INTERVAL       轮询间隔秒，默认 5
-  HEARTBEAT_INTERVAL  心跳间隔秒，默认 15
-  REQUEST_TIMEOUT     HTTP 请求超时秒，默认 60
-  TASK_MAX_RETRY      单个任务最大重试次数（在 Worker 本地的重试），默认 2
+  HEARTBEAT_INTERVAL  空闲心跳间隔秒，默认 15
+  REQUEST_TIMEOUT     HTTP 请求超时秒（下载/推理/上传统一使用），默认 900
+                      需与服务端 GPU_TASK_TIMEOUT_SECONDS（默认 900）对齐
+  TASK_MAX_RETRY      单任务本地最大重试次数（仅对瞬时错误重试），默认 1
+  TASK_HEARTBEAT_INTERVAL  推理期间任务心跳间隔秒，默认 30
+  MIN_VIDEO_DURATION_SECONDS  最短输入视频时长秒，小于则直接上报失败，默认 3
 
 用法：
   python gpu_worker.py
@@ -19,13 +22,13 @@
 
 from __future__ import annotations
 
-import json
 import logging
 import os
 import platform
 import socket
 import sys
 import tempfile
+import threading
 import time
 import uuid
 from pathlib import Path
@@ -54,8 +57,17 @@ class Config:
     muse_talk_url: str = _env("MUSE_TALK_URL", "http://127.0.0.1:7861").rstrip("/")
     poll_interval: float = float(_env("POLL_INTERVAL", "5"))
     heartbeat_interval: float = float(_env("HEARTBEAT_INTERVAL", "15"))
-    request_timeout: float = float(_env("REQUEST_TIMEOUT", "300"))
-    task_max_retry: int = int(_env("TASK_MAX_RETRY", "2"))
+    # #1970：RTX2060 6G 处理 720p 长视频可能 >5min；与服务端
+    # GPU_TASK_TIMEOUT_SECONDS 默认值对齐为 900，避免推理被本地/服务端先掐断。
+    request_timeout: float = float(_env("REQUEST_TIMEOUT", "900"))
+    # 本地只在网络/MuseTalk 瞬时错误时重试 1 次；服务端 MAX_ATTEMPTS=3
+    # 负责跨 worker/真正超时后的重派发，总尝试次数不再相乘放大。
+    task_max_retry: int = int(_env("TASK_MAX_RETRY", "1"))
+    # 推理期间任务心跳间隔（独立线程 POST /gpu/register 带 task_id）
+    task_heartbeat_interval: float = float(_env("TASK_HEARTBEAT_INTERVAL", "30"))
+    # 输入视频最短时长（秒）：过短（如 1s）MuseTalk 会 division by zero，
+    # 本地前置拦截，直接上报 failed，不浪费 GPU 时间
+    min_video_duration_seconds: float = float(_env("MIN_VIDEO_DURATION_SECONDS", "3"))
     worker_id: str = _env("WORKER_ID", "")
 
     @classmethod
@@ -96,11 +108,20 @@ def _check_musetalk_health() -> tuple[bool, dict]:
         return False, {"error": str(exc)}
 
 
-def _register() -> bool:
-    """向服务端注册 / 心跳，附带 GPU 信息."""
+def _register(task_id: Optional[str] = None) -> bool:
+    """向服务端注册 / 心跳，附带 GPU 信息。
+
+    推理期间的心跳线程传 task_id：服务端会同步刷新该 processing 任务的
+    last_heartbeat_at，防止长推理被误判超时回收。
+    """
     ok, info = _check_musetalk_health()
-    free_vram = int(info.get("free_vram_mb", 0) or 0) if isinstance(info, dict) else 0
-    gpu_name = info.get("gpu_name", "") if isinstance(info, dict) else ""
+    if isinstance(info, dict):
+        gpu_info = info.get("gpu", info)
+        free_vram = int(gpu_info.get("free_vram_mb", gpu_info.get("memory_free_mb", 0)) or 0)
+        gpu_name = gpu_info.get("gpu_name", info.get("gpu_name", ""))
+    else:
+        free_vram = 0
+        gpu_name = ""
     if not gpu_name:
         # 尝试在 Windows 上读 nvidia-smi
         gpu_name = _probe_gpu_name()
@@ -111,6 +132,8 @@ def _register() -> bool:
         "free_vram_mb": free_vram,
         "capabilities": "musetalk",
     }
+    if task_id:
+        payload["task_id"] = task_id
     try:
         r = requests.post(
             f"{Config.api_base_url}/api/v1/gpu/register",
@@ -181,11 +204,13 @@ def _download(url: str, path: Path) -> bool:
         return False
 
 
-def _call_musetalk(video_path: Path, audio_path: Path, out_path: Path) -> tuple[bool, float, str]:
+def _call_musetalk(video_path: Path, audio_path: Path, out_path: Path) -> tuple[bool, float, str, bool]:
     """调用本地 MuseTalk /inference.
 
-    返回 (success, duration_seconds, error_msg).
+    返回 (success, duration_seconds, error_msg, retryable)。
     duration 用 ffprobe 读结果视频，失败填 0。
+    retryable 仅对瞬时错误（连接失败/超时/5xx）为 True；HTTP 4xx、结果过小
+    等确定性失败不重试，直接上报服务端（服务端 MAX_ATTEMPTS 再决定是否重派发）。
     """
     try:
         with open(video_path, "rb") as vf, open(audio_path, "rb") as af:
@@ -199,17 +224,34 @@ def _call_musetalk(video_path: Path, audio_path: Path, out_path: Path) -> tuple[
                 timeout=Config.request_timeout,
             )
         if r.status_code != 200:
-            return False, 0.0, f"MuseTalk HTTP {r.status_code}: {r.text[:500]}"
+            retryable = r.status_code >= 500
+            return False, 0.0, f"MuseTalk HTTP {r.status_code}: {r.text[:500]}", retryable
         out_path.parent.mkdir(parents=True, exist_ok=True)
         out_path.write_bytes(r.content)
         if out_path.stat().st_size < 1024:
-            return False, 0.0, f"MuseTalk 返回结果过小 ({out_path.stat().st_size} bytes)"
+            # 确定性失败（推理产物异常），本地重试大概率还是坏的，不重试
+            return False, 0.0, f"MuseTalk 返回结果过小 ({out_path.stat().st_size} bytes)", False
         duration = _probe_duration(out_path)
-        return True, duration, ""
-    except requests.exceptions.Timeout:
-        return False, 0.0, f"MuseTalk 推理超时（>{Config.request_timeout}s）"
+        return True, duration, "", False
+    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError):
+        # 瞬时网络/超时错误，允许本地重试 1 次；同时调 /cancel 让服务端终止僵尸推理
+        _cancel_musetalk()
+        return False, 0.0, f"MuseTalk 推理超时或连接失败（>{Config.request_timeout}s）", True
     except Exception as exc:
-        return False, 0.0, f"MuseTalk 调用异常: {exc}"
+        return False, 0.0, f"MuseTalk 调用异常: {exc}", False
+
+
+def _cancel_musetalk() -> None:
+    """调 MuseTalk /cancel 端点终止服务端僵尸推理进程，避免超时后任务还在跑占显存."""
+    try:
+        r = requests.post(f"{Config.muse_talk_url}/cancel", timeout=10)
+        if r.status_code == 200:
+            logger.info("已调 MuseTalk /cancel，服务端终止推理")
+        else:
+            logger.warning("MuseTalk /cancel 返回 %d: %s", r.status_code, r.text[:200])
+    except Exception as exc:
+        # /cancel 失败不应影响主流程上报
+        logger.warning("调 MuseTalk /cancel 异常（忽略）: %s", exc)
 
 
 def _probe_duration(path: Path) -> float:
@@ -219,9 +261,13 @@ def _probe_duration(path: Path) -> float:
 
         out = subprocess.check_output(
             [
-                "ffprobe", "-v", "error",
-                "-show_entries", "format=duration",
-                "-of", "default=noprint_wrappers=1:nokey=1",
+                "ffprobe",
+                "-v",
+                "error",
+                "-show_entries",
+                "format=duration",
+                "-of",
+                "default=noprint_wrappers=1:nokey=1",
                 str(path),
             ],
             stderr=subprocess.DEVNULL,
@@ -276,42 +322,91 @@ def _report_result(task_id: str, success: bool, duration: float = 0.0, error_msg
         return False
 
 
+class TaskHeartbeat(threading.Thread):
+    """推理期间的任务心跳线程。
+
+    主循环的空闲心跳在 ``_handle_task`` 同步阻塞（下载/推理/上传最长 900s）
+    期间无法发送，服务端会因任务 last_heartbeat_at 停滞而误判超时回退 pending。
+    本线程每 task_heartbeat_interval 秒（默认 30s）POST /gpu/register 并
+    携带当前 task_id，让服务端持续续期任务心跳；任务处理结束 stop()。
+    """
+
+    def __init__(self, task_id: str, interval: float):
+        super().__init__(daemon=True, name=f"hb-{task_id[:8]}")
+        self.task_id = task_id
+        self.interval = max(5.0, interval)
+        self._stop_event = threading.Event()
+
+    def run(self) -> None:
+        # 先立即发一次，再按间隔循环（首次心跳失败不影响主流程）
+        while not self._stop_event.is_set():
+            try:
+                if _register(self.task_id):
+                    logger.debug("任务 %s 心跳已发送", self.task_id)
+            except Exception as exc:  # noqa: BLE001
+                logger.warning("任务 %s 心跳异常（忽略）: %s", self.task_id, exc)
+            self._stop_event.wait(self.interval)
+
+    def stop(self) -> None:
+        self._stop_event.set()
+
+
 def _handle_task(task: dict) -> None:
-    """处理一条任务（整个串行流程：下载→推理→上传→上报）."""
+    """处理一条任务（整个串行流程：下载→时长校验→推理→上传→上报）。"""
     task_id = task["task_id"]
     logger.info("开始处理任务 %s", task_id)
-    with tempfile.TemporaryDirectory(prefix="musetalk_") as tmpdir:
-        tmp = Path(tmpdir)
-        video_path = tmp / "input.mp4"
-        audio_path = tmp / "input_audio.bin"
-        out_path = tmp / "output.mp4"
+    # 领取任务后立即启动任务级心跳线程，覆盖下载/推理/上报全过程
+    hb = TaskHeartbeat(task_id, Config.task_heartbeat_interval)
+    hb.start()
+    try:
+        with tempfile.TemporaryDirectory(prefix="musetalk_") as tmpdir:
+            tmp = Path(tmpdir)
+            video_path = tmp / "input.mp4"
+            audio_path = tmp / "input_audio.bin"
+            out_path = tmp / "output.mp4"
 
-        # 1. 下载
-        if not _download(task["video_url"], video_path):
-            _report_result(task_id, False, 0.0, "下载人物视频失败")
-            return
-        if not _download(task["audio_url"], audio_path):
-            _report_result(task_id, False, 0.0, "下载驱动音频失败")
-            return
+            # 1. 下载
+            if not _download(task["video_url"], video_path):
+                _report_result(task_id, False, 0.0, "下载人物视频失败")
+                return
+            if not _download(task["audio_url"], audio_path):
+                _report_result(task_id, False, 0.0, "下载驱动音频失败")
+                return
 
-        # 2. 推理（本地重试）
-        success = False
-        duration = 0.0
-        err = ""
-        for attempt in range(Config.task_max_retry + 1):
-            if attempt > 0:
-                logger.info("任务 %s 第 %d 次重试...", task_id, attempt + 1)
-                time.sleep(2)
-            success, duration, err = _call_musetalk(video_path, audio_path, out_path)
-            if success:
-                break
-        if not success:
-            logger.error("任务 %s 推理失败: %s", task_id, err)
-            _report_result(task_id, False, 0.0, err)
-            return
+            # 2. 输入时长前置校验：短视频 MuseTalk 会 division by zero，
+            #    直接上报 failed，不浪费 GPU 时间。ffprobe 不可用/读失败（0.0）
+            #    时不拦截，交给 MuseTalk 处理，避免误杀。
+            video_duration = _probe_duration(video_path)
+            if video_duration and video_duration < Config.min_video_duration_seconds:
+                msg = (
+                    f"视频过短（{video_duration:.2f}s < {Config.min_video_duration_seconds:.0f}s），"
+                    "MuseTalk 无法处理"
+                )
+                logger.error("任务 %s %s", task_id, msg)
+                _report_result(task_id, False, 0.0, msg)
+                return
 
-        # 3. 上报结果（multipart 同时上传文件 → API 代为 PUT 到 OSS，逻辑最稳）
-        _report_success_with_file(task_id, duration, out_path)
+            # 3. 推理（本地仅对瞬时错误重试）
+            success = False
+            duration = 0.0
+            err = ""
+            retryable = False
+            for attempt in range(Config.task_max_retry + 1):
+                if attempt > 0:
+                    logger.info("任务 %s 第 %d 次重试（瞬时错误）...", task_id, attempt + 1)
+                    time.sleep(2)
+                success, duration, err, retryable = _call_musetalk(video_path, audio_path, out_path)
+                if success or not retryable:
+                    break
+            if not success:
+                logger.error("任务 %s 推理失败: %s", task_id, err)
+                _report_result(task_id, False, 0.0, err)
+                return
+
+            # 4. 上报结果（multipart 同时上传文件 → API 代为 PUT 到 OSS，逻辑最稳）
+            _report_success_with_file(task_id, duration, out_path)
+    finally:
+        hb.stop()
 
 
 def _report_success_with_file(task_id: str, duration: float, file_path: Path) -> None: