feat(generation): worker孤儿任务自动恢复 + 429限流结构化提示 (#1677) (#1710)

Co-authored-by: xiaoxia <dev@xiaoxiajianji.com>
Co-committed-by: xiaoxia <dev@xiaoxiajianji.com>

apps/api/app/api/routes/generation_preview.py

@@ -14,6 +14,7 @@ from app.core.task_enqueue import (
     USER_PENDING_LIMIT,
     GlobalQueueFull,
     UserPendingLimitExceeded,
+    build_rate_limit_detail,
     safe_enqueue_generation_task,
 )
 from app.dependencies import (
@@ -312,12 +313,12 @@ def create_preview_generation_task(
     except UserPendingLimitExceeded as e:
         raise HTTPException(
             status_code=429,
-            detail=f"您的待处理任务过多（当前 {e.pending_count - count}/{e.limit}，本次提交 {count} 个），请等待后再提交",
+            detail=build_rate_limit_detail(e, generation_task_repository, scope="user"),
         ) from e
     except GlobalQueueFull as e:
         raise HTTPException(
             status_code=503,
-            detail="系统繁忙，请稍后再试",
+            detail=build_rate_limit_detail(e, generation_task_repository, scope="global"),
         ) from e
 
     # 确定视频比例：优先前端传入，否则从模板 mode 推断
@@ -498,6 +499,7 @@ def create_preview_generation_task(
 
     # ── 入队 ──
     responses: list[PreviewGenerationTaskResponse] = []
+    rate_limit_exc: Exception | None = None  # 记录首个限流异常，全部失败时返回结构化提示
     for variant_index, task in enumerate(created_tasks):
         try:
             enqueued = safe_enqueue_generation_task(
@@ -510,23 +512,29 @@ def create_preview_generation_task(
             if not enqueued:
                 logger.warning("[预览生成] 任务入队失败: task_id=%s", task.id)
                 _mark_task_failed(generation_task_repository, task, "任务入队失败")
-        except UserPendingLimitExceeded:
+        except UserPendingLimitExceeded as e:
             _mark_task_failed(generation_task_repository, task, "待处理任务超限")
-        except GlobalQueueFull:
+            rate_limit_exc = rate_limit_exc or e
+        except GlobalQueueFull as e:
             _mark_task_failed(generation_task_repository, task, "系统队列已满")
+            rate_limit_exc = rate_limit_exc or e
         except Exception:
             logger.exception("[预览生成] 入队异常: task_id=%s", task.id)
             _mark_task_failed(generation_task_repository, task, "任务入队异常")
         # enqueue 会原地更新 task 状态/进度，直接用 task 构造响应
         responses.append(_to_preview_response(task))
 
-    # 队列满/限流时若全部失败，返回明确错误码
-    if all(r.status == "failed" for r in responses):
-        first_err = next((r.error_message for r in responses if r.error_message), "")
-        if "待处理任务" in first_err:
-            raise HTTPException(status_code=429, detail=first_err or "待处理任务超限")
-        if "队列" in first_err:
-            raise HTTPException(status_code=503, detail=first_err or "系统繁忙，请稍后再试")
+    # 队列满/限流时若全部失败，返回结构化错误码（前端区分"排队"与"创建失败"）
+    if all(r.status == "failed" for r in responses) and rate_limit_exc is not None:
+        if isinstance(rate_limit_exc, UserPendingLimitExceeded):
+            raise HTTPException(
+                status_code=429,
+                detail=build_rate_limit_detail(rate_limit_exc, generation_task_repository, scope="user"),
+            )
+        raise HTTPException(
+            status_code=503,
+            detail=build_rate_limit_detail(rate_limit_exc, generation_task_repository, scope="global"),
+        )
 
     logger.info(
         "[预览生成] 创建完成: %d 个变体任务, task_ids=%s",

apps/api/app/api/routes/generation_tasks.py

@@ -10,6 +10,7 @@ from app.core.task_enqueue import (
     USER_PENDING_LIMIT,
     GlobalQueueFull,
     UserPendingLimitExceeded,
+    build_rate_limit_detail,
     safe_enqueue_generation_task,
 )
 from app.dependencies import (
@@ -417,12 +418,12 @@ def create_generation_task(
     except UserPendingLimitExceeded as e:
         raise HTTPException(
             status_code=429,
-            detail=f"您的待处理任务过多（当前 {e.pending_count - count}/{e.limit}，本次提交 {count} 个），请等待完成后再提交",
+            detail=build_rate_limit_detail(e, generation_task_repository, scope="user"),
         ) from e
     except GlobalQueueFull as e:
         raise HTTPException(
             status_code=503,
-            detail="系统繁忙，请稍后再试",
+            detail=build_rate_limit_detail(e, generation_task_repository, scope="global"),
         ) from e
 
     # 画中画已下线：strategy_id 中的 pip/voice_pip 统一映射为 one_take
@@ -579,7 +580,7 @@ def create_generation_task(
                 if not created_tasks:
                     raise HTTPException(
                         status_code=429,
-                        detail="您的待处理任务过多，请等待完成后再提交",
+                        detail=build_rate_limit_detail(_e, generation_task_repository, scope="user"),
                     ) from _e
                 break
             except GlobalQueueFull as _e:
@@ -587,7 +588,7 @@ def create_generation_task(
                 if not created_tasks:
                     raise HTTPException(
                         status_code=503,
-                        detail="系统繁忙，请稍后再试",
+                        detail=build_rate_limit_detail(_e, generation_task_repository, scope="global"),
                     ) from _e
                 break
     except HTTPException:
@@ -713,15 +714,15 @@ def confirm_generation(
             log_task_status=True,
         ):
             logger.warning("[确认生成] 入队失败: task_id=%s", new_task.id)
-    except UserPendingLimitExceeded:
+    except UserPendingLimitExceeded as _e:
         raise HTTPException(
             status_code=429,
-            detail="您的待处理任务过多，请等待完成后再提交",
+            detail=build_rate_limit_detail(_e, generation_task_repository, scope="user"),
         ) from None
-    except GlobalQueueFull:
+    except GlobalQueueFull as _e:
         raise HTTPException(
             status_code=503,
-            detail="系统繁忙，请稍后再试",
+            detail=build_rate_limit_detail(_e, generation_task_repository, scope="global"),
         ) from None
 
     return BatchGenerationTaskResponse(
@@ -803,12 +804,24 @@ def retry_generation_task(
     if user_pending >= USER_PENDING_LIMIT:
         raise HTTPException(
             status_code=429,
-            detail=f"您的待处理任务过多（当前 {user_pending}/{USER_PENDING_LIMIT}），请等待完成后再提交",
+            detail=build_rate_limit_detail(
+                UserPendingLimitExceeded(
+                    user_id=user_id,
+                    pending_count=user_pending,
+                    limit=USER_PENDING_LIMIT,
+                ),
+                generation_task_repository,
+                scope="user",
+            ),
         )
     if global_pending >= GLOBAL_PENDING_LIMIT:
         raise HTTPException(
             status_code=503,
-            detail="系统繁忙，请稍后再试",
+            detail=build_rate_limit_detail(
+                GlobalQueueFull(pending_count=global_pending, limit=GLOBAL_PENDING_LIMIT),
+                generation_task_repository,
+                scope="global",
+            ),
         )
 
     use_case = CreateGenerationTaskUseCase(generation_task_repository)
@@ -843,15 +856,15 @@ def retry_generation_task(
             log_task_status=True,
         ):
             logger.warning("[生成任务] 重试入队失败: task_id=%s", retried.id)
-    except UserPendingLimitExceeded:
+    except UserPendingLimitExceeded as _e:
         raise HTTPException(
             status_code=429,
-            detail="您的待处理任务过多，请等待完成后再提交",
+            detail=build_rate_limit_detail(_e, generation_task_repository, scope="user"),
         ) from None
-    except GlobalQueueFull:
+    except GlobalQueueFull as _e:
         raise HTTPException(
             status_code=503,
-            detail="系统繁忙，请稍后再试",
+            detail=build_rate_limit_detail(_e, generation_task_repository, scope="global"),
         ) from None
     return _to_generation_task_response(retried)
 

apps/api/app/core/task_enqueue.py

@@ -8,27 +8,145 @@ logger = logging.getLogger(__name__)
 # ── 限流阈值常量（全系统统一管理，不要在业务代码里硬编码） ──
 USER_PENDING_LIMIT = 3  # 单用户 pending 上限
 GLOBAL_PENDING_LIMIT = 20  # 全局 pending 上限
+WORKER_CONCURRENCY = 4  # worker 渲染并发数（infra/docker/compose.yml WORKER_CONCURRENCY 默认值）
+
+# 限流错误码：前端据此区分"排队等待"与"创建失败"
+ERROR_CODE_USER_QUEUE_FULL = "USER_QUEUE_FULL"  # 429：用户自己的任务排队中
+ERROR_CODE_SYSTEM_QUEUE_FULL = "SYSTEM_QUEUE_FULL"  # 503：系统整体繁忙
 
 
 class UserPendingLimitExceeded(Exception):
     """用户 pending 任务数超限，返回 429。"""
 
-    def __init__(self, user_id: str, pending_count: int, limit: int):
+    def __init__(
+        self,
+        user_id: str,
+        pending_count: int,
+        limit: int,
+        *,
+        running_count: int = 0,
+        requested_count: int = 1,
+        queue_ahead: int = 0,
+        estimated_wait_seconds: int = 0,
+    ):
         self.user_id = user_id
         self.pending_count = pending_count
         self.limit = limit
+        # 排队上下文（用于 429 结构化提示，前端展示"排队中"而非"创建失败"）
+        self.running_count = running_count
+        self.requested_count = requested_count
+        self.queue_ahead = queue_ahead
+        self.estimated_wait_seconds = estimated_wait_seconds
         super().__init__(f"用户 {user_id} pending 任务数 {pending_count} 超过上限 {limit}")
 
 
 class GlobalQueueFull(Exception):
     """全局限流，返回 503。"""
 
-    def __init__(self, pending_count: int, limit: int):
+    def __init__(
+        self,
+        pending_count: int,
+        limit: int,
+        *,
+        running_count: int = 0,
+        queue_ahead: int = 0,
+        estimated_wait_seconds: int = 0,
+    ):
         self.pending_count = pending_count
         self.limit = limit
+        self.running_count = running_count
+        self.queue_ahead = queue_ahead
+        self.estimated_wait_seconds = estimated_wait_seconds
         super().__init__(f"系统 pending 任务数 {pending_count} 超过上限 {limit}")
 
 
+def _estimate_wait_seconds(queue_ahead: int, generation_task_repository: Any) -> int:
+    """根据排队任务数 + worker 并发数 + 历史平均任务耗时估算等待秒数。
+
+    估算公式：ceil(排队任务数 / 并发数) × 平均单任务耗时。
+    拿不到历史数据时仓储层返回默认 120 秒。
+    """
+    import math
+
+    if queue_ahead <= 0:
+        return 0
+    try:
+        estimator = getattr(generation_task_repository, "estimate_avg_duration_seconds", None)
+        avg_seconds = estimator() if estimator is not None else 120.0
+    except Exception:
+        avg_seconds = 120.0
+    return int(math.ceil(queue_ahead / WORKER_CONCURRENCY) * avg_seconds)
+
+
+def build_rate_limit_detail(
+    exc: Exception,
+    generation_task_repository: Any,
+    *,
+    scope: str = "user",
+) -> dict:
+    """构造结构化限流响应体（HTTPException 的 detail）。
+
+    前端按 detail.code 判断场景：
+    - USER_QUEUE_FULL (429)：用户自己的任务在排队，应提示"等待/继续排队"，不是创建失败
+    - SYSTEM_QUEUE_FULL (503)：系统繁忙，稍后重试
+
+    detail 字段：
+    - code: 错误码
+    - message: 可读中文提示（可直接展示）
+    - queued_count: 当前排队(pending)任务数
+    - running_count: 当前渲染中(running)任务数
+    - queue_ahead: 前方排队任务数（预计等待批次依据）
+    - estimated_wait_seconds: 预计等待秒数
+    - limit: 对应限流上限
+    """
+    if scope == "user" and isinstance(exc, UserPendingLimitExceeded):
+        running = exc.running_count
+        if not running:
+            try:
+                counter = getattr(generation_task_repository, "count_running_by_user", None)
+                running = counter(exc.user_id) if counter is not None else 0
+            except Exception:
+                running = 0
+        queue_ahead = exc.queue_ahead or max(exc.pending_count, 0)
+        wait = exc.estimated_wait_seconds or _estimate_wait_seconds(queue_ahead, generation_task_repository)
+        wait_minutes = max(1, round(wait / 60))
+        message = (
+            f"您有 {exc.pending_count} 个任务正在排队、{running} 个正在渲染，"
+            f"同一时间最多提交 {exc.limit} 个任务。请等待约 {wait_minutes} 分钟后再提交"
+        )
+        return {
+            "code": ERROR_CODE_USER_QUEUE_FULL,
+            "message": message,
+            "queued_count": exc.pending_count,
+            "running_count": running,
+            "queue_ahead": queue_ahead,
+            "estimated_wait_seconds": wait,
+            "limit": exc.limit,
+        }
+
+    # 全局繁忙
+    pending = getattr(exc, "pending_count", 0)
+    running = getattr(exc, "running_count", 0)
+    if not running:
+        try:
+            counter = getattr(generation_task_repository, "count_running_total", None)
+            running = counter() if counter is not None else 0
+        except Exception:
+            running = 0
+    queue_ahead = getattr(exc, "queue_ahead", 0) or pending
+    wait = getattr(exc, "estimated_wait_seconds", 0) or _estimate_wait_seconds(queue_ahead, generation_task_repository)
+    wait_minutes = max(1, round(wait / 60))
+    return {
+        "code": ERROR_CODE_SYSTEM_QUEUE_FULL,
+        "message": f"系统繁忙：当前 {pending} 个任务排队中、{running} 个渲染中，预计等待约 {wait_minutes} 分钟，请稍后再试",
+        "queued_count": pending,
+        "running_count": running,
+        "queue_ahead": queue_ahead,
+        "estimated_wait_seconds": wait,
+        "limit": getattr(exc, "limit", GLOBAL_PENDING_LIMIT),
+    }
+
+
 def check_queue_limits(
     user_id: str,
     generation_task_repository: Any,