feat(generation): worker孤儿任务自动恢复 + 429限流结构化提示 (#1677) (#1710)

Co-authored-by: xiaoxia <dev@xiaoxiajianji.com>
Co-committed-by: xiaoxia <dev@xiaoxiajianji.com>

apps/api/app/core/task_enqueue.py

@@ -8,27 +8,145 @@ logger = logging.getLogger(__name__)
 # ── 限流阈值常量（全系统统一管理，不要在业务代码里硬编码） ──
 USER_PENDING_LIMIT = 3  # 单用户 pending 上限
 GLOBAL_PENDING_LIMIT = 20  # 全局 pending 上限
+WORKER_CONCURRENCY = 4  # worker 渲染并发数（infra/docker/compose.yml WORKER_CONCURRENCY 默认值）
+
+# 限流错误码：前端据此区分"排队等待"与"创建失败"
+ERROR_CODE_USER_QUEUE_FULL = "USER_QUEUE_FULL"  # 429：用户自己的任务排队中
+ERROR_CODE_SYSTEM_QUEUE_FULL = "SYSTEM_QUEUE_FULL"  # 503：系统整体繁忙
 
 
 class UserPendingLimitExceeded(Exception):
     """用户 pending 任务数超限，返回 429。"""
 
-    def __init__(self, user_id: str, pending_count: int, limit: int):
+    def __init__(
+        self,
+        user_id: str,
+        pending_count: int,
+        limit: int,
+        *,
+        running_count: int = 0,
+        requested_count: int = 1,
+        queue_ahead: int = 0,
+        estimated_wait_seconds: int = 0,
+    ):
         self.user_id = user_id
         self.pending_count = pending_count
         self.limit = limit
+        # 排队上下文（用于 429 结构化提示，前端展示"排队中"而非"创建失败"）
+        self.running_count = running_count
+        self.requested_count = requested_count
+        self.queue_ahead = queue_ahead
+        self.estimated_wait_seconds = estimated_wait_seconds
         super().__init__(f"用户 {user_id} pending 任务数 {pending_count} 超过上限 {limit}")
 
 
 class GlobalQueueFull(Exception):
     """全局限流，返回 503。"""
 
-    def __init__(self, pending_count: int, limit: int):
+    def __init__(
+        self,
+        pending_count: int,
+        limit: int,
+        *,
+        running_count: int = 0,
+        queue_ahead: int = 0,
+        estimated_wait_seconds: int = 0,
+    ):
         self.pending_count = pending_count
         self.limit = limit
+        self.running_count = running_count
+        self.queue_ahead = queue_ahead
+        self.estimated_wait_seconds = estimated_wait_seconds
         super().__init__(f"系统 pending 任务数 {pending_count} 超过上限 {limit}")
 
 
+def _estimate_wait_seconds(queue_ahead: int, generation_task_repository: Any) -> int:
+    """根据排队任务数 + worker 并发数 + 历史平均任务耗时估算等待秒数。
+
+    估算公式：ceil(排队任务数 / 并发数) × 平均单任务耗时。
+    拿不到历史数据时仓储层返回默认 120 秒。
+    """
+    import math
+
+    if queue_ahead <= 0:
+        return 0
+    try:
+        estimator = getattr(generation_task_repository, "estimate_avg_duration_seconds", None)
+        avg_seconds = estimator() if estimator is not None else 120.0
+    except Exception:
+        avg_seconds = 120.0
+    return int(math.ceil(queue_ahead / WORKER_CONCURRENCY) * avg_seconds)
+
+
+def build_rate_limit_detail(
+    exc: Exception,
+    generation_task_repository: Any,
+    *,
+    scope: str = "user",
+) -> dict:
+    """构造结构化限流响应体（HTTPException 的 detail）。
+
+    前端按 detail.code 判断场景：
+    - USER_QUEUE_FULL (429)：用户自己的任务在排队，应提示"等待/继续排队"，不是创建失败
+    - SYSTEM_QUEUE_FULL (503)：系统繁忙，稍后重试
+
+    detail 字段：
+    - code: 错误码
+    - message: 可读中文提示（可直接展示）
+    - queued_count: 当前排队(pending)任务数
+    - running_count: 当前渲染中(running)任务数
+    - queue_ahead: 前方排队任务数（预计等待批次依据）
+    - estimated_wait_seconds: 预计等待秒数
+    - limit: 对应限流上限
+    """
+    if scope == "user" and isinstance(exc, UserPendingLimitExceeded):
+        running = exc.running_count
+        if not running:
+            try:
+                counter = getattr(generation_task_repository, "count_running_by_user", None)
+                running = counter(exc.user_id) if counter is not None else 0
+            except Exception:
+                running = 0
+        queue_ahead = exc.queue_ahead or max(exc.pending_count, 0)
+        wait = exc.estimated_wait_seconds or _estimate_wait_seconds(queue_ahead, generation_task_repository)
+        wait_minutes = max(1, round(wait / 60))
+        message = (
+            f"您有 {exc.pending_count} 个任务正在排队、{running} 个正在渲染，"
+            f"同一时间最多提交 {exc.limit} 个任务。请等待约 {wait_minutes} 分钟后再提交"
+        )
+        return {
+            "code": ERROR_CODE_USER_QUEUE_FULL,
+            "message": message,
+            "queued_count": exc.pending_count,
+            "running_count": running,
+            "queue_ahead": queue_ahead,
+            "estimated_wait_seconds": wait,
+            "limit": exc.limit,
+        }
+
+    # 全局繁忙
+    pending = getattr(exc, "pending_count", 0)
+    running = getattr(exc, "running_count", 0)
+    if not running:
+        try:
+            counter = getattr(generation_task_repository, "count_running_total", None)
+            running = counter() if counter is not None else 0
+        except Exception:
+            running = 0
+    queue_ahead = getattr(exc, "queue_ahead", 0) or pending
+    wait = getattr(exc, "estimated_wait_seconds", 0) or _estimate_wait_seconds(queue_ahead, generation_task_repository)
+    wait_minutes = max(1, round(wait / 60))
+    return {
+        "code": ERROR_CODE_SYSTEM_QUEUE_FULL,
+        "message": f"系统繁忙：当前 {pending} 个任务排队中、{running} 个渲染中，预计等待约 {wait_minutes} 分钟，请稍后再试",
+        "queued_count": pending,
+        "running_count": running,
+        "queue_ahead": queue_ahead,
+        "estimated_wait_seconds": wait,
+        "limit": getattr(exc, "limit", GLOBAL_PENDING_LIMIT),
+    }
+
+
 def check_queue_limits(
     user_id: str,
     generation_task_repository: Any,