feat(generation): worker孤儿任务自动恢复 + 429限流结构化提示 (#1677) (#1710)

Co-authored-by: xiaoxia <dev@xiaoxiajianji.com>
Co-committed-by: xiaoxia <dev@xiaoxiajianji.com>

apps/api/app/api/routes/generation_preview.py

@@ -14,6 +14,7 @@ from app.core.task_enqueue import (
     USER_PENDING_LIMIT,
     GlobalQueueFull,
     UserPendingLimitExceeded,
+    build_rate_limit_detail,
     safe_enqueue_generation_task,
 )
 from app.dependencies import (
@@ -312,12 +313,12 @@ def create_preview_generation_task(
     except UserPendingLimitExceeded as e:
         raise HTTPException(
             status_code=429,
-            detail=f"您的待处理任务过多（当前 {e.pending_count - count}/{e.limit}，本次提交 {count} 个），请等待后再提交",
+            detail=build_rate_limit_detail(e, generation_task_repository, scope="user"),
         ) from e
     except GlobalQueueFull as e:
         raise HTTPException(
             status_code=503,
-            detail="系统繁忙，请稍后再试",
+            detail=build_rate_limit_detail(e, generation_task_repository, scope="global"),
         ) from e
 
     # 确定视频比例：优先前端传入，否则从模板 mode 推断
@@ -498,6 +499,7 @@ def create_preview_generation_task(
 
     # ── 入队 ──
     responses: list[PreviewGenerationTaskResponse] = []
+    rate_limit_exc: Exception | None = None  # 记录首个限流异常，全部失败时返回结构化提示
     for variant_index, task in enumerate(created_tasks):
         try:
             enqueued = safe_enqueue_generation_task(
@@ -510,23 +512,29 @@ def create_preview_generation_task(
             if not enqueued:
                 logger.warning("[预览生成] 任务入队失败: task_id=%s", task.id)
                 _mark_task_failed(generation_task_repository, task, "任务入队失败")
-        except UserPendingLimitExceeded:
+        except UserPendingLimitExceeded as e:
             _mark_task_failed(generation_task_repository, task, "待处理任务超限")
-        except GlobalQueueFull:
+            rate_limit_exc = rate_limit_exc or e
+        except GlobalQueueFull as e:
             _mark_task_failed(generation_task_repository, task, "系统队列已满")
+            rate_limit_exc = rate_limit_exc or e
         except Exception:
             logger.exception("[预览生成] 入队异常: task_id=%s", task.id)
             _mark_task_failed(generation_task_repository, task, "任务入队异常")
         # enqueue 会原地更新 task 状态/进度，直接用 task 构造响应
         responses.append(_to_preview_response(task))
 
-    # 队列满/限流时若全部失败，返回明确错误码
-    if all(r.status == "failed" for r in responses):
-        first_err = next((r.error_message for r in responses if r.error_message), "")
-        if "待处理任务" in first_err:
-            raise HTTPException(status_code=429, detail=first_err or "待处理任务超限")
-        if "队列" in first_err:
-            raise HTTPException(status_code=503, detail=first_err or "系统繁忙，请稍后再试")
+    # 队列满/限流时若全部失败，返回结构化错误码（前端区分"排队"与"创建失败"）
+    if all(r.status == "failed" for r in responses) and rate_limit_exc is not None:
+        if isinstance(rate_limit_exc, UserPendingLimitExceeded):
+            raise HTTPException(
+                status_code=429,
+                detail=build_rate_limit_detail(rate_limit_exc, generation_task_repository, scope="user"),
+            )
+        raise HTTPException(
+            status_code=503,
+            detail=build_rate_limit_detail(rate_limit_exc, generation_task_repository, scope="global"),
+        )
 
     logger.info(
         "[预览生成] 创建完成: %d 个变体任务, task_ids=%s",

apps/api/app/api/routes/generation_tasks.py

@@ -10,6 +10,7 @@ from app.core.task_enqueue import (
     USER_PENDING_LIMIT,
     GlobalQueueFull,
     UserPendingLimitExceeded,
+    build_rate_limit_detail,
     safe_enqueue_generation_task,
 )
 from app.dependencies import (
@@ -417,12 +418,12 @@ def create_generation_task(
     except UserPendingLimitExceeded as e:
         raise HTTPException(
             status_code=429,
-            detail=f"您的待处理任务过多（当前 {e.pending_count - count}/{e.limit}，本次提交 {count} 个），请等待完成后再提交",
+            detail=build_rate_limit_detail(e, generation_task_repository, scope="user"),
         ) from e
     except GlobalQueueFull as e:
         raise HTTPException(
             status_code=503,
-            detail="系统繁忙，请稍后再试",
+            detail=build_rate_limit_detail(e, generation_task_repository, scope="global"),
         ) from e
 
     # 画中画已下线：strategy_id 中的 pip/voice_pip 统一映射为 one_take
@@ -579,7 +580,7 @@ def create_generation_task(
                 if not created_tasks:
                     raise HTTPException(
                         status_code=429,
-                        detail="您的待处理任务过多，请等待完成后再提交",
+                        detail=build_rate_limit_detail(_e, generation_task_repository, scope="user"),
                     ) from _e
                 break
             except GlobalQueueFull as _e:
@@ -587,7 +588,7 @@ def create_generation_task(
                 if not created_tasks:
                     raise HTTPException(
                         status_code=503,
-                        detail="系统繁忙，请稍后再试",
+                        detail=build_rate_limit_detail(_e, generation_task_repository, scope="global"),
                     ) from _e
                 break
     except HTTPException:
@@ -713,15 +714,15 @@ def confirm_generation(
             log_task_status=True,
         ):
             logger.warning("[确认生成] 入队失败: task_id=%s", new_task.id)
-    except UserPendingLimitExceeded:
+    except UserPendingLimitExceeded as _e:
         raise HTTPException(
             status_code=429,
-            detail="您的待处理任务过多，请等待完成后再提交",
+            detail=build_rate_limit_detail(_e, generation_task_repository, scope="user"),
         ) from None
-    except GlobalQueueFull:
+    except GlobalQueueFull as _e:
         raise HTTPException(
             status_code=503,
-            detail="系统繁忙，请稍后再试",
+            detail=build_rate_limit_detail(_e, generation_task_repository, scope="global"),
         ) from None
 
     return BatchGenerationTaskResponse(
@@ -803,12 +804,24 @@ def retry_generation_task(
     if user_pending >= USER_PENDING_LIMIT:
         raise HTTPException(
             status_code=429,
-            detail=f"您的待处理任务过多（当前 {user_pending}/{USER_PENDING_LIMIT}），请等待完成后再提交",
+            detail=build_rate_limit_detail(
+                UserPendingLimitExceeded(
+                    user_id=user_id,
+                    pending_count=user_pending,
+                    limit=USER_PENDING_LIMIT,
+                ),
+                generation_task_repository,
+                scope="user",
+            ),
         )
     if global_pending >= GLOBAL_PENDING_LIMIT:
         raise HTTPException(
             status_code=503,
-            detail="系统繁忙，请稍后再试",
+            detail=build_rate_limit_detail(
+                GlobalQueueFull(pending_count=global_pending, limit=GLOBAL_PENDING_LIMIT),
+                generation_task_repository,
+                scope="global",
+            ),
         )
 
     use_case = CreateGenerationTaskUseCase(generation_task_repository)
@@ -843,15 +856,15 @@ def retry_generation_task(
             log_task_status=True,
         ):
             logger.warning("[生成任务] 重试入队失败: task_id=%s", retried.id)
-    except UserPendingLimitExceeded:
+    except UserPendingLimitExceeded as _e:
         raise HTTPException(
             status_code=429,
-            detail="您的待处理任务过多，请等待完成后再提交",
+            detail=build_rate_limit_detail(_e, generation_task_repository, scope="user"),
         ) from None
-    except GlobalQueueFull:
+    except GlobalQueueFull as _e:
         raise HTTPException(
             status_code=503,
-            detail="系统繁忙，请稍后再试",
+            detail=build_rate_limit_detail(_e, generation_task_repository, scope="global"),
         ) from None
     return _to_generation_task_response(retried)