Files
xiaoxia-saas/apps/api/app/api/routes/gpu_relay.py
T
saas-backend-agent dfa9e46edc
CI/CD Pipeline / Check push changed paths (pull_request) Has been skipped
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (pull_request) Successful in 1s
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 2s
CI/CD Pipeline / Frontend Lint (pull_request) Has been skipped
CI/CD Pipeline / Frontend Unit Tests (pull_request) Has been skipped
CI/CD Pipeline / PR Build Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging API Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging Web Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Has been skipped
CI/CD Pipeline / Staging E2E Tests (pull_request) Has been skipped
CI/CD Pipeline / Staging API Integration Tests (pull_request) Has been skipped
CI/CD Pipeline / ACR Image Cleanup (pull_request) Has been skipped
CI/CD Pipeline / PR Build API Image (pull_request) Successful in 51s
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 1m22s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 1m50s
CI/CD Pipeline / Validate - Python (mypy + alembic) (pull_request) Successful in 1m59s
PR Automation / Auto Approve on CI Green (pull_request) Successful in 3m0s
CI/CD Pipeline / Validate - Security (pull_request) Successful in 4m56s
AI Code Review / AI Code Review (pull_request) Successful in 8m14s
CI/CD Pipeline / PR Build Worker Image (pull_request) Failing after 9m8s
CI/CD Pipeline / Validate - Style (pull_request) Failing after 9m43s
PR Automation / Auto Merge on CI Green + Approved (pull_request) Successful in 11m19s
CI/CD Pipeline / Unit Tests (pull_request) Failing after 40m55s
CI/CD Pipeline / Build Production Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Deploy Production (pull_request) Has been skipped
CI/CD Pipeline / Canary Release to Production (pull_request) Has been skipped
CI/CD Pipeline / Production Browser E2E (pull_request) Has been skipped
CI/CD Pipeline / CI Gate (pull_request) Failing after 2s
feat(gpu): 接入 P4000 NVENC 硬件编码加速
- packages/config/base.py: 新增 GPU_ENCODE_* 配置项(开关/endpoint/relay/secret/编码参数/fallback)
- packages/shared/gpu_encoder.py: 新增 GpuEncoderClient,封装 health 探测 + mezzanine 上传 OSS + P4000 nvenc 编码 + relay 回传下载,失败抛 GpuEncodeError 触发 CPU 降级
- apps/api/app/api/routes/gpu_relay.py: 新增 internal PUT/GET/DELETE /api/v1/internal/gpu-relay/{key}(token 鉴权),P4000 PUT 编码结果,worker GET 下载
- apps/api/app/api/router.py: 注册 gpu_relay_router
- apps/worker/video_processing/unified_render_service.py: _execute_ffmpeg 和 _render_pass_through 尝试 GPU 路径:CPU ultrafast mezzanine → P4000 nvenc → 输出到最终路径;任何失败自动回退到原 CPU libx264 路径
- apps/worker/worker_app/tasks/_startup.py: worker_ready 时探测 P4000 健康并打日志
- tests/unit/test_gpu_encoder.py: GpuEncoderClient 单测(health/sync 调用/失败/fallback/relay URL)

架构:
- 输入:CPU 输出 libx264 ultrafast mezzanine → 上传 OSS 临时前缀 → P4000 签名 URL 下载
- 输出:P4000 PUT → 宿主机 nginx(tailscale:80)→ API /api/ 反代 → gpu_relay 路由落盘到 generated/gpu_relay/
- 回传:worker 通过 docker 网络 http://xiaoxia-api-staging:8000 GET 下载最终 mp4 到 output_path
- 降级:GPU 任何环节异常(health/上传/编码/回传/下载)→ 原 CPU 路径继续执行,不影响成片
2026-09-26 18:39:42 +08:00

174 lines
5.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""GPU 编码回传 relay 端点。
P4000 编码完成后通过 HTTP PUT 把结果 mp4 写到这里;Worker 在发起 GPU 请求时携带
带签名(token + 随机 key)的 URL,等待 P4000 写入后用同 URL 把文件 GET 回本地。
安全:
- 生产环境必须配置 GPU_ENCODE_RELAY_SECRET;token=xxx 查询参数必须匹配。
- key 为随机 hex,无法被枚举。
- 写入/读取后 worker 会调用 DELETE 主动清理;文件落地在 generated-files/gpu_relay/,
跟 generated-files 同卷,nginx 已对 generated-files 做静态挂载,但 gpu_relay/ 子目录
通过本接口走鉴权,不直接暴露为静态目录(文件名随机 + token 保护双重保险)。
"""
from __future__ import annotations
import logging
import os
import secrets
import time
import uuid
from pathlib import Path
from typing import Optional
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import FileResponse, Response
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/internal/gpu-relay", tags=["Internal-GpuRelay"])
_DEFAULT_SECRET_LOGGED = False
def _relay_dir() -> Path:
base = os.getenv("GENERATED_FILES_DIR", "/app/generated")
sub = os.getenv("GPU_ENCODE_RELAY_DIR", "gpu_relay")
p = Path(base) / sub
p.mkdir(parents=True, exist_ok=True)
return p
def _secret() -> str:
global _DEFAULT_SECRET_LOGGED
secret = (os.getenv("GPU_ENCODE_RELAY_SECRET", "") or "").strip()
if not secret:
env = (os.getenv("APP_ENV", os.getenv("ENV", "development"))).lower()
if env in ("production", "prod"):
# Production: raise so deployment fails fast
raise RuntimeError("GPU_ENCODE_RELAY_SECRET must be set in production")
# Dev: ephemeral random secret, log once
secret = os.environ.setdefault("GPU_ENCODE_RELAY_SECRET", secrets.token_urlsafe(32))
if not _DEFAULT_SECRET_LOGGED:
logger.warning(
"[gpu-relay] GPU_ENCODE_RELAY_SECRET not set; using ephemeral dev token (%s...)",
secret[:8],
)
_DEFAULT_SECRET_LOGGED = True
return secret
def _safe_key(key: str) -> str:
"""只允许合法文件名字符,防 path traversal。"""
k = key.strip()
if not k or "/" in k or "\\" in k or k in (".", "..") or not all(
c.isalnum() or c in "-_" for c in k
):
raise HTTPException(status_code=400, detail="invalid key")
return k
def _check_token(tok: Optional[str]) -> None:
if not tok or tok != _secret():
raise HTTPException(status_code=401, detail="unauthorized")
# ── Worker 侧:生成一个一次性 PUT URL ───────────────────────────────────
def build_relay_put_url(base_url: str, key: str, secret: str) -> str:
"""给 P4000 用的 PUT URL(含 token)。"""
return f"{base_url.rstrip('/')}/api/v1/internal/gpu-relay/{key}?token={secret}"
def build_relay_get_url(base_url: str, key: str, secret: str) -> str:
"""Worker 取回结果用的 GET URL。"""
return build_relay_put_url(base_url, key, secret)
def generate_key() -> str:
return uuid.uuid4().hex
# ── HTTP endpoints ──────────────────────────────────────────────────────
@router.put("/{key}")
async def put_object(
key: str,
request: Request,
token: Optional[str] = Query(None),
):
_check_token(token)
safe = _safe_key(key)
dst = _relay_dir() / safe
tmp = dst.with_suffix(dst.suffix + ".part")
size = 0
t0 = time.time()
try:
with open(tmp, "wb") as f:
async for chunk in request.stream():
f.write(chunk)
size += len(chunk)
os.replace(tmp, dst)
except Exception as e: # noqa: BLE001
if tmp.exists():
try:
tmp.unlink()
except OSError:
pass
logger.exception("[gpu-relay] PUT failed key=%s", safe)
raise HTTPException(status_code=500, detail=f"write failed: {e}") from e
logger.info(
"[gpu-relay] PUT key=%s size=%d took=%.2fs",
safe, size, time.time() - t0,
)
return {"ok": True, "key": safe, "size": size}
@router.get("/{key}")
async def get_object(
key: str,
token: Optional[str] = Query(None),
):
_check_token(token)
safe = _safe_key(key)
path = _relay_dir() / safe
if not path.exists():
raise HTTPException(status_code=404, detail="not found")
return FileResponse(
path=path,
media_type="video/mp4",
filename=f"{safe}.mp4",
)
@router.head("/{key}")
async def head_object(
key: str,
token: Optional[str] = Query(None),
):
_check_token(token)
safe = _safe_key(key)
path = _relay_dir() / safe
if not path.exists():
return Response(status_code=404)
return Response(
status_code=200,
media_type="video/mp4",
headers={"Content-Length": str(path.stat().st_size)},
)
@router.delete("/{key}")
async def delete_object(
key: str,
token: Optional[str] = Query(None),
):
_check_token(token)
safe = _safe_key(key)
path = _relay_dir() / safe
try:
if path.exists():
path.unlink()
except OSError as e:
raise HTTPException(status_code=500, detail=f"delete failed: {e}") from e
return {"ok": True, "key": safe}