Compare commits

..

1 Commits

Author SHA1 Message Date
CI Bot 077f7710df test: cleanup workflow验证用临时PR
Preview Cleanup / Cleanup Preview Environment (pull_request) Failing after 1s
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 23s
CI/CD Pipeline / Frontend Lint (pull_request) Successful in 52s
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 54s
AI Code Review / AI Code Review (pull_request) Successful in 2m34s
CI/CD Pipeline / Validate Code Quality And Tests (pull_request) Successful in 2m48s
CI/CD Pipeline / Unit Tests (pull_request) Successful in 3m10s
Auto Approve CI PRs / Auto Approve on CI Green (pull_request) Failing after 3m47s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 1m43s
Auto Merge CI PRs / Auto Merge on CI Green + Approved (pull_request) Successful in 34m5s
CI Build & Deploy Pipeline / Production Browser E2E (pull_request) Failing after 1441h10m5s
CI Build & Deploy Pipeline / Build Production Worker Image (pull_request) Failing after 1441h10m7s
CI Build & Deploy Pipeline / Build Production Web Image (pull_request) Failing after 1441h10m8s
CI Build & Deploy Pipeline / Staging E2E Tests (pull_request) Failing after 1441h10m8s
CI Build & Deploy Pipeline / Build Production API Image (pull_request) Failing after 1441h10m9s
CI Build & Deploy Pipeline / Build Staging Worker Image (pull_request) Failing after 1441h10m11s
CI Build & Deploy Pipeline / Build Staging Web Image (pull_request) Failing after 1441h10m11s
CI Build & Deploy Pipeline / Build Staging API Image (pull_request) Failing after 1441h10m11s
CI Build & Deploy Pipeline / Deploy Production (pull_request) Failing after 1441h42m7s
CI Build & Deploy Pipeline / Staging API Integration Tests (pull_request) Failing after 1441h42m9s
CI Build & Deploy Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Failing after 1441h42m10s
2026-07-18 14:19:15 +08:00
1919 changed files with 74020 additions and 360800 deletions
-1
View File
@@ -1 +0,0 @@
CI re-trigger after runner add-host/DNS fix. This file is harmless and not referenced.
+1 -2
View File
@@ -1,2 +1 @@
CI trigger file - safe to delete
retrigger at 2026-09-15 20:31:24 UTC
trigger: 1784009947
+27 -235
View File
@@ -1,268 +1,60 @@
# ============================================================
# 小虾 SaaS 环境变量完整配置
# ============================================================
# 本文件列出所有可配置的环境变量及默认值。
# 复制为 .env 后按需修改;生产环境务必覆盖所有密钥类配置。
#
# 配置读取规则(pydantic-settings,大小写不敏感):
# 1. 系统环境变量(最高优先级)
# 2. .env.{APP_ENV} 文件(如 .env.staging)
# 3. .env 文件
# 4. 代码中的默认值(最低优先级)
# ============================================================
# 小虾 SaaS 环境变量配置
# ==================== 应用基本配置 ====================
# 应用名称
APP_NAME=xiaoxia-saas
# 应用版本号(展示用,代码中已内置默认)
APP_VERSION=0.1.61
# 环境标识:development / staging / production
# 决定读取 .env.{APP_ENV} 还是 .env,也影响部分配置的严格校验
# ==================== 应用配置 ====================
APP_NAME=小虾 SaaS
APP_BASE_URL=http://localhost:3000
APP_ENV=development
# 是否开启 Debug 模式(开发环境 true,生产环境 false)
DEBUG=true
# 应用基础 URL,用于生成认证邮件、回调链接等
APP_BASE_URL=http://localhost:3000
# API 服务监听地址(容器内绑定,外部暴露由 Docker/Nginx 控制)
API_HOST=0.0.0.0
# API 服务监听端口
API_PORT=8000
# 是否自动创建数据库表结构(开发环境可开启,生产环境用 alembic migration)
AUTO_CREATE_SCHEMA=false
# ==================== 数据库配置 ====================
DATABASE_URL=postgresql://xiaoxia_user:your_password@localhost:5432/xiaoxia_saas
# 数据库连接串(格式:postgresql+psycopg://user:password@host:port/dbname)
DATABASE_URL=postgresql+psycopg://postgres:postgres@localhost:5432/xiaoxia_saas
# 连接池大小(常驻连接数)
DATABASE_POOL_SIZE=20
# 连接池最大溢出连接数(pool_size + max_overflow = 最大并发连接数)
DATABASE_MAX_OVERFLOW=10
# 获取连接超时时间(秒)
DATABASE_POOL_TIMEOUT=30
# 连接回收时间(秒),防止数据库端主动断开导致的死连接
DATABASE_POOL_RECYCLE=3600
# 是否使用内存数据库(SQLite,仅开发/测试可用;生产务必 false)
USE_IN_MEMORY_DB=false
# 开发环境:使用内存数据库(不需要 PostgreSQL)
USE_IN_MEMORY_DB=true
# 生产环境:使用 PostgreSQL
# USE_IN_MEMORY_DB=false
# ==================== Redis 配置 ====================
# Redis 连接 URL(格式:redis://[:password@]host:port/db)
REDIS_URL=redis://localhost:6379/0
# 是否使用 Redis 存储 Session(多实例部署时必须开启;开发可用内存存储)
ENABLE_REDIS_SESSIONS=false
# ==================== Celery 任务队列 ====================
# Celery Broker(任务分发),默认用 Redis db0
CELERY_BROKER_URL=redis://localhost:6379/0
# Celery Result Backend(任务结果存储),默认用 Redis db1
CELERY_RESULT_BACKEND=redis://localhost:6379/1
# ==================== Worker 配置(#2073 队列分流) ====================
#
# 容器内跑三个独立进程:beat(只发定时任务)+ generation worker(实时高优)
# + transcode worker(后台批量/清理)。三个进程的并发与开关独立配置。
# Worker 进程名称
WORKER_NAME=xiaoxia-saas-worker
# 总并发参考(兼容旧变量):
# - 若 GENERATION_CONCURRENCY 与 TRANSCODE_CONCURRENCY 都未显式设置,
# entrypoint 会按此总数对半分配(gen=ceil(total/2), trans=剩余,各至少 1);
# - 任一个 *_CONCURRENCY 显式设置后,按显式值生效,忽略此变量对应部分。
WORKER_CONCURRENCY=4
# Generation worker 并发数(用户实时任务:视频生成/TTS/音色克隆/lipsync/数字人)
# 实时链路对延迟敏感,建议 2C 以上机器设为 2;高负载场景可加到 4。
GENERATION_CONCURRENCY=2
# Transcode worker 并发数(后台批量:素材入库转码/AI 分类打标/质量评分/查重/批量下载)
# 后台任务可排队,独立伸缩;素材入库量大时可加到 4。
TRANSCODE_CONCURRENCY=2
# 是否在本容器启动 celery beat 进程(默认 1)。
# 默认 beat 与 worker 同容器部署;若要独立 beat 容器部署,worker 容器设为 0、
# beat 容器单独跑 `celery -A worker_app.celery_app beat` 并设 BEAT_ENABLED=1。
BEAT_ENABLED=1
# 每个子进程最多处理多少任务后重启(防止内存泄漏)
WORKER_MAX_TASKS_PER_CHILD=1000
# ==================== JWT 认证配置 ====================
# JWT 签名密钥 — 生产环境必须设置为强随机字符串(至少32字符)
# 内置不安全值会被拒绝:secret / changeme / password / your-secret-key 等
# ==================== JWT 配置 ====================
JWT_SECRET_KEY=your-super-secret-key-change-this-in-production-min-32-chars
# JWT 签名算法
JWT_ALGORITHM=HS256
# Access Token 过期时间(分钟)
JWT_ACCESS_TOKEN_EXPIRE_MINUTES=30
# Refresh Token 过期时间(天)
JWT_REFRESH_TOKEN_EXPIRE_DAYS=30
# ==================== 邮件配置 ====================
# 是否启用邮件投递(关闭时邮件内容打印到日志,开发调试用)
ENABLE_EMAIL_DELIVERY=false
# SMTP 服务器地址
SMTP_HOST=smtp.gmail.com
# SMTP 端口
SMTP_PORT=587
# SMTP 用户名
SMTP_USER=your-email@gmail.com
# SMTP 密码 / 应用专用密码
SMTP_PASSWORD=your-app-specific-password
# 发件人邮箱
SMTP_FROM_EMAIL=noreply@xiaoxia-saas.com
# 发件人显示名称
SMTP_FROM_NAME=小虾 SaaS
# 是否启用 TLS
SMTP_USE_TLS=true
# ==================== 阿里云 OSS 配置 ====================
# OSS 区域 endpoint
OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
# OSS Access Key ID — 非开发环境必须设置
OSS_ACCESS_KEY_ID=your-access-key-id
# OSS Access Key Secret — 非开发环境必须设置
OSS_ACCESS_KEY_SECRET=your-access-key-secret
# OSS Bucket 名称
OSS_BUCKET_NAME=xiaoxia-autocut
# 直传最大文件大小(MB)
OSS_DIRECT_UPLOAD_MAX_MB=2000
# 直传签名有效期(秒)
OSS_DIRECT_UPLOAD_EXPIRE_SECONDS=900
# ==================== 环境配置 ====================
ENVIRONMENT=development
DEBUG=true
# ==================== CORS 配置 ====================
# 逗号分隔的域名列表(Settings 读取 CORS_ORIGINS_RAW)
CORS_ORIGINS_RAW=http://localhost:3000,http://localhost:5173
# 允许跨域的前端域名列表,逗号分隔
CORS_ORIGINS_RAW=http://localhost:3000,http://localhost:5173,http://localhost:8000
# ==================== 渲染引擎配置 ====================
# 渲染引擎选择:
# legacy — 旧 VideoComposeService(稳定,功能完整)
# unified — 新 UnifiedRenderService(新架构,部分场景仍在验证)
RENDER_ENGINE=legacy
# ==================== CosyVoice 语音合成 ====================
# 阿里云百灵语音合成服务
# 模型选择:
# cosyvoice-v3-flash — 推荐,系统音色多,性价比高
# cosyvoice-v3-plus — 高质量,系统音色少
# cosyvoice-v3.5-flash / cosyvoice-v3.5-plus — 仅支持克隆/设计音色,无系统音色
# 音色:v3 系列系统音色带 _v3 后缀,如 longxiaochun_v3 / longxiaoxia_v3 / longanyang
# ==================== 阿里云 OSS 配置 ====================
OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
OSS_ACCESS_KEY_ID=your-access-key-id
OSS_ACCESS_KEY_SECRET=your-access-key-secret
OSS_BUCKET_NAME=xiaoxia-autocut
# ==================== CosyVoice 语音合成配置 ====================
# 注意:base_url 只需写到 /api/v1,具体路径由代码拼接
# 模型: cosyvoice-v3-flash (推荐,支持系统音色,性价比高)
# cosyvoice-v3-plus (高质量,系统音色少)
# cosyvoice-v3.5-flash / cosyvoice-v3.5-plus (仅支持克隆/设计音色,无系统音色)
# 音色: v3系列系统音色带 _v3 后缀,如 longxiaochun_v3, longxiaoxia_v3, longanyang (无后缀)
# 注意:COSYVOICE_* 变量由 packages/shared/config.py 的 SharedSettings 读取
COSYVOICE_API_KEY=your-cosyvoice-api-key
COSYVOICE_BASE_URL=https://dashscope.aliyuncs.com/api/v1
COSYVOICE_MODEL=cosyvoice-v3-flash
COSYVOICE_VOICE=longxiaochun_v3
COSYVOICE_SAMPLE_RATE=22050
COSYVOICE_FORMAT=mp3
# 音色克隆模型名(固定为 voice-enrollment,通常不需修改)
COSYVOICE_CLONE_MODEL=voice-enrollment
# ==================== 豆包大模型(火山引擎方舟) ====================
# 用于 AI 文案生成、智能剪辑等需要大模型能力的场景
DOUBAO_API_KEY=your-doubao-api-key
DOUBAO_MODEL=doubao-seed-2-1-pro-260915
DOUBAO_FAST_MODEL=doubao-seed-2-1-lite-260915
DOUBAO_BASE_URL=https://ark.cn-beijing.volces.com/api/v3
DOUBAO_TIMEOUT=30
DOUBAO_MAX_RETRIES=2
# 视觉模型:pro 精度高,lite 速度快(viral-video 商品识别默认用 lite 提速)
DOUBAO_VISION_MODEL=doubao-seed-2-1-pro-260915
DOUBAO_VISION_LITE_MODEL=doubao-seed-2-1-lite-260915
DOUBAO_VISION_USE_LITE=true
# Embedding 向量化模型
DOUBAO_EMBEDDING_MODEL=doubao-embedding-vision-251215
# 视频模型(Seedance 2.5,统一走方舟;真人参考图通过信任链自动 AI 化)
DOUBAO_VIDEO_MODEL=doubao-seedance-2-5-260628
DOUBAO_VIDEO_TIMEOUT=480
DOUBAO_VIDEO_POLL_INTERVAL=10
# 图片模型(Seedream 5.0 Pro,用于信任链真人 AI 化 + 文生图)
DOUBAO_IMAGE_MODEL=doubao-seedream-5-0-pro-260628
DOUBAO_IMAGE_TIMEOUT=120
# ==================== 积分/会员系统 (#1895) ====================
# 积分系统总开关:默认 false(暂停积分系统)。
# - false:生成视频/口型同步/数字人/AI标题/TTS/克隆音色等所有功能对登录
# 用户免费放行,不扣积分、不做余额拦截;积分余额/流水/会员状态查询接口
# 保留可用,但数据不再变动。积分相关的表、代码、接口均保留不删除。
# - 恢复积分:设置 ENABLE_CREDIT_SYSTEM=true 即可,无需改代码。
ENABLE_CREDIT_SYSTEM=false
# 旧开关名(兼容别名):与 ENABLE_CREDIT_SYSTEM 任一为 true 即启用。
POINTS_ENABLED=false
# ==================== 抖音解析多源轮询 (#1963) ====================
# 无需配置 Key 也可使用(P0 免费源可用),配置 Key 可增加兜底能力
# TikHub API Key (https://tikhub.io) — $0.001/次起,注册送$0.05
TIKHUB_API_KEY=
# apizero.cn API Key (https://v1.apizero.cn) — 国内抖音解析服务
APIZERO_API_KEY=
# ==================== GPU MuseTalk Worker(反向轮询口型同步)====================
# GPU Worker 长期鉴权 Token,Worker 端 .env 的 GPU_WORKER_TOKEN 必须与此一致
# 留空时 development 环境允许匿名访问(仅本地调试),staging/production 必须配置
GPU_WORKER_TOKEN=
# 单任务超时(秒),processing 超过此时长无任务心跳才回退 pending 或标记 failed
# #1970:RTX2060 6G 推理 720p 长视频需 5 分钟以上,默认 900
GPU_TASK_TIMEOUT_SECONDS=900
# 是否启用 GPU 口型同步(开关)。开启后需同时有 Worker 在心跳窗口内(5分钟)才会走 GPU 路径;
# 开关关闭 / 无可用 Worker / GPU 任务失败或超时 → 自动回退现有 MediaKit 云端 lipsync
USE_GPU_LIPSYNC=false
# 业务侧轮询 GPU 任务结果的间隔(秒)
GPU_LIPSYNC_POLL_INTERVAL=5
# 业务侧等待 GPU 任务总超时(秒);超时回退 MediaKit
GPU_LIPSYNC_WAIT_TIMEOUT=1200
# Worker 心跳新鲜度窗口(秒),last_heartbeat_at 在此窗口内视为在线
GPU_WORKER_STALE_SECONDS=300
-161
View File
@@ -1,161 +0,0 @@
name: ACR Cleanup
on:
schedule:
- cron: '0 19 * * *' # UTC 19:00 = 北京时间凌晨3:00
workflow_dispatch:
inputs:
pr_sha:
description: "PR commit SHA(仅清理指定PR镜像,留空则全量清理)"
required: false
default: ""
dry_run:
description: "预览模式(dry-run),不实际删除"
required: false
default: "true"
pull_request_target:
types: [closed]
branches: [develop, main]
concurrency:
group: acr-cleanup-${{ gitea.ref }}
cancel-in-progress: false
jobs:
cleanup:
name: ACR Image Cleanup
runs-on: ci-l2
timeout-minutes: 20
permissions:
contents: read
env:
ACR_REGISTRY: xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com
ACR_NAMESPACE: xiaoxiakeji
ACR_SERVICE: registry.aliyuncs.com:cn-hangzhou:china:cri-fvec8o9q4mmxrkaa
GITEA_URL: https://git.xiaoxiajianji.com
GITEA_REPO: xiaoxia/xiaoxia-saas
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
# ====== Cron模式:获取staging运行中镜像作为白名单 ======
- name: Get staging running images (whitelist)
id: protected_images
if: gitea.event_name != 'pull_request_target' && !gitea.event.inputs.pr_sha
env:
STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
run: |
set +e
echo "获取staging服务器运行中镜像作为白名单..."
mkdir -p ~/.ssh
echo "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
staging_host="${STAGING_SSH_HOST:-47.98.113.167}"
staging_port="${STAGING_SSH_PORT:-22222}"
ssh-keyscan -p "$staging_port" -H "$staging_host" >> ~/.ssh/known_hosts 2>/dev/null
# 获取所有运行容器的镜像,提取tag部分
IMAGES=$(ssh -p "$staging_port" -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no \
"root@$staging_host" "docker ps --format '{{.Image}}' 2>/dev/null" 2>/dev/null | grep -v "^$" | sort -u)
PROTECTED_TAGS=""
if [ -n "$IMAGES" ]; then
while IFS= read -r img; do
# 从完整镜像名中提取tag(最后一个冒号后)
tag=$(echo "$img" | rev | cut -d: -f1 | rev)
if [ -n "$tag" ] && [ "$tag" != "latest" ] && [ ${#tag} -gt 5 ]; then
if [ -z "$PROTECTED_TAGS" ]; then
PROTECTED_TAGS="$tag"
else
PROTECTED_TAGS="$PROTECTED_TAGS,$tag"
fi
fi
done <<< "$IMAGES"
fi
echo "staging运行中镜像tag: ${PROTECTED_TAGS:-(无)}"
echo "protected_tags=$PROTECTED_TAGS" >> $GITEA_OUTPUT
# ====== Docker登录 ======
- name: Docker login to ACR
env:
ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
run: |
printf '%s' "$ACR_PASSWORD" | docker login "$ACR_REGISTRY" -u "$ACR_USERNAME" --password-stdin
# ====== 模式1:PR关闭时清理 ======
- name: Cleanup PR images (PR closed)
if: gitea.event_name == 'pull_request_target'
env:
ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
PR_SHA: ${{ gitea.event.pull_request.head.sha }}
PR_NUMBER: ${{ gitea.event.pull_request.number }}
run: |
echo "============================================"
echo " PR #$PR_NUMBER 已关闭,清理对应镜像"
echo " Head SHA: ${PR_SHA::12}"
echo "============================================"
echo ""
python3 scripts/ci/acr_cleanup.py \
--pr-sha "$PR_SHA" \
--execute
# ====== 模式2:Cron全量清理 ======
- name: Full cleanup (cron / manual)
if: gitea.event_name != 'pull_request_target' && !gitea.event.inputs.pr_sha
env:
ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
PROTECTED_TAGS: ${{ steps.protected_images.outputs.protected_tags }}
DRY_RUN_INPUT: ${{ gitea.event.inputs.dry_run }}
run: |
echo "============================================"
echo " ACR 全量清理(${{ gitea.event_name }})"
echo "============================================"
echo ""
# 决定是否dry-run
DRY_RUN_FLAG=""
if [ "$DRY_RUN_INPUT" = "true" ]; then
DRY_RUN_FLAG="--dry-run"
echo "模式: 预览模式 (dry-run)"
else
echo "模式: 执行模式"
fi
echo ""
python3 scripts/ci/acr_cleanup.py \
--keep 20 \
--protected-tags "$PROTECTED_TAGS" \
$DRY_RUN_FLAG
# ====== 模式3:手动指定PR SHA清理 ======
- name: Cleanup specific PR image (manual)
if: gitea.event_name == 'workflow_dispatch' && gitea.event.inputs.pr_sha
env:
ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
PR_SHA: ${{ gitea.event.inputs.pr_sha }}
DRY_RUN_INPUT: ${{ gitea.event.inputs.dry_run }}
run: |
echo "手动清理PR镜像: ${PR_SHA::12}"
echo ""
DRY_RUN_FLAG=""
if [ "$DRY_RUN_INPUT" = "true" ]; then
DRY_RUN_FLAG="--dry-run"
fi
python3 scripts/ci/acr_cleanup.py \
--pr-sha "$PR_SHA" \
$DRY_RUN_FLAG
-84
View File
@@ -1,84 +0,0 @@
name: API Base Image Build
on:
push:
branches:
- develop
- main
paths:
- 'requirements-base.txt'
- 'requirements.txt'
- 'infra/docker/api-base.Dockerfile'
workflow_dispatch:
jobs:
build-api-base:
name: Build API Base Image
runs-on: runtime-builder
timeout-minutes: 45
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" \
| bash
- name: Docker login to Registry
shell: sh
env:
ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
GITEA_REGISTRY_USER: xiaoxia
GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
run: |
set -eu
for i in 1 2 3; do
echo "=== Docker login 尝试 $i/3 ==="
if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin \
&& docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then
echo "✅ Docker login successful"
break
fi
echo "❌ Docker login 失败(尝试 $i/3),5s 后重试..."
sleep 5
done
- name: Build and push API base image
shell: sh
run: |
set -eu
ACR_IMAGE="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/saas-api-base:latest"
GITEA_IMAGE="git.xiaoxiajianji.com/xiaoxia-saas/saas-api-base:latest"
echo "=== Building API base image ==="
# 使用普通 docker build(单平台不需要 buildx)
docker build \
-f infra/docker/api-base.Dockerfile \
-t "${ACR_IMAGE}" \
.
echo ""
echo "✅ Image built successfully"
# 推送到 ACR
echo "=== Pushing to ACR ==="
docker push "${ACR_IMAGE}"
echo "✅ Pushed to ACR"
# 打标签并推送到 Gitea Packages 作为备份
echo "=== Pushing to Gitea Packages ==="
docker tag "${ACR_IMAGE}" "${GITEA_IMAGE}"
docker push "${GITEA_IMAGE}" || echo "⚠️ Gitea Packages push failed (non-fatal)"
echo "✅ Gitea backup push completed"
- name: Cleanup
if: always()
shell: sh
run: |
ACR_IMAGE="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/saas-api-base:latest"
docker rmi "${ACR_IMAGE}" 2>/dev/null || true
echo "Cleanup done"
+174
View File
@@ -0,0 +1,174 @@
name: Auto Approve CI PRs
on:
pull_request:
types: [synchronize, opened, ready_for_review]
jobs:
auto-approve:
name: Auto Approve on CI Green
runs-on: ci-check
if: github.event_name == 'pull_request' && !github.event.pull_request.draft
timeout-minutes: 20
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: "set -eu\npython3 - <<'PY'\nimport io, os, tarfile, time, urllib.request, urllib.error\nurl = f\"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz\"\nrequest = urllib.request.Request(url, headers={\"Authorization\": f\"token {os.environ['GITHUB_TOKEN']}\"})\nlast_err = None\nfor attempt in range(5):\n try:\n with urllib.request.urlopen(request, timeout=120) as response:\n archive = response.read()\n break\n except urllib.error.HTTPError as e:\n last_err = e\n if e.code >= 500 and attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...\")\n time.sleep(wait)\n continue\n raise\n except Exception as e:\n last_err = e\n if attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...\")\n time.sleep(wait)\n continue\n raise\nelse:\n raise last_err\nwith tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:\n root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'\n for member in tar.getmembers():\n name = member.name\n if name == root_prefix[:-1]:\n continue\n if name.startswith(root_prefix):\n member.name = name[len(root_prefix):]\n if member.name:\n tar.extract(member, '.')\nPY\n"
- name: Auto approve when CI passes
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
REVIEW_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
set -eu
echo "PR #${PR_NUMBER} - 检查CI状态并自动审批"
# 检查是否纯前端改动
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300"
FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin)]")
FRONTEND_COUNT=$(echo "$FILES" | grep -c '^apps/web/' || true)
BACKEND_COUNT=$(echo "$FILES" | grep -cv '^apps/web/' || true)
TOTAL=$(echo "$FILES" | grep -cv '^$' || true)
echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT})"
if [ "$BACKEND_COUNT" = "0" ] && [ "$FRONTEND_COUNT" -gt "0" ]; then
SKIP_BACKEND=true
echo "✅ 纯前端改动,只检查Frontend Lint"
else
SKIP_BACKEND=false
echo "🔧 包含后端/公共变更,检查全部CI"
fi
# 定义需要检查的context
if [ "$SKIP_BACKEND" = "true" ]; then
CONTEXTS=("CI/CD Pipeline / Frontend Lint (pull_request)")
else
CONTEXTS=(
"CI/CD Pipeline / Validate Code Quality And Tests (pull_request)"
"CI/CD Pipeline / Unit Tests (pull_request)"
"CI/CD Pipeline / Frontend Lint (pull_request)"
)
fi
echo "需要通过的CI检查: ${#CONTEXTS[@]} 项"
for ctx in "${CONTEXTS[@]}"; do
echo " - $ctx"
done
echo
# 初始等待30秒,给CI启动写status的时间,避免checkout太快导致全找不到context误判
echo "等待30秒让CI启动..."
sleep 30
# 轮询等待,最多20分钟(120次x10秒)
for attempt in $(seq 1 120); do
ALL_SUCCESS=true
ANY_FAILED=false
ANY_PENDING=false
echo "--- 第${attempt}次检查 ($(date '+%H:%M:%S')) ---"
# 调用辅助脚本检查每个context状态
for ctx in "${CONTEXTS[@]}"; do
STATE=$(python3 scripts/check_ci_status.py "$GITHUB_TOKEN" "$GITHUB_REPOSITORY" "$PR_HEAD_SHA" "$ctx")
echo " $ctx: $STATE"
if [ "$STATE" != "success" ]; then
ALL_SUCCESS=false
fi
if [ "$STATE" = "failure" ] || [ "$STATE" = "error" ]; then
ANY_FAILED=true
fi
if [ "$STATE" = "pending" ] || [ "$STATE" = "null" ]; then
ANY_PENDING=true
fi
done
if [ "$ALL_SUCCESS" = "true" ]; then
echo
echo "✅ 所有CI检查通过,自动审批 PR #${PR_NUMBER}"
# 检查是否已有审批(任何用户的APPROVED都算,避免重复审批)
EXISTING=$(curl -s -H "Authorization: token ${REVIEW_TOKEN}" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" \
| python3 -c "import sys,json; reviews=json.load(sys.stdin); print('yes' if any(r.get('state')=='APPROVED' for r in reviews) else 'no')")
if [ "$EXISTING" = "yes" ]; then
echo "ℹ️ PR #${PR_NUMBER} 已有审批,跳过"
exit 0
fi
# 第一步:创建PENDING review(Gitea API需要先创建再提交)
echo "创建review..."
REVIEW_CREATE=$(curl -s -X POST \
-H "Authorization: token ${REVIEW_TOKEN}" \
-H "Content-Type: application/json" \
-d '{"event": "PENDING", "body": "CI全绿,自动审批通过。"}' \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews")
REVIEW_ID=$(echo "$REVIEW_CREATE" | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))")
REVIEW_STATE=$(echo "$REVIEW_CREATE" | python3 -c "import sys,json; print(json.load(sys.stdin).get('state',''))")
echo "创建结果: id=$REVIEW_ID state=$REVIEW_STATE"
if [ -z "$REVIEW_ID" ]; then
echo "❌ 创建review失败"
echo "$REVIEW_CREATE"
exit 1
fi
# 如果已经是APPROVED就不用再submit了(兼容不同Gitea版本)
if [ "$REVIEW_STATE" = "APPROVED" ]; then
echo "✅ 自动审批成功(直接创建为APPROVED)"
exit 0
fi
# 第二步:submit review为APPROVED
echo "提交review审批..."
SUBMIT_CODE=$(curl -s -o /tmp/submit_resp.json -w "%{http_code}" \
-X POST \
-H "Authorization: token ${REVIEW_TOKEN}" \
-H "Content-Type: application/json" \
-d '{"event": "APPROVED", "body": "CI全绿,自动审批通过。"}' \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews/${REVIEW_ID}")
echo "提交API HTTP状态: $SUBMIT_CODE"
cat /tmp/submit_resp.json 2>/dev/null || true
echo
if [ "$SUBMIT_CODE" = "200" ] || [ "$SUBMIT_CODE" = "201" ]; then
FINAL_STATE=$(python3 -c "import json; print(json.load(open('/tmp/submit_resp.json')).get('state',''))" 2>/dev/null || echo "?")
echo "✅ 自动审批成功 (state: $FINAL_STATE)"
exit 0
else
echo "❌ 提交审批失败"
exit 1
fi
fi
# 还有CI在跑(pending状态)→ 继续等
if [ "$ANY_PENDING" = "true" ]; then
echo "⏳ CI仍在运行中,继续等待(第${attempt}/120次轮询)..."
sleep 10
continue
fi
# 所有CI都跑完了但有失败 → 退出
if [ "$ANY_FAILED" = "true" ]; then
echo
echo "❌ CI检查有失败项,不自动审批"
exit 0
fi
# 其他情况继续等
sleep 10
done
echo
echo "⏰ 等待超时(20分钟),CI尚未全部完成"
exit 0
+170
View File
@@ -0,0 +1,170 @@
name: Auto Merge CI PRs
on:
pull_request:
types: [synchronize, opened, ready_for_review, review_requested]
jobs:
auto-merge:
name: Auto Merge on CI Green + Approved
runs-on: ci-check
if: github.event_name == 'pull_request' && !github.event.pull_request.draft && github.event.pull_request.base.ref == 'develop'
timeout-minutes: 30
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: "set -eu\npython3 - <<'PY'\nimport io, os, tarfile, time, urllib.request, urllib.error\nurl = f\"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz\"\nrequest = urllib.request.Request(url, headers={\"Authorization\": f\"token {os.environ['GITHUB_TOKEN']}\"})\nlast_err = None\nfor attempt in range(5):\n try:\n with urllib.request.urlopen(request, timeout=120) as response:\n archive = response.read()\n break\n except urllib.error.HTTPError as e:\n last_err = e\n if e.code >= 500 and attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...\")\n time.sleep(wait)\n continue\n raise\n except Exception as e:\n last_err = e\n if attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...\")\n time.sleep(wait)\n continue\n raise\nelse:\n raise last_err\nwith tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:\n root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'\n for member in tar.getmembers():\n name = member.name\n if name == root_prefix[:-1]:\n continue\n if name.startswith(root_prefix):\n member.name = name[len(root_prefix):]\n if member.name:\n tar.extract(member, '.')\nPY\n"
- name: Auto merge when CI passes and approved
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
MERGE_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
BASE_REF: ${{ github.event.pull_request.base.ref }}
run: |
set -eu
echo "PR #${PR_NUMBER} - 检查CI状态+审批并自动合并到${BASE_REF}"
echo
# 只合develop分支
if [ "$BASE_REF" != "develop" ]; then
echo "Skip: 目标分支不是develop"
exit 0
fi
# 判断是否纯前端改动
FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300" \
| python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin)]")
TOTAL=$(echo "$FILES" | grep -cv '^$' || true)
FRONTEND_COUNT=$(echo "$FILES" | grep -c '^apps/web/' || true)
BACKEND_COUNT=$((TOTAL - FRONTEND_COUNT))
echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT})"
if [ "$BACKEND_COUNT" = "0" ] && [ "$FRONTEND_COUNT" -gt "0" ]; then
CONTEXTS=("CI/CD Pipeline / Frontend Lint (pull_request)")
echo "纯前端改动,只检查Frontend Lint"
else
CONTEXTS=(
"CI/CD Pipeline / Validate Code Quality And Tests (pull_request)"
"CI/CD Pipeline / Unit Tests (pull_request)"
"CI/CD Pipeline / Frontend Lint (pull_request)"
"CI/CD Pipeline / Integration Tests (pull_request)"
)
echo "检查全部四门禁"
fi
echo
# 初始等待30秒,给CI启动写status的时间,避免checkout太快导致全找不到context误判
echo "等待30秒让CI启动..."
sleep 30
# 405连续计数器:连续多次合并返回405才放弃
MERGE_405_COUNT=0
MAX_405_RETRIES=6
# 轮询等待,最多30分钟(180次x10秒)
for attempt in $(seq 1 180); do
ALL_SUCCESS=true
ANY_FAILED=false
ANY_PENDING=false
echo "--- 第${attempt}次检查 ($(date '+%H:%M:%S')) ---"
# 检查CI状态
for ctx in "${CONTEXTS[@]}"; do
STATE=$(python3 scripts/check_ci_status.py "$GITHUB_TOKEN" "$GITHUB_REPOSITORY" "$PR_HEAD_SHA" "$ctx")
echo " CI: ${ctx##*/}: $STATE"
if [ "$STATE" != "success" ]; then
ALL_SUCCESS=false
fi
if [ "$STATE" = "failure" ] || [ "$STATE" = "error" ]; then
ANY_FAILED=true
fi
if [ "$STATE" = "pending" ]; then
ANY_PENDING=true
fi
done
# 检查审批状态
APPROVAL_RESULT=$(python3 scripts/check_pr_approval.py "$MERGE_TOKEN" "$GITHUB_REPOSITORY" "$PR_NUMBER" 1)
echo " 审批: $APPROVAL_RESULT"
HAS_APPROVAL=false
if echo "$APPROVAL_RESULT" | grep -q '^approved'; then
HAS_APPROVAL=true
fi
# 全部满足 → 合并
if [ "$ALL_SUCCESS" = "true" ] && [ "$HAS_APPROVAL" = "true" ]; then
echo
echo "CI全绿 + 审批通过,执行自动合并"
# 幂等检查:PR是否还是open
PR_STATE=$(curl -s -H "Authorization: token ${MERGE_TOKEN}" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" \
| python3 -c "import sys,json; print(json.load(sys.stdin).get('state',''))")
if [ "$PR_STATE" != "open" ]; then
echo "PR状态为 ${PR_STATE},无需合并"
exit 0
fi
# 执行squash merge
HTTP_CODE=$(curl -s -o /tmp/merge_resp.json -w "%{http_code}" \
-X POST \
-H "Authorization: token ${MERGE_TOKEN}" \
-H "Content-Type: application/json" \
-d '{"do":"squash","merge_title_field":"","merge_message_field":"","delete_branch_after_merge":true,"force_merge":false}' \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/merge")
echo "合并API HTTP状态: $HTTP_CODE"
if [ "$HTTP_CODE" = "200" ]; then
echo "自动合并成功"
exit 0
elif [ "$HTTP_CODE" = "405" ]; then
MERGE_405_COUNT=$((MERGE_405_COUNT + 1))
echo "⚠️ 合并返回405(第${MERGE_405_COUNT}次),可能CI状态尚未同步或有未解决的门禁,继续等待重试..."
cat /tmp/merge_resp.json 2>/dev/null || true
echo
if [ "$MERGE_405_COUNT" -ge "$MAX_405_RETRIES" ]; then
echo "❌ 连续${MAX_405_RETRIES}次合并返回405,放弃自动合并"
curl -s -X POST \
-H "Authorization: token ${MERGE_TOKEN}" \
-H "Content-Type: application/json" \
-d '{"body": "Auto merge failed after multiple 405 errors: PR may have conflicts or unresolved checks. Please review manually."}' \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" > /dev/null 2>&1 || true
exit 1
fi
else
echo "自动合并失败 (HTTP $HTTP_CODE)"
cat /tmp/merge_resp.json 2>/dev/null || true
curl -s -X POST \
-H "Authorization: token ${MERGE_TOKEN}" \
-H "Content-Type: application/json" \
-d "{\"body\": \"Auto merge failed (HTTP ${HTTP_CODE}), please check manually.\"}" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" > /dev/null 2>&1 || true
exit 1
fi
else
# 本轮不满足合并条件,重置405计数器
MERGE_405_COUNT=0
fi
if [ "$ANY_FAILED" = "true" ]; then
echo
echo "CI有失败项,不自动合并"
exit 0
fi
sleep 10
done
echo
echo "等待超时(30分钟)"
exit 0
-105
View File
@@ -1,105 +0,0 @@
name: CI Base Image Build
on:
push:
branches:
- develop
- main
paths:
- 'requirements-base.txt'
- 'requirements-dev.txt'
- 'infra/docker/ci.Dockerfile'
workflow_dispatch:
inputs:
reason:
description: "触发原因"
required: false
default: "手动触发 - ci-base 镜像重建"
concurrency:
group: ci-base-image-build
cancel-in-progress: false
jobs:
build-ci-base:
name: Build CI Base Image
runs-on: runtime-builder
timeout-minutes: 60
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" \
| bash
- name: Docker login to Gitea Registry
shell: sh
env:
GITEA_REGISTRY_USER: xiaoxia
GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
run: |
set -eu
for i in 1 2 3; do
echo "=== Docker login 尝试 $i/3 ==="
if docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then
echo "✅ Docker login successful"
break
fi
echo "❌ Docker login 失败(尝试 $i/3),5s 后重试..."
sleep 5
done
- name: Build and push CI base image
shell: sh
run: |
set -eu
IMAGE="git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/ci-base"
VERSION_TAG="deps-$(date +%Y%m%d-%H%M)-${GITHUB_SHA::8}"
echo "=== Building CI base image (tags: latest, ${VERSION_TAG}) ==="
docker build --progress=plain \
-f infra/docker/ci.Dockerfile \
-t "${IMAGE}:latest" \
-t "${IMAGE}:${VERSION_TAG}" \
.
echo "✅ Image built successfully"
echo "=== Pushing ${VERSION_TAG} ==="
docker push "${IMAGE}:${VERSION_TAG}"
echo "=== Pushing latest ==="
docker push "${IMAGE}:latest"
echo "✅ Pushed to Gitea Registry"
- name: Verify image
shell: sh
run: |
set -eu
IMAGE="git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/ci-base:latest"
echo "=== Verifying pinned deps in fresh image ==="
docker run --rm "${IMAGE}" /opt/xiaoxia-ci-venv/bin/python -c \
"import httpcore, h2, numpy, httpx; print('VERSIONS:', httpcore.__version__, h2.__version__, numpy.__version__, httpx.__version__)"
- name: Notify result
if: always()
continue-on-error: true
shell: sh
env:
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
run: |
set +e
if [ "${{ job.status }}" = "success" ]; then
NOTIFY_MODE=success JOB_NAME="CI Base Image Build" python3 scripts/ci_notify.py
else
NOTIFY_MODE=failure JOB_NAME="CI Base Image Build" python3 scripts/ci_notify.py
fi
- name: Cleanup
if: always()
shell: sh
run: |
IMAGE="git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/ci-base"
docker rmi "${IMAGE}:latest" 2>/dev/null || true
echo "Cleanup done"
File diff suppressed because one or more lines are too long
-51
View File
@@ -1,51 +0,0 @@
name: CI Canary Check
on:
schedule:
- cron: '*/30 * * * *'
workflow_dispatch:
jobs:
canary:
runs-on: ci-l2
timeout-minutes: 10
steps:
- name: Canary (runner -> docker -> network -> gitea)
run: |
set -e
echo "== runner/container basic =="
date; hostname; whoami
echo "== gitea api reachability =="
code=$(curl -s -o /tmp/v.json -w '%{http_code}' -m 15 "$GITHUB_API_URL/version")
echo "gitea api http_code=$code"
[ "$code" = "200" ] || { echo "::error::Gitea API unreachable, http_code=$code"; exit 1; }
cat /tmp/v.json; echo
echo "== external egress =="
ext=$(curl -s -o /dev/null -w '%{http_code}' -m 15 https://www.baidu.com || echo 000)
echo "external http_code=$ext"
echo "== gitea domain resolves NOT to loopback =="
set -o pipefail
ip=$(getent hosts git.xiaoxiajianji.com | awk '{print $1}' | head -1)
echo "git.xiaoxiajianji.com -> $ip"
if [ -z "$ip" ]; then
echo "::error::DNS resolution failed, git.xiaoxiajianji.com unresolvable"; exit 1
fi
if [ "$ip" = "127.0.0.1" ] || [ "$ip" = "::1" ]; then
echo "::error::Gitea domain resolves to loopback inside job container (hosts/DNS leak)"; exit 1
fi
echo "CANARY OK"
- name: Notify failure
if: failure()
env:
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
run: |
set +e
if [ -n "$CI_NOTIFY_WEBHOOK" ]; then
MSG="🚨 CI 金丝雀失败:runner->docker->网络->Gitea 链路异常,时间 $(date '+%Y-%m-%d %H:%M:%S'),请立即检查构建服务器"
python3 - "$CI_NOTIFY_WEBHOOK" "$MSG" <<'PY'
import json,sys,urllib.request
hook,msg=sys.argv[1],sys.argv[2]
data=json.dumps({"msg_type":"text","content":{"text":msg}}).encode()
urllib.request.urlopen(urllib.request.Request(hook,data=data,headers={"Content-Type":"application/json"}),timeout=10)
PY
fi
exit 0
+589
View File
@@ -0,0 +1,589 @@
name: CI/CD Pipeline
on:
push:
branches:
- main
- develop
tags:
- v*
pull_request:
branches:
- main
- develop
workflow_dispatch:
inputs:
reason:
description: "触发原因"
required: false
default: "手动触发 - CI漏触发补跑"
permissions:
contents: read
concurrency:
group: ci-cd-${{ gitea.event_name }}-${{ gitea.ref }}
cancel-in-progress: true
jobs:
check-frontend-only:
name: Check if frontend-only change
runs-on: ci-check
if: github.event_name == 'pull_request'
outputs:
skip_backend: ${{ steps.check.outputs.skip_backend }}
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: "set -eu\npython3 - <<'PY'\nimport io, os, tarfile, time, urllib.request, urllib.error\nurl = f\"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz\"\nrequest = urllib.request.Request(url, headers={\"Authorization\": f\"token {os.environ['GITHUB_TOKEN']}\"})\nlast_err = None\nfor attempt in range(5):\n try:\n with urllib.request.urlopen(request, timeout=120) as response:\n archive = response.read()\n break\n except urllib.error.HTTPError as e:\n last_err = e\n if e.code >= 500 and attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...\")\n time.sleep(wait)\n continue\n raise\n except Exception as e:\n last_err = e\n if attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...\")\n time.sleep(wait)\n continue\n raise\nelse:\n raise last_err\nwith tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:\n root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'\n for member in tar.getmembers():\n name = member.name\n if name == root_prefix[:-1]:\n continue\n if name.startswith(root_prefix):\n member.name = name[len(root_prefix):]\n if member.name:\n tar.extract(member, '.')\nPY\n"
- name: Check changed files
id: check
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
set -eu
PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||')
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300"
FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin)]")
FRONTEND_COUNT=$(echo "$FILES" | grep -c '^apps/web/' || true)
BACKEND_COUNT=$(echo "$FILES" | grep -cv '^apps/web/' || true)
TOTAL=$(echo "$FILES" | grep -cv '^$' || true)
echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT})"
if [ "$BACKEND_COUNT" = "0" ] && [ "$FRONTEND_COUNT" -gt "0" ]; then
echo "skip_backend=true" >> $GITHUB_OUTPUT
echo "✅ 纯前端改动,跳过后端检查"
else
echo "skip_backend=false" >> $GITHUB_OUTPUT
echo "🔧 包含后端/公共变更,运行完整CI"
fi
validate:
needs: check-frontend-only
if: always() && needs.check-frontend-only.outputs.skip_backend != 'true'
name: Validate Code Quality And Tests
runs-on: ci-check
timeout-minutes: 10
permissions:
contents: write
env:
DATABASE_URL: postgresql+psycopg://postgres:postgres@127.0.0.1:5432/xiaoxia_saas
USE_IN_MEMORY_DB: 'false'
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: "set -eu\npython3 - <<'PY'\nimport io, os, tarfile, time, urllib.request, urllib.error\nurl = f\"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz\"\nrequest = urllib.request.Request(url, headers={\"Authorization\": f\"token {os.environ['GITHUB_TOKEN']}\"})\nlast_err = None\nfor attempt in range(5):\n try:\n with urllib.request.urlopen(request, timeout=120) as response:\n archive = response.read()\n break\n except urllib.error.HTTPError as e:\n last_err = e\n if e.code >= 500 and attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...\")\n time.sleep(wait)\n continue\n raise\n except Exception as e:\n last_err = e\n if attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...\"\
)\n time.sleep(wait)\n continue\n raise\nelse:\n raise last_err\nwith tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:\n root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'\n for member in tar.getmembers():\n name = member.name\n if name == root_prefix[:-1]:\n continue\n if name.startswith(root_prefix):\n member.name = name[len(root_prefix):]\n if member.name:\n tar.extract(member, '.')\nPY\n"
- name: Record job start time
shell: sh
run: 'set -eu
echo "JOB_START_TIME=$(date +%s)" >> $GITHUB_ENV
echo "Job started at $(date)"
'
- name: Verify CI environment
shell: sh
run: 'set -eu
python3 --version
python3 -m pip --version
echo "CI environment is ready"
'
- name: Install dependencies
shell: sh
run: 'set -eu
python3 -m pip install -q -r requirements-base.txt
python3 -m pip install -q -r requirements.txt
python3 -m pip install -q -r requirements-dev.txt
# Force source install of black/isort to ensure consistent formatting
# across compiled/source installations on different machines
python3 -m pip install --no-binary :all: black==26.5.1 isort==8.0.1
python3 -m black --version
python3 -m isort --version-number
python3 -m ruff --version
bandit --version
pytest --version
'
- name: Secret detection (detect-secrets)
shell: sh
run: "set -eu\necho \"=== Installing detect-secrets ===\"\npython3 -m pip install -q detect-secrets\ndetect-secrets --version\necho \"\"\necho \"=== Running secret scan ===\"\ndetect-secrets scan \\\n --all-files \\\n --exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \\\n --exclude-files '\\.(md|rst|txt|lock|example|sample|min\\.js|min\\.css|spec\\.ts|test\\.ts|test\\.py)$' \\\n --exclude-files '(package-lock|yarn\\.lock|poetry\\.lock|Pipfile\\.lock)$' \\\n --disable-plugin Base64HighEntropyString \\\n --disable-plugin HexHighEntropyString \\\n --disable-plugin BasicAuthDetector \\\n --disable-plugin KeywordDetector \\\n --disable-plugin IPPublicDetector \\\n > /tmp/secrets-scan.json 2>&1\n\nFOUND=$(python3 -c \"\nimport json\ntry:\n with open('/tmp/secrets-scan.json') as f:\n data = json.load(f)\n results = data.get('results', {})\n total = sum(len(v) for\
\ v in results.values())\n print(total)\nexcept Exception:\n print('error')\n\")\necho \"\"\necho \"Secrets detected: $FOUND\"\nif [ \"$FOUND\" != \"0\" ] && [ \"$FOUND\" != \"error\" ]; then\n echo \"\"\n echo \"=== Secret details ===\"\n python3 -c \"\nimport json\nwith open('/tmp/secrets-scan.json') as f:\n data = json.load(f)\nfor fpath, items in data.get('results', {}).items():\n for item in items:\n line = item.get('line_number', '?')\n stype = item.get('type', '?')\n hashed = item.get('hashed_secret', '')[:16]\n print(f' {fpath}:{line} [{stype}] {hashed}...')\n\"\n echo \"\"\n echo \"ERROR: Potential secrets detected in code!\"\n echo \"If these are false positives, add exclusions in the CI workflow.\"\n exit 1\nfi\necho \"Secret scan completed - no secrets detected\"\n"
- name: Calculate changed Python files (incremental scan)
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: "set -eu\nSCAN_MODE=\"full\"\nCHANGED_PY_FILES=\"\"\n\nif [ \"${GITHUB_EVENT_NAME:-}\" = \"pull_request\" ] && [ -n \"${GITHUB_REF_NAME:-}\" ]; then\n echo \"PR mode (#${GITHUB_REF_NAME}) - fetching changed files from API\"\n\n PR_NUMBER=$(echo \"$GITHUB_REF\" | sed 's|refs/pull/||; s|/.*||')\n API_URL=\"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100\"\n\n set +e\n RESPONSE=$(curl -s -w \"\\n%{http_code}\" -H \"Authorization: token ${GITHUB_TOKEN}\" \"${API_URL}\")\n HTTP_CODE=$(echo \"$RESPONSE\" | tail -n1)\n BODY=$(echo \"$RESPONSE\" | sed '$d')\n set -e\n\n if [ \"$HTTP_CODE\" = \"200\" ]; then\n CHANGED_PY_FILES=$(echo \"$BODY\" | python3 -c \"\nimport json, sys\ntry:\n files = json.load(sys.stdin)\n py_files = [f['filename'] for f in files\n if f['filename'].endswith('.py') and f['status'] != 'removed']\n print(' '.join(py_files))\nexcept Exception:\n print('')\n\")\n if [ -n \"$CHANGED_PY_FILES\" ]; then\n SCAN_MODE=\"incremental\"\n FILE_COUNT=$(echo \"$CHANGED_PY_FILES\" | wc -w)\n echo \"Changed Python files: ${FILE_COUNT}\"\n echo \"$CHANGED_PY_FILES\" | tr ' ' '\\n' | grep -v '^$'\n else\n SCAN_MODE=\"skip_py\"\n echo \"No Python files changed in this PR\"\n fi\n else\n echo \"WARN: API returned HTTP $HTTP_CODE, falling back to full scan\"\n fi\nelse\n echo \"Full scan mode (not a PR event)\"\nfi\n\necho \"SCAN_MODE=$SCAN_MODE\" >> $GITHUB_ENV\necho \"CHANGED_PY_FILES=$CHANGED_PY_FILES\" >> $GITHUB_ENV\n"
- name: Run code quality checks
shell: sh
run: "set -eu\n\nif [ \"$SCAN_MODE\" = \"incremental\" ]; then\n echo \"=== Incremental scan mode ===\"\n\n python3 -m compileall -q $CHANGED_PY_FILES\n\n python3 -m black --check --fast $CHANGED_PY_FILES\n\n python3 -m isort --check-only $CHANGED_PY_FILES\n\n RUFF_FILES=$(echo \"$CHANGED_PY_FILES\" | tr ' ' '\\n' | grep -v '^scripts/' | tr '\\n' ' ')\n if [ -n \"$RUFF_FILES\" ]; then\n python3 -m ruff check $RUFF_FILES --statistics\n else\n echo \"No ruff-checkable files changed, skipping\"\n fi\n\nelif [ \"$SCAN_MODE\" = \"skip_py\" ]; then\n echo \"No Python files changed - skipping Python lint checks\"\n\nelse\n echo \"=== Full scan mode ===\"\n\n python3 -m compileall -q alembic apps packages tests scripts\n\n python3 -m black --check --fast alembic apps packages tests scripts\n\n python3 -m isort --check-only alembic apps packages tests scripts\n\n python3 -m ruff check apps packages tests --statistics\nfi\n"
- name: Auto-fix formatting (black + isort)
if: failure()
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: python3 scripts/ci/auto_fix_formatting.py
- name: Type check (mypy, hard gate)
shell: sh
run: "bash scripts/ci/mypy_check.sh"
- name: Run security scan (bandit)
shell: sh
run: 'set -eu
bandit -r apps packages -q -ll
'
- name: Python dependency vulnerability scan (pip-audit)
shell: sh
run: "set -eu\necho \"=== Installing pip-audit ===\"\npython3 -m pip install -q pip-audit\npip-audit --version\necho \"\"\necho \"=== Scanning Python dependencies ===\"\nEXIT_CODE=0\nfor req_file in requirements.txt requirements-base.txt requirements-dev.txt; do\n if [ -f \"$req_file\" ]; then\n echo \"--- Scanning $req_file ---\"\n pip-audit -r \"$req_file\" --desc on 2>&1 | head -40 || EXIT_CODE=$?\n echo \"\"\n fi\ndone\necho \"pip-audit scan completed (advisory mode - warnings only, not blocking CI)\"\nif [ \"$EXIT_CODE\" != \"0\" ]; then\n echo \"WARNING: Potential vulnerabilities found in dependencies.\"\nfi\nexit 0\n"
- name: Dead code detection (vulture)
if: always()
shell: sh
run: "set +e\necho \"=== Installing vulture ===\"\npython3 -m pip install -q vulture\nvulture --version\necho \"\"\necho \"=== Running vulture dead code scan (confidence >= 70%) ===\"\necho \"告警模式,不阻断CI。置信度>=90%建议尽快确认。\"\necho \"\"\n# 按置信度从高到低输出,便于优先查看高价值条目\nvulture apps packages scripts \\\n --exclude \"tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py\" \\\n --min-confidence 70 \\\n 2>&1 | sort -t'(' -k2 -rn | head -80\nEXIT_CODE=$?\necho \"\"\necho \"=== vulture scan summary ===\"\nif [ \"$EXIT_CODE\" != \"0\" ]; then\n echo \"发现潜在死代码(可能包含框架装饰器注册的函数,为误报)\"\n echo \"建议:定期人工审查高置信度(>=90%)条目\"\nelse\n echo \"未发现明显死代码 ✅\"\nfi\nexit 0\n"
- name: Validate release scripts syntax
shell: sh
run: 'set -eu
bash -n scripts/backup_postgres.sh
bash -n scripts/restore_postgres_plan.sh
bash -n scripts/init_production_env.sh
'
- name: Validate Alembic migrations (with isolated PG)
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
set -eu
PG_CONTAINER=ci-pg-validate-${GITHUB_RUN_ID:-$$}
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker run -d --name "$PG_CONTAINER" \
--shm-size=256m \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=xiaoxia_saas \
-P \
--health-cmd "pg_isready -U postgres" \
--health-interval 3s \
--health-timeout 3s \
--health-retries 20 \
postgres:16-alpine
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
echo "PostgreSQL port: $PG_PORT"
export DATABASE_URL=postgresql+psycopg://postgres:postgres@127.0.0.1:$PG_PORT/xiaoxia_saas
for i in $(seq 1 30); do
if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
echo "PostgreSQL is ready on port $PG_PORT"
break
fi
echo "Waiting for PostgreSQL... ($i/30)"
sleep 2
done
docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy
python3 -m alembic upgrade head --sql > /tmp/alembic-upgrade.sql
test -s /tmp/alembic-upgrade.sql
grep -q "Running upgrade" /tmp/alembic-upgrade.sql
python3 scripts/check_schema_metadata.py
# Initialize git for migration safety diff (CI checkout is tar.gz without .git)
git init > /dev/null 2>&1
git remote add origin https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas.git > /dev/null 2>&1
git fetch origin develop:refs/remotes/origin/develop --depth=100 > /dev/null 2>&1
git add -A > /dev/null 2>&1
git -c user.email=ci@local -c user.name=CI commit -m "ci-tmp" > /dev/null 2>&1
python3 scripts/check_migration_safety.py --allow-medium-risk --diff-against origin/develop
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
echo "PostgreSQL container cleaned up"
- name: Job duration summary
if: always()
shell: sh
run: "set +eu\nif [ -n \"$JOB_START_TIME\" ]; then\n END_TIME=$(date +%s)\n DURATION=$((END_TIME - JOB_START_TIME))\n MINS=$((DURATION / 60))\n SECS=$((DURATION % 60))\n echo \"JOB_DURATION_SECONDS=$DURATION\" >> $GITHUB_ENV\n echo \"=== Job Duration: ${MINS}m${SECS}s ===\"\nelse\n echo \"JOB_DURATION_SECONDS=0\" >> $GITHUB_ENV\n echo \"=== Job Duration: unknown ===\"\nfi\n"
- name: Notify on failure
continue-on-error: true
if: failure()
shell: sh
env:
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
run: 'set +e
NOTIFY_MODE=failure JOB_NAME="Validate Code Quality And Tests" python3 scripts/ci_notify.py
'
unit-tests:
needs: check-frontend-only
if: always() && needs.check-frontend-only.outputs.skip_backend != 'true'
name: Unit Tests
runs-on: ci-l2
timeout-minutes: 8
env:
USE_IN_MEMORY_DB: 'true'
OSS_ACCESS_KEY_ID: placeholder
OSS_ACCESS_KEY_SECRET: placeholder
OSS_BUCKET_NAME: xiaoxia-autocut
OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: "set -eu\npython3 - <<'PY'\nimport io, os, tarfile, time, urllib.request, urllib.error\nurl = f\"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz\"\nrequest = urllib.request.Request(url, headers={\"Authorization\": f\"token {os.environ['GITHUB_TOKEN']}\"})\nlast_err = None\nfor attempt in range(5):\n try:\n with urllib.request.urlopen(request, timeout=120) as response:\n archive = response.read()\n break\n except urllib.error.HTTPError as e:\n last_err = e\n if e.code >= 500 and attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...\")\n time.sleep(wait)\n continue\n raise\n except Exception as e:\n last_err = e\n if attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...\"\
)\n time.sleep(wait)\n continue\n raise\nelse:\n raise last_err\nwith tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:\n root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'\n for member in tar.getmembers():\n name = member.name\n if name == root_prefix[:-1]:\n continue\n if name.startswith(root_prefix):\n member.name = name[len(root_prefix):]\n if member.name:\n tar.extract(member, '.')\nPY\n"
- name: Record job start time
shell: sh
run: 'set -eu
echo "JOB_START_TIME=$(date +%s)" >> $GITHUB_ENV
echo "Job started at $(date)"
'
- name: Install ffmpeg
shell: sh
run: "set +e\nif command -v ffmpeg > /dev/null 2>&1; then\n echo \"ffmpeg already installed: $(ffmpeg -version | head -1)\"\n exit 0\nfi\nif command -v apt-get > /dev/null 2>&1; then\n apt-get update -qq && apt-get install -y -qq ffmpeg\nelif command -v yum > /dev/null 2>&1; then\n yum install -y -q epel-release 2>/dev/null\n yum install -y -q ffmpeg 2>/dev/null\n if [ $? -ne 0 ] && command -v dnf > /dev/null 2>&1; then\n dnf install -y -q --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfusion-free-release-$(rpm -E %rhel).noarch.rpm 2>/dev/null\n dnf install -y -q ffmpeg 2>/dev/null\n fi\nelif command -v dnf > /dev/null 2>&1; then\n dnf install -y -q ffmpeg 2>/dev/null\nfi\nif command -v ffmpeg > /dev/null 2>&1; then\n echo \"ffmpeg installed successfully: $(ffmpeg -version | head -1)\"\nelse\n echo \"Warning: ffmpeg installation failed or not available, some tests may be skipped\"\nfi\n"
- name: Install dependencies
shell: sh
run: 'set -eu
python3 -m pip install -q -r requirements-base.txt
python3 -m pip install -q -r requirements.txt
python3 -m pip install -q -r requirements-dev.txt
pytest --version
'
- name: Select incremental test files
if: github.event_name == 'pull_request'
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
set +e
PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||')
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300"
CHANGED_FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin) if f['status'] != 'removed']")
echo "改动文件数: $(echo "$CHANGED_FILES" | grep -c . || echo 0)"
CHANGED_FILES="$CHANGED_FILES" \
SELECTED_TESTS_OUTPUT=/tmp/selected_tests.txt \
python3 scripts/ci/select_unit_tests.py
SELECT_EXIT=$?
if [ $SELECT_EXIT -eq 0 ]; then
echo "UNIT_TEST_MODE=incremental" >> $GITHUB_ENV
TEST_FILES=$(cat /tmp/selected_tests.txt | tr '\n' ' ')
echo "SELECTED_TEST_FILES=$TEST_FILES" >> $GITHUB_ENV
echo "增量模式: $(cat /tmp/selected_tests.txt | wc -l) 个测试文件"
else
echo "UNIT_TEST_MODE=full" >> $GITHUB_ENV
echo "SELECTED_TEST_FILES=tests/unit" >> $GITHUB_ENV
echo "全量模式"
fi
- name: Run unit tests with coverage
shell: sh
run: |
set -eu
if [ "${UNIT_TEST_MODE:-full}" = "incremental" ]; then
echo "=== 增量测试模式 ==="
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m coverage run \
--source=apps/api/app,packages \
--omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \
--branch \
-m pytest $SELECTED_TEST_FILES -q
python3 -m coverage report --show-missing
python3 -m coverage xml -o coverage.xml
# 增量模式下调低覆盖率门槛(跑的文件少覆盖率自然低,不做强校验)
python3 -m coverage report --fail-under=10 > /dev/null || true
else
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m coverage run \
--source=apps/api/app,packages \
--omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \
--branch \
-m pytest tests/unit -q
python3 -m coverage report --show-missing
python3 -m coverage xml -o coverage.xml
python3 -m coverage report --fail-under=65 > /dev/null
fi
- name: Diff coverage check (增量行覆盖率)
if: github.event_name == 'pull_request' && env.HAS_APP_CHANGES == 'true'
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
set -eu
PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||')
# 获取base分支
BASE_BRANCH="${{ github.base_ref }}"
echo "Base branch: $BASE_BRANCH"
# 初始化git (CI tarball checkout没有.git目录)
# 先备份PR代码,再基于base分支建分支,确保HEAD与base有共同祖先
PR_CODE_DIR="/tmp/pr-code-$$"
mkdir -p "$PR_CODE_DIR"
# 排除隐藏文件(如.env)和后续生成的coverage文件,只备份源码
find . -maxdepth 1 -mindepth 1 ! -name 'coverage.xml' ! -name 'diff_coverage.html' -exec cp -r {} "$PR_CODE_DIR/" \;
rm -rf .git
git init > /dev/null 2>&1
git remote add origin https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas.git > /dev/null 2>&1
git config user.email "ci@local"
git config user.name "CI"
# 拉取base分支用于对比
git fetch origin $BASE_BRANCH --depth=200
# 基于base分支创建当前分支,确保有共同祖先
git checkout -b ci-pr-branch "origin/$BASE_BRANCH" > /dev/null 2>&1
# 清除base分支的源码,用PR代码覆盖
find . -mindepth 1 -maxdepth 1 ! -name '.git' -exec rm -rf {} +
cp -r "$PR_CODE_DIR"/. .
rm -rf "$PR_CODE_DIR"
# 提交当前代码
git add -A > /dev/null 2>&1
git commit -m "ci-tmp" > /dev/null 2>&1
# 根据模式设置门槛
if [ "${UNIT_TEST_MODE:-full}" = "incremental" ]; then
# 增量测试模式覆盖不全,门槛设低一些
THRESHOLD=40
echo "增量测试模式,增量覆盖率门槛: ${THRESHOLD}%"
else
THRESHOLD=60
echo "全量测试模式,增量覆盖率门槛: ${THRESHOLD}%"
fi
# 运行diff-cover
set +e
python3 -m diff_cover.diff_cover_tool coverage.xml \
--compare-branch="origin/$BASE_BRANCH" \
--fail-under=$THRESHOLD \
--html-report diff_coverage.html \
2>&1
DIFF_EXIT=$?
set -e
if [ $DIFF_EXIT -ne 0 ]; then
echo ""
echo "❌ 增量覆盖率未达到门槛 (${THRESHOLD}%)"
echo " 请为改动的代码添加单元测试后再提交"
echo ""
echo "=== 覆盖率报告 ==="
python3 -m diff_cover.diff_cover_tool coverage.xml \
--compare-branch="origin/$BASE_BRANCH" 2>&1 | tail -30
exit 1
fi
echo "✅ 增量覆盖率达标"
- name: CI failure notification
if: failure()
shell: sh
env:
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
run: 'set +e
FAILED_JOB="Unit Tests" python3 scripts/ci_notify_failure.py
'
- name: Job duration summary
if: always()
shell: sh
run: "set +eu\nif [ -n \"$JOB_START_TIME\" ]; then\n END_TIME=$(date +%s)\n DURATION=$((END_TIME - JOB_START_TIME))\n MINS=$((DURATION / 60))\n SECS=$((DURATION % 60))\n echo \"JOB_DURATION_SECONDS=$DURATION\" >> $GITHUB_ENV\n echo \"=== Job Duration: ${MINS}m${SECS}s ===\"\nelse\n echo \"JOB_DURATION_SECONDS=0\" >> $GITHUB_ENV\n echo \"=== Job Duration: unknown ===\"\nfi\n"
- name: Notify on failure
continue-on-error: true
if: failure()
shell: sh
env:
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
run: 'set +e
NOTIFY_MODE=failure JOB_NAME="Unit Tests" python3 scripts/ci_notify.py
'
integration-tests:
name: Integration Tests
runs-on: ci-l2
timeout-minutes: 30
if: always() && needs.check-frontend-only.outputs.skip_backend != 'true'
needs:
- check-frontend-only
- validate
env:
DATABASE_URL: postgresql+psycopg://postgres:postgres@127.0.0.1:5432/xiaoxia_saas
USE_IN_MEMORY_DB: 'false'
OSS_ACCESS_KEY_ID: placeholder
OSS_ACCESS_KEY_SECRET: placeholder
OSS_BUCKET_NAME: xiaoxia-autocut
OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: "set -eu\npython3 - <<'PY'\nimport io, os, tarfile, time, urllib.request, urllib.error\nurl = f\"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz\"\nrequest = urllib.request.Request(url, headers={\"Authorization\": f\"token {os.environ['GITHUB_TOKEN']}\"})\nlast_err = None\nfor attempt in range(5):\n try:\n with urllib.request.urlopen(request, timeout=120) as response:\n archive = response.read()\n break\n except urllib.error.HTTPError as e:\n last_err = e\n if e.code >= 500 and attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...\")\n time.sleep(wait)\n continue\n raise\n except Exception as e:\n last_err = e\n if attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...\"\
)\n time.sleep(wait)\n continue\n raise\nelse:\n raise last_err\nwith tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:\n root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'\n for member in tar.getmembers():\n name = member.name\n if name == root_prefix[:-1]:\n continue\n if name.startswith(root_prefix):\n member.name = name[len(root_prefix):]\n if member.name:\n tar.extract(member, '.')\nPY\n"
- name: Record job start time
shell: sh
run: 'set -eu
echo "JOB_START_TIME=$(date +%s)" >> $GITHUB_ENV
echo "Job started at $(date)"
'
- name: Verify CI environment
shell: sh
run: 'set -eu
python3 --version
python3 -m pip --version
echo "CI environment is ready"
'
- name: Install dependencies
shell: sh
run: 'set -eu
python3 -m pip install -q -r requirements-base.txt
python3 -m pip install -q -r requirements.txt
python3 -m pip install -q -r requirements-dev.txt
pytest --version
'
- name: Install ffmpeg
shell: sh
run: "set +e\nif command -v ffmpeg > /dev/null 2>&1; then\n echo \"ffmpeg already installed: $(ffmpeg -version | head -1)\"\n exit 0\nfi\nif command -v apt-get > /dev/null 2>&1; then\n apt-get update -qq && apt-get install -y -qq ffmpeg\nelif command -v yum > /dev/null 2>&1; then\n yum install -y -q epel-release 2>/dev/null\n yum install -y -q ffmpeg 2>/dev/null\n if [ $? -ne 0 ] && command -v dnf > /dev/null 2>&1; then\n dnf install -y -q --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfusion-free-release-$(rpm -E %rhel).noarch.rpm 2>/dev/null\n dnf install -y -q ffmpeg 2>/dev/null\n fi\nelif command -v dnf > /dev/null 2>&1; then\n dnf install -y -q ffmpeg 2>/dev/null\nfi\nif command -v ffmpeg > /dev/null 2>&1; then\n echo \"ffmpeg installed successfully: $(ffmpeg -version | head -1)\"\nelse\n echo \"Warning: ffmpeg installation failed or not available, some tests may be skipped\"\nfi\n"
- name: Start Redis
shell: sh
run: "set -eu\nREDIS_CONTAINER=\"ci-redis-${GITHUB_RUN_ID:-$$}\"\necho \"REDIS_CONTAINER=$REDIS_CONTAINER\" >> \"$GITHUB_ENV\"\ndocker rm -f \"$REDIS_CONTAINER\" 2>/dev/null || true\ndocker run -d --name \"$REDIS_CONTAINER\" \\\n -P \\\n --health-cmd \"redis-cli ping\" \\\n --health-interval 2s \\\n --health-timeout 2s \\\n --health-retries 10 \\\n redis:7-alpine\nREDIS_PORT=$(docker port \"$REDIS_CONTAINER\" 6379/tcp | cut -d: -f2)\necho \"Redis port: $REDIS_PORT\"\necho \"REDIS_URL=redis://127.0.0.1:$REDIS_PORT/0\" >> \"$GITHUB_ENV\"\nfor i in $(seq 1 15); do\n if docker inspect --format='{{.State.Health.Status}}' \"$REDIS_CONTAINER\" 2>/dev/null | grep -q healthy; then\n echo \"Redis is ready on port $REDIS_PORT\"\n break\n fi\n echo \"Waiting for Redis... ($i/15)\"\n sleep 2\ndone\ndocker inspect --format='{{.State.Health.Status}}' \"$REDIS_CONTAINER\" | grep -q healthy\n"
- name: Start PostgreSQL for integration tests
shell: sh
run: "set -eu\nPG_CONTAINER=\"ci-pg-${GITHUB_RUN_ID:-$$}\"\necho \"PG_CONTAINER=$PG_CONTAINER\" >> \"$GITHUB_ENV\"\ndocker rm -f \"$PG_CONTAINER\" 2>/dev/null || true\ndocker run -d --name \"$PG_CONTAINER\" \\\n --shm-size=256m \\\n -e POSTGRES_USER=postgres \\\n -e POSTGRES_PASSWORD=postgres \\\n -e POSTGRES_DB=xiaoxia_saas \\\n -P \\\n --health-cmd \"pg_isready -U postgres\" \\\n --health-interval 5s \\\n --health-timeout 5s \\\n --health-retries 12 \\\n postgres:16\n# 获取随机映射的端口\nPG_PORT=$(docker port \"$PG_CONTAINER\" 5432/tcp | cut -d: -f2)\necho \"PostgreSQL port: $PG_PORT\"\necho \"DATABASE_URL=postgresql+psycopg://postgres:postgres@127.0.0.1:$PG_PORT/xiaoxia_saas\" >> \"$GITHUB_ENV\"\nfor i in $(seq 1 30); do\n if docker inspect --format='{{.State.Health.Status}}' \"$PG_CONTAINER\" 2>/dev/null | grep -q healthy; then\n echo \"PostgreSQL is ready on port $PG_PORT\"\n break\n fi\n echo \"Waiting for PostgreSQL... ($i/30)\"\n sleep 2\ndone\ndocker inspect --format='{{.State.Health.Status}}'\
\ \"$PG_CONTAINER\" | grep -q healthy\n"
- name: Apply migrations for integration tests
shell: sh
run: 'set -eu
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
'
- name: Run integration tests
shell: sh
run: "set -eu\npython3 -m pip install -q pytest-rerunfailures\nPYTHONPATH=\"$PWD/apps/api:$PWD\" python3 -m coverage run --append \\\n --source=apps/api/app,packages \\\n --omit=\"*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*\" \\\n --branch \\\n -m pytest tests/integration -q --timeout=60 -x --reruns 2 --reruns-delay 1 -m \"not performance\"\npython3 -m coverage report --show-missing\npython3 -m coverage xml -o coverage.xml\npython3 -m coverage report --fail-under=40 > /dev/null # 集成测试覆盖率门槛较低,核心目标是功能验证\n"
- name: Run API performance baseline tests
shell: sh
continue-on-error: true
run: "set +e\necho \"=== API 性能基线测试 ===\"\nPERF_OUTPUT=$(mktemp)\nPYTHONPATH=\"$PWD/apps/api:$PWD\" python3 -m pytest tests/integration/test_api_performance.py \\\n -v --timeout=120 -p no:cacheprovider 2>&1 | tee \"$PERF_OUTPUT\"\nPERF_EXIT=$?\n\n# 提取性能统计\necho \"\"\necho \"=== 性能测试摘要 ===\"\ngrep \"PERF_STATS:\" \"$PERF_OUTPUT\" || echo \"PERF_STATS: 未找到统计数据\"\ngrep \"PERF_RESULT:\" \"$PERF_OUTPUT\" || echo \"PERF_RESULT: 未找到详细结果\"\n\n# 统计通过率\nTOTAL=$(grep -c \"PERF_RESULT:\" \"$PERF_OUTPUT\" || echo 0)\nPASSED=$(grep \"PERF_RESULT: PASS\" \"$PERF_OUTPUT\" | wc -l)\nFAILED=$(grep \"PERF_RESULT: FAIL\" \"$PERF_OUTPUT\" | wc -l)\n\necho \"\"\necho \"性能测试结果: $PASSED/$TOTAL 通过, $FAILED 未达标\"\n\nif [ \"$FAILED\" -gt 0 ]; then\n echo \"\"\n echo \"⚠️ 警告: $FAILED 个接口性能未达标,请关注以下接口:\"\n grep \"PERF_RESULT: FAIL\" \"$PERF_OUTPUT\" | while read line; do\n echo \" $line\"\n done\n echo \"\"\n echo \"性能测试失败不阻塞主流水线,但建议尽快优化。\"\nelse\n echo \"✅ 所有接口性能达标!\"\nfi\n\nrm -f \"$PERF_OUTPUT\"\
\n# 始终返回 0,不阻塞流水线\nexit 0\n"
- name: Cleanup PostgreSQL & Redis
if: always()
shell: sh
run: 'docker rm -f "${PG_CONTAINER:-ci-pg-validate}" 2>/dev/null || true
docker rm -f "${REDIS_CONTAINER:-ci-redis-int}" 2>/dev/null || true
echo "PostgreSQL container cleaned up"
echo "Redis container cleaned up"
'
- name: Coverage summary
if: always()
shell: sh
env:
COVERAGE_THRESHOLD: '40'
run: 'set +e
echo "=== 覆盖率汇总 ==="
python3 scripts/ci_coverage_summary.py
'
- name: Job duration summary
if: always()
shell: sh
run: "set +eu\nif [ -n \"$JOB_START_TIME\" ]; then\n END_TIME=$(date +%s)\n DURATION=$((END_TIME - JOB_START_TIME))\n MINS=$((DURATION / 60))\n SECS=$((DURATION % 60))\n echo \"JOB_DURATION_SECONDS=$DURATION\" >> $GITHUB_ENV\n echo \"=== Job Duration: ${MINS}m${SECS}s ===\"\nelse\n echo \"JOB_DURATION_SECONDS=0\" >> $GITHUB_ENV\n echo \"=== Job Duration: unknown ===\"\nfi\n"
- name: Notify on failure
continue-on-error: true
if: failure()
shell: sh
env:
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
run: 'set +e
NOTIFY_MODE=failure JOB_NAME="Integration Tests" python3 scripts/ci_notify.py
'
frontend-lint:
name: Frontend Lint
runs-on: ci-check
timeout-minutes: 10
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: "set -eu\npython3 - <<'PY'\nimport io, os, tarfile, time, urllib.request, urllib.error\nurl = f\"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz\"\nrequest = urllib.request.Request(url, headers={\"Authorization\": f\"token {os.environ['GITHUB_TOKEN']}\"})\nlast_err = None\nfor attempt in range(5):\n try:\n with urllib.request.urlopen(request, timeout=120) as response:\n archive = response.read()\n break\n except urllib.error.HTTPError as e:\n last_err = e\n if e.code >= 500 and attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...\")\n time.sleep(wait)\n continue\n raise\n except Exception as e:\n last_err = e\n if attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...\"\
)\n time.sleep(wait)\n continue\n raise\nelse:\n raise last_err\nwith tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:\n root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'\n for member in tar.getmembers():\n name = member.name\n if name == root_prefix[:-1]:\n continue\n if name.startswith(root_prefix):\n member.name = name[len(root_prefix):]\n if member.name:\n tar.extract(member, '.')\nPY\n"
- name: Record job start time
shell: sh
run: 'set -eu
echo "JOB_START_TIME=$(date +%s)" >> $GITHUB_ENV
echo "Job started at $(date)"
'
- name: Install dependencies
shell: sh
run: "set -eu\nNPM_CACHE_VOLUME=\"xiaoxia-npm-cache\"\nif ! docker volume inspect \"$NPM_CACHE_VOLUME\" >/dev/null 2>&1; then\n docker volume create \"$NPM_CACHE_VOLUME\" >/dev/null\n echo \"Created npm cache volume: $NPM_CACHE_VOLUME\"\nfi\n\ndocker run --rm \\\n -v \"$PWD:/workspace\" \\\n -v \"$NPM_CACHE_VOLUME:/workspace/apps/web/node_modules\" -w /workspace/apps/web \\\n docker.m.daocloud.io/library/node:20 \\\n sh -lc 'PACKAGE_LOCK_HASH=$(md5sum package-lock.json 2>/dev/null | cut -d\" \" -f1)\nCACHE_HASH_FILE=\"node_modules/.package-lock-hash\"\nCACHE_VALID=false\nif [ -f \"$CACHE_HASH_FILE\" ] && [ \"$(cat \"$CACHE_HASH_FILE\")\" = \"$PACKAGE_LOCK_HASH\" ] && [ -x \"node_modules/.bin/eslint\" ] && [ -x \"node_modules/.bin/tsc\" ] && [ -x \"node_modules/.bin/prettier\" ] && [ -x \"node_modules/.bin/vitest\" ]; then\n CACHE_VALID=true\n echo \"Cache hit: dependencies valid, skipping npm ci\"\nfi\nif [ \"$CACHE_VALID\" = \"false\" ]; then\n echo \"Cache miss or invalid: running npm ci...\"\n if ! npm ci --include=dev; then\n echo \"npm ci failed, cleaning node_modules and retrying...\"\n rm -rf node_modules\n mkdir -p node_modules\n npm ci --include=dev\n fi\n # Post-install integrity check: verify all critical tools exist\n if [ ! -x \"node_modules/.bin/eslint\" ] || [ ! -x \"node_modules/.bin/tsc\" ] || [ ! -x \"node_modules/.bin/prettier\" ] || [ ! -x \"node_modules/.bin/vitest\" ]; then\n echo \"Post-install check failed: critical binaries missing, cleaning and retrying...\"\n rm -rf node_modules\n mkdir -p node_modules\n npm ci --include=dev\n fi\n echo \"$PACKAGE_LOCK_HASH\" > \"$CACHE_HASH_FILE\"\n echo \"Dependencies installed, cache updated\"\nfi'\n"
- name: Run ESLint
shell: sh
run: "set -eu\nNPM_CACHE_VOLUME=\"xiaoxia-npm-cache\"\nif ! docker volume inspect \"$NPM_CACHE_VOLUME\" >/dev/null 2>&1; then\n docker volume create \"$NPM_CACHE_VOLUME\" >/dev/null\n echo \"Created npm cache volume: $NPM_CACHE_VOLUME\"\nfi\n\ndocker run --rm \\\n -v \"$PWD:/workspace\" \\\n -v \"$NPM_CACHE_VOLUME:/workspace/apps/web/node_modules\" -w /workspace/apps/web \\\n docker.m.daocloud.io/library/node:20 \\\n sh -lc 'npx --no-install eslint src --ext .ts,.tsx --max-warnings 0'\n"
- name: Run TypeScript type check
shell: sh
run: "set -eu\nNPM_CACHE_VOLUME=\"xiaoxia-npm-cache\"\nif ! docker volume inspect \"$NPM_CACHE_VOLUME\" >/dev/null 2>&1; then\n docker volume create \"$NPM_CACHE_VOLUME\" >/dev/null\n echo \"Created npm cache volume: $NPM_CACHE_VOLUME\"\nfi\n\ndocker run --rm \\\n -v \"$PWD:/workspace\" \\\n -v \"$NPM_CACHE_VOLUME:/workspace/apps/web/node_modules\" -w /workspace/apps/web \\\n docker.m.daocloud.io/library/node:20 \\\n sh -lc 'npx --no-install tsc --noEmit'\n"
- name: Run Prettier check
shell: sh
run: "set -eu\nNPM_CACHE_VOLUME=\"xiaoxia-npm-cache\"\nif ! docker volume inspect \"$NPM_CACHE_VOLUME\" >/dev/null 2>&1; then\n docker volume create \"$NPM_CACHE_VOLUME\" >/dev/null\n echo \"Created npm cache volume: $NPM_CACHE_VOLUME\"\nfi\n\ndocker run --rm \\\n -v \"$PWD:/workspace\" \\\n -v \"$NPM_CACHE_VOLUME:/workspace/apps/web/node_modules\" -w /workspace/apps/web \\\n docker.m.daocloud.io/library/node:20 \\\n sh -lc 'npx --no-install prettier --check \"src/**/*.{ts,tsx,md}\"'\n"
- name: Run Vitest tests
shell: sh
run: "set -eu\nNPM_CACHE_VOLUME=\"xiaoxia-npm-cache\"\nif ! docker volume inspect \"$NPM_CACHE_VOLUME\" >/dev/null 2>&1; then\n docker volume create \"$NPM_CACHE_VOLUME\" >/dev/null\n echo \"Created npm cache volume: $NPM_CACHE_VOLUME\"\nfi\n\ndocker run --rm \\\n -v \"$PWD:/workspace\" \\\n -v \"$NPM_CACHE_VOLUME:/workspace/apps/web/node_modules\" -w /workspace/apps/web \\\n docker.m.daocloud.io/library/node:20 \\\n sh -lc 'npx --no-install vitest run src/test'\n"
- name: Job duration summary
if: always()
shell: sh
run: "set +eu\nif [ -n \"$JOB_START_TIME\" ]; then\n END_TIME=$(date +%s)\n DURATION=$((END_TIME - JOB_START_TIME))\n MINS=$((DURATION / 60))\n SECS=$((DURATION % 60))\n echo \"JOB_DURATION_SECONDS=$DURATION\" >> $GITHUB_ENV\n echo \"=== Job Duration: ${MINS}m${SECS}s ===\"\nelse\n echo \"JOB_DURATION_SECONDS=0\" >> $GITHUB_ENV\n echo \"=== Job Duration: unknown ===\"\nfi\n"
- name: Notify on failure
continue-on-error: true
if: failure()
shell: sh
env:
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
run: 'set +e
NOTIFY_MODE=failure JOB_NAME="Frontend Lint" python3 scripts/ci_notify.py
'
-78
View File
@@ -1,78 +0,0 @@
name: CI Failure Monitor
on:
schedule:
- cron: '0 */6 * * *' # 每6小时检查一次
workflow_dispatch:
inputs:
days:
description: '统计最近N天的失败'
required: false
default: '7'
fail_threshold:
description: '失败次数阈值'
required: false
default: '3'
fail_rate_threshold:
description: '失败率阈值(%)'
required: false
default: '30'
permissions:
contents: read
jobs:
monitor:
name: CI重复失败检测
runs-on: ci-l2
timeout-minutes: 10
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" \
| bash
- name: Record job start time
shell: sh
run: bash scripts/ci/step_timer_start.sh
- name: Run failure detection
shell: sh
env:
GITEA_API_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
GITEA_URL: https://git.xiaoxiajianji.com
GITEA_REPO: xiaoxia/xiaoxia-saas
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
FAIL_CHECK_DAYS: ${{ inputs.days || 7 }}
FAIL_THRESHOLD: ${{ inputs.fail_threshold || 3 }}
FAIL_RATE_THRESHOLD: ${{ inputs.fail_rate_threshold || 30 }}
run: |
set +e
python3 scripts/ci/ci_repeated_failure_detector.py
EXIT_CODE=$?
echo "检测完成,退出码: $EXIT_CODE"
# 0=无异常, 1=有警告, 2=有严重问题
# 监控脚本永远不fail,避免告警风暴
exit 0
- name: Job duration summary
if: always()
shell: sh
run: bash scripts/ci/step_timer_end.sh
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
-103
View File
@@ -1,103 +0,0 @@
name: CI Health Daily Report
on:
schedule:
- cron: '0 1 * * *' # UTC 01:00 = 北京时间 09:00
workflow_dispatch:
permissions:
contents: read
jobs:
ci-health-report:
name: CI健康度每日巡检
runs-on: ci-l2
timeout-minutes: 15
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
set -eu
python3 - <<'PY'
import io, os, tarfile, time, urllib.request, urllib.error
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
last_err = None
for attempt in range(5):
try:
with urllib.request.urlopen(request, timeout=120) as response:
archive = response.read()
break
except urllib.error.HTTPError as e:
last_err = e
if e.code >= 500 and attempt < 4:
wait = 2 ** attempt
print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
time.sleep(wait)
continue
raise
except Exception as e:
last_err = e
if attempt < 4:
wait = 2 ** attempt
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
time.sleep(wait)
continue
raise
else:
raise last_err
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
for member in tar.getmembers():
name = member.name
if name == root_prefix[:-1]:
continue
if name.startswith(root_prefix):
member.name = name[len(root_prefix):]
if member.name:
tar.extract(member, '.')
PY
- name: Generate CI Dashboard HTML
shell: sh
env:
GITEA_TOKEN: ${{ github.token }}
run: |
set +e
echo "=== 生成 CI 健康度 HTML 看板 ==="
echo "时间: $(date '+%Y-%m-%d %H:%M:%S')"
echo ""
python3 scripts/ci/ci_dashboard.py --days 7 --html --html-output ci_dashboard.html
EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ] && [ -f ci_dashboard.html ]; then
HTML_SIZE=$(wc -c < ci_dashboard.html)
echo ""
echo "✅ HTML 看板生成成功 (${HTML_SIZE} bytes)"
echo "路径: $(pwd)/ci_dashboard.html"
# 输出文件内容前几行,方便在 Actions 日志中确认
echo ""
echo "--- 看板预览 (前 5 行) ---"
head -5 ci_dashboard.html
echo "...(完整内容见产物文件)"
else
echo "❌ HTML 看板生成失败 (exit code: $EXIT_CODE)"
fi
echo ""
# 永远成功,看板生成失败不影响主流程
exit 0
- name: Run CI health check and report
shell: sh
env:
GITEA_TOKEN: ${{ github.token }}
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
run: |
set +e
echo "=== CI健康度每日巡检 ==="
echo "时间: $(date '+%Y-%m-%d %H:%M:%S')"
echo ""
python3 scripts/ci/ci_health_report.py --limit 30
EXIT_CODE=$?
echo ""
echo "巡检完成 (exit code: $EXIT_CODE)"
# 永远成功,不影响CI状态(通知失败不应该标红)
exit 0
File diff suppressed because it is too large Load Diff
+4 -20
View File
@@ -2,13 +2,13 @@ name: CI Trigger Monitor
on:
schedule:
- cron: '*/10 * * * *' # 每10分钟检查一次(与pr-auto-scan同步降频)
- cron: '*/5 * * * *' # 每5分钟检查一次
workflow_dispatch:
inputs:
stale_threshold:
description: 'CI未触发告警阈值(分钟)'
required: false
default: '10'
default: '5'
permissions:
contents: read
@@ -16,15 +16,11 @@ permissions:
jobs:
monitor:
name: Monitor CI Trigger Reliability
runs-on: ci-l2
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
uses: actions/checkout@v3
- name: Check CI trigger status for all open PRs
env:
@@ -38,15 +34,3 @@ jobs:
python3 scripts/ci_trigger_monitor.py
# 监控脚本永远不fail,避免告警风暴
exit 0
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
+7 -33
View File
@@ -15,29 +15,18 @@ concurrency:
jobs:
code-review:
name: AI Code Review
runs-on: ci-l2
runs-on: ubuntu-latest
# 跳过草稿 PR
if: ${{ !gitea.event.pull_request.draft }}
steps:
# actions/checkout 由 runner 在宿主机层面处理,不受容器网络影响
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install dependencies
run: |
# 确保 python3-pip 可用(兼容不同基础镜像)
if ! python3 -m pip --version >/dev/null 2>&1; then
apt-get update -qq && apt-get install -y -qq python3-pip python3-venv >/dev/null 2>&1
fi
# 部分镜像 ensurepip 方式兜底
if ! python3 -m pip --version >/dev/null 2>&1; then
python3 -m ensurepip --upgrade 2>/dev/null || curl -sS https://bootstrap.pypa.io/get-pip.py | python3
fi
python3 -m pip install --upgrade pip
python3 -m pip install requests
@@ -48,7 +37,6 @@ jobs:
GITEA_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
REPO_NAME: ${{ gitea.repository }}
PR_NUMBER: ${{ gitea.event.pull_request.number }}
PR_HEAD_SHA: ${{ gitea.event.pull_request.head.sha }}
# LLM 提供商: coze (扣子原生Bot) / openai (OpenAI兼容)
LLM_PROVIDER: "coze"
# 扣子模式配置(默认国内站 api.coze.cn)
@@ -61,19 +49,5 @@ jobs:
LLM_TIMEOUT: "120"
run: |
python3 scripts/ci_code_review.py
# 注意:脚本退出码决定job状态
# - 有阻塞级问题 → exit 1 → job失败 → 门禁拦截
# - 无阻塞级问题/LLM异常 → exit 0 → 通过(fail-open)
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
# 审查脚本异常不影响 CI 通过
continue-on-error: true
+136 -94
View File
@@ -1,5 +1,4 @@
name: Daily Health Check
# 注意:使用 curl step_checkout.sh 方式以兼容 docker runner
on:
schedule:
@@ -13,7 +12,7 @@ jobs:
# ── 1. 生产环境冒烟测试 ─────────────────────────────────────────────
production-smoke:
name: Production Smoke Test
runs-on: ci-l2
runs-on: saas
timeout-minutes: 8
outputs:
report: ${{ steps.smoke.outputs.report }}
@@ -24,9 +23,47 @@ jobs:
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" \
| bash
set -eu
python3 - <<'PY'
import io, os, tarfile, time, urllib.request, urllib.error
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
last_err = None
for attempt in range(5):
try:
with urllib.request.urlopen(request, timeout=120) as response:
archive = response.read()
break
except urllib.error.HTTPError as e:
last_err = e
if e.code >= 500 and attempt < 4:
wait = 2 ** attempt
print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
time.sleep(wait)
continue
raise
except Exception as e:
last_err = e
if attempt < 4:
wait = 2 ** attempt
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
time.sleep(wait)
continue
raise
else:
raise last_err
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
for member in tar.getmembers():
name = member.name
if name == root_prefix[:-1]:
continue
if name.startswith(root_prefix):
member.name = name[len(root_prefix):]
if member.name:
tar.extract(member, '.')
PY
- name: Production health check & smoke test
id: smoke
shell: sh
@@ -69,25 +106,13 @@ jobs:
echo "======================================"
exit $SMOKE_EXIT
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
# ── 2. Staging API 集成测试 ─────────────────────────────────────────
staging-api-tests:
name: Staging API Integration Tests
runs-on: ci-l2
runs-on: saas
timeout-minutes: 10
outputs:
report: ${{ steps.report.outputs.report }}
report: ${{ steps.smoke.outputs.report }}
steps:
- name: Checkout code
@@ -95,15 +120,50 @@ jobs:
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" \
| bash
set -eu
python3 - <<'PY'
import io, os, tarfile, time, urllib.request, urllib.error
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
last_err = None
for attempt in range(5):
try:
with urllib.request.urlopen(request, timeout=120) as response:
archive = response.read()
break
except urllib.error.HTTPError as e:
last_err = e
if e.code >= 500 and attempt < 4:
wait = 2 ** attempt
print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
time.sleep(wait)
continue
raise
except Exception as e:
last_err = e
if attempt < 4:
wait = 2 ** attempt
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
time.sleep(wait)
continue
raise
else:
raise last_err
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
for member in tar.getmembers():
name = member.name
if name == root_prefix[:-1]:
continue
if name.startswith(root_prefix):
member.name = name[len(root_prefix):]
if member.name:
tar.extract(member, '.')
PY
- name: Run API smoke test on staging
id: smoke
shell: sh
env:
STAGING_TEST_USER: ${{ secrets.STAGING_TEST_USER }}
STAGING_TEST_PASSWORD: ${{ secrets.STAGING_TEST_PASSWORD }}
run: |
set +e
START_TIME=$(date +%s)
@@ -111,8 +171,8 @@ jobs:
docker run --rm \
-e BASE_URL=https://staging-api.xiaoxiajianji.com \
-e WEB_URL=https://staging.xiaoxiajianji.com \
-e TEST_USER="$STAGING_TEST_USER" \
-e TEST_PASSWORD="$STAGING_TEST_PASSWORD" \
-e TEST_USER=18314979086@163.com \
-e TEST_PASSWORD=Ying1234 \
-e CLEANUP_ENABLED=1 \
-e PERF_CHECK_ENABLED=1 \
-e PERF_WARN_THRESHOLD_MS=500 \
@@ -183,25 +243,13 @@ jobs:
echo "report=FAIL" >> "${GITHUB_OUTPUT}"
fi
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
# ── 3. Staging 浏览器 E2E ──────────────────────────────────────────
staging-e2e:
name: Staging Browser E2E
runs-on: ci-l2
runs-on: saas
timeout-minutes: 15
outputs:
report: ${{ steps.e2e.outputs.report }}
report: ${{ steps.smoke.outputs.report }}
steps:
- name: Checkout code
@@ -209,9 +257,47 @@ jobs:
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" \
| bash
set -eu
python3 - <<'PY'
import io, os, tarfile, time, urllib.request, urllib.error
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
last_err = None
for attempt in range(5):
try:
with urllib.request.urlopen(request, timeout=120) as response:
archive = response.read()
break
except urllib.error.HTTPError as e:
last_err = e
if e.code >= 500 and attempt < 4:
wait = 2 ** attempt
print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
time.sleep(wait)
continue
raise
except Exception as e:
last_err = e
if attempt < 4:
wait = 2 ** attempt
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
time.sleep(wait)
continue
raise
else:
raise last_err
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
for member in tar.getmembers():
name = member.name
if name == root_prefix[:-1]:
continue
if name.startswith(root_prefix):
member.name = name[len(root_prefix):]
if member.name:
tar.extract(member, '.')
PY
- name: Run Playwright E2E on staging
id: e2e
shell: sh
@@ -246,22 +332,10 @@ jobs:
echo "=========================================="
exit $EXIT_CODE
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
# ── 4. 性能基线巡检 ────────────────────────────────────────────────
performance-check:
name: Performance Baseline Check
runs-on: ci-l2
runs-on: saas
timeout-minutes: 8
outputs:
report: ${{ steps.report.outputs.report }}
@@ -270,9 +344,6 @@ jobs:
- name: Run performance baseline checks
id: perf
shell: sh
env:
STAGING_TEST_USER: ${{ secrets.STAGING_TEST_USER }}
STAGING_TEST_PASSWORD: ${{ secrets.STAGING_TEST_PASSWORD }}
run: |
set +e
START_TIME=$(date +%s)
@@ -308,10 +379,9 @@ jobs:
# 先登录获取 token
echo "--- 准备: 获取测试 Token ---"
LOGIN_BODY="{\"email\":\"$STAGING_TEST_USER\",\"password\":\"$STAGING_TEST_PASSWORD\"}"
AUTH_RESP=$(curl -s -w "\n%{http_code}" -X POST \
-H "Content-Type: application/json" \
-d "$LOGIN_BODY" \
-d '{"email":"18314979086@163.com","password":"Ying1234"}' \
"https://staging-api.xiaoxiajianji.com/api/v1/auth/login" \
--max-time 10 2>&1)
AUTH_CODE=$(echo "$AUTH_RESP" | tail -1)
@@ -341,7 +411,7 @@ jobs:
# 构建 curl 命令
CURL_ARGS="-s -o /dev/null -w '%{http_code} %{time_total}' --max-time 30"
if [ "$method" = "POST" ]; then
CURL_ARGS="$CURL_ARGS -X POST -H 'Content-Type: application/json' -d \"$LOGIN_BODY\""
CURL_ARGS="$CURL_ARGS -X POST -H 'Content-Type: application/json' -d '{\"email\":\"18314979086@163.com\",\"password\":\"Ying1234\"}'"
fi
if [ -n "$TOKEN" ] && [ "$name" != "健康检查" ]; then
CURL_ARGS="$CURL_ARGS -H 'Authorization: Bearer $TOKEN'"
@@ -389,9 +459,6 @@ jobs:
- name: Generate performance report
id: report
shell: sh
env:
STAGING_TEST_USER: ${{ secrets.STAGING_TEST_USER }}
STAGING_TEST_PASSWORD: ${{ secrets.STAGING_TEST_PASSWORD }}
run: |
set +e
echo ""
@@ -406,11 +473,10 @@ jobs:
RESULTS=""
START_TIME=$(date +%s)
LOGIN_BODY="{\"email\":\"$STAGING_TEST_USER\",\"password\":\"$STAGING_TEST_PASSWORD\"}"
# 先登录获取 token
AUTH_RESP=$(curl -s -w "\n%{http_code}" -X POST \
-H "Content-Type: application/json" \
-d "$LOGIN_BODY" \
-d '{"email":"18314979086@163.com","password":"Ying1234"}' \
"https://staging-api.xiaoxiajianji.com/api/v1/auth/login" \
--max-time 10 2>&1)
AUTH_CODE=$(echo "$AUTH_RESP" | tail -1)
@@ -426,7 +492,7 @@ jobs:
local CURL_ARGS="-s -o /dev/null -w '%{http_code} %{time_total}' --max-time 30"
if [ "$method" = "POST" ]; then
CURL_ARGS="$CURL_ARGS -X POST -H 'Content-Type: application/json' -d \"$LOGIN_BODY\""
CURL_ARGS="$CURL_ARGS -X POST -H 'Content-Type: application/json' -d '{\"email\":\"18314979086@163.com\",\"password\":\"Ying1234\"}'"
fi
if [ -n "$TOKEN" ] && [ "$name" != "健康检查" ]; then
CURL_ARGS="$CURL_ARGS -H 'Authorization: Bearer $TOKEN'"
@@ -514,22 +580,10 @@ jobs:
exit 0
fi
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
# ── 5. 每日巡检汇总报告 ────────────────────────────────────────────
daily-report:
name: Daily Check Report
runs-on: ci-l2
runs-on: saas
timeout-minutes: 2
if: always()
needs:
@@ -602,15 +656,3 @@ jobs:
# 不 exit 1,因为我们用了 always(),保持 report job 成功,
# 但其他失败的 job 已经让整体流水线标记为失败
fi
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
-60
View File
@@ -1,60 +0,0 @@
name: "Debug: Web container v2 (mount conflict)"
on:
push:
branches: [debug/web-crash-v2]
workflow_dispatch:
jobs:
web-diag:
runs-on: runtime-builder
timeout-minutes: 10
steps:
- name: Setup SSH and diagnose
shell: bash
env:
STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
run: |
set -x
which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
mkdir -p ~/.ssh && chmod 700 ~/.ssh
printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
H=47.98.113.167; P=22222
ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null
ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H 'bash -s' <<'REMOTE'
set -x
echo "=== Current staging containers ==="
docker ps -a --filter name=xiaoxia-*-staging --format "table {{.Names}}\t{{.Status}}\t{{.Image}}"
echo ""
echo "=== Web container logs (current/current-rolledback) ==="
docker logs xiaoxia-web-staging 2>&1 | tail -40
echo ""
echo "=== Web inspect: env & mounts ==="
docker inspect xiaoxia-web-staging --format 'Entrypoint: {{.Config.Entrypoint}} Cmd: {{.Config.Cmd}}'
docker inspect xiaoxia-web-staging --format '{{range .Config.Env}}{{.}}{{"\n"}}{{end}}' | grep -E "APP_ENV|VERSION"
echo "Mounts:"
docker inspect xiaoxia-web-staging --format '{{range .Mounts}}{{.Type}} {{.Source}} -> {{.Destination}} (rw={{.RW}}){{"\n"}}{{end}}'
echo ""
echo "=== Reproduce: rm on read-only bind mount ==="
docker run --rm --name nginx-ro-test \
-v /var/lib/xiaoxia-saas-staging/nginx-staging.conf:/etc/nginx/conf.d/default.conf:ro \
git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/xiaoxia-saas-web:387514c \
sh -c '
set -x
echo "Before:"
ls -la /etc/nginx/conf.d/
echo "Try rm (as entrypoint does):"
rm -f /etc/nginx/conf.d/default.conf
echo "rm exitcode=$?"
echo "After rm:"
ls -la /etc/nginx/conf.d/
echo "Test ln:"
ln -s /etc/nginx/nginx-staging.conf /etc/nginx/conf.d/default.conf
echo "ln exitcode=$?"
ls -la /etc/nginx/conf.d/
echo "nginx -t:"
nginx -t 2>&1
' 2>&1
echo ""
echo "=== Also test with NEW fixed image (9c0d4b1 if present) ==="
docker images | grep xiaoxia-saas-web | head -5
REMOTE
@@ -1,59 +0,0 @@
name: Playwright Base Image Build
on:
workflow_dispatch:
inputs:
reason:
description: "触发原因"
required: false
default: "构建 playwright 基础镜像"
jobs:
build-playwright:
name: Build Playwright Base Image
runs-on: runtime-builder
timeout-minutes: 30
steps:
- name: Docker login to Gitea Registry
shell: sh
env:
GITEA_REGISTRY_USER: xiaoxia
GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
run: |
set -eu
for i in 1 2 3; do
echo "=== Docker login attempt $i/3 ==="
if printf '%s' "${GITEA_REGISTRY_TOKEN}" | docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" --password-stdin; then
echo "Docker login successful"
break
fi
echo "Docker login failed (attempt $i/3), retrying in 5s..."
sleep 5
[ $i -eq 3 ] && exit 1
done
- name: Pull, retag and push Playwright image
shell: sh
run: |
set -eu
OFFICIAL_IMAGE="mcr.microsoft.com/playwright:v1.45.0-jammy"
GITEA_IMAGE="git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy"
echo "=== Pulling official Playwright image ==="
docker pull "${OFFICIAL_IMAGE}"
echo "=== Tagging ==="
docker tag "${OFFICIAL_IMAGE}" "${GITEA_IMAGE}"
echo "=== Pushing to Gitea Registry ==="
docker push "${GITEA_IMAGE}"
echo "Done: ${GITEA_IMAGE}"
- name: Cleanup
if: always()
shell: sh
run: |
docker rmi "mcr.microsoft.com/playwright:v1.45.0-jammy" 2>/dev/null || true
docker rmi "git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy" 2>/dev/null || true
echo "Cleanup done"
-56
View File
@@ -1,56 +0,0 @@
name: PR Auto Scan
# 定时扫描所有open PR,对CI全绿的触发审批/合并
# 作为短作业模式的兜底,防止事件驱动遗漏
on:
schedule:
# - cron: "*/15 * * * *" # DISABLED: temporarily to stop failure spam (2026-09-02) # 每10分钟扫描一次(脚本自带240s墙钟上限,降频减负)
workflow_dispatch:
permissions:
contents: read
jobs:
auto-scan:
name: Auto Scan Open PRs
runs-on: ci-check
timeout-minutes: 5
if: github.repository == 'xiaoxia/xiaoxia-saas'
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/pr_auto_scan.py?ref=develop" -o /tmp/pr_auto_scan.py
python3 /tmp/pr_auto_scan.py --help > /dev/null 2>&1 || {
# fallback: checkout
echo "使用checkout方式"
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=develop" | bash
}
- name: Scan and auto process PRs
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
REVIEW_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
MERGE_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
run: |
set -eu
echo "=== 扫描所有open PR并自动处理 ==="
echo "时间: $(date)"
echo
python3 /tmp/pr_auto_scan.py --token "$REVIEW_TOKEN" --repo "$GITHUB_REPOSITORY" --base develop --approve --merge --dry-run false
echo ""
echo "✅ 扫描完成"
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "" || true
-119
View File
@@ -1,119 +0,0 @@
name: PR Automation
on:
pull_request:
types: [synchronize, opened, ready_for_review, review_requested]
workflow_dispatch:
permissions:
contents: read
concurrency:
group: pr-automation-${{ gitea.event.pull_request.number }}
cancel-in-progress: true
jobs:
auto-approve:
name: Auto Approve on CI Green
runs-on: ci-check
if: github.event_name == 'pull_request' && !github.event.pull_request.draft
timeout-minutes: 10 # 等待CI全绿+审批,需要充足时间
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
- name: "🔍 脚本语法自检"
shell: bash
run: |
ERROR=0
for f in scripts/ci/*.sh; do [ -f "$f" ] && bash -n "$f" 2>&1 || ERROR=$((ERROR+1)); done
for f in scripts/ci/*.py; do [ -f "$f" ] && python3 -m py_compile "$f" 2>&1 || ERROR=$((ERROR+1)); done
if [ "$ERROR" -ne 0 ]; then echo "❌ 语法自检失败 ($ERROR个)"; exit 1; fi
echo "✅ 脚本语法自检通过"
- name: Auto approve when CI passes
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
REVIEW_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
bash scripts/ci/auto_approve.sh
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
auto-merge:
name: Auto Merge on CI Green + Approved
runs-on: ci-check
if: github.event_name == 'pull_request' && !github.event.pull_request.draft && github.event.pull_request.base.ref == 'develop'
needs: [auto-approve] # 修复竞态:必须等审批完成后再尝试合并
timeout-minutes: 15 # 等待审批+CI就绪+合并,需要充足时间
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
- name: "🔍 脚本语法自检(防止脚本bug导致所有PR挂掉)"
shell: bash
run: |
echo "=== CI脚本语法自检 ==="
ERROR=0
for f in scripts/ci/*.sh; do
[ -f "$f" ] || continue
if ! bash -n "$f" 2>&1; then
echo "FAIL: $f"
ERROR=1
fi
done
for f in scripts/ci/*.py; do
[ -f "$f" ] || continue
if ! python3 -m py_compile "$f" 2>&1; then
echo "FAIL: $f"
ERROR=1
fi
done
if [ "$ERROR" -ne 0 ]; then
echo "❌ 脚本语法自检失败"
exit 1
fi
echo "✅ 所有CI脚本语法自检通过"
- name: Auto merge when CI passes and approved
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
MERGE_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
BASE_REF: ${{ github.event.pull_request.base.ref }}
run: |
bash scripts/ci/auto_merge.sh
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
+8 -66
View File
@@ -15,52 +15,6 @@ jobs:
runs-on: runtime-builder
timeout-minutes: 10
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
set -eu
python3 - <<'PY'
import io, os, tarfile, time, urllib.request, urllib.error
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
last_err = None
for attempt in range(5):
try:
with urllib.request.urlopen(request, timeout=120) as response:
archive = response.read()
break
except urllib.error.HTTPError as e:
last_err = e
if e.code >= 500 and attempt < 4:
wait = 2 ** attempt
print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
time.sleep(wait)
continue
raise
except Exception as e:
last_err = e
if attempt < 4:
wait = 2 ** attempt
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
time.sleep(wait)
continue
raise
else:
raise last_err
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
for member in tar.getmembers():
name = member.name
if name == root_prefix[:-1]:
continue
if name.startswith(root_prefix):
member.name = name[len(root_prefix):]
if member.name:
tar.extract(member, '.')
PY
- name: Extract PR number
shell: sh
run: |
@@ -120,26 +74,26 @@ jobs:
PREVIEW_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
run: |
set -eux
preview_host="${PREVIEW_SSH_HOST:-47.98.113.167}"
preview_host="${PREVIEW_SSH_HOST:-172.30.18.197}"
preview_user="${PREVIEW_SSH_USER:-deploy}"
preview_port="${PREVIEW_SSH_PORT:-22222}"
preview_dir="/var/www/preview/pr-${PR_NUMBER}"
mkdir -p ~/.ssh
# 查找可用的SSH密钥(优先用 secret 里专门为 preview 配置的 key)
# 查找可用的SSH密钥
key_path=""
if [ -n "${PREVIEW_SSH_KEY:-}" ]; then
key_path="$HOME/.ssh/id_ed25519"
printf '%s\n' "$PREVIEW_SSH_KEY" > "$key_path"
chmod 600 "$key_path"
echo "Using key from PREVIEW_SSH_KEY secret"
elif [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
key_path="/root/.ssh/xiaoxia_runtime_builder"
echo "Using key: $key_path (builder key)"
elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then
key_path="$HOME/.ssh/xiaoxia_runtime_builder"
echo "Using key: $key_path (home key)"
elif [ -n "${PREVIEW_SSH_KEY:-}" ]; then
key_path="$HOME/.ssh/id_ed25519"
printf '%s\n' "$PREVIEW_SSH_KEY" > "$key_path"
chmod 600 "$key_path"
echo "Using key from PREVIEW_SSH_KEY secret"
else
echo "ERROR: No SSH key available"
ls -la ~/.ssh/ 2>/dev/null || true
@@ -193,15 +147,3 @@ jobs:
> /dev/null
echo "Cleanup comment posted"
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
+46 -52
View File
@@ -93,28 +93,44 @@ jobs:
shell: sh
run: |
set -eu
cd apps/web
NPM_CACHE_VOLUME="xiaoxia-npm-cache"
if ! docker volume inspect "$NPM_CACHE_VOLUME" >/dev/null 2>&1; then
docker volume create "$NPM_CACHE_VOLUME" >/dev/null
echo "Created npm cache volume: $NPM_CACHE_VOLUME"
fi
# Install dependencies with retry
for i in 1 2 3; do
npm ci --registry=https://registry.npmmirror.com --no-audit --no-fund && break
echo "npm install failed, retry $i/3..."
[ $i -eq 3 ] && exit 1
rm -rf node_modules
sleep 5
done
# TypeScript check
echo "=== TypeScript check ==="
./node_modules/.bin/tsc --noEmit
# Vite build
echo "=== Vite build ==="
export VITE_API_URL=https://staging-api.xiaoxiajianji.com
./node_modules/.bin/vite build
echo "=== Build completed ==="
ls -la dist/
docker run --rm \
-v "$PWD:/workspace" \
-v "$NPM_CACHE_VOLUME:/workspace/apps/web/node_modules" \
-w /workspace/apps/web \
-e VITE_API_URL=https://staging-api.xiaoxiajianji.com \
docker.m.daocloud.io/library/node:20 \
sh -lc '
PACKAGE_LOCK_HASH=$(md5sum package-lock.json 2>/dev/null | cut -d" " -f1)
CACHE_HASH_FILE="node_modules/.package-lock-hash"
CACHE_VALID=false
if [ -f "$CACHE_HASH_FILE" ] && [ "$(cat "$CACHE_HASH_FILE")" = "$PACKAGE_LOCK_HASH" ] && [ -x "node_modules/.bin/vite" ] && [ -x "node_modules/.bin/tsc" ]; then
CACHE_VALID=true
echo "Cache hit: dependencies valid, skipping npm ci"
fi
if [ "$CACHE_VALID" = "false" ]; then
echo "Cache miss or invalid: running npm ci..."
if ! npm ci --include=dev; then
echo "npm ci failed, cleaning node_modules and retrying..."
rm -rf node_modules
mkdir -p node_modules
npm ci --include=dev
fi
echo "$PACKAGE_LOCK_HASH" > "$CACHE_HASH_FILE"
echo "Dependencies installed, cache updated"
fi
echo "Running TypeScript check..."
npx --no-install tsc
echo "Running Vite build..."
npx --no-install vite build
echo "Build completed successfully"
ls -la dist/
'
- name: Install SSH client and rsync
shell: sh
@@ -141,26 +157,26 @@ jobs:
PREVIEW_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
run: |
set -eux
preview_host="${PREVIEW_SSH_HOST:-47.98.113.167}"
preview_user="${PREVIEW_SSH_USER:-root}"
preview_host="${PREVIEW_SSH_HOST:-172.30.18.197}"
preview_user="${PREVIEW_SSH_USER:-deploy}"
preview_port="${PREVIEW_SSH_PORT:-22222}"
preview_dir="/var/www/preview/pr-${PR_NUMBER}"
mkdir -p ~/.ssh
# 查找可用的SSH密钥(优先用 secret 里专门为 preview 配置的 key)
# 查找可用的SSH密钥
key_path=""
if [ -n "${PREVIEW_SSH_KEY:-}" ]; then
key_path="$HOME/.ssh/id_ed25519"
printf '%s\n' "$PREVIEW_SSH_KEY" > "$key_path"
chmod 600 "$key_path"
echo "Using key from PREVIEW_SSH_KEY secret"
elif [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
key_path="/root/.ssh/xiaoxia_runtime_builder"
echo "Using key: $key_path (builder key)"
elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then
key_path="$HOME/.ssh/xiaoxia_runtime_builder"
echo "Using key: $key_path (home key)"
elif [ -n "${PREVIEW_SSH_KEY:-}" ]; then
key_path="$HOME/.ssh/id_ed25519"
printf '%s\n' "$PREVIEW_SSH_KEY" > "$key_path"
chmod 600 "$key_path"
echo "Using key from PREVIEW_SSH_KEY secret"
else
echo "ERROR: No SSH key available"
ls -la ~/.ssh/ 2>/dev/null || true
@@ -168,16 +184,6 @@ jobs:
exit 1
fi
# SSH密钥完整性自检
if ! ssh-keygen -y -f "$key_path" > /dev/null 2>&1; then
echo "ERROR: SSH密钥损坏(private key contents do not match public)"
echo "请检查 PREVIEW_SSH_KEY secret 中的私钥是否完整正确"
echo "私钥文件大小: $(wc -c < "$key_path") 字节"
head -2 "$key_path"
exit 1
fi
echo "SSH key integrity check passed"
ssh-keyscan -p "$preview_port" -H "$preview_host" >> ~/.ssh/known_hosts 2>/dev/null
echo "SSH keyscan done"
@@ -266,15 +272,3 @@ jobs:
run: |
set +e
NOTIFY_MODE=failure JOB_NAME="Deploy Preview Environment" python3 scripts/ci_notify.py
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
-86
View File
@@ -1,86 +0,0 @@
name: Worker Base Image Build
on:
push:
branches:
- develop
- main
paths:
- 'requirements-base.txt'
- 'requirements.txt'
- 'requirements-worker.txt'
- 'infra/docker/worker-base.Dockerfile'
workflow_dispatch:
jobs:
build-worker-base:
name: Build Worker Base Image
runs-on: runtime-builder
timeout-minutes: 45
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" \
| bash
- name: Docker login to Registry
shell: sh
env:
ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
GITEA_REGISTRY_USER: xiaoxia
GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
run: |
set -eu
for i in 1 2 3; do
echo "=== Docker login 尝试 $i/3 ==="
if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin \
&& docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then
echo "✅ Docker login successful"
break
fi
echo "❌ Docker login 失败(尝试 $i/3),5s 后重试..."
sleep 5
done
- name: Build and push Worker base image
shell: sh
run: |
set -eu
ACR_IMAGE="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/saas-worker-base:latest"
GITEA_IMAGE="git.xiaoxiajianji.com/xiaoxia-saas/saas-worker-base:latest"
echo "=== Building Worker base image ==="
# 使用普通 docker build(单平台不需要 buildx)
docker build \
-f infra/docker/worker-base.Dockerfile \
-t "${ACR_IMAGE}" \
.
echo ""
echo "✅ Image built successfully"
# 推送到 ACR
echo "=== Pushing to ACR ==="
docker push "${ACR_IMAGE}"
echo "✅ Pushed to ACR"
# 打标签并推送到 Gitea Packages 作为备份
echo "=== Pushing to Gitea Packages ==="
docker tag "${ACR_IMAGE}" "${GITEA_IMAGE}"
docker push "${GITEA_IMAGE}" || echo "⚠️ Gitea Packages push failed (non-fatal)"
echo "✅ Gitea backup push completed"
- name: Cleanup
if: always()
shell: sh
run: |
ACR_IMAGE="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/saas-worker-base:latest"
docker rmi "${ACR_IMAGE}" 2>/dev/null || true
docker image prune -f 2>/dev/null || true
echo "Cleanup done"
-7
View File
@@ -24,11 +24,6 @@ ruff_cache/
.env.production
.env.staging
!.env.example
# 配置模板不受忽略规则限制
!deploy/configs/.env.staging
!deploy/configs/.env.production
# 渲染后的 env 文件包含真实密钥,绝不能提交
.env.rendered
# OS / editor
.DS_Store
@@ -58,5 +53,3 @@ frontend-v21-ui-prototype-final.html
!.vscode/
!.vscode/settings.json
.vscode/extensions.json
.coverage
.env.current
-9
View File
@@ -18,12 +18,3 @@ repos:
- id: ruff
args: [--fix]
language_version: python3.12
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v4.0.0-alpha.11
hooks:
- id: prettier
name: prettier (frontend)
files: ^apps/web/.*\.(ts|tsx|js|jsx|css|scss|less|json|html|md|yaml|yml)$
additional_dependencies:
- prettier@3.4.2
Vendored Executable → Regular
-37
View File
@@ -6,43 +6,6 @@
"source.organizeImports": "explicit"
}
},
"[typescript]": {
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true
},
"[typescriptreact]": {
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true
},
"[javascript]": {
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true
},
"[javascriptreact]": {
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true
},
"[json]": {
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true
},
"[css]": {
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true
},
"[scss]": {
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true
},
"[html]": {
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true
},
"[markdown]": {
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true
},
"prettier.requireConfig": true,
"isort.args": ["--profile", "black"],
"python.linting.ruffEnabled": true,
"python.analysis.typeCheckingMode": "basic"
-222
View File
@@ -1,222 +0,0 @@
---
AIGC:
Label: "1"
ContentProducer: 001191110102MACQD9K64018705
ProduceID: 15868733686388_0/project_7655981463858544923-files/docs/1197_preview_generation_proposal.md
ReservedCode1: ""
ContentPropagator: 001191110102MACQD9K64028705
PropagateID: 15868733686388#1785468313901
ReservedCode2: ""
---
# #1197 预览生成接口方案评估
## 背景
智能剪辑「一键生成」流程中,第3步预览生成当前被跳过,直接进入下一步。需要实现真正的预览生成功能,让用户在正式生成前能看到效果预览。
## 现状分析
### 现有生成链路
```
API 触发生成 → GenerationTask入库 → Celery异步任务 → UnifiedRenderService渲染 → OSS上传 → 更新状态
```
**关键节点:**
1. **API层**:`POST /generation-tasks` 或 `POST /templates/{id}/generate` 触发生成
2. **任务调度**:Celery task `worker.generate_video`
3. **渲染引擎**:`UnifiedRenderService`(统一渲染引擎,已接入9个效果层)
4. **输出配置**:默认 720p (1280x720),支持 `resolution` 字段自定义
5. **产物存储**:`GeneratedVideo` 表记录,OSS 存储视频文件
### 已有可复用能力
| 能力 | 位置 | 是否可复用 |
|------|------|-----------|
| 任务创建与状态管理 | `GenerationTask` + `CreateGenerationTaskUseCase` | ✅ 是 |
| 素材下载与预处理 | `_download_video_assets` / `_download_voice_asset` | ✅ 是 |
| 统一渲染引擎 | `UnifiedRenderService` | ✅ 是 |
| 分辨率配置 | `resolution` 字段已支持 | ✅ 是 |
| 混音与后处理 | `_render_video` 内流程 | ✅ 是 |
| OSS 上传与查重 | `_upload_and_dedup` | ✅ 是 |
| 进度追踪 | `append_log` / `progress` 字段 | ✅ 是 |
## 方案对比
### 方案A:复用现有生成链路 + is_preview 标记(推荐)
**思路**:在现有 GenerationTask 上加 `is_preview` 标记,预览生成走完整链路但参数降级。
**改动点:**
1. **数据模型**:`GenerationTask` 加 `is_preview: bool` 字段(默认 false);`GeneratedVideo` 加 `is_preview: bool`
2. **API 层**:生成接口加 `is_preview` 参数,预览任务不计入配额
3. **渲染参数**:预览模式下自动调整
- 分辨率:480p (854x480)
- 时长:限制前 15 秒(或模板第一个片段)
- 码率:降低至 1.5Mbps(正式 4Mbps)
- 效果层:跳过高级转场/粒子特效等耗时效果
4. **任务调度**:预览任务走低优先级队列(或复用现有队列,标记优先级)
5. **前端对接**:预览生成结果带 `is_preview=true` 标记,前端展示"预览"标签
**优点:**
- 代码复用率 90%+,改动最小
- 与正式生成逻辑一致,预览效果真实可信
- 进度查询、结果展示等功能直接复用
- 后续可平滑升级:预览满意后一键转正式生成
**缺点:**
- 需要区分预览和正式任务,避免数据混淆
- 预览任务和正式任务竞争同一队列资源(可后续优化为独立队列)
**开发量估算**:2-3 天
- 数据模型 + 迁移:0.5 天
- API 层改造:0.5 天
- 渲染参数降级:1 天
- 测试 + 联调:1 天
---
### 方案B:新建独立预览接口 + 轻量渲染逻辑
**思路**:新建独立的预览生成接口,使用简化的渲染逻辑(如只拼接素材+基础配音,跳过大部分效果)。
**改动点:**
1. 新增 `PreviewTask` 数据模型
2. 新增 `POST /api/v1/preview/generate` 接口
3. 新增独立的 Celery task `worker.generate_preview`
4. 简化渲染流程:只做素材裁剪+拼接+配音,跳过转场/滤镜/字幕特效等
**优点:**
- 完全隔离,不影响正式生成链路
- 可以做极致优化,预览生成速度快
- 数据模型清晰,不会混淆
**缺点:**
- 代码重复率高,两套生成逻辑维护成本翻倍
- 预览效果与正式生成可能不一致(效果层差异)
- 前端需要对接两套接口
- 无法从预览升级为正式生成(需重新走完整流程)
**开发量估算**:4-5 天
- 数据模型 + 接口:1 天
- 简化渲染逻辑:2 天
- 测试 + 联调:1-2 天
---
### 方案C:图片预览(首帧/关键帧截图)
**思路**:不生成视频,只生成几张关键帧的预览图片。
**优点:**
- 生成速度极快(秒级)
- 资源消耗小
**缺点:**
- 预览效果差,用户无法感知动态效果
- 无法验证配音、转场、节奏等时间维度的效果
- 用户体验不佳,不如"真预览"有说服力
**开发量估算**:1-2 天
---
## 推荐方案:方案A(复用现有生成链路)
### 核心理由
1. **效果保真**:预览和正式生成用同一套渲染引擎,效果一致,用户信任度高
2. **开发效率**:90% 代码复用,2-3 天可上线
3. **可扩展性强**:后续可加「预览转正式」「低分辨率快速预览」等增强功能
4. **维护成本低**:一套生成逻辑,bug 修复和新功能同时生效
### 详细设计
#### 1. 数据模型变更
```python
# GenerationTask 新增字段
is_preview: bool = False
"""是否为预览生成"""
preview_of: str = ""
"""预览对应的正式任务 ID(或反向关联)"""
# GeneratedVideo 新增字段
is_preview: bool = False
"""是否为预览视频"""
```
**迁移**:alembic 新增 migration,两个表各加 1-2 个字段。
#### 2. API 层
```
POST /api/v1/generation-tasks
Body 增加 is_preview: bool = false
POST /api/v1/templates/{id}/generate
Query 增加 is_preview: bool = false
```
**配额处理**:预览生成不计入用户配额,不占用生成次数限制。
#### 3. 渲染参数降级
| 参数 | 正式生成 | 预览生成 |
|------|---------|---------|
| 分辨率 | 720p (1280x720) | 480p (854x480) |
| 码率 | 4 Mbps | 1.5 Mbps |
| 时长 | 完整时长 | 前 15 秒(或第一段) |
| 帧率 | 30 fps | 24 fps |
| 转场效果 | 完整转场 | 仅淡入淡出(或简单切) |
| 特效滤镜 | 全部启用 | 跳过粒子/光效等高级效果 |
| 字幕 | 完整渲染 | 正常渲染(字幕是核心信息) |
| 配音 | 完整混音 | 正常混音(配音是核心信息) |
**实现方式**:在 `_render_video` 或 UnifiedRenderService 入口处,根据 `is_preview` 标记调整渲染配置。
#### 4. 任务调度
- 初期复用现有队列,预览任务正常排队
- 后续如需优化,可拆分独立预览队列(低优先级)
- 预览任务可设置较短超时时间
#### 5. 前端对接
- 调用生成接口时传 `is_preview=true`
- 结果列表中预览视频带「预览」标签
- 预览满意后可一键「升级为正式生成」(重新触发全分辨率生成,可复用素材下载缓存)
### 实施步骤
**Phase 1(MVP,2天):**
1. 数据模型 + 迁移
2. API 层支持 is_preview 参数
3. 渲染分辨率降级(480p)
4. 不计入配额
5. 基础测试
**Phase 2(优化,1-2天):**
1. 时长限制(前15秒)
2. 效果层降级(跳高级效果)
3. 预览任务低优先级队列
4. 预览转正式生成功能
## 与前端对齐点
1. 预览生成的触发时机(第3步自动生成?用户点击才生成?)
2. 预览时长是固定15秒还是完整但低清?
3. 是否需要「预览转正式生成」功能
4. 预览视频的展示形态(和正式视频一样还是有特殊UI)
## 风险与注意事项
1. **数据混淆**:确保统计、计费、列表展示时正确区分预览和正式任务
2. **存储成本**:预览视频也占 OSS 空间,可设置自动清理(7天后自动删除)
3. **用户预期**:要明确告诉用户这是预览,效果和正式生成一致但清晰度低
4. **并发压力**:如果用户频繁生成预览,可能增加系统负载,需要限流
---
> 本内容由 Coze AI 生成,请遵循相关法律法规及《人工智能生成合成内容标识办法》使用与传播。
-382
View File
@@ -1,382 +0,0 @@
# #1197 预览生成接口技术方案(v2)
> 更新说明:v2 新增「多版本预览生成」能力,支持一个模板生成多个不重复的预览视频,左侧列表展示,用户可挑选满意的版本转正式生成。
## 1. 背景与目标
**现状**:智能剪辑「一键生成」第3步预览生成被跳过,用户直接进入正式生成,缺少效果预览环节。
**目标**:
1. ✅ 实现真正的预览生成(低分辨率快速出片)
2. ✅ **支持生成 1~N 个不重复的预览版本**(默认 3 个),左侧列表展示
3. ✅ 预览满意后可一键转正式生成(复用素材下载缓存)
4. ✅ 不计入用户配额,不占用正式生成次数
---
## 2. 现有生成链路分析
### 2.1 链路总览
```
API 触发生成 → GenerationTask入库 → Celery异步任务
→ 下载素材 → 构建plan/clips → UnifiedRenderService渲染
→ 混音后处理 → OSS上传 + 查重 → 更新状态
```
### 2.2 决定视频差异的变量
要做"多个不重复版本",先分析哪些环节可以引入变化:
| 变量 | 当前行为 | 能否引入变化 | 影响程度 |
|------|---------|------------|---------|
| 素材选择 | 按 asset_ids 顺序全用 | ✅ 可随机选择子集/不同组合 | 大 |
| 素材排序 | 按 asset_ids 顺序 | ✅ 可 shuffle 重排 | 大 |
| 配音选择 | 固定 voice_library_id | ✅ 可选不同音色 | 中 |
| 标题选择 | 固定 title_ids 或随机选 | ✅ 可选不同标题 | 中 |
| BGM | 固定 bgm_config | ✅ 可选不同BGM | 小 |
| 转场效果 | 模板固定 | ✅ 可随机化转场类型 | 小 |
| 播放速度 | 模板固定 | ✅ 可微调速度 | 小 |
| 分辨率/码率 | 固定 | ✅ 预览可降级 | 不影响内容 |
### 2.3 可复用能力
- 任务创建与状态管理:`GenerationTask` + `CreateGenerationTaskUseCase`
- 素材下载与预处理:`_download_all_assets`
- 统一渲染引擎:`UnifiedRenderService`
- 分辨率配置:`resolution` 字段已支持
- 批量任务:`batch_id` 字段已存在(可用于预览组)
---
## 3. 总体方案:复用现有链路 + 多变体引擎
**核心思路**:沿用 v1 的"复用现有生成链路 + is_preview 标记"方案,在此基础上增加「多版本生成」能力。
**架构**:
```
预览生成请求(count=N)
↓
创建预览批次(preview_batch)
↓
变体引擎生成 N 个变体参数(variation seed + 参数组合)
↓
为每个变体创建 1 个 GenerationTask(is_preview=true)
↓
N 个 Celery 任务并行执行(走现有生成链路,参数降级)
↓
N 个结果汇聚,前端左侧列表展示
```
---
## 4. 详细设计
### 4.1 数据模型变更
#### 4.1.1 GenerationTask 新增字段
```python
# 现有字段保留,新增:
is_preview: bool = False
"""是否为预览生成"""
preview_batch_id: str = ""
"""预览批次 ID(同批次的 N 个预览共享一个 batch)"""
variant_seed: int = 0
"""变体种子,用于控制随机化行为(素材选择、排序、转场等)"""
variant_params: dict = field(default_factory=dict)
"""变体参数快照(记录本次使用了哪些素材、标题、配音等,可追溯)
{
"asset_ids": [...], # 实际选用的素材子集
"title_id": "", # 选用的标题
"voice_id": "", # 选用的配音
"transition_style": "", # 转场风格
"bgm_track": "", # BGM 音轨
}
"""
```
#### 4.1.2 GeneratedVideo 新增字段
```python
is_preview: bool = False
"""是否为预览视频"""
preview_batch_id: str = ""
"""所属预览批次"""
variant_index: int = 0
"""在批次中的序号(0, 1, 2...)"""
```
#### 4.1.3 迁移方案
alembic 新增 migration,两个表各加 4 个字段,默认值为空/false,无数据回填成本。
---
### 4.2 变体引擎(Variant Engine)
**核心组件**:根据 count 和 seed,生成 N 组互不相同的生成参数。
#### 4.2.1 变纬度设计
| 维度 | 策略 | 说明 |
|------|------|------|
| **素材子集选择** | 从素材池中随机选 M 个(M=min(素材数, 模板clip数*2)) | 版本差异最大的来源 |
| **素材排序** | 随机打乱顺序 | 影响叙事节奏 |
| **标题选择** | 从 title_ids 中随机选 1 个 | 影响文案内容 |
| **配音选择** | 从 voice_ids 中随机选 1 个(如有多个) | 影响听觉体验 |
| **转场风格** | 从预设转场池中随机选 1 种 | 影响视觉过渡 |
| **BGM 选择** | 从 bgm 列表中随机选 1 首(如有配置) | 影响氛围 |
#### 4.2.2 去重机制
- 同一批次内,变体参数必须两两不同(至少素材组合或排序不同)
- 使用 `variant_seed` 保证可复现(相同 seed → 相同变体)
- 如果素材数量不足导致无法生成 N 个不同版本,按实际能生成的数量返回
#### 4.2.3 接口设计
```python
def generate_variants(
count: int,
seed: int,
asset_pool: list[str], # 可用素材 ID 列表
title_pool: list[str] = [], # 可用标题 ID 列表
voice_pool: list[str] = [], # 可用配音 ID 列表
template_id: str = "",
) -> list[dict]:
"""
生成 count 组变体参数。
每组参数包含:asset_ids(选用的素材+排序)、title_id、voice_id、
transition_style 等,确保两两不同。
"""
```
---
### 4.3 API 层设计
#### 4.3.1 预览生成接口
```
POST /api/v1/templates/{template_id}/generate-preview
```
**请求体**:
```json
{
"asset_library_id": "lib_xxx",
"asset_ids": ["asset_1", "asset_2", ...],
"title_ids": ["title_1", "title_2"],
"voice_ids": ["voice_1", "voice_2"],
"bgm_config": {},
"count": 3,
"seed": 0
}
```
| 参数 | 类型 | 必填 | 默认 | 说明 |
|------|------|------|------|------|
| template_id | path | ✅ | - | 模板 ID |
| asset_library_id | body | ✅ | - | 素材库 ID |
| asset_ids | body | ✅ | - | 素材池(从中选子集/排序) |
| title_ids | body | - | [] | 标题池(可选,不传则不用标题) |
| voice_ids | body | - | [] | 配音池(可选) |
| bgm_config | body | - | {} | BGM 配置 |
| count | body | - | 3 | 生成几个预览版本(1~10) |
| seed | body | - | 0 | 随机种子,0 表示随机 |
**响应**:
```json
{
"preview_batch_id": "pb_xxx",
"count": 3,
"tasks": [
{
"task_id": "gen_xxx_0",
"variant_index": 0,
"status": "processing"
},
{
"task_id": "gen_xxx_1",
"variant_index": 1,
"status": "processing"
},
...
]
}
```
#### 4.3.2 预览批次查询接口
```
GET /api/v1/preview-batches/{batch_id}
```
返回批次内所有预览任务的状态、结果(已完成的带 video_url)。
**响应**:
```json
{
"preview_batch_id": "pb_xxx",
"count": 3,
"completed_count": 2,
"tasks": [
{
"task_id": "gen_xxx_0",
"variant_index": 0,
"status": "completed",
"video_url": "https://oss.xxx/preview/xxx.mp4",
"duration": 15.5,
"thumbnail_url": "https://oss.xxx/preview/xxx.jpg"
},
...
]
}
```
#### 4.3.3 预览转正式生成
```
POST /api/v1/preview-batches/{batch_id}/tasks/{task_id}/promote
```
将某个预览版本升级为正式生成(复用素材缓存,重新全分辨率渲染)。
---
### 4.4 渲染参数降级
预览模式下自动调整以下参数:
| 参数 | 正式生成 | 预览生成 |
|------|---------|---------|
| 分辨率 | 720p (1280x720) | 480p (854x480) |
| 码率 | 4 Mbps | 1.5 Mbps |
| 帧率 | 30 fps | 24 fps |
| 时长 | 完整时长 | 前 15 秒(或第一段完整clip) |
| 转场效果 | 完整转场 | 仅淡入淡出 |
| 高级特效 | 全部启用 | 跳过粒子/光效等 |
| 字幕 | 完整渲染 | 正常渲染 |
| 配音 | 完整混音 | 正常混音 |
| 输出质量 | high | medium |
**实现位置**:`_render_video` 函数入口处,根据 `is_preview` 标记调整渲染配置。
---
### 4.5 任务调度
- **并行执行**:N 个预览任务并行提交到 Celery,不排队等待
- **低优先级**:预览任务走独立队列(`preview_queue`),不抢占正式生成资源
- **超时控制**:预览任务超时时间 5 分钟(正式 30 分钟)
- **自动清理**:预览视频 7 天后自动从 OSS 删除,任务记录标记为 archived
---
## 5. 前端对接要点
### 5.1 交互流程
```
第2步选素材 → 第3步点击"生成预览"
→ 显示 loading + 进度
→ 预览陆续完成,左侧列表逐张出现
→ 用户点击左侧不同版本,右侧预览区切换
→ 用户选中满意版本 → 点击"正式生成"
```
### 5.2 需要对齐的接口
1. **预览创建**:`POST /templates/{id}/generate-preview`
2. **批次状态轮询**:`GET /preview-batches/{id}`(建议 2s 轮询,或走 SSE)
3. **预览转正式**:`POST /preview-batches/{id}/tasks/{task_id}/promote`
### 5.3 数据格式对齐
预览视频条目结构:
```json
{
"id": "gen_xxx",
"variant_index": 0,
"status": "completed",
"video_url": "https://...",
"duration": 15.5,
"file_size": 2850000,
"thumbnail_url": "https://...",
"is_preview": true
}
```
---
## 6. 配额与计费
- 预览生成**不计入**用户配额
- 同一模板 + 同一素材池,每天最多生成 3 次多版本预览(防滥用)
- 单个预览批次最多 10 个版本
---
## 7. 实施步骤
### Phase 1:单版本预览(MVP,2 天)
1. 数据模型 + 迁移(is_preview 字段)
2. API 层支持 is_preview 参数
3. 渲染分辨率降级(480p)
4. 不计入配额
5. 基础测试
### Phase 2:多版本预览(3 天)
1. 变体引擎实现(素材随机选择 + 排序 + 去重)
2. preview_batch 批次管理
3. 批量创建 N 个预览任务
4. 批次查询接口
5. 前端联调
### Phase 3:预览转正式 + 优化(2 天)
1. 预览转正式生成接口(promote)
2. 素材下载缓存复用
3. 独立预览队列(低优先级)
4. 自动清理机制
5. 完整测试 + 压测
---
## 8. 风险与注意事项
| 风险 | 影响 | 应对 |
|------|------|------|
| 并发预览任务过多打满 worker | 正式生成被阻塞 | 独立预览队列 + 限流 |
| 变体生成的视频差异不够大 | 用户觉得"都一样" | 优先素材子集+排序差异,保证视觉差异 |
| 预览视频占用 OSS 存储 | 存储成本上升 | 7 天自动清理 + 低码率 |
| N 个版本同时下载重复素材 | 带宽浪费 | 批次内共享一次下载(Phase 3 优化) |
| 用户预期管理 | 以为预览就是最终效果 | 明确标注"预览版",说明分辨率差异 |
---
## 9. 开发量估算
| 阶段 | 后端 | 前端 | 合计 |
|------|------|------|------|
| Phase 1 单版本预览 | 2 天 | 1 天 | 3 天 |
| Phase 2 多版本预览 | 3 天 | 2 天 | 5 天 |
| Phase 3 转正式+优化 | 2 天 | 1 天 | 3 天 |
| **总计** | **7 天** | **4 天** | **~7 天(并行)** |
---
## 10. 与 v1 方案的差异总结
1. **新增多版本能力**:从"生成1个预览"升级为"生成N个不重复预览"
2. **新增变体引擎**:负责素材选择/排序/配音/标题的随机化
3. **新增批次概念**:preview_batch 管理一组预览任务
4. **新增 promote 接口**:预览转正式生成
5. **独立队列**:预览不抢占正式生成资源
6. **开发量**:从 2-3 天增加到约 7 天(后端)
-2
View File
@@ -494,5 +494,3 @@
- [Fixed] Bug 修复
- [Security] 安全相关更新
- [Performance] 性能优化
---
- 2026-09-16: fix extract-from-douyin 异常路径全部返回业务码(消除500) #1963
-2
View File
@@ -1,2 +0,0 @@
Mon Oct 5 04:09:11 PM CST 2026
2198 lite/pro并行竞速 (commit 9699a1f) — CI rebuild trigger Mon Oct 5 08:09:11 AM UTC 2026
-1
View File
@@ -1 +0,0 @@
retrigger3
-1
View File
@@ -263,4 +263,3 @@ pytest --cov=packages --cov-report=html
---
**License**: MIT
<!-- CI trigger: 1788229339 -->
Executable → Regular
+4 -7
View File
@@ -1,3 +1,4 @@
import os
from logging.config import fileConfig
from sqlalchemy import engine_from_config, pool
@@ -7,17 +8,13 @@ from alembic import context
# Import your models' Base here
from packages.adapters.sqlalchemy_impl.models import Base
# 使用统一配置入口获取 database_url,而非直接读环境变量
from packages.config import get_shared_settings
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# 从统一配置系统获取 database_url,确保与应用使用同一配置源
settings = get_shared_settings()
if settings.database_url:
config.set_main_option("sqlalchemy.url", settings.database_url)
database_url = os.getenv("DATABASE_URL")
if database_url:
config.set_main_option("sqlalchemy.url", database_url)
# Interpret the config file for Python logging.
# This line sets up loggers basically.
@@ -1,33 +0,0 @@
"""add updated_at to generation_tasks
Revision ID: 043_updated_at_generation_tasks
Revises: 042_storage_key
Create Date: 2026-07-18 19:30:00.000000
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "043_updated_at_generation_tasks"
down_revision = "042_storage_key"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"generation_tasks",
sa.Column(
"updated_at",
sa.DateTime,
nullable=False,
server_default=sa.func.now(),
),
)
def downgrade() -> None:
op.drop_column("generation_tasks", "updated_at")
@@ -1,34 +0,0 @@
"""add user_id to generated_videos
Revision ID: 044_user_id_generated_videos
Revises: 043_updated_at_generation_tasks
Create Date: 2026-07-19 08:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "044_user_id_generated_videos"
down_revision = "043_updated_at_generation_tasks"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"generated_videos",
sa.Column(
"user_id",
sa.String(36),
nullable=False,
server_default="",
index=True,
),
)
def downgrade() -> None:
op.drop_column("generated_videos", "user_id")
-36
View File
@@ -1,36 +0,0 @@
"""backfill user_id for generated_videos from generation_tasks
Revision ID: 045_backfill_user_id_generated_videos
Revises: 044_user_id_generated_videos
Create Date: 2026-07-19 10:50:00.000000
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "045_backfill_user_id"
down_revision = "044_user_id_generated_videos"
branch_labels = None
depends_on = None
def upgrade() -> None:
# 回填 generated_videos.user_id:通过 generation_task_id 关联 generation_tasks 表
# 取 generation_tasks.created_by_user_id 作为 user_id
# 回填不到的(无关联task的兜底记录)保持空字符串
op.execute("""
UPDATE generated_videos gv
SET user_id = gt.created_by_user_id
FROM generation_tasks gt
WHERE gv.generation_task_id = gt.id
AND gv.user_id = ''
AND gt.created_by_user_id != ''
""")
def downgrade() -> None:
# 降级不做处理(无法精确区分哪些是回填的)
pass
-33
View File
@@ -1,33 +0,0 @@
"""add video_title to generation_tasks
Revision ID: 046_add_video_title_to_generation_tasks
Revises: 045_backfill_user_id_generated_videos
Create Date: 2026-07-19 11:20:00.000000
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "046_task_title"
down_revision = "045_backfill_user_id"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"generation_tasks",
sa.Column(
"video_title",
sa.String(255),
nullable=False,
server_default="",
),
)
def downgrade() -> None:
op.drop_column("generation_tasks", "video_title")
@@ -1,64 +0,0 @@
"""Phase 2 - 模板发布版本化:version字段 + 发布历史表
Revision ID: 047
Revises: 046
Create Date: 2026-07-20
Changes:
1. edit_templates 加 version 字段(INT,默认1,每次发布+1)
2. 新建 edit_template_versions 表存发布历史快照,支持回滚
"""
import sqlalchemy as sa
from alembic import op
revision = "047_template_versioning"
down_revision = "046_task_title"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
# 1. edit_templates 加 version 字段
op.add_column(
"edit_templates",
sa.Column("version", sa.Integer, nullable=False, server_default="1"),
)
# 2. 新建 edit_template_versions 发布历史表
conn.execute(sa.text("""
CREATE TABLE IF NOT EXISTS edit_template_versions (
id VARCHAR(36) PRIMARY KEY,
template_id VARCHAR(32) NOT NULL,
version INTEGER NOT NULL,
name VARCHAR(200) NOT NULL DEFAULT '',
editing_mode VARCHAR(30) NOT NULL DEFAULT 'one_take',
config JSONB NOT NULL DEFAULT '{}',
clip_configs JSONB NOT NULL DEFAULT '[]',
change_note VARCHAR(500) NOT NULL DEFAULT '',
published_by VARCHAR(36) NOT NULL DEFAULT '',
created_at TIMESTAMP NOT NULL DEFAULT NOW()
)
"""))
conn.execute(
sa.text(
"CREATE INDEX IF NOT EXISTS ix_edit_template_versions_template_id " "ON edit_template_versions(template_id)"
)
)
conn.execute(
sa.text(
"CREATE UNIQUE INDEX IF NOT EXISTS ix_edit_template_versions_template_version "
"ON edit_template_versions(template_id, version)"
)
)
def downgrade() -> None:
conn = op.get_bind()
conn.execute(sa.text("DROP TABLE IF EXISTS edit_template_versions"))
op.drop_column("edit_templates", "version")
@@ -1,37 +0,0 @@
"""Phase 3 - 清理 EditPlan 表冗余字段
Revision ID: 048
Revises: 047
Create Date: 2026-07-21
Changes:
1. 删除 edit_plans.result_count 字段(剪辑计划独立功能遗留,模板草稿不用,
生成结果数由 generation_tasks.result_count 承载)
"""
import sqlalchemy as sa
from alembic import op
revision = "048_cleanup_result_count"
down_revision = "047_template_versioning"
branch_labels = None
depends_on = None
def upgrade() -> None:
# 删除 result_count 字段(剪辑计划独立功能遗留字段)
op.drop_column("edit_plans", "result_count")
def downgrade() -> None:
# 回滚:恢复 result_count 字段,默认值 0
op.add_column(
"edit_plans",
sa.Column(
"result_count",
sa.Integer,
nullable=False,
server_default="0",
),
)
@@ -1,97 +0,0 @@
"""#558 - 微信登录:手机号绑定字段 + 验证码表
Revision ID: 049
Revises: 048
Create Date: 2026-07-21
Changes:
1. users 表新增 phone_verified / binding_completed_at 字段(phone 字段已在 029 中添加)
2. users 表 phone 字段添加唯一索引(幂等)
3. 新建 verification_codes 表(统一管理邮箱+手机验证码)
"""
import sqlalchemy as sa
from alembic import context, op
revision = "049_wechat_login_phone"
down_revision = "048_cleanup_result_count"
branch_labels = None
depends_on = None
def _column_exists(table: str, column: str) -> bool:
"""检查列是否已存在。离线模式下返回 False。"""
if context.is_offline_mode():
return False
conn = op.get_bind()
result = conn.execute(
sa.text("SELECT 1 FROM information_schema.columns " "WHERE table_name = :table AND column_name = :column"),
{"table": table, "column": column},
)
return result.first() is not None
def _index_exists(index_name: str) -> bool:
"""检查索引是否已存在。离线模式下返回 False。"""
if context.is_offline_mode():
return False
conn = op.get_bind()
result = conn.execute(
sa.text("SELECT 1 FROM pg_indexes WHERE indexname = :index_name"),
{"index_name": index_name},
)
return result.first() is not None
def upgrade() -> None:
# 1. users 表新增手机号验证状态字段(幂等)
if not _column_exists("users", "phone_verified"):
op.add_column(
"users",
sa.Column(
"phone_verified",
sa.Boolean,
nullable=False,
server_default=sa.text("false"),
),
)
if not _column_exists("users", "binding_completed_at"):
op.add_column(
"users",
sa.Column("binding_completed_at", sa.DateTime, nullable=True),
)
# 2. phone 字段唯一索引(幂等 - 029 加了字段但没加索引)
if not _index_exists("ix_users_phone"):
op.create_index("ix_users_phone", "users", ["phone"], unique=True)
# 3. verification_codes 表
op.create_table(
"verification_codes",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("recipient", sa.String(255), nullable=False, index=True),
sa.Column("code", sa.String(10), nullable=False),
sa.Column("code_type", sa.String(32), nullable=False, index=True),
sa.Column("expires_at", sa.DateTime, nullable=False),
sa.Column("used_at", sa.DateTime, nullable=True),
sa.Column("attempts", sa.Integer, nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime, nullable=False),
sa.Index(
"ix_verification_recipient_type",
"recipient",
"code_type",
"created_at",
),
)
def downgrade() -> None:
op.drop_table("verification_codes")
if _index_exists("ix_users_phone"):
op.drop_index("ix_users_phone", table_name="users")
if _column_exists("users", "binding_completed_at"):
op.drop_column("users", "binding_completed_at")
if _column_exists("users", "phone_verified"):
op.drop_column("users", "phone_verified")
-48
View File
@@ -1,48 +0,0 @@
"""#P3-2 - 视频分享表 video_shares
Revision ID: 050
Revises: 049
Create Date: 2026-07-22
Changes:
1. 新建 video_shares 表,支持视频匿名分享链接
2. share_token 唯一索引,用于公开分享URL
3. 支持密码保护、有效期、浏览/下载计数
"""
import sqlalchemy as sa
from alembic import context, op
revision = "050_video_shares"
down_revision = "049_wechat_login_phone"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
if context.get_context().dialect.name == "postgresql":
# 检查表是否已存在(幂等)
result = conn.execute(sa.text("SELECT to_regclass('public.video_shares')"))
if result.scalar() is not None:
return
op.create_table(
"video_shares",
sa.Column("id", sa.String(32), primary_key=True),
sa.Column("video_id", sa.String(32), nullable=False, index=True),
sa.Column("user_id", sa.String(32), nullable=False, index=True),
sa.Column("share_token", sa.String(16), nullable=False, unique=True),
sa.Column("password_hash", sa.String(255), nullable=True),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("view_count", sa.Integer, nullable=False, server_default="0"),
sa.Column("download_count", sa.Integer, nullable=False, server_default="0"),
sa.Column("is_active", sa.Boolean, nullable=False, server_default=sa.true()),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
)
def downgrade() -> None:
op.drop_table("video_shares")
@@ -1,53 +0,0 @@
"""#632 - 一键生成输出分辨率可配置
Revision ID: 051
Revises: 050
Create Date: 2026-07-23
Changes:
1. generation_tasks 表新增 resolution 字段,存储用户指定的输出分辨率(如 "1280x720")
2. 为空时使用默认值(1280x720)
"""
import sqlalchemy as sa
from alembic import context, op
revision = "051_generation_task_resolution"
down_revision = "050_video_shares"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
if context.get_context().dialect.name == "postgresql":
# 检查列是否已存在(幂等)
result = conn.execute(
sa.text(
"SELECT column_name FROM information_schema.columns "
"WHERE table_name = 'generation_tasks' AND column_name = 'resolution'"
)
)
if result.scalar() is not None:
return
op.add_column(
"generation_tasks",
sa.Column("resolution", sa.String(20), nullable=False, server_default=""),
)
def downgrade() -> None:
conn = op.get_bind()
if context.get_context().dialect.name == "postgresql":
result = conn.execute(
sa.text(
"SELECT column_name FROM information_schema.columns "
"WHERE table_name = 'generation_tasks' AND column_name = 'resolution'"
)
)
if result.scalar() is None:
return
op.drop_column("generation_tasks", "resolution")
@@ -1,64 +0,0 @@
"""#642 - 生成任务新增 bgm_config 字段
Revision ID: 052_generation_task_bgm_config
Revises: 051_generation_task_resolution
Create Date: 2026-07-25
Changes:
1. generation_tasks 表新增 bgm_config 字段(JSON类型),存储用户自定义BGM配置
2. 为空时使用默认空字典
背景:
#642 一键生成支持自定义BGM 功能在 SQLAlchemy 模型中加了 bgm_config 字段,
但遗漏了 alembic migration,导致 staging 环境数据库没有该列,
创建生成任务时直接 500。
"""
import sqlalchemy as sa
from alembic import context, op
revision = "052_generation_task_bgm_config"
down_revision = "051_generation_task_resolution"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
if context.get_context().dialect.name == "postgresql":
# 检查列是否已存在(幂等)
result = conn.execute(
sa.text(
"SELECT column_name FROM information_schema.columns "
"WHERE table_name = 'generation_tasks' AND column_name = 'bgm_config'"
)
)
if result.scalar() is not None:
return
op.add_column(
"generation_tasks",
sa.Column(
"bgm_config",
sa.JSON,
nullable=False,
server_default=sa.text("'{}'::json"),
),
)
def downgrade() -> None:
conn = op.get_bind()
if context.get_context().dialect.name == "postgresql":
# 检查列是否存在(幂等)
result = conn.execute(
sa.text(
"SELECT column_name FROM information_schema.columns "
"WHERE table_name = 'generation_tasks' AND column_name = 'bgm_config'"
)
)
if result.scalar() is None:
return
op.drop_column("generation_tasks", "bgm_config")
@@ -1,61 +0,0 @@
"""#1197 - 预览生成:generation_tasks 表新增 is_preview 字段
Revision ID: 053
Revises: 052
Create Date: 2026-08-15
Changes:
1. generation_tasks 表新增 is_preview 字段,标记是否为预览生成任务(低清 480p)
2. 默认 False,与现有正式生成任务兼容
3. 加索引以支持按预览/正式任务筛选
"""
import sqlalchemy as sa
from alembic import context, op
revision = "053_generation_task_is_preview"
down_revision = "052_generation_task_bgm_config"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
if context.get_context().dialect.name == "postgresql":
# 检查列是否已存在(幂等)
result = conn.execute(
sa.text(
"SELECT column_name FROM information_schema.columns "
"WHERE table_name = 'generation_tasks' AND column_name = 'is_preview'"
)
)
if result.scalar() is not None:
return
op.add_column(
"generation_tasks",
sa.Column("is_preview", sa.Boolean, nullable=False, server_default=sa.text("false")),
)
# 加索引
op.create_index(
"ix_generation_tasks_is_preview",
"generation_tasks",
["is_preview"],
)
def downgrade() -> None:
conn = op.get_bind()
if context.get_context().dialect.name == "postgresql":
result = conn.execute(
sa.text(
"SELECT column_name FROM information_schema.columns "
"WHERE table_name = 'generation_tasks' AND column_name = 'is_preview'"
)
)
if result.scalar() is None:
return
op.drop_index("ix_generation_tasks_is_preview", table_name="generation_tasks")
op.drop_column("generation_tasks", "is_preview")
@@ -1,82 +0,0 @@
"""确认生成 API 改造:为 generation_tasks 表添加 source_task_id、output_width、output_height、cover_url、custom_title 字段
Revision ID: 054_confirm_gen_fields
Revises: 053_generation_task_is_preview
Create Date: 2026-08-16
Changes:
1. generation_tasks 表新增 source_task_id(来源预览任务 ID,带索引)
2. generation_tasks 表新增 output_width / output_height(动态输出分辨率)
3. generation_tasks 表新增 cover_url / custom_title(自定义封面和标题)
"""
import sqlalchemy as sa
from alembic import op
revision = "054_confirm_gen_fields"
down_revision = "053_generation_task_is_preview"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
is_pg = conn.dialect.name == "postgresql"
if is_pg:
# 幂等检查:source_task_id 列是否已存在
result = conn.execute(
sa.text(
"SELECT column_name FROM information_schema.columns "
"WHERE table_name = 'generation_tasks' AND column_name = 'source_task_id'"
)
)
if result.scalar() is not None:
return
# source_task_id
op.add_column(
"generation_tasks",
sa.Column("source_task_id", sa.String(32), nullable=False, server_default=""),
)
# output_width
op.add_column(
"generation_tasks",
sa.Column("output_width", sa.Integer, nullable=False, server_default=sa.text("1280")),
)
# output_height
op.add_column(
"generation_tasks",
sa.Column("output_height", sa.Integer, nullable=False, server_default=sa.text("720")),
)
# cover_url
op.add_column(
"generation_tasks",
sa.Column("cover_url", sa.String(1000), nullable=False, server_default=""),
)
# custom_title
op.add_column(
"generation_tasks",
sa.Column("custom_title", sa.String(500), nullable=False, server_default=""),
)
# 索引
op.create_index(
"ix_generation_tasks_source_task_id",
"generation_tasks",
["source_task_id"],
)
def downgrade() -> None:
op.drop_index("ix_generation_tasks_source_task_id", table_name="generation_tasks")
op.drop_column("generation_tasks", "custom_title")
op.drop_column("generation_tasks", "cover_url")
op.drop_column("generation_tasks", "output_height")
op.drop_column("generation_tasks", "output_width")
op.drop_column("generation_tasks", "source_task_id")
-82
View File
@@ -1,82 +0,0 @@
"""封面模板表 cover_templates
Revision ID: 055_cover_templates
Revises: 054_confirm_gen_fields
Create Date: 2026-08-09
Changes:
1. 新建 cover_templates 表,支持系统预置和用户自定义封面模板
2. user_id 为 NULL 表示系统模板,is_system 标记区分
3. config 为 JSON 字段,存储封面配置信息
"""
import sqlalchemy as sa
from alembic import context, op
revision = "055_cover_templates"
down_revision = "054_confirm_gen_fields"
branch_labels = None
depends_on = None
SYSTEM_TEMPLATES = [
("a8b0120fd98e44788f5a6590f983d327", "默认模板", {}),
("6d8c501b11424432b3df3a45ae89b1a9", "大胆红", {"background_color": "#ef4444"}),
("04937fb57fea4bad95e7883e71a6b246", "优雅黑", {"background_color": "#111827"}),
("3ff9cc821174437ca53931073e7f536e", "渐变蓝", {"background_color": "#3b82f6"}),
("db51b3ea8f1a4f4caa94bf2d51f27d11", "渐变紫", {"background_color": "#8b5cf6"}),
("5027d113432a4f798a3b4ee1644d66af", "暖橙", {"background_color": "#f97316"}),
("0e10def2b5a148d686416494474726c2", "清新绿", {"background_color": "#22c55e"}),
("38ea98ac00c04bada064006d880546f0", "科技蓝", {"background_color": "#06b6d4"}),
]
def upgrade() -> None:
conn = op.get_bind()
if context.get_context().dialect.name == "postgresql":
result = conn.execute(sa.text("SELECT to_regclass('public.cover_templates')"))
if result.scalar() is not None:
return
op.create_table(
"cover_templates",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("user_id", sa.String(36), nullable=True, index=True),
sa.Column("name", sa.String(200), nullable=False),
sa.Column("thumbnail_url", sa.String(1000), nullable=False, server_default=""),
sa.Column("is_system", sa.Boolean, nullable=False, server_default=sa.false(), index=True),
sa.Column("config", sa.JSON, nullable=False, server_default="{}"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
)
# 预置系统模板 seed 数据
cover_templates = sa.table(
"cover_templates",
sa.column("id", sa.String),
sa.column("user_id", sa.String),
sa.column("name", sa.String),
sa.column("thumbnail_url", sa.String),
sa.column("is_system", sa.Boolean),
sa.column("config", sa.JSON),
sa.column("created_at", sa.DateTime),
sa.column("updated_at", sa.DateTime),
)
for tid, name, config in SYSTEM_TEMPLATES:
conn.execute(
cover_templates.insert().values(
id=tid,
user_id=None,
name=name,
thumbnail_url="",
is_system=True,
config=config,
created_at=sa.func.now(),
updated_at=sa.func.now(),
)
)
def downgrade() -> None:
op.drop_table("cover_templates")
@@ -1,39 +0,0 @@
"""修复 cover_templates.config 双重序列化
Revision ID: 056_fix_cover_templates_config
Revises: 055_cover_templates
Create Date: 2026-08-13
问题: 055 迁移 seed 数据时 json.dumps(config) 导致 config 被双重序列化为 JSON 字符串
例如 "{}"(字符串)而不是 {}(对象),导致 Pydantic CoverTemplateResponse 校验失败 500。
修复: 从 JSON 字符串中提取文本值,再 cast 回 json 对象类型。
"""
import sqlalchemy as sa
from alembic import op
revision = "056_fix_cover_templates_config"
down_revision = "055_cover_templates"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
# PostgreSQL: 从 JSON string scalar 中提取文本内容,cast 为 json object
# 例如: JSON string "{}" -> text "{}" -> JSON object {}
if conn.dialect.name == "postgresql":
conn.execute(
sa.text(
"UPDATE cover_templates SET config = (config#>>'{}')::json "
"WHERE jsonb_typeof(config::jsonb) = 'string'"
)
)
def downgrade() -> None:
# No safe rollback — the original data was incorrect
pass
@@ -1,26 +0,0 @@
"""Add title_config to generation_tasks
Revision ID: 057_title_config
Revises: 056_fix_cover_templates_config
Create Date: 2026-08-23
"""
import sqlalchemy as sa
from alembic import op
revision = "057_title_config"
down_revision = "056_fix_cover_templates_config"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"generation_tasks",
sa.Column("title_config", sa.JSON(), nullable=False, server_default="{}"),
)
def downgrade() -> None:
op.drop_column("generation_tasks", "title_config")
@@ -1,49 +0,0 @@
"""Add unique index on asset_libraries(project_id, kind)
Revision ID: 058_uq_asset_lib_project_kind
Revises: 057_title_config
Create Date: 2026-08-30
同一项目下同 kind 的素材库业务上唯一(前端 getOrCreate 语义、TTS 保存自动建库)。
加唯一索引兜底并发创建竞态,避免重复素材库。
"""
import sqlalchemy as sa
from alembic import op
revision = "058_uq_asset_lib_project_kind"
down_revision = "057_title_config"
branch_labels = None
depends_on = None
def upgrade() -> None:
# 建唯一索引前清洗历史重复:同 (project_id, kind) 只保留 created_at 最新的一条。
# project_id 为 NULL 的系统级行不参与去重(NULL 在唯一索引中互不冲突)。
op.execute("""
DELETE FROM asset_libraries
WHERE id IN (
SELECT id FROM (
SELECT id,
ROW_NUMBER() OVER (
PARTITION BY project_id, kind
ORDER BY created_at DESC, id DESC
) AS rn
FROM asset_libraries
WHERE project_id IS NOT NULL
) t
WHERE t.rn > 1
)
""")
# 与 model 的 UniqueConstraint 定义保持一致(pg_constraint + pg_index 同时注册),
# 避免 Alembic autogenerate 检测到 schema drift
op.create_unique_constraint(
"uq_asset_libraries_project_kind",
"asset_libraries",
["project_id", "kind"],
)
def downgrade() -> None:
op.drop_constraint("uq_asset_libraries_project_kind", "asset_libraries", type_="unique")
@@ -1,23 +0,0 @@
"""add duplicate_rate to generated_videos
Revision ID: 059_duplicate_rate
Revises: 058_uq_asset_lib_project_kind
Create Date: 2026-08-31
"""
import sqlalchemy as sa
from alembic import op
revision = "059_duplicate_rate"
down_revision = "058_uq_asset_lib_project_kind"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("generated_videos", sa.Column("duplicate_rate", sa.Float(), nullable=True))
def downgrade() -> None:
op.drop_column("generated_videos", "duplicate_rate")
@@ -1,57 +0,0 @@
"""migrate template_segments data to template_clip_configs
Revision ID: 060_migrate_segments
Revises: 059_duplicate_rate
Create Date: 2026-08-31
"""
import sqlalchemy as sa
from alembic import op
revision = "060_migrate_segments"
down_revision = "059_duplicate_rate"
branch_labels = None
depends_on = None
def upgrade() -> None:
dialect = op.get_bind().dialect.name
if dialect == "postgresql":
config_expr = (
"CASE WHEN s.material_type IS NOT NULL AND s.material_type != '' "
"THEN json_build_object('material_type', s.material_type)::jsonb "
"ELSE '{}'::jsonb END"
)
empty_json = "'{}'::jsonb"
else:
config_expr = (
"CASE WHEN s.material_type IS NOT NULL AND s.material_type != '' "
"THEN JSON_OBJECT('material_type', s.material_type) "
"ELSE '{}' END"
)
empty_json = "'{}'"
sql_str = (
"INSERT INTO template_clip_configs "
'(id, template_id, clip_type, "order", min_duration, max_duration, '
"text_template, material_requirements, transition_effect, config, "
"created_at, updated_at) "
"SELECT "
"s.id, s.template_id, 'main', s.segment_order, "
"s.duration_min, s.duration_max, "
"'', " + empty_json + ", "
"'cut', " + config_expr + ", "
"s.created_at, s.updated_at "
"FROM template_segments s "
"WHERE NOT EXISTS ("
" SELECT 1 FROM template_clip_configs c "
" WHERE c.template_id = s.template_id"
")"
)
op.execute(sa.text(sql_str))
def downgrade() -> None:
pass
@@ -1,26 +0,0 @@
"""add sort_order to template_categories
Revision ID: 061_sort_order
Revises: 060_migrate_segments
Create Date: 2026-09-02
"""
import sqlalchemy as sa
from alembic import op
revision = "061_sort_order"
down_revision = "060_migrate_segments"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"template_categories",
sa.Column("sort_order", sa.Integer, nullable=False, server_default="0"),
)
def downgrade() -> None:
op.drop_column("template_categories", "sort_order")
@@ -1,28 +0,0 @@
"""re-add edit_plan_id to generation_tasks (align staging with production)
Revision ID: 062_edit_plan_id
Revises: 061_sort_order
Create Date: 2026-09-02
"""
import sqlalchemy as sa
from alembic import op
revision = "062_edit_plan_id"
down_revision = "061_sort_order"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"generation_tasks",
sa.Column("edit_plan_id", sa.String(36), nullable=True),
)
op.create_index("ix_generation_tasks_edit_plan_id_2", "generation_tasks", ["edit_plan_id"])
def downgrade() -> None:
op.drop_index("ix_generation_tasks_edit_plan_id_2", table_name="generation_tasks")
op.drop_column("generation_tasks", "edit_plan_id")
@@ -1,46 +0,0 @@
"""add video_fingerprint_chunks table for per-chunk fingerprint storage
Revision ID: 063_fingerprint_chunks
Revises: 062_edit_plan_id
Create Date: 2026-09-03
"""
import sqlalchemy as sa
from alembic import op
revision = "063_fingerprint_chunks"
down_revision = "062_edit_plan_id"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"video_fingerprint_chunks",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("video_id", sa.String(36), nullable=False),
sa.Column("project_id", sa.String(36), nullable=False),
sa.Column("user_id", sa.String(36), nullable=False, server_default=""),
sa.Column("start_time_ms", sa.Integer, nullable=False),
sa.Column("end_time_ms", sa.Integer, nullable=False),
sa.Column("phash_binary", sa.String(16), nullable=False),
sa.Column("color_histogram", sa.JSON, nullable=False),
sa.Column("frame_count", sa.Integer, nullable=False, server_default="1"),
sa.Column(
"created_at",
sa.DateTime,
nullable=False,
server_default=sa.func.now(),
),
)
op.create_index("ix_vfc_video_id", "video_fingerprint_chunks", ["video_id"])
op.create_index("ix_vfc_project_id", "video_fingerprint_chunks", ["project_id"])
op.create_index("ix_vfc_user_id", "video_fingerprint_chunks", ["user_id"])
def downgrade() -> None:
op.drop_index("ix_vfc_user_id", table_name="video_fingerprint_chunks")
op.drop_index("ix_vfc_project_id", table_name="video_fingerprint_chunks")
op.drop_index("ix_vfc_video_id", table_name="video_fingerprint_chunks")
op.drop_table("video_fingerprint_chunks")
@@ -1,25 +0,0 @@
"""add match_count and visual_similarity to generated_videos
Revision ID: 064_match_count_visual_sim
Revises: 063_fingerprint_chunks
Create Date: 2026-09-03
"""
import sqlalchemy as sa
from alembic import op
revision = "064_match_count_visual_sim"
down_revision = "063_fingerprint_chunks"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("generated_videos", sa.Column("match_count", sa.Integer(), nullable=True, server_default="0"))
op.add_column("generated_videos", sa.Column("visual_similarity", sa.Float(), nullable=True, server_default="0.0"))
def downgrade() -> None:
op.drop_column("generated_videos", "visual_similarity")
op.drop_column("generated_videos", "match_count")
@@ -1,25 +0,0 @@
"""add visual_similarity and match_count to duplication_records
Revision ID: 065_dup_record_sim_match
Revises: 064_match_count_visual_sim
Create Date: 2026-09-04
"""
import sqlalchemy as sa
from alembic import op
revision = "065_dup_record_sim_match"
down_revision = "064_match_count_visual_sim"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("duplication_records", sa.Column("visual_similarity", sa.Float(), nullable=True))
op.add_column("duplication_records", sa.Column("match_count", sa.Integer(), nullable=True))
def downgrade() -> None:
op.drop_column("duplication_records", "match_count")
op.drop_column("duplication_records", "visual_similarity")
@@ -1,34 +0,0 @@
"""add client_upload_id to assets and asset_id to ingest_jobs
Issue #1714:上传 complete 幂等 + worker 转码回写关联。
- assets.client_upload_id:客户端幂等 token(complete 去重)
- ingest_jobs.asset_id:complete 阶段创建的占位 asset id(worker 回写关联,
防止 HEVC 转码改写 storage_key 后找不到占位而兜底新建 READY 记录)
Revision ID: 066_upload_idempotency
Revises: 065_dup_record_sim_match
Create Date: 2026-09-05
"""
import sqlalchemy as sa
from alembic import op
revision = "066_upload_idempotency"
down_revision = "065_dup_record_sim_match"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("assets", sa.Column("client_upload_id", sa.String(64), nullable=True))
op.create_index("ix_assets_client_upload_id", "assets", ["client_upload_id"])
op.add_column("ingest_jobs", sa.Column("asset_id", sa.String(36), nullable=False, server_default=""))
op.create_index("ix_ingest_jobs_asset_id", "ingest_jobs", ["asset_id"])
def downgrade() -> None:
op.drop_index("ix_ingest_jobs_asset_id", table_name="ingest_jobs")
op.drop_column("ingest_jobs", "asset_id")
op.drop_index("ix_assets_client_upload_id", table_name="assets")
op.drop_column("assets", "client_upload_id")
@@ -1,35 +0,0 @@
"""add celery_task_id to generation_tasks and ingest_jobs
Issue #1714:孤儿恢复/超时清理撤销队列消息。
- generation_tasks.celery_task_id:入队时记录的 Celery 消息 ID,清理时 revoke
- ingest_jobs.celery_task_id:同上(素材转码任务)
Revision ID: 067_celery_task_id
Revises: 066_upload_idempotency
Create Date: 2026-09-05
"""
import sqlalchemy as sa
from alembic import op
revision = "067_celery_task_id"
down_revision = "066_upload_idempotency"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"generation_tasks",
sa.Column("celery_task_id", sa.String(64), nullable=False, server_default=""),
)
op.add_column(
"ingest_jobs",
sa.Column("celery_task_id", sa.String(64), nullable=False, server_default=""),
)
def downgrade() -> None:
op.drop_column("ingest_jobs", "celery_task_id")
op.drop_column("generation_tasks", "celery_task_id")
@@ -1,26 +0,0 @@
"""add profile_completed to users
Issue #1718:微信新用户首次登录需设置昵称(PATCH /auth/me)。
- users.profile_completed:资料是否已完善;存量行默认 True(不触发引导),
微信新建用户在应用层置 False。
"""
import sqlalchemy as sa
from alembic import op
revision = "068_user_profile_completed"
down_revision = "067_celery_task_id"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"users",
sa.Column("profile_completed", sa.Boolean(), nullable=False, server_default=sa.text("true")),
)
def downgrade() -> None:
op.drop_column("users", "profile_completed")
@@ -1,72 +0,0 @@
"""Projects is_default + partial unique index for idempotent default project (Issue #1775)
Revision ID: 069_project_is_default
Revises: 068_user_profile_completed
Create Date: 2026-09-08
背景:
小程序端 getOrCreateDefaultProject 在重试/并发/前端重复调用下,
仅靠应用层"先查再插"不保证幂等,会给同一用户重复创建默认项目。
改动:
1. projects 表新增 is_default 布尔列(默认 false)
2. 部分唯一索引 uq_projects_owner_default:(owner_user_id) WHERE is_default = true
—— 保证每个用户至多一个默认项目
3. 存量数据回填:把名为"默认项目"的存量项目按创建时间最早者标记为 is_default=true
(只标记不删除;存量重复项目的清理另行确认后单独执行)
注意:部分唯一索引依赖 PostgreSQL,不支持 downgrade 到其他方言。
"""
import sqlalchemy as sa
from alembic import op
revision = "069_project_is_default"
down_revision = "068_user_profile_completed"
branch_labels = None
depends_on = None
def upgrade() -> None:
# 1. 新增 is_default 列
op.add_column(
"projects",
sa.Column(
"is_default",
sa.Boolean(),
nullable=False,
server_default=sa.text("false"),
),
)
# 2. 存量回填:每个拥有"默认项目"的用户,只把最早创建的那一个标记为默认。
# 用 ROW_NUMBER() 取每组第一条;非"默认项目"命名的项目不标记(保守,不动用户自建项目)。
op.execute("""
UPDATE projects p
SET is_default = true
WHERE p.id IN (
SELECT id FROM (
SELECT id,
ROW_NUMBER() OVER (
PARTITION BY owner_user_id
ORDER BY created_at ASC, id ASC
) AS rn
FROM projects
WHERE name = '默认项目'
) t
WHERE t.rn = 1
)
""")
# 3. 部分唯一索引:每用户至多一个默认项目(只约束 is_default = true 的行)
op.execute("""
CREATE UNIQUE INDEX uq_projects_owner_default
ON projects (owner_user_id)
WHERE is_default = true
""")
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS uq_projects_owner_default")
op.drop_column("projects", "is_default")
-48
View File
@@ -1,48 +0,0 @@
"""Add scripts table for oral broadcast script library (Issue #1795)
Revision ID: 070_add_scripts
Revises: 069_project_is_default
Create Date: 2026-09-08
新建 scripts 表,支持口播文案 CRUD + 分段存储。
"""
import sqlalchemy as sa
from alembic import op
revision = "070_add_scripts"
down_revision = "069_project_is_default"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"scripts",
sa.Column("id", sa.String(36), nullable=False),
sa.Column("user_id", sa.String(36), nullable=False),
sa.Column("title", sa.String(255), nullable=False),
sa.Column("content", sa.Text(), nullable=False, server_default=""),
sa.Column("segments", sa.JSON(), nullable=False, server_default="[]"),
sa.Column("tags", sa.JSON(), nullable=False, server_default="[]"),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_scripts_user_id", "scripts", ["user_id"])
def downgrade() -> None:
op.drop_index("ix_scripts_user_id", table_name="scripts")
op.drop_table("scripts")
@@ -1,47 +0,0 @@
"""add lipsync jobs table
Revision ID: 071_add_lipsync_jobs
Revises: 070_add_scripts
Create Date: 2026-09-08
"""
import sqlalchemy as sa
from alembic import op
revision = "071_add_lipsync_jobs"
down_revision = "070_add_scripts"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"lipsync_jobs",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("user_id", sa.String(36), nullable=False, index=True),
sa.Column("project_id", sa.String(36), nullable=False, server_default=""),
sa.Column("video_url", sa.Text(), nullable=False),
sa.Column("audio_url", sa.Text(), nullable=False),
sa.Column("enable_video_loop", sa.Boolean(), nullable=False, server_default=sa.text("false")),
sa.Column("mediakit_task_id", sa.String(200), nullable=False, server_default="", index=True),
sa.Column("status", sa.String(20), nullable=False, server_default="pending", index=True),
sa.Column("output_video_url", sa.Text(), nullable=False, server_default=""),
sa.Column("output_duration", sa.Float(), nullable=False, server_default=sa.text("0.0")),
sa.Column("error_message", sa.Text(), nullable=False, server_default=""),
sa.Column("error_code", sa.String(100), nullable=False, server_default=""),
sa.Column("submitted_at", sa.DateTime(), nullable=True),
sa.Column("completed_at", sa.DateTime(), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False, server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(), nullable=False, server_default=sa.func.now()),
)
# 复合索引:用户 + 状态(列表查询常用)
op.create_index("ix_lipsync_jobs_user_status", "lipsync_jobs", ["user_id", "status"])
# 项目 + 用户(项目维度查询)
op.create_index("ix_lipsync_jobs_project_user", "lipsync_jobs", ["project_id", "user_id"])
def downgrade() -> None:
op.drop_index("ix_lipsync_jobs_project_user", table_name="lipsync_jobs")
op.drop_index("ix_lipsync_jobs_user_status", table_name="lipsync_jobs")
op.drop_table("lipsync_jobs")
@@ -1,48 +0,0 @@
"""add ai avatar render jobs table
Revision ID: 072_add_ai_avatar_render
Revises: 071_add_lipsync_jobs
Create Date: 2026-09-09
"""
import sqlalchemy as sa
from alembic import op
revision = "072_add_ai_avatar_render"
down_revision = "071_add_lipsync_jobs"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"ai_avatar_render_jobs",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("user_id", sa.String(36), nullable=False, index=True),
sa.Column("project_id", sa.String(36), nullable=False, server_default=""),
sa.Column("lipsync_job_id", sa.String(36), nullable=False),
sa.Column("script_id", sa.String(36), nullable=False),
sa.Column("b_roll_segments", sa.JSON(), nullable=False, server_default="[]"),
sa.Column("title_config", sa.JSON(), nullable=False, server_default="{}"),
sa.Column("cover_config", sa.JSON(), nullable=False, server_default="{}"),
sa.Column("status", sa.String(20), nullable=False, server_default="pending", index=True),
sa.Column("progress", sa.Integer(), nullable=False, server_default=sa.text("0")),
sa.Column("output_video_url", sa.Text(), nullable=False, server_default=""),
sa.Column("output_cover_url", sa.Text(), nullable=False, server_default=""),
sa.Column("output_duration", sa.Float(), nullable=False, server_default=sa.text("0.0")),
sa.Column("error_message", sa.Text(), nullable=False, server_default=""),
sa.Column("submitted_at", sa.DateTime(), nullable=True),
sa.Column("started_at", sa.DateTime(), nullable=True),
sa.Column("completed_at", sa.DateTime(), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False, server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(), nullable=False, server_default=sa.func.now()),
)
op.create_index("ix_ai_avatar_render_user_status", "ai_avatar_render_jobs", ["user_id", "status"])
op.create_index("ix_ai_avatar_render_project_user", "ai_avatar_render_jobs", ["project_id", "user_id"])
def downgrade() -> None:
op.drop_index("ix_ai_avatar_render_project_user", table_name="ai_avatar_render_jobs")
op.drop_index("ix_ai_avatar_render_user_status", table_name="ai_avatar_render_jobs")
op.drop_table("ai_avatar_render_jobs")
@@ -1,45 +0,0 @@
"""lipsync_jobs 增加 TTS 直生字段(voice_id/script_text/speed/emotion)
Revision ID: 073_add_lipsync_tts_fields
Revises: 072_add_ai_avatar_render
Create Date: 2026-09-09
"""
import sqlalchemy as sa
from alembic import op
revision = "073_add_lipsync_tts_fields"
down_revision = "072_add_ai_avatar_render"
branch_labels = None
depends_on = None
def upgrade() -> None:
# 对口型支持「传音色 + 文案直接生成」:后端内部先 TTS 合成音频再提交对口型
op.add_column(
"lipsync_jobs",
sa.Column("voice_id", sa.String(200), nullable=False, server_default=""),
)
op.add_column(
"lipsync_jobs",
sa.Column("script_text", sa.Text(), nullable=False, server_default=""),
)
op.add_column(
"lipsync_jobs",
sa.Column("speed", sa.Float(), nullable=False, server_default=sa.text("1.0")),
)
op.add_column(
"lipsync_jobs",
sa.Column("emotion", sa.String(20), nullable=False, server_default=""),
)
# audio_url 改为可空:直生模式下音频由后端 TTS 合成后回填
op.alter_column("lipsync_jobs", "audio_url", existing_type=sa.Text(), nullable=True)
def downgrade() -> None:
op.alter_column("lipsync_jobs", "audio_url", existing_type=sa.Text(), nullable=False)
op.drop_column("lipsync_jobs", "emotion")
op.drop_column("lipsync_jobs", "speed")
op.drop_column("lipsync_jobs", "script_text")
op.drop_column("lipsync_jobs", "voice_id")
@@ -1,36 +0,0 @@
"""ai_avatar_render_jobs.script_id 放宽为可空串(手动文案直生场景不关联文案库)
Revision ID: 074_render_script_id_optional
Revises: 073_add_lipsync_tts_fields
Create Date: 2026-09-09
"""
import sqlalchemy as sa
from alembic import op
revision = "074_render_script_id_optional"
down_revision = "073_add_lipsync_tts_fields"
branch_labels = None
depends_on = None
def upgrade() -> None:
# 列保持 NOT NULL(空串占位),仅应用层允许不传;这里显式补 server_default 防止历史约束歧义
with op.batch_alter_table("ai_avatar_render_jobs") as batch:
batch.alter_column(
"script_id",
existing_type=sa.String(length=36),
nullable=False,
server_default="",
)
def downgrade() -> None:
with op.batch_alter_table("ai_avatar_render_jobs") as batch:
batch.alter_column(
"script_id",
existing_type=sa.String(length=36),
nullable=False,
server_default=None,
)
@@ -1,27 +0,0 @@
"""add sentence_timings to lipsync_jobs
Revision ID: 075_add_sentence_timings
Revises: 074_ai_avatar_render_script_id_optional
Create Date: 2026-09-12
"""
import sqlalchemy as sa
from alembic import op
revision = "075_add_sentence_timings"
down_revision = "074_render_script_id_optional"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("lipsync_jobs") as batch:
batch.add_column(
sa.Column("sentence_timings", sa.JSON(), nullable=True),
)
def downgrade() -> None:
with op.batch_alter_table("lipsync_jobs") as batch:
batch.drop_column("sentence_timings")
-133
View File
@@ -1,133 +0,0 @@
"""add membership & points system
Revision ID: 076_membership_points
Revises: 075_add_sentence_timings
Create Date: 2026-09-15
"""
import sqlalchemy as sa
from sqlalchemy import text
from alembic import op
revision = "076_membership_points"
down_revision = "075_add_sentence_timings"
branch_labels = None
depends_on = None
def upgrade() -> None:
# 1. users 表新增字段
with op.batch_alter_table("users") as batch:
batch.add_column(
sa.Column("is_member", sa.Boolean(), nullable=False, server_default=sa.text("false")),
)
batch.add_column(
sa.Column("member_type", sa.String(20), nullable=True),
)
batch.add_column(
sa.Column("member_expires_at", sa.DateTime(), nullable=True),
)
batch.add_column(
sa.Column("points_balance", sa.Integer(), nullable=False, server_default=sa.text("0")),
)
# 2. points_accounts 积分账户表
op.create_table(
"points_accounts",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("user_id", sa.String(36), nullable=False, unique=True, index=True),
sa.Column("balance", sa.Integer(), nullable=False, server_default=sa.text("0")),
sa.Column("total_earned", sa.Integer(), nullable=False, server_default=sa.text("0")),
sa.Column("total_spent", sa.Integer(), nullable=False, server_default=sa.text("0")),
sa.Column(
"created_at",
sa.DateTime(),
nullable=False,
server_default=sa.text("NOW()"),
),
sa.Column(
"updated_at",
sa.DateTime(),
nullable=False,
server_default=sa.text("NOW()"),
),
)
# 3. points_transactions 积分流水表
op.create_table(
"points_transactions",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("user_id", sa.String(36), nullable=False, index=True),
sa.Column("account_id", sa.String(36), nullable=False, index=True),
sa.Column("type", sa.String(20), nullable=False, index=True),
sa.Column("source", sa.String(50), nullable=False, index=True),
sa.Column("amount", sa.Integer(), nullable=False),
sa.Column("balance_after", sa.Integer(), nullable=False),
sa.Column("description", sa.String(255), nullable=False, server_default=""),
sa.Column("ref_id", sa.String(100), nullable=False, server_default=""),
sa.Column(
"created_at",
sa.DateTime(),
nullable=False,
server_default=sa.text("NOW()"),
),
)
# 4. points_orders 积分/会员订单表
op.create_table(
"points_orders",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("user_id", sa.String(36), nullable=False, index=True),
sa.Column("order_type", sa.String(20), nullable=False),
sa.Column("product_code", sa.String(50), nullable=False),
sa.Column("amount_cents", sa.Integer(), nullable=False),
sa.Column("original_amount_cents", sa.Integer(), nullable=False, server_default=sa.text("0")),
sa.Column("discount", sa.Float(), nullable=False, server_default=sa.text("1.0")),
sa.Column("points_amount", sa.Integer(), nullable=False, server_default=sa.text("0")),
sa.Column("status", sa.String(20), nullable=False, server_default="pending", index=True),
sa.Column("payment_method", sa.String(50), nullable=True),
sa.Column("payment_id", sa.String(100), nullable=True),
sa.Column("paid_at", sa.DateTime(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(),
nullable=False,
server_default=sa.text("NOW()"),
),
)
# 5. daily_usage_records 每日使用记录表
op.create_table(
"daily_usage_records",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("user_id", sa.String(36), nullable=False, index=True),
sa.Column("usage_date", sa.DateTime(), nullable=False),
sa.Column("usage_type", sa.String(50), nullable=False, server_default="free_clip"),
sa.Column("count", sa.Integer(), nullable=False, server_default=sa.text("0")),
sa.Column(
"updated_at",
sa.DateTime(),
nullable=False,
server_default=sa.text("NOW()"),
),
sa.UniqueConstraint(
"user_id",
"usage_date",
"usage_type",
name="uq_daily_usage_user_date_type",
),
)
def downgrade() -> None:
op.drop_table("daily_usage_records")
op.drop_table("points_orders")
op.drop_table("points_transactions")
op.drop_table("points_accounts")
with op.batch_alter_table("users") as batch:
batch.drop_column("points_balance")
batch.drop_column("member_expires_at")
batch.drop_column("member_type")
batch.drop_column("is_member")
-67
View File
@@ -1,67 +0,0 @@
"""#1894: merge title_libraries into scripts — add title_text/title_category/title_config
Revision ID: 077_merge_title_libs
Revises: 076_membership_points
Create Date: 2026-09-15
"""
import sqlalchemy as sa
from alembic import context, op
revision = "077_merge_title_libs"
down_revision = "076_membership_points"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("scripts") as batch:
batch.add_column(
sa.Column("title_text", sa.String(500), nullable=False, server_default=""),
)
batch.add_column(
sa.Column("title_category", sa.String(50), nullable=False, server_default=""),
)
batch.add_column(
sa.Column("title_config", sa.JSON, nullable=False, server_default="{}"),
)
if context.get_context().dialect.name == "postgresql":
conn = op.get_bind()
result = conn.execute(sa.text("SELECT to_regclass('public.title_libraries')"))
if result.scalar() is not None:
conn.execute(sa.text("""
INSERT INTO scripts
(id, user_id, title, content, segments, tags,
title_text, title_category, title_config,
created_at, updated_at)
SELECT
gen_random_uuid()::TEXT,
tl.user_id,
COALESCE(tl.name, '迁移标题'),
COALESCE(tl.text, ''),
'[]'::JSONB,
COALESCE(tl.tags, '[]'::JSONB),
COALESCE(tl.text, ''),
COALESCE(tl.category, ''),
COALESCE(tl."metadata", '{}'::JSONB),
tl.created_at,
tl.updated_at
FROM title_libraries tl
WHERE tl.is_active = true
AND NOT EXISTS (
SELECT 1 FROM scripts s
WHERE s.user_id = tl.user_id
AND s.title_text = COALESCE(tl.text, '')
AND s.title_category = COALESCE(tl.category, '')
AND s.created_at = tl.created_at
)
"""))
def downgrade() -> None:
with op.batch_alter_table("scripts") as batch:
batch.drop_column("title_config")
batch.drop_column("title_category")
batch.drop_column("title_text")
@@ -1,33 +0,0 @@
"""#1894: drop obsolete script title fields (title_text/title_category/title_config)
Revision ID: 078_drop_script_title_fields
Revises: 077_merge_title_libs
Create Date: 2026-09-16
口播文案(scripts)不再自带配套标题、标题分类和标题样式字段。
智能剪辑 / AI 数字人等生成场景各自通过入参配置标题,不再从文案读取。
保留字段:title(名称)、content(正文)、segments(分段)、tags(标签)。
"""
import sqlalchemy as sa
from alembic import op
revision = "078_drop_script_title_fields"
down_revision = "077_merge_title_libs"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("scripts") as batch:
batch.drop_column("title_config")
batch.drop_column("title_category")
batch.drop_column("title_text")
def downgrade() -> None:
with op.batch_alter_table("scripts") as batch:
batch.add_column(sa.Column("title_text", sa.String(500), nullable=False, server_default=""))
batch.add_column(sa.Column("title_category", sa.String(50), nullable=False, server_default=""))
batch.add_column(sa.Column("title_config", sa.JSON, nullable=False, server_default="{}"))
-58
View File
@@ -1,58 +0,0 @@
"""add asset_atom_clips table
Revision ID: 079_asset_atom_clips
Revises: 078_drop_script_title_fields
Create Date: 2026-09-17
"""
import sqlalchemy as sa
from alembic import op
revision = "079_asset_atom_clips"
down_revision = "078_drop_script_title_fields"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"asset_atom_clips",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column(
"asset_id",
sa.String(36),
sa.ForeignKey("assets.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("start_time", sa.Float(), nullable=False),
sa.Column("end_time", sa.Float(), nullable=False),
sa.Column("duration", sa.Float(), nullable=False),
sa.Column("clip_index", sa.Integer(), nullable=False),
sa.Column("tags", sa.JSON(), nullable=False, server_default=sa.text("'[]'")),
sa.Column("scene_change_at", sa.Float(), nullable=True),
sa.Column(
"is_fallback",
sa.Boolean(),
nullable=False,
server_default=sa.text("false"),
),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.text("NOW()"),
),
)
# 按素材查片段并按索引排序(复合索引前缀可独立用于 asset_id 过滤)
op.create_index(
"ix_asset_atom_clips_asset_index",
"asset_atom_clips",
["asset_id", "clip_index"],
unique=True,
)
def downgrade() -> None:
op.drop_index("ix_asset_atom_clips_asset_index", table_name="asset_atom_clips")
op.drop_table("asset_atom_clips")
@@ -1,37 +0,0 @@
"""add edit_plan_clips.atom_clip_id for #1970
Revision ID: 080_edit_plan_clips_atom_clip_id
Revises: 079_asset_atom_clips
Create Date: 2026-09-17
"""
import sqlalchemy as sa
from alembic import op
revision = "080_edit_plan_clips_atom_clip_id"
down_revision = "079_asset_atom_clips"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"edit_plan_clips",
sa.Column(
"atom_clip_id",
sa.String(36),
nullable=False,
server_default=sa.text("''"),
),
)
op.create_index(
"ix_edit_plan_clips_atom_clip_id",
"edit_plan_clips",
["atom_clip_id"],
)
def downgrade() -> None:
op.drop_index("ix_edit_plan_clips_atom_clip_id", table_name="edit_plan_clips")
op.drop_column("edit_plan_clips", "atom_clip_id")
@@ -1,58 +0,0 @@
"""add gpu_lipsync_tasks and gpu_workers tables for MuseTalk reverse-poll worker
Revision ID: 081_add_gpu_lipsync
Revises: 080_edit_plan_clips_atom_clip_id
Create Date: 2026-09-18
"""
import sqlalchemy as sa
from alembic import op
revision = "081_add_gpu_lipsync"
down_revision = "080_edit_plan_clips_atom_clip_id"
branch_labels = None
depends_on = None
def upgrade() -> None:
# GPU Worker 注册表
op.create_table(
"gpu_workers",
sa.Column("worker_id", sa.String(100), primary_key=True),
sa.Column("hostname", sa.String(200), nullable=False, server_default=""),
sa.Column("gpu_name", sa.String(200), nullable=False, server_default=""),
sa.Column("free_vram_mb", sa.Integer(), nullable=False, server_default=sa.text("0")),
sa.Column("capabilities", sa.String(500), nullable=False, server_default=""),
sa.Column("last_heartbeat_at", sa.DateTime(), nullable=True, index=True),
sa.Column("created_at", sa.DateTime(), nullable=False, server_default=sa.func.now()),
)
# GPU 口型同步任务表
op.create_table(
"gpu_lipsync_tasks",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("lipsync_job_id", sa.String(36), nullable=False, server_default="", index=True),
sa.Column("user_id", sa.String(36), nullable=False, server_default="", index=True),
sa.Column("project_id", sa.String(36), nullable=False, server_default="", index=True),
sa.Column("video_url", sa.Text(), nullable=False),
sa.Column("audio_url", sa.Text(), nullable=False),
sa.Column("result_url", sa.Text(), nullable=False, server_default=""),
sa.Column("result_duration", sa.Float(), nullable=False, server_default=sa.text("0.0")),
sa.Column("status", sa.String(20), nullable=False, server_default="pending", index=True),
sa.Column("worker_id", sa.String(100), nullable=False, server_default="", index=True),
sa.Column("attempt", sa.Integer(), nullable=False, server_default=sa.text("0")),
sa.Column("error_msg", sa.Text(), nullable=False, server_default=""),
sa.Column("created_at", sa.DateTime(), nullable=False, server_default=sa.func.now()),
sa.Column("started_at", sa.DateTime(), nullable=True),
sa.Column("finished_at", sa.DateTime(), nullable=True),
sa.Column("updated_at", sa.DateTime(), nullable=False, server_default=sa.func.now()),
sa.Column("last_heartbeat_at", sa.DateTime(), nullable=True),
)
op.create_index("ix_gpu_lipsync_status_created", "gpu_lipsync_tasks", ["status", "created_at"])
def downgrade() -> None:
op.drop_index("ix_gpu_lipsync_status_created", table_name="gpu_lipsync_tasks")
op.drop_table("gpu_lipsync_tasks")
op.drop_table("gpu_workers")
-26
View File
@@ -1,26 +0,0 @@
"""add ai_tags to asset_atom_clips for #1970 fragment-level AI tagging
Revision ID: 082_atom_clip_ai_tags
Revises: 081_add_gpu_lipsync
Create Date: 2026-09-18
"""
import sqlalchemy as sa
from alembic import op
revision = "082_atom_clip_ai_tags"
down_revision = "081_add_gpu_lipsync"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"asset_atom_clips",
sa.Column("ai_tags", sa.JSON(), nullable=True),
)
def downgrade() -> None:
op.drop_column("asset_atom_clips", "ai_tags")
@@ -1,26 +0,0 @@
"""#2001 爆款标题样式面板升级: ai_avatar_render_jobs 新增 cover_title_config
Revision ID: 083_cover_title_config
Revises: 082_atom_clip_ai_tags
Create Date: 2026-09-20
"""
import sqlalchemy as sa
from alembic import op
revision = "083_cover_title_config"
down_revision = "082_atom_clip_ai_tags"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"ai_avatar_render_jobs",
sa.Column("cover_title_config", sa.JSON(), nullable=False, server_default=sa.text("'{}'")),
)
def downgrade() -> None:
op.drop_column("ai_avatar_render_jobs", "cover_title_config")
@@ -1,26 +0,0 @@
"""lipsync_jobs 新增 style 字段(TTS 语气风格)
Revision ID: 084_lipsync_jobs_style
Revises: 083_cover_title_config
Create Date: 2026-09-21
"""
import sqlalchemy as sa
from alembic import op
revision = "084_lipsync_jobs_style"
down_revision = "083_cover_title_config"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"lipsync_jobs",
sa.Column("style", sa.String(length=32), nullable=False, server_default=""),
)
def downgrade() -> None:
op.drop_column("lipsync_jobs", "style")
@@ -1,33 +0,0 @@
"""asset_atom_clips 新增 caption/embedding 字段(#2035 语义标签增强)
Revision ID: 085_atom_clip_caption_embedding
Revises: 084_lipsync_jobs_style
Create Date: 2026-09-25
"""
import sqlalchemy as sa
from alembic import op
revision = "085_atom_clip_caption_embedding"
down_revision = "084_lipsync_jobs_style"
branch_labels = None
depends_on = None
def upgrade() -> None:
# caption: 中文画面描述(10-30字)
op.add_column(
"asset_atom_clips",
sa.Column("caption", sa.Text(), nullable=True),
)
# embedding: caption 对应的向量(豆包 embedding 接口返回,JSON 存 float 数组)
op.add_column(
"asset_atom_clips",
sa.Column("embedding", sa.JSON(), nullable=True),
)
def downgrade() -> None:
op.drop_column("asset_atom_clips", "embedding")
op.drop_column("asset_atom_clips", "caption")
@@ -1,100 +0,0 @@
"""add viral video tables
Revision ID: 086_add_viral_video_tables
Revises: 085_atom_clip_caption_embedding
Create Date: 2026-09-28
新增爆款视频相关表:
- viral_video_jobs: 爆款视频任务
- viral_video_style_templates: 风格模板配置
- viral_video_prompt_templates: Prompt 模板(由 #2040 seed)
"""
import sqlalchemy as sa
from alembic import op
revision = "086_add_viral_video_tables"
down_revision = "085_atom_clip_caption_embedding"
branch_labels = None
depends_on = None
def upgrade() -> None:
# viral_video_jobs
op.create_table(
"viral_video_jobs",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("user_id", sa.String(36), nullable=False, index=True),
sa.Column("images", sa.JSON(), nullable=False, server_default="[]"),
sa.Column("industry", sa.String(100), nullable=False, server_default=""),
sa.Column("target_customer", sa.String(500), nullable=False, server_default=""),
sa.Column("persona_id", sa.String(36), nullable=False, server_default=""),
sa.Column("viral_structure", sa.String(50), nullable=False, server_default=""),
sa.Column("marketing_purpose", sa.String(100), nullable=False, server_default=""),
sa.Column("bgm_preference", sa.String(50), nullable=False, server_default=""),
sa.Column("duration", sa.Integer(), nullable=False, server_default="30"),
sa.Column("user_copy_text", sa.Text(), nullable=False, server_default=""),
sa.Column("fusion_level", sa.String(20), nullable=False, server_default="ai_polish"),
sa.Column("reference_audio_path", sa.String(1000), nullable=False, server_default=""),
# v1.3 新增
sa.Column("reference_video_url", sa.String(1000), nullable=False, server_default=""),
sa.Column("style_strength", sa.String(20), nullable=False, server_default="medium"),
sa.Column("style_guide", sa.JSON(), nullable=True),
sa.Column("style_template_id", sa.String(36), nullable=False, server_default="", index=True),
# 状态与结果
sa.Column("status", sa.String(30), nullable=False, server_default="pending", index=True),
sa.Column("intent_result", sa.JSON(), nullable=True),
sa.Column("result_video_url", sa.String(1000), nullable=False, server_default=""),
sa.Column("credits_cost", sa.Integer(), nullable=False, server_default="0"),
sa.Column("error_msg", sa.Text(), nullable=False, server_default=""),
sa.Column("retry_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
)
# viral_video_style_templates
op.create_table(
"viral_video_style_templates",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("name", sa.String(200), nullable=False),
sa.Column("description", sa.Text(), nullable=False, server_default=""),
sa.Column("thumbnail_url", sa.String(1000), nullable=False, server_default=""),
sa.Column("style_config", sa.JSON(), nullable=False, server_default="{}"),
sa.Column("is_system", sa.Boolean(), nullable=False, server_default=sa.text("true"), index=True),
sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
)
# viral_video_prompt_templates
op.create_table(
"viral_video_prompt_templates",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("prompt_type", sa.String(50), nullable=False, index=True),
sa.Column("name", sa.String(200), nullable=False),
sa.Column("content", sa.Text(), nullable=False, server_default=""),
sa.Column("variables", sa.JSON(), nullable=False, server_default="[]"),
sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.text("true"), index=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
)
# Seed 默认风格模板
op.execute("""
INSERT INTO viral_video_style_templates (id, name, description, style_config, is_system, sort_order)
VALUES
('style-tpl-001', '快节奏冲击', '高频切镜+动感BGM,适合食品饮料等快消品', '{"cut_speed": "fast", "transition": "jump_cut", "energy": "high"}', true, 1),
('style-tpl-002', '质感慢镜', '慢节奏+电影感调色,适合美妆护肤珠宝', '{"cut_speed": "slow", "transition": "dissolve", "energy": "low", "color_grade": "cinematic"}', true, 2),
('style-tpl-003', '口播种草', '数字人口播+产品特写穿插', '{"cut_speed": "medium", "transition": "cross_dissolve", "has_talking_head": true}', true, 3),
('style-tpl-004', '场景叙事', '多场景切换+故事线叙述', '{"cut_speed": "medium", "transition": "wipe", "narrative": true}', true, 4)
""")
def downgrade() -> None:
op.drop_table("viral_video_prompt_templates")
op.drop_table("viral_video_style_templates")
op.drop_table("viral_video_jobs")
@@ -1,25 +0,0 @@
"""viral video add image_analysis column
Revision ID: 087_viral_video_image_analysis
Revises: 086_add_viral_video_tables
Create Date: 2026-09-30
#2106 爆款视频 P0:持久化图片分析结果(image_analysis JSON),供 resume 阶段使用。
"""
import sqlalchemy as sa
from alembic import op
revision = "087_viral_video_image_analysis"
down_revision = "086_add_viral_video_tables"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("viral_video_jobs", sa.Column("image_analysis", sa.JSON(), nullable=True))
def downgrade() -> None:
op.drop_column("viral_video_jobs", "image_analysis")
@@ -1,51 +0,0 @@
"""viral video add copy_result + voice/video columns
Revision ID: 088_viral_video_copy_result
Revises: 087_viral_video_image_analysis
Create Date: 2026-10-01
v1.6 爆款视频字段补齐:
- copy_result JSON: 编导分镜脚本完整结构(overview/scene_and_lighting/shots/hard_constraints/negative_prompts/voiceover_script)
- voice_id/voice_source: TTS 音色参数
- video_ratio/video_model: Seedance 视频比例/模型
注意:线上启动也有幂等 ADD COLUMN 补列逻辑 (_ensure_viral_video_columns),本 migration 提供标准 Alembic 路径,
两套机制互不冲突(IF NOT EXISTS 等价行为)。
"""
import sqlalchemy as sa
from alembic import op
revision = "088_viral_video_copy_result"
down_revision = "087_viral_video_image_analysis"
branch_labels = None
depends_on = None
def upgrade() -> None:
# 幂等添加列(通过单独执行 + 异常忽略兼容已由 backfill 补上的环境)
cols = [
("voice_id", "VARCHAR(200) NOT NULL DEFAULT ''"),
("voice_source", "VARCHAR(20) NOT NULL DEFAULT ''"),
("video_ratio", "VARCHAR(10) NOT NULL DEFAULT '9:16'"),
("video_model", "VARCHAR(100) NOT NULL DEFAULT ''"),
("copy_result", "JSON"),
]
conn = op.get_bind()
for name, ddl in cols:
try:
conn.execute(sa.text(f"ALTER TABLE viral_video_jobs ADD COLUMN IF NOT EXISTS {name} {ddl}"))
except Exception:
# 不支持 IF NOT EXISTS 的库(如老版本 SQLite)直接尝试 ADD COLUMN,失败则忽略
try:
conn.execute(sa.text(f"ALTER TABLE viral_video_jobs ADD COLUMN {name} {ddl}"))
except Exception:
pass
def downgrade() -> None:
for name in ("copy_result", "video_model", "video_ratio", "voice_source", "voice_id"):
try:
op.drop_column("viral_video_jobs", name)
except Exception:
pass
-62
View File
@@ -1,62 +0,0 @@
"""viral video add storyboard + generated_copy_text (complement 088)
Revision ID: 089_viral_video_cols
Revises: 088_viral_video_copy_result
Create Date: 2026-10-01
#2129 兜底迁移:补齐 _VIRAL_VIDEO_BACKFILL_COLS 中所有列,覆盖
# watchtower 自动部署未跑历史 migration、且 AUTO_CREATE_SCHEMA=false 时
# _ensure_viral_video_columns 未执行的场景。
# 幂等 ADD COLUMN IF NOT EXISTS,已存在则跳过。
"""
import sqlalchemy as sa
from alembic import op
revision = "089_viral_video_cols"
down_revision = "088_viral_video_copy_result"
branch_labels = None
depends_on = None
def upgrade() -> None:
# 扩展 alembic_version.version_num 字段长度(原来 VARCHAR(32) 装不下长 revision id)
conn = op.get_bind()
try:
conn.execute(sa.text("ALTER TABLE alembic_version ALTER COLUMN version_num TYPE VARCHAR(256)"))
except Exception:
pass
cols = [
("storyboard", "JSON"),
("generated_copy_text", "TEXT NOT NULL DEFAULT ''"),
("voice_id", "VARCHAR(200) NOT NULL DEFAULT ''"),
("voice_source", "VARCHAR(20) NOT NULL DEFAULT ''"),
("video_ratio", "VARCHAR(10) NOT NULL DEFAULT '9:16'"),
("video_model", "VARCHAR(100) NOT NULL DEFAULT ''"),
("copy_result", "JSON"),
]
for name, ddl in cols:
try:
conn.execute(sa.text(f"ALTER TABLE viral_video_jobs ADD COLUMN IF NOT EXISTS {name} {ddl}"))
except Exception:
try:
conn.execute(sa.text(f"ALTER TABLE viral_video_jobs ADD COLUMN {name} {ddl}"))
except Exception:
pass
def downgrade() -> None:
for name in (
"copy_result",
"video_model",
"video_ratio",
"voice_source",
"voice_id",
"generated_copy_text",
"storyboard",
):
try:
op.drop_column("viral_video_jobs", name)
except Exception:
pass
@@ -1,35 +0,0 @@
"""viral video add phase_message column (#2134)
Revision ID: 090_viral_video_phase_msg
Revises: 089_viral_video_cols
Create Date: 2026-10-02
#2134 阶段细粒度提示:viral_video 表新增 phase_message 列(中文阶段提示文案)。
current_stage 列已在之前版本存在,本迁移只补 phase_message。
幂等 ADD COLUMN IF NOT EXISTS。
"""
import sqlalchemy as sa
from alembic import op
revision = "090_viral_video_phase_msg"
down_revision = "089_viral_video_cols"
branch_labels = None
depends_on = None
def upgrade() -> None:
# SQLite/PostgreSQL 兼容的幂等添加列
conn = op.get_bind()
inspector = sa.inspect(conn)
cols = {c["name"] for c in inspector.get_columns("viral_video_jobs")}
if "phase_message" not in cols:
op.add_column(
"viral_video_jobs",
sa.Column("phase_message", sa.String(length=500), nullable=False, server_default=""),
)
def downgrade() -> None:
op.drop_column("viral_video_jobs", "phase_message")
-49
View File
@@ -1,49 +0,0 @@
"""viral video add current_stage column (#2137 follow-up)
Revision ID: 091_viral_video_stage
Revises: 090_viral_video_phase_msg
Create Date: 2026-10-02
#2137 follow-up fix: 090 migration missed current_stage column on viral_video_jobs,
causing UndefinedColumn errors and 500s on all authenticated viral-video endpoints.
Idempotently add current_stage and double-check phase_message.
"""
import sqlalchemy as sa
from alembic import op
revision = "091_viral_video_stage"
down_revision = "090_viral_video_phase_msg"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
cols = {c["name"] for c in inspector.get_columns("viral_video_jobs")}
if "current_stage" not in cols:
op.add_column(
"viral_video_jobs",
sa.Column(
"current_stage",
sa.String(length=200),
nullable=False,
server_default="",
),
)
if "phase_message" not in cols:
op.add_column(
"viral_video_jobs",
sa.Column(
"phase_message",
sa.String(length=500),
nullable=False,
server_default="",
),
)
def downgrade() -> None:
op.drop_column("viral_video_jobs", "current_stage")
@@ -1,42 +0,0 @@
"""viral_video_jobs 增加 heartbeat_at 列(worker 心跳,用于僵尸任务超时回收)
Revision ID: 092_viral_video_heartbeat
Revises: 091_viral_video_stage
Create Date: 2026-10-02
"""
import sqlalchemy as sa
from alembic import op
revision = "092_viral_video_heartbeat"
down_revision = "091_viral_video_stage"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
cols = {c["name"] for c in inspector.get_columns("viral_video_jobs")}
if "heartbeat_at" not in cols:
op.add_column("viral_video_jobs", sa.Column("heartbeat_at", sa.DateTime(), nullable=True))
op.execute(
"UPDATE viral_video_jobs SET heartbeat_at = updated_at " "WHERE status = 'running' AND heartbeat_at IS NULL"
)
try:
op.create_index("ix_viral_video_jobs_heartbeat_at", "viral_video_jobs", ["heartbeat_at"])
except Exception:
pass
def downgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
cols = {c["name"] for c in inspector.get_columns("viral_video_jobs")}
if "heartbeat_at" in cols:
try:
op.drop_index("ix_viral_video_jobs_heartbeat_at", table_name="viral_video_jobs")
except Exception:
pass
op.drop_column("viral_video_jobs", "heartbeat_at")
@@ -1,87 +0,0 @@
"""viral_video 动态积分定价 + 积分字段从 Integer 改为 Float (#2151)
Revision ID: 093
Revises: 092_viral_video_heartbeat
Create Date: 2026-10-02
"""
import sqlalchemy as sa
from alembic import op
revision = "093"
down_revision = "092_viral_video_heartbeat"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
# 1) points_accounts 三列 Integer -> Float
pa_cols = {c["name"]: c for c in inspector.get_columns("points_accounts")}
for col in ("balance", "total_earned", "total_spent"):
if col in pa_cols:
op.alter_column(
"points_accounts",
col,
existing_type=sa.Integer(),
type_=sa.Float(),
existing_nullable=False,
)
# 2) points_transactions amount/balance_after Integer -> Float
pt_cols = {c["name"]: c for c in inspector.get_columns("points_transactions")}
for col in ("amount", "balance_after"):
if col in pt_cols:
op.alter_column(
"points_transactions",
col,
existing_type=sa.Integer(),
type_=sa.Float(),
existing_nullable=False,
)
# 3) users.points_balance Integer -> Float
user_cols = {c["name"]: c for c in inspector.get_columns("users")}
if "points_balance" in user_cols:
op.alter_column(
"users",
"points_balance",
existing_type=sa.Integer(),
type_=sa.Float(),
existing_nullable=False,
)
# 4) viral_video_jobs.credits_cost Integer -> Float
vv_cols = {c["name"]: c for c in inspector.get_columns("viral_video_jobs")}
if "credits_cost" in vv_cols:
op.alter_column(
"viral_video_jobs",
"credits_cost",
existing_type=sa.Integer(),
type_=sa.Float(),
existing_nullable=False,
)
# 5) viral_video_jobs 新增列
if "video_resolution" not in vv_cols:
op.add_column(
"viral_video_jobs",
sa.Column("video_resolution", sa.String(20), nullable=False, server_default="720p"),
)
if "credits_prepaid" not in vv_cols:
op.add_column(
"viral_video_jobs",
sa.Column("credits_prepaid", sa.Float(), nullable=False, server_default="0"),
)
if "credits_transaction_id" not in vv_cols:
op.add_column(
"viral_video_jobs",
sa.Column("credits_transaction_id", sa.String(36), nullable=False, server_default=""),
)
def downgrade() -> None:
pass
@@ -1,31 +0,0 @@
"""viral_video_jobs 增加 pre_trusted_images 列(信任链Seedream预热结果)
Revision ID: 094_viral_video_pre_trusted
Revises: 093_viral_video_pricing_points_float
Create Date: 2026-10-04
"""
import sqlalchemy as sa
from alembic import op
revision = "094_viral_video_pre_trusted"
down_revision = "093"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
cols = {c["name"] for c in inspector.get_columns("viral_video_jobs")}
if "pre_trusted_images" not in cols:
op.add_column("viral_video_jobs", sa.Column("pre_trusted_images", sa.Text(), nullable=True))
def downgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
cols = {c["name"] for c in inspector.get_columns("viral_video_jobs")}
if "pre_trusted_images" in cols:
op.drop_column("viral_video_jobs", "pre_trusted_images")
@@ -1,102 +0,0 @@
"""爆款视频 Prompt 模板配置表(#2040)。
086 曾预留同名旧表(id varchar / content / variables json),从未被业务使用;
本迁移将其替换为 #2040 新结构。
Revision ID: 095_viral_video_prompt_templates
Revises: 094_viral_video_pre_trusted
Create Date: 2026-10-04
"""
import sqlalchemy as sa
from alembic import op
revision = "095_viral_video_prompt_templates"
down_revision = "094_viral_video_pre_trusted"
branch_labels = None
depends_on = None
def _table_exists(conn, name: str) -> bool:
return name in sa.inspect(conn).get_table_names()
def upgrade() -> None:
conn = op.get_bind()
# 086 预留的旧结构表:先删除(无业务数据、无任何引用)
if _table_exists(conn, "viral_video_prompt_templates"):
op.drop_table("viral_video_prompt_templates")
op.create_table(
"viral_video_prompt_templates",
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
sa.Column("name", sa.String(128), nullable=False),
sa.Column("prompt_type", sa.String(32), nullable=False),
sa.Column("version", sa.Integer, nullable=False, server_default="1"),
sa.Column("system_prompt", sa.Text, nullable=False),
sa.Column("user_prompt_template", sa.Text, nullable=False),
sa.Column("example_output", sa.Text, nullable=True),
sa.Column("is_active", sa.Boolean, nullable=False, server_default=sa.text("true")),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.func.now(),
nullable=False,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.func.now(),
nullable=False,
),
)
op.create_index(
"ix_vvpt_type_active",
"viral_video_prompt_templates",
["prompt_type", "is_active"],
)
op.create_index(
"uq_vvpt_type_version",
"viral_video_prompt_templates",
["prompt_type", "version"],
unique=True,
)
def downgrade() -> None:
conn = op.get_bind()
if _table_exists(conn, "viral_video_prompt_templates"):
op.drop_index("uq_vvpt_type_version", table_name="viral_video_prompt_templates")
op.drop_index("ix_vvpt_type_active", table_name="viral_video_prompt_templates")
op.drop_table("viral_video_prompt_templates")
# 恢复 086 的旧预留结构
op.create_table(
"viral_video_prompt_templates",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("prompt_type", sa.String(50), nullable=False, index=True),
sa.Column("name", sa.String(200), nullable=False),
sa.Column("content", sa.Text, nullable=False, server_default=""),
sa.Column("variables", sa.JSON, nullable=False, server_default="[]"),
sa.Column("version", sa.Integer, nullable=False, server_default="1"),
sa.Column(
"is_active",
sa.Boolean,
nullable=False,
server_default=sa.text("true"),
index=True,
),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
)
@@ -1,61 +0,0 @@
"""功能计费积分字段(爆款/对口型/智能剪辑 DB 化计费)。
给 gpu_lipsync_tasks / generation_tasks / lipsync_jobs 三张表加积分字段:
- credits_prepaid: 提交任务时预扣积分
- credits_cost: 最终结算积分
- credits_transaction_id: 预扣流水 ID
注意:feature_pricing_configs 配置表由 xiaoxia-admin 侧 migration 建立,
本仓库只读,不在此创建。
Revision ID: 096_feature_billing_fields
Revises: 095_viral_video_prompt_templates
Create Date: 2026-10-05
"""
import sqlalchemy as sa
from alembic import op
revision = "096_feature_billing_fields"
down_revision = "095_viral_video_prompt_templates"
branch_labels = None
depends_on = None
_TABLES = ("gpu_lipsync_tasks", "generation_tasks", "lipsync_jobs")
_COLUMNS = (
("credits_prepaid", sa.Float(), "0"),
("credits_cost", sa.Float(), "0"),
("credits_transaction_id", sa.String(36), ""),
)
def _table_exists(conn, name: str) -> bool:
return name in sa.inspect(conn).get_table_names()
def upgrade() -> None:
conn = op.get_bind()
for table in _TABLES:
if not _table_exists(conn, table):
continue
existing = {c["name"] for c in sa.inspect(conn).get_columns(table)}
for col_name, col_type, default in _COLUMNS:
if col_name in existing:
continue
op.add_column(
table,
sa.Column(col_name, col_type, nullable=False, server_default=default),
)
def downgrade() -> None:
conn = op.get_bind()
for table in _TABLES:
if not _table_exists(conn, table):
continue
existing = {c["name"] for c in sa.inspect(conn).get_columns(table)}
for col_name, _col_type, _default in _COLUMNS:
if col_name not in existing:
continue
op.drop_column(table, col_name)
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,222 +0,0 @@
# -*- coding: utf-8 -*-
"""099: AI 模型路由层 seed — 补齐缺失模型和能力配置.
幂等:所有 INSERT 先检查存在性。
- ai_models: 补齐 qwen3.7-plus, seedream, seedance, embedding, wan3.0 等
- ai_capability_configs: 补齐 image_generation, video_generation, embedding
- 更新已有 capability 的 lite_model_id
"""
import json
import sqlalchemy as sa
from alembic import op
revision = "099_ai_model_router_seed"
down_revision = "098_viral_video_image_analysis_v5"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
# CI 环境下 ai_models 表可能尚未创建(由 ORM 自动建表,非 migration)
# 如果表不存在则跳过 seed,由应用启动时 ORM 建表后首次访问时生效
table_check = conn.execute(sa.text("SELECT to_regclass('public.ai_models')")).scalar()
if not table_check:
# ai_models 表不存在,跳过所有 seed(CI 环境)
return
# ── 1. 补齐 ai_models 缺失记录 ────────────────────────────────────────────
existing_models = {
row[0]
for row in conn.execute(
sa.text("SELECT model_key FROM ai_models WHERE deleted_at IS NULL")
).fetchall()
}
# 从已有 active 记录获取 API key(复用,不硬编码)
dashscope_key_row = conn.execute(
sa.text(
"SELECT api_key FROM ai_models WHERE provider='dashscope' AND deleted_at IS NULL AND api_key IS NOT NULL AND api_key != '' LIMIT 1"
)
).first()
dashscope_key = dashscope_key_row[0] if dashscope_key_row else ""
volcengine_key_row = conn.execute(
sa.text(
"SELECT api_key FROM ai_models WHERE provider='volcengine' AND deleted_at IS NULL AND api_key IS NOT NULL AND api_key != '' LIMIT 1"
)
).first()
volcengine_key = volcengine_key_row[0] if volcengine_key_row else ""
new_models = [
{
"model_key": "qwen3.7-plus",
"name": "通义千问3.7 Plus(VLM 兜底)",
"provider": "dashscope",
"api_key": dashscope_key,
"api_base": "https://dashscope.aliyuncs.com/compatible-mode/v1",
"description": "阿里云百炼 Qwen3.7 Plus 多模态模型,用于 VLM 兜底分析",
},
{
"model_key": "doubao-seedream-5-0-flash-260915",
"name": "Seedream 5.0 Flash(图片生成)",
"provider": "volcengine",
"api_key": volcengine_key,
"api_base": "https://ark.cn-beijing.volces.com/api/v3",
"description": "火山引擎 Seedream 5.0 Flash 文生图模型",
},
{
"model_key": "doubao-seedance-2-5-260628",
"name": "Seedance 2.5(视频生成)",
"provider": "volcengine",
"api_key": volcengine_key,
"api_base": "https://ark.cn-beijing.volces.com/api/v3",
"description": "火山引擎 Seedance 2.5 图/文生视频模型",
},
{
"model_key": "doubao-embedding-vision-251215",
"name": "豆包多模态向量嵌入",
"provider": "volcengine",
"api_key": volcengine_key,
"api_base": "https://ark.cn-beijing.volces.com/api/v3",
"description": "火山引擎豆包多模态向量嵌入模型",
},
{
"model_key": "wan3.0-video",
"name": "Wan 3.0 视频生成",
"provider": "dashscope",
"api_key": dashscope_key,
"api_base": "https://dashscope.aliyuncs.com/api/v1",
"description": "阿里云百炼 Wan 3.0 视频生成模型",
},
{
"model_key": "doubao-seed-2-1-pro-260915",
"name": "豆包 Seed 2.1 Pro(高精度推理)",
"provider": "volcengine",
"api_key": volcengine_key,
"api_base": "https://ark.cn-beijing.volces.com/api/v3",
"description": "火山引擎豆包 Seed 2.1 Pro 深度思考+多模态",
},
]
for m in new_models:
if m["model_key"] not in existing_models:
conn.execute(
sa.text(
"""
INSERT INTO ai_models (id, name, provider, model_key, api_key, api_base, description, status, is_default, usage_today, created_at, updated_at)
VALUES (gen_random_uuid()::text, :name, :provider, :model_key, :api_key, :api_base, :description, 'active', false, 0, now(), now())
"""
),
m,
)
# ── 2. 补齐 ai_capability_configs 缺失项 ──────────────────────────────────
cap_table_check = conn.execute(sa.text("SELECT to_regclass('public.ai_capability_configs')")).scalar()
if not cap_table_check:
return
existing_caps = {
row[0]
for row in conn.execute(
sa.text("SELECT capability_key FROM ai_capability_configs")
).fetchall()
}
def _get_model_id(model_key: str) -> str | None:
row = conn.execute(
sa.text(
"SELECT id FROM ai_models WHERE model_key = :key AND deleted_at IS NULL AND status = 'active' LIMIT 1"
),
{"key": model_key},
).first()
return row[0] if row else None
# image_generation
if "image_generation" not in existing_caps:
mid = _get_model_id("doubao-seedream-5-0-flash-260915")
if mid:
conn.execute(
sa.text(
"""
INSERT INTO ai_capability_configs (id, capability_key, capability_name, primary_model_id, timeout_seconds, max_retries, concurrency, extra_params, is_enabled, created_at, updated_at)
VALUES (gen_random_uuid()::text, :ck, :cn, :pm, 60, 1, 2, :ep, true, now(), now())
"""
),
{
"ck": "image_generation",
"cn": "图片生成(Seedream)",
"pm": mid,
"ep": json.dumps({"size": "1K"}),
},
)
# video_generation
if "video_generation" not in existing_caps:
mid = _get_model_id("doubao-seedance-2-5-260628")
fb_mid = _get_model_id("wan3.0-video")
if mid:
conn.execute(
sa.text(
"""
INSERT INTO ai_capability_configs (id, capability_key, capability_name, primary_model_id, fallback_model_id, timeout_seconds, max_retries, concurrency, extra_params, is_enabled, created_at, updated_at)
VALUES (gen_random_uuid()::text, :ck, :cn, :pm, :fm, 600, 1, 1, :ep, true, now(), now())
"""
),
{
"ck": "video_generation",
"cn": "视频生成(Seedance/Wan)",
"pm": mid,
"fm": fb_mid,
"ep": json.dumps({}),
},
)
# embedding
if "embedding" not in existing_caps:
mid = _get_model_id("doubao-embedding-vision-251215")
if mid:
conn.execute(
sa.text(
"""
INSERT INTO ai_capability_configs (id, capability_key, capability_name, primary_model_id, timeout_seconds, max_retries, concurrency, extra_params, is_enabled, created_at, updated_at)
VALUES (gen_random_uuid()::text, :ck, :cn, :pm, 30, 2, 5, :ep, true, now(), now())
"""
),
{
"ck": "embedding",
"cn": "向量嵌入",
"pm": mid,
"ep": json.dumps({}),
},
)
# ── 3. 更新 image_analysis 的 lite_model_id ─────────────────────────────
lite_model_id = _get_model_id("qwen3.8-flash")
if lite_model_id:
conn.execute(
sa.text(
"UPDATE ai_capability_configs SET lite_model_id = :lite WHERE capability_key = 'image_analysis' AND lite_model_id IS NULL"
),
{"lite": lite_model_id},
)
def downgrade() -> None:
conn = op.get_bind()
# 安全检查表是否存在
table_check = conn.execute(sa.text("SELECT to_regclass('public.ai_models')")).scalar()
if not table_check:
return
conn.execute(
sa.text("DELETE FROM ai_capability_configs WHERE capability_key IN ('image_generation', 'video_generation', 'embedding')")
)
conn.execute(
sa.text(
"DELETE FROM ai_models WHERE model_key IN ('qwen3.7-plus', 'doubao-seedream-5-0-flash-260915', 'doubao-seedance-2-5-260628', 'doubao-embedding-vision-251215', 'wan3.0-video', 'doubao-seed-2-1-pro-260915') AND deleted_at IS NULL"
)
)
@@ -1,107 +0,0 @@
# -*- coding: utf-8 -*-
"""100: 修正已有 capability 的模型绑定.
幂等:仅当 primary_model_id 当前绑定到旧模型 (doubao-seed-1-6) 时才更新,
避免覆盖用户在后台的自定义配置。
- 更新 5 个 LLM capability (intent_parsing, copy_fusion, storyboard, copy_review, asset_classify)
的 primary_model_id 从 doubao-seed-1-6 改为 doubao-seed-2-1-pro-260915
- 更新 image_analysis 的 primary/lite/fallback 模型绑定
"""
import sqlalchemy as sa
from alembic import op
revision = "100_fix_capability_model_bindings"
down_revision = "099_ai_model_router_seed"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
# Check tables exist
table_check = conn.execute(sa.text("SELECT to_regclass('public.ai_models')")).scalar()
if not table_check:
return
config_table_check = conn.execute(sa.text("SELECT to_regclass('public.ai_capability_configs')")).scalar()
if not config_table_check:
return
# Look up model IDs by model_key (not hardcoded UUIDs)
pro_model_row = conn.execute(
sa.text(
"SELECT id FROM ai_models WHERE model_key = 'doubao-seed-2-1-pro-260915' AND deleted_at IS NULL LIMIT 1"
)
).first()
if not pro_model_row:
return
pro_model_id = pro_model_row[0]
old_model_row = conn.execute(
sa.text("SELECT id FROM ai_models WHERE model_key = 'doubao-seed-1-6-250615' LIMIT 1")
).first()
old_model_id = old_model_row[0] if old_model_row else None
llm_capabilities = [
"intent_parsing",
"copy_fusion",
"storyboard",
"copy_review",
"asset_classify",
]
for cap_key in llm_capabilities:
if old_model_id:
conn.execute(
sa.text(
"UPDATE ai_capability_configs SET primary_model_id = :new_id, updated_at = NOW() "
"WHERE capability_key = :cap_key AND primary_model_id = :old_id"
),
{"new_id": pro_model_id, "old_id": old_model_id, "cap_key": cap_key},
)
# Update image_analysis
qwen38_row = conn.execute(
sa.text("SELECT id FROM ai_models WHERE model_key = 'qwen3.8-flash' AND deleted_at IS NULL LIMIT 1")
).first()
qwen37_row = conn.execute(
sa.text("SELECT id FROM ai_models WHERE model_key = 'qwen3.7-plus' AND deleted_at IS NULL LIMIT 1")
).first()
if qwen38_row and qwen37_row:
qwen38_id = qwen38_row[0]
qwen37_id = qwen37_row[0]
current_ia = conn.execute(
sa.text(
"SELECT primary_model_id, lite_model_id, fallback_model_id "
"FROM ai_capability_configs WHERE capability_key = 'image_analysis'"
)
).first()
if current_ia:
current_primary, current_lite, current_fallback = current_ia
updates = {}
if current_primary != qwen38_id:
updates["primary_model_id"] = qwen38_id
if current_lite != qwen38_id:
updates["lite_model_id"] = qwen38_id
if current_fallback != qwen37_id:
updates["fallback_model_id"] = qwen37_id
if updates:
set_clause = ", ".join([f"{k} = :{k}" for k in updates.keys()])
set_clause += ", updated_at = NOW()"
updates["cap_key"] = "image_analysis"
conn.execute(
sa.text(f"UPDATE ai_capability_configs SET {set_clause} WHERE capability_key = :cap_key"),
updates,
)
def downgrade() -> None:
pass
@@ -1,153 +0,0 @@
# -*- coding: utf-8 -*-
"""101: 补齐 qwen-vl-plus 视觉模型并修正 image_analysis 绑定与 max_tokens.
背景:
- qwen-vl-plus 做图片识别时返回 JSON 约 500-600 tokens,旧硬编码
max_tokens=350 导致 JSON 被截断、解析失败返回"未识别"。
- 代码侧已移除硬编码,改由 capability 的 DB 配置决定 max_tokens。
幂等:
- qwen-vl-plus 已存在则不插入;
- 仅当 image_analysis 当前 primary_model 不是 qwen-vl-plus 时才更新绑定,
避免覆盖后台手动配置。
"""
import sqlalchemy as sa
from alembic import op
revision = "101_qwen_vl_plus_and_max_tokens"
down_revision = "100_fix_capability_model_bindings"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
models_table = conn.execute(sa.text("SELECT to_regclass('public.ai_models')")).scalar()
if not models_table:
return
caps_table = conn.execute(sa.text("SELECT to_regclass('public.ai_capability_configs')")).scalar()
if not caps_table:
return
# ── c. 补全其他 capability 的 max_tokens 默认值(幂等)──────────────────
# 放在 image_analysis 特定逻辑之前,确保任何分支 return 都不会跳过本段。
# 仅在当前值为 NULL 或过小 (<100) 时更新,不覆盖已有合理配置。
# embedding / tts / voice_clone 不走 chat 接口,无需设置。
default_max_tokens = {
"intent_parsing": 500,
"copy_fusion": 2500,
"storyboard": 4000,
"copy_review": 1000,
"asset_classify": 500,
"image_generation": 500,
"video_generation": 500,
}
for cap_key, mt in default_max_tokens.items():
conn.execute(
sa.text(
"UPDATE ai_capability_configs "
"SET max_tokens = :mt, updated_at = now() "
"WHERE capability_key = :key "
"AND (max_tokens IS NULL OR max_tokens < 100)"
),
{"mt": mt, "key": cap_key},
)
# ── a. 确保 qwen-vl-plus 模型存在 ────────────────────────────────────────
conn.execute(sa.text("""
INSERT INTO ai_models (id, name, provider, model_key, api_key, api_base,
description, status, is_default, usage_today,
created_at, updated_at)
SELECT gen_random_uuid()::text,
'通义千问VL Plus',
'dashscope',
'qwen-vl-plus',
COALESCE(
(SELECT api_key FROM ai_models
WHERE provider = 'dashscope' AND deleted_at IS NULL
AND api_key IS NOT NULL AND api_key != ''
LIMIT 1),
''
),
'https://dashscope.aliyuncs.com/compatible-mode/v1',
'阿里云视觉理解模型(图片识别/分析)',
'active', false, 0, now(), now()
WHERE NOT EXISTS (
SELECT 1 FROM ai_models
WHERE model_key = 'qwen-vl-plus' AND deleted_at IS NULL
)
"""))
qwen_vl_row = conn.execute(
sa.text(
"SELECT id FROM ai_models WHERE model_key = 'qwen-vl-plus' "
"AND deleted_at IS NULL AND status = 'active' LIMIT 1"
)
).first()
if not qwen_vl_row:
return
qwen_vl_id = qwen_vl_row[0]
qwen37_row = conn.execute(
sa.text(
"SELECT id FROM ai_models WHERE model_key = 'qwen3.7-plus' "
"AND deleted_at IS NULL AND status = 'active' LIMIT 1"
)
).first()
qwen37_id = qwen37_row[0] if qwen37_row else None
# ── b. 仅当当前 primary 不是 qwen-vl-plus 时修正绑定与 max_tokens ───────
current = conn.execute(
sa.text(
"SELECT primary_model_id, lite_model_id, fallback_model_id, max_tokens "
"FROM ai_capability_configs WHERE capability_key = 'image_analysis'"
)
).first()
if current is None:
# capability 不存在则创建
conn.execute(
sa.text("""
INSERT INTO ai_capability_configs
(id, capability_key, capability_name, primary_model_id,
lite_model_id, fallback_model_id, timeout_seconds,
max_retries, max_tokens, concurrency, extra_params,
is_enabled, created_at, updated_at)
VALUES (gen_random_uuid()::text, 'image_analysis', '图片分析',
:primary, :primary, :fallback, 30, 1, 1000, 2,
'{}'::jsonb, true, now(), now())
"""),
{"primary": qwen_vl_id, "fallback": qwen37_id},
)
return
current_primary = current[0]
if current_primary == qwen_vl_id:
# 已经绑定 qwen-vl-plus:视为后台/数据迁移已处理,不覆盖任何配置
return
set_parts = [
"primary_model_id = :vl_id",
"lite_model_id = :vl_id",
"max_tokens = 1000",
"updated_at = now()",
]
params: dict = {"vl_id": qwen_vl_id}
if qwen37_id is not None:
set_parts.insert(2, "fallback_model_id = :qwen37_id")
params["qwen37_id"] = qwen37_id
conn.execute(
sa.text(
"UPDATE ai_capability_configs SET " + ", ".join(set_parts) + " WHERE capability_key = 'image_analysis'"
),
params,
)
def downgrade() -> None:
pass
@@ -1,36 +0,0 @@
# -*- coding: utf-8 -*-
"""102: image_analysis max_tokens 1200 -> 1500.
v6 prompt 更长、字段更多,旧 max_tokens 容易截断 JSON。
仅在 image_analysis 当前 max_tokens < 1500 时更新(幂等,不覆盖后台已调到 >=1500 的配置)。
"""
import sqlalchemy as sa
from alembic import op
revision = "102_image_analysis_max_tokens_1500"
down_revision = "101_qwen_vl_plus_and_max_tokens"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
caps_table = conn.execute(sa.text("SELECT to_regclass('public.ai_capability_configs')")).scalar()
if not caps_table:
return
conn.execute(
sa.text(
"UPDATE ai_capability_configs "
"SET max_tokens = 1500, updated_at = now() "
"WHERE capability_key = 'image_analysis' "
"AND (max_tokens IS NULL OR max_tokens < 1500)"
)
)
def downgrade() -> None:
pass
BIN
View File
Binary file not shown.
View File
View File
+4 -109
View File
@@ -1,42 +1,25 @@
from app.api.routes.ai import router as ai_router
from app.api.routes.ai_avatar_render import router as ai_avatar_render_router
from app.api.routes.asset_diagnosis import router as asset_diagnosis_router
from app.api.routes.asset_libraries import router as asset_libraries_router
from app.api.routes.assets import router as assets_router
from app.api.routes.auth import router as auth_router
from app.api.routes.chunked_upload import router as chunked_upload_router
from app.api.routes.classification_jobs import router as classification_jobs_router
from app.api.routes.clips_standalone import router as clips_standalone_router
from app.api.routes.cover_templates import router as cover_templates_router
from app.api.routes.drafts_standalone import router as drafts_standalone_router
from app.api.routes.duplication import router as duplication_router
from app.api.routes.edit_plans import router as edit_plans_router
from app.api.routes.feature_flags import router as feature_flags_router
from app.api.routes.generation_cover import router as generation_cover_router
from app.api.routes.generation_preview import router as generation_preview_router
from app.api.routes.generation_tasks import router as generation_tasks_router
from app.api.routes.generation_variant_plans import router as generation_variant_plans_router
from app.api.routes.gpu_lipsync import router as gpu_lipsync_router
from app.api.routes.gpu_relay import router as gpu_relay_router
from app.api.routes.health import router as health_check_router
from app.api.routes.ingest_jobs import router as ingest_jobs_router
from app.api.routes.internal_render import router as internal_render_router
from app.api.routes.lipsync import router as lipsync_router
from app.api.routes.points import router as points_router
from app.api.routes.points import usage_router
from app.api.routes.projects import router as projects_router
from app.api.routes.scripts import router as scripts_router
from app.api.routes.scripts_ai import router as scripts_ai_router
from app.api.routes.share import router as share_router
from app.api.routes.subscription import router as subscription_router
from app.api.routes.tags import router as tags_router
from app.api.routes.task_center import router as task_center_router
from app.api.routes.templates import router as templates_router
from app.api.routes.templates_editor import router as templates_editor_router
from app.api.routes.titles import router as titles_router
from app.api.routes.tts import router as tts_router
from app.api.routes.upload import router as upload_router
from app.api.routes.videos import router as videos_router
from app.api.routes.viral_video import router as viral_video_router
from app.api.routes.voice_clones import router as voice_clones_router
from app.api.routes.voices import router as voices_router
from fastapi import APIRouter
@@ -45,28 +28,10 @@ api_router = APIRouter(prefix="/api/v1")
health_router = APIRouter()
health_router.include_router(health_check_router)
# ── /api/health 别名:部分前端/探针把 health 放在 /api 前缀下 ──────────────
# 原来 /health 在根路径;额外加一个 /api/health 别名避免 404。
api_health_router = APIRouter(prefix="/api")
api_health_router.include_router(health_check_router)
health_router.include_router(api_health_router)
# ── 旧前端路径别名(无需 template_id 路径参数)────────────────────────────
# /api/v1/clips/from-assets 已有 clips_standalone;此处额外挂 /api/v1/editor/*,
# 解决前端调 /api/v1/editor/clips/from-assets 和 /api/v1/editor/drafts 的 404。
editor_legacy_router = APIRouter(prefix="/editor", tags=["Editor Legacy Alias"])
editor_legacy_router.include_router(clips_standalone_router)
editor_legacy_router.include_router(drafts_standalone_router)
api_router.include_router(
auth_router,
tags=["Auth"],
)
api_router.include_router(
lipsync_router,
prefix="/lipsync",
tags=["Lipsync"],
)
api_router.include_router(
projects_router,
prefix="/projects",
@@ -77,10 +42,6 @@ api_router.include_router(
prefix="/tags",
tags=["Tag"],
)
api_router.include_router(
cover_templates_router,
tags=["CoverTemplate"],
)
api_router.include_router(
task_center_router,
tags=["TaskCenter"],
@@ -124,21 +85,6 @@ api_router.include_router(
prefix="/generation",
tags=["Generation"],
)
api_router.include_router(
generation_preview_router,
prefix="/generation",
tags=["Generation"],
)
api_router.include_router(
generation_variant_plans_router,
prefix="/generation",
tags=["Generation"],
)
api_router.include_router(
generation_cover_router,
prefix="/generation",
tags=["Generation"],
)
api_router.include_router(
titles_router,
prefix="/titles",
@@ -158,10 +104,6 @@ api_router.include_router(
videos_router,
tags=["VideoCenter"],
)
api_router.include_router(
share_router,
tags=["Share"],
)
api_router.include_router(
duplication_router,
prefix="/duplication",
@@ -178,27 +120,15 @@ api_router.include_router(
tags=["Template"],
)
api_router.include_router(
clips_standalone_router,
tags=["Clips"],
)
api_router.include_router(
templates_editor_router,
prefix="/templates/{template_id}/editor",
tags=["TemplateEditor"],
)
api_router.include_router(
editor_legacy_router,
edit_plans_router,
prefix="/edit-plans",
tags=["EditPlan"],
)
api_router.include_router(
tts_router,
prefix="/tts",
tags=["TTS"],
)
api_router.include_router(
ai_router,
prefix="/ai",
tags=["AI"],
)
api_router.include_router(
feature_flags_router,
tags=["Internal"],
@@ -207,38 +137,3 @@ api_router.include_router(
internal_render_router,
tags=["Internal"],
)
api_router.include_router(
gpu_relay_router,
tags=["GpuRelay"],
)
api_router.include_router(
scripts_router,
prefix="/scripts",
tags=["ScriptLibrary"],
)
api_router.include_router(
scripts_ai_router,
prefix="/scripts",
tags=["ScriptLibrary AI"],
)
api_router.include_router(
ai_avatar_render_router,
prefix="/ai-avatar/render",
tags=["AI Avatar Render"],
)
api_router.include_router(
points_router,
prefix="/points",
tags=["Points"],
)
api_router.include_router(
usage_router,
prefix="/usage",
tags=["Usage"],
)
api_router.include_router(
gpu_lipsync_router,
prefix="/gpu",
tags=["GPU Worker"],
)
api_router.include_router(viral_video_router, prefix="/viral-video", tags=["爆款视频"])
@@ -1,91 +0,0 @@
"""默认模板兜底共享逻辑(P0 #1922).
提供 get_or_create_default_template_id(db, user_id) 共享函数,
供 templates.py 列表查询、clips_standalone.py 独立端点、dependencies.py
resolve_draft_plan_id 三处复用,避免三处各写一套兜底逻辑产生分叉。
根因:PR#1918 清理模板管理 API 时误删了 GET /templates 自动创建默认模板
兜底,前端 PR#1913 去掉空 tid 拦截后首次进入生成页拼出
/templates//editor/clips/from-assets(双斜杠)→ FastAPI 404,阻断新用户首次
生成。
"""
from __future__ import annotations
import logging
from typing import Optional
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
def get_or_create_default_template_id(db: Session, user_id: str) -> Optional[str]:
"""获取或自动创建默认配音模板的 id。
判定逻辑(不做异常降级,只有确实创建失败时才回滚重查):
1. 查用户名下 is_active=True 且有 TemplateClipConfig 的模板 → 返回其 id;
2. 无则调用 CreateTemplateUseCase 创建一条默认 voice_over 模板;
3. 创建异常时 rollback 再重查一次(防并发唯一键冲突),重查仍无返回 None。
"""
from packages.adapters.sqlalchemy_impl.models import (
TemplateClipConfigModel,
TemplateModel,
)
from packages.adapters.sqlalchemy_impl.template_repository import (
SQLAlchemyTemplateRepository,
)
from packages.application.template.commands import (
CreateTemplateCommand,
SegmentCommand,
)
from packages.application.template.use_cases import CreateTemplateUseCase
existing = (
db.query(TemplateModel)
.filter(TemplateModel.user_id == user_id, TemplateModel.is_active.is_(True))
.order_by(TemplateModel.created_at.asc())
.first()
)
if existing is not None:
has_seg = (
db.query(TemplateClipConfigModel.id).filter(TemplateClipConfigModel.template_id == existing.id).first()
)
if has_seg:
return existing.id
try:
repo = SQLAlchemyTemplateRepository(db)
cmd = CreateTemplateCommand(
user_id=user_id,
name="默认配音模板",
mode="voice_over",
category="default",
tags=[],
title_config={},
subtitle_config={},
bgm_config={},
estimated_duration=0.0,
segments=[SegmentCommand(segment_order=0, duration_min=1.0, duration_max=30.0)],
)
tpl = CreateTemplateUseCase(repo).execute(cmd)
db.commit()
logger.info("auto-created default voice_over template: id=%s user=%s", tpl.id, user_id)
return tpl.id
except Exception:
db.rollback()
# 重查:可能并发请求已建好
existing = (
db.query(TemplateModel)
.filter(TemplateModel.user_id == user_id, TemplateModel.is_active.is_(True))
.order_by(TemplateModel.created_at.asc())
.first()
)
if existing is not None:
has_seg = (
db.query(TemplateClipConfigModel.id).filter(TemplateClipConfigModel.template_id == existing.id).first()
)
if has_seg:
return existing.id
logger.exception("failed to auto-create default template user=%s", user_id)
return None
+22 -36
View File
@@ -1,6 +1,5 @@
"""路由层共享辅助函数 — 消除跨文件重复定义。"""
from datetime import UTC, datetime
from typing import Any
from fastapi import HTTPException, status
@@ -25,27 +24,12 @@ def check_project_access(project_id: str, user_id: str, project_repository) -> N
raise HTTPException(status_code=403, detail="无权访问该项目")
_LEGACY_PLANS = {"standard", "pro", "enterprise", "basic", "premium"}
def get_user_plan(user_id: str, user_repository: UserRepository) -> str:
"""获取用户的会员类型,兼容旧档位值。
旧档位 standard/pro/enterprise/basic/premium 统一映射到当前体系:
- standard/basic → monthly
- pro/premium/enterprise → quarterly
"""
"""获取用户的订阅计划名称。"""
user = user_repository.find_by_id(user_id)
if user is None:
return "free"
plan = getattr(user, "subscription_plan", "free") or "free"
if plan in {"standard", "basic"}:
return "monthly"
if plan in {"pro", "premium", "enterprise"}:
return "quarterly"
if plan not in {"free", "monthly", "quarterly", "yearly"}:
return "free"
return plan
return getattr(user, "subscription_plan", "free") or "free"
def require_project_and_library(
@@ -82,6 +66,9 @@ def auto_select_video_assets(
Returns:
选中的素材 ID 列表,无可用素材时返回空列表
"""
# 明确不支持的视频编码(会导致渲染失败)
UNSUPPORTED_CODECS = {"hevc", "h265", "hev1", "hvc1", "vp9", "vp09", "av1", "av01"}
if not project_id:
return []
@@ -109,12 +96,27 @@ def auto_select_video_assets(
and a.mime_type.startswith("video")
]
# 过滤不支持的编码格式(HEVC/VP9/AV1 等会导致渲染失败)
# 优先读 asset.codec 字段,其次从 metadata 里取(兼容存量数据)
codec_filtered = []
skipped_codec = 0
for a in ready_videos:
codec = (a.codec or "").lower()
if not codec and a.metadata and isinstance(a.metadata, dict):
codec = str(a.metadata.get("codec", "")).lower()
if codec and codec in UNSUPPORTED_CODECS:
skipped_codec += 1
continue
codec_filtered.append(a)
if skipped_codec and logger:
logger.warning("自动选素材: 跳过 %d 个不支持编码的素材", skipped_codec)
# 过滤横屏素材(只保留竖屏/正方形)
# 移动端短视频场景默认竖屏,横屏素材裁剪后画面不可用
# 注意:这只是选素材优化,渲染引擎本身支持任何分辨率的素材
filtered_videos = []
skipped_landscape = 0
for a in ready_videos:
for a in codec_filtered:
width = a.width if hasattr(a, "width") and a.width else 0
height = a.height if hasattr(a, "height") and a.height else 0
if not width or not height:
@@ -138,19 +140,3 @@ def auto_select_video_assets(
# 按创建时间降序(新素材在前)
filtered_videos.sort(key=lambda a: a.created_at, reverse=True)
return [a.id for a in filtered_videos]
def format_utc_datetime(dt: datetime | None) -> str:
"""将数据库读出的 UTC naive datetime 格式化为带时区的 ISO 8601 字符串。
数据库 DateTime 列不带时区信息,但存的是 UTC 时间。
直接 .isoformat() 输出无时区标识,前端会按本地时间解析,导致差 8 小时。
输出带 Z 后缀,前端 new Date() 自动转本地时间。
"""
if dt is None:
return ""
if isinstance(dt, str):
return dt
if dt.tzinfo is None:
return dt.isoformat() + "Z"
return dt.astimezone(UTC).isoformat().replace("+00:00", "Z")

Some files were not shown because too many files have changed in this diff Show More