From 6d4b9a0e5113ebbc673ed9b9c8e00e39f658c56e Mon Sep 17 00:00:00 2001 From: XiaoXia Bot Date: Thu, 16 Jul 2026 08:18:05 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat(ci):=20staging=E9=83=A8=E7=BD=B2?= =?UTF-8?q?=E8=84=9A=E6=9C=AC=E5=A2=9E=E5=8A=A0=E6=9C=AC=E5=9C=B0=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E5=9B=9E=E6=BB=9A=E8=83=BD=E5=8A=9B-=20=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E5=89=8D=E8=AE=B0=E5=BD=95=E5=BD=93=E5=89=8D=E4=B8=89?= =?UTF-8?q?=E4=B8=AA=E6=9C=8D=E5=8A=A1=E7=9A=84=E9=95=9C=E5=83=8F=E7=89=88?= =?UTF-8?q?=E6=9C=AC=EF=BC=88=E5=9B=9E=E6=BB=9A=E7=9B=AE=E6=A0=87=EF=BC=89?= =?UTF-8?q?-=20=E6=96=B0=E5=A2=9Erollback()=E5=87=BD=E6=95=B0=EF=BC=9A?= =?UTF-8?q?=E5=AE=B9=E5=99=A8=E5=90=AF=E5=8A=A8=E5=A4=B1=E8=B4=A5/?= =?UTF-8?q?=E5=81=A5=E5=BA=B7=E6=A3=80=E6=9F=A5=E4=B8=8D=E9=80=9A=E8=BF=87?= =?UTF-8?q?=E6=97=B6=E8=87=AA=E5=8A=A8=E5=9B=9E=E6=BB=9A-=20=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E5=81=A5=E5=BA=B7=E6=A3=80=E6=9F=A5=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E7=AB=8B=E5=8D=B3=E5=9B=9E=E6=BB=9A=EF=BC=8C=E4=B8=8D=E7=94=A8?= =?UTF-8?q?=E7=AD=89CI=E4=BE=A7SSH=E5=9B=9E=E6=BB=9A=EF=BC=88=E6=9B=B4?= =?UTF-8?q?=E5=BF=AB=EF=BC=89-=20=E8=BF=81=E7=A7=BB=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E4=B8=8D=E8=87=AA=E5=8A=A8=E5=9B=9E=E6=BB=9A=EF=BC=88=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=AE=89=E5=85=A8=E8=80=83=E8=99=91=EF=BC=8C=E9=9C=80?= =?UTF-8?q?=E4=BA=BA=E5=B7=A5=E4=BB=8B=E5=85=A5=EF=BC=89-=20=E6=94=AF?= =?UTF-8?q?=E6=8C=81SKIP=5FROLLBACK=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E5=85=B3=E9=97=AD=E8=87=AA=E5=8A=A8=E5=9B=9E=E6=BB=9A-=20?= =?UTF-8?q?=E4=B8=8ECI=E4=BE=A7=E5=85=AC=E7=BD=91=E5=81=A5=E5=BA=B7?= =?UTF-8?q?=E6=A3=80=E6=9F=A5=E5=9B=9E=E6=BB=9A=E5=BD=A2=E6=88=90=E4=B8=A4?= =?UTF-8?q?=E9=81=93=E9=98=B2=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/ci_staging_deploy.sh | 173 +++++++++++++++++++++++++++++++++-- 1 file changed, 164 insertions(+), 9 deletions(-) diff --git a/scripts/ci_staging_deploy.sh b/scripts/ci_staging_deploy.sh index e90d6c1b6..4a1070062 100755 --- a/scripts/ci_staging_deploy.sh +++ b/scripts/ci_staging_deploy.sh @@ -1,6 +1,6 @@ #!/bin/sh # =========================================== -# Staging 部署脚本（SSH 模式） +# Staging 部署脚本（SSH 模式，支持自动回滚） # =========================================== # 通过 SSH 在 staging 服务器上执行 # @@ -12,6 +12,7 @@ # ENV_FILE - 环境变量文件路径 # GENERATED_DIR - 生成文件目录 # SKIP_MIGRATION - 跳过数据库迁移（true/false，默认 false） +# SKIP_ROLLBACK - 失败时跳过自动回滚（true/false，默认 false） set -eu @@ -25,6 +26,7 @@ GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-staging/generated}" LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-staging/legacy-assets}" SKIP_MIGRATION="${SKIP_MIGRATION:-false}" +SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}" if [ -z "$IMAGE_TAG" ]; then echo "ERROR: IMAGE_TAG is required" @@ -35,9 +37,157 @@ test -f "$ENV_FILE" mkdir -p "$GENERATED_DIR" mkdir -p "$LEGACY_ASSETS_DIR" -echo "==========================================" +echo "===========================================" echo " Staging 部署 - $IMAGE_TAG" -echo "==========================================" +echo "===========================================" + +# ---- 记录当前运行的镜像版本（用于回滚） ---- +echo "Recording current image versions for rollback..." +PREV_API_IMAGE="" +PREV_WORKER_IMAGE="" +PREV_WEB_IMAGE="" +for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do + if docker inspect "$c" >/dev/null 2>&1; then + img=$(docker inspect -f '{{.Config.Image}}' "$c") + case "$c" in + xiaoxia-api-staging) PREV_API_IMAGE="$img" ;; + xiaoxia-worker-staging) PREV_WORKER_IMAGE="$img" ;; + xiaoxia-web-staging) PREV_WEB_IMAGE="$img" ;; + esac + echo " $c -> $img" + else + echo " $c -> (not running)" + fi +done + +# ---- 回滚函数 ---- +rollback() { + echo "" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo " 部署失败，正在自动回滚到上一版本..." + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "" + + if [ "$SKIP_ROLLBACK" = "true" ]; then + echo "SKIP_ROLLBACK=true，跳过自动回滚" + exit 1 + fi + + # 停止当前（失败的）新容器 + echo "Stopping new containers..." + docker rm -f xiaoxia-api-staging 2>/dev/null || true + docker rm -f xiaoxia-worker-staging 2>/dev/null || true + docker rm -f xiaoxia-web-staging 2>/dev/null || true + + LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" + + # 恢复 API + if [ -n "$PREV_API_IMAGE" ]; then + echo "Rolling back API to: $PREV_API_IMAGE" + docker run -d \ + --name xiaoxia-api-staging \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -p 127.0.0.1:8000:8000 \ + -e APP_ENV=staging \ + -e APP_VERSION="$(echo $PREV_API_IMAGE | grep -oE '[^:]+$')" \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 40s \ + $LOG_OPTS \ + "$PREV_API_IMAGE" + else + echo "No previous API image to roll back to" + fi + + # 恢复 Worker + if [ -n "$PREV_WORKER_IMAGE" ]; then + echo "Rolling back Worker to: $PREV_WORKER_IMAGE" + docker run -d \ + --name xiaoxia-worker-staging \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -e APP_ENV=staging \ + -e APP_VERSION="$(echo $PREV_WORKER_IMAGE | grep -oE '[^:]+$')" \ + -e WORKER_CONCURRENCY=1 \ + -e WORKER_MAX_TASKS_PER_CHILD=100 \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 30s \ + $LOG_OPTS \ + "$PREV_WORKER_IMAGE" + else + echo "No previous Worker image to roll back to" + fi + + # 恢复 Web + if [ -n "$PREV_WEB_IMAGE" ]; then + echo "Rolling back Web to: $PREV_WEB_IMAGE" + LEGACY_VOLUME="" + if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then + LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" + fi + docker run -d \ + --name xiaoxia-web-staging \ + --network xiaoxia-net-staging \ + -p 127.0.0.1:3001:80 \ + --restart unless-stopped \ + $LEGACY_VOLUME \ + --health-cmd "wget --spider -q http://127.0.0.1:80" \ + --health-interval 30s \ + --health-timeout 5s \ + --health-retries 3 \ + $LOG_OPTS \ + "$PREV_WEB_IMAGE" + else + echo "No previous Web image to roll back to" + fi + + # 等待 API 回滚后恢复健康 + if [ -n "$PREV_API_IMAGE" ]; then + echo "Waiting for rolled-back API to become healthy..." + i=0 + while [ "$i" -lt 40 ]; do + if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then + echo "Rolled-back API is healthy!" + break + fi + i=$((i + 1)) + echo " Waiting... ($i/40)" + sleep 3 + done + if [ "$i" -ge 40 ]; then + echo "WARN: Rolled-back API did not become healthy within 120s" + docker logs --tail 30 xiaoxia-api-staging + fi + fi + + echo "" + echo "===========================================" + echo " 回滚完成" + echo "===========================================" + echo "Previous API: ${PREV_API_IMAGE:-none}" + echo "Previous Worker: ${PREV_WORKER_IMAGE:-none}" + echo "Previous Web: ${PREV_WEB_IMAGE:-none}" + echo "" + echo "部署失败，已自动回滚到上一版本" + docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging + exit 1 +} # ---- 登录 Registry ---- if [ -n "$REGISTRY_TOKEN" ]; then @@ -117,7 +267,12 @@ if [ "$SKIP_MIGRATION" != "true" ]; then --env-file "$ENV_FILE" \ --network xiaoxia-net-staging \ -e APP_ENV=staging \ - "$LOCAL_API" sh -c "cd /app && alembic upgrade head" + "$LOCAL_API" sh -c "cd /app && alembic upgrade head" || { + echo "ERROR: Database migration failed" + echo "Note: Migration failures are NOT automatically rolled back (data safety)" + echo "Please manually check and fix the migration, then redeploy" + exit 1 + } echo "Migrations completed." else echo "Skipping migrations (SKIP_MIGRATION=true)" @@ -151,7 +306,7 @@ docker run -d \ --health-retries 3 \ --health-start-period 40s \ $LOG_OPTS \ - "$LOCAL_API" + "$LOCAL_API" || rollback # ---- 启动 Worker ---- echo "Starting Worker container..." @@ -174,7 +329,7 @@ docker run -d \ --health-retries 3 \ --health-start-period 30s \ $LOG_OPTS \ - "$LOCAL_WORKER" + "$LOCAL_WORKER" || rollback # ---- 启动 Web ---- LEGACY_VOLUME="" @@ -197,7 +352,7 @@ docker run -d \ --health-timeout 5s \ --health-retries 3 \ $LOG_OPTS \ - "$LOCAL_WEB" + "$LOCAL_WEB" || rollback # ---- 等待 API 健康 ---- echo "Waiting for API to become healthy..." @@ -215,7 +370,7 @@ done if [ "$i" -ge 40 ]; then echo "ERROR: API did not become healthy within 120s" docker logs --tail 50 xiaoxia-api-staging - exit 1 + rollback fi # ---- 等待 Web 健康 ---- @@ -234,7 +389,7 @@ done if [ "$i" -ge 15 ]; then echo "ERROR: Web did not become healthy within 30s" docker logs --tail 30 xiaoxia-web-staging - exit 1 + rollback fi # ---- 清理旧镜像 ---- -- 2.54.0 From 58e4e4a18d38b998484cf2a97b05130b7049378f Mon Sep 17 00:00:00 2001 From: XiaoXia Bot Date: Thu, 16 Jul 2026 09:50:34 +0800 Subject: [PATCH 2/2] chore(ci): retry CI run to avoid flaky runner -- 2.54.0