Merge remote-tracking branch 'origin/develop' into feat/staging-auto-rollback

scripts/ci_staging_deploy.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 # ===========================================
-# Staging 部署脚本（SSH 模式）
+# Staging 部署脚本（SSH 模式，支持自动回滚）
 # ===========================================
 # 通过 SSH 在 staging 服务器上执行
 #
@@ -12,6 +12,7 @@
 #   ENV_FILE        - 环境变量文件路径
 #   GENERATED_DIR   - 生成文件目录
 #   SKIP_MIGRATION  - 跳过数据库迁移（true/false，默认 false）
+#   SKIP_ROLLBACK   - 失败时跳过自动回滚（true/false，默认 false）
 
 set -eu
 
@@ -25,6 +26,7 @@ GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-staging/generated}"
 LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-staging/legacy-assets}"
 
 SKIP_MIGRATION="${SKIP_MIGRATION:-false}"
+SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}"
 
 if [ -z "$IMAGE_TAG" ]; then
   echo "ERROR: IMAGE_TAG is required"
@@ -35,9 +37,157 @@ test -f "$ENV_FILE"
 mkdir -p "$GENERATED_DIR"
 mkdir -p "$LEGACY_ASSETS_DIR"
 
-echo "=========================================="
+echo "==========================================="
 echo "  Staging 部署 - $IMAGE_TAG"
-echo "=========================================="
+echo "==========================================="
+
+# ---- 记录当前运行的镜像版本（用于回滚） ----
+echo "Recording current image versions for rollback..."
+PREV_API_IMAGE=""
+PREV_WORKER_IMAGE=""
+PREV_WEB_IMAGE=""
+for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do
+  if docker inspect "$c" >/dev/null 2>&1; then
+    img=$(docker inspect -f '{{.Config.Image}}' "$c")
+    case "$c" in
+      xiaoxia-api-staging) PREV_API_IMAGE="$img" ;;
+      xiaoxia-worker-staging) PREV_WORKER_IMAGE="$img" ;;
+      xiaoxia-web-staging) PREV_WEB_IMAGE="$img" ;;
+    esac
+    echo "  $c -> $img"
+  else
+    echo "  $c -> (not running)"
+  fi
+done
+
+# ---- 回滚函数 ----
+rollback() {
+  echo ""
+  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+  echo "  部署失败，正在自动回滚到上一版本..."
+  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+  echo ""
+
+  if [ "$SKIP_ROLLBACK" = "true" ]; then
+    echo "SKIP_ROLLBACK=true，跳过自动回滚"
+    exit 1
+  fi
+
+  # 停止当前（失败的）新容器
+  echo "Stopping new containers..."
+  docker rm -f xiaoxia-api-staging 2>/dev/null || true
+  docker rm -f xiaoxia-worker-staging 2>/dev/null || true
+  docker rm -f xiaoxia-web-staging 2>/dev/null || true
+
+  LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
+
+  # 恢复 API
+  if [ -n "$PREV_API_IMAGE" ]; then
+    echo "Rolling back API to: $PREV_API_IMAGE"
+    docker run -d \
+      --name xiaoxia-api-staging \
+      --env-file "$ENV_FILE" \
+      --network xiaoxia-net-staging \
+      -p 127.0.0.1:8000:8000 \
+      -e APP_ENV=staging \
+      -e APP_VERSION="$(echo $PREV_API_IMAGE | grep -oE '[^:]+$')" \
+      -e GENERATED_FILES_DIR=/app/generated \
+      -e GENERATED_FILES_URL_PREFIX=/generated-files \
+      -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \
+      -v "$GENERATED_DIR:/app/generated" \
+      --restart unless-stopped \
+      --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \
+      --health-interval 30s \
+      --health-timeout 10s \
+      --health-retries 3 \
+      --health-start-period 40s \
+      $LOG_OPTS \
+      "$PREV_API_IMAGE"
+  else
+    echo "No previous API image to roll back to"
+  fi
+
+  # 恢复 Worker
+  if [ -n "$PREV_WORKER_IMAGE" ]; then
+    echo "Rolling back Worker to: $PREV_WORKER_IMAGE"
+    docker run -d \
+      --name xiaoxia-worker-staging \
+      --env-file "$ENV_FILE" \
+      --network xiaoxia-net-staging \
+      -e APP_ENV=staging \
+      -e APP_VERSION="$(echo $PREV_WORKER_IMAGE | grep -oE '[^:]+$')" \
+      -e WORKER_CONCURRENCY=1 \
+      -e WORKER_MAX_TASKS_PER_CHILD=100 \
+      -e GENERATED_FILES_DIR=/app/generated \
+      -e GENERATED_FILES_URL_PREFIX=/generated-files \
+      -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \
+      -v "$GENERATED_DIR:/app/generated" \
+      --restart unless-stopped \
+      --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \
+      --health-interval 30s \
+      --health-timeout 10s \
+      --health-retries 3 \
+      --health-start-period 30s \
+      $LOG_OPTS \
+      "$PREV_WORKER_IMAGE"
+  else
+    echo "No previous Worker image to roll back to"
+  fi
+
+  # 恢复 Web
+  if [ -n "$PREV_WEB_IMAGE" ]; then
+    echo "Rolling back Web to: $PREV_WEB_IMAGE"
+    LEGACY_VOLUME=""
+    if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
+      LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
+    fi
+    docker run -d \
+      --name xiaoxia-web-staging \
+      --network xiaoxia-net-staging \
+      -p 127.0.0.1:3001:80 \
+      --restart unless-stopped \
+      $LEGACY_VOLUME \
+      --health-cmd "wget --spider -q http://127.0.0.1:80" \
+      --health-interval 30s \
+      --health-timeout 5s \
+      --health-retries 3 \
+      $LOG_OPTS \
+      "$PREV_WEB_IMAGE"
+  else
+    echo "No previous Web image to roll back to"
+  fi
+
+  # 等待 API 回滚后恢复健康
+  if [ -n "$PREV_API_IMAGE" ]; then
+    echo "Waiting for rolled-back API to become healthy..."
+    i=0
+    while [ "$i" -lt 40 ]; do
+      if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then
+        echo "Rolled-back API is healthy!"
+        break
+      fi
+      i=$((i + 1))
+      echo "  Waiting... ($i/40)"
+      sleep 3
+    done
+    if [ "$i" -ge 40 ]; then
+      echo "WARN: Rolled-back API did not become healthy within 120s"
+      docker logs --tail 30 xiaoxia-api-staging
+    fi
+  fi
+
+  echo ""
+  echo "==========================================="
+  echo "  回滚完成"
+  echo "==========================================="
+  echo "Previous API:    ${PREV_API_IMAGE:-none}"
+  echo "Previous Worker: ${PREV_WORKER_IMAGE:-none}"
+  echo "Previous Web:    ${PREV_WEB_IMAGE:-none}"
+  echo ""
+  echo "部署失败，已自动回滚到上一版本"
+  docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging
+  exit 1
+}
 
 # ---- 登录 Registry ----
 if [ -n "$REGISTRY_TOKEN" ]; then
@@ -117,7 +267,12 @@ if [ "$SKIP_MIGRATION" != "true" ]; then
     --env-file "$ENV_FILE" \
     --network xiaoxia-net-staging \
     -e APP_ENV=staging \
-    "$LOCAL_API" sh -c "cd /app && alembic upgrade head"
+    "$LOCAL_API" sh -c "cd /app && alembic upgrade head" || {
+    echo "ERROR: Database migration failed"
+    echo "Note: Migration failures are NOT automatically rolled back (data safety)"
+    echo "Please manually check and fix the migration, then redeploy"
+    exit 1
+  }
   echo "Migrations completed."
 else
   echo "Skipping migrations (SKIP_MIGRATION=true)"
@@ -151,7 +306,7 @@ docker run -d \
   --health-retries 3 \
   --health-start-period 40s \
   $LOG_OPTS \
-  "$LOCAL_API"
+  "$LOCAL_API" || rollback
 
 # ---- 启动 Worker ----
 echo "Starting Worker container..."
@@ -174,7 +329,7 @@ docker run -d \
   --health-retries 3 \
   --health-start-period 30s \
   $LOG_OPTS \
-  "$LOCAL_WORKER"
+  "$LOCAL_WORKER" || rollback
 
 # ---- 启动 Web ----
 LEGACY_VOLUME=""
@@ -197,7 +352,7 @@ docker run -d \
   --health-timeout 5s \
   --health-retries 3 \
   $LOG_OPTS \
-  "$LOCAL_WEB"
+  "$LOCAL_WEB" || rollback
 
 # ---- 等待 API 健康 ----
 echo "Waiting for API to become healthy..."
@@ -215,7 +370,7 @@ done
 if [ "$i" -ge 40 ]; then
   echo "ERROR: API did not become healthy within 120s"
   docker logs --tail 50 xiaoxia-api-staging
-  exit 1
+  rollback
 fi
 
 # ---- 等待 Web 健康 ----
@@ -234,7 +389,7 @@ done
 if [ "$i" -ge 15 ]; then
   echo "ERROR: Web did not become healthy within 30s"
   docker logs --tail 30 xiaoxia-web-staging
-  exit 1
+  rollback
 fi
 
 # ---- 清理旧镜像 ----