diff --git a/scripts/ci_staging_deploy.sh b/scripts/ci_staging_deploy.sh index e90d6c1b6..4a1070062 100755 --- a/scripts/ci_staging_deploy.sh +++ b/scripts/ci_staging_deploy.sh @@ -1,6 +1,6 @@ #!/bin/sh # =========================================== -# Staging 部署脚本(SSH 模式) +# Staging 部署脚本(SSH 模式,支持自动回滚) # =========================================== # 通过 SSH 在 staging 服务器上执行 # @@ -12,6 +12,7 @@ # ENV_FILE - 环境变量文件路径 # GENERATED_DIR - 生成文件目录 # SKIP_MIGRATION - 跳过数据库迁移(true/false,默认 false) +# SKIP_ROLLBACK - 失败时跳过自动回滚(true/false,默认 false) set -eu @@ -25,6 +26,7 @@ GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-staging/generated}" LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-staging/legacy-assets}" SKIP_MIGRATION="${SKIP_MIGRATION:-false}" +SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}" if [ -z "$IMAGE_TAG" ]; then echo "ERROR: IMAGE_TAG is required" @@ -35,9 +37,157 @@ test -f "$ENV_FILE" mkdir -p "$GENERATED_DIR" mkdir -p "$LEGACY_ASSETS_DIR" -echo "==========================================" +echo "===========================================" echo " Staging 部署 - $IMAGE_TAG" -echo "==========================================" +echo "===========================================" + +# ---- 记录当前运行的镜像版本(用于回滚) ---- +echo "Recording current image versions for rollback..." +PREV_API_IMAGE="" +PREV_WORKER_IMAGE="" +PREV_WEB_IMAGE="" +for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do + if docker inspect "$c" >/dev/null 2>&1; then + img=$(docker inspect -f '{{.Config.Image}}' "$c") + case "$c" in + xiaoxia-api-staging) PREV_API_IMAGE="$img" ;; + xiaoxia-worker-staging) PREV_WORKER_IMAGE="$img" ;; + xiaoxia-web-staging) PREV_WEB_IMAGE="$img" ;; + esac + echo " $c -> $img" + else + echo " $c -> (not running)" + fi +done + +# ---- 回滚函数 ---- +rollback() { + echo "" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo " 部署失败,正在自动回滚到上一版本..." + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "" + + if [ "$SKIP_ROLLBACK" = "true" ]; then + echo "SKIP_ROLLBACK=true,跳过自动回滚" + exit 1 + fi + + # 停止当前(失败的)新容器 + echo "Stopping new containers..." + docker rm -f xiaoxia-api-staging 2>/dev/null || true + docker rm -f xiaoxia-worker-staging 2>/dev/null || true + docker rm -f xiaoxia-web-staging 2>/dev/null || true + + LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" + + # 恢复 API + if [ -n "$PREV_API_IMAGE" ]; then + echo "Rolling back API to: $PREV_API_IMAGE" + docker run -d \ + --name xiaoxia-api-staging \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -p 127.0.0.1:8000:8000 \ + -e APP_ENV=staging \ + -e APP_VERSION="$(echo $PREV_API_IMAGE | grep -oE '[^:]+$')" \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 40s \ + $LOG_OPTS \ + "$PREV_API_IMAGE" + else + echo "No previous API image to roll back to" + fi + + # 恢复 Worker + if [ -n "$PREV_WORKER_IMAGE" ]; then + echo "Rolling back Worker to: $PREV_WORKER_IMAGE" + docker run -d \ + --name xiaoxia-worker-staging \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -e APP_ENV=staging \ + -e APP_VERSION="$(echo $PREV_WORKER_IMAGE | grep -oE '[^:]+$')" \ + -e WORKER_CONCURRENCY=1 \ + -e WORKER_MAX_TASKS_PER_CHILD=100 \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 30s \ + $LOG_OPTS \ + "$PREV_WORKER_IMAGE" + else + echo "No previous Worker image to roll back to" + fi + + # 恢复 Web + if [ -n "$PREV_WEB_IMAGE" ]; then + echo "Rolling back Web to: $PREV_WEB_IMAGE" + LEGACY_VOLUME="" + if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then + LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" + fi + docker run -d \ + --name xiaoxia-web-staging \ + --network xiaoxia-net-staging \ + -p 127.0.0.1:3001:80 \ + --restart unless-stopped \ + $LEGACY_VOLUME \ + --health-cmd "wget --spider -q http://127.0.0.1:80" \ + --health-interval 30s \ + --health-timeout 5s \ + --health-retries 3 \ + $LOG_OPTS \ + "$PREV_WEB_IMAGE" + else + echo "No previous Web image to roll back to" + fi + + # 等待 API 回滚后恢复健康 + if [ -n "$PREV_API_IMAGE" ]; then + echo "Waiting for rolled-back API to become healthy..." + i=0 + while [ "$i" -lt 40 ]; do + if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then + echo "Rolled-back API is healthy!" + break + fi + i=$((i + 1)) + echo " Waiting... ($i/40)" + sleep 3 + done + if [ "$i" -ge 40 ]; then + echo "WARN: Rolled-back API did not become healthy within 120s" + docker logs --tail 30 xiaoxia-api-staging + fi + fi + + echo "" + echo "===========================================" + echo " 回滚完成" + echo "===========================================" + echo "Previous API: ${PREV_API_IMAGE:-none}" + echo "Previous Worker: ${PREV_WORKER_IMAGE:-none}" + echo "Previous Web: ${PREV_WEB_IMAGE:-none}" + echo "" + echo "部署失败,已自动回滚到上一版本" + docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging + exit 1 +} # ---- 登录 Registry ---- if [ -n "$REGISTRY_TOKEN" ]; then @@ -117,7 +267,12 @@ if [ "$SKIP_MIGRATION" != "true" ]; then --env-file "$ENV_FILE" \ --network xiaoxia-net-staging \ -e APP_ENV=staging \ - "$LOCAL_API" sh -c "cd /app && alembic upgrade head" + "$LOCAL_API" sh -c "cd /app && alembic upgrade head" || { + echo "ERROR: Database migration failed" + echo "Note: Migration failures are NOT automatically rolled back (data safety)" + echo "Please manually check and fix the migration, then redeploy" + exit 1 + } echo "Migrations completed." else echo "Skipping migrations (SKIP_MIGRATION=true)" @@ -151,7 +306,7 @@ docker run -d \ --health-retries 3 \ --health-start-period 40s \ $LOG_OPTS \ - "$LOCAL_API" + "$LOCAL_API" || rollback # ---- 启动 Worker ---- echo "Starting Worker container..." @@ -174,7 +329,7 @@ docker run -d \ --health-retries 3 \ --health-start-period 30s \ $LOG_OPTS \ - "$LOCAL_WORKER" + "$LOCAL_WORKER" || rollback # ---- 启动 Web ---- LEGACY_VOLUME="" @@ -197,7 +352,7 @@ docker run -d \ --health-timeout 5s \ --health-retries 3 \ $LOG_OPTS \ - "$LOCAL_WEB" + "$LOCAL_WEB" || rollback # ---- 等待 API 健康 ---- echo "Waiting for API to become healthy..." @@ -215,7 +370,7 @@ done if [ "$i" -ge 40 ]; then echo "ERROR: API did not become healthy within 120s" docker logs --tail 50 xiaoxia-api-staging - exit 1 + rollback fi # ---- 等待 Web 健康 ---- @@ -234,7 +389,7 @@ done if [ "$i" -ge 15 ]; then echo "ERROR: Web did not become healthy within 30s" docker logs --tail 30 xiaoxia-web-staging - exit 1 + rollback fi # ---- 清理旧镜像 ----