xiaoxia-saas/scripts/ci_production_deploy.sh

#!/bin/sh
# ===========================================
# Production 部署脚本（SSH 模式，支持自动回滚）
# ===========================================
# 通过 SSH 在 production 服务器上执行
#
# 环境变量:
#   IMAGE_TAG       - 镜像版本 tag（如 commit SHA 或分支名）
#   REGISTRY_TOKEN  - Registry 访问令牌
#   REGISTRY        - Registry 地址（默认 xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji）
#   REGISTRY_USER   - Registry 用户名（默认 xiaoxia）
#   ENV_FILE        - 环境变量文件路径
#   GENERATED_DIR   - 生成文件目录
#   SKIP_MIGRATION  - 跳过数据库迁移（true/false，默认 false）
#   SKIP_ROLLBACK   - 失败时跳过自动回滚（true/false，默认 false）

set -eu

# ---- 重试工具函数 ----
retry_cmd() {
  local max_attempts=$1
  local backoff=$2
  shift 2
  local attempt=1
  while [ $attempt -le $max_attempts ]; do
    if "$@"; then
      return 0
    fi
    echo "  attempt $attempt/$max_attempts failed, retrying in ${backoff}s..."
    sleep $backoff
    backoff=$((backoff * 2))
    attempt=$((attempt + 1))
  done
  echo "  ERROR: failed after $max_attempts retries"
  return 1
}

retry_docker_login() {
  echo "Logging in to registry (up to 3 retries)"
  export REGISTRY_TOKEN REGISTRY_HOST REGISTRY_USER
  if retry_cmd 3 5 sh -c 'printf "%s" "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin'; then
    return 0
  fi
  echo "WARN: docker login failed after retries, will try pull anyway"
  return 0
}

retry_docker_pull() {
  local image=$1
  echo "Pulling $image (up to 3 retries)"
  retry_cmd 3 10 docker pull "$image"
}

IMAGE_TAG="${IMAGE_TAG:-}"
REGISTRY="${REGISTRY:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji}"
REGISTRY_USER="${ACR_USERNAME:-${REGISTRY_USER:-nick0415343655}}"
REGISTRY_TOKEN="${ACR_PASSWORD:-${REGISTRY_TOKEN:-}}"

ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-production/.env}"
GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-production/generated}"
LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-production/legacy-assets}"

SKIP_MIGRATION="${SKIP_MIGRATION:-false}"
SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}"

if [ -z "$IMAGE_TAG" ]; then
  echo "ERROR: IMAGE_TAG is required"
  exit 1
fi

test -f "$ENV_FILE"
mkdir -p "$GENERATED_DIR"
mkdir -p "$LEGACY_ASSETS_DIR"

echo "==========================================="
echo "  Production 部署 - $IMAGE_TAG"
echo "==========================================="

# ---- 记录当前运行的镜像版本（用于回滚） ----
echo "Recording current image versions for rollback..."
PREV_API_IMAGE=""
PREV_WORKER_IMAGE=""
PREV_WEB_IMAGE=""
for c in xiaoxia-api-production xiaoxia-worker-production xiaoxia-web-production; do
  if docker inspect "$c" >/dev/null 2>&1; then
    img=$(docker inspect -f '{{.Config.Image}}' "$c")
    case "$c" in
      xiaoxia-api-production) PREV_API_IMAGE="$img" ;;
      xiaoxia-worker-production) PREV_WORKER_IMAGE="$img" ;;
      xiaoxia-web-production) PREV_WEB_IMAGE="$img" ;;
    esac
    echo "  $c -> $img"
  else
    echo "  $c -> (not running)"
  fi
done

# ---- 回滚函数 ----
rollback() {
  echo ""
  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
  echo "  部署失败，正在自动回滚到上一版本..."
  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
  echo ""

  if [ "$SKIP_ROLLBACK" = "true" ]; then
    echo "SKIP_ROLLBACK=true，跳过自动回滚"
    exit 1
  fi

  # 停止当前（失败的）新容器
  echo "Stopping new containers..."
  docker rm -f xiaoxia-api-production 2>/dev/null || true
  docker rm -f xiaoxia-worker-production 2>/dev/null || true
  docker rm -f xiaoxia-web-production 2>/dev/null || true

  LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"

  # 恢复 API
  if [ -n "$PREV_API_IMAGE" ]; then
    echo "Rolling back API to: $PREV_API_IMAGE"
    docker run -d \
      --name xiaoxia-api-production \
      --env-file "$ENV_FILE" \
      --network xiaoxia-net-production \
      -p 127.0.0.1:8001:8000 \
      -e APP_ENV=production \
      -e APP_VERSION="$(echo $PREV_API_IMAGE | grep -oE '[^:]+$')" \
      -e GENERATED_FILES_DIR=/app/generated \
      -e GENERATED_FILES_URL_PREFIX=/generated-files \
      -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \
      -v "$GENERATED_DIR:/app/generated" \
      --restart unless-stopped \
  --cpus 2 \
  --memory 2g \
      --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \
      --health-interval 30s \
      --health-timeout 10s \
      --health-retries 3 \
      --health-start-period 40s \
      $LOG_OPTS \
      "$PREV_API_IMAGE"
  else
    echo "No previous API image to roll back to"
  fi

  # 恢复 Worker
  if [ -n "$PREV_WORKER_IMAGE" ]; then
    echo "Rolling back Worker to: $PREV_WORKER_IMAGE"
    docker run -d \
      --name xiaoxia-worker-production \
      --env-file "$ENV_FILE" \
      --network xiaoxia-net-production \
      -e APP_ENV=production \
      -e APP_VERSION="$(echo $PREV_WORKER_IMAGE | grep -oE '[^:]+$')" \
      -e WORKER_CONCURRENCY=1 \
      -e WORKER_MAX_TASKS_PER_CHILD=100 \
      -e GENERATED_FILES_DIR=/app/generated \
      -e GENERATED_FILES_URL_PREFIX=/generated-files \
      -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \
      -v "$GENERATED_DIR:/app/generated" \
      --restart unless-stopped \
  --cpus 2 \
  --memory 2g \
      --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \
      --health-interval 30s \
      --health-timeout 10s \
      --health-retries 3 \
      --health-start-period 30s \
      $LOG_OPTS \
      "$PREV_WORKER_IMAGE"
  else
    echo "No previous Worker image to roll back to"
  fi

  # 恢复 Web
  if [ -n "$PREV_WEB_IMAGE" ]; then
    echo "Rolling back Web to: $PREV_WEB_IMAGE"
    LEGACY_VOLUME=""
    if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
      LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
    fi
    docker run -d \
      --name xiaoxia-web-production \
      --network xiaoxia-net-production \
      -p 127.0.0.1:3002:80 \
      --restart unless-stopped \
  --cpus 0.5 \
  --memory 512m \
      $LEGACY_VOLUME \
      --health-cmd "wget --spider -q http://127.0.0.1:80" \
      --health-interval 30s \
      --health-timeout 5s \
      --health-retries 3 \
      $LOG_OPTS \
      "$PREV_WEB_IMAGE"
  else
    echo "No previous Web image to roll back to"
  fi

  # 等待 API 回滚后恢复健康
  if [ -n "$PREV_API_IMAGE" ]; then
    echo "Waiting for rolled-back API to become healthy..."
    i=0
    while [ "$i" -lt 40 ]; do
      if curl -sf --max-time 5 http://127.0.0.1:8001/health >/dev/null 2>&1; then
        echo "Rolled-back API is healthy!"
        break
      fi
      i=$((i + 1))
      echo "  Waiting... ($i/40)"
      sleep 3
    done
    if [ "$i" -ge 40 ]; then
      echo "WARN: Rolled-back API did not become healthy within 120s"
      docker logs --tail 30 xiaoxia-api-production
    fi
  fi

  echo ""
  echo "==========================================="
  echo "  回滚完成"
  echo "==========================================="
  echo "Previous API:    ${PREV_API_IMAGE:-none}"
  echo "Previous Worker: ${PREV_WORKER_IMAGE:-none}"
  echo "Previous Web:    ${PREV_WEB_IMAGE:-none}"
  echo ""
  echo "部署失败，已自动回滚到上一版本"
  docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep production
  exit 1
}

# ---- 登录 Registry ----
if [ -n "$REGISTRY_TOKEN" ]; then
  echo "=========================================="
  echo "  Login to Registry (with retries)"
  echo "=========================================="
  REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1)
  retry_docker_login
fi

# ---- Pull 新版本镜像 ----
REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}"
REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}"
REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}"

echo "=========================================="
echo "  Pull images (with retries)"
echo "=========================================="
retry_docker_pull "$REGISTRY_API"
retry_docker_pull "$REGISTRY_WORKER"
retry_docker_pull "$REGISTRY_WEB"

echo "All images pulled."

# ---- 备份 legacy assets ----
echo "Backing up legacy assets from current web container..."
if docker inspect xiaoxia-web-production >/dev/null 2>&1; then
  _tmpdir="/tmp/legacy-assets-$$"
  rm -rf "$_tmpdir"
  mkdir -p "$_tmpdir"
  docker cp xiaoxia-web-production:/usr/share/nginx/html/assets/. "$_tmpdir/" 2>/dev/null || true
  # 只有目录非空才拷贝，避免覆盖有内容的 legacy assets
  if [ -d "$_tmpdir" ] && [ "$(ls -A "$_tmpdir" 2>/dev/null)" ]; then
    cp -an "$_tmpdir"/. "$LEGACY_ASSETS_DIR"/ 2>/dev/null || true
    echo "Legacy assets backed up: $(ls "$_tmpdir" | wc -l) files"
  fi
  rm -rf "$_tmpdir"
else
  echo "No existing web container, skipping legacy assets backup"
fi

# 清理 7 天前的 legacy assets
if [ -d "$LEGACY_ASSETS_DIR" ]; then
  find "$LEGACY_ASSETS_DIR" -type f -mtime +7 -delete 2>/dev/null || true
  echo "Legacy assets cleanup done (retain 7 days)"
fi

# ---- 检查基础设施容器 ----
echo "Checking infrastructure containers..."
for c in xiaoxia-postgres-production xiaoxia-redis-production; do
  if ! docker inspect "$c" >/dev/null 2>&1; then
    echo "ERROR: Required container not found: $c"
    exit 1
  fi
  state=$(docker inspect -f '{{.State.Status}}' "$c")
  if [ "$state" != "running" ]; then
    echo "ERROR: Container not running: $c ($state)"
    exit 1
  fi
done

# ---- 创建网络（不存在则创建） ----
docker network create xiaoxia-net-production 2>/dev/null || true

# ---- 数据库迁移 ----
if [ "$SKIP_MIGRATION" != "true" ]; then
  echo "Running database migrations..."
  docker run --rm \
    --env-file "$ENV_FILE" \
    --network xiaoxia-net-production \
    -e APP_ENV=production \
    "$REGISTRY_API" sh -c "cd /app && alembic upgrade head" || {
    echo "ERROR: Database migration failed"
    echo "Note: Migration failures are NOT automatically rolled back (data safety)"
    echo "Please manually check and fix the migration, then redeploy"
    exit 1
  }
  echo "Migrations completed."
else
  echo "Skipping migrations (SKIP_MIGRATION=true)"
fi

# ---- 停止旧容器 ----
echo "Stopping old containers..."
docker rm -f xiaoxia-api-production 2>/dev/null || true
docker rm -f xiaoxia-worker-production 2>/dev/null || true
docker rm -f xiaoxia-web-production 2>/dev/null || true

LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"

# ---- 启动 API ----
echo "Starting API container..."
docker run -d \
  --name xiaoxia-api-production \
  --env-file "$ENV_FILE" \
  --network xiaoxia-net-production \
  -p 127.0.0.1:8001:8000 \
  -e APP_ENV=production \
  -e APP_VERSION="$IMAGE_TAG" \
  -e GENERATED_FILES_DIR=/app/generated \
  -e GENERATED_FILES_URL_PREFIX=/generated-files \
  -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \
  -v "$GENERATED_DIR:/app/generated" \
  --restart unless-stopped \
  --cpus 2 \
  --memory 2g \
  --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \
  --health-interval 30s \
  --health-timeout 10s \
  --health-retries 3 \
  --health-start-period 40s \
  $LOG_OPTS \
  "$REGISTRY_API" || rollback

# ---- 启动 Worker ----
echo "Starting Worker container..."
docker run -d \
  --name xiaoxia-worker-production \
  --env-file "$ENV_FILE" \
  --network xiaoxia-net-production \
  -e APP_ENV=production \
  -e APP_VERSION="$IMAGE_TAG" \
  -e WORKER_CONCURRENCY=1 \
  -e WORKER_MAX_TASKS_PER_CHILD=100 \
  -e GENERATED_FILES_DIR=/app/generated \
  -e GENERATED_FILES_URL_PREFIX=/generated-files \
  -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \
  -v "$GENERATED_DIR:/app/generated" \
  --restart unless-stopped \
  --cpus 2 \
  --memory 2g \
  --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \
  --health-interval 30s \
  --health-timeout 10s \
  --health-retries 3 \
  --health-start-period 30s \
  $LOG_OPTS \
  "$REGISTRY_WORKER" || rollback

# ---- 启动 Web ----
LEGACY_VOLUME=""
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
  LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
  echo "Web container: legacy assets mounted (fallback)"
else
  echo "Web container: no legacy assets to mount"
fi

echo "Starting Web container..."
docker run -d \
  --name xiaoxia-web-production \
  --network xiaoxia-net-production \
  -p 127.0.0.1:3002:80 \
  --restart unless-stopped \
  --cpus 0.5 \
  --memory 512m \
  $LEGACY_VOLUME \
  --health-cmd "wget --spider -q http://127.0.0.1:80" \
  --health-interval 30s \
  --health-timeout 5s \
  --health-retries 3 \
  $LOG_OPTS \
  "$REGISTRY_WEB" || rollback

# ---- 等待 API 健康 ----
echo "Waiting for API to become healthy..."
i=0
while [ "$i" -lt 40 ]; do
  if curl -sf --max-time 5 http://127.0.0.1:8001/health >/dev/null 2>&1; then
    echo "API is healthy!"
    break
  fi
  i=$((i + 1))
  echo "  Waiting... ($i/40)"
  sleep 3
done

if [ "$i" -ge 40 ]; then
  echo "ERROR: API did not become healthy within 120s"
  docker logs --tail 50 xiaoxia-api-production
  rollback
fi

# ---- 等待 Web 健康 ----
echo "Waiting for Web to become healthy..."
i=0
while [ "$i" -lt 15 ]; do
  if curl -sf --max-time 5 http://127.0.0.1:3002/ >/dev/null 2>&1; then
    echo "Web is healthy!"
    break
  fi
  i=$((i + 1))
  echo "  Waiting... ($i/15)"
  sleep 2
done

if [ "$i" -ge 15 ]; then
  echo "ERROR: Web did not become healthy within 30s"
  docker logs --tail 30 xiaoxia-web-production
  rollback
fi

# ---- 清理旧镜像 ----
echo "Cleaning up old images..."
docker image prune -af --filter "until=168h" 2>/dev/null || true
docker builder prune -af --filter "until=168h" 2>/dev/null || true

echo ""
echo "=== Production deployment complete ==="
echo "API:      http://127.0.0.1:8000"
echo "Web:      http://127.0.0.1:3001"
echo "Version:  $IMAGE_TAG"
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep production