#!/bin/sh # =========================================== # Production 部署脚本(SSH 模式,支持自动回滚) # =========================================== # 通过 SSH 在 production 服务器上执行 # # 环境变量: # IMAGE_TAG - 镜像版本 tag(如 commit SHA 或分支名) # REGISTRY_TOKEN - Registry 访问令牌 # REGISTRY - Registry 地址(默认 xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji) # REGISTRY_USER - Registry 用户名(默认 xiaoxia) # ENV_FILE - 环境变量文件路径 # GENERATED_DIR - 生成文件目录 # SKIP_MIGRATION - 跳过数据库迁移(true/false,默认 false) # SKIP_ROLLBACK - 失败时跳过自动回滚(true/false,默认 false) set -eu # ---- 重试工具函数 ---- retry_cmd() { local max_attempts=$1 local backoff=$2 shift 2 local attempt=1 while [ $attempt -le $max_attempts ]; do if "$@"; then return 0 fi echo " attempt $attempt/$max_attempts failed, retrying in ${backoff}s..." sleep $backoff backoff=$((backoff * 2)) attempt=$((attempt + 1)) done echo " ERROR: failed after $max_attempts retries" return 1 } retry_docker_login() { echo "Logging in to registry (up to 3 retries)" export REGISTRY_TOKEN REGISTRY_HOST REGISTRY_USER if retry_cmd 3 5 sh -c 'printf "%s" "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin'; then return 0 fi echo "WARN: docker login failed after retries, will try pull anyway" return 0 } retry_docker_pull() { local image=$1 echo "Pulling $image (up to 3 retries)" retry_cmd 3 10 docker pull "$image" } IMAGE_TAG="${IMAGE_TAG:-}" REGISTRY="${REGISTRY:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji}" REGISTRY_USER="${ACR_USERNAME:-${REGISTRY_USER:-nick0415343655}}" REGISTRY_TOKEN="${ACR_PASSWORD:-${REGISTRY_TOKEN:-}}" ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-production/.env}" GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-production/generated}" LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-production/legacy-assets}" NGINX_CONF_FILE="${NGINX_CONF_FILE:-/var/lib/xiaoxia-saas-production/nginx-production.conf}" SKIP_MIGRATION="${SKIP_MIGRATION:-false}" SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}" if [ -z "$IMAGE_TAG" ]; then echo "ERROR: IMAGE_TAG is required" exit 1 fi test -f "$ENV_FILE" mkdir -p "$GENERATED_DIR" mkdir -p "$LEGACY_ASSETS_DIR" # ── 写入 Production Nginx 配置 ── echo "Writing production nginx config..." cat > "$NGINX_CONF_FILE" << 'NGINX_EOF' server { listen 80; server_name _; root /usr/share/nginx/html; index index.html; gzip on; gzip_vary on; gzip_min_length 1024; gzip_types text/plain text/css text/xml text/javascript application/javascript application/json application/xml+rss; client_max_body_size 800m; location / { try_files $uri /index.html; } resolver 127.0.0.11 valid=10s; resolver_timeout 5s; location /api/ { proxy_pass http://xiaoxia-api-production:8000/api/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 300s; proxy_send_timeout 300s; proxy_request_buffering off; } location /generated-files/ { alias /app/generated/; } location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ { expires 1y; add_header Cache-Control "public, immutable"; } } NGINX_EOF echo "✅ Nginx config written: $NGINX_CONF_FILE" echo "===========================================" echo " Production 部署 - $IMAGE_TAG" echo "===========================================" # ---- 记录当前运行的镜像版本(用于回滚) ---- echo "Recording current image versions for rollback..." PREV_API_IMAGE="" PREV_WORKER_IMAGE="" PREV_WEB_IMAGE="" for c in xiaoxia-api-production xiaoxia-worker-production xiaoxia-web-production; do if docker inspect "$c" >/dev/null 2>&1; then img=$(docker inspect -f '{{.Config.Image}}' "$c") case "$c" in xiaoxia-api-production) PREV_API_IMAGE="$img" ;; xiaoxia-worker-production) PREV_WORKER_IMAGE="$img" ;; xiaoxia-web-production) PREV_WEB_IMAGE="$img" ;; esac echo " $c -> $img" else echo " $c -> (not running)" fi done # ---- 回滚函数 ---- rollback() { echo "" echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" echo " 部署失败,正在自动回滚到上一版本..." echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" echo "" if [ "$SKIP_ROLLBACK" = "true" ]; then echo "SKIP_ROLLBACK=true,跳过自动回滚" exit 1 fi # 停止当前(失败的)新容器 echo "Stopping new containers..." docker rm -f xiaoxia-api-production 2>/dev/null || true docker rm -f xiaoxia-worker-production 2>/dev/null || true docker rm -f xiaoxia-web-production 2>/dev/null || true LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" # 恢复 API if [ -n "$PREV_API_IMAGE" ]; then echo "Rolling back API to: $PREV_API_IMAGE" docker run -d \ --name xiaoxia-api-production \ --env-file "$ENV_FILE" \ --network xiaoxia-net-production \ -p 127.0.0.1:8001:8000 \ -e APP_ENV=production \ -e APP_VERSION="$(echo $PREV_API_IMAGE | grep -oE '[^:]+$')" \ -e GENERATED_FILES_DIR=/app/generated \ -e GENERATED_FILES_URL_PREFIX=/generated-files \ -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \ -v "$GENERATED_DIR:/app/generated" \ --restart unless-stopped \ --cpus 2 \ --memory 2g \ --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \ --health-start-period 40s \ $LOG_OPTS \ "$PREV_API_IMAGE" else echo "No previous API image to roll back to" fi # 恢复 Worker if [ -n "$PREV_WORKER_IMAGE" ]; then echo "Rolling back Worker to: $PREV_WORKER_IMAGE" docker run -d \ --name xiaoxia-worker-production \ --env-file "$ENV_FILE" \ --network xiaoxia-net-production \ -e APP_ENV=production \ -e APP_VERSION="$(echo $PREV_WORKER_IMAGE | grep -oE '[^:]+$')" \ -e WORKER_CONCURRENCY=1 \ -e WORKER_MAX_TASKS_PER_CHILD=100 \ -e GENERATED_FILES_DIR=/app/generated \ -e GENERATED_FILES_URL_PREFIX=/generated-files \ -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \ -v "$GENERATED_DIR:/app/generated" \ --restart unless-stopped \ --cpus 2 \ --memory 2g \ --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \ --health-start-period 30s \ $LOG_OPTS \ "$PREV_WORKER_IMAGE" else echo "No previous Worker image to roll back to" fi # 恢复 Web if [ -n "$PREV_WEB_IMAGE" ]; then echo "Rolling back Web to: $PREV_WEB_IMAGE" LEGACY_VOLUME="" if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" fi docker run -d \ --name xiaoxia-web-production \ --network xiaoxia-net-production \ -p 127.0.0.1:3002:80 \ --restart unless-stopped \ --cpus 0.5 \ --memory 512m \ $LEGACY_VOLUME \ -v "$NGINX_CONF_FILE:/etc/nginx/conf.d/default.conf:ro" \ --health-cmd "wget --spider -q http://127.0.0.1:80" \ --health-interval 30s \ --health-timeout 5s \ --health-retries 3 \ $LOG_OPTS \ "$PREV_WEB_IMAGE" else echo "No previous Web image to roll back to" fi # 等待 API 回滚后恢复健康 if [ -n "$PREV_API_IMAGE" ]; then echo "Waiting for rolled-back API to become healthy..." i=0 while [ "$i" -lt 40 ]; do if curl -sf --max-time 5 http://127.0.0.1:8001/health >/dev/null 2>&1; then echo "Rolled-back API is healthy!" break fi i=$((i + 1)) echo " Waiting... ($i/40)" sleep 3 done if [ "$i" -ge 40 ]; then echo "WARN: Rolled-back API did not become healthy within 120s" docker logs --tail 30 xiaoxia-api-production fi fi echo "" echo "===========================================" echo " 回滚完成" echo "===========================================" echo "Previous API: ${PREV_API_IMAGE:-none}" echo "Previous Worker: ${PREV_WORKER_IMAGE:-none}" echo "Previous Web: ${PREV_WEB_IMAGE:-none}" echo "" echo "部署失败,已自动回滚到上一版本" docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep production exit 1 } # ---- 登录 Registry ---- if [ -n "$REGISTRY_TOKEN" ]; then echo "==========================================" echo " Login to Registry (with retries)" echo "==========================================" REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1) retry_docker_login fi # ---- Pull 新版本镜像 ---- REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" echo "==========================================" echo " Pull images (with retries)" echo "==========================================" retry_docker_pull "$REGISTRY_API" retry_docker_pull "$REGISTRY_WORKER" retry_docker_pull "$REGISTRY_WEB" echo "All images pulled." # ---- 备份 legacy assets ---- echo "Backing up legacy assets from current web container..." if docker inspect xiaoxia-web-production >/dev/null 2>&1; then _tmpdir="/tmp/legacy-assets-$$" rm -rf "$_tmpdir" mkdir -p "$_tmpdir" docker cp xiaoxia-web-production:/usr/share/nginx/html/assets/. "$_tmpdir/" 2>/dev/null || true # 只有目录非空才拷贝,避免覆盖有内容的 legacy assets if [ -d "$_tmpdir" ] && [ "$(ls -A "$_tmpdir" 2>/dev/null)" ]; then cp -an "$_tmpdir"/. "$LEGACY_ASSETS_DIR"/ 2>/dev/null || true echo "Legacy assets backed up: $(ls "$_tmpdir" | wc -l) files" fi rm -rf "$_tmpdir" else echo "No existing web container, skipping legacy assets backup" fi # 清理 7 天前的 legacy assets if [ -d "$LEGACY_ASSETS_DIR" ]; then find "$LEGACY_ASSETS_DIR" -type f -mtime +7 -delete 2>/dev/null || true echo "Legacy assets cleanup done (retain 7 days)" fi # ---- 检查基础设施容器 ---- echo "Checking infrastructure containers..." for c in xiaoxia-postgres-production xiaoxia-redis-production; do if ! docker inspect "$c" >/dev/null 2>&1; then echo "ERROR: Required container not found: $c" exit 1 fi state=$(docker inspect -f '{{.State.Status}}' "$c") if [ "$state" != "running" ]; then echo "ERROR: Container not running: $c ($state)" exit 1 fi done # ---- 创建网络(不存在则创建) ---- docker network create xiaoxia-net-production 2>/dev/null || true # ---- 数据库迁移 ---- if [ "$SKIP_MIGRATION" != "true" ]; then echo "Running database migrations..." docker run --rm \ --env-file "$ENV_FILE" \ --network xiaoxia-net-production \ -e APP_ENV=production \ "$REGISTRY_API" sh -c "cd /app && alembic upgrade head" || { echo "ERROR: Database migration failed" echo "Note: Migration failures are NOT automatically rolled back (data safety)" echo "Please manually check and fix the migration, then redeploy" exit 1 } echo "Migrations completed." else echo "Skipping migrations (SKIP_MIGRATION=true)" fi # ---- 停止旧容器 ---- echo "Stopping old containers..." docker rm -f xiaoxia-api-production 2>/dev/null || true docker rm -f xiaoxia-worker-production 2>/dev/null || true docker rm -f xiaoxia-web-production 2>/dev/null || true LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" # ---- 启动 API ---- echo "Starting API container..." docker run -d \ --name xiaoxia-api-production \ --env-file "$ENV_FILE" \ --network xiaoxia-net-production \ -p 127.0.0.1:8001:8000 \ -e APP_ENV=production \ -e APP_VERSION="$IMAGE_TAG" \ -e GENERATED_FILES_DIR=/app/generated \ -e GENERATED_FILES_URL_PREFIX=/generated-files \ -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \ -v "$GENERATED_DIR:/app/generated" \ --restart unless-stopped \ --cpus 2 \ --memory 2g \ --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \ --health-start-period 40s \ $LOG_OPTS \ "$REGISTRY_API" || rollback # ---- 启动 Worker ---- echo "Starting Worker container..." docker run -d \ --name xiaoxia-worker-production \ --env-file "$ENV_FILE" \ --network xiaoxia-net-production \ -e APP_ENV=production \ -e APP_VERSION="$IMAGE_TAG" \ -e WORKER_CONCURRENCY=1 \ -e WORKER_MAX_TASKS_PER_CHILD=100 \ -e GENERATED_FILES_DIR=/app/generated \ -e GENERATED_FILES_URL_PREFIX=/generated-files \ -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \ -v "$GENERATED_DIR:/app/generated" \ --restart unless-stopped \ --cpus 2 \ --memory 2g \ --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \ --health-start-period 30s \ $LOG_OPTS \ "$REGISTRY_WORKER" || rollback # ---- 启动 Web ---- LEGACY_VOLUME="" if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" echo "Web container: legacy assets mounted (fallback)" else echo "Web container: no legacy assets to mount" fi echo "Starting Web container..." docker run -d \ --name xiaoxia-web-production \ --network xiaoxia-net-production \ -p 127.0.0.1:3002:80 \ --restart unless-stopped \ --cpus 0.5 \ --memory 512m \ -v "$NGINX_CONF_FILE:/etc/nginx/conf.d/default.conf:ro" \ $LEGACY_VOLUME \ --health-cmd "wget --spider -q http://127.0.0.1:80" \ --health-interval 30s \ --health-timeout 5s \ --health-retries 3 \ $LOG_OPTS \ "$REGISTRY_WEB" || rollback # ---- 等待 API 健康 ---- echo "Waiting for API to become healthy..." i=0 while [ "$i" -lt 40 ]; do if curl -sf --max-time 5 http://127.0.0.1:8001/health >/dev/null 2>&1; then echo "API is healthy!" break fi i=$((i + 1)) echo " Waiting... ($i/40)" sleep 3 done if [ "$i" -ge 40 ]; then echo "ERROR: API did not become healthy within 120s" docker logs --tail 50 xiaoxia-api-production rollback fi # ---- 等待 Web 健康 ---- echo "Waiting for Web to become healthy..." i=0 while [ "$i" -lt 15 ]; do if curl -sf --max-time 5 http://127.0.0.1:3002/ >/dev/null 2>&1; then echo "Web is healthy!" break fi i=$((i + 1)) echo " Waiting... ($i/15)" sleep 2 done if [ "$i" -ge 15 ]; then echo "ERROR: Web did not become healthy within 30s" docker logs --tail 30 xiaoxia-web-production rollback fi # ---- 清理旧镜像 ---- echo "Cleaning up old images..." docker image prune -af --filter "until=168h" 2>/dev/null || true docker builder prune -af --filter "until=168h" 2>/dev/null || true echo "" echo "=== Production deployment complete ===" echo "API: http://127.0.0.1:8000" echo "Web: http://127.0.0.1:3001" echo "Version: $IMAGE_TAG" docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep production