xiaoxia-saas/scripts/ci_production_healthcheck.sh

#!/bin/bash
# ===========================================
# CI Production 健康检查 + 自动回滚脚本（SSH 部署模式）
# ===========================================
#
# 在 CI Runner 上执行，通过公网 URL 检查 Production 部署健康状态。
# 不健康则通过 SSH 自动回滚到上一个版本的镜像。
#
# 用法:
#   ./ci_production_healthcheck.sh
#
# 环境变量:
#   PROD_API_URL          - Production API 公网地址 (默认 https://api.xiaoxiajianji.com)
#   PROD_WEB_URL          - Production Web 公网地址 (默认 https://saas.xiaoxiajianji.com)
#   HEALTH_CHECK_TIMEOUT  - 健康检查总超时秒数 (默认 300)
#   SKIP_ROLLBACK         - 失败时不自动回滚 (true/false, 默认 false)
#   SKIP_NOTIFY           - 跳过通知 (true/false, 默认 false)
#   CI_NOTIFY_WEBHOOK     - 通知 Webhook URL
#
#   PRODUCTION_SSH_HOST   - 生产服务器 SSH 地址
#   PRODUCTION_SSH_USER   - SSH 用户名 (默认 root)
#   PRODUCTION_SSH_PORT   - SSH 端口 (默认 22222)
#   PRODUCTION_SSH_KEY    - SSH 私钥内容
#   REGISTRY_TOKEN        - Registry Token（回滚时拉取旧镜像需要）
#
#   GITHUB_SHA            - 当前 commit SHA
#   GITHUB_REF_NAME       - tag 名 (如 v0.1.100)
#   GITHUB_RUN_ID         - CI Run ID
#   GITHUB_REPOSITORY     - 仓库名
#   GITHUB_ACTOR          - 提交者

set -eu

SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"

# 配置
PROD_API_URL="${PROD_API_URL:-https://api.xiaoxiajianji.com}"
PROD_WEB_URL="${PROD_WEB_URL:-https://saas.xiaoxiajianji.com}"
HEALTH_CHECK_TIMEOUT="${HEALTH_CHECK_TIMEOUT:-300}"
SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}"
SKIP_NOTIFY="${SKIP_NOTIFY:-false}"

PRODUCTION_SSH_HOST="${PRODUCTION_SSH_HOST:-47.98.113.167}"
PRODUCTION_SSH_USER="${PRODUCTION_SSH_USER:-root}"
PRODUCTION_SSH_PORT="${PRODUCTION_SSH_PORT:-22222}"

REGISTRY="${REGISTRY:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji}"
REGISTRY_USER="${REGISTRY_USER:-xiaoxia}"

# 颜色
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info()  { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn()  { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_step()  { echo -e "${BLUE}[STEP]${NC} $1"; }

# ===========================================
# SSH 工具函数
# ===========================================
SSH_KEY_PATH=""

setup_ssh() {
    # 查找或创建 SSH 密钥
    if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
        SSH_KEY_PATH="/root/.ssh/xiaoxia_runtime_builder"
    elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then
        SSH_KEY_PATH="$HOME/.ssh/xiaoxia_runtime_builder"
    elif [ -n "${PRODUCTION_SSH_KEY:-}" ]; then
        SSH_KEY_PATH="$HOME/.ssh/prod_deploy_key"
        mkdir -p "$HOME/.ssh"
        printf '%s\n' "$PRODUCTION_SSH_KEY" > "$SSH_KEY_PATH"
        chmod 600 "$SSH_KEY_PATH"
    else
        log_error "没有可用的 SSH 密钥"
        return 1
    fi

    ssh-keyscan -p "$PRODUCTION_SSH_PORT" -H "$PRODUCTION_SSH_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true
    log_info "SSH 已配置: ${PRODUCTION_SSH_USER}@${PRODUCTION_SSH_HOST}:${PRODUCTION_SSH_PORT}"
}

run_ssh() {
    local cmd="$1"
    ssh -p "$PRODUCTION_SSH_PORT" -i "$SSH_KEY_PATH" -o StrictHostKeyChecking=no \
        "${PRODUCTION_SSH_USER}@${PRODUCTION_SSH_HOST}" "$cmd"
}

# ===========================================
# 1. 记录部署前各服务的镜像版本（用于回滚）
# ===========================================
ROLLBACK_API_TAG=""
ROLLBACK_WORKER_TAG=""
ROLLBACK_WEB_TAG=""

save_rollback_target() {
    log_step "记录当前生产环境各服务镜像版本（回滚目标）..."

    # 通过 SSH 获取当前运行的容器镜像
    local api_image worker_image web_image
    api_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-api-production 2>/dev/null || echo ''")
    worker_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-worker-production 2>/dev/null || echo ''")
    web_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-web-production 2>/dev/null || echo ''")

    # 提取 tag（镜像名是 xiaoxia-saas-api:v0.1.100 格式）
    ROLLBACK_API_TAG=$(echo "$api_image" | sed 's/.*://' || echo "")
    ROLLBACK_WORKER_TAG=$(echo "$worker_image" | sed 's/.*://' || echo "")
    ROLLBACK_WEB_TAG=$(echo "$web_image" | sed 's/.*://' || echo "")

    log_info "  API:     ${ROLLBACK_API_TAG:-未知}"
    log_info "  Worker:  ${ROLLBACK_WORKER_TAG:-未知}"
    log_info "  Web:     ${ROLLBACK_WEB_TAG:-未知}"

    # 验证三个服务版本是否一致
    if [ -n "$ROLLBACK_API_TAG" ] && [ -n "$ROLLBACK_WORKER_TAG" ] && [ -n "$ROLLBACK_WEB_TAG" ]; then
        if [ "$ROLLBACK_API_TAG" = "$ROLLBACK_WORKER_TAG" ] && [ "$ROLLBACK_API_TAG" = "$ROLLBACK_WEB_TAG" ]; then
            log_info "  ✅ 三个服务版本一致: $ROLLBACK_API_TAG"
            export ROLLBACK_TAG="$ROLLBACK_API_TAG"
        else
            log_warn "  ⚠️  三个服务版本不一致，回滚时将分别使用各自版本"
            export ROLLBACK_API_TAG ROLLBACK_WORKER_TAG ROLLBACK_WEB_TAG
            export ROLLBACK_TAG_MIXED="true"
        fi
    else
        log_warn "  ⚠️  未能获取全部服务版本，回滚功能可能受限"
    fi
}

# ===========================================
# 2. 健康检查（公网视角）
# ===========================================
health_check() {
    local timeout="$HEALTH_CHECK_TIMEOUT"
    local start_time
    start_time=$(date +%s)

    log_step "公网健康检查（超时 ${timeout}s）..."
    log_info "  API: ${PROD_API_URL}/health"
    log_info "  Web: ${PROD_WEB_URL}/"

    local api_ok=false
    local web_ok=false
    local api_docs_ok=false
    local login_api_ok=false

    while [ $(( $(date +%s) - start_time )) -lt "$timeout" ]; do
        # 检查 API health
        if [ "$api_ok" = false ] && curl -sf --max-time 10 "${PROD_API_URL}/health" >/dev/null 2>&1; then
            log_info "✅ API 健康检查通过"
            api_ok=true
        fi

        # 检查 Web 首页
        if [ "$web_ok" = false ] && curl -sf --max-time 10 "$PROD_WEB_URL/" >/dev/null 2>&1; then
            log_info "✅ Web 前端检查通过"
            web_ok=true
        fi

        # 检查 API docs（生产环境禁用 /docs，404 表示 API 在正常响应，视为健康）
        if [ "$api_docs_ok" = false ]; then
            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "${PROD_API_URL}/docs" 2>/dev/null || echo "000")
            if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "404" ]; then
                log_info "✅ API Docs 检查通过（HTTP $HTTP_CODE）"
                api_docs_ok=true
            fi
        fi

        # 检查登录 API
        if [ "$login_api_ok" = false ]; then
            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 -X POST \
                "${PROD_API_URL}/api/v1/auth/login" \
                -H "Content-Type: application/json" \
                -d '{"email":"smoke@test.com","password":"wrong"}' 2>/dev/null || echo "000")
            if [ "$HTTP_CODE" = "401" ] || [ "$HTTP_CODE" = "422" ]; then
                log_info "✅ 登录 API 检查通过（HTTP $HTTP_CODE，符合预期）"
                login_api_ok=true
            fi
        fi

        # 都通过了就退出
        if [ "$api_ok" = true ] && [ "$web_ok" = true ] && [ "$api_docs_ok" = true ] && [ "$login_api_ok" = true ]; then
            log_info "🎉 所有健康检查通过！"
            return 0
        fi

        sleep 5
    done

    # 超时了
    log_error "❌ 健康检查超时 (${timeout}s)"
    [ "$api_ok" = false ]       && log_error "  - API health 未通过"
    [ "$web_ok" = false ]       && log_error "  - Web 前端未通过"
    [ "$api_docs_ok" = false ]  && log_error "  - API Docs 未通过"
    [ "$login_api_ok" = false ] && log_error "  - 登录 API 未通过"

    return 1
}

# ===========================================
# 3. 执行回滚（SSH 重新部署旧版本）
# ===========================================
do_rollback() {
    log_step "执行回滚：通过 SSH 重新部署旧版本镜像..."

    local rollback_tag="${ROLLBACK_TAG:-}"
    if [ -z "$rollback_tag" ] && [ "${ROLLBACK_TAG_MIXED:-}" != "true" ]; then
        log_error "没有可回滚的版本记录，无法自动回滚"
        return 1
    fi

    # 如果版本不一致，用 API 的版本作为回滚目标
    if [ -z "$rollback_tag" ]; then
        rollback_tag="$ROLLBACK_API_TAG"
    fi

    if [ -z "$rollback_tag" ]; then
        log_error "无法确定回滚版本"
        return 1
    fi

    log_info "回滚目标版本: $rollback_tag"

    # 通过 SSH 在生产服务器上执行回滚部署
    # 复用 Registry 方式部署脚本的逻辑，用旧版本 tag 重新部署
    local rollback_script=$(cat << 'ROLLBACK_EOF'
#!/bin/sh
set -eu

IMAGE_TAG="$1"
REGISTRY_TOKEN="$2"
REGISTRY="${REGISTRY:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji}"
REGISTRY_USER="${REGISTRY_USER:-xiaoxia}"

ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-production/.env}"
GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-production/generated}"
LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-production/legacy-assets}"

echo "=== Rollback to $IMAGE_TAG ==="

# 登录 Registry
if [ -n "$REGISTRY_TOKEN" ]; then
    REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1)
    printf %s "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin 2>/dev/null || true
fi

# Pull 旧版本镜像
LOCAL_API="xiaoxia-saas-api:${IMAGE_TAG}"
LOCAL_WORKER="xiaoxia-saas-worker:${IMAGE_TAG}"
LOCAL_WEB="xiaoxia-saas-web:${IMAGE_TAG}"

docker pull "${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}"
docker pull "${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}"
docker pull "${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}"

docker tag "${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" "$LOCAL_API"
docker tag "${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" "$LOCAL_WORKER"
docker tag "${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" "$LOCAL_WEB"

echo "Rollback images pulled."

# 停止当前容器
docker rm -f xiaoxia-api-production 2>/dev/null || true
docker rm -f xiaoxia-worker-production 2>/dev/null || true
docker rm -f xiaoxia-web-production 2>/dev/null || true

LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"

# 启动 API（回滚不跑 migration，因为新版本可能加了字段，回滚后代码是旧的但数据还在）
echo "Starting API (rollback)..."
docker run -d \
  --name xiaoxia-api-production \
  --env-file "$ENV_FILE" \
  --network xiaoxia-net-production \
  -p 127.0.0.1:8001:8000 \
  -e APP_ENV=production \
  -e APP_VERSION="$IMAGE_TAG" \
  -e GENERATED_FILES_DIR=/app/generated \
  -e GENERATED_FILES_URL_PREFIX=/generated-files \
  -e PUBLIC_API_BASE_URL=https://api.xiaoxiajianji.com \
  -v "$GENERATED_DIR:/app/generated" \
  --restart unless-stopped \
  --cpus 2 \
  --memory 2g \
  --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \
  --health-interval 30s \
  --health-timeout 10s \
  --health-retries 3 \
  --health-start-period 40s \
  $LOG_OPTS \
  "$LOCAL_API"

# 启动 Worker
echo "Starting Worker (rollback)..."
docker run -d \
  --name xiaoxia-worker-production \
  --env-file "$ENV_FILE" \
  --network xiaoxia-net-production \
  -e APP_ENV=production \
  -e APP_VERSION="$IMAGE_TAG" \
  -e WORKER_CONCURRENCY=1 \
  -e WORKER_MAX_TASKS_PER_CHILD=100 \
  -e GENERATED_FILES_DIR=/app/generated \
  -e GENERATED_FILES_URL_PREFIX=/generated-files \
  -e PUBLIC_API_BASE_URL=https://api.xiaoxiajianji.com \
  -v "$GENERATED_DIR:/app/generated" \
  --restart unless-stopped \
  --cpus 2 \
  --memory 2g \
  --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \
  --health-interval 30s \
  --health-timeout 10s \
  --health-retries 3 \
  --health-start-period 30s \
  $LOG_OPTS \
  "$LOCAL_WORKER"

# 启动 Web
LEGACY_VOLUME=""
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
  LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
fi

echo "Starting Web (rollback)..."
docker run -d \
  --name xiaoxia-web-production \
  --network xiaoxia-net-production \
  -p 127.0.0.1:3002:80 \
  --restart unless-stopped \
  --cpus 0.5 \
  --memory 512m \
  $LEGACY_VOLUME \
  --health-cmd "wget --spider -q http://127.0.0.1:80" \
  --health-interval 30s \
  --health-timeout 5s \
  --health-retries 3 \
  $LOG_OPTS \
  "$LOCAL_WEB"

# 等待 API 健康
echo "Waiting for API (rollback)..."
i=0
while [ "$i" -lt 40 ]; do
  if curl -sf --max-time 5 http://127.0.0.1:8001/health >/dev/null 2>&1; then
    echo "API healthy (rollback)."
    break
  fi
  i=$((i + 1))
  sleep 3
done

# 等待 Web 健康
echo "Waiting for Web (rollback)..."
i=0
while [ "$i" -lt 15 ]; do
  if curl -sf --max-time 5 http://127.0.0.1:3002/ >/dev/null 2>&1; then
    echo "Web healthy (rollback)."
    break
  fi
  i=$((i + 1))
  sleep 2
done

echo "=== Rollback complete: $IMAGE_TAG ==="
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep production
ROLLBACK_EOF
)

    # 将脚本 base64 编码后通过 SSH 执行
    local script_b64
    script_b64=$(echo "$rollback_script" | base64 -w 0)

    log_info "在生产服务器上执行回滚脚本..."
    if run_ssh "echo '$script_b64' | base64 -d | sh -s -- '$rollback_tag' '${REGISTRY_TOKEN:-}'" 2>&1; then
        log_info "✅ 回滚命令执行完成"
        return 0
    else
        log_error "❌ 回滚命令执行失败"
        return 1
    fi
}

# ===========================================
# 4. 发送通知
# ===========================================
send_notification() {
    local status="$1"   # success / failure / rollback
    local detail="$2"

    if [ "${SKIP_NOTIFY:-false}" = "true" ]; then
        log_info "跳过通知（SKIP_NOTIFY=true）"
        return 0
    fi

    local webhook="${CI_NOTIFY_WEBHOOK:-}"
    if [ -z "$webhook" ]; then
        log_warn "未配置 CI_NOTIFY_WEBHOOK，跳过通知"
        return 0
    fi

    if [ -f "$SCRIPT_DIR/deploy_notify.py" ]; then
        python3 "$SCRIPT_DIR/deploy_notify.py" \
            --status "$status" \
            --detail "$detail" \
            --webhook "$webhook" \
            --env production \
            2>/dev/null || log_warn "通知发送失败（非致命）"
    else
        log_warn "找不到 deploy_notify.py，跳过通知"
    fi

    # 标记：通知已由健康检查脚本发出，避免 CI 兜底通知重复发送
    echo "$status" > /tmp/prod_deploy_notification_sent
}

# ===========================================
# 主流程
# ===========================================
main() {
    echo ""
    echo "=========================================="
    echo "  CI Production 健康检查 + 自动回滚"
    echo "=========================================="
    echo ""

    local deploy_status="success"
    local deploy_detail=""

    # 1. 设置 SSH
    if ! setup_ssh; then
        log_error "SSH 配置失败，无法执行回滚"
    fi

    # 2. 记录部署前状态（回滚目标）
    save_rollback_target || true

    # 3. 健康检查（公网视角）
    if ! health_check; then
        log_error "健康检查失败"
        deploy_status="failure"
        deploy_detail="公网健康检查超时，部署后服务未正常响应"

        # 自动回滚
        if [ "${SKIP_ROLLBACK:-false}" != "true" ]; then
            log_warn "开始自动回滚..."
            if do_rollback; then
                deploy_status="rollback"
                deploy_detail="健康检查失败，已自动回滚到上一版本 (${ROLLBACK_TAG:-未知})"

                # 回滚后再检查一下公网状态
                log_info "回滚完成，重新检查公网健康状态..."
                if health_check; then
                    log_info "✅ 回滚后服务已恢复"
                    deploy_detail="${deploy_detail}，回滚后服务已恢复"
                else
                    log_error "⚠️  回滚后健康检查仍未通过，请手动排查"
                    deploy_detail="${deploy_detail}，但回滚后仍未恢复，请紧急排查"
                fi
            else
                deploy_detail="健康检查失败且回滚失败，请手动排查"
            fi
        fi

        send_notification "$deploy_status" "$deploy_detail"

        # 失败时退出非零，让 CI Job 标记为失败
        exit 1
    fi

    # 4. 成功
    log_info ""
    log_info "=================================="
    log_info "  ✅ Production 部署成功！"
    log_info "=================================="

    deploy_detail="部署成功，所有健康检查通过 (${GITHUB_REF_NAME:-未知版本})"
    send_notification "success" "$deploy_detail"
}

main "$@"