diff --git a/scripts/ci_production_healthcheck.sh b/scripts/ci_production_healthcheck.sh new file mode 100644 index 000000000..7d76c7651 --- /dev/null +++ b/scripts/ci_production_healthcheck.sh @@ -0,0 +1,483 @@ +#!/bin/bash +# =========================================== +# CI Production 健康检查 + 自动回滚脚本(SSH 部署模式) +# =========================================== +# +# 在 CI Runner 上执行,通过公网 URL 检查 Production 部署健康状态。 +# 不健康则通过 SSH 自动回滚到上一个版本的镜像。 +# +# 用法: +# ./ci_production_healthcheck.sh +# +# 环境变量: +# PROD_API_URL - Production API 公网地址 (默认 https://api.xiaoxiajianji.com) +# PROD_WEB_URL - Production Web 公网地址 (默认 https://saas.xiaoxiajianji.com) +# HEALTH_CHECK_TIMEOUT - 健康检查总超时秒数 (默认 180) +# SKIP_ROLLBACK - 失败时不自动回滚 (true/false, 默认 false) +# SKIP_NOTIFY - 跳过通知 (true/false, 默认 false) +# CI_NOTIFY_WEBHOOK - 通知 Webhook URL +# +# PRODUCTION_SSH_HOST - 生产服务器 SSH 地址 +# PRODUCTION_SSH_USER - SSH 用户名 (默认 root) +# PRODUCTION_SSH_PORT - SSH 端口 (默认 22222) +# PRODUCTION_SSH_KEY - SSH 私钥内容 +# REGISTRY_TOKEN - Registry Token(回滚时拉取旧镜像需要) +# +# GITHUB_SHA - 当前 commit SHA +# GITHUB_REF_NAME - tag 名 (如 v0.1.100) +# GITHUB_RUN_ID - CI Run ID +# GITHUB_REPOSITORY - 仓库名 +# GITHUB_ACTOR - 提交者 + +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" + +# 配置 +PROD_API_URL="${PROD_API_URL:-https://api.xiaoxiajianji.com}" +PROD_WEB_URL="${PROD_WEB_URL:-https://saas.xiaoxiajianji.com}" +HEALTH_CHECK_TIMEOUT="${HEALTH_CHECK_TIMEOUT:-180}" +SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}" +SKIP_NOTIFY="${SKIP_NOTIFY:-false}" + +PRODUCTION_SSH_HOST="${PRODUCTION_SSH_HOST:-47.98.113.167}" +PRODUCTION_SSH_USER="${PRODUCTION_SSH_USER:-root}" +PRODUCTION_SSH_PORT="${PRODUCTION_SSH_PORT:-22222}" + +REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}" +REGISTRY_USER="${REGISTRY_USER:-xiaoxia}" + +# 颜色 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } +log_step() { echo -e "${BLUE}[STEP]${NC} $1"; } + +# =========================================== +# SSH 工具函数 +# =========================================== +SSH_KEY_PATH="" + +setup_ssh() { + # 查找或创建 SSH 密钥 + if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then + SSH_KEY_PATH="/root/.ssh/xiaoxia_runtime_builder" + elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then + SSH_KEY_PATH="$HOME/.ssh/xiaoxia_runtime_builder" + elif [ -n "${PRODUCTION_SSH_KEY:-}" ]; then + SSH_KEY_PATH="$HOME/.ssh/prod_deploy_key" + mkdir -p "$HOME/.ssh" + printf '%s\n' "$PRODUCTION_SSH_KEY" > "$SSH_KEY_PATH" + chmod 600 "$SSH_KEY_PATH" + else + log_error "没有可用的 SSH 密钥" + return 1 + fi + + ssh-keyscan -p "$PRODUCTION_SSH_PORT" -H "$PRODUCTION_SSH_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true + log_info "SSH 已配置: ${PRODUCTION_SSH_USER}@${PRODUCTION_SSH_HOST}:${PRODUCTION_SSH_PORT}" +} + +run_ssh() { + local cmd="$1" + ssh -p "$PRODUCTION_SSH_PORT" -i "$SSH_KEY_PATH" -o StrictHostKeyChecking=no \ + "${PRODUCTION_SSH_USER}@${PRODUCTION_SSH_HOST}" "$cmd" +} + +# =========================================== +# 1. 记录部署前各服务的镜像版本(用于回滚) +# =========================================== +ROLLBACK_API_TAG="" +ROLLBACK_WORKER_TAG="" +ROLLBACK_WEB_TAG="" + +save_rollback_target() { + log_step "记录当前生产环境各服务镜像版本(回滚目标)..." + + # 通过 SSH 获取当前运行的容器镜像 + local api_image worker_image web_image + api_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-api-production 2>/dev/null || echo ''") + worker_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-worker-production 2>/dev/null || echo ''") + web_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-web-production 2>/dev/null || echo ''") + + # 提取 tag(镜像名是 xiaoxia-saas-api:v0.1.100 格式) + ROLLBACK_API_TAG=$(echo "$api_image" | sed 's/.*://' || echo "") + ROLLBACK_WORKER_TAG=$(echo "$worker_image" | sed 's/.*://' || echo "") + ROLLBACK_WEB_TAG=$(echo "$web_image" | sed 's/.*://' || echo "") + + log_info " API: ${ROLLBACK_API_TAG:-未知}" + log_info " Worker: ${ROLLBACK_WORKER_TAG:-未知}" + log_info " Web: ${ROLLBACK_WEB_TAG:-未知}" + + # 验证三个服务版本是否一致 + if [ -n "$ROLLBACK_API_TAG" ] && [ -n "$ROLLBACK_WORKER_TAG" ] && [ -n "$ROLLBACK_WEB_TAG" ]; then + if [ "$ROLLBACK_API_TAG" = "$ROLLBACK_WORKER_TAG" ] && [ "$ROLLBACK_API_TAG" = "$ROLLBACK_WEB_TAG" ]; then + log_info " ✅ 三个服务版本一致: $ROLLBACK_API_TAG" + export ROLLBACK_TAG="$ROLLBACK_API_TAG" + else + log_warn " ⚠️ 三个服务版本不一致,回滚时将分别使用各自版本" + export ROLLBACK_API_TAG ROLLBACK_WORKER_TAG ROLLBACK_WEB_TAG + export ROLLBACK_TAG_MIXED="true" + fi + else + log_warn " ⚠️ 未能获取全部服务版本,回滚功能可能受限" + fi +} + +# =========================================== +# 2. 健康检查(公网视角) +# =========================================== +health_check() { + local timeout="$HEALTH_CHECK_TIMEOUT" + local start_time + start_time=$(date +%s) + + log_step "公网健康检查(超时 ${timeout}s)..." + log_info " API: ${PROD_API_URL}/health" + log_info " Web: ${PROD_WEB_URL}/" + + local api_ok=false + local web_ok=false + local api_docs_ok=false + local login_api_ok=false + + while [ $(( $(date +%s) - start_time )) -lt "$timeout" ]; do + # 检查 API health + if [ "$api_ok" = false ] && curl -sf --max-time 10 "${PROD_API_URL}/health" >/dev/null 2>&1; then + log_info "✅ API 健康检查通过" + api_ok=true + fi + + # 检查 Web 首页 + if [ "$web_ok" = false ] && curl -sf --max-time 10 "$PROD_WEB_URL/" >/dev/null 2>&1; then + log_info "✅ Web 前端检查通过" + web_ok=true + fi + + # 检查 API docs + if [ "$api_docs_ok" = false ]; then + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "${PROD_API_URL}/docs" 2>/dev/null || echo "000") + if [ "$HTTP_CODE" = "200" ]; then + log_info "✅ API Docs 检查通过" + api_docs_ok=true + fi + fi + + # 检查登录 API + if [ "$login_api_ok" = false ]; then + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 -X POST \ + "${PROD_API_URL}/api/v1/auth/login" \ + -H "Content-Type: application/json" \ + -d '{"email":"smoke@test.com","password":"wrong"}' 2>/dev/null || echo "000") + if [ "$HTTP_CODE" = "401" ] || [ "$HTTP_CODE" = "422" ]; then + log_info "✅ 登录 API 检查通过(HTTP $HTTP_CODE,符合预期)" + login_api_ok=true + fi + fi + + # 都通过了就退出 + if [ "$api_ok" = true ] && [ "$web_ok" = true ] && [ "$api_docs_ok" = true ] && [ "$login_api_ok" = true ]; then + log_info "🎉 所有健康检查通过!" + return 0 + fi + + sleep 5 + done + + # 超时了 + log_error "❌ 健康检查超时 (${timeout}s)" + [ "$api_ok" = false ] && log_error " - API health 未通过" + [ "$web_ok" = false ] && log_error " - Web 前端未通过" + [ "$api_docs_ok" = false ] && log_error " - API Docs 未通过" + [ "$login_api_ok" = false ] && log_error " - 登录 API 未通过" + + return 1 +} + +# =========================================== +# 3. 执行回滚(SSH 重新部署旧版本) +# =========================================== +do_rollback() { + log_step "执行回滚:通过 SSH 重新部署旧版本镜像..." + + local rollback_tag="${ROLLBACK_TAG:-}" + if [ -z "$rollback_tag" ] && [ "${ROLLBACK_TAG_MIXED:-}" != "true" ]; then + log_error "没有可回滚的版本记录,无法自动回滚" + return 1 + fi + + # 如果版本不一致,用 API 的版本作为回滚目标 + if [ -z "$rollback_tag" ]; then + rollback_tag="$ROLLBACK_API_TAG" + fi + + if [ -z "$rollback_tag" ]; then + log_error "无法确定回滚版本" + return 1 + fi + + log_info "回滚目标版本: $rollback_tag" + + # 通过 SSH 在生产服务器上执行回滚部署 + # 复用 Registry 方式部署脚本的逻辑,用旧版本 tag 重新部署 + local rollback_script=$(cat << 'ROLLBACK_EOF' +#!/bin/sh +set -eu + +IMAGE_TAG="$1" +REGISTRY_TOKEN="$2" +REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}" +REGISTRY_USER="${REGISTRY_USER:-xiaoxia}" + +ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-production/.env}" +GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-production/generated}" +LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-production/legacy-assets}" + +echo "=== Rollback to $IMAGE_TAG ===" + +# 登录 Registry +if [ -n "$REGISTRY_TOKEN" ]; then + REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1) + printf %s "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin 2>/dev/null || true +fi + +# Pull 旧版本镜像 +LOCAL_API="xiaoxia-saas-api:${IMAGE_TAG}" +LOCAL_WORKER="xiaoxia-saas-worker:${IMAGE_TAG}" +LOCAL_WEB="xiaoxia-saas-web:${IMAGE_TAG}" + +docker pull "${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" +docker pull "${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" +docker pull "${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" + +docker tag "${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" "$LOCAL_API" +docker tag "${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" "$LOCAL_WORKER" +docker tag "${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" "$LOCAL_WEB" + +echo "Rollback images pulled." + +# 停止当前容器 +docker rm -f xiaoxia-api-production 2>/dev/null || true +docker rm -f xiaoxia-worker-production 2>/dev/null || true +docker rm -f xiaoxia-web-production 2>/dev/null || true + +LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" + +# 启动 API(回滚不跑 migration,因为新版本可能加了字段,回滚后代码是旧的但数据还在) +echo "Starting API (rollback)..." +docker run -d \ + --name xiaoxia-api-production \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-production \ + -p 127.0.0.1:8001:8000 \ + -e APP_ENV=production \ + -e APP_VERSION="$IMAGE_TAG" \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --cpus 2 \ + --memory 2g \ + --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 40s \ + $LOG_OPTS \ + "$LOCAL_API" + +# 启动 Worker +echo "Starting Worker (rollback)..." +docker run -d \ + --name xiaoxia-worker-production \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-production \ + -e APP_ENV=production \ + -e APP_VERSION="$IMAGE_TAG" \ + -e WORKER_CONCURRENCY=1 \ + -e WORKER_MAX_TASKS_PER_CHILD=100 \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --cpus 2 \ + --memory 2g \ + --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 30s \ + $LOG_OPTS \ + "$LOCAL_WORKER" + +# 启动 Web +LEGACY_VOLUME="" +if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then + LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" +fi + +echo "Starting Web (rollback)..." +docker run -d \ + --name xiaoxia-web-production \ + --network xiaoxia-net-production \ + -p 127.0.0.1:3002:80 \ + --restart unless-stopped \ + --cpus 0.5 \ + --memory 512m \ + $LEGACY_VOLUME \ + --health-cmd "wget --spider -q http://127.0.0.1:80" \ + --health-interval 30s \ + --health-timeout 5s \ + --health-retries 3 \ + $LOG_OPTS \ + "$LOCAL_WEB" + +# 等待 API 健康 +echo "Waiting for API (rollback)..." +i=0 +while [ "$i" -lt 40 ]; do + if curl -sf --max-time 5 http://127.0.0.1:8001/health >/dev/null 2>&1; then + echo "API healthy (rollback)." + break + fi + i=$((i + 1)) + sleep 3 +done + +# 等待 Web 健康 +echo "Waiting for Web (rollback)..." +i=0 +while [ "$i" -lt 15 ]; do + if curl -sf --max-time 5 http://127.0.0.1:3002/ >/dev/null 2>&1; then + echo "Web healthy (rollback)." + break + fi + i=$((i + 1)) + sleep 2 +done + +echo "=== Rollback complete: $IMAGE_TAG ===" +docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep production +ROLLBACK_EOF +) + + # 将脚本 base64 编码后通过 SSH 执行 + local script_b64 + script_b64=$(echo "$rollback_script" | base64 -w 0) + + log_info "在生产服务器上执行回滚脚本..." + if run_ssh "echo '$script_b64' | base64 -d | sh -s -- '$rollback_tag' '${REGISTRY_TOKEN:-}'" 2>&1; then + log_info "✅ 回滚命令执行完成" + return 0 + else + log_error "❌ 回滚命令执行失败" + return 1 + fi +} + +# =========================================== +# 4. 发送通知 +# =========================================== +send_notification() { + local status="$1" # success / failure / rollback + local detail="$2" + + if [ "${SKIP_NOTIFY:-false}" = "true" ]; then + log_info "跳过通知(SKIP_NOTIFY=true)" + return 0 + fi + + local webhook="${CI_NOTIFY_WEBHOOK:-}" + if [ -z "$webhook" ]; then + log_warn "未配置 CI_NOTIFY_WEBHOOK,跳过通知" + return 0 + fi + + if [ -f "$SCRIPT_DIR/deploy_notify.py" ]; then + python3 "$SCRIPT_DIR/deploy_notify.py" \ + --status "$status" \ + --detail "$detail" \ + --webhook "$webhook" \ + --env production \ + 2>/dev/null || log_warn "通知发送失败(非致命)" + else + log_warn "找不到 deploy_notify.py,跳过通知" + fi + + # 标记:通知已由健康检查脚本发出,避免 CI 兜底通知重复发送 + echo "$status" > /tmp/prod_deploy_notification_sent +} + +# =========================================== +# 主流程 +# =========================================== +main() { + echo "" + echo "==========================================" + echo " CI Production 健康检查 + 自动回滚" + echo "==========================================" + echo "" + + local deploy_status="success" + local deploy_detail="" + + # 1. 设置 SSH + if ! setup_ssh; then + log_error "SSH 配置失败,无法执行回滚" + fi + + # 2. 记录部署前状态(回滚目标) + save_rollback_target || true + + # 3. 健康检查(公网视角) + if ! health_check; then + log_error "健康检查失败" + deploy_status="failure" + deploy_detail="公网健康检查超时,部署后服务未正常响应" + + # 自动回滚 + if [ "${SKIP_ROLLBACK:-false}" != "true" ]; then + log_warn "开始自动回滚..." + if do_rollback; then + deploy_status="rollback" + deploy_detail="健康检查失败,已自动回滚到上一版本 (${ROLLBACK_TAG:-未知})" + + # 回滚后再检查一下公网状态 + log_info "回滚完成,重新检查公网健康状态..." + if health_check; then + log_info "✅ 回滚后服务已恢复" + deploy_detail="${deploy_detail},回滚后服务已恢复" + else + log_error "⚠️ 回滚后健康检查仍未通过,请手动排查" + deploy_detail="${deploy_detail},但回滚后仍未恢复,请紧急排查" + fi + else + deploy_detail="健康检查失败且回滚失败,请手动排查" + fi + fi + + send_notification "$deploy_status" "$deploy_detail" + + # 失败时退出非零,让 CI Job 标记为失败 + exit 1 + fi + + # 4. 成功 + log_info "" + log_info "==================================" + log_info " ✅ Production 部署成功!" + log_info "==================================" + + deploy_detail="部署成功,所有健康检查通过 (${GITHUB_REF_NAME:-未知版本})" + send_notification "success" "$deploy_detail" +} + +main "$@"