From e45426fe7d1914007a6839ff39df87ecde10a246 Mon Sep 17 00:00:00 2001
From: xiaoxia <dev@xiaoxiajianji.com>
Date: Tue, 14 Jul 2026 21:33:43 +0800
Subject: [PATCH] =?UTF-8?q?feat(ci):=20=E6=96=B0=E5=A2=9EStaging=E5=81=A5?=
 =?UTF-8?q?=E5=BA=B7=E6=A3=80=E6=9F=A5=E8=84=9A=E6=9C=AC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/ci_staging_healthcheck.sh | 328 ++++++++++++++++++++++++++++++
 1 file changed, 328 insertions(+)
 create mode 100644 scripts/ci_staging_healthcheck.sh

diff --git a/scripts/ci_staging_healthcheck.sh b/scripts/ci_staging_healthcheck.sh
new file mode 100644
index 000000000..9373132e9
--- /dev/null
+++ b/scripts/ci_staging_healthcheck.sh
@@ -0,0 +1,328 @@
+#!/bin/bash
+# ===========================================
+# CI Staging 健康检查 + 自动回滚脚本（Watchtower 模式）
+# ===========================================
+#
+# 在 CI Runner 上执行，通过公网 URL 检查 Staging 部署健康状态。
+# 不健康则自动回滚（把 :staging tag 指回旧版本镜像，Watchtower 检测到后自动回滚）。
+#
+# 用法:
+#   ./ci_staging_healthcheck.sh
+#
+# 环境变量:
+#   STAGING_API_URL       - Staging API 地址 (默认 https://staging-api.xiaoxiajianji.com)
+#   STAGING_WEB_URL       - Staging Web 地址 (默认 https://staging.xiaoxiajianji.com)
+#   HEALTH_CHECK_TIMEOUT  - 健康检查总超时秒数 (默认 120)
+#   WATCHTOWER_WAIT       - 等待 Watchtower 检测更新的秒数 (默认 90)
+#   SKIP_ROLLBACK         - 失败时不自动回滚 (true/false, 默认 false)
+#   SKIP_NOTIFY           - 跳过通知 (true/false, 默认 false)
+#   CI_NOTIFY_WEBHOOK     - 通知 Webhook URL
+#
+#   REGISTRY_TOKEN        - Gitea Registry Token（回滚时需要）
+#   REGISTRY_HOST         - Registry 地址 (默认 git.xiaoxiajianji.com)
+#   REGISTRY_REPO         - 仓库路径 (默认 xiaoxia/xiaoxia-saas)
+#
+#   GITHUB_SHA            - 当前 commit SHA
+#   GITHUB_REF_NAME       - 分支名
+#   GITHUB_RUN_ID         - CI Run ID
+#   GITHUB_REPOSITORY     - 仓库名
+#   GITHUB_ACTOR          - 提交者
+
+set -eu
+
+SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
+
+# 配置
+STAGING_API_URL="${STAGING_API_URL:-https://staging-api.xiaoxiajianji.com}"
+STAGING_WEB_URL="${STAGING_WEB_URL:-https://staging.xiaoxiajianji.com}"
+HEALTH_CHECK_TIMEOUT="${HEALTH_CHECK_TIMEOUT:-120}"
+WATCHTOWER_WAIT="${WATCHTOWER_WAIT:-90}"
+SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}"
+SKIP_NOTIFY="${SKIP_NOTIFY:-false}"
+
+REGISTRY_HOST="${REGISTRY_HOST:-git.xiaoxiajianji.com}"
+REGISTRY_REPO="${REGISTRY_REPO:-xiaoxia/xiaoxia-saas}"
+
+# 颜色
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+log_info()  { echo -e "${GREEN}[INFO]${NC} $1"; }
+log_warn()  { echo -e "${YELLOW}[WARN]${NC} $1"; }
+log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
+log_step()  { echo -e "${BLUE}[STEP]${NC} $1"; }
+
+# ===========================================
+# 1. 记录部署前 :staging 镜像 digest（用于回滚）
+# ===========================================
+save_rollback_target() {
+    log_step "记录当前 :staging 镜像 digest（回滚目标）..."
+
+    # 登录 Registry
+    if [ -n "${REGISTRY_TOKEN:-}" ]; then
+        printf '%s' "${REGISTRY_TOKEN}" | docker login "${REGISTRY_HOST}" -u xiaoxia --password-stdin 2>/dev/null || true
+    fi
+
+    ROLLBACK_IMAGES=""
+
+    for svc in api worker web; do
+        image="${REGISTRY_HOST}/${REGISTRY_REPO}/xiaoxia-saas-${svc}:staging"
+        # pull 当前 :staging 镜像获取 digest
+        if docker pull "$image" 2>/dev/null; then
+            digest=$(docker inspect -f '{{index .RepoDigests 0}}' "$image" 2>/dev/null || echo "")
+            if [ -n "$digest" ]; then
+                ROLLBACK_IMAGES="${ROLLBACK_IMAGES}${svc}:${digest} "
+                log_info "  $svc: $digest"
+            else
+                log_warn "  $svc: 无法获取 digest，该服务将不参与回滚"
+            fi
+        else
+            log_warn "  $svc: 拉取 :staging 镜像失败，该服务将不参与回滚"
+        fi
+    done
+
+    if [ -z "$ROLLBACK_IMAGES" ]; then
+        log_warn "没有获取到任何可回滚镜像，回滚功能将不可用"
+    fi
+
+    export ROLLBACK_IMAGES
+}
+
+# ===========================================
+# 2. 等待 Watchtower 更新
+# ===========================================
+wait_for_watchtower() {
+    log_step "等待 Watchtower 检测新镜像并更新（${WATCHTOWER_WAIT}s）..."
+    sleep "$WATCHTOWER_WAIT"
+    log_info "等待结束，开始健康检查"
+}
+
+# ===========================================
+# 3. 健康检查
+# ===========================================
+health_check() {
+    local timeout="$HEALTH_CHECK_TIMEOUT"
+    local start_time
+    start_time=$(date +%s)
+
+    log_step "健康检查（超时 ${timeout}s）..."
+    log_info "  API: ${STAGING_API_URL}/health"
+    log_info "  Web: ${STAGING_WEB_URL}/"
+
+    local api_ok=false
+    local web_ok=false
+    local api_docs_ok=false
+    local login_api_ok=false
+
+    while [ $(( $(date +%s) - start_time )) -lt "$timeout" ]; do
+        # 检查 API health
+        if [ "$api_ok" = false ] && curl -sf --max-time 10 "${STAGING_API_URL}/health" >/dev/null 2>&1; then
+            log_info "✅ API 健康检查通过"
+            api_ok=true
+        fi
+
+        # 检查 Web 首页
+        if [ "$web_ok" = false ] && curl -sf --max-time 10 "$STAGING_WEB_URL/" >/dev/null 2>&1; then
+            log_info "✅ Web 前端检查通过"
+            web_ok=true
+        fi
+
+        # 检查 API docs（服务完全启动的标志）
+        if [ "$api_docs_ok" = false ]; then
+            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "${STAGING_API_URL}/docs" 2>/dev/null || echo "000")
+            if [ "$HTTP_CODE" = "200" ]; then
+                log_info "✅ API Docs 检查通过"
+                api_docs_ok=true
+            fi
+        fi
+
+        # 检查登录 API（业务逻辑正常的标志）
+        if [ "$login_api_ok" = false ]; then
+            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 -X POST \
+                "${STAGING_API_URL}/api/v1/auth/login" \
+                -H "Content-Type: application/json" \
+                -d '{"email":"smoke@test.com","password":"wrong"}' 2>/dev/null || echo "000")
+            if [ "$HTTP_CODE" = "401" ] || [ "$HTTP_CODE" = "422" ]; then
+                log_info "✅ 登录 API 检查通过（HTTP $HTTP_CODE，符合预期）"
+                login_api_ok=true
+            fi
+        fi
+
+        # 都通过了就退出
+        if [ "$api_ok" = true ] && [ "$web_ok" = true ] && [ "$api_docs_ok" = true ] && [ "$login_api_ok" = true ]; then
+            log_info "🎉 所有健康检查通过！"
+            return 0
+        fi
+
+        sleep 5
+    done
+
+    # 超时了
+    log_error "❌ 健康检查超时 (${timeout}s)"
+    [ "$api_ok" = false ]       && log_error "  - API health 未通过"
+    [ "$web_ok" = false ]       && log_error "  - Web 前端未通过"
+    [ "$api_docs_ok" = false ]  && log_error "  - API Docs 未通过"
+    [ "$login_api_ok" = false ] && log_error "  - 登录 API 未通过"
+
+    return 1
+}
+
+# ===========================================
+# 4. 执行回滚（把 :staging tag 改回旧版本）
+# ===========================================
+do_rollback() {
+    log_step "执行回滚：将 :staging tag 指回旧版本镜像..."
+
+    if [ -z "${ROLLBACK_IMAGES:-}" ]; then
+        log_error "没有可回滚的镜像记录，无法自动回滚"
+        return 1
+    fi
+
+    local rollback_ok=true
+
+    for entry in $ROLLBACK_IMAGES; do
+        svc="${entry%%:*}"
+        digest="${entry#*:}"
+        # digest 是 host/repo/image@sha256:xxx 格式，entry 里只取了第一部分，需要重新组合
+        # 实际上 ROLLBACK_IMAGES 存的是 svc:repo/image@sha256:xxx
+        # 让我们重新从完整digest中提取
+        image_digest="${entry#*:}:${entry#*:*:}"
+        # 不对，格式是 svc:REGISTRY_HOST/REGISTRY_REPO/xiaoxia-saas-svc@sha256:xxx
+        # 让我重新组织一下
+        :
+    done
+
+    # 上面的解析有问题，重新来
+    # ROLLBACK_IMAGES 格式: "api:digest1 worker:digest2 web:digest3"
+    # 但digest本身带冒号(@sha256:xxx)，所以分隔有问题
+    # 改用数组方式
+    log_info "开始回滚各服务..."
+
+    local IFS=' '
+    for pair in $ROLLBACK_IMAGES; do
+        svc="${pair%%:*}"
+        # 剩余部分是完整的 digest 字符串（含 @sha256:xxx）
+        digest="${pair#*:}"
+
+        image_name="${REGISTRY_HOST}/${REGISTRY_REPO}/xiaoxia-saas-${svc}"
+
+        log_info "  回滚 $svc 到: $digest"
+
+        # 通过 digest 拉取旧镜像
+        if docker pull "$digest" 2>/dev/null; then
+            # 重新打 :staging tag
+            docker tag "$digest" "${image_name}:staging"
+            docker push "${image_name}:staging"
+            log_info "  ✅ $svc 已回滚"
+        else
+            log_error "  ❌ $svc 回滚失败：无法拉取旧镜像"
+            rollback_ok=false
+        fi
+    done
+
+    if [ "$rollback_ok" = true ]; then
+        log_info "所有服务回滚命令执行完成，Watchtower 将在 ~60s 内检测到并更新"
+        return 0
+    else
+        log_error "部分服务回滚失败"
+        return 1
+    fi
+}
+
+# ===========================================
+# 5. 发送通知
+# ===========================================
+send_notification() {
+    local status="$1"   # success / failure / rollback
+    local detail="$2"
+
+    if [ "${SKIP_NOTIFY:-false}" = "true" ]; then
+        log_info "跳过通知（SKIP_NOTIFY=true）"
+        return 0
+    fi
+
+    local webhook="${CI_NOTIFY_WEBHOOK:-}"
+    if [ -z "$webhook" ]; then
+        log_warn "未配置 CI_NOTIFY_WEBHOOK，跳过通知"
+        return 0
+    fi
+
+    # 调用通知脚本
+    if [ -f "$SCRIPT_DIR/deploy_notify.py" ]; then
+        python3 "$SCRIPT_DIR/deploy_notify.py" \
+            --status "$status" \
+            --detail "$detail" \
+            --webhook "$webhook" \
+            2>/dev/null || log_warn "通知发送失败（非致命）"
+    else
+        log_warn "找不到 deploy_notify.py，跳过通知"
+    fi
+}
+
+# ===========================================
+# 主流程
+# ===========================================
+main() {
+    echo ""
+    echo "=========================================="
+    echo "  CI Staging 健康检查 + 自动回滚"
+    echo "=========================================="
+    echo ""
+
+    local deploy_status="success"
+    local deploy_detail=""
+
+    # 1. 记录部署前状态（回滚目标）
+    save_rollback_target
+
+    # 2. 等待 Watchtower 更新
+    wait_for_watchtower
+
+    # 3. 健康检查
+    if ! health_check; then
+        log_error "健康检查失败"
+        deploy_status="failure"
+        deploy_detail="健康检查超时，部署后服务未正常启动"
+
+        # 自动回滚
+        if [ "${SKIP_ROLLBACK:-false}" != "true" ]; then
+            log_warn "开始自动回滚..."
+            if do_rollback; then
+                deploy_status="rollback"
+                deploy_detail="健康检查失败，已自动回滚到上一版本"
+
+                # 回滚后再检查一下回滚后的状态
+                log_info "等待 Watchtower 应用回滚（90s）..."
+                sleep 90
+                if health_check; then
+                    log_info "✅ 回滚后服务已恢复"
+                    deploy_detail="${deploy_detail}，回滚后服务已恢复"
+                else
+                    log_error "⚠️ 回滚后健康检查仍未通过，请手动排查"
+                    deploy_detail="${deploy_detail}，但回滚后仍未恢复，请紧急排查"
+                fi
+            else
+                deploy_detail="健康检查失败且回滚失败，请手动排查"
+            fi
+        fi
+
+        send_notification "$deploy_status" "$deploy_detail"
+
+        # 失败时退出非零，让 CI Job 标记为失败
+        exit 1
+    fi
+
+    # 4. 成功
+    log_info ""
+    log_info "=================================="
+    log_info "  ✅ Staging 部署成功！"
+    log_info "=================================="
+
+    deploy_detail="部署成功，所有健康检查通过"
+    send_notification "success" "$deploy_detail"
+}
+
+main "$@"