feat(ci): 新增Staging健康检查脚本
This commit is contained in:
XiaoXia Bot
@@ -0,0 +1,328 @@
|
||||
#!/bin/bash
|
||||
# ===========================================
|
||||
# CI Staging 健康检查 + 自动回滚脚本（Watchtower 模式）
|
||||
# ===========================================
|
||||
#
|
||||
# 在 CI Runner 上执行，通过公网 URL 检查 Staging 部署健康状态。
|
||||
# 不健康则自动回滚（把 :staging tag 指回旧版本镜像，Watchtower 检测到后自动回滚）。
|
||||
#
|
||||
# 用法:
|
||||
# ./ci_staging_healthcheck.sh
|
||||
#
|
||||
# 环境变量:
|
||||
# STAGING_API_URL - Staging API 地址 (默认 https://staging-api.xiaoxiajianji.com)
|
||||
# STAGING_WEB_URL - Staging Web 地址 (默认 https://staging.xiaoxiajianji.com)
|
||||
# HEALTH_CHECK_TIMEOUT - 健康检查总超时秒数 (默认 120)
|
||||
# WATCHTOWER_WAIT - 等待 Watchtower 检测更新的秒数 (默认 90)
|
||||
# SKIP_ROLLBACK - 失败时不自动回滚 (true/false, 默认 false)
|
||||
# SKIP_NOTIFY - 跳过通知 (true/false, 默认 false)
|
||||
# CI_NOTIFY_WEBHOOK - 通知 Webhook URL
|
||||
#
|
||||
# REGISTRY_TOKEN - Gitea Registry Token（回滚时需要）
|
||||
# REGISTRY_HOST - Registry 地址 (默认 git.xiaoxiajianji.com)
|
||||
# REGISTRY_REPO - 仓库路径 (默认 xiaoxia/xiaoxia-saas)
|
||||
#
|
||||
# GITHUB_SHA - 当前 commit SHA
|
||||
# GITHUB_REF_NAME - 分支名
|
||||
# GITHUB_RUN_ID - CI Run ID
|
||||
# GITHUB_REPOSITORY - 仓库名
|
||||
# GITHUB_ACTOR - 提交者
|
||||
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
||||
|
||||
# 配置
|
||||
STAGING_API_URL="${STAGING_API_URL:-https://staging-api.xiaoxiajianji.com}"
|
||||
STAGING_WEB_URL="${STAGING_WEB_URL:-https://staging.xiaoxiajianji.com}"
|
||||
HEALTH_CHECK_TIMEOUT="${HEALTH_CHECK_TIMEOUT:-120}"
|
||||
WATCHTOWER_WAIT="${WATCHTOWER_WAIT:-90}"
|
||||
SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}"
|
||||
SKIP_NOTIFY="${SKIP_NOTIFY:-false}"
|
||||
|
||||
REGISTRY_HOST="${REGISTRY_HOST:-git.xiaoxiajianji.com}"
|
||||
REGISTRY_REPO="${REGISTRY_REPO:-xiaoxia/xiaoxia-saas}"
|
||||
|
||||
# 颜色
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
||||
log_step() { echo -e "${BLUE}[STEP]${NC} $1"; }
|
||||
|
||||
# ===========================================
|
||||
# 1. 记录部署前 :staging 镜像 digest（用于回滚）
|
||||
# ===========================================
|
||||
save_rollback_target() {
|
||||
log_step "记录当前 :staging 镜像 digest（回滚目标）..."
|
||||
|
||||
# 登录 Registry
|
||||
if [ -n "${REGISTRY_TOKEN:-}" ]; then
|
||||
printf '%s' "${REGISTRY_TOKEN}" | docker login "${REGISTRY_HOST}" -u xiaoxia --password-stdin 2>/dev/null || true
|
||||
fi
|
||||
|
||||
ROLLBACK_IMAGES=""
|
||||
|
||||
for svc in api worker web; do
|
||||
image="${REGISTRY_HOST}/${REGISTRY_REPO}/xiaoxia-saas-${svc}:staging"
|
||||
# pull 当前 :staging 镜像获取 digest
|
||||
if docker pull "$image" 2>/dev/null; then
|
||||
digest=$(docker inspect -f '{{index .RepoDigests 0}}' "$image" 2>/dev/null || echo "")
|
||||
if [ -n "$digest" ]; then
|
||||
ROLLBACK_IMAGES="${ROLLBACK_IMAGES}${svc}:${digest} "
|
||||
log_info " $svc: $digest"
|
||||
else
|
||||
log_warn " $svc: 无法获取 digest，该服务将不参与回滚"
|
||||
fi
|
||||
else
|
||||
log_warn " $svc: 拉取 :staging 镜像失败，该服务将不参与回滚"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$ROLLBACK_IMAGES" ]; then
|
||||
log_warn "没有获取到任何可回滚镜像，回滚功能将不可用"
|
||||
fi
|
||||
|
||||
export ROLLBACK_IMAGES
|
||||
}
|
||||
|
||||
# ===========================================
|
||||
# 2. 等待 Watchtower 更新
|
||||
# ===========================================
|
||||
wait_for_watchtower() {
|
||||
log_step "等待 Watchtower 检测新镜像并更新（${WATCHTOWER_WAIT}s）..."
|
||||
sleep "$WATCHTOWER_WAIT"
|
||||
log_info "等待结束，开始健康检查"
|
||||
}
|
||||
|
||||
# ===========================================
|
||||
# 3. 健康检查
|
||||
# ===========================================
|
||||
health_check() {
|
||||
local timeout="$HEALTH_CHECK_TIMEOUT"
|
||||
local start_time
|
||||
start_time=$(date +%s)
|
||||
|
||||
log_step "健康检查（超时 ${timeout}s）..."
|
||||
log_info " API: ${STAGING_API_URL}/health"
|
||||
log_info " Web: ${STAGING_WEB_URL}/"
|
||||
|
||||
local api_ok=false
|
||||
local web_ok=false
|
||||
local api_docs_ok=false
|
||||
local login_api_ok=false
|
||||
|
||||
while [ $(( $(date +%s) - start_time )) -lt "$timeout" ]; do
|
||||
# 检查 API health
|
||||
if [ "$api_ok" = false ] && curl -sf --max-time 10 "${STAGING_API_URL}/health" >/dev/null 2>&1; then
|
||||
log_info "✅ API 健康检查通过"
|
||||
api_ok=true
|
||||
fi
|
||||
|
||||
# 检查 Web 首页
|
||||
if [ "$web_ok" = false ] && curl -sf --max-time 10 "$STAGING_WEB_URL/" >/dev/null 2>&1; then
|
||||
log_info "✅ Web 前端检查通过"
|
||||
web_ok=true
|
||||
fi
|
||||
|
||||
# 检查 API docs（服务完全启动的标志）
|
||||
if [ "$api_docs_ok" = false ]; then
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "${STAGING_API_URL}/docs" 2>/dev/null || echo "000")
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
log_info "✅ API Docs 检查通过"
|
||||
api_docs_ok=true
|
||||
fi
|
||||
fi
|
||||
|
||||
# 检查登录 API（业务逻辑正常的标志）
|
||||
if [ "$login_api_ok" = false ]; then
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 -X POST \
|
||||
"${STAGING_API_URL}/api/v1/auth/login" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"email":"smoke@test.com","password":"wrong"}' 2>/dev/null || echo "000")
|
||||
if [ "$HTTP_CODE" = "401" ] || [ "$HTTP_CODE" = "422" ]; then
|
||||
log_info "✅ 登录 API 检查通过（HTTP $HTTP_CODE，符合预期）"
|
||||
login_api_ok=true
|
||||
fi
|
||||
fi
|
||||
|
||||
# 都通过了就退出
|
||||
if [ "$api_ok" = true ] && [ "$web_ok" = true ] && [ "$api_docs_ok" = true ] && [ "$login_api_ok" = true ]; then
|
||||
log_info "🎉 所有健康检查通过！"
|
||||
return 0
|
||||
fi
|
||||
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# 超时了
|
||||
log_error "❌ 健康检查超时 (${timeout}s)"
|
||||
[ "$api_ok" = false ] && log_error " - API health 未通过"
|
||||
[ "$web_ok" = false ] && log_error " - Web 前端未通过"
|
||||
[ "$api_docs_ok" = false ] && log_error " - API Docs 未通过"
|
||||
[ "$login_api_ok" = false ] && log_error " - 登录 API 未通过"
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# ===========================================
|
||||
# 4. 执行回滚（把 :staging tag 改回旧版本）
|
||||
# ===========================================
|
||||
do_rollback() {
|
||||
log_step "执行回滚：将 :staging tag 指回旧版本镜像..."
|
||||
|
||||
if [ -z "${ROLLBACK_IMAGES:-}" ]; then
|
||||
log_error "没有可回滚的镜像记录，无法自动回滚"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local rollback_ok=true
|
||||
|
||||
for entry in $ROLLBACK_IMAGES; do
|
||||
svc="${entry%%:*}"
|
||||
digest="${entry#*:}"
|
||||
# digest 是 host/repo/image@sha256:xxx 格式，entry 里只取了第一部分，需要重新组合
|
||||
# 实际上 ROLLBACK_IMAGES 存的是 svc:repo/image@sha256:xxx
|
||||
# 让我们重新从完整digest中提取
|
||||
image_digest="${entry#*:}:${entry#*:*:}"
|
||||
# 不对，格式是 svc:REGISTRY_HOST/REGISTRY_REPO/xiaoxia-saas-svc@sha256:xxx
|
||||
# 让我重新组织一下
|
||||
:
|
||||
done
|
||||
|
||||
# 上面的解析有问题，重新来
|
||||
# ROLLBACK_IMAGES 格式: "api:digest1 worker:digest2 web:digest3"
|
||||
# 但digest本身带冒号(@sha256:xxx)，所以分隔有问题
|
||||
# 改用数组方式
|
||||
log_info "开始回滚各服务..."
|
||||
|
||||
local IFS=' '
|
||||
for pair in $ROLLBACK_IMAGES; do
|
||||
svc="${pair%%:*}"
|
||||
# 剩余部分是完整的 digest 字符串（含 @sha256:xxx）
|
||||
digest="${pair#*:}"
|
||||
|
||||
image_name="${REGISTRY_HOST}/${REGISTRY_REPO}/xiaoxia-saas-${svc}"
|
||||
|
||||
log_info " 回滚 $svc 到: $digest"
|
||||
|
||||
# 通过 digest 拉取旧镜像
|
||||
if docker pull "$digest" 2>/dev/null; then
|
||||
# 重新打 :staging tag
|
||||
docker tag "$digest" "${image_name}:staging"
|
||||
docker push "${image_name}:staging"
|
||||
log_info " ✅ $svc 已回滚"
|
||||
else
|
||||
log_error " ❌ $svc 回滚失败：无法拉取旧镜像"
|
||||
rollback_ok=false
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$rollback_ok" = true ]; then
|
||||
log_info "所有服务回滚命令执行完成，Watchtower 将在 ~60s 内检测到并更新"
|
||||
return 0
|
||||
else
|
||||
log_error "部分服务回滚失败"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ===========================================
|
||||
# 5. 发送通知
|
||||
# ===========================================
|
||||
send_notification() {
|
||||
local status="$1" # success / failure / rollback
|
||||
local detail="$2"
|
||||
|
||||
if [ "${SKIP_NOTIFY:-false}" = "true" ]; then
|
||||
log_info "跳过通知（SKIP_NOTIFY=true）"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local webhook="${CI_NOTIFY_WEBHOOK:-}"
|
||||
if [ -z "$webhook" ]; then
|
||||
log_warn "未配置 CI_NOTIFY_WEBHOOK，跳过通知"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 调用通知脚本
|
||||
if [ -f "$SCRIPT_DIR/deploy_notify.py" ]; then
|
||||
python3 "$SCRIPT_DIR/deploy_notify.py" \
|
||||
--status "$status" \
|
||||
--detail "$detail" \
|
||||
--webhook "$webhook" \
|
||||
2>/dev/null || log_warn "通知发送失败（非致命）"
|
||||
else
|
||||
log_warn "找不到 deploy_notify.py，跳过通知"
|
||||
fi
|
||||
}
|
||||
|
||||
# ===========================================
|
||||
# 主流程
|
||||
# ===========================================
|
||||
main() {
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo " CI Staging 健康检查 + 自动回滚"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
local deploy_status="success"
|
||||
local deploy_detail=""
|
||||
|
||||
# 1. 记录部署前状态（回滚目标）
|
||||
save_rollback_target
|
||||
|
||||
# 2. 等待 Watchtower 更新
|
||||
wait_for_watchtower
|
||||
|
||||
# 3. 健康检查
|
||||
if ! health_check; then
|
||||
log_error "健康检查失败"
|
||||
deploy_status="failure"
|
||||
deploy_detail="健康检查超时，部署后服务未正常启动"
|
||||
|
||||
# 自动回滚
|
||||
if [ "${SKIP_ROLLBACK:-false}" != "true" ]; then
|
||||
log_warn "开始自动回滚..."
|
||||
if do_rollback; then
|
||||
deploy_status="rollback"
|
||||
deploy_detail="健康检查失败，已自动回滚到上一版本"
|
||||
|
||||
# 回滚后再检查一下回滚后的状态
|
||||
log_info "等待 Watchtower 应用回滚（90s）..."
|
||||
sleep 90
|
||||
if health_check; then
|
||||
log_info "✅ 回滚后服务已恢复"
|
||||
deploy_detail="${deploy_detail}，回滚后服务已恢复"
|
||||
else
|
||||
log_error "⚠️ 回滚后健康检查仍未通过，请手动排查"
|
||||
deploy_detail="${deploy_detail}，但回滚后仍未恢复，请紧急排查"
|
||||
fi
|
||||
else
|
||||
deploy_detail="健康检查失败且回滚失败，请手动排查"
|
||||
fi
|
||||
fi
|
||||
|
||||
send_notification "$deploy_status" "$deploy_detail"
|
||||
|
||||
# 失败时退出非零，让 CI Job 标记为失败
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 4. 成功
|
||||
log_info ""
|
||||
log_info "=================================="
|
||||
log_info " ✅ Staging 部署成功！"
|
||||
log_info "=================================="
|
||||
|
||||
deploy_detail="部署成功，所有健康检查通过"
|
||||
send_notification "success" "$deploy_detail"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user