#!/bin/bash # =========================================== # 金丝雀发布脚本 - 分阶段灰度到全量 # =========================================== # 在 CI Runner 上执行,通过 SSH 控制生产服务器执行灰度发布。 # 流程:5%灰度 → 20%灰度 → 50%灰度 → 100%全量 # 每阶段自动健康检查,失败自动回滚。 # # 用法: # IMAGE_TAG=v0.1.130 ./scripts/ci/canary_release.sh # # 环境变量: # IMAGE_TAG - 新版本镜像标签 (必填) # CANARY_STAGES - 灰度阶段配置,格式: "百分比:等待秒数" 用逗号分隔 # 默认: "5:600,20:900,50:1200" # PROD_API_URL - Production API 公网地址 # PROD_WEB_URL - Production Web 公网地址 # PRODUCTION_SSH_HOST - 生产服务器 SSH 地址 # PRODUCTION_SSH_USER - SSH 用户名 # PRODUCTION_SSH_PORT - SSH 端口 # PRODUCTION_SSH_KEY - SSH 私钥内容 # ACR_USERNAME - 容器镜像仓库用户名 # ACR_PASSWORD - 容器镜像仓库密码 # CI_NOTIFY_WEBHOOK - 通知 Webhook # SKIP_ROLLBACK - 失败时不自动回滚 (调试用) set -euo pipefail SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" # 配置 IMAGE_TAG="${IMAGE_TAG:-}" CANARY_STAGES="${CANARY_STAGES:-5:600,20:900,50:1200}" PROD_API_URL="${PROD_API_URL:-https://api.xiaoxiajianji.com}" PROD_WEB_URL="${PROD_WEB_URL:-https://saas.xiaoxiajianji.com}" PRODUCTION_SSH_HOST="${PRODUCTION_SSH_HOST:-47.98.113.167}" PRODUCTION_SSH_USER="${PRODUCTION_SSH_USER:-root}" PRODUCTION_SSH_PORT="${PRODUCTION_SSH_PORT:-22222}" # gray_deploy.sh 的镜像命名格式是 ${REGISTRY}-component:tag # 需要与 ACR 镜像名 xiaoxia-registry.../xiaoxiakeji/xiaoxia-saas-api:tag 匹配 GRAY_REGISTRY="${GRAY_REGISTRY:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/xiaoxia-saas}" ACR_REGISTRY_HOST="${ACR_REGISTRY_HOST:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com}" ACR_USERNAME="${ACR_USERNAME:-}" ACR_PASSWORD="${ACR_PASSWORD:-}" SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}" if [[ -z "$IMAGE_TAG" ]]; then echo "ERROR: IMAGE_TAG is required" exit 1 fi # 颜色 RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $1"; } log_step() { echo -e "${BLUE}[STEP]${NC} $1"; } # =========================================== # SSH 配置 # =========================================== SSH_KEY_PATH="" setup_ssh() { if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then SSH_KEY_PATH="/root/.ssh/xiaoxia_runtime_builder" elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then SSH_KEY_PATH="$HOME/.ssh/xiaoxia_runtime_builder" elif [ -n "${PRODUCTION_SSH_KEY:-}" ]; then SSH_KEY_PATH="$HOME/.ssh/canary_deploy_key" mkdir -p "$HOME/.ssh" printf '%s\n' "$PRODUCTION_SSH_KEY" > "$SSH_KEY_PATH" chmod 600 "$SSH_KEY_PATH" else log_error "没有可用的 SSH 密钥" return 1 fi ssh-keyscan -p "$PRODUCTION_SSH_PORT" -H "$PRODUCTION_SSH_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true log_info "SSH 已配置: ${PRODUCTION_SSH_USER}@${PRODUCTION_SSH_HOST}:${PRODUCTION_SSH_PORT}" } run_ssh() { local cmd="$1" ssh -p "$PRODUCTION_SSH_PORT" -i "$SSH_KEY_PATH" -o StrictHostKeyChecking=no \ "${PRODUCTION_SSH_USER}@${PRODUCTION_SSH_HOST}" "$cmd" } # =========================================== # 上传脚本 + Docker登录 # =========================================== prepare_server() { log_step "准备生产服务器环境" # 创建临时目录 run_ssh "mkdir -p /tmp/canary-release" # 上传 gray_deploy.sh local gray_script="$REPO_ROOT/scripts/gray_deploy.sh" if [[ -f "$gray_script" ]]; then cat "$gray_script" | run_ssh "cat > /tmp/canary-release/gray_deploy.sh && chmod +x /tmp/canary-release/gray_deploy.sh" log_info " gray_deploy.sh 已上传" else log_error "找不到 gray_deploy.sh: $gray_script" return 1 fi # 上传 rollback_gray.sh local rollback_script="$REPO_ROOT/scripts/rollback_gray.sh" if [[ -f "$rollback_script" ]]; then cat "$rollback_script" | run_ssh "cat > /tmp/canary-release/rollback_gray.sh && chmod +x /tmp/canary-release/rollback_gray.sh" log_info " rollback_gray.sh 已上传" else log_warn "找不到 rollback_gray.sh" fi # 上传 ci_production_deploy.sh local prod_deploy="$REPO_ROOT/scripts/ci_production_deploy.sh" if [[ -f "$prod_deploy" ]]; then cat "$prod_deploy" | run_ssh "cat > /tmp/canary-release/ci_production_deploy.sh && chmod +x /tmp/canary-release/ci_production_deploy.sh" log_info " ci_production_deploy.sh 已上传" else log_error "找不到 ci_production_deploy.sh: $prod_deploy" return 1 fi # Docker 登录到 ACR if [[ -n "$ACR_USERNAME" && -n "$ACR_PASSWORD" ]]; then log_info " Docker 登录到 ACR..." run_ssh "docker login '$ACR_REGISTRY_HOST' -u '$ACR_USERNAME' -p '$ACR_PASSWORD' 2>/dev/null" || \ log_warn " Docker login 失败(可能已有凭证),将尝试直接 pull" fi log_info "✅ 服务器环境准备完成" } # =========================================== # 健康检查(公网访问) # =========================================== health_check() { local stage_name="$1" local timeout="${2:-120}" local interval=5 local elapsed=0 log_step "健康检查 - $stage_name (超时 ${timeout}s)" while [ $elapsed -lt $timeout ]; do local api_ok=false local web_ok=false # 检查 API local api_code=$(curl -s -o /dev/null -w "%{http_code}" \ --connect-timeout 5 --max-time 10 \ "${PROD_API_URL}/health" 2>/dev/null || echo "000") if [[ "$api_code" == "200" ]]; then api_ok=true fi # 检查 Web local web_code=$(curl -s -o /dev/null -w "%{http_code}" \ --connect-timeout 5 --max-time 10 \ "$PROD_WEB_URL" 2>/dev/null || echo "000") if [[ "$web_code" == "200" || "$web_code" == "301" || "$web_code" == "302" ]]; then web_ok=true fi if $api_ok && $web_ok; then log_info "✅ 健康检查通过 (API=$api_code, Web=$web_code)" return 0 fi log_warn " 等待中... API=$api_code, Web=$web_code (${elapsed}s/${timeout}s)" sleep $interval elapsed=$((elapsed + interval)) done log_error "❌ 健康检查超时" return 1 } # =========================================== # 灰度发布 # =========================================== gray_deploy() { local pct="$1" log_step "灰度发布 ${pct}% - $IMAGE_TAG" run_ssh "cd /tmp/canary-release && \ REGISTRY='$GRAY_REGISTRY' \ ./gray_deploy.sh '$IMAGE_TAG' '$pct'" } # =========================================== # 全量部署 # =========================================== full_deploy() { log_step "全量部署 - $IMAGE_TAG" run_ssh "cd /tmp/canary-release && \ IMAGE_TAG='$IMAGE_TAG' \ ACR_USERNAME='$ACR_USERNAME' \ ACR_PASSWORD='$ACR_PASSWORD' \ sh ./ci_production_deploy.sh" } # =========================================== # 灰度回滚 # =========================================== rollback_gray() { log_error "执行灰度回滚..." if [[ "$SKIP_ROLLBACK" == "true" ]]; then log_warn "SKIP_ROLLBACK=true,跳过回滚" return fi if run_ssh "test -f /tmp/canary-release/rollback_gray.sh"; then run_ssh "cd /tmp/canary-release && ./rollback_gray.sh" || \ log_error "回滚脚本执行失败,请手动处理" else # 内联回滚逻辑 log_warn "使用内联回滚逻辑" run_ssh ' NGINX_CONF="/etc/nginx/sites-enabled/00-xiaoxia-saas" LATEST_BAK=$(ls -t "${NGINX_CONF}".bak.gray.* 2>/dev/null | head -1 || true) if [[ -n "$LATEST_BAK" ]]; then cp "$LATEST_BAK" "$NGINX_CONF" else sed -i "s|proxy_pass http://saas_api_backend|proxy_pass http://127.0.0.1:8001|g" "$NGINX_CONF" sed -i "s|proxy_pass http://saas_web_backend/|proxy_pass http://127.0.0.1:3002/|g" "$NGINX_CONF" fi nginx -t && nginx -s reload docker rm -f xiaoxia-api-canary xiaoxia-web-canary 2>/dev/null || true ' || log_error "回滚失败,请手动处理" fi } # =========================================== # 通知 # =========================================== notify_status() { local status="$1" local message="$2" if [ -n "${CI_NOTIFY_WEBHOOK:-}" ]; then NOTIFY_MODE="$status" JOB_NAME="Canary Release - $message" \ python3 "$REPO_ROOT/scripts/ci_notify.py" 2>/dev/null || true fi } # =========================================== # 清理 # =========================================== cleanup() { log_step "清理生产服务器临时文件" run_ssh "rm -rf /tmp/canary-release" 2>/dev/null || true } # =========================================== # 主流程 # =========================================== main() { echo "===========================================" echo " 🐦 金丝雀发布" echo " 版本: $IMAGE_TAG" echo " 阶段: $CANARY_STAGES" echo "===========================================" echo "" setup_ssh prepare_server trap cleanup EXIT # 解析灰度阶段 IFS=',' read -ra STAGES <<< "$CANARY_STAGES" local total_stages=${#STAGES[@]} local current_stage=0 # 逐阶段灰度 for stage in "${STAGES[@]}"; do current_stage=$((current_stage + 1)) local pct=$(echo "$stage" | cut -d: -f1) local wait_time=$(echo "$stage" | cut -d: -f2) echo "" echo "--- 阶段 $current_stage/$total_stages: ${pct}% 灰度 ---" # 执行灰度发布 if ! gray_deploy "$pct"; then log_error "灰度发布 ${pct}% 失败" rollback_gray notify_status "failure" "Stage ${pct}% Deploy Failed" exit 1 fi # 健康检查 if ! health_check "${pct}%灰度"; then log_error "${pct}%灰度健康检查失败" rollback_gray notify_status "failure" "Stage ${pct}% Health Check Failed" exit 1 fi # 观察期 log_info "⏳ 观察期 ${wait_time}s,监控流量稳定性..." local waited=0 local check_interval=60 while [ $waited -lt $wait_time ]; do sleep $check_interval waited=$((waited + check_interval)) # 每隔一段时间做一次快速健康检查 local api_code=$(curl -s -o /dev/null -w "%{http_code}" \ --connect-timeout 5 --max-time 10 \ "${PROD_API_URL}/health" 2>/dev/null || echo "000") if [[ "$api_code" != "200" ]]; then log_error "❌ 观察期内 API 异常 (HTTP $api_code),触发回滚" rollback_gray notify_status "failure" "Stage ${pct}% Watch Period Failed" exit 1 fi log_info " 观察中... ${waited}s/${wait_time}s (API=$api_code)" done log_info "✅ ${pct}%灰度阶段完成,稳定运行 ${wait_time}s" done # 全量部署 echo "" echo "--- 最终阶段: 100% 全量部署 ---" if ! full_deploy; then log_error "全量部署失败" notify_status "failure" "Full Deploy Failed" exit 1 fi # 最终健康检查 if ! health_check "全量部署" "180"; then log_error "全量部署后健康检查失败" notify_status "failure" "Full Deploy Health Check Failed" exit 1 fi # 清理 canary 容器 log_step "清理 Canary 容器" run_ssh "docker rm -f xiaoxia-api-canary xiaoxia-web-canary 2>/dev/null || true" || true echo "" echo "===========================================" echo " ✅ 金丝雀发布完成" echo " 版本: $IMAGE_TAG" echo " 状态: 100%全量运行" echo "===========================================" notify_status "success" "$IMAGE_TAG Fully Deployed" } main "$@"