xiaoxia-saas/scripts/ci/canary_release.sh

#!/bin/bash
# ===========================================
# 金丝雀发布脚本 - 分阶段灰度到全量
# ===========================================
# 在 CI Runner 上执行，通过 SSH 控制生产服务器执行灰度发布。
# 流程：5%灰度 → 20%灰度 → 50%灰度 → 100%全量
# 每阶段自动健康检查，失败自动回滚。
#
# 用法:
#   IMAGE_TAG=v0.1.130 ./scripts/ci/canary_release.sh
#
# 环境变量:
#   IMAGE_TAG              - 新版本镜像标签 (必填)
#   CANARY_STAGES          - 灰度阶段配置，格式: "百分比:等待秒数" 用逗号分隔
#                            默认: "5:600,20:900,50:1200"
#   PROD_API_URL           - Production API 公网地址
#   PROD_WEB_URL           - Production Web 公网地址
#   PRODUCTION_SSH_HOST    - 生产服务器 SSH 地址
#   PRODUCTION_SSH_USER    - SSH 用户名
#   PRODUCTION_SSH_PORT    - SSH 端口
#   PRODUCTION_SSH_KEY     - SSH 私钥内容
#   ACR_USERNAME           - 容器镜像仓库用户名
#   ACR_PASSWORD           - 容器镜像仓库密码
#   CI_NOTIFY_WEBHOOK      - 通知 Webhook
#   SKIP_ROLLBACK          - 失败时不自动回滚 (调试用)

set -euo pipefail

SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"

# 配置
IMAGE_TAG="${IMAGE_TAG:-}"
CANARY_STAGES="${CANARY_STAGES:-5:600,20:900,50:1200}"
PROD_API_URL="${PROD_API_URL:-https://api.xiaoxiajianji.com}"
PROD_WEB_URL="${PROD_WEB_URL:-https://saas.xiaoxiajianji.com}"
PRODUCTION_SSH_HOST="${PRODUCTION_SSH_HOST:-47.98.113.167}"
PRODUCTION_SSH_USER="${PRODUCTION_SSH_USER:-root}"
PRODUCTION_SSH_PORT="${PRODUCTION_SSH_PORT:-22222}"
# gray_deploy.sh 的镜像命名格式是 ${REGISTRY}-component:tag
# 需要与 ACR 镜像名 xiaoxia-registry.../xiaoxiakeji/xiaoxia-saas-api:tag 匹配
GRAY_REGISTRY="${GRAY_REGISTRY:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/xiaoxia-saas}"
ACR_REGISTRY_HOST="${ACR_REGISTRY_HOST:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com}"
ACR_USERNAME="${ACR_USERNAME:-}"
ACR_PASSWORD="${ACR_PASSWORD:-}"
SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}"

if [[ -z "$IMAGE_TAG" ]]; then
    echo "ERROR: IMAGE_TAG is required"
    exit 1
fi

# 颜色
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info()  { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn()  { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_step()  { echo -e "${BLUE}[STEP]${NC} $1"; }

# ===========================================
# SSH 配置
# ===========================================
SSH_KEY_PATH=""

setup_ssh() {
    if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
        SSH_KEY_PATH="/root/.ssh/xiaoxia_runtime_builder"
    elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then
        SSH_KEY_PATH="$HOME/.ssh/xiaoxia_runtime_builder"
    elif [ -n "${PRODUCTION_SSH_KEY:-}" ]; then
        SSH_KEY_PATH="$HOME/.ssh/canary_deploy_key"
        mkdir -p "$HOME/.ssh"
        printf '%s\n' "$PRODUCTION_SSH_KEY" > "$SSH_KEY_PATH"
        chmod 600 "$SSH_KEY_PATH"
    else
        log_error "没有可用的 SSH 密钥"
        return 1
    fi

    ssh-keyscan -p "$PRODUCTION_SSH_PORT" -H "$PRODUCTION_SSH_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true
    log_info "SSH 已配置: ${PRODUCTION_SSH_USER}@${PRODUCTION_SSH_HOST}:${PRODUCTION_SSH_PORT}"
}

run_ssh() {
    local cmd="$1"
    ssh -p "$PRODUCTION_SSH_PORT" -i "$SSH_KEY_PATH" -o StrictHostKeyChecking=no \
        "${PRODUCTION_SSH_USER}@${PRODUCTION_SSH_HOST}" "$cmd"
}

# ===========================================
# 上传脚本 + Docker登录
# ===========================================
prepare_server() {
    log_step "准备生产服务器环境"
    
    # 创建临时目录
    run_ssh "mkdir -p /tmp/canary-release"
    
    # 上传 gray_deploy.sh
    local gray_script="$REPO_ROOT/scripts/gray_deploy.sh"
    if [[ -f "$gray_script" ]]; then
        cat "$gray_script" | run_ssh "cat > /tmp/canary-release/gray_deploy.sh && chmod +x /tmp/canary-release/gray_deploy.sh"
        log_info "  gray_deploy.sh 已上传"
    else
        log_error "找不到 gray_deploy.sh: $gray_script"
        return 1
    fi
    
    # 上传 rollback_gray.sh
    local rollback_script="$REPO_ROOT/scripts/rollback_gray.sh"
    if [[ -f "$rollback_script" ]]; then
        cat "$rollback_script" | run_ssh "cat > /tmp/canary-release/rollback_gray.sh && chmod +x /tmp/canary-release/rollback_gray.sh"
        log_info "  rollback_gray.sh 已上传"
    else
        log_warn "找不到 rollback_gray.sh"
    fi
    
    # 上传 ci_production_deploy.sh
    local prod_deploy="$REPO_ROOT/scripts/ci_production_deploy.sh"
    if [[ -f "$prod_deploy" ]]; then
        cat "$prod_deploy" | run_ssh "cat > /tmp/canary-release/ci_production_deploy.sh && chmod +x /tmp/canary-release/ci_production_deploy.sh"
        log_info "  ci_production_deploy.sh 已上传"
    else
        log_error "找不到 ci_production_deploy.sh: $prod_deploy"
        return 1
    fi
    
    # Docker 登录到 ACR
    if [[ -n "$ACR_USERNAME" && -n "$ACR_PASSWORD" ]]; then
        log_info "  Docker 登录到 ACR..."
        run_ssh "docker login '$ACR_REGISTRY_HOST' -u '$ACR_USERNAME' -p '$ACR_PASSWORD' 2>/dev/null" || \
            log_warn "  Docker login 失败（可能已有凭证），将尝试直接 pull"
    fi
    
    log_info "✅ 服务器环境准备完成"
}

# ===========================================
# 健康检查（公网访问）
# ===========================================
health_check() {
    local stage_name="$1"
    local timeout="${2:-120}"
    local interval=5
    local elapsed=0

    log_step "健康检查 - $stage_name (超时 ${timeout}s)"

    while [ $elapsed -lt $timeout ]; do
        local api_ok=false
        local web_ok=false

        # 检查 API
        local api_code=$(curl -s -o /dev/null -w "%{http_code}" \
            --connect-timeout 5 --max-time 10 \
            "${PROD_API_URL}/health" 2>/dev/null || echo "000")
        if [[ "$api_code" == "200" ]]; then
            api_ok=true
        fi

        # 检查 Web
        local web_code=$(curl -s -o /dev/null -w "%{http_code}" \
            --connect-timeout 5 --max-time 10 \
            "$PROD_WEB_URL" 2>/dev/null || echo "000")
        if [[ "$web_code" == "200" || "$web_code" == "301" || "$web_code" == "302" ]]; then
            web_ok=true
        fi

        if $api_ok && $web_ok; then
            log_info "✅ 健康检查通过 (API=$api_code, Web=$web_code)"
            return 0
        fi

        log_warn "  等待中... API=$api_code, Web=$web_code (${elapsed}s/${timeout}s)"
        sleep $interval
        elapsed=$((elapsed + interval))
    done

    log_error "❌ 健康检查超时"
    return 1
}

# ===========================================
# 灰度发布
# ===========================================
gray_deploy() {
    local pct="$1"
    log_step "灰度发布 ${pct}% - $IMAGE_TAG"

    run_ssh "cd /tmp/canary-release && \
        REGISTRY='$GRAY_REGISTRY' \
        ./gray_deploy.sh '$IMAGE_TAG' '$pct'"
}

# ===========================================
# 全量部署
# ===========================================
full_deploy() {
    log_step "全量部署 - $IMAGE_TAG"

    run_ssh "cd /tmp/canary-release && \
        IMAGE_TAG='$IMAGE_TAG' \
        ACR_USERNAME='$ACR_USERNAME' \
        ACR_PASSWORD='$ACR_PASSWORD' \
        sh ./ci_production_deploy.sh"
}

# ===========================================
# 灰度回滚
# ===========================================
rollback_gray() {
    log_error "执行灰度回滚..."
    if [[ "$SKIP_ROLLBACK" == "true" ]]; then
        log_warn "SKIP_ROLLBACK=true，跳过回滚"
        return
    fi

    if run_ssh "test -f /tmp/canary-release/rollback_gray.sh"; then
        run_ssh "cd /tmp/canary-release && ./rollback_gray.sh" || \
            log_error "回滚脚本执行失败，请手动处理"
    else
        # 内联回滚逻辑
        log_warn "使用内联回滚逻辑"
        run_ssh '
            NGINX_CONF="/etc/nginx/sites-enabled/00-xiaoxia-saas"
            LATEST_BAK=$(ls -t "${NGINX_CONF}".bak.gray.* 2>/dev/null | head -1 || true)
            if [[ -n "$LATEST_BAK" ]]; then
                cp "$LATEST_BAK" "$NGINX_CONF"
            else
                sed -i "s|proxy_pass http://saas_api_backend|proxy_pass http://127.0.0.1:8001|g" "$NGINX_CONF"
                sed -i "s|proxy_pass http://saas_web_backend/|proxy_pass http://127.0.0.1:3002/|g" "$NGINX_CONF"
            fi
            nginx -t && nginx -s reload
            docker rm -f xiaoxia-api-canary xiaoxia-web-canary 2>/dev/null || true
        ' || log_error "回滚失败，请手动处理"
    fi
}

# ===========================================
# 通知
# ===========================================
notify_status() {
    local status="$1"
    local message="$2"
    if [ -n "${CI_NOTIFY_WEBHOOK:-}" ]; then
        NOTIFY_MODE="$status" JOB_NAME="Canary Release - $message" \
            python3 "$REPO_ROOT/scripts/ci_notify.py" 2>/dev/null || true
    fi
}

# ===========================================
# 清理
# ===========================================
cleanup() {
    log_step "清理生产服务器临时文件"
    run_ssh "rm -rf /tmp/canary-release" 2>/dev/null || true
}

# ===========================================
# 主流程
# ===========================================
main() {
    echo "==========================================="
    echo "  🐦 金丝雀发布"
    echo "  版本: $IMAGE_TAG"
    echo "  阶段: $CANARY_STAGES"
    echo "==========================================="
    echo ""

    setup_ssh
    prepare_server
    trap cleanup EXIT

    # 解析灰度阶段
    IFS=',' read -ra STAGES <<< "$CANARY_STAGES"
    local total_stages=${#STAGES[@]}
    local current_stage=0

    # 逐阶段灰度
    for stage in "${STAGES[@]}"; do
        current_stage=$((current_stage + 1))
        local pct=$(echo "$stage" | cut -d: -f1)
        local wait_time=$(echo "$stage" | cut -d: -f2)

        echo ""
        echo "--- 阶段 $current_stage/$total_stages: ${pct}% 灰度 ---"

        # 执行灰度发布
        if ! gray_deploy "$pct"; then
            log_error "灰度发布 ${pct}% 失败"
            rollback_gray
            notify_status "failure" "Stage ${pct}% Deploy Failed"
            exit 1
        fi

        # 健康检查
        if ! health_check "${pct}%灰度"; then
            log_error "${pct}%灰度健康检查失败"
            rollback_gray
            notify_status "failure" "Stage ${pct}% Health Check Failed"
            exit 1
        fi

        # 观察期
        log_info "⏳ 观察期 ${wait_time}s，监控流量稳定性..."
        local waited=0
        local check_interval=60
        while [ $waited -lt $wait_time ]; do
            sleep $check_interval
            waited=$((waited + check_interval))
            # 每隔一段时间做一次快速健康检查
            local api_code=$(curl -s -o /dev/null -w "%{http_code}" \
                --connect-timeout 5 --max-time 10 \
                "${PROD_API_URL}/health" 2>/dev/null || echo "000")
            if [[ "$api_code" != "200" ]]; then
                log_error "❌ 观察期内 API 异常 (HTTP $api_code)，触发回滚"
                rollback_gray
                notify_status "failure" "Stage ${pct}% Watch Period Failed"
                exit 1
            fi
            log_info "  观察中... ${waited}s/${wait_time}s (API=$api_code)"
        done

        log_info "✅ ${pct}%灰度阶段完成，稳定运行 ${wait_time}s"
    done

    # 全量部署
    echo ""
    echo "--- 最终阶段: 100% 全量部署 ---"

    if ! full_deploy; then
        log_error "全量部署失败"
        notify_status "failure" "Full Deploy Failed"
        exit 1
    fi

    # 最终健康检查
    if ! health_check "全量部署" "180"; then
        log_error "全量部署后健康检查失败"
        notify_status "failure" "Full Deploy Health Check Failed"
        exit 1
    fi

    # 清理 canary 容器
    log_step "清理 Canary 容器"
    run_ssh "docker rm -f xiaoxia-api-canary xiaoxia-web-canary 2>/dev/null || true" || true

    echo ""
    echo "==========================================="
    echo "  ✅ 金丝雀发布完成"
    echo "  版本: $IMAGE_TAG"
    echo "  状态: 100%全量运行"
    echo "==========================================="

    notify_status "success" "$IMAGE_TAG Fully Deployed"
}

main "$@"