feat(ci): 金丝雀发布流水线 #450

.gitea/workflows/ci-pipeline.yml

@@ -1230,7 +1230,7 @@ jobs:
     runs-on: runtime-builder
     timeout-minutes: ${{ matrix.timeout }}
     needs:
-    if: startsWith(github.ref, 'refs/tags/v')
+    if: startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'push' && github.ref_name == 'main')
     strategy:
       fail-fast: false
       matrix:
@@ -1307,10 +1307,16 @@ jobs:
       run: |
         set -eu
         REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
-        IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:${GITHUB_REF_NAME}"
+        # 根据ref类型设置镜像标签：tag用版本号，分支用分支名+sha
+        if [[ "$GITHUB_REF" == refs/tags/* ]]; then
+          TAG_NAME="${GITHUB_REF_NAME}"
+        else
+          TAG_NAME="${GITHUB_REF_NAME}-${GITHUB_SHA::8}"
+        fi
+        IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:${TAG_NAME}"
         CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:main"
 
-        EXTRA_BUILD_ARGS="APP_VERSION=\"${GITHUB_REF_NAME}\""
+        EXTRA_BUILD_ARGS="APP_VERSION=\"${TAG_NAME}\""
         if [ "${{ matrix.service }}" = "web" ]; then
           EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-production.conf"
         fi
@@ -1608,4 +1614,89 @@ jobs:
         [ ${{ job.status }} = "success" ] || STATUS="error"
         START_TIME=""
         [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
-        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
+        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
+
+  canary-release:
+    name: Canary Release to Production
+    runs-on: runtime-builder
+    timeout-minutes: 120
+    concurrency:
+      group: canary-release-production
+      cancel-in-progress: false
+    if: github.event_name == 'push' && github.ref_name == 'main'
+    needs:
+    - build-production
+    - staging-api-tests
+    steps:
+    - name: Checkout code
+      shell: sh
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
+    - name: Record job start time
+      shell: sh
+      run: bash scripts/ci/step_timer_start.sh
+    - name: Notify canary release start
+      continue-on-error: true
+      shell: sh
+      env:
+        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
+      run: |
+        set +e
+        NOTIFY_MODE=start JOB_NAME="Canary Release" python3 scripts/ci_notify.py
+    - name: Install SSH client
+      shell: sh
+      run: |
+        set -eu
+        apt-get update -qq && apt-get install -y -qq openssh-client curl >/dev/null 2>&1
+        echo "openssh-client installed"
+    - name: Run canary release
+      shell: bash
+      env:
+        PRODUCTION_SSH_HOST: ${{ secrets.PRODUCTION_SSH_HOST }}
+        PRODUCTION_SSH_USER: ${{ secrets.PRODUCTION_SSH_USER }}
+        PRODUCTION_SSH_PORT: ${{ secrets.PRODUCTION_SSH_PORT }}
+        PRODUCTION_SSH_KEY: ${{ secrets.PRODUCTION_SSH_KEY }}
+        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
+        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
+        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
+      run: |
+        set -eu
+        IMAGE_TAG="main-${GITHUB_SHA::8}"
+        export IMAGE_TAG
+        echo "Canary release version: $IMAGE_TAG"
+        bash scripts/ci/canary_release.sh
+    - name: Job duration summary
+      if: always()
+      shell: sh
+      run: bash scripts/ci/step_timer_end.sh
+    - name: Notify on success
+      continue-on-error: true
+      if: success()
+      shell: sh
+      env:
+        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
+      run: |
+        set +e
+        NOTIFY_MODE=success JOB_NAME="Canary Release" python3 scripts/ci_notify.py
+    - name: Notify on failure
+      continue-on-error: true
+      if: failure()
+      shell: sh
+      env:
+        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
+      run: |
+        set +e
+        NOTIFY_MODE=failure JOB_NAME="Canary Release" python3 scripts/ci_notify.py
+    - name: Report CI trace
+      if: always()
+      shell: sh
+      env:
+        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
+      run: |
+        STATUS="ok"
+        [ ${{ job.status }} = "success" ] || STATUS="error"
+        START_TIME=""
+        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
+        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

scripts/ci/canary_release.sh

@@ -0,0 +1,363 @@
+#!/bin/bash
+# ===========================================
+# 金丝雀发布脚本 - 分阶段灰度到全量
+# ===========================================
+# 在 CI Runner 上执行，通过 SSH 控制生产服务器执行灰度发布。
+# 流程：5%灰度 → 20%灰度 → 50%灰度 → 100%全量
+# 每阶段自动健康检查，失败自动回滚。
+#
+# 用法:
+#   IMAGE_TAG=v0.1.130 ./scripts/ci/canary_release.sh
+#
+# 环境变量:
+#   IMAGE_TAG              - 新版本镜像标签 (必填)
+#   CANARY_STAGES          - 灰度阶段配置，格式: "百分比:等待秒数" 用逗号分隔
+#                            默认: "5:600,20:900,50:1200"
+#   PROD_API_URL           - Production API 公网地址
+#   PROD_WEB_URL           - Production Web 公网地址
+#   PRODUCTION_SSH_HOST    - 生产服务器 SSH 地址
+#   PRODUCTION_SSH_USER    - SSH 用户名
+#   PRODUCTION_SSH_PORT    - SSH 端口
+#   PRODUCTION_SSH_KEY     - SSH 私钥内容
+#   ACR_USERNAME           - 容器镜像仓库用户名
+#   ACR_PASSWORD           - 容器镜像仓库密码
+#   CI_NOTIFY_WEBHOOK      - 通知 Webhook
+#   SKIP_ROLLBACK          - 失败时不自动回滚 (调试用)
+
+set -euo pipefail
+
+SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+# 配置
+IMAGE_TAG="${IMAGE_TAG:-}"
+CANARY_STAGES="${CANARY_STAGES:-5:600,20:900,50:1200}"
+PROD_API_URL="${PROD_API_URL:-https://api.xiaoxiajianji.com}"
+PROD_WEB_URL="${PROD_WEB_URL:-https://saas.xiaoxiajianji.com}"
+PRODUCTION_SSH_HOST="${PRODUCTION_SSH_HOST:-47.98.113.167}"
+PRODUCTION_SSH_USER="${PRODUCTION_SSH_USER:-root}"
+PRODUCTION_SSH_PORT="${PRODUCTION_SSH_PORT:-22222}"
+# gray_deploy.sh 的镜像命名格式是 ${REGISTRY}-component:tag
+# 需要与 ACR 镜像名 xiaoxia-registry.../xiaoxiakeji/xiaoxia-saas-api:tag 匹配
+GRAY_REGISTRY="${GRAY_REGISTRY:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/xiaoxia-saas}"
+ACR_REGISTRY_HOST="${ACR_REGISTRY_HOST:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com}"
+ACR_USERNAME="${ACR_USERNAME:-}"
+ACR_PASSWORD="${ACR_PASSWORD:-}"
+SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}"
+
+if [[ -z "$IMAGE_TAG" ]]; then
+    echo "ERROR: IMAGE_TAG is required"
+    exit 1
+fi
+
+# 颜色
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+log_info()  { echo -e "${GREEN}[INFO]${NC} $1"; }
+log_warn()  { echo -e "${YELLOW}[WARN]${NC} $1"; }
+log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
+log_step()  { echo -e "${BLUE}[STEP]${NC} $1"; }
+
+# ===========================================
+# SSH 配置
+# ===========================================
+SSH_KEY_PATH=""
+
+setup_ssh() {
+    if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
+        SSH_KEY_PATH="/root/.ssh/xiaoxia_runtime_builder"
+    elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then
+        SSH_KEY_PATH="$HOME/.ssh/xiaoxia_runtime_builder"
+    elif [ -n "${PRODUCTION_SSH_KEY:-}" ]; then
+        SSH_KEY_PATH="$HOME/.ssh/canary_deploy_key"
+        mkdir -p "$HOME/.ssh"
+        printf '%s\n' "$PRODUCTION_SSH_KEY" > "$SSH_KEY_PATH"
+        chmod 600 "$SSH_KEY_PATH"
+    else
+        log_error "没有可用的 SSH 密钥"
+        return 1
+    fi
+
+    ssh-keyscan -p "$PRODUCTION_SSH_PORT" -H "$PRODUCTION_SSH_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true
+    log_info "SSH 已配置: ${PRODUCTION_SSH_USER}@${PRODUCTION_SSH_HOST}:${PRODUCTION_SSH_PORT}"
+}
+
+run_ssh() {
+    local cmd="$1"
+    ssh -p "$PRODUCTION_SSH_PORT" -i "$SSH_KEY_PATH" -o StrictHostKeyChecking=no \
+        "${PRODUCTION_SSH_USER}@${PRODUCTION_SSH_HOST}" "$cmd"
+}
+
+# ===========================================
+# 上传脚本 + Docker登录
+# ===========================================
+prepare_server() {
+    log_step "准备生产服务器环境"
+    
+    # 创建临时目录
+    run_ssh "mkdir -p /tmp/canary-release"
+    
+    # 上传 gray_deploy.sh
+    local gray_script="$REPO_ROOT/scripts/gray_deploy.sh"
+    if [[ -f "$gray_script" ]]; then
+        cat "$gray_script" | run_ssh "cat > /tmp/canary-release/gray_deploy.sh && chmod +x /tmp/canary-release/gray_deploy.sh"
+        log_info "  gray_deploy.sh 已上传"
+    else
+        log_error "找不到 gray_deploy.sh: $gray_script"
+        return 1
+    fi
+    
+    # 上传 rollback_gray.sh
+    local rollback_script="$REPO_ROOT/scripts/rollback_gray.sh"
+    if [[ -f "$rollback_script" ]]; then
+        cat "$rollback_script" | run_ssh "cat > /tmp/canary-release/rollback_gray.sh && chmod +x /tmp/canary-release/rollback_gray.sh"
+        log_info "  rollback_gray.sh 已上传"
+    else
+        log_warn "找不到 rollback_gray.sh"
+    fi
+    
+    # 上传 ci_production_deploy.sh
+    local prod_deploy="$REPO_ROOT/scripts/ci_production_deploy.sh"
+    if [[ -f "$prod_deploy" ]]; then
+        cat "$prod_deploy" | run_ssh "cat > /tmp/canary-release/ci_production_deploy.sh && chmod +x /tmp/canary-release/ci_production_deploy.sh"
+        log_info "  ci_production_deploy.sh 已上传"
+    else
+        log_error "找不到 ci_production_deploy.sh: $prod_deploy"
+        return 1
+    fi
+    
+    # Docker 登录到 ACR
+    if [[ -n "$ACR_USERNAME" && -n "$ACR_PASSWORD" ]]; then
+        log_info "  Docker 登录到 ACR..."
+        run_ssh "docker login '$ACR_REGISTRY_HOST' -u '$ACR_USERNAME' -p '$ACR_PASSWORD' 2>/dev/null" || \
+            log_warn "  Docker login 失败（可能已有凭证），将尝试直接 pull"
+    fi
+    
+    log_info "✅ 服务器环境准备完成"
+}
+
+# ===========================================
+# 健康检查（公网访问）
+# ===========================================
+health_check() {
+    local stage_name="$1"
+    local timeout="${2:-120}"
+    local interval=5
+    local elapsed=0
+
+    log_step "健康检查 - $stage_name (超时 ${timeout}s)"
+
+    while [ $elapsed -lt $timeout ]; do
+        local api_ok=false
+        local web_ok=false
+
+        # 检查 API
+        local api_code=$(curl -s -o /dev/null -w "%{http_code}" \
+            --connect-timeout 5 --max-time 10 \
+            "${PROD_API_URL}/health" 2>/dev/null || echo "000")
+        if [[ "$api_code" == "200" ]]; then
+            api_ok=true
+        fi
+
+        # 检查 Web
+        local web_code=$(curl -s -o /dev/null -w "%{http_code}" \
+            --connect-timeout 5 --max-time 10 \
+            "$PROD_WEB_URL" 2>/dev/null || echo "000")
+        if [[ "$web_code" == "200" || "$web_code" == "301" || "$web_code" == "302" ]]; then
+            web_ok=true
+        fi
+
+        if $api_ok && $web_ok; then
+            log_info "✅ 健康检查通过 (API=$api_code, Web=$web_code)"
+            return 0
+        fi
+
+        log_warn "  等待中... API=$api_code, Web=$web_code (${elapsed}s/${timeout}s)"
+        sleep $interval
+        elapsed=$((elapsed + interval))
+    done
+
+    log_error "❌ 健康检查超时"
+    return 1
+}
+
+# ===========================================
+# 灰度发布
+# ===========================================
+gray_deploy() {
+    local pct="$1"
+    log_step "灰度发布 ${pct}% - $IMAGE_TAG"
+
+    run_ssh "cd /tmp/canary-release && \
+        REGISTRY='$GRAY_REGISTRY' \
+        ./gray_deploy.sh '$IMAGE_TAG' '$pct'"
+}
+
+# ===========================================
+# 全量部署
+# ===========================================
+full_deploy() {
+    log_step "全量部署 - $IMAGE_TAG"
+
+    run_ssh "cd /tmp/canary-release && \
+        IMAGE_TAG='$IMAGE_TAG' \
+        ACR_USERNAME='$ACR_USERNAME' \
+        ACR_PASSWORD='$ACR_PASSWORD' \
+        sh ./ci_production_deploy.sh"
+}
+
+# ===========================================
+# 灰度回滚
+# ===========================================
+rollback_gray() {
+    log_error "执行灰度回滚..."
+    if [[ "$SKIP_ROLLBACK" == "true" ]]; then
+        log_warn "SKIP_ROLLBACK=true，跳过回滚"
+        return
+    fi
+
+    if run_ssh "test -f /tmp/canary-release/rollback_gray.sh"; then
+        run_ssh "cd /tmp/canary-release && ./rollback_gray.sh" || \
+            log_error "回滚脚本执行失败，请手动处理"
+    else
+        # 内联回滚逻辑
+        log_warn "使用内联回滚逻辑"
+        run_ssh '
+            NGINX_CONF="/etc/nginx/sites-enabled/00-xiaoxia-saas"
+            LATEST_BAK=$(ls -t "${NGINX_CONF}".bak.gray.* 2>/dev/null | head -1 || true)
+            if [[ -n "$LATEST_BAK" ]]; then
+                cp "$LATEST_BAK" "$NGINX_CONF"
+            else
+                sed -i "s|proxy_pass http://saas_api_backend|proxy_pass http://127.0.0.1:8001|g" "$NGINX_CONF"
+                sed -i "s|proxy_pass http://saas_web_backend/|proxy_pass http://127.0.0.1:3002/|g" "$NGINX_CONF"
+            fi
+            nginx -t && nginx -s reload
+            docker rm -f xiaoxia-api-canary xiaoxia-web-canary 2>/dev/null || true
+        ' || log_error "回滚失败，请手动处理"
+    fi
+}
+
+# ===========================================
+# 通知
+# ===========================================
+notify_status() {
+    local status="$1"
+    local message="$2"
+    if [ -n "${CI_NOTIFY_WEBHOOK:-}" ]; then
+        NOTIFY_MODE="$status" JOB_NAME="Canary Release - $message" \
+            python3 "$REPO_ROOT/scripts/ci_notify.py" 2>/dev/null || true
+    fi
+}
+
+# ===========================================
+# 清理
+# ===========================================
+cleanup() {
+    log_step "清理生产服务器临时文件"
+    run_ssh "rm -rf /tmp/canary-release" 2>/dev/null || true
+}
+
+# ===========================================
+# 主流程
+# ===========================================
+main() {
+    echo "==========================================="
+    echo "  🐦 金丝雀发布"
+    echo "  版本: $IMAGE_TAG"
+    echo "  阶段: $CANARY_STAGES"
+    echo "==========================================="
+    echo ""
+
+    setup_ssh
+    prepare_server
+    trap cleanup EXIT
+
+    # 解析灰度阶段
+    IFS=',' read -ra STAGES <<< "$CANARY_STAGES"
+    local total_stages=${#STAGES[@]}
+    local current_stage=0
+
+    # 逐阶段灰度
+    for stage in "${STAGES[@]}"; do
+        current_stage=$((current_stage + 1))
+        local pct=$(echo "$stage" | cut -d: -f1)
+        local wait_time=$(echo "$stage" | cut -d: -f2)
+
+        echo ""
+        echo "--- 阶段 $current_stage/$total_stages: ${pct}% 灰度 ---"
+
+        # 执行灰度发布
+        if ! gray_deploy "$pct"; then
+            log_error "灰度发布 ${pct}% 失败"
+            rollback_gray
+            notify_status "failure" "Stage ${pct}% Deploy Failed"
+            exit 1
+        fi
+
+        # 健康检查
+        if ! health_check "${pct}%灰度"; then
+            log_error "${pct}%灰度健康检查失败"
+            rollback_gray
+            notify_status "failure" "Stage ${pct}% Health Check Failed"
+            exit 1
+        fi
+
+        # 观察期
+        log_info "⏳ 观察期 ${wait_time}s，监控流量稳定性..."
+        local waited=0
+        local check_interval=60
+        while [ $waited -lt $wait_time ]; do
+            sleep $check_interval
+            waited=$((waited + check_interval))
+            # 每隔一段时间做一次快速健康检查
+            local api_code=$(curl -s -o /dev/null -w "%{http_code}" \
+                --connect-timeout 5 --max-time 10 \
+                "${PROD_API_URL}/health" 2>/dev/null || echo "000")
+            if [[ "$api_code" != "200" ]]; then
+                log_error "❌ 观察期内 API 异常 (HTTP $api_code)，触发回滚"
+                rollback_gray
+                notify_status "failure" "Stage ${pct}% Watch Period Failed"
+                exit 1
+            fi
+            log_info "  观察中... ${waited}s/${wait_time}s (API=$api_code)"
+        done
+
+        log_info "✅ ${pct}%灰度阶段完成，稳定运行 ${wait_time}s"
+    done
+
+    # 全量部署
+    echo ""
+    echo "--- 最终阶段: 100% 全量部署 ---"
+
+    if ! full_deploy; then
+        log_error "全量部署失败"
+        notify_status "failure" "Full Deploy Failed"
+        exit 1
+    fi
+
+    # 最终健康检查
+    if ! health_check "全量部署" "180"; then
+        log_error "全量部署后健康检查失败"
+        notify_status "failure" "Full Deploy Health Check Failed"
+        exit 1
+    fi
+
+    # 清理 canary 容器
+    log_step "清理 Canary 容器"
+    run_ssh "docker rm -f xiaoxia-api-canary xiaoxia-web-canary 2>/dev/null || true" || true
+
+    echo ""
+    echo "==========================================="
+    echo "  ✅ 金丝雀发布完成"
+    echo "  版本: $IMAGE_TAG"
+    echo "  状态: 100%全量运行"
+    echo "==========================================="
+
+    notify_status "success" "$IMAGE_TAG Fully Deployed"
+}
+
+main "$@"