diff --git a/scripts/ci/auto_merge.sh b/scripts/ci/auto_merge.sh index 3d40674be..bcf7747ab 100755 --- a/scripts/ci/auto_merge.sh +++ b/scripts/ci/auto_merge.sh @@ -4,8 +4,6 @@ # 环境变量:GITHUB_TOKEN, MERGE_TOKEN, PR_NUMBER, PR_HEAD_SHA, BASE_REF, GITHUB_API_URL, GITHUB_REPOSITORY set -eu -set -eu - echo "PR #${PR_NUMBER} - 检查CI状态+审批并自动合并到${BASE_REF}" echo echo "模式: 短作业(只检查一次,不满足则退出,由pr-auto-scan定时兜底)" @@ -25,6 +23,7 @@ TOTAL=$(echo "$FILES" | grep -cv '^$' || true) FRONTEND_COUNT=$(echo "$FILES" | grep -c '^apps/web/' || true) BACKEND_COUNT=$((TOTAL - FRONTEND_COUNT)) echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT})" +echo # 使用统一的CI Gate门禁(单一检查点,自动处理前端/后端/全栈跳过逻辑) CONTEXTS=( @@ -115,6 +114,11 @@ check_and_merge() { if [ "$MERGE_405_COUNT" -ge "$MAX_405_RETRIES" ]; then echo "⚠️ 连续${MAX_405_RETRIES}次合并返回405,放弃本次自动合并" echo " (pr-auto-scan会继续尝试,需人工确认是否有冲突或门禁问题)" + curl -s -X POST \ + -H "Authorization: token ${MERGE_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"body": "Auto merge skipped after multiple 405 errors: PR may have conflicts or unresolved checks. Please review manually. This is not a CI failure."}' \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" > /dev/null 2>&1 || true exit 0 fi echo "30秒后重试..." diff --git a/scripts/ci/ci_health_report.py b/scripts/ci/ci_health_report.py index 12b52bfb4..0ccf0e446 100644 --- a/scripts/ci/ci_health_report.py +++ b/scripts/ci/ci_health_report.py @@ -96,7 +96,7 @@ def build_feishu_card(data: dict) -> dict: # 失败详情(最多显示5条) fail_detail_lines = [] - for i, run in enumerate(failed_runs[:5]): + for _i, run in enumerate(failed_runs[:5]): run_id = run["id"] title = run.get("title", "")[:35] branch = run.get("branch", "") diff --git a/scripts/ci/ci_repeated_failure_detector.py b/scripts/ci/ci_repeated_failure_detector.py index 5a4c8b556..b5dc95d34 100644 --- a/scripts/ci/ci_repeated_failure_detector.py +++ b/scripts/ci/ci_repeated_failure_detector.py @@ -122,8 +122,8 @@ def analyze_failures(runs): for run in sorted_runs: run_id = run.get("id") - run_status = run.get("status", "") - run_conclusion = run.get("conclusion", "") + run.get("status", "") + run.get("conclusion", "") run_started = run.get("started_at", run.get("created_at", "")) event = run.get("event", "") @@ -135,7 +135,7 @@ def analyze_failures(runs): for job in jobs: name = job.get("name", "") - status = job.get("status", "") + job.get("status", "") conclusion = job.get("conclusion", "") # 跳过非CI核心job(如AI Code Review、Preview等) @@ -176,7 +176,7 @@ def analyze_failures(runs): # cancelled不算失败也不打断 # 计算失败率 - for name, stats in job_stats.items(): + for _name, stats in job_stats.items(): total_actual = stats["total"] - stats["skipped"] - stats["cancelled"] if total_actual > 0: stats["failure_rate"] = round((stats["failure"] + stats["error"]) / total_actual * 100, 1) @@ -240,10 +240,10 @@ def generate_report(critical, warning, info, days, total_runs): lines.append(f"**生成时间**: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}") lines.append("") - lines.append(f"## 概览") + lines.append("## 概览") lines.append("") - lines.append(f"| 级别 | 数量 |") - lines.append(f"|------|------|") + lines.append("| 级别 | 数量 |") + lines.append("|------|------|") lines.append(f"| 🔴 严重 (连续失败≥{CONSECUTIVE_FAIL_THRESHOLD}次 或 失败率≥50%) | {len(critical)} |") lines.append(f"| 🟡 警告 (失败率≥{FAIL_RATE_THRESHOLD}% 且 失败≥{FAIL_THRESHOLD}次) | {len(warning)} |") lines.append(f"| 🔵 关注 (失败≥2次) | {len(info)} |") @@ -350,7 +350,7 @@ def send_feishu_notification(critical, warning, info, days): def main(): - print(f"=== CI重复失败检测 ===") + print("=== CI重复失败检测 ===") print(f"统计周期: 最近{DAYS}天") print(f"仓库: {REPO}") print() diff --git a/scripts/ci/docker_build_only.sh b/scripts/ci/docker_build_only.sh index 7eda422ab..d0aaa18d7 100755 --- a/scripts/ci/docker_build_only.sh +++ b/scripts/ci/docker_build_only.sh @@ -1,6 +1,6 @@ #!/bin/bash # PR构建专用:只构建不输出,验证Dockerfile能否正常构建 -# 优先用buildx + 远程缓存,失败自动回退到普通docker build(DooD模式下buildx builder偶发崩溃) +# 无本地缓存(12个runner不共享,反而添乱),只用ACR远程缓存 set -eu NO_CACHE_FLAG="" @@ -19,64 +19,25 @@ for arg in "$@"; do done BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}" +if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then + docker buildx create --use --name "$BUILDER_NAME" --driver docker-container +else + docker buildx use "$BUILDER_NAME" +fi +docker buildx inspect --bootstrap -echo "=== PR Build: buildx + remote cache (attempt 1) ===" +echo "=== PR Build: build only, no output, remote cache only ===" echo "Dockerfile: ${DOCKERFILE}" echo "Image tag: ${IMAGE_TAG}" echo "" -# --- 尝试 buildx docker-container driver --- -if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then - docker buildx create --use --name "$BUILDER_NAME" --driver docker-container 2>/dev/null || true -else - docker buildx use "$BUILDER_NAME" 2>/dev/null || true -fi -docker buildx inspect --bootstrap > /dev/null 2>&1 || true - -set +e docker buildx build \ $NO_CACHE_FLAG \ $BUILD_ARGS \ --cache-from "type=registry,ref=${CACHE_REF}" \ -f "${DOCKERFILE}" \ -t "${IMAGE_TAG}" \ - --load \ . -BUILDX_EXIT=$? -set -e - -if [ $BUILDX_EXIT -eq 0 ]; then - echo "" - echo "PR build OK (buildx): ${IMAGE_TAG}" - exit 0 -fi echo "" -echo "⚠️ buildx build失败,回退到普通docker build" -echo " 原因:buildx builder在DooD模式下偶发不稳定(graceful_stop / buildkitd.sock)" -echo "" - -# 清理 buildx builder -docker buildx rm "$BUILDER_NAME" 2>/dev/null || true - -# --- 回退:普通 docker build --- -# 注意:普通docker build不支持远程缓存,但更稳定 -set +e -docker build \ - $NO_CACHE_FLAG \ - $BUILD_ARGS \ - -f "${DOCKERFILE}" \ - -t "${IMAGE_TAG}" \ - . -DOCKER_EXIT=$? -set -e - -if [ $DOCKER_EXIT -eq 0 ]; then - echo "" - echo "PR build OK (fallback docker build): ${IMAGE_TAG}" - exit 0 -fi - -echo "" -echo "❌ PR build failed (both buildx and docker build)" -exit 1 +echo "PR build OK (build only, no output): ${IMAGE_TAG}" diff --git a/scripts/ci/run_integration_tests.sh b/scripts/ci/run_integration_tests.sh index 49aa82ad8..192b538c4 100755 --- a/scripts/ci/run_integration_tests.sh +++ b/scripts/ci/run_integration_tests.sh @@ -4,6 +4,11 @@ # 支持 pytest-xdist 并行执行:每个 worker 使用独立数据库,预期加速 2-4 倍 set -eu +# 加载CI共享常量 +SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")" +# shellcheck source=ci_env.sh +source "${SCRIPT_DIR}/ci_env.sh" + echo "=== CI Integration Tests 开始 ===" # --- 安装依赖 --- @@ -47,7 +52,7 @@ bash scripts/ci/step_install_ffmpeg.sh # 需要用宿主机IP访问映射端口 # 检测策略:host.docker.internal -> docker0桥接IP -> 容器IP直连 -> 默认网关 -> 127.0.0.1 detect_docker_host() { - local test_port="${1:-5432}" + local test_port="${1:-${CI_LOCAL_PG_PORT}}" # 候选IP列表 local candidates=() @@ -106,7 +111,7 @@ except: # 获取宿主机IP(先尝试用共享PG端口5433测试,再回退到其他端口) if [ -S /var/run/docker.sock ]; then # 先用共享PG端口5433探测 - DOCKER_HOST_IP=$(detect_docker_host 5433) + DOCKER_HOST_IP=$(detect_docker_host "${CI_SHARED_PG_PORT}") if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then # 如果共享PG端口探测失败,说明不在DooD或共享PG不可用,再试其他端口 DOCKER_HOST_IP=$(detect_docker_host 22) @@ -182,9 +187,9 @@ if [ "$USE_SHARED_PG" = "true" ]; then # 使用常驻共享PG实例 echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true)" SHARED_PG_HOST="$PG_HOST" - SHARED_PG_PORT="5433" - SHARED_PG_USER="postgres" - SHARED_PG_PASSWORD="ci_pg_2026!" + SHARED_PG_PORT="${CI_SHARED_PG_PORT}" + SHARED_PG_USER="${CI_SHARED_PG_USER}" + SHARED_PG_PASSWORD="${CI_SHARED_PG_PASSWORD}" echo "等待共享PG连接就绪..." wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5 @@ -220,9 +225,9 @@ else --health-timeout 5s \ --health-retries 12 \ postgres:16 - PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2) + PG_PORT=$(docker port "$PG_CONTAINER" ${CI_LOCAL_PG_PORT}/tcp | cut -d: -f2) echo "PostgreSQL port: $PG_PORT" - export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas" + export DATABASE_URL="postgresql+psycopg://${CI_SHARED_PG_USER}:${CI_SHARED_PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${CI_DEFAULT_DB}" # 等待容器健康 for i in $(seq 1 30); do diff --git a/scripts/ci/run_unit_tests.sh b/scripts/ci/run_unit_tests.sh index ae3dc2e4b..3a0faadcf 100755 --- a/scripts/ci/run_unit_tests.sh +++ b/scripts/ci/run_unit_tests.sh @@ -3,6 +3,9 @@ # 包含:依赖安装、增量测试选择、覆盖率测试、diff覆盖率门禁 set -eu +# 测试环境必须的密钥变量 +export JWT_SECRET_KEY=${JWT_SECRET_KEY:-test-jwt-secret-for-ci-only-2026} + JOB_NAME="${1:-Unit Tests}" echo "=== CI Unit Tests 开始 ===" @@ -42,6 +45,9 @@ for i in 1 2 3; do done pytest --version +# --- 安装 ffmpeg(视频处理相关测试依赖)--- +bash scripts/ci/step_install_ffmpeg.sh + # 双保险:确保numpy已安装 python3 -m pip install -q numpy==1.26.4 || true @@ -94,7 +100,7 @@ else -m pytest tests/unit -q python3 -m coverage report --show-missing python3 -m coverage xml -o coverage.xml - python3 -m coverage report --fail-under=65 > /dev/null || true # 全量覆盖率仅作参考,不阻塞合并 + python3 -m coverage report --fail-under=65 > /dev/null fi # --- Diff 覆盖率检查(仅PR) --- diff --git a/scripts/ci/run_validate.sh b/scripts/ci/run_validate.sh index 050a82b94..c85c299a1 100755 --- a/scripts/ci/run_validate.sh +++ b/scripts/ci/run_validate.sh @@ -14,6 +14,11 @@ # 所有子任务同时启动,最后汇总结果。 set -eu +# 加载CI共享常量 +SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")" +# shellcheck source=ci_env.sh +source "${SCRIPT_DIR}/ci_env.sh" + echo "=== CI Validate: 并行化代码质量检查 ===" echo "" @@ -302,7 +307,7 @@ task_alembic() { # --- DooD模式检测:确定宿主机访问地址 --- detect_docker_host() { - local test_port="${1:-5432}" + local test_port="${1:-${CI_LOCAL_PG_PORT}}" local candidates=() # 1. host.docker.internal @@ -376,7 +381,7 @@ except: # 获取宿主机IP local PG_HOST if [ -S /var/run/docker.sock ]; then - PG_HOST=$(detect_docker_host 5433) + PG_HOST=$(detect_docker_host "${CI_SHARED_PG_PORT}") if [ "$PG_HOST" = "127.0.0.1" ]; then PG_HOST=$(detect_docker_host 22) fi @@ -394,9 +399,9 @@ except: # 使用常驻共享PG实例 echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true)" local SHARED_PG_HOST="$PG_HOST" - local SHARED_PG_PORT="5433" - local SHARED_PG_USER="postgres" - local SHARED_PG_PASSWORD="ci_pg_2026!" + local SHARED_PG_PORT="${CI_SHARED_PG_PORT}" + local SHARED_PG_USER="${CI_SHARED_PG_USER}" + local SHARED_PG_PASSWORD="${CI_SHARED_PG_PASSWORD}" local CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}" echo "等待共享PG连接就绪..." @@ -456,9 +461,9 @@ conn.close() if [ $exit_code -eq 0 ]; then local PG_PORT - PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2) + PG_PORT=$(docker port "$PG_CONTAINER" ${CI_LOCAL_PG_PORT}/tcp | cut -d: -f2) echo "PostgreSQL port: $PG_PORT" - export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas" + export DATABASE_URL="postgresql+psycopg://${CI_SHARED_PG_USER}:${CI_SHARED_PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${CI_DEFAULT_DB}" # 等待容器健康 local i diff --git a/scripts/ci/validate_code_quality.sh b/scripts/ci/validate_code_quality.sh index ce67490eb..874c9c6cf 100644 --- a/scripts/ci/validate_code_quality.sh +++ b/scripts/ci/validate_code_quality.sh @@ -56,95 +56,18 @@ for fpath, items in data.get('results', {}).items(): fi echo "✅ Secret scan passed" -# --- 增量/全量模式判断 --- +# --- 代码质量检查(全量,PR 和 push 统一标准)--- +# 历史:PR 侧用增量检查以加速,但会导致 push 侧全量检查失败时 PR 侧感知不到 +# 现在统一全量检查,确保 CI 真正保护主分支(black/isort/ruff 全量仅多几十秒) echo "" -echo "=== [2/6] Code quality checks ===" +echo "=== [2/6] Code quality checks (full scan) ===" SCAN_MODE="full" -CHANGED_PY_FILES="" +echo "Full scan mode" +python3 -m compileall -q alembic apps packages tests scripts +python3 -m black --check --fast alembic apps packages tests scripts +python3 -m isort --check-only alembic apps packages tests scripts +python3 -m ruff check apps packages tests --statistics -if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then - PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') - API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100" - set +e - RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL") - HTTP_CODE=$(echo "$RESPONSE" | tail -n1) - BODY=$(echo "$RESPONSE" | sed '$d') - set -e - if [ "$HTTP_CODE" = "200" ]; then - CHANGED_PY_FILES=$(echo "$BODY" | python3 -c " -import json, sys -try: - files = json.load(sys.stdin) - py_files = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] != 'removed'] - print(' '.join(py_files)) -except Exception: - print('') -") - # 新增文件(added)强制全量检查,防止增量漏检 - ADDED_PY_FILES=$(echo "$BODY" | python3 -c " -import json, sys -try: - files = json.load(sys.stdin) - added = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] == 'added'] - print(' '.join(added)) -except Exception: - print('') -") - MODIFIED_PY_FILES=$(echo "$BODY" | python3 -c " -import json, sys -try: - files = json.load(sys.stdin) - modified = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] not in ('removed', 'added')] - print(' '.join(modified)) -except Exception: - print('') -") - if [ -n "$CHANGED_PY_FILES" ]; then - SCAN_MODE="incremental" - echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed" - else - SCAN_MODE="skip_py" - echo "No Python files changed in this PR" - fi - else - echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan" - fi -else - echo "Full scan mode (not a PR event)" -fi - -if [ "$SCAN_MODE" = "incremental" ]; then - # 防御性过滤 - EXISTING_PY_FILES="" - for f in $CHANGED_PY_FILES; do - if [ -f "$f" ]; then - if [ -z "$EXISTING_PY_FILES" ]; then - EXISTING_PY_FILES="$f" - else - EXISTING_PY_FILES="$EXISTING_PY_FILES $f" - fi - fi - done - CHANGED_PY_FILES="$EXISTING_PY_FILES" - - python3 -m compileall -q $CHANGED_PY_FILES - python3 -m black --check --fast $CHANGED_PY_FILES - python3 -m isort --check-only $CHANGED_PY_FILES - RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | grep -v '^$' | xargs) - if [ -n "$RUFF_FILES" ]; then - python3 -m ruff check $RUFF_FILES --statistics - else - echo "No ruff-checkable files changed, skipping" - fi -elif [ "$SCAN_MODE" = "skip_py" ]; then - echo "No Python files changed - skipping Python lint checks" -else - echo "Full scan mode" - python3 -m compileall -q alembic apps packages tests scripts - python3 -m black --check --fast alembic apps packages tests scripts - python3 -m isort --check-only alembic apps packages tests scripts - python3 -m ruff check apps packages tests --statistics -fi echo "✅ Code quality checks passed" # --- Bandit 安全扫描(仅告警) --- diff --git a/scripts/ci_notify.py b/scripts/ci_notify.py old mode 100755 new mode 100644 index 2cc0c64f7..f15e84802 --- a/scripts/ci_notify.py +++ b/scripts/ci_notify.py @@ -1,68 +1,185 @@ #!/usr/bin/env python3 """ -CI 通知脚本 - 发送飞书消息通知 +统一CI通知脚本 - 发送飞书卡片通知 +支持三种模式: start / success / failure +包含: PR链接、耗时、失败阶段、分支、提交者、Run链接、Runner信息 + 用法: - NOTIFY_MODE=start JOB_NAME="Build API" python3 scripts/ci_notify.py - NOTIFY_MODE=success JOB_NAME="Deploy Staging" python3 scripts/ci_notify.py - NOTIFY_MODE=failure JOB_NAME="Unit Tests" python3 scripts/ci_notify.py + NOTIFY_MODE=start JOB_NAME="xxx" python3 scripts/ci_notify.py + NOTIFY_MODE=success JOB_NAME="xxx" JOB_DURATION="2m30s" python3 scripts/ci_notify.py + NOTIFY_MODE=failure JOB_NAME="xxx" FAILED_STEP="xxx" JOB_DURATION="2m30s" python3 scripts/ci_notify.py 环境变量: - NOTIFY_MODE - 通知类型: start/success/failure - JOB_NAME - Job名称 - CI_NOTIFY_WEBHOOK - 飞书Webhook地址 - GITHUB_SHA - Commit SHA (可选) - GITHUB_REF_NAME - 分支名 (可选) - GITHUB_RUN_ID - Run ID (可选) - GITHUB_REPOSITORY - 仓库名 (可选) - GITHUB_SERVER_URL - Gitea地址 (可选) + CI_NOTIFY_WEBHOOK - 飞书webhook地址 (必填) + NOTIFY_MODE - 通知模式: start / success / failure (必填) + JOB_NAME - Job名称 (必填) + JOB_DURATION - 耗时,如"2m30s" (成功/失败时建议传) + FAILED_STEP - 失败的步骤名 (失败时建议传) + GITHUB_REF_NAME - 分支名 + GITHUB_SHA - commit SHA + GITHUB_ACTOR - 提交者 + GITHUB_RUN_ID - Run ID + GITHUB_REPOSITORY - 仓库路径 + GITHUB_EVENT_NAME - 事件类型 (pull_request / push / ...) + GITHUB_PR_NUMBER - PR编号 (PR事件时) + GITHUB_PR_TITLE - PR标题 (PR事件时) + RUNNER_NAME - Runner名称 (可选,自动获取) 设计原则: - 通知失败永远不阻断主流程(永远返回0) + 1. 通知失败永远不阻断CI主流程(返回exit code 0) + 2. 标题包含"CI通知"/"CI告警"关键词,适配飞书webhook关键词校验 + 3. 卡片信息尽量丰富,方便快速定位问题 """ import json import os import sys import urllib.request -from datetime import datetime def get_env(name, default=""): + """读取环境变量""" return os.environ.get(name, default) -def send_feishu_notify(webhook_url, title, content, color="blue"): - """发送飞书通知(简单卡片格式)""" - if not webhook_url: - print("[INFO] 未配置 CI_NOTIFY_WEBHOOK,跳过通知") - return True +def format_duration(seconds_str): + """将秒数格式化为易读形式""" + try: + seconds = int(float(seconds_str)) + mins = seconds // 60 + secs = seconds % 60 + if mins > 0: + return f"{mins}m{secs}s" + return f"{secs}s" + except (ValueError, TypeError): + return seconds_str or "未知" - # 状态颜色映射 - color_map = { - "green": "green", - "red": "red", - "blue": "blue", - "yellow": "yellow", - } - header_color = color_map.get(color, "blue") - # 构造卡片 - card = { - "config": {"wide_screen_mode": True}, - "header": { - "title": {"tag": "plain_text", "content": title}, - "template": header_color, +def classify_job(job_name): + """根据Job名称判断所属阶段""" + name = job_name.lower() + if any(k in name for k in ["validate", "lint", "unit test", "integration test"]): + return "门禁检查" + if any(k in name for k in ["build", "image"]): + return "镜像构建" + if any(k in name for k in ["deploy", "staging", "production"]): + return "部署发布" + if any(k in name for k in ["e2e", "test", "smoke"]): + return "测试验证" + return "其他" + + +def main() -> int: + webhook = get_env("CI_NOTIFY_WEBHOOK") + if not webhook: + print("未配置 CI_NOTIFY_WEBHOOK,跳过通知") + print("如需启用,请在仓库 Settings -> Secrets and variables -> Actions 中添加 CI_NOTIFY_WEBHOOK") + return 0 + + mode = get_env("NOTIFY_MODE", "failure").lower() + job_name = get_env("JOB_NAME", "Unknown Job") + duration = get_env("JOB_DURATION") + if not duration: + duration_sec = get_env("JOB_DURATION_SECONDS") + duration = format_duration(duration_sec) if duration_sec else "计算中..." + + failed_step = get_env("FAILED_STEP", "") + branch = get_env("GITHUB_REF_NAME", "unknown") + commit = get_env("GITHUB_SHA", "unknown")[:8] + actor = get_env("GITHUB_ACTOR", "unknown") + run_id = get_env("GITHUB_RUN_ID", "unknown") + repo = get_env("GITHUB_REPOSITORY", "unknown") + event_name = get_env("GITHUB_EVENT_NAME", "") + pr_number = get_env("GITHUB_PR_NUMBER", "") + pr_title = get_env("GITHUB_PR_TITLE", "") + runner_name = get_env("RUNNER_NAME", "") + + run_url = f"https://git.xiaoxiajianji.com/{repo}/actions/runs/{run_id}" + job_stage = classify_job(job_name) + + # 根据模式设置标题、状态、颜色 + # 注意:标题中必须包含飞书webhook配置的关键词,否则会报"Key Words Not Found" + # 这里加入"CI通知"/"CI告警"关键词提高命中率 + if mode == "start": + title = f"🔄 CI通知:{job_name} 开始构建" + status = "blue" + button_text = "查看进度" + button_type = "primary" + elif mode == "success": + title = f"✅ CI通知:{job_name} 构建成功" + status = "green" + button_text = "查看详情" + button_type = "primary" + else: # failure + title = f"❌ CI告警:{job_name} 构建失败" + status = "red" + button_text = "查看失败日志" + button_type = "danger" + + # 构建卡片内容 - 左侧标签+右侧值的结构化展示 + fields = [] + + fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**阶段**\n{job_stage}"}}) + fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**任务**\n{job_name}"}}) + + if mode != "start": + fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**耗时**\n{duration}"}}) + else: + fields.append({"is_short": True, "text": {"tag": "lark_md", "content": "**状态**\n进行中"}}) + + if runner_name: + fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**Runner**\n{runner_name}"}}) + + if mode == "failure" and failed_step: + fields.append({"is_short": False, "text": {"tag": "lark_md", "content": f"**失败步骤**\n{failed_step}"}}) + + # PR/分支信息 + if event_name == "pull_request" and pr_number: + pr_url = f"https://git.xiaoxiajianji.com/{repo}/pulls/{pr_number}" + pr_display = f"#{pr_number}" + if pr_title: + pr_display += f" {pr_title[:30]}" + fields.append({"is_short": False, "text": {"tag": "lark_md", "content": f"**PR**\n[{pr_display}]({pr_url})"}}) + elif event_name == "push": + fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**分支**\n{branch}"}}) + + fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**提交**\n`{commit}`"}}) + fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**提交者**\n{actor}"}}) + fields.append({"is_short": True, "text": {"tag": "lark_md", "content": f"**Run ID**\n{run_id}"}}) + + payload = { + "msg_type": "interactive", + "card": { + "header": { + "title": { + "tag": "plain_text", + "content": title, + }, + "status": status, + }, + "elements": [ + { + "tag": "div", + "fields": fields, + }, + { + "tag": "action", + "actions": [ + { + "tag": "button", + "text": {"tag": "plain_text", "content": button_text}, + "url": run_url, + "type": button_type, + } + ], + }, + ], }, - "elements": [ - {"tag": "div", "text": {"tag": "lark_md", "content": content}}, - ], } - payload = {"msg_type": "interactive", "card": card} data = json.dumps(payload).encode("utf-8") - req = urllib.request.Request( - webhook_url, + webhook, data=data, headers={"Content-Type": "application/json"}, method="POST", @@ -70,64 +187,20 @@ def send_feishu_notify(webhook_url, title, content, color="blue"): try: with urllib.request.urlopen(req, timeout=10) as resp: resp_body = resp.read().decode("utf-8") - result = json.loads(resp_body) - if result.get("code", 0) != 0: - print(f"[WARN] 飞书通知返回错误: {result.get('msg', resp_body)}", file=sys.stderr) - return False - print("[INFO] 飞书通知发送成功") - return True + # 飞书返回code=0表示成功 + try: + result = json.loads(resp_body) + if result.get("code", 0) != 0: + print(f"通知发送告警: 飞书返回错误 - {result.get('msg', resp_body)}", file=sys.stderr) + print(f"通知已发送 ({mode}) - 飞书返回非0,但不阻断CI流程") + else: + print(f"通知已发送 ({mode})") + except json.JSONDecodeError: + print(f"通知已发送 ({mode})") except Exception as e: - print(f"[WARN] 飞书通知发送失败: {e}", file=sys.stderr) - return False + print(f"通知发送告警: {e}", file=sys.stderr) - -def build_message(): - """根据环境变量构造通知消息""" - notify_mode = get_env("NOTIFY_MODE", "info").lower() - job_name = get_env("JOB_NAME", "未知Job") - branch = get_env("GITHUB_REF_NAME", "未知分支") - sha = get_env("GITHUB_SHA", "")[:8] - run_id = get_env("GITHUB_RUN_ID", "") - repo = get_env("GITHUB_REPOSITORY", "") - server_url = get_env("GITHUB_SERVER_URL", "https://git.xiaoxiajianji.com") - - # 状态映射 - status_map = { - "start": ("🔔 CI 任务开始", "blue", "开始执行"), - "success": ("✅ CI 任务成功", "green", "执行成功"), - "failure": ("❌ CI 任务失败", "red", "执行失败"), - "info": ("ℹ️ CI 通知", "blue", "通知"), - } - title, color, status_text = status_map.get(notify_mode, status_map["info"]) - - # 构造内容 - content_lines = [ - f"**任务**: {job_name}", - f"**状态**: {status_text}", - f"**分支**: {branch}", - ] - if sha: - content_lines.append(f"**Commit**: `{sha}`") - if run_id and repo and server_url: - run_url = f"{server_url}/{repo}/actions/runs/{run_id}" - content_lines.append(f"**详情**: [点击查看]({run_url})") - - content_lines.append(f"**时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") - - content = "\n".join(content_lines) - return title, content, color - - -def main(): - webhook = get_env("CI_NOTIFY_WEBHOOK", "") - title, content, color = build_message() - - print(f"[CI Notify] 模式: {get_env('NOTIFY_MODE')}") - print(f"[CI Notify] 任务: {get_env('JOB_NAME')}") - - send_feishu_notify(webhook, title, content, color) - - # 永远返回0,不阻断主流程 + # 通知无论成功失败都不阻断CI主流程,统一返回0 return 0 diff --git a/scripts/ci_staging_deploy.sh b/scripts/ci_staging_deploy.sh index f2aeca7f5..71444545f 100755 --- a/scripts/ci_staging_deploy.sh +++ b/scripts/ci_staging_deploy.sh @@ -1,99 +1,340 @@ #!/bin/sh -# ============================================ -# CI Staging 部署脚本 - 通过 SSH 在 Staging 服务器执行 -# 用法:IMAGE_TAG= ACR_USERNAME= ACR_PASSWORD= sh ci_staging_deploy.sh -# ============================================ - +# =========================================== +# Staging 部署脚本(SSH 模式,并行优化版) +# =========================================== set -eu +retry_cmd() { + local max_attempts=$1 + local backoff=$2 + shift 2 + local attempt=1 + while [ $attempt -le $max_attempts ]; do + if "$@"; then + return 0 + fi + echo " attempt $attempt/$max_attempts failed, retrying in ${backoff}s..." + sleep $backoff + backoff=$((backoff * 2)) + attempt=$((attempt + 1)) + done + echo " ERROR: failed after $max_attempts retries" + return 1 +} + +retry_docker_login() { + echo "Logging in to registry (up to 3 retries)" + export REGISTRY_TOKEN REGISTRY_HOST REGISTRY_USER + if retry_cmd 3 5 sh -c 'printf "%s" "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin'; then + return 0 + fi + echo "WARN: docker login failed after retries, will try pull anyway" + return 0 +} + +retry_docker_pull() { + local image=$1 + echo "Pulling $image (up to 3 retries)" + retry_cmd 3 10 docker pull "$image" +} + IMAGE_TAG="${IMAGE_TAG:-}" -ACR_REGISTRY="${ACR_REGISTRY:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji}" -ACR_USERNAME="${ACR_USERNAME:-}" -ACR_PASSWORD="${ACR_PASSWORD:-}" +REGISTRY="${REGISTRY:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji}" +REGISTRY_USER="${ACR_USERNAME:-${REGISTRY_USER:-nick0415343655}}" +REGISTRY_TOKEN="${ACR_PASSWORD:-${REGISTRY_TOKEN:-}}" ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-staging/.env}" GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-staging/generated}" -STAGING_NETWORK="${STAGING_NETWORK:-xiaoxia-net-staging}" +LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-staging/legacy-assets}" + +SKIP_MIGRATION="${SKIP_MIGRATION:-false}" +SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}" if [ -z "$IMAGE_TAG" ]; then echo "ERROR: IMAGE_TAG is required" exit 1 fi -echo "============================================" -echo " Staging 部署 - CI触发" -echo " Version: $IMAGE_TAG" -echo "============================================" -echo "" +test -f "$ENV_FILE" +mkdir -p "$GENERATED_DIR" +mkdir -p "$LEGACY_ASSETS_DIR" -# ---- 登录 ACR ---- -if [ -n "$ACR_PASSWORD" ] && [ -n "$ACR_USERNAME" ]; then - echo "登录 ACR..." - printf '%s' "$ACR_PASSWORD" | docker login "$(echo "$ACR_REGISTRY" | cut -d/ -f1)" -u "$ACR_USERNAME" --password-stdin 2>/dev/null || { - echo "WARN: docker login failed, will try to pull anyway" - } +echo "===========================================" +echo " Staging 部署 - $IMAGE_TAG (并行优化版)" +echo "===========================================" + +echo "Recording current image versions for rollback..." +PREV_API_IMAGE="" +PREV_WORKER_IMAGE="" +PREV_WEB_IMAGE="" +for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do + if docker inspect "$c" >/dev/null 2>&1; then + img=$(docker inspect -f '{{.Config.Image}}' "$c") + case "$c" in + xiaoxia-api-staging) PREV_API_IMAGE="$img" ;; + xiaoxia-worker-staging) PREV_WORKER_IMAGE="$img" ;; + xiaoxia-web-staging) PREV_WEB_IMAGE="$img" ;; + esac + echo " $c -> $img" + else + echo " $c -> (not running)" + fi +done + +rollback() { + echo "" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo " 部署失败,正在自动回滚到上一版本..." + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "" + + if [ "$SKIP_ROLLBACK" = "true" ]; then + echo "SKIP_ROLLBACK=true,跳过自动回滚" + exit 1 + fi + + echo "Stopping new containers..." + docker rm -f xiaoxia-api-staging 2>/dev/null || true + docker rm -f xiaoxia-worker-staging 2>/dev/null || true + docker rm -f xiaoxia-web-staging 2>/dev/null || true + + LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" + + if [ -n "$PREV_API_IMAGE" ]; then + echo "Rolling back API to: $PREV_API_IMAGE" + docker run -d \ + --name xiaoxia-api-staging \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -p 127.0.0.1:8000:8000 \ + -e APP_ENV=staging \ + -e APP_VERSION="$(echo $PREV_API_IMAGE | grep -oE '[^:]+$')" \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 40s \ + $LOG_OPTS \ + "$PREV_API_IMAGE" & + fi + + if [ -n "$PREV_WORKER_IMAGE" ]; then + echo "Rolling back Worker to: $PREV_WORKER_IMAGE" + docker run -d \ + --name xiaoxia-worker-staging \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -e APP_ENV=staging \ + -e APP_VERSION="$(echo $PREV_WORKER_IMAGE | grep -oE '[^:]+$')" \ + -e WORKER_CONCURRENCY=1 \ + -e WORKER_MAX_TASKS_PER_CHILD=100 \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 30s \ + $LOG_OPTS \ + "$PREV_WORKER_IMAGE" & + fi + + if [ -n "$PREV_WEB_IMAGE" ]; then + echo "Rolling back Web to: $PREV_WEB_IMAGE" + LEGACY_VOLUME="" + if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then + LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" + fi + docker run -d \ + --name xiaoxia-web-staging \ + --network xiaoxia-net-staging \ + -p 127.0.0.1:3001:80 \ + --restart unless-stopped \ + $LEGACY_VOLUME \ + --health-cmd "wget --spider -q http://127.0.0.1:80" \ + --health-interval 30s \ + --health-timeout 5s \ + --health-retries 3 \ + $LOG_OPTS \ + "$PREV_WEB_IMAGE" & + fi + + wait + + if [ -n "$PREV_API_IMAGE" ]; then + echo "Waiting for rolled-back API to become healthy..." + i=0 + while [ "$i" -lt 40 ]; do + if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then + echo "Rolled-back API is healthy!" + break + fi + i=$((i + 1)) + echo " Waiting... ($i/40)" + sleep 3 + done + if [ "$i" -ge 40 ]; then + echo "WARN: Rolled-back API did not become healthy within 120s" + docker logs --tail 30 xiaoxia-api-staging + fi + fi + + echo "" + echo "===========================================" + echo " 回滚完成" + echo "===========================================" + echo "Previous API: ${PREV_API_IMAGE:-none}" + echo "Previous Worker: ${PREV_WORKER_IMAGE:-none}" + echo "Previous Web: ${PREV_WEB_IMAGE:-none}" + echo "" + echo "部署失败,已自动回滚到上一版本" + docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging + exit 1 +} + +if [ -n "$REGISTRY_TOKEN" ]; then + echo "==========================================" + echo " Login to Registry (with retries)" + echo "==========================================" + REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1) + retry_docker_login fi -# ---- Pull 3个镜像 ---- -API_IMAGE="${ACR_REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" -WORKER_IMAGE="${ACR_REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" -WEB_IMAGE="${ACR_REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" +# ---- 并行 Pull 三个镜像 ---- +REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" +REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" +REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" + +echo "==========================================" +echo " Pull images (parallel, up to 3 retries each)" +echo "==========================================" +PULL_LOG_DIR="/tmp/staging-pull-$$" +mkdir -p "$PULL_LOG_DIR" + +retry_docker_pull "$REGISTRY_API" > "$PULL_LOG_DIR/api.log" 2>&1 & +PID_API=$! +retry_docker_pull "$REGISTRY_WORKER" > "$PULL_LOG_DIR/worker.log" 2>&1 & +PID_WORKER=$! +retry_docker_pull "$REGISTRY_WEB" > "$PULL_LOG_DIR/web.log" 2>&1 & +PID_WEB=$! + +wait $PID_API $PID_WORKER $PID_WEB echo "" -echo "拉取 API 镜像..." -docker pull "$API_IMAGE" -echo "拉取 Worker 镜像..." -docker pull "$WORKER_IMAGE" -echo "拉取 Web 镜像..." -docker pull "$WEB_IMAGE" -echo "" -echo "所有镜像拉取完成" +echo "Pull 结果:" +PULL_FAILED=0 +for svc in api worker web; do + if tail -1 "$PULL_LOG_DIR/$svc.log" 2>/dev/null | grep -qE "Status:|Downloaded|already exists|is up to date"; then + echo " OK $svc" + elif grep -qE "Digest:|Status: Downloaded" "$PULL_LOG_DIR/$svc.log" 2>/dev/null; then + echo " OK $svc" + else + # 检查docker pull返回值不直接,用镜像是否存在来判断 + img_var="REGISTRY_$(echo $svc | tr '[:lower:]' '[:upper:]')" + img_val=$(eval echo "\$$img_var") + if docker image inspect "$img_val" >/dev/null 2>&1; then + echo " OK $svc" + else + echo " FAIL $svc" + tail -5 "$PULL_LOG_DIR/$svc.log" 2>/dev/null || true + PULL_FAILED=$((PULL_FAILED + 1)) + fi + fi +done -# ---- 确保基础设施在运行 ---- -echo "" -echo "检查基础设施容器..." +rm -rf "$PULL_LOG_DIR" + +if [ "$PULL_FAILED" -gt 0 ]; then + echo "" + echo "ERROR: $PULL_FAILED 个镜像 pull 失败" + exit 1 +fi + +echo "All images pulled." + +echo "Backing up legacy assets from current web container..." +if docker inspect xiaoxia-web-staging >/dev/null 2>&1; then + _tmpdir="/tmp/legacy-assets-$$" + rm -rf "$_tmpdir" + mkdir -p "$_tmpdir" + docker cp xiaoxia-web-staging:/usr/share/nginx/html/assets/. "$_tmpdir/" 2>/dev/null || true + if [ -d "$_tmpdir" ] && [ "$(ls -A "$_tmpdir" 2>/dev/null)" ]; then + cp -an "$_tmpdir"/. "$LEGACY_ASSETS_DIR"/ 2>/dev/null || true + echo "Legacy assets backed up: $(ls "$_tmpdir" | wc -l) files" + fi + rm -rf "$_tmpdir" +else + echo "No existing web container, skipping legacy assets backup" +fi + +if [ -d "$LEGACY_ASSETS_DIR" ]; then + find "$LEGACY_ASSETS_DIR" -type f -mtime +7 -delete 2>/dev/null || true + echo "Legacy assets cleanup done (retain 7 days)" +fi + +echo "Checking infrastructure containers..." for c in xiaoxia-postgres-staging xiaoxia-redis-staging; do if ! docker inspect "$c" >/dev/null 2>&1; then - echo "ERROR: 基础设施容器不存在: $c" + echo "ERROR: Required container not found: $c" exit 1 fi state=$(docker inspect -f '{{.State.Status}}' "$c") if [ "$state" != "running" ]; then - echo "ERROR: 基础设施容器未运行: $c ($state)" + echo "ERROR: Container not running: $c ($state)" exit 1 fi done -echo "基础设施正常" -# ---- 确保 staging 网络存在 ---- -docker network create "$STAGING_NETWORK" 2>/dev/null || true +docker network create xiaoxia-net-staging 2>/dev/null || true -mkdir -p "$GENERATED_DIR" +if [ "$SKIP_MIGRATION" != "true" ]; then + echo "Running database migrations..." + docker run --rm \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -e APP_ENV=staging \ + "$REGISTRY_API" sh -c "cd /app && alembic upgrade head" || { + echo "ERROR: Database migration failed" + exit 1 + } + echo "Migrations completed." +else + echo "Skipping migrations (SKIP_MIGRATION=true)" +fi -# ---- 数据库 Migration ---- -echo "" -echo "执行数据库迁移..." -docker run --rm --env-file "$ENV_FILE" --network "$STAGING_NETWORK" "$API_IMAGE" sh -c "cd /app && /opt/venv/bin/alembic upgrade head" -echo "数据库迁移完成" - -# ---- 停止旧容器 ---- -echo "" -echo "停止旧容器..." +echo "Stopping old containers..." docker rm -f xiaoxia-api-staging 2>/dev/null || true docker rm -f xiaoxia-worker-staging 2>/dev/null || true docker rm -f xiaoxia-web-staging 2>/dev/null || true -# ---- 启动 API ---- -echo "启动 API 容器..." +LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" + +# ---- 并行启动三个容器 ---- +echo "Starting all containers (parallel)..." + +LEGACY_VOLUME="" +if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then + LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" +fi + docker run -d \ --name xiaoxia-api-staging \ --env-file "$ENV_FILE" \ - --network "$STAGING_NETWORK" \ + --network xiaoxia-net-staging \ -p 127.0.0.1:8000:8000 \ -e APP_ENV=staging \ -e APP_VERSION="$IMAGE_TAG" \ -e GENERATED_FILES_DIR=/app/generated \ -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ -v "$GENERATED_DIR:/app/generated" \ --restart unless-stopped \ --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ @@ -101,20 +342,21 @@ docker run -d \ --health-timeout 10s \ --health-retries 3 \ --health-start-period 40s \ - "$API_IMAGE" + $LOG_OPTS \ + "$REGISTRY_API" & +PID_API_START=$! -# ---- 启动 Worker ---- -echo "启动 Worker 容器..." docker run -d \ --name xiaoxia-worker-staging \ --env-file "$ENV_FILE" \ - --network "$STAGING_NETWORK" \ + --network xiaoxia-net-staging \ -e APP_ENV=staging \ -e APP_VERSION="$IMAGE_TAG" \ -e WORKER_CONCURRENCY=1 \ -e WORKER_MAX_TASKS_PER_CHILD=100 \ -e GENERATED_FILES_DIR=/app/generated \ -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ -v "$GENERATED_DIR:/app/generated" \ --restart unless-stopped \ --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ @@ -122,52 +364,127 @@ docker run -d \ --health-timeout 10s \ --health-retries 3 \ --health-start-period 30s \ - "$WORKER_IMAGE" + $LOG_OPTS \ + "$REGISTRY_WORKER" & +PID_WORKER_START=$! -# ---- 启动 Web ---- -echo "启动 Web 容器..." docker run -d \ --name xiaoxia-web-staging \ - --network "$STAGING_NETWORK" \ + --network xiaoxia-net-staging \ -p 127.0.0.1:3001:80 \ --restart unless-stopped \ + $LEGACY_VOLUME \ --health-cmd "wget --spider -q http://127.0.0.1:80" \ --health-interval 30s \ --health-timeout 5s \ --health-retries 3 \ - "$WEB_IMAGE" + $LOG_OPTS \ + "$REGISTRY_WEB" & +PID_WEB_START=$! -# ---- 等待 API 健康 ---- -echo "" -echo "等待 API 健康检查..." -i=0 -while [ "$i" -lt 30 ]; do - if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then - echo "API 健康检查通过" - break +wait $PID_API_START $PID_WORKER_START $PID_WEB_START + +START_FAILED=0 +for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do + if ! docker inspect "$c" >/dev/null 2>&1; then + echo " FAIL $c: not created" + START_FAILED=$((START_FAILED + 1)) + else + state=$(docker inspect -f '{{.State.Status}}' "$c") + if [ "$state" = "running" ] || [ "$state" = "starting" ]; then + echo " OK $c: $state" + else + echo " FAIL $c: $state" + docker logs --tail 20 "$c" 2>/dev/null || true + START_FAILED=$((START_FAILED + 1)) + fi fi - i=$((i + 1)) - echo " 等待中... ($i/30)" - sleep 2 done -if [ "$i" -ge 30 ]; then - echo "ERROR: API 在 60s 内未通过健康检查" - echo "" - echo "=== API 最近日志 ===" - docker logs --tail 50 xiaoxia-api-staging - exit 1 +if [ "$START_FAILED" -gt 0 ]; then + echo "ERROR: $START_FAILED 个容器启动失败" + rollback fi -# ---- 清理旧镜像 ---- -docker image prune -af --filter "until=72h" 2>/dev/null || true +# ---- 并行等待 API 和 Web 健康 ---- +echo "" +echo "Waiting for API + Web health (parallel)..." + +HEALTH_LOG_DIR="/tmp/staging-health-$$" +mkdir -p "$HEALTH_LOG_DIR" + +( + i=0 + while [ "$i" -lt 40 ]; do + if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then + echo "API healthy after $((i * 3))s" + exit 0 + fi + i=$((i + 1)) + sleep 3 + done + echo "API FAILED after 120s" + exit 1 +) > "$HEALTH_LOG_DIR/api.log" 2>&1 & +PID_API_HEALTH=$! + +( + i=0 + while [ "$i" -lt 15 ]; do + if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then + echo "Web healthy after $((i * 2))s" + exit 0 + fi + i=$((i + 1)) + sleep 2 + done + echo "Web FAILED after 30s" + exit 1 +) > "$HEALTH_LOG_DIR/web.log" 2>&1 & +PID_WEB_HEALTH=$! + +set +e +wait $PID_API_HEALTH +API_EXIT=$? +wait $PID_WEB_HEALTH +WEB_EXIT=$? +set -e echo "" -echo "============================================" -echo " ✅ Staging 部署完成" -echo "============================================" +echo "健康检查结果:" +API_OK=0 +WEB_OK=0 +if [ "$API_EXIT" -eq 0 ]; then + echo " OK API: $(cat "$HEALTH_LOG_DIR/api.log")" + API_OK=1 +else + echo " FAIL API: 120s未就绪" + docker logs --tail 50 xiaoxia-api-staging +fi + +if [ "$WEB_EXIT" -eq 0 ]; then + echo " OK Web: $(cat "$HEALTH_LOG_DIR/web.log")" + WEB_OK=1 +else + echo " FAIL Web: 30s未就绪" + docker logs --tail 30 xiaoxia-web-staging +fi + +rm -rf "$HEALTH_LOG_DIR" + +if [ "$API_OK" -eq 0 ] || [ "$WEB_OK" -eq 0 ]; then + echo "" + echo "ERROR: 健康检查失败" + rollback +fi + +echo "Cleaning up old images..." +docker image prune -af --filter "until=168h" 2>/dev/null || true +docker builder prune -af --filter "until=168h" 2>/dev/null || true + +echo "" +echo "=== Staging deployment complete (并行优化版) ===" echo "API: http://127.0.0.1:8000" echo "Web: http://127.0.0.1:3001" echo "Version: $IMAGE_TAG" -echo "" -docker ps --format "table {{.Names}}\t{{.Status}}" | grep staging +docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging diff --git a/scripts/ci_staging_healthcheck.sh b/scripts/ci_staging_healthcheck.sh old mode 100755 new mode 100644 index aa20b0abe..f8609f956 --- a/scripts/ci_staging_healthcheck.sh +++ b/scripts/ci_staging_healthcheck.sh @@ -1,110 +1,473 @@ #!/bin/bash -# ============================================ -# CI Staging 健康检查脚本 - 在 CI Runner 上执行 -# 通过 SSH 连接 Staging 服务器,检查 API/Worker/Web 健康状态 -# ============================================ +# =========================================== +# CI Staging 健康检查 + 自动回滚脚本(SSH 部署模式) +# =========================================== +# +# 在 CI Runner 上执行,通过公网 URL 检查 Staging 部署健康状态。 +# 不健康则通过 SSH 自动回滚到上一个版本的镜像。 +# +# 用法: +# ./ci_staging_healthcheck.sh +# +# 环境变量: +# STAGING_API_URL - Staging API 地址 (默认 https://staging-api.xiaoxiajianji.com) +# STAGING_WEB_URL - Staging Web 地址 (默认 https://staging.xiaoxiajianji.com) +# HEALTH_CHECK_TIMEOUT - 健康检查总超时秒数 (默认 120) +# SKIP_ROLLBACK - 失败时不自动回滚 (true/false, 默认 false) +# SKIP_NOTIFY - 跳过通知 (true/false, 默认 false) +# CI_NOTIFY_WEBHOOK - 通知 Webhook URL +# +# STAGING_SSH_HOST - Staging 服务器 SSH 地址 (默认 47.98.113.167) +# STAGING_SSH_USER - SSH 用户名 (默认 root) +# STAGING_SSH_PORT - SSH 端口 (默认 22222) +# STAGING_SSH_KEY - SSH 私钥内容 +# REGISTRY_TOKEN - Registry Token(回滚时拉取旧镜像需要) +# +# GITHUB_SHA - 当前 commit SHA +# GITHUB_REF_NAME - 分支名 +# GITHUB_RUN_ID - CI Run ID +# GITHUB_REPOSITORY - 仓库名 +# GITHUB_ACTOR - 提交者 set -eu +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" + +# 配置 +STAGING_API_URL="${STAGING_API_URL:-https://staging-api.xiaoxiajianji.com}" +STAGING_WEB_URL="${STAGING_WEB_URL:-https://staging.xiaoxiajianji.com}" +HEALTH_CHECK_TIMEOUT="${HEALTH_CHECK_TIMEOUT:-120}" +SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}" +SKIP_NOTIFY="${SKIP_NOTIFY:-false}" + STAGING_SSH_HOST="${STAGING_SSH_HOST:-47.98.113.167}" STAGING_SSH_USER="${STAGING_SSH_USER:-root}" STAGING_SSH_PORT="${STAGING_SSH_PORT:-22222}" -STAGING_SSH_KEY="${STAGING_SSH_KEY:-}" -HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-120}" -HEALTH_INTERVAL="${HEALTH_INTERVAL:-5}" +REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}" +REGISTRY_USER="${REGISTRY_USER:-xiaoxia}" -echo "============================================" -echo " Staging 健康检查" -echo " Host: $STAGING_SSH_HOST:$STAGING_SSH_PORT" -echo "============================================" -echo "" +# 颜色 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' -# ---- 准备 SSH key ---- -mkdir -p ~/.ssh -key_path="" -if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then - key_path="/root/.ssh/xiaoxia_runtime_builder" -elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then - key_path="$HOME/.ssh/xiaoxia_runtime_builder" -elif [ -n "${STAGING_SSH_KEY:-}" ]; then - key_path="$HOME/.ssh/staging_deploy_key" - printf '%s\n' "$STAGING_SSH_KEY" > "$key_path" - chmod 600 "$key_path" -else - echo "ERROR: 没有可用的 SSH key" - exit 1 +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } +log_step() { echo -e "${BLUE}[STEP]${NC} $1"; } + +# =========================================== +# SSH 工具函数 +# =========================================== +SSH_KEY_PATH="" + +setup_ssh() { + # 查找或创建 SSH 密钥 + if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then + SSH_KEY_PATH="/root/.ssh/xiaoxia_runtime_builder" + elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then + SSH_KEY_PATH="$HOME/.ssh/xiaoxia_runtime_builder" + elif [ -n "${STAGING_SSH_KEY:-}" ]; then + SSH_KEY_PATH="$HOME/.ssh/staging_deploy_key" + mkdir -p "$HOME/.ssh" + printf '%s\n' "$STAGING_SSH_KEY" > "$SSH_KEY_PATH" + chmod 600 "$SSH_KEY_PATH" + else + log_error "没有可用的 SSH 密钥" + return 1 + fi + + ssh-keyscan -p "$STAGING_SSH_PORT" -H "$STAGING_SSH_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true + log_info "SSH 已配置: ${STAGING_SSH_USER}@${STAGING_SSH_HOST}:${STAGING_SSH_PORT}" +} + +run_ssh() { + local cmd="$1" + ssh -p "$STAGING_SSH_PORT" -i "$SSH_KEY_PATH" -o StrictHostKeyChecking=no \ + "${STAGING_SSH_USER}@${STAGING_SSH_HOST}" "$cmd" +} + +# =========================================== +# 1. 记录部署前各服务的镜像版本(用于回滚) +# =========================================== +ROLLBACK_API_TAG="" +ROLLBACK_WORKER_TAG="" +ROLLBACK_WEB_TAG="" + +save_rollback_target() { + log_step "记录当前 staging 各服务镜像版本(回滚目标)..." + + # 通过 SSH 获取当前运行的容器镜像 + local api_image worker_image web_image + api_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-api-staging 2>/dev/null || echo ''") + worker_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-worker-staging 2>/dev/null || echo ''") + web_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-web-staging 2>/dev/null || echo ''") + + # 提取 tag(镜像名是 xiaoxia-saas-api:abc123 或 git.xiaoxiajianji.com/.../xiaoxia-saas-api:staging 格式) + ROLLBACK_API_TAG=$(echo "$api_image" | sed 's/.*://' || echo "") + ROLLBACK_WORKER_TAG=$(echo "$worker_image" | sed 's/.*://' || echo "") + ROLLBACK_WEB_TAG=$(echo "$web_image" | sed 's/.*://' || echo "") + + log_info " API: ${ROLLBACK_API_TAG:-未知}" + log_info " Worker: ${ROLLBACK_WORKER_TAG:-未知}" + log_info " Web: ${ROLLBACK_WEB_TAG:-未知}" + + # 验证三个服务版本是否一致 + if [ -n "$ROLLBACK_API_TAG" ] && [ -n "$ROLLBACK_WORKER_TAG" ] && [ -n "$ROLLBACK_WEB_TAG" ]; then + if [ "$ROLLBACK_API_TAG" = "$ROLLBACK_WORKER_TAG" ] && [ "$ROLLBACK_API_TAG" = "$ROLLBACK_WEB_TAG" ]; then + log_info " ✅ 三个服务版本一致: $ROLLBACK_API_TAG" + export ROLLBACK_TAG="$ROLLBACK_API_TAG" + else + log_warn " ⚠️ 三个服务版本不一致,回滚时将分别使用各自版本" + export ROLLBACK_API_TAG ROLLBACK_WORKER_TAG ROLLBACK_WEB_TAG + export ROLLBACK_TAG_MIXED="true" + fi + else + log_warn " ⚠️ 未能获取全部服务版本,回滚功能可能受限" + fi +} + +# =========================================== +# 2. 健康检查(公网视角) +# =========================================== +health_check() { + local timeout="$HEALTH_CHECK_TIMEOUT" + local start_time + start_time=$(date +%s) + + log_step "公网健康检查(超时 ${timeout}s)..." + log_info " API: ${STAGING_API_URL}/health" + log_info " Web: ${STAGING_WEB_URL}/" + + local api_ok=false + local web_ok=false + local api_docs_ok=false + local login_api_ok=false + + while [ $(( $(date +%s) - start_time )) -lt "$timeout" ]; do + # 检查 API health + if [ "$api_ok" = false ] && curl -sf --max-time 10 "${STAGING_API_URL}/health" >/dev/null 2>&1; then + log_info "✅ API 健康检查通过" + api_ok=true + fi + + # 检查 Web 首页 + if [ "$web_ok" = false ] && curl -sf --max-time 10 "$STAGING_WEB_URL/" >/dev/null 2>&1; then + log_info "✅ Web 前端检查通过" + web_ok=true + fi + + # 检查 API docs(服务完全启动的标志) + if [ "$api_docs_ok" = false ]; then + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "${STAGING_API_URL}/docs" 2>/dev/null || echo "000") + if [ "$HTTP_CODE" = "200" ]; then + log_info "✅ API Docs 检查通过" + api_docs_ok=true + fi + fi + + # 检查登录 API(业务逻辑正常的标志) + if [ "$login_api_ok" = false ]; then + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 -X POST \ + "${STAGING_API_URL}/api/v1/auth/login" \ + -H "Content-Type: application/json" \ + -d '{"email":"smoke@test.com","password":"wrong"}' 2>/dev/null || echo "000") + if [ "$HTTP_CODE" = "401" ] || [ "$HTTP_CODE" = "422" ]; then + log_info "✅ 登录 API 检查通过(HTTP $HTTP_CODE,符合预期)" + login_api_ok=true + fi + fi + + # 都通过了就退出 + if [ "$api_ok" = true ] && [ "$web_ok" = true ] && [ "$api_docs_ok" = true ] && [ "$login_api_ok" = true ]; then + log_info "🎉 所有健康检查通过!" + return 0 + fi + + sleep 5 + done + + # 超时了 + log_error "❌ 健康检查超时 (${timeout}s)" + [ "$api_ok" = false ] && log_error " - API health 未通过" + [ "$web_ok" = false ] && log_error " - Web 前端未通过" + [ "$api_docs_ok" = false ] && log_error " - API Docs 未通过" + [ "$login_api_ok" = false ] && log_error " - 登录 API 未通过" + + return 1 +} + +# =========================================== +# 3. 执行回滚(SSH 重新部署旧版本) +# =========================================== +do_rollback() { + log_step "执行回滚:通过 SSH 重新部署旧版本镜像..." + + local rollback_tag="${ROLLBACK_TAG:-}" + if [ -z "$rollback_tag" ] && [ "${ROLLBACK_TAG_MIXED:-}" != "true" ]; then + log_error "没有可回滚的版本记录,无法自动回滚" + return 1 + fi + + # 如果版本不一致,用 API 的版本作为回滚目标 + if [ -z "$rollback_tag" ]; then + rollback_tag="$ROLLBACK_API_TAG" + fi + + if [ -z "$rollback_tag" ]; then + log_error "无法确定回滚版本" + return 1 + fi + + log_info "回滚目标版本: $rollback_tag" + + # 构建回滚脚本(直接部署旧版本镜像,不跑 migration) + local rollback_script=$(cat << 'ROLLBACK_EOF' +#!/bin/sh +set -eu + +IMAGE_TAG="$1" +REGISTRY_TOKEN="$2" +REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}" +REGISTRY_USER="${REGISTRY_USER:-xiaoxia}" + +ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-staging/.env}" +GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-staging/generated}" +LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-staging/legacy-assets}" + +echo "=== Rollback to $IMAGE_TAG ===" + +# 登录 Registry +if [ -n "$REGISTRY_TOKEN" ]; then + REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1) + printf %s "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin 2>/dev/null || true fi -ssh-keyscan -p "$STAGING_SSH_PORT" -H "$STAGING_SSH_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true +# Pull 旧版本镜像 +LOCAL_API="xiaoxia-saas-api:${IMAGE_TAG}" +LOCAL_WORKER="xiaoxia-saas-worker:${IMAGE_TAG}" +LOCAL_WEB="xiaoxia-saas-web:${IMAGE_TAG}" -SSH_CMD="ssh -p $STAGING_SSH_PORT -i $key_path -o StrictHostKeyChecking=no ${STAGING_SSH_USER}@${STAGING_SSH_HOST}" +docker pull "${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" +docker pull "${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" +docker pull "${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" -# ---- 检查函数 ---- -check_api() { - $SSH_CMD "curl -sf --max-time 5 http://127.0.0.1:8000/health" 2>/dev/null >/dev/null -} +docker tag "${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" "$LOCAL_API" +docker tag "${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" "$LOCAL_WORKER" +docker tag "${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" "$LOCAL_WEB" -check_web() { - $SSH_CMD "curl -sf --max-time 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:3001/" 2>/dev/null | grep -q "200" -} +echo "Rollback images pulled." -check_worker() { - $SSH_CMD "docker inspect --format='{{.State.Status}}' xiaoxia-worker-staging 2>/dev/null | grep -q running" 2>/dev/null -} +# 停止当前容器(回滚不跑 migration,避免数据问题) +docker rm -f xiaoxia-api-staging 2>/dev/null || true +docker rm -f xiaoxia-worker-staging 2>/dev/null || true +docker rm -f xiaoxia-web-staging 2>/dev/null || true -# ---- 循环等待健康 ---- -echo "等待 Staging 环境健康 (超时: ${HEALTH_TIMEOUT}s)..." -elapsed=0 -api_ok=0 -web_ok=0 -worker_ok=0 +LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" -while [ "$elapsed" -lt "$HEALTH_TIMEOUT" ]; do - # 检查 API - if [ "$api_ok" -eq 0 ] && check_api; then - echo " ✅ API 健康" - api_ok=1 +# 启动 API(回滚不跑 migration) +echo "Starting API (rollback)..." +docker run -d \ + --name xiaoxia-api-staging \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -p 127.0.0.1:8000:8000 \ + -e APP_ENV=staging \ + -e APP_VERSION="$IMAGE_TAG" \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 40s \ + $LOG_OPTS \ + "$LOCAL_API" + +# 启动 Worker +echo "Starting Worker (rollback)..." +docker run -d \ + --name xiaoxia-worker-staging \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -e APP_ENV=staging \ + -e APP_VERSION="$IMAGE_TAG" \ + -e WORKER_CONCURRENCY=1 \ + -e WORKER_MAX_TASKS_PER_CHILD=100 \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 30s \ + $LOG_OPTS \ + "$LOCAL_WORKER" + +# 启动 Web +LEGACY_VOLUME="" +if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then + LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" +fi + +echo "Starting Web (rollback)..." +docker run -d \ + --name xiaoxia-web-staging \ + --network xiaoxia-net-staging \ + -p 127.0.0.1:3001:80 \ + --restart unless-stopped \ + $LEGACY_VOLUME \ + --health-cmd "wget --spider -q http://127.0.0.1:80" \ + --health-interval 30s \ + --health-timeout 5s \ + --health-retries 3 \ + $LOG_OPTS \ + "$LOCAL_WEB" + +# 等待 API 健康 +echo "Waiting for API (rollback)..." +i=0 +while [ "$i" -lt 40 ]; do + if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then + echo "API healthy (rollback)." + break fi - - # 检查 Web - if [ "$web_ok" -eq 0 ] && check_web; then - echo " ✅ Web 健康" - web_ok=1 - fi - - # 检查 Worker - if [ "$worker_ok" -eq 0 ] && check_worker; then - echo " ✅ Worker 运行中" - worker_ok=1 - fi - - # 全部通过 - if [ "$api_ok" -eq 1 ] && [ "$web_ok" -eq 1 ] && [ "$worker_ok" -eq 1 ]; then - echo "" - echo "============================================" - echo " ✅ 全部健康检查通过" - echo "============================================" - exit 0 - fi - - sleep "$HEALTH_INTERVAL" - elapsed=$((elapsed + HEALTH_INTERVAL)) - echo " 等待中... (${elapsed}s/${HEALTH_TIMEOUT}s)" + i=$((i + 1)) + sleep 3 done -echo "" -echo "============================================" -echo " ❌ 健康检查超时" -echo "============================================" -echo "" -echo "未通过的检查项:" -[ "$api_ok" -eq 0 ] && echo " - API (http://127.0.0.1:8000/health)" -[ "$web_ok" -eq 0 ] && echo " - Web (http://127.0.0.1:3001/)" -[ "$worker_ok" -eq 0 ] && echo " - Worker 容器" +# 等待 Web 健康 +echo "Waiting for Web (rollback)..." +i=0 +while [ "$i" -lt 15 ]; do + if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then + echo "Web healthy (rollback)." + break + fi + i=$((i + 1)) + sleep 2 +done -echo "" -echo "=== API 最近日志 ===" -$SSH_CMD "docker logs --tail 30 xiaoxia-api-staging" 2>/dev/null || echo "无法获取日志" +echo "=== Rollback complete: $IMAGE_TAG ===" +docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging +ROLLBACK_EOF +) -exit 1 + # 将脚本 base64 编码后通过 SSH 执行 + local script_b64 + script_b64=$(echo "$rollback_script" | base64 -w 0) + + log_info "在 staging 服务器上执行回滚脚本..." + if run_ssh "echo '$script_b64' | base64 -d | sh -s -- '$rollback_tag' '${REGISTRY_TOKEN:-}'" 2>&1; then + log_info "✅ 回滚命令执行完成" + return 0 + else + log_error "❌ 回滚命令执行失败" + return 1 + fi +} + +# =========================================== +# 4. 发送通知 +# =========================================== +send_notification() { + local status="$1" # success / failure / rollback + local detail="$2" + + if [ "${SKIP_NOTIFY:-false}" = "true" ]; then + log_info "跳过通知(SKIP_NOTIFY=true)" + return 0 + fi + + local webhook="${CI_NOTIFY_WEBHOOK:-}" + if [ -z "$webhook" ]; then + log_warn "未配置 CI_NOTIFY_WEBHOOK,跳过通知" + return 0 + fi + + if [ -f "$SCRIPT_DIR/deploy_notify.py" ]; then + python3 "$SCRIPT_DIR/deploy_notify.py" \ + --status "$status" \ + --detail "$detail" \ + --webhook "$webhook" \ + --env staging \ + 2>/dev/null || log_warn "通知发送失败(非致命)" + else + log_warn "找不到 deploy_notify.py,跳过通知" + fi +} + +# =========================================== +# 主流程 +# =========================================== +main() { + echo "" + echo "===========================================" + echo " CI Staging 健康检查 + 自动回滚(SSH模式)" + echo "===========================================" + echo "" + + local deploy_status="success" + local deploy_detail="" + + # 1. 设置 SSH + if ! setup_ssh; then + log_error "SSH 配置失败,无法执行回滚" + fi + + # 2. 记录部署前状态(回滚目标) + save_rollback_target || true + + # 3. 健康检查(公网视角) + if ! health_check; then + log_error "健康检查失败" + deploy_status="failure" + deploy_detail="公网健康检查超时,部署后服务未正常响应" + + # 自动回滚 + if [ "${SKIP_ROLLBACK:-false}" != "true" ]; then + log_warn "开始自动回滚..." + if do_rollback; then + deploy_status="rollback" + deploy_detail="健康检查失败,已自动回滚到上一版本 (${ROLLBACK_TAG:-未知})" + + # 回滚后再检查一下公网状态 + log_info "回滚完成,重新检查公网健康状态..." + if health_check; then + log_info "✅ 回滚后服务已恢复" + deploy_detail="${deploy_detail},回滚后服务已恢复" + else + log_error "⚠️ 回滚后健康检查仍未通过,请手动排查" + deploy_detail="${deploy_detail},但回滚后仍未恢复,请紧急排查" + fi + else + deploy_detail="健康检查失败且回滚失败,请手动排查" + fi + fi + + send_notification "$deploy_status" "$deploy_detail" + + # 失败时退出非零,让 CI Job 标记为失败 + exit 1 + fi + + # 4. 成功 + log_info "" + log_info "==================================" + log_info " ✅ Staging 部署成功!" + log_info "==================================" + + deploy_detail="部署成功,所有健康检查通过 (${GITHUB_SHA:-未知版本})" + send_notification "success" "$deploy_detail" +} + +main "$@"