diff --git a/scripts/ci/run_validate.sh b/scripts/ci/run_validate.sh index 543f1848f..da70cf245 100755 --- a/scripts/ci/run_validate.sh +++ b/scripts/ci/run_validate.sh @@ -1,29 +1,75 @@ #!/bin/bash -# CI Validate Job 主脚本:代码质量全量检查 -# 包含:密钥扫描、格式检查、类型检查、安全扫描、依赖漏洞检查、死代码检测、Alembic迁移验证 +# CI Validate Job 主脚本:并行化代码质量检查 +# 将 8 项检查分为 2 组并行执行,预计耗时从 ~1.8min 降至 ~1min +# +# 并行分组: +# Group A(独立并行): +# A1: Secret detection (detect-secrets) +# A2: Code quality checks (black/isort/ruff/compileall) +# A3: Mypy type check +# A4: Advisory checks (bandit + pip-audit + vulture + release scripts syntax) +# Group B(PG 依赖,独立并行): +# B1: Alembic migrations validation(需要 PG) +# +# 所有子任务同时启动,最后汇总结果。 set -eu -echo "=== CI Validate: 开始全量代码质量检查 ===" - -# --- 密钥检测 --- +echo "=== CI Validate: 并行化代码质量检查 ===" echo "" -echo "=== [1/8] Secret detection (detect-secrets) ===" -python3 -m pip install -q detect-secrets -detect-secrets --version -detect-secrets scan \ - --all-files \ - --exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \ - --exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \ - --exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \ - --disable-plugin Base64HighEntropyString \ - --disable-plugin HexHighEntropyString \ - --disable-plugin BasicAuthDetector \ - --disable-plugin KeywordDetector \ - --disable-plugin IPPublicDetector \ - > /tmp/secrets-scan.json 2>&1 +# ============================================================ +# 配置 +# ============================================================ +LOG_DIR="/tmp/validate_logs" +rm -rf "$LOG_DIR" +mkdir -p "$LOG_DIR" -FOUND=$(python3 -c " +# 子任务结果文件(每个记录 exit code) +RESULT_FILE="$LOG_DIR/results.json" +echo '{}' > "$RESULT_FILE" + +# ============================================================ +# 工具函数 +# ============================================================ + +# 记录子任务结果 +# 用法: record_result +record_result() { + local name="$1" + local exit_code="$2" + local blocking="$3" # "yes" or "no" + # 写入独立文件,避免并发写 JSON 冲突 + echo "${exit_code}" > "$LOG_DIR/exit_${name}" + echo "${blocking}" > "$LOG_DIR/blocking_${name}" +} + +# ============================================================ +# 子任务定义(每个子任务输出写入独立日志文件) +# ============================================================ + +# --- A1: Secret detection --- +task_secret_detection() { + local log="$LOG_DIR/task_secret_detection.log" + exec > "$log" 2>&1 + set +e + + echo "=== [A1] Secret detection (detect-secrets) ===" + python3 -m pip install -q detect-secrets + detect-secrets --version + + detect-secrets scan \ + --all-files \ + --exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \ + --exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \ + --exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \ + --disable-plugin Base64HighEntropyString \ + --disable-plugin HexHighEntropyString \ + --disable-plugin BasicAuthDetector \ + --disable-plugin KeywordDetector \ + --disable-plugin IPPublicDetector \ + > /tmp/secrets-scan.json 2>&1 + + FOUND=$(python3 -c " import json try: with open('/tmp/secrets-scan.json') as f: @@ -35,11 +81,12 @@ except Exception: print('error') ") -echo "Secrets detected: $FOUND" -if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then - echo "" - echo "=== Secret details ===" - python3 -c " + echo "Secrets detected: $FOUND" + local exit_code=0 + if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then + echo "" + echo "=== Secret details ===" + python3 -c " import json with open('/tmp/secrets-scan.json') as f: data = json.load(f) @@ -50,28 +97,37 @@ for fpath, items in data.get('results', {}).items(): hashed = item.get('hashed_secret', '')[:16] print(f' {fpath}:{line} [{stype}] {hashed}...') " - echo "" - echo "ERROR: Potential secrets detected in code!" - exit 1 -fi -echo "✅ Secret scan passed" + echo "" + echo "ERROR: Potential secrets detected in code!" + exit_code=1 + else + echo "✅ Secret scan passed" + fi -# --- 增量/全量模式判断 --- -echo "" -echo "=== [2/8] Code quality checks ===" -SCAN_MODE="full" -CHANGED_PY_FILES="" + record_result "secret_detection" "$exit_code" "yes" + exit $exit_code +} -if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then - PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') - API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100" +# --- A2: Code quality checks --- +task_code_quality() { + local log="$LOG_DIR/task_code_quality.log" + exec > "$log" 2>&1 set +e - RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL") - HTTP_CODE=$(echo "$RESPONSE" | tail -n1) - BODY=$(echo "$RESPONSE" | sed '$d') - set -e - if [ "$HTTP_CODE" = "200" ]; then - CHANGED_PY_FILES=$(echo "$BODY" | python3 -c " + + echo "=== [A2] Code quality checks (black/isort/ruff/compileall) ===" + + # --- 增量/全量模式判断 --- + local SCAN_MODE="full" + local CHANGED_PY_FILES="" + + if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then + PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') + API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100" + RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL") + HTTP_CODE=$(echo "$RESPONSE" | tail -n1) + BODY=$(echo "$RESPONSE" | sed '$d') + if [ "$HTTP_CODE" = "200" ]; then + CHANGED_PY_FILES=$(echo "$BODY" | python3 -c " import json, sys try: files = json.load(sys.stdin) @@ -80,160 +136,205 @@ try: except Exception: print('') ") - if [ -n "$CHANGED_PY_FILES" ]; then - SCAN_MODE="incremental" - echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed" + if [ -n "$CHANGED_PY_FILES" ]; then + SCAN_MODE="incremental" + echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed" + else + SCAN_MODE="skip_py" + echo "No Python files changed in this PR" + fi else - SCAN_MODE="skip_py" - echo "No Python files changed in this PR" + echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan" fi else - echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan" + echo "Full scan mode (not a PR event)" fi -else - echo "Full scan mode (not a PR event)" -fi -if [ "$SCAN_MODE" = "incremental" ]; then - # 防御性过滤:磁盘上不存在的文件(已删除文件)不参与检查, - # 避免 black/isort/ruff 报 "Path does not exist" 错误。 - EXISTING_PY_FILES="" - for f in $CHANGED_PY_FILES; do - if [ -f "$f" ]; then - if [ -z "$EXISTING_PY_FILES" ]; then - EXISTING_PY_FILES="$f" + local exit_code=0 + + if [ "$SCAN_MODE" = "incremental" ]; then + # 防御性过滤:磁盘上不存在的文件(已删除文件)不参与检查 + local EXISTING_PY_FILES="" + for f in $CHANGED_PY_FILES; do + if [ -f "$f" ]; then + if [ -z "$EXISTING_PY_FILES" ]; then + EXISTING_PY_FILES="$f" + else + EXISTING_PY_FILES="$EXISTING_PY_FILES $f" + fi + fi + done + CHANGED_PY_FILES="$EXISTING_PY_FILES" + + python3 -m compileall -q $CHANGED_PY_FILES || exit_code=$? + if [ $exit_code -eq 0 ]; then + python3 -m black --check --fast $CHANGED_PY_FILES || exit_code=$? + fi + if [ $exit_code -eq 0 ]; then + python3 -m isort --check-only $CHANGED_PY_FILES || exit_code=$? + fi + if [ $exit_code -eq 0 ]; then + local RUFF_FILES + RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | grep -v '^$' | xargs) + if [ -n "$RUFF_FILES" ]; then + python3 -m ruff check $RUFF_FILES --statistics || exit_code=$? else - EXISTING_PY_FILES="$EXISTING_PY_FILES $f" + echo "No ruff-checkable files changed, skipping" fi fi - done - CHANGED_PY_FILES="$EXISTING_PY_FILES" - - python3 -m compileall -q $CHANGED_PY_FILES - python3 -m black --check --fast $CHANGED_PY_FILES - python3 -m isort --check-only $CHANGED_PY_FILES - RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | grep -v '^$' | xargs) - if [ -n "$RUFF_FILES" ]; then - python3 -m ruff check $RUFF_FILES --statistics + elif [ "$SCAN_MODE" = "skip_py" ]; then + echo "No Python files changed - skipping Python lint checks" else - echo "No ruff-checkable files changed, skipping" - fi -elif [ "$SCAN_MODE" = "skip_py" ]; then - echo "No Python files changed - skipping Python lint checks" -else - echo "Full scan mode" - python3 -m compileall -q alembic apps packages tests scripts - python3 -m black --check --fast alembic apps packages tests scripts - python3 -m isort --check-only alembic apps packages tests scripts - python3 -m ruff check apps packages tests --statistics -fi -echo "✅ Code quality checks passed" - -# --- Mypy 类型检查 --- -echo "" -echo "=== [3/8] Type check (mypy) ===" -bash scripts/ci/mypy_check.sh -echo "✅ Mypy type check passed" - -# --- Bandit 安全扫描(仅告警) --- -echo "" -echo "=== [4/8] Security scan (bandit, advisory only) ===" -set +e -bandit -r apps packages -q -ll -BANDIT_EXIT=$? -set -e -if [ "$BANDIT_EXIT" -ne 0 ]; then - echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)" -else - echo "✅ Bandit security scan passed" -fi - -# --- Pip-audit 依赖漏洞扫描(仅告警) --- -echo "" -echo "=== [5/8] Python dependency vulnerability scan (pip-audit, advisory only) ===" -python3 -m pip install -q pip-audit -pip-audit --version -EXIT_CODE=0 -for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do - if [ -f "$req_file" ]; then - echo "--- Scanning $req_file ---" - pip-audit -r "$req_file" --desc on 2>&1 | head -40 || EXIT_CODE=$? - echo "" - fi -done -echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)" - -# --- Vulture 死代码检测(仅告警) --- -echo "" -echo "=== [6/8] Dead code detection (vulture, advisory only) ===" -set +e -python3 -m pip install -q vulture -vulture --version -echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。" -echo "" -vulture apps packages scripts \ - --exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \ - --min-confidence 70 \ - 2>&1 | sort -t'(' -k2 -rn | head -80 -echo "" -echo "=== vulture scan summary ===" -echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)" -echo "建议:定期人工审查高置信度(>=90%)条目" -set -e - -# --- Release 脚本语法校验 --- -echo "" -echo "=== [7/8] Release scripts syntax validation ===" -bash -n scripts/backup_postgres.sh -bash -n scripts/restore_postgres_plan.sh -bash -n scripts/init_production_env.sh -echo "✅ Release scripts syntax OK" - -# --- Alembic 迁移验证 --- -echo "" -echo "=== [8/8] Alembic migrations validation ===" - -# --- DooD模式检测:确定宿主机访问地址 --- -# DooD模式下,docker run启动的容器跑在宿主机Docker上 -# 需要用宿主机IP访问映射端口 -# 检测策略:host.docker.internal -> docker0桥接IP -> 容器IP直连 -> 默认网关 -> 127.0.0.1 -detect_docker_host() { - local test_port="${1:-5432}" - - # 候选IP列表 - local candidates=() - - # 1. host.docker.internal(runner配置了--add-host时可用) - if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then - candidates+=("host.docker.internal") + echo "Full scan mode" + python3 -m compileall -q alembic apps packages tests scripts || exit_code=$? + if [ $exit_code -eq 0 ]; then + python3 -m black --check --fast alembic apps packages tests scripts || exit_code=$? + fi + if [ $exit_code -eq 0 ]; then + python3 -m isort --check-only alembic apps packages tests scripts || exit_code=$? + fi + if [ $exit_code -eq 0 ]; then + python3 -m ruff check apps packages tests --statistics || exit_code=$? + fi fi - # 2. docker0 桥接网关 (172.17.0.1) - candidates+=("172.17.0.1") - - # 3. 默认网关(容器网络的网关即宿主机) - local gw="" - gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1) - if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then - candidates+=("$gw") + if [ $exit_code -eq 0 ]; then + echo "✅ Code quality checks passed" + else + echo "❌ Code quality checks FAILED" fi - # 4. 宿主机可能的IP:容器同网段的.1或.254 - local my_ip="" - my_ip=$(hostname -I 2>/dev/null | awk '{print $1}') - if [ -n "$my_ip" ]; then - # 尝试同网段的常见宿主机IP - local subnet=$(echo "$my_ip" | cut -d. -f1-3) - candidates+=("${subnet}.1") - candidates+=("${subnet}.254") + record_result "code_quality" "$exit_code" "yes" + exit $exit_code +} + +# --- A3: Mypy type check --- +task_mypy() { + local log="$LOG_DIR/task_mypy.log" + exec > "$log" 2>&1 + set +e + + echo "=== [A3] Type check (mypy) ===" + bash scripts/ci/mypy_check.sh + local exit_code=$? + + if [ $exit_code -eq 0 ]; then + echo "✅ Mypy type check passed" + else + echo "❌ Mypy type check FAILED" fi - # 5. 127.0.0.1 最后尝试 - candidates+=("127.0.0.1") + record_result "mypy" "$exit_code" "yes" + exit $exit_code +} - # 测试每个候选IP - for candidate in "${candidates[@]}"; do - if python3 -c " +# --- A4: Advisory checks (bandit + pip-audit + vulture + release scripts syntax) --- +task_advisory() { + local log="$LOG_DIR/task_advisory.log" + exec > "$log" 2>&1 + set +e + + # --- Bandit 安全扫描(仅告警) --- + echo "=== [A4a] Security scan (bandit, advisory only) ===" + bandit -r apps packages -q -ll + local BANDIT_EXIT=$? + if [ "$BANDIT_EXIT" -ne 0 ]; then + echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)" + else + echo "✅ Bandit security scan passed" + fi + + # --- Pip-audit 依赖漏洞扫描(仅告警) --- + echo "" + echo "=== [A4b] Python dependency vulnerability scan (pip-audit, advisory only) ===" + python3 -m pip install -q pip-audit + pip-audit --version + for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do + if [ -f "$req_file" ]; then + echo "--- Scanning $req_file ---" + pip-audit -r "$req_file" --desc on 2>&1 | head -40 || true + echo "" + fi + done + echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)" + + # --- Vulture 死代码检测(仅告警) --- + echo "" + echo "=== [A4c] Dead code detection (vulture, advisory only) ===" + python3 -m pip install -q vulture + vulture --version + echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。" + echo "" + vulture apps packages scripts \ + --exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \ + --min-confidence 70 \ + 2>&1 | sort -t'(' -k2 -rn | head -80 + echo "" + echo "=== vulture scan summary ===" + echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)" + echo "建议:定期人工审查高置信度(>=90%)条目" + + # --- Release 脚本语法校验(不阻断) --- + echo "" + echo "=== [A4d] Release scripts syntax validation ===" + local syntax_exit=0 + bash -n scripts/backup_postgres.sh || syntax_exit=$? + bash -n scripts/restore_postgres_plan.sh || syntax_exit=$? + bash -n scripts/init_production_env.sh || syntax_exit=$? + if [ $syntax_exit -eq 0 ]; then + echo "✅ Release scripts syntax OK" + else + echo "⚠️ Release scripts have syntax issues (advisory)" + fi + + # Advisory checks never block + record_result "advisory" 0 "no" + exit 0 +} + +# --- B1: Alembic migrations validation (needs PG) --- +task_alembic() { + local log="$LOG_DIR/task_alembic.log" + exec > "$log" 2>&1 + set +e + + echo "=== [B1] Alembic migrations validation ===" + + # --- DooD模式检测:确定宿主机访问地址 --- + detect_docker_host() { + local test_port="${1:-5432}" + local candidates=() + + # 1. host.docker.internal + if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then + candidates+=("host.docker.internal") + fi + + # 2. docker0 桥接网关 (172.17.0.1) + candidates+=("172.17.0.1") + + # 3. 默认网关 + local gw="" + gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1) + if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then + candidates+=("$gw") + fi + + # 4. 宿主机可能的IP + local my_ip="" + my_ip=$(hostname -I 2>/dev/null | awk '{print $1}') + if [ -n "$my_ip" ]; then + local subnet + subnet=$(echo "$my_ip" | cut -d. -f1-3) + candidates+=("${subnet}.1") + candidates+=("${subnet}.254") + fi + + # 5. 127.0.0.1 + candidates+=("127.0.0.1") + + for candidate in "${candidates[@]}"; do + if python3 -c " import socket s = socket.socket() s.settimeout(2) @@ -244,69 +345,65 @@ try: except: pass " 2>/dev/null | grep -q ok; then - echo "$candidate" - return 0 + echo "$candidate" + return 0 + fi + done + + echo "127.0.0.1" + return 1 + } + + # 指数退避TCP连接检查函数 + wait_tcp_ready() { + local host="$1" + local port="$2" + local max_attempts="${3:-5}" + local delay=1 + local attempt=1 + while [ "$attempt" -le "$max_attempts" ]; do + if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then + return 0 + fi + echo "TCP连接尝试 $attempt/$max_attempts 失败,${delay}s后重试..." + sleep "$delay" + delay=$((delay * 2)) + attempt=$((attempt + 1)) + done + return 1 + } + + # 获取宿主机IP + local PG_HOST + if [ -S /var/run/docker.sock ]; then + PG_HOST=$(detect_docker_host 5433) + if [ "$PG_HOST" = "127.0.0.1" ]; then + PG_HOST=$(detect_docker_host 22) fi - done - - # 都失败则返回127.0.0.1 - echo "127.0.0.1" - return 1 -} - -# 获取宿主机IP(先尝试用共享PG端口5433测试,再回退到其他端口) -if [ -S /var/run/docker.sock ]; then - # 先用共享PG端口5433探测 - DOCKER_HOST_IP=$(detect_docker_host 5433) - if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then - # 如果共享PG端口探测失败,说明不在DooD或共享PG不可用,再试其他端口 - DOCKER_HOST_IP=$(detect_docker_host 22) + echo "检测到DooD模式(/var/run/docker.sock已挂载),宿主机地址: $PG_HOST" + else + PG_HOST="127.0.0.1" + echo "非DooD模式,使用 127.0.0.1" fi - echo "检测到DooD模式(/var/run/docker.sock已挂载),宿主机地址: $DOCKER_HOST_IP" -else - DOCKER_HOST_IP="127.0.0.1" - echo "非DooD模式,使用 127.0.0.1" -fi -PG_HOST="$DOCKER_HOST_IP" -echo "PG host: $PG_HOST" + echo "PG host: $PG_HOST" -# 指数退避TCP连接检查函数 -# 用法: wait_tcp_ready host port max_attempts -wait_tcp_ready() { - local host="$1" - local port="$2" - local max_attempts="${3:-5}" - local delay=1 - local attempt=1 - while [ "$attempt" -le "$max_attempts" ]; do - if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then - return 0 - fi - echo "TCP连接尝试 $attempt/$max_attempts 失败,${delay}s后重试..." - sleep "$delay" - delay=$((delay * 2)) - attempt=$((attempt + 1)) - done - return 1 -} + local USE_SHARED_PG="${CI_USE_SHARED_PG:-false}" + local exit_code=0 -USE_SHARED_PG="${CI_USE_SHARED_PG:-false}" + if [ "$USE_SHARED_PG" = "true" ]; then + # 使用常驻共享PG实例 + echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true)" + local SHARED_PG_HOST="$PG_HOST" + local SHARED_PG_PORT="5433" + local SHARED_PG_USER="postgres" + local SHARED_PG_PASSWORD="ci_pg_2026!" + local CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}" -if [ "$USE_SHARED_PG" = "true" ]; then - # 使用常驻共享PG实例(host.docker.internal:5433) - echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true)" - SHARED_PG_HOST="$PG_HOST" - SHARED_PG_PORT="5433" - SHARED_PG_USER="postgres" - SHARED_PG_PASSWORD="ci_pg_2026!" - CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}" + echo "等待共享PG连接就绪..." + wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5 - echo "等待共享PG连接就绪..." - wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5 - - # 创建独立数据库 - echo "创建测试数据库: $CI_DB_NAME" - PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c " + echo "创建测试数据库: $CI_DB_NAME" + PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c " import psycopg2 conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres') conn.autocommit = True @@ -314,17 +411,20 @@ cur = conn.cursor() cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"') cur.close() conn.close() -" - export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}" - echo "✅ 共享PG数据库已创建: $CI_DB_NAME" +" || exit_code=$? - # 执行迁移 - PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head - echo "✅ Alembic migrations applied successfully" + if [ $exit_code -eq 0 ]; then + export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}" + echo "✅ 共享PG数据库已创建: $CI_DB_NAME" - # 清理数据库 - echo "清理测试数据库: $CI_DB_NAME" - PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c " + PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head || exit_code=$? + if [ $exit_code -eq 0 ]; then + echo "✅ Alembic migrations applied successfully" + fi + + # 清理数据库 + echo "清理测试数据库: $CI_DB_NAME" + PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c " import psycopg2 conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres') conn.autocommit = True @@ -333,49 +433,220 @@ cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)') cur.close() conn.close() " 2>/dev/null || echo "WARN: 数据库清理失败(可能已被清理)" - echo "✅ 共享PG数据库已清理" -else - # 使用临时PG容器(默认模式) - echo "使用临时PG容器模式" - PG_CONTAINER=ci-pg-validate-${GITHUB_RUN_ID:-$$} - docker rm -f "$PG_CONTAINER" 2>/dev/null || true - docker run -d --name "$PG_CONTAINER" \ - --shm-size=256m \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=xiaoxia_saas \ - -P \ - --health-cmd "pg_isready -U postgres" \ - --health-interval 3s \ - --health-timeout 3s \ - --health-retries 20 \ - postgres:16-alpine - PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2) - echo "PostgreSQL port: $PG_PORT" - export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas" - - # 等待容器健康 - for i in $(seq 1 30); do - if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then - echo "PostgreSQL container is healthy on port $PG_PORT" - break + echo "✅ 共享PG数据库已清理" fi - echo "Waiting for PostgreSQL container health... ($i/30)" - sleep 2 - done - docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy - # TCP连通性检查(指数退避) - echo "验证TCP连通性 ($PG_HOST:$PG_PORT)..." - wait_tcp_ready "$PG_HOST" "$PG_PORT" 5 - echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT" + else + # 使用临时PG容器 + echo "使用临时PG容器模式" + local PG_CONTAINER="ci-pg-validate-${GITHUB_RUN_ID:-$$}" + docker rm -f "$PG_CONTAINER" 2>/dev/null || true + docker run -d --name "$PG_CONTAINER" \ + --shm-size=256m \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=xiaoxia_saas \ + -P \ + --health-cmd "pg_isready -U postgres" \ + --health-interval 3s \ + --health-timeout 3s \ + --health-retries 20 \ + postgres:16-alpine || exit_code=$? - # 执行迁移 - PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head - echo "✅ Alembic migrations applied successfully" + if [ $exit_code -eq 0 ]; then + local PG_PORT + PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2) + echo "PostgreSQL port: $PG_PORT" + export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas" - docker rm -f "$PG_CONTAINER" 2>/dev/null || true -fi + # 等待容器健康 + local i + for i in $(seq 1 30); do + if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then + echo "PostgreSQL container is healthy on port $PG_PORT" + break + fi + echo "Waiting for PostgreSQL container health... ($i/30)" + sleep 2 + done + + if ! docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then + echo "❌ PostgreSQL container failed health check" + exit_code=1 + else + # TCP连通性检查 + echo "验证TCP连通性 ($PG_HOST:$PG_PORT)..." + if wait_tcp_ready "$PG_HOST" "$PG_PORT" 5; then + echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT" + + # 执行迁移 + PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head || exit_code=$? + if [ $exit_code -eq 0 ]; then + echo "✅ Alembic migrations applied successfully" + fi + else + echo "❌ TCP connectivity to PostgreSQL failed" + exit_code=1 + fi + fi + + # 清理 + docker rm -f "$PG_CONTAINER" 2>/dev/null || true + fi + fi + + if [ $exit_code -eq 0 ]; then + echo "✅ Alembic migrations validation passed" + else + echo "❌ Alembic migrations validation FAILED" + fi + + record_result "alembic" "$exit_code" "yes" + exit $exit_code +} + +# ============================================================ +# 主流程:并行启动所有子任务 +# ============================================================ + +echo "启动并行检查(5 个子任务同时运行)..." +echo "" + +# 记录开始时间 +START_TIME=$(date +%s) + +# 启动所有子任务(后台运行) +task_secret_detection & +PID_A1=$! + +task_code_quality & +PID_A2=$! + +task_mypy & +PID_A3=$! + +task_advisory & +PID_A4=$! + +task_alembic & +PID_B1=$! + +echo "子任务 PID: A1=$PID_A1 A2=$PID_A2 A3=$PID_A3 A4=$PID_A4 B1=$PID_B1" +echo "" + +# 等待所有后台任务完成(不因单个失败而中断) +# 使用 set +e 临时取消 errexit +set +e +wait $PID_A1; EXIT_A1=$? +wait $PID_A2; EXIT_A2=$? +wait $PID_A3; EXIT_A3=$? +wait $PID_A4; EXIT_A4=$? +wait $PID_B1; EXIT_B1=$? +set -e + +# 计算耗时 +END_TIME=$(date +%s) +ELAPSED=$((END_TIME - START_TIME)) + +# ============================================================ +# 结果汇总 +# ============================================================ echo "" -echo "=== CI Validate: 所有检查通过 ✅ ===" +echo "============================================" +echo " CI Validate 结果汇总(耗时 ${ELAPSED}s)" +echo "============================================" +echo "" + +# 定义任务信息:名称 | PID | 退出码 | 描述 | 是否阻断 +declare -A TASK_DESC +TASK_DESC[A1]="Secret detection" +TASK_DESC[A2]="Code quality (black/isort/ruff)" +TASK_DESC[A3]="Mypy type check" +TASK_DESC[A4]="Advisory (bandit/pip-audit/vulture/syntax)" +TASK_DESC[B1]="Alembic migrations" + +declare -A TASK_PID +TASK_PID[A1]=$PID_A1 +TASK_PID[A2]=$PID_A2 +TASK_PID[A3]=$PID_A3 +TASK_PID[A4]=$PID_A4 +TASK_PID[B1]=$PID_B1 + +declare -A TASK_EXIT +TASK_EXIT[A1]=$EXIT_A1 +TASK_EXIT[A2]=$EXIT_A2 +TASK_EXIT[A3]=$EXIT_A3 +TASK_EXIT[A4]=$EXIT_A4 +TASK_EXIT[B1]=$EXIT_B1 + +declare -A TASK_LOG +TASK_LOG[A1]="task_secret_detection" +TASK_LOG[A2]="task_code_quality" +TASK_LOG[A3]="task_mypy" +TASK_LOG[A4]="task_advisory" +TASK_LOG[B1]="task_alembic" + +declare -A TASK_BLOCKING +TASK_BLOCKING[A1]="yes" +TASK_BLOCKING[A2]="yes" +TASK_BLOCKING[A3]="yes" +TASK_BLOCKING[A4]="no" +TASK_BLOCKING[B1]="yes" + +OVERALL_EXIT=0 +FAILED_TASKS=() + +# 按固定顺序打印摘要 +for task_id in A1 A2 A3 A4 B1; do + local_exit=${TASK_EXIT[$task_id]} + local_desc=${TASK_DESC[$task_id]} + local_blocking=${TASK_BLOCKING[$task_id]} + + if [ "$local_exit" -eq 0 ]; then + echo " ✅ $task_id: $local_desc — PASSED" + else + if [ "$local_blocking" = "yes" ]; then + echo " ❌ $task_id: $local_desc — FAILED (blocking)" + OVERALL_EXIT=1 + FAILED_TASKS+=("$task_id") + else + echo " ⚠️ $task_id: $local_desc — FAILED (advisory, not blocking)" + # Advisory tasks don't cause overall failure + if [ "$local_blocking" = "no" ]; then + echo " → 告警类检查,不阻断流水线" + fi + fi + fi +done + +echo "" + +# 打印失败任务的完整日志 +if [ ${#FAILED_TASKS[@]} -gt 0 ]; then + echo "============================================" + echo " 失败任务详细日志" + echo "============================================" + for task_id in "${FAILED_TASKS[@]}"; do + local_log="${TASK_LOG[$task_id]}" + local_desc="${TASK_DESC[$task_id]}" + echo "" + echo "--- $task_id: $local_desc ---" + if [ -f "$LOG_DIR/${local_log}.log" ]; then + cat "$LOG_DIR/${local_log}.log" + else + echo "(日志文件不存在)" + fi + echo "" + done +fi + +# 最终结论 +echo "" +if [ $OVERALL_EXIT -eq 0 ]; then + echo "=== CI Validate: 所有检查通过 ✅ (并行耗时 ${ELAPSED}s) ===" +else + echo "=== CI Validate: 存在阻断性检查失败 ❌ (并行耗时 ${ELAPSED}s) ===" +fi + +exit $OVERALL_EXIT