xiaoxia-saas/scripts/ci/run_validate.sh

#!/bin/bash
# CI Validate Job 主脚本：代码质量全量检查
# 包含：密钥扫描、格式检查、类型检查、安全扫描、依赖漏洞检查、死代码检测、Alembic迁移验证
set -eu

echo "=== CI Validate: 开始全量代码质量检查 ==="

# --- 密钥检测 ---
echo ""
echo "=== [1/8] Secret detection (detect-secrets) ==="
python3 -m pip install -q detect-secrets
detect-secrets --version

detect-secrets scan \
  --all-files \
  --exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \
  --exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \
  --exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \
  --disable-plugin Base64HighEntropyString \
  --disable-plugin HexHighEntropyString \
  --disable-plugin BasicAuthDetector \
  --disable-plugin KeywordDetector \
  --disable-plugin IPPublicDetector \
  > /tmp/secrets-scan.json 2>&1

FOUND=$(python3 -c "
import json
try:
    with open('/tmp/secrets-scan.json') as f:
        data = json.load(f)
    results = data.get('results', {})
    total = sum(len(v) for v in results.values())
    print(total)
except Exception:
    print('error')
")

echo "Secrets detected: $FOUND"
if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then
  echo ""
  echo "=== Secret details ==="
  python3 -c "
import json
with open('/tmp/secrets-scan.json') as f:
    data = json.load(f)
for fpath, items in data.get('results', {}).items():
    for item in items:
        line = item.get('line_number', '?')
        stype = item.get('type', '?')
        hashed = item.get('hashed_secret', '')[:16]
        print(f'  {fpath}:{line} [{stype}] {hashed}...')
"
  echo ""
  echo "ERROR: Potential secrets detected in code!"
  exit 1
fi
echo "✅ Secret scan passed"

# --- 增量/全量模式判断 ---
echo ""
echo "=== [2/8] Code quality checks ==="
SCAN_MODE="full"
CHANGED_PY_FILES=""

if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then
  PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||')
  API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100"
  set +e
  RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL")
  HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
  BODY=$(echo "$RESPONSE" | sed '$d')
  set -e
  if [ "$HTTP_CODE" = "200" ]; then
    CHANGED_PY_FILES=$(echo "$BODY" | python3 -c "
import json, sys
try:
    files = json.load(sys.stdin)
    py_files = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] != 'removed']
    print(' '.join(py_files))
except Exception:
    print('')
")
    if [ -n "$CHANGED_PY_FILES" ]; then
      SCAN_MODE="incremental"
      echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed"
    else
      SCAN_MODE="skip_py"
      echo "No Python files changed in this PR"
    fi
  else
    echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan"
  fi
else
  echo "Full scan mode (not a PR event)"
fi

if [ "$SCAN_MODE" = "incremental" ]; then
  python3 -m compileall -q $CHANGED_PY_FILES
  python3 -m black --check --fast $CHANGED_PY_FILES
  python3 -m isort --check-only $CHANGED_PY_FILES
  RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | tr '\n' ' ')
  if [ -n "$RUFF_FILES" ]; then
    python3 -m ruff check $RUFF_FILES --statistics
  else
    echo "No ruff-checkable files changed, skipping"
  fi
elif [ "$SCAN_MODE" = "skip_py" ]; then
  echo "No Python files changed - skipping Python lint checks"
else
  echo "Full scan mode"
  python3 -m compileall -q alembic apps packages tests scripts
  python3 -m black --check --fast alembic apps packages tests scripts
  python3 -m isort --check-only alembic apps packages tests scripts
  python3 -m ruff check apps packages tests --statistics
fi
echo "✅ Code quality checks passed"

# --- Mypy 类型检查 ---
echo ""
echo "=== [3/8] Type check (mypy) ==="
bash scripts/ci/mypy_check.sh
echo "✅ Mypy type check passed"

# --- Bandit 安全扫描（仅告警） ---
echo ""
echo "=== [4/8] Security scan (bandit, advisory only) ==="
set +e
bandit -r apps packages -q -ll
BANDIT_EXIT=$?
set -e
if [ "$BANDIT_EXIT" -ne 0 ]; then
  echo "⚠️  Bandit found security issues (advisory mode - not blocking CI)"
else
  echo "✅ Bandit security scan passed"
fi

# --- Pip-audit 依赖漏洞扫描（仅告警） ---
echo ""
echo "=== [5/8] Python dependency vulnerability scan (pip-audit, advisory only) ==="
python3 -m pip install -q pip-audit
pip-audit --version
EXIT_CODE=0
for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do
  if [ -f "$req_file" ]; then
    echo "--- Scanning $req_file ---"
    pip-audit -r "$req_file" --desc on 2>&1 | head -40 || EXIT_CODE=$?
    echo ""
  fi
done
echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)"

# --- Vulture 死代码检测（仅告警） ---
echo ""
echo "=== [6/8] Dead code detection (vulture, advisory only) ==="
set +e
python3 -m pip install -q vulture
vulture --version
echo "告警模式，不阻断CI。置信度>=90%建议尽快确认。"
echo ""
vulture apps packages scripts \
  --exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \
  --min-confidence 70 \
  2>&1 | sort -t'(' -k2 -rn | head -80
echo ""
echo "=== vulture scan summary ==="
echo "发现潜在死代码（可能包含框架装饰器注册的函数，为误报）"
echo "建议：定期人工审查高置信度（>=90%）条目"
set -e

# --- Release 脚本语法校验 ---
echo ""
echo "=== [7/8] Release scripts syntax validation ==="
bash -n scripts/backup_postgres.sh
bash -n scripts/restore_postgres_plan.sh
bash -n scripts/init_production_env.sh
echo "✅ Release scripts syntax OK"

# --- Alembic 迁移验证 ---
echo ""
echo "=== [8/8] Alembic migrations validation ==="

# --- DooD模式检测：确定宿主机访问地址 ---
# DooD模式下，docker run启动的容器跑在宿主机Docker上
# 需要用宿主机IP访问映射端口
# 检测策略：host.docker.internal -> 默认网关 -> 127.0.0.1
detect_docker_host() {
  # 1. 尝试 host.docker.internal（runner配置了--add-host时可用）
  if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then
    # 确认端口可达性（用一个常用端口快速检测）
    if python3 -c "
import socket
s = socket.socket()
s.settimeout(1)
try:
    s.connect(('host.docker.internal', 5433))
    s.close()
    print('ok')
except:
    # 端口不通不代表host不对，可能只是没服务
    print('ok')
" 2>/dev/null; then
      echo "host.docker.internal"
      return 0
    fi
  fi

  # 2. 尝试默认网关（Docker容器默认网关即宿主机）
  local gateway=""
  gateway=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1)
  if [ -n "$gateway" ] && [ "$gateway" != "127.0.0.1" ]; then
    echo "$gateway"
    return 0
  fi

  # 3. 回退到 127.0.0.1
  echo "127.0.0.1"
  return 0
}

if [ -S /var/run/docker.sock ]; then
  PG_HOST=$(detect_docker_host)
  echo "检测到DooD模式（/var/run/docker.sock已挂载），宿主机地址: $PG_HOST"
else
  PG_HOST="127.0.0.1"
  echo "非DooD模式，使用 127.0.0.1"
fi
echo "PG host: $PG_HOST"

# 指数退避TCP连接检查函数
# 用法: wait_tcp_ready host port max_attempts
wait_tcp_ready() {
  local host="$1"
  local port="$2"
  local max_attempts="${3:-5}"
  local delay=1
  local attempt=1
  while [ "$attempt" -le "$max_attempts" ]; do
    if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
      return 0
    fi
    echo "TCP连接尝试 $attempt/$max_attempts 失败，${delay}s后重试..."
    sleep "$delay"
    delay=$((delay * 2))
    attempt=$((attempt + 1))
  done
  return 1
}

USE_SHARED_PG="${CI_USE_SHARED_PG:-false}"

if [ "$USE_SHARED_PG" = "true" ]; then
  # 使用常驻共享PG实例（host.docker.internal:5433）
  echo "使用常驻共享PG实例（CI_USE_SHARED_PG=true）"
  SHARED_PG_HOST="$PG_HOST"
  SHARED_PG_PORT="5433"
  SHARED_PG_USER="postgres"
  SHARED_PG_PASSWORD="ci_pg_2026!"
  CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}"

  echo "等待共享PG连接就绪..."
  wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5

  # 创建独立数据库
  echo "创建测试数据库: $CI_DB_NAME"
  PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
import psycopg2
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
conn.autocommit = True
cur = conn.cursor()
cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"')
cur.close()
conn.close()
"
  export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}"
  echo "✅ 共享PG数据库已创建: $CI_DB_NAME"

  # 执行迁移
  PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
  echo "✅ Alembic migrations applied successfully"

  # 清理数据库
  echo "清理测试数据库: $CI_DB_NAME"
  PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
import psycopg2
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
conn.autocommit = True
cur = conn.cursor()
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
cur.close()
conn.close()
" 2>/dev/null || echo "WARN: 数据库清理失败（可能已被清理）"
  echo "✅ 共享PG数据库已清理"
else
  # 使用临时PG容器（默认模式）
  echo "使用临时PG容器模式"
  PG_CONTAINER=ci-pg-validate-${GITHUB_RUN_ID:-$$}
  docker rm -f "$PG_CONTAINER" 2>/dev/null || true
  docker run -d --name "$PG_CONTAINER" \
    --shm-size=256m \
    -e POSTGRES_USER=postgres \
    -e POSTGRES_PASSWORD=postgres \
    -e POSTGRES_DB=xiaoxia_saas \
    -P \
    --health-cmd "pg_isready -U postgres" \
    --health-interval 3s \
    --health-timeout 3s \
    --health-retries 20 \
    postgres:16-alpine
  PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
  echo "PostgreSQL port: $PG_PORT"
  export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas"

  # 等待容器健康
  for i in $(seq 1 30); do
    if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
      echo "PostgreSQL container is healthy on port $PG_PORT"
      break
    fi
    echo "Waiting for PostgreSQL container health... ($i/30)"
    sleep 2
  done
  docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy

  # TCP连通性检查（指数退避）
  echo "验证TCP连通性 ($PG_HOST:$PG_PORT)..."
  wait_tcp_ready "$PG_HOST" "$PG_PORT" 5
  echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"

  # 执行迁移
  PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
  echo "✅ Alembic migrations applied successfully"

  docker rm -f "$PG_CONTAINER" 2>/dev/null || true
fi

echo ""
echo "=== CI Validate: 所有检查通过 ✅ ==="