#!/bin/bash # CI Integration Tests Job 主脚本 # 包含:依赖安装、ffmpeg安装、Redis启动、PG启动、迁移、测试、清理、覆盖率 # 支持 pytest-xdist 并行执行:每个 worker 使用独立数据库,预期加速 2-4 倍 set -eu # 加载CI共享常量 SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")" # shellcheck source=ci_env.sh source "${SCRIPT_DIR}/ci_env.sh" echo "=== CI Integration Tests 开始 ===" # --- 安装依赖 --- echo "" echo "=== 安装 Python 依赖 ===" # pip install 带重试(网络不稳定时自动重试) for i in 1 2 3; do python3 -m pip install -q -r requirements-base.txt && break echo "pip install requirements-base.txt 失败,重试 $i/3..." [ $i -eq 3 ] && exit 1 sleep 5 done for i in 1 2 3; do python3 -m pip install -q -r requirements.txt && break echo "pip install requirements.txt 失败,重试 $i/3..." [ $i -eq 3 ] && exit 1 sleep 5 done for i in 1 2 3; do python3 -m pip install -q -r requirements-dev.txt && break echo "pip install requirements-dev.txt 失败,重试 $i/3..." [ $i -eq 3 ] && exit 1 sleep 5 done for i in 1 2 3; do python3 -m pip install -q pytest-rerunfailures pytest-xdist && break echo "pip install pytest-rerunfailures/pytest-xdist 失败,重试 $i/3..." [ $i -eq 3 ] && exit 1 sleep 5 done pytest --version echo "pytest-xdist: $(python3 -c "import xdist; print(xdist.__version__)" 2>/dev/null || echo 'not installed')" # --- 安装 ffmpeg --- echo "" echo "=== 安装 ffmpeg ===" bash scripts/ci/step_install_ffmpeg.sh # --- DooD模式检测:确定宿主机访问地址 --- # DooD模式下,docker run启动的容器跑在宿主机Docker上 # 需要用宿主机IP访问映射端口 # 检测策略:host.docker.internal -> docker0桥接IP -> 容器IP直连 -> 默认网关 -> 127.0.0.1 detect_docker_host() { local test_port="${1:-${CI_LOCAL_PG_PORT}}" # 候选IP列表 local candidates=() # 1. host.docker.internal(runner配置了--add-host时可用) if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then candidates+=("host.docker.internal") fi # 2. docker0 桥接网关 (172.17.0.1) candidates+=("172.17.0.1") # 3. 默认网关(容器网络的网关即宿主机) local gw="" gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1) if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then candidates+=("$gw") fi # 4. 宿主机可能的IP:容器同网段的.1或.254 local my_ip="" my_ip=$(hostname -I 2>/dev/null | awk '{print $1}') if [ -n "$my_ip" ]; then # 尝试同网段的常见宿主机IP local subnet=$(echo "$my_ip" | cut -d. -f1-3) candidates+=("${subnet}.1") candidates+=("${subnet}.254") fi # 5. 127.0.0.1 最后尝试 candidates+=("127.0.0.1") # 测试每个候选IP for candidate in "${candidates[@]}"; do if python3 -c " import socket s = socket.socket() s.settimeout(2) try: s.connect(('$candidate', $test_port)) s.close() print('ok') except: pass " 2>/dev/null | grep -q ok; then echo "$candidate" return 0 fi done # 都失败则返回127.0.0.1 echo "127.0.0.1" return 1 } # 获取宿主机IP(先尝试用共享PG端口5433测试,再回退到其他端口) if [ -S /var/run/docker.sock ]; then # 先用共享PG端口5433探测 DOCKER_HOST_IP=$(detect_docker_host "${CI_SHARED_PG_PORT}") if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then # 如果共享PG端口探测失败,说明不在DooD或共享PG不可用,再试其他端口 DOCKER_HOST_IP=$(detect_docker_host 22) fi echo "检测到DooD模式(/var/run/docker.sock已挂载),宿主机地址: $DOCKER_HOST_IP" else DOCKER_HOST_IP="127.0.0.1" echo "非DooD模式,使用 127.0.0.1" fi PG_HOST="$DOCKER_HOST_IP" REDIS_HOST="$DOCKER_HOST_IP" echo "PG host: $PG_HOST, Redis host: $REDIS_HOST" # --- 指数退避TCP连接检查函数 --- # 用法: wait_tcp_ready host port max_attempts wait_tcp_ready() { local host="$1" local port="$2" local max_attempts="${3:-5}" local delay=1 local attempt=1 while [ "$attempt" -le "$max_attempts" ]; do if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then return 0 fi echo "TCP连接尝试 $attempt/$max_attempts 失败,${delay}s后重试..." sleep "$delay" delay=$((delay * 2)) attempt=$((attempt + 1)) done return 1 } # --- 启动 Redis --- echo "" echo "=== 启动 Redis ===" REDIS_CONTAINER="ci-redis-${GITHUB_RUN_ID:-$$}" docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true docker run -d --name "$REDIS_CONTAINER" \ -P \ --health-cmd "redis-cli ping" \ --health-interval 2s \ --health-timeout 2s \ --health-retries 10 \ redis:7-alpine REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | cut -d: -f2) echo "Redis port: $REDIS_PORT" export REDIS_URL="redis://${REDIS_HOST}:${REDIS_PORT}/0" # 等待容器健康 for i in $(seq 1 15); do if docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" 2>/dev/null | grep -q healthy; then echo "Redis container is ready on port $REDIS_PORT" break fi echo "Waiting for Redis container health... ($i/15)" sleep 2 done docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" | grep -q healthy # TCP连通性检查(指数退避) echo "验证Redis TCP连通性 ($REDIS_HOST:$REDIS_PORT)..." wait_tcp_ready "$REDIS_HOST" "$REDIS_PORT" 5 echo "TCP connectivity to Redis confirmed on port $REDIS_PORT" # --- 启动/连接 PostgreSQL --- echo "" echo "=== 准备 PostgreSQL ===" USE_SHARED_PG="${CI_USE_SHARED_PG:-false}" CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}" if [ "$USE_SHARED_PG" = "true" ]; then # 使用常驻共享PG实例 echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true)" SHARED_PG_HOST="$PG_HOST" SHARED_PG_PORT="${CI_SHARED_PG_PORT}" SHARED_PG_USER="${CI_SHARED_PG_USER}" SHARED_PG_PASSWORD="${CI_SHARED_PG_PASSWORD}" echo "等待共享PG连接就绪..." wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5 # 创建主数据库(xdist 模式下各 worker 会创建自己的数据库,主库作为 fallback) echo "创建主测试数据库: $CI_DB_NAME" PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c " import psycopg2 conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres') conn.autocommit = True cur = conn.cursor() cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)') cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"') cur.close() conn.close() " export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}" echo "✅ 共享PG数据库已创建: $CI_DB_NAME" PG_CONTAINER="" else # 使用临时PG容器 echo "使用临时PG容器模式" PG_CONTAINER="ci-pg-${GITHUB_RUN_ID:-$$}" docker rm -f "$PG_CONTAINER" 2>/dev/null || true docker run -d --name "$PG_CONTAINER" \ --shm-size=256m \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=xiaoxia_saas \ -P \ --health-cmd "pg_isready -U postgres" \ --health-interval 5s \ --health-timeout 5s \ --health-retries 12 \ postgres:16 PG_PORT=$(docker port "$PG_CONTAINER" ${CI_LOCAL_PG_PORT}/tcp | cut -d: -f2) echo "PostgreSQL port: $PG_PORT" export DATABASE_URL="postgresql+psycopg://${CI_SHARED_PG_USER}:${CI_SHARED_PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${CI_DEFAULT_DB}" # 等待容器健康 for i in $(seq 1 30); do if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then echo "PostgreSQL container is ready on port $PG_PORT" break fi echo "Waiting for PostgreSQL container health... ($i/30)" sleep 2 done docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy # TCP连通性检查(指数退避) echo "验证PostgreSQL TCP连通性 ($PG_HOST:$PG_PORT)..." wait_tcp_ready "$PG_HOST" "$PG_PORT" 5 echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT" fi # --- 执行迁移(主数据库,xdist worker 会各自创建自己的库并迁移) --- echo "" echo "=== 执行 Alembic 迁移(主数据库) ===" PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head echo "✅ 迁移完成" # --- 运行集成测试(pytest-xdist 并行) --- echo "" echo "=== 运行集成测试(pytest-xdist 并行模式) ===" echo "CPU 核数: $(nproc 2>/dev/null || echo 'unknown')" # 集成测试使用 pytest-xdist 并行加速(coverage 由单元测试负责,并行模式下 coverage 不稳定) # -n auto: 自动使用 CPU 核数(DooD模式下加--maxprocesses=4防止OOM # --dist loadfile: 同一测试文件分配到同一 worker(共享 fixture 更高效) # --maxfail=1: 遇到失败停止调度新测试(并行模式下等价于 -x) PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration \ -q --timeout=60 --maxfail=1 --reruns 3 --reruns-delay 5 \ -m "not performance" \ -n auto --maxprocesses=4 --dist loadfile \ -p no:cacheprovider echo "✅ 集成测试通过" # --- API 性能基线测试(仅告警,串行执行) --- echo "" echo "=== API 性能基线测试(仅告警) ===" set +e PERF_OUTPUT=$(mktemp) # 性能测试单独串行运行(不参与并行,避免资源竞争影响测量结果) PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration/test_api_performance.py \ -v --timeout=120 -p no:cacheprovider 2>&1 | tee "$PERF_OUTPUT" \ --reruns 3 \ --reruns-delay=10 echo "" echo "=== 性能测试摘要 ===" grep "PERF_STATS:" "$PERF_OUTPUT" || echo "PERF_STATS: 未找到统计数据" grep "PERF_RESULT:" "$PERF_OUTPUT" || echo "PERF_RESULT: 未找到详细结果" TOTAL=$(grep -c "PERF_RESULT:" "$PERF_OUTPUT" || echo 0) PASSED=$(grep "PERF_RESULT: PASS" "$PERF_OUTPUT" | wc -l) FAILED=$(grep "PERF_RESULT: FAIL" "$PERF_OUTPUT" | wc -l) echo "" echo "性能测试结果: $PASSED/$TOTAL 通过, $FAILED 未达标" if [ "$FAILED" -gt 0 ]; then echo "" echo "⚠️ 警告: $FAILED 个接口性能未达标" fi rm -f "$PERF_OUTPUT" set -e # --- 清理 --- echo "" echo "=== 清理 ===" if [ "$USE_SHARED_PG" = "true" ]; then # 清理共享PG上的测试数据库(主库 + 可能残留的 worker 库) echo "清理共享PG测试数据库..." # 清理所有以 CI_DB_NAME 开头的数据库(主库 + worker 库) PGPASSWORD="${SHARED_PG_PASSWORD}" python3 -c " import psycopg2 conn = psycopg2.connect(host='${SHARED_PG_HOST}', port=${SHARED_PG_PORT}, user='${SHARED_PG_USER}', password='${SHARED_PG_PASSWORD}', dbname='postgres') conn.autocommit = True cur = conn.cursor() # 查找所有需要清理的数据库(主库 + worker 库) cur.execute(\"SELECT datname FROM pg_database WHERE datname LIKE '$CI_DB_NAME%'\") dbs = [row[0] for row in cur.fetchall()] for db in dbs: try: # 强制断开所有连接 cur.execute(f\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{db}' AND pid <> pg_backend_pid()\") cur.execute(f'DROP DATABASE IF EXISTS \"{db}\" WITH (FORCE)') print(f' 已清理: {db}') except Exception as e: print(f' 警告: 清理 {db} 失败: {e}') cur.close() conn.close() " 2>/dev/null || echo "WARN: 数据库清理失败(可能已被清理)" echo "✅ 共享PG数据库已清理" else # 清理临时PG容器 docker rm -f "$PG_CONTAINER" 2>/dev/null || true echo "✅ PG容器已清理" fi # 清理Redis容器 docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true echo "✅ Redis容器已清理" # --- 覆盖率汇总 --- echo "" echo "=== 覆盖率汇总 ===" set +e python3 scripts/ci_coverage_summary.py set -e echo "" echo "=== CI Integration Tests 全部通过 ✅ ==="