322 lines
11 KiB
Bash
Executable File
322 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
||
# CI Integration Tests Job 主脚本
|
||
# 包含：依赖安装、ffmpeg安装、Redis启动、PG启动、迁移、测试、清理、覆盖率
|
||
# 支持 pytest-xdist 并行执行：每个 worker 使用独立数据库，预期加速 2-4 倍
|
||
set -eu
|
||
|
||
# 加载CI共享常量
|
||
SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
|
||
# shellcheck source=ci_env.sh
|
||
source "${SCRIPT_DIR}/ci_env.sh"
|
||
|
||
echo "=== CI Integration Tests 开始 ==="
|
||
|
||
# --- 安装依赖 ---
|
||
echo ""
|
||
echo "=== 安装 Python 依赖 ==="
|
||
# pip install 带重试（网络不稳定时自动重试）
|
||
for i in 1 2 3; do
|
||
python3 -m pip install -q -r requirements-base.txt && break
|
||
echo "pip install requirements-base.txt 失败，重试 $i/3..."
|
||
[ $i -eq 3 ] && exit 1
|
||
sleep 5
|
||
done
|
||
for i in 1 2 3; do
|
||
python3 -m pip install -q -r requirements.txt && break
|
||
echo "pip install requirements.txt 失败，重试 $i/3..."
|
||
[ $i -eq 3 ] && exit 1
|
||
sleep 5
|
||
done
|
||
for i in 1 2 3; do
|
||
python3 -m pip install -q -r requirements-dev.txt && break
|
||
echo "pip install requirements-dev.txt 失败，重试 $i/3..."
|
||
[ $i -eq 3 ] && exit 1
|
||
sleep 5
|
||
done
|
||
for i in 1 2 3; do
|
||
python3 -m pip install -q pytest-rerunfailures pytest-xdist && break
|
||
echo "pip install pytest-rerunfailures/pytest-xdist 失败，重试 $i/3..."
|
||
[ $i -eq 3 ] && exit 1
|
||
sleep 5
|
||
done
|
||
pytest --version
|
||
echo "pytest-xdist: $(python3 -c "import xdist; print(xdist.__version__)" 2>/dev/null || echo 'not installed')"
|
||
|
||
# --- 安装 ffmpeg ---
|
||
echo ""
|
||
echo "=== 安装 ffmpeg ==="
|
||
bash scripts/ci/step_install_ffmpeg.sh
|
||
|
||
# --- DooD模式检测：确定宿主机访问地址 ---
|
||
# DooD模式下，docker run启动的容器跑在宿主机Docker上
|
||
# 需要用宿主机IP访问映射端口
|
||
# 检测策略：host.docker.internal -> docker0桥接IP -> 容器IP直连 -> 默认网关 -> 127.0.0.1
|
||
detect_docker_host() {
|
||
# 目标：找到宿主机IP（DooD模式下CI容器访问宿主机上其他容器用）
|
||
# 不依赖特定端口TCP探测，直接用网络拓扑信息
|
||
|
||
# 1. host.docker.internal（runner配置了--add-host时可用）
|
||
if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then
|
||
echo "host.docker.internal"
|
||
return 0
|
||
fi
|
||
|
||
# 2. 默认网关（Docker bridge模式下网关即宿主机）
|
||
local gw=""
|
||
gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1)
|
||
if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then
|
||
echo "$gw"
|
||
return 0
|
||
fi
|
||
|
||
# 3. docker0 桥接网关
|
||
if [ -n "$(ip addr show docker0 2>/dev/null)" ]; then
|
||
echo "172.17.0.1"
|
||
return 0
|
||
fi
|
||
|
||
# 4. 通过 git server hostname 反查（runner 配置了 ExtraHosts host-gateway）
|
||
local git_host_ip=""
|
||
git_host_ip=$(python3 -c "import socket; print(socket.gethostbyname('git.xiaoxiajianji.com'))" 2>/dev/null || true)
|
||
if [ -n "$git_host_ip" ] && [ "$git_host_ip" != "127.0.0.1" ]; then
|
||
echo "$git_host_ip"
|
||
return 0
|
||
fi
|
||
|
||
# 5. 最终 fallback
|
||
echo "127.0.0.1"
|
||
return 0
|
||
}
|
||
|
||
# 获取宿主机IP（先尝试用共享PG端口5433测试，再回退到其他端口）
|
||
if [ -S /var/run/docker.sock ]; then
|
||
DOCKER_HOST_IP=$(detect_docker_host)
|
||
echo "检测到DooD模式（/var/run/docker.sock已挂载），宿主机地址: $DOCKER_HOST_IP"
|
||
else
|
||
DOCKER_HOST_IP="127.0.0.1"
|
||
echo "非DooD模式，使用 127.0.0.1"
|
||
fi
|
||
PG_HOST="$DOCKER_HOST_IP"
|
||
REDIS_HOST="$DOCKER_HOST_IP"
|
||
echo "PG host: $PG_HOST, Redis host: $REDIS_HOST"
|
||
|
||
# --- 指数退避TCP连接检查函数 ---
|
||
# 用法: wait_tcp_ready host port max_attempts
|
||
wait_tcp_ready() {
|
||
local host="$1"
|
||
local port="$2"
|
||
local max_attempts="${3:-5}"
|
||
local delay=1
|
||
local attempt=1
|
||
while [ "$attempt" -le "$max_attempts" ]; do
|
||
if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
|
||
return 0
|
||
fi
|
||
echo "TCP连接尝试 $attempt/$max_attempts 失败，${delay}s后重试..."
|
||
sleep "$delay"
|
||
delay=$((delay * 2))
|
||
attempt=$((attempt + 1))
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# --- 启动 Redis ---
|
||
echo ""
|
||
echo "=== 启动 Redis ==="
|
||
REDIS_CONTAINER="ci-redis-${GITHUB_RUN_ID:-$$}"
|
||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||
docker run -d --name "$REDIS_CONTAINER" \
|
||
-P \
|
||
--health-cmd "redis-cli ping" \
|
||
--health-interval 2s \
|
||
--health-timeout 2s \
|
||
--health-retries 10 \
|
||
redis:7-alpine
|
||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | cut -d: -f2)
|
||
echo "Redis port: $REDIS_PORT"
|
||
export REDIS_URL="redis://${REDIS_HOST}:${REDIS_PORT}/0"
|
||
|
||
# 等待容器健康
|
||
for i in $(seq 1 15); do
|
||
if docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" 2>/dev/null | grep -q healthy; then
|
||
echo "Redis container is ready on port $REDIS_PORT"
|
||
break
|
||
fi
|
||
echo "Waiting for Redis container health... ($i/15)"
|
||
sleep 2
|
||
done
|
||
docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" | grep -q healthy
|
||
|
||
# TCP连通性检查（指数退避）
|
||
echo "验证Redis TCP连通性 ($REDIS_HOST:$REDIS_PORT)..."
|
||
wait_tcp_ready "$REDIS_HOST" "$REDIS_PORT" 5
|
||
echo "TCP connectivity to Redis confirmed on port $REDIS_PORT"
|
||
|
||
# --- 启动/连接 PostgreSQL ---
|
||
echo ""
|
||
echo "=== 准备 PostgreSQL ==="
|
||
USE_SHARED_PG="${CI_USE_SHARED_PG:-false}"
|
||
CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}"
|
||
|
||
if [ "$USE_SHARED_PG" = "true" ]; then
|
||
# 使用常驻共享PG实例
|
||
echo "使用常驻共享PG实例（CI_USE_SHARED_PG=true）"
|
||
SHARED_PG_HOST="$PG_HOST"
|
||
SHARED_PG_PORT="${CI_SHARED_PG_PORT}"
|
||
SHARED_PG_USER="${CI_SHARED_PG_USER}"
|
||
SHARED_PG_PASSWORD="${CI_SHARED_PG_PASSWORD}"
|
||
|
||
echo "等待共享PG连接就绪..."
|
||
wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5
|
||
|
||
# 创建主数据库（xdist 模式下各 worker 会创建自己的数据库，主库作为 fallback）
|
||
echo "创建主测试数据库: $CI_DB_NAME"
|
||
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||
import psycopg
|
||
conn = psycopg.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
|
||
conn.autocommit = True
|
||
cur = conn.cursor()
|
||
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
|
||
cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"')
|
||
cur.close()
|
||
conn.close()
|
||
"
|
||
export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}"
|
||
echo "✅ 共享PG数据库已创建: $CI_DB_NAME"
|
||
PG_CONTAINER=""
|
||
else
|
||
# 使用临时PG容器
|
||
echo "使用临时PG容器模式"
|
||
PG_CONTAINER="ci-pg-${GITHUB_RUN_ID:-$$}"
|
||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||
docker run -d --name "$PG_CONTAINER" \
|
||
--shm-size=256m \
|
||
-e POSTGRES_USER=postgres \
|
||
-e POSTGRES_PASSWORD=postgres \
|
||
-e POSTGRES_DB=xiaoxia_saas \
|
||
-P \
|
||
--health-cmd "pg_isready -U postgres" \
|
||
--health-interval 5s \
|
||
--health-timeout 5s \
|
||
--health-retries 12 \
|
||
postgres:16
|
||
PG_PORT=$(docker port "$PG_CONTAINER" ${CI_LOCAL_PG_PORT}/tcp | cut -d: -f2)
|
||
echo "PostgreSQL port: $PG_PORT"
|
||
export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/${CI_DEFAULT_DB}"
|
||
|
||
# 等待容器健康
|
||
for i in $(seq 1 30); do
|
||
if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
|
||
echo "PostgreSQL container is ready on port $PG_PORT"
|
||
break
|
||
fi
|
||
echo "Waiting for PostgreSQL container health... ($i/30)"
|
||
sleep 2
|
||
done
|
||
docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy
|
||
|
||
# TCP连通性检查（指数退避）
|
||
echo "验证PostgreSQL TCP连通性 ($PG_HOST:$PG_PORT)..."
|
||
wait_tcp_ready "$PG_HOST" "$PG_PORT" 5
|
||
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
|
||
fi
|
||
|
||
# --- 执行迁移（主数据库，xdist worker 会各自创建自己的库并迁移） ---
|
||
echo ""
|
||
echo "=== 执行 Alembic 迁移（主数据库） ==="
|
||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
|
||
echo "✅ 迁移完成"
|
||
|
||
# --- 运行集成测试（pytest-xdist 并行） ---
|
||
echo ""
|
||
echo "=== 运行集成测试（pytest-xdist 并行模式） ==="
|
||
echo "CPU 核数: $(nproc 2>/dev/null || echo 'unknown')"
|
||
|
||
# 集成测试使用 pytest-xdist 并行加速（coverage 由单元测试负责，并行模式下 coverage 不稳定）
|
||
# -n auto: 自动使用 CPU 核数（DooD模式下加--maxprocesses=4防止OOM
|
||
# --dist loadfile: 同一测试文件分配到同一 worker（共享 fixture 更高效）
|
||
# --maxfail=1: 遇到失败停止调度新测试（并行模式下等价于 -x）
|
||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration \
|
||
-q --timeout=60 --maxfail=1 --reruns 3 --reruns-delay 5 \
|
||
-m "not performance" \
|
||
-n auto --maxprocesses=4 --dist loadfile \
|
||
-p no:cacheprovider
|
||
|
||
echo "✅ 集成测试通过"
|
||
|
||
# --- API 性能基线测试（仅告警，串行执行） ---
|
||
echo ""
|
||
echo "=== API 性能基线测试（仅告警） ==="
|
||
set +e
|
||
PERF_OUTPUT=$(mktemp)
|
||
# 性能测试单独串行运行（不参与并行，避免资源竞争影响测量结果）
|
||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration/test_api_performance.py \
|
||
-v --timeout=120 -p no:cacheprovider 2>&1 | tee "$PERF_OUTPUT" \
|
||
--reruns 3 \
|
||
--reruns-delay=10
|
||
echo ""
|
||
echo "=== 性能测试摘要 ==="
|
||
grep "PERF_STATS:" "$PERF_OUTPUT" || echo "PERF_STATS: 未找到统计数据"
|
||
grep "PERF_RESULT:" "$PERF_OUTPUT" || echo "PERF_RESULT: 未找到详细结果"
|
||
TOTAL=$(grep -c "PERF_RESULT:" "$PERF_OUTPUT" || echo 0)
|
||
PASSED=$(grep "PERF_RESULT: PASS" "$PERF_OUTPUT" | wc -l)
|
||
FAILED=$(grep "PERF_RESULT: FAIL" "$PERF_OUTPUT" | wc -l)
|
||
echo ""
|
||
echo "性能测试结果: $PASSED/$TOTAL 通过, $FAILED 未达标"
|
||
if [ "$FAILED" -gt 0 ]; then
|
||
echo ""
|
||
echo "⚠️ 警告: $FAILED 个接口性能未达标"
|
||
fi
|
||
rm -f "$PERF_OUTPUT"
|
||
set -e
|
||
|
||
# --- 清理 ---
|
||
echo ""
|
||
echo "=== 清理 ==="
|
||
if [ "$USE_SHARED_PG" = "true" ]; then
|
||
# 清理共享PG上的测试数据库（主库 + 可能残留的 worker 库）
|
||
echo "清理共享PG测试数据库..."
|
||
|
||
# 清理所有以 CI_DB_NAME 开头的数据库（主库 + worker 库）
|
||
PGPASSWORD="${SHARED_PG_PASSWORD}" python3 -c "
|
||
import psycopg
|
||
conn = psycopg.connect(host='${SHARED_PG_HOST}', port=${SHARED_PG_PORT}, user='${SHARED_PG_USER}', password='${SHARED_PG_PASSWORD}', dbname='postgres')
|
||
conn.autocommit = True
|
||
cur = conn.cursor()
|
||
|
||
# 查找所有需要清理的数据库（主库 + worker 库）
|
||
cur.execute(\"SELECT datname FROM pg_database WHERE datname LIKE '$CI_DB_NAME%'\")
|
||
dbs = [row[0] for row in cur.fetchall()]
|
||
|
||
for db in dbs:
|
||
try:
|
||
# 强制断开所有连接
|
||
cur.execute(f\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{db}' AND pid <> pg_backend_pid()\")
|
||
cur.execute(f'DROP DATABASE IF EXISTS \"{db}\" WITH (FORCE)')
|
||
print(f' 已清理: {db}')
|
||
except Exception as e:
|
||
print(f' 警告: 清理 {db} 失败: {e}')
|
||
|
||
cur.close()
|
||
conn.close()
|
||
" 2>/dev/null || echo "WARN: 数据库清理失败（可能已被清理）"
|
||
echo "✅ 共享PG数据库已清理"
|
||
else
|
||
# 清理临时PG容器
|
||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||
echo "✅ PG容器已清理"
|
||
fi
|
||
|
||
# 清理Redis容器
|
||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||
echo "✅ Redis容器已清理"
|
||
|
||
# --- 覆盖率汇总 ---
|
||
echo ""
|
||
echo "=== 覆盖率汇总 ==="
|
||
set +e
|
||
python3 scripts/ci_coverage_summary.py
|
||
set -e
|
||
|
||
echo ""
|
||
echo "=== CI Integration Tests 全部通过 ✅ ===" |