xiaoxia-saas/scripts/ci/run_integration_tests.sh

#!/bin/bash
# CI Integration Tests Job 主脚本
# 包含：依赖安装、ffmpeg安装、Redis启动、PG启动、迁移、测试、清理、覆盖率
# 支持 pytest-xdist 并行执行：每个 worker 使用独立数据库，预期加速 2-4 倍
set -eu

# 加载CI共享常量
SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
# shellcheck source=ci_env.sh
source "${SCRIPT_DIR}/ci_env.sh"

echo "=== CI Integration Tests 开始 ==="

# --- 安装依赖 ---
echo ""
echo "=== 安装 Python 依赖 ==="
# pip install 带重试（网络不稳定时自动重试）
for i in 1 2 3; do
  python3 -m pip install -q -r requirements-base.txt && break
  echo "pip install requirements-base.txt 失败，重试 $i/3..."
  [ $i -eq 3 ] && exit 1
  sleep 5
done
for i in 1 2 3; do
  python3 -m pip install -q -r requirements.txt && break
  echo "pip install requirements.txt 失败，重试 $i/3..."
  [ $i -eq 3 ] && exit 1
  sleep 5
done
for i in 1 2 3; do
  python3 -m pip install -q -r requirements-dev.txt && break
  echo "pip install requirements-dev.txt 失败，重试 $i/3..."
  [ $i -eq 3 ] && exit 1
  sleep 5
done
for i in 1 2 3; do
  python3 -m pip install -q pytest-rerunfailures pytest-xdist && break
  echo "pip install pytest-rerunfailures/pytest-xdist 失败，重试 $i/3..."
  [ $i -eq 3 ] && exit 1
  sleep 5
done
pytest --version
echo "pytest-xdist: $(python3 -c "import xdist; print(xdist.__version__)" 2>/dev/null || echo 'not installed')"

# --- 安装 ffmpeg ---
echo ""
echo "=== 安装 ffmpeg ==="
bash scripts/ci/step_install_ffmpeg.sh

# --- DooD模式检测：确定宿主机访问地址 ---
# DooD模式下，docker run启动的容器跑在宿主机Docker上
# 需要用宿主机IP访问映射端口
# 检测策略：host.docker.internal -> docker0桥接IP -> 容器IP直连 -> 默认网关 -> 127.0.0.1
detect_docker_host() {
  local test_port="${1:-${CI_LOCAL_PG_PORT}}"

  # 候选IP列表
  local candidates=()

  # 1. host.docker.internal（runner配置了--add-host时可用）
  if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then
    candidates+=("host.docker.internal")
  fi

  # 2. docker0 桥接网关 (172.17.0.1)
  candidates+=("172.17.0.1")

  # 3. 默认网关（容器网络的网关即宿主机）
  local gw=""
  gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1)
  if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then
    candidates+=("$gw")
  fi

  # 4. 宿主机可能的IP：容器同网段的.1或.254
  local my_ip=""
  my_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
  if [ -n "$my_ip" ]; then
    # 尝试同网段的常见宿主机IP
    local subnet=$(echo "$my_ip" | cut -d. -f1-3)
    candidates+=("${subnet}.1")
    candidates+=("${subnet}.254")
  fi

  # 5. 127.0.0.1 最后尝试
  candidates+=("127.0.0.1")

  # 测试每个候选IP
  for candidate in "${candidates[@]}"; do
    if python3 -c "
import socket
s = socket.socket()
s.settimeout(2)
try:
    s.connect(('$candidate', $test_port))
    s.close()
    print('ok')
except:
    pass
" 2>/dev/null | grep -q ok; then
      echo "$candidate"
      return 0
    fi
  done

  # 都失败则返回127.0.0.1
  echo "127.0.0.1"
  return 1
}

# 获取宿主机IP（先尝试用共享PG端口5433测试，再回退到其他端口）
if [ -S /var/run/docker.sock ]; then
  # 先用共享PG端口5433探测
  DOCKER_HOST_IP=$(detect_docker_host "${CI_SHARED_PG_PORT}")
  if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then
    # 如果共享PG端口探测失败，说明不在DooD或共享PG不可用，再试其他端口
    DOCKER_HOST_IP=$(detect_docker_host 22)
  fi
  echo "检测到DooD模式（/var/run/docker.sock已挂载），宿主机地址: $DOCKER_HOST_IP"
else
  DOCKER_HOST_IP="127.0.0.1"
  echo "非DooD模式，使用 127.0.0.1"
fi
PG_HOST="$DOCKER_HOST_IP"
REDIS_HOST="$DOCKER_HOST_IP"
echo "PG host: $PG_HOST, Redis host: $REDIS_HOST"

# --- 指数退避TCP连接检查函数 ---
# 用法: wait_tcp_ready host port max_attempts
wait_tcp_ready() {
  local host="$1"
  local port="$2"
  local max_attempts="${3:-5}"
  local delay=1
  local attempt=1
  while [ "$attempt" -le "$max_attempts" ]; do
    if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
      return 0
    fi
    echo "TCP连接尝试 $attempt/$max_attempts 失败，${delay}s后重试..."
    sleep "$delay"
    delay=$((delay * 2))
    attempt=$((attempt + 1))
  done
  return 1
}

# --- 启动 Redis ---
echo ""
echo "=== 启动 Redis ==="
REDIS_CONTAINER="ci-redis-${GITHUB_RUN_ID:-$$}"
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
docker run -d --name "$REDIS_CONTAINER" \
  -P \
  --health-cmd "redis-cli ping" \
  --health-interval 2s \
  --health-timeout 2s \
  --health-retries 10 \
  redis:7-alpine
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | cut -d: -f2)
echo "Redis port: $REDIS_PORT"
export REDIS_URL="redis://${REDIS_HOST}:${REDIS_PORT}/0"

# 等待容器健康
for i in $(seq 1 15); do
  if docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" 2>/dev/null | grep -q healthy; then
    echo "Redis container is ready on port $REDIS_PORT"
    break
  fi
  echo "Waiting for Redis container health... ($i/15)"
  sleep 2
done
docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" | grep -q healthy

# TCP连通性检查（指数退避）
echo "验证Redis TCP连通性 ($REDIS_HOST:$REDIS_PORT)..."
wait_tcp_ready "$REDIS_HOST" "$REDIS_PORT" 5
echo "TCP connectivity to Redis confirmed on port $REDIS_PORT"

# --- 启动/连接 PostgreSQL ---
echo ""
echo "=== 准备 PostgreSQL ==="
USE_SHARED_PG="${CI_USE_SHARED_PG:-false}"
CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}"

if [ "$USE_SHARED_PG" = "true" ]; then
  # 使用常驻共享PG实例
  echo "使用常驻共享PG实例（CI_USE_SHARED_PG=true）"
  SHARED_PG_HOST="$PG_HOST"
  SHARED_PG_PORT="${CI_SHARED_PG_PORT}"
  SHARED_PG_USER="${CI_SHARED_PG_USER}"
  SHARED_PG_PASSWORD="${CI_SHARED_PG_PASSWORD}"

  echo "等待共享PG连接就绪..."
  wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5

  # 创建主数据库（xdist 模式下各 worker 会创建自己的数据库，主库作为 fallback）
  echo "创建主测试数据库: $CI_DB_NAME"
  PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
import psycopg2
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
conn.autocommit = True
cur = conn.cursor()
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"')
cur.close()
conn.close()
"
  export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}"
  echo "✅ 共享PG数据库已创建: $CI_DB_NAME"
  PG_CONTAINER=""
else
  # 使用临时PG容器
  echo "使用临时PG容器模式"
  PG_CONTAINER="ci-pg-${GITHUB_RUN_ID:-$$}"
  docker rm -f "$PG_CONTAINER" 2>/dev/null || true
  docker run -d --name "$PG_CONTAINER" \
    --shm-size=256m \
    -e POSTGRES_USER=postgres \
    -e POSTGRES_PASSWORD=postgres \
    -e POSTGRES_DB=xiaoxia_saas \
    -P \
    --health-cmd "pg_isready -U postgres" \
    --health-interval 5s \
    --health-timeout 5s \
    --health-retries 12 \
    postgres:16
  PG_PORT=$(docker port "$PG_CONTAINER" ${CI_LOCAL_PG_PORT}/tcp | cut -d: -f2)
  echo "PostgreSQL port: $PG_PORT"
  export DATABASE_URL="postgresql+psycopg://${CI_SHARED_PG_USER}:${CI_SHARED_PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${CI_DEFAULT_DB}"

  # 等待容器健康
  for i in $(seq 1 30); do
    if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
      echo "PostgreSQL container is ready on port $PG_PORT"
      break
    fi
    echo "Waiting for PostgreSQL container health... ($i/30)"
    sleep 2
  done
  docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy

  # TCP连通性检查（指数退避）
  echo "验证PostgreSQL TCP连通性 ($PG_HOST:$PG_PORT)..."
  wait_tcp_ready "$PG_HOST" "$PG_PORT" 5
  echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
fi

# --- 执行迁移（主数据库，xdist worker 会各自创建自己的库并迁移） ---
echo ""
echo "=== 执行 Alembic 迁移（主数据库） ==="
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
echo "✅ 迁移完成"

# --- 运行集成测试（pytest-xdist 并行） ---
echo ""
echo "=== 运行集成测试（pytest-xdist 并行模式） ==="
echo "CPU 核数: $(nproc 2>/dev/null || echo 'unknown')"

# 集成测试使用 pytest-xdist 并行加速（coverage 由单元测试负责，并行模式下 coverage 不稳定）
# -n auto: 自动使用 CPU 核数（DooD模式下加--maxprocesses=4防止OOM
# --dist loadfile: 同一测试文件分配到同一 worker（共享 fixture 更高效）
# --maxfail=1: 遇到失败停止调度新测试（并行模式下等价于 -x）
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration \
  -q --timeout=60 --maxfail=1 --reruns 3 --reruns-delay 5 \
  -m "not performance" \
  -n auto --maxprocesses=4 --dist loadfile \
  -p no:cacheprovider

echo "✅ 集成测试通过"

# --- API 性能基线测试（仅告警，串行执行） ---
echo ""
echo "=== API 性能基线测试（仅告警） ==="
set +e
PERF_OUTPUT=$(mktemp)
# 性能测试单独串行运行（不参与并行，避免资源竞争影响测量结果）
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration/test_api_performance.py \
  -v --timeout=120 -p no:cacheprovider 2>&1 | tee "$PERF_OUTPUT" \
  --reruns 3 \
  --reruns-delay=10
echo ""
echo "=== 性能测试摘要 ==="
grep "PERF_STATS:" "$PERF_OUTPUT" || echo "PERF_STATS: 未找到统计数据"
grep "PERF_RESULT:" "$PERF_OUTPUT" || echo "PERF_RESULT: 未找到详细结果"
TOTAL=$(grep -c "PERF_RESULT:" "$PERF_OUTPUT" || echo 0)
PASSED=$(grep "PERF_RESULT: PASS" "$PERF_OUTPUT" | wc -l)
FAILED=$(grep "PERF_RESULT: FAIL" "$PERF_OUTPUT" | wc -l)
echo ""
echo "性能测试结果: $PASSED/$TOTAL 通过, $FAILED 未达标"
if [ "$FAILED" -gt 0 ]; then
  echo ""
  echo "⚠️  警告: $FAILED 个接口性能未达标"
fi
rm -f "$PERF_OUTPUT"
set -e

# --- 清理 ---
echo ""
echo "=== 清理 ==="
if [ "$USE_SHARED_PG" = "true" ]; then
  # 清理共享PG上的测试数据库（主库 + 可能残留的 worker 库）
  echo "清理共享PG测试数据库..."

  # 清理所有以 CI_DB_NAME 开头的数据库（主库 + worker 库）
  PGPASSWORD="${SHARED_PG_PASSWORD}" python3 -c "
import psycopg2
conn = psycopg2.connect(host='${SHARED_PG_HOST}', port=${SHARED_PG_PORT}, user='${SHARED_PG_USER}', password='${SHARED_PG_PASSWORD}', dbname='postgres')
conn.autocommit = True
cur = conn.cursor()

# 查找所有需要清理的数据库（主库 + worker 库）
cur.execute(\"SELECT datname FROM pg_database WHERE datname LIKE '$CI_DB_NAME%'\")
dbs = [row[0] for row in cur.fetchall()]

for db in dbs:
    try:
        # 强制断开所有连接
        cur.execute(f\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{db}' AND pid <> pg_backend_pid()\")
        cur.execute(f'DROP DATABASE IF EXISTS \"{db}\" WITH (FORCE)')
        print(f'  已清理: {db}')
    except Exception as e:
        print(f'  警告: 清理 {db} 失败: {e}')

cur.close()
conn.close()
" 2>/dev/null || echo "WARN: 数据库清理失败（可能已被清理）"
  echo "✅ 共享PG数据库已清理"
else
  # 清理临时PG容器
  docker rm -f "$PG_CONTAINER" 2>/dev/null || true
  echo "✅ PG容器已清理"
fi

# 清理Redis容器
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
echo "✅ Redis容器已清理"

# --- 覆盖率汇总 ---
echo ""
echo "=== 覆盖率汇总 ==="
set +e
python3 scripts/ci_coverage_summary.py
set -e

echo ""
echo "=== CI Integration Tests 全部通过 ✅ ==="