ci: 全面根治CI稳定性问题 (#619) - DooD模式PG端口映射 + 常驻PG + 重试机制 #621

Merged
auto-approve-bot merged 4 commits from ci/stability-fix into develop 2026-07-20 01:08:33 +08:00
4 changed files with 510 additions and 149 deletions
+88 -16
View File
@@ -73,8 +73,9 @@ jobs:
permissions:
contents: write
env:
DATABASE_URL: postgresql+psycopg://postgres:postgres@127.0.0.1:5432/xiaoxia_saas
DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas
USE_IN_MEMORY_DB: 'false'
CI_USE_SHARED_PG: 'true'
steps:
- name: Checkout code
shell: sh
@@ -89,10 +90,31 @@ jobs:
shell: sh
run: |
set -eu
python3 -m pip install -q -r requirements-base.txt
python3 -m pip install -q -r requirements.txt
python3 -m pip install -q -r requirements-dev.txt
python3 -m pip install --no-binary :all: black==26.5.1 isort==8.0.1
# pip install 带重试(网络不稳定时自动重试)
for i in 1 2 3; do
python3 -m pip install -q -r requirements-base.txt && break
echo "pip install requirements-base.txt 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
for i in 1 2 3; do
python3 -m pip install -q -r requirements.txt && break
echo "pip install requirements.txt 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
for i in 1 2 3; do
python3 -m pip install -q -r requirements-dev.txt && break
echo "pip install requirements-dev.txt 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
for i in 1 2 3; do
python3 -m pip install --no-binary :all: black==26.5.1 isort==8.0.1 && break
echo "pip install black/isort 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
- name: Run all quality checks
shell: bash
env:
@@ -188,8 +210,9 @@ jobs:
- check-frontend-only
- validate
env:
DATABASE_URL: postgresql+psycopg://postgres:postgres@127.0.0.1:5432/xiaoxia_saas
DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas
USE_IN_MEMORY_DB: 'false'
CI_USE_SHARED_PG: 'true'
OSS_ACCESS_KEY_ID: placeholder
OSS_ACCESS_KEY_SECRET: placeholder
OSS_BUCKET_NAME: xiaoxia-autocut
@@ -244,9 +267,17 @@ jobs:
- name: Record job start time
shell: sh
run: bash scripts/ci/step_timer_start.sh
- name: Install frontend dependencies
- name: Install frontend dependencies (with retry)
shell: sh
run: bash scripts/ci/step_frontend_install.sh
run: |
set -eu
# npm install 带重试(网络不稳定时自动重试)
for i in 1 2 3; do
bash scripts/ci/step_frontend_install.sh && break
echo "前端依赖安装失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
- name: Run ESLint
shell: sh
run: bash scripts/ci/step_frontend_run.sh "npx --no-install eslint src --ext .ts,.tsx --max-warnings 0"
@@ -289,9 +320,16 @@ jobs:
- name: Record job start time
shell: sh
run: bash scripts/ci/step_timer_start.sh
- name: Install frontend dependencies (vitest only)
- name: Install frontend dependencies (vitest only, with retry)
shell: sh
run: bash scripts/ci/step_frontend_install.sh vitest
run: |
set -eu
for i in 1 2 3; do
bash scripts/ci/step_frontend_install.sh vitest && break
echo "前端依赖安装失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
- name: Run Vitest with coverage
shell: sh
run: bash scripts/ci/step_frontend_run.sh "npx --no-install vitest run --coverage"
@@ -383,7 +421,7 @@ jobs:
fi
docker buildx inspect --bootstrap
- name: Build and push ${{ matrix.service_display }} image
- name: Build and push ${{ matrix.service_display }} image (with retry)
shell: sh
run: |
set -eu
@@ -396,7 +434,23 @@ jobs:
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
fi
bash scripts/ci/docker_build_push.sh ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS
# Docker build 带重试:失败自动重试2次,第2次重试加--no-cache
NO_CACHE_FLAG=""
for i in 1 2 3; do
echo "=== Docker build 尝试 $i/3 ==="
if bash scripts/ci/docker_build_push.sh $NO_CACHE_FLAG ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS; then
echo "✅ Docker build 成功"
break
fi
echo "❌ Docker build 失败(尝试 $i/3"
[ $i -eq 3 ] && exit 1
sleep 10
# 第2次重试使用 --no-cache
if [ $i -eq 2 ]; then
NO_CACHE_FLAG="--no-cache"
echo "下次重试将使用 --no-cache"
fi
done
echo
echo "${{ matrix.service_display }} image pushed: ${IMAGE_TAG}"
@@ -507,7 +561,8 @@ jobs:
ssh -p "$staging_port" -i "$key_path" -o StrictHostKeyChecking=no "${staging_user}@${staging_host}" "echo SSH_CONNECTION_OK && hostname"
echo "SSH connection verified"
cat scripts/ci_staging_deploy.sh | ssh -p "$staging_port" -i "$key_path" -o StrictHostKeyChecking=no "${staging_user}@${staging_host}" "IMAGE_TAG='${GITHUB_SHA}' ACR_USERNAME='${ACR_USERNAME}' ACR_PASSWORD='${ACR_PASSWORD}' sh"
# 通过环境变量传递凭证,避免命令行引号转义问题
cat scripts/ci_staging_deploy.sh | ssh -p "$staging_port" -i "$key_path" -o StrictHostKeyChecking=no "${staging_user}@${staging_host}" "IMAGE_TAG=${GITHUB_SHA} ACR_USERNAME=${ACR_USERNAME} ACR_PASSWORD=${ACR_PASSWORD} sh"
- name: Staging health check + auto rollback
if: success()
@@ -706,7 +761,7 @@ jobs:
fi
docker buildx inspect --bootstrap
- name: Build and push production ${{ matrix.service_display }} image
- name: Build and push production ${{ matrix.service_display }} image (with retry)
shell: sh
run: |
set -eu
@@ -719,7 +774,23 @@ jobs:
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-production.conf"
fi
bash scripts/ci/docker_build_push.sh ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS
# Docker build 带重试:失败自动重试2次,第2次重试加--no-cache
NO_CACHE_FLAG=""
for i in 1 2 3; do
echo "=== Docker build 尝试 $i/3 ==="
if bash scripts/ci/docker_build_push.sh $NO_CACHE_FLAG ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS; then
echo "✅ Docker build 成功"
break
fi
echo "❌ Docker build 失败(尝试 $i/3"
[ $i -eq 3 ] && exit 1
sleep 10
# 第2次重试使用 --no-cache
if [ $i -eq 2 ]; then
NO_CACHE_FLAG="--no-cache"
echo "下次重试将使用 --no-cache"
fi
done
echo
echo "${{ matrix.service_display }} production image pushed: ${IMAGE_TAG}"
@@ -816,7 +887,8 @@ jobs:
ssh -p "$production_port" -i "$key_path" -o StrictHostKeyChecking=no "${production_user}@${production_host}" "echo SSH_CONNECTION_OK && hostname"
echo "SSH connection verified"
cat scripts/ci_production_deploy.sh | ssh -p "$production_port" -i "$key_path" -o StrictHostKeyChecking=no "${production_user}@${production_host}" "IMAGE_TAG='${GITHUB_REF_NAME}' ACR_USERNAME='${ACR_USERNAME}' ACR_PASSWORD='${ACR_PASSWORD}' sh"
# 通过环境变量传递凭证,避免命令行引号转义问题
cat scripts/ci_production_deploy.sh | ssh -p "$production_port" -i "$key_path" -o StrictHostKeyChecking=no "${production_user}@${production_host}" "IMAGE_TAG=${GITHUB_REF_NAME} ACR_USERNAME=${ACR_USERNAME} ACR_PASSWORD=${ACR_PASSWORD} sh"
- name: Production health check + auto rollback
if: success()
+10 -34
View File
@@ -1,9 +1,16 @@
#!/bin/bash
# 通用Docker镜像构建+推送脚本(local cache为主 + registry cache兜底)
# M-2优化:解决registry缓存导入慢(247s)和推送不稳定问题
# 用法: docker_build_push.sh <Dockerfile> <image_tag> <cache_ref> [build_arg...]
# 用法: docker_build_push.sh [--no-cache] <Dockerfile> <image_tag> <cache_ref> [build_arg...]
set -eu
NO_CACHE_FLAG=""
if [ "$1" = "--no-cache" ]; then
NO_CACHE_FLAG="--no-cache"
shift
echo "模式: --no-cache (不使用缓存,全新构建)"
fi
DOCKERFILE="$1"
IMAGE_TAG="$2"
CACHE_REF="$3"
@@ -38,6 +45,7 @@ build_with_cache_retry() {
local exit_code
set +e
build_output=$(docker buildx build \
$NO_CACHE_FLAG \
$BUILD_ARGS \
--cache-from "type=local,src=${LOCAL_CACHE_DIR}" \
--cache-from "type=registry,ref=${CACHE_REF},ignore-error=true" \
@@ -71,6 +79,7 @@ build_with_cache_retry() {
# 重试完还是失败,不用本地缓存最后试一次(只从registry读)
echo "⚠️ All cached attempts failed, building without local cache..."
docker buildx build \
$NO_CACHE_FLAG \
$BUILD_ARGS \
--cache-from "type=registry,ref=${CACHE_REF},ignore-error=true" \
--cache-to "type=local,dest=${LOCAL_CACHE_DIR},mode=max" \
@@ -91,38 +100,5 @@ echo ""
echo "Image pushed: ${IMAGE_TAG}"
echo "Local cache updated"
# DISABLED: registry cache too slow echo ""
# DISABLED: registry cache too slow echo "=== Step 2: Sync registry cache (best effort, retries 3x) ==="
# DISABLED: registry cache too slow CACHE_TO_REGISTRY="type=registry,ref=${CACHE_REF},mode=max,compression=zstd"
# DISABLED: registry cache too slow
# DISABLED: registry cache too slow MAX_RETRIES=3
# DISABLED: registry cache too slow SUCCESS=0
# DISABLED: registry cache too slow for attempt in $(seq 1 $MAX_RETRIES); do
# DISABLED: registry cache too slow echo "Registry cache sync attempt $attempt/$MAX_RETRIES"
# DISABLED: registry cache too slow if docker buildx build \
# DISABLED: registry cache too slow $BUILD_ARGS \
# DISABLED: registry cache too slow --cache-from "${CACHE_FROM_LOCAL}" \
# DISABLED: registry cache too slow --cache-to "${CACHE_TO_REGISTRY}" \
# DISABLED: registry cache too slow -f "${DOCKERFILE}" \
# DISABLED: registry cache too slow -t "${IMAGE_TAG}" \
# DISABLED: registry cache too slow --push \
# DISABLED: registry cache too slow .; then
# DISABLED: registry cache too slow echo "Registry cache synced (attempt $attempt)"
# DISABLED: registry cache too slow SUCCESS=1
# DISABLED: registry cache too slow break
# DISABLED: registry cache too slow else
# DISABLED: registry cache too slow echo "Registry cache sync failed (attempt $attempt)"
# DISABLED: registry cache too slow if [ $attempt -lt $MAX_RETRIES ]; then
# DISABLED: registry cache too slow WAIT=$((attempt * 5))
# DISABLED: registry cache too slow echo "Retrying in ${WAIT}s..."
# DISABLED: registry cache too slow sleep $WAIT
# DISABLED: registry cache too slow fi
# DISABLED: registry cache too slow fi
# DISABLED: registry cache too slow done
# DISABLED: registry cache too slow
# DISABLED: registry cache too slow if [ $SUCCESS -eq 0 ]; then
# DISABLED: registry cache too slow echo "WARNING: Registry cache sync failed after $MAX_RETRIES attempts (non-fatal, local cache still works)"
# DISABLED: registry cache too slow fi
echo ""
echo "Build completed: ${IMAGE_TAG}"
+219 -57
View File
@@ -8,10 +8,31 @@ echo "=== CI Integration Tests 开始 ==="
# --- 安装依赖 ---
echo ""
echo "=== 安装 Python 依赖 ==="
python3 -m pip install -q -r requirements-base.txt
python3 -m pip install -q -r requirements.txt
python3 -m pip install -q -r requirements-dev.txt
python3 -m pip install -q pytest-rerunfailures
# pip install 带重试(网络不稳定时自动重试)
for i in 1 2 3; do
python3 -m pip install -q -r requirements-base.txt && break
echo "pip install requirements-base.txt 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
for i in 1 2 3; do
python3 -m pip install -q -r requirements.txt && break
echo "pip install requirements.txt 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
for i in 1 2 3; do
python3 -m pip install -q -r requirements-dev.txt && break
echo "pip install requirements-dev.txt 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
for i in 1 2 3; do
python3 -m pip install -q pytest-rerunfailures && break
echo "pip install pytest-rerunfailures 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
pytest --version
# --- 安装 ffmpeg ---
@@ -19,6 +40,104 @@ echo ""
echo "=== 安装 ffmpeg ==="
bash scripts/ci/step_install_ffmpeg.sh
# --- DooD模式检测:确定宿主机访问地址 ---
# DooD模式下,docker run启动的容器跑在宿主机Docker上
# 需要用宿主机IP访问映射端口
# 检测策略:host.docker.internal -> docker0桥接IP -> 容器IP直连 -> 默认网关 -> 127.0.0.1
detect_docker_host() {
local test_port="${1:-5432}"
# 候选IP列表
local candidates=()
# 1. host.docker.internalrunner配置了--add-host时可用)
if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then
candidates+=("host.docker.internal")
fi
# 2. docker0 桥接网关 (172.17.0.1)
candidates+=("172.17.0.1")
# 3. 默认网关(容器网络的网关即宿主机)
local gw=""
gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1)
if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then
candidates+=("$gw")
fi
# 4. 宿主机可能的IP:容器同网段的.1或.254
local my_ip=""
my_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
if [ -n "$my_ip" ]; then
# 尝试同网段的常见宿主机IP
local subnet=$(echo "$my_ip" | cut -d. -f1-3)
candidates+=("${subnet}.1")
candidates+=("${subnet}.254")
fi
# 5. 127.0.0.1 最后尝试
candidates+=("127.0.0.1")
# 测试每个候选IP
for candidate in "${candidates[@]}"; do
if python3 -c "
import socket
s = socket.socket()
s.settimeout(2)
try:
s.connect(('$candidate', $test_port))
s.close()
print('ok')
except:
pass
" 2>/dev/null | grep -q ok; then
echo "$candidate"
return 0
fi
done
# 都失败则返回127.0.0.1
echo "127.0.0.1"
return 1
}
# 获取宿主机IP(先尝试用共享PG端口5433测试,再回退到其他端口)
if [ -S /var/run/docker.sock ]; then
# 先用共享PG端口5433探测
DOCKER_HOST_IP=$(detect_docker_host 5433)
if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then
# 如果共享PG端口探测失败,说明不在DooD或共享PG不可用,再试其他端口
DOCKER_HOST_IP=$(detect_docker_host 22)
fi
echo "检测到DooD模式(/var/run/docker.sock已挂载),宿主机地址: $DOCKER_HOST_IP"
else
DOCKER_HOST_IP="127.0.0.1"
echo "非DooD模式,使用 127.0.0.1"
fi
PG_HOST="$DOCKER_HOST_IP"
REDIS_HOST="$DOCKER_HOST_IP"
echo "PG host: $PG_HOST, Redis host: $REDIS_HOST"
# --- 指数退避TCP连接检查函数 ---
# 用法: wait_tcp_ready host port max_attempts
wait_tcp_ready() {
local host="$1"
local port="$2"
local max_attempts="${3:-5}"
local delay=1
local attempt=1
while [ "$attempt" -le "$max_attempts" ]; do
if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
return 0
fi
echo "TCP连接尝试 $attempt/$max_attempts 失败,${delay}s后重试..."
sleep "$delay"
delay=$((delay * 2))
attempt=$((attempt + 1))
done
return 1
}
# --- 启动 Redis ---
echo ""
echo "=== 启动 Redis ==="
@@ -33,67 +152,91 @@ docker run -d --name "$REDIS_CONTAINER" \
redis:7-alpine
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | cut -d: -f2)
echo "Redis port: $REDIS_PORT"
export REDIS_URL="redis://127.0.0.1:$REDIS_PORT/0"
export REDIS_URL="redis://${REDIS_HOST}:${REDIS_PORT}/0"
# 等待容器健康
for i in $(seq 1 15); do
if docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" 2>/dev/null | grep -q healthy; then
echo "Redis is ready on port $REDIS_PORT"
echo "Redis container is ready on port $REDIS_PORT"
break
fi
echo "Waiting for Redis... ($i/15)"
echo "Waiting for Redis container health... ($i/15)"
sleep 2
done
docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" | grep -q healthy
# 额外验证:确保从宿主侧通过映射端口实际能连上 Redis
for i in $(seq 1 20); do
if python3 -c "import socket; s=socket.socket(); s.settimeout(2); s.connect(('127.0.0.1', $REDIS_PORT)); s.close()" 2>/dev/null; then
echo "TCP connectivity to Redis confirmed on port $REDIS_PORT"
break
fi
echo "Waiting for TCP connectivity to Redis on port $REDIS_PORT... ($i/20)"
sleep 2
done
python3 -c "import socket; s=socket.socket(); s.settimeout(2); s.connect(('127.0.0.1', $REDIS_PORT)); s.close()"
# TCP连通性检查(指数退避)
echo "验证Redis TCP连通性 ($REDIS_HOST:$REDIS_PORT)..."
wait_tcp_ready "$REDIS_HOST" "$REDIS_PORT" 5
echo "TCP connectivity to Redis confirmed on port $REDIS_PORT"
# --- 启动 PostgreSQL ---
# --- 启动/连接 PostgreSQL ---
echo ""
echo "=== 启动 PostgreSQL ==="
PG_CONTAINER="ci-pg-${GITHUB_RUN_ID:-$$}"
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker run -d --name "$PG_CONTAINER" \
--shm-size=256m \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=xiaoxia_saas \
-P \
--health-cmd "pg_isready -U postgres" \
--health-interval 5s \
--health-timeout 5s \
--health-retries 12 \
postgres:16
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
echo "PostgreSQL port: $PG_PORT"
export DATABASE_URL="postgresql+psycopg://postgres:postgres@127.0.0.1:$PG_PORT/xiaoxia_saas"
for i in $(seq 1 30); do
if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
echo "PostgreSQL is ready on port $PG_PORT"
break
fi
echo "Waiting for PostgreSQL... ($i/30)"
sleep 2
done
docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy
echo "=== 准备 PostgreSQL ==="
USE_SHARED_PG="${CI_USE_SHARED_PG:-false}"
CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}"
# 额外验证:确保从宿主侧通过映射端口实际能连上 PG(端口映射可能有延迟)
for i in $(seq 1 30); do
if python3 -c "import socket; s=socket.socket(); s.settimeout(2); s.connect(('127.0.0.1', $PG_PORT)); s.close()" 2>/dev/null; then
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
break
fi
echo "Waiting for TCP connectivity to PostgreSQL on port $PG_PORT... ($i/30)"
sleep 2
done
python3 -c "import socket; s=socket.socket(); s.settimeout(2); s.connect(('127.0.0.1', $PG_PORT)); s.close()"
if [ "$USE_SHARED_PG" = "true" ]; then
# 使用常驻共享PG实例
echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true"
SHARED_PG_HOST="$PG_HOST"
SHARED_PG_PORT="5433"
SHARED_PG_USER="postgres"
SHARED_PG_PASSWORD="ci_pg_2026!"
echo "等待共享PG连接就绪..."
wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5
# 创建独立数据库
echo "创建测试数据库: $CI_DB_NAME"
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
import psycopg2
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
conn.autocommit = True
cur = conn.cursor()
cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"')
cur.close()
conn.close()
"
export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}"
echo "✅ 共享PG数据库已创建: $CI_DB_NAME"
PG_CONTAINER=""
else
# 使用临时PG容器
echo "使用临时PG容器模式"
PG_CONTAINER="ci-pg-${GITHUB_RUN_ID:-$$}"
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker run -d --name "$PG_CONTAINER" \
--shm-size=256m \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=xiaoxia_saas \
-P \
--health-cmd "pg_isready -U postgres" \
--health-interval 5s \
--health-timeout 5s \
--health-retries 12 \
postgres:16
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
echo "PostgreSQL port: $PG_PORT"
export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas"
# 等待容器健康
for i in $(seq 1 30); do
if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
echo "PostgreSQL container is ready on port $PG_PORT"
break
fi
echo "Waiting for PostgreSQL container health... ($i/30)"
sleep 2
done
docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy
# TCP连通性检查(指数退避)
echo "验证PostgreSQL TCP连通性 ($PG_HOST:$PG_PORT)..."
wait_tcp_ready "$PG_HOST" "$PG_PORT" 5
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
fi
# --- 执行迁移 ---
echo ""
@@ -139,10 +282,29 @@ set -e
# --- 清理 ---
echo ""
echo "=== 清理容器 ==="
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
echo "=== 清理 ==="
if [ "$USE_SHARED_PG" = "true" ]; then
# 清理共享PG上的测试数据库
echo "清理共享PG测试数据库: $CI_DB_NAME"
PGPASSWORD="${SHARED_PG_PASSWORD}" python3 -c "
import psycopg2
conn = psycopg2.connect(host='${SHARED_PG_HOST}', port=${SHARED_PG_PORT}, user='${SHARED_PG_USER}', password='${SHARED_PG_PASSWORD}', dbname='postgres')
conn.autocommit = True
cur = conn.cursor()
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
cur.close()
conn.close()
" 2>/dev/null || echo "WARN: 数据库清理失败(可能已被清理)"
echo "✅ 共享PG数据库已清理"
else
# 清理临时PG容器
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
echo "✅ PG容器已清理"
fi
# 清理Redis容器
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
echo "✅ 清理完成"
echo "✅ Redis容器已清理"
# --- 覆盖率汇总 ---
echo ""
+193 -42
View File
@@ -121,11 +121,18 @@ echo "=== [3/8] Type check (mypy) ==="
bash scripts/ci/mypy_check.sh
echo "✅ Mypy type check passed"
# --- Bandit 安全扫描 ---
# --- Bandit 安全扫描(仅告警) ---
echo ""
echo "=== [4/8] Security scan (bandit) ==="
echo "=== [4/8] Security scan (bandit, advisory only) ==="
set +e
bandit -r apps packages -q -ll
echo "✅ Bandit security scan passed"
BANDIT_EXIT=$?
set -e
if [ "$BANDIT_EXIT" -ne 0 ]; then
echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)"
else
echo "✅ Bandit security scan passed"
fi
# --- Pip-audit 依赖漏洞扫描(仅告警) ---
echo ""
@@ -170,47 +177,191 @@ echo "✅ Release scripts syntax OK"
# --- Alembic 迁移验证 ---
echo ""
echo "=== [8/8] Alembic migrations validation (with isolated PG) ==="
PG_CONTAINER=ci-pg-validate-${GITHUB_RUN_ID:-$$}
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker run -d --name "$PG_CONTAINER" \
--shm-size=256m \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=xiaoxia_saas \
-P \
--health-cmd "pg_isready -U postgres" \
--health-interval 3s \
--health-timeout 3s \
--health-retries 20 \
postgres:16-alpine
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
echo "PostgreSQL port: $PG_PORT"
export DATABASE_URL=postgresql+psycopg://postgres:postgres@127.0.0.1:$PG_PORT/xiaoxia_saas
for i in $(seq 1 30); do
if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
echo "PostgreSQL is ready on port $PG_PORT"
break
fi
echo "Waiting for PostgreSQL... ($i/30)"
sleep 2
done
docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy
echo "=== [8/8] Alembic migrations validation ==="
# 额外验证:确保从宿主侧通过映射端口实际能连上(端口映射可能有延迟)
for i in $(seq 1 30); do
if python3 -c "import socket; s=socket.socket(); s.settimeout(2); s.connect(('127.0.0.1', $PG_PORT)); s.close()" 2>/dev/null; then
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
break
fi
echo "Waiting for TCP connectivity to PostgreSQL on port $PG_PORT... ($i/30)"
sleep 2
done
python3 -c "import socket; s=socket.socket(); s.settimeout(2); s.connect(('127.0.0.1', $PG_PORT)); s.close()"
# --- DooD模式检测:确定宿主机访问地址 ---
# DooD模式下,docker run启动的容器跑在宿主机Docker上
# 需要用宿主机IP访问映射端口
# 检测策略:host.docker.internal -> docker0桥接IP -> 容器IP直连 -> 默认网关 -> 127.0.0.1
detect_docker_host() {
local test_port="${1:-5432}"
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
echo "✅ Alembic migrations applied successfully"
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
# 候选IP列表
local candidates=()
# 1. host.docker.internalrunner配置了--add-host时可用)
if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then
candidates+=("host.docker.internal")
fi
# 2. docker0 桥接网关 (172.17.0.1)
candidates+=("172.17.0.1")
# 3. 默认网关(容器网络的网关即宿主机)
local gw=""
gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1)
if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then
candidates+=("$gw")
fi
# 4. 宿主机可能的IP:容器同网段的.1或.254
local my_ip=""
my_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
if [ -n "$my_ip" ]; then
# 尝试同网段的常见宿主机IP
local subnet=$(echo "$my_ip" | cut -d. -f1-3)
candidates+=("${subnet}.1")
candidates+=("${subnet}.254")
fi
# 5. 127.0.0.1 最后尝试
candidates+=("127.0.0.1")
# 测试每个候选IP
for candidate in "${candidates[@]}"; do
if python3 -c "
import socket
s = socket.socket()
s.settimeout(2)
try:
s.connect(('$candidate', $test_port))
s.close()
print('ok')
except:
pass
" 2>/dev/null | grep -q ok; then
echo "$candidate"
return 0
fi
done
# 都失败则返回127.0.0.1
echo "127.0.0.1"
return 1
}
# 获取宿主机IP(先尝试用共享PG端口5433测试,再回退到其他端口)
if [ -S /var/run/docker.sock ]; then
# 先用共享PG端口5433探测
DOCKER_HOST_IP=$(detect_docker_host 5433)
if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then
# 如果共享PG端口探测失败,说明不在DooD或共享PG不可用,再试其他端口
DOCKER_HOST_IP=$(detect_docker_host 22)
fi
echo "检测到DooD模式(/var/run/docker.sock已挂载),宿主机地址: $DOCKER_HOST_IP"
else
DOCKER_HOST_IP="127.0.0.1"
echo "非DooD模式,使用 127.0.0.1"
fi
PG_HOST="$DOCKER_HOST_IP"
echo "PG host: $PG_HOST"
# 指数退避TCP连接检查函数
# 用法: wait_tcp_ready host port max_attempts
wait_tcp_ready() {
local host="$1"
local port="$2"
local max_attempts="${3:-5}"
local delay=1
local attempt=1
while [ "$attempt" -le "$max_attempts" ]; do
if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
return 0
fi
echo "TCP连接尝试 $attempt/$max_attempts 失败,${delay}s后重试..."
sleep "$delay"
delay=$((delay * 2))
attempt=$((attempt + 1))
done
return 1
}
USE_SHARED_PG="${CI_USE_SHARED_PG:-false}"
if [ "$USE_SHARED_PG" = "true" ]; then
# 使用常驻共享PG实例(host.docker.internal:5433
echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true"
SHARED_PG_HOST="$PG_HOST"
SHARED_PG_PORT="5433"
SHARED_PG_USER="postgres"
SHARED_PG_PASSWORD="ci_pg_2026!"
CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}"
echo "等待共享PG连接就绪..."
wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5
# 创建独立数据库
echo "创建测试数据库: $CI_DB_NAME"
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
import psycopg2
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
conn.autocommit = True
cur = conn.cursor()
cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"')
cur.close()
conn.close()
"
export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}"
echo "✅ 共享PG数据库已创建: $CI_DB_NAME"
# 执行迁移
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
echo "✅ Alembic migrations applied successfully"
# 清理数据库
echo "清理测试数据库: $CI_DB_NAME"
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
import psycopg2
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
conn.autocommit = True
cur = conn.cursor()
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
cur.close()
conn.close()
" 2>/dev/null || echo "WARN: 数据库清理失败(可能已被清理)"
echo "✅ 共享PG数据库已清理"
else
# 使用临时PG容器(默认模式)
echo "使用临时PG容器模式"
PG_CONTAINER=ci-pg-validate-${GITHUB_RUN_ID:-$$}
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker run -d --name "$PG_CONTAINER" \
--shm-size=256m \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=xiaoxia_saas \
-P \
--health-cmd "pg_isready -U postgres" \
--health-interval 3s \
--health-timeout 3s \
--health-retries 20 \
postgres:16-alpine
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
echo "PostgreSQL port: $PG_PORT"
export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas"
# 等待容器健康
for i in $(seq 1 30); do
if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
echo "PostgreSQL container is healthy on port $PG_PORT"
break
fi
echo "Waiting for PostgreSQL container health... ($i/30)"
sleep 2
done
docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy
# TCP连通性检查(指数退避)
echo "验证TCP连通性 ($PG_HOST:$PG_PORT)..."
wait_tcp_ready "$PG_HOST" "$PG_PORT" 5
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
# 执行迁移
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
echo "✅ Alembic migrations applied successfully"
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
fi
echo ""
echo "=== CI Validate: 所有检查通过 ✅ ==="