From 8b661691370a209d76047d5a254ee52d3a0296c8 Mon Sep 17 00:00:00 2001 From: CI Bot Date: Sun, 19 Jul 2026 23:21:00 +0800 Subject: [PATCH] =?UTF-8?q?fix(ci):=20=E5=A4=9A=E7=B3=BB=E7=BB=9F=E5=85=BC?= =?UTF-8?q?=E5=AE=B9=E7=9A=84PG/Redis=E5=AE=89=E8=A3=85=E6=96=B9=E5=BC=8F?= =?UTF-8?q?=EF=BC=88apk/apt/yum/dnf=EF=BC=89=EF=BC=8C=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E4=BC=98=E5=85=88Docker=20fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runner容器是Alpine系统，没有apt-get，导致127错误。 改为多包管理器兼容：apk(Alpine) / apt-get(Debian/Ubuntu) / yum/dnf(RHEL/CentOS) 优先本地安装，失败则fallback到Docker方式。 这样在任何runner环境下都能最大限度保证PG和Redis可用。 --- scripts/ci/run_integration_tests.sh | 148 ++++++++++++++++++++-------- scripts/ci/run_validate.sh | 94 +++++++++++------- 2 files changed, 168 insertions(+), 74 deletions(-) diff --git a/scripts/ci/run_integration_tests.sh b/scripts/ci/run_integration_tests.sh index 3a664821a..87b4b3388 100755 --- a/scripts/ci/run_integration_tests.sh +++ b/scripts/ci/run_integration_tests.sh @@ -22,55 +22,123 @@ bash scripts/ci/step_install_ffmpeg.sh # --- 安装并启动 Redis --- echo "" echo "=== 安装并启动 Redis ===" -export DEBIAN_FRONTEND=noninteractive -apt-get update -qq > /dev/null 2>&1 -apt-get install -y -qq redis-server > /dev/null 2>&1 -redis-server --daemonize yes --port 6379 -sleep 1 -for i in $(seq 1 10); do - if redis-cli ping 2>/dev/null | grep -q PONG; then - echo "Redis is ready on 127.0.0.1:6379" - break +# 兼容多种包管理器 +install_redis() { + if command -v apk > /dev/null 2>&1; then + apk add --no-cache redis > /dev/null 2>&1 + elif command -v apt-get > /dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq > /dev/null 2>&1 + apt-get install -y -qq redis-server > /dev/null 2>&1 + elif command -v yum > /dev/null 2>&1; then + yum install -y -q redis > /dev/null 2>&1 + elif command -v dnf > /dev/null 2>&1; then + dnf install -y -q redis > /dev/null 2>&1 + else + return 1 fi - echo "Waiting for Redis... ($i/10)" +} +# 如果redis-cli不存在才安装 +if ! command -v redis-cli > /dev/null 2>&1; then + install_redis || echo "Warning: Redis installation failed, trying Docker..." +fi +# 优先用本地redis-server，失败再fallback +if command -v redis-server > /dev/null 2>&1; then + redis-server --daemonize yes --port 6379 2>/dev/null || true sleep 1 -done -redis-cli ping | grep -q PONG -export REDIS_URL="redis://127.0.0.1:6379/0" + for i in $(seq 1 10); do + if redis-cli ping 2>/dev/null | grep -q PONG; then + echo "Redis is ready on 127.0.0.1:6379" + break + fi + sleep 1 + done +fi +# 如果本地redis连不上，fallback到Docker +if ! redis-cli ping 2>/dev/null | grep -q PONG; then + echo "Local Redis not available, falling back to Docker..." + REDIS_CONTAINER="ci-redis-${GITHUB_RUN_ID:-$$}" + docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true + docker run -d --name "$REDIS_CONTAINER" -P --health-cmd "redis-cli ping" --health-interval 2s --health-timeout 2s --health-retries 10 redis:7-alpine + for i in $(seq 1 20); do + if docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" 2>/dev/null | grep -q healthy; then + break + fi + sleep 2 + done + REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | cut -d: -f2) + export REDIS_URL="redis://127.0.0.1:$REDIS_PORT/0" + echo "Redis (Docker) ready on 127.0.0.1:$REDIS_PORT" +else + export REDIS_URL="redis://127.0.0.1:6379/0" +fi # --- 安装并启动 PostgreSQL --- echo "" echo "=== 安装并启动 PostgreSQL ===" -apt-get install -y -qq postgresql postgresql-client > /dev/null 2>&1 -PG_VERSION=$(pg_lsclusters -h | head -1 | awk '{print $1}') -PG_CLUSTER=$(pg_lsclusters -h | head -1 | awk '{print $2}') -echo "PostgreSQL version: $PG_VERSION, cluster: $PG_CLUSTER" - -# 修改认证方式为trust -PG_HBA="/etc/postgresql/$PG_VERSION/$PG_CLUSTER/pg_hba.conf" -sed -i "s/local all all peer/local all all trust/" "$PG_HBA" -sed -i "s/host all all 127.0.0.1\/32 scram-sha-256/host all all 127.0.0.1\/32 trust/" "$PG_HBA" - -pg_ctlcluster $PG_VERSION $PG_CLUSTER start -sleep 2 - -# 创建用户和数据库 -su - postgres -c "psql -c "CREATE USER postgres WITH SUPERUSER PASSWORD 'postgres';"" 2>/dev/null || true -su - postgres -c "psql -c "CREATE DATABASE xiaoxia_saas OWNER postgres;"" 2>/dev/null || true - -# 等PG就绪 -for i in $(seq 1 20); do - if pg_isready -U postgres -h 127.0.0.1 -p 5432 2>/dev/null | grep -q "accepting connections"; then - echo "PostgreSQL is ready on 127.0.0.1:5432" - break +install_pg_local() { + if command -v pg_isready > /dev/null 2>&1; then + return 0 fi - echo "Waiting for PostgreSQL... ($i/20)" - sleep 2 -done -pg_isready -U postgres -h 127.0.0.1 -p 5432 | grep -q "accepting connections" + if command -v apk > /dev/null 2>&1; then + apk add --no-cache postgresql postgresql-client > /dev/null 2>&1 + # Alpine需要手动初始化 + mkdir -p /var/lib/postgresql/data + chown postgres:postgres /var/lib/postgresql/data + su - postgres -c "initdb -D /var/lib/postgresql/data" > /dev/null 2>&1 + su - postgres -c "pg_ctl -D /var/lib/postgresql/data -l /var/lib/postgresql/logfile start" > /dev/null 2>&1 + sleep 2 + su - postgres -c "psql -c "CREATE USER postgres WITH SUPERUSER PASSWORD 'postgres';"" > /dev/null 2>&1 + su - postgres -c "psql -c "CREATE DATABASE xiaoxia_saas OWNER postgres;"" > /dev/null 2>&1 + elif command -v apt-get > /dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq > /dev/null 2>&1 + apt-get install -y -qq postgresql postgresql-client > /dev/null 2>&1 + PG_VERSION=$(pg_lsclusters -h | head -1 | awk '{print $1}') + PG_CLUSTER=$(pg_lsclusters -h | head -1 | awk '{print $2}') + PG_HBA="/etc/postgresql/$PG_VERSION/$PG_CLUSTER/pg_hba.conf" + sed -i "s/local.*all.*all.*peer/local all all trust/" "$PG_HBA" 2>/dev/null || true + sed -i "s/host.*all.*all.*127.0.0.1.*scram-sha-256/host all all 127.0.0.1/32 trust/" "$PG_HBA" 2>/dev/null || true + pg_ctlcluster "$PG_VERSION" "$PG_CLUSTER" start 2>/dev/null || true + sleep 2 + su - postgres -c "psql -c "CREATE USER postgres WITH SUPERUSER PASSWORD 'postgres';"" 2>/dev/null || true + su - postgres -c "psql -c "CREATE DATABASE xiaoxia_saas OWNER postgres;"" 2>/dev/null || true + else + return 1 + fi +} -export DATABASE_URL="postgresql+psycopg://postgres:postgres@127.0.0.1:5432/xiaoxia_saas" +PG_LOCAL_OK=0 +if install_pg_local; then + for i in $(seq 1 15); do + if pg_isready -U postgres -h 127.0.0.1 -p 5432 2>/dev/null | grep -q "accepting connections"; then + echo "PostgreSQL is ready on 127.0.0.1:5432 (local install)" + PG_LOCAL_OK=1 + break + fi + sleep 2 + done +fi + +# 本地安装失败则fallback到Docker +if [ "$PG_LOCAL_OK" != "1" ]; then + echo "Local PostgreSQL not available, falling back to Docker..." + PG_CONTAINER="ci-pg-${GITHUB_RUN_ID:-$$}" + docker rm -f "$PG_CONTAINER" 2>/dev/null || true + docker run -d --name "$PG_CONTAINER" -P --shm-size=256m -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=xiaoxia_saas --health-cmd "pg_isready -U postgres" --health-interval 3s --health-timeout 3s --health-retries 20 postgres:16-alpine + for i in $(seq 1 30); do + if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then + break + fi + sleep 2 + done + PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2) + echo "PostgreSQL (Docker) ready on 127.0.0.1:$PG_PORT" + export DATABASE_URL="postgresql+psycopg://postgres:postgres@127.0.0.1:$PG_PORT/xiaoxia_saas" +else + export DATABASE_URL="postgresql+psycopg://postgres:postgres@127.0.0.1:5432/xiaoxia_saas" +fi echo "=== 执行 Alembic 迁移 ===" PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head diff --git a/scripts/ci/run_validate.sh b/scripts/ci/run_validate.sh index 222335f7f..ae986b89a 100755 --- a/scripts/ci/run_validate.sh +++ b/scripts/ci/run_validate.sh @@ -169,44 +169,70 @@ bash -n scripts/init_production_env.sh echo "✅ Release scripts syntax OK" # --- Alembic 迁移验证 --- -# 直接在runner容器中安装PG，不用Docker跑PG，彻底避免DinD网络问题 echo "" -echo "=== [8/8] Alembic migrations validation (with local PG) ===" +echo "=== [8/8] Alembic migrations validation ===" -# 安装并启动PostgreSQL -export DEBIAN_FRONTEND=noninteractive -apt-get update -qq > /dev/null 2>&1 -apt-get install -y -qq postgresql postgresql-client > /dev/null 2>&1 - -# 启动PG服务 -pg_lsclusters -PG_VERSION=$(pg_lsclusters -h | head -1 | awk '{print $1}') -PG_CLUSTER=$(pg_lsclusters -h | head -1 | awk '{print $2}') -echo "PostgreSQL version: $PG_VERSION, cluster: $PG_CLUSTER" - -# 修改认证方式为trust（本地连接不需要密码） -sed -i "s/local all all peer/local all all trust/" /etc/postgresql/$PG_VERSION/$PG_CLUSTER/pg_hba.conf -sed -i "s/host all all 127.0.0.1\/32 scram-sha-256/host all all 127.0.0.1\/32 trust/" /etc/postgresql/$PG_VERSION/$PG_CLUSTER/pg_hba.conf - -pg_ctlcluster $PG_VERSION $PG_CLUSTER start -sleep 2 - -# 创建用户和数据库 -su - postgres -c "psql -c "CREATE USER postgres WITH SUPERUSER PASSWORD 'postgres';"" 2>/dev/null || true -su - postgres -c "psql -c "CREATE DATABASE xiaoxia_saas OWNER postgres;"" 2>/dev/null || true - -# 等PG就绪 -for i in $(seq 1 20); do - if pg_isready -U postgres -h 127.0.0.1 -p 5432 2>/dev/null | grep -q "accepting connections"; then - echo "PostgreSQL is ready on 127.0.0.1:5432" - break +install_pg_local() { + if command -v pg_isready > /dev/null 2>&1; then + return 0 fi - echo "Waiting for PostgreSQL... ($i/20)" - sleep 2 -done -pg_isready -U postgres -h 127.0.0.1 -p 5432 | grep -q "accepting connections" + if command -v apk > /dev/null 2>&1; then + apk add --no-cache postgresql postgresql-client > /dev/null 2>&1 + mkdir -p /var/lib/postgresql/data + chown postgres:postgres /var/lib/postgresql/data + su - postgres -c "initdb -D /var/lib/postgresql/data" > /dev/null 2>&1 + su - postgres -c "pg_ctl -D /var/lib/postgresql/data -l /tmp/pg.log start" > /dev/null 2>&1 + sleep 2 + su - postgres -c "psql -c "CREATE USER postgres WITH SUPERUSER PASSWORD 'postgres';"" > /dev/null 2>&1 + su - postgres -c "psql -c "CREATE DATABASE xiaoxia_saas OWNER postgres;"" > /dev/null 2>&1 + elif command -v apt-get > /dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq > /dev/null 2>&1 + apt-get install -y -qq postgresql postgresql-client > /dev/null 2>&1 + PG_VERSION=$(pg_lsclusters -h | head -1 | awk '{print $1}') + PG_CLUSTER=$(pg_lsclusters -h | head -1 | awk '{print $2}') + PG_HBA="/etc/postgresql/$PG_VERSION/$PG_CLUSTER/pg_hba.conf" + sed -i "s/local.*all.*all.*peer/local all all trust/" "$PG_HBA" 2>/dev/null || true + sed -i "s/host.*all.*all.*127.0.0.1.*scram-sha-256/host all all 127.0.0.1/32 trust/" "$PG_HBA" 2>/dev/null || true + pg_ctlcluster "$PG_VERSION" "$PG_CLUSTER" start 2>/dev/null || true + sleep 2 + su - postgres -c "psql -c "CREATE USER postgres WITH SUPERUSER PASSWORD 'postgres';"" 2>/dev/null || true + su - postgres -c "psql -c "CREATE DATABASE xiaoxia_saas OWNER postgres;"" 2>/dev/null || true + else + return 1 + fi +} -export DATABASE_URL="postgresql+psycopg://postgres:postgres@127.0.0.1:5432/xiaoxia_saas" +# 优先本地安装，失败则fallback到Docker +PG_LOCAL_OK=0 +if install_pg_local; then + for i in $(seq 1 20); do + if pg_isready -U postgres -h 127.0.0.1 -p 5432 2>/dev/null | grep -q "accepting connections"; then + echo "PostgreSQL is ready on 127.0.0.1:5432 (local install)" + PG_LOCAL_OK=1 + break + fi + sleep 2 + done +fi + +if [ "$PG_LOCAL_OK" != "1" ]; then + echo "Local PG not available, falling back to Docker..." + PG_CONTAINER="ci-pg-validate-${GITHUB_RUN_ID:-$$}" + docker rm -f "$PG_CONTAINER" 2>/dev/null || true + docker run -d --name "$PG_CONTAINER" -P --shm-size=256m -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=xiaoxia_saas --health-cmd "pg_isready -U postgres" --health-interval 3s --health-timeout 3s --health-retries 20 postgres:16-alpine + for i in $(seq 1 30); do + if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then + break + fi + sleep 2 + done + PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2) + echo "PostgreSQL (Docker) ready on port $PG_PORT" + export DATABASE_URL="postgresql+psycopg://postgres:postgres@127.0.0.1:$PG_PORT/xiaoxia_saas" +else + export DATABASE_URL="postgresql+psycopg://postgres:postgres@127.0.0.1:5432/xiaoxia_saas" +fi PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head echo "✅ Alembic migrations applied successfully"