Compare commits

..

2 Commits

Author SHA1 Message Date
xiaoxia d1c2443fae Merge branch 'develop' into ci/cache-optimization-plus
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 36s
CI/CD Pipeline / Build Staging API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Frontend Lint (pull_request) Successful in 59s
CI/CD Pipeline / PR Build Web Image (pull_request) Successful in 43s
CI/CD Pipeline / Build Production API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production Web Image (pull_request) Has been skipped
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 52s
CI/CD Pipeline / Build Production Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Has been skipped
CI/CD Pipeline / Frontend Unit Tests (pull_request) Has been skipped
AI Code Review / AI Code Review (pull_request) Successful in 4m36s
CI/CD Pipeline / Validate Code Quality And Tests (pull_request) Successful in 1m40s
CI/CD Pipeline / Deploy Production (pull_request) Has been skipped
CI/CD Pipeline / Staging E2E Tests (pull_request) Has been skipped
PR Automation / Auto Approve on CI Green (pull_request) Successful in 5m6s
CI/CD Pipeline / Staging API Integration Tests (pull_request) Has been skipped
CI/CD Pipeline / ACR Image Cleanup (pull_request) Has been skipped
CI/CD Pipeline / PR Build API Image (pull_request) Successful in 6m18s
CI/CD Pipeline / Production Browser E2E (pull_request) Has been skipped
CI/CD Pipeline / Unit Tests (pull_request) Successful in 3m4s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 1m29s
CI/CD Pipeline / PR Build Worker Image (pull_request) Successful in 12m16s
PR Automation / Auto Merge on CI Green + Approved (pull_request) Successful in 35m3s
Preview Cleanup / Cleanup Preview Environment (pull_request) Successful in 47s
2026-07-22 01:04:47 +08:00
xiaoxia 43815e5f20 ci: 前端npm安装增加国内镜像源,加重试间隔
CI/CD Pipeline / Build Staging Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Has been skipped
CI/CD Pipeline / Build Production API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Staging E2E Tests (pull_request) Has been skipped
CI/CD Pipeline / Staging API Integration Tests (pull_request) Has been skipped
CI/CD Pipeline / ACR Image Cleanup (pull_request) Has been skipped
CI/CD Pipeline / Deploy Production (pull_request) Has been skipped
CI/CD Pipeline / Production Browser E2E (pull_request) Has been skipped
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 28s
CI/CD Pipeline / Frontend Unit Tests (pull_request) Has been skipped
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 1m4s
CI/CD Pipeline / Frontend Lint (pull_request) Successful in 1m9s
CI/CD Pipeline / Validate Code Quality And Tests (pull_request) Successful in 1m41s
PR Automation / Auto Approve on CI Green (pull_request) Successful in 2m15s
CI/CD Pipeline / Unit Tests (pull_request) Successful in 2m48s
AI Code Review / AI Code Review (pull_request) Successful in 3m34s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 1m24s
PR Automation / Auto Merge on CI Green + Approved (pull_request) Successful in 16m53s
- npm registry改为npmmirror.com国内镜像,加速下载减少失败
- 重试间隔从5s增加到10s,给网络恢复更多时间
2026-07-22 00:17:56 +08:00
23 changed files with 250 additions and 2392 deletions
-78
View File
@@ -1,78 +0,0 @@
name: CI Failure Monitor
on:
schedule:
- cron: '0 */6 * * *' # 每6小时检查一次
workflow_dispatch:
inputs:
days:
description: '统计最近N天的失败'
required: false
default: '7'
fail_threshold:
description: '失败次数阈值'
required: false
default: '3'
fail_rate_threshold:
description: '失败率阈值(%)'
required: false
default: '30'
permissions:
contents: read
jobs:
monitor:
name: CI重复失败检测
runs-on: ci-l2
timeout-minutes: 10
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" \
| bash
- name: Record job start time
shell: sh
run: bash scripts/ci/step_timer_start.sh
- name: Run failure detection
shell: sh
env:
GITEA_API_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
GITEA_URL: https://git.xiaoxiajianji.com
GITEA_REPO: xiaoxia/xiaoxia-saas
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
FAIL_CHECK_DAYS: ${{ inputs.days || 7 }}
FAIL_THRESHOLD: ${{ inputs.fail_threshold || 3 }}
FAIL_RATE_THRESHOLD: ${{ inputs.fail_rate_threshold || 30 }}
run: |
set +e
python3 scripts/ci/ci_repeated_failure_detector.py
EXIT_CODE=$?
echo "检测完成,退出码: $EXIT_CODE"
# 0=无异常, 1=有警告, 2=有严重问题
# 监控脚本永远不fail,避免告警风暴
exit 0
- name: Job duration summary
if: always()
shell: sh
run: bash scripts/ci/step_timer_end.sh
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
+18 -237
View File
@@ -76,12 +76,18 @@ jobs:
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
validate-code-quality:
name: Validate - Code Quality
validate:
needs: check-frontend-only
if: always() && needs.check-frontend-only.outputs.skip_backend != 'true'
name: Validate Code Quality And Tests
runs-on: ci-l2
timeout-minutes: 8
timeout-minutes: 10
permissions:
contents: write
env:
DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas
USE_IN_MEMORY_DB: 'false'
CI_USE_SHARED_PG: 'true'
steps:
- name: Checkout code
shell: sh
@@ -96,6 +102,7 @@ jobs:
shell: sh
run: |
set -eu
# pip install 带重试(网络不稳定时自动重试)
for i in 1 2 3; do
python3 -m pip install -q -r requirements-base.txt && break
echo "pip install requirements-base.txt 失败,重试 $i/3..."
@@ -120,11 +127,11 @@ jobs:
[ $i -eq 3 ] && exit 1
sleep 5
done
- name: Run code quality and security checks
- name: Run all quality checks
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
run: bash scripts/ci/validate_code_quality.sh
run: bash scripts/ci/run_validate.sh
- name: Auto-fix formatting (black + isort)
if: failure()
shell: sh
@@ -139,7 +146,7 @@ jobs:
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
run: |
set +e
FAILED_JOB="Validate - Code Quality" python3 scripts/ci_notify_failure.py
FAILED_JOB="Validate Code Quality And Tests" python3 scripts/ci_notify_failure.py
- name: Job duration summary
if: always()
shell: sh
@@ -152,161 +159,7 @@ jobs:
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
run: |
set +e
NOTIFY_MODE=failure JOB_NAME="Validate - Code Quality" python3 scripts/ci_notify.py
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
validate-type-check:
name: Validate - Type Check (mypy)
runs-on: ci-l2
timeout-minutes: 8
permissions:
contents: read
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
- name: Record job start time
shell: sh
run: bash scripts/ci/step_timer_start.sh
- name: Install dependencies
shell: sh
run: |
set -eu
for i in 1 2 3; do
python3 -m pip install -q -r requirements-base.txt && break
echo "pip install requirements-base.txt 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
for i in 1 2 3; do
python3 -m pip install -q -r requirements.txt && break
echo "pip install requirements.txt 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
for i in 1 2 3; do
python3 -m pip install -q -r requirements-dev.txt && break
echo "pip install requirements-dev.txt 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
- name: Run mypy type check
shell: bash
run: bash scripts/ci/validate_mypy.sh
- name: CI failure notification
if: failure()
shell: sh
env:
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
run: |
set +e
FAILED_JOB="Validate - Type Check (mypy)" python3 scripts/ci_notify_failure.py
- name: Job duration summary
if: always()
shell: sh
run: bash scripts/ci/step_timer_end.sh
- name: Notify on failure
continue-on-error: true
if: failure()
shell: sh
env:
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
run: |
set +e
NOTIFY_MODE=failure JOB_NAME="Validate - Type Check (mypy)" python3 scripts/ci_notify.py
- name: Report CI trace
if: always()
shell: sh
env:
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
run: |
STATUS="ok"
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
validate-migration:
name: Validate - Migration (alembic)
runs-on: ci-l2
timeout-minutes: 8
permissions:
contents: read
env:
DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas
USE_IN_MEMORY_DB: 'false'
CI_USE_SHARED_PG: 'true'
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
- name: Record job start time
shell: sh
run: bash scripts/ci/step_timer_start.sh
- name: Install dependencies
shell: sh
run: |
set -eu
for i in 1 2 3; do
python3 -m pip install -q -r requirements-base.txt && break
echo "pip install requirements-base.txt 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
for i in 1 2 3; do
python3 -m pip install -q -r requirements.txt && break
echo "pip install requirements.txt 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
for i in 1 2 3; do
python3 -m pip install -q -r requirements-dev.txt && break
echo "pip install requirements-dev.txt 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
- name: Run alembic migration validation
shell: bash
run: bash scripts/ci/validate_migration.sh
- name: CI failure notification
if: failure()
shell: sh
env:
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
run: |
set +e
FAILED_JOB="Validate - Migration (alembic)" python3 scripts/ci_notify_failure.py
- name: Job duration summary
if: always()
shell: sh
run: bash scripts/ci/step_timer_end.sh
- name: Notify on failure
continue-on-error: true
if: failure()
shell: sh
env:
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
run: |
set +e
NOTIFY_MODE=failure JOB_NAME="Validate - Migration (alembic)" python3 scripts/ci_notify.py
NOTIFY_MODE=failure JOB_NAME="Validate Code Quality And Tests" python3 scripts/ci_notify.py
- name: Report CI trace
if: always()
shell: sh
@@ -534,11 +387,9 @@ jobs:
[ $i -eq 3 ] && exit 1
sleep 5
done
- name: Run Vitest (incremental for PRs, full for main branches)
- name: Run Vitest with coverage
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: bash scripts/ci/vitest_incremental.sh
run: bash scripts/ci/step_frontend_run.sh "npx --no-install vitest run --coverage"
- name: Job duration summary
if: always()
shell: sh
@@ -621,65 +472,6 @@ jobs:
echo "Docker login failed ($i/3), retrying in 5s..."
sleep 5
done
- name: Pre-build worker base images (fallback if not exist)
if: matrix.service == 'worker'
id: prebuild
shell: sh
run: |
set -eu
REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas"
BASE_BUILDER="${REGISTRY}/worker-base-builder:latest"
BASE_RUNTIME="${REGISTRY}/worker-base-runtime:latest"
# 尝试拉取基础镜像
echo "检查基础镜像..."
if docker pull "$BASE_BUILDER" 2>/dev/null && docker pull "$BASE_RUNTIME" 2>/dev/null; then
echo "基础镜像已存在,使用远程镜像"
echo "fallback=false" >> $GITHUB_OUTPUT
else
echo "基础镜像不存在,本地构建(fallback模式)..."
# 构建builder基础镜像
echo "构建 worker-base-builder..."
# 用buildx docker-container驱动构建(兼容DooD模式:普通docker build看不到容器内文件)
BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}"
if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
else
docker buildx use "$BUILDER_NAME"
fi
docker buildx inspect --bootstrap > /dev/null 2>&1
# 构建builder基础镜像(带重试,buildx容器偶发不稳定)
echo "构建 worker-base-builder..."
for attempt in 1 2 3; do
if docker buildx build --load -f infra/docker/worker-base-builder.Dockerfile -t "$BASE_BUILDER" .; then
echo "worker-base-builder 构建成功"
break
fi
echo "worker-base-builder 构建失败,重试 $attempt/3..."
docker buildx rm "$BUILDER_NAME" 2>/dev/null || true
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
sleep 3
done
# 构建runtime基础镜像
echo "构建 worker-base-runtime..."
for attempt in 1 2 3; do
if docker buildx build --load -f infra/docker/worker-base-runtime.Dockerfile -t "$BASE_RUNTIME" .; then
echo "worker-base-runtime 构建成功"
break
fi
echo "worker-base-runtime 构建失败,重试 $attempt/3..."
docker buildx rm "$BUILDER_NAME" 2>/dev/null || true
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
sleep 3
done
echo "fallback=true" >> $GITHUB_OUTPUT
echo "基础镜像本地构建完成"
fi
- name: Build PR image (verify only, no push)
shell: sh
run: |
@@ -693,18 +485,6 @@ jobs:
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
fi
# Worker fallback模式:基础镜像本地已构建,用普通docker build绕过buildx
if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.fallback }}" = "true" ]; then
echo "Fallback模式:用普通docker build(基础镜像本地已构建)"
BUILD_ARG_STR=""
for arg in $EXTRA_BUILD_ARGS; do
BUILD_ARG_STR="$BUILD_ARG_STR --build-arg $arg"
done
docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR .
echo "Fallback PR Build successful"
exit 0
fi
NO_CACHE_FLAG=""
for i in 1 2 3; do
echo "PR Build attempt $i/3"
@@ -1553,4 +1333,5 @@ jobs:
[ ${{ job.status }} = "success" ] || STATUS="error"
START_TIME=""
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
+2 -6
View File
@@ -54,9 +54,7 @@ jobs:
CONTEXTS=("CI/CD Pipeline / Frontend Lint (pull_request)")
else
CONTEXTS=(
"CI/CD Pipeline / Validate - Code Quality (pull_request)"
"CI/CD Pipeline / Validate - Type Check (mypy) (pull_request)"
"CI/CD Pipeline / Validate - Migration (alembic) (pull_request)"
"CI/CD Pipeline / Validate Code Quality And Tests (pull_request)"
"CI/CD Pipeline / Frontend Lint (pull_request)"
)
fi
@@ -235,9 +233,7 @@ jobs:
echo "纯前端改动,只检查Frontend Lint"
else
CONTEXTS=(
"CI/CD Pipeline / Validate - Code Quality (pull_request)"
"CI/CD Pipeline / Validate - Type Check (mypy) (pull_request)"
"CI/CD Pipeline / Validate - Migration (alembic) (pull_request)"
"CI/CD Pipeline / Validate Code Quality And Tests (pull_request)"
"CI/CD Pipeline / Frontend Lint (pull_request)"
"CI/CD Pipeline / PR Build API Image (pull_request)"
"CI/CD Pipeline / PR Build Worker Image (pull_request)"
-103
View File
@@ -1,103 +0,0 @@
name: Worker Base Image Build
on:
push:
branches:
- develop
- main
paths:
- 'requirements-base.txt'
- 'requirements-worker.txt'
- 'infra/docker/worker-base-builder.Dockerfile'
- 'infra/docker/worker-base-runtime.Dockerfile'
workflow_dispatch: # 支持手动触发
jobs:
build-worker-base:
name: Build Worker Base Images
runs-on: runtime-builder
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- name: builder
dockerfile: infra/docker/worker-base-builder.Dockerfile
image_name: worker-base-builder
cache_name: worker-base-builder-cache
- name: runtime
dockerfile: infra/docker/worker-base-runtime.Dockerfile
image_name: worker-base-runtime
cache_name: worker-base-runtime-cache
steps:
- name: Checkout code
shell: sh
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
- name: Docker login to Registry
shell: sh
env:
ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
GITEA_REGISTRY_USER: xiaoxia
GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
run: |
set -eu
for i in 1 2 3; do
echo "=== Docker login 尝试 $i/3 ==="
if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then
echo "✅ Docker login successful"
break
fi
echo "❌ Docker login 失败(尝试 $i/3),5s 后重试..."
sleep 5
done
- name: Setup buildx builder
shell: sh
run: |
set -eu
BUILDER_NAME="ci-builder-${GITHUB_RUN_ID}-${{ matrix.name }}"
if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
echo "Created $BUILDER_NAME"
else
docker buildx use "$BUILDER_NAME"
echo "Using existing $BUILDER_NAME"
fi
docker buildx inspect --bootstrap
- name: Build and push base image
shell: sh
run: |
set -eu
REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:latest"
SAFE_REF_NAME=$(echo "${GITHUB_REF_NAME}" | tr '/' '-')
CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:${SAFE_REF_NAME}"
echo "=== Building ${{ matrix.name }} base image ==="
echo "Image: ${IMAGE_TAG}"
echo "Cache: ${CACHE_REF}"
# 用通用构建脚本
bash scripts/ci/docker_build_push.sh ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}"
# 同时推送到 Gitea Packages 作为备份(可选)
GITEA_IMAGE="git.xiaoxiajianji.com/xiaoxia-saas/${{ matrix.image_name }}:latest"
docker tag "${IMAGE_TAG}" "${GITEA_IMAGE}"
docker push "${GITEA_IMAGE}" || echo "Gitea Packages push failed (non-fatal)"
echo ""
echo "✅ ${{ matrix.name }} base image built and pushed"
- name: Cleanup buildx builder
if: always()
shell: sh
run: |
docker buildx rm "ci-builder-${GITHUB_RUN_ID}-${{ matrix.name }}" 2>/dev/null || true
docker buildx prune -f 2>/dev/null || true
echo "Builder cleanup done"
@@ -1,43 +0,0 @@
# ============================================================
# Worker Builder 基础镜像
# 预编译:编译工具 + 基础依赖 + Worker大包
# 当 requirements-base.txt 或 requirements-worker.txt 变更时重新构建
# 业务构建从此镜像开始,只需要安装业务依赖,节省15+分钟
# ============================================================
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim
# 使用阿里云镜像加速
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
# 安装编译工具
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc \
g++ \
python3-dev \
binutils \
&& rm -rf /var/lib/apt/lists/*
# 创建 venv
RUN python -m venv /opt/venv
ENV PATH="/opt/venv/bin:$PATH"
WORKDIR /tmp
# 基础依赖(变化极少)
COPY requirements-base.txt /tmp/requirements-base.txt
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
-r /tmp/requirements-base.txt \
&& rm /tmp/requirements-base.txt
# Worker 大包(变化少)
COPY requirements-worker.txt /tmp/requirements-worker.txt
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
-r /tmp/requirements-worker.txt \
&& rm /tmp/requirements-worker.txt
# 预先做一次 strip(基础层瘦身,业务层增量)
RUN find /opt/venv -name "*.so" -type f -exec strip --strip-all {} \; 2>/dev/null || true
@@ -1,17 +0,0 @@
# ============================================================
# Worker Runtime 基础镜像
# 预安装:ffmpeg + 运行时依赖
# 变化极少,业务构建从此镜像开始
# ============================================================
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim
# 使用阿里云镜像加速
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
# 运行时依赖:ffmpeg + opencv需要的libglib
RUN apt-get update && apt-get install -y --no-install-recommends \
ffmpeg \
libglib2.0-0 \
&& rm -rf /var/lib/apt/lists/*
+62 -17
View File
@@ -1,43 +1,88 @@
# ============================================================
# Worker Dockerfile - 分层缓存优化版
# 优化:基础依赖 + Worker大包预构建为基础镜像,业务构建仅叠加业务依赖
# 基础镜像:worker-base-builder / worker-base-runtime
# 预计节省:依赖不变时构建时间从23min降至5min以内
# Worker Dockerfile - 优化版(多阶段构建 + 镜像瘦身 + cache mount加速)
# 优化
# 1. 多阶段构建:builder 阶段安装编译依赖,runtime 阶段只保留运行时
# 2. ffmpeg 通过 apt 安装(阿里云镜像加速,几秒完成,稳定可靠)
# 3. Python 依赖瘦身:strip .so 调试符号 + 清理测试文件 + 清理缓存
# 4. pip cache mount:加速依赖下载(跨构建共享pip wheel缓存)
# ============================================================
# ==================== Builder 阶段 ====================
# 从预构建的builder基础镜像开始,已经包含:
# - 编译工具 (gcc/g++/python3-dev/binutils)
# - requirements-base.txt 全部依赖
# - requirements-worker.txt 全部依赖 (numpy/scipy/opencv)
# - 预strip的.so文件
FROM xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/worker-base-builder:latest AS builder
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim AS builder
ENV PATH="/opt/venv/bin:$PATH"
# 使用阿里云镜像加速
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
# 安装编译工具(仅 builder 需要)
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc \
g++ \
python3-dev \
binutils \
&& rm -rf /var/lib/apt/lists/*
# ---- 安装 Python 依赖 ----
WORKDIR /tmp
# ---- 安装业务依赖(变化频繁,单独一层)----
# 创建 venv
RUN python -m venv /opt/venv
ENV PATH="/opt/venv/bin:$PATH"
# 基础依赖
COPY requirements-base.txt /tmp/requirements-base.txt
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
-r /tmp/requirements-base.txt \
&& rm /tmp/requirements-base.txt
# Worker 专属大包
COPY requirements-worker.txt /tmp/requirements-worker.txt
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
-r /tmp/requirements-worker.txt \
&& rm /tmp/requirements-worker.txt
# 业务依赖
COPY requirements.txt /tmp/requirements.txt
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
-r /tmp/requirements.txt \
&& rm /tmp/requirements.txt
# ---- 增量瘦身(只处理新增的业务依赖)----
# ---- Python 依赖瘦身 ----
# 1. strip .so 文件的调试符号(节省约 80-100MB)
RUN find /opt/venv -name "*.so" -type f -exec strip --strip-all {} \; 2>/dev/null || true
# 2. 清理测试文件(节省约 20MB)
RUN find /opt/venv -type d -name "tests" -exec rm -rf {} + 2>/dev/null; \
find /opt/venv -type d -name "test" -exec rm -rf {} + 2>/dev/null; \
find /opt/venv -name "test_*.py" -delete 2>/dev/null || true
# 3. 清理 .pyc 缓存和 __pycache__(节省约 10MB,运行时按需生成)
RUN find /opt/venv -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null; \
find /opt/venv -name "*.pyc" -delete 2>/dev/null || true
# 4. 清理 dist-info 中的文档
RUN find /opt/venv -name "*.dist-info" -type d -exec sh -c 'rm -f "$1"/DESCRIPTION.rst "$1"/INSTALLER "$1"/LICENSE* "$1"/WHEEL "$1"/entry_points.txt' _ {} \; 2>/dev/null || true
# ==================== Runtime 阶段 ====================
# 从预构建的runtime基础镜像开始,已经包含:
# - ffmpeg
# - libglib2.0-0
FROM xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/worker-base-runtime:latest AS runtime
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim AS runtime
# 构建参数:版本号
ARG APP_VERSION=dev
# 使用阿里云镜像加速
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
# 安装最小运行时依赖(opencv-python-headless 需要 libglib2.0-0
RUN apt-get update && apt-get install -y --no-install-recommends \
ffmpeg \
libglib2.0-0 \
&& rm -rf /var/lib/apt/lists/*
# 从 builder 复制 Python 虚拟环境
COPY --from=builder /opt/venv /opt/venv
+1 -1
View File
@@ -3,7 +3,7 @@
# 数据库(基础层)
psycopg2-binary==2.9.10
psycopg[binary]==3.2.2
psycopg[binary]>=3.2.2
sqlalchemy==2.0.35
alembic==1.13.3
+1 -2
View File
@@ -11,5 +11,4 @@ pytest==8.3.3
pytest-asyncio==0.24.0
pytest-cov==6.0.0
pytest-timeout==2.3.1
pytest-xdist==3.6.1
diff-cover==8.0.3
diff-cover>=8.0
+3 -3
View File
@@ -2,13 +2,13 @@
# 这些包体积大,API 服务不需要安装
# 数值计算
numpy==1.26.4
numpy>=1.24.0
# 科学计算
scipy==1.13.1
scipy>=1.10.0
# 计算机视觉(视频去重、帧处理)
opencv-python-headless==4.10.0.84
opencv-python-headless>=4.8.0
# 图像处理
Pillow==10.4.0
+13 -15
View File
@@ -31,22 +31,20 @@ def main():
print("pending")
return
# 筛选目标context,按时间倒序取最新的
matching = [s for s in statuses if s.get("context") == target_context]
if not matching:
# 找不到说明CI还没开始写状态,返回pending继续等待
print("pending")
return
# API返回按时间倒序,第一个就是最新的
for s in statuses:
if s.get("context") == target_context:
status = s.get("status", "pending")
# skipped 视为通过(条件跳过的任务不需要等)
if status == "skipped":
print("success")
else:
print(status)
return
# Gitea statuses API按时间正序返回,必须取最新的一条
latest = max(matching, key=lambda s: s.get("created_at", ""))
status = latest.get("status", "pending")
# skipped 视为通过(条件跳过的任务不需要等)
if status == "skipped":
print("success")
else:
print(status)
# 找不到这个context说明CI还没开始写状态,返回pending继续等待
# (如果workflow真的被跳过,它会有一条status为skipped的记录)
print("pending")
if __name__ == "__main__":
+1 -43
View File
@@ -168,28 +168,6 @@ def main():
return
api_url = os.environ.get("GITHUB_API_URL", "")
# 获取PR作者信息,判断是人还是Agent提交的
pr_info_url = f"{api_url}/repos/{repo}/pulls/{pr_number}"
req_pr = urllib.request.Request(pr_info_url, headers={"Authorization": f"token {token}"})
with urllib.request.urlopen(req_pr) as resp:
pr_info = json.loads(resp.read())
pr_author = pr_info.get("user", {}).get("login", "")
print(f"PR作者: {pr_author}")
# 判断是否为Agent提交的PR
# Agent账号:actions, auto-approve-bot 等bot用户
# 人提交的PR(如xiaoxia):只诊断不自动修
agent_authors = {"actions", "auto-approve-bot", "gitea-actions"}
is_agent_pr = pr_author in agent_authors or "bot" in pr_author.lower()
if is_agent_pr:
print(f"检测到Agent提交的PR(作者: {pr_author}),将自动修复并推送")
fix_mode = "auto_fix_and_push"
else:
print(f"检测到人提交的PR(作者: {pr_author}),仅诊断不自动修改")
print("(如需自动修复,请用Agent账号提交PR,或手动运行格式化脚本)")
fix_mode = "diagnose_only"
repo = os.environ.get("GITHUB_REPOSITORY", "")
token = os.environ.get("GITHUB_TOKEN", "")
scan_mode = os.environ.get("SCAN_MODE", "full")
@@ -253,26 +231,6 @@ def main():
print("没有需要提交的格式改动")
return
# 诊断模式:只报告问题,不修改不推送
if fix_mode == "diagnose_only":
print()
print("=" * 50)
print("📋 格式问题诊断报告(人提交的PR,仅诊断不自动修复)")
print("=" * 50)
print()
print("以下文件存在格式问题,建议手动修复:")
for line in result.stdout.strip().split("\n"):
print(f" {line}")
print()
print("修复方式:")
print(" 后端(Python): 运行 black + isort")
print(" 前端: 运行 prettier --write")
print(" 或使用 scripts/agent-commit.sh 提交(自动格式化)")
print()
print("=" * 50)
# 以非0状态码退出,让CI继续报失败(因为问题没修)
sys.exit(1)
print()
print("变更文件:")
for line in result.stdout.strip().split("\n"):
@@ -280,7 +238,7 @@ def main():
# 提交修复
run("git add -A")
run('git commit -m "style: auto-format with black + isort + prettier"')
run('git commit -m "style: auto-format with black + isort + prettier [ci skip]"')
# 推送(head_branch已从ensure_git_repo获取)
print(f"\nPR来源分支: {head_branch}")
-447
View File
@@ -1,447 +0,0 @@
#!/usr/bin/env python3
"""CI失败诊断增强脚本:自动分类失败原因 + 提取关键错误 + 给出修复建议。
# Trigger CI after auto-format fix
支持的失败类型:
1. Lint/格式问题 (ruff/black/eslint/prettier)
2. 单元测试失败
3. Docker构建失败
4. 依赖安装失败 (pip/npm)
5. 超时
6. 缓存问题
7. 数据库/迁移问题
8. 网络问题
9. 其他
用法:
python3 scripts/ci/ci_failure_diagnosis.py [--job-name "Job Name"] [--log-file /path/to/log]
如果不传--log-file,会尝试从Gitea API获取失败job的日志。
"""
import json
import os
import re
import sys
import urllib.request
from dataclasses import dataclass, field
from typing import List, Optional
@dataclass
class FailureDiagnosis:
"""失败诊断结果"""
category: str # 失败分类
category_cn: str # 中文分类名
severity: str # 严重程度: high / medium / low
summary: str # 一句话摘要
error_lines: List[str] = field(default_factory=list) # 关键错误行
suggestions: List[str] = field(default_factory=list) # 修复建议
auto_fixable: bool = False # 是否可以自动修复
related_docs: str = "" # 相关文档链接
# ============================================================
# 失败模式定义
# ============================================================
FAILURE_PATTERNS = [
# ===== Lint / 格式问题 =====
{
"pattern": r"(ruff|black|isort)\b.*(error|failed|Error)",
"category": "lint_python",
"category_cn": "Python代码质量检查",
"severity": "low",
"summary_contains": ["ruff", "black", "isort"],
"suggestions": [
"本地运行 `black . && isort . && ruff check --fix .` 自动修复",
"使用 `scripts/agent-commit.sh` 提交(自动格式化)",
"如确认无误,可加 `# noqa: xxx` 忽略特定规则",
],
"auto_fixable": True,
},
{
"pattern": r"ESLint|prettier|eslint",
"category": "lint_frontend",
"category_cn": "前端代码检查",
"severity": "low",
"summary_contains": ["eslint", "prettier"],
"suggestions": [
"本地运行 `cd apps/web && npm run lint:fix` 自动修复",
"Prettier问题: `cd apps/web && npx prettier --write .`",
],
"auto_fixable": True,
},
{
"pattern": r"F\d{3}|E\d{3}|W\d{3}.*ruff|ruff.*F\d{3}",
"category": "lint_python",
"category_cn": "Python代码质量检查",
"severity": "low",
"suggestions": [
"F401: 删除未使用的import",
"F841: 删除未使用的变量或加下划线前缀",
"E501: 行超长,加 `# noqa: E501`",
"F811: 删重复import",
"运行 `ruff check --fix .` 自动修复大部分问题",
],
"auto_fixable": True,
},
# ===== 单元测试失败 =====
{
"pattern": r"FAILED|assert.*Error|AssertionError",
"category": "unit_test",
"category_cn": "单元测试失败",
"severity": "high",
"suggestions": [
"检查相关测试文件,确认是代码问题还是测试用例问题",
"本地运行对应测试:`pytest path/to/test.py -v`",
"如测试依赖外部服务,检查mock是否正确",
],
"auto_fixable": False,
},
{
"pattern": r"pytest.*failed|\d+ failed.*\d+ passed",
"category": "unit_test",
"category_cn": "单元测试失败",
"severity": "high",
"suggestions": [
"查看上方日志中的FAILED测试用例",
"检查失败断言的期望值 vs 实际值",
"新代码影响了现有测试行为,确认是预期内变更吗?",
],
"auto_fixable": False,
},
# ===== Docker 构建失败 =====
{
"pattern": r"Dockerfile.*not found|docker build.*failed|ERROR: failed to solve",
"category": "docker_build",
"category_cn": "Docker构建失败",
"severity": "high",
"suggestions": [
"检查Dockerfile语法是否正确",
"检查引用的基础镜像是否存在",
"本地运行 `docker build -f path/to/Dockerfile .` 复现",
],
"auto_fixable": False,
},
{
"pattern": r"manifest.*not found|no such image|image.*not found",
"category": "docker_build",
"category_cn": "镜像不存在",
"severity": "medium",
"suggestions": [
"检查基础镜像名称和tag是否正确",
"确认镜像仓库可访问,登录是否有效",
"如为新基础镜像,需先手动构建一次基础镜像",
],
"auto_fixable": False,
},
{
"pattern": r"ETXTBSY|text file busy",
"category": "docker_build",
"category_cn": "文件锁冲突(ETXTBSY",
"severity": "low",
"summary": "esbuild并发构建冲突,重试即可",
"suggestions": ["偶发问题,点击Rerun重新运行即可", "如频繁出现,检查是否有多个job并发写入同一文件"],
"auto_fixable": True,
},
# ===== 依赖安装失败 =====
{
"pattern": r"pip install.*error|Could not find a version|No matching distribution",
"category": "dependency",
"category_cn": "pip依赖安装失败",
"severity": "medium",
"suggestions": [
"检查requirements.txt中的版本号是否正确",
"如为新版本刚发布,可能源还没同步,稍后重试",
"检查网络连接,可尝试切换pip镜像源",
],
"auto_fixable": False,
},
{
"pattern": r"npm.*ERR|npm install.*failed|E404|ECONNREFUSED.*npm",
"category": "dependency",
"category_cn": "npm依赖安装失败",
"severity": "medium",
"suggestions": [
"检查package.json中的版本号是否存在",
"网络问题:检查npm registry是否可访问",
"国内网络建议配置npmmirror镜像源",
],
"auto_fixable": False,
},
{
"pattern": r"Connection refused|timed out|network.*unreachable",
"category": "network",
"category_cn": "网络问题",
"severity": "medium",
"summary": "网络连接失败,可能是源站问题或DNS问题",
"suggestions": [
"点击Rerun重试,网络问题通常是临时的",
"如持续失败,检查对应服务是否正常",
"检查Runner网络配置",
],
"auto_fixable": True,
},
# ===== 超时 =====
{
"pattern": r"timeout|timed out|exceeded.*time limit|job.*cancelled.*timeout",
"category": "timeout",
"category_cn": "执行超时",
"severity": "medium",
"suggestions": [
"如首次出现:重试一次,可能是临时性能波动",
"频繁出现:检查构建是否变慢了,最近是否加了新依赖",
"可适当增加timeout-minutes配置",
],
"auto_fixable": False,
},
# ===== 数据库/迁移 =====
{
"pattern": r"alembic.*error|migration.*failed|relation.*does not exist|column.*does not exist",
"category": "migration",
"category_cn": "数据库迁移失败",
"severity": "high",
"suggestions": [
"检查迁移脚本是否正确,down_revision是否对",
"确认数据库中是否有脏数据或残留表",
"迁移脚本合并冲突时,重新生成迁移文件",
],
"auto_fixable": False,
},
# ===== 缓存问题 =====
{
"pattern": r"cache.*corrupt|cache.*invalid|snapshot.*not found|failed to compute cache key",
"category": "cache",
"category_cn": "缓存损坏",
"severity": "low",
"suggestions": ["构建系统会自动清理损坏缓存并重试,通常无需干预", "如持续失败,手动清理Runner上的缓存目录"],
"auto_fixable": True,
},
# ===== Checkout 失败 =====
{
"pattern": r"Could not resolve host|fatal:.*repository|SSL.*problem",
"category": "checkout",
"category_cn": "代码拉取失败",
"severity": "low",
"suggestions": ["临时网络问题,点击Rerun重试", "如持续失败,检查Gitea服务状态"],
"auto_fixable": True,
},
]
def analyze_log(log_text: str, job_name: str = "") -> FailureDiagnosis:
"""分析日志,返回诊断结果"""
lines = log_text.strip().split("\n")
# 收集所有匹配的模式
matched = []
error_lines = []
for line in lines:
line_stripped = line.strip()
# 收集ERROR/FAILED/Failed等错误行(最多20行)
if re.search(r"(ERROR|FAILED|Error|error:|FAIL:|Traceback)", line_stripped):
if len(error_lines) < 20:
error_lines.append(line_stripped)
for pattern_info in FAILURE_PATTERNS:
if re.search(pattern_info["pattern"], line_stripped, re.IGNORECASE):
matched.append(pattern_info)
break # 一行只匹配一个模式
if not matched:
# 未识别的失败类型
return FailureDiagnosis(
category="unknown",
category_cn="未知错误",
severity="medium",
summary="未识别的失败类型,需要人工查看日志",
error_lines=error_lines[:10],
suggestions=[
"点击'查看失败日志'查看完整日志",
"如为偶发问题,可先重试一次",
"常见原因:环境问题、配置问题、新增逻辑引入的bug",
],
auto_fixable=False,
)
# 选最严重、最具体的那个
severity_order = {"high": 3, "medium": 2, "low": 1}
matched.sort(key=lambda x: severity_order.get(x["severity"], 0), reverse=True)
best_match = matched[0]
# 生成摘要
if "summary" in best_match:
summary = best_match["summary"]
else:
summary = f"{best_match['category_cn']}检查失败"
if job_name:
summary = f"[{job_name}] {summary}"
# 从error_lines中过滤出与该分类相关的
relevant_errors = error_lines[:10]
return FailureDiagnosis(
category=best_match["category"],
category_cn=best_match["category_cn"],
severity=best_match["severity"],
summary=summary,
error_lines=relevant_errors,
suggestions=best_match["suggestions"],
auto_fixable=best_match.get("auto_fixable", False),
)
def fetch_failed_job_log(run_id: str, job_id: str, token: str, repo: str) -> Optional[str]:
"""从Gitea API获取失败job的日志"""
api_base = f"https://git.xiaoxiajianji.com/api/v1/repos/{repo}"
# 尝试获取job的日志
url = f"{api_base}/actions/runs/{run_id}/jobs/{job_id}/log"
req = urllib.request.Request(url)
req.add_header("Authorization", f"token {token}")
try:
with urllib.request.urlopen(req, timeout=15) as resp:
return resp.read().decode("utf-8", errors="replace")
except Exception as e:
print(f"获取日志失败: {e}", file=sys.stderr)
return None
def format_diagnosis_markdown(d: FailureDiagnosis, job_name: str = "", run_url: str = "") -> str:
"""将诊断结果格式化为飞书卡片markdown"""
severity_emoji = {"high": "🔴", "medium": "🟡", "low": "🟢"}
emoji = severity_emoji.get(d.severity, "")
lines = []
lines.append(f"**分类**: {emoji} {d.category_cn}")
lines.append(f"**问题**: {d.summary}")
if d.error_lines:
lines.append("")
lines.append("**关键错误行**:")
for err in d.error_lines[:5]:
# 截断过长的行
if len(err) > 150:
err = err[:147] + "..."
lines.append(f" `{err}`")
lines.append("")
lines.append("**修复建议**:")
for i, s in enumerate(d.suggestions[:5], 1):
lines.append(f" {i}. {s}")
if d.auto_fixable:
lines.append("")
lines.append("💡 **可自动修复**:如格式问题,可尝试点击Rerun让auto-fix自动处理")
if run_url:
lines.append("")
lines.append(f"[查看完整日志]({run_url})")
return "\n".join(lines)
def main():
job_name = os.environ.get("FAILED_JOB", "")
run_id = os.environ.get("GITHUB_RUN_ID", "")
repo = os.environ.get("GITHUB_REPOSITORY", "xiaoxia/xiaoxia-saas")
token = os.environ.get("GITHUB_TOKEN", "")
# 1. 尝试获取日志
log_text = ""
# 优先从环境变量或文件读取
log_file = os.environ.get("CI_LOG_FILE", "")
if log_file and os.path.exists(log_file):
with open(log_file) as f:
log_text = f.read()
elif run_id and token:
# 尝试从API获取(需要job_id,这里简化处理)
pass
# 如果没有日志,用job_name做粗略分类
if not log_text:
# 基于job名做初始判断
if any(k in job_name.lower() for k in ["validate", "lint", "quality"]):
d = FailureDiagnosis(
category="lint_general",
category_cn="代码质量检查",
severity="low",
summary=f"{job_name} 检查失败(日志不可用,基于job名初步诊断)",
suggestions=["点击查看日志获取具体错误信息", "格式类问题通常可自动修复"],
auto_fixable=True,
)
elif "build" in job_name.lower():
d = FailureDiagnosis(
category="build_general",
category_cn="构建失败",
severity="high",
summary=f"{job_name} 构建失败(日志不可用)",
suggestions=["点击查看日志获取具体构建错误", "常见原因:Dockerfile错误、依赖安装失败、网络问题"],
auto_fixable=False,
)
elif "test" in job_name.lower():
d = FailureDiagnosis(
category="test_general",
category_cn="测试失败",
severity="high",
summary=f"{job_name} 测试失败(日志不可用)",
suggestions=["点击查看日志获取具体失败的测试用例", "检查最近代码改动是否影响了测试"],
auto_fixable=False,
)
elif "deploy" in job_name.lower():
d = FailureDiagnosis(
category="deploy_general",
category_cn="部署失败",
severity="high",
summary=f"{job_name} 部署失败(日志不可用)",
suggestions=["检查目标服务器状态和网络", "检查镜像是否正确推送", "查看服务器上的容器日志"],
auto_fixable=False,
)
else:
d = FailureDiagnosis(
category="unknown",
category_cn="未知错误",
severity="medium",
summary=f"{job_name} 失败",
suggestions=["点击查看日志获取详细信息"],
auto_fixable=False,
)
else:
d = analyze_log(log_text, job_name)
# 输出诊断结果
run_url = f"https://git.xiaoxiajianji.com/{repo}/actions/runs/{run_id}" if run_id else ""
print("=" * 60)
print(" CI 失败诊断报告")
print("=" * 60)
print()
print(format_diagnosis_markdown(d, job_name, run_url))
print()
print("=" * 60)
# 将诊断结果写入文件(供通知脚本读取)
output_file = os.environ.get("DIAGNOSIS_OUTPUT", "/tmp/ci_diagnosis.json")
result = {
"category": d.category,
"category_cn": d.category_cn,
"severity": d.severity,
"summary": d.summary,
"error_lines": d.error_lines,
"suggestions": d.suggestions,
"auto_fixable": d.auto_fixable,
}
with open(output_file, "w") as f:
json.dump(result, f, ensure_ascii=False, indent=2)
print(f"\n诊断结果已保存到: {output_file}")
if __name__ == "__main__":
main()
-417
View File
@@ -1,417 +0,0 @@
#!/usr/bin/env python3
"""
CI重复失败检测脚本
- 扫描最近N天的CI失败
- 按job名称分组统计失败率
- 识别高失败率job(系统性故障)
- 飞书通知告警
"""
import json
import os
import sys
import time
import urllib.error
import urllib.request
from collections import defaultdict
from datetime import datetime, timedelta, timezone
def get_env(name, default=None, required=False):
val = os.environ.get(name, default)
if required and not val:
print(f"❌ 缺少环境变量: {name}")
sys.exit(1)
return val
GITEA_URL = get_env("GITEA_URL", "https://git.xiaoxiajianji.com")
GITEA_TOKEN = get_env("GITEA_API_TOKEN", required=False) or get_env("GITHUB_TOKEN", "")
REPO = get_env("GITEA_REPO", "xiaoxia/xiaoxia-saas")
DAYS = int(get_env("FAIL_CHECK_DAYS", "7"))
FAIL_THRESHOLD = int(get_env("FAIL_THRESHOLD", 3)) # 失败次数阈值
FAIL_RATE_THRESHOLD = float(get_env("FAIL_RATE_THRESHOLD", "30")) # 失败率阈值%
CONSECUTIVE_FAIL_THRESHOLD = int(get_env("CONSECUTIVE_FAIL_THRESHOLD", "3")) # 连续失败阈值
WEBHOOK = get_env("CI_NOTIFY_WEBHOOK", "")
def api_get(path):
"""调用Gitea API"""
url = f"{GITEA_URL}/api/v1{path}"
req = urllib.request.Request(url)
if GITEA_TOKEN:
req.add_header("Authorization", f"token {GITEA_TOKEN}")
try:
with urllib.request.urlopen(req, timeout=30) as resp:
return json.loads(resp.read())
except urllib.error.HTTPError as e:
print(f" HTTP {e.code}: {path}")
return None
except Exception as e:
print(f" 错误: {e}")
return None
def fetch_recent_runs(days=7, per_page=50, max_pages=10):
"""获取最近N天的runs"""
since = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
all_runs = []
for page in range(1, max_pages + 1):
path = f"/repos/{REPO}/actions/runs?page={page}&limit={per_page}"
data = api_get(path)
if not data:
break
runs = data.get("workflow_runs", data.get("runs", []))
if not runs:
break
# 检查时间范围(Gitea用started_at,格式2026-07-22T10:58:10+08:00
oldest = None
for r in runs:
started = r.get("started_at", r.get("created_at", ""))
if started and started >= since:
all_runs.append(r)
else:
oldest = started
if oldest and oldest < since:
break
if len(runs) < per_page:
break
return all_runs
def fetch_run_jobs(run_id):
"""获取run的所有jobs"""
path = f"/repos/{REPO}/actions/runs/{run_id}/jobs"
data = api_get(path)
if not data:
return []
return data.get("jobs", [])
def analyze_failures(runs):
"""
分析失败情况
返回:
- job_stats: {job_name: {total, success, failure, skipped, failure_rate, failures: [...]}}
- consecutive_failures: {job_name: current_streak, max_streak, last_status}
"""
job_stats = defaultdict(
lambda: {
"total": 0,
"success": 0,
"failure": 0,
"error": 0,
"skipped": 0,
"cancelled": 0,
"failures": [],
}
)
# 按时间正序排列(旧→新)用于连续失败计算
sorted_runs = sorted(runs, key=lambda r: r.get("started_at", r.get("created_at", "")))
# 连续失败跟踪 {job_name: streak}
consecutive = defaultdict(lambda: {"current": 0, "max": 0, "last_run": None})
for run in sorted_runs:
run_id = run.get("id")
run_status = run.get("status", "")
run_conclusion = run.get("conclusion", "")
run_started = run.get("started_at", run.get("created_at", ""))
event = run.get("event", "")
# 只统计pull_request和push事件的CI
if event not in ("pull_request", "push"):
continue
jobs = fetch_run_jobs(run_id)
for job in jobs:
name = job.get("name", "")
status = job.get("status", "")
conclusion = job.get("conclusion", "")
# 跳过非CI核心job(如AI Code Review、Preview等)
skip_prefixes = ("AI Code Review", "Preview", "PR Automation", "Auto")
if any(name.startswith(p) for p in skip_prefixes):
continue
stats = job_stats[name]
stats["total"] += 1
if conclusion == "success":
stats["success"] += 1
consecutive[name]["current"] = 0
elif conclusion == "failure":
stats["failure"] += 1
stats["failures"].append(
{
"run_id": run_id,
"time": run_started,
"event": event,
}
)
consecutive[name]["current"] += 1
if consecutive[name]["current"] > consecutive[name]["max"]:
consecutive[name]["max"] = consecutive[name]["current"]
consecutive[name]["last_run"] = run_id
elif conclusion == "error":
stats["error"] += 1
# error也算失败的一种
consecutive[name]["current"] += 1
if consecutive[name]["current"] > consecutive[name]["max"]:
consecutive[name]["max"] = consecutive[name]["current"]
elif conclusion == "skipped":
stats["skipped"] += 1
# skipped不算也不打断连续失败
elif conclusion == "cancelled":
stats["cancelled"] += 1
# cancelled不算失败也不打断
# 计算失败率
for name, stats in job_stats.items():
total_actual = stats["total"] - stats["skipped"] - stats["cancelled"]
if total_actual > 0:
stats["failure_rate"] = round((stats["failure"] + stats["error"]) / total_actual * 100, 1)
else:
stats["failure_rate"] = 0.0
return dict(job_stats), dict(consecutive)
def find_high_failures(job_stats, consecutive):
"""
找出高风险job
告警级别:
- critical: 连续失败 >= CONSECUTIVE_FAIL_THRESHOLD,或 失败率>=50%且失败次数>=5
- warning: 失败率>=FAIL_RATE_THRESHOLD且失败次数>=FAIL_THRESHOLD
- info: 失败次数>=2
"""
critical = []
warning = []
info = []
for name, stats in job_stats.items():
fail_count = stats["failure"] + stats["error"]
rate = stats["failure_rate"]
streak = consecutive.get(name, {}).get("current", 0)
max_streak = consecutive.get(name, {}).get("max", 0)
issue = {
"name": name,
"fail_count": fail_count,
"total": stats["total"],
"failure_rate": rate,
"current_streak": streak,
"max_streak": max_streak,
"recent_failures": stats["failures"][-5:], # 最近5次
}
if streak >= CONSECUTIVE_FAIL_THRESHOLD or (rate >= 50 and fail_count >= 5):
critical.append(issue)
elif rate >= FAIL_RATE_THRESHOLD and fail_count >= FAIL_THRESHOLD:
warning.append(issue)
elif fail_count >= 2:
info.append(issue)
# 按失败次数倒序
critical.sort(key=lambda x: x["fail_count"], reverse=True)
warning.sort(key=lambda x: x["fail_count"], reverse=True)
info.sort(key=lambda x: x["fail_count"], reverse=True)
return critical, warning, info
def generate_report(critical, warning, info, days, total_runs):
"""生成Markdown报告"""
lines = []
lines.append("# CI重复失败检测报告")
lines.append("")
lines.append(f"**统计周期**: 最近{days}")
lines.append(f"**扫描Runs**: {total_runs}")
lines.append(f"**生成时间**: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
lines.append("")
lines.append(f"## 概览")
lines.append("")
lines.append(f"| 级别 | 数量 |")
lines.append(f"|------|------|")
lines.append(f"| 🔴 严重 (连续失败≥{CONSECUTIVE_FAIL_THRESHOLD}次 或 失败率≥50%) | {len(critical)} |")
lines.append(f"| 🟡 警告 (失败率≥{FAIL_RATE_THRESHOLD}% 且 失败≥{FAIL_THRESHOLD}次) | {len(warning)} |")
lines.append(f"| 🔵 关注 (失败≥2次) | {len(info)} |")
lines.append("")
if critical:
lines.append("## 🔴 严重问题")
lines.append("")
for item in critical:
lines.append(f"### {item['name']}")
lines.append("")
lines.append(f"- 失败次数: **{item['fail_count']}** / {item['total']} 次运行")
lines.append(f"- 失败率: **{item['failure_rate']}%**")
lines.append(f"- 当前连续失败: **{item['current_streak']}** 次 (历史最高: {item['max_streak']} 次)")
lines.append("")
if item["recent_failures"]:
lines.append("最近失败:")
lines.append("")
for f in item["recent_failures"]:
lines.append(f"- [{f['time'][:16]}] run #{f['run_id']} ({f['event']})")
lines.append("")
if warning:
lines.append("## 🟡 警告")
lines.append("")
for item in warning:
lines.append(
f"- **{item['name']}**: {item['fail_count']}次失败 / {item['total']}次运行 ({item['failure_rate']}%)"
)
lines.append("")
if info:
lines.append("## 🔵 关注列表")
lines.append("")
lines.append("| Job名称 | 失败次数 | 总次数 | 失败率 | 当前连续 |")
lines.append("|---------|----------|--------|--------|----------|")
for item in info[:20]: # 最多显示20个
lines.append(
f"| {item['name']} | {item['fail_count']} | {item['total']} | {item['failure_rate']}% | {item['current_streak']} |"
)
lines.append("")
return "\n".join(lines)
def send_feishu_notification(critical, warning, info, days):
"""发送飞书通知"""
if not WEBHOOK:
print(" ⚠️ 未配置WEBHOOK,跳过飞书通知")
return False
total_issues = len(critical) + len(warning) + len(info)
if total_issues == 0:
print(" ✅ 无异常,不发送通知")
return True
level = "🔴 严重告警" if critical else "🟡 警告" if warning else "🔵 关注"
title = f"CI重复失败检测 - {level}"
text = f"统计周期: 最近{days}\n\n"
if critical:
text += "【严重问题】\n"
for item in critical[:5]:
text += f"{item['name']}\n"
text += f" 失败 {item['fail_count']}/{item['total']} ({item['failure_rate']}%) 连续{item['current_streak']}\n"
if len(critical) > 5:
text += f" ...还有{len(critical)-5}\n"
text += "\n"
if warning:
text += "【警告】\n"
for item in warning[:5]:
text += f"{item['name']}: {item['fail_count']}次失败 ({item['failure_rate']}%)\n"
if len(warning) > 5:
text += f" ...还有{len(warning)-5}\n"
text += "\n"
if info and not critical and not warning:
text += "【关注列表】\n"
for item in info[:10]:
text += f"{item['name']}: {item['fail_count']}次失败\n"
text += "\n"
text += f"共发现 {total_issues} 个异常job"
payload = {"msg_type": "text", "content": {"text": f"{title}\n\n{text}"}}
data = json.dumps(payload).encode()
req = urllib.request.Request(WEBHOOK, data=data, headers={"Content-Type": "application/json"})
try:
with urllib.request.urlopen(req, timeout=10) as resp:
result = json.loads(resp.read())
if result.get("code") == 0 or result.get("StatusCode") == 0:
print(" ✅ 飞书通知已发送")
return True
else:
print(f" ⚠️ 飞书返回: {result}")
return False
except Exception as e:
print(f" ❌ 飞书通知失败: {e}")
return False
def main():
print(f"=== CI重复失败检测 ===")
print(f"统计周期: 最近{DAYS}")
print(f"仓库: {REPO}")
print()
print("1. 获取最近的Runs...")
runs = fetch_recent_runs(days=DAYS)
print(f" 找到 {len(runs)} 个runs")
if not runs:
print("⚠️ 没有找到runs,退出")
return
print()
print("2. 分析job失败情况(可能需要点时间)...")
job_stats, consecutive = analyze_failures(runs)
print(f" 共统计 {len(job_stats)} 个job")
print()
print("3. 识别高风险job...")
critical, warning, info = find_high_failures(job_stats, consecutive)
print(f" 🔴 严重: {len(critical)}")
print(f" 🟡 警告: {len(warning)}")
print(f" 🔵 关注: {len(info)}")
print()
print("4. 生成报告...")
report = generate_report(critical, warning, info, DAYS, len(runs))
# 保存报告
report_path = os.environ.get("REPORT_PATH", f"/tmp/ci_failure_report_{int(time.time())}.md")
with open(report_path, "w") as f:
f.write(report)
print(f" 报告已保存: {report_path}")
# 打印摘要
print()
print("=== 摘要 ===")
if critical:
print("🔴 严重问题:")
for item in critical[:5]:
print(
f" {item['name']}: {item['fail_count']}次失败, {item['failure_rate']}%, 连续{item['current_streak']}"
)
if warning:
print("🟡 警告:")
for item in warning[:5]:
print(f" {item['name']}: {item['fail_count']}次失败, {item['failure_rate']}%")
print()
print("5. 发送飞书通知...")
send_feishu_notification(critical, warning, info, DAYS)
print()
print("✅ 检测完成")
# 有严重问题时退出码非零,方便workflow标记
if critical:
sys.exit(2)
elif warning:
sys.exit(1)
if __name__ == "__main__":
main()
+20 -44
View File
@@ -1,7 +1,6 @@
#!/bin/bash
# CI Integration Tests Job 主脚本
# 包含:依赖安装、ffmpeg安装、Redis启动、PG启动、迁移、测试、清理、覆盖率
# 支持 pytest-xdist 并行执行:每个 worker 使用独立数据库,预期加速 2-4 倍
set -eu
echo "=== CI Integration Tests 开始 ==="
@@ -29,13 +28,12 @@ for i in 1 2 3; do
sleep 5
done
for i in 1 2 3; do
python3 -m pip install -q pytest-rerunfailures pytest-xdist && break
echo "pip install pytest-rerunfailures/pytest-xdist 失败,重试 $i/3..."
python3 -m pip install -q pytest-rerunfailures && break
echo "pip install pytest-rerunfailures 失败,重试 $i/3..."
[ $i -eq 3 ] && exit 1
sleep 5
done
pytest --version
echo "pytest-xdist: $(python3 -c "import xdist; print(xdist.__version__)" 2>/dev/null || echo 'not installed')"
# --- 安装 ffmpeg ---
echo ""
@@ -189,8 +187,8 @@ if [ "$USE_SHARED_PG" = "true" ]; then
echo "等待共享PG连接就绪..."
wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5
# 创建主数据库(xdist 模式下各 worker 会创建自己的数据库,主库作为 fallback)
echo "创建测试数据库: $CI_DB_NAME"
# 创建独立数据库
echo "创建测试数据库: $CI_DB_NAME"
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
import psycopg2
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
@@ -240,37 +238,30 @@ else
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
fi
# --- 执行迁移(主数据库,xdist worker 会各自创建自己的库并迁移) ---
# --- 执行迁移 ---
echo ""
echo "=== 执行 Alembic 迁移(主数据库) ==="
echo "=== 执行 Alembic 迁移 ==="
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
echo "✅ 迁移完成"
# --- 运行集成测试pytest-xdist 并行) ---
# --- 运行集成测试 ---
echo ""
echo "=== 运行集成测试pytest-xdist 并行模式) ==="
echo "CPU 核数: $(nproc 2>/dev/null || echo 'unknown')"
# 集成测试使用 pytest-xdist 并行加速(coverage 由单元测试负责,并行模式下 coverage 不稳定)
# -n 2: 限制2个worker,避免DooD模式下读到宿主机全部核数导致OOM
# 集成测试涉及ffmpeg编码+PG多库,内存开销大,2worker较稳妥
# (auto模式下读到宿主机4核8线程=8workerOOM直接杀worker进程)
# --dist loadfile: 同一测试文件分配到同一 worker(共享 fixture 更高效)
# --maxfail=1: 遇到失败停止调度新测试(并行模式下等价于 -x)
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration \
-q --timeout=60 --maxfail=1 --reruns 2 --reruns-delay 1 \
-m "not performance" \
-n 2 --dist loadfile \
-p no:cacheprovider
echo "=== 运行集成测试 ==="
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m coverage run \
--source=apps/api/app,packages \
--omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \
--branch \
-m pytest tests/integration -q --timeout=60 -x --reruns 2 --reruns-delay 1 -m "not performance"
python3 -m coverage report --show-missing
python3 -m coverage xml -o coverage.xml
python3 -m coverage report --fail-under=40 > /dev/null
echo "✅ 集成测试通过"
# --- API 性能基线测试(仅告警,串行执行 ---
# --- API 性能基线测试(仅告警) ---
echo ""
echo "=== API 性能基线测试(仅告警) ==="
set +e
PERF_OUTPUT=$(mktemp)
# 性能测试单独串行运行(不参与并行,避免资源竞争影响测量结果)
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration/test_api_performance.py \
-v --timeout=120 -p no:cacheprovider 2>&1 | tee "$PERF_OUTPUT"
echo ""
@@ -293,29 +284,14 @@ set -e
echo ""
echo "=== 清理 ==="
if [ "$USE_SHARED_PG" = "true" ]; then
# 清理共享PG上的测试数据库(主库 + 可能残留的 worker 库)
echo "清理共享PG测试数据库..."
# 清理所有以 CI_DB_NAME 开头的数据库(主库 + worker 库)
# 清理共享PG上的测试数据库
echo "清理共享PG测试数据库: $CI_DB_NAME"
PGPASSWORD="${SHARED_PG_PASSWORD}" python3 -c "
import psycopg2
conn = psycopg2.connect(host='${SHARED_PG_HOST}', port=${SHARED_PG_PORT}, user='${SHARED_PG_USER}', password='${SHARED_PG_PASSWORD}', dbname='postgres')
conn.autocommit = True
cur = conn.cursor()
# 查找所有需要清理的数据库(主库 + worker 库)
cur.execute(\"SELECT datname FROM pg_database WHERE datname LIKE '$CI_DB_NAME%'\")
dbs = [row[0] for row in cur.fetchall()]
for db in dbs:
try:
# 强制断开所有连接
cur.execute(f\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{db}' AND pid <> pg_backend_pid()\")
cur.execute(f'DROP DATABASE IF EXISTS \"{db}\" WITH (FORCE)')
print(f' 已清理: {db}')
except Exception as e:
print(f' 警告: 清理 {db} 失败: {e}')
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
cur.close()
conn.close()
" 2>/dev/null || echo "WARN: 数据库清理失败(可能已被清理)"
-2
View File
@@ -2,5 +2,3 @@
# CI 公共步骤:Job 开始计时
echo "JOB_START_TIME=$(date +%s)" >> $GITHUB_ENV
echo "Job started at $(date)"
# trigger CI run for PR validation
# trigger CI - worker dood fallback fix test
-186
View File
@@ -1,186 +0,0 @@
#!/bin/bash
# CI Validate: 代码质量与安全扫描(并行Job 1/3)
# 包含:密钥扫描、格式检查、安全扫描、依赖漏洞、死代码检测、脚本语法校验
set -eu
echo "=== CI Validate: 代码质量与安全扫描 ==="
# --- 密钥检测 ---
echo ""
echo "=== [1/6] Secret detection (detect-secrets) ==="
python3 -m pip install -q detect-secrets
detect-secrets --version
detect-secrets scan \
--all-files \
--exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \
--exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \
--exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \
--disable-plugin Base64HighEntropyString \
--disable-plugin HexHighEntropyString \
--disable-plugin BasicAuthDetector \
--disable-plugin KeywordDetector \
--disable-plugin IPPublicDetector \
> /tmp/secrets-scan.json 2>&1
FOUND=$(python3 -c "
import json
try:
with open('/tmp/secrets-scan.json') as f:
data = json.load(f)
results = data.get('results', {})
total = sum(len(v) for v in results.values())
print(total)
except Exception:
print('error')
")
echo "Secrets detected: $FOUND"
if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then
echo ""
echo "=== Secret details ==="
python3 -c "
import json
with open('/tmp/secrets-scan.json') as f:
data = json.load(f)
for fpath, items in data.get('results', {}).items():
for item in items:
line = item.get('line_number', '?')
stype = item.get('type', '?')
hashed = item.get('hashed_secret', '')[:16]
print(f' {fpath}:{line} [{stype}] {hashed}...')
"
echo ""
echo "ERROR: Potential secrets detected in code!"
exit 1
fi
echo "✅ Secret scan passed"
# --- 增量/全量模式判断 ---
echo ""
echo "=== [2/6] Code quality checks ==="
SCAN_MODE="full"
CHANGED_PY_FILES=""
if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then
PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||')
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100"
set +e
RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL")
HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
BODY=$(echo "$RESPONSE" | sed '$d')
set -e
if [ "$HTTP_CODE" = "200" ]; then
CHANGED_PY_FILES=$(echo "$BODY" | python3 -c "
import json, sys
try:
files = json.load(sys.stdin)
py_files = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] != 'removed']
print(' '.join(py_files))
except Exception:
print('')
")
if [ -n "$CHANGED_PY_FILES" ]; then
SCAN_MODE="incremental"
echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed"
else
SCAN_MODE="skip_py"
echo "No Python files changed in this PR"
fi
else
echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan"
fi
else
echo "Full scan mode (not a PR event)"
fi
if [ "$SCAN_MODE" = "incremental" ]; then
# 防御性过滤
EXISTING_PY_FILES=""
for f in $CHANGED_PY_FILES; do
if [ -f "$f" ]; then
if [ -z "$EXISTING_PY_FILES" ]; then
EXISTING_PY_FILES="$f"
else
EXISTING_PY_FILES="$EXISTING_PY_FILES $f"
fi
fi
done
CHANGED_PY_FILES="$EXISTING_PY_FILES"
python3 -m compileall -q $CHANGED_PY_FILES
python3 -m black --check --fast $CHANGED_PY_FILES
python3 -m isort --check-only $CHANGED_PY_FILES
RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | grep -v '^$' | xargs)
if [ -n "$RUFF_FILES" ]; then
python3 -m ruff check $RUFF_FILES --statistics
else
echo "No ruff-checkable files changed, skipping"
fi
elif [ "$SCAN_MODE" = "skip_py" ]; then
echo "No Python files changed - skipping Python lint checks"
else
echo "Full scan mode"
python3 -m compileall -q alembic apps packages tests scripts
python3 -m black --check --fast alembic apps packages tests scripts
python3 -m isort --check-only alembic apps packages tests scripts
python3 -m ruff check apps packages tests --statistics
fi
echo "✅ Code quality checks passed"
# --- Bandit 安全扫描(仅告警) ---
echo ""
echo "=== [3/6] Security scan (bandit, advisory only) ==="
set +e
bandit -r apps packages -q -ll
BANDIT_EXIT=$?
set -e
if [ "$BANDIT_EXIT" -ne 0 ]; then
echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)"
else
echo "✅ Bandit security scan passed"
fi
# --- Pip-audit 依赖漏洞扫描(仅告警) ---
echo ""
echo "=== [4/6] Python dependency vulnerability scan (pip-audit, advisory only) ==="
python3 -m pip install -q pip-audit
pip-audit --version
EXIT_CODE=0
for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do
if [ -f "$req_file" ]; then
echo "--- Scanning $req_file ---"
pip-audit -r "$req_file" --desc on 2>&1 | head -40 || EXIT_CODE=$?
echo ""
fi
done
echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)"
# --- Vulture 死代码检测(仅告警) ---
echo ""
echo "=== [5/6] Dead code detection (vulture, advisory only) ==="
set +e
python3 -m pip install -q vulture
vulture --version
echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。"
echo ""
vulture apps packages scripts \
--exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \
--min-confidence 70 \
2>&1 | sort -t'(' -k2 -rn | head -80
echo ""
echo "=== vulture scan summary ==="
echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)"
echo "建议:定期人工审查高置信度(>=90%)条目"
set -e
# --- Release 脚本语法校验 ---
echo ""
echo "=== [6/6] Release scripts syntax validation ==="
bash -n scripts/backup_postgres.sh
bash -n scripts/restore_postgres_plan.sh
bash -n scripts/init_production_env.sh
echo "✅ Release scripts syntax OK"
echo ""
echo "=== CI Validate: 代码质量与安全扫描 全部通过 ✅ ==="
-182
View File
@@ -1,182 +0,0 @@
#!/bin/bash
# CI Validate: Alembic迁移验证(并行Job 3/3
# 需要PostgreSQL数据库
set -eu
echo "=== CI Validate: Alembic迁移验证 ==="
# --- DooD模式检测:确定宿主机访问地址 ---
detect_docker_host() {
local test_port="${1:-5432}"
local candidates=()
# 1. host.docker.internal
if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then
candidates+=("host.docker.internal")
fi
# 2. docker0 桥接网关
candidates+=("172.17.0.1")
# 3. 默认网关
local gw=""
gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1)
if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then
candidates+=("$gw")
fi
# 4. 宿主机同网段的.1或.254
local my_ip=""
my_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
if [ -n "$my_ip" ]; then
local subnet=$(echo "$my_ip" | cut -d. -f1-3)
candidates+=("${subnet}.1")
candidates+=("${subnet}.254")
fi
# 5. 127.0.0.1 最后尝试
candidates+=("127.0.0.1")
for candidate in "${candidates[@]}"; do
if python3 -c "
import socket
s = socket.socket()
s.settimeout(2)
try:
s.connect(('$candidate', $test_port))
s.close()
print('ok')
except:
pass
" 2>/dev/null | grep -q ok; then
echo "$candidate"
return 0
fi
done
echo "127.0.0.1"
return 1
}
# 获取宿主机IP
if [ -S /var/run/docker.sock ]; then
DOCKER_HOST_IP=$(detect_docker_host 5433)
if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then
DOCKER_HOST_IP=$(detect_docker_host 22)
fi
echo "检测到DooD模式,宿主机地址: $DOCKER_HOST_IP"
else
DOCKER_HOST_IP="127.0.0.1"
echo "非DooD模式,使用 127.0.0.1"
fi
PG_HOST="$DOCKER_HOST_IP"
echo "PG host: $PG_HOST"
# 指数退避TCP连接检查
wait_tcp_ready() {
local host="$1"
local port="$2"
local max_attempts="${3:-5}"
local delay=1
local attempt=1
while [ "$attempt" -le "$max_attempts" ]; do
if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
return 0
fi
echo "TCP连接尝试 $attempt/$max_attempts 失败,${delay}s后重试..."
sleep "$delay"
delay=$((delay * 2))
attempt=$((attempt + 1))
done
return 1
}
USE_SHARED_PG="${CI_USE_SHARED_PG:-false}"
if [ "$USE_SHARED_PG" = "true" ]; then
# 使用常驻共享PG实例
echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true"
SHARED_PG_HOST="$PG_HOST"
SHARED_PG_PORT="5433"
SHARED_PG_USER="postgres"
SHARED_PG_PASSWORD="ci_pg_2026!"
CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}"
echo "等待共享PG连接就绪..."
wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5
echo "创建测试数据库: $CI_DB_NAME"
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
import psycopg2
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
conn.autocommit = True
cur = conn.cursor()
cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"')
cur.close()
conn.close()
"
export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}"
echo "✅ 共享PG数据库已创建: $CI_DB_NAME"
# 执行迁移
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
echo "✅ Alembic migrations applied successfully"
# 清理数据库
echo "清理测试数据库: $CI_DB_NAME"
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
import psycopg2
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
conn.autocommit = True
cur = conn.cursor()
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
cur.close()
conn.close()
" 2>/dev/null || echo "WARN: 数据库清理失败"
echo "✅ 共享PG数据库已清理"
else
# 使用临时PG容器(默认模式)
echo "使用临时PG容器模式"
PG_CONTAINER=ci-pg-validate-migration-${GITHUB_RUN_ID:-$$}
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker run -d --name "$PG_CONTAINER" \
--shm-size=256m \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=xiaoxia_saas \
-P \
--health-cmd "pg_isready -U postgres" \
--health-interval 3s \
--health-timeout 3s \
--health-retries 20 \
postgres:16-alpine
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
echo "PostgreSQL port: $PG_PORT"
export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas"
# 等待容器健康
for i in $(seq 1 30); do
if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
echo "PostgreSQL container is healthy on port $PG_PORT"
break
fi
echo "Waiting for PostgreSQL container health... ($i/30)"
sleep 2
done
docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy
# TCP连通性检查
echo "验证TCP连通性 ($PG_HOST:$PG_PORT)..."
wait_tcp_ready "$PG_HOST" "$PG_PORT" 5
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
# 执行迁移
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
echo "✅ Alembic migrations applied successfully"
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
fi
echo ""
echo "=== CI Validate: Alembic迁移验证 通过 ✅ ==="
-10
View File
@@ -1,10 +0,0 @@
#!/bin/bash
# CI Validate: Mypy类型检查(并行Job 2/3
set -eu
echo "=== CI Validate: Mypy类型检查 ==="
bash scripts/ci/mypy_check.sh
echo ""
echo "=== CI Validate: Mypy类型检查 通过 ✅ ==="
-78
View File
@@ -1,78 +0,0 @@
#!/bin/bash
# Vitest 增量执行脚本
# PR模式下只跑与改动文件相关的测试,大幅节省时间
# 用法: bash scripts/ci/vitest_incremental.sh
set -eu
cd apps/web
# 如果不是PR事件,直接全量跑
if [ "${GITHUB_EVENT_NAME:-}" != "pull_request" ]; then
echo "非PR模式,全量执行Vitest"
npx --no-install vitest run --coverage
exit $?
fi
# 获取PR改动的文件列表
PR_NUMBER=$(echo "${GITHUB_REF:-}" | sed 's|refs/pull/||; s|/.*||')
if [ -z "$PR_NUMBER" ]; then
echo "无法获取PR编号,全量执行Vitest"
npx --no-install vitest run --coverage
exit $?
fi
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100"
CHANGED_FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "
import json, sys
try:
files = json.load(sys.stdin)
web_files = []
for f in files:
fname = f['filename']
# 只关注前端源码文件
if fname.startswith('apps/web/src/') and fname.endswith(('.ts', '.tsx', '.js', '.jsx')) and f['status'] != 'removed':
# 去掉apps/web/前缀,变成相对路径
web_files.append(fname.replace('apps/web/', ''))
print(' '.join(web_files))
except Exception as e:
print('')
")
if [ -z "$CHANGED_FILES" ]; then
echo "PR未改动前端源码文件,跳过Vitest"
echo "(如果配置了前端单测门禁,请确保至少有一个相关测试)"
exit 0
fi
FILE_COUNT=$(echo "$CHANGED_FILES" | wc -w)
echo "PR改动了 $FILE_COUNT 个前端文件"
echo "改动文件: $CHANGED_FILES"
# 如果改动文件太多(超过30个),全量跑更可靠
if [ "$FILE_COUNT" -gt 30 ]; then
echo "改动文件较多(>$FILE_COUNT),降级为全量执行以确保覆盖"
npx --no-install vitest run --coverage
exit $?
fi
# 使用vitest --related 跑增量测试
echo ""
echo "=== 增量执行 Vitest(只跑相关测试)==="
echo "相关源文件: $CHANGED_FILES"
echo ""
set +e
npx --no-install vitest run --related $CHANGED_FILES
VITEST_EXIT=$?
set -e
if [ "$VITEST_EXIT" -eq 0 ]; then
echo ""
echo "✅ 增量测试通过"
echo "(仅覆盖与改动相关的测试用例)"
exit 0
else
echo ""
echo "❌ 增量测试失败"
exit $VITEST_EXIT
fi
+31 -128
View File
@@ -1,57 +1,17 @@
#!/usr/bin/env python3
"""发送 CI 失败通知到飞书/项目群 webhook(增强版:带失败诊断)。
诊断功能:自动分析失败原因,给出分类和修复建议。
"""
"""发送 CI 失败通知到飞书/项目群 webhook"""
import json
import os
import subprocess
import sys
import urllib.request
def run_diagnosis() -> dict:
"""运行失败诊断脚本,返回诊断结果"""
diag_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ci/ci_failure_diagnosis.py")
if not os.path.exists(diag_script):
diag_script = "scripts/ci/ci_failure_diagnosis.py"
result = {
"category": "unknown",
"category_cn": "未知",
"severity": "medium",
"summary": "",
"error_lines": [],
"suggestions": [],
"auto_fixable": False,
}
try:
# 运行诊断脚本
env = os.environ.copy()
env["DIAGNOSIS_OUTPUT"] = "/tmp/ci_diagnosis_result.json"
proc = subprocess.run([sys.executable, diag_script], capture_output=True, text=True, timeout=30, env=env)
# 尝试读取结果文件
output_file = "/tmp/ci_diagnosis_result.json"
if os.path.exists(output_file):
with open(output_file) as f:
result = json.load(f)
elif proc.stdout:
# 从stdout解析
pass
except Exception as e:
print(f"诊断脚本执行失败: {e}", file=sys.stderr)
return result
def main() -> int:
webhook = os.environ.get("CI_NOTIFY_WEBHOOK", "")
if not webhook:
print("未配置 CI_NOTIFY_WEBHOOK,跳过通知")
print("如需启用,请在仓库 Settings -> Secrets and variables -> Actions 中添加 CI_NOTIFY_WEBHOOK")
return 0
failed_job = os.environ.get("FAILED_JOB", "Unknown Job")
@@ -61,86 +21,6 @@ def main() -> int:
run_id = os.environ.get("GITHUB_RUN_ID", "unknown")
repo = os.environ.get("GITHUB_REPOSITORY", "unknown")
run_url = f"https://git.xiaoxiajianji.com/{repo}/actions/runs/{run_id}"
pr_number = os.environ.get("PR_NUMBER", "")
# 运行诊断
diagnosis = run_diagnosis()
# 构建卡片内容
severity_color = {"high": "red", "medium": "orange", "low": "blue"}
card_status = severity_color.get(diagnosis.get("severity", "medium"), "red")
# 标题
title = f"❌ CI失败 - {diagnosis.get('category_cn', '未知')}"
# 诊断部分
diag_lines = []
diag_lines.append(f"**任务**: {failed_job}")
diag_lines.append(f"**分类**: {diagnosis.get('category_cn', '未知')}")
if diagnosis.get("summary"):
diag_lines.append(f"**问题**: {diagnosis['summary']}")
# 错误行
error_lines = diagnosis.get("error_lines", [])
if error_lines:
diag_lines.append("")
diag_lines.append("**关键错误**:")
for err in error_lines[:3]:
if len(err) > 100:
err = err[:97] + "..."
diag_lines.append(f"`{err}`")
# 修复建议
suggestions = diagnosis.get("suggestions", [])
if suggestions:
diag_lines.append("")
diag_lines.append("**修复建议**:")
for i, s in enumerate(suggestions[:3], 1):
diag_lines.append(f"{i}. {s}")
if diagnosis.get("auto_fixable"):
diag_lines.append("")
diag_lines.append("💡 *可自动修复的问题,试试Rerun*")
# 基本信息
info_lines = [
f"**分支**: {branch}",
f"**提交**: `{commit}`",
f"**提交者**: {actor}",
]
if pr_number:
info_lines.append(f"**PR**: #{pr_number}")
elements = [
{
"tag": "div",
"text": {
"tag": "lark_md",
"content": "\n".join(diag_lines),
},
},
{
"tag": "hr",
},
{
"tag": "div",
"text": {
"tag": "lark_md",
"content": "\n".join(info_lines),
},
},
{
"tag": "action",
"actions": [
{
"tag": "button",
"text": {"tag": "plain_text", "content": "查看失败日志"},
"url": run_url,
"type": "danger",
},
],
},
]
payload = {
"msg_type": "interactive",
@@ -148,11 +28,36 @@ def main() -> int:
"header": {
"title": {
"tag": "plain_text",
"content": title,
"content": "❌ CI 构建失败",
},
"status": card_status,
"status": "red",
},
"elements": elements,
"elements": [
{
"tag": "div",
"text": {
"tag": "lark_md",
"content": (
f"**任务**: {failed_job}\n"
f"**分支**: {branch}\n"
f"**提交**: {commit}\n"
f"**提交者**: {actor}\n"
f"**Run ID**: {run_id}"
),
},
},
{
"tag": "action",
"actions": [
{
"tag": "button",
"text": {"tag": "plain_text", "content": "查看失败日志"},
"url": run_url,
"type": "danger",
}
],
},
],
},
}
@@ -166,7 +71,7 @@ def main() -> int:
try:
with urllib.request.urlopen(req, timeout=10) as resp:
resp.read()
print("通知已发送(带诊断信息)")
print("通知已发送")
except Exception as e:
print(f"通知发送失败: {e}", file=sys.stderr)
return 1
@@ -176,5 +81,3 @@ def main() -> int:
if __name__ == "__main__":
sys.exit(main())
# trigger CI - bypass [ci skip] bug
+96 -154
View File
@@ -1,9 +1,22 @@
#!/bin/sh
# ===========================================
# Staging 部署脚本(SSH 模式,并行优化版
# Staging 部署脚本(SSH 模式,支持自动回滚
# ===========================================
# 通过 SSH 在 staging 服务器上执行
#
# 环境变量:
# IMAGE_TAG - 镜像版本 tag(如 commit SHA 或分支名)
# REGISTRY_TOKEN - Registry 访问令牌
# REGISTRY - Registry 地址(默认 xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji
# REGISTRY_USER - Registry 用户名(默认 xiaoxia
# ENV_FILE - 环境变量文件路径
# GENERATED_DIR - 生成文件目录
# SKIP_MIGRATION - 跳过数据库迁移(true/false,默认 false
# SKIP_ROLLBACK - 失败时跳过自动回滚(true/false,默认 false
set -eu
# ---- 重试工具函数 ----
retry_cmd() {
local max_attempts=$1
local backoff=$2
@@ -60,9 +73,10 @@ mkdir -p "$GENERATED_DIR"
mkdir -p "$LEGACY_ASSETS_DIR"
echo "==========================================="
echo " Staging 部署 - $IMAGE_TAG (并行优化版)"
echo " Staging 部署 - $IMAGE_TAG"
echo "==========================================="
# ---- 记录当前运行的镜像版本(用于回滚) ----
echo "Recording current image versions for rollback..."
PREV_API_IMAGE=""
PREV_WORKER_IMAGE=""
@@ -81,6 +95,7 @@ for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do
fi
done
# ---- 回滚函数 ----
rollback() {
echo ""
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
@@ -93,6 +108,7 @@ rollback() {
exit 1
fi
# 停止当前(失败的)新容器
echo "Stopping new containers..."
docker rm -f xiaoxia-api-staging 2>/dev/null || true
docker rm -f xiaoxia-worker-staging 2>/dev/null || true
@@ -100,6 +116,7 @@ rollback() {
LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
# 恢复 API
if [ -n "$PREV_API_IMAGE" ]; then
echo "Rolling back API to: $PREV_API_IMAGE"
docker run -d \
@@ -120,9 +137,12 @@ rollback() {
--health-retries 3 \
--health-start-period 40s \
$LOG_OPTS \
"$PREV_API_IMAGE" &
"$PREV_API_IMAGE"
else
echo "No previous API image to roll back to"
fi
# 恢复 Worker
if [ -n "$PREV_WORKER_IMAGE" ]; then
echo "Rolling back Worker to: $PREV_WORKER_IMAGE"
docker run -d \
@@ -144,9 +164,12 @@ rollback() {
--health-retries 3 \
--health-start-period 30s \
$LOG_OPTS \
"$PREV_WORKER_IMAGE" &
"$PREV_WORKER_IMAGE"
else
echo "No previous Worker image to roll back to"
fi
# 恢复 Web
if [ -n "$PREV_WEB_IMAGE" ]; then
echo "Rolling back Web to: $PREV_WEB_IMAGE"
LEGACY_VOLUME=""
@@ -164,11 +187,12 @@ rollback() {
--health-timeout 5s \
--health-retries 3 \
$LOG_OPTS \
"$PREV_WEB_IMAGE" &
"$PREV_WEB_IMAGE"
else
echo "No previous Web image to roll back to"
fi
wait
# 等待 API 回滚后恢复健康
if [ -n "$PREV_API_IMAGE" ]; then
echo "Waiting for rolled-back API to become healthy..."
i=0
@@ -200,6 +224,7 @@ rollback() {
exit 1
}
# ---- 登录 Registry ----
if [ -n "$REGISTRY_TOKEN" ]; then
echo "=========================================="
echo " Login to Registry (with retries)"
@@ -208,64 +233,28 @@ if [ -n "$REGISTRY_TOKEN" ]; then
retry_docker_login
fi
# ---- 并行 Pull 三个镜像 ----
# ---- Pull 新版本镜像 ----
REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}"
REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}"
REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}"
echo "=========================================="
echo " Pull images (parallel, up to 3 retries each)"
echo " Pull images (with retries)"
echo "=========================================="
PULL_LOG_DIR="/tmp/staging-pull-$$"
mkdir -p "$PULL_LOG_DIR"
retry_docker_pull "$REGISTRY_API" > "$PULL_LOG_DIR/api.log" 2>&1 &
PID_API=$!
retry_docker_pull "$REGISTRY_WORKER" > "$PULL_LOG_DIR/worker.log" 2>&1 &
PID_WORKER=$!
retry_docker_pull "$REGISTRY_WEB" > "$PULL_LOG_DIR/web.log" 2>&1 &
PID_WEB=$!
wait $PID_API $PID_WORKER $PID_WEB
echo ""
echo "Pull 结果:"
PULL_FAILED=0
for svc in api worker web; do
if tail -1 "$PULL_LOG_DIR/$svc.log" 2>/dev/null | grep -qE "Status:|Downloaded|already exists|is up to date"; then
echo " OK $svc"
elif grep -qE "Digest:|Status: Downloaded" "$PULL_LOG_DIR/$svc.log" 2>/dev/null; then
echo " OK $svc"
else
# 检查docker pull返回值不直接,用镜像是否存在来判断
img_var="REGISTRY_$(echo $svc | tr '[:lower:]' '[:upper:]')"
img_val=$(eval echo "\$$img_var")
if docker image inspect "$img_val" >/dev/null 2>&1; then
echo " OK $svc"
else
echo " FAIL $svc"
tail -5 "$PULL_LOG_DIR/$svc.log" 2>/dev/null || true
PULL_FAILED=$((PULL_FAILED + 1))
fi
fi
done
rm -rf "$PULL_LOG_DIR"
if [ "$PULL_FAILED" -gt 0 ]; then
echo ""
echo "ERROR: $PULL_FAILED 个镜像 pull 失败"
exit 1
fi
retry_docker_pull "$REGISTRY_API"
retry_docker_pull "$REGISTRY_WORKER"
retry_docker_pull "$REGISTRY_WEB"
echo "All images pulled."
# ---- 备份 legacy assets ----
echo "Backing up legacy assets from current web container..."
if docker inspect xiaoxia-web-staging >/dev/null 2>&1; then
_tmpdir="/tmp/legacy-assets-$$"
rm -rf "$_tmpdir"
mkdir -p "$_tmpdir"
docker cp xiaoxia-web-staging:/usr/share/nginx/html/assets/. "$_tmpdir/" 2>/dev/null || true
# 只有目录非空才拷贝,避免覆盖有内容的 legacy assets
if [ -d "$_tmpdir" ] && [ "$(ls -A "$_tmpdir" 2>/dev/null)" ]; then
cp -an "$_tmpdir"/. "$LEGACY_ASSETS_DIR"/ 2>/dev/null || true
echo "Legacy assets backed up: $(ls "$_tmpdir" | wc -l) files"
@@ -275,11 +264,13 @@ else
echo "No existing web container, skipping legacy assets backup"
fi
# 清理 7 天前的 legacy assets
if [ -d "$LEGACY_ASSETS_DIR" ]; then
find "$LEGACY_ASSETS_DIR" -type f -mtime +7 -delete 2>/dev/null || true
echo "Legacy assets cleanup done (retain 7 days)"
fi
# ---- 检查基础设施容器 ----
echo "Checking infrastructure containers..."
for c in xiaoxia-postgres-staging xiaoxia-redis-staging; do
if ! docker inspect "$c" >/dev/null 2>&1; then
@@ -293,8 +284,10 @@ for c in xiaoxia-postgres-staging xiaoxia-redis-staging; do
fi
done
# ---- 创建网络(不存在则创建) ----
docker network create xiaoxia-net-staging 2>/dev/null || true
# ---- 数据库迁移 ----
if [ "$SKIP_MIGRATION" != "true" ]; then
echo "Running database migrations..."
docker run --rm \
@@ -303,6 +296,8 @@ if [ "$SKIP_MIGRATION" != "true" ]; then
-e APP_ENV=staging \
"$REGISTRY_API" sh -c "cd /app && alembic upgrade head" || {
echo "ERROR: Database migration failed"
echo "Note: Migration failures are NOT automatically rolled back (data safety)"
echo "Please manually check and fix the migration, then redeploy"
exit 1
}
echo "Migrations completed."
@@ -310,6 +305,7 @@ else
echo "Skipping migrations (SKIP_MIGRATION=true)"
fi
# ---- 停止旧容器 ----
echo "Stopping old containers..."
docker rm -f xiaoxia-api-staging 2>/dev/null || true
docker rm -f xiaoxia-worker-staging 2>/dev/null || true
@@ -317,14 +313,8 @@ docker rm -f xiaoxia-web-staging 2>/dev/null || true
LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
# ---- 并行启动三个容器 ----
echo "Starting all containers (parallel)..."
LEGACY_VOLUME=""
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
fi
# ---- 启动 API ----
echo "Starting API container..."
docker run -d \
--name xiaoxia-api-staging \
--env-file "$ENV_FILE" \
@@ -343,9 +333,10 @@ docker run -d \
--health-retries 3 \
--health-start-period 40s \
$LOG_OPTS \
"$REGISTRY_API" &
PID_API_START=$!
"$REGISTRY_API" || rollback
# ---- 启动 Worker ----
echo "Starting Worker container..."
docker run -d \
--name xiaoxia-worker-staging \
--env-file "$ENV_FILE" \
@@ -365,9 +356,18 @@ docker run -d \
--health-retries 3 \
--health-start-period 30s \
$LOG_OPTS \
"$REGISTRY_WORKER" &
PID_WORKER_START=$!
"$REGISTRY_WORKER" || rollback
# ---- 启动 Web ----
LEGACY_VOLUME=""
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
echo "Web container: legacy assets mounted (fallback)"
else
echo "Web container: no legacy assets to mount"
fi
echo "Starting Web container..."
docker run -d \
--name xiaoxia-web-staging \
--network xiaoxia-net-staging \
@@ -379,111 +379,53 @@ docker run -d \
--health-timeout 5s \
--health-retries 3 \
$LOG_OPTS \
"$REGISTRY_WEB" &
PID_WEB_START=$!
"$REGISTRY_WEB" || rollback
wait $PID_API_START $PID_WORKER_START $PID_WEB_START
START_FAILED=0
for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do
if ! docker inspect "$c" >/dev/null 2>&1; then
echo " FAIL $c: not created"
START_FAILED=$((START_FAILED + 1))
else
state=$(docker inspect -f '{{.State.Status}}' "$c")
if [ "$state" = "running" ] || [ "$state" = "starting" ]; then
echo " OK $c: $state"
else
echo " FAIL $c: $state"
docker logs --tail 20 "$c" 2>/dev/null || true
START_FAILED=$((START_FAILED + 1))
fi
# ---- 等待 API 健康 ----
echo "Waiting for API to become healthy..."
i=0
while [ "$i" -lt 40 ]; do
if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then
echo "API is healthy!"
break
fi
i=$((i + 1))
echo " Waiting... ($i/40)"
sleep 3
done
if [ "$START_FAILED" -gt 0 ]; then
echo "ERROR: $START_FAILED 个容器启动失败"
rollback
fi
# ---- 并行等待 API 和 Web 健康 ----
echo ""
echo "Waiting for API + Web health (parallel)..."
HEALTH_LOG_DIR="/tmp/staging-health-$$"
mkdir -p "$HEALTH_LOG_DIR"
(
i=0
while [ "$i" -lt 40 ]; do
if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then
echo "API healthy after $((i * 3))s"
exit 0
fi
i=$((i + 1))
sleep 3
done
echo "API FAILED after 120s"
exit 1
) > "$HEALTH_LOG_DIR/api.log" 2>&1 &
PID_API_HEALTH=$!
(
i=0
while [ "$i" -lt 15 ]; do
if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then
echo "Web healthy after $((i * 2))s"
exit 0
fi
i=$((i + 1))
sleep 2
done
echo "Web FAILED after 30s"
exit 1
) > "$HEALTH_LOG_DIR/web.log" 2>&1 &
PID_WEB_HEALTH=$!
set +e
wait $PID_API_HEALTH
API_EXIT=$?
wait $PID_WEB_HEALTH
WEB_EXIT=$?
set -e
echo ""
echo "健康检查结果:"
API_OK=0
WEB_OK=0
if [ "$API_EXIT" -eq 0 ]; then
echo " OK API: $(cat "$HEALTH_LOG_DIR/api.log")"
API_OK=1
else
echo " FAIL API: 120s未就绪"
if [ "$i" -ge 40 ]; then
echo "ERROR: API did not become healthy within 120s"
docker logs --tail 50 xiaoxia-api-staging
fi
if [ "$WEB_EXIT" -eq 0 ]; then
echo " OK Web: $(cat "$HEALTH_LOG_DIR/web.log")"
WEB_OK=1
else
echo " FAIL Web: 30s未就绪"
docker logs --tail 30 xiaoxia-web-staging
fi
rm -rf "$HEALTH_LOG_DIR"
if [ "$API_OK" -eq 0 ] || [ "$WEB_OK" -eq 0 ]; then
echo ""
echo "ERROR: 健康检查失败"
rollback
fi
# ---- 等待 Web 健康 ----
echo "Waiting for Web to become healthy..."
i=0
while [ "$i" -lt 15 ]; do
if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then
echo "Web is healthy!"
break
fi
i=$((i + 1))
echo " Waiting... ($i/15)"
sleep 2
done
if [ "$i" -ge 15 ]; then
echo "ERROR: Web did not become healthy within 30s"
docker logs --tail 30 xiaoxia-web-staging
rollback
fi
# ---- 清理旧镜像 ----
echo "Cleaning up old images..."
docker image prune -af --filter "until=168h" 2>/dev/null || true
docker builder prune -af --filter "until=168h" 2>/dev/null || true
echo ""
echo "=== Staging deployment complete (并行优化版) ==="
echo "=== Staging deployment complete ==="
echo "API: http://127.0.0.1:8000"
echo "Web: http://127.0.0.1:3001"
echo "Version: $IMAGE_TAG"
+2 -179
View File
@@ -2,167 +2,18 @@
集成测试公共 fixtures
提供性能测试相关的工具、fixture 和 marker。
支持 pytest-xdist 并行执行:每个 worker 使用独立数据库,数据完全隔离。
"""
from __future__ import annotations
import os
import sys
import time
from contextlib import contextmanager
from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable, Dict, List, Optional
import pytest
# ── xdist 并行数据库隔离 ──────────────────────────────────────────────────
# 每个 xdist worker 进程创建独立的数据库并执行迁移,确保测试数据完全隔离
# 通过 PYTEST_XDIST_WORKER 环境变量识别 worker(如 gw0, gw1, ...
_WORKER_DB_NAME: Optional[str] = None
def _get_worker_id() -> Optional[str]:
"""获取当前 xdist worker ID,非 worker 模式返回 None"""
return os.environ.get("PYTEST_XDIST_WORKER")
def _parse_database_url(url: str) -> Dict[str, str]:
"""
解析 DATABASE_URL,返回各组件。
支持 postgresql+psycopg://user:pass@host:port/dbname 格式
"""
from urllib.parse import urlparse
parsed = urlparse(url)
return {
"driver": parsed.scheme,
"user": parsed.username or "",
"password": parsed.password or "",
"host": parsed.hostname or "",
"port": str(parsed.port or 5432),
"dbname": parsed.path.lstrip("/") or "",
}
def _create_worker_database(worker_id: str) -> str:
"""
为 xdist worker 创建独立数据库并执行迁移。
返回新的 DATABASE_URL。
"""
base_url = os.environ.get(
"DATABASE_URL",
"postgresql+psycopg://postgres:postgres@localhost:5432/xiaoxia_saas",
)
db_info = _parse_database_url(base_url)
# 生成 worker 专属数据库名
base_db = db_info["dbname"]
worker_db = f"{base_db}_{worker_id}"
global _WORKER_DB_NAME
_WORKER_DB_NAME = worker_db
# 使用 psycopg 创建数据库(连接到 postgres 库)
try:
import psycopg
conn_str = (
f"host={db_info['host']} port={db_info['port']} "
f"user={db_info['user']} password={db_info['password']} "
f"dbname=postgres"
)
conn = psycopg.connect(conn_str, autocommit=True)
cur = conn.cursor()
# 先尝试删除(防止残留)
cur.execute(f'DROP DATABASE IF EXISTS "{worker_db}" WITH (FORCE)')
# 创建新数据库
cur.execute(f'CREATE DATABASE "{worker_db}"')
cur.close()
conn.close()
print(f"[xdist {worker_id}] ✅ 创建数据库: {worker_db}")
except ImportError:
print(f"[xdist {worker_id}] ⚠️ psycopg 未安装,跳过数据库创建")
return base_url
except Exception as e:
print(f"[xdist {worker_id}] ⚠️ 创建数据库失败: {e}")
return base_url
# 构建新的 DATABASE_URL
new_url = (
f"{db_info['driver']}://{db_info['user']}:{db_info['password']}"
f"@{db_info['host']}:{db_info['port']}/{worker_db}"
)
# 执行 alembic 迁移
print(f"[xdist {worker_id}] 🔄 执行 Alembic 迁移...")
try:
ROOT = Path(__file__).resolve().parents[2]
api_path = str(ROOT / "apps" / "api")
if api_path not in sys.path:
sys.path.insert(0, api_path)
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from alembic import command as alembic_command
from alembic.config import Config as AlembicConfig
alembic_cfg = AlembicConfig(str(ROOT / "alembic.ini"))
alembic_cfg.set_main_option("sqlalchemy.url", new_url)
# 兼容不同的脚本路径配置
alembic_cfg.set_main_option("script_location", str(ROOT / "alembic"))
# 临时设置环境变量供 alembic env.py 使用
os.environ["DATABASE_URL"] = new_url
alembic_command.upgrade(alembic_cfg, "head")
print(f"[xdist {worker_id}] ✅ 迁移完成")
except Exception as e:
print(f"[xdist {worker_id}] ❌ 迁移失败: {e}")
raise
return new_url
def _cleanup_worker_database(worker_id: str):
"""清理 xdist worker 的数据库"""
global _WORKER_DB_NAME
if not _WORKER_DB_NAME:
return
base_url = os.environ.get(
"DATABASE_URL",
"postgresql+psycopg://postgres:postgres@localhost:5432/xiaoxia_saas",
)
db_info = _parse_database_url(base_url)
try:
import psycopg
conn_str = (
f"host={db_info['host']} port={db_info['port']} "
f"user={db_info['user']} password={db_info['password']} "
f"dbname=postgres"
)
conn = psycopg.connect(conn_str, autocommit=True)
cur = conn.cursor()
# 强制断开所有连接后删除
cur.execute(
f"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
f"WHERE datname = '{_WORKER_DB_NAME}' AND pid <> pg_backend_pid()"
)
cur.execute(f'DROP DATABASE IF EXISTS "{_WORKER_DB_NAME}" WITH (FORCE)')
cur.close()
conn.close()
print(f"[xdist {worker_id}] 🧹 已清理数据库: {_WORKER_DB_NAME}")
except Exception as e:
print(f"[xdist {worker_id}] ⚠️ 清理数据库失败: {e}")
finally:
_WORKER_DB_NAME = None
# ── 性能阈值配置 ──────────────────────────────────────────────────────────
PERF_THRESHOLDS: Dict[str, int] = {
"core": 500, # 核心接口:500ms
@@ -332,41 +183,16 @@ class PerfAssert:
return "\n".join(lines)
# ── pytest hooks ──────────────────────────────────────────────────────────
# ── pytest fixtures ──────────────────────────────────────────────────────
def pytest_configure(config):
"""
pytest 配置钩子。
- 注册自定义 marker
- xdist worker 模式下:创建独立数据库 + 执行迁移
"""
# 注册自定义 marker
"""注册自定义 marker"""
config.addinivalue_line("markers", "performance: 标记为性能测试(可通过 -m 'not performance' 跳过)")
config.addinivalue_line("markers", "perf_core: 核心接口性能测试(阈值 500ms)")
config.addinivalue_line("markers", "perf_normal: 普通接口性能测试(阈值 1000ms)")
config.addinivalue_line("markers", "perf_heavy: 重操作接口性能测试(阈值 3000ms)")
# xdist worker 模式:创建独立数据库并执行迁移
worker_id = _get_worker_id()
if worker_id:
# 只有当 USE_IN_MEMORY_DB 不为 true 时才创建独立数据库
use_in_memory = os.environ.get("USE_IN_MEMORY_DB", "true").lower() == "true"
if not use_in_memory:
print(f"[xdist {worker_id}] 🚀 worker 启动,准备独立数据库...")
new_db_url = _create_worker_database(worker_id)
os.environ["DATABASE_URL"] = new_db_url
else:
print(f"[xdist {worker_id}] ️ USE_IN_MEMORY_DB=true,跳过 worker 数据库创建")
def pytest_unconfigure(config):
"""pytest 结束钩子:清理 xdist worker 数据库"""
worker_id = _get_worker_id()
if worker_id and _WORKER_DB_NAME:
_cleanup_worker_database(worker_id)
def pytest_collection_modifyitems(config, items):
"""根据环境变量自动跳过性能测试"""
@@ -377,9 +203,6 @@ def pytest_collection_modifyitems(config, items):
item.add_marker(skip_perf)
# ── pytest fixtures ──────────────────────────────────────────────────────
@pytest.fixture
def perf_assert():
"""