Compare commits
53 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a4fc1fb1bc | |||
| 0c5e1c4b72 | |||
| 2400947bff | |||
| 4fde26b0bf | |||
| f2dd0862b7 | |||
| d24caefbca | |||
| 50250536a9 | |||
| d3660df129 | |||
| e4ce3caad1 | |||
| 35f92f8101 | |||
| ed061aae3e | |||
| 6d8656b3f6 | |||
| 52e866b750 | |||
| 2cb273b472 | |||
| 7a1f7c89b5 | |||
| 8c1566cec3 | |||
| 246406eeb0 | |||
| a0c27e3132 | |||
| 61f0601cd4 | |||
| a39680dbca | |||
| 319347cb41 | |||
| 8a59fff246 | |||
| 13e5c03e77 | |||
| 83d373c840 | |||
| 7b2d794126 | |||
| 6ce7db7f8e | |||
| 6b45ddb6fc | |||
| d3438b9786 | |||
| 1669e787c2 | |||
| 7cffe564a9 | |||
| c2ad5003df | |||
| 14be2b654b | |||
| 92c11ced63 | |||
| 3379f07a15 | |||
| cfe75543a1 | |||
| 5e44f3dd1a | |||
| 95683ab5ef | |||
| 6bde806462 | |||
| 0ae105628d | |||
| 24efd5121f | |||
| e0c3bae072 | |||
| 05b84ac669 | |||
| 809339fee3 | |||
| 6fe648432a | |||
| 81ae089985 | |||
| 524dbc002a | |||
| df0e95ee58 | |||
| b59cf8aab5 | |||
| 906eb5de9c | |||
| f22a6fd90e | |||
| a18244b8a4 | |||
| 48102ca2e2 | |||
| baf6a4143c |
@@ -0,0 +1,78 @@
|
||||
name: CI Failure Monitor
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 */6 * * *' # 每6小时检查一次
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
days:
|
||||
description: '统计最近N天的失败'
|
||||
required: false
|
||||
default: '7'
|
||||
fail_threshold:
|
||||
description: '失败次数阈值'
|
||||
required: false
|
||||
default: '3'
|
||||
fail_rate_threshold:
|
||||
description: '失败率阈值(%)'
|
||||
required: false
|
||||
default: '30'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
monitor:
|
||||
name: CI重复失败检测
|
||||
runs-on: ci-l2
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
curl -sH "Authorization: token $GITHUB_TOKEN" \
|
||||
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" \
|
||||
| bash
|
||||
|
||||
- name: Record job start time
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_timer_start.sh
|
||||
|
||||
- name: Run failure detection
|
||||
shell: sh
|
||||
env:
|
||||
GITEA_API_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
|
||||
GITEA_URL: https://git.xiaoxiajianji.com
|
||||
GITEA_REPO: xiaoxia/xiaoxia-saas
|
||||
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
|
||||
FAIL_CHECK_DAYS: ${{ inputs.days || 7 }}
|
||||
FAIL_THRESHOLD: ${{ inputs.fail_threshold || 3 }}
|
||||
FAIL_RATE_THRESHOLD: ${{ inputs.fail_rate_threshold || 30 }}
|
||||
run: |
|
||||
set +e
|
||||
python3 scripts/ci/ci_repeated_failure_detector.py
|
||||
EXIT_CODE=$?
|
||||
echo "检测完成,退出码: $EXIT_CODE"
|
||||
# 0=无异常, 1=有警告, 2=有严重问题
|
||||
# 监控脚本永远不fail,避免告警风暴
|
||||
exit 0
|
||||
|
||||
- name: Job duration summary
|
||||
if: always()
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_timer_end.sh
|
||||
|
||||
- name: Report CI trace
|
||||
if: always()
|
||||
shell: sh
|
||||
env:
|
||||
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
|
||||
run: |
|
||||
STATUS="ok"
|
||||
[ ${{ job.status }} = "success" ] || STATUS="error"
|
||||
START_TIME=""
|
||||
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||
@@ -76,18 +76,12 @@ jobs:
|
||||
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||
|
||||
validate:
|
||||
needs: check-frontend-only
|
||||
if: always() && needs.check-frontend-only.outputs.skip_backend != 'true'
|
||||
name: Validate Code Quality And Tests
|
||||
validate-code-quality:
|
||||
name: Validate - Code Quality
|
||||
runs-on: ci-l2
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 8
|
||||
permissions:
|
||||
contents: write
|
||||
env:
|
||||
DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas
|
||||
USE_IN_MEMORY_DB: 'false'
|
||||
CI_USE_SHARED_PG: 'true'
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
@@ -102,7 +96,6 @@ jobs:
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
# pip install 带重试(网络不稳定时自动重试)
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements-base.txt && break
|
||||
echo "pip install requirements-base.txt 失败,重试 $i/3..."
|
||||
@@ -127,11 +120,11 @@ jobs:
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
- name: Run all quality checks
|
||||
- name: Run code quality and security checks
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: bash scripts/ci/run_validate.sh
|
||||
run: bash scripts/ci/validate_code_quality.sh
|
||||
- name: Auto-fix formatting (black + isort)
|
||||
if: failure()
|
||||
shell: sh
|
||||
@@ -146,7 +139,7 @@ jobs:
|
||||
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
|
||||
run: |
|
||||
set +e
|
||||
FAILED_JOB="Validate Code Quality And Tests" python3 scripts/ci_notify_failure.py
|
||||
FAILED_JOB="Validate - Code Quality" python3 scripts/ci_notify_failure.py
|
||||
- name: Job duration summary
|
||||
if: always()
|
||||
shell: sh
|
||||
@@ -159,7 +152,161 @@ jobs:
|
||||
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
|
||||
run: |
|
||||
set +e
|
||||
NOTIFY_MODE=failure JOB_NAME="Validate Code Quality And Tests" python3 scripts/ci_notify.py
|
||||
NOTIFY_MODE=failure JOB_NAME="Validate - Code Quality" python3 scripts/ci_notify.py
|
||||
- name: Report CI trace
|
||||
if: always()
|
||||
shell: sh
|
||||
env:
|
||||
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
|
||||
run: |
|
||||
STATUS="ok"
|
||||
[ ${{ job.status }} = "success" ] || STATUS="error"
|
||||
START_TIME=""
|
||||
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||
|
||||
validate-type-check:
|
||||
name: Validate - Type Check (mypy)
|
||||
runs-on: ci-l2
|
||||
timeout-minutes: 8
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
|
||||
- name: Record job start time
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_timer_start.sh
|
||||
- name: Install dependencies
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements-base.txt && break
|
||||
echo "pip install requirements-base.txt 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements.txt && break
|
||||
echo "pip install requirements.txt 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements-dev.txt && break
|
||||
echo "pip install requirements-dev.txt 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
- name: Run mypy type check
|
||||
shell: bash
|
||||
run: bash scripts/ci/validate_mypy.sh
|
||||
- name: CI failure notification
|
||||
if: failure()
|
||||
shell: sh
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
|
||||
run: |
|
||||
set +e
|
||||
FAILED_JOB="Validate - Type Check (mypy)" python3 scripts/ci_notify_failure.py
|
||||
- name: Job duration summary
|
||||
if: always()
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_timer_end.sh
|
||||
- name: Notify on failure
|
||||
continue-on-error: true
|
||||
if: failure()
|
||||
shell: sh
|
||||
env:
|
||||
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
|
||||
run: |
|
||||
set +e
|
||||
NOTIFY_MODE=failure JOB_NAME="Validate - Type Check (mypy)" python3 scripts/ci_notify.py
|
||||
- name: Report CI trace
|
||||
if: always()
|
||||
shell: sh
|
||||
env:
|
||||
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
|
||||
run: |
|
||||
STATUS="ok"
|
||||
[ ${{ job.status }} = "success" ] || STATUS="error"
|
||||
START_TIME=""
|
||||
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||
|
||||
validate-migration:
|
||||
name: Validate - Migration (alembic)
|
||||
runs-on: ci-l2
|
||||
timeout-minutes: 8
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas
|
||||
USE_IN_MEMORY_DB: 'false'
|
||||
CI_USE_SHARED_PG: 'true'
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
|
||||
- name: Record job start time
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_timer_start.sh
|
||||
- name: Install dependencies
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements-base.txt && break
|
||||
echo "pip install requirements-base.txt 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements.txt && break
|
||||
echo "pip install requirements.txt 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements-dev.txt && break
|
||||
echo "pip install requirements-dev.txt 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
- name: Run alembic migration validation
|
||||
shell: bash
|
||||
run: bash scripts/ci/validate_migration.sh
|
||||
- name: CI failure notification
|
||||
if: failure()
|
||||
shell: sh
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
|
||||
run: |
|
||||
set +e
|
||||
FAILED_JOB="Validate - Migration (alembic)" python3 scripts/ci_notify_failure.py
|
||||
- name: Job duration summary
|
||||
if: always()
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_timer_end.sh
|
||||
- name: Notify on failure
|
||||
continue-on-error: true
|
||||
if: failure()
|
||||
shell: sh
|
||||
env:
|
||||
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
|
||||
run: |
|
||||
set +e
|
||||
NOTIFY_MODE=failure JOB_NAME="Validate - Migration (alembic)" python3 scripts/ci_notify.py
|
||||
- name: Report CI trace
|
||||
if: always()
|
||||
shell: sh
|
||||
@@ -387,9 +534,11 @@ jobs:
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
- name: Run Vitest with coverage
|
||||
- name: Run Vitest (incremental for PRs, full for main branches)
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_frontend_run.sh "npx --no-install vitest run --coverage"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: bash scripts/ci/vitest_incremental.sh
|
||||
- name: Job duration summary
|
||||
if: always()
|
||||
shell: sh
|
||||
@@ -472,6 +621,65 @@ jobs:
|
||||
echo "Docker login failed ($i/3), retrying in 5s..."
|
||||
sleep 5
|
||||
done
|
||||
- name: Pre-build worker base images (fallback if not exist)
|
||||
if: matrix.service == 'worker'
|
||||
id: prebuild
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas"
|
||||
BASE_BUILDER="${REGISTRY}/worker-base-builder:latest"
|
||||
BASE_RUNTIME="${REGISTRY}/worker-base-runtime:latest"
|
||||
|
||||
# 尝试拉取基础镜像
|
||||
echo "检查基础镜像..."
|
||||
if docker pull "$BASE_BUILDER" 2>/dev/null && docker pull "$BASE_RUNTIME" 2>/dev/null; then
|
||||
echo "基础镜像已存在,使用远程镜像"
|
||||
echo "fallback=false" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "基础镜像不存在,本地构建(fallback模式)..."
|
||||
|
||||
# 构建builder基础镜像
|
||||
echo "构建 worker-base-builder..."
|
||||
# 用buildx docker-container驱动构建(兼容DooD模式:普通docker build看不到容器内文件)
|
||||
BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}"
|
||||
if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
else
|
||||
docker buildx use "$BUILDER_NAME"
|
||||
fi
|
||||
docker buildx inspect --bootstrap > /dev/null 2>&1
|
||||
|
||||
# 构建builder基础镜像(带重试,buildx容器偶发不稳定)
|
||||
echo "构建 worker-base-builder..."
|
||||
for attempt in 1 2 3; do
|
||||
if docker buildx build --load -f infra/docker/worker-base-builder.Dockerfile -t "$BASE_BUILDER" .; then
|
||||
echo "worker-base-builder 构建成功"
|
||||
break
|
||||
fi
|
||||
echo "worker-base-builder 构建失败,重试 $attempt/3..."
|
||||
docker buildx rm "$BUILDER_NAME" 2>/dev/null || true
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# 构建runtime基础镜像
|
||||
echo "构建 worker-base-runtime..."
|
||||
for attempt in 1 2 3; do
|
||||
if docker buildx build --load -f infra/docker/worker-base-runtime.Dockerfile -t "$BASE_RUNTIME" .; then
|
||||
echo "worker-base-runtime 构建成功"
|
||||
break
|
||||
fi
|
||||
echo "worker-base-runtime 构建失败,重试 $attempt/3..."
|
||||
docker buildx rm "$BUILDER_NAME" 2>/dev/null || true
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
sleep 3
|
||||
done
|
||||
|
||||
echo "fallback=true" >> $GITHUB_OUTPUT
|
||||
echo "基础镜像本地构建完成"
|
||||
fi
|
||||
|
||||
- name: Build PR image (verify only, no push)
|
||||
shell: sh
|
||||
run: |
|
||||
@@ -485,6 +693,18 @@ jobs:
|
||||
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
|
||||
fi
|
||||
|
||||
# Worker fallback模式:基础镜像本地已构建,用普通docker build绕过buildx
|
||||
if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.fallback }}" = "true" ]; then
|
||||
echo "Fallback模式:用普通docker build(基础镜像本地已构建)"
|
||||
BUILD_ARG_STR=""
|
||||
for arg in $EXTRA_BUILD_ARGS; do
|
||||
BUILD_ARG_STR="$BUILD_ARG_STR --build-arg $arg"
|
||||
done
|
||||
docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR .
|
||||
echo "Fallback PR Build successful"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
NO_CACHE_FLAG=""
|
||||
for i in 1 2 3; do
|
||||
echo "PR Build attempt $i/3"
|
||||
@@ -1333,5 +1553,4 @@ jobs:
|
||||
[ ${{ job.status }} = "success" ] || STATUS="error"
|
||||
START_TIME=""
|
||||
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||
@@ -54,7 +54,9 @@ jobs:
|
||||
CONTEXTS=("CI/CD Pipeline / Frontend Lint (pull_request)")
|
||||
else
|
||||
CONTEXTS=(
|
||||
"CI/CD Pipeline / Validate Code Quality And Tests (pull_request)"
|
||||
"CI/CD Pipeline / Validate - Code Quality (pull_request)"
|
||||
"CI/CD Pipeline / Validate - Type Check (mypy) (pull_request)"
|
||||
"CI/CD Pipeline / Validate - Migration (alembic) (pull_request)"
|
||||
"CI/CD Pipeline / Frontend Lint (pull_request)"
|
||||
)
|
||||
fi
|
||||
@@ -233,7 +235,9 @@ jobs:
|
||||
echo "纯前端改动,只检查Frontend Lint"
|
||||
else
|
||||
CONTEXTS=(
|
||||
"CI/CD Pipeline / Validate Code Quality And Tests (pull_request)"
|
||||
"CI/CD Pipeline / Validate - Code Quality (pull_request)"
|
||||
"CI/CD Pipeline / Validate - Type Check (mypy) (pull_request)"
|
||||
"CI/CD Pipeline / Validate - Migration (alembic) (pull_request)"
|
||||
"CI/CD Pipeline / Frontend Lint (pull_request)"
|
||||
"CI/CD Pipeline / PR Build API Image (pull_request)"
|
||||
"CI/CD Pipeline / PR Build Worker Image (pull_request)"
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
name: Worker Base Image Build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- develop
|
||||
- main
|
||||
paths:
|
||||
- 'requirements-base.txt'
|
||||
- 'requirements-worker.txt'
|
||||
- 'infra/docker/worker-base-builder.Dockerfile'
|
||||
- 'infra/docker/worker-base-runtime.Dockerfile'
|
||||
workflow_dispatch: # 支持手动触发
|
||||
|
||||
jobs:
|
||||
build-worker-base:
|
||||
name: Build Worker Base Images
|
||||
runs-on: runtime-builder
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- name: builder
|
||||
dockerfile: infra/docker/worker-base-builder.Dockerfile
|
||||
image_name: worker-base-builder
|
||||
cache_name: worker-base-builder-cache
|
||||
- name: runtime
|
||||
dockerfile: infra/docker/worker-base-runtime.Dockerfile
|
||||
image_name: worker-base-runtime
|
||||
cache_name: worker-base-runtime-cache
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
|
||||
|
||||
- name: Docker login to Registry
|
||||
shell: sh
|
||||
env:
|
||||
ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
|
||||
ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
|
||||
GITEA_REGISTRY_USER: xiaoxia
|
||||
GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
for i in 1 2 3; do
|
||||
echo "=== Docker login 尝试 $i/3 ==="
|
||||
if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then
|
||||
echo "✅ Docker login successful"
|
||||
break
|
||||
fi
|
||||
echo "❌ Docker login 失败(尝试 $i/3),5s 后重试..."
|
||||
sleep 5
|
||||
done
|
||||
|
||||
- name: Setup buildx builder
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
BUILDER_NAME="ci-builder-${GITHUB_RUN_ID}-${{ matrix.name }}"
|
||||
if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
echo "Created $BUILDER_NAME"
|
||||
else
|
||||
docker buildx use "$BUILDER_NAME"
|
||||
echo "Using existing $BUILDER_NAME"
|
||||
fi
|
||||
docker buildx inspect --bootstrap
|
||||
|
||||
- name: Build and push base image
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
|
||||
IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:latest"
|
||||
SAFE_REF_NAME=$(echo "${GITHUB_REF_NAME}" | tr '/' '-')
|
||||
CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:${SAFE_REF_NAME}"
|
||||
|
||||
echo "=== Building ${{ matrix.name }} base image ==="
|
||||
echo "Image: ${IMAGE_TAG}"
|
||||
echo "Cache: ${CACHE_REF}"
|
||||
|
||||
# 用通用构建脚本
|
||||
bash scripts/ci/docker_build_push.sh ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}"
|
||||
|
||||
# 同时推送到 Gitea Packages 作为备份(可选)
|
||||
GITEA_IMAGE="git.xiaoxiajianji.com/xiaoxia-saas/${{ matrix.image_name }}:latest"
|
||||
docker tag "${IMAGE_TAG}" "${GITEA_IMAGE}"
|
||||
docker push "${GITEA_IMAGE}" || echo "Gitea Packages push failed (non-fatal)"
|
||||
|
||||
echo ""
|
||||
echo "✅ ${{ matrix.name }} base image built and pushed"
|
||||
|
||||
- name: Cleanup buildx builder
|
||||
if: always()
|
||||
shell: sh
|
||||
run: |
|
||||
docker buildx rm "ci-builder-${GITHUB_RUN_ID}-${{ matrix.name }}" 2>/dev/null || true
|
||||
docker buildx prune -f 2>/dev/null || true
|
||||
echo "Builder cleanup done"
|
||||
@@ -0,0 +1,43 @@
|
||||
# ============================================================
|
||||
# Worker Builder 基础镜像
|
||||
# 预编译:编译工具 + 基础依赖 + Worker大包
|
||||
# 当 requirements-base.txt 或 requirements-worker.txt 变更时重新构建
|
||||
# 业务构建从此镜像开始,只需要安装业务依赖,节省15+分钟
|
||||
# ============================================================
|
||||
|
||||
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim
|
||||
|
||||
# 使用阿里云镜像加速
|
||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
|
||||
|
||||
# 安装编译工具
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
python3-dev \
|
||||
binutils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 创建 venv
|
||||
RUN python -m venv /opt/venv
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
|
||||
WORKDIR /tmp
|
||||
|
||||
# 基础依赖(变化极少)
|
||||
COPY requirements-base.txt /tmp/requirements-base.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
|
||||
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
||||
-r /tmp/requirements-base.txt \
|
||||
&& rm /tmp/requirements-base.txt
|
||||
|
||||
# Worker 大包(变化少)
|
||||
COPY requirements-worker.txt /tmp/requirements-worker.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
|
||||
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
||||
-r /tmp/requirements-worker.txt \
|
||||
&& rm /tmp/requirements-worker.txt
|
||||
|
||||
# 预先做一次 strip(基础层瘦身,业务层增量)
|
||||
RUN find /opt/venv -name "*.so" -type f -exec strip --strip-all {} \; 2>/dev/null || true
|
||||
@@ -0,0 +1,17 @@
|
||||
# ============================================================
|
||||
# Worker Runtime 基础镜像
|
||||
# 预安装:ffmpeg + 运行时依赖
|
||||
# 变化极少,业务构建从此镜像开始
|
||||
# ============================================================
|
||||
|
||||
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim
|
||||
|
||||
# 使用阿里云镜像加速
|
||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
|
||||
|
||||
# 运行时依赖:ffmpeg + opencv需要的libglib
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
libglib2.0-0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
@@ -1,88 +1,43 @@
|
||||
# ============================================================
|
||||
# Worker Dockerfile - 优化版(多阶段构建 + 镜像瘦身 + cache mount加速)
|
||||
# 优化项:
|
||||
# 1. 多阶段构建:builder 阶段安装编译依赖,runtime 阶段只保留运行时
|
||||
# 2. ffmpeg 通过 apt 安装(阿里云镜像加速,几秒完成,稳定可靠)
|
||||
# 3. Python 依赖瘦身:strip .so 调试符号 + 清理测试文件 + 清理缓存
|
||||
# 4. pip cache mount:加速依赖下载(跨构建共享pip wheel缓存)
|
||||
# Worker Dockerfile - 分层缓存优化版
|
||||
# 优化:基础依赖 + Worker大包预构建为基础镜像,业务构建仅叠加业务依赖
|
||||
# 基础镜像:worker-base-builder / worker-base-runtime
|
||||
# 预计节省:依赖不变时构建时间从23min降至5min以内
|
||||
# ============================================================
|
||||
|
||||
# ==================== Builder 阶段 ====================
|
||||
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim AS builder
|
||||
# 从预构建的builder基础镜像开始,已经包含:
|
||||
# - 编译工具 (gcc/g++/python3-dev/binutils)
|
||||
# - requirements-base.txt 全部依赖
|
||||
# - requirements-worker.txt 全部依赖 (numpy/scipy/opencv)
|
||||
# - 预strip的.so文件
|
||||
FROM xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/worker-base-builder:latest AS builder
|
||||
|
||||
# 使用阿里云镜像加速
|
||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
|
||||
|
||||
# 安装编译工具(仅 builder 需要)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
python3-dev \
|
||||
binutils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
# ---- 安装 Python 依赖 ----
|
||||
WORKDIR /tmp
|
||||
|
||||
# 创建 venv
|
||||
RUN python -m venv /opt/venv
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
|
||||
# 基础依赖
|
||||
COPY requirements-base.txt /tmp/requirements-base.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
|
||||
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
||||
-r /tmp/requirements-base.txt \
|
||||
&& rm /tmp/requirements-base.txt
|
||||
WORKDIR /tmp
|
||||
|
||||
# Worker 专属大包
|
||||
COPY requirements-worker.txt /tmp/requirements-worker.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
|
||||
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
||||
-r /tmp/requirements-worker.txt \
|
||||
&& rm /tmp/requirements-worker.txt
|
||||
|
||||
# 业务依赖
|
||||
# ---- 安装业务依赖(变化频繁,单独一层)----
|
||||
COPY requirements.txt /tmp/requirements.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
|
||||
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
||||
-r /tmp/requirements.txt \
|
||||
&& rm /tmp/requirements.txt
|
||||
|
||||
# ---- Python 依赖瘦身 ----
|
||||
# 1. strip .so 文件的调试符号(节省约 80-100MB)
|
||||
# ---- 增量瘦身(只处理新增的业务依赖)----
|
||||
RUN find /opt/venv -name "*.so" -type f -exec strip --strip-all {} \; 2>/dev/null || true
|
||||
|
||||
# 2. 清理测试文件(节省约 20MB)
|
||||
RUN find /opt/venv -type d -name "tests" -exec rm -rf {} + 2>/dev/null; \
|
||||
find /opt/venv -type d -name "test" -exec rm -rf {} + 2>/dev/null; \
|
||||
find /opt/venv -name "test_*.py" -delete 2>/dev/null || true
|
||||
|
||||
# 3. 清理 .pyc 缓存和 __pycache__(节省约 10MB,运行时按需生成)
|
||||
RUN find /opt/venv -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null; \
|
||||
find /opt/venv -name "*.pyc" -delete 2>/dev/null || true
|
||||
|
||||
# 4. 清理 dist-info 中的文档
|
||||
RUN find /opt/venv -name "*.dist-info" -type d -exec sh -c 'rm -f "$1"/DESCRIPTION.rst "$1"/INSTALLER "$1"/LICENSE* "$1"/WHEEL "$1"/entry_points.txt' _ {} \; 2>/dev/null || true
|
||||
|
||||
# ==================== Runtime 阶段 ====================
|
||||
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim AS runtime
|
||||
# 从预构建的runtime基础镜像开始,已经包含:
|
||||
# - ffmpeg
|
||||
# - libglib2.0-0
|
||||
FROM xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/worker-base-runtime:latest AS runtime
|
||||
|
||||
# 构建参数:版本号
|
||||
ARG APP_VERSION=dev
|
||||
|
||||
# 使用阿里云镜像加速
|
||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
|
||||
|
||||
# 安装最小运行时依赖(opencv-python-headless 需要 libglib2.0-0)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
libglib2.0-0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 从 builder 复制 Python 虚拟环境
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
# 数据库(基础层)
|
||||
psycopg2-binary==2.9.10
|
||||
psycopg[binary]>=3.2.2
|
||||
psycopg[binary]==3.2.2
|
||||
sqlalchemy==2.0.35
|
||||
alembic==1.13.3
|
||||
|
||||
|
||||
@@ -11,4 +11,5 @@ pytest==8.3.3
|
||||
pytest-asyncio==0.24.0
|
||||
pytest-cov==6.0.0
|
||||
pytest-timeout==2.3.1
|
||||
diff-cover>=8.0
|
||||
pytest-xdist==3.6.1
|
||||
diff-cover==8.0.3
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
# 这些包体积大,API 服务不需要安装
|
||||
|
||||
# 数值计算
|
||||
numpy>=1.24.0
|
||||
numpy==1.26.4
|
||||
|
||||
# 科学计算
|
||||
scipy>=1.10.0
|
||||
scipy==1.13.1
|
||||
|
||||
# 计算机视觉(视频去重、帧处理)
|
||||
opencv-python-headless>=4.8.0
|
||||
opencv-python-headless==4.10.0.84
|
||||
|
||||
# 图像处理
|
||||
Pillow==10.4.0
|
||||
|
||||
+15
-13
@@ -31,20 +31,22 @@ def main():
|
||||
print("pending")
|
||||
return
|
||||
|
||||
# API返回按时间倒序,第一个就是最新的
|
||||
for s in statuses:
|
||||
if s.get("context") == target_context:
|
||||
status = s.get("status", "pending")
|
||||
# skipped 视为通过(条件跳过的任务不需要等)
|
||||
if status == "skipped":
|
||||
print("success")
|
||||
else:
|
||||
print(status)
|
||||
return
|
||||
# 筛选目标context,按时间倒序取最新的
|
||||
matching = [s for s in statuses if s.get("context") == target_context]
|
||||
if not matching:
|
||||
# 找不到说明CI还没开始写状态,返回pending继续等待
|
||||
print("pending")
|
||||
return
|
||||
|
||||
# 找不到这个context说明CI还没开始写状态,返回pending继续等待
|
||||
# (如果workflow真的被跳过,它会有一条status为skipped的记录)
|
||||
print("pending")
|
||||
# Gitea statuses API按时间正序返回,必须取最新的一条
|
||||
latest = max(matching, key=lambda s: s.get("created_at", ""))
|
||||
status = latest.get("status", "pending")
|
||||
|
||||
# skipped 视为通过(条件跳过的任务不需要等)
|
||||
if status == "skipped":
|
||||
print("success")
|
||||
else:
|
||||
print(status)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -168,6 +168,28 @@ def main():
|
||||
return
|
||||
|
||||
api_url = os.environ.get("GITHUB_API_URL", "")
|
||||
|
||||
# 获取PR作者信息,判断是人还是Agent提交的
|
||||
pr_info_url = f"{api_url}/repos/{repo}/pulls/{pr_number}"
|
||||
req_pr = urllib.request.Request(pr_info_url, headers={"Authorization": f"token {token}"})
|
||||
with urllib.request.urlopen(req_pr) as resp:
|
||||
pr_info = json.loads(resp.read())
|
||||
pr_author = pr_info.get("user", {}).get("login", "")
|
||||
print(f"PR作者: {pr_author}")
|
||||
|
||||
# 判断是否为Agent提交的PR
|
||||
# Agent账号:actions, auto-approve-bot 等bot用户
|
||||
# 人提交的PR(如xiaoxia):只诊断不自动修
|
||||
agent_authors = {"actions", "auto-approve-bot", "gitea-actions"}
|
||||
is_agent_pr = pr_author in agent_authors or "bot" in pr_author.lower()
|
||||
|
||||
if is_agent_pr:
|
||||
print(f"检测到Agent提交的PR(作者: {pr_author}),将自动修复并推送")
|
||||
fix_mode = "auto_fix_and_push"
|
||||
else:
|
||||
print(f"检测到人提交的PR(作者: {pr_author}),仅诊断不自动修改")
|
||||
print("(如需自动修复,请用Agent账号提交PR,或手动运行格式化脚本)")
|
||||
fix_mode = "diagnose_only"
|
||||
repo = os.environ.get("GITHUB_REPOSITORY", "")
|
||||
token = os.environ.get("GITHUB_TOKEN", "")
|
||||
scan_mode = os.environ.get("SCAN_MODE", "full")
|
||||
@@ -231,6 +253,26 @@ def main():
|
||||
print("没有需要提交的格式改动")
|
||||
return
|
||||
|
||||
# 诊断模式:只报告问题,不修改不推送
|
||||
if fix_mode == "diagnose_only":
|
||||
print()
|
||||
print("=" * 50)
|
||||
print("📋 格式问题诊断报告(人提交的PR,仅诊断不自动修复)")
|
||||
print("=" * 50)
|
||||
print()
|
||||
print("以下文件存在格式问题,建议手动修复:")
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
print(f" {line}")
|
||||
print()
|
||||
print("修复方式:")
|
||||
print(" 后端(Python): 运行 black + isort")
|
||||
print(" 前端: 运行 prettier --write")
|
||||
print(" 或使用 scripts/agent-commit.sh 提交(自动格式化)")
|
||||
print()
|
||||
print("=" * 50)
|
||||
# 以非0状态码退出,让CI继续报失败(因为问题没修)
|
||||
sys.exit(1)
|
||||
|
||||
print()
|
||||
print("变更文件:")
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
@@ -238,7 +280,7 @@ def main():
|
||||
|
||||
# 提交修复
|
||||
run("git add -A")
|
||||
run('git commit -m "style: auto-format with black + isort + prettier [ci skip]"')
|
||||
run('git commit -m "style: auto-format with black + isort + prettier"')
|
||||
|
||||
# 推送(head_branch已从ensure_git_repo获取)
|
||||
print(f"\nPR来源分支: {head_branch}")
|
||||
|
||||
@@ -0,0 +1,447 @@
|
||||
#!/usr/bin/env python3
|
||||
"""CI失败诊断增强脚本:自动分类失败原因 + 提取关键错误 + 给出修复建议。
|
||||
# Trigger CI after auto-format fix
|
||||
|
||||
支持的失败类型:
|
||||
1. Lint/格式问题 (ruff/black/eslint/prettier)
|
||||
2. 单元测试失败
|
||||
3. Docker构建失败
|
||||
4. 依赖安装失败 (pip/npm)
|
||||
5. 超时
|
||||
6. 缓存问题
|
||||
7. 数据库/迁移问题
|
||||
8. 网络问题
|
||||
9. 其他
|
||||
|
||||
用法:
|
||||
python3 scripts/ci/ci_failure_diagnosis.py [--job-name "Job Name"] [--log-file /path/to/log]
|
||||
|
||||
如果不传--log-file,会尝试从Gitea API获取失败job的日志。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class FailureDiagnosis:
|
||||
"""失败诊断结果"""
|
||||
|
||||
category: str # 失败分类
|
||||
category_cn: str # 中文分类名
|
||||
severity: str # 严重程度: high / medium / low
|
||||
summary: str # 一句话摘要
|
||||
error_lines: List[str] = field(default_factory=list) # 关键错误行
|
||||
suggestions: List[str] = field(default_factory=list) # 修复建议
|
||||
auto_fixable: bool = False # 是否可以自动修复
|
||||
related_docs: str = "" # 相关文档链接
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 失败模式定义
|
||||
# ============================================================
|
||||
|
||||
FAILURE_PATTERNS = [
|
||||
# ===== Lint / 格式问题 =====
|
||||
{
|
||||
"pattern": r"(ruff|black|isort)\b.*(error|failed|Error)",
|
||||
"category": "lint_python",
|
||||
"category_cn": "Python代码质量检查",
|
||||
"severity": "low",
|
||||
"summary_contains": ["ruff", "black", "isort"],
|
||||
"suggestions": [
|
||||
"本地运行 `black . && isort . && ruff check --fix .` 自动修复",
|
||||
"使用 `scripts/agent-commit.sh` 提交(自动格式化)",
|
||||
"如确认无误,可加 `# noqa: xxx` 忽略特定规则",
|
||||
],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
{
|
||||
"pattern": r"ESLint|prettier|eslint",
|
||||
"category": "lint_frontend",
|
||||
"category_cn": "前端代码检查",
|
||||
"severity": "low",
|
||||
"summary_contains": ["eslint", "prettier"],
|
||||
"suggestions": [
|
||||
"本地运行 `cd apps/web && npm run lint:fix` 自动修复",
|
||||
"Prettier问题: `cd apps/web && npx prettier --write .`",
|
||||
],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
{
|
||||
"pattern": r"F\d{3}|E\d{3}|W\d{3}.*ruff|ruff.*F\d{3}",
|
||||
"category": "lint_python",
|
||||
"category_cn": "Python代码质量检查",
|
||||
"severity": "low",
|
||||
"suggestions": [
|
||||
"F401: 删除未使用的import",
|
||||
"F841: 删除未使用的变量或加下划线前缀",
|
||||
"E501: 行超长,加 `# noqa: E501`",
|
||||
"F811: 删重复import",
|
||||
"运行 `ruff check --fix .` 自动修复大部分问题",
|
||||
],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
# ===== 单元测试失败 =====
|
||||
{
|
||||
"pattern": r"FAILED|assert.*Error|AssertionError",
|
||||
"category": "unit_test",
|
||||
"category_cn": "单元测试失败",
|
||||
"severity": "high",
|
||||
"suggestions": [
|
||||
"检查相关测试文件,确认是代码问题还是测试用例问题",
|
||||
"本地运行对应测试:`pytest path/to/test.py -v`",
|
||||
"如测试依赖外部服务,检查mock是否正确",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
{
|
||||
"pattern": r"pytest.*failed|\d+ failed.*\d+ passed",
|
||||
"category": "unit_test",
|
||||
"category_cn": "单元测试失败",
|
||||
"severity": "high",
|
||||
"suggestions": [
|
||||
"查看上方日志中的FAILED测试用例",
|
||||
"检查失败断言的期望值 vs 实际值",
|
||||
"新代码影响了现有测试行为,确认是预期内变更吗?",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
# ===== Docker 构建失败 =====
|
||||
{
|
||||
"pattern": r"Dockerfile.*not found|docker build.*failed|ERROR: failed to solve",
|
||||
"category": "docker_build",
|
||||
"category_cn": "Docker构建失败",
|
||||
"severity": "high",
|
||||
"suggestions": [
|
||||
"检查Dockerfile语法是否正确",
|
||||
"检查引用的基础镜像是否存在",
|
||||
"本地运行 `docker build -f path/to/Dockerfile .` 复现",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
{
|
||||
"pattern": r"manifest.*not found|no such image|image.*not found",
|
||||
"category": "docker_build",
|
||||
"category_cn": "镜像不存在",
|
||||
"severity": "medium",
|
||||
"suggestions": [
|
||||
"检查基础镜像名称和tag是否正确",
|
||||
"确认镜像仓库可访问,登录是否有效",
|
||||
"如为新基础镜像,需先手动构建一次基础镜像",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
{
|
||||
"pattern": r"ETXTBSY|text file busy",
|
||||
"category": "docker_build",
|
||||
"category_cn": "文件锁冲突(ETXTBSY)",
|
||||
"severity": "low",
|
||||
"summary": "esbuild并发构建冲突,重试即可",
|
||||
"suggestions": ["偶发问题,点击Rerun重新运行即可", "如频繁出现,检查是否有多个job并发写入同一文件"],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
# ===== 依赖安装失败 =====
|
||||
{
|
||||
"pattern": r"pip install.*error|Could not find a version|No matching distribution",
|
||||
"category": "dependency",
|
||||
"category_cn": "pip依赖安装失败",
|
||||
"severity": "medium",
|
||||
"suggestions": [
|
||||
"检查requirements.txt中的版本号是否正确",
|
||||
"如为新版本刚发布,可能源还没同步,稍后重试",
|
||||
"检查网络连接,可尝试切换pip镜像源",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
{
|
||||
"pattern": r"npm.*ERR|npm install.*failed|E404|ECONNREFUSED.*npm",
|
||||
"category": "dependency",
|
||||
"category_cn": "npm依赖安装失败",
|
||||
"severity": "medium",
|
||||
"suggestions": [
|
||||
"检查package.json中的版本号是否存在",
|
||||
"网络问题:检查npm registry是否可访问",
|
||||
"国内网络建议配置npmmirror镜像源",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
{
|
||||
"pattern": r"Connection refused|timed out|network.*unreachable",
|
||||
"category": "network",
|
||||
"category_cn": "网络问题",
|
||||
"severity": "medium",
|
||||
"summary": "网络连接失败,可能是源站问题或DNS问题",
|
||||
"suggestions": [
|
||||
"点击Rerun重试,网络问题通常是临时的",
|
||||
"如持续失败,检查对应服务是否正常",
|
||||
"检查Runner网络配置",
|
||||
],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
# ===== 超时 =====
|
||||
{
|
||||
"pattern": r"timeout|timed out|exceeded.*time limit|job.*cancelled.*timeout",
|
||||
"category": "timeout",
|
||||
"category_cn": "执行超时",
|
||||
"severity": "medium",
|
||||
"suggestions": [
|
||||
"如首次出现:重试一次,可能是临时性能波动",
|
||||
"频繁出现:检查构建是否变慢了,最近是否加了新依赖",
|
||||
"可适当增加timeout-minutes配置",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
# ===== 数据库/迁移 =====
|
||||
{
|
||||
"pattern": r"alembic.*error|migration.*failed|relation.*does not exist|column.*does not exist",
|
||||
"category": "migration",
|
||||
"category_cn": "数据库迁移失败",
|
||||
"severity": "high",
|
||||
"suggestions": [
|
||||
"检查迁移脚本是否正确,down_revision是否对",
|
||||
"确认数据库中是否有脏数据或残留表",
|
||||
"迁移脚本合并冲突时,重新生成迁移文件",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
# ===== 缓存问题 =====
|
||||
{
|
||||
"pattern": r"cache.*corrupt|cache.*invalid|snapshot.*not found|failed to compute cache key",
|
||||
"category": "cache",
|
||||
"category_cn": "缓存损坏",
|
||||
"severity": "low",
|
||||
"suggestions": ["构建系统会自动清理损坏缓存并重试,通常无需干预", "如持续失败,手动清理Runner上的缓存目录"],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
# ===== Checkout 失败 =====
|
||||
{
|
||||
"pattern": r"Could not resolve host|fatal:.*repository|SSL.*problem",
|
||||
"category": "checkout",
|
||||
"category_cn": "代码拉取失败",
|
||||
"severity": "low",
|
||||
"suggestions": ["临时网络问题,点击Rerun重试", "如持续失败,检查Gitea服务状态"],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def analyze_log(log_text: str, job_name: str = "") -> FailureDiagnosis:
|
||||
"""分析日志,返回诊断结果"""
|
||||
|
||||
lines = log_text.strip().split("\n")
|
||||
|
||||
# 收集所有匹配的模式
|
||||
matched = []
|
||||
error_lines = []
|
||||
|
||||
for line in lines:
|
||||
line_stripped = line.strip()
|
||||
# 收集ERROR/FAILED/Failed等错误行(最多20行)
|
||||
if re.search(r"(ERROR|FAILED|Error|error:|FAIL:|Traceback)", line_stripped):
|
||||
if len(error_lines) < 20:
|
||||
error_lines.append(line_stripped)
|
||||
|
||||
for pattern_info in FAILURE_PATTERNS:
|
||||
if re.search(pattern_info["pattern"], line_stripped, re.IGNORECASE):
|
||||
matched.append(pattern_info)
|
||||
break # 一行只匹配一个模式
|
||||
|
||||
if not matched:
|
||||
# 未识别的失败类型
|
||||
return FailureDiagnosis(
|
||||
category="unknown",
|
||||
category_cn="未知错误",
|
||||
severity="medium",
|
||||
summary="未识别的失败类型,需要人工查看日志",
|
||||
error_lines=error_lines[:10],
|
||||
suggestions=[
|
||||
"点击'查看失败日志'查看完整日志",
|
||||
"如为偶发问题,可先重试一次",
|
||||
"常见原因:环境问题、配置问题、新增逻辑引入的bug",
|
||||
],
|
||||
auto_fixable=False,
|
||||
)
|
||||
|
||||
# 选最严重、最具体的那个
|
||||
severity_order = {"high": 3, "medium": 2, "low": 1}
|
||||
matched.sort(key=lambda x: severity_order.get(x["severity"], 0), reverse=True)
|
||||
best_match = matched[0]
|
||||
|
||||
# 生成摘要
|
||||
if "summary" in best_match:
|
||||
summary = best_match["summary"]
|
||||
else:
|
||||
summary = f"{best_match['category_cn']}检查失败"
|
||||
if job_name:
|
||||
summary = f"[{job_name}] {summary}"
|
||||
|
||||
# 从error_lines中过滤出与该分类相关的
|
||||
relevant_errors = error_lines[:10]
|
||||
|
||||
return FailureDiagnosis(
|
||||
category=best_match["category"],
|
||||
category_cn=best_match["category_cn"],
|
||||
severity=best_match["severity"],
|
||||
summary=summary,
|
||||
error_lines=relevant_errors,
|
||||
suggestions=best_match["suggestions"],
|
||||
auto_fixable=best_match.get("auto_fixable", False),
|
||||
)
|
||||
|
||||
|
||||
def fetch_failed_job_log(run_id: str, job_id: str, token: str, repo: str) -> Optional[str]:
|
||||
"""从Gitea API获取失败job的日志"""
|
||||
api_base = f"https://git.xiaoxiajianji.com/api/v1/repos/{repo}"
|
||||
|
||||
# 尝试获取job的日志
|
||||
url = f"{api_base}/actions/runs/{run_id}/jobs/{job_id}/log"
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header("Authorization", f"token {token}")
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
print(f"获取日志失败: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def format_diagnosis_markdown(d: FailureDiagnosis, job_name: str = "", run_url: str = "") -> str:
|
||||
"""将诊断结果格式化为飞书卡片markdown"""
|
||||
|
||||
severity_emoji = {"high": "🔴", "medium": "🟡", "low": "🟢"}
|
||||
emoji = severity_emoji.get(d.severity, "⚪")
|
||||
|
||||
lines = []
|
||||
lines.append(f"**分类**: {emoji} {d.category_cn}")
|
||||
lines.append(f"**问题**: {d.summary}")
|
||||
|
||||
if d.error_lines:
|
||||
lines.append("")
|
||||
lines.append("**关键错误行**:")
|
||||
for err in d.error_lines[:5]:
|
||||
# 截断过长的行
|
||||
if len(err) > 150:
|
||||
err = err[:147] + "..."
|
||||
lines.append(f" `{err}`")
|
||||
|
||||
lines.append("")
|
||||
lines.append("**修复建议**:")
|
||||
for i, s in enumerate(d.suggestions[:5], 1):
|
||||
lines.append(f" {i}. {s}")
|
||||
|
||||
if d.auto_fixable:
|
||||
lines.append("")
|
||||
lines.append("💡 **可自动修复**:如格式问题,可尝试点击Rerun让auto-fix自动处理")
|
||||
|
||||
if run_url:
|
||||
lines.append("")
|
||||
lines.append(f"[查看完整日志]({run_url})")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
job_name = os.environ.get("FAILED_JOB", "")
|
||||
run_id = os.environ.get("GITHUB_RUN_ID", "")
|
||||
repo = os.environ.get("GITHUB_REPOSITORY", "xiaoxia/xiaoxia-saas")
|
||||
token = os.environ.get("GITHUB_TOKEN", "")
|
||||
|
||||
# 1. 尝试获取日志
|
||||
log_text = ""
|
||||
|
||||
# 优先从环境变量或文件读取
|
||||
log_file = os.environ.get("CI_LOG_FILE", "")
|
||||
if log_file and os.path.exists(log_file):
|
||||
with open(log_file) as f:
|
||||
log_text = f.read()
|
||||
elif run_id and token:
|
||||
# 尝试从API获取(需要job_id,这里简化处理)
|
||||
pass
|
||||
|
||||
# 如果没有日志,用job_name做粗略分类
|
||||
if not log_text:
|
||||
# 基于job名做初始判断
|
||||
if any(k in job_name.lower() for k in ["validate", "lint", "quality"]):
|
||||
d = FailureDiagnosis(
|
||||
category="lint_general",
|
||||
category_cn="代码质量检查",
|
||||
severity="low",
|
||||
summary=f"{job_name} 检查失败(日志不可用,基于job名初步诊断)",
|
||||
suggestions=["点击查看日志获取具体错误信息", "格式类问题通常可自动修复"],
|
||||
auto_fixable=True,
|
||||
)
|
||||
elif "build" in job_name.lower():
|
||||
d = FailureDiagnosis(
|
||||
category="build_general",
|
||||
category_cn="构建失败",
|
||||
severity="high",
|
||||
summary=f"{job_name} 构建失败(日志不可用)",
|
||||
suggestions=["点击查看日志获取具体构建错误", "常见原因:Dockerfile错误、依赖安装失败、网络问题"],
|
||||
auto_fixable=False,
|
||||
)
|
||||
elif "test" in job_name.lower():
|
||||
d = FailureDiagnosis(
|
||||
category="test_general",
|
||||
category_cn="测试失败",
|
||||
severity="high",
|
||||
summary=f"{job_name} 测试失败(日志不可用)",
|
||||
suggestions=["点击查看日志获取具体失败的测试用例", "检查最近代码改动是否影响了测试"],
|
||||
auto_fixable=False,
|
||||
)
|
||||
elif "deploy" in job_name.lower():
|
||||
d = FailureDiagnosis(
|
||||
category="deploy_general",
|
||||
category_cn="部署失败",
|
||||
severity="high",
|
||||
summary=f"{job_name} 部署失败(日志不可用)",
|
||||
suggestions=["检查目标服务器状态和网络", "检查镜像是否正确推送", "查看服务器上的容器日志"],
|
||||
auto_fixable=False,
|
||||
)
|
||||
else:
|
||||
d = FailureDiagnosis(
|
||||
category="unknown",
|
||||
category_cn="未知错误",
|
||||
severity="medium",
|
||||
summary=f"{job_name} 失败",
|
||||
suggestions=["点击查看日志获取详细信息"],
|
||||
auto_fixable=False,
|
||||
)
|
||||
else:
|
||||
d = analyze_log(log_text, job_name)
|
||||
|
||||
# 输出诊断结果
|
||||
run_url = f"https://git.xiaoxiajianji.com/{repo}/actions/runs/{run_id}" if run_id else ""
|
||||
|
||||
print("=" * 60)
|
||||
print(" CI 失败诊断报告")
|
||||
print("=" * 60)
|
||||
print()
|
||||
print(format_diagnosis_markdown(d, job_name, run_url))
|
||||
print()
|
||||
print("=" * 60)
|
||||
|
||||
# 将诊断结果写入文件(供通知脚本读取)
|
||||
output_file = os.environ.get("DIAGNOSIS_OUTPUT", "/tmp/ci_diagnosis.json")
|
||||
result = {
|
||||
"category": d.category,
|
||||
"category_cn": d.category_cn,
|
||||
"severity": d.severity,
|
||||
"summary": d.summary,
|
||||
"error_lines": d.error_lines,
|
||||
"suggestions": d.suggestions,
|
||||
"auto_fixable": d.auto_fixable,
|
||||
}
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
print(f"\n诊断结果已保存到: {output_file}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,417 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
CI重复失败检测脚本
|
||||
- 扫描最近N天的CI失败
|
||||
- 按job名称分组统计失败率
|
||||
- 识别高失败率job(系统性故障)
|
||||
- 飞书通知告警
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
|
||||
def get_env(name, default=None, required=False):
|
||||
val = os.environ.get(name, default)
|
||||
if required and not val:
|
||||
print(f"❌ 缺少环境变量: {name}")
|
||||
sys.exit(1)
|
||||
return val
|
||||
|
||||
|
||||
GITEA_URL = get_env("GITEA_URL", "https://git.xiaoxiajianji.com")
|
||||
GITEA_TOKEN = get_env("GITEA_API_TOKEN", required=False) or get_env("GITHUB_TOKEN", "")
|
||||
REPO = get_env("GITEA_REPO", "xiaoxia/xiaoxia-saas")
|
||||
DAYS = int(get_env("FAIL_CHECK_DAYS", "7"))
|
||||
FAIL_THRESHOLD = int(get_env("FAIL_THRESHOLD", 3)) # 失败次数阈值
|
||||
FAIL_RATE_THRESHOLD = float(get_env("FAIL_RATE_THRESHOLD", "30")) # 失败率阈值%
|
||||
CONSECUTIVE_FAIL_THRESHOLD = int(get_env("CONSECUTIVE_FAIL_THRESHOLD", "3")) # 连续失败阈值
|
||||
WEBHOOK = get_env("CI_NOTIFY_WEBHOOK", "")
|
||||
|
||||
|
||||
def api_get(path):
|
||||
"""调用Gitea API"""
|
||||
url = f"{GITEA_URL}/api/v1{path}"
|
||||
req = urllib.request.Request(url)
|
||||
if GITEA_TOKEN:
|
||||
req.add_header("Authorization", f"token {GITEA_TOKEN}")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return json.loads(resp.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
print(f" HTTP {e.code}: {path}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f" 错误: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_recent_runs(days=7, per_page=50, max_pages=10):
|
||||
"""获取最近N天的runs"""
|
||||
since = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
|
||||
all_runs = []
|
||||
|
||||
for page in range(1, max_pages + 1):
|
||||
path = f"/repos/{REPO}/actions/runs?page={page}&limit={per_page}"
|
||||
data = api_get(path)
|
||||
if not data:
|
||||
break
|
||||
|
||||
runs = data.get("workflow_runs", data.get("runs", []))
|
||||
if not runs:
|
||||
break
|
||||
|
||||
# 检查时间范围(Gitea用started_at,格式2026-07-22T10:58:10+08:00)
|
||||
oldest = None
|
||||
for r in runs:
|
||||
started = r.get("started_at", r.get("created_at", ""))
|
||||
if started and started >= since:
|
||||
all_runs.append(r)
|
||||
else:
|
||||
oldest = started
|
||||
|
||||
if oldest and oldest < since:
|
||||
break
|
||||
|
||||
if len(runs) < per_page:
|
||||
break
|
||||
|
||||
return all_runs
|
||||
|
||||
|
||||
def fetch_run_jobs(run_id):
|
||||
"""获取run的所有jobs"""
|
||||
path = f"/repos/{REPO}/actions/runs/{run_id}/jobs"
|
||||
data = api_get(path)
|
||||
if not data:
|
||||
return []
|
||||
return data.get("jobs", [])
|
||||
|
||||
|
||||
def analyze_failures(runs):
|
||||
"""
|
||||
分析失败情况
|
||||
|
||||
返回:
|
||||
- job_stats: {job_name: {total, success, failure, skipped, failure_rate, failures: [...]}}
|
||||
- consecutive_failures: {job_name: current_streak, max_streak, last_status}
|
||||
"""
|
||||
job_stats = defaultdict(
|
||||
lambda: {
|
||||
"total": 0,
|
||||
"success": 0,
|
||||
"failure": 0,
|
||||
"error": 0,
|
||||
"skipped": 0,
|
||||
"cancelled": 0,
|
||||
"failures": [],
|
||||
}
|
||||
)
|
||||
|
||||
# 按时间正序排列(旧→新)用于连续失败计算
|
||||
sorted_runs = sorted(runs, key=lambda r: r.get("started_at", r.get("created_at", "")))
|
||||
|
||||
# 连续失败跟踪 {job_name: streak}
|
||||
consecutive = defaultdict(lambda: {"current": 0, "max": 0, "last_run": None})
|
||||
|
||||
for run in sorted_runs:
|
||||
run_id = run.get("id")
|
||||
run_status = run.get("status", "")
|
||||
run_conclusion = run.get("conclusion", "")
|
||||
run_started = run.get("started_at", run.get("created_at", ""))
|
||||
event = run.get("event", "")
|
||||
|
||||
# 只统计pull_request和push事件的CI
|
||||
if event not in ("pull_request", "push"):
|
||||
continue
|
||||
|
||||
jobs = fetch_run_jobs(run_id)
|
||||
|
||||
for job in jobs:
|
||||
name = job.get("name", "")
|
||||
status = job.get("status", "")
|
||||
conclusion = job.get("conclusion", "")
|
||||
|
||||
# 跳过非CI核心job(如AI Code Review、Preview等)
|
||||
skip_prefixes = ("AI Code Review", "Preview", "PR Automation", "Auto")
|
||||
if any(name.startswith(p) for p in skip_prefixes):
|
||||
continue
|
||||
|
||||
stats = job_stats[name]
|
||||
stats["total"] += 1
|
||||
|
||||
if conclusion == "success":
|
||||
stats["success"] += 1
|
||||
consecutive[name]["current"] = 0
|
||||
elif conclusion == "failure":
|
||||
stats["failure"] += 1
|
||||
stats["failures"].append(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"time": run_started,
|
||||
"event": event,
|
||||
}
|
||||
)
|
||||
consecutive[name]["current"] += 1
|
||||
if consecutive[name]["current"] > consecutive[name]["max"]:
|
||||
consecutive[name]["max"] = consecutive[name]["current"]
|
||||
consecutive[name]["last_run"] = run_id
|
||||
elif conclusion == "error":
|
||||
stats["error"] += 1
|
||||
# error也算失败的一种
|
||||
consecutive[name]["current"] += 1
|
||||
if consecutive[name]["current"] > consecutive[name]["max"]:
|
||||
consecutive[name]["max"] = consecutive[name]["current"]
|
||||
elif conclusion == "skipped":
|
||||
stats["skipped"] += 1
|
||||
# skipped不算也不打断连续失败
|
||||
elif conclusion == "cancelled":
|
||||
stats["cancelled"] += 1
|
||||
# cancelled不算失败也不打断
|
||||
|
||||
# 计算失败率
|
||||
for name, stats in job_stats.items():
|
||||
total_actual = stats["total"] - stats["skipped"] - stats["cancelled"]
|
||||
if total_actual > 0:
|
||||
stats["failure_rate"] = round((stats["failure"] + stats["error"]) / total_actual * 100, 1)
|
||||
else:
|
||||
stats["failure_rate"] = 0.0
|
||||
|
||||
return dict(job_stats), dict(consecutive)
|
||||
|
||||
|
||||
def find_high_failures(job_stats, consecutive):
|
||||
"""
|
||||
找出高风险job
|
||||
|
||||
告警级别:
|
||||
- critical: 连续失败 >= CONSECUTIVE_FAIL_THRESHOLD,或 失败率>=50%且失败次数>=5
|
||||
- warning: 失败率>=FAIL_RATE_THRESHOLD且失败次数>=FAIL_THRESHOLD
|
||||
- info: 失败次数>=2
|
||||
"""
|
||||
critical = []
|
||||
warning = []
|
||||
info = []
|
||||
|
||||
for name, stats in job_stats.items():
|
||||
fail_count = stats["failure"] + stats["error"]
|
||||
rate = stats["failure_rate"]
|
||||
streak = consecutive.get(name, {}).get("current", 0)
|
||||
max_streak = consecutive.get(name, {}).get("max", 0)
|
||||
|
||||
issue = {
|
||||
"name": name,
|
||||
"fail_count": fail_count,
|
||||
"total": stats["total"],
|
||||
"failure_rate": rate,
|
||||
"current_streak": streak,
|
||||
"max_streak": max_streak,
|
||||
"recent_failures": stats["failures"][-5:], # 最近5次
|
||||
}
|
||||
|
||||
if streak >= CONSECUTIVE_FAIL_THRESHOLD or (rate >= 50 and fail_count >= 5):
|
||||
critical.append(issue)
|
||||
elif rate >= FAIL_RATE_THRESHOLD and fail_count >= FAIL_THRESHOLD:
|
||||
warning.append(issue)
|
||||
elif fail_count >= 2:
|
||||
info.append(issue)
|
||||
|
||||
# 按失败次数倒序
|
||||
critical.sort(key=lambda x: x["fail_count"], reverse=True)
|
||||
warning.sort(key=lambda x: x["fail_count"], reverse=True)
|
||||
info.sort(key=lambda x: x["fail_count"], reverse=True)
|
||||
|
||||
return critical, warning, info
|
||||
|
||||
|
||||
def generate_report(critical, warning, info, days, total_runs):
|
||||
"""生成Markdown报告"""
|
||||
lines = []
|
||||
lines.append("# CI重复失败检测报告")
|
||||
lines.append("")
|
||||
lines.append(f"**统计周期**: 最近{days}天")
|
||||
lines.append(f"**扫描Runs**: {total_runs}个")
|
||||
lines.append(f"**生成时间**: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
|
||||
lines.append("")
|
||||
|
||||
lines.append(f"## 概览")
|
||||
lines.append("")
|
||||
lines.append(f"| 级别 | 数量 |")
|
||||
lines.append(f"|------|------|")
|
||||
lines.append(f"| 🔴 严重 (连续失败≥{CONSECUTIVE_FAIL_THRESHOLD}次 或 失败率≥50%) | {len(critical)} |")
|
||||
lines.append(f"| 🟡 警告 (失败率≥{FAIL_RATE_THRESHOLD}% 且 失败≥{FAIL_THRESHOLD}次) | {len(warning)} |")
|
||||
lines.append(f"| 🔵 关注 (失败≥2次) | {len(info)} |")
|
||||
lines.append("")
|
||||
|
||||
if critical:
|
||||
lines.append("## 🔴 严重问题")
|
||||
lines.append("")
|
||||
for item in critical:
|
||||
lines.append(f"### {item['name']}")
|
||||
lines.append("")
|
||||
lines.append(f"- 失败次数: **{item['fail_count']}** / {item['total']} 次运行")
|
||||
lines.append(f"- 失败率: **{item['failure_rate']}%**")
|
||||
lines.append(f"- 当前连续失败: **{item['current_streak']}** 次 (历史最高: {item['max_streak']} 次)")
|
||||
lines.append("")
|
||||
if item["recent_failures"]:
|
||||
lines.append("最近失败:")
|
||||
lines.append("")
|
||||
for f in item["recent_failures"]:
|
||||
lines.append(f"- [{f['time'][:16]}] run #{f['run_id']} ({f['event']})")
|
||||
lines.append("")
|
||||
|
||||
if warning:
|
||||
lines.append("## 🟡 警告")
|
||||
lines.append("")
|
||||
for item in warning:
|
||||
lines.append(
|
||||
f"- **{item['name']}**: {item['fail_count']}次失败 / {item['total']}次运行 ({item['failure_rate']}%)"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
if info:
|
||||
lines.append("## 🔵 关注列表")
|
||||
lines.append("")
|
||||
lines.append("| Job名称 | 失败次数 | 总次数 | 失败率 | 当前连续 |")
|
||||
lines.append("|---------|----------|--------|--------|----------|")
|
||||
for item in info[:20]: # 最多显示20个
|
||||
lines.append(
|
||||
f"| {item['name']} | {item['fail_count']} | {item['total']} | {item['failure_rate']}% | {item['current_streak']} |"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def send_feishu_notification(critical, warning, info, days):
|
||||
"""发送飞书通知"""
|
||||
if not WEBHOOK:
|
||||
print(" ⚠️ 未配置WEBHOOK,跳过飞书通知")
|
||||
return False
|
||||
|
||||
total_issues = len(critical) + len(warning) + len(info)
|
||||
if total_issues == 0:
|
||||
print(" ✅ 无异常,不发送通知")
|
||||
return True
|
||||
|
||||
level = "🔴 严重告警" if critical else "🟡 警告" if warning else "🔵 关注"
|
||||
|
||||
title = f"CI重复失败检测 - {level}"
|
||||
text = f"统计周期: 最近{days}天\n\n"
|
||||
|
||||
if critical:
|
||||
text += "【严重问题】\n"
|
||||
for item in critical[:5]:
|
||||
text += f"• {item['name']}\n"
|
||||
text += f" 失败 {item['fail_count']}/{item['total']} ({item['failure_rate']}%) 连续{item['current_streak']}次\n"
|
||||
if len(critical) > 5:
|
||||
text += f" ...还有{len(critical)-5}个\n"
|
||||
text += "\n"
|
||||
|
||||
if warning:
|
||||
text += "【警告】\n"
|
||||
for item in warning[:5]:
|
||||
text += f"• {item['name']}: {item['fail_count']}次失败 ({item['failure_rate']}%)\n"
|
||||
if len(warning) > 5:
|
||||
text += f" ...还有{len(warning)-5}个\n"
|
||||
text += "\n"
|
||||
|
||||
if info and not critical and not warning:
|
||||
text += "【关注列表】\n"
|
||||
for item in info[:10]:
|
||||
text += f"• {item['name']}: {item['fail_count']}次失败\n"
|
||||
text += "\n"
|
||||
|
||||
text += f"共发现 {total_issues} 个异常job"
|
||||
|
||||
payload = {"msg_type": "text", "content": {"text": f"{title}\n\n{text}"}}
|
||||
|
||||
data = json.dumps(payload).encode()
|
||||
req = urllib.request.Request(WEBHOOK, data=data, headers={"Content-Type": "application/json"})
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
result = json.loads(resp.read())
|
||||
if result.get("code") == 0 or result.get("StatusCode") == 0:
|
||||
print(" ✅ 飞书通知已发送")
|
||||
return True
|
||||
else:
|
||||
print(f" ⚠️ 飞书返回: {result}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f" ❌ 飞书通知失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
print(f"=== CI重复失败检测 ===")
|
||||
print(f"统计周期: 最近{DAYS}天")
|
||||
print(f"仓库: {REPO}")
|
||||
print()
|
||||
|
||||
print("1. 获取最近的Runs...")
|
||||
runs = fetch_recent_runs(days=DAYS)
|
||||
print(f" 找到 {len(runs)} 个runs")
|
||||
|
||||
if not runs:
|
||||
print("⚠️ 没有找到runs,退出")
|
||||
return
|
||||
|
||||
print()
|
||||
print("2. 分析job失败情况(可能需要点时间)...")
|
||||
job_stats, consecutive = analyze_failures(runs)
|
||||
print(f" 共统计 {len(job_stats)} 个job")
|
||||
|
||||
print()
|
||||
print("3. 识别高风险job...")
|
||||
critical, warning, info = find_high_failures(job_stats, consecutive)
|
||||
print(f" 🔴 严重: {len(critical)}")
|
||||
print(f" 🟡 警告: {len(warning)}")
|
||||
print(f" 🔵 关注: {len(info)}")
|
||||
|
||||
print()
|
||||
print("4. 生成报告...")
|
||||
report = generate_report(critical, warning, info, DAYS, len(runs))
|
||||
|
||||
# 保存报告
|
||||
report_path = os.environ.get("REPORT_PATH", f"/tmp/ci_failure_report_{int(time.time())}.md")
|
||||
with open(report_path, "w") as f:
|
||||
f.write(report)
|
||||
print(f" 报告已保存: {report_path}")
|
||||
|
||||
# 打印摘要
|
||||
print()
|
||||
print("=== 摘要 ===")
|
||||
if critical:
|
||||
print("🔴 严重问题:")
|
||||
for item in critical[:5]:
|
||||
print(
|
||||
f" {item['name']}: {item['fail_count']}次失败, {item['failure_rate']}%, 连续{item['current_streak']}次"
|
||||
)
|
||||
if warning:
|
||||
print("🟡 警告:")
|
||||
for item in warning[:5]:
|
||||
print(f" {item['name']}: {item['fail_count']}次失败, {item['failure_rate']}%")
|
||||
|
||||
print()
|
||||
print("5. 发送飞书通知...")
|
||||
send_feishu_notification(critical, warning, info, DAYS)
|
||||
|
||||
print()
|
||||
print("✅ 检测完成")
|
||||
|
||||
# 有严重问题时退出码非零,方便workflow标记
|
||||
if critical:
|
||||
sys.exit(2)
|
||||
elif warning:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/bin/bash
|
||||
# CI Integration Tests Job 主脚本
|
||||
# 包含:依赖安装、ffmpeg安装、Redis启动、PG启动、迁移、测试、清理、覆盖率
|
||||
# 支持 pytest-xdist 并行执行:每个 worker 使用独立数据库,预期加速 2-4 倍
|
||||
set -eu
|
||||
|
||||
echo "=== CI Integration Tests 开始 ==="
|
||||
@@ -28,12 +29,13 @@ for i in 1 2 3; do
|
||||
sleep 5
|
||||
done
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q pytest-rerunfailures && break
|
||||
echo "pip install pytest-rerunfailures 失败,重试 $i/3..."
|
||||
python3 -m pip install -q pytest-rerunfailures pytest-xdist && break
|
||||
echo "pip install pytest-rerunfailures/pytest-xdist 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
pytest --version
|
||||
echo "pytest-xdist: $(python3 -c "import xdist; print(xdist.__version__)" 2>/dev/null || echo 'not installed')"
|
||||
|
||||
# --- 安装 ffmpeg ---
|
||||
echo ""
|
||||
@@ -187,8 +189,8 @@ if [ "$USE_SHARED_PG" = "true" ]; then
|
||||
echo "等待共享PG连接就绪..."
|
||||
wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5
|
||||
|
||||
# 创建独立数据库
|
||||
echo "创建测试数据库: $CI_DB_NAME"
|
||||
# 创建主数据库(xdist 模式下各 worker 会创建自己的数据库,主库作为 fallback)
|
||||
echo "创建主测试数据库: $CI_DB_NAME"
|
||||
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
|
||||
@@ -238,30 +240,37 @@ else
|
||||
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
|
||||
fi
|
||||
|
||||
# --- 执行迁移 ---
|
||||
# --- 执行迁移(主数据库,xdist worker 会各自创建自己的库并迁移) ---
|
||||
echo ""
|
||||
echo "=== 执行 Alembic 迁移 ==="
|
||||
echo "=== 执行 Alembic 迁移(主数据库) ==="
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
|
||||
echo "✅ 迁移完成"
|
||||
|
||||
# --- 运行集成测试 ---
|
||||
# --- 运行集成测试(pytest-xdist 并行) ---
|
||||
echo ""
|
||||
echo "=== 运行集成测试 ==="
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m coverage run \
|
||||
--source=apps/api/app,packages \
|
||||
--omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \
|
||||
--branch \
|
||||
-m pytest tests/integration -q --timeout=60 -x --reruns 2 --reruns-delay 1 -m "not performance"
|
||||
python3 -m coverage report --show-missing
|
||||
python3 -m coverage xml -o coverage.xml
|
||||
python3 -m coverage report --fail-under=40 > /dev/null
|
||||
echo "=== 运行集成测试(pytest-xdist 并行模式) ==="
|
||||
echo "CPU 核数: $(nproc 2>/dev/null || echo 'unknown')"
|
||||
|
||||
# 集成测试使用 pytest-xdist 并行加速(coverage 由单元测试负责,并行模式下 coverage 不稳定)
|
||||
# -n 2: 限制2个worker,避免DooD模式下读到宿主机全部核数导致OOM
|
||||
# 集成测试涉及ffmpeg编码+PG多库,内存开销大,2worker较稳妥
|
||||
# (auto模式下读到宿主机4核8线程=8worker,OOM直接杀worker进程)
|
||||
# --dist loadfile: 同一测试文件分配到同一 worker(共享 fixture 更高效)
|
||||
# --maxfail=1: 遇到失败停止调度新测试(并行模式下等价于 -x)
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration \
|
||||
-q --timeout=60 --maxfail=1 --reruns 2 --reruns-delay 1 \
|
||||
-m "not performance" \
|
||||
-n 2 --dist loadfile \
|
||||
-p no:cacheprovider
|
||||
|
||||
echo "✅ 集成测试通过"
|
||||
|
||||
# --- API 性能基线测试(仅告警) ---
|
||||
# --- API 性能基线测试(仅告警,串行执行) ---
|
||||
echo ""
|
||||
echo "=== API 性能基线测试(仅告警) ==="
|
||||
set +e
|
||||
PERF_OUTPUT=$(mktemp)
|
||||
# 性能测试单独串行运行(不参与并行,避免资源竞争影响测量结果)
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration/test_api_performance.py \
|
||||
-v --timeout=120 -p no:cacheprovider 2>&1 | tee "$PERF_OUTPUT"
|
||||
echo ""
|
||||
@@ -284,14 +293,29 @@ set -e
|
||||
echo ""
|
||||
echo "=== 清理 ==="
|
||||
if [ "$USE_SHARED_PG" = "true" ]; then
|
||||
# 清理共享PG上的测试数据库
|
||||
echo "清理共享PG测试数据库: $CI_DB_NAME"
|
||||
# 清理共享PG上的测试数据库(主库 + 可能残留的 worker 库)
|
||||
echo "清理共享PG测试数据库..."
|
||||
|
||||
# 清理所有以 CI_DB_NAME 开头的数据库(主库 + worker 库)
|
||||
PGPASSWORD="${SHARED_PG_PASSWORD}" python3 -c "
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(host='${SHARED_PG_HOST}', port=${SHARED_PG_PORT}, user='${SHARED_PG_USER}', password='${SHARED_PG_PASSWORD}', dbname='postgres')
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
|
||||
|
||||
# 查找所有需要清理的数据库(主库 + worker 库)
|
||||
cur.execute(\"SELECT datname FROM pg_database WHERE datname LIKE '$CI_DB_NAME%'\")
|
||||
dbs = [row[0] for row in cur.fetchall()]
|
||||
|
||||
for db in dbs:
|
||||
try:
|
||||
# 强制断开所有连接
|
||||
cur.execute(f\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{db}' AND pid <> pg_backend_pid()\")
|
||||
cur.execute(f'DROP DATABASE IF EXISTS \"{db}\" WITH (FORCE)')
|
||||
print(f' 已清理: {db}')
|
||||
except Exception as e:
|
||||
print(f' 警告: 清理 {db} 失败: {e}')
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
" 2>/dev/null || echo "WARN: 数据库清理失败(可能已被清理)"
|
||||
|
||||
@@ -1,24 +1,25 @@
|
||||
#!/bin/sh
|
||||
# CI 公共步骤:前端依赖安装(在 docker node 容器中运行)
|
||||
# 用法:step_frontend_install.sh [模式]
|
||||
# 模式: full (默认) - 完整安装所有依赖
|
||||
# vitest - 同full(保持接口兼容)
|
||||
# 优化:增加国内npm镜像源,加重试间隔
|
||||
set -eu
|
||||
|
||||
MODE="${1:-full}"
|
||||
|
||||
echo "=== 前端依赖安装开始 (模式: $MODE) ==="
|
||||
|
||||
# npm ci 带重试(网络不稳定时自动重试)
|
||||
# npm国内镜像源(加速下载,减少网络失败)
|
||||
NPM_REGISTRY="https://registry.npmmirror.com"
|
||||
|
||||
for i in 1 2 3; do
|
||||
echo "npm ci 尝试 $i/3 (镜像: $NPM_REGISTRY)"
|
||||
docker run --rm \
|
||||
-v "$PWD:/workspace" \
|
||||
-w /workspace/apps/web \
|
||||
docker.m.daocloud.io/library/node:20 \
|
||||
sh -lc "npm ci --no-audit --no-fund" && break
|
||||
sh -lc "npm config set registry $NPM_REGISTRY && npm ci --no-audit --no-fund" && break
|
||||
echo "npm ci 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
sleep 10
|
||||
done
|
||||
|
||||
echo "=== 前端依赖安装完成 ==="
|
||||
|
||||
@@ -2,3 +2,5 @@
|
||||
# CI 公共步骤:Job 开始计时
|
||||
echo "JOB_START_TIME=$(date +%s)" >> $GITHUB_ENV
|
||||
echo "Job started at $(date)"
|
||||
# trigger CI run for PR validation
|
||||
# trigger CI - worker dood fallback fix test
|
||||
@@ -0,0 +1,186 @@
|
||||
#!/bin/bash
|
||||
# CI Validate: 代码质量与安全扫描(并行Job 1/3)
|
||||
# 包含:密钥扫描、格式检查、安全扫描、依赖漏洞、死代码检测、脚本语法校验
|
||||
set -eu
|
||||
|
||||
echo "=== CI Validate: 代码质量与安全扫描 ==="
|
||||
|
||||
# --- 密钥检测 ---
|
||||
echo ""
|
||||
echo "=== [1/6] Secret detection (detect-secrets) ==="
|
||||
python3 -m pip install -q detect-secrets
|
||||
detect-secrets --version
|
||||
|
||||
detect-secrets scan \
|
||||
--all-files \
|
||||
--exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \
|
||||
--exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \
|
||||
--exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \
|
||||
--disable-plugin Base64HighEntropyString \
|
||||
--disable-plugin HexHighEntropyString \
|
||||
--disable-plugin BasicAuthDetector \
|
||||
--disable-plugin KeywordDetector \
|
||||
--disable-plugin IPPublicDetector \
|
||||
> /tmp/secrets-scan.json 2>&1
|
||||
|
||||
FOUND=$(python3 -c "
|
||||
import json
|
||||
try:
|
||||
with open('/tmp/secrets-scan.json') as f:
|
||||
data = json.load(f)
|
||||
results = data.get('results', {})
|
||||
total = sum(len(v) for v in results.values())
|
||||
print(total)
|
||||
except Exception:
|
||||
print('error')
|
||||
")
|
||||
|
||||
echo "Secrets detected: $FOUND"
|
||||
if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then
|
||||
echo ""
|
||||
echo "=== Secret details ==="
|
||||
python3 -c "
|
||||
import json
|
||||
with open('/tmp/secrets-scan.json') as f:
|
||||
data = json.load(f)
|
||||
for fpath, items in data.get('results', {}).items():
|
||||
for item in items:
|
||||
line = item.get('line_number', '?')
|
||||
stype = item.get('type', '?')
|
||||
hashed = item.get('hashed_secret', '')[:16]
|
||||
print(f' {fpath}:{line} [{stype}] {hashed}...')
|
||||
"
|
||||
echo ""
|
||||
echo "ERROR: Potential secrets detected in code!"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ Secret scan passed"
|
||||
|
||||
# --- 增量/全量模式判断 ---
|
||||
echo ""
|
||||
echo "=== [2/6] Code quality checks ==="
|
||||
SCAN_MODE="full"
|
||||
CHANGED_PY_FILES=""
|
||||
|
||||
if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then
|
||||
PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||')
|
||||
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100"
|
||||
set +e
|
||||
RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL")
|
||||
HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
|
||||
BODY=$(echo "$RESPONSE" | sed '$d')
|
||||
set -e
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
CHANGED_PY_FILES=$(echo "$BODY" | python3 -c "
|
||||
import json, sys
|
||||
try:
|
||||
files = json.load(sys.stdin)
|
||||
py_files = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] != 'removed']
|
||||
print(' '.join(py_files))
|
||||
except Exception:
|
||||
print('')
|
||||
")
|
||||
if [ -n "$CHANGED_PY_FILES" ]; then
|
||||
SCAN_MODE="incremental"
|
||||
echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed"
|
||||
else
|
||||
SCAN_MODE="skip_py"
|
||||
echo "No Python files changed in this PR"
|
||||
fi
|
||||
else
|
||||
echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan"
|
||||
fi
|
||||
else
|
||||
echo "Full scan mode (not a PR event)"
|
||||
fi
|
||||
|
||||
if [ "$SCAN_MODE" = "incremental" ]; then
|
||||
# 防御性过滤
|
||||
EXISTING_PY_FILES=""
|
||||
for f in $CHANGED_PY_FILES; do
|
||||
if [ -f "$f" ]; then
|
||||
if [ -z "$EXISTING_PY_FILES" ]; then
|
||||
EXISTING_PY_FILES="$f"
|
||||
else
|
||||
EXISTING_PY_FILES="$EXISTING_PY_FILES $f"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
CHANGED_PY_FILES="$EXISTING_PY_FILES"
|
||||
|
||||
python3 -m compileall -q $CHANGED_PY_FILES
|
||||
python3 -m black --check --fast $CHANGED_PY_FILES
|
||||
python3 -m isort --check-only $CHANGED_PY_FILES
|
||||
RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | grep -v '^$' | xargs)
|
||||
if [ -n "$RUFF_FILES" ]; then
|
||||
python3 -m ruff check $RUFF_FILES --statistics
|
||||
else
|
||||
echo "No ruff-checkable files changed, skipping"
|
||||
fi
|
||||
elif [ "$SCAN_MODE" = "skip_py" ]; then
|
||||
echo "No Python files changed - skipping Python lint checks"
|
||||
else
|
||||
echo "Full scan mode"
|
||||
python3 -m compileall -q alembic apps packages tests scripts
|
||||
python3 -m black --check --fast alembic apps packages tests scripts
|
||||
python3 -m isort --check-only alembic apps packages tests scripts
|
||||
python3 -m ruff check apps packages tests --statistics
|
||||
fi
|
||||
echo "✅ Code quality checks passed"
|
||||
|
||||
# --- Bandit 安全扫描(仅告警) ---
|
||||
echo ""
|
||||
echo "=== [3/6] Security scan (bandit, advisory only) ==="
|
||||
set +e
|
||||
bandit -r apps packages -q -ll
|
||||
BANDIT_EXIT=$?
|
||||
set -e
|
||||
if [ "$BANDIT_EXIT" -ne 0 ]; then
|
||||
echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)"
|
||||
else
|
||||
echo "✅ Bandit security scan passed"
|
||||
fi
|
||||
|
||||
# --- Pip-audit 依赖漏洞扫描(仅告警) ---
|
||||
echo ""
|
||||
echo "=== [4/6] Python dependency vulnerability scan (pip-audit, advisory only) ==="
|
||||
python3 -m pip install -q pip-audit
|
||||
pip-audit --version
|
||||
EXIT_CODE=0
|
||||
for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do
|
||||
if [ -f "$req_file" ]; then
|
||||
echo "--- Scanning $req_file ---"
|
||||
pip-audit -r "$req_file" --desc on 2>&1 | head -40 || EXIT_CODE=$?
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)"
|
||||
|
||||
# --- Vulture 死代码检测(仅告警) ---
|
||||
echo ""
|
||||
echo "=== [5/6] Dead code detection (vulture, advisory only) ==="
|
||||
set +e
|
||||
python3 -m pip install -q vulture
|
||||
vulture --version
|
||||
echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。"
|
||||
echo ""
|
||||
vulture apps packages scripts \
|
||||
--exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \
|
||||
--min-confidence 70 \
|
||||
2>&1 | sort -t'(' -k2 -rn | head -80
|
||||
echo ""
|
||||
echo "=== vulture scan summary ==="
|
||||
echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)"
|
||||
echo "建议:定期人工审查高置信度(>=90%)条目"
|
||||
set -e
|
||||
|
||||
# --- Release 脚本语法校验 ---
|
||||
echo ""
|
||||
echo "=== [6/6] Release scripts syntax validation ==="
|
||||
bash -n scripts/backup_postgres.sh
|
||||
bash -n scripts/restore_postgres_plan.sh
|
||||
bash -n scripts/init_production_env.sh
|
||||
echo "✅ Release scripts syntax OK"
|
||||
|
||||
echo ""
|
||||
echo "=== CI Validate: 代码质量与安全扫描 全部通过 ✅ ==="
|
||||
@@ -0,0 +1,182 @@
|
||||
#!/bin/bash
|
||||
# CI Validate: Alembic迁移验证(并行Job 3/3)
|
||||
# 需要PostgreSQL数据库
|
||||
set -eu
|
||||
|
||||
echo "=== CI Validate: Alembic迁移验证 ==="
|
||||
|
||||
# --- DooD模式检测:确定宿主机访问地址 ---
|
||||
detect_docker_host() {
|
||||
local test_port="${1:-5432}"
|
||||
|
||||
local candidates=()
|
||||
|
||||
# 1. host.docker.internal
|
||||
if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then
|
||||
candidates+=("host.docker.internal")
|
||||
fi
|
||||
|
||||
# 2. docker0 桥接网关
|
||||
candidates+=("172.17.0.1")
|
||||
|
||||
# 3. 默认网关
|
||||
local gw=""
|
||||
gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1)
|
||||
if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then
|
||||
candidates+=("$gw")
|
||||
fi
|
||||
|
||||
# 4. 宿主机同网段的.1或.254
|
||||
local my_ip=""
|
||||
my_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
|
||||
if [ -n "$my_ip" ]; then
|
||||
local subnet=$(echo "$my_ip" | cut -d. -f1-3)
|
||||
candidates+=("${subnet}.1")
|
||||
candidates+=("${subnet}.254")
|
||||
fi
|
||||
|
||||
# 5. 127.0.0.1 最后尝试
|
||||
candidates+=("127.0.0.1")
|
||||
|
||||
for candidate in "${candidates[@]}"; do
|
||||
if python3 -c "
|
||||
import socket
|
||||
s = socket.socket()
|
||||
s.settimeout(2)
|
||||
try:
|
||||
s.connect(('$candidate', $test_port))
|
||||
s.close()
|
||||
print('ok')
|
||||
except:
|
||||
pass
|
||||
" 2>/dev/null | grep -q ok; then
|
||||
echo "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo "127.0.0.1"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 获取宿主机IP
|
||||
if [ -S /var/run/docker.sock ]; then
|
||||
DOCKER_HOST_IP=$(detect_docker_host 5433)
|
||||
if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then
|
||||
DOCKER_HOST_IP=$(detect_docker_host 22)
|
||||
fi
|
||||
echo "检测到DooD模式,宿主机地址: $DOCKER_HOST_IP"
|
||||
else
|
||||
DOCKER_HOST_IP="127.0.0.1"
|
||||
echo "非DooD模式,使用 127.0.0.1"
|
||||
fi
|
||||
PG_HOST="$DOCKER_HOST_IP"
|
||||
echo "PG host: $PG_HOST"
|
||||
|
||||
# 指数退避TCP连接检查
|
||||
wait_tcp_ready() {
|
||||
local host="$1"
|
||||
local port="$2"
|
||||
local max_attempts="${3:-5}"
|
||||
local delay=1
|
||||
local attempt=1
|
||||
while [ "$attempt" -le "$max_attempts" ]; do
|
||||
if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
echo "TCP连接尝试 $attempt/$max_attempts 失败,${delay}s后重试..."
|
||||
sleep "$delay"
|
||||
delay=$((delay * 2))
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
USE_SHARED_PG="${CI_USE_SHARED_PG:-false}"
|
||||
|
||||
if [ "$USE_SHARED_PG" = "true" ]; then
|
||||
# 使用常驻共享PG实例
|
||||
echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true)"
|
||||
SHARED_PG_HOST="$PG_HOST"
|
||||
SHARED_PG_PORT="5433"
|
||||
SHARED_PG_USER="postgres"
|
||||
SHARED_PG_PASSWORD="ci_pg_2026!"
|
||||
CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}"
|
||||
|
||||
echo "等待共享PG连接就绪..."
|
||||
wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5
|
||||
|
||||
echo "创建测试数据库: $CI_DB_NAME"
|
||||
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"')
|
||||
cur.close()
|
||||
conn.close()
|
||||
"
|
||||
export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}"
|
||||
echo "✅ 共享PG数据库已创建: $CI_DB_NAME"
|
||||
|
||||
# 执行迁移
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
|
||||
echo "✅ Alembic migrations applied successfully"
|
||||
|
||||
# 清理数据库
|
||||
echo "清理测试数据库: $CI_DB_NAME"
|
||||
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
|
||||
cur.close()
|
||||
conn.close()
|
||||
" 2>/dev/null || echo "WARN: 数据库清理失败"
|
||||
echo "✅ 共享PG数据库已清理"
|
||||
else
|
||||
# 使用临时PG容器(默认模式)
|
||||
echo "使用临时PG容器模式"
|
||||
PG_CONTAINER=ci-pg-validate-migration-${GITHUB_RUN_ID:-$$}
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
--shm-size=256m \
|
||||
-e POSTGRES_USER=postgres \
|
||||
-e POSTGRES_PASSWORD=postgres \
|
||||
-e POSTGRES_DB=xiaoxia_saas \
|
||||
-P \
|
||||
--health-cmd "pg_isready -U postgres" \
|
||||
--health-interval 3s \
|
||||
--health-timeout 3s \
|
||||
--health-retries 20 \
|
||||
postgres:16-alpine
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
|
||||
echo "PostgreSQL port: $PG_PORT"
|
||||
export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas"
|
||||
|
||||
# 等待容器健康
|
||||
for i in $(seq 1 30); do
|
||||
if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
|
||||
echo "PostgreSQL container is healthy on port $PG_PORT"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for PostgreSQL container health... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy
|
||||
|
||||
# TCP连通性检查
|
||||
echo "验证TCP连通性 ($PG_HOST:$PG_PORT)..."
|
||||
wait_tcp_ready "$PG_HOST" "$PG_PORT" 5
|
||||
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
|
||||
|
||||
# 执行迁移
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
|
||||
echo "✅ Alembic migrations applied successfully"
|
||||
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== CI Validate: Alembic迁移验证 通过 ✅ ==="
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
# CI Validate: Mypy类型检查(并行Job 2/3)
|
||||
set -eu
|
||||
|
||||
echo "=== CI Validate: Mypy类型检查 ==="
|
||||
|
||||
bash scripts/ci/mypy_check.sh
|
||||
|
||||
echo ""
|
||||
echo "=== CI Validate: Mypy类型检查 通过 ✅ ==="
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/bin/bash
|
||||
# Vitest 增量执行脚本
|
||||
# PR模式下只跑与改动文件相关的测试,大幅节省时间
|
||||
# 用法: bash scripts/ci/vitest_incremental.sh
|
||||
set -eu
|
||||
|
||||
cd apps/web
|
||||
|
||||
# 如果不是PR事件,直接全量跑
|
||||
if [ "${GITHUB_EVENT_NAME:-}" != "pull_request" ]; then
|
||||
echo "非PR模式,全量执行Vitest"
|
||||
npx --no-install vitest run --coverage
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# 获取PR改动的文件列表
|
||||
PR_NUMBER=$(echo "${GITHUB_REF:-}" | sed 's|refs/pull/||; s|/.*||')
|
||||
if [ -z "$PR_NUMBER" ]; then
|
||||
echo "无法获取PR编号,全量执行Vitest"
|
||||
npx --no-install vitest run --coverage
|
||||
exit $?
|
||||
fi
|
||||
|
||||
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100"
|
||||
CHANGED_FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "
|
||||
import json, sys
|
||||
try:
|
||||
files = json.load(sys.stdin)
|
||||
web_files = []
|
||||
for f in files:
|
||||
fname = f['filename']
|
||||
# 只关注前端源码文件
|
||||
if fname.startswith('apps/web/src/') and fname.endswith(('.ts', '.tsx', '.js', '.jsx')) and f['status'] != 'removed':
|
||||
# 去掉apps/web/前缀,变成相对路径
|
||||
web_files.append(fname.replace('apps/web/', ''))
|
||||
print(' '.join(web_files))
|
||||
except Exception as e:
|
||||
print('')
|
||||
")
|
||||
|
||||
if [ -z "$CHANGED_FILES" ]; then
|
||||
echo "PR未改动前端源码文件,跳过Vitest"
|
||||
echo "(如果配置了前端单测门禁,请确保至少有一个相关测试)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
FILE_COUNT=$(echo "$CHANGED_FILES" | wc -w)
|
||||
echo "PR改动了 $FILE_COUNT 个前端文件"
|
||||
echo "改动文件: $CHANGED_FILES"
|
||||
|
||||
# 如果改动文件太多(超过30个),全量跑更可靠
|
||||
if [ "$FILE_COUNT" -gt 30 ]; then
|
||||
echo "改动文件较多(>$FILE_COUNT),降级为全量执行以确保覆盖"
|
||||
npx --no-install vitest run --coverage
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# 使用vitest --related 跑增量测试
|
||||
echo ""
|
||||
echo "=== 增量执行 Vitest(只跑相关测试)==="
|
||||
echo "相关源文件: $CHANGED_FILES"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
npx --no-install vitest run --related $CHANGED_FILES
|
||||
VITEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
if [ "$VITEST_EXIT" -eq 0 ]; then
|
||||
echo ""
|
||||
echo "✅ 增量测试通过"
|
||||
echo "(仅覆盖与改动相关的测试用例)"
|
||||
exit 0
|
||||
else
|
||||
echo ""
|
||||
echo "❌ 增量测试失败"
|
||||
exit $VITEST_EXIT
|
||||
fi
|
||||
+128
-31
@@ -1,17 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""发送 CI 失败通知到飞书/项目群 webhook。"""
|
||||
"""发送 CI 失败通知到飞书/项目群 webhook(增强版:带失败诊断)。
|
||||
|
||||
诊断功能:自动分析失败原因,给出分类和修复建议。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
|
||||
def run_diagnosis() -> dict:
|
||||
"""运行失败诊断脚本,返回诊断结果"""
|
||||
diag_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ci/ci_failure_diagnosis.py")
|
||||
if not os.path.exists(diag_script):
|
||||
diag_script = "scripts/ci/ci_failure_diagnosis.py"
|
||||
|
||||
result = {
|
||||
"category": "unknown",
|
||||
"category_cn": "未知",
|
||||
"severity": "medium",
|
||||
"summary": "",
|
||||
"error_lines": [],
|
||||
"suggestions": [],
|
||||
"auto_fixable": False,
|
||||
}
|
||||
|
||||
try:
|
||||
# 运行诊断脚本
|
||||
env = os.environ.copy()
|
||||
env["DIAGNOSIS_OUTPUT"] = "/tmp/ci_diagnosis_result.json"
|
||||
|
||||
proc = subprocess.run([sys.executable, diag_script], capture_output=True, text=True, timeout=30, env=env)
|
||||
|
||||
# 尝试读取结果文件
|
||||
output_file = "/tmp/ci_diagnosis_result.json"
|
||||
if os.path.exists(output_file):
|
||||
with open(output_file) as f:
|
||||
result = json.load(f)
|
||||
elif proc.stdout:
|
||||
# 从stdout解析
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"诊断脚本执行失败: {e}", file=sys.stderr)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
webhook = os.environ.get("CI_NOTIFY_WEBHOOK", "")
|
||||
if not webhook:
|
||||
print("未配置 CI_NOTIFY_WEBHOOK,跳过通知")
|
||||
print("如需启用,请在仓库 Settings -> Secrets and variables -> Actions 中添加 CI_NOTIFY_WEBHOOK")
|
||||
return 0
|
||||
|
||||
failed_job = os.environ.get("FAILED_JOB", "Unknown Job")
|
||||
@@ -21,6 +61,86 @@ def main() -> int:
|
||||
run_id = os.environ.get("GITHUB_RUN_ID", "unknown")
|
||||
repo = os.environ.get("GITHUB_REPOSITORY", "unknown")
|
||||
run_url = f"https://git.xiaoxiajianji.com/{repo}/actions/runs/{run_id}"
|
||||
pr_number = os.environ.get("PR_NUMBER", "")
|
||||
|
||||
# 运行诊断
|
||||
diagnosis = run_diagnosis()
|
||||
|
||||
# 构建卡片内容
|
||||
severity_color = {"high": "red", "medium": "orange", "low": "blue"}
|
||||
card_status = severity_color.get(diagnosis.get("severity", "medium"), "red")
|
||||
|
||||
# 标题
|
||||
title = f"❌ CI失败 - {diagnosis.get('category_cn', '未知')}"
|
||||
|
||||
# 诊断部分
|
||||
diag_lines = []
|
||||
diag_lines.append(f"**任务**: {failed_job}")
|
||||
diag_lines.append(f"**分类**: {diagnosis.get('category_cn', '未知')}")
|
||||
if diagnosis.get("summary"):
|
||||
diag_lines.append(f"**问题**: {diagnosis['summary']}")
|
||||
|
||||
# 错误行
|
||||
error_lines = diagnosis.get("error_lines", [])
|
||||
if error_lines:
|
||||
diag_lines.append("")
|
||||
diag_lines.append("**关键错误**:")
|
||||
for err in error_lines[:3]:
|
||||
if len(err) > 100:
|
||||
err = err[:97] + "..."
|
||||
diag_lines.append(f"`{err}`")
|
||||
|
||||
# 修复建议
|
||||
suggestions = diagnosis.get("suggestions", [])
|
||||
if suggestions:
|
||||
diag_lines.append("")
|
||||
diag_lines.append("**修复建议**:")
|
||||
for i, s in enumerate(suggestions[:3], 1):
|
||||
diag_lines.append(f"{i}. {s}")
|
||||
|
||||
if diagnosis.get("auto_fixable"):
|
||||
diag_lines.append("")
|
||||
diag_lines.append("💡 *可自动修复的问题,试试Rerun*")
|
||||
|
||||
# 基本信息
|
||||
info_lines = [
|
||||
f"**分支**: {branch}",
|
||||
f"**提交**: `{commit}`",
|
||||
f"**提交者**: {actor}",
|
||||
]
|
||||
if pr_number:
|
||||
info_lines.append(f"**PR**: #{pr_number}")
|
||||
|
||||
elements = [
|
||||
{
|
||||
"tag": "div",
|
||||
"text": {
|
||||
"tag": "lark_md",
|
||||
"content": "\n".join(diag_lines),
|
||||
},
|
||||
},
|
||||
{
|
||||
"tag": "hr",
|
||||
},
|
||||
{
|
||||
"tag": "div",
|
||||
"text": {
|
||||
"tag": "lark_md",
|
||||
"content": "\n".join(info_lines),
|
||||
},
|
||||
},
|
||||
{
|
||||
"tag": "action",
|
||||
"actions": [
|
||||
{
|
||||
"tag": "button",
|
||||
"text": {"tag": "plain_text", "content": "查看失败日志"},
|
||||
"url": run_url,
|
||||
"type": "danger",
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
payload = {
|
||||
"msg_type": "interactive",
|
||||
@@ -28,36 +148,11 @@ def main() -> int:
|
||||
"header": {
|
||||
"title": {
|
||||
"tag": "plain_text",
|
||||
"content": "❌ CI 构建失败",
|
||||
"content": title,
|
||||
},
|
||||
"status": "red",
|
||||
"status": card_status,
|
||||
},
|
||||
"elements": [
|
||||
{
|
||||
"tag": "div",
|
||||
"text": {
|
||||
"tag": "lark_md",
|
||||
"content": (
|
||||
f"**任务**: {failed_job}\n"
|
||||
f"**分支**: {branch}\n"
|
||||
f"**提交**: {commit}\n"
|
||||
f"**提交者**: {actor}\n"
|
||||
f"**Run ID**: {run_id}"
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
"tag": "action",
|
||||
"actions": [
|
||||
{
|
||||
"tag": "button",
|
||||
"text": {"tag": "plain_text", "content": "查看失败日志"},
|
||||
"url": run_url,
|
||||
"type": "danger",
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
"elements": elements,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -71,7 +166,7 @@ def main() -> int:
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
resp.read()
|
||||
print("通知已发送")
|
||||
print("通知已发送(带诊断信息)")
|
||||
except Exception as e:
|
||||
print(f"通知发送失败: {e}", file=sys.stderr)
|
||||
return 1
|
||||
@@ -81,3 +176,5 @@ def main() -> int:
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
# trigger CI - bypass [ci skip] bug
|
||||
|
||||
+151
-93
@@ -1,22 +1,9 @@
|
||||
#!/bin/sh
|
||||
# ===========================================
|
||||
# Staging 部署脚本(SSH 模式,支持自动回滚)
|
||||
# Staging 部署脚本(SSH 模式,并行优化版)
|
||||
# ===========================================
|
||||
# 通过 SSH 在 staging 服务器上执行
|
||||
#
|
||||
# 环境变量:
|
||||
# IMAGE_TAG - 镜像版本 tag(如 commit SHA 或分支名)
|
||||
# REGISTRY_TOKEN - Registry 访问令牌
|
||||
# REGISTRY - Registry 地址(默认 xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji)
|
||||
# REGISTRY_USER - Registry 用户名(默认 xiaoxia)
|
||||
# ENV_FILE - 环境变量文件路径
|
||||
# GENERATED_DIR - 生成文件目录
|
||||
# SKIP_MIGRATION - 跳过数据库迁移(true/false,默认 false)
|
||||
# SKIP_ROLLBACK - 失败时跳过自动回滚(true/false,默认 false)
|
||||
|
||||
set -eu
|
||||
|
||||
# ---- 重试工具函数 ----
|
||||
retry_cmd() {
|
||||
local max_attempts=$1
|
||||
local backoff=$2
|
||||
@@ -73,10 +60,9 @@ mkdir -p "$GENERATED_DIR"
|
||||
mkdir -p "$LEGACY_ASSETS_DIR"
|
||||
|
||||
echo "==========================================="
|
||||
echo " Staging 部署 - $IMAGE_TAG"
|
||||
echo " Staging 部署 - $IMAGE_TAG (并行优化版)"
|
||||
echo "==========================================="
|
||||
|
||||
# ---- 记录当前运行的镜像版本(用于回滚) ----
|
||||
echo "Recording current image versions for rollback..."
|
||||
PREV_API_IMAGE=""
|
||||
PREV_WORKER_IMAGE=""
|
||||
@@ -95,7 +81,6 @@ for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do
|
||||
fi
|
||||
done
|
||||
|
||||
# ---- 回滚函数 ----
|
||||
rollback() {
|
||||
echo ""
|
||||
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||
@@ -108,7 +93,6 @@ rollback() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 停止当前(失败的)新容器
|
||||
echo "Stopping new containers..."
|
||||
docker rm -f xiaoxia-api-staging 2>/dev/null || true
|
||||
docker rm -f xiaoxia-worker-staging 2>/dev/null || true
|
||||
@@ -116,7 +100,6 @@ rollback() {
|
||||
|
||||
LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
|
||||
|
||||
# 恢复 API
|
||||
if [ -n "$PREV_API_IMAGE" ]; then
|
||||
echo "Rolling back API to: $PREV_API_IMAGE"
|
||||
docker run -d \
|
||||
@@ -137,12 +120,9 @@ rollback() {
|
||||
--health-retries 3 \
|
||||
--health-start-period 40s \
|
||||
$LOG_OPTS \
|
||||
"$PREV_API_IMAGE"
|
||||
else
|
||||
echo "No previous API image to roll back to"
|
||||
"$PREV_API_IMAGE" &
|
||||
fi
|
||||
|
||||
# 恢复 Worker
|
||||
if [ -n "$PREV_WORKER_IMAGE" ]; then
|
||||
echo "Rolling back Worker to: $PREV_WORKER_IMAGE"
|
||||
docker run -d \
|
||||
@@ -164,12 +144,9 @@ rollback() {
|
||||
--health-retries 3 \
|
||||
--health-start-period 30s \
|
||||
$LOG_OPTS \
|
||||
"$PREV_WORKER_IMAGE"
|
||||
else
|
||||
echo "No previous Worker image to roll back to"
|
||||
"$PREV_WORKER_IMAGE" &
|
||||
fi
|
||||
|
||||
# 恢复 Web
|
||||
if [ -n "$PREV_WEB_IMAGE" ]; then
|
||||
echo "Rolling back Web to: $PREV_WEB_IMAGE"
|
||||
LEGACY_VOLUME=""
|
||||
@@ -187,12 +164,11 @@ rollback() {
|
||||
--health-timeout 5s \
|
||||
--health-retries 3 \
|
||||
$LOG_OPTS \
|
||||
"$PREV_WEB_IMAGE"
|
||||
else
|
||||
echo "No previous Web image to roll back to"
|
||||
"$PREV_WEB_IMAGE" &
|
||||
fi
|
||||
|
||||
# 等待 API 回滚后恢复健康
|
||||
wait
|
||||
|
||||
if [ -n "$PREV_API_IMAGE" ]; then
|
||||
echo "Waiting for rolled-back API to become healthy..."
|
||||
i=0
|
||||
@@ -224,7 +200,6 @@ rollback() {
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ---- 登录 Registry ----
|
||||
if [ -n "$REGISTRY_TOKEN" ]; then
|
||||
echo "=========================================="
|
||||
echo " Login to Registry (with retries)"
|
||||
@@ -233,28 +208,64 @@ if [ -n "$REGISTRY_TOKEN" ]; then
|
||||
retry_docker_login
|
||||
fi
|
||||
|
||||
# ---- Pull 新版本镜像 ----
|
||||
# ---- 并行 Pull 三个镜像 ----
|
||||
REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}"
|
||||
REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}"
|
||||
REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}"
|
||||
|
||||
echo "=========================================="
|
||||
echo " Pull images (with retries)"
|
||||
echo " Pull images (parallel, up to 3 retries each)"
|
||||
echo "=========================================="
|
||||
retry_docker_pull "$REGISTRY_API"
|
||||
retry_docker_pull "$REGISTRY_WORKER"
|
||||
retry_docker_pull "$REGISTRY_WEB"
|
||||
PULL_LOG_DIR="/tmp/staging-pull-$$"
|
||||
mkdir -p "$PULL_LOG_DIR"
|
||||
|
||||
retry_docker_pull "$REGISTRY_API" > "$PULL_LOG_DIR/api.log" 2>&1 &
|
||||
PID_API=$!
|
||||
retry_docker_pull "$REGISTRY_WORKER" > "$PULL_LOG_DIR/worker.log" 2>&1 &
|
||||
PID_WORKER=$!
|
||||
retry_docker_pull "$REGISTRY_WEB" > "$PULL_LOG_DIR/web.log" 2>&1 &
|
||||
PID_WEB=$!
|
||||
|
||||
wait $PID_API $PID_WORKER $PID_WEB
|
||||
|
||||
echo ""
|
||||
echo "Pull 结果:"
|
||||
PULL_FAILED=0
|
||||
for svc in api worker web; do
|
||||
if tail -1 "$PULL_LOG_DIR/$svc.log" 2>/dev/null | grep -qE "Status:|Downloaded|already exists|is up to date"; then
|
||||
echo " OK $svc"
|
||||
elif grep -qE "Digest:|Status: Downloaded" "$PULL_LOG_DIR/$svc.log" 2>/dev/null; then
|
||||
echo " OK $svc"
|
||||
else
|
||||
# 检查docker pull返回值不直接,用镜像是否存在来判断
|
||||
img_var="REGISTRY_$(echo $svc | tr '[:lower:]' '[:upper:]')"
|
||||
img_val=$(eval echo "\$$img_var")
|
||||
if docker image inspect "$img_val" >/dev/null 2>&1; then
|
||||
echo " OK $svc"
|
||||
else
|
||||
echo " FAIL $svc"
|
||||
tail -5 "$PULL_LOG_DIR/$svc.log" 2>/dev/null || true
|
||||
PULL_FAILED=$((PULL_FAILED + 1))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
rm -rf "$PULL_LOG_DIR"
|
||||
|
||||
if [ "$PULL_FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "ERROR: $PULL_FAILED 个镜像 pull 失败"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "All images pulled."
|
||||
|
||||
# ---- 备份 legacy assets ----
|
||||
echo "Backing up legacy assets from current web container..."
|
||||
if docker inspect xiaoxia-web-staging >/dev/null 2>&1; then
|
||||
_tmpdir="/tmp/legacy-assets-$$"
|
||||
rm -rf "$_tmpdir"
|
||||
mkdir -p "$_tmpdir"
|
||||
docker cp xiaoxia-web-staging:/usr/share/nginx/html/assets/. "$_tmpdir/" 2>/dev/null || true
|
||||
# 只有目录非空才拷贝,避免覆盖有内容的 legacy assets
|
||||
if [ -d "$_tmpdir" ] && [ "$(ls -A "$_tmpdir" 2>/dev/null)" ]; then
|
||||
cp -an "$_tmpdir"/. "$LEGACY_ASSETS_DIR"/ 2>/dev/null || true
|
||||
echo "Legacy assets backed up: $(ls "$_tmpdir" | wc -l) files"
|
||||
@@ -264,13 +275,11 @@ else
|
||||
echo "No existing web container, skipping legacy assets backup"
|
||||
fi
|
||||
|
||||
# 清理 7 天前的 legacy assets
|
||||
if [ -d "$LEGACY_ASSETS_DIR" ]; then
|
||||
find "$LEGACY_ASSETS_DIR" -type f -mtime +7 -delete 2>/dev/null || true
|
||||
echo "Legacy assets cleanup done (retain 7 days)"
|
||||
fi
|
||||
|
||||
# ---- 检查基础设施容器 ----
|
||||
echo "Checking infrastructure containers..."
|
||||
for c in xiaoxia-postgres-staging xiaoxia-redis-staging; do
|
||||
if ! docker inspect "$c" >/dev/null 2>&1; then
|
||||
@@ -284,10 +293,8 @@ for c in xiaoxia-postgres-staging xiaoxia-redis-staging; do
|
||||
fi
|
||||
done
|
||||
|
||||
# ---- 创建网络(不存在则创建) ----
|
||||
docker network create xiaoxia-net-staging 2>/dev/null || true
|
||||
|
||||
# ---- 数据库迁移 ----
|
||||
if [ "$SKIP_MIGRATION" != "true" ]; then
|
||||
echo "Running database migrations..."
|
||||
docker run --rm \
|
||||
@@ -296,8 +303,6 @@ if [ "$SKIP_MIGRATION" != "true" ]; then
|
||||
-e APP_ENV=staging \
|
||||
"$REGISTRY_API" sh -c "cd /app && alembic upgrade head" || {
|
||||
echo "ERROR: Database migration failed"
|
||||
echo "Note: Migration failures are NOT automatically rolled back (data safety)"
|
||||
echo "Please manually check and fix the migration, then redeploy"
|
||||
exit 1
|
||||
}
|
||||
echo "Migrations completed."
|
||||
@@ -305,7 +310,6 @@ else
|
||||
echo "Skipping migrations (SKIP_MIGRATION=true)"
|
||||
fi
|
||||
|
||||
# ---- 停止旧容器 ----
|
||||
echo "Stopping old containers..."
|
||||
docker rm -f xiaoxia-api-staging 2>/dev/null || true
|
||||
docker rm -f xiaoxia-worker-staging 2>/dev/null || true
|
||||
@@ -313,8 +317,14 @@ docker rm -f xiaoxia-web-staging 2>/dev/null || true
|
||||
|
||||
LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
|
||||
|
||||
# ---- 启动 API ----
|
||||
echo "Starting API container..."
|
||||
# ---- 并行启动三个容器 ----
|
||||
echo "Starting all containers (parallel)..."
|
||||
|
||||
LEGACY_VOLUME=""
|
||||
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
|
||||
LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
|
||||
fi
|
||||
|
||||
docker run -d \
|
||||
--name xiaoxia-api-staging \
|
||||
--env-file "$ENV_FILE" \
|
||||
@@ -333,10 +343,9 @@ docker run -d \
|
||||
--health-retries 3 \
|
||||
--health-start-period 40s \
|
||||
$LOG_OPTS \
|
||||
"$REGISTRY_API" || rollback
|
||||
"$REGISTRY_API" &
|
||||
PID_API_START=$!
|
||||
|
||||
# ---- 启动 Worker ----
|
||||
echo "Starting Worker container..."
|
||||
docker run -d \
|
||||
--name xiaoxia-worker-staging \
|
||||
--env-file "$ENV_FILE" \
|
||||
@@ -356,18 +365,9 @@ docker run -d \
|
||||
--health-retries 3 \
|
||||
--health-start-period 30s \
|
||||
$LOG_OPTS \
|
||||
"$REGISTRY_WORKER" || rollback
|
||||
"$REGISTRY_WORKER" &
|
||||
PID_WORKER_START=$!
|
||||
|
||||
# ---- 启动 Web ----
|
||||
LEGACY_VOLUME=""
|
||||
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
|
||||
LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
|
||||
echo "Web container: legacy assets mounted (fallback)"
|
||||
else
|
||||
echo "Web container: no legacy assets to mount"
|
||||
fi
|
||||
|
||||
echo "Starting Web container..."
|
||||
docker run -d \
|
||||
--name xiaoxia-web-staging \
|
||||
--network xiaoxia-net-staging \
|
||||
@@ -379,53 +379,111 @@ docker run -d \
|
||||
--health-timeout 5s \
|
||||
--health-retries 3 \
|
||||
$LOG_OPTS \
|
||||
"$REGISTRY_WEB" || rollback
|
||||
"$REGISTRY_WEB" &
|
||||
PID_WEB_START=$!
|
||||
|
||||
# ---- 等待 API 健康 ----
|
||||
echo "Waiting for API to become healthy..."
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then
|
||||
echo "API is healthy!"
|
||||
break
|
||||
wait $PID_API_START $PID_WORKER_START $PID_WEB_START
|
||||
|
||||
START_FAILED=0
|
||||
for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do
|
||||
if ! docker inspect "$c" >/dev/null 2>&1; then
|
||||
echo " FAIL $c: not created"
|
||||
START_FAILED=$((START_FAILED + 1))
|
||||
else
|
||||
state=$(docker inspect -f '{{.State.Status}}' "$c")
|
||||
if [ "$state" = "running" ] || [ "$state" = "starting" ]; then
|
||||
echo " OK $c: $state"
|
||||
else
|
||||
echo " FAIL $c: $state"
|
||||
docker logs --tail 20 "$c" 2>/dev/null || true
|
||||
START_FAILED=$((START_FAILED + 1))
|
||||
fi
|
||||
fi
|
||||
i=$((i + 1))
|
||||
echo " Waiting... ($i/40)"
|
||||
sleep 3
|
||||
done
|
||||
|
||||
if [ "$i" -ge 40 ]; then
|
||||
echo "ERROR: API did not become healthy within 120s"
|
||||
if [ "$START_FAILED" -gt 0 ]; then
|
||||
echo "ERROR: $START_FAILED 个容器启动失败"
|
||||
rollback
|
||||
fi
|
||||
|
||||
# ---- 并行等待 API 和 Web 健康 ----
|
||||
echo ""
|
||||
echo "Waiting for API + Web health (parallel)..."
|
||||
|
||||
HEALTH_LOG_DIR="/tmp/staging-health-$$"
|
||||
mkdir -p "$HEALTH_LOG_DIR"
|
||||
|
||||
(
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then
|
||||
echo "API healthy after $((i * 3))s"
|
||||
exit 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 3
|
||||
done
|
||||
echo "API FAILED after 120s"
|
||||
exit 1
|
||||
) > "$HEALTH_LOG_DIR/api.log" 2>&1 &
|
||||
PID_API_HEALTH=$!
|
||||
|
||||
(
|
||||
i=0
|
||||
while [ "$i" -lt 15 ]; do
|
||||
if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then
|
||||
echo "Web healthy after $((i * 2))s"
|
||||
exit 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 2
|
||||
done
|
||||
echo "Web FAILED after 30s"
|
||||
exit 1
|
||||
) > "$HEALTH_LOG_DIR/web.log" 2>&1 &
|
||||
PID_WEB_HEALTH=$!
|
||||
|
||||
set +e
|
||||
wait $PID_API_HEALTH
|
||||
API_EXIT=$?
|
||||
wait $PID_WEB_HEALTH
|
||||
WEB_EXIT=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
echo "健康检查结果:"
|
||||
API_OK=0
|
||||
WEB_OK=0
|
||||
if [ "$API_EXIT" -eq 0 ]; then
|
||||
echo " OK API: $(cat "$HEALTH_LOG_DIR/api.log")"
|
||||
API_OK=1
|
||||
else
|
||||
echo " FAIL API: 120s未就绪"
|
||||
docker logs --tail 50 xiaoxia-api-staging
|
||||
rollback
|
||||
fi
|
||||
|
||||
# ---- 等待 Web 健康 ----
|
||||
echo "Waiting for Web to become healthy..."
|
||||
i=0
|
||||
while [ "$i" -lt 15 ]; do
|
||||
if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then
|
||||
echo "Web is healthy!"
|
||||
break
|
||||
fi
|
||||
i=$((i + 1))
|
||||
echo " Waiting... ($i/15)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [ "$i" -ge 15 ]; then
|
||||
echo "ERROR: Web did not become healthy within 30s"
|
||||
if [ "$WEB_EXIT" -eq 0 ]; then
|
||||
echo " OK Web: $(cat "$HEALTH_LOG_DIR/web.log")"
|
||||
WEB_OK=1
|
||||
else
|
||||
echo " FAIL Web: 30s未就绪"
|
||||
docker logs --tail 30 xiaoxia-web-staging
|
||||
fi
|
||||
|
||||
rm -rf "$HEALTH_LOG_DIR"
|
||||
|
||||
if [ "$API_OK" -eq 0 ] || [ "$WEB_OK" -eq 0 ]; then
|
||||
echo ""
|
||||
echo "ERROR: 健康检查失败"
|
||||
rollback
|
||||
fi
|
||||
|
||||
# ---- 清理旧镜像 ----
|
||||
echo "Cleaning up old images..."
|
||||
docker image prune -af --filter "until=168h" 2>/dev/null || true
|
||||
docker builder prune -af --filter "until=168h" 2>/dev/null || true
|
||||
|
||||
echo ""
|
||||
echo "=== Staging deployment complete ==="
|
||||
echo "=== Staging deployment complete (并行优化版) ==="
|
||||
echo "API: http://127.0.0.1:8000"
|
||||
echo "Web: http://127.0.0.1:3001"
|
||||
echo "Version: $IMAGE_TAG"
|
||||
|
||||
@@ -2,18 +2,167 @@
|
||||
集成测试公共 fixtures
|
||||
|
||||
提供性能测试相关的工具、fixture 和 marker。
|
||||
支持 pytest-xdist 并行执行:每个 worker 使用独立数据库,数据完全隔离。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
import pytest
|
||||
|
||||
# ── xdist 并行数据库隔离 ──────────────────────────────────────────────────
|
||||
# 每个 xdist worker 进程创建独立的数据库并执行迁移,确保测试数据完全隔离
|
||||
# 通过 PYTEST_XDIST_WORKER 环境变量识别 worker(如 gw0, gw1, ...)
|
||||
|
||||
_WORKER_DB_NAME: Optional[str] = None
|
||||
|
||||
|
||||
def _get_worker_id() -> Optional[str]:
|
||||
"""获取当前 xdist worker ID,非 worker 模式返回 None"""
|
||||
return os.environ.get("PYTEST_XDIST_WORKER")
|
||||
|
||||
|
||||
def _parse_database_url(url: str) -> Dict[str, str]:
|
||||
"""
|
||||
解析 DATABASE_URL,返回各组件。
|
||||
支持 postgresql+psycopg://user:pass@host:port/dbname 格式
|
||||
"""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(url)
|
||||
return {
|
||||
"driver": parsed.scheme,
|
||||
"user": parsed.username or "",
|
||||
"password": parsed.password or "",
|
||||
"host": parsed.hostname or "",
|
||||
"port": str(parsed.port or 5432),
|
||||
"dbname": parsed.path.lstrip("/") or "",
|
||||
}
|
||||
|
||||
|
||||
def _create_worker_database(worker_id: str) -> str:
|
||||
"""
|
||||
为 xdist worker 创建独立数据库并执行迁移。
|
||||
返回新的 DATABASE_URL。
|
||||
"""
|
||||
base_url = os.environ.get(
|
||||
"DATABASE_URL",
|
||||
"postgresql+psycopg://postgres:postgres@localhost:5432/xiaoxia_saas",
|
||||
)
|
||||
db_info = _parse_database_url(base_url)
|
||||
|
||||
# 生成 worker 专属数据库名
|
||||
base_db = db_info["dbname"]
|
||||
worker_db = f"{base_db}_{worker_id}"
|
||||
global _WORKER_DB_NAME
|
||||
_WORKER_DB_NAME = worker_db
|
||||
|
||||
# 使用 psycopg 创建数据库(连接到 postgres 库)
|
||||
try:
|
||||
import psycopg
|
||||
|
||||
conn_str = (
|
||||
f"host={db_info['host']} port={db_info['port']} "
|
||||
f"user={db_info['user']} password={db_info['password']} "
|
||||
f"dbname=postgres"
|
||||
)
|
||||
conn = psycopg.connect(conn_str, autocommit=True)
|
||||
cur = conn.cursor()
|
||||
|
||||
# 先尝试删除(防止残留)
|
||||
cur.execute(f'DROP DATABASE IF EXISTS "{worker_db}" WITH (FORCE)')
|
||||
|
||||
# 创建新数据库
|
||||
cur.execute(f'CREATE DATABASE "{worker_db}"')
|
||||
cur.close()
|
||||
conn.close()
|
||||
print(f"[xdist {worker_id}] ✅ 创建数据库: {worker_db}")
|
||||
except ImportError:
|
||||
print(f"[xdist {worker_id}] ⚠️ psycopg 未安装,跳过数据库创建")
|
||||
return base_url
|
||||
except Exception as e:
|
||||
print(f"[xdist {worker_id}] ⚠️ 创建数据库失败: {e}")
|
||||
return base_url
|
||||
|
||||
# 构建新的 DATABASE_URL
|
||||
new_url = (
|
||||
f"{db_info['driver']}://{db_info['user']}:{db_info['password']}"
|
||||
f"@{db_info['host']}:{db_info['port']}/{worker_db}"
|
||||
)
|
||||
|
||||
# 执行 alembic 迁移
|
||||
print(f"[xdist {worker_id}] 🔄 执行 Alembic 迁移...")
|
||||
try:
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
api_path = str(ROOT / "apps" / "api")
|
||||
if api_path not in sys.path:
|
||||
sys.path.insert(0, api_path)
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from alembic import command as alembic_command
|
||||
from alembic.config import Config as AlembicConfig
|
||||
|
||||
alembic_cfg = AlembicConfig(str(ROOT / "alembic.ini"))
|
||||
alembic_cfg.set_main_option("sqlalchemy.url", new_url)
|
||||
# 兼容不同的脚本路径配置
|
||||
alembic_cfg.set_main_option("script_location", str(ROOT / "alembic"))
|
||||
|
||||
# 临时设置环境变量供 alembic env.py 使用
|
||||
os.environ["DATABASE_URL"] = new_url
|
||||
alembic_command.upgrade(alembic_cfg, "head")
|
||||
print(f"[xdist {worker_id}] ✅ 迁移完成")
|
||||
except Exception as e:
|
||||
print(f"[xdist {worker_id}] ❌ 迁移失败: {e}")
|
||||
raise
|
||||
|
||||
return new_url
|
||||
|
||||
|
||||
def _cleanup_worker_database(worker_id: str):
|
||||
"""清理 xdist worker 的数据库"""
|
||||
global _WORKER_DB_NAME
|
||||
if not _WORKER_DB_NAME:
|
||||
return
|
||||
|
||||
base_url = os.environ.get(
|
||||
"DATABASE_URL",
|
||||
"postgresql+psycopg://postgres:postgres@localhost:5432/xiaoxia_saas",
|
||||
)
|
||||
db_info = _parse_database_url(base_url)
|
||||
|
||||
try:
|
||||
import psycopg
|
||||
|
||||
conn_str = (
|
||||
f"host={db_info['host']} port={db_info['port']} "
|
||||
f"user={db_info['user']} password={db_info['password']} "
|
||||
f"dbname=postgres"
|
||||
)
|
||||
conn = psycopg.connect(conn_str, autocommit=True)
|
||||
cur = conn.cursor()
|
||||
# 强制断开所有连接后删除
|
||||
cur.execute(
|
||||
f"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
||||
f"WHERE datname = '{_WORKER_DB_NAME}' AND pid <> pg_backend_pid()"
|
||||
)
|
||||
cur.execute(f'DROP DATABASE IF EXISTS "{_WORKER_DB_NAME}" WITH (FORCE)')
|
||||
cur.close()
|
||||
conn.close()
|
||||
print(f"[xdist {worker_id}] 🧹 已清理数据库: {_WORKER_DB_NAME}")
|
||||
except Exception as e:
|
||||
print(f"[xdist {worker_id}] ⚠️ 清理数据库失败: {e}")
|
||||
finally:
|
||||
_WORKER_DB_NAME = None
|
||||
|
||||
|
||||
# ── 性能阈值配置 ──────────────────────────────────────────────────────────
|
||||
PERF_THRESHOLDS: Dict[str, int] = {
|
||||
"core": 500, # 核心接口:500ms
|
||||
@@ -183,16 +332,41 @@ class PerfAssert:
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ── pytest fixtures ──────────────────────────────────────────────────────
|
||||
# ── pytest hooks ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""注册自定义 marker"""
|
||||
"""
|
||||
pytest 配置钩子。
|
||||
|
||||
- 注册自定义 marker
|
||||
- xdist worker 模式下:创建独立数据库 + 执行迁移
|
||||
"""
|
||||
# 注册自定义 marker
|
||||
config.addinivalue_line("markers", "performance: 标记为性能测试(可通过 -m 'not performance' 跳过)")
|
||||
config.addinivalue_line("markers", "perf_core: 核心接口性能测试(阈值 500ms)")
|
||||
config.addinivalue_line("markers", "perf_normal: 普通接口性能测试(阈值 1000ms)")
|
||||
config.addinivalue_line("markers", "perf_heavy: 重操作接口性能测试(阈值 3000ms)")
|
||||
|
||||
# xdist worker 模式:创建独立数据库并执行迁移
|
||||
worker_id = _get_worker_id()
|
||||
if worker_id:
|
||||
# 只有当 USE_IN_MEMORY_DB 不为 true 时才创建独立数据库
|
||||
use_in_memory = os.environ.get("USE_IN_MEMORY_DB", "true").lower() == "true"
|
||||
if not use_in_memory:
|
||||
print(f"[xdist {worker_id}] 🚀 worker 启动,准备独立数据库...")
|
||||
new_db_url = _create_worker_database(worker_id)
|
||||
os.environ["DATABASE_URL"] = new_db_url
|
||||
else:
|
||||
print(f"[xdist {worker_id}] ℹ️ USE_IN_MEMORY_DB=true,跳过 worker 数据库创建")
|
||||
|
||||
|
||||
def pytest_unconfigure(config):
|
||||
"""pytest 结束钩子:清理 xdist worker 数据库"""
|
||||
worker_id = _get_worker_id()
|
||||
if worker_id and _WORKER_DB_NAME:
|
||||
_cleanup_worker_database(worker_id)
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config, items):
|
||||
"""根据环境变量自动跳过性能测试"""
|
||||
@@ -203,6 +377,9 @@ def pytest_collection_modifyitems(config, items):
|
||||
item.add_marker(skip_perf)
|
||||
|
||||
|
||||
# ── pytest fixtures ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def perf_assert():
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user