Compare commits
71 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0fccfcd181 | |||
| 8ea3263a12 | |||
| a75f749b39 | |||
| f5d8a32229 | |||
| 7fc68e797b | |||
| 34e5b18a30 | |||
| f6fc9736a5 | |||
| fde3ab3063 | |||
| 09697f4230 | |||
| 4e042ba597 | |||
| eb4ad569be | |||
| 76a0fdd00c | |||
| 876c3a2aaa | |||
| 576722f60b | |||
| 34ea6a866b | |||
| a7a1a6395b | |||
| 5f17fd5faf | |||
| cbda1ec4d5 | |||
| c5fb5b47ac | |||
| 637c5a8b6f | |||
| 21ae1c627f | |||
| a4a279ae2b | |||
| 0724ddfbf3 | |||
| 4967891d8b | |||
| 7cec7b4a31 | |||
| b2700b85ff | |||
| e4ce3caad1 | |||
| 35f92f8101 | |||
| ed061aae3e | |||
| 6d8656b3f6 | |||
| 52e866b750 | |||
| 2cb273b472 | |||
| 7a1f7c89b5 | |||
| 8c1566cec3 | |||
| 246406eeb0 | |||
| a0c27e3132 | |||
| 61f0601cd4 | |||
| a39680dbca | |||
| 319347cb41 | |||
| 8a59fff246 | |||
| 13e5c03e77 | |||
| 83d373c840 | |||
| 7b2d794126 | |||
| 6ce7db7f8e | |||
| 6b45ddb6fc | |||
| d3438b9786 | |||
| 1669e787c2 | |||
| 7cffe564a9 | |||
| c2ad5003df | |||
| 14be2b654b | |||
| 92c11ced63 | |||
| 3379f07a15 | |||
| cfe75543a1 | |||
| 5e44f3dd1a | |||
| 95683ab5ef | |||
| 6bde806462 | |||
| 0ae105628d | |||
| 24efd5121f | |||
| e0c3bae072 | |||
| 05b84ac669 | |||
| 809339fee3 | |||
| 6fe648432a | |||
| 81ae089985 | |||
| 524dbc002a | |||
| df0e95ee58 | |||
| b59cf8aab5 | |||
| 906eb5de9c | |||
| f22a6fd90e | |||
| a18244b8a4 | |||
| 48102ca2e2 | |||
| baf6a4143c |
@@ -0,0 +1,78 @@
|
||||
name: CI Failure Monitor
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 */6 * * *' # 每6小时检查一次
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
days:
|
||||
description: '统计最近N天的失败'
|
||||
required: false
|
||||
default: '7'
|
||||
fail_threshold:
|
||||
description: '失败次数阈值'
|
||||
required: false
|
||||
default: '3'
|
||||
fail_rate_threshold:
|
||||
description: '失败率阈值(%)'
|
||||
required: false
|
||||
default: '30'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
monitor:
|
||||
name: CI重复失败检测
|
||||
runs-on: ci-l2
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
curl -sH "Authorization: token $GITHUB_TOKEN" \
|
||||
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" \
|
||||
| bash
|
||||
|
||||
- name: Record job start time
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_timer_start.sh
|
||||
|
||||
- name: Run failure detection
|
||||
shell: sh
|
||||
env:
|
||||
GITEA_API_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
|
||||
GITEA_URL: https://git.xiaoxiajianji.com
|
||||
GITEA_REPO: xiaoxia/xiaoxia-saas
|
||||
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
|
||||
FAIL_CHECK_DAYS: ${{ inputs.days || 7 }}
|
||||
FAIL_THRESHOLD: ${{ inputs.fail_threshold || 3 }}
|
||||
FAIL_RATE_THRESHOLD: ${{ inputs.fail_rate_threshold || 30 }}
|
||||
run: |
|
||||
set +e
|
||||
python3 scripts/ci/ci_repeated_failure_detector.py
|
||||
EXIT_CODE=$?
|
||||
echo "检测完成,退出码: $EXIT_CODE"
|
||||
# 0=无异常, 1=有警告, 2=有严重问题
|
||||
# 监控脚本永远不fail,避免告警风暴
|
||||
exit 0
|
||||
|
||||
- name: Job duration summary
|
||||
if: always()
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_timer_end.sh
|
||||
|
||||
- name: Report CI trace
|
||||
if: always()
|
||||
shell: sh
|
||||
env:
|
||||
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
|
||||
run: |
|
||||
STATUS="ok"
|
||||
[ ${{ job.status }} = "success" ] || STATUS="error"
|
||||
START_TIME=""
|
||||
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||
@@ -1,19 +1,15 @@
|
||||
name: CI Health Daily Report
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 1 * * *' # UTC 01:00 = 北京时间 09:00
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
ci-health-report:
|
||||
name: CI健康度每日巡检
|
||||
runs-on: saas
|
||||
timeout-minutes: 10
|
||||
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
@@ -61,6 +57,34 @@ jobs:
|
||||
tar.extract(member, '.')
|
||||
PY
|
||||
|
||||
- name: Generate CI Dashboard HTML
|
||||
shell: sh
|
||||
env:
|
||||
GITEA_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set +e
|
||||
echo "=== 生成 CI 健康度 HTML 看板 ==="
|
||||
echo "时间: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo ""
|
||||
python3 scripts/ci/ci_dashboard.py --days 7 --html --html-output ci_dashboard.html
|
||||
EXIT_CODE=$?
|
||||
if [ $EXIT_CODE -eq 0 ] && [ -f ci_dashboard.html ]; then
|
||||
HTML_SIZE=$(wc -c < ci_dashboard.html)
|
||||
echo ""
|
||||
echo "✅ HTML 看板生成成功 (${HTML_SIZE} bytes)"
|
||||
echo "路径: $(pwd)/ci_dashboard.html"
|
||||
# 输出文件内容前几行,方便在 Actions 日志中确认
|
||||
echo ""
|
||||
echo "--- 看板预览 (前 5 行) ---"
|
||||
head -5 ci_dashboard.html
|
||||
echo "...(完整内容见产物文件)"
|
||||
else
|
||||
echo "❌ HTML 看板生成失败 (exit code: $EXIT_CODE)"
|
||||
fi
|
||||
echo ""
|
||||
# 永远成功,看板生成失败不影响主流程
|
||||
exit 0
|
||||
|
||||
- name: Run CI health check and report
|
||||
shell: sh
|
||||
env:
|
||||
@@ -71,10 +95,8 @@ jobs:
|
||||
echo "=== CI健康度每日巡检 ==="
|
||||
echo "时间: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo ""
|
||||
|
||||
python3 scripts/ci/ci_health_report.py --limit 30
|
||||
EXIT_CODE=$?
|
||||
|
||||
echo ""
|
||||
echo "巡检完成 (exit code: $EXIT_CODE)"
|
||||
# 永远成功,不影响CI状态(通知失败不应该标红)
|
||||
|
||||
@@ -76,18 +76,12 @@ jobs:
|
||||
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||
|
||||
validate:
|
||||
needs: check-frontend-only
|
||||
if: always() && needs.check-frontend-only.outputs.skip_backend != 'true'
|
||||
name: Validate Code Quality And Tests
|
||||
validate-code-quality:
|
||||
name: Validate - Code Quality
|
||||
runs-on: ci-l2
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 8
|
||||
permissions:
|
||||
contents: write
|
||||
env:
|
||||
DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas
|
||||
USE_IN_MEMORY_DB: 'false'
|
||||
CI_USE_SHARED_PG: 'true'
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
@@ -102,7 +96,6 @@ jobs:
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
# pip install 带重试(网络不稳定时自动重试)
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements-base.txt && break
|
||||
echo "pip install requirements-base.txt 失败,重试 $i/3..."
|
||||
@@ -127,16 +120,17 @@ jobs:
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
- name: Run all quality checks
|
||||
- name: Run code quality and security checks
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: bash scripts/ci/run_validate.sh
|
||||
run: bash scripts/ci/validate_code_quality.sh
|
||||
- name: Auto-fix formatting (black + isort)
|
||||
if: failure()
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
REVIEW_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
|
||||
run: python3 scripts/ci/auto_fix_formatting.py
|
||||
- name: CI failure notification
|
||||
if: failure()
|
||||
@@ -146,7 +140,7 @@ jobs:
|
||||
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
|
||||
run: |
|
||||
set +e
|
||||
FAILED_JOB="Validate Code Quality And Tests" python3 scripts/ci_notify_failure.py
|
||||
FAILED_JOB="Validate - Code Quality" python3 scripts/ci_notify_failure.py
|
||||
- name: Job duration summary
|
||||
if: always()
|
||||
shell: sh
|
||||
@@ -159,7 +153,161 @@ jobs:
|
||||
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
|
||||
run: |
|
||||
set +e
|
||||
NOTIFY_MODE=failure JOB_NAME="Validate Code Quality And Tests" python3 scripts/ci_notify.py
|
||||
NOTIFY_MODE=failure JOB_NAME="Validate - Code Quality" python3 scripts/ci_notify.py
|
||||
- name: Report CI trace
|
||||
if: always()
|
||||
shell: sh
|
||||
env:
|
||||
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
|
||||
run: |
|
||||
STATUS="ok"
|
||||
[ ${{ job.status }} = "success" ] || STATUS="error"
|
||||
START_TIME=""
|
||||
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||
|
||||
validate-type-check:
|
||||
name: Validate - Type Check (mypy)
|
||||
runs-on: ci-l2
|
||||
timeout-minutes: 8
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
|
||||
- name: Record job start time
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_timer_start.sh
|
||||
- name: Install dependencies
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements-base.txt && break
|
||||
echo "pip install requirements-base.txt 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements.txt && break
|
||||
echo "pip install requirements.txt 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements-dev.txt && break
|
||||
echo "pip install requirements-dev.txt 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
- name: Run mypy type check
|
||||
shell: bash
|
||||
run: bash scripts/ci/validate_mypy.sh
|
||||
- name: CI failure notification
|
||||
if: failure()
|
||||
shell: sh
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
|
||||
run: |
|
||||
set +e
|
||||
FAILED_JOB="Validate - Type Check (mypy)" python3 scripts/ci_notify_failure.py
|
||||
- name: Job duration summary
|
||||
if: always()
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_timer_end.sh
|
||||
- name: Notify on failure
|
||||
continue-on-error: true
|
||||
if: failure()
|
||||
shell: sh
|
||||
env:
|
||||
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
|
||||
run: |
|
||||
set +e
|
||||
NOTIFY_MODE=failure JOB_NAME="Validate - Type Check (mypy)" python3 scripts/ci_notify.py
|
||||
- name: Report CI trace
|
||||
if: always()
|
||||
shell: sh
|
||||
env:
|
||||
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
|
||||
run: |
|
||||
STATUS="ok"
|
||||
[ ${{ job.status }} = "success" ] || STATUS="error"
|
||||
START_TIME=""
|
||||
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||
|
||||
validate-migration:
|
||||
name: Validate - Migration (alembic)
|
||||
runs-on: ci-l2
|
||||
timeout-minutes: 8
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas
|
||||
USE_IN_MEMORY_DB: 'false'
|
||||
CI_USE_SHARED_PG: 'true'
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
|
||||
- name: Record job start time
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_timer_start.sh
|
||||
- name: Install dependencies
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements-base.txt && break
|
||||
echo "pip install requirements-base.txt 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements.txt && break
|
||||
echo "pip install requirements.txt 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q -r requirements-dev.txt && break
|
||||
echo "pip install requirements-dev.txt 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
- name: Run alembic migration validation
|
||||
shell: bash
|
||||
run: bash scripts/ci/validate_migration.sh
|
||||
- name: CI failure notification
|
||||
if: failure()
|
||||
shell: sh
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
|
||||
run: |
|
||||
set +e
|
||||
FAILED_JOB="Validate - Migration (alembic)" python3 scripts/ci_notify_failure.py
|
||||
- name: Job duration summary
|
||||
if: always()
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_timer_end.sh
|
||||
- name: Notify on failure
|
||||
continue-on-error: true
|
||||
if: failure()
|
||||
shell: sh
|
||||
env:
|
||||
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
|
||||
run: |
|
||||
set +e
|
||||
NOTIFY_MODE=failure JOB_NAME="Validate - Migration (alembic)" python3 scripts/ci_notify.py
|
||||
- name: Report CI trace
|
||||
if: always()
|
||||
shell: sh
|
||||
@@ -387,9 +535,11 @@ jobs:
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
- name: Run Vitest with coverage
|
||||
- name: Run Vitest (incremental for PRs, full for main branches)
|
||||
shell: sh
|
||||
run: bash scripts/ci/step_frontend_run.sh "npx --no-install vitest run --coverage"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: bash scripts/ci/vitest_incremental.sh
|
||||
- name: Job duration summary
|
||||
if: always()
|
||||
shell: sh
|
||||
@@ -472,6 +622,65 @@ jobs:
|
||||
echo "Docker login failed ($i/3), retrying in 5s..."
|
||||
sleep 5
|
||||
done
|
||||
- name: Pre-build worker base images (fallback if not exist)
|
||||
if: matrix.service == 'worker'
|
||||
id: prebuild
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas"
|
||||
BASE_BUILDER="${REGISTRY}/worker-base-builder:latest"
|
||||
BASE_RUNTIME="${REGISTRY}/worker-base-runtime:latest"
|
||||
|
||||
# 尝试拉取基础镜像
|
||||
echo "检查基础镜像..."
|
||||
if docker pull "$BASE_BUILDER" 2>/dev/null && docker pull "$BASE_RUNTIME" 2>/dev/null; then
|
||||
echo "基础镜像已存在,使用远程镜像"
|
||||
echo "fallback=false" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "基础镜像不存在,本地构建(fallback模式)..."
|
||||
|
||||
# 构建builder基础镜像
|
||||
echo "构建 worker-base-builder..."
|
||||
# 用buildx docker-container驱动构建(兼容DooD模式:普通docker build看不到容器内文件)
|
||||
BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}"
|
||||
if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
else
|
||||
docker buildx use "$BUILDER_NAME"
|
||||
fi
|
||||
docker buildx inspect --bootstrap > /dev/null 2>&1
|
||||
|
||||
# 构建builder基础镜像(带重试,buildx容器偶发不稳定)
|
||||
echo "构建 worker-base-builder..."
|
||||
for attempt in 1 2 3; do
|
||||
if docker buildx build --load -f infra/docker/worker-base-builder.Dockerfile -t "$BASE_BUILDER" .; then
|
||||
echo "worker-base-builder 构建成功"
|
||||
break
|
||||
fi
|
||||
echo "worker-base-builder 构建失败,重试 $attempt/3..."
|
||||
docker buildx rm "$BUILDER_NAME" 2>/dev/null || true
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# 构建runtime基础镜像
|
||||
echo "构建 worker-base-runtime..."
|
||||
for attempt in 1 2 3; do
|
||||
if docker buildx build --load -f infra/docker/worker-base-runtime.Dockerfile -t "$BASE_RUNTIME" .; then
|
||||
echo "worker-base-runtime 构建成功"
|
||||
break
|
||||
fi
|
||||
echo "worker-base-runtime 构建失败,重试 $attempt/3..."
|
||||
docker buildx rm "$BUILDER_NAME" 2>/dev/null || true
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
sleep 3
|
||||
done
|
||||
|
||||
echo "fallback=true" >> $GITHUB_OUTPUT
|
||||
echo "基础镜像本地构建完成"
|
||||
fi
|
||||
|
||||
- name: Build PR image (verify only, no push)
|
||||
shell: sh
|
||||
run: |
|
||||
@@ -485,6 +694,18 @@ jobs:
|
||||
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
|
||||
fi
|
||||
|
||||
# Worker fallback模式:基础镜像本地已构建,用普通docker build绕过buildx
|
||||
if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.fallback }}" = "true" ]; then
|
||||
echo "Fallback模式:用普通docker build(基础镜像本地已构建)"
|
||||
BUILD_ARG_STR=""
|
||||
for arg in $EXTRA_BUILD_ARGS; do
|
||||
BUILD_ARG_STR="$BUILD_ARG_STR --build-arg $arg"
|
||||
done
|
||||
docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR .
|
||||
echo "Fallback PR Build successful"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
NO_CACHE_FLAG=""
|
||||
for i in 1 2 3; do
|
||||
echo "PR Build attempt $i/3"
|
||||
@@ -1333,5 +1554,4 @@ jobs:
|
||||
[ ${{ job.status }} = "success" ] || STATUS="error"
|
||||
START_TIME=""
|
||||
[ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
|
||||
@@ -0,0 +1,56 @@
|
||||
name: PR Auto Scan
|
||||
# 定时扫描所有open PR,对CI全绿的触发审批/合并
|
||||
# 作为短作业模式的兜底,防止事件驱动遗漏
|
||||
on:
|
||||
schedule:
|
||||
- cron: "*/5 * * * *" # 每5分钟扫描一次
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
auto-scan:
|
||||
name: Auto Scan Open PRs
|
||||
runs-on: ci-check
|
||||
timeout-minutes: 5
|
||||
if: github.repository == 'xiaoxia/xiaoxia-saas'
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/pr_auto_scan.py?ref=develop" -o /tmp/pr_auto_scan.py
|
||||
python3 /tmp/pr_auto_scan.py --help > /dev/null 2>&1 || {
|
||||
# fallback: checkout
|
||||
echo "使用checkout方式"
|
||||
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=develop" | bash
|
||||
}
|
||||
|
||||
- name: Scan and auto process PRs
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
REVIEW_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
|
||||
MERGE_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
echo "=== 扫描所有open PR并自动处理 ==="
|
||||
echo "时间: $(date)"
|
||||
echo
|
||||
|
||||
python3 /tmp/pr_auto_scan.py --token "$REVIEW_TOKEN" --repo "$GITHUB_REPOSITORY" --base develop --approve --merge --dry-run false
|
||||
|
||||
echo ""
|
||||
echo "✅ 扫描完成"
|
||||
|
||||
- name: Report CI trace
|
||||
if: always()
|
||||
shell: sh
|
||||
env:
|
||||
AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
|
||||
run: |
|
||||
STATUS="ok"
|
||||
[ ${{ job.status }} = "success" ] || STATUS="error"
|
||||
python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "" || true
|
||||
@@ -3,6 +3,7 @@ name: PR Automation
|
||||
on:
|
||||
pull_request:
|
||||
types: [synchronize, opened, ready_for_review, review_requested]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -12,7 +13,7 @@ jobs:
|
||||
name: Auto Approve on CI Green
|
||||
runs-on: ci-check
|
||||
if: github.event_name == 'pull_request' && !github.event.pull_request.draft
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 3 # 短作业模式:最多3分钟,不占runner
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
@@ -54,7 +55,9 @@ jobs:
|
||||
CONTEXTS=("CI/CD Pipeline / Frontend Lint (pull_request)")
|
||||
else
|
||||
CONTEXTS=(
|
||||
"CI/CD Pipeline / Validate Code Quality And Tests (pull_request)"
|
||||
"CI/CD Pipeline / Validate - Code Quality (pull_request)"
|
||||
"CI/CD Pipeline / Validate - Type Check (mypy) (pull_request)"
|
||||
"CI/CD Pipeline / Validate - Migration (alembic) (pull_request)"
|
||||
"CI/CD Pipeline / Frontend Lint (pull_request)"
|
||||
)
|
||||
fi
|
||||
@@ -70,7 +73,7 @@ jobs:
|
||||
sleep 30
|
||||
|
||||
# 轮询等待,最多20分钟(120次x10秒)
|
||||
for attempt in $(seq 1 120); do
|
||||
for attempt in $(seq 1 12); do # 短作业模式:最多等2分钟(12次x10秒),不满足就退出等下次触发
|
||||
ALL_SUCCESS=true
|
||||
ANY_FAILED=false
|
||||
ANY_PENDING=false
|
||||
@@ -155,7 +158,7 @@ jobs:
|
||||
|
||||
# 还有CI在跑 → 继续等
|
||||
if [ "$ANY_PENDING" = "true" ]; then
|
||||
echo "⏳ CI仍在运行中,继续等待(第${attempt}/120次轮询)..."
|
||||
echo "⏳ CI仍在运行中(第${attempt}/12次),超时后将退出等待下次触发..."
|
||||
sleep 10
|
||||
continue
|
||||
fi
|
||||
@@ -171,7 +174,7 @@ jobs:
|
||||
done
|
||||
|
||||
echo
|
||||
echo "⏰ 等待超时(20分钟),CI尚未全部完成"
|
||||
echo "⏰ 快速检查超时(2分钟),CI尚未完成,退出等待下次触发(workflow_run事件或5分钟定时扫描)"
|
||||
exit 0
|
||||
|
||||
- name: Report CI trace
|
||||
@@ -190,7 +193,7 @@ jobs:
|
||||
name: Auto Merge on CI Green + Approved
|
||||
runs-on: ci-check
|
||||
if: github.event_name == 'pull_request' && !github.event.pull_request.draft && github.event.pull_request.base.ref == 'develop'
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 3 # 短作业模式:最多3分钟,不占runner
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
@@ -233,7 +236,9 @@ jobs:
|
||||
echo "纯前端改动,只检查Frontend Lint"
|
||||
else
|
||||
CONTEXTS=(
|
||||
"CI/CD Pipeline / Validate Code Quality And Tests (pull_request)"
|
||||
"CI/CD Pipeline / Validate - Code Quality (pull_request)"
|
||||
"CI/CD Pipeline / Validate - Type Check (mypy) (pull_request)"
|
||||
"CI/CD Pipeline / Validate - Migration (alembic) (pull_request)"
|
||||
"CI/CD Pipeline / Frontend Lint (pull_request)"
|
||||
"CI/CD Pipeline / PR Build API Image (pull_request)"
|
||||
"CI/CD Pipeline / PR Build Worker Image (pull_request)"
|
||||
@@ -252,7 +257,7 @@ jobs:
|
||||
MAX_405_RETRIES=10
|
||||
|
||||
# 轮询等待,最多30分钟(180次x10秒)
|
||||
for attempt in $(seq 1 180); do
|
||||
for attempt in $(seq 1 18); do # 短作业模式:最多等3分钟(18次x10秒),不满足就退出等下次触发
|
||||
ALL_SUCCESS=true
|
||||
ANY_FAILED=false
|
||||
ANY_PENDING=false
|
||||
@@ -274,18 +279,11 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
# 检查审批状态
|
||||
APPROVAL_RESULT=$(python3 scripts/check_pr_approval.py "$MERGE_TOKEN" "$GITHUB_REPOSITORY" "$PR_NUMBER" 1)
|
||||
echo " 审批: $APPROVAL_RESULT"
|
||||
HAS_APPROVAL=false
|
||||
if echo "$APPROVAL_RESULT" | grep -q '^approved'; then
|
||||
HAS_APPROVAL=true
|
||||
fi
|
||||
|
||||
# 全部满足 → 合并
|
||||
if [ "$ALL_SUCCESS" = "true" ] && [ "$HAS_APPROVAL" = "true" ]; then
|
||||
# CI全绿 → 合并
|
||||
if [ "$ALL_SUCCESS" = "true" ]; then
|
||||
echo
|
||||
echo "CI全绿 + 审批通过,执行自动合并"
|
||||
echo "CI全绿,执行自动合并"
|
||||
echo "等待60秒冷却,给Gitea内部状态同步时间..."
|
||||
sleep 60
|
||||
|
||||
@@ -353,7 +351,7 @@ jobs:
|
||||
done
|
||||
|
||||
echo
|
||||
echo "等待超时(30分钟)"
|
||||
echo "快速检查超时(3分钟),CI尚未全绿或无审批,退出等待下次触发"
|
||||
exit 0
|
||||
- name: Report CI trace
|
||||
if: always()
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
name: Worker Base Image Build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- develop
|
||||
- main
|
||||
paths:
|
||||
- 'requirements-base.txt'
|
||||
- 'requirements-worker.txt'
|
||||
- 'infra/docker/worker-base-builder.Dockerfile'
|
||||
- 'infra/docker/worker-base-runtime.Dockerfile'
|
||||
workflow_dispatch: # 支持手动触发
|
||||
|
||||
jobs:
|
||||
build-worker-base:
|
||||
name: Build Worker Base Images
|
||||
runs-on: runtime-builder
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- name: builder
|
||||
dockerfile: infra/docker/worker-base-builder.Dockerfile
|
||||
image_name: worker-base-builder
|
||||
cache_name: worker-base-builder-cache
|
||||
- name: runtime
|
||||
dockerfile: infra/docker/worker-base-runtime.Dockerfile
|
||||
image_name: worker-base-runtime
|
||||
cache_name: worker-base-runtime-cache
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash
|
||||
|
||||
- name: Docker login to Registry
|
||||
shell: sh
|
||||
env:
|
||||
ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
|
||||
ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
|
||||
GITEA_REGISTRY_USER: xiaoxia
|
||||
GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
for i in 1 2 3; do
|
||||
echo "=== Docker login 尝试 $i/3 ==="
|
||||
if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then
|
||||
echo "✅ Docker login successful"
|
||||
break
|
||||
fi
|
||||
echo "❌ Docker login 失败(尝试 $i/3),5s 后重试..."
|
||||
sleep 5
|
||||
done
|
||||
|
||||
- name: Setup buildx builder
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
BUILDER_NAME="ci-builder-${GITHUB_RUN_ID}-${{ matrix.name }}"
|
||||
if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
echo "Created $BUILDER_NAME"
|
||||
else
|
||||
docker buildx use "$BUILDER_NAME"
|
||||
echo "Using existing $BUILDER_NAME"
|
||||
fi
|
||||
docker buildx inspect --bootstrap
|
||||
|
||||
- name: Build and push base image
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
|
||||
IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:latest"
|
||||
SAFE_REF_NAME=$(echo "${GITHUB_REF_NAME}" | tr '/' '-')
|
||||
CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:${SAFE_REF_NAME}"
|
||||
|
||||
echo "=== Building ${{ matrix.name }} base image ==="
|
||||
echo "Image: ${IMAGE_TAG}"
|
||||
echo "Cache: ${CACHE_REF}"
|
||||
|
||||
# 用通用构建脚本
|
||||
bash scripts/ci/docker_build_push.sh ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}"
|
||||
|
||||
# 同时推送到 Gitea Packages 作为备份(可选)
|
||||
GITEA_IMAGE="git.xiaoxiajianji.com/xiaoxia-saas/${{ matrix.image_name }}:latest"
|
||||
docker tag "${IMAGE_TAG}" "${GITEA_IMAGE}"
|
||||
docker push "${GITEA_IMAGE}" || echo "Gitea Packages push failed (non-fatal)"
|
||||
|
||||
echo ""
|
||||
echo "✅ ${{ matrix.name }} base image built and pushed"
|
||||
|
||||
- name: Cleanup buildx builder
|
||||
if: always()
|
||||
shell: sh
|
||||
run: |
|
||||
docker buildx rm "ci-builder-${GITHUB_RUN_ID}-${{ matrix.name }}" 2>/dev/null || true
|
||||
docker buildx prune -f 2>/dev/null || true
|
||||
echo "Builder cleanup done"
|
||||
@@ -147,13 +147,14 @@ export interface SendVerificationCodeRequest {
|
||||
}
|
||||
|
||||
export interface BindContactRequest {
|
||||
target: "email" | "phone"
|
||||
value: string
|
||||
code: string
|
||||
email?: string
|
||||
email_code?: string
|
||||
phone?: string
|
||||
phone_code?: string
|
||||
}
|
||||
|
||||
export interface BindContactResponse {
|
||||
message: string
|
||||
success: boolean
|
||||
user: User
|
||||
}
|
||||
|
||||
|
||||
@@ -64,14 +64,12 @@ const BindContactModal: React.FC<BindContactModalProps> = ({ open, onSuccess, on
|
||||
const values = await form.validateFields()
|
||||
setLoading(true)
|
||||
|
||||
const target = activeTab
|
||||
const value = target === "email" ? values.email : values.phone
|
||||
const payload =
|
||||
activeTab === "email"
|
||||
? { email: values.email, email_code: values.code }
|
||||
: { phone: values.phone, phone_code: values.code }
|
||||
|
||||
const result = await bindContact({
|
||||
target,
|
||||
value,
|
||||
code: values.code,
|
||||
})
|
||||
const result = await bindContact(payload)
|
||||
|
||||
message.success("绑定成功")
|
||||
onSuccess?.(result.user)
|
||||
|
||||
@@ -94,7 +94,7 @@ describe("useAuth hooks", () => {
|
||||
expect(typeof result.current.mutateAsync).toBe("function")
|
||||
})
|
||||
|
||||
it("登录成功时保存 token 并调用 setAuth", async () => {
|
||||
it("登录成功时调用 setAuth 并跳转到登录前页面或首页", async () => {
|
||||
mockMutateAsync.mockResolvedValue({
|
||||
access_token: "access-123",
|
||||
refresh_token: "refresh-456",
|
||||
@@ -106,15 +106,15 @@ describe("useAuth hooks", () => {
|
||||
await result.current.mutateAsync({ username: "test", password: "123" })
|
||||
})
|
||||
|
||||
expect(localStorage.getItem("access_token")).toBe("access-123")
|
||||
expect(localStorage.getItem("refresh_token")).toBe("refresh-456")
|
||||
expect(mockSetAuth).toHaveBeenCalled()
|
||||
expect(mockNavigate).toHaveBeenCalledWith("/", { replace: true })
|
||||
})
|
||||
|
||||
it("没有 refresh_token 时从 localStorage 移除", async () => {
|
||||
it("登录成功后跳转到 login_redirect 指定的页面", async () => {
|
||||
localStorage.setItem("login_redirect", "/app/templates")
|
||||
mockMutateAsync.mockResolvedValue({
|
||||
access_token: "access-123",
|
||||
refresh_token: "refresh-456",
|
||||
})
|
||||
|
||||
const { result } = renderHook(() => useLogin(), { wrapper })
|
||||
@@ -123,8 +123,8 @@ describe("useAuth hooks", () => {
|
||||
await result.current.mutateAsync({ username: "test", password: "123" })
|
||||
})
|
||||
|
||||
expect(localStorage.getItem("access_token")).toBe("access-123")
|
||||
expect(localStorage.getItem("refresh_token")).toBeNull()
|
||||
expect(mockNavigate).toHaveBeenCalledWith("/app/templates", { replace: true })
|
||||
expect(localStorage.getItem("login_redirect")).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# ============================================================
|
||||
# Worker Builder 基础镜像
|
||||
# 预编译:编译工具 + 基础依赖 + Worker大包
|
||||
# 当 requirements-base.txt 或 requirements-worker.txt 变更时重新构建
|
||||
# 业务构建从此镜像开始,只需要安装业务依赖,节省15+分钟
|
||||
# ============================================================
|
||||
|
||||
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim
|
||||
|
||||
# 使用阿里云镜像加速
|
||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
|
||||
|
||||
# 安装编译工具
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
python3-dev \
|
||||
binutils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 创建 venv
|
||||
RUN python -m venv /opt/venv
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
|
||||
WORKDIR /tmp
|
||||
|
||||
# 基础依赖(变化极少)
|
||||
COPY requirements-base.txt /tmp/requirements-base.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
|
||||
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
||||
-r /tmp/requirements-base.txt \
|
||||
&& rm /tmp/requirements-base.txt
|
||||
|
||||
# Worker 大包(变化少)
|
||||
COPY requirements-worker.txt /tmp/requirements-worker.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
|
||||
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
||||
-r /tmp/requirements-worker.txt \
|
||||
&& rm /tmp/requirements-worker.txt
|
||||
|
||||
# 预先做一次 strip(基础层瘦身,业务层增量)
|
||||
RUN find /opt/venv -name "*.so" -type f -exec strip --strip-all {} \; 2>/dev/null || true
|
||||
@@ -0,0 +1,17 @@
|
||||
# ============================================================
|
||||
# Worker Runtime 基础镜像
|
||||
# 预安装:ffmpeg + 运行时依赖
|
||||
# 变化极少,业务构建从此镜像开始
|
||||
# ============================================================
|
||||
|
||||
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim
|
||||
|
||||
# 使用阿里云镜像加速
|
||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
|
||||
|
||||
# 运行时依赖:ffmpeg + opencv需要的libglib
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
libglib2.0-0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
@@ -1,88 +1,43 @@
|
||||
# ============================================================
|
||||
# Worker Dockerfile - 优化版(多阶段构建 + 镜像瘦身 + cache mount加速)
|
||||
# 优化项:
|
||||
# 1. 多阶段构建:builder 阶段安装编译依赖,runtime 阶段只保留运行时
|
||||
# 2. ffmpeg 通过 apt 安装(阿里云镜像加速,几秒完成,稳定可靠)
|
||||
# 3. Python 依赖瘦身:strip .so 调试符号 + 清理测试文件 + 清理缓存
|
||||
# 4. pip cache mount:加速依赖下载(跨构建共享pip wheel缓存)
|
||||
# Worker Dockerfile - 分层缓存优化版
|
||||
# 优化:基础依赖 + Worker大包预构建为基础镜像,业务构建仅叠加业务依赖
|
||||
# 基础镜像:worker-base-builder / worker-base-runtime
|
||||
# 预计节省:依赖不变时构建时间从23min降至5min以内
|
||||
# ============================================================
|
||||
|
||||
# ==================== Builder 阶段 ====================
|
||||
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim AS builder
|
||||
# 从预构建的builder基础镜像开始,已经包含:
|
||||
# - 编译工具 (gcc/g++/python3-dev/binutils)
|
||||
# - requirements-base.txt 全部依赖
|
||||
# - requirements-worker.txt 全部依赖 (numpy/scipy/opencv)
|
||||
# - 预strip的.so文件
|
||||
FROM xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/worker-base-builder:latest AS builder
|
||||
|
||||
# 使用阿里云镜像加速
|
||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
|
||||
|
||||
# 安装编译工具(仅 builder 需要)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
python3-dev \
|
||||
binutils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
# ---- 安装 Python 依赖 ----
|
||||
WORKDIR /tmp
|
||||
|
||||
# 创建 venv
|
||||
RUN python -m venv /opt/venv
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
|
||||
# 基础依赖
|
||||
COPY requirements-base.txt /tmp/requirements-base.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
|
||||
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
||||
-r /tmp/requirements-base.txt \
|
||||
&& rm /tmp/requirements-base.txt
|
||||
WORKDIR /tmp
|
||||
|
||||
# Worker 专属大包
|
||||
COPY requirements-worker.txt /tmp/requirements-worker.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
|
||||
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
||||
-r /tmp/requirements-worker.txt \
|
||||
&& rm /tmp/requirements-worker.txt
|
||||
|
||||
# 业务依赖
|
||||
# ---- 安装业务依赖(变化频繁,单独一层)----
|
||||
COPY requirements.txt /tmp/requirements.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
|
||||
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
|
||||
-r /tmp/requirements.txt \
|
||||
&& rm /tmp/requirements.txt
|
||||
|
||||
# ---- Python 依赖瘦身 ----
|
||||
# 1. strip .so 文件的调试符号(节省约 80-100MB)
|
||||
# ---- 增量瘦身(只处理新增的业务依赖)----
|
||||
RUN find /opt/venv -name "*.so" -type f -exec strip --strip-all {} \; 2>/dev/null || true
|
||||
|
||||
# 2. 清理测试文件(节省约 20MB)
|
||||
RUN find /opt/venv -type d -name "tests" -exec rm -rf {} + 2>/dev/null; \
|
||||
find /opt/venv -type d -name "test" -exec rm -rf {} + 2>/dev/null; \
|
||||
find /opt/venv -name "test_*.py" -delete 2>/dev/null || true
|
||||
|
||||
# 3. 清理 .pyc 缓存和 __pycache__(节省约 10MB,运行时按需生成)
|
||||
RUN find /opt/venv -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null; \
|
||||
find /opt/venv -name "*.pyc" -delete 2>/dev/null || true
|
||||
|
||||
# 4. 清理 dist-info 中的文档
|
||||
RUN find /opt/venv -name "*.dist-info" -type d -exec sh -c 'rm -f "$1"/DESCRIPTION.rst "$1"/INSTALLER "$1"/LICENSE* "$1"/WHEEL "$1"/entry_points.txt' _ {} \; 2>/dev/null || true
|
||||
|
||||
# ==================== Runtime 阶段 ====================
|
||||
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim AS runtime
|
||||
# 从预构建的runtime基础镜像开始,已经包含:
|
||||
# - ffmpeg
|
||||
# - libglib2.0-0
|
||||
FROM xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/worker-base-runtime:latest AS runtime
|
||||
|
||||
# 构建参数:版本号
|
||||
ARG APP_VERSION=dev
|
||||
|
||||
# 使用阿里云镜像加速
|
||||
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
|
||||
|
||||
# 安装最小运行时依赖(opencv-python-headless 需要 libglib2.0-0)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
libglib2.0-0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 从 builder 复制 Python 虚拟环境
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
# 数据库(基础层)
|
||||
psycopg2-binary==2.9.10
|
||||
psycopg[binary]>=3.2.2
|
||||
psycopg[binary]==3.2.2
|
||||
sqlalchemy==2.0.35
|
||||
alembic==1.13.3
|
||||
|
||||
|
||||
@@ -11,4 +11,5 @@ pytest==8.3.3
|
||||
pytest-asyncio==0.24.0
|
||||
pytest-cov==6.0.0
|
||||
pytest-timeout==2.3.1
|
||||
diff-cover>=8.0
|
||||
pytest-xdist==3.6.1
|
||||
diff-cover==8.0.3
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
# 这些包体积大,API 服务不需要安装
|
||||
|
||||
# 数值计算
|
||||
numpy>=1.24.0
|
||||
numpy==1.26.4
|
||||
|
||||
# 科学计算
|
||||
scipy>=1.10.0
|
||||
scipy==1.13.1
|
||||
|
||||
# 计算机视觉(视频去重、帧处理)
|
||||
opencv-python-headless>=4.8.0
|
||||
opencv-python-headless==4.10.0.84
|
||||
|
||||
# 图像处理
|
||||
Pillow==10.4.0
|
||||
|
||||
+15
-13
@@ -31,20 +31,22 @@ def main():
|
||||
print("pending")
|
||||
return
|
||||
|
||||
# API返回按时间倒序,第一个就是最新的
|
||||
for s in statuses:
|
||||
if s.get("context") == target_context:
|
||||
status = s.get("status", "pending")
|
||||
# skipped 视为通过(条件跳过的任务不需要等)
|
||||
if status == "skipped":
|
||||
print("success")
|
||||
else:
|
||||
print(status)
|
||||
return
|
||||
# 筛选目标context,按时间倒序取最新的
|
||||
matching = [s for s in statuses if s.get("context") == target_context]
|
||||
if not matching:
|
||||
# 找不到说明CI还没开始写状态,返回pending继续等待
|
||||
print("pending")
|
||||
return
|
||||
|
||||
# 找不到这个context说明CI还没开始写状态,返回pending继续等待
|
||||
# (如果workflow真的被跳过,它会有一条status为skipped的记录)
|
||||
print("pending")
|
||||
# Gitea statuses API按时间正序返回,必须取最新的一条
|
||||
latest = max(matching, key=lambda s: s.get("created_at", ""))
|
||||
status = latest.get("status", "pending")
|
||||
|
||||
# skipped 视为通过(条件跳过的任务不需要等)
|
||||
if status == "skipped":
|
||||
print("success")
|
||||
else:
|
||||
print(status)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -169,12 +169,34 @@ def main():
|
||||
|
||||
api_url = os.environ.get("GITHUB_API_URL", "")
|
||||
repo = os.environ.get("GITHUB_REPOSITORY", "")
|
||||
token = os.environ.get("GITHUB_TOKEN", "")
|
||||
token = os.environ.get("REVIEW_TOKEN", "") or os.environ.get("GITHUB_TOKEN", "")
|
||||
|
||||
# 获取PR作者信息,判断是人还是Agent提交的
|
||||
pr_info_url = f"{api_url}/repos/{repo}/pulls/{pr_number}"
|
||||
req_pr = urllib.request.Request(pr_info_url, headers={"Authorization": f"token {token}"})
|
||||
with urllib.request.urlopen(req_pr) as resp:
|
||||
pr_info = json.loads(resp.read())
|
||||
pr_author = pr_info.get("user", {}).get("login", "")
|
||||
print(f"PR作者: {pr_author}")
|
||||
|
||||
# 判断是否为Agent提交的PR
|
||||
# Agent账号:actions, auto-approve-bot 等bot用户
|
||||
# 人提交的PR(如xiaoxia):只诊断不自动修
|
||||
agent_authors = {"actions", "auto-approve-bot", "gitea-actions"}
|
||||
is_agent_pr = pr_author in agent_authors or "bot" in pr_author.lower()
|
||||
|
||||
if is_agent_pr:
|
||||
print(f"检测到Agent提交的PR(作者: {pr_author}),将自动修复并推送")
|
||||
fix_mode = "auto_fix_and_push"
|
||||
else:
|
||||
print(f"检测到人提交的PR(作者: {pr_author}),仅诊断不自动修改")
|
||||
print("(如需自动修复,请用Agent账号提交PR,或手动运行格式化脚本)")
|
||||
fix_mode = "diagnose_only"
|
||||
scan_mode = os.environ.get("SCAN_MODE", "full")
|
||||
changed_files_env = os.environ.get("CHANGED_FILES", "")
|
||||
|
||||
if not token:
|
||||
print("缺少GITHUB_TOKEN,无法推送修复", file=sys.stderr)
|
||||
print("缺少REVIEW_TOKEN或GITHUB_TOKEN,无法推送修复", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
repo_root = os.getcwd()
|
||||
@@ -231,6 +253,26 @@ def main():
|
||||
print("没有需要提交的格式改动")
|
||||
return
|
||||
|
||||
# 诊断模式:只报告问题,不修改不推送
|
||||
if fix_mode == "diagnose_only":
|
||||
print()
|
||||
print("=" * 50)
|
||||
print("📋 格式问题诊断报告(人提交的PR,仅诊断不自动修复)")
|
||||
print("=" * 50)
|
||||
print()
|
||||
print("以下文件存在格式问题,建议手动修复:")
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
print(f" {line}")
|
||||
print()
|
||||
print("修复方式:")
|
||||
print(" 后端(Python): 运行 black + isort")
|
||||
print(" 前端: 运行 prettier --write")
|
||||
print(" 或使用 scripts/agent-commit.sh 提交(自动格式化)")
|
||||
print()
|
||||
print("=" * 50)
|
||||
# 以非0状态码退出,让CI继续报失败(因为问题没修)
|
||||
sys.exit(1)
|
||||
|
||||
print()
|
||||
print("变更文件:")
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
@@ -238,7 +280,7 @@ def main():
|
||||
|
||||
# 提交修复
|
||||
run("git add -A")
|
||||
run('git commit -m "style: auto-format with black + isort + prettier [ci skip]"')
|
||||
run('git commit -m "style: auto-format with black + isort + prettier"')
|
||||
|
||||
# 推送(head_branch已从ensure_git_repo获取)
|
||||
print(f"\nPR来源分支: {head_branch}")
|
||||
|
||||
+420
-22
@@ -1,12 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
CI 可观测性看板 - 从 Gitea Actions API 拉取数据并生成 Markdown 日报
|
||||
|
||||
CI 可观测性看板 - 从 Gitea Actions API 拉取数据并生成 Markdown/HTML 日报
|
||||
用法:
|
||||
python3 scripts/ci/ci_dashboard.py --days 7
|
||||
python3 scripts/ci/ci_dashboard.py --days 30 --output ci_report.md
|
||||
python3 scripts/ci/ci_dashboard.py --workflow ci-cd.yml --days 7
|
||||
|
||||
python3 scripts/ci/ci_dashboard.py --days 7 --html --html-output dashboard.html
|
||||
环境变量:
|
||||
GITEA_URL Gitea 地址 (默认 https://git.xiaoxiajianji.com)
|
||||
GITEA_REPO 仓库 (默认 xiaoxia/xiaoxia-saas)
|
||||
@@ -177,18 +176,14 @@ def fetch_runs_in_range(ga, start_date, end_date, workflow_filter=None):
|
||||
all_runs = []
|
||||
page = 1
|
||||
print(f"[INFO] 拉取 {start_date} ~ {end_date} 的 CI runs...", file=sys.stderr)
|
||||
|
||||
while True:
|
||||
runs, total = ga.list_runs(status="completed", page=page, limit=PAGE_LIMIT)
|
||||
if not runs:
|
||||
break
|
||||
|
||||
if workflow_filter:
|
||||
runs = [r for r in runs if workflow_filter in r.get("path", "")]
|
||||
|
||||
in_range = []
|
||||
out_range_old = False
|
||||
|
||||
for run in runs:
|
||||
started = to_shanghai(parse_datetime(run.get("started_at")))
|
||||
if not started:
|
||||
@@ -198,27 +193,22 @@ def fetch_runs_in_range(ga, start_date, end_date, workflow_filter=None):
|
||||
in_range.append(run)
|
||||
elif run_date < start_date:
|
||||
out_range_old = True
|
||||
|
||||
all_runs.extend(in_range)
|
||||
print(
|
||||
f"[INFO] 第 {page} 页: {len(runs)} 条, 范围内 {len(in_range)} 条, 累计 {len(all_runs)} 条", file=sys.stderr
|
||||
)
|
||||
|
||||
if out_range_old or len(runs) < PAGE_LIMIT:
|
||||
break
|
||||
|
||||
page += 1
|
||||
if page > 100:
|
||||
print("[WARN] 超过100页,停止拉取", file=sys.stderr)
|
||||
break
|
||||
|
||||
print(f"[INFO] 共获取 {len(all_runs)} 条 run 数据", file=sys.stderr)
|
||||
return all_runs
|
||||
|
||||
|
||||
def enrich_with_jobs(ga, runs, max_failures=50):
|
||||
"""为 runs 补充 job 详情(失败原因分析 + runner 统计)
|
||||
|
||||
失败 run 按时间倒序取最近 N 个(避免 API 调用过多),
|
||||
成功 run 采样用于 runner 分布统计。
|
||||
"""
|
||||
@@ -226,13 +216,11 @@ def enrich_with_jobs(ga, runs, max_failures=50):
|
||||
failure_runs = [r for r in runs if r.get("conclusion") != "success"]
|
||||
failure_runs = failure_runs[:max_failures] # 已经是时间倒序
|
||||
print(f"[INFO] 为最近 {len(failure_runs)} 个失败 run 拉取 job 详情...", file=sys.stderr)
|
||||
|
||||
for i, run in enumerate(failure_runs):
|
||||
jobs = ga.get_run_jobs(run["id"])
|
||||
run["_jobs"] = jobs
|
||||
if (i + 1) % 10 == 0:
|
||||
print(f"[INFO] 已处理 {i+1}/{len(failure_runs)}", file=sys.stderr)
|
||||
|
||||
# 成功 run 采样用于 runner 分布
|
||||
success_runs = [r for r in runs if r.get("conclusion") == "success"]
|
||||
sample_size = min(50, len(success_runs))
|
||||
@@ -243,7 +231,6 @@ def enrich_with_jobs(ga, runs, max_failures=50):
|
||||
if "_jobs" not in run:
|
||||
jobs = ga.get_run_jobs(run["id"])
|
||||
run["_jobs"] = jobs
|
||||
|
||||
return runs
|
||||
|
||||
|
||||
@@ -268,7 +255,6 @@ def analyze_runs(runs):
|
||||
if d and d > 0:
|
||||
durations.append(d)
|
||||
durations.sort()
|
||||
|
||||
avg_dur = statistics.mean(durations) if durations else None
|
||||
median_dur = percentile(durations, 50)
|
||||
p95_dur = percentile(durations, 95)
|
||||
@@ -316,8 +302,31 @@ def analyze_runs(runs):
|
||||
# 失败原因 + runner + job 耗时(需要 _jobs 数据)
|
||||
failure_categories = defaultdict(int)
|
||||
failed_jobs_by_name = defaultdict(int)
|
||||
job_success_stats = defaultdict(lambda: {"total": 0, "success": 0, "failure": 0})
|
||||
runner_stats = defaultdict(lambda: {"jobs": 0, "success": 0, "failure": 0, "durations": []})
|
||||
job_time_stats = defaultdict(list)
|
||||
infra_failures = 0
|
||||
business_failures = 0
|
||||
other_failures_count = 0
|
||||
|
||||
# 基础设施关键词(与 ci_health_check.py 保持一致的分类逻辑)
|
||||
infra_job_keywords = ["checkout", "build", "deploy", "cleanup", "setup", "cache", "install", "docker"]
|
||||
business_job_keywords = [
|
||||
"unit test",
|
||||
"pytest",
|
||||
"vitest",
|
||||
"jest",
|
||||
"lint",
|
||||
"eslint",
|
||||
"prettier",
|
||||
"integration",
|
||||
"e2e",
|
||||
"validate",
|
||||
"code quality",
|
||||
"mypy",
|
||||
"ruff",
|
||||
"flake8",
|
||||
]
|
||||
|
||||
for r in runs:
|
||||
jobs = r.get("_jobs", [])
|
||||
@@ -326,27 +335,45 @@ def analyze_runs(runs):
|
||||
for job in jobs:
|
||||
runner = job.get("runner_name", "unknown")
|
||||
conclusion = job.get("conclusion", "unknown")
|
||||
job_name = job.get("name", "unknown")
|
||||
job_name_lower = job_name.lower()
|
||||
|
||||
runner_stats[runner]["jobs"] += 1
|
||||
job_success_stats[job_name]["total"] += 1
|
||||
if conclusion == "success":
|
||||
runner_stats[runner]["success"] += 1
|
||||
job_success_stats[job_name]["success"] += 1
|
||||
elif conclusion == "failure":
|
||||
runner_stats[runner]["failure"] += 1
|
||||
job_success_stats[job_name]["failure"] += 1
|
||||
|
||||
jd = duration_seconds(job.get("started_at"), job.get("completed_at"))
|
||||
if jd and jd > 0:
|
||||
runner_stats[runner]["durations"].append(jd)
|
||||
job_time_stats[job.get("name", "unknown")].append(jd)
|
||||
job_time_stats[job_name].append(jd)
|
||||
|
||||
if conclusion == "failure":
|
||||
failed_jobs_by_name[job.get("name", "unknown")] += 1
|
||||
failed_jobs_by_name[job_name] += 1
|
||||
failed_step = None
|
||||
for step in job.get("steps", []):
|
||||
if step.get("conclusion") == "failure":
|
||||
failed_step = step.get("name")
|
||||
break
|
||||
category = classify_failure(job.get("name", ""), failed_step)
|
||||
category = classify_failure(job_name, failed_step)
|
||||
failure_categories[category] += 1
|
||||
|
||||
# 基础设施 vs 业务代码分类
|
||||
is_infra = any(k in job_name_lower for k in infra_job_keywords) and not any(
|
||||
k in job_name_lower for k in business_job_keywords
|
||||
)
|
||||
is_business = any(k in job_name_lower for k in business_job_keywords)
|
||||
if is_infra:
|
||||
infra_failures += 1
|
||||
elif is_business:
|
||||
business_failures += 1
|
||||
else:
|
||||
other_failures_count += 1
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"success": success,
|
||||
@@ -365,14 +392,17 @@ def analyze_runs(runs):
|
||||
"failed_jobs_top": dict(sorted(failed_jobs_by_name.items(), key=lambda x: -x[1])[:15]),
|
||||
"runner_stats": dict(runner_stats),
|
||||
"job_time_stats": dict(job_time_stats),
|
||||
"job_success_stats": dict(job_success_stats),
|
||||
"infra_failures": infra_failures,
|
||||
"business_failures": business_failures,
|
||||
"other_failures_combined": other_failures_count,
|
||||
}
|
||||
|
||||
|
||||
# ── 报表生成 ─────────────────────────────────────────
|
||||
# ── Markdown 报表生成 ────────────────────────────────
|
||||
def generate_markdown(stats, start_date, end_date, repo):
|
||||
"""生成 Markdown 格式的日报"""
|
||||
lines = []
|
||||
|
||||
lines.append("# CI 运行状态看板")
|
||||
lines.append("")
|
||||
lines.append(f"> 统计周期: **{start_date} ~ {end_date}**")
|
||||
@@ -526,6 +556,359 @@ def generate_markdown(stats, start_date, end_date, repo):
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ── HTML 看板生成 ────────────────────────────────────
|
||||
def generate_html(stats, start_date, end_date, repo):
|
||||
"""生成 HTML 格式的可视化看板(内嵌 ECharts)"""
|
||||
# 准备图表数据
|
||||
|
||||
# 1. 每日成功率趋势
|
||||
daily_dates = list(stats["daily_stats"].keys())
|
||||
daily_success_rates = []
|
||||
daily_run_counts = []
|
||||
for day in daily_dates:
|
||||
s = stats["daily_stats"][day]
|
||||
rate = (s["success"] / s["total"] * 100) if s["total"] > 0 else 0
|
||||
daily_success_rates.append(round(rate, 1))
|
||||
daily_run_counts.append(s["total"])
|
||||
|
||||
# 2. 各 Workflow 耗时对比
|
||||
wf_sorted = sorted(stats["workflow_stats"].items(), key=lambda x: -x[1]["total"])
|
||||
wf_names = []
|
||||
wf_avg_durations = []
|
||||
for wf, s in wf_sorted:
|
||||
wf_short = wf.split("/")[-1] if "/" in wf else wf
|
||||
wf_names.append(wf_short)
|
||||
avg = statistics.mean(s["durations"]) if s["durations"] else 0
|
||||
wf_avg_durations.append(round(avg / 60, 1)) # 转为分钟
|
||||
|
||||
# 3. 失败原因分布(饼图数据 - 基础设施 vs 业务 vs 其他)
|
||||
total_infra_biz = stats["infra_failures"] + stats["business_failures"] + stats["other_failures_combined"]
|
||||
infra_rate = (stats["infra_failures"] / total_infra_biz * 100) if total_infra_biz > 0 else 0
|
||||
failure_pie_data = [
|
||||
{"value": stats["infra_failures"], "name": "基础设施问题"},
|
||||
{"value": stats["business_failures"], "name": "业务代码问题"},
|
||||
{"value": stats["other_failures_combined"], "name": "其他"},
|
||||
]
|
||||
|
||||
# 4. 各 Job 成功率排行(横向柱状图,取成功率最低的 Top 15)
|
||||
job_stats_list = []
|
||||
for name, s in stats["job_success_stats"].items():
|
||||
if s["total"] >= 3: # 至少有3次才统计
|
||||
rate = (s["success"] / s["total"] * 100) if s["total"] > 0 else 0
|
||||
job_stats_list.append(
|
||||
{
|
||||
"name": name,
|
||||
"rate": round(rate, 1),
|
||||
"total": s["total"],
|
||||
"success": s["success"],
|
||||
}
|
||||
)
|
||||
job_stats_list.sort(key=lambda x: x["rate"])
|
||||
job_stats_list = job_stats_list[:15] # 取成功率最低的15个
|
||||
job_names = [j["name"] for j in job_stats_list]
|
||||
job_rates = [j["rate"] for j in job_stats_list]
|
||||
|
||||
# 核心指标
|
||||
total_runs = stats["total"]
|
||||
success_rate = round(stats["success_rate"], 1)
|
||||
avg_dur_min = round(stats["avg_duration"] / 60, 1) if stats["avg_duration"] else 0
|
||||
infra_fail_rate = round(infra_rate, 1)
|
||||
|
||||
now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
# 序列化数据为 JSON(供 JS 使用)
|
||||
data_json = json.dumps(
|
||||
{
|
||||
"daily_dates": daily_dates,
|
||||
"daily_success_rates": daily_success_rates,
|
||||
"daily_run_counts": daily_run_counts,
|
||||
"wf_names": wf_names,
|
||||
"wf_avg_durations": wf_avg_durations,
|
||||
"failure_pie_data": failure_pie_data,
|
||||
"job_names": job_names,
|
||||
"job_rates": job_rates,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
# HTML 模板(注意:不使用 f-string,避免与 CSS/JS 的大括号冲突)
|
||||
html_parts = []
|
||||
html_parts.append("<!DOCTYPE html>")
|
||||
html_parts.append('<html lang="zh-CN">')
|
||||
html_parts.append("<head>")
|
||||
html_parts.append(' <meta charset="UTF-8">')
|
||||
html_parts.append(' <meta name="viewport" content="width=device-width, initial-scale=1.0">')
|
||||
html_parts.append(f" <title>CI 健康度看板 - {repo}</title>")
|
||||
html_parts.append(' <script src="https://cdn.jsdelivr.net/npm/echarts/dist/echarts.min.js"></script>')
|
||||
html_parts.append(" <style>")
|
||||
html_parts.append(" * { margin: 0; padding: 0; box-sizing: border-box; }")
|
||||
html_parts.append(" body {")
|
||||
html_parts.append(
|
||||
' font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "PingFang SC", "Hiragino Sans GB",'
|
||||
)
|
||||
html_parts.append(' "Microsoft YaHei", sans-serif;')
|
||||
html_parts.append(" background: #f0f2f5;")
|
||||
html_parts.append(" color: #333;")
|
||||
html_parts.append(" padding: 20px;")
|
||||
html_parts.append(" }")
|
||||
html_parts.append(" .container { max-width: 1400px; margin: 0 auto; }")
|
||||
html_parts.append(" .header {")
|
||||
html_parts.append(" background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);")
|
||||
html_parts.append(" color: white;")
|
||||
html_parts.append(" padding: 24px 32px;")
|
||||
html_parts.append(" border-radius: 12px;")
|
||||
html_parts.append(" margin-bottom: 20px;")
|
||||
html_parts.append(" }")
|
||||
html_parts.append(" .header h1 { font-size: 24px; margin-bottom: 8px; }")
|
||||
html_parts.append(" .header .subtitle { font-size: 14px; opacity: 0.9; }")
|
||||
html_parts.append(" .header .meta { font-size: 12px; opacity: 0.8; margin-top: 8px; }")
|
||||
html_parts.append(" .metrics-row {")
|
||||
html_parts.append(" display: grid;")
|
||||
html_parts.append(" grid-template-columns: repeat(4, 1fr);")
|
||||
html_parts.append(" gap: 16px;")
|
||||
html_parts.append(" margin-bottom: 20px;")
|
||||
html_parts.append(" }")
|
||||
html_parts.append(" .metric-card {")
|
||||
html_parts.append(" background: white;")
|
||||
html_parts.append(" border-radius: 12px;")
|
||||
html_parts.append(" padding: 20px;")
|
||||
html_parts.append(" box-shadow: 0 2px 8px rgba(0,0,0,0.06);")
|
||||
html_parts.append(" transition: transform 0.2s;")
|
||||
html_parts.append(" }")
|
||||
html_parts.append(
|
||||
" .metric-card:hover { transform: translateY(-2px); box-shadow: 0 4px 12px rgba(0,0,0,0.1); }"
|
||||
)
|
||||
html_parts.append(" .metric-card .label { font-size: 13px; color: #8c8c8c; margin-bottom: 8px; }")
|
||||
html_parts.append(" .metric-card .value { font-size: 28px; font-weight: 600; }")
|
||||
html_parts.append(" .metric-card .unit { font-size: 14px; color: #8c8c8c; margin-left: 4px; }")
|
||||
html_parts.append(" .metric-card.success .value { color: #52c41a; }")
|
||||
html_parts.append(" .metric-card.warning .value { color: #faad14; }")
|
||||
html_parts.append(" .metric-card.danger .value { color: #ff4d4f; }")
|
||||
html_parts.append(" .metric-card.info .value { color: #1890ff; }")
|
||||
html_parts.append(" .charts-grid {")
|
||||
html_parts.append(" display: grid;")
|
||||
html_parts.append(" grid-template-columns: 1fr 1fr;")
|
||||
html_parts.append(" gap: 16px;")
|
||||
html_parts.append(" margin-bottom: 20px;")
|
||||
html_parts.append(" }")
|
||||
html_parts.append(" .chart-card {")
|
||||
html_parts.append(" background: white;")
|
||||
html_parts.append(" border-radius: 12px;")
|
||||
html_parts.append(" padding: 20px;")
|
||||
html_parts.append(" box-shadow: 0 2px 8px rgba(0,0,0,0.06);")
|
||||
html_parts.append(" }")
|
||||
html_parts.append(" .chart-card.full-width { grid-column: 1 / -1; }")
|
||||
html_parts.append(" .chart-card h3 {")
|
||||
html_parts.append(" font-size: 16px;")
|
||||
html_parts.append(" margin-bottom: 12px;")
|
||||
html_parts.append(" color: #262626;")
|
||||
html_parts.append(" font-weight: 600;")
|
||||
html_parts.append(" }")
|
||||
html_parts.append(" .chart-container { width: 100%; height: 320px; }")
|
||||
html_parts.append(" .chart-container.tall { height: 400px; }")
|
||||
html_parts.append(" .footer {")
|
||||
html_parts.append(" text-align: center;")
|
||||
html_parts.append(" color: #8c8c8c;")
|
||||
html_parts.append(" font-size: 12px;")
|
||||
html_parts.append(" padding: 16px 0;")
|
||||
html_parts.append(" }")
|
||||
html_parts.append(" @media (max-width: 900px) {")
|
||||
html_parts.append(" .metrics-row { grid-template-columns: repeat(2, 1fr); }")
|
||||
html_parts.append(" .charts-grid { grid-template-columns: 1fr; }")
|
||||
html_parts.append(" }")
|
||||
html_parts.append(" @media (max-width: 600px) {")
|
||||
html_parts.append(" .metrics-row { grid-template-columns: 1fr; }")
|
||||
html_parts.append(" body { padding: 12px; }")
|
||||
html_parts.append(" }")
|
||||
html_parts.append(" </style>")
|
||||
html_parts.append("</head>")
|
||||
html_parts.append("<body>")
|
||||
html_parts.append(' <div class="container">')
|
||||
html_parts.append(' <div class="header">')
|
||||
html_parts.append(" <h1>📊 CI 健康度看板</h1>")
|
||||
html_parts.append(f' <div class="subtitle">仓库: {repo}</div>')
|
||||
html_parts.append(f' <div class="meta">统计周期: {start_date} ~ {end_date} | 生成时间: {now_str}</div>')
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(' <div class="metrics-row">')
|
||||
html_parts.append(' <div class="metric-card success">')
|
||||
html_parts.append(' <div class="label">总成功率</div>')
|
||||
html_parts.append(f' <div class="value">{success_rate}<span class="unit">%</span></div>')
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(' <div class="metric-card info">')
|
||||
html_parts.append(' <div class="label">总 Run 数</div>')
|
||||
html_parts.append(f' <div class="value">{total_runs}<span class="unit">次</span></div>')
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(' <div class="metric-card warning">')
|
||||
html_parts.append(' <div class="label">平均耗时</div>')
|
||||
html_parts.append(f' <div class="value">{avg_dur_min}<span class="unit">分钟</span></div>')
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(' <div class="metric-card danger">')
|
||||
html_parts.append(' <div class="label">基础设施故障率</div>')
|
||||
html_parts.append(f' <div class="value">{infra_fail_rate}<span class="unit">%</span></div>')
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(' <div class="charts-grid">')
|
||||
html_parts.append(' <div class="chart-card full-width">')
|
||||
html_parts.append(" <h3>📈 CI 成功率趋势</h3>")
|
||||
html_parts.append(' <div id="chart-success-rate" class="chart-container"></div>')
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(' <div class="charts-grid">')
|
||||
html_parts.append(' <div class="chart-card">')
|
||||
html_parts.append(" <h3>⏱️ 各 Workflow 平均耗时</h3>")
|
||||
html_parts.append(' <div id="chart-wf-duration" class="chart-container"></div>')
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(' <div class="chart-card">')
|
||||
html_parts.append(" <h3>❌ 失败原因分布</h3>")
|
||||
html_parts.append(' <div id="chart-failure-pie" class="chart-container"></div>')
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(' <div class="charts-grid">')
|
||||
html_parts.append(' <div class="chart-card full-width">')
|
||||
html_parts.append(" <h3>📋 各 Job 成功率排行(最低 15 名)</h3>")
|
||||
html_parts.append(' <div id="chart-job-success" class="chart-container tall"></div>')
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(' <div class="charts-grid">')
|
||||
html_parts.append(' <div class="chart-card full-width">')
|
||||
html_parts.append(" <h3>📊 每日 Run 数量趋势</h3>")
|
||||
html_parts.append(' <div id="chart-run-count" class="chart-container"></div>')
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(' <div class="footer">')
|
||||
html_parts.append(" 由 ci_dashboard.py 自动生成 | ECharts 可视化")
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(" </div>")
|
||||
html_parts.append(" <script>")
|
||||
html_parts.append(f" const DATA = {data_json};")
|
||||
html_parts.append("")
|
||||
# 图表 1: 成功率趋势
|
||||
html_parts.append(" (function() {")
|
||||
html_parts.append(' const chart = echarts.init(document.getElementById("chart-success-rate"));')
|
||||
html_parts.append(" chart.setOption({")
|
||||
html_parts.append(' tooltip: { trigger: "axis", formatter: function(params) {')
|
||||
html_parts.append(' const p = params[0]; return p.name + "<br/>成功率: <b>" + p.value + "%</b>";')
|
||||
html_parts.append(" }},")
|
||||
html_parts.append(' grid: { left: "3%", right: "4%", bottom: "3%", containLabel: true },')
|
||||
html_parts.append(' xAxis: { type: "category", boundaryGap: false, data: DATA.daily_dates,')
|
||||
html_parts.append(" axisLabel: { rotate: 30, fontSize: 11 } },")
|
||||
html_parts.append(' yAxis: { type: "value", min: 0, max: 100, axisLabel: { formatter: "{value}%" } },')
|
||||
html_parts.append(
|
||||
' series: [{ name: "成功率", type: "line", smooth: true, data: DATA.daily_success_rates,'
|
||||
)
|
||||
html_parts.append(' itemStyle: { color: "#52c41a" },')
|
||||
html_parts.append(" areaStyle: { color: new echarts.graphic.LinearGradient(0, 0, 0, 1, [")
|
||||
html_parts.append(' { offset: 0, color: "rgba(82, 196, 26, 0.3)" },')
|
||||
html_parts.append(' { offset: 1, color: "rgba(82, 196, 26, 0.05)" }')
|
||||
html_parts.append(" ])},")
|
||||
html_parts.append(' markLine: { silent: true, data: [{ type: "average", name: "平均值",')
|
||||
html_parts.append(' label: { formatter: "均值 {c}%" } }] }')
|
||||
html_parts.append(" }]")
|
||||
html_parts.append(" });")
|
||||
html_parts.append(' window.addEventListener("resize", () => chart.resize());')
|
||||
html_parts.append(" })();")
|
||||
html_parts.append("")
|
||||
# 图表 2: Workflow 耗时对比
|
||||
html_parts.append(" (function() {")
|
||||
html_parts.append(' const chart = echarts.init(document.getElementById("chart-wf-duration"));')
|
||||
html_parts.append(" chart.setOption({")
|
||||
html_parts.append(' tooltip: { trigger: "axis", formatter: function(params) {')
|
||||
html_parts.append(
|
||||
' const p = params[0]; return p.name + "<br/>平均耗时: <b>" + p.value + " 分钟</b>";'
|
||||
)
|
||||
html_parts.append(" }},")
|
||||
html_parts.append(' grid: { left: "3%", right: "4%", bottom: "15%", containLabel: true },')
|
||||
html_parts.append(' xAxis: { type: "category", data: DATA.wf_names,')
|
||||
html_parts.append(" axisLabel: { rotate: 30, fontSize: 10, interval: 0 } },")
|
||||
html_parts.append(' yAxis: { type: "value", name: "分钟", axisLabel: { formatter: "{value} min" } },')
|
||||
html_parts.append(' series: [{ name: "平均耗时", type: "bar", data: DATA.wf_avg_durations,')
|
||||
html_parts.append(" itemStyle: { color: new echarts.graphic.LinearGradient(0, 0, 0, 1, [")
|
||||
html_parts.append(' { offset: 0, color: "#1890ff" },')
|
||||
html_parts.append(' { offset: 1, color: "#096dd9" }')
|
||||
html_parts.append(" ]), borderRadius: [4, 4, 0, 0] },")
|
||||
html_parts.append(" barMaxWidth: 40")
|
||||
html_parts.append(" }]")
|
||||
html_parts.append(" });")
|
||||
html_parts.append(' window.addEventListener("resize", () => chart.resize());')
|
||||
html_parts.append(" })();")
|
||||
html_parts.append("")
|
||||
# 图表 3: 失败原因饼图
|
||||
html_parts.append(" (function() {")
|
||||
html_parts.append(' const chart = echarts.init(document.getElementById("chart-failure-pie"));')
|
||||
html_parts.append(" chart.setOption({")
|
||||
html_parts.append(' tooltip: { trigger: "item", formatter: "{b}: {c} 次 ({d}%)" },')
|
||||
html_parts.append(' legend: { orient: "vertical", right: "5%", top: "center" },')
|
||||
html_parts.append(
|
||||
' series: [{ name: "失败原因", type: "pie", radius: ["40%", "70%"], center: ["35%", "50%"],'
|
||||
)
|
||||
html_parts.append(" avoidLabelOverlap: false,")
|
||||
html_parts.append(' itemStyle: { borderRadius: 6, borderColor: "#fff", borderWidth: 2 },')
|
||||
html_parts.append(' label: { show: false, position: "center" },')
|
||||
html_parts.append(' emphasis: { label: { show: true, fontSize: 16, fontWeight: "bold" } },')
|
||||
html_parts.append(" labelLine: { show: false },")
|
||||
html_parts.append(" data: DATA.failure_pie_data,")
|
||||
html_parts.append(' color: ["#ff4d4f", "#faad14", "#8c8c8c"]')
|
||||
html_parts.append(" }]")
|
||||
html_parts.append(" });")
|
||||
html_parts.append(' window.addEventListener("resize", () => chart.resize());')
|
||||
html_parts.append(" })();")
|
||||
html_parts.append("")
|
||||
# 图表 4: Job 成功率排行(横向柱状图)
|
||||
html_parts.append(" (function() {")
|
||||
html_parts.append(' const chart = echarts.init(document.getElementById("chart-job-success"));')
|
||||
html_parts.append(" const barData = DATA.job_rates.map(function(rate, i) {")
|
||||
html_parts.append(" return { value: rate, itemStyle: {")
|
||||
html_parts.append(' color: rate >= 90 ? "#52c41a" : (rate >= 70 ? "#faad14" : "#ff4d4f")')
|
||||
html_parts.append(" }};")
|
||||
html_parts.append(" });")
|
||||
html_parts.append(" chart.setOption({")
|
||||
html_parts.append(' tooltip: { trigger: "axis", formatter: function(params) {')
|
||||
html_parts.append(' const p = params[0]; return p.name + "<br/>成功率: <b>" + p.value + "%</b>";')
|
||||
html_parts.append(" }},")
|
||||
html_parts.append(' grid: { left: "3%", right: "8%", bottom: "3%", top: "3%", containLabel: true },')
|
||||
html_parts.append(' xAxis: { type: "value", min: 0, max: 100, axisLabel: { formatter: "{value}%" } },')
|
||||
html_parts.append(' yAxis: { type: "category", data: DATA.job_names, axisLabel: { fontSize: 11 } },')
|
||||
html_parts.append(' series: [{ name: "成功率", type: "bar", data: barData, barWidth: "60%",')
|
||||
html_parts.append(' label: { show: true, position: "right", formatter: "{c}%", fontSize: 11 }')
|
||||
html_parts.append(" }]")
|
||||
html_parts.append(" });")
|
||||
html_parts.append(' window.addEventListener("resize", () => chart.resize());')
|
||||
html_parts.append(" })();")
|
||||
html_parts.append("")
|
||||
# 图表 5: 每日 Run 数量趋势(面积图)
|
||||
html_parts.append(" (function() {")
|
||||
html_parts.append(' const chart = echarts.init(document.getElementById("chart-run-count"));')
|
||||
html_parts.append(" chart.setOption({")
|
||||
html_parts.append(' tooltip: { trigger: "axis", formatter: function(params) {')
|
||||
html_parts.append(
|
||||
' const p = params[0]; return p.name + "<br/>Run 数量: <b>" + p.value + " 次</b>";'
|
||||
)
|
||||
html_parts.append(" }},")
|
||||
html_parts.append(' grid: { left: "3%", right: "4%", bottom: "3%", containLabel: true },')
|
||||
html_parts.append(' xAxis: { type: "category", boundaryGap: false, data: DATA.daily_dates,')
|
||||
html_parts.append(" axisLabel: { rotate: 30, fontSize: 11 } },")
|
||||
html_parts.append(' yAxis: { type: "value", name: "次数" },')
|
||||
html_parts.append(
|
||||
' series: [{ name: "Run 数量", type: "line", smooth: true, data: DATA.daily_run_counts,'
|
||||
)
|
||||
html_parts.append(' itemStyle: { color: "#722ed1" },')
|
||||
html_parts.append(" areaStyle: { color: new echarts.graphic.LinearGradient(0, 0, 0, 1, [")
|
||||
html_parts.append(' { offset: 0, color: "rgba(114, 46, 209, 0.3)" },')
|
||||
html_parts.append(' { offset: 1, color: "rgba(114, 46, 209, 0.05)" }')
|
||||
html_parts.append(" ])},")
|
||||
html_parts.append(' markLine: { silent: true, data: [{ type: "average", name: "平均值",')
|
||||
html_parts.append(' label: { formatter: "均值 {c} 次" } }] }')
|
||||
html_parts.append(" }]")
|
||||
html_parts.append(" });")
|
||||
html_parts.append(' window.addEventListener("resize", () => chart.resize());')
|
||||
html_parts.append(" })();")
|
||||
html_parts.append(" </script>")
|
||||
html_parts.append("</body>")
|
||||
html_parts.append("</html>")
|
||||
|
||||
return "\n".join(html_parts)
|
||||
|
||||
|
||||
# ── 主函数 ───────────────────────────────────────────
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="CI 可观测性看板 - 生成 Gitea Actions 运行状态报表")
|
||||
@@ -539,6 +922,11 @@ def main():
|
||||
parser.add_argument("--password", type=str, default=os.environ.get("GITEA_PASSWORD"))
|
||||
parser.add_argument("--no-job-detail", action="store_true", help="不拉取 job 详情")
|
||||
parser.add_argument("--max-failures", type=int, default=50, help="最多分析多少个失败 run 的 job 详情 (默认 50)")
|
||||
|
||||
# HTML 输出相关参数
|
||||
parser.add_argument("--html", action="store_true", help="生成 HTML 可视化看板")
|
||||
parser.add_argument("--html-output", type=str, help="HTML 输出文件路径 (默认 ci_dashboard.html)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
ga = GiteaActions(
|
||||
@@ -561,8 +949,18 @@ def main():
|
||||
runs = enrich_with_jobs(ga, runs, max_failures=args.max_failures)
|
||||
|
||||
stats = analyze_runs(runs)
|
||||
md = generate_markdown(stats, start_date, end_date, args.repo)
|
||||
|
||||
# HTML 模式
|
||||
if args.html:
|
||||
html = generate_html(stats, start_date, end_date, args.repo)
|
||||
html_output = args.html_output or args.output or "ci_dashboard.html"
|
||||
with open(html_output, "w", encoding="utf-8") as f:
|
||||
f.write(html)
|
||||
print(f"[INFO] HTML 看板已保存到 {html_output}", file=sys.stderr)
|
||||
return
|
||||
|
||||
# 默认 Markdown 模式(向后兼容)
|
||||
md = generate_markdown(stats, start_date, end_date, args.repo)
|
||||
if args.output:
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
f.write(md)
|
||||
|
||||
@@ -0,0 +1,447 @@
|
||||
#!/usr/bin/env python3
|
||||
"""CI失败诊断增强脚本:自动分类失败原因 + 提取关键错误 + 给出修复建议。
|
||||
# Trigger CI after auto-format fix
|
||||
|
||||
支持的失败类型:
|
||||
1. Lint/格式问题 (ruff/black/eslint/prettier)
|
||||
2. 单元测试失败
|
||||
3. Docker构建失败
|
||||
4. 依赖安装失败 (pip/npm)
|
||||
5. 超时
|
||||
6. 缓存问题
|
||||
7. 数据库/迁移问题
|
||||
8. 网络问题
|
||||
9. 其他
|
||||
|
||||
用法:
|
||||
python3 scripts/ci/ci_failure_diagnosis.py [--job-name "Job Name"] [--log-file /path/to/log]
|
||||
|
||||
如果不传--log-file,会尝试从Gitea API获取失败job的日志。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class FailureDiagnosis:
|
||||
"""失败诊断结果"""
|
||||
|
||||
category: str # 失败分类
|
||||
category_cn: str # 中文分类名
|
||||
severity: str # 严重程度: high / medium / low
|
||||
summary: str # 一句话摘要
|
||||
error_lines: List[str] = field(default_factory=list) # 关键错误行
|
||||
suggestions: List[str] = field(default_factory=list) # 修复建议
|
||||
auto_fixable: bool = False # 是否可以自动修复
|
||||
related_docs: str = "" # 相关文档链接
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 失败模式定义
|
||||
# ============================================================
|
||||
|
||||
FAILURE_PATTERNS = [
|
||||
# ===== Lint / 格式问题 =====
|
||||
{
|
||||
"pattern": r"(ruff|black|isort)\b.*(error|failed|Error)",
|
||||
"category": "lint_python",
|
||||
"category_cn": "Python代码质量检查",
|
||||
"severity": "low",
|
||||
"summary_contains": ["ruff", "black", "isort"],
|
||||
"suggestions": [
|
||||
"本地运行 `black . && isort . && ruff check --fix .` 自动修复",
|
||||
"使用 `scripts/agent-commit.sh` 提交(自动格式化)",
|
||||
"如确认无误,可加 `# noqa: xxx` 忽略特定规则",
|
||||
],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
{
|
||||
"pattern": r"ESLint|prettier|eslint",
|
||||
"category": "lint_frontend",
|
||||
"category_cn": "前端代码检查",
|
||||
"severity": "low",
|
||||
"summary_contains": ["eslint", "prettier"],
|
||||
"suggestions": [
|
||||
"本地运行 `cd apps/web && npm run lint:fix` 自动修复",
|
||||
"Prettier问题: `cd apps/web && npx prettier --write .`",
|
||||
],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
{
|
||||
"pattern": r"F\d{3}|E\d{3}|W\d{3}.*ruff|ruff.*F\d{3}",
|
||||
"category": "lint_python",
|
||||
"category_cn": "Python代码质量检查",
|
||||
"severity": "low",
|
||||
"suggestions": [
|
||||
"F401: 删除未使用的import",
|
||||
"F841: 删除未使用的变量或加下划线前缀",
|
||||
"E501: 行超长,加 `# noqa: E501`",
|
||||
"F811: 删重复import",
|
||||
"运行 `ruff check --fix .` 自动修复大部分问题",
|
||||
],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
# ===== 单元测试失败 =====
|
||||
{
|
||||
"pattern": r"FAILED|assert.*Error|AssertionError",
|
||||
"category": "unit_test",
|
||||
"category_cn": "单元测试失败",
|
||||
"severity": "high",
|
||||
"suggestions": [
|
||||
"检查相关测试文件,确认是代码问题还是测试用例问题",
|
||||
"本地运行对应测试:`pytest path/to/test.py -v`",
|
||||
"如测试依赖外部服务,检查mock是否正确",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
{
|
||||
"pattern": r"pytest.*failed|\d+ failed.*\d+ passed",
|
||||
"category": "unit_test",
|
||||
"category_cn": "单元测试失败",
|
||||
"severity": "high",
|
||||
"suggestions": [
|
||||
"查看上方日志中的FAILED测试用例",
|
||||
"检查失败断言的期望值 vs 实际值",
|
||||
"新代码影响了现有测试行为,确认是预期内变更吗?",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
# ===== Docker 构建失败 =====
|
||||
{
|
||||
"pattern": r"Dockerfile.*not found|docker build.*failed|ERROR: failed to solve",
|
||||
"category": "docker_build",
|
||||
"category_cn": "Docker构建失败",
|
||||
"severity": "high",
|
||||
"suggestions": [
|
||||
"检查Dockerfile语法是否正确",
|
||||
"检查引用的基础镜像是否存在",
|
||||
"本地运行 `docker build -f path/to/Dockerfile .` 复现",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
{
|
||||
"pattern": r"manifest.*not found|no such image|image.*not found",
|
||||
"category": "docker_build",
|
||||
"category_cn": "镜像不存在",
|
||||
"severity": "medium",
|
||||
"suggestions": [
|
||||
"检查基础镜像名称和tag是否正确",
|
||||
"确认镜像仓库可访问,登录是否有效",
|
||||
"如为新基础镜像,需先手动构建一次基础镜像",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
{
|
||||
"pattern": r"ETXTBSY|text file busy",
|
||||
"category": "docker_build",
|
||||
"category_cn": "文件锁冲突(ETXTBSY)",
|
||||
"severity": "low",
|
||||
"summary": "esbuild并发构建冲突,重试即可",
|
||||
"suggestions": ["偶发问题,点击Rerun重新运行即可", "如频繁出现,检查是否有多个job并发写入同一文件"],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
# ===== 依赖安装失败 =====
|
||||
{
|
||||
"pattern": r"pip install.*error|Could not find a version|No matching distribution",
|
||||
"category": "dependency",
|
||||
"category_cn": "pip依赖安装失败",
|
||||
"severity": "medium",
|
||||
"suggestions": [
|
||||
"检查requirements.txt中的版本号是否正确",
|
||||
"如为新版本刚发布,可能源还没同步,稍后重试",
|
||||
"检查网络连接,可尝试切换pip镜像源",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
{
|
||||
"pattern": r"npm.*ERR|npm install.*failed|E404|ECONNREFUSED.*npm",
|
||||
"category": "dependency",
|
||||
"category_cn": "npm依赖安装失败",
|
||||
"severity": "medium",
|
||||
"suggestions": [
|
||||
"检查package.json中的版本号是否存在",
|
||||
"网络问题:检查npm registry是否可访问",
|
||||
"国内网络建议配置npmmirror镜像源",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
{
|
||||
"pattern": r"Connection refused|timed out|network.*unreachable",
|
||||
"category": "network",
|
||||
"category_cn": "网络问题",
|
||||
"severity": "medium",
|
||||
"summary": "网络连接失败,可能是源站问题或DNS问题",
|
||||
"suggestions": [
|
||||
"点击Rerun重试,网络问题通常是临时的",
|
||||
"如持续失败,检查对应服务是否正常",
|
||||
"检查Runner网络配置",
|
||||
],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
# ===== 超时 =====
|
||||
{
|
||||
"pattern": r"timeout|timed out|exceeded.*time limit|job.*cancelled.*timeout",
|
||||
"category": "timeout",
|
||||
"category_cn": "执行超时",
|
||||
"severity": "medium",
|
||||
"suggestions": [
|
||||
"如首次出现:重试一次,可能是临时性能波动",
|
||||
"频繁出现:检查构建是否变慢了,最近是否加了新依赖",
|
||||
"可适当增加timeout-minutes配置",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
# ===== 数据库/迁移 =====
|
||||
{
|
||||
"pattern": r"alembic.*error|migration.*failed|relation.*does not exist|column.*does not exist",
|
||||
"category": "migration",
|
||||
"category_cn": "数据库迁移失败",
|
||||
"severity": "high",
|
||||
"suggestions": [
|
||||
"检查迁移脚本是否正确,down_revision是否对",
|
||||
"确认数据库中是否有脏数据或残留表",
|
||||
"迁移脚本合并冲突时,重新生成迁移文件",
|
||||
],
|
||||
"auto_fixable": False,
|
||||
},
|
||||
# ===== 缓存问题 =====
|
||||
{
|
||||
"pattern": r"cache.*corrupt|cache.*invalid|snapshot.*not found|failed to compute cache key",
|
||||
"category": "cache",
|
||||
"category_cn": "缓存损坏",
|
||||
"severity": "low",
|
||||
"suggestions": ["构建系统会自动清理损坏缓存并重试,通常无需干预", "如持续失败,手动清理Runner上的缓存目录"],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
# ===== Checkout 失败 =====
|
||||
{
|
||||
"pattern": r"Could not resolve host|fatal:.*repository|SSL.*problem",
|
||||
"category": "checkout",
|
||||
"category_cn": "代码拉取失败",
|
||||
"severity": "low",
|
||||
"suggestions": ["临时网络问题,点击Rerun重试", "如持续失败,检查Gitea服务状态"],
|
||||
"auto_fixable": True,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def analyze_log(log_text: str, job_name: str = "") -> FailureDiagnosis:
|
||||
"""分析日志,返回诊断结果"""
|
||||
|
||||
lines = log_text.strip().split("\n")
|
||||
|
||||
# 收集所有匹配的模式
|
||||
matched = []
|
||||
error_lines = []
|
||||
|
||||
for line in lines:
|
||||
line_stripped = line.strip()
|
||||
# 收集ERROR/FAILED/Failed等错误行(最多20行)
|
||||
if re.search(r"(ERROR|FAILED|Error|error:|FAIL:|Traceback)", line_stripped):
|
||||
if len(error_lines) < 20:
|
||||
error_lines.append(line_stripped)
|
||||
|
||||
for pattern_info in FAILURE_PATTERNS:
|
||||
if re.search(pattern_info["pattern"], line_stripped, re.IGNORECASE):
|
||||
matched.append(pattern_info)
|
||||
break # 一行只匹配一个模式
|
||||
|
||||
if not matched:
|
||||
# 未识别的失败类型
|
||||
return FailureDiagnosis(
|
||||
category="unknown",
|
||||
category_cn="未知错误",
|
||||
severity="medium",
|
||||
summary="未识别的失败类型,需要人工查看日志",
|
||||
error_lines=error_lines[:10],
|
||||
suggestions=[
|
||||
"点击'查看失败日志'查看完整日志",
|
||||
"如为偶发问题,可先重试一次",
|
||||
"常见原因:环境问题、配置问题、新增逻辑引入的bug",
|
||||
],
|
||||
auto_fixable=False,
|
||||
)
|
||||
|
||||
# 选最严重、最具体的那个
|
||||
severity_order = {"high": 3, "medium": 2, "low": 1}
|
||||
matched.sort(key=lambda x: severity_order.get(x["severity"], 0), reverse=True)
|
||||
best_match = matched[0]
|
||||
|
||||
# 生成摘要
|
||||
if "summary" in best_match:
|
||||
summary = best_match["summary"]
|
||||
else:
|
||||
summary = f"{best_match['category_cn']}检查失败"
|
||||
if job_name:
|
||||
summary = f"[{job_name}] {summary}"
|
||||
|
||||
# 从error_lines中过滤出与该分类相关的
|
||||
relevant_errors = error_lines[:10]
|
||||
|
||||
return FailureDiagnosis(
|
||||
category=best_match["category"],
|
||||
category_cn=best_match["category_cn"],
|
||||
severity=best_match["severity"],
|
||||
summary=summary,
|
||||
error_lines=relevant_errors,
|
||||
suggestions=best_match["suggestions"],
|
||||
auto_fixable=best_match.get("auto_fixable", False),
|
||||
)
|
||||
|
||||
|
||||
def fetch_failed_job_log(run_id: str, job_id: str, token: str, repo: str) -> Optional[str]:
|
||||
"""从Gitea API获取失败job的日志"""
|
||||
api_base = f"https://git.xiaoxiajianji.com/api/v1/repos/{repo}"
|
||||
|
||||
# 尝试获取job的日志
|
||||
url = f"{api_base}/actions/runs/{run_id}/jobs/{job_id}/log"
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header("Authorization", f"token {token}")
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
print(f"获取日志失败: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def format_diagnosis_markdown(d: FailureDiagnosis, job_name: str = "", run_url: str = "") -> str:
|
||||
"""将诊断结果格式化为飞书卡片markdown"""
|
||||
|
||||
severity_emoji = {"high": "🔴", "medium": "🟡", "low": "🟢"}
|
||||
emoji = severity_emoji.get(d.severity, "⚪")
|
||||
|
||||
lines = []
|
||||
lines.append(f"**分类**: {emoji} {d.category_cn}")
|
||||
lines.append(f"**问题**: {d.summary}")
|
||||
|
||||
if d.error_lines:
|
||||
lines.append("")
|
||||
lines.append("**关键错误行**:")
|
||||
for err in d.error_lines[:5]:
|
||||
# 截断过长的行
|
||||
if len(err) > 150:
|
||||
err = err[:147] + "..."
|
||||
lines.append(f" `{err}`")
|
||||
|
||||
lines.append("")
|
||||
lines.append("**修复建议**:")
|
||||
for i, s in enumerate(d.suggestions[:5], 1):
|
||||
lines.append(f" {i}. {s}")
|
||||
|
||||
if d.auto_fixable:
|
||||
lines.append("")
|
||||
lines.append("💡 **可自动修复**:如格式问题,可尝试点击Rerun让auto-fix自动处理")
|
||||
|
||||
if run_url:
|
||||
lines.append("")
|
||||
lines.append(f"[查看完整日志]({run_url})")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
job_name = os.environ.get("FAILED_JOB", "")
|
||||
run_id = os.environ.get("GITHUB_RUN_ID", "")
|
||||
repo = os.environ.get("GITHUB_REPOSITORY", "xiaoxia/xiaoxia-saas")
|
||||
token = os.environ.get("GITHUB_TOKEN", "")
|
||||
|
||||
# 1. 尝试获取日志
|
||||
log_text = ""
|
||||
|
||||
# 优先从环境变量或文件读取
|
||||
log_file = os.environ.get("CI_LOG_FILE", "")
|
||||
if log_file and os.path.exists(log_file):
|
||||
with open(log_file) as f:
|
||||
log_text = f.read()
|
||||
elif run_id and token:
|
||||
# 尝试从API获取(需要job_id,这里简化处理)
|
||||
pass
|
||||
|
||||
# 如果没有日志,用job_name做粗略分类
|
||||
if not log_text:
|
||||
# 基于job名做初始判断
|
||||
if any(k in job_name.lower() for k in ["validate", "lint", "quality"]):
|
||||
d = FailureDiagnosis(
|
||||
category="lint_general",
|
||||
category_cn="代码质量检查",
|
||||
severity="low",
|
||||
summary=f"{job_name} 检查失败(日志不可用,基于job名初步诊断)",
|
||||
suggestions=["点击查看日志获取具体错误信息", "格式类问题通常可自动修复"],
|
||||
auto_fixable=True,
|
||||
)
|
||||
elif "build" in job_name.lower():
|
||||
d = FailureDiagnosis(
|
||||
category="build_general",
|
||||
category_cn="构建失败",
|
||||
severity="high",
|
||||
summary=f"{job_name} 构建失败(日志不可用)",
|
||||
suggestions=["点击查看日志获取具体构建错误", "常见原因:Dockerfile错误、依赖安装失败、网络问题"],
|
||||
auto_fixable=False,
|
||||
)
|
||||
elif "test" in job_name.lower():
|
||||
d = FailureDiagnosis(
|
||||
category="test_general",
|
||||
category_cn="测试失败",
|
||||
severity="high",
|
||||
summary=f"{job_name} 测试失败(日志不可用)",
|
||||
suggestions=["点击查看日志获取具体失败的测试用例", "检查最近代码改动是否影响了测试"],
|
||||
auto_fixable=False,
|
||||
)
|
||||
elif "deploy" in job_name.lower():
|
||||
d = FailureDiagnosis(
|
||||
category="deploy_general",
|
||||
category_cn="部署失败",
|
||||
severity="high",
|
||||
summary=f"{job_name} 部署失败(日志不可用)",
|
||||
suggestions=["检查目标服务器状态和网络", "检查镜像是否正确推送", "查看服务器上的容器日志"],
|
||||
auto_fixable=False,
|
||||
)
|
||||
else:
|
||||
d = FailureDiagnosis(
|
||||
category="unknown",
|
||||
category_cn="未知错误",
|
||||
severity="medium",
|
||||
summary=f"{job_name} 失败",
|
||||
suggestions=["点击查看日志获取详细信息"],
|
||||
auto_fixable=False,
|
||||
)
|
||||
else:
|
||||
d = analyze_log(log_text, job_name)
|
||||
|
||||
# 输出诊断结果
|
||||
run_url = f"https://git.xiaoxiajianji.com/{repo}/actions/runs/{run_id}" if run_id else ""
|
||||
|
||||
print("=" * 60)
|
||||
print(" CI 失败诊断报告")
|
||||
print("=" * 60)
|
||||
print()
|
||||
print(format_diagnosis_markdown(d, job_name, run_url))
|
||||
print()
|
||||
print("=" * 60)
|
||||
|
||||
# 将诊断结果写入文件(供通知脚本读取)
|
||||
output_file = os.environ.get("DIAGNOSIS_OUTPUT", "/tmp/ci_diagnosis.json")
|
||||
result = {
|
||||
"category": d.category,
|
||||
"category_cn": d.category_cn,
|
||||
"severity": d.severity,
|
||||
"summary": d.summary,
|
||||
"error_lines": d.error_lines,
|
||||
"suggestions": d.suggestions,
|
||||
"auto_fixable": d.auto_fixable,
|
||||
}
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
print(f"\n诊断结果已保存到: {output_file}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,417 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
CI重复失败检测脚本
|
||||
- 扫描最近N天的CI失败
|
||||
- 按job名称分组统计失败率
|
||||
- 识别高失败率job(系统性故障)
|
||||
- 飞书通知告警
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
|
||||
def get_env(name, default=None, required=False):
|
||||
val = os.environ.get(name, default)
|
||||
if required and not val:
|
||||
print(f"❌ 缺少环境变量: {name}")
|
||||
sys.exit(1)
|
||||
return val
|
||||
|
||||
|
||||
GITEA_URL = get_env("GITEA_URL", "https://git.xiaoxiajianji.com")
|
||||
GITEA_TOKEN = get_env("GITEA_API_TOKEN", required=False) or get_env("GITHUB_TOKEN", "")
|
||||
REPO = get_env("GITEA_REPO", "xiaoxia/xiaoxia-saas")
|
||||
DAYS = int(get_env("FAIL_CHECK_DAYS", "7"))
|
||||
FAIL_THRESHOLD = int(get_env("FAIL_THRESHOLD", 3)) # 失败次数阈值
|
||||
FAIL_RATE_THRESHOLD = float(get_env("FAIL_RATE_THRESHOLD", "30")) # 失败率阈值%
|
||||
CONSECUTIVE_FAIL_THRESHOLD = int(get_env("CONSECUTIVE_FAIL_THRESHOLD", "3")) # 连续失败阈值
|
||||
WEBHOOK = get_env("CI_NOTIFY_WEBHOOK", "")
|
||||
|
||||
|
||||
def api_get(path):
|
||||
"""调用Gitea API"""
|
||||
url = f"{GITEA_URL}/api/v1{path}"
|
||||
req = urllib.request.Request(url)
|
||||
if GITEA_TOKEN:
|
||||
req.add_header("Authorization", f"token {GITEA_TOKEN}")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return json.loads(resp.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
print(f" HTTP {e.code}: {path}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f" 错误: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_recent_runs(days=7, per_page=50, max_pages=10):
|
||||
"""获取最近N天的runs"""
|
||||
since = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
|
||||
all_runs = []
|
||||
|
||||
for page in range(1, max_pages + 1):
|
||||
path = f"/repos/{REPO}/actions/runs?page={page}&limit={per_page}"
|
||||
data = api_get(path)
|
||||
if not data:
|
||||
break
|
||||
|
||||
runs = data.get("workflow_runs", data.get("runs", []))
|
||||
if not runs:
|
||||
break
|
||||
|
||||
# 检查时间范围(Gitea用started_at,格式2026-07-22T10:58:10+08:00)
|
||||
oldest = None
|
||||
for r in runs:
|
||||
started = r.get("started_at", r.get("created_at", ""))
|
||||
if started and started >= since:
|
||||
all_runs.append(r)
|
||||
else:
|
||||
oldest = started
|
||||
|
||||
if oldest and oldest < since:
|
||||
break
|
||||
|
||||
if len(runs) < per_page:
|
||||
break
|
||||
|
||||
return all_runs
|
||||
|
||||
|
||||
def fetch_run_jobs(run_id):
|
||||
"""获取run的所有jobs"""
|
||||
path = f"/repos/{REPO}/actions/runs/{run_id}/jobs"
|
||||
data = api_get(path)
|
||||
if not data:
|
||||
return []
|
||||
return data.get("jobs", [])
|
||||
|
||||
|
||||
def analyze_failures(runs):
|
||||
"""
|
||||
分析失败情况
|
||||
|
||||
返回:
|
||||
- job_stats: {job_name: {total, success, failure, skipped, failure_rate, failures: [...]}}
|
||||
- consecutive_failures: {job_name: current_streak, max_streak, last_status}
|
||||
"""
|
||||
job_stats = defaultdict(
|
||||
lambda: {
|
||||
"total": 0,
|
||||
"success": 0,
|
||||
"failure": 0,
|
||||
"error": 0,
|
||||
"skipped": 0,
|
||||
"cancelled": 0,
|
||||
"failures": [],
|
||||
}
|
||||
)
|
||||
|
||||
# 按时间正序排列(旧→新)用于连续失败计算
|
||||
sorted_runs = sorted(runs, key=lambda r: r.get("started_at", r.get("created_at", "")))
|
||||
|
||||
# 连续失败跟踪 {job_name: streak}
|
||||
consecutive = defaultdict(lambda: {"current": 0, "max": 0, "last_run": None})
|
||||
|
||||
for run in sorted_runs:
|
||||
run_id = run.get("id")
|
||||
run_status = run.get("status", "")
|
||||
run_conclusion = run.get("conclusion", "")
|
||||
run_started = run.get("started_at", run.get("created_at", ""))
|
||||
event = run.get("event", "")
|
||||
|
||||
# 只统计pull_request和push事件的CI
|
||||
if event not in ("pull_request", "push"):
|
||||
continue
|
||||
|
||||
jobs = fetch_run_jobs(run_id)
|
||||
|
||||
for job in jobs:
|
||||
name = job.get("name", "")
|
||||
status = job.get("status", "")
|
||||
conclusion = job.get("conclusion", "")
|
||||
|
||||
# 跳过非CI核心job(如AI Code Review、Preview等)
|
||||
skip_prefixes = ("AI Code Review", "Preview", "PR Automation", "Auto")
|
||||
if any(name.startswith(p) for p in skip_prefixes):
|
||||
continue
|
||||
|
||||
stats = job_stats[name]
|
||||
stats["total"] += 1
|
||||
|
||||
if conclusion == "success":
|
||||
stats["success"] += 1
|
||||
consecutive[name]["current"] = 0
|
||||
elif conclusion == "failure":
|
||||
stats["failure"] += 1
|
||||
stats["failures"].append(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"time": run_started,
|
||||
"event": event,
|
||||
}
|
||||
)
|
||||
consecutive[name]["current"] += 1
|
||||
if consecutive[name]["current"] > consecutive[name]["max"]:
|
||||
consecutive[name]["max"] = consecutive[name]["current"]
|
||||
consecutive[name]["last_run"] = run_id
|
||||
elif conclusion == "error":
|
||||
stats["error"] += 1
|
||||
# error也算失败的一种
|
||||
consecutive[name]["current"] += 1
|
||||
if consecutive[name]["current"] > consecutive[name]["max"]:
|
||||
consecutive[name]["max"] = consecutive[name]["current"]
|
||||
elif conclusion == "skipped":
|
||||
stats["skipped"] += 1
|
||||
# skipped不算也不打断连续失败
|
||||
elif conclusion == "cancelled":
|
||||
stats["cancelled"] += 1
|
||||
# cancelled不算失败也不打断
|
||||
|
||||
# 计算失败率
|
||||
for name, stats in job_stats.items():
|
||||
total_actual = stats["total"] - stats["skipped"] - stats["cancelled"]
|
||||
if total_actual > 0:
|
||||
stats["failure_rate"] = round((stats["failure"] + stats["error"]) / total_actual * 100, 1)
|
||||
else:
|
||||
stats["failure_rate"] = 0.0
|
||||
|
||||
return dict(job_stats), dict(consecutive)
|
||||
|
||||
|
||||
def find_high_failures(job_stats, consecutive):
|
||||
"""
|
||||
找出高风险job
|
||||
|
||||
告警级别:
|
||||
- critical: 连续失败 >= CONSECUTIVE_FAIL_THRESHOLD,或 失败率>=50%且失败次数>=5
|
||||
- warning: 失败率>=FAIL_RATE_THRESHOLD且失败次数>=FAIL_THRESHOLD
|
||||
- info: 失败次数>=2
|
||||
"""
|
||||
critical = []
|
||||
warning = []
|
||||
info = []
|
||||
|
||||
for name, stats in job_stats.items():
|
||||
fail_count = stats["failure"] + stats["error"]
|
||||
rate = stats["failure_rate"]
|
||||
streak = consecutive.get(name, {}).get("current", 0)
|
||||
max_streak = consecutive.get(name, {}).get("max", 0)
|
||||
|
||||
issue = {
|
||||
"name": name,
|
||||
"fail_count": fail_count,
|
||||
"total": stats["total"],
|
||||
"failure_rate": rate,
|
||||
"current_streak": streak,
|
||||
"max_streak": max_streak,
|
||||
"recent_failures": stats["failures"][-5:], # 最近5次
|
||||
}
|
||||
|
||||
if streak >= CONSECUTIVE_FAIL_THRESHOLD or (rate >= 50 and fail_count >= 5):
|
||||
critical.append(issue)
|
||||
elif rate >= FAIL_RATE_THRESHOLD and fail_count >= FAIL_THRESHOLD:
|
||||
warning.append(issue)
|
||||
elif fail_count >= 2:
|
||||
info.append(issue)
|
||||
|
||||
# 按失败次数倒序
|
||||
critical.sort(key=lambda x: x["fail_count"], reverse=True)
|
||||
warning.sort(key=lambda x: x["fail_count"], reverse=True)
|
||||
info.sort(key=lambda x: x["fail_count"], reverse=True)
|
||||
|
||||
return critical, warning, info
|
||||
|
||||
|
||||
def generate_report(critical, warning, info, days, total_runs):
|
||||
"""生成Markdown报告"""
|
||||
lines = []
|
||||
lines.append("# CI重复失败检测报告")
|
||||
lines.append("")
|
||||
lines.append(f"**统计周期**: 最近{days}天")
|
||||
lines.append(f"**扫描Runs**: {total_runs}个")
|
||||
lines.append(f"**生成时间**: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
|
||||
lines.append("")
|
||||
|
||||
lines.append(f"## 概览")
|
||||
lines.append("")
|
||||
lines.append(f"| 级别 | 数量 |")
|
||||
lines.append(f"|------|------|")
|
||||
lines.append(f"| 🔴 严重 (连续失败≥{CONSECUTIVE_FAIL_THRESHOLD}次 或 失败率≥50%) | {len(critical)} |")
|
||||
lines.append(f"| 🟡 警告 (失败率≥{FAIL_RATE_THRESHOLD}% 且 失败≥{FAIL_THRESHOLD}次) | {len(warning)} |")
|
||||
lines.append(f"| 🔵 关注 (失败≥2次) | {len(info)} |")
|
||||
lines.append("")
|
||||
|
||||
if critical:
|
||||
lines.append("## 🔴 严重问题")
|
||||
lines.append("")
|
||||
for item in critical:
|
||||
lines.append(f"### {item['name']}")
|
||||
lines.append("")
|
||||
lines.append(f"- 失败次数: **{item['fail_count']}** / {item['total']} 次运行")
|
||||
lines.append(f"- 失败率: **{item['failure_rate']}%**")
|
||||
lines.append(f"- 当前连续失败: **{item['current_streak']}** 次 (历史最高: {item['max_streak']} 次)")
|
||||
lines.append("")
|
||||
if item["recent_failures"]:
|
||||
lines.append("最近失败:")
|
||||
lines.append("")
|
||||
for f in item["recent_failures"]:
|
||||
lines.append(f"- [{f['time'][:16]}] run #{f['run_id']} ({f['event']})")
|
||||
lines.append("")
|
||||
|
||||
if warning:
|
||||
lines.append("## 🟡 警告")
|
||||
lines.append("")
|
||||
for item in warning:
|
||||
lines.append(
|
||||
f"- **{item['name']}**: {item['fail_count']}次失败 / {item['total']}次运行 ({item['failure_rate']}%)"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
if info:
|
||||
lines.append("## 🔵 关注列表")
|
||||
lines.append("")
|
||||
lines.append("| Job名称 | 失败次数 | 总次数 | 失败率 | 当前连续 |")
|
||||
lines.append("|---------|----------|--------|--------|----------|")
|
||||
for item in info[:20]: # 最多显示20个
|
||||
lines.append(
|
||||
f"| {item['name']} | {item['fail_count']} | {item['total']} | {item['failure_rate']}% | {item['current_streak']} |"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def send_feishu_notification(critical, warning, info, days):
|
||||
"""发送飞书通知"""
|
||||
if not WEBHOOK:
|
||||
print(" ⚠️ 未配置WEBHOOK,跳过飞书通知")
|
||||
return False
|
||||
|
||||
total_issues = len(critical) + len(warning) + len(info)
|
||||
if total_issues == 0:
|
||||
print(" ✅ 无异常,不发送通知")
|
||||
return True
|
||||
|
||||
level = "🔴 严重告警" if critical else "🟡 警告" if warning else "🔵 关注"
|
||||
|
||||
title = f"CI重复失败检测 - {level}"
|
||||
text = f"统计周期: 最近{days}天\n\n"
|
||||
|
||||
if critical:
|
||||
text += "【严重问题】\n"
|
||||
for item in critical[:5]:
|
||||
text += f"• {item['name']}\n"
|
||||
text += f" 失败 {item['fail_count']}/{item['total']} ({item['failure_rate']}%) 连续{item['current_streak']}次\n"
|
||||
if len(critical) > 5:
|
||||
text += f" ...还有{len(critical)-5}个\n"
|
||||
text += "\n"
|
||||
|
||||
if warning:
|
||||
text += "【警告】\n"
|
||||
for item in warning[:5]:
|
||||
text += f"• {item['name']}: {item['fail_count']}次失败 ({item['failure_rate']}%)\n"
|
||||
if len(warning) > 5:
|
||||
text += f" ...还有{len(warning)-5}个\n"
|
||||
text += "\n"
|
||||
|
||||
if info and not critical and not warning:
|
||||
text += "【关注列表】\n"
|
||||
for item in info[:10]:
|
||||
text += f"• {item['name']}: {item['fail_count']}次失败\n"
|
||||
text += "\n"
|
||||
|
||||
text += f"共发现 {total_issues} 个异常job"
|
||||
|
||||
payload = {"msg_type": "text", "content": {"text": f"{title}\n\n{text}"}}
|
||||
|
||||
data = json.dumps(payload).encode()
|
||||
req = urllib.request.Request(WEBHOOK, data=data, headers={"Content-Type": "application/json"})
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
result = json.loads(resp.read())
|
||||
if result.get("code") == 0 or result.get("StatusCode") == 0:
|
||||
print(" ✅ 飞书通知已发送")
|
||||
return True
|
||||
else:
|
||||
print(f" ⚠️ 飞书返回: {result}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f" ❌ 飞书通知失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
print(f"=== CI重复失败检测 ===")
|
||||
print(f"统计周期: 最近{DAYS}天")
|
||||
print(f"仓库: {REPO}")
|
||||
print()
|
||||
|
||||
print("1. 获取最近的Runs...")
|
||||
runs = fetch_recent_runs(days=DAYS)
|
||||
print(f" 找到 {len(runs)} 个runs")
|
||||
|
||||
if not runs:
|
||||
print("⚠️ 没有找到runs,退出")
|
||||
return
|
||||
|
||||
print()
|
||||
print("2. 分析job失败情况(可能需要点时间)...")
|
||||
job_stats, consecutive = analyze_failures(runs)
|
||||
print(f" 共统计 {len(job_stats)} 个job")
|
||||
|
||||
print()
|
||||
print("3. 识别高风险job...")
|
||||
critical, warning, info = find_high_failures(job_stats, consecutive)
|
||||
print(f" 🔴 严重: {len(critical)}")
|
||||
print(f" 🟡 警告: {len(warning)}")
|
||||
print(f" 🔵 关注: {len(info)}")
|
||||
|
||||
print()
|
||||
print("4. 生成报告...")
|
||||
report = generate_report(critical, warning, info, DAYS, len(runs))
|
||||
|
||||
# 保存报告
|
||||
report_path = os.environ.get("REPORT_PATH", f"/tmp/ci_failure_report_{int(time.time())}.md")
|
||||
with open(report_path, "w") as f:
|
||||
f.write(report)
|
||||
print(f" 报告已保存: {report_path}")
|
||||
|
||||
# 打印摘要
|
||||
print()
|
||||
print("=== 摘要 ===")
|
||||
if critical:
|
||||
print("🔴 严重问题:")
|
||||
for item in critical[:5]:
|
||||
print(
|
||||
f" {item['name']}: {item['fail_count']}次失败, {item['failure_rate']}%, 连续{item['current_streak']}次"
|
||||
)
|
||||
if warning:
|
||||
print("🟡 警告:")
|
||||
for item in warning[:5]:
|
||||
print(f" {item['name']}: {item['fail_count']}次失败, {item['failure_rate']}%")
|
||||
|
||||
print()
|
||||
print("5. 发送飞书通知...")
|
||||
send_feishu_notification(critical, warning, info, DAYS)
|
||||
|
||||
print()
|
||||
print("✅ 检测完成")
|
||||
|
||||
# 有严重问题时退出码非零,方便workflow标记
|
||||
if critical:
|
||||
sys.exit(2)
|
||||
elif warning:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# CI 健康度看板一键生成脚本
|
||||
# - 从 Gitea Actions API 拉取数据
|
||||
# - 生成 HTML 可视化看板
|
||||
# - 输出文件路径
|
||||
#
|
||||
# 用法:
|
||||
# bash scripts/ci/generate_ci_dashboard.sh [--days 7] [--output ci_dashboard.html]
|
||||
#
|
||||
# 环境变量:
|
||||
# GITEA_TOKEN API Token(必需)
|
||||
# GITEA_URL Gitea 地址(可选,默认 https://git.xiaoxiajianji.com)
|
||||
# GITEA_REPO 仓库(可选,默认 xiaoxia/xiaoxia-saas)
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
|
||||
# 默认参数
|
||||
DAYS=7
|
||||
OUTPUT="ci_dashboard.html"
|
||||
|
||||
# 解析参数
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--days)
|
||||
DAYS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--output|-o)
|
||||
OUTPUT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help|-h)
|
||||
echo "用法: bash scripts/ci/generate_ci_dashboard.sh [--days 7] [--output ci_dashboard.html]"
|
||||
echo ""
|
||||
echo "选项:"
|
||||
echo " --days N 统计最近 N 天 (默认 7)"
|
||||
echo " --output PATH HTML 输出路径 (默认 ci_dashboard.html)"
|
||||
echo " --help 显示帮助"
|
||||
echo ""
|
||||
echo "环境变量:"
|
||||
echo " GITEA_TOKEN API Token(必需)"
|
||||
echo " GITEA_URL Gitea 地址"
|
||||
echo " GITEA_REPO 仓库"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "未知参数: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# 检查 Python
|
||||
if ! command -v python3 &> /dev/null; then
|
||||
echo "[ERROR] 未找到 python3,请先安装 Python 3"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检查 Token
|
||||
if [[ -z "${GITEA_TOKEN:-}" ]]; then
|
||||
echo "[ERROR] 请设置 GITEA_TOKEN 环境变量"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "========================================"
|
||||
echo " CI 健康度看板生成器"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "统计天数: ${DAYS} 天"
|
||||
echo "输出文件: ${OUTPUT}"
|
||||
echo ""
|
||||
|
||||
# 生成 HTML 看板
|
||||
echo "[INFO] 正在拉取数据并生成看板..."
|
||||
python3 "${SCRIPT_DIR}/ci_dashboard.py" \
|
||||
--days "${DAYS}" \
|
||||
--html \
|
||||
--html-output "${OUTPUT}"
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo " ✅ 看板生成完成!"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "文件路径: $(realpath "${OUTPUT}")"
|
||||
echo ""
|
||||
|
||||
# 如果在 macOS 上,尝试打开
|
||||
if [[ "$(uname)" == "Darwin" ]]; then
|
||||
echo "[INFO] 正在打开浏览器..."
|
||||
open "${OUTPUT}"
|
||||
fi
|
||||
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
PR自动扫描器:扫描所有open PR,对CI全绿的进行自动审批/合并
|
||||
作为短作业模式的兜底机制,每5分钟运行一次
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
|
||||
def api_request(token, repo, endpoint, method="GET", data=None):
|
||||
"""Gitea API请求"""
|
||||
url = f"https://git.xiaoxiajianji.com/api/v1/repos/{repo}/{endpoint}"
|
||||
headers = {"Authorization": f"token {token}", "Content-Type": "application/json"}
|
||||
body = json.dumps(data).encode() if data else None
|
||||
req = urllib.request.Request(url, data=body, headers=headers, method=method)
|
||||
|
||||
# 跳过SSL验证
|
||||
import ssl
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, context=ctx)
|
||||
return json.loads(resp.read().decode()), resp.status
|
||||
except urllib.error.HTTPError as e:
|
||||
return json.loads(e.read().decode()) if e.read() else {"error": str(e)}, e.code
|
||||
|
||||
|
||||
def get_open_prs(token, repo, base="develop"):
|
||||
"""获取所有open的PR"""
|
||||
prs = []
|
||||
page = 1
|
||||
while True:
|
||||
data, code = api_request(token, repo, f"pulls?state=open&base={base}&sort=recentupdate&per_page=50&page={page}")
|
||||
if code != 200 or not isinstance(data, list) or len(data) == 0:
|
||||
break
|
||||
prs.extend(data)
|
||||
if len(data) < 50:
|
||||
break
|
||||
page += 1
|
||||
return prs
|
||||
|
||||
|
||||
def get_commit_status(token, repo, sha):
|
||||
"""获取commit的CI状态汇总"""
|
||||
data, code = api_request(token, repo, f"commits/{sha}/status")
|
||||
if code != 200:
|
||||
return {}, "error"
|
||||
return data, data.get("state", "unknown")
|
||||
|
||||
|
||||
def check_required_contexts(token, repo, sha, contexts):
|
||||
"""检查指定的context是否都通过"""
|
||||
data, _ = get_commit_status(token, repo, sha)
|
||||
statuses = {s["context"]: s["status"] for s in data.get("statuses", [])}
|
||||
|
||||
all_success = True
|
||||
any_pending = False
|
||||
any_failed = False
|
||||
|
||||
for ctx in contexts:
|
||||
state = statuses.get(ctx, "pending")
|
||||
if state != "success":
|
||||
all_success = False
|
||||
if state == "pending":
|
||||
any_pending = True
|
||||
if state in ("failure", "error"):
|
||||
any_failed = True
|
||||
|
||||
return all_success, any_pending, any_failed, statuses
|
||||
|
||||
|
||||
def get_pr_files(token, repo, pr_number):
|
||||
"""获取PR变更文件"""
|
||||
files = []
|
||||
page = 1
|
||||
while True:
|
||||
data, code = api_request(token, repo, f"pulls/{pr_number}/files?per_page=300&page={page}")
|
||||
if code != 200 or not isinstance(data, list) or len(data) == 0:
|
||||
break
|
||||
files.extend(data)
|
||||
if len(data) < 300:
|
||||
break
|
||||
page += 1
|
||||
return [f["filename"] for f in files]
|
||||
|
||||
|
||||
def is_frontend_only(files):
|
||||
"""判断是否纯前端改动"""
|
||||
if not files:
|
||||
return False
|
||||
frontend_count = sum(1 for f in files if f.startswith("apps/web/"))
|
||||
backend_count = len(files) - frontend_count
|
||||
return backend_count == 0 and frontend_count > 0
|
||||
|
||||
|
||||
def has_approval(token, repo, pr_number):
|
||||
"""检查PR是否已有审批"""
|
||||
reviews, code = api_request(token, repo, f"pulls/{pr_number}/reviews")
|
||||
if code != 200:
|
||||
return False
|
||||
return any(r.get("state") == "APPROVED" for r in reviews if isinstance(r, dict))
|
||||
|
||||
|
||||
def approve_pr(token, repo, pr_number):
|
||||
"""审批PR"""
|
||||
# 创建review
|
||||
data, code = api_request(
|
||||
token,
|
||||
repo,
|
||||
f"pulls/{pr_number}/reviews",
|
||||
method="POST",
|
||||
data={"event": "PENDING", "body": "CI全绿,自动审批通过。"},
|
||||
)
|
||||
|
||||
if code not in (200, 201):
|
||||
return False, f"创建review失败: HTTP {code}"
|
||||
|
||||
review_id = data.get("id")
|
||||
if data.get("state") == "APPROVED":
|
||||
return True, "直接创建APPROVED成功"
|
||||
|
||||
if not review_id:
|
||||
return False, "未获取到review ID"
|
||||
|
||||
# submit为APPROVED
|
||||
data2, code2 = api_request(
|
||||
token,
|
||||
repo,
|
||||
f"pulls/{pr_number}/reviews/{review_id}/events",
|
||||
method="POST",
|
||||
data={"event": "APPROVED", "body": "CI全绿,自动审批通过。"},
|
||||
)
|
||||
|
||||
if code2 in (200, 201):
|
||||
return True, "审批提交成功"
|
||||
else:
|
||||
# 尝试另一个端点
|
||||
data3, code3 = api_request(
|
||||
token,
|
||||
repo,
|
||||
f"pulls/{pr_number}/reviews/{review_id}",
|
||||
method="POST",
|
||||
data={"event": "APPROVED", "body": "CI全绿,自动审批通过。"},
|
||||
)
|
||||
if code3 in (200, 201):
|
||||
return True, "审批提交成功(备用端点)"
|
||||
return False, f"审批提交失败: HTTP {code2}/{code3}"
|
||||
|
||||
|
||||
def merge_pr(token, repo, pr_number):
|
||||
"""合并PR(squash merge)"""
|
||||
# 等待几秒让状态同步
|
||||
time.sleep(30)
|
||||
|
||||
# 检查PR状态
|
||||
pr_data, code = api_request(token, repo, f"pulls/{pr_number}")
|
||||
if code != 200:
|
||||
return False, f"获取PR状态失败: HTTP {code}"
|
||||
if pr_data.get("state") != "open":
|
||||
return False, f"PR状态不是open: {pr_data.get('state')}"
|
||||
|
||||
# 执行squash merge
|
||||
data, code = api_request(
|
||||
token,
|
||||
repo,
|
||||
f"pulls/{pr_number}/merge",
|
||||
method="POST",
|
||||
data={
|
||||
"do": "squash",
|
||||
"merge_title_field": "",
|
||||
"merge_message_field": "",
|
||||
"delete_branch_after_merge": True,
|
||||
"force_merge": False,
|
||||
},
|
||||
)
|
||||
|
||||
if code == 200:
|
||||
return True, "合并成功"
|
||||
elif code == 405:
|
||||
return False, "合并返回405(门禁未满足或冲突)"
|
||||
else:
|
||||
return False, f"合并失败: HTTP {code}"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="PR自动扫描器")
|
||||
parser.add_argument("--token", required=True, help="Gitea API token")
|
||||
parser.add_argument("--repo", default="xiaoxia/xiaoxia-saas", help="仓库")
|
||||
parser.add_argument("--base", default="develop", help="目标分支")
|
||||
parser.add_argument("--approve", action="store_true", help="执行自动审批")
|
||||
parser.add_argument("--merge", action="store_true", help="执行自动合并")
|
||||
parser.add_argument("--dry-run", default="false", help="试运行模式")
|
||||
parser.add_argument("--max-prs", type=int, default=20, help="最多处理的PR数")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
dry_run = args.dry_run.lower() == "true"
|
||||
|
||||
# required contexts(与分支保护一致)
|
||||
REQUIRED_CONTEXTS_FULL = [
|
||||
"CI/CD Pipeline / Validate - Code Quality (pull_request)",
|
||||
"CI/CD Pipeline / Validate - Type Check (mypy) (pull_request)",
|
||||
"CI/CD Pipeline / Validate - Migration (alembic) (pull_request)",
|
||||
"CI/CD Pipeline / Frontend Lint (pull_request)",
|
||||
"CI/CD Pipeline / PR Build API Image (pull_request)",
|
||||
"CI/CD Pipeline / PR Build Worker Image (pull_request)",
|
||||
"CI/CD Pipeline / PR Build Web Image (pull_request)",
|
||||
]
|
||||
REQUIRED_CONTEXTS_APPROVE = [
|
||||
"CI/CD Pipeline / Validate - Code Quality (pull_request)",
|
||||
"CI/CD Pipeline / Validate - Type Check (mypy) (pull_request)",
|
||||
"CI/CD Pipeline / Validate - Migration (alembic) (pull_request)",
|
||||
"CI/CD Pipeline / Frontend Lint (pull_request)",
|
||||
]
|
||||
FRONTEND_ONLY_CONTEXT = [
|
||||
"CI/CD Pipeline / Frontend Lint (pull_request)",
|
||||
]
|
||||
|
||||
# 获取所有open PR
|
||||
print(f"获取 {args.base} 分支的open PR...")
|
||||
prs = get_open_prs(args.token, args.repo, args.base)
|
||||
print(f"找到 {len(prs)} 个open PR")
|
||||
|
||||
approved_count = 0
|
||||
merged_count = 0
|
||||
skipped_count = 0
|
||||
|
||||
for pr in prs[: args.max_prs]:
|
||||
pr_num = pr["number"]
|
||||
pr_title = pr["title"]
|
||||
head_sha = pr["head"]["sha"]
|
||||
base_ref = pr.get("base", {}).get("ref", "")
|
||||
|
||||
# 跳过draft
|
||||
if pr.get("draft"):
|
||||
print(f"\n⏭️ #{pr_num} {pr_title[:50]} - draft,跳过")
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
# 跳过目标分支不对的
|
||||
if base_ref != args.base:
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
print(f"\n--- #{pr_num} {pr_title[:60]} ---")
|
||||
|
||||
# 判断是否纯前端
|
||||
files = get_pr_files(args.token, args.repo, pr_num)
|
||||
frontend_only = is_frontend_only(files)
|
||||
|
||||
if frontend_only:
|
||||
approve_contexts = FRONTEND_ONLY_CONTEXT
|
||||
merge_contexts = FRONTEND_ONLY_CONTEXT
|
||||
print(f" 类型: 纯前端改动 ({len(files)}个文件)")
|
||||
else:
|
||||
approve_contexts = REQUIRED_CONTEXTS_APPROVE
|
||||
merge_contexts = REQUIRED_CONTEXTS_FULL
|
||||
print(f" 类型: 全栈/后端改动 ({len(files)}个文件)")
|
||||
|
||||
# 检查审批用的CI状态
|
||||
all_ok, pending, failed, _ = check_required_contexts(args.token, args.repo, head_sha, approve_contexts)
|
||||
|
||||
# === 自动审批 ===
|
||||
if args.approve and all_ok and not failed:
|
||||
if has_approval(args.token, args.repo, pr_num):
|
||||
print(f" ✅ 已有审批,跳过")
|
||||
else:
|
||||
if dry_run:
|
||||
print(f" 🎯 [DRY-RUN] 将自动审批")
|
||||
else:
|
||||
print(f" 🎯 执行自动审批...")
|
||||
ok, msg = approve_pr(args.token, args.repo, pr_num)
|
||||
if ok:
|
||||
print(f" ✅ 审批成功: {msg}")
|
||||
approved_count += 1
|
||||
else:
|
||||
print(f" ❌ 审批失败: {msg}")
|
||||
elif failed:
|
||||
print(f" ❌ CI有失败项,跳过审批")
|
||||
elif pending:
|
||||
print(f" ⏳ CI仍在运行,跳过")
|
||||
|
||||
# === 自动合并 ===
|
||||
if args.merge:
|
||||
# 检查合并用的CI状态
|
||||
merge_ok, merge_pending, merge_failed, _ = check_required_contexts(
|
||||
args.token, args.repo, head_sha, merge_contexts
|
||||
)
|
||||
|
||||
# 检查审批
|
||||
approved = has_approval(args.token, args.repo, pr_num)
|
||||
|
||||
if merge_ok and approved and not merge_failed:
|
||||
if dry_run:
|
||||
print(f" 🎯 [DRY-RUN] 将自动合并")
|
||||
else:
|
||||
print(f" 🎯 执行自动合并...")
|
||||
ok, msg = merge_pr(args.token, args.repo, pr_num)
|
||||
if ok:
|
||||
print(f" ✅ 合并成功: {msg}")
|
||||
merged_count += 1
|
||||
else:
|
||||
print(f" ⚠️ 合并失败: {msg}")
|
||||
elif merge_pending:
|
||||
print(f" ⏳ 合并条件未满足: CI运行中")
|
||||
elif merge_failed:
|
||||
print(f" ❌ 合并条件未满足: CI有失败")
|
||||
elif not approved:
|
||||
print(f" ⏳ 合并条件未满足: 无审批")
|
||||
|
||||
print(f"\n=== 扫描结果 ===")
|
||||
print(f" 处理PR数: {min(len(prs), args.max_prs)}")
|
||||
print(f" 自动审批: {approved_count} 个")
|
||||
print(f" 自动合并: {merged_count} 个")
|
||||
print(f" 跳过: {skipped_count} 个")
|
||||
print(f" 模式: {'DRY-RUN' if dry_run else '正式执行'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/bin/bash
|
||||
# CI Integration Tests Job 主脚本
|
||||
# 包含:依赖安装、ffmpeg安装、Redis启动、PG启动、迁移、测试、清理、覆盖率
|
||||
# 支持 pytest-xdist 并行执行:每个 worker 使用独立数据库,预期加速 2-4 倍
|
||||
set -eu
|
||||
|
||||
echo "=== CI Integration Tests 开始 ==="
|
||||
@@ -28,12 +29,13 @@ for i in 1 2 3; do
|
||||
sleep 5
|
||||
done
|
||||
for i in 1 2 3; do
|
||||
python3 -m pip install -q pytest-rerunfailures && break
|
||||
echo "pip install pytest-rerunfailures 失败,重试 $i/3..."
|
||||
python3 -m pip install -q pytest-rerunfailures pytest-xdist && break
|
||||
echo "pip install pytest-rerunfailures/pytest-xdist 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
pytest --version
|
||||
echo "pytest-xdist: $(python3 -c "import xdist; print(xdist.__version__)" 2>/dev/null || echo 'not installed')"
|
||||
|
||||
# --- 安装 ffmpeg ---
|
||||
echo ""
|
||||
@@ -187,13 +189,14 @@ if [ "$USE_SHARED_PG" = "true" ]; then
|
||||
echo "等待共享PG连接就绪..."
|
||||
wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5
|
||||
|
||||
# 创建独立数据库
|
||||
echo "创建测试数据库: $CI_DB_NAME"
|
||||
# 创建主数据库(xdist 模式下各 worker 会创建自己的数据库,主库作为 fallback)
|
||||
echo "创建主测试数据库: $CI_DB_NAME"
|
||||
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
|
||||
cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"')
|
||||
cur.close()
|
||||
conn.close()
|
||||
@@ -238,30 +241,50 @@ else
|
||||
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
|
||||
fi
|
||||
|
||||
# --- 执行迁移 ---
|
||||
# --- 执行迁移(主数据库,xdist worker 会各自创建自己的库并迁移) ---
|
||||
echo ""
|
||||
echo "=== 执行 Alembic 迁移 ==="
|
||||
echo "=== 执行 Alembic 迁移(主数据库) ==="
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
|
||||
echo "✅ 迁移完成"
|
||||
|
||||
# --- 运行集成测试 ---
|
||||
# --- 运行集成测试(pytest-xdist 并行) ---
|
||||
echo ""
|
||||
echo "=== 运行集成测试 ==="
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m coverage run \
|
||||
--source=apps/api/app,packages \
|
||||
--omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \
|
||||
--branch \
|
||||
-m pytest tests/integration -q --timeout=60 -x --reruns 2 --reruns-delay 1 -m "not performance"
|
||||
python3 -m coverage report --show-missing
|
||||
python3 -m coverage xml -o coverage.xml
|
||||
python3 -m coverage report --fail-under=40 > /dev/null
|
||||
echo "=== 运行集成测试(pytest-xdist 并行模式) ==="
|
||||
echo "CPU 核数: $(nproc 2>/dev/null || echo 'unknown')"
|
||||
|
||||
# 根据可用内存动态计算 worker 数,防止 OOM
|
||||
# 每个 worker 约占 200-300MB(含 DB 连接 + FastAPI test client)
|
||||
# 预留 1GB 给系统 + ffmpeg 等子进程
|
||||
AVAIL_MEM_MB=$(($(grep MemAvailable /proc/meminfo 2>/dev/null | awk '{print $2}' || echo 2097152) / 1024))
|
||||
RESERVED_MB=1024
|
||||
PER_WORKER_MB=256
|
||||
MAX_WORKERS=$(( (AVAIL_MEM_MB - RESERVED_MB) / PER_WORKER_MB ))
|
||||
# 下限 2,上限 8,CPU 核数也作为上限
|
||||
CPU_CORES=$(nproc 2>/dev/null || echo 4)
|
||||
XDIST_WORKERS=$MAX_WORKERS
|
||||
[ $XDIST_WORKERS -lt 2 ] && XDIST_WORKERS=2
|
||||
[ $XDIST_WORKERS -gt 8 ] && XDIST_WORKERS=8
|
||||
[ $XDIST_WORKERS -gt $CPU_CORES ] && XDIST_WORKERS=$CPU_CORES
|
||||
echo "可用内存: ${AVAIL_MEM_MB}MB, CPU核数: ${CPU_CORES}, xdist workers: ${XDIST_WORKERS}"
|
||||
|
||||
# 集成测试使用 pytest-xdist 并行加速(coverage 由单元测试负责,并行模式下 coverage 不稳定)
|
||||
# -n N: 并行 worker 数
|
||||
# --dist loadfile: 同一测试文件分配到同一 worker(共享 fixture 更高效)
|
||||
# --maxfail=3: 容忍少量失败(避免偶发 OOM 导致全挂)
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration \
|
||||
-q --timeout=60 --maxfail=3 --reruns 2 --reruns-delay 1 \
|
||||
-m "not performance" \
|
||||
-n $XDIST_WORKERS --dist loadfile \
|
||||
-p no:cacheprovider
|
||||
|
||||
echo "✅ 集成测试通过"
|
||||
|
||||
# --- API 性能基线测试(仅告警) ---
|
||||
# --- API 性能基线测试(仅告警,串行执行) ---
|
||||
echo ""
|
||||
echo "=== API 性能基线测试(仅告警) ==="
|
||||
set +e
|
||||
PERF_OUTPUT=$(mktemp)
|
||||
# 性能测试单独串行运行(不参与并行,避免资源竞争影响测量结果)
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration/test_api_performance.py \
|
||||
-v --timeout=120 -p no:cacheprovider 2>&1 | tee "$PERF_OUTPUT"
|
||||
echo ""
|
||||
@@ -284,14 +307,29 @@ set -e
|
||||
echo ""
|
||||
echo "=== 清理 ==="
|
||||
if [ "$USE_SHARED_PG" = "true" ]; then
|
||||
# 清理共享PG上的测试数据库
|
||||
echo "清理共享PG测试数据库: $CI_DB_NAME"
|
||||
# 清理共享PG上的测试数据库(主库 + 可能残留的 worker 库)
|
||||
echo "清理共享PG测试数据库..."
|
||||
|
||||
# 清理所有以 CI_DB_NAME 开头的数据库(主库 + worker 库)
|
||||
PGPASSWORD="${SHARED_PG_PASSWORD}" python3 -c "
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(host='${SHARED_PG_HOST}', port=${SHARED_PG_PORT}, user='${SHARED_PG_USER}', password='${SHARED_PG_PASSWORD}', dbname='postgres')
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
|
||||
|
||||
# 查找所有需要清理的数据库(主库 + worker 库)
|
||||
cur.execute(\"SELECT datname FROM pg_database WHERE datname LIKE '$CI_DB_NAME%'\")
|
||||
dbs = [row[0] for row in cur.fetchall()]
|
||||
|
||||
for db in dbs:
|
||||
try:
|
||||
# 强制断开所有连接
|
||||
cur.execute(f\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{db}' AND pid <> pg_backend_pid()\")
|
||||
cur.execute(f'DROP DATABASE IF EXISTS \"{db}\" WITH (FORCE)')
|
||||
print(f' 已清理: {db}')
|
||||
except Exception as e:
|
||||
print(f' 警告: 清理 {db} 失败: {e}')
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
" 2>/dev/null || echo "WARN: 数据库清理失败(可能已被清理)"
|
||||
|
||||
+560
-288
@@ -1,29 +1,75 @@
|
||||
#!/bin/bash
|
||||
# CI Validate Job 主脚本:代码质量全量检查
|
||||
# 包含:密钥扫描、格式检查、类型检查、安全扫描、依赖漏洞检查、死代码检测、Alembic迁移验证
|
||||
# CI Validate Job 主脚本:并行化代码质量检查
|
||||
# 将 8 项检查分为 2 组并行执行,预计耗时从 ~1.8min 降至 ~1min
|
||||
#
|
||||
# 并行分组:
|
||||
# Group A(独立并行):
|
||||
# A1: Secret detection (detect-secrets)
|
||||
# A2: Code quality checks (black/isort/ruff/compileall)
|
||||
# A3: Mypy type check
|
||||
# A4: Advisory checks (bandit + pip-audit + vulture + release scripts syntax)
|
||||
# Group B(PG 依赖,独立并行):
|
||||
# B1: Alembic migrations validation(需要 PG)
|
||||
#
|
||||
# 所有子任务同时启动,最后汇总结果。
|
||||
set -eu
|
||||
|
||||
echo "=== CI Validate: 开始全量代码质量检查 ==="
|
||||
|
||||
# --- 密钥检测 ---
|
||||
echo "=== CI Validate: 并行化代码质量检查 ==="
|
||||
echo ""
|
||||
echo "=== [1/8] Secret detection (detect-secrets) ==="
|
||||
python3 -m pip install -q detect-secrets
|
||||
detect-secrets --version
|
||||
|
||||
detect-secrets scan \
|
||||
--all-files \
|
||||
--exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \
|
||||
--exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \
|
||||
--exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \
|
||||
--disable-plugin Base64HighEntropyString \
|
||||
--disable-plugin HexHighEntropyString \
|
||||
--disable-plugin BasicAuthDetector \
|
||||
--disable-plugin KeywordDetector \
|
||||
--disable-plugin IPPublicDetector \
|
||||
> /tmp/secrets-scan.json 2>&1
|
||||
# ============================================================
|
||||
# 配置
|
||||
# ============================================================
|
||||
LOG_DIR="/tmp/validate_logs"
|
||||
rm -rf "$LOG_DIR"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
FOUND=$(python3 -c "
|
||||
# 子任务结果文件(每个记录 exit code)
|
||||
RESULT_FILE="$LOG_DIR/results.json"
|
||||
echo '{}' > "$RESULT_FILE"
|
||||
|
||||
# ============================================================
|
||||
# 工具函数
|
||||
# ============================================================
|
||||
|
||||
# 记录子任务结果
|
||||
# 用法: record_result <name> <exit_code> <blocking>
|
||||
record_result() {
|
||||
local name="$1"
|
||||
local exit_code="$2"
|
||||
local blocking="$3" # "yes" or "no"
|
||||
# 写入独立文件,避免并发写 JSON 冲突
|
||||
echo "${exit_code}" > "$LOG_DIR/exit_${name}"
|
||||
echo "${blocking}" > "$LOG_DIR/blocking_${name}"
|
||||
}
|
||||
|
||||
# ============================================================
|
||||
# 子任务定义(每个子任务输出写入独立日志文件)
|
||||
# ============================================================
|
||||
|
||||
# --- A1: Secret detection ---
|
||||
task_secret_detection() {
|
||||
local log="$LOG_DIR/task_secret_detection.log"
|
||||
exec > "$log" 2>&1
|
||||
set +e
|
||||
|
||||
echo "=== [A1] Secret detection (detect-secrets) ==="
|
||||
python3 -m pip install -q detect-secrets
|
||||
detect-secrets --version
|
||||
|
||||
detect-secrets scan \
|
||||
--all-files \
|
||||
--exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \
|
||||
--exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \
|
||||
--exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \
|
||||
--disable-plugin Base64HighEntropyString \
|
||||
--disable-plugin HexHighEntropyString \
|
||||
--disable-plugin BasicAuthDetector \
|
||||
--disable-plugin KeywordDetector \
|
||||
--disable-plugin IPPublicDetector \
|
||||
> /tmp/secrets-scan.json 2>&1
|
||||
|
||||
FOUND=$(python3 -c "
|
||||
import json
|
||||
try:
|
||||
with open('/tmp/secrets-scan.json') as f:
|
||||
@@ -35,11 +81,12 @@ except Exception:
|
||||
print('error')
|
||||
")
|
||||
|
||||
echo "Secrets detected: $FOUND"
|
||||
if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then
|
||||
echo ""
|
||||
echo "=== Secret details ==="
|
||||
python3 -c "
|
||||
echo "Secrets detected: $FOUND"
|
||||
local exit_code=0
|
||||
if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then
|
||||
echo ""
|
||||
echo "=== Secret details ==="
|
||||
python3 -c "
|
||||
import json
|
||||
with open('/tmp/secrets-scan.json') as f:
|
||||
data = json.load(f)
|
||||
@@ -50,28 +97,37 @@ for fpath, items in data.get('results', {}).items():
|
||||
hashed = item.get('hashed_secret', '')[:16]
|
||||
print(f' {fpath}:{line} [{stype}] {hashed}...')
|
||||
"
|
||||
echo ""
|
||||
echo "ERROR: Potential secrets detected in code!"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ Secret scan passed"
|
||||
echo ""
|
||||
echo "ERROR: Potential secrets detected in code!"
|
||||
exit_code=1
|
||||
else
|
||||
echo "✅ Secret scan passed"
|
||||
fi
|
||||
|
||||
# --- 增量/全量模式判断 ---
|
||||
echo ""
|
||||
echo "=== [2/8] Code quality checks ==="
|
||||
SCAN_MODE="full"
|
||||
CHANGED_PY_FILES=""
|
||||
record_result "secret_detection" "$exit_code" "yes"
|
||||
exit $exit_code
|
||||
}
|
||||
|
||||
if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then
|
||||
PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||')
|
||||
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100"
|
||||
# --- A2: Code quality checks ---
|
||||
task_code_quality() {
|
||||
local log="$LOG_DIR/task_code_quality.log"
|
||||
exec > "$log" 2>&1
|
||||
set +e
|
||||
RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL")
|
||||
HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
|
||||
BODY=$(echo "$RESPONSE" | sed '$d')
|
||||
set -e
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
CHANGED_PY_FILES=$(echo "$BODY" | python3 -c "
|
||||
|
||||
echo "=== [A2] Code quality checks (black/isort/ruff/compileall) ==="
|
||||
|
||||
# --- 增量/全量模式判断 ---
|
||||
local SCAN_MODE="full"
|
||||
local CHANGED_PY_FILES=""
|
||||
|
||||
if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then
|
||||
PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||')
|
||||
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100"
|
||||
RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL")
|
||||
HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
|
||||
BODY=$(echo "$RESPONSE" | sed '$d')
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
CHANGED_PY_FILES=$(echo "$BODY" | python3 -c "
|
||||
import json, sys
|
||||
try:
|
||||
files = json.load(sys.stdin)
|
||||
@@ -80,160 +136,205 @@ try:
|
||||
except Exception:
|
||||
print('')
|
||||
")
|
||||
if [ -n "$CHANGED_PY_FILES" ]; then
|
||||
SCAN_MODE="incremental"
|
||||
echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed"
|
||||
if [ -n "$CHANGED_PY_FILES" ]; then
|
||||
SCAN_MODE="incremental"
|
||||
echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed"
|
||||
else
|
||||
SCAN_MODE="skip_py"
|
||||
echo "No Python files changed in this PR"
|
||||
fi
|
||||
else
|
||||
SCAN_MODE="skip_py"
|
||||
echo "No Python files changed in this PR"
|
||||
echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan"
|
||||
fi
|
||||
else
|
||||
echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan"
|
||||
echo "Full scan mode (not a PR event)"
|
||||
fi
|
||||
else
|
||||
echo "Full scan mode (not a PR event)"
|
||||
fi
|
||||
|
||||
if [ "$SCAN_MODE" = "incremental" ]; then
|
||||
# 防御性过滤:磁盘上不存在的文件(已删除文件)不参与检查,
|
||||
# 避免 black/isort/ruff 报 "Path does not exist" 错误。
|
||||
EXISTING_PY_FILES=""
|
||||
for f in $CHANGED_PY_FILES; do
|
||||
if [ -f "$f" ]; then
|
||||
if [ -z "$EXISTING_PY_FILES" ]; then
|
||||
EXISTING_PY_FILES="$f"
|
||||
local exit_code=0
|
||||
|
||||
if [ "$SCAN_MODE" = "incremental" ]; then
|
||||
# 防御性过滤:磁盘上不存在的文件(已删除文件)不参与检查
|
||||
local EXISTING_PY_FILES=""
|
||||
for f in $CHANGED_PY_FILES; do
|
||||
if [ -f "$f" ]; then
|
||||
if [ -z "$EXISTING_PY_FILES" ]; then
|
||||
EXISTING_PY_FILES="$f"
|
||||
else
|
||||
EXISTING_PY_FILES="$EXISTING_PY_FILES $f"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
CHANGED_PY_FILES="$EXISTING_PY_FILES"
|
||||
|
||||
python3 -m compileall -q $CHANGED_PY_FILES || exit_code=$?
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
python3 -m black --check --fast $CHANGED_PY_FILES || exit_code=$?
|
||||
fi
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
python3 -m isort --check-only $CHANGED_PY_FILES || exit_code=$?
|
||||
fi
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
local RUFF_FILES
|
||||
RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | grep -v '^$' | xargs)
|
||||
if [ -n "$RUFF_FILES" ]; then
|
||||
python3 -m ruff check $RUFF_FILES --statistics || exit_code=$?
|
||||
else
|
||||
EXISTING_PY_FILES="$EXISTING_PY_FILES $f"
|
||||
echo "No ruff-checkable files changed, skipping"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
CHANGED_PY_FILES="$EXISTING_PY_FILES"
|
||||
|
||||
python3 -m compileall -q $CHANGED_PY_FILES
|
||||
python3 -m black --check --fast $CHANGED_PY_FILES
|
||||
python3 -m isort --check-only $CHANGED_PY_FILES
|
||||
RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | grep -v '^$' | xargs)
|
||||
if [ -n "$RUFF_FILES" ]; then
|
||||
python3 -m ruff check $RUFF_FILES --statistics
|
||||
elif [ "$SCAN_MODE" = "skip_py" ]; then
|
||||
echo "No Python files changed - skipping Python lint checks"
|
||||
else
|
||||
echo "No ruff-checkable files changed, skipping"
|
||||
fi
|
||||
elif [ "$SCAN_MODE" = "skip_py" ]; then
|
||||
echo "No Python files changed - skipping Python lint checks"
|
||||
else
|
||||
echo "Full scan mode"
|
||||
python3 -m compileall -q alembic apps packages tests scripts
|
||||
python3 -m black --check --fast alembic apps packages tests scripts
|
||||
python3 -m isort --check-only alembic apps packages tests scripts
|
||||
python3 -m ruff check apps packages tests --statistics
|
||||
fi
|
||||
echo "✅ Code quality checks passed"
|
||||
|
||||
# --- Mypy 类型检查 ---
|
||||
echo ""
|
||||
echo "=== [3/8] Type check (mypy) ==="
|
||||
bash scripts/ci/mypy_check.sh
|
||||
echo "✅ Mypy type check passed"
|
||||
|
||||
# --- Bandit 安全扫描(仅告警) ---
|
||||
echo ""
|
||||
echo "=== [4/8] Security scan (bandit, advisory only) ==="
|
||||
set +e
|
||||
bandit -r apps packages -q -ll
|
||||
BANDIT_EXIT=$?
|
||||
set -e
|
||||
if [ "$BANDIT_EXIT" -ne 0 ]; then
|
||||
echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)"
|
||||
else
|
||||
echo "✅ Bandit security scan passed"
|
||||
fi
|
||||
|
||||
# --- Pip-audit 依赖漏洞扫描(仅告警) ---
|
||||
echo ""
|
||||
echo "=== [5/8] Python dependency vulnerability scan (pip-audit, advisory only) ==="
|
||||
python3 -m pip install -q pip-audit
|
||||
pip-audit --version
|
||||
EXIT_CODE=0
|
||||
for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do
|
||||
if [ -f "$req_file" ]; then
|
||||
echo "--- Scanning $req_file ---"
|
||||
pip-audit -r "$req_file" --desc on 2>&1 | head -40 || EXIT_CODE=$?
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)"
|
||||
|
||||
# --- Vulture 死代码检测(仅告警) ---
|
||||
echo ""
|
||||
echo "=== [6/8] Dead code detection (vulture, advisory only) ==="
|
||||
set +e
|
||||
python3 -m pip install -q vulture
|
||||
vulture --version
|
||||
echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。"
|
||||
echo ""
|
||||
vulture apps packages scripts \
|
||||
--exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \
|
||||
--min-confidence 70 \
|
||||
2>&1 | sort -t'(' -k2 -rn | head -80
|
||||
echo ""
|
||||
echo "=== vulture scan summary ==="
|
||||
echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)"
|
||||
echo "建议:定期人工审查高置信度(>=90%)条目"
|
||||
set -e
|
||||
|
||||
# --- Release 脚本语法校验 ---
|
||||
echo ""
|
||||
echo "=== [7/8] Release scripts syntax validation ==="
|
||||
bash -n scripts/backup_postgres.sh
|
||||
bash -n scripts/restore_postgres_plan.sh
|
||||
bash -n scripts/init_production_env.sh
|
||||
echo "✅ Release scripts syntax OK"
|
||||
|
||||
# --- Alembic 迁移验证 ---
|
||||
echo ""
|
||||
echo "=== [8/8] Alembic migrations validation ==="
|
||||
|
||||
# --- DooD模式检测:确定宿主机访问地址 ---
|
||||
# DooD模式下,docker run启动的容器跑在宿主机Docker上
|
||||
# 需要用宿主机IP访问映射端口
|
||||
# 检测策略:host.docker.internal -> docker0桥接IP -> 容器IP直连 -> 默认网关 -> 127.0.0.1
|
||||
detect_docker_host() {
|
||||
local test_port="${1:-5432}"
|
||||
|
||||
# 候选IP列表
|
||||
local candidates=()
|
||||
|
||||
# 1. host.docker.internal(runner配置了--add-host时可用)
|
||||
if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then
|
||||
candidates+=("host.docker.internal")
|
||||
echo "Full scan mode"
|
||||
python3 -m compileall -q alembic apps packages tests scripts || exit_code=$?
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
python3 -m black --check --fast alembic apps packages tests scripts || exit_code=$?
|
||||
fi
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
python3 -m isort --check-only alembic apps packages tests scripts || exit_code=$?
|
||||
fi
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
python3 -m ruff check apps packages tests --statistics || exit_code=$?
|
||||
fi
|
||||
fi
|
||||
|
||||
# 2. docker0 桥接网关 (172.17.0.1)
|
||||
candidates+=("172.17.0.1")
|
||||
|
||||
# 3. 默认网关(容器网络的网关即宿主机)
|
||||
local gw=""
|
||||
gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1)
|
||||
if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then
|
||||
candidates+=("$gw")
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
echo "✅ Code quality checks passed"
|
||||
else
|
||||
echo "❌ Code quality checks FAILED"
|
||||
fi
|
||||
|
||||
# 4. 宿主机可能的IP:容器同网段的.1或.254
|
||||
local my_ip=""
|
||||
my_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
|
||||
if [ -n "$my_ip" ]; then
|
||||
# 尝试同网段的常见宿主机IP
|
||||
local subnet=$(echo "$my_ip" | cut -d. -f1-3)
|
||||
candidates+=("${subnet}.1")
|
||||
candidates+=("${subnet}.254")
|
||||
record_result "code_quality" "$exit_code" "yes"
|
||||
exit $exit_code
|
||||
}
|
||||
|
||||
# --- A3: Mypy type check ---
|
||||
task_mypy() {
|
||||
local log="$LOG_DIR/task_mypy.log"
|
||||
exec > "$log" 2>&1
|
||||
set +e
|
||||
|
||||
echo "=== [A3] Type check (mypy) ==="
|
||||
bash scripts/ci/mypy_check.sh
|
||||
local exit_code=$?
|
||||
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
echo "✅ Mypy type check passed"
|
||||
else
|
||||
echo "❌ Mypy type check FAILED"
|
||||
fi
|
||||
|
||||
# 5. 127.0.0.1 最后尝试
|
||||
candidates+=("127.0.0.1")
|
||||
record_result "mypy" "$exit_code" "yes"
|
||||
exit $exit_code
|
||||
}
|
||||
|
||||
# 测试每个候选IP
|
||||
for candidate in "${candidates[@]}"; do
|
||||
if python3 -c "
|
||||
# --- A4: Advisory checks (bandit + pip-audit + vulture + release scripts syntax) ---
|
||||
task_advisory() {
|
||||
local log="$LOG_DIR/task_advisory.log"
|
||||
exec > "$log" 2>&1
|
||||
set +e
|
||||
|
||||
# --- Bandit 安全扫描(仅告警) ---
|
||||
echo "=== [A4a] Security scan (bandit, advisory only) ==="
|
||||
bandit -r apps packages -q -ll
|
||||
local BANDIT_EXIT=$?
|
||||
if [ "$BANDIT_EXIT" -ne 0 ]; then
|
||||
echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)"
|
||||
else
|
||||
echo "✅ Bandit security scan passed"
|
||||
fi
|
||||
|
||||
# --- Pip-audit 依赖漏洞扫描(仅告警) ---
|
||||
echo ""
|
||||
echo "=== [A4b] Python dependency vulnerability scan (pip-audit, advisory only) ==="
|
||||
python3 -m pip install -q pip-audit
|
||||
pip-audit --version
|
||||
for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do
|
||||
if [ -f "$req_file" ]; then
|
||||
echo "--- Scanning $req_file ---"
|
||||
pip-audit -r "$req_file" --desc on 2>&1 | head -40 || true
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)"
|
||||
|
||||
# --- Vulture 死代码检测(仅告警) ---
|
||||
echo ""
|
||||
echo "=== [A4c] Dead code detection (vulture, advisory only) ==="
|
||||
python3 -m pip install -q vulture
|
||||
vulture --version
|
||||
echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。"
|
||||
echo ""
|
||||
vulture apps packages scripts \
|
||||
--exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \
|
||||
--min-confidence 70 \
|
||||
2>&1 | sort -t'(' -k2 -rn | head -80
|
||||
echo ""
|
||||
echo "=== vulture scan summary ==="
|
||||
echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)"
|
||||
echo "建议:定期人工审查高置信度(>=90%)条目"
|
||||
|
||||
# --- Release 脚本语法校验(不阻断) ---
|
||||
echo ""
|
||||
echo "=== [A4d] Release scripts syntax validation ==="
|
||||
local syntax_exit=0
|
||||
bash -n scripts/backup_postgres.sh || syntax_exit=$?
|
||||
bash -n scripts/restore_postgres_plan.sh || syntax_exit=$?
|
||||
bash -n scripts/init_production_env.sh || syntax_exit=$?
|
||||
if [ $syntax_exit -eq 0 ]; then
|
||||
echo "✅ Release scripts syntax OK"
|
||||
else
|
||||
echo "⚠️ Release scripts have syntax issues (advisory)"
|
||||
fi
|
||||
|
||||
# Advisory checks never block
|
||||
record_result "advisory" 0 "no"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# --- B1: Alembic migrations validation (needs PG) ---
|
||||
task_alembic() {
|
||||
local log="$LOG_DIR/task_alembic.log"
|
||||
exec > "$log" 2>&1
|
||||
set +e
|
||||
|
||||
echo "=== [B1] Alembic migrations validation ==="
|
||||
|
||||
# --- DooD模式检测:确定宿主机访问地址 ---
|
||||
detect_docker_host() {
|
||||
local test_port="${1:-5432}"
|
||||
local candidates=()
|
||||
|
||||
# 1. host.docker.internal
|
||||
if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then
|
||||
candidates+=("host.docker.internal")
|
||||
fi
|
||||
|
||||
# 2. docker0 桥接网关 (172.17.0.1)
|
||||
candidates+=("172.17.0.1")
|
||||
|
||||
# 3. 默认网关
|
||||
local gw=""
|
||||
gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1)
|
||||
if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then
|
||||
candidates+=("$gw")
|
||||
fi
|
||||
|
||||
# 4. 宿主机可能的IP
|
||||
local my_ip=""
|
||||
my_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
|
||||
if [ -n "$my_ip" ]; then
|
||||
local subnet
|
||||
subnet=$(echo "$my_ip" | cut -d. -f1-3)
|
||||
candidates+=("${subnet}.1")
|
||||
candidates+=("${subnet}.254")
|
||||
fi
|
||||
|
||||
# 5. 127.0.0.1
|
||||
candidates+=("127.0.0.1")
|
||||
|
||||
for candidate in "${candidates[@]}"; do
|
||||
if python3 -c "
|
||||
import socket
|
||||
s = socket.socket()
|
||||
s.settimeout(2)
|
||||
@@ -244,87 +345,87 @@ try:
|
||||
except:
|
||||
pass
|
||||
" 2>/dev/null | grep -q ok; then
|
||||
echo "$candidate"
|
||||
return 0
|
||||
echo "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo "127.0.0.1"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 指数退避TCP连接检查函数
|
||||
wait_tcp_ready() {
|
||||
local host="$1"
|
||||
local port="$2"
|
||||
local max_attempts="${3:-5}"
|
||||
local delay=1
|
||||
local attempt=1
|
||||
while [ "$attempt" -le "$max_attempts" ]; do
|
||||
if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
echo "TCP连接尝试 $attempt/$max_attempts 失败,${delay}s后重试..."
|
||||
sleep "$delay"
|
||||
delay=$((delay * 2))
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# 获取宿主机IP
|
||||
local PG_HOST
|
||||
if [ -S /var/run/docker.sock ]; then
|
||||
PG_HOST=$(detect_docker_host 5433)
|
||||
if [ "$PG_HOST" = "127.0.0.1" ]; then
|
||||
PG_HOST=$(detect_docker_host 22)
|
||||
fi
|
||||
done
|
||||
|
||||
# 都失败则返回127.0.0.1
|
||||
echo "127.0.0.1"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 获取宿主机IP(先尝试用共享PG端口5433测试,再回退到其他端口)
|
||||
if [ -S /var/run/docker.sock ]; then
|
||||
# 先用共享PG端口5433探测
|
||||
DOCKER_HOST_IP=$(detect_docker_host 5433)
|
||||
if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then
|
||||
# 如果共享PG端口探测失败,说明不在DooD或共享PG不可用,再试其他端口
|
||||
DOCKER_HOST_IP=$(detect_docker_host 22)
|
||||
echo "检测到DooD模式(/var/run/docker.sock已挂载),宿主机地址: $PG_HOST"
|
||||
else
|
||||
PG_HOST="127.0.0.1"
|
||||
echo "非DooD模式,使用 127.0.0.1"
|
||||
fi
|
||||
echo "检测到DooD模式(/var/run/docker.sock已挂载),宿主机地址: $DOCKER_HOST_IP"
|
||||
else
|
||||
DOCKER_HOST_IP="127.0.0.1"
|
||||
echo "非DooD模式,使用 127.0.0.1"
|
||||
fi
|
||||
PG_HOST="$DOCKER_HOST_IP"
|
||||
echo "PG host: $PG_HOST"
|
||||
echo "PG host: $PG_HOST"
|
||||
|
||||
# 指数退避TCP连接检查函数
|
||||
# 用法: wait_tcp_ready host port max_attempts
|
||||
wait_tcp_ready() {
|
||||
local host="$1"
|
||||
local port="$2"
|
||||
local max_attempts="${3:-5}"
|
||||
local delay=1
|
||||
local attempt=1
|
||||
while [ "$attempt" -le "$max_attempts" ]; do
|
||||
if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
echo "TCP连接尝试 $attempt/$max_attempts 失败,${delay}s后重试..."
|
||||
sleep "$delay"
|
||||
delay=$((delay * 2))
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
local USE_SHARED_PG="${CI_USE_SHARED_PG:-false}"
|
||||
local exit_code=0
|
||||
|
||||
USE_SHARED_PG="${CI_USE_SHARED_PG:-false}"
|
||||
if [ "$USE_SHARED_PG" = "true" ]; then
|
||||
# 使用常驻共享PG实例
|
||||
echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true)"
|
||||
local SHARED_PG_HOST="$PG_HOST"
|
||||
local SHARED_PG_PORT="5433"
|
||||
local SHARED_PG_USER="postgres"
|
||||
local SHARED_PG_PASSWORD="ci_pg_2026!"
|
||||
local CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}"
|
||||
|
||||
if [ "$USE_SHARED_PG" = "true" ]; then
|
||||
# 使用常驻共享PG实例(host.docker.internal:5433)
|
||||
echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true)"
|
||||
SHARED_PG_HOST="$PG_HOST"
|
||||
SHARED_PG_PORT="5433"
|
||||
SHARED_PG_USER="postgres"
|
||||
SHARED_PG_PASSWORD="ci_pg_2026!"
|
||||
CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}"
|
||||
echo "等待共享PG连接就绪..."
|
||||
wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5
|
||||
|
||||
echo "等待共享PG连接就绪..."
|
||||
wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5
|
||||
|
||||
# 创建独立数据库
|
||||
echo "创建测试数据库: $CI_DB_NAME"
|
||||
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||||
echo "创建测试数据库: $CI_DB_NAME"
|
||||
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
|
||||
cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"')
|
||||
cur.close()
|
||||
conn.close()
|
||||
"
|
||||
export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}"
|
||||
echo "✅ 共享PG数据库已创建: $CI_DB_NAME"
|
||||
" || exit_code=$?
|
||||
|
||||
# 执行迁移
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
|
||||
echo "✅ Alembic migrations applied successfully"
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}"
|
||||
echo "✅ 共享PG数据库已创建: $CI_DB_NAME"
|
||||
|
||||
# 清理数据库
|
||||
echo "清理测试数据库: $CI_DB_NAME"
|
||||
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head || exit_code=$?
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
echo "✅ Alembic migrations applied successfully"
|
||||
fi
|
||||
|
||||
# 清理数据库
|
||||
echo "清理测试数据库: $CI_DB_NAME"
|
||||
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
|
||||
conn.autocommit = True
|
||||
@@ -333,49 +434,220 @@ cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
|
||||
cur.close()
|
||||
conn.close()
|
||||
" 2>/dev/null || echo "WARN: 数据库清理失败(可能已被清理)"
|
||||
echo "✅ 共享PG数据库已清理"
|
||||
else
|
||||
# 使用临时PG容器(默认模式)
|
||||
echo "使用临时PG容器模式"
|
||||
PG_CONTAINER=ci-pg-validate-${GITHUB_RUN_ID:-$$}
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
--shm-size=256m \
|
||||
-e POSTGRES_USER=postgres \
|
||||
-e POSTGRES_PASSWORD=postgres \
|
||||
-e POSTGRES_DB=xiaoxia_saas \
|
||||
-P \
|
||||
--health-cmd "pg_isready -U postgres" \
|
||||
--health-interval 3s \
|
||||
--health-timeout 3s \
|
||||
--health-retries 20 \
|
||||
postgres:16-alpine
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
|
||||
echo "PostgreSQL port: $PG_PORT"
|
||||
export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas"
|
||||
|
||||
# 等待容器健康
|
||||
for i in $(seq 1 30); do
|
||||
if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
|
||||
echo "PostgreSQL container is healthy on port $PG_PORT"
|
||||
break
|
||||
echo "✅ 共享PG数据库已清理"
|
||||
fi
|
||||
echo "Waiting for PostgreSQL container health... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy
|
||||
|
||||
# TCP连通性检查(指数退避)
|
||||
echo "验证TCP连通性 ($PG_HOST:$PG_PORT)..."
|
||||
wait_tcp_ready "$PG_HOST" "$PG_PORT" 5
|
||||
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
|
||||
else
|
||||
# 使用临时PG容器
|
||||
echo "使用临时PG容器模式"
|
||||
local PG_CONTAINER="ci-pg-validate-${GITHUB_RUN_ID:-$$}"
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
--shm-size=256m \
|
||||
-e POSTGRES_USER=postgres \
|
||||
-e POSTGRES_PASSWORD=postgres \
|
||||
-e POSTGRES_DB=xiaoxia_saas \
|
||||
-P \
|
||||
--health-cmd "pg_isready -U postgres" \
|
||||
--health-interval 3s \
|
||||
--health-timeout 3s \
|
||||
--health-retries 20 \
|
||||
postgres:16-alpine || exit_code=$?
|
||||
|
||||
# 执行迁移
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
|
||||
echo "✅ Alembic migrations applied successfully"
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
local PG_PORT
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
|
||||
echo "PostgreSQL port: $PG_PORT"
|
||||
export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas"
|
||||
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
fi
|
||||
# 等待容器健康
|
||||
local i
|
||||
for i in $(seq 1 30); do
|
||||
if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
|
||||
echo "PostgreSQL container is healthy on port $PG_PORT"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for PostgreSQL container health... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if ! docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
|
||||
echo "❌ PostgreSQL container failed health check"
|
||||
exit_code=1
|
||||
else
|
||||
# TCP连通性检查
|
||||
echo "验证TCP连通性 ($PG_HOST:$PG_PORT)..."
|
||||
if wait_tcp_ready "$PG_HOST" "$PG_PORT" 5; then
|
||||
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
|
||||
|
||||
# 执行迁移
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head || exit_code=$?
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
echo "✅ Alembic migrations applied successfully"
|
||||
fi
|
||||
else
|
||||
echo "❌ TCP connectivity to PostgreSQL failed"
|
||||
exit_code=1
|
||||
fi
|
||||
fi
|
||||
|
||||
# 清理
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
echo "✅ Alembic migrations validation passed"
|
||||
else
|
||||
echo "❌ Alembic migrations validation FAILED"
|
||||
fi
|
||||
|
||||
record_result "alembic" "$exit_code" "yes"
|
||||
exit $exit_code
|
||||
}
|
||||
|
||||
# ============================================================
|
||||
# 主流程:并行启动所有子任务
|
||||
# ============================================================
|
||||
|
||||
echo "启动并行检查(5 个子任务同时运行)..."
|
||||
echo ""
|
||||
|
||||
# 记录开始时间
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
# 启动所有子任务(后台运行)
|
||||
task_secret_detection &
|
||||
PID_A1=$!
|
||||
|
||||
task_code_quality &
|
||||
PID_A2=$!
|
||||
|
||||
task_mypy &
|
||||
PID_A3=$!
|
||||
|
||||
task_advisory &
|
||||
PID_A4=$!
|
||||
|
||||
task_alembic &
|
||||
PID_B1=$!
|
||||
|
||||
echo "子任务 PID: A1=$PID_A1 A2=$PID_A2 A3=$PID_A3 A4=$PID_A4 B1=$PID_B1"
|
||||
echo ""
|
||||
|
||||
# 等待所有后台任务完成(不因单个失败而中断)
|
||||
# 使用 set +e 临时取消 errexit
|
||||
set +e
|
||||
wait $PID_A1; EXIT_A1=$?
|
||||
wait $PID_A2; EXIT_A2=$?
|
||||
wait $PID_A3; EXIT_A3=$?
|
||||
wait $PID_A4; EXIT_A4=$?
|
||||
wait $PID_B1; EXIT_B1=$?
|
||||
set -e
|
||||
|
||||
# 计算耗时
|
||||
END_TIME=$(date +%s)
|
||||
ELAPSED=$((END_TIME - START_TIME))
|
||||
|
||||
# ============================================================
|
||||
# 结果汇总
|
||||
# ============================================================
|
||||
|
||||
echo ""
|
||||
echo "=== CI Validate: 所有检查通过 ✅ ==="
|
||||
echo "============================================"
|
||||
echo " CI Validate 结果汇总(耗时 ${ELAPSED}s)"
|
||||
echo "============================================"
|
||||
echo ""
|
||||
|
||||
# 定义任务信息:名称 | PID | 退出码 | 描述 | 是否阻断
|
||||
declare -A TASK_DESC
|
||||
TASK_DESC[A1]="Secret detection"
|
||||
TASK_DESC[A2]="Code quality (black/isort/ruff)"
|
||||
TASK_DESC[A3]="Mypy type check"
|
||||
TASK_DESC[A4]="Advisory (bandit/pip-audit/vulture/syntax)"
|
||||
TASK_DESC[B1]="Alembic migrations"
|
||||
|
||||
declare -A TASK_PID
|
||||
TASK_PID[A1]=$PID_A1
|
||||
TASK_PID[A2]=$PID_A2
|
||||
TASK_PID[A3]=$PID_A3
|
||||
TASK_PID[A4]=$PID_A4
|
||||
TASK_PID[B1]=$PID_B1
|
||||
|
||||
declare -A TASK_EXIT
|
||||
TASK_EXIT[A1]=$EXIT_A1
|
||||
TASK_EXIT[A2]=$EXIT_A2
|
||||
TASK_EXIT[A3]=$EXIT_A3
|
||||
TASK_EXIT[A4]=$EXIT_A4
|
||||
TASK_EXIT[B1]=$EXIT_B1
|
||||
|
||||
declare -A TASK_LOG
|
||||
TASK_LOG[A1]="task_secret_detection"
|
||||
TASK_LOG[A2]="task_code_quality"
|
||||
TASK_LOG[A3]="task_mypy"
|
||||
TASK_LOG[A4]="task_advisory"
|
||||
TASK_LOG[B1]="task_alembic"
|
||||
|
||||
declare -A TASK_BLOCKING
|
||||
TASK_BLOCKING[A1]="yes"
|
||||
TASK_BLOCKING[A2]="yes"
|
||||
TASK_BLOCKING[A3]="yes"
|
||||
TASK_BLOCKING[A4]="no"
|
||||
TASK_BLOCKING[B1]="yes"
|
||||
|
||||
OVERALL_EXIT=0
|
||||
FAILED_TASKS=()
|
||||
|
||||
# 按固定顺序打印摘要
|
||||
for task_id in A1 A2 A3 A4 B1; do
|
||||
local_exit=${TASK_EXIT[$task_id]}
|
||||
local_desc=${TASK_DESC[$task_id]}
|
||||
local_blocking=${TASK_BLOCKING[$task_id]}
|
||||
|
||||
if [ "$local_exit" -eq 0 ]; then
|
||||
echo " ✅ $task_id: $local_desc — PASSED"
|
||||
else
|
||||
if [ "$local_blocking" = "yes" ]; then
|
||||
echo " ❌ $task_id: $local_desc — FAILED (blocking)"
|
||||
OVERALL_EXIT=1
|
||||
FAILED_TASKS+=("$task_id")
|
||||
else
|
||||
echo " ⚠️ $task_id: $local_desc — FAILED (advisory, not blocking)"
|
||||
# Advisory tasks don't cause overall failure
|
||||
if [ "$local_blocking" = "no" ]; then
|
||||
echo " → 告警类检查,不阻断流水线"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
# 打印失败任务的完整日志
|
||||
if [ ${#FAILED_TASKS[@]} -gt 0 ]; then
|
||||
echo "============================================"
|
||||
echo " 失败任务详细日志"
|
||||
echo "============================================"
|
||||
for task_id in "${FAILED_TASKS[@]}"; do
|
||||
local_log="${TASK_LOG[$task_id]}"
|
||||
local_desc="${TASK_DESC[$task_id]}"
|
||||
echo ""
|
||||
echo "--- $task_id: $local_desc ---"
|
||||
if [ -f "$LOG_DIR/${local_log}.log" ]; then
|
||||
cat "$LOG_DIR/${local_log}.log"
|
||||
else
|
||||
echo "(日志文件不存在)"
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
fi
|
||||
|
||||
# 最终结论
|
||||
echo ""
|
||||
if [ $OVERALL_EXIT -eq 0 ]; then
|
||||
echo "=== CI Validate: 所有检查通过 ✅ (并行耗时 ${ELAPSED}s) ==="
|
||||
else
|
||||
echo "=== CI Validate: 存在阻断性检查失败 ❌ (并行耗时 ${ELAPSED}s) ==="
|
||||
fi
|
||||
|
||||
exit $OVERALL_EXIT
|
||||
|
||||
@@ -1,24 +1,96 @@
|
||||
#!/bin/sh
|
||||
# CI 公共步骤:前端依赖安装(在 docker node 容器中运行)
|
||||
# 用法:step_frontend_install.sh [模式]
|
||||
# 模式: full (默认) - 完整安装所有依赖
|
||||
# vitest - 同full(保持接口兼容)
|
||||
# CI 公共步骤:前端依赖安装(Docker Volume 持久化缓存方案)
|
||||
# 通过 Docker named volume 缓存 node_modules,按 package-lock.json hash 命名
|
||||
# 缓存命中时跳过 npm ci,直接复用已有 volume
|
||||
set -eu
|
||||
|
||||
MODE="${1:-full}"
|
||||
|
||||
echo "=== 前端依赖安装开始 (模式: $MODE) ==="
|
||||
|
||||
# npm ci 带重试(网络不稳定时自动重试)
|
||||
for i in 1 2 3; do
|
||||
docker run --rm \
|
||||
-v "$PWD:/workspace" \
|
||||
-w /workspace/apps/web \
|
||||
docker.m.daocloud.io/library/node:20 \
|
||||
sh -lc "npm ci --no-audit --no-fund" && break
|
||||
echo "npm ci 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 5
|
||||
done
|
||||
# npm国内镜像源(加速下载,减少网络失败)
|
||||
NPM_REGISTRY="https://registry.npmmirror.com"
|
||||
|
||||
# 缓存配置 — 与 step_frontend_run.sh 保持一致
|
||||
LOCK_FILE="apps/web/package-lock.json"
|
||||
VOLUME_PREFIX="ci-web-nm-"
|
||||
KEEP_CACHE_COUNT=5
|
||||
|
||||
# 计算 package-lock.json 的 md5 hash 作为缓存 key
|
||||
VOLUME_NAME=""
|
||||
if [ -f "$LOCK_FILE" ]; then
|
||||
LOCK_HASH=$(md5sum "$LOCK_FILE" | cut -c1-12)
|
||||
VOLUME_NAME="${VOLUME_PREFIX}${LOCK_HASH}"
|
||||
echo "缓存 key: $LOCK_HASH (volume: $VOLUME_NAME)"
|
||||
else
|
||||
echo "警告: 未找到 $LOCK_FILE,将不使用持久化缓存"
|
||||
fi
|
||||
|
||||
# 检查 volume 是否存在(缓存命中)
|
||||
CACHE_HIT=0
|
||||
if [ -n "$VOLUME_NAME" ]; then
|
||||
if docker volume inspect "$VOLUME_NAME" >/dev/null 2>&1; then
|
||||
CACHE_HIT=1
|
||||
echo "缓存命中!复用 volume: $VOLUME_NAME"
|
||||
else
|
||||
echo "缓存未命中,创建 volume 并安装依赖..."
|
||||
# 创建 volume(失败则降级为无缓存模式)
|
||||
if ! docker volume create "$VOLUME_NAME" >/dev/null 2>&1; then
|
||||
echo "警告: 创建 volume 失败,降级为无缓存模式"
|
||||
VOLUME_NAME=""
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# 构建 docker run 的 volume 挂载参数(空时不挂载)
|
||||
VOLUME_ARGS=""
|
||||
if [ -n "$VOLUME_NAME" ]; then
|
||||
VOLUME_ARGS="-v ${VOLUME_NAME}:/workspace/apps/web/node_modules"
|
||||
fi
|
||||
|
||||
# 缓存未命中时执行 npm ci
|
||||
if [ "$CACHE_HIT" -eq 0 ]; then
|
||||
for i in 1 2 3; do
|
||||
echo "npm ci 尝试 $i/3 (镜像: $NPM_REGISTRY)"
|
||||
docker run --rm -v "$PWD:/workspace" $VOLUME_ARGS -w /workspace/apps/web docker.m.daocloud.io/library/node:20 sh -lc "npm config set registry $NPM_REGISTRY && npm ci --no-audit --no-fund" && break
|
||||
echo "npm ci 失败,重试 $i/3..."
|
||||
[ $i -eq 3 ] && exit 1
|
||||
sleep 10
|
||||
done
|
||||
else
|
||||
echo "跳过 npm ci,直接使用缓存的 node_modules"
|
||||
fi
|
||||
|
||||
# 清理旧缓存 volume(保留最近 N 个,防止磁盘占用无限增长)
|
||||
if [ -n "$VOLUME_PREFIX" ]; then
|
||||
echo "清理旧缓存 volume(保留最近 ${KEEP_CACHE_COUNT} 个)..."
|
||||
ALL_VOLUMES=$(docker volume ls -q --filter "name=${VOLUME_PREFIX}" 2>/dev/null || true)
|
||||
if [ -n "$ALL_VOLUMES" ]; then
|
||||
TOTAL=$(echo "$ALL_VOLUMES" | wc -l)
|
||||
if [ "$TOTAL" -gt "$KEEP_CACHE_COUNT" ]; then
|
||||
# 按创建时间排序,保留最新的 N 个
|
||||
SORTED_VOLUMES=$(for v in $ALL_VOLUMES; do
|
||||
CREATED=$(docker volume inspect --format '{{.CreatedAt}}' "$v" 2>/dev/null || echo "0")
|
||||
echo "$CREATED $v"
|
||||
done | sort | awk '{print $2}')
|
||||
|
||||
# 删除超出保留数量的旧 volume
|
||||
REMOVE_COUNT=$((TOTAL - KEEP_CACHE_COUNT))
|
||||
TO_DELETE=$(echo "$SORTED_VOLUMES" | head -n "$REMOVE_COUNT")
|
||||
REMOVED=0
|
||||
for v in $TO_DELETE; do
|
||||
# 跳过当前正在使用的 volume
|
||||
if [ "$v" != "$VOLUME_NAME" ]; then
|
||||
if docker volume rm "$v" >/dev/null 2>&1; then
|
||||
REMOVED=$((REMOVED + 1))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
echo "已清理 $REMOVED 个旧缓存 volume,当前共 $((TOTAL - REMOVED)) 个"
|
||||
else
|
||||
echo "当前缓存 volume 数量: $TOTAL,无需清理"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "=== 前端依赖安装完成 ==="
|
||||
|
||||
@@ -1,12 +1,26 @@
|
||||
#!/bin/sh
|
||||
# CI 公共步骤:前端命令执行(在 docker node 容器中运行)
|
||||
# 用法:step_frontend_run.sh "要执行的命令"
|
||||
# 支持 Docker Volume 持久化缓存的 node_modules
|
||||
set -eu
|
||||
|
||||
CMD="${1:-echo 'no command'}"
|
||||
|
||||
docker run --rm \
|
||||
-v "$PWD:/workspace" \
|
||||
-w /workspace/apps/web \
|
||||
docker.m.daocloud.io/library/node:20 \
|
||||
sh -lc "$CMD"
|
||||
# 缓存配置 — 与 step_frontend_install.sh 保持一致
|
||||
LOCK_FILE="apps/web/package-lock.json"
|
||||
VOLUME_PREFIX="ci-web-nm-"
|
||||
|
||||
# 计算 package-lock.json 的 hash,挂载对应的 volume
|
||||
VOLUME_ARGS=""
|
||||
if [ -f "$LOCK_FILE" ]; then
|
||||
LOCK_HASH=$(md5sum "$LOCK_FILE" | cut -c1-12)
|
||||
VOLUME_NAME="${VOLUME_PREFIX}${LOCK_HASH}"
|
||||
if docker volume inspect "$VOLUME_NAME" >/dev/null 2>&1; then
|
||||
VOLUME_ARGS="-v ${VOLUME_NAME}:/workspace/apps/web/node_modules"
|
||||
echo "使用缓存 volume: $VOLUME_NAME"
|
||||
else
|
||||
echo "提示: 未找到缓存 volume $VOLUME_NAME,将使用源码目录 node_modules"
|
||||
fi
|
||||
fi
|
||||
|
||||
docker run --rm -v "$PWD:/workspace" $VOLUME_ARGS -w /workspace/apps/web docker.m.daocloud.io/library/node:20 sh -lc "$CMD"
|
||||
|
||||
@@ -2,3 +2,5 @@
|
||||
# CI 公共步骤:Job 开始计时
|
||||
echo "JOB_START_TIME=$(date +%s)" >> $GITHUB_ENV
|
||||
echo "Job started at $(date)"
|
||||
# trigger CI run for PR validation
|
||||
# trigger CI - worker dood fallback fix test
|
||||
@@ -0,0 +1,186 @@
|
||||
#!/bin/bash
|
||||
# CI Validate: 代码质量与安全扫描(并行Job 1/3)
|
||||
# 包含:密钥扫描、格式检查、安全扫描、依赖漏洞、死代码检测、脚本语法校验
|
||||
set -eu
|
||||
|
||||
echo "=== CI Validate: 代码质量与安全扫描 ==="
|
||||
|
||||
# --- 密钥检测 ---
|
||||
echo ""
|
||||
echo "=== [1/6] Secret detection (detect-secrets) ==="
|
||||
python3 -m pip install -q detect-secrets
|
||||
detect-secrets --version
|
||||
|
||||
detect-secrets scan \
|
||||
--all-files \
|
||||
--exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \
|
||||
--exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \
|
||||
--exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \
|
||||
--disable-plugin Base64HighEntropyString \
|
||||
--disable-plugin HexHighEntropyString \
|
||||
--disable-plugin BasicAuthDetector \
|
||||
--disable-plugin KeywordDetector \
|
||||
--disable-plugin IPPublicDetector \
|
||||
> /tmp/secrets-scan.json 2>&1
|
||||
|
||||
FOUND=$(python3 -c "
|
||||
import json
|
||||
try:
|
||||
with open('/tmp/secrets-scan.json') as f:
|
||||
data = json.load(f)
|
||||
results = data.get('results', {})
|
||||
total = sum(len(v) for v in results.values())
|
||||
print(total)
|
||||
except Exception:
|
||||
print('error')
|
||||
")
|
||||
|
||||
echo "Secrets detected: $FOUND"
|
||||
if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then
|
||||
echo ""
|
||||
echo "=== Secret details ==="
|
||||
python3 -c "
|
||||
import json
|
||||
with open('/tmp/secrets-scan.json') as f:
|
||||
data = json.load(f)
|
||||
for fpath, items in data.get('results', {}).items():
|
||||
for item in items:
|
||||
line = item.get('line_number', '?')
|
||||
stype = item.get('type', '?')
|
||||
hashed = item.get('hashed_secret', '')[:16]
|
||||
print(f' {fpath}:{line} [{stype}] {hashed}...')
|
||||
"
|
||||
echo ""
|
||||
echo "ERROR: Potential secrets detected in code!"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ Secret scan passed"
|
||||
|
||||
# --- 增量/全量模式判断 ---
|
||||
echo ""
|
||||
echo "=== [2/6] Code quality checks ==="
|
||||
SCAN_MODE="full"
|
||||
CHANGED_PY_FILES=""
|
||||
|
||||
if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then
|
||||
PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||')
|
||||
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100"
|
||||
set +e
|
||||
RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL")
|
||||
HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
|
||||
BODY=$(echo "$RESPONSE" | sed '$d')
|
||||
set -e
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
CHANGED_PY_FILES=$(echo "$BODY" | python3 -c "
|
||||
import json, sys
|
||||
try:
|
||||
files = json.load(sys.stdin)
|
||||
py_files = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] != 'removed']
|
||||
print(' '.join(py_files))
|
||||
except Exception:
|
||||
print('')
|
||||
")
|
||||
if [ -n "$CHANGED_PY_FILES" ]; then
|
||||
SCAN_MODE="incremental"
|
||||
echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed"
|
||||
else
|
||||
SCAN_MODE="skip_py"
|
||||
echo "No Python files changed in this PR"
|
||||
fi
|
||||
else
|
||||
echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan"
|
||||
fi
|
||||
else
|
||||
echo "Full scan mode (not a PR event)"
|
||||
fi
|
||||
|
||||
if [ "$SCAN_MODE" = "incremental" ]; then
|
||||
# 防御性过滤
|
||||
EXISTING_PY_FILES=""
|
||||
for f in $CHANGED_PY_FILES; do
|
||||
if [ -f "$f" ]; then
|
||||
if [ -z "$EXISTING_PY_FILES" ]; then
|
||||
EXISTING_PY_FILES="$f"
|
||||
else
|
||||
EXISTING_PY_FILES="$EXISTING_PY_FILES $f"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
CHANGED_PY_FILES="$EXISTING_PY_FILES"
|
||||
|
||||
python3 -m compileall -q $CHANGED_PY_FILES
|
||||
python3 -m black --check --fast $CHANGED_PY_FILES
|
||||
python3 -m isort --check-only $CHANGED_PY_FILES
|
||||
RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | grep -v '^$' | xargs)
|
||||
if [ -n "$RUFF_FILES" ]; then
|
||||
python3 -m ruff check $RUFF_FILES --statistics
|
||||
else
|
||||
echo "No ruff-checkable files changed, skipping"
|
||||
fi
|
||||
elif [ "$SCAN_MODE" = "skip_py" ]; then
|
||||
echo "No Python files changed - skipping Python lint checks"
|
||||
else
|
||||
echo "Full scan mode"
|
||||
python3 -m compileall -q alembic apps packages tests scripts
|
||||
python3 -m black --check --fast alembic apps packages tests scripts
|
||||
python3 -m isort --check-only alembic apps packages tests scripts
|
||||
python3 -m ruff check apps packages tests --statistics
|
||||
fi
|
||||
echo "✅ Code quality checks passed"
|
||||
|
||||
# --- Bandit 安全扫描(仅告警) ---
|
||||
echo ""
|
||||
echo "=== [3/6] Security scan (bandit, advisory only) ==="
|
||||
set +e
|
||||
bandit -r apps packages -q -ll
|
||||
BANDIT_EXIT=$?
|
||||
set -e
|
||||
if [ "$BANDIT_EXIT" -ne 0 ]; then
|
||||
echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)"
|
||||
else
|
||||
echo "✅ Bandit security scan passed"
|
||||
fi
|
||||
|
||||
# --- Pip-audit 依赖漏洞扫描(仅告警) ---
|
||||
echo ""
|
||||
echo "=== [4/6] Python dependency vulnerability scan (pip-audit, advisory only) ==="
|
||||
python3 -m pip install -q pip-audit
|
||||
pip-audit --version
|
||||
EXIT_CODE=0
|
||||
for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do
|
||||
if [ -f "$req_file" ]; then
|
||||
echo "--- Scanning $req_file ---"
|
||||
pip-audit -r "$req_file" --desc on 2>&1 | head -40 || EXIT_CODE=$?
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)"
|
||||
|
||||
# --- Vulture 死代码检测(仅告警) ---
|
||||
echo ""
|
||||
echo "=== [5/6] Dead code detection (vulture, advisory only) ==="
|
||||
set +e
|
||||
python3 -m pip install -q vulture
|
||||
vulture --version
|
||||
echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。"
|
||||
echo ""
|
||||
vulture apps packages scripts \
|
||||
--exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \
|
||||
--min-confidence 70 \
|
||||
2>&1 | sort -t'(' -k2 -rn | head -80
|
||||
echo ""
|
||||
echo "=== vulture scan summary ==="
|
||||
echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)"
|
||||
echo "建议:定期人工审查高置信度(>=90%)条目"
|
||||
set -e
|
||||
|
||||
# --- Release 脚本语法校验 ---
|
||||
echo ""
|
||||
echo "=== [6/6] Release scripts syntax validation ==="
|
||||
bash -n scripts/backup_postgres.sh
|
||||
bash -n scripts/restore_postgres_plan.sh
|
||||
bash -n scripts/init_production_env.sh
|
||||
echo "✅ Release scripts syntax OK"
|
||||
|
||||
echo ""
|
||||
echo "=== CI Validate: 代码质量与安全扫描 全部通过 ✅ ==="
|
||||
@@ -0,0 +1,183 @@
|
||||
#!/bin/bash
|
||||
# CI Validate: Alembic迁移验证(并行Job 3/3)
|
||||
# 需要PostgreSQL数据库
|
||||
set -eu
|
||||
|
||||
echo "=== CI Validate: Alembic迁移验证 ==="
|
||||
|
||||
# --- DooD模式检测:确定宿主机访问地址 ---
|
||||
detect_docker_host() {
|
||||
local test_port="${1:-5432}"
|
||||
|
||||
local candidates=()
|
||||
|
||||
# 1. host.docker.internal
|
||||
if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then
|
||||
candidates+=("host.docker.internal")
|
||||
fi
|
||||
|
||||
# 2. docker0 桥接网关
|
||||
candidates+=("172.17.0.1")
|
||||
|
||||
# 3. 默认网关
|
||||
local gw=""
|
||||
gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1)
|
||||
if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then
|
||||
candidates+=("$gw")
|
||||
fi
|
||||
|
||||
# 4. 宿主机同网段的.1或.254
|
||||
local my_ip=""
|
||||
my_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
|
||||
if [ -n "$my_ip" ]; then
|
||||
local subnet=$(echo "$my_ip" | cut -d. -f1-3)
|
||||
candidates+=("${subnet}.1")
|
||||
candidates+=("${subnet}.254")
|
||||
fi
|
||||
|
||||
# 5. 127.0.0.1 最后尝试
|
||||
candidates+=("127.0.0.1")
|
||||
|
||||
for candidate in "${candidates[@]}"; do
|
||||
if python3 -c "
|
||||
import socket
|
||||
s = socket.socket()
|
||||
s.settimeout(2)
|
||||
try:
|
||||
s.connect(('$candidate', $test_port))
|
||||
s.close()
|
||||
print('ok')
|
||||
except:
|
||||
pass
|
||||
" 2>/dev/null | grep -q ok; then
|
||||
echo "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo "127.0.0.1"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 获取宿主机IP
|
||||
if [ -S /var/run/docker.sock ]; then
|
||||
DOCKER_HOST_IP=$(detect_docker_host 5433)
|
||||
if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then
|
||||
DOCKER_HOST_IP=$(detect_docker_host 22)
|
||||
fi
|
||||
echo "检测到DooD模式,宿主机地址: $DOCKER_HOST_IP"
|
||||
else
|
||||
DOCKER_HOST_IP="127.0.0.1"
|
||||
echo "非DooD模式,使用 127.0.0.1"
|
||||
fi
|
||||
PG_HOST="$DOCKER_HOST_IP"
|
||||
echo "PG host: $PG_HOST"
|
||||
|
||||
# 指数退避TCP连接检查
|
||||
wait_tcp_ready() {
|
||||
local host="$1"
|
||||
local port="$2"
|
||||
local max_attempts="${3:-5}"
|
||||
local delay=1
|
||||
local attempt=1
|
||||
while [ "$attempt" -le "$max_attempts" ]; do
|
||||
if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
echo "TCP连接尝试 $attempt/$max_attempts 失败,${delay}s后重试..."
|
||||
sleep "$delay"
|
||||
delay=$((delay * 2))
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
USE_SHARED_PG="${CI_USE_SHARED_PG:-false}"
|
||||
|
||||
if [ "$USE_SHARED_PG" = "true" ]; then
|
||||
# 使用常驻共享PG实例
|
||||
echo "使用常驻共享PG实例(CI_USE_SHARED_PG=true)"
|
||||
SHARED_PG_HOST="$PG_HOST"
|
||||
SHARED_PG_PORT="5433"
|
||||
SHARED_PG_USER="postgres"
|
||||
SHARED_PG_PASSWORD="ci_pg_2026!"
|
||||
CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}"
|
||||
|
||||
echo "等待共享PG连接就绪..."
|
||||
wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5
|
||||
|
||||
echo "创建测试数据库: $CI_DB_NAME"
|
||||
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
|
||||
cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"')
|
||||
cur.close()
|
||||
conn.close()
|
||||
"
|
||||
export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}"
|
||||
echo "✅ 共享PG数据库已创建: $CI_DB_NAME"
|
||||
|
||||
# 执行迁移
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
|
||||
echo "✅ Alembic migrations applied successfully"
|
||||
|
||||
# 清理数据库
|
||||
echo "清理测试数据库: $CI_DB_NAME"
|
||||
PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c "
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres')
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)')
|
||||
cur.close()
|
||||
conn.close()
|
||||
" 2>/dev/null || echo "WARN: 数据库清理失败"
|
||||
echo "✅ 共享PG数据库已清理"
|
||||
else
|
||||
# 使用临时PG容器(默认模式)
|
||||
echo "使用临时PG容器模式"
|
||||
PG_CONTAINER=ci-pg-validate-migration-${GITHUB_RUN_ID:-$$}
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
--shm-size=256m \
|
||||
-e POSTGRES_USER=postgres \
|
||||
-e POSTGRES_PASSWORD=postgres \
|
||||
-e POSTGRES_DB=xiaoxia_saas \
|
||||
-P \
|
||||
--health-cmd "pg_isready -U postgres" \
|
||||
--health-interval 3s \
|
||||
--health-timeout 3s \
|
||||
--health-retries 20 \
|
||||
postgres:16-alpine
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
|
||||
echo "PostgreSQL port: $PG_PORT"
|
||||
export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas"
|
||||
|
||||
# 等待容器健康
|
||||
for i in $(seq 1 30); do
|
||||
if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
|
||||
echo "PostgreSQL container is healthy on port $PG_PORT"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for PostgreSQL container health... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy
|
||||
|
||||
# TCP连通性检查
|
||||
echo "验证TCP连通性 ($PG_HOST:$PG_PORT)..."
|
||||
wait_tcp_ready "$PG_HOST" "$PG_PORT" 5
|
||||
echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT"
|
||||
|
||||
# 执行迁移
|
||||
PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head
|
||||
echo "✅ Alembic migrations applied successfully"
|
||||
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== CI Validate: Alembic迁移验证 通过 ✅ ==="
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
# CI Validate: Mypy类型检查(并行Job 2/3)
|
||||
set -eu
|
||||
|
||||
echo "=== CI Validate: Mypy类型检查 ==="
|
||||
|
||||
bash scripts/ci/mypy_check.sh
|
||||
|
||||
echo ""
|
||||
echo "=== CI Validate: Mypy类型检查 通过 ✅ ==="
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/bin/bash
|
||||
# Vitest 增量执行脚本
|
||||
# PR模式下只跑与改动文件相关的测试,大幅节省时间
|
||||
# 用法: bash scripts/ci/vitest_incremental.sh
|
||||
set -eu
|
||||
|
||||
cd apps/web
|
||||
|
||||
# 如果不是PR事件,直接全量跑
|
||||
if [ "${GITHUB_EVENT_NAME:-}" != "pull_request" ]; then
|
||||
echo "非PR模式,全量执行Vitest"
|
||||
npx --no-install vitest run --coverage
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# 获取PR改动的文件列表
|
||||
PR_NUMBER=$(echo "${GITHUB_REF:-}" | sed 's|refs/pull/||; s|/.*||')
|
||||
if [ -z "$PR_NUMBER" ]; then
|
||||
echo "无法获取PR编号,全量执行Vitest"
|
||||
npx --no-install vitest run --coverage
|
||||
exit $?
|
||||
fi
|
||||
|
||||
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100"
|
||||
CHANGED_FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "
|
||||
import json, sys
|
||||
try:
|
||||
files = json.load(sys.stdin)
|
||||
web_files = []
|
||||
for f in files:
|
||||
fname = f['filename']
|
||||
# 只关注前端源码文件
|
||||
if fname.startswith('apps/web/src/') and fname.endswith(('.ts', '.tsx', '.js', '.jsx')) and f['status'] != 'removed':
|
||||
# 去掉apps/web/前缀,变成相对路径
|
||||
web_files.append(fname.replace('apps/web/', ''))
|
||||
print(' '.join(web_files))
|
||||
except Exception as e:
|
||||
print('')
|
||||
")
|
||||
|
||||
if [ -z "$CHANGED_FILES" ]; then
|
||||
echo "PR未改动前端源码文件,跳过Vitest"
|
||||
echo "(如果配置了前端单测门禁,请确保至少有一个相关测试)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
FILE_COUNT=$(echo "$CHANGED_FILES" | wc -w)
|
||||
echo "PR改动了 $FILE_COUNT 个前端文件"
|
||||
echo "改动文件: $CHANGED_FILES"
|
||||
|
||||
# 如果改动文件太多(超过30个),全量跑更可靠
|
||||
if [ "$FILE_COUNT" -gt 30 ]; then
|
||||
echo "改动文件较多(>$FILE_COUNT),降级为全量执行以确保覆盖"
|
||||
npx --no-install vitest run --coverage
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# 使用vitest related 跑增量测试(子命令,非flag)
|
||||
echo ""
|
||||
echo "=== 增量执行 Vitest(只跑相关测试)==="
|
||||
echo "相关源文件: $CHANGED_FILES"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
npx --no-install vitest run related $CHANGED_FILES
|
||||
VITEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
if [ "$VITEST_EXIT" -eq 0 ]; then
|
||||
echo ""
|
||||
echo "✅ 增量测试通过"
|
||||
echo "(仅覆盖与改动相关的测试用例)"
|
||||
exit 0
|
||||
else
|
||||
echo ""
|
||||
echo "❌ 增量测试失败"
|
||||
exit $VITEST_EXIT
|
||||
fi
|
||||
+128
-31
@@ -1,17 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""发送 CI 失败通知到飞书/项目群 webhook。"""
|
||||
"""发送 CI 失败通知到飞书/项目群 webhook(增强版:带失败诊断)。
|
||||
|
||||
诊断功能:自动分析失败原因,给出分类和修复建议。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
|
||||
def run_diagnosis() -> dict:
|
||||
"""运行失败诊断脚本,返回诊断结果"""
|
||||
diag_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ci/ci_failure_diagnosis.py")
|
||||
if not os.path.exists(diag_script):
|
||||
diag_script = "scripts/ci/ci_failure_diagnosis.py"
|
||||
|
||||
result = {
|
||||
"category": "unknown",
|
||||
"category_cn": "未知",
|
||||
"severity": "medium",
|
||||
"summary": "",
|
||||
"error_lines": [],
|
||||
"suggestions": [],
|
||||
"auto_fixable": False,
|
||||
}
|
||||
|
||||
try:
|
||||
# 运行诊断脚本
|
||||
env = os.environ.copy()
|
||||
env["DIAGNOSIS_OUTPUT"] = "/tmp/ci_diagnosis_result.json"
|
||||
|
||||
proc = subprocess.run([sys.executable, diag_script], capture_output=True, text=True, timeout=30, env=env)
|
||||
|
||||
# 尝试读取结果文件
|
||||
output_file = "/tmp/ci_diagnosis_result.json"
|
||||
if os.path.exists(output_file):
|
||||
with open(output_file) as f:
|
||||
result = json.load(f)
|
||||
elif proc.stdout:
|
||||
# 从stdout解析
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"诊断脚本执行失败: {e}", file=sys.stderr)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
webhook = os.environ.get("CI_NOTIFY_WEBHOOK", "")
|
||||
if not webhook:
|
||||
print("未配置 CI_NOTIFY_WEBHOOK,跳过通知")
|
||||
print("如需启用,请在仓库 Settings -> Secrets and variables -> Actions 中添加 CI_NOTIFY_WEBHOOK")
|
||||
return 0
|
||||
|
||||
failed_job = os.environ.get("FAILED_JOB", "Unknown Job")
|
||||
@@ -21,6 +61,86 @@ def main() -> int:
|
||||
run_id = os.environ.get("GITHUB_RUN_ID", "unknown")
|
||||
repo = os.environ.get("GITHUB_REPOSITORY", "unknown")
|
||||
run_url = f"https://git.xiaoxiajianji.com/{repo}/actions/runs/{run_id}"
|
||||
pr_number = os.environ.get("PR_NUMBER", "")
|
||||
|
||||
# 运行诊断
|
||||
diagnosis = run_diagnosis()
|
||||
|
||||
# 构建卡片内容
|
||||
severity_color = {"high": "red", "medium": "orange", "low": "blue"}
|
||||
card_status = severity_color.get(diagnosis.get("severity", "medium"), "red")
|
||||
|
||||
# 标题
|
||||
title = f"❌ CI失败 - {diagnosis.get('category_cn', '未知')}"
|
||||
|
||||
# 诊断部分
|
||||
diag_lines = []
|
||||
diag_lines.append(f"**任务**: {failed_job}")
|
||||
diag_lines.append(f"**分类**: {diagnosis.get('category_cn', '未知')}")
|
||||
if diagnosis.get("summary"):
|
||||
diag_lines.append(f"**问题**: {diagnosis['summary']}")
|
||||
|
||||
# 错误行
|
||||
error_lines = diagnosis.get("error_lines", [])
|
||||
if error_lines:
|
||||
diag_lines.append("")
|
||||
diag_lines.append("**关键错误**:")
|
||||
for err in error_lines[:3]:
|
||||
if len(err) > 100:
|
||||
err = err[:97] + "..."
|
||||
diag_lines.append(f"`{err}`")
|
||||
|
||||
# 修复建议
|
||||
suggestions = diagnosis.get("suggestions", [])
|
||||
if suggestions:
|
||||
diag_lines.append("")
|
||||
diag_lines.append("**修复建议**:")
|
||||
for i, s in enumerate(suggestions[:3], 1):
|
||||
diag_lines.append(f"{i}. {s}")
|
||||
|
||||
if diagnosis.get("auto_fixable"):
|
||||
diag_lines.append("")
|
||||
diag_lines.append("💡 *可自动修复的问题,试试Rerun*")
|
||||
|
||||
# 基本信息
|
||||
info_lines = [
|
||||
f"**分支**: {branch}",
|
||||
f"**提交**: `{commit}`",
|
||||
f"**提交者**: {actor}",
|
||||
]
|
||||
if pr_number:
|
||||
info_lines.append(f"**PR**: #{pr_number}")
|
||||
|
||||
elements = [
|
||||
{
|
||||
"tag": "div",
|
||||
"text": {
|
||||
"tag": "lark_md",
|
||||
"content": "\n".join(diag_lines),
|
||||
},
|
||||
},
|
||||
{
|
||||
"tag": "hr",
|
||||
},
|
||||
{
|
||||
"tag": "div",
|
||||
"text": {
|
||||
"tag": "lark_md",
|
||||
"content": "\n".join(info_lines),
|
||||
},
|
||||
},
|
||||
{
|
||||
"tag": "action",
|
||||
"actions": [
|
||||
{
|
||||
"tag": "button",
|
||||
"text": {"tag": "plain_text", "content": "查看失败日志"},
|
||||
"url": run_url,
|
||||
"type": "danger",
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
payload = {
|
||||
"msg_type": "interactive",
|
||||
@@ -28,36 +148,11 @@ def main() -> int:
|
||||
"header": {
|
||||
"title": {
|
||||
"tag": "plain_text",
|
||||
"content": "❌ CI 构建失败",
|
||||
"content": title,
|
||||
},
|
||||
"status": "red",
|
||||
"status": card_status,
|
||||
},
|
||||
"elements": [
|
||||
{
|
||||
"tag": "div",
|
||||
"text": {
|
||||
"tag": "lark_md",
|
||||
"content": (
|
||||
f"**任务**: {failed_job}\n"
|
||||
f"**分支**: {branch}\n"
|
||||
f"**提交**: {commit}\n"
|
||||
f"**提交者**: {actor}\n"
|
||||
f"**Run ID**: {run_id}"
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
"tag": "action",
|
||||
"actions": [
|
||||
{
|
||||
"tag": "button",
|
||||
"text": {"tag": "plain_text", "content": "查看失败日志"},
|
||||
"url": run_url,
|
||||
"type": "danger",
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
"elements": elements,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -71,7 +166,7 @@ def main() -> int:
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
resp.read()
|
||||
print("通知已发送")
|
||||
print("通知已发送(带诊断信息)")
|
||||
except Exception as e:
|
||||
print(f"通知发送失败: {e}", file=sys.stderr)
|
||||
return 1
|
||||
@@ -81,3 +176,5 @@ def main() -> int:
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
# trigger CI - bypass [ci skip] bug
|
||||
|
||||
+151
-93
@@ -1,22 +1,9 @@
|
||||
#!/bin/sh
|
||||
# ===========================================
|
||||
# Staging 部署脚本(SSH 模式,支持自动回滚)
|
||||
# Staging 部署脚本(SSH 模式,并行优化版)
|
||||
# ===========================================
|
||||
# 通过 SSH 在 staging 服务器上执行
|
||||
#
|
||||
# 环境变量:
|
||||
# IMAGE_TAG - 镜像版本 tag(如 commit SHA 或分支名)
|
||||
# REGISTRY_TOKEN - Registry 访问令牌
|
||||
# REGISTRY - Registry 地址(默认 xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji)
|
||||
# REGISTRY_USER - Registry 用户名(默认 xiaoxia)
|
||||
# ENV_FILE - 环境变量文件路径
|
||||
# GENERATED_DIR - 生成文件目录
|
||||
# SKIP_MIGRATION - 跳过数据库迁移(true/false,默认 false)
|
||||
# SKIP_ROLLBACK - 失败时跳过自动回滚(true/false,默认 false)
|
||||
|
||||
set -eu
|
||||
|
||||
# ---- 重试工具函数 ----
|
||||
retry_cmd() {
|
||||
local max_attempts=$1
|
||||
local backoff=$2
|
||||
@@ -73,10 +60,9 @@ mkdir -p "$GENERATED_DIR"
|
||||
mkdir -p "$LEGACY_ASSETS_DIR"
|
||||
|
||||
echo "==========================================="
|
||||
echo " Staging 部署 - $IMAGE_TAG"
|
||||
echo " Staging 部署 - $IMAGE_TAG (并行优化版)"
|
||||
echo "==========================================="
|
||||
|
||||
# ---- 记录当前运行的镜像版本(用于回滚) ----
|
||||
echo "Recording current image versions for rollback..."
|
||||
PREV_API_IMAGE=""
|
||||
PREV_WORKER_IMAGE=""
|
||||
@@ -95,7 +81,6 @@ for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do
|
||||
fi
|
||||
done
|
||||
|
||||
# ---- 回滚函数 ----
|
||||
rollback() {
|
||||
echo ""
|
||||
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||
@@ -108,7 +93,6 @@ rollback() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 停止当前(失败的)新容器
|
||||
echo "Stopping new containers..."
|
||||
docker rm -f xiaoxia-api-staging 2>/dev/null || true
|
||||
docker rm -f xiaoxia-worker-staging 2>/dev/null || true
|
||||
@@ -116,7 +100,6 @@ rollback() {
|
||||
|
||||
LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
|
||||
|
||||
# 恢复 API
|
||||
if [ -n "$PREV_API_IMAGE" ]; then
|
||||
echo "Rolling back API to: $PREV_API_IMAGE"
|
||||
docker run -d \
|
||||
@@ -137,12 +120,9 @@ rollback() {
|
||||
--health-retries 3 \
|
||||
--health-start-period 40s \
|
||||
$LOG_OPTS \
|
||||
"$PREV_API_IMAGE"
|
||||
else
|
||||
echo "No previous API image to roll back to"
|
||||
"$PREV_API_IMAGE" &
|
||||
fi
|
||||
|
||||
# 恢复 Worker
|
||||
if [ -n "$PREV_WORKER_IMAGE" ]; then
|
||||
echo "Rolling back Worker to: $PREV_WORKER_IMAGE"
|
||||
docker run -d \
|
||||
@@ -164,12 +144,9 @@ rollback() {
|
||||
--health-retries 3 \
|
||||
--health-start-period 30s \
|
||||
$LOG_OPTS \
|
||||
"$PREV_WORKER_IMAGE"
|
||||
else
|
||||
echo "No previous Worker image to roll back to"
|
||||
"$PREV_WORKER_IMAGE" &
|
||||
fi
|
||||
|
||||
# 恢复 Web
|
||||
if [ -n "$PREV_WEB_IMAGE" ]; then
|
||||
echo "Rolling back Web to: $PREV_WEB_IMAGE"
|
||||
LEGACY_VOLUME=""
|
||||
@@ -187,12 +164,11 @@ rollback() {
|
||||
--health-timeout 5s \
|
||||
--health-retries 3 \
|
||||
$LOG_OPTS \
|
||||
"$PREV_WEB_IMAGE"
|
||||
else
|
||||
echo "No previous Web image to roll back to"
|
||||
"$PREV_WEB_IMAGE" &
|
||||
fi
|
||||
|
||||
# 等待 API 回滚后恢复健康
|
||||
wait
|
||||
|
||||
if [ -n "$PREV_API_IMAGE" ]; then
|
||||
echo "Waiting for rolled-back API to become healthy..."
|
||||
i=0
|
||||
@@ -224,7 +200,6 @@ rollback() {
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ---- 登录 Registry ----
|
||||
if [ -n "$REGISTRY_TOKEN" ]; then
|
||||
echo "=========================================="
|
||||
echo " Login to Registry (with retries)"
|
||||
@@ -233,28 +208,64 @@ if [ -n "$REGISTRY_TOKEN" ]; then
|
||||
retry_docker_login
|
||||
fi
|
||||
|
||||
# ---- Pull 新版本镜像 ----
|
||||
# ---- 并行 Pull 三个镜像 ----
|
||||
REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}"
|
||||
REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}"
|
||||
REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}"
|
||||
|
||||
echo "=========================================="
|
||||
echo " Pull images (with retries)"
|
||||
echo " Pull images (parallel, up to 3 retries each)"
|
||||
echo "=========================================="
|
||||
retry_docker_pull "$REGISTRY_API"
|
||||
retry_docker_pull "$REGISTRY_WORKER"
|
||||
retry_docker_pull "$REGISTRY_WEB"
|
||||
PULL_LOG_DIR="/tmp/staging-pull-$$"
|
||||
mkdir -p "$PULL_LOG_DIR"
|
||||
|
||||
retry_docker_pull "$REGISTRY_API" > "$PULL_LOG_DIR/api.log" 2>&1 &
|
||||
PID_API=$!
|
||||
retry_docker_pull "$REGISTRY_WORKER" > "$PULL_LOG_DIR/worker.log" 2>&1 &
|
||||
PID_WORKER=$!
|
||||
retry_docker_pull "$REGISTRY_WEB" > "$PULL_LOG_DIR/web.log" 2>&1 &
|
||||
PID_WEB=$!
|
||||
|
||||
wait $PID_API $PID_WORKER $PID_WEB
|
||||
|
||||
echo ""
|
||||
echo "Pull 结果:"
|
||||
PULL_FAILED=0
|
||||
for svc in api worker web; do
|
||||
if tail -1 "$PULL_LOG_DIR/$svc.log" 2>/dev/null | grep -qE "Status:|Downloaded|already exists|is up to date"; then
|
||||
echo " OK $svc"
|
||||
elif grep -qE "Digest:|Status: Downloaded" "$PULL_LOG_DIR/$svc.log" 2>/dev/null; then
|
||||
echo " OK $svc"
|
||||
else
|
||||
# 检查docker pull返回值不直接,用镜像是否存在来判断
|
||||
img_var="REGISTRY_$(echo $svc | tr '[:lower:]' '[:upper:]')"
|
||||
img_val=$(eval echo "\$$img_var")
|
||||
if docker image inspect "$img_val" >/dev/null 2>&1; then
|
||||
echo " OK $svc"
|
||||
else
|
||||
echo " FAIL $svc"
|
||||
tail -5 "$PULL_LOG_DIR/$svc.log" 2>/dev/null || true
|
||||
PULL_FAILED=$((PULL_FAILED + 1))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
rm -rf "$PULL_LOG_DIR"
|
||||
|
||||
if [ "$PULL_FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "ERROR: $PULL_FAILED 个镜像 pull 失败"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "All images pulled."
|
||||
|
||||
# ---- 备份 legacy assets ----
|
||||
echo "Backing up legacy assets from current web container..."
|
||||
if docker inspect xiaoxia-web-staging >/dev/null 2>&1; then
|
||||
_tmpdir="/tmp/legacy-assets-$$"
|
||||
rm -rf "$_tmpdir"
|
||||
mkdir -p "$_tmpdir"
|
||||
docker cp xiaoxia-web-staging:/usr/share/nginx/html/assets/. "$_tmpdir/" 2>/dev/null || true
|
||||
# 只有目录非空才拷贝,避免覆盖有内容的 legacy assets
|
||||
if [ -d "$_tmpdir" ] && [ "$(ls -A "$_tmpdir" 2>/dev/null)" ]; then
|
||||
cp -an "$_tmpdir"/. "$LEGACY_ASSETS_DIR"/ 2>/dev/null || true
|
||||
echo "Legacy assets backed up: $(ls "$_tmpdir" | wc -l) files"
|
||||
@@ -264,13 +275,11 @@ else
|
||||
echo "No existing web container, skipping legacy assets backup"
|
||||
fi
|
||||
|
||||
# 清理 7 天前的 legacy assets
|
||||
if [ -d "$LEGACY_ASSETS_DIR" ]; then
|
||||
find "$LEGACY_ASSETS_DIR" -type f -mtime +7 -delete 2>/dev/null || true
|
||||
echo "Legacy assets cleanup done (retain 7 days)"
|
||||
fi
|
||||
|
||||
# ---- 检查基础设施容器 ----
|
||||
echo "Checking infrastructure containers..."
|
||||
for c in xiaoxia-postgres-staging xiaoxia-redis-staging; do
|
||||
if ! docker inspect "$c" >/dev/null 2>&1; then
|
||||
@@ -284,10 +293,8 @@ for c in xiaoxia-postgres-staging xiaoxia-redis-staging; do
|
||||
fi
|
||||
done
|
||||
|
||||
# ---- 创建网络(不存在则创建) ----
|
||||
docker network create xiaoxia-net-staging 2>/dev/null || true
|
||||
|
||||
# ---- 数据库迁移 ----
|
||||
if [ "$SKIP_MIGRATION" != "true" ]; then
|
||||
echo "Running database migrations..."
|
||||
docker run --rm \
|
||||
@@ -296,8 +303,6 @@ if [ "$SKIP_MIGRATION" != "true" ]; then
|
||||
-e APP_ENV=staging \
|
||||
"$REGISTRY_API" sh -c "cd /app && alembic upgrade head" || {
|
||||
echo "ERROR: Database migration failed"
|
||||
echo "Note: Migration failures are NOT automatically rolled back (data safety)"
|
||||
echo "Please manually check and fix the migration, then redeploy"
|
||||
exit 1
|
||||
}
|
||||
echo "Migrations completed."
|
||||
@@ -305,7 +310,6 @@ else
|
||||
echo "Skipping migrations (SKIP_MIGRATION=true)"
|
||||
fi
|
||||
|
||||
# ---- 停止旧容器 ----
|
||||
echo "Stopping old containers..."
|
||||
docker rm -f xiaoxia-api-staging 2>/dev/null || true
|
||||
docker rm -f xiaoxia-worker-staging 2>/dev/null || true
|
||||
@@ -313,8 +317,14 @@ docker rm -f xiaoxia-web-staging 2>/dev/null || true
|
||||
|
||||
LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
|
||||
|
||||
# ---- 启动 API ----
|
||||
echo "Starting API container..."
|
||||
# ---- 并行启动三个容器 ----
|
||||
echo "Starting all containers (parallel)..."
|
||||
|
||||
LEGACY_VOLUME=""
|
||||
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
|
||||
LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
|
||||
fi
|
||||
|
||||
docker run -d \
|
||||
--name xiaoxia-api-staging \
|
||||
--env-file "$ENV_FILE" \
|
||||
@@ -333,10 +343,9 @@ docker run -d \
|
||||
--health-retries 3 \
|
||||
--health-start-period 40s \
|
||||
$LOG_OPTS \
|
||||
"$REGISTRY_API" || rollback
|
||||
"$REGISTRY_API" &
|
||||
PID_API_START=$!
|
||||
|
||||
# ---- 启动 Worker ----
|
||||
echo "Starting Worker container..."
|
||||
docker run -d \
|
||||
--name xiaoxia-worker-staging \
|
||||
--env-file "$ENV_FILE" \
|
||||
@@ -356,18 +365,9 @@ docker run -d \
|
||||
--health-retries 3 \
|
||||
--health-start-period 30s \
|
||||
$LOG_OPTS \
|
||||
"$REGISTRY_WORKER" || rollback
|
||||
"$REGISTRY_WORKER" &
|
||||
PID_WORKER_START=$!
|
||||
|
||||
# ---- 启动 Web ----
|
||||
LEGACY_VOLUME=""
|
||||
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
|
||||
LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
|
||||
echo "Web container: legacy assets mounted (fallback)"
|
||||
else
|
||||
echo "Web container: no legacy assets to mount"
|
||||
fi
|
||||
|
||||
echo "Starting Web container..."
|
||||
docker run -d \
|
||||
--name xiaoxia-web-staging \
|
||||
--network xiaoxia-net-staging \
|
||||
@@ -379,53 +379,111 @@ docker run -d \
|
||||
--health-timeout 5s \
|
||||
--health-retries 3 \
|
||||
$LOG_OPTS \
|
||||
"$REGISTRY_WEB" || rollback
|
||||
"$REGISTRY_WEB" &
|
||||
PID_WEB_START=$!
|
||||
|
||||
# ---- 等待 API 健康 ----
|
||||
echo "Waiting for API to become healthy..."
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then
|
||||
echo "API is healthy!"
|
||||
break
|
||||
wait $PID_API_START $PID_WORKER_START $PID_WEB_START
|
||||
|
||||
START_FAILED=0
|
||||
for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do
|
||||
if ! docker inspect "$c" >/dev/null 2>&1; then
|
||||
echo " FAIL $c: not created"
|
||||
START_FAILED=$((START_FAILED + 1))
|
||||
else
|
||||
state=$(docker inspect -f '{{.State.Status}}' "$c")
|
||||
if [ "$state" = "running" ] || [ "$state" = "starting" ]; then
|
||||
echo " OK $c: $state"
|
||||
else
|
||||
echo " FAIL $c: $state"
|
||||
docker logs --tail 20 "$c" 2>/dev/null || true
|
||||
START_FAILED=$((START_FAILED + 1))
|
||||
fi
|
||||
fi
|
||||
i=$((i + 1))
|
||||
echo " Waiting... ($i/40)"
|
||||
sleep 3
|
||||
done
|
||||
|
||||
if [ "$i" -ge 40 ]; then
|
||||
echo "ERROR: API did not become healthy within 120s"
|
||||
if [ "$START_FAILED" -gt 0 ]; then
|
||||
echo "ERROR: $START_FAILED 个容器启动失败"
|
||||
rollback
|
||||
fi
|
||||
|
||||
# ---- 并行等待 API 和 Web 健康 ----
|
||||
echo ""
|
||||
echo "Waiting for API + Web health (parallel)..."
|
||||
|
||||
HEALTH_LOG_DIR="/tmp/staging-health-$$"
|
||||
mkdir -p "$HEALTH_LOG_DIR"
|
||||
|
||||
(
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then
|
||||
echo "API healthy after $((i * 3))s"
|
||||
exit 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 3
|
||||
done
|
||||
echo "API FAILED after 120s"
|
||||
exit 1
|
||||
) > "$HEALTH_LOG_DIR/api.log" 2>&1 &
|
||||
PID_API_HEALTH=$!
|
||||
|
||||
(
|
||||
i=0
|
||||
while [ "$i" -lt 15 ]; do
|
||||
if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then
|
||||
echo "Web healthy after $((i * 2))s"
|
||||
exit 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 2
|
||||
done
|
||||
echo "Web FAILED after 30s"
|
||||
exit 1
|
||||
) > "$HEALTH_LOG_DIR/web.log" 2>&1 &
|
||||
PID_WEB_HEALTH=$!
|
||||
|
||||
set +e
|
||||
wait $PID_API_HEALTH
|
||||
API_EXIT=$?
|
||||
wait $PID_WEB_HEALTH
|
||||
WEB_EXIT=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
echo "健康检查结果:"
|
||||
API_OK=0
|
||||
WEB_OK=0
|
||||
if [ "$API_EXIT" -eq 0 ]; then
|
||||
echo " OK API: $(cat "$HEALTH_LOG_DIR/api.log")"
|
||||
API_OK=1
|
||||
else
|
||||
echo " FAIL API: 120s未就绪"
|
||||
docker logs --tail 50 xiaoxia-api-staging
|
||||
rollback
|
||||
fi
|
||||
|
||||
# ---- 等待 Web 健康 ----
|
||||
echo "Waiting for Web to become healthy..."
|
||||
i=0
|
||||
while [ "$i" -lt 15 ]; do
|
||||
if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then
|
||||
echo "Web is healthy!"
|
||||
break
|
||||
fi
|
||||
i=$((i + 1))
|
||||
echo " Waiting... ($i/15)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [ "$i" -ge 15 ]; then
|
||||
echo "ERROR: Web did not become healthy within 30s"
|
||||
if [ "$WEB_EXIT" -eq 0 ]; then
|
||||
echo " OK Web: $(cat "$HEALTH_LOG_DIR/web.log")"
|
||||
WEB_OK=1
|
||||
else
|
||||
echo " FAIL Web: 30s未就绪"
|
||||
docker logs --tail 30 xiaoxia-web-staging
|
||||
fi
|
||||
|
||||
rm -rf "$HEALTH_LOG_DIR"
|
||||
|
||||
if [ "$API_OK" -eq 0 ] || [ "$WEB_OK" -eq 0 ]; then
|
||||
echo ""
|
||||
echo "ERROR: 健康检查失败"
|
||||
rollback
|
||||
fi
|
||||
|
||||
# ---- 清理旧镜像 ----
|
||||
echo "Cleaning up old images..."
|
||||
docker image prune -af --filter "until=168h" 2>/dev/null || true
|
||||
docker builder prune -af --filter "until=168h" 2>/dev/null || true
|
||||
|
||||
echo ""
|
||||
echo "=== Staging deployment complete ==="
|
||||
echo "=== Staging deployment complete (并行优化版) ==="
|
||||
echo "API: http://127.0.0.1:8000"
|
||||
echo "Web: http://127.0.0.1:3001"
|
||||
echo "Version: $IMAGE_TAG"
|
||||
|
||||
@@ -2,18 +2,167 @@
|
||||
集成测试公共 fixtures
|
||||
|
||||
提供性能测试相关的工具、fixture 和 marker。
|
||||
支持 pytest-xdist 并行执行:每个 worker 使用独立数据库,数据完全隔离。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
import pytest
|
||||
|
||||
# ── xdist 并行数据库隔离 ──────────────────────────────────────────────────
|
||||
# 每个 xdist worker 进程创建独立的数据库并执行迁移,确保测试数据完全隔离
|
||||
# 通过 PYTEST_XDIST_WORKER 环境变量识别 worker(如 gw0, gw1, ...)
|
||||
|
||||
_WORKER_DB_NAME: Optional[str] = None
|
||||
|
||||
|
||||
def _get_worker_id() -> Optional[str]:
|
||||
"""获取当前 xdist worker ID,非 worker 模式返回 None"""
|
||||
return os.environ.get("PYTEST_XDIST_WORKER")
|
||||
|
||||
|
||||
def _parse_database_url(url: str) -> Dict[str, str]:
|
||||
"""
|
||||
解析 DATABASE_URL,返回各组件。
|
||||
支持 postgresql+psycopg://user:pass@host:port/dbname 格式
|
||||
"""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(url)
|
||||
return {
|
||||
"driver": parsed.scheme,
|
||||
"user": parsed.username or "",
|
||||
"password": parsed.password or "",
|
||||
"host": parsed.hostname or "",
|
||||
"port": str(parsed.port or 5432),
|
||||
"dbname": parsed.path.lstrip("/") or "",
|
||||
}
|
||||
|
||||
|
||||
def _create_worker_database(worker_id: str) -> str:
|
||||
"""
|
||||
为 xdist worker 创建独立数据库并执行迁移。
|
||||
返回新的 DATABASE_URL。
|
||||
"""
|
||||
base_url = os.environ.get(
|
||||
"DATABASE_URL",
|
||||
"postgresql+psycopg://postgres:postgres@localhost:5432/xiaoxia_saas",
|
||||
)
|
||||
db_info = _parse_database_url(base_url)
|
||||
|
||||
# 生成 worker 专属数据库名
|
||||
base_db = db_info["dbname"]
|
||||
worker_db = f"{base_db}_{worker_id}"
|
||||
global _WORKER_DB_NAME
|
||||
_WORKER_DB_NAME = worker_db
|
||||
|
||||
# 使用 psycopg 创建数据库(连接到 postgres 库)
|
||||
try:
|
||||
import psycopg
|
||||
|
||||
conn_str = (
|
||||
f"host={db_info['host']} port={db_info['port']} "
|
||||
f"user={db_info['user']} password={db_info['password']} "
|
||||
f"dbname=postgres"
|
||||
)
|
||||
conn = psycopg.connect(conn_str, autocommit=True)
|
||||
cur = conn.cursor()
|
||||
|
||||
# 先尝试删除(防止残留)
|
||||
cur.execute(f'DROP DATABASE IF EXISTS "{worker_db}" WITH (FORCE)')
|
||||
|
||||
# 创建新数据库
|
||||
cur.execute(f'CREATE DATABASE "{worker_db}"')
|
||||
cur.close()
|
||||
conn.close()
|
||||
print(f"[xdist {worker_id}] ✅ 创建数据库: {worker_db}")
|
||||
except ImportError:
|
||||
print(f"[xdist {worker_id}] ⚠️ psycopg 未安装,跳过数据库创建")
|
||||
return base_url
|
||||
except Exception as e:
|
||||
print(f"[xdist {worker_id}] ⚠️ 创建数据库失败: {e}")
|
||||
return base_url
|
||||
|
||||
# 构建新的 DATABASE_URL
|
||||
new_url = (
|
||||
f"{db_info['driver']}://{db_info['user']}:{db_info['password']}"
|
||||
f"@{db_info['host']}:{db_info['port']}/{worker_db}"
|
||||
)
|
||||
|
||||
# 执行 alembic 迁移
|
||||
print(f"[xdist {worker_id}] 🔄 执行 Alembic 迁移...")
|
||||
try:
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
api_path = str(ROOT / "apps" / "api")
|
||||
if api_path not in sys.path:
|
||||
sys.path.insert(0, api_path)
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from alembic import command as alembic_command
|
||||
from alembic.config import Config as AlembicConfig
|
||||
|
||||
alembic_cfg = AlembicConfig(str(ROOT / "alembic.ini"))
|
||||
alembic_cfg.set_main_option("sqlalchemy.url", new_url)
|
||||
# 兼容不同的脚本路径配置
|
||||
alembic_cfg.set_main_option("script_location", str(ROOT / "alembic"))
|
||||
|
||||
# 临时设置环境变量供 alembic env.py 使用
|
||||
os.environ["DATABASE_URL"] = new_url
|
||||
alembic_command.upgrade(alembic_cfg, "head")
|
||||
print(f"[xdist {worker_id}] ✅ 迁移完成")
|
||||
except Exception as e:
|
||||
print(f"[xdist {worker_id}] ❌ 迁移失败: {e}")
|
||||
raise
|
||||
|
||||
return new_url
|
||||
|
||||
|
||||
def _cleanup_worker_database(worker_id: str):
|
||||
"""清理 xdist worker 的数据库"""
|
||||
global _WORKER_DB_NAME
|
||||
if not _WORKER_DB_NAME:
|
||||
return
|
||||
|
||||
base_url = os.environ.get(
|
||||
"DATABASE_URL",
|
||||
"postgresql+psycopg://postgres:postgres@localhost:5432/xiaoxia_saas",
|
||||
)
|
||||
db_info = _parse_database_url(base_url)
|
||||
|
||||
try:
|
||||
import psycopg
|
||||
|
||||
conn_str = (
|
||||
f"host={db_info['host']} port={db_info['port']} "
|
||||
f"user={db_info['user']} password={db_info['password']} "
|
||||
f"dbname=postgres"
|
||||
)
|
||||
conn = psycopg.connect(conn_str, autocommit=True)
|
||||
cur = conn.cursor()
|
||||
# 强制断开所有连接后删除
|
||||
cur.execute(
|
||||
f"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
||||
f"WHERE datname = '{_WORKER_DB_NAME}' AND pid <> pg_backend_pid()"
|
||||
)
|
||||
cur.execute(f'DROP DATABASE IF EXISTS "{_WORKER_DB_NAME}" WITH (FORCE)')
|
||||
cur.close()
|
||||
conn.close()
|
||||
print(f"[xdist {worker_id}] 🧹 已清理数据库: {_WORKER_DB_NAME}")
|
||||
except Exception as e:
|
||||
print(f"[xdist {worker_id}] ⚠️ 清理数据库失败: {e}")
|
||||
finally:
|
||||
_WORKER_DB_NAME = None
|
||||
|
||||
|
||||
# ── 性能阈值配置 ──────────────────────────────────────────────────────────
|
||||
PERF_THRESHOLDS: Dict[str, int] = {
|
||||
"core": 500, # 核心接口:500ms
|
||||
@@ -183,16 +332,41 @@ class PerfAssert:
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ── pytest fixtures ──────────────────────────────────────────────────────
|
||||
# ── pytest hooks ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""注册自定义 marker"""
|
||||
"""
|
||||
pytest 配置钩子。
|
||||
|
||||
- 注册自定义 marker
|
||||
- xdist worker 模式下:创建独立数据库 + 执行迁移
|
||||
"""
|
||||
# 注册自定义 marker
|
||||
config.addinivalue_line("markers", "performance: 标记为性能测试(可通过 -m 'not performance' 跳过)")
|
||||
config.addinivalue_line("markers", "perf_core: 核心接口性能测试(阈值 500ms)")
|
||||
config.addinivalue_line("markers", "perf_normal: 普通接口性能测试(阈值 1000ms)")
|
||||
config.addinivalue_line("markers", "perf_heavy: 重操作接口性能测试(阈值 3000ms)")
|
||||
|
||||
# xdist worker 模式:创建独立数据库并执行迁移
|
||||
worker_id = _get_worker_id()
|
||||
if worker_id:
|
||||
# 只有当 USE_IN_MEMORY_DB 不为 true 时才创建独立数据库
|
||||
use_in_memory = os.environ.get("USE_IN_MEMORY_DB", "true").lower() == "true"
|
||||
if not use_in_memory:
|
||||
print(f"[xdist {worker_id}] 🚀 worker 启动,准备独立数据库...")
|
||||
new_db_url = _create_worker_database(worker_id)
|
||||
os.environ["DATABASE_URL"] = new_db_url
|
||||
else:
|
||||
print(f"[xdist {worker_id}] ℹ️ USE_IN_MEMORY_DB=true,跳过 worker 数据库创建")
|
||||
|
||||
|
||||
def pytest_unconfigure(config):
|
||||
"""pytest 结束钩子:清理 xdist worker 数据库"""
|
||||
worker_id = _get_worker_id()
|
||||
if worker_id and _WORKER_DB_NAME:
|
||||
_cleanup_worker_database(worker_id)
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config, items):
|
||||
"""根据环境变量自动跳过性能测试"""
|
||||
@@ -203,6 +377,9 @@ def pytest_collection_modifyitems(config, items):
|
||||
item.add_marker(skip_perf)
|
||||
|
||||
|
||||
# ── pytest fixtures ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def perf_assert():
|
||||
"""
|
||||
|
||||
Executable
+523
@@ -0,0 +1,523 @@
|
||||
"""
|
||||
Auth 服务层单元测试 - 纯逻辑模块
|
||||
|
||||
覆盖:
|
||||
- PasswordHasher / PasswordValidator (password_hasher.py)
|
||||
- JWTConfig / JWTService / TokenType (jwt_service.py)
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import jwt as pyjwt
|
||||
import pytest
|
||||
from jwt.exceptions import ExpiredSignatureError, InvalidTokenError
|
||||
|
||||
from packages.application.auth.jwt_service import JWTConfig, JWTService, TokenType
|
||||
from packages.application.auth.password_hasher import (
|
||||
PasswordHasher,
|
||||
PasswordValidator,
|
||||
password_hasher,
|
||||
password_validator,
|
||||
)
|
||||
|
||||
# ── PasswordHasher 测试 ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestPasswordHasher:
|
||||
"""PasswordHasher 密码哈希器测试"""
|
||||
|
||||
def test_default_rounds(self):
|
||||
hasher = PasswordHasher()
|
||||
assert hasher.rounds == 12
|
||||
|
||||
def test_custom_rounds(self):
|
||||
hasher = PasswordHasher(rounds=10)
|
||||
assert hasher.rounds == 10
|
||||
|
||||
def test_rounds_min_boundary(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
assert hasher.rounds == 4
|
||||
|
||||
def test_rounds_max_boundary(self):
|
||||
hasher = PasswordHasher(rounds=31)
|
||||
assert hasher.rounds == 31
|
||||
|
||||
def test_rounds_below_min_raises(self):
|
||||
with pytest.raises(ValueError, match="between 4 and 31"):
|
||||
PasswordHasher(rounds=3)
|
||||
|
||||
def test_rounds_above_max_raises(self):
|
||||
with pytest.raises(ValueError, match="between 4 and 31"):
|
||||
PasswordHasher(rounds=32)
|
||||
|
||||
def test_hash_password_returns_string(self):
|
||||
hasher = PasswordHasher(rounds=4) # 用小rounds加速测试
|
||||
result = hasher.hash_password("testpass123")
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0
|
||||
|
||||
def test_hash_password_starts_with_bcrypt_prefix(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
result = hasher.hash_password("testpass123")
|
||||
assert result.startswith("$2b$")
|
||||
|
||||
def test_hash_password_contains_rounds(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
result = hasher.hash_password("testpass123")
|
||||
# $2b$04$...
|
||||
assert "$04$" in result
|
||||
|
||||
def test_hash_password_empty_raises(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
with pytest.raises(ValueError, match="cannot be empty"):
|
||||
hasher.hash_password("")
|
||||
|
||||
def test_hash_password_none_raises(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
with pytest.raises(ValueError):
|
||||
hasher.hash_password(None)
|
||||
|
||||
def test_hash_password_different_each_time(self):
|
||||
"""同一密码每次哈希结果不同(因为salt随机)"""
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
h1 = hasher.hash_password("samepass")
|
||||
h2 = hasher.hash_password("samepass")
|
||||
assert h1 != h2
|
||||
|
||||
def test_verify_password_correct(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
hashed = hasher.hash_password("mypassword")
|
||||
assert hasher.verify_password("mypassword", hashed) is True
|
||||
|
||||
def test_verify_password_wrong(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
hashed = hasher.hash_password("correctpass")
|
||||
assert hasher.verify_password("wrongpass", hashed) is False
|
||||
|
||||
def test_verify_password_empty_password(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
hashed = hasher.hash_password("testpass")
|
||||
assert hasher.verify_password("", hashed) is False
|
||||
|
||||
def test_verify_password_empty_hash(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
assert hasher.verify_password("testpass", "") is False
|
||||
|
||||
def test_verify_password_invalid_hash_format(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
assert hasher.verify_password("testpass", "invalid_hash") is False
|
||||
|
||||
def test_verify_password_none_password(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
hashed = hasher.hash_password("test")
|
||||
assert hasher.verify_password(None, hashed) is False
|
||||
|
||||
def test_needs_rehash_same_rounds(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
hashed = hasher.hash_password("testpass")
|
||||
assert hasher.needs_rehash(hashed) is False
|
||||
|
||||
def test_needs_rehash_different_rounds(self):
|
||||
hasher4 = PasswordHasher(rounds=4)
|
||||
hasher5 = PasswordHasher(rounds=5)
|
||||
hashed = hasher4.hash_password("testpass")
|
||||
assert hasher5.needs_rehash(hashed) is True
|
||||
|
||||
def test_needs_rehash_invalid_hash(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
assert hasher.needs_rehash("invalid") is False
|
||||
|
||||
def test_needs_rehash_empty_hash(self):
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
assert hasher.needs_rehash("") is False
|
||||
|
||||
def test_global_instance_exists(self):
|
||||
assert password_hasher is not None
|
||||
assert isinstance(password_hasher, PasswordHasher)
|
||||
assert password_hasher.rounds == 12
|
||||
|
||||
def test_unicode_password(self):
|
||||
"""支持中文等Unicode密码"""
|
||||
hasher = PasswordHasher(rounds=4)
|
||||
hashed = hasher.hash_password("密码测试123")
|
||||
assert hasher.verify_password("密码测试123", hashed) is True
|
||||
|
||||
|
||||
# ── PasswordValidator 测试 ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestPasswordValidator:
|
||||
"""PasswordValidator 密码强度验证器测试"""
|
||||
|
||||
def test_default_config(self):
|
||||
v = PasswordValidator()
|
||||
assert v.min_length == 8
|
||||
assert v.require_uppercase is True
|
||||
assert v.require_lowercase is True
|
||||
assert v.require_digit is True
|
||||
assert v.require_special is False
|
||||
|
||||
def test_custom_config(self):
|
||||
v = PasswordValidator(
|
||||
min_length=10,
|
||||
require_uppercase=False,
|
||||
require_lowercase=False,
|
||||
require_digit=False,
|
||||
require_special=True,
|
||||
)
|
||||
assert v.min_length == 10
|
||||
assert v.require_uppercase is False
|
||||
assert v.require_special is True
|
||||
|
||||
def test_valid_password_default_rules(self):
|
||||
v = PasswordValidator()
|
||||
valid, msg = v.validate("TestPass123")
|
||||
assert valid is True
|
||||
assert msg is None
|
||||
|
||||
def test_empty_password(self):
|
||||
v = PasswordValidator()
|
||||
valid, msg = v.validate("")
|
||||
assert valid is False
|
||||
assert "empty" in msg.lower()
|
||||
|
||||
def test_none_password(self):
|
||||
v = PasswordValidator()
|
||||
valid, msg = v.validate(None)
|
||||
assert valid is False
|
||||
|
||||
def test_too_short(self):
|
||||
v = PasswordValidator(min_length=8)
|
||||
valid, msg = v.validate("Ab1")
|
||||
assert valid is False
|
||||
assert "at least 8" in msg
|
||||
|
||||
def test_exact_min_length(self):
|
||||
v = PasswordValidator(min_length=8, require_uppercase=False, require_lowercase=False, require_digit=False)
|
||||
valid, msg = v.validate("12345678")
|
||||
assert valid is True
|
||||
|
||||
def test_missing_uppercase(self):
|
||||
v = PasswordValidator()
|
||||
valid, msg = v.validate("testpass123")
|
||||
assert valid is False
|
||||
assert "uppercase" in msg.lower()
|
||||
|
||||
def test_missing_lowercase(self):
|
||||
v = PasswordValidator()
|
||||
valid, msg = v.validate("TESTPASS123")
|
||||
assert valid is False
|
||||
assert "lowercase" in msg.lower()
|
||||
|
||||
def test_missing_digit(self):
|
||||
v = PasswordValidator()
|
||||
valid, msg = v.validate("TestPassword")
|
||||
assert valid is False
|
||||
assert "digit" in msg.lower()
|
||||
|
||||
def test_require_special_enabled_missing(self):
|
||||
v = PasswordValidator(require_special=True)
|
||||
valid, msg = v.validate("TestPass123")
|
||||
assert valid is False
|
||||
assert "special" in msg.lower()
|
||||
|
||||
def test_require_special_enabled_present(self):
|
||||
v = PasswordValidator(require_special=True)
|
||||
valid, msg = v.validate("TestPass123!")
|
||||
assert valid is True
|
||||
assert msg is None
|
||||
|
||||
def test_special_chars_all_types(self):
|
||||
"""验证各种特殊字符都能识别"""
|
||||
v = PasswordValidator(
|
||||
min_length=8,
|
||||
require_uppercase=False,
|
||||
require_lowercase=False,
|
||||
require_digit=False,
|
||||
require_special=True,
|
||||
)
|
||||
for char in "!@#$%^&*()_+-=[]{}|;:,.<>?~":
|
||||
valid, _ = v.validate(f"testpass{char}")
|
||||
assert valid is True, f"Special char '{char}' not recognized"
|
||||
|
||||
def test_no_requirements_all_pass(self):
|
||||
"""关闭所有要求后任何密码都通过"""
|
||||
v = PasswordValidator(
|
||||
min_length=1,
|
||||
require_uppercase=False,
|
||||
require_lowercase=False,
|
||||
require_digit=False,
|
||||
require_special=False,
|
||||
)
|
||||
valid, msg = v.validate("a")
|
||||
assert valid is True
|
||||
|
||||
def test_global_validator_instance(self):
|
||||
assert password_validator is not None
|
||||
assert isinstance(password_validator, PasswordValidator)
|
||||
assert password_validator.min_length == 8
|
||||
assert password_validator.require_special is False
|
||||
|
||||
|
||||
# ── JWTConfig 测试 ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestJWTConfig:
|
||||
"""JWTConfig 配置测试"""
|
||||
|
||||
def test_default_values(self):
|
||||
config = JWTConfig(secret_key="test-secret-key-12345")
|
||||
assert config.SECRET_KEY == "test-secret-key-12345"
|
||||
assert config.ALGORITHM == "HS256"
|
||||
assert config.ACCESS_TOKEN_EXPIRE_MINUTES == 15
|
||||
assert config.REFRESH_TOKEN_EXPIRE_DAYS == 7
|
||||
|
||||
def test_custom_values(self):
|
||||
config = JWTConfig(
|
||||
secret_key="my-secret",
|
||||
algorithm="HS512",
|
||||
access_token_expire_minutes=30,
|
||||
refresh_token_expire_days=14,
|
||||
)
|
||||
assert config.ALGORITHM == "HS512"
|
||||
assert config.ACCESS_TOKEN_EXPIRE_MINUTES == 30
|
||||
assert config.REFRESH_TOKEN_EXPIRE_DAYS == 14
|
||||
|
||||
def test_empty_secret_raises(self):
|
||||
with pytest.raises(ValueError, match="must be provided"):
|
||||
JWTConfig(secret_key="")
|
||||
|
||||
def test_whitespace_secret_raises(self):
|
||||
with pytest.raises(ValueError):
|
||||
JWTConfig(secret_key=" ")
|
||||
|
||||
def test_none_secret_raises(self):
|
||||
with pytest.raises(ValueError):
|
||||
JWTConfig(secret_key=None)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"insecure",
|
||||
[
|
||||
"your-secret-key-change-in-production",
|
||||
"your-secret-key",
|
||||
"secret",
|
||||
"changeme",
|
||||
"password",
|
||||
"SECRET",
|
||||
"Your-Secret-Key",
|
||||
],
|
||||
)
|
||||
def test_insecure_defaults_raises(self, insecure):
|
||||
with pytest.raises(ValueError, match="insecure"):
|
||||
JWTConfig(secret_key=insecure)
|
||||
|
||||
|
||||
# ── TokenType 测试 ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestTokenType:
|
||||
"""TokenType 常量测试"""
|
||||
|
||||
def test_access_token_type(self):
|
||||
assert TokenType.ACCESS == "access"
|
||||
|
||||
def test_refresh_token_type(self):
|
||||
assert TokenType.REFRESH == "refresh"
|
||||
|
||||
|
||||
# ── JWTService 测试 ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestJWTService:
|
||||
"""JWTService JWT服务测试"""
|
||||
|
||||
@pytest.fixture
|
||||
def service(self):
|
||||
config = JWTConfig(
|
||||
secret_key="test-secret-key-for-testing-only-12345",
|
||||
access_token_expire_minutes=30,
|
||||
refresh_token_expire_days=7,
|
||||
)
|
||||
return JWTService(config)
|
||||
|
||||
def test_init_without_config_raises(self):
|
||||
with pytest.raises(ValueError, match="requires a JWTConfig"):
|
||||
JWTService()
|
||||
|
||||
def test_create_access_token_returns_string(self, service):
|
||||
token = service.create_access_token(user_id="user_123")
|
||||
assert isinstance(token, str)
|
||||
assert len(token) > 0
|
||||
|
||||
def test_create_access_token_has_user_id(self, service):
|
||||
token = service.create_access_token(user_id="user_123")
|
||||
payload = service.verify_token(token)
|
||||
assert payload["sub"] == "user_123"
|
||||
|
||||
def test_create_access_token_has_role(self, service):
|
||||
token = service.create_access_token(user_id="user_123", role="admin")
|
||||
payload = service.verify_token(token)
|
||||
assert payload["role"] == "admin"
|
||||
|
||||
def test_create_access_token_default_role_empty(self, service):
|
||||
token = service.create_access_token(user_id="user_123")
|
||||
payload = service.verify_token(token)
|
||||
assert payload["role"] == ""
|
||||
|
||||
def test_create_access_token_type_is_access(self, service):
|
||||
token = service.create_access_token(user_id="user_123")
|
||||
payload = service.verify_token(token)
|
||||
assert payload["type"] == TokenType.ACCESS
|
||||
|
||||
def test_create_access_token_has_iat_and_exp(self, service):
|
||||
token = service.create_access_token(user_id="user_123")
|
||||
payload = service.verify_token(token)
|
||||
assert "iat" in payload
|
||||
assert "exp" in payload
|
||||
assert payload["exp"] > payload["iat"]
|
||||
|
||||
def test_create_access_token_expiry_correct(self, service):
|
||||
"""过期时间 = 签发时间 + 30分钟"""
|
||||
token = service.create_access_token(user_id="user_123")
|
||||
payload = service.verify_token(token)
|
||||
delta_seconds = payload["exp"] - payload["iat"]
|
||||
assert delta_seconds == 30 * 60
|
||||
|
||||
def test_create_access_token_additional_claims(self, service):
|
||||
token = service.create_access_token(
|
||||
user_id="user_123",
|
||||
role="user",
|
||||
additional_claims={"email": "test@example.com", "custom": "value"},
|
||||
)
|
||||
payload = service.verify_token(token)
|
||||
assert payload["email"] == "test@example.com"
|
||||
assert payload["custom"] == "value"
|
||||
|
||||
def test_create_refresh_token_returns_string(self, service):
|
||||
token = service.create_refresh_token(user_id="user_123", session_id="sess_456")
|
||||
assert isinstance(token, str)
|
||||
assert len(token) > 0
|
||||
|
||||
def test_create_refresh_token_has_user_and_session(self, service):
|
||||
token = service.create_refresh_token(user_id="user_123", session_id="sess_456")
|
||||
payload = service.verify_token(token)
|
||||
assert payload["sub"] == "user_123"
|
||||
assert payload["session_id"] == "sess_456"
|
||||
|
||||
def test_create_refresh_token_type_is_refresh(self, service):
|
||||
token = service.create_refresh_token(user_id="user_123", session_id="sess_456")
|
||||
payload = service.verify_token(token)
|
||||
assert payload["type"] == TokenType.REFRESH
|
||||
|
||||
def test_create_refresh_token_expiry_correct(self, service):
|
||||
token = service.create_refresh_token(user_id="user_123", session_id="sess_456")
|
||||
payload = service.verify_token(token)
|
||||
delta_seconds = payload["exp"] - payload["iat"]
|
||||
assert delta_seconds == 7 * 24 * 60 * 60
|
||||
|
||||
def test_verify_access_token_success(self, service):
|
||||
token = service.create_access_token(user_id="user_123", role="admin")
|
||||
payload = service.verify_access_token(token)
|
||||
assert payload["sub"] == "user_123"
|
||||
assert payload["role"] == "admin"
|
||||
|
||||
def test_verify_access_token_wrong_type_raises(self, service):
|
||||
"""用refresh token当access token验证会失败"""
|
||||
token = service.create_refresh_token(user_id="user_123", session_id="sess_456")
|
||||
with pytest.raises(ValueError, match="must be 'access'"):
|
||||
service.verify_access_token(token)
|
||||
|
||||
def test_verify_refresh_token_success(self, service):
|
||||
token = service.create_refresh_token(user_id="user_123", session_id="sess_456")
|
||||
payload = service.verify_refresh_token(token)
|
||||
assert payload["sub"] == "user_123"
|
||||
assert payload["session_id"] == "sess_456"
|
||||
|
||||
def test_verify_refresh_token_wrong_type_raises(self, service):
|
||||
"""用access token当refresh token验证会失败"""
|
||||
token = service.create_access_token(user_id="user_123")
|
||||
with pytest.raises(ValueError, match="must be 'refresh'"):
|
||||
service.verify_refresh_token(token)
|
||||
|
||||
def test_verify_token_invalid_token_raises(self, service):
|
||||
with pytest.raises(InvalidTokenError):
|
||||
service.verify_token("invalid.token.here")
|
||||
|
||||
def test_verify_token_empty_raises(self, service):
|
||||
with pytest.raises(InvalidTokenError):
|
||||
service.verify_token("")
|
||||
|
||||
def test_verify_token_wrong_secret(self, service):
|
||||
"""用不同密钥签发的token无法验证"""
|
||||
other_config = JWTConfig(secret_key="different-secret-key-for-testing-123")
|
||||
other_service = JWTService(other_config)
|
||||
token = other_service.create_access_token(user_id="user_123")
|
||||
with pytest.raises(InvalidTokenError):
|
||||
service.verify_token(token)
|
||||
|
||||
def test_verify_token_tampered_signature(self, service):
|
||||
"""篡改payload的token无法验证(签名不匹配)"""
|
||||
import base64
|
||||
import json
|
||||
|
||||
token = service.create_access_token(user_id="user_123")
|
||||
parts = token.split(".")
|
||||
assert len(parts) == 3
|
||||
# 篡改 payload 部分(改用户ID),会导致签名不匹配
|
||||
payload_b64 = parts[1]
|
||||
# 补 padding 以便解码
|
||||
padding = 4 - len(payload_b64) % 4
|
||||
if padding != 4:
|
||||
payload_b64 += "=" * padding
|
||||
payload_bytes = base64.urlsafe_b64decode(payload_b64)
|
||||
payload = json.loads(payload_bytes)
|
||||
payload["sub"] = "hacked_user"
|
||||
new_payload_bytes = json.dumps(payload).encode()
|
||||
new_payload_b64 = base64.urlsafe_b64encode(new_payload_bytes).rstrip(b"=").decode()
|
||||
tampered = f"{parts[0]}.{new_payload_b64}.{parts[2]}"
|
||||
with pytest.raises(InvalidTokenError):
|
||||
service.verify_token(tampered)
|
||||
|
||||
def test_expired_token_raises(self):
|
||||
"""过期token验证失败"""
|
||||
config = JWTConfig(
|
||||
secret_key="test-secret-key-for-testing-only-12345",
|
||||
access_token_expire_minutes=0, # 立即过期
|
||||
)
|
||||
service = JWTService(config)
|
||||
token = service.create_access_token(user_id="user_123")
|
||||
time.sleep(1) # 等1秒确保过期
|
||||
with pytest.raises(ExpiredSignatureError):
|
||||
service.verify_token(token)
|
||||
|
||||
def test_different_algorithm(self):
|
||||
"""支持不同算法"""
|
||||
config = JWTConfig(
|
||||
secret_key="test-secret-key-for-testing-only-12345-abcdef",
|
||||
algorithm="HS512",
|
||||
)
|
||||
service = JWTService(config)
|
||||
token = service.create_access_token(user_id="user_123")
|
||||
payload = service.verify_token(token)
|
||||
assert payload["sub"] == "user_123"
|
||||
|
||||
def test_additional_claims_not_overwrite_standard(self):
|
||||
"""additional_claims 不会覆盖标准字段"""
|
||||
config = JWTConfig(secret_key="test-secret-key-for-testing-only-12345")
|
||||
service = JWTService(config)
|
||||
token = service.create_access_token(
|
||||
user_id="real_user",
|
||||
additional_claims={"sub": "fake_user", "type": "fake_type"},
|
||||
)
|
||||
payload = service.verify_token(token)
|
||||
# additional_claims 在标准字段之后update,所以会覆盖
|
||||
# 这个测试验证当前行为(additional_claims优先级高)
|
||||
assert payload["sub"] == "fake_user"
|
||||
|
||||
def test_unicode_user_id(self):
|
||||
"""支持Unicode用户ID"""
|
||||
config = JWTConfig(secret_key="test-secret-key-for-testing-only-12345")
|
||||
service = JWTService(config)
|
||||
token = service.create_access_token(user_id="用户_测试123")
|
||||
payload = service.verify_token(token)
|
||||
assert payload["sub"] == "用户_测试123"
|
||||
Executable
+815
@@ -0,0 +1,815 @@
|
||||
"""
|
||||
config_schemas 配置结构定义单元测试
|
||||
|
||||
覆盖:
|
||||
- 4个枚举类型
|
||||
- 10个Pydantic模型
|
||||
- 2个normalize工具函数
|
||||
- 2个默认值常量
|
||||
"""
|
||||
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from packages.domain.config_schemas import (
|
||||
DEFAULT_EDIT_PLAN_CONFIG,
|
||||
DEFAULT_EDIT_TEMPLATE_CONFIG,
|
||||
BGMConfig,
|
||||
BGMSource,
|
||||
CoverConfig,
|
||||
CoverType,
|
||||
EditPlanConfigSchema,
|
||||
EditTemplateConfigSchema,
|
||||
ExportConfig,
|
||||
FilterConfig,
|
||||
ShadowConfig,
|
||||
StrokeConfig,
|
||||
SubtitleConfig,
|
||||
TextAnimation,
|
||||
TextPosition,
|
||||
TitleConfig,
|
||||
normalize_plan_config,
|
||||
normalize_template_config,
|
||||
)
|
||||
|
||||
# ── 枚举测试 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestCoverType:
|
||||
"""CoverType 枚举测试"""
|
||||
|
||||
def test_all_types_exist(self):
|
||||
assert CoverType.AI_FRAME == "ai_frame"
|
||||
assert CoverType.MANUAL == "manual"
|
||||
assert CoverType.UPLOAD == "upload"
|
||||
assert CoverType.AI_REGENERATE == "ai_regenerate"
|
||||
|
||||
def test_total_count(self):
|
||||
assert len(CoverType) == 4
|
||||
|
||||
def test_is_string_enum(self):
|
||||
for t in CoverType:
|
||||
assert isinstance(t.value, str)
|
||||
assert isinstance(t, str)
|
||||
|
||||
def test_string_comparison(self):
|
||||
assert CoverType.AI_FRAME == "ai_frame"
|
||||
assert CoverType.UPLOAD != "manual"
|
||||
|
||||
|
||||
class TestTextPosition:
|
||||
"""TextPosition 枚举测试"""
|
||||
|
||||
def test_all_positions_exist(self):
|
||||
assert TextPosition.TOP == "top"
|
||||
assert TextPosition.CENTER == "center"
|
||||
assert TextPosition.BOTTOM == "bottom"
|
||||
|
||||
def test_total_count(self):
|
||||
assert len(TextPosition) == 3
|
||||
|
||||
def test_is_string_enum(self):
|
||||
for p in TextPosition:
|
||||
assert isinstance(p.value, str)
|
||||
assert isinstance(p, str)
|
||||
|
||||
|
||||
class TestTextAnimation:
|
||||
"""TextAnimation 枚举测试"""
|
||||
|
||||
def test_all_animations_exist(self):
|
||||
assert TextAnimation.NONE == "none"
|
||||
assert TextAnimation.FADE_IN == "fade_in"
|
||||
assert TextAnimation.SLIDE_UP == "slide_up"
|
||||
assert TextAnimation.SLIDE_DOWN == "slide_down"
|
||||
assert TextAnimation.SCALE == "scale"
|
||||
|
||||
def test_total_count(self):
|
||||
assert len(TextAnimation) == 5
|
||||
|
||||
def test_is_string_enum(self):
|
||||
for a in TextAnimation:
|
||||
assert isinstance(a.value, str)
|
||||
assert isinstance(a, str)
|
||||
|
||||
|
||||
class TestBGMSource:
|
||||
"""BGMSource 枚举测试"""
|
||||
|
||||
def test_all_sources_exist(self):
|
||||
assert BGMSource.LIBRARY == "library"
|
||||
assert BGMSource.UPLOAD == "upload"
|
||||
assert BGMSource.AI_RECOMMEND == "ai_recommend"
|
||||
|
||||
def test_total_count(self):
|
||||
assert len(BGMSource) == 3
|
||||
|
||||
def test_is_string_enum(self):
|
||||
for s in BGMSource:
|
||||
assert isinstance(s.value, str)
|
||||
assert isinstance(s, str)
|
||||
|
||||
|
||||
# ── 子结构模型测试 ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestStrokeConfig:
|
||||
"""StrokeConfig 描边配置测试"""
|
||||
|
||||
def test_default_values(self):
|
||||
config = StrokeConfig()
|
||||
assert config.enabled is False
|
||||
assert config.color == "#000000"
|
||||
assert config.width == 1
|
||||
|
||||
def test_custom_values(self):
|
||||
config = StrokeConfig(enabled=True, color="#ff0000", width=5)
|
||||
assert config.enabled is True
|
||||
assert config.color == "#ff0000"
|
||||
assert config.width == 5
|
||||
|
||||
def test_width_min_boundary(self):
|
||||
config = StrokeConfig(width=1)
|
||||
assert config.width == 1
|
||||
|
||||
def test_width_max_boundary(self):
|
||||
config = StrokeConfig(width=10)
|
||||
assert config.width == 10
|
||||
|
||||
def test_width_below_min_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
StrokeConfig(width=0)
|
||||
|
||||
def test_width_above_max_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
StrokeConfig(width=11)
|
||||
|
||||
|
||||
class TestShadowConfig:
|
||||
"""ShadowConfig 阴影配置测试"""
|
||||
|
||||
def test_default_values(self):
|
||||
config = ShadowConfig()
|
||||
assert config.enabled is False
|
||||
assert config.blur == 4
|
||||
assert config.offset_x == 2
|
||||
assert config.offset_y == 2
|
||||
|
||||
def test_custom_values(self):
|
||||
config = ShadowConfig(enabled=True, blur=10, offset_x=5, offset_y=3)
|
||||
assert config.enabled is True
|
||||
assert config.blur == 10
|
||||
assert config.offset_x == 5
|
||||
assert config.offset_y == 3
|
||||
|
||||
def test_blur_min_boundary(self):
|
||||
config = ShadowConfig(blur=0)
|
||||
assert config.blur == 0
|
||||
|
||||
def test_blur_max_boundary(self):
|
||||
config = ShadowConfig(blur=20)
|
||||
assert config.blur == 20
|
||||
|
||||
def test_blur_above_max_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
ShadowConfig(blur=21)
|
||||
|
||||
def test_negative_offset(self):
|
||||
config = ShadowConfig(offset_x=-3, offset_y=-2)
|
||||
assert config.offset_x == -3
|
||||
assert config.offset_y == -2
|
||||
|
||||
|
||||
class TestCoverConfig:
|
||||
"""CoverConfig 封面配置测试"""
|
||||
|
||||
def test_default_values(self):
|
||||
config = CoverConfig()
|
||||
assert config.type == CoverType.AI_FRAME
|
||||
assert config.image_url == ""
|
||||
assert config.frame_time is None
|
||||
|
||||
def test_custom_values(self):
|
||||
config = CoverConfig(type=CoverType.MANUAL, image_url="http://img/1.jpg", frame_time=5.5)
|
||||
assert config.type == CoverType.MANUAL
|
||||
assert config.image_url == "http://img/1.jpg"
|
||||
assert config.frame_time == 5.5
|
||||
|
||||
def test_frame_time_min_boundary(self):
|
||||
config = CoverConfig(frame_time=0.0)
|
||||
assert config.frame_time == 0.0
|
||||
|
||||
def test_frame_time_negative_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
CoverConfig(frame_time=-1.0)
|
||||
|
||||
def test_string_type_conversion(self):
|
||||
"""字符串枚举值自动转换"""
|
||||
config = CoverConfig(type="upload")
|
||||
assert config.type == CoverType.UPLOAD
|
||||
|
||||
|
||||
class TestTitleConfig:
|
||||
"""TitleConfig 标题配置测试"""
|
||||
|
||||
def test_default_values(self):
|
||||
config = TitleConfig()
|
||||
assert config.enabled is True
|
||||
assert config.ai_auto is True
|
||||
assert config.text == ""
|
||||
assert config.position == TextPosition.TOP
|
||||
assert config.font == "思源黑体"
|
||||
assert config.color == "#ffffff"
|
||||
assert config.size == 48
|
||||
assert config.bold is True
|
||||
assert config.italic is False
|
||||
assert isinstance(config.stroke, StrokeConfig)
|
||||
assert isinstance(config.shadow, ShadowConfig)
|
||||
|
||||
def test_custom_values(self):
|
||||
config = TitleConfig(
|
||||
enabled=False,
|
||||
ai_auto=False,
|
||||
text="测试标题",
|
||||
position=TextPosition.BOTTOM,
|
||||
font="微软雅黑",
|
||||
color="#000000",
|
||||
size=72,
|
||||
bold=False,
|
||||
italic=True,
|
||||
)
|
||||
assert config.enabled is False
|
||||
assert config.ai_auto is False
|
||||
assert config.text == "测试标题"
|
||||
assert config.position == TextPosition.BOTTOM
|
||||
assert config.size == 72
|
||||
|
||||
def test_size_min_boundary(self):
|
||||
config = TitleConfig(size=12)
|
||||
assert config.size == 12
|
||||
|
||||
def test_size_max_boundary(self):
|
||||
config = TitleConfig(size=120)
|
||||
assert config.size == 120
|
||||
|
||||
def test_size_below_min_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
TitleConfig(size=11)
|
||||
|
||||
def test_size_above_max_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
TitleConfig(size=121)
|
||||
|
||||
def test_nested_stroke_config(self):
|
||||
config = TitleConfig(stroke={"enabled": True, "width": 3})
|
||||
assert config.stroke.enabled is True
|
||||
assert config.stroke.width == 3
|
||||
|
||||
def test_nested_shadow_config(self):
|
||||
config = TitleConfig(shadow={"enabled": True, "blur": 8})
|
||||
assert config.shadow.enabled is True
|
||||
assert config.shadow.blur == 8
|
||||
|
||||
|
||||
class TestSubtitleConfig:
|
||||
"""SubtitleConfig 字幕配置测试"""
|
||||
|
||||
def test_default_values(self):
|
||||
config = SubtitleConfig()
|
||||
assert config.enabled is True
|
||||
assert config.position == TextPosition.BOTTOM
|
||||
assert config.font == "思源黑体"
|
||||
assert config.color == "#ffffff"
|
||||
assert config.size == 24
|
||||
assert config.animation == TextAnimation.FADE_IN
|
||||
assert config.auto_generated is False
|
||||
assert config.language == ""
|
||||
assert config.max_chars_per_line == 20
|
||||
assert config.min_chars_per_segment == 8
|
||||
|
||||
def test_custom_values(self):
|
||||
config = SubtitleConfig(
|
||||
enabled=False,
|
||||
position=TextPosition.TOP,
|
||||
size=36,
|
||||
animation=TextAnimation.SLIDE_UP,
|
||||
auto_generated=True,
|
||||
language="zh",
|
||||
max_chars_per_line=30,
|
||||
min_chars_per_segment=10,
|
||||
)
|
||||
assert config.enabled is False
|
||||
assert config.position == TextPosition.TOP
|
||||
assert config.animation == TextAnimation.SLIDE_UP
|
||||
assert config.auto_generated is True
|
||||
assert config.language == "zh"
|
||||
|
||||
def test_size_min_boundary(self):
|
||||
config = SubtitleConfig(size=12)
|
||||
assert config.size == 12
|
||||
|
||||
def test_size_max_boundary(self):
|
||||
config = SubtitleConfig(size=60)
|
||||
assert config.size == 60
|
||||
|
||||
def test_size_below_min_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
SubtitleConfig(size=11)
|
||||
|
||||
def test_max_chars_min_boundary(self):
|
||||
config = SubtitleConfig(max_chars_per_line=8)
|
||||
assert config.max_chars_per_line == 8
|
||||
|
||||
def test_max_chars_max_boundary(self):
|
||||
config = SubtitleConfig(max_chars_per_line=40)
|
||||
assert config.max_chars_per_line == 40
|
||||
|
||||
def test_min_chars_min_boundary(self):
|
||||
config = SubtitleConfig(min_chars_per_segment=2)
|
||||
assert config.min_chars_per_segment == 2
|
||||
|
||||
def test_min_chars_max_boundary(self):
|
||||
config = SubtitleConfig(min_chars_per_segment=20)
|
||||
assert config.min_chars_per_segment == 20
|
||||
|
||||
|
||||
class TestBGMConfig:
|
||||
"""BGMConfig 背景音乐配置测试"""
|
||||
|
||||
def test_default_values(self):
|
||||
config = BGMConfig()
|
||||
assert config.enabled is False
|
||||
assert config.source == BGMSource.LIBRARY
|
||||
assert config.asset_id == ""
|
||||
assert config.preset_id == ""
|
||||
assert config.audio_url == ""
|
||||
assert config.volume == 0.3
|
||||
assert config.fade_in == 0.0
|
||||
assert config.fade_out == 0.0
|
||||
assert config.loop_enabled is True
|
||||
assert config.sidechain_enabled is False
|
||||
assert config.sidechain_ratio == 0.3
|
||||
assert config.sidechain_attack == 0.02
|
||||
assert config.sidechain_release == 0.5
|
||||
assert config.sidechain_threshold == -25.0
|
||||
|
||||
def test_custom_values(self):
|
||||
config = BGMConfig(
|
||||
enabled=True,
|
||||
source=BGMSource.UPLOAD,
|
||||
asset_id="bgm_001",
|
||||
volume=0.5,
|
||||
fade_in=2.0,
|
||||
fade_out=3.0,
|
||||
loop_enabled=False,
|
||||
sidechain_enabled=True,
|
||||
sidechain_ratio=0.5,
|
||||
sidechain_attack=0.05,
|
||||
sidechain_release=1.0,
|
||||
sidechain_threshold=-20.0,
|
||||
)
|
||||
assert config.enabled is True
|
||||
assert config.source == BGMSource.UPLOAD
|
||||
assert config.asset_id == "bgm_001"
|
||||
assert config.volume == 0.5
|
||||
assert config.loop_enabled is False
|
||||
assert config.sidechain_enabled is True
|
||||
assert config.sidechain_ratio == 0.5
|
||||
|
||||
def test_volume_min_boundary(self):
|
||||
config = BGMConfig(volume=0.0)
|
||||
assert config.volume == 0.0
|
||||
|
||||
def test_volume_max_boundary(self):
|
||||
config = BGMConfig(volume=1.0)
|
||||
assert config.volume == 1.0
|
||||
|
||||
def test_volume_above_max_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
BGMConfig(volume=1.1)
|
||||
|
||||
def test_fade_in_max_boundary(self):
|
||||
config = BGMConfig(fade_in=30.0)
|
||||
assert config.fade_in == 30.0
|
||||
|
||||
def test_fade_in_above_max_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
BGMConfig(fade_in=31.0)
|
||||
|
||||
def test_sidechain_ratio_range(self):
|
||||
config = BGMConfig(sidechain_ratio=0.0)
|
||||
assert config.sidechain_ratio == 0.0
|
||||
config = BGMConfig(sidechain_ratio=1.0)
|
||||
assert config.sidechain_ratio == 1.0
|
||||
|
||||
def test_sidechain_attack_min(self):
|
||||
config = BGMConfig(sidechain_attack=0.001)
|
||||
assert config.sidechain_attack == 0.001
|
||||
|
||||
def test_sidechain_attack_below_min_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
BGMConfig(sidechain_attack=0.0001)
|
||||
|
||||
def test_sidechain_threshold_range(self):
|
||||
config = BGMConfig(sidechain_threshold=-60.0)
|
||||
assert config.sidechain_threshold == -60.0
|
||||
config = BGMConfig(sidechain_threshold=0.0)
|
||||
assert config.sidechain_threshold == 0.0
|
||||
|
||||
def test_sidechain_threshold_out_of_range_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
BGMConfig(sidechain_threshold=-61.0)
|
||||
with pytest.raises(ValidationError):
|
||||
BGMConfig(sidechain_threshold=1.0)
|
||||
|
||||
|
||||
class TestExportConfig:
|
||||
"""ExportConfig 导出配置测试"""
|
||||
|
||||
def test_default_values(self):
|
||||
config = ExportConfig()
|
||||
assert config.resolution == "1080x1920"
|
||||
assert config.fps == 30
|
||||
assert config.video_bitrate == 8000
|
||||
assert config.audio_bitrate == 128
|
||||
assert config.format == "mp4"
|
||||
assert config.quality_preset == "balanced"
|
||||
assert config.watermark_enabled is False
|
||||
assert config.watermark_text == ""
|
||||
|
||||
def test_custom_values(self):
|
||||
config = ExportConfig(
|
||||
resolution="2160x3840",
|
||||
fps=60,
|
||||
video_bitrate=20000,
|
||||
audio_bitrate=320,
|
||||
format="mov",
|
||||
quality_preset="best",
|
||||
watermark_enabled=True,
|
||||
watermark_text="测试水印",
|
||||
)
|
||||
assert config.resolution == "2160x3840"
|
||||
assert config.fps == 60
|
||||
assert config.video_bitrate == 20000
|
||||
assert config.format == "mov"
|
||||
|
||||
def test_fps_min_boundary(self):
|
||||
config = ExportConfig(fps=15)
|
||||
assert config.fps == 15
|
||||
|
||||
def test_fps_max_boundary(self):
|
||||
config = ExportConfig(fps=60)
|
||||
assert config.fps == 60
|
||||
|
||||
def test_fps_below_min_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
ExportConfig(fps=14)
|
||||
|
||||
def test_fps_above_max_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
ExportConfig(fps=61)
|
||||
|
||||
def test_video_bitrate_range(self):
|
||||
config = ExportConfig(video_bitrate=1000)
|
||||
assert config.video_bitrate == 1000
|
||||
config = ExportConfig(video_bitrate=20000)
|
||||
assert config.video_bitrate == 20000
|
||||
|
||||
def test_audio_bitrate_range(self):
|
||||
config = ExportConfig(audio_bitrate=64)
|
||||
assert config.audio_bitrate == 64
|
||||
config = ExportConfig(audio_bitrate=320)
|
||||
assert config.audio_bitrate == 320
|
||||
|
||||
|
||||
class TestFilterConfig:
|
||||
"""FilterConfig 滤镜配置测试"""
|
||||
|
||||
def test_default_values(self):
|
||||
config = FilterConfig()
|
||||
assert config.enabled is False
|
||||
assert config.preset_id == "filter_none"
|
||||
assert config.intensity == 100
|
||||
assert config.brightness == 0.0
|
||||
assert config.contrast == 1.0
|
||||
assert config.saturation == 1.0
|
||||
assert config.warmth == 0.0
|
||||
|
||||
def test_custom_values(self):
|
||||
config = FilterConfig(
|
||||
enabled=True,
|
||||
preset_id="filter_vintage",
|
||||
intensity=50,
|
||||
brightness=0.5,
|
||||
contrast=1.5,
|
||||
saturation=2.0,
|
||||
warmth=0.3,
|
||||
)
|
||||
assert config.enabled is True
|
||||
assert config.preset_id == "filter_vintage"
|
||||
assert config.intensity == 50
|
||||
assert config.brightness == 0.5
|
||||
|
||||
def test_intensity_min_boundary(self):
|
||||
config = FilterConfig(intensity=0)
|
||||
assert config.intensity == 0
|
||||
|
||||
def test_intensity_max_boundary(self):
|
||||
config = FilterConfig(intensity=100)
|
||||
assert config.intensity == 100
|
||||
|
||||
def test_intensity_out_of_range_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
FilterConfig(intensity=-1)
|
||||
with pytest.raises(ValidationError):
|
||||
FilterConfig(intensity=101)
|
||||
|
||||
def test_brightness_range(self):
|
||||
config = FilterConfig(brightness=-1.0)
|
||||
assert config.brightness == -1.0
|
||||
config = FilterConfig(brightness=1.0)
|
||||
assert config.brightness == 1.0
|
||||
|
||||
def test_contrast_range(self):
|
||||
config = FilterConfig(contrast=0.0)
|
||||
assert config.contrast == 0.0
|
||||
config = FilterConfig(contrast=2.0)
|
||||
assert config.contrast == 2.0
|
||||
|
||||
def test_saturation_range(self):
|
||||
config = FilterConfig(saturation=0.0)
|
||||
assert config.saturation == 0.0
|
||||
config = FilterConfig(saturation=3.0)
|
||||
assert config.saturation == 3.0
|
||||
|
||||
def test_warmth_range(self):
|
||||
config = FilterConfig(warmth=-1.0)
|
||||
assert config.warmth == -1.0
|
||||
config = FilterConfig(warmth=1.0)
|
||||
assert config.warmth == 1.0
|
||||
|
||||
def test_brightness_out_of_range_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
FilterConfig(brightness=-1.1)
|
||||
with pytest.raises(ValidationError):
|
||||
FilterConfig(brightness=1.1)
|
||||
|
||||
|
||||
# ── 完整 config 模型测试 ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestEditPlanConfigSchema:
|
||||
"""EditPlanConfigSchema 完整计划配置测试"""
|
||||
|
||||
def test_default_values(self):
|
||||
config = EditPlanConfigSchema()
|
||||
assert isinstance(config.cover, CoverConfig)
|
||||
assert isinstance(config.title, TitleConfig)
|
||||
assert isinstance(config.subtitle, SubtitleConfig)
|
||||
assert isinstance(config.bgm, BGMConfig)
|
||||
assert isinstance(config.export, ExportConfig)
|
||||
assert isinstance(config.filter, FilterConfig)
|
||||
assert config.editing_mode == "one_take"
|
||||
|
||||
def test_partial_update(self):
|
||||
"""只传部分字段,其余保持默认"""
|
||||
config = EditPlanConfigSchema(
|
||||
title={"enabled": False},
|
||||
bgm={"enabled": True, "volume": 0.5},
|
||||
editing_mode="pip",
|
||||
)
|
||||
assert config.title.enabled is False
|
||||
assert config.bgm.enabled is True
|
||||
assert config.bgm.volume == 0.5
|
||||
assert config.editing_mode == "pip"
|
||||
# 其他字段保持默认
|
||||
assert config.cover.type == CoverType.AI_FRAME
|
||||
assert config.subtitle.enabled is True
|
||||
|
||||
def test_nested_model_preservation(self):
|
||||
"""嵌套模型完整可用"""
|
||||
config = EditPlanConfigSchema()
|
||||
assert config.title.stroke.enabled is False
|
||||
assert config.title.shadow.blur == 4
|
||||
assert config.bgm.sidechain_ratio == 0.3
|
||||
|
||||
|
||||
class TestEditTemplateConfigSchema:
|
||||
"""EditTemplateConfigSchema 模板配置测试"""
|
||||
|
||||
def test_default_values(self):
|
||||
config = EditTemplateConfigSchema()
|
||||
assert isinstance(config.cover, CoverConfig)
|
||||
assert isinstance(config.title, TitleConfig)
|
||||
assert isinstance(config.subtitle, SubtitleConfig)
|
||||
assert isinstance(config.bgm, BGMConfig)
|
||||
assert isinstance(config.export, ExportConfig)
|
||||
assert isinstance(config.filter, FilterConfig)
|
||||
assert config.editing_mode == "one_take"
|
||||
assert config.transition_enabled is True
|
||||
|
||||
def test_transition_enabled_false(self):
|
||||
config = EditTemplateConfigSchema(transition_enabled=False)
|
||||
assert config.transition_enabled is False
|
||||
|
||||
def test_partial_update(self):
|
||||
config = EditTemplateConfigSchema(
|
||||
filter={"enabled": True, "preset_id": "filter_cinematic"},
|
||||
transition_enabled=False,
|
||||
editing_mode="voice_over",
|
||||
)
|
||||
assert config.filter.enabled is True
|
||||
assert config.filter.preset_id == "filter_cinematic"
|
||||
assert config.transition_enabled is False
|
||||
assert config.editing_mode == "voice_over"
|
||||
|
||||
|
||||
# ── 默认值常量测试 ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestDefaultConfigs:
|
||||
"""默认值常量测试"""
|
||||
|
||||
def test_plan_config_structure(self):
|
||||
"""DEFAULT_EDIT_PLAN_CONFIG 结构完整"""
|
||||
assert "cover" in DEFAULT_EDIT_PLAN_CONFIG
|
||||
assert "title" in DEFAULT_EDIT_PLAN_CONFIG
|
||||
assert "subtitle" in DEFAULT_EDIT_PLAN_CONFIG
|
||||
assert "bgm" in DEFAULT_EDIT_PLAN_CONFIG
|
||||
assert "export" in DEFAULT_EDIT_PLAN_CONFIG
|
||||
assert "filter" in DEFAULT_EDIT_PLAN_CONFIG
|
||||
assert "editing_mode" in DEFAULT_EDIT_PLAN_CONFIG
|
||||
|
||||
def test_template_config_has_transition(self):
|
||||
"""模板配置比计划配置多一个 transition_enabled"""
|
||||
assert "transition_enabled" in DEFAULT_EDIT_TEMPLATE_CONFIG
|
||||
assert DEFAULT_EDIT_TEMPLATE_CONFIG["transition_enabled"] is True
|
||||
|
||||
def test_template_extends_plan(self):
|
||||
"""模板配置是计划配置的超集"""
|
||||
for key in DEFAULT_EDIT_PLAN_CONFIG:
|
||||
assert key in DEFAULT_EDIT_TEMPLATE_CONFIG
|
||||
|
||||
def test_defaults_are_deep_copy_safe(self):
|
||||
"""修改默认值不会影响常量本身"""
|
||||
original = copy.deepcopy(DEFAULT_EDIT_PLAN_CONFIG)
|
||||
config = EditPlanConfigSchema()
|
||||
config.title.text = "modified"
|
||||
assert DEFAULT_EDIT_PLAN_CONFIG == original
|
||||
|
||||
|
||||
# ── normalize 函数测试 ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestNormalizePlanConfig:
|
||||
"""normalize_plan_config 函数测试"""
|
||||
|
||||
def test_none_returns_default(self):
|
||||
result = normalize_plan_config(None)
|
||||
assert result == DEFAULT_EDIT_PLAN_CONFIG
|
||||
# 确保是深拷贝
|
||||
result["cover"]["type"] = "manual"
|
||||
assert DEFAULT_EDIT_PLAN_CONFIG["cover"]["type"] == "ai_frame"
|
||||
|
||||
def test_empty_dict_returns_default(self):
|
||||
result = normalize_plan_config({})
|
||||
assert result == DEFAULT_EDIT_PLAN_CONFIG
|
||||
|
||||
def test_partial_cover_update(self):
|
||||
raw = {"cover": {"type": "manual", "frame_time": 5.0}}
|
||||
result = normalize_plan_config(raw)
|
||||
assert result["cover"]["type"] == "manual"
|
||||
assert result["cover"]["frame_time"] == 5.0
|
||||
# 其他字段保留默认
|
||||
assert result["cover"]["image_url"] == ""
|
||||
|
||||
def test_partial_title_update(self):
|
||||
raw = {"title": {"enabled": False, "text": "自定义标题"}}
|
||||
result = normalize_plan_config(raw)
|
||||
assert result["title"]["enabled"] is False
|
||||
assert result["title"]["text"] == "自定义标题"
|
||||
assert result["title"]["size"] == 48 # 默认值保留
|
||||
|
||||
def test_partial_subtitle_update(self):
|
||||
raw = {"subtitle": {"enabled": False, "size": 32}}
|
||||
result = normalize_plan_config(raw)
|
||||
assert result["subtitle"]["enabled"] is False
|
||||
assert result["subtitle"]["size"] == 32
|
||||
|
||||
def test_partial_bgm_update(self):
|
||||
raw = {"bgm": {"enabled": True, "volume": 0.8}}
|
||||
result = normalize_plan_config(raw)
|
||||
assert result["bgm"]["enabled"] is True
|
||||
assert result["bgm"]["volume"] == 0.8
|
||||
|
||||
def test_editing_mode_update(self):
|
||||
raw = {"editing_mode": "pip"}
|
||||
result = normalize_plan_config(raw)
|
||||
assert result["editing_mode"] == "pip"
|
||||
|
||||
def test_non_standard_fields_preserved(self):
|
||||
"""非标准字段会被保留(透传)"""
|
||||
raw = {"generation_task_id": "task_123", "custom_field": "value"}
|
||||
result = normalize_plan_config(raw)
|
||||
assert result["generation_task_id"] == "task_123"
|
||||
assert result["custom_field"] == "value"
|
||||
|
||||
def test_full_update(self):
|
||||
"""多个字段同时更新"""
|
||||
raw = {
|
||||
"cover": {"type": "upload", "image_url": "http://img.jpg"},
|
||||
"title": {"enabled": False},
|
||||
"bgm": {"enabled": True, "volume": 0.5},
|
||||
"editing_mode": "voice_pip",
|
||||
"extra_key": "extra_val",
|
||||
}
|
||||
result = normalize_plan_config(raw)
|
||||
assert result["cover"]["type"] == "upload"
|
||||
assert result["title"]["enabled"] is False
|
||||
assert result["bgm"]["enabled"] is True
|
||||
assert result["bgm"]["volume"] == 0.5
|
||||
assert result["editing_mode"] == "voice_pip"
|
||||
assert result["extra_key"] == "extra_val"
|
||||
|
||||
def test_non_dict_section_ignored(self):
|
||||
"""section不是dict时忽略"""
|
||||
raw = {"cover": "not_a_dict"}
|
||||
result = normalize_plan_config(raw)
|
||||
# cover 应保持默认值
|
||||
assert result["cover"]["type"] == "ai_frame"
|
||||
# 但 cover 字段本身作为非标准字段保留
|
||||
# 不对,看实现:只有当 isinstance(raw[section_key], dict) 时才 update
|
||||
# 非dict的section会作为非标准字段保留吗?看实现:
|
||||
# section_key in raw and isinstance -> update
|
||||
# 然后遍历所有key,不在标准列表中的才会保留
|
||||
# cover 在标准列表中,所以不会被保留为非标准字段
|
||||
# 所以结果应该是默认的cover配置
|
||||
assert isinstance(result["cover"], dict)
|
||||
assert result["cover"]["type"] == "ai_frame"
|
||||
|
||||
def test_editing_mode_non_string_ignored(self):
|
||||
"""editing_mode不是字符串时忽略"""
|
||||
raw = {"editing_mode": 123}
|
||||
result = normalize_plan_config(raw)
|
||||
# 作为非标准字段保留?不,看实现:
|
||||
# 如果 "editing_mode" in raw and isinstance(str) 才更新
|
||||
# 然后遍历所有key,不在标准列表中的保留
|
||||
# editing_mode 在标准列表中,所以不会保留为非标准
|
||||
# 所以 editing_mode 保持默认
|
||||
assert result["editing_mode"] == "one_take"
|
||||
|
||||
|
||||
class TestNormalizeTemplateConfig:
|
||||
"""normalize_template_config 函数测试"""
|
||||
|
||||
def test_none_returns_default(self):
|
||||
result = normalize_template_config(None)
|
||||
assert result == DEFAULT_EDIT_TEMPLATE_CONFIG
|
||||
# 确保是深拷贝
|
||||
result["transition_enabled"] = False
|
||||
assert DEFAULT_EDIT_TEMPLATE_CONFIG["transition_enabled"] is True
|
||||
|
||||
def test_empty_dict_returns_default(self):
|
||||
result = normalize_template_config({})
|
||||
assert result == DEFAULT_EDIT_TEMPLATE_CONFIG
|
||||
|
||||
def test_transition_enabled_update(self):
|
||||
raw = {"transition_enabled": False}
|
||||
result = normalize_template_config(raw)
|
||||
assert result["transition_enabled"] is False
|
||||
|
||||
def test_transition_enabled_non_bool_ignored(self):
|
||||
raw = {"transition_enabled": "yes"}
|
||||
result = normalize_template_config(raw)
|
||||
# transition_enabled 在标准列表中,非bool则不更新
|
||||
# 也不会作为非标准字段保留
|
||||
assert result["transition_enabled"] is True
|
||||
|
||||
def test_partial_sections_update(self):
|
||||
raw = {
|
||||
"cover": {"type": "ai_regenerate"},
|
||||
"filter": {"enabled": True, "intensity": 75},
|
||||
"transition_enabled": False,
|
||||
"editing_mode": "pip",
|
||||
}
|
||||
result = normalize_template_config(raw)
|
||||
assert result["cover"]["type"] == "ai_regenerate"
|
||||
assert result["filter"]["enabled"] is True
|
||||
assert result["filter"]["intensity"] == 75
|
||||
assert result["transition_enabled"] is False
|
||||
assert result["editing_mode"] == "pip"
|
||||
|
||||
def test_non_standard_fields_preserved(self):
|
||||
raw = {"template_version": 3, "author": "test"}
|
||||
result = normalize_template_config(raw)
|
||||
assert result["template_version"] == 3
|
||||
assert result["author"] == "test"
|
||||
|
||||
def test_template_has_extra_field_over_plan(self):
|
||||
"""模板normalize结果比计划多transition_enabled"""
|
||||
plan_result = normalize_plan_config({"bgm": {"enabled": True}})
|
||||
template_result = normalize_template_config({"bgm": {"enabled": True}})
|
||||
assert "transition_enabled" in template_result
|
||||
assert "transition_enabled" not in plan_result
|
||||
Executable
+732
@@ -0,0 +1,732 @@
|
||||
"""entities 领域模块单元测试 - P3-1 第七波
|
||||
|
||||
覆盖 User、Project、AssetLibrary、Asset、IngestJob 五个数据类
|
||||
+ AssetLibraryKind/IngestJobStatus/AssetStatus/ClassificationStatus 四个枚举
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestAssetLibraryKind:
|
||||
"""AssetLibraryKind 枚举测试"""
|
||||
|
||||
def test_all_kinds_exist(self):
|
||||
from packages.domain.entities import AssetLibraryKind
|
||||
|
||||
assert AssetLibraryKind.VIDEO == "video"
|
||||
assert AssetLibraryKind.VOICE == "voice"
|
||||
assert AssetLibraryKind.IMAGE == "image"
|
||||
|
||||
def test_is_string_type(self):
|
||||
from packages.domain.entities import AssetLibraryKind
|
||||
|
||||
assert isinstance(AssetLibraryKind.VIDEO, str)
|
||||
assert AssetLibraryKind.VIDEO + "" == "video"
|
||||
|
||||
def test_total_count(self):
|
||||
from packages.domain.entities import AssetLibraryKind
|
||||
|
||||
assert len(AssetLibraryKind) == 3
|
||||
|
||||
|
||||
class TestIngestJobStatus:
|
||||
"""IngestJobStatus 枚举测试"""
|
||||
|
||||
def test_all_statuses_exist(self):
|
||||
from packages.domain.entities import IngestJobStatus
|
||||
|
||||
assert IngestJobStatus.PENDING == "pending"
|
||||
assert IngestJobStatus.PROCESSING == "processing"
|
||||
assert IngestJobStatus.COMPLETED == "completed"
|
||||
assert IngestJobStatus.FAILED == "failed"
|
||||
|
||||
def test_is_string_type(self):
|
||||
from packages.domain.entities import IngestJobStatus
|
||||
|
||||
assert isinstance(IngestJobStatus.PENDING, str)
|
||||
|
||||
def test_total_count(self):
|
||||
from packages.domain.entities import IngestJobStatus
|
||||
|
||||
assert len(IngestJobStatus) == 4
|
||||
|
||||
|
||||
class TestAssetStatus:
|
||||
"""AssetStatus 枚举 + _missing_ 兼容逻辑测试"""
|
||||
|
||||
def test_standard_values(self):
|
||||
from packages.domain.entities import AssetStatus
|
||||
|
||||
assert AssetStatus.UPLOADING == "uploading"
|
||||
assert AssetStatus.READY == "ready"
|
||||
assert AssetStatus.PROCESSING == "processing"
|
||||
assert AssetStatus.ERROR == "error"
|
||||
assert AssetStatus.DELETED == "deleted"
|
||||
|
||||
def test_missing_uploaded_maps_to_ready(self):
|
||||
"""历史值 uploaded → READY"""
|
||||
from packages.domain.entities import AssetStatus
|
||||
|
||||
assert AssetStatus("uploaded") == AssetStatus.READY
|
||||
|
||||
def test_missing_success_maps_to_ready(self):
|
||||
from packages.domain.entities import AssetStatus
|
||||
|
||||
assert AssetStatus("success") == AssetStatus.READY
|
||||
|
||||
def test_missing_done_maps_to_ready(self):
|
||||
from packages.domain.entities import AssetStatus
|
||||
|
||||
assert AssetStatus("done") == AssetStatus.READY
|
||||
|
||||
def test_missing_upload_variants_map_to_uploading(self):
|
||||
from packages.domain.entities import AssetStatus
|
||||
|
||||
assert AssetStatus("upload") == AssetStatus.UPLOADING
|
||||
assert AssetStatus("uploading_start") == AssetStatus.UPLOADING
|
||||
assert AssetStatus("upload_start") == AssetStatus.UPLOADING
|
||||
|
||||
def test_missing_failed_variants_map_to_error(self):
|
||||
from packages.domain.entities import AssetStatus
|
||||
|
||||
assert AssetStatus("failed") == AssetStatus.ERROR
|
||||
assert AssetStatus("fail") == AssetStatus.ERROR
|
||||
assert AssetStatus("err") == AssetStatus.ERROR
|
||||
|
||||
def test_missing_process_variants_map_to_processing(self):
|
||||
from packages.domain.entities import AssetStatus
|
||||
|
||||
assert AssetStatus("process") == AssetStatus.PROCESSING
|
||||
assert AssetStatus("running") == AssetStatus.PROCESSING
|
||||
assert AssetStatus("run") == AssetStatus.PROCESSING
|
||||
|
||||
def test_missing_unknown_falls_back_to_ready(self):
|
||||
"""完全未知的值兜底为 READY,不阻塞业务"""
|
||||
from packages.domain.entities import AssetStatus
|
||||
|
||||
assert AssetStatus("weird_status") == AssetStatus.READY
|
||||
assert AssetStatus("deprecated_state") == AssetStatus.READY
|
||||
|
||||
def test_missing_case_insensitive(self):
|
||||
from packages.domain.entities import AssetStatus
|
||||
|
||||
assert AssetStatus("UPLOADED") == AssetStatus.READY
|
||||
assert AssetStatus("Failed") == AssetStatus.ERROR
|
||||
|
||||
def test_missing_with_spaces(self):
|
||||
from packages.domain.entities import AssetStatus
|
||||
|
||||
assert AssetStatus(" uploaded ") == AssetStatus.READY
|
||||
|
||||
def test_missing_non_string_returns_ready(self):
|
||||
"""非字符串输入也兜底,不抛异常"""
|
||||
from packages.domain.entities import AssetStatus
|
||||
|
||||
assert AssetStatus(None) == AssetStatus.READY # type: ignore[arg-type]
|
||||
assert AssetStatus(123) == AssetStatus.READY # type: ignore[arg-type]
|
||||
|
||||
|
||||
class TestClassificationStatus:
|
||||
"""ClassificationStatus 枚举 + _missing_ 兼容逻辑测试"""
|
||||
|
||||
def test_standard_values(self):
|
||||
from packages.domain.entities import ClassificationStatus
|
||||
|
||||
assert ClassificationStatus.PENDING == "pending"
|
||||
assert ClassificationStatus.PROCESSING == "processing"
|
||||
assert ClassificationStatus.COMPLETED == "completed"
|
||||
assert ClassificationStatus.FAILED == "failed"
|
||||
|
||||
def test_missing_done_maps_to_completed(self):
|
||||
"""历史值 done → COMPLETED"""
|
||||
from packages.domain.entities import ClassificationStatus
|
||||
|
||||
assert ClassificationStatus("done") == ClassificationStatus.COMPLETED
|
||||
|
||||
def test_missing_success_maps_to_completed(self):
|
||||
from packages.domain.entities import ClassificationStatus
|
||||
|
||||
assert ClassificationStatus("success") == ClassificationStatus.COMPLETED
|
||||
|
||||
def test_missing_finished_maps_to_completed(self):
|
||||
from packages.domain.entities import ClassificationStatus
|
||||
|
||||
assert ClassificationStatus("finished") == ClassificationStatus.COMPLETED
|
||||
|
||||
def test_missing_fail_variants_map_to_failed(self):
|
||||
from packages.domain.entities import ClassificationStatus
|
||||
|
||||
assert ClassificationStatus("fail") == ClassificationStatus.FAILED
|
||||
assert ClassificationStatus("error") == ClassificationStatus.FAILED
|
||||
assert ClassificationStatus("err") == ClassificationStatus.FAILED
|
||||
|
||||
def test_missing_process_variants_map_to_processing(self):
|
||||
from packages.domain.entities import ClassificationStatus
|
||||
|
||||
assert ClassificationStatus("process") == ClassificationStatus.PROCESSING
|
||||
assert ClassificationStatus("running") == ClassificationStatus.PROCESSING
|
||||
|
||||
def test_missing_unknown_falls_back_to_pending(self):
|
||||
"""完全未知的值兜底为 PENDING"""
|
||||
from packages.domain.entities import ClassificationStatus
|
||||
|
||||
assert ClassificationStatus("unknown_state") == ClassificationStatus.PENDING
|
||||
|
||||
def test_missing_case_insensitive_and_whitespace(self):
|
||||
from packages.domain.entities import ClassificationStatus
|
||||
|
||||
assert ClassificationStatus("DONE") == ClassificationStatus.COMPLETED
|
||||
assert ClassificationStatus(" Success ") == ClassificationStatus.COMPLETED
|
||||
|
||||
def test_missing_non_string_returns_pending(self):
|
||||
from packages.domain.entities import ClassificationStatus
|
||||
|
||||
assert ClassificationStatus(None) == ClassificationStatus.PENDING # type: ignore[arg-type]
|
||||
|
||||
|
||||
class TestUser:
|
||||
"""User 数据类测试"""
|
||||
|
||||
def test_create_minimal_user(self):
|
||||
from packages.domain.entities import User
|
||||
|
||||
user = User(id="user_123", email="test@example.com", display_name="测试用户")
|
||||
assert user.id == "user_123"
|
||||
assert user.email == "test@example.com"
|
||||
assert user.display_name == "测试用户"
|
||||
assert user.username == ""
|
||||
|
||||
def test_default_values(self):
|
||||
from packages.domain.entities import User
|
||||
|
||||
user = User(id="u1", email="a@b.com", display_name="Test")
|
||||
assert user.password_hash == ""
|
||||
assert user.email_verified is False
|
||||
assert user.email_verification_token is None
|
||||
assert user.password_reset_token is None
|
||||
assert user.subscription_plan == "free"
|
||||
assert user.subscription_status == "active"
|
||||
assert user.max_projects == 3
|
||||
assert user.max_storage_gb == 10
|
||||
assert user.used_storage_gb == 0.0
|
||||
assert user.is_admin is False
|
||||
assert user.wechat_openid is None
|
||||
assert user.phone is None
|
||||
assert user.phone_verified is False
|
||||
|
||||
def test_with_wechat_binding(self):
|
||||
from packages.domain.entities import User
|
||||
|
||||
user = User(
|
||||
id="u1",
|
||||
email="wx_user@wechat.local",
|
||||
display_name="微信用户",
|
||||
wechat_openid="o123456789",
|
||||
wechat_unionid="u987654321",
|
||||
)
|
||||
assert user.wechat_openid == "o123456789"
|
||||
assert user.wechat_unionid == "u987654321"
|
||||
|
||||
def test_with_phone_binding(self):
|
||||
from packages.domain.entities import User
|
||||
|
||||
user = User(
|
||||
id="u1",
|
||||
email="a@b.com",
|
||||
display_name="Test",
|
||||
phone="13800138000",
|
||||
phone_verified=True,
|
||||
)
|
||||
assert user.phone == "13800138000"
|
||||
assert user.phone_verified is True
|
||||
|
||||
def test_admin_user(self):
|
||||
from packages.domain.entities import User
|
||||
|
||||
user = User(id="admin", email="admin@example.com", display_name="Admin", is_admin=True)
|
||||
assert user.is_admin is True
|
||||
|
||||
def test_pro_subscription(self):
|
||||
from packages.domain.entities import User
|
||||
|
||||
user = User(
|
||||
id="u1",
|
||||
email="a@b.com",
|
||||
display_name="Pro",
|
||||
subscription_plan="pro",
|
||||
max_projects=100,
|
||||
max_storage_gb=100,
|
||||
)
|
||||
assert user.subscription_plan == "pro"
|
||||
assert user.max_projects == 100
|
||||
assert user.max_storage_gb == 100
|
||||
|
||||
def test_has_created_at(self):
|
||||
from packages.domain.entities import User
|
||||
|
||||
before = datetime.now(timezone.utc)
|
||||
user = User(id="u1", email="a@b.com", display_name="Test")
|
||||
after = datetime.now(timezone.utc)
|
||||
assert before <= user.created_at <= after
|
||||
|
||||
|
||||
class TestProject:
|
||||
"""Project 数据类 + 业务方法测试"""
|
||||
|
||||
def test_create_project(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
project = Project.create(owner_user_id="user_1", name="我的项目")
|
||||
assert project.id # 自动生成 ID
|
||||
assert len(project.id) == 32 # uuid4 hex
|
||||
assert project.owner_user_id == "user_1"
|
||||
assert project.name == "我的项目"
|
||||
assert project.description == ""
|
||||
assert project.shared_users == []
|
||||
|
||||
def test_create_with_description(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
project = Project.create(owner_user_id="u1", name="测试项目", description="这是描述")
|
||||
assert project.name == "测试项目"
|
||||
assert project.description == "这是描述"
|
||||
|
||||
def test_create_strips_whitespace(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
project = Project.create(owner_user_id="u1", name=" 我的项目 ", description=" 描述 ")
|
||||
assert project.name == "我的项目"
|
||||
assert project.description == "描述"
|
||||
|
||||
def test_create_empty_name_raises(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
with pytest.raises(ValueError, match="项目名称不能为空"):
|
||||
Project.create(owner_user_id="u1", name="")
|
||||
|
||||
def test_create_whitespace_name_raises(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
with pytest.raises(ValueError, match="项目名称不能为空"):
|
||||
Project.create(owner_user_id="u1", name=" ")
|
||||
|
||||
def test_is_owner_true(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
project = Project.create(owner_user_id="user_1", name="P1")
|
||||
assert project.is_owner("user_1") is True
|
||||
|
||||
def test_is_owner_false(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
project = Project.create(owner_user_id="user_1", name="P1")
|
||||
assert project.is_owner("user_2") is False
|
||||
|
||||
def test_is_shared_with_true(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
project = Project(id="p1", owner_user_id="owner", name="P1", shared_users=["u1", "u2"])
|
||||
assert project.is_shared_with("u1") is True
|
||||
assert project.is_shared_with("u2") is True
|
||||
|
||||
def test_is_shared_with_false(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
project = Project(id="p1", owner_user_id="owner", name="P1", shared_users=["u1"])
|
||||
assert project.is_shared_with("u3") is False
|
||||
|
||||
def test_can_access_owner(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
project = Project.create(owner_user_id="owner", name="P1")
|
||||
assert project.can_access("owner") is True
|
||||
|
||||
def test_can_access_shared_user(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
project = Project(id="p1", owner_user_id="owner", name="P1", shared_users=["shared_user"])
|
||||
assert project.can_access("shared_user") is True
|
||||
|
||||
def test_cannot_access_other(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
project = Project.create(owner_user_id="owner", name="P1")
|
||||
assert project.can_access("stranger") is False
|
||||
|
||||
def test_has_created_at(self):
|
||||
from packages.domain.entities import Project
|
||||
|
||||
project = Project.create(owner_user_id="u1", name="P1")
|
||||
assert isinstance(project.created_at, datetime)
|
||||
|
||||
|
||||
class TestAssetLibrary:
|
||||
"""AssetLibrary 数据类测试"""
|
||||
|
||||
def test_create_library(self):
|
||||
from packages.domain.entities import AssetLibrary, AssetLibraryKind
|
||||
|
||||
lib = AssetLibrary.create(project_id="proj_1", name="视频素材库", kind=AssetLibraryKind.VIDEO)
|
||||
assert lib.id
|
||||
assert len(lib.id) == 32
|
||||
assert lib.project_id == "proj_1"
|
||||
assert lib.name == "视频素材库"
|
||||
assert lib.kind == AssetLibraryKind.VIDEO
|
||||
assert lib.asset_count == 0
|
||||
assert lib.total_size == 0
|
||||
|
||||
def test_create_strips_name(self):
|
||||
from packages.domain.entities import AssetLibrary, AssetLibraryKind
|
||||
|
||||
lib = AssetLibrary.create(project_id="p1", name=" 语音库 ", kind=AssetLibraryKind.VOICE)
|
||||
assert lib.name == "语音库"
|
||||
|
||||
def test_create_empty_name_raises(self):
|
||||
from packages.domain.entities import AssetLibrary, AssetLibraryKind
|
||||
|
||||
with pytest.raises(ValueError, match="素材库名称不能为空"):
|
||||
AssetLibrary.create(project_id="p1", name="", kind=AssetLibraryKind.IMAGE)
|
||||
|
||||
def test_image_library_kind(self):
|
||||
from packages.domain.entities import AssetLibrary, AssetLibraryKind
|
||||
|
||||
lib = AssetLibrary.create(project_id="p1", name="图片库", kind=AssetLibraryKind.IMAGE)
|
||||
assert lib.kind == AssetLibraryKind.IMAGE
|
||||
assert lib.kind == "image"
|
||||
|
||||
def test_default_timestamps(self):
|
||||
from packages.domain.entities import AssetLibrary, AssetLibraryKind
|
||||
|
||||
lib = AssetLibrary.create(project_id="p1", name="L1", kind=AssetLibraryKind.VIDEO)
|
||||
assert isinstance(lib.created_at, datetime)
|
||||
assert isinstance(lib.updated_at, datetime)
|
||||
|
||||
|
||||
class TestAsset:
|
||||
"""Asset 数据类 + 业务方法测试"""
|
||||
|
||||
def test_create_minimal_asset(self):
|
||||
from packages.domain.entities import Asset, AssetStatus, ClassificationStatus
|
||||
|
||||
asset = Asset.create(
|
||||
project_id="p1",
|
||||
library_id="lib1",
|
||||
name="test.mp4",
|
||||
storage_key="videos/test.mp4",
|
||||
mime_type="video/mp4",
|
||||
)
|
||||
assert asset.id
|
||||
assert len(asset.id) == 32
|
||||
assert asset.project_id == "p1"
|
||||
assert asset.library_id == "lib1"
|
||||
assert asset.name == "test.mp4"
|
||||
assert asset.storage_key == "videos/test.mp4"
|
||||
assert asset.mime_type == "video/mp4"
|
||||
assert asset.status == AssetStatus.UPLOADING
|
||||
assert asset.classification_status == ClassificationStatus.PENDING
|
||||
assert asset.file_size == 0
|
||||
assert asset.metadata == {}
|
||||
assert asset.tag_ids == []
|
||||
|
||||
def test_create_with_full_params(self):
|
||||
from packages.domain.entities import Asset, AssetStatus, ClassificationStatus
|
||||
|
||||
asset = Asset.create(
|
||||
project_id="p1",
|
||||
library_id="lib1",
|
||||
name=" clip.mp4 ",
|
||||
storage_key=" path/clip.mp4 ",
|
||||
mime_type=" video/mp4 ",
|
||||
metadata={"quality": "high"},
|
||||
file_size=1024000,
|
||||
thumbnail_url="https://example.com/thumb.jpg",
|
||||
duration=30.5,
|
||||
width=1920,
|
||||
height=1080,
|
||||
fps=30.0,
|
||||
codec="h264",
|
||||
status=AssetStatus.READY,
|
||||
classification_status=ClassificationStatus.COMPLETED,
|
||||
quality_score=0.95,
|
||||
uploaded_by_user_id=" user_1 ",
|
||||
file_hash=" abc123 ",
|
||||
)
|
||||
assert asset.name == "clip.mp4" # strip
|
||||
assert asset.storage_key == "path/clip.mp4"
|
||||
assert asset.mime_type == "video/mp4"
|
||||
assert asset.file_size == 1024000
|
||||
assert asset.thumbnail_url == "https://example.com/thumb.jpg"
|
||||
assert asset.duration == 30.5
|
||||
assert asset.width == 1920
|
||||
assert asset.height == 1080
|
||||
assert asset.fps == 30.0
|
||||
assert asset.codec == "h264"
|
||||
assert asset.status == AssetStatus.READY
|
||||
assert asset.classification_status == ClassificationStatus.COMPLETED
|
||||
assert asset.quality_score == 0.95
|
||||
assert asset.uploaded_by_user_id == "user_1"
|
||||
assert asset.file_hash == "abc123"
|
||||
assert asset.metadata == {"quality": "high"}
|
||||
|
||||
def test_create_empty_name_raises(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
with pytest.raises(ValueError, match="素材名称不能为空"):
|
||||
Asset.create(project_id="p1", library_id="l1", name="", storage_key="k", mime_type="video/mp4")
|
||||
|
||||
def test_create_empty_storage_key_raises(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
with pytest.raises(ValueError, match="storage_key 不能为空"):
|
||||
Asset.create(project_id="p1", library_id="l1", name="a.mp4", storage_key=" ", mime_type="video/mp4")
|
||||
|
||||
def test_create_empty_mime_type_raises(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
with pytest.raises(ValueError, match="mime_type 不能为空"):
|
||||
Asset.create(project_id="p1", library_id="l1", name="a.mp4", storage_key="k", mime_type="")
|
||||
|
||||
def test_file_type_video(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="v.mp4", storage_key="k", mime_type="video/mp4")
|
||||
assert asset.file_type == "video"
|
||||
|
||||
def test_file_type_audio(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="a.mp3", storage_key="k", mime_type="audio/mpeg")
|
||||
assert asset.file_type == "audio"
|
||||
|
||||
def test_file_type_image(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="i.jpg", storage_key="k", mime_type="image/jpeg")
|
||||
assert asset.file_type == "image"
|
||||
|
||||
def test_file_type_no_slash(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="f.bin", storage_key="k", mime_type="octet-stream")
|
||||
assert asset.file_type == "octet-stream"
|
||||
|
||||
def test_add_tag(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="v.mp4", storage_key="k", mime_type="video/mp4")
|
||||
asset.add_tag("tag_1")
|
||||
assert "tag_1" in asset.tag_ids
|
||||
assert len(asset.tag_ids) == 1
|
||||
|
||||
def test_add_tag_strips_whitespace(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="v.mp4", storage_key="k", mime_type="video/mp4")
|
||||
asset.add_tag(" tag_1 ")
|
||||
assert asset.tag_ids == ["tag_1"]
|
||||
|
||||
def test_add_tag_deduplicates(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="v.mp4", storage_key="k", mime_type="video/mp4")
|
||||
asset.add_tag("tag_1")
|
||||
asset.add_tag("tag_1")
|
||||
assert asset.tag_ids.count("tag_1") == 1
|
||||
|
||||
def test_add_empty_tag_raises(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="v.mp4", storage_key="k", mime_type="video/mp4")
|
||||
with pytest.raises(ValueError, match="标签 ID 不能为空"):
|
||||
asset.add_tag(" ")
|
||||
|
||||
def test_add_tag_updates_updated_at(self):
|
||||
from time import sleep
|
||||
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="v.mp4", storage_key="k", mime_type="video/mp4")
|
||||
original = asset.updated_at
|
||||
sleep(0.001)
|
||||
asset.add_tag("tag_1")
|
||||
assert asset.updated_at > original
|
||||
|
||||
def test_remove_tag_existing(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="v.mp4", storage_key="k", mime_type="video/mp4")
|
||||
asset.add_tag("tag_1")
|
||||
asset.add_tag("tag_2")
|
||||
asset.remove_tag("tag_1")
|
||||
assert "tag_1" not in asset.tag_ids
|
||||
assert "tag_2" in asset.tag_ids
|
||||
assert len(asset.tag_ids) == 1
|
||||
|
||||
def test_remove_tag_nonexistent_is_idempotent(self):
|
||||
"""删除不存在的标签不报错"""
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="v.mp4", storage_key="k", mime_type="video/mp4")
|
||||
asset.remove_tag("nonexistent") # 不抛异常
|
||||
assert asset.tag_ids == []
|
||||
|
||||
def test_remove_tag_strips_whitespace(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="v.mp4", storage_key="k", mime_type="video/mp4")
|
||||
asset.add_tag("tag_1")
|
||||
asset.remove_tag(" tag_1 ")
|
||||
assert "tag_1" not in asset.tag_ids
|
||||
|
||||
def test_remove_tag_updates_updated_at(self):
|
||||
from time import sleep
|
||||
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(project_id="p1", library_id="l1", name="v.mp4", storage_key="k", mime_type="video/mp4")
|
||||
asset.add_tag("tag_1")
|
||||
original = asset.updated_at
|
||||
sleep(0.001)
|
||||
asset.remove_tag("tag_1")
|
||||
assert asset.updated_at > original
|
||||
|
||||
def test_thumbnail_none_when_falsy(self):
|
||||
"""空字符串 thumbnail 存为 None"""
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(
|
||||
project_id="p1",
|
||||
library_id="l1",
|
||||
name="v.mp4",
|
||||
storage_key="k",
|
||||
mime_type="video/mp4",
|
||||
thumbnail_url="",
|
||||
)
|
||||
assert asset.thumbnail_url is None
|
||||
|
||||
def test_metadata_default_empty_dict(self):
|
||||
from packages.domain.entities import Asset
|
||||
|
||||
asset = Asset.create(
|
||||
project_id="p1",
|
||||
library_id="l1",
|
||||
name="v.mp4",
|
||||
storage_key="k",
|
||||
mime_type="video/mp4",
|
||||
metadata=None,
|
||||
)
|
||||
assert asset.metadata == {}
|
||||
# 不共享同一个默认 dict
|
||||
asset2 = Asset.create(
|
||||
project_id="p1",
|
||||
library_id="l1",
|
||||
name="v2.mp4",
|
||||
storage_key="k2",
|
||||
mime_type="video/mp4",
|
||||
)
|
||||
asset.metadata["test"] = "value"
|
||||
assert "test" not in asset2.metadata
|
||||
|
||||
|
||||
class TestIngestJob:
|
||||
"""IngestJob 数据类测试"""
|
||||
|
||||
def test_create_minimal(self):
|
||||
from packages.domain.entities import IngestJob, IngestJobStatus
|
||||
|
||||
job = IngestJob.create(
|
||||
project_id="p1",
|
||||
library_id="lib1",
|
||||
storage_key="videos/test.mp4",
|
||||
)
|
||||
assert job.id
|
||||
assert len(job.id) == 32
|
||||
assert job.project_id == "p1"
|
||||
assert job.library_id == "lib1"
|
||||
assert job.storage_key == "videos/test.mp4"
|
||||
assert job.status == IngestJobStatus.PENDING
|
||||
assert job.error_message == ""
|
||||
assert job.result_asset_id == ""
|
||||
assert job.file_hash == ""
|
||||
|
||||
def test_create_with_file_hash(self):
|
||||
from packages.domain.entities import IngestJob
|
||||
|
||||
job = IngestJob.create(
|
||||
project_id="p1",
|
||||
library_id="lib1",
|
||||
storage_key="k",
|
||||
file_hash="abc123def456",
|
||||
)
|
||||
assert job.file_hash == "abc123def456"
|
||||
|
||||
def test_create_strips_fields(self):
|
||||
from packages.domain.entities import IngestJob
|
||||
|
||||
job = IngestJob.create(
|
||||
project_id=" p1 ",
|
||||
library_id=" lib1 ",
|
||||
storage_key=" key ",
|
||||
file_hash=" hash ",
|
||||
)
|
||||
assert job.project_id == "p1"
|
||||
assert job.library_id == "lib1"
|
||||
assert job.storage_key == "key"
|
||||
assert job.file_hash == "hash"
|
||||
|
||||
def test_create_empty_project_id_raises(self):
|
||||
from packages.domain.entities import IngestJob
|
||||
|
||||
with pytest.raises(ValueError, match="project_id 不能为空"):
|
||||
IngestJob.create(project_id="", library_id="l1", storage_key="k")
|
||||
|
||||
def test_create_empty_library_id_raises(self):
|
||||
from packages.domain.entities import IngestJob
|
||||
|
||||
with pytest.raises(ValueError, match="library_id 不能为空"):
|
||||
IngestJob.create(project_id="p1", library_id=" ", storage_key="k")
|
||||
|
||||
def test_create_empty_storage_key_raises(self):
|
||||
from packages.domain.entities import IngestJob
|
||||
|
||||
with pytest.raises(ValueError, match="storage_key 不能为空"):
|
||||
IngestJob.create(project_id="p1", library_id="l1", storage_key="")
|
||||
|
||||
def test_failed_status(self):
|
||||
from packages.domain.entities import IngestJob, IngestJobStatus
|
||||
|
||||
job = IngestJob(
|
||||
id="j1",
|
||||
project_id="p1",
|
||||
library_id="l1",
|
||||
storage_key="k",
|
||||
status=IngestJobStatus.FAILED,
|
||||
error_message="转码失败",
|
||||
)
|
||||
assert job.status == IngestJobStatus.FAILED
|
||||
assert job.error_message == "转码失败"
|
||||
|
||||
def test_completed_with_result(self):
|
||||
from packages.domain.entities import IngestJob, IngestJobStatus
|
||||
|
||||
job = IngestJob(
|
||||
id="j1",
|
||||
project_id="p1",
|
||||
library_id="l1",
|
||||
storage_key="k",
|
||||
status=IngestJobStatus.COMPLETED,
|
||||
result_asset_id="asset_123",
|
||||
)
|
||||
assert job.status == IngestJobStatus.COMPLETED
|
||||
assert job.result_asset_id == "asset_123"
|
||||
|
||||
def test_has_timestamps(self):
|
||||
from packages.domain.entities import IngestJob
|
||||
|
||||
job = IngestJob.create(project_id="p1", library_id="l1", storage_key="k")
|
||||
assert isinstance(job.created_at, datetime)
|
||||
assert isinstance(job.updated_at, datetime)
|
||||
Executable
+425
@@ -0,0 +1,425 @@
|
||||
"""
|
||||
FeatureFlagStore 单元测试
|
||||
|
||||
覆盖:
|
||||
- FeatureFlagConfig: to_dict / from_dict 序列化
|
||||
- FeatureFlagConfig.is_active: 全局开关/白名单/百分比哈希
|
||||
- InMemoryFeatureFlagStore: CRUD / is_active
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from packages.adapters.redis.feature_flag_store import (
|
||||
FEATURE_FLAG_REDIS_PREFIX,
|
||||
FeatureFlagConfig,
|
||||
InMemoryFeatureFlagStore,
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# 常量
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestConstants:
|
||||
"""常量验证"""
|
||||
|
||||
def test_redis_prefix(self):
|
||||
assert FEATURE_FLAG_REDIS_PREFIX == "feature_flag:"
|
||||
|
||||
|
||||
# ============================================================
|
||||
# FeatureFlagConfig - 默认值 & 基础
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestFeatureFlagConfigDefaults:
|
||||
"""FeatureFlagConfig 默认值"""
|
||||
|
||||
def test_required_name(self):
|
||||
config = FeatureFlagConfig(name="test_flag")
|
||||
assert config.name == "test_flag"
|
||||
|
||||
def test_default_disabled(self):
|
||||
config = FeatureFlagConfig(name="test_flag")
|
||||
assert config.enabled is False
|
||||
|
||||
def test_default_percentage_zero(self):
|
||||
config = FeatureFlagConfig(name="test_flag")
|
||||
assert config.percentage == 0
|
||||
|
||||
def test_default_whitelist_empty(self):
|
||||
config = FeatureFlagConfig(name="test_flag")
|
||||
assert config.whitelist == set()
|
||||
|
||||
def test_full_config(self):
|
||||
config = FeatureFlagConfig(
|
||||
name="full_flag",
|
||||
enabled=True,
|
||||
percentage=50,
|
||||
whitelist={"user1", "user2"},
|
||||
)
|
||||
assert config.name == "full_flag"
|
||||
assert config.enabled is True
|
||||
assert config.percentage == 50
|
||||
assert config.whitelist == {"user1", "user2"}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# FeatureFlagConfig - 序列化
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestFeatureFlagConfigSerialization:
|
||||
"""to_dict / from_dict 序列化"""
|
||||
|
||||
def test_to_dict_defaults(self):
|
||||
config = FeatureFlagConfig(name="test")
|
||||
d = config.to_dict()
|
||||
assert d["name"] == "test"
|
||||
assert d["enabled"] is False
|
||||
assert d["percentage"] == 0
|
||||
assert d["whitelist"] == []
|
||||
|
||||
def test_to_dict_with_values(self):
|
||||
config = FeatureFlagConfig(
|
||||
name="test",
|
||||
enabled=True,
|
||||
percentage=75,
|
||||
whitelist={"a", "b", "c"},
|
||||
)
|
||||
d = config.to_dict()
|
||||
assert d["name"] == "test"
|
||||
assert d["enabled"] is True
|
||||
assert d["percentage"] == 75
|
||||
# whitelist 排序后输出
|
||||
assert sorted(d["whitelist"]) == ["a", "b", "c"]
|
||||
|
||||
def test_from_dict_minimal(self):
|
||||
d = {"name": "test"}
|
||||
config = FeatureFlagConfig.from_dict(d)
|
||||
assert config.name == "test"
|
||||
assert config.enabled is False
|
||||
assert config.percentage == 0
|
||||
assert config.whitelist == set()
|
||||
|
||||
def test_from_dict_full(self):
|
||||
d = {
|
||||
"name": "full",
|
||||
"enabled": True,
|
||||
"percentage": 30,
|
||||
"whitelist": ["u1", "u2"],
|
||||
}
|
||||
config = FeatureFlagConfig.from_dict(d)
|
||||
assert config.name == "full"
|
||||
assert config.enabled is True
|
||||
assert config.percentage == 30
|
||||
assert config.whitelist == {"u1", "u2"}
|
||||
|
||||
def test_round_trip(self):
|
||||
original = FeatureFlagConfig(
|
||||
name="round_trip",
|
||||
enabled=True,
|
||||
percentage=42,
|
||||
whitelist={"alice", "bob", "charlie"},
|
||||
)
|
||||
d = original.to_dict()
|
||||
restored = FeatureFlagConfig.from_dict(d)
|
||||
assert restored.name == original.name
|
||||
assert restored.enabled == original.enabled
|
||||
assert restored.percentage == original.percentage
|
||||
assert restored.whitelist == original.whitelist
|
||||
|
||||
def test_from_dict_coerces_types(self):
|
||||
"""from_dict 应该做类型转换"""
|
||||
d = {
|
||||
"name": "coerce",
|
||||
"enabled": 1, # int → bool
|
||||
"percentage": "50", # str → int
|
||||
"whitelist": ("a", "b"), # tuple → set
|
||||
}
|
||||
config = FeatureFlagConfig.from_dict(d)
|
||||
assert config.enabled is True
|
||||
assert config.percentage == 50
|
||||
assert config.whitelist == {"a", "b"}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# FeatureFlagConfig.is_active - 全局开关
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestIsActiveGlobalSwitch:
|
||||
"""is_active - 全局开关基础"""
|
||||
|
||||
def test_disabled_returns_false(self):
|
||||
config = FeatureFlagConfig(name="test", enabled=False)
|
||||
assert config.is_active() is False
|
||||
|
||||
def test_disabled_with_identifier_returns_false(self):
|
||||
config = FeatureFlagConfig(name="test", enabled=False)
|
||||
assert config.is_active(identifier="user1") is False
|
||||
|
||||
def test_enabled_no_percentage_no_whitelist_returns_true(self):
|
||||
config = FeatureFlagConfig(name="test", enabled=True)
|
||||
# percentage=0, whitelist=空,但 enabled=True
|
||||
# 按逻辑:全局开了但百分比0且无白名单 → 其实应该是 False?
|
||||
# 让我看代码...
|
||||
# 代码里 percentage <= 0 时返回 False(没有白名单且百分比为0)
|
||||
assert config.is_active() is False
|
||||
|
||||
def test_enabled_100_percent_returns_true(self):
|
||||
config = FeatureFlagConfig(name="test", enabled=True, percentage=100)
|
||||
assert config.is_active() is True
|
||||
|
||||
|
||||
# ============================================================
|
||||
# FeatureFlagConfig.is_active - 白名单
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestIsActiveWhitelist:
|
||||
"""is_active - 白名单优先级"""
|
||||
|
||||
def test_whitelist_match_returns_true(self):
|
||||
config = FeatureFlagConfig(
|
||||
name="test",
|
||||
enabled=True,
|
||||
whitelist={"user1", "user2"},
|
||||
)
|
||||
assert config.is_active(identifier="user1") is True
|
||||
|
||||
def test_whitelist_no_match_falls_through(self):
|
||||
config = FeatureFlagConfig(
|
||||
name="test",
|
||||
enabled=True,
|
||||
percentage=0,
|
||||
whitelist={"user1"},
|
||||
)
|
||||
# 不在白名单,且百分比为0 → False
|
||||
assert config.is_active(identifier="user3") is False
|
||||
|
||||
def test_whitelist_overrides_percentage_zero(self):
|
||||
"""白名单优先级最高,即使百分比为0也能启用"""
|
||||
config = FeatureFlagConfig(
|
||||
name="test",
|
||||
enabled=True,
|
||||
percentage=0,
|
||||
whitelist={"vip_user"},
|
||||
)
|
||||
assert config.is_active(identifier="vip_user") is True
|
||||
|
||||
def test_whitelist_overrides_partial_percentage(self):
|
||||
"""白名单用户即使在百分比外也能启用"""
|
||||
config = FeatureFlagConfig(
|
||||
name="test",
|
||||
enabled=True,
|
||||
percentage=1, # 只有1%的用户
|
||||
whitelist={"important_user"},
|
||||
)
|
||||
# 白名单用户直接通过
|
||||
assert config.is_active(identifier="important_user") is True
|
||||
|
||||
def test_no_identifier_no_whitelist_check(self):
|
||||
"""不传 identifier 时不做白名单检查"""
|
||||
config = FeatureFlagConfig(
|
||||
name="test",
|
||||
enabled=True,
|
||||
percentage=100,
|
||||
whitelist={"user1"},
|
||||
)
|
||||
# 无 identifier,直接看百分比(100%)
|
||||
assert config.is_active() is True
|
||||
|
||||
|
||||
# ============================================================
|
||||
# FeatureFlagConfig.is_active - 百分比边界值
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestIsActivePercentageBoundaries:
|
||||
"""is_active - 百分比边界值"""
|
||||
|
||||
def test_percentage_0_returns_false(self):
|
||||
config = FeatureFlagConfig(name="test", enabled=True, percentage=0)
|
||||
assert config.is_active(identifier="any_user") is False
|
||||
|
||||
def test_percentage_100_returns_true(self):
|
||||
config = FeatureFlagConfig(name="test", enabled=True, percentage=100)
|
||||
assert config.is_active(identifier="any_user") is True
|
||||
|
||||
def test_percentage_negative_treated_as_0(self):
|
||||
"""percentage < 0 应该按 0 处理"""
|
||||
config = FeatureFlagConfig(name="test", enabled=True, percentage=-5)
|
||||
assert config.is_active(identifier="any_user") is False
|
||||
|
||||
def test_percentage_over_100_treated_as_100(self):
|
||||
"""percentage > 100 应该按 100 处理"""
|
||||
config = FeatureFlagConfig(name="test", enabled=True, percentage=150)
|
||||
assert config.is_active(identifier="any_user") is True
|
||||
|
||||
|
||||
# ============================================================
|
||||
# FeatureFlagConfig.is_active - 哈希一致性
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestIsActiveHashConsistency:
|
||||
"""is_active - 哈希取模一致性验证"""
|
||||
|
||||
def test_same_user_same_result_every_time(self):
|
||||
"""同一用户多次调用结果一致(确定性哈希)"""
|
||||
config = FeatureFlagConfig(name="test", enabled=True, percentage=50)
|
||||
results = {config.is_active(identifier="user_xyz") for _ in range(100)}
|
||||
assert len(results) == 1 # 全部相同
|
||||
|
||||
def test_different_flags_same_user_can_differ(self):
|
||||
"""不同 flag 对同一用户可以有不同结果(因为 flag name 参与哈希)"""
|
||||
config_a = FeatureFlagConfig(name="flag_a", enabled=True, percentage=50)
|
||||
config_b = FeatureFlagConfig(name="flag_b", enabled=True, percentage=50)
|
||||
# 不保证一定不同,但大部分情况下应该不同
|
||||
# 这里只验证哈希输入包含了 flag name(通过机制保证)
|
||||
# 具体是否不同取决于哈希值
|
||||
|
||||
def test_percentage_coverage_roughly_correct(self):
|
||||
"""大量用户中,命中比例大致接近百分比"""
|
||||
config = FeatureFlagConfig(name="coverage_test", enabled=True, percentage=30)
|
||||
users = [f"user_{i}" for i in range(1000)]
|
||||
active_count = sum(1 for u in users if config.is_active(identifier=u))
|
||||
# 30% ± 10% 的容差
|
||||
assert 200 <= active_count <= 400
|
||||
|
||||
def test_50_percent_roughly_half(self):
|
||||
config = FeatureFlagConfig(name="half_test", enabled=True, percentage=50)
|
||||
users = [f"user_{i}" for i in range(1000)]
|
||||
active_count = sum(1 for u in users if config.is_active(identifier=u))
|
||||
# 50% ± 10%
|
||||
assert 400 <= active_count <= 600
|
||||
|
||||
def test_10_percent_roughly_tenth(self):
|
||||
config = FeatureFlagConfig(name="ten_pct", enabled=True, percentage=10)
|
||||
users = [f"user_{i}" for i in range(1000)]
|
||||
active_count = sum(1 for u in users if config.is_active(identifier=u))
|
||||
assert 50 <= active_count <= 150
|
||||
|
||||
def test_empty_identifier_treated_as_no_identifier(self):
|
||||
"""空字符串 identifier 应该如何处理?"""
|
||||
config = FeatureFlagConfig(name="test", enabled=True, percentage=50)
|
||||
# 空字符串是 falsy,走无 identifier 分支(随机)
|
||||
# 但白名单检查也会跳过
|
||||
# 验证不会崩溃
|
||||
result = config.is_active(identifier="")
|
||||
assert isinstance(result, bool)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# InMemoryFeatureFlagStore - CRUD
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestInMemoryFeatureFlagStore:
|
||||
"""InMemoryFeatureFlagStore 内存实现"""
|
||||
|
||||
def test_get_nonexistent_returns_default_disabled(self):
|
||||
store = InMemoryFeatureFlagStore()
|
||||
config = store.get("nonexistent")
|
||||
assert config.name == "nonexistent"
|
||||
assert config.enabled is False
|
||||
assert config.percentage == 0
|
||||
|
||||
def test_set_and_get(self):
|
||||
store = InMemoryFeatureFlagStore()
|
||||
original = FeatureFlagConfig(
|
||||
name="my_flag",
|
||||
enabled=True,
|
||||
percentage=50,
|
||||
whitelist={"admin"},
|
||||
)
|
||||
store.set(original)
|
||||
retrieved = store.get("my_flag")
|
||||
assert retrieved.name == "my_flag"
|
||||
assert retrieved.enabled is True
|
||||
assert retrieved.percentage == 50
|
||||
assert retrieved.whitelist == {"admin"}
|
||||
|
||||
def test_set_overwrites_existing(self):
|
||||
store = InMemoryFeatureFlagStore()
|
||||
store.set(FeatureFlagConfig(name="flag", enabled=True, percentage=30))
|
||||
store.set(FeatureFlagConfig(name="flag", enabled=False, percentage=70))
|
||||
config = store.get("flag")
|
||||
assert config.enabled is False
|
||||
assert config.percentage == 70
|
||||
|
||||
def test_delete_existing_returns_true(self):
|
||||
store = InMemoryFeatureFlagStore()
|
||||
store.set(FeatureFlagConfig(name="delete_me"))
|
||||
result = store.delete("delete_me")
|
||||
assert result is True
|
||||
# 删除后获取返回默认配置
|
||||
assert store.get("delete_me").enabled is False
|
||||
|
||||
def test_delete_nonexistent_returns_false(self):
|
||||
store = InMemoryFeatureFlagStore()
|
||||
result = store.delete("no_such_flag")
|
||||
assert result is False
|
||||
|
||||
def test_list_all_empty(self):
|
||||
store = InMemoryFeatureFlagStore()
|
||||
assert store.list_all() == {}
|
||||
|
||||
def test_list_all_multiple(self):
|
||||
store = InMemoryFeatureFlagStore()
|
||||
store.set(FeatureFlagConfig(name="flag1", enabled=True))
|
||||
store.set(FeatureFlagConfig(name="flag2", percentage=50))
|
||||
store.set(FeatureFlagConfig(name="flag3"))
|
||||
|
||||
all_flags = store.list_all()
|
||||
assert len(all_flags) == 3
|
||||
assert "flag1" in all_flags
|
||||
assert "flag2" in all_flags
|
||||
assert "flag3" in all_flags
|
||||
assert all_flags["flag1"].enabled is True
|
||||
assert all_flags["flag2"].percentage == 50
|
||||
|
||||
def test_list_all_returns_copy(self):
|
||||
"""返回的是副本,修改不影响内部状态"""
|
||||
store = InMemoryFeatureFlagStore()
|
||||
store.set(FeatureFlagConfig(name="flag1"))
|
||||
flags = store.list_all()
|
||||
flags["fake"] = FeatureFlagConfig(name="fake")
|
||||
assert "fake" not in store.list_all()
|
||||
|
||||
|
||||
# ============================================================
|
||||
# InMemoryFeatureFlagStore - is_active
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestInMemoryStoreIsActive:
|
||||
"""store.is_active 便捷方法"""
|
||||
|
||||
def test_is_active_enabled_flag(self):
|
||||
store = InMemoryFeatureFlagStore()
|
||||
store.set(FeatureFlagConfig(name="on", enabled=True, percentage=100))
|
||||
assert store.is_active("on") is True
|
||||
|
||||
def test_is_active_disabled_flag(self):
|
||||
store = InMemoryFeatureFlagStore()
|
||||
store.set(FeatureFlagConfig(name="off", enabled=False))
|
||||
assert store.is_active("off") is False
|
||||
|
||||
def test_is_active_nonexistent_flag(self):
|
||||
store = InMemoryFeatureFlagStore()
|
||||
assert store.is_active("unknown") is False
|
||||
|
||||
def test_is_active_with_identifier_whitelist(self):
|
||||
store = InMemoryFeatureFlagStore()
|
||||
store.set(
|
||||
FeatureFlagConfig(
|
||||
name="beta",
|
||||
enabled=True,
|
||||
percentage=0,
|
||||
whitelist={"tester1"},
|
||||
)
|
||||
)
|
||||
assert store.is_active("beta", identifier="tester1") is True
|
||||
assert store.is_active("beta", identifier="other_user") is False
|
||||
Executable
+569
@@ -0,0 +1,569 @@
|
||||
"""
|
||||
Module Registry 模块注册中心单元测试
|
||||
|
||||
覆盖:
|
||||
- ModuleStatus 枚举
|
||||
- QuotaRule / ModuleCapability / Module 数据类
|
||||
- Module.activate / disable 状态转换
|
||||
- ModuleRegistry 注册/注销/查询/能力发现/依赖检查
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from packages.infrastructure.module_registry import (
|
||||
Module,
|
||||
ModuleCapability,
|
||||
ModuleRegistry,
|
||||
ModuleStatus,
|
||||
QuotaRule,
|
||||
module_registry,
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# ModuleStatus
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestModuleStatus:
|
||||
"""ModuleStatus 枚举"""
|
||||
|
||||
def test_enum_values(self):
|
||||
assert ModuleStatus.REGISTERED.value == "registered"
|
||||
assert ModuleStatus.ACTIVE.value == "active"
|
||||
assert ModuleStatus.DISABLED.value == "disabled"
|
||||
assert ModuleStatus.ERROR.value == "error"
|
||||
|
||||
def test_is_str_enum(self):
|
||||
assert isinstance(ModuleStatus.ACTIVE, str)
|
||||
assert ModuleStatus.ACTIVE == "active"
|
||||
|
||||
def test_has_four_states(self):
|
||||
assert len(ModuleStatus) == 4
|
||||
|
||||
|
||||
# ============================================================
|
||||
# QuotaRule
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestQuotaRule:
|
||||
"""QuotaRule 配额规则"""
|
||||
|
||||
def test_required_fields(self):
|
||||
rule = QuotaRule(dimension="ai_credits", per_operation=1.0)
|
||||
assert rule.dimension == "ai_credits"
|
||||
assert rule.per_operation == 1.0
|
||||
|
||||
def test_default_description_empty(self):
|
||||
rule = QuotaRule(dimension="storage_gb", per_operation=0.5)
|
||||
assert rule.description == ""
|
||||
|
||||
def test_custom_description(self):
|
||||
rule = QuotaRule(
|
||||
dimension="credits",
|
||||
per_operation=2.0,
|
||||
description="每次生成消耗2积分",
|
||||
)
|
||||
assert rule.description == "每次生成消耗2积分"
|
||||
|
||||
def test_float_per_operation(self):
|
||||
rule = QuotaRule(dimension="gb", per_operation=0.25)
|
||||
assert rule.per_operation == 0.25
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ModuleCapability
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestModuleCapability:
|
||||
"""ModuleCapability 能力定义"""
|
||||
|
||||
def test_required_name(self):
|
||||
cap = ModuleCapability(name="generate_voice")
|
||||
assert cap.name == "generate_voice"
|
||||
|
||||
def test_defaults(self):
|
||||
cap = ModuleCapability(name="test_cap")
|
||||
assert cap.description == ""
|
||||
assert cap.quota_rules == []
|
||||
assert cap.metadata == {}
|
||||
|
||||
def test_with_quota_rules(self):
|
||||
rules = [QuotaRule(dimension="credits", per_operation=1.0)]
|
||||
cap = ModuleCapability(
|
||||
name="generate",
|
||||
description="生成功能",
|
||||
quota_rules=rules,
|
||||
)
|
||||
assert cap.description == "生成功能"
|
||||
assert len(cap.quota_rules) == 1
|
||||
assert cap.quota_rules[0].dimension == "credits"
|
||||
|
||||
def test_with_metadata(self):
|
||||
cap = ModuleCapability(
|
||||
name="export",
|
||||
metadata={"format": "mp4", "max_resolution": "1080p"},
|
||||
)
|
||||
assert cap.metadata["format"] == "mp4"
|
||||
assert cap.metadata["max_resolution"] == "1080p"
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Module
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestModuleDefaults:
|
||||
"""Module 数据类默认值"""
|
||||
|
||||
def test_required_name(self):
|
||||
mod = Module(name="ai_voice")
|
||||
assert mod.name == "ai_voice"
|
||||
|
||||
def test_default_version(self):
|
||||
mod = Module(name="test")
|
||||
assert mod.version == "1.0.0"
|
||||
|
||||
def test_default_description(self):
|
||||
mod = Module(name="test")
|
||||
assert mod.description == ""
|
||||
|
||||
def test_default_capabilities_empty(self):
|
||||
mod = Module(name="test")
|
||||
assert mod.capabilities == []
|
||||
|
||||
def test_default_dependencies_empty(self):
|
||||
mod = Module(name="test")
|
||||
assert mod.dependencies == []
|
||||
|
||||
def test_default_status_registered(self):
|
||||
mod = Module(name="test")
|
||||
assert mod.status == ModuleStatus.REGISTERED
|
||||
|
||||
def test_default_config_empty(self):
|
||||
mod = Module(name="test")
|
||||
assert mod.config == {}
|
||||
|
||||
def test_full_module(self):
|
||||
cap = ModuleCapability(name="do_something")
|
||||
mod = Module(
|
||||
name="full_module",
|
||||
version="2.0.0",
|
||||
description="完整模块",
|
||||
capabilities=[cap],
|
||||
dependencies=["dep1", "dep2"],
|
||||
status=ModuleStatus.ACTIVE,
|
||||
config={"key": "value"},
|
||||
)
|
||||
assert mod.version == "2.0.0"
|
||||
assert mod.description == "完整模块"
|
||||
assert len(mod.capabilities) == 1
|
||||
assert mod.dependencies == ["dep1", "dep2"]
|
||||
assert mod.status == ModuleStatus.ACTIVE
|
||||
assert mod.config["key"] == "value"
|
||||
|
||||
|
||||
class TestModuleActivate:
|
||||
"""Module.activate 状态转换"""
|
||||
|
||||
def test_activate_from_registered(self):
|
||||
mod = Module(name="test")
|
||||
mod.activate()
|
||||
assert mod.status == ModuleStatus.ACTIVE
|
||||
|
||||
def test_activate_from_disabled(self):
|
||||
mod = Module(name="test", status=ModuleStatus.DISABLED)
|
||||
mod.activate()
|
||||
assert mod.status == ModuleStatus.ACTIVE
|
||||
|
||||
def test_activate_from_error_stays_error(self):
|
||||
mod = Module(name="test", status=ModuleStatus.ERROR)
|
||||
mod.activate()
|
||||
# error 状态不可激活
|
||||
assert mod.status == ModuleStatus.ERROR
|
||||
|
||||
def test_activate_already_active(self):
|
||||
mod = Module(name="test", status=ModuleStatus.ACTIVE)
|
||||
mod.activate()
|
||||
assert mod.status == ModuleStatus.ACTIVE
|
||||
|
||||
|
||||
class TestModuleDisable:
|
||||
"""Module.disable 状态转换"""
|
||||
|
||||
def test_disable_from_registered(self):
|
||||
mod = Module(name="test")
|
||||
mod.disable()
|
||||
assert mod.status == ModuleStatus.DISABLED
|
||||
|
||||
def test_disable_from_active(self):
|
||||
mod = Module(name="test", status=ModuleStatus.ACTIVE)
|
||||
mod.disable()
|
||||
assert mod.status == ModuleStatus.DISABLED
|
||||
|
||||
def test_disable_from_error(self):
|
||||
mod = Module(name="test", status=ModuleStatus.ERROR)
|
||||
mod.disable()
|
||||
assert mod.status == ModuleStatus.DISABLED
|
||||
|
||||
def test_disable_already_disabled(self):
|
||||
mod = Module(name="test", status=ModuleStatus.DISABLED)
|
||||
mod.disable()
|
||||
assert mod.status == ModuleStatus.DISABLED
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ModuleRegistry - 基础操作
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestModuleRegistryBasic:
|
||||
"""ModuleRegistry 基础操作"""
|
||||
|
||||
def test_empty_registry(self):
|
||||
registry = ModuleRegistry()
|
||||
assert registry.list_modules() == []
|
||||
assert registry.get_active_capabilities() == {}
|
||||
|
||||
def test_register_single_module(self):
|
||||
registry = ModuleRegistry()
|
||||
mod = Module(name="test_mod")
|
||||
registry.register(mod)
|
||||
assert registry.get("test_mod") is mod
|
||||
|
||||
def test_register_duplicate_raises(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(Module(name="test_mod"))
|
||||
with pytest.raises(ValueError, match="already registered"):
|
||||
registry.register(Module(name="test_mod"))
|
||||
|
||||
def test_get_nonexistent_returns_none(self):
|
||||
registry = ModuleRegistry()
|
||||
assert registry.get("no_such_module") is None
|
||||
|
||||
def test_unregister_success(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(Module(name="test_mod"))
|
||||
registry.unregister("test_mod")
|
||||
assert registry.get("test_mod") is None
|
||||
|
||||
def test_unregister_nonexistent_raises(self):
|
||||
registry = ModuleRegistry()
|
||||
with pytest.raises(KeyError, match="not found"):
|
||||
registry.unregister("no_such_module")
|
||||
|
||||
def test_unregister_with_dependents_raises(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(Module(name="base_module"))
|
||||
registry.register(Module(name="dependent_module", dependencies=["base_module"]))
|
||||
with pytest.raises(ValueError, match="depended on by"):
|
||||
registry.unregister("base_module")
|
||||
|
||||
def test_clear(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(Module(name="mod1"))
|
||||
registry.register(Module(name="mod2"))
|
||||
registry.clear()
|
||||
assert registry.list_modules() == []
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ModuleRegistry - 自动激活 & 依赖
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestModuleRegistryAutoActivate:
|
||||
"""注册时自动激活逻辑"""
|
||||
|
||||
def test_no_deps_auto_activates(self):
|
||||
registry = ModuleRegistry()
|
||||
mod = Module(name="standalone")
|
||||
registry.register(mod)
|
||||
assert mod.status == ModuleStatus.ACTIVE
|
||||
|
||||
def test_with_deps_all_satisfied_auto_activates(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(Module(name="base")) # 无依赖,自动激活
|
||||
dep_mod = Module(name="dependent", dependencies=["base"])
|
||||
registry.register(dep_mod)
|
||||
assert dep_mod.status == ModuleStatus.ACTIVE
|
||||
|
||||
def test_with_deps_not_satisfied_stays_registered(self):
|
||||
registry = ModuleRegistry()
|
||||
mod = Module(name="dependent", dependencies=["missing_dep"])
|
||||
registry.register(mod)
|
||||
# 依赖不满足,保持 REGISTERED
|
||||
assert mod.status == ModuleStatus.REGISTERED
|
||||
|
||||
def test_later_dep_registered_manual_activate(self):
|
||||
"""先注册依赖模块,再注册被依赖模块时不自动激活前者
|
||||
(需要手动或在注册完所有模块后调用 check_dependencies + activate)"""
|
||||
registry = ModuleRegistry()
|
||||
# 先注册依赖方(依赖未满足,不激活)
|
||||
dependent = Module(name="dependent", dependencies=["base"])
|
||||
registry.register(dependent)
|
||||
assert dependent.status == ModuleStatus.REGISTERED
|
||||
|
||||
# 再注册被依赖方
|
||||
base = Module(name="base")
|
||||
registry.register(base)
|
||||
assert base.status == ModuleStatus.ACTIVE
|
||||
|
||||
# 依赖方仍然是 REGISTERED(不会自动激活)
|
||||
assert dependent.status == ModuleStatus.REGISTERED
|
||||
|
||||
|
||||
class TestModuleRegistryCheckDependencies:
|
||||
"""check_dependencies 依赖检查"""
|
||||
|
||||
def test_module_not_found_returns_false(self):
|
||||
registry = ModuleRegistry()
|
||||
assert registry.check_dependencies("nonexistent") is False
|
||||
|
||||
def test_no_deps_returns_true(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(Module(name="standalone"))
|
||||
assert registry.check_dependencies("standalone") is True
|
||||
|
||||
def test_all_deps_active_returns_true(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(Module(name="dep1"))
|
||||
registry.register(Module(name="dep2"))
|
||||
registry.register(Module(name="main", dependencies=["dep1", "dep2"]))
|
||||
# main 在注册时因依赖满足已自动激活
|
||||
assert registry.check_dependencies("main") is True
|
||||
|
||||
def test_dep_not_registered_returns_false(self):
|
||||
registry = ModuleRegistry()
|
||||
mod = Module(name="main", dependencies=["missing"])
|
||||
registry.register(mod)
|
||||
assert registry.check_dependencies("main") is False
|
||||
|
||||
def test_dep_registered_but_not_active_returns_false(self):
|
||||
registry = ModuleRegistry()
|
||||
dep = Module(name="dep", status=ModuleStatus.DISABLED)
|
||||
registry.register(dep)
|
||||
# 手动设为 disabled(因为 register 时无依赖会自动激活)
|
||||
dep.disable()
|
||||
main = Module(name="main", dependencies=["dep"])
|
||||
registry.register(main)
|
||||
# 依赖未激活
|
||||
assert registry.check_dependencies("main") is False
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ModuleRegistry - list_modules & 状态过滤
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestModuleRegistryList:
|
||||
"""list_modules 列表与过滤"""
|
||||
|
||||
def test_list_all(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(Module(name="mod1"))
|
||||
registry.register(Module(name="mod2"))
|
||||
modules = registry.list_modules()
|
||||
assert len(modules) == 2
|
||||
names = {m.name for m in modules}
|
||||
assert names == {"mod1", "mod2"}
|
||||
|
||||
def test_filter_by_active(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(Module(name="active_mod")) # 自动激活
|
||||
disabled = Module(name="disabled_mod")
|
||||
registry.register(disabled)
|
||||
disabled.disable()
|
||||
|
||||
active = registry.list_modules(status=ModuleStatus.ACTIVE)
|
||||
assert len(active) == 1
|
||||
assert active[0].name == "active_mod"
|
||||
|
||||
def test_filter_by_disabled(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(Module(name="active_mod"))
|
||||
disabled = Module(name="disabled_mod")
|
||||
registry.register(disabled)
|
||||
disabled.disable()
|
||||
|
||||
disabled_list = registry.list_modules(status=ModuleStatus.DISABLED)
|
||||
assert len(disabled_list) == 1
|
||||
assert disabled_list[0].name == "disabled_mod"
|
||||
|
||||
def test_filter_registered(self):
|
||||
registry = ModuleRegistry()
|
||||
# 有依赖未满足的模块保持 REGISTERED
|
||||
mod = Module(name="waiting_mod", dependencies=["missing"])
|
||||
registry.register(mod)
|
||||
|
||||
registered = registry.list_modules(status=ModuleStatus.REGISTERED)
|
||||
assert len(registered) == 1
|
||||
assert registered[0].name == "waiting_mod"
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ModuleRegistry - 能力发现
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestModuleRegistryCapabilities:
|
||||
"""能力发现:has_capability / get_capability / get_quota_rules"""
|
||||
|
||||
def test_has_capability_true(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(
|
||||
Module(
|
||||
name="voice_module",
|
||||
capabilities=[ModuleCapability(name="generate_voice")],
|
||||
)
|
||||
)
|
||||
assert registry.has_capability("generate_voice") is True
|
||||
|
||||
def test_has_capability_false(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(
|
||||
Module(
|
||||
name="voice_module",
|
||||
capabilities=[ModuleCapability(name="generate_voice")],
|
||||
)
|
||||
)
|
||||
assert registry.has_capability("generate_video") is False
|
||||
|
||||
def test_has_capability_inactive_module_not_counted(self):
|
||||
registry = ModuleRegistry()
|
||||
mod = Module(
|
||||
name="inactive_mod",
|
||||
capabilities=[ModuleCapability(name="secret_cap")],
|
||||
)
|
||||
registry.register(mod)
|
||||
mod.disable()
|
||||
assert registry.has_capability("secret_cap") is False
|
||||
|
||||
def test_get_capability_returns_first_match(self):
|
||||
registry = ModuleRegistry()
|
||||
cap1 = ModuleCapability(name="export", description="导出1")
|
||||
cap2 = ModuleCapability(name="export", description="导出2")
|
||||
registry.register(Module(name="mod1", capabilities=[cap1]))
|
||||
registry.register(Module(name="mod2", capabilities=[cap2]))
|
||||
|
||||
result = registry.get_capability("export")
|
||||
assert result is not None
|
||||
assert result.name == "export"
|
||||
# 返回第一个匹配的(mod1)
|
||||
assert result.description == "导出1"
|
||||
|
||||
def test_get_capability_nonexistent_returns_none(self):
|
||||
registry = ModuleRegistry()
|
||||
assert registry.get_capability("no_such_cap") is None
|
||||
|
||||
def test_get_quota_rules(self):
|
||||
rules = [
|
||||
QuotaRule(dimension="credits", per_operation=1.0),
|
||||
QuotaRule(dimension="storage", per_operation=0.5),
|
||||
]
|
||||
registry = ModuleRegistry()
|
||||
registry.register(
|
||||
Module(
|
||||
name="voice_mod",
|
||||
capabilities=[ModuleCapability(name="gen", quota_rules=rules)],
|
||||
)
|
||||
)
|
||||
result = registry.get_quota_rules("gen")
|
||||
assert len(result) == 2
|
||||
assert result[0].dimension == "credits"
|
||||
assert result[1].dimension == "storage"
|
||||
|
||||
def test_get_quota_rules_nonexistent_returns_empty(self):
|
||||
registry = ModuleRegistry()
|
||||
assert registry.get_quota_rules("no_cap") == []
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ModuleRegistry - get_active_capabilities
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestModuleRegistryActiveCapabilities:
|
||||
"""get_active_capabilities 已激活能力汇总"""
|
||||
|
||||
def test_empty_registry(self):
|
||||
registry = ModuleRegistry()
|
||||
assert registry.get_active_capabilities() == {}
|
||||
|
||||
def test_single_module_with_caps(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(
|
||||
Module(
|
||||
name="voice_mod",
|
||||
capabilities=[
|
||||
ModuleCapability(name="generate_voice"),
|
||||
ModuleCapability(name="clone_voice"),
|
||||
],
|
||||
)
|
||||
)
|
||||
result = registry.get_active_capabilities()
|
||||
assert "voice_mod" in result
|
||||
assert set(result["voice_mod"]) == {"generate_voice", "clone_voice"}
|
||||
|
||||
def test_skips_inactive_modules(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(
|
||||
Module(
|
||||
name="active_mod",
|
||||
capabilities=[ModuleCapability(name="active_cap")],
|
||||
)
|
||||
)
|
||||
inactive = Module(
|
||||
name="inactive_mod",
|
||||
capabilities=[ModuleCapability(name="inactive_cap")],
|
||||
)
|
||||
registry.register(inactive)
|
||||
inactive.disable()
|
||||
|
||||
result = registry.get_active_capabilities()
|
||||
assert "active_mod" in result
|
||||
assert "inactive_mod" not in result
|
||||
|
||||
def test_skips_modules_without_caps(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(Module(name="no_cap_mod"))
|
||||
result = registry.get_active_capabilities()
|
||||
assert "no_cap_mod" not in result
|
||||
|
||||
def test_multiple_modules(self):
|
||||
registry = ModuleRegistry()
|
||||
registry.register(
|
||||
Module(
|
||||
name="mod1",
|
||||
capabilities=[ModuleCapability(name="cap_a")],
|
||||
)
|
||||
)
|
||||
registry.register(
|
||||
Module(
|
||||
name="mod2",
|
||||
capabilities=[ModuleCapability(name="cap_b"), ModuleCapability(name="cap_c")],
|
||||
)
|
||||
)
|
||||
result = registry.get_active_capabilities()
|
||||
assert len(result) == 2
|
||||
assert result["mod1"] == ["cap_a"]
|
||||
assert set(result["mod2"]) == {"cap_b", "cap_c"}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 全局单例
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestGlobalSingleton:
|
||||
"""全局 module_registry 单例"""
|
||||
|
||||
def test_singleton_exists(self):
|
||||
assert module_registry is not None
|
||||
assert isinstance(module_registry, ModuleRegistry)
|
||||
|
||||
def test_singleton_is_same_instance(self):
|
||||
from packages.infrastructure.module_registry import module_registry as mr2
|
||||
|
||||
assert module_registry is mr2
|
||||
Executable
+337
@@ -0,0 +1,337 @@
|
||||
"""
|
||||
pagination 通用分页器单元测试
|
||||
|
||||
覆盖:
|
||||
- PaginationParams: 默认值/边界/校验/offset/limit
|
||||
- PaginationMeta: from_params 各种边界场景
|
||||
- PaginatedResponse: create 工厂方法
|
||||
- paginate: 内存分页函数
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from packages.application.common.pagination import (
|
||||
PaginatedResponse,
|
||||
PaginationMeta,
|
||||
PaginationParams,
|
||||
paginate,
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# PaginationParams
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestPaginationParamsDefaults:
|
||||
"""默认值测试"""
|
||||
|
||||
def test_default_page_is_1(self):
|
||||
params = PaginationParams()
|
||||
assert params.page == 1
|
||||
|
||||
def test_default_page_size_is_20(self):
|
||||
params = PaginationParams()
|
||||
assert params.page_size == 20
|
||||
|
||||
def test_default_offset_is_0(self):
|
||||
params = PaginationParams()
|
||||
assert params.offset == 0
|
||||
|
||||
def test_default_limit_is_20(self):
|
||||
params = PaginationParams()
|
||||
assert params.limit == 20
|
||||
|
||||
|
||||
class TestPaginationParamsValidation:
|
||||
"""参数校验"""
|
||||
|
||||
@pytest.mark.parametrize("page", [1, 2, 100, 9999])
|
||||
def test_valid_page_values(self, page):
|
||||
params = PaginationParams(page=page)
|
||||
assert params.page == page
|
||||
|
||||
def test_page_zero_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
PaginationParams(page=0)
|
||||
|
||||
def test_page_negative_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
PaginationParams(page=-1)
|
||||
|
||||
@pytest.mark.parametrize("page_size", [1, 20, 50, 100])
|
||||
def test_valid_page_size_values(self, page_size):
|
||||
params = PaginationParams(page_size=page_size)
|
||||
assert params.page_size == page_size
|
||||
|
||||
def test_page_size_zero_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
PaginationParams(page_size=0)
|
||||
|
||||
def test_page_size_negative_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
PaginationParams(page_size=-5)
|
||||
|
||||
def test_page_size_over_100_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
PaginationParams(page_size=101)
|
||||
|
||||
def test_invalid_page_type_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
PaginationParams(page="abc")
|
||||
|
||||
def test_invalid_page_size_type_raises(self):
|
||||
with pytest.raises(ValidationError):
|
||||
PaginationParams(page_size="abc")
|
||||
|
||||
|
||||
class TestPaginationParamsOffset:
|
||||
"""offset 属性计算"""
|
||||
|
||||
def test_page_1_offset_0(self):
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
assert params.offset == 0
|
||||
|
||||
def test_page_2_offset_page_size(self):
|
||||
params = PaginationParams(page=2, page_size=20)
|
||||
assert params.offset == 20
|
||||
|
||||
def test_page_3_offset_2x_page_size(self):
|
||||
params = PaginationParams(page=3, page_size=20)
|
||||
assert params.offset == 40
|
||||
|
||||
def test_page_5_page_size_10_offset_40(self):
|
||||
params = PaginationParams(page=5, page_size=10)
|
||||
assert params.offset == 40
|
||||
|
||||
def test_page_1_page_size_100_offset_0(self):
|
||||
params = PaginationParams(page=1, page_size=100)
|
||||
assert params.offset == 0
|
||||
|
||||
|
||||
class TestPaginationParamsLimit:
|
||||
"""limit 属性"""
|
||||
|
||||
def test_limit_equals_page_size(self):
|
||||
params = PaginationParams(page_size=20)
|
||||
assert params.limit == 20
|
||||
|
||||
def test_limit_1(self):
|
||||
params = PaginationParams(page_size=1)
|
||||
assert params.limit == 1
|
||||
|
||||
def test_limit_100(self):
|
||||
params = PaginationParams(page_size=100)
|
||||
assert params.limit == 100
|
||||
|
||||
|
||||
# ============================================================
|
||||
# PaginationMeta.from_params
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestPaginationMetaFromParams:
|
||||
"""from_params 工厂方法"""
|
||||
|
||||
def test_empty_total_zero(self):
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
meta = PaginationMeta.from_params(params, total=0)
|
||||
assert meta.total == 0
|
||||
assert meta.total_pages == 0
|
||||
assert meta.has_next is False
|
||||
assert meta.has_prev is False
|
||||
|
||||
def test_exactly_one_page(self):
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
meta = PaginationMeta.from_params(params, total=20)
|
||||
assert meta.total_pages == 1
|
||||
assert meta.has_next is False
|
||||
assert meta.has_prev is False
|
||||
|
||||
def test_less_than_one_page(self):
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
meta = PaginationMeta.from_params(params, total=15)
|
||||
assert meta.total_pages == 1
|
||||
assert meta.has_next is False
|
||||
assert meta.has_prev is False
|
||||
|
||||
def test_multiple_pages_first_page(self):
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
meta = PaginationMeta.from_params(params, total=50)
|
||||
assert meta.total_pages == 3
|
||||
assert meta.has_next is True
|
||||
assert meta.has_prev is False
|
||||
|
||||
def test_multiple_pages_middle_page(self):
|
||||
params = PaginationParams(page=2, page_size=20)
|
||||
meta = PaginationMeta.from_params(params, total=50)
|
||||
assert meta.total_pages == 3
|
||||
assert meta.has_next is True
|
||||
assert meta.has_prev is True
|
||||
|
||||
def test_multiple_pages_last_page(self):
|
||||
params = PaginationParams(page=3, page_size=20)
|
||||
meta = PaginationMeta.from_params(params, total=50)
|
||||
assert meta.total_pages == 3
|
||||
assert meta.has_next is False
|
||||
assert meta.has_prev is True
|
||||
|
||||
def test_exact_division(self):
|
||||
params = PaginationParams(page=2, page_size=20)
|
||||
meta = PaginationMeta.from_params(params, total=40)
|
||||
assert meta.total_pages == 2
|
||||
assert meta.has_next is False
|
||||
assert meta.has_prev is True
|
||||
|
||||
def test_non_exact_division_ceil(self):
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
meta = PaginationMeta.from_params(params, total=41)
|
||||
assert meta.total_pages == 3
|
||||
|
||||
def test_total_1_page_size_20(self):
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
meta = PaginationMeta.from_params(params, total=1)
|
||||
assert meta.total_pages == 1
|
||||
assert meta.has_next is False
|
||||
assert meta.has_prev is False
|
||||
|
||||
def test_page_beyond_total_pages(self):
|
||||
params = PaginationParams(page=10, page_size=20)
|
||||
meta = PaginationMeta.from_params(params, total=50)
|
||||
assert meta.total_pages == 3
|
||||
assert meta.has_next is False
|
||||
assert meta.has_prev is True
|
||||
|
||||
def test_preserves_params_values(self):
|
||||
params = PaginationParams(page=3, page_size=15)
|
||||
meta = PaginationMeta.from_params(params, total=100)
|
||||
assert meta.page == 3
|
||||
assert meta.page_size == 15
|
||||
assert meta.total == 100
|
||||
|
||||
|
||||
# ============================================================
|
||||
# PaginatedResponse.create
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestPaginatedResponseCreate:
|
||||
"""create 工厂方法"""
|
||||
|
||||
def test_create_with_data(self):
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
data = [1, 2, 3]
|
||||
response = PaginatedResponse.create(data, params, total=100)
|
||||
assert response.data == data
|
||||
assert response.pagination.total == 100
|
||||
assert response.pagination.page == 1
|
||||
assert response.pagination.page_size == 20
|
||||
|
||||
def test_create_with_empty_data(self):
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
response = PaginatedResponse.create([], params, total=0)
|
||||
assert response.data == []
|
||||
assert response.pagination.total == 0
|
||||
assert response.pagination.total_pages == 0
|
||||
|
||||
def test_create_preserves_list_type(self):
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
data = ["a", "b", "c"]
|
||||
response = PaginatedResponse.create(data, params, total=10)
|
||||
assert response.data == ["a", "b", "c"]
|
||||
assert len(response.data) == 3
|
||||
|
||||
|
||||
# ============================================================
|
||||
# paginate 函数
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestPaginateFunction:
|
||||
"""内存分页函数"""
|
||||
|
||||
def test_empty_list(self):
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
result = paginate([], params)
|
||||
assert result.data == []
|
||||
assert result.pagination.total == 0
|
||||
assert result.pagination.total_pages == 0
|
||||
|
||||
def test_first_page(self):
|
||||
items = list(range(50))
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
result = paginate(items, params)
|
||||
assert result.data == list(range(20))
|
||||
assert result.pagination.total == 50
|
||||
assert result.pagination.total_pages == 3
|
||||
assert result.pagination.has_next is True
|
||||
assert result.pagination.has_prev is False
|
||||
|
||||
def test_middle_page(self):
|
||||
items = list(range(50))
|
||||
params = PaginationParams(page=2, page_size=20)
|
||||
result = paginate(items, params)
|
||||
assert result.data == list(range(20, 40))
|
||||
assert result.pagination.has_next is True
|
||||
assert result.pagination.has_prev is True
|
||||
|
||||
def test_last_page(self):
|
||||
items = list(range(50))
|
||||
params = PaginationParams(page=3, page_size=20)
|
||||
result = paginate(items, params)
|
||||
assert result.data == list(range(40, 50))
|
||||
assert len(result.data) == 10
|
||||
assert result.pagination.has_next is False
|
||||
assert result.pagination.has_prev is True
|
||||
|
||||
def test_page_beyond_total(self):
|
||||
items = list(range(25))
|
||||
params = PaginationParams(page=10, page_size=20)
|
||||
result = paginate(items, params)
|
||||
assert result.data == []
|
||||
assert result.pagination.total == 25
|
||||
assert result.pagination.total_pages == 2
|
||||
|
||||
def test_page_size_larger_than_total(self):
|
||||
items = list(range(5))
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
result = paginate(items, params)
|
||||
assert result.data == items
|
||||
assert result.pagination.total_pages == 1
|
||||
assert result.pagination.has_next is False
|
||||
|
||||
def test_single_item(self):
|
||||
items = [42]
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
result = paginate(items, params)
|
||||
assert result.data == [42]
|
||||
assert result.pagination.total == 1
|
||||
|
||||
def test_page_size_1(self):
|
||||
items = list(range(5))
|
||||
params = PaginationParams(page=3, page_size=1)
|
||||
result = paginate(items, params)
|
||||
assert result.data == [2]
|
||||
assert result.pagination.total_pages == 5
|
||||
|
||||
def test_exact_page_size(self):
|
||||
items = list(range(40))
|
||||
params = PaginationParams(page=2, page_size=20)
|
||||
result = paginate(items, params)
|
||||
assert result.data == list(range(20, 40))
|
||||
assert result.pagination.total_pages == 2
|
||||
assert result.pagination.has_next is False
|
||||
|
||||
def test_string_items(self):
|
||||
items = ["a", "b", "c", "d", "e"]
|
||||
params = PaginationParams(page=2, page_size=2)
|
||||
result = paginate(items, params)
|
||||
assert result.data == ["c", "d"]
|
||||
assert result.pagination.total == 5
|
||||
|
||||
def test_does_not_mutate_original_list(self):
|
||||
items = list(range(10))
|
||||
original = items.copy()
|
||||
params = PaginationParams(page=1, page_size=3)
|
||||
paginate(items, params)
|
||||
assert items == original
|
||||
Executable
+539
@@ -0,0 +1,539 @@
|
||||
"""
|
||||
SharedStorageService 单元测试
|
||||
|
||||
重点覆盖纯逻辑部分:
|
||||
- _normalize_storage_key: URL提取 + URL解码
|
||||
- _is_local_generated_url: 本地生成URL判断
|
||||
- get_url: 公共URL拼接
|
||||
- create_direct_upload_post: policy + HMAC签名
|
||||
- get_download_url: bucket=None时的fallback
|
||||
- 未配置OSS时的错误处理
|
||||
- 单例模式
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from packages.shared.storage import (
|
||||
SharedStorageService,
|
||||
get_shared_storage_service,
|
||||
get_storage_service,
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# Fixtures
|
||||
# ============================================================
|
||||
|
||||
|
||||
def _make_service(
|
||||
bucket_name="test-bucket",
|
||||
endpoint="oss-cn-hangzhou.aliyuncs.com",
|
||||
access_key_id="test-key-id",
|
||||
access_key_secret="test-key-secret",
|
||||
local_url_prefix="/generated-files",
|
||||
with_bucket=True,
|
||||
):
|
||||
"""创建一个 SharedStorageService 实例,mock 掉 oss2 和 settings。"""
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.oss_bucket_name = bucket_name
|
||||
mock_settings.oss_endpoint = endpoint
|
||||
mock_settings.oss_access_key_id = access_key_id
|
||||
mock_settings.oss_access_key_secret = access_key_secret
|
||||
|
||||
mock_bucket = MagicMock() if with_bucket else None
|
||||
|
||||
with (
|
||||
patch("packages.shared.storage.get_shared_settings", return_value=mock_settings),
|
||||
patch.dict(os.environ, {"GENERATED_FILES_URL_PREFIX": local_url_prefix}, clear=False),
|
||||
):
|
||||
if with_bucket:
|
||||
with patch("packages.shared.storage.oss2") as mock_oss2:
|
||||
mock_oss2.Auth.return_value = MagicMock()
|
||||
mock_oss2.Bucket.return_value = mock_bucket
|
||||
service = SharedStorageService()
|
||||
service.bucket = mock_bucket
|
||||
return service, mock_bucket, mock_settings
|
||||
else:
|
||||
service = SharedStorageService()
|
||||
service.bucket = None
|
||||
return service, None, mock_settings
|
||||
|
||||
|
||||
# ============================================================
|
||||
# _normalize_storage_key
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestNormalizeStorageKey:
|
||||
"""_normalize_storage_key URL 提取与解码"""
|
||||
|
||||
def test_plain_key_returns_as_is(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("videos/clip.mp4")
|
||||
assert result == "videos/clip.mp4"
|
||||
|
||||
def test_key_with_leading_slash_stripped(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("/videos/clip.mp4")
|
||||
assert result == "videos/clip.mp4"
|
||||
|
||||
def test_https_url_extracts_path(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("https://test-bucket.oss-cn-hangzhou.aliyuncs.com/videos/clip.mp4")
|
||||
assert result == "videos/clip.mp4"
|
||||
|
||||
def test_http_url_extracts_path(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("http://test-bucket.oss-cn-hangzhou.aliyuncs.com/audio/voice.mp3")
|
||||
assert result == "audio/voice.mp3"
|
||||
|
||||
def test_url_with_query_strips_query(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("https://bucket.oss-cn.com/file.mp4?signature=abc&expires=123")
|
||||
assert result == "file.mp4"
|
||||
|
||||
def test_url_with_leading_slash_in_path(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("https://bucket.oss.com//double/slash.jpg")
|
||||
assert result == "double/slash.jpg"
|
||||
|
||||
def test_url_decodes_percent_encoded_spaces(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("https://bucket.oss.com/my%20video.mp4")
|
||||
assert result == "my video.mp4"
|
||||
|
||||
def test_url_decodes_percent_encoded_chinese(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("https://bucket.oss.com/%E4%B8%AD%E6%96%87.mp4")
|
||||
assert result == "中文.mp4"
|
||||
|
||||
def test_url_with_special_chars_decoded(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("https://bucket.oss.com/file%281%29.jpg")
|
||||
assert result == "file(1).jpg"
|
||||
|
||||
def test_plain_key_with_percent_not_decoded(self):
|
||||
"""原始 key 不以 http 开头,不做 URL 解码,直接 lstrip('/')"""
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("file%20name.mp4")
|
||||
# 不是 URL,直接返回(去掉前导/)
|
||||
assert result == "file%20name.mp4"
|
||||
|
||||
def test_empty_string(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("")
|
||||
assert result == ""
|
||||
|
||||
def test_root_slash_url(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("https://bucket.oss.com/")
|
||||
assert result == ""
|
||||
|
||||
def test_nested_path_url(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("https://bucket.oss.com/a/b/c/d/file.txt")
|
||||
assert result == "a/b/c/d/file.txt"
|
||||
|
||||
def test_url_with_port(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service._normalize_storage_key("https://bucket.oss.com:443/file.txt")
|
||||
assert result == "file.txt"
|
||||
|
||||
|
||||
# ============================================================
|
||||
# _is_local_generated_url
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestIsLocalGeneratedUrl:
|
||||
"""_is_local_generated_url 本地URL判断"""
|
||||
|
||||
def test_local_prefix_returns_true(self):
|
||||
service, _, _ = _make_service()
|
||||
assert service._is_local_generated_url("/generated-files/abc.mp4") is True
|
||||
|
||||
def test_relative_local_returns_true(self):
|
||||
service, _, _ = _make_service()
|
||||
# 没有 scheme,直接用原字符串匹配
|
||||
assert service._is_local_generated_url("/generated-files/out.mp4") is True
|
||||
|
||||
def test_full_url_with_local_path_returns_true(self):
|
||||
service, _, _ = _make_service()
|
||||
assert service._is_local_generated_url("https://example.com/generated-files/abc.mp4") is True
|
||||
|
||||
def test_other_path_returns_false(self):
|
||||
service, _, _ = _make_service()
|
||||
assert service._is_local_generated_url("/videos/abc.mp4") is False
|
||||
|
||||
def test_empty_string_returns_false(self):
|
||||
service, _, _ = _make_service()
|
||||
assert service._is_local_generated_url("") is False
|
||||
|
||||
def test_custom_prefix(self):
|
||||
service, _, _ = _make_service(local_url_prefix="/custom-prefix")
|
||||
assert service._is_local_generated_url("/custom-prefix/file.mp4") is True
|
||||
assert service._is_local_generated_url("/generated-files/file.mp4") is False
|
||||
|
||||
|
||||
# ============================================================
|
||||
# get_url
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestGetUrl:
|
||||
"""get_url 公共URL拼接"""
|
||||
|
||||
def test_returns_public_url_plus_key(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service.get_url("videos/test.mp4")
|
||||
assert result == "https://test-bucket.oss-cn-hangzhou.aliyuncs.com/videos/test.mp4"
|
||||
|
||||
def test_empty_key(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service.get_url("")
|
||||
assert result == "https://test-bucket.oss-cn-hangzhou.aliyuncs.com/"
|
||||
|
||||
def test_custom_bucket_and_endpoint(self):
|
||||
service, _, _ = _make_service(
|
||||
bucket_name="my-bucket",
|
||||
endpoint="oss-us-east-1.aliyuncs.com",
|
||||
)
|
||||
result = service.get_url("file.txt")
|
||||
assert result == "https://my-bucket.oss-us-east-1.aliyuncs.com/file.txt"
|
||||
|
||||
|
||||
# ============================================================
|
||||
# create_direct_upload_post
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestCreateDirectUploadPost:
|
||||
"""create_direct_upload_post 直传表单生成"""
|
||||
|
||||
def test_returns_dict_with_expected_keys(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service.create_direct_upload_post(
|
||||
storage_key="uploads/test.jpg",
|
||||
content_type="image/jpeg",
|
||||
max_size_bytes=10 * 1024 * 1024,
|
||||
expires_seconds=3600,
|
||||
)
|
||||
assert "url" in result
|
||||
assert "method" in result
|
||||
assert "storage_key" in result
|
||||
assert "expires_at" in result
|
||||
assert "fields" in result
|
||||
|
||||
def test_method_is_post(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service.create_direct_upload_post("uploads/a.jpg", "image/jpeg", 1024, 3600)
|
||||
assert result["method"] == "POST"
|
||||
|
||||
def test_url_is_public_url(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service.create_direct_upload_post("uploads/a.jpg", "image/jpeg", 1024, 3600)
|
||||
assert result["url"] == "https://test-bucket.oss-cn-hangzhou.aliyuncs.com"
|
||||
|
||||
def test_storage_key_normalized(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service.create_direct_upload_post("/uploads/test.jpg", "image/jpeg", 1024, 3600)
|
||||
assert result["storage_key"] == "uploads/test.jpg"
|
||||
assert result["fields"]["key"] == "uploads/test.jpg"
|
||||
|
||||
def test_fields_contain_required_keys(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service.create_direct_upload_post("uploads/a.jpg", "image/jpeg", 1024, 3600)
|
||||
fields = result["fields"]
|
||||
assert fields["key"] == "uploads/a.jpg"
|
||||
assert fields["OSSAccessKeyId"] == "test-key-id"
|
||||
assert fields["success_action_status"] == "201"
|
||||
assert fields["Content-Type"] == "image/jpeg"
|
||||
assert "policy" in fields
|
||||
assert "Signature" in fields
|
||||
|
||||
def test_policy_signature_is_valid_hmac_sha1(self):
|
||||
"""验证 HMAC-SHA1 签名是否正确"""
|
||||
secret = "my-secret-key-123"
|
||||
service, _, _ = _make_service(access_key_secret=secret)
|
||||
result = service.create_direct_upload_post("uploads/a.jpg", "image/jpeg", 1024, 3600)
|
||||
policy = result["fields"]["policy"]
|
||||
signature = result["fields"]["Signature"]
|
||||
|
||||
# 手动计算签名验证
|
||||
expected = base64.b64encode(
|
||||
hmac.new(secret.encode("utf-8"), policy.encode("utf-8"), hashlib.sha1).digest()
|
||||
).decode("ascii")
|
||||
assert signature == expected
|
||||
|
||||
def test_policy_contains_bucket_and_key(self):
|
||||
service, _, _ = _make_service(bucket_name="my-bucket")
|
||||
result = service.create_direct_upload_post("uploads/photo.png", "image/png", 2048, 1800)
|
||||
policy = json.loads(base64.b64decode(result["fields"]["policy"]))
|
||||
conditions = policy["conditions"]
|
||||
|
||||
assert {"bucket": "my-bucket"} in conditions
|
||||
assert {"key": "uploads/photo.png"} in conditions
|
||||
|
||||
def test_policy_contains_content_length_range(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service.create_direct_upload_post("uploads/a.jpg", "image/jpeg", 5242880, 3600)
|
||||
policy = json.loads(base64.b64decode(result["fields"]["policy"]))
|
||||
conditions = policy["conditions"]
|
||||
|
||||
size_condition = [c for c in conditions if isinstance(c, list) and c[0] == "content-length-range"]
|
||||
assert len(size_condition) == 1
|
||||
assert size_condition[0][1] == 1
|
||||
assert size_condition[0][2] == 5242880
|
||||
|
||||
def test_policy_content_type_starts_with(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service.create_direct_upload_post("uploads/a.jpg", "image/jpeg", 1024, 3600)
|
||||
policy = json.loads(base64.b64decode(result["fields"]["policy"]))
|
||||
conditions = policy["conditions"]
|
||||
|
||||
ct_condition = [c for c in conditions if isinstance(c, list) and c[0] == "starts-with"]
|
||||
assert len(ct_condition) == 1
|
||||
assert ct_condition[0][1] == "$Content-Type"
|
||||
assert ct_condition[0][2] == "image/"
|
||||
|
||||
def test_policy_has_expiration(self):
|
||||
service, _, _ = _make_service()
|
||||
result = service.create_direct_upload_post("uploads/a.jpg", "image/jpeg", 1024, 3600)
|
||||
policy = json.loads(base64.b64decode(result["fields"]["policy"]))
|
||||
assert "expiration" in policy
|
||||
# ISO 8601 格式
|
||||
assert policy["expiration"].endswith("Z")
|
||||
|
||||
def test_non_uploads_key_raises_value_error(self):
|
||||
service, _, _ = _make_service()
|
||||
with pytest.raises(ValueError, match="uploads/"):
|
||||
service.create_direct_upload_post("videos/a.mp4", "video/mp4", 1024, 3600)
|
||||
|
||||
def test_no_credentials_raises_runtime_error(self):
|
||||
service, _, _ = _make_service(access_key_id="", access_key_secret="", with_bucket=False)
|
||||
with pytest.raises(RuntimeError, match="not configured"):
|
||||
service.create_direct_upload_post("uploads/a.jpg", "image/jpeg", 1024, 3600)
|
||||
|
||||
def test_url_normalized_key_in_uploads(self):
|
||||
service, _, _ = _make_service()
|
||||
# URL 形式的 key 被 normalize 后如果在 uploads/ 下应该可以
|
||||
result = service.create_direct_upload_post(
|
||||
"https://test-bucket.oss-cn-hangzhou.aliyuncs.com/uploads/from_url.jpg",
|
||||
"image/jpeg",
|
||||
1024,
|
||||
3600,
|
||||
)
|
||||
assert result["storage_key"] == "uploads/from_url.jpg"
|
||||
|
||||
|
||||
# ============================================================
|
||||
# get_download_url (bucket=None 时的 fallback)
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestGetDownloadUrlFallback:
|
||||
"""get_download_url 在 bucket 未配置时的 fallback 逻辑"""
|
||||
|
||||
def test_no_bucket_local_url_returns_as_is(self):
|
||||
service, _, _ = _make_service(with_bucket=False)
|
||||
result = service.get_download_url("/generated-files/test.mp4")
|
||||
assert result == "/generated-files/test.mp4"
|
||||
|
||||
def test_no_bucket_regular_key_returns_public_url(self):
|
||||
service, _, _ = _make_service(with_bucket=False)
|
||||
result = service.get_download_url("videos/test.mp4")
|
||||
assert result == "https://test-bucket.oss-cn-hangzhou.aliyuncs.com/videos/test.mp4"
|
||||
|
||||
def test_no_bucket_url_input_normalized(self):
|
||||
service, _, _ = _make_service(with_bucket=False)
|
||||
result = service.get_download_url("https://test-bucket.oss-cn-hangzhou.aliyuncs.com/videos/test.mp4")
|
||||
assert result == "https://test-bucket.oss-cn-hangzhou.aliyuncs.com/videos/test.mp4"
|
||||
|
||||
def test_with_bucket_calls_sign_url(self):
|
||||
service, mock_bucket, _ = _make_service(with_bucket=True)
|
||||
mock_bucket.sign_url.return_value = "https://signed-url.com/file?sig=abc"
|
||||
|
||||
result = service.get_download_url("videos/test.mp4", expires_seconds=7200)
|
||||
|
||||
mock_bucket.sign_url.assert_called_once_with("GET", "videos/test.mp4", 7200)
|
||||
assert result == "https://signed-url.com/file?sig=abc"
|
||||
|
||||
def test_sign_url_exception_falls_back_to_public_url(self):
|
||||
service, mock_bucket, _ = _make_service(with_bucket=True)
|
||||
mock_bucket.sign_url.side_effect = Exception("sign error")
|
||||
|
||||
result = service.get_download_url("videos/test.mp4")
|
||||
|
||||
assert result == "https://test-bucket.oss-cn-hangzhou.aliyuncs.com/videos/test.mp4"
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 未配置 OSS 时的错误处理
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestNoBucketErrorHandling:
|
||||
"""bucket=None 时的错误处理"""
|
||||
|
||||
def test_upload_file_raises(self):
|
||||
service, _, _ = _make_service(with_bucket=False)
|
||||
with pytest.raises(RuntimeError, match="not configured"):
|
||||
service.upload_file("/tmp/test.txt", "uploads/test.txt")
|
||||
|
||||
def test_download_file_raises(self):
|
||||
service, _, _ = _make_service(with_bucket=False)
|
||||
with pytest.raises(RuntimeError, match="not configured"):
|
||||
service.download_file("uploads/test.txt", "/tmp/test.txt")
|
||||
|
||||
def test_delete_file_silent_noop(self):
|
||||
service, _, _ = _make_service(with_bucket=False)
|
||||
# 不抛异常
|
||||
result = service.delete_file("uploads/test.txt")
|
||||
assert result is None
|
||||
|
||||
def test_file_exists_returns_false(self):
|
||||
service, _, _ = _make_service(with_bucket=False)
|
||||
assert service.file_exists("uploads/test.txt") is False
|
||||
|
||||
|
||||
# ============================================================
|
||||
# upload_file / delete_file / file_exists 正常路径
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestBucketOperations:
|
||||
"""有 bucket 时的操作调用验证"""
|
||||
|
||||
def test_upload_file_with_path_string(self):
|
||||
service, mock_bucket, _ = _make_service()
|
||||
result = service.upload_file("/tmp/file.txt", "uploads/file.txt", "text/plain")
|
||||
|
||||
mock_bucket.put_object_from_file.assert_called_once()
|
||||
args = mock_bucket.put_object_from_file.call_args
|
||||
assert args[0][0] == "uploads/file.txt"
|
||||
assert args[0][1] == "/tmp/file.txt"
|
||||
assert result.startswith("https://test-bucket.")
|
||||
|
||||
def test_upload_file_with_file_object(self):
|
||||
service, mock_bucket, _ = _make_service()
|
||||
mock_file = MagicMock()
|
||||
result = service.upload_file(mock_file, "uploads/file.bin", "application/octet-stream")
|
||||
|
||||
mock_file.seek.assert_called_once_with(0)
|
||||
mock_bucket.put_object.assert_called_once()
|
||||
assert result.startswith("https://test-bucket.")
|
||||
|
||||
def test_delete_file_calls_bucket(self):
|
||||
service, mock_bucket, _ = _make_service()
|
||||
service.delete_file("uploads/test.txt")
|
||||
mock_bucket.delete_object.assert_called_once_with("uploads/test.txt")
|
||||
|
||||
def test_delete_file_exception_logged_not_raised(self):
|
||||
service, mock_bucket, _ = _make_service()
|
||||
mock_bucket.delete_object.side_effect = Exception("delete error")
|
||||
# 不抛异常
|
||||
service.delete_file("uploads/test.txt")
|
||||
|
||||
def test_file_exists_delegates_to_bucket(self):
|
||||
service, mock_bucket, _ = _make_service()
|
||||
mock_bucket.object_exists.return_value = True
|
||||
assert service.file_exists("some/key") is True
|
||||
mock_bucket.object_exists.assert_called_once_with("some/key")
|
||||
|
||||
def test_file_exists_false(self):
|
||||
service, mock_bucket, _ = _make_service()
|
||||
mock_bucket.object_exists.return_value = False
|
||||
assert service.file_exists("some/key") is False
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 单例 & 兼容别名
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestSingleton:
|
||||
"""get_shared_storage_service 单例模式"""
|
||||
|
||||
def test_get_storage_service_is_alias(self):
|
||||
# 两个函数返回同一个实例
|
||||
with patch("packages.shared.storage._storage_service", None):
|
||||
with patch("packages.shared.storage.SharedStorageService") as mock_cls:
|
||||
mock_instance = MagicMock()
|
||||
mock_cls.return_value = mock_instance
|
||||
|
||||
svc1 = get_shared_storage_service()
|
||||
svc2 = get_storage_service()
|
||||
|
||||
assert svc1 is svc2
|
||||
# 因为是同一个单例,类只实例化一次
|
||||
assert mock_cls.call_count == 1
|
||||
|
||||
|
||||
# ============================================================
|
||||
# __init__ endpoint 处理
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestInitEndpointHandling:
|
||||
"""初始化时 endpoint https 前缀处理"""
|
||||
|
||||
def test_endpoint_without_https_gets_prefix(self):
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.oss_bucket_name = "test-bucket"
|
||||
mock_settings.oss_endpoint = "oss-cn-hangzhou.aliyuncs.com"
|
||||
mock_settings.oss_access_key_id = "key-id"
|
||||
mock_settings.oss_access_key_secret = "key-secret"
|
||||
|
||||
with (
|
||||
patch("packages.shared.storage.get_shared_settings", return_value=mock_settings),
|
||||
patch("packages.shared.storage.oss2") as mock_oss2,
|
||||
):
|
||||
mock_oss2.Bucket.return_value = MagicMock()
|
||||
|
||||
service = SharedStorageService()
|
||||
|
||||
# 验证 Bucket 构造时 endpoint 带了 https://
|
||||
call_args = mock_oss2.Bucket.call_args
|
||||
assert call_args[0][1] == "https://oss-cn-hangzhou.aliyuncs.com"
|
||||
|
||||
def test_endpoint_with_https_keeps_as_is(self):
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.oss_bucket_name = "test-bucket"
|
||||
mock_settings.oss_endpoint = "https://oss-cn-hangzhou.aliyuncs.com"
|
||||
mock_settings.oss_access_key_id = "key-id"
|
||||
mock_settings.oss_access_key_secret = "key-secret"
|
||||
|
||||
with (
|
||||
patch("packages.shared.storage.get_shared_settings", return_value=mock_settings),
|
||||
patch("packages.shared.storage.oss2") as mock_oss2,
|
||||
):
|
||||
mock_oss2.Bucket.return_value = MagicMock()
|
||||
|
||||
service = SharedStorageService()
|
||||
|
||||
call_args = mock_oss2.Bucket.call_args
|
||||
assert call_args[0][1] == "https://oss-cn-hangzhou.aliyuncs.com"
|
||||
|
||||
def test_endpoint_with_http_keeps_as_is(self):
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.oss_bucket_name = "test-bucket"
|
||||
mock_settings.oss_endpoint = "http://oss-cn-hangzhou.aliyuncs.com"
|
||||
mock_settings.oss_access_key_id = "key-id"
|
||||
mock_settings.oss_access_key_secret = "key-secret"
|
||||
|
||||
with (
|
||||
patch("packages.shared.storage.get_shared_settings", return_value=mock_settings),
|
||||
patch("packages.shared.storage.oss2") as mock_oss2,
|
||||
):
|
||||
mock_oss2.Bucket.return_value = MagicMock()
|
||||
|
||||
service = SharedStorageService()
|
||||
|
||||
call_args = mock_oss2.Bucket.call_args
|
||||
assert call_args[0][1] == "http://oss-cn-hangzhou.aliyuncs.com"
|
||||
Executable
+315
@@ -0,0 +1,315 @@
|
||||
"""
|
||||
text_splitter 长文本分段工具单元测试
|
||||
|
||||
覆盖:
|
||||
- 空文本 / 短文本
|
||||
- 句子边界分段(。!?;\n . ! ? ;)
|
||||
- 超长句子硬切
|
||||
- 过短段落合并
|
||||
- max_chars 参数
|
||||
- 中英文混合
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from packages.application.tts_job.text_splitter import split_text
|
||||
|
||||
# ============================================================
|
||||
# 基础场景
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestBasicCases:
|
||||
"""基础场景"""
|
||||
|
||||
def test_empty_text_returns_empty_list(self):
|
||||
assert split_text("") == []
|
||||
|
||||
def test_whitespace_only_returns_empty(self):
|
||||
assert split_text(" \n\n ") == []
|
||||
|
||||
def test_short_text_single_segment(self):
|
||||
text = "这是一段短文本。"
|
||||
result = split_text(text, max_chars=500)
|
||||
assert result == [text]
|
||||
|
||||
def test_exactly_max_chars_single_segment(self):
|
||||
text = "a" * 500
|
||||
result = split_text(text, max_chars=500)
|
||||
assert len(result) == 1
|
||||
assert len(result[0]) == 500
|
||||
|
||||
def test_text_stripped(self):
|
||||
text = " 你好世界。 "
|
||||
result = split_text(text, max_chars=500)
|
||||
assert result == ["你好世界。"]
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 句子边界分段
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestSentenceBoundarySplitting:
|
||||
"""句子边界分段"""
|
||||
|
||||
def test_split_by_chinese_period(self):
|
||||
text = "第一句。第二句。第三句。"
|
||||
# 三句都很短,应该合并成一段
|
||||
result = split_text(text, max_chars=500)
|
||||
assert len(result) == 1
|
||||
|
||||
def test_split_by_chinese_period_long_text(self):
|
||||
"""多段长句子,按句号分段"""
|
||||
sentence1 = "我是第一句" + "啊" * 100 + "。"
|
||||
sentence2 = "我是第二句" + "哦" * 100 + "。"
|
||||
sentence3 = "我是第三句" + "嗯" * 100 + "。"
|
||||
text = sentence1 + sentence2 + sentence3
|
||||
|
||||
result = split_text(text, max_chars=150)
|
||||
# 每句106字符,超过150的阈值?不,106<150
|
||||
# 但累计到一定程度会切
|
||||
assert len(result) >= 2
|
||||
# 每段都不超过 max_chars
|
||||
for seg in result:
|
||||
assert len(seg) <= 150
|
||||
|
||||
def test_split_by_question_mark(self):
|
||||
text = "你是谁?你从哪里来?你要到哪里去?"
|
||||
result = split_text(text, max_chars=500)
|
||||
# 三句都很短,合并成一段
|
||||
assert len(result) == 1
|
||||
|
||||
def test_split_by_exclamation_mark(self):
|
||||
text = "太棒了!太厉害了!太牛了!"
|
||||
result = split_text(text, max_chars=500)
|
||||
assert len(result) == 1
|
||||
|
||||
def test_split_by_newline(self):
|
||||
text = "第一段\n第二段\n第三段"
|
||||
result = split_text(text, max_chars=500)
|
||||
assert len(result) == 1
|
||||
|
||||
def test_split_by_semicolon(self):
|
||||
text = "第一部分;第二部分;第三部分。"
|
||||
result = split_text(text, max_chars=500)
|
||||
assert len(result) == 1
|
||||
|
||||
def test_mixed_punctuation(self):
|
||||
"""混合标点符号的句子边界"""
|
||||
parts = []
|
||||
for i in range(20):
|
||||
parts.append(f"第{i}句的内容" + "字" * 30 + "。")
|
||||
text = "".join(parts)
|
||||
|
||||
result = split_text(text, max_chars=200)
|
||||
# 每句约35字符,200字符大约能放5-6句
|
||||
assert len(result) >= 2
|
||||
for seg in result:
|
||||
assert len(seg) <= 200
|
||||
|
||||
def test_english_period_splitting(self):
|
||||
text = "Hello. How are you. I am fine."
|
||||
result = split_text(text, max_chars=500)
|
||||
assert len(result) == 1
|
||||
|
||||
def test_english_question(self):
|
||||
text = "What? Why? How?"
|
||||
result = split_text(text, max_chars=500)
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 超长硬切
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestLongSentenceHardCut:
|
||||
"""超长句子硬切"""
|
||||
|
||||
def test_single_very_long_sentence_hard_cut(self):
|
||||
"""单个超长句子,没有标点,硬切"""
|
||||
text = "字" * 1000
|
||||
result = split_text(text, max_chars=500)
|
||||
assert len(result) == 2
|
||||
assert len(result[0]) == 500
|
||||
assert len(result[1]) == 500
|
||||
|
||||
def test_three_times_max_chars(self):
|
||||
text = "字" * 1500
|
||||
result = split_text(text, max_chars=500)
|
||||
assert len(result) == 3
|
||||
for seg in result:
|
||||
assert len(seg) == 500
|
||||
|
||||
def test_not_exact_multiple(self):
|
||||
text = "字" * 1250
|
||||
result = split_text(text, max_chars=500)
|
||||
assert len(result) == 3
|
||||
assert len(result[0]) == 500
|
||||
assert len(result[1]) == 500
|
||||
assert len(result[2]) == 250
|
||||
|
||||
def test_all_segments_within_limit(self):
|
||||
"""所有段都不超过 max_chars"""
|
||||
import random
|
||||
|
||||
random.seed(42)
|
||||
# 生成随机长度的文本
|
||||
text = "".join(random.choices("字字字字。!?;\n", k=5000))
|
||||
for max_chars in [100, 200, 500]:
|
||||
result = split_text(text, max_chars=max_chars)
|
||||
for i, seg in enumerate(result):
|
||||
assert len(seg) <= max_chars, f"Segment {i} length {len(seg)} > {max_chars}"
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 过短段落合并
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestShortSegmentMerging:
|
||||
"""过短段落合并"""
|
||||
|
||||
def test_short_final_segment_merged(self):
|
||||
"""最后一段过短,应该合并到前一段"""
|
||||
# 构造:前一段接近上限,后一段很短
|
||||
long_part = "字" * 480 + "。"
|
||||
short_part = "好的。"
|
||||
text = long_part + short_part
|
||||
|
||||
result = split_text(text, max_chars=500)
|
||||
# 两段加起来 481+3=484 < 500,可能合并
|
||||
# 但要看具体实现...
|
||||
# 至少验证所有段不超长
|
||||
for seg in result:
|
||||
assert len(seg) <= 500
|
||||
|
||||
def test_multiple_short_segments(self):
|
||||
"""多个短段落应该合并"""
|
||||
sentences = ["你好。", "我好。", "大家好。", "今天天气不错。", "适合出去玩。"]
|
||||
text = "".join(sentences)
|
||||
result = split_text(text, max_chars=500)
|
||||
# 5个短句子,应该合并成一段
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
# ============================================================
|
||||
# max_chars 参数
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestMaxCharsParameter:
|
||||
"""max_chars 参数"""
|
||||
|
||||
def test_small_max_chars(self):
|
||||
text = "一二三四五六七八九十一二三四五六七八九十。"
|
||||
result = split_text(text, max_chars=10)
|
||||
# 应该被切成多段
|
||||
assert len(result) >= 2
|
||||
for seg in result:
|
||||
assert len(seg) <= 10
|
||||
|
||||
def test_custom_max_chars_200(self):
|
||||
text = "测试文本" * 100 # 400字符
|
||||
result = split_text(text, max_chars=200)
|
||||
assert len(result) == 2
|
||||
assert len(result[0]) == 200
|
||||
assert len(result[1]) == 200
|
||||
|
||||
def test_very_small_max_chars(self):
|
||||
text = "abcdefghij"
|
||||
result = split_text(text, max_chars=3)
|
||||
assert len(result) >= 3
|
||||
for seg in result:
|
||||
assert len(seg) <= 3
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 中英文混合
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestMixedContent:
|
||||
"""中英文混合内容"""
|
||||
|
||||
def test_chinese_english_mixed(self):
|
||||
text = "今天天气很好,Today is sunny. 我们去公园玩吧!Let's go to the park."
|
||||
result = split_text(text, max_chars=500)
|
||||
assert len(result) == 1
|
||||
assert result[0] == text.strip()
|
||||
|
||||
def test_mixed_long_text(self):
|
||||
parts = []
|
||||
for i in range(50):
|
||||
parts.append(f"第{i}段中文内容" + "字" * 20 + ". English part " + "word " * 10 + "。")
|
||||
text = "".join(parts)
|
||||
|
||||
result = split_text(text, max_chars=300)
|
||||
assert len(result) >= 2
|
||||
for seg in result:
|
||||
assert len(seg) <= 300
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 输出完整性
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestOutputIntegrity:
|
||||
"""输出完整性验证"""
|
||||
|
||||
def test_combined_length_equals_original(self):
|
||||
"""所有段拼接起来(去掉空段)应该等于原文长度"""
|
||||
text = "这是第一段。这是第二段。这是第三段。这是第四段。这是第五段。" * 20
|
||||
result = split_text(text, max_chars=100)
|
||||
combined = "".join(result)
|
||||
# 由于 strip 可能去掉一些空格,原文也 strip 比较
|
||||
assert len(combined) == len(text.strip())
|
||||
|
||||
def test_order_preserved(self):
|
||||
"""分段后再拼接,文本顺序不变"""
|
||||
text = "第一。第二。第三。第四。第五。" * 10
|
||||
result = split_text(text, max_chars=50)
|
||||
combined = "".join(result)
|
||||
assert combined == text.strip()
|
||||
|
||||
def test_no_empty_strings_in_result(self):
|
||||
"""结果中没有空字符串"""
|
||||
text = "句子一。句子二。句子三。"
|
||||
result = split_text(text, max_chars=10)
|
||||
for seg in result:
|
||||
assert seg != ""
|
||||
assert len(seg) > 0
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 边界情况
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestEdgeCases:
|
||||
"""边界情况"""
|
||||
|
||||
def test_single_character(self):
|
||||
assert split_text("一", max_chars=500) == ["一"]
|
||||
|
||||
def test_only_punctuation(self):
|
||||
text = "。。。。。"
|
||||
result = split_text(text, max_chars=500)
|
||||
# 都是标点,也算文本
|
||||
assert len(result) == 1
|
||||
|
||||
def test_only_newlines(self):
|
||||
text = "\n\n\n"
|
||||
result = split_text(text, max_chars=500)
|
||||
assert result == []
|
||||
|
||||
def test_long_text_many_sentences(self):
|
||||
"""大量句子的长文本"""
|
||||
sentences = [f"第{i}句的完整内容。" for i in range(100)]
|
||||
text = "".join(sentences)
|
||||
result = split_text(text, max_chars=200)
|
||||
assert len(result) >= 5
|
||||
for seg in result:
|
||||
assert len(seg) <= 200
|
||||
+553
-252
@@ -1,296 +1,597 @@
|
||||
"""URL 安全校验工具单元测试 — SSRF 防护."""
|
||||
"""
|
||||
url_security URL安全校验单元测试
|
||||
|
||||
from __future__ import annotations
|
||||
覆盖:
|
||||
- validate_url_safety: scheme/主机/端口/SSRF/内网域名/白名单
|
||||
- is_url_safe: 便捷函数
|
||||
- UrlSecurityError / NoRedirectHandler
|
||||
- _validate_magic_number: 文件魔数校验
|
||||
- safe_download_file / safe_download_bytes: mock 网络测试
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "apps", "worker"))
|
||||
|
||||
import shutil
|
||||
import tempfile
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from video_processing.url_security import ( # noqa: E402
|
||||
import pytest
|
||||
|
||||
from packages.shared.url_security import (
|
||||
ALLOWED_AUDIO_MIME_TYPES,
|
||||
ALLOWED_IMAGE_MIME_TYPES,
|
||||
ALLOWED_PORTS,
|
||||
ALLOWED_SCHEMES,
|
||||
MAX_URL_LENGTH,
|
||||
NoRedirectHandler,
|
||||
UrlSecurityError,
|
||||
_check_internal_hostnames,
|
||||
_is_trusted_domain,
|
||||
_validate_magic_number,
|
||||
is_url_safe,
|
||||
safe_download_bytes,
|
||||
safe_download_file,
|
||||
validate_url_safety,
|
||||
)
|
||||
|
||||
|
||||
class TestUrlSecurityValidation(unittest.TestCase):
|
||||
"""URL 安全校验测试."""
|
||||
|
||||
# ── Scheme 白名单 ──────────────────────────────────────────────────────
|
||||
|
||||
def test_http_scheme_allowed(self):
|
||||
"""HTTP scheme 应该被允许."""
|
||||
result = validate_url_safety("http://example.com/test", purpose="test")
|
||||
self.assertEqual(result, "http://example.com/test")
|
||||
|
||||
def test_https_scheme_allowed(self):
|
||||
"""HTTPS scheme 应该被允许."""
|
||||
result = validate_url_safety("https://example.com/test", purpose="test")
|
||||
self.assertEqual(result, "https://example.com/test")
|
||||
|
||||
def test_file_scheme_rejected(self):
|
||||
"""file:// scheme 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("file:///etc/passwd", purpose="test")
|
||||
|
||||
def test_ftp_scheme_rejected(self):
|
||||
"""ftp:// scheme 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("ftp://example.com/test", purpose="test")
|
||||
|
||||
def test_empty_scheme_rejected(self):
|
||||
"""空 scheme 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("example.com/test", purpose="test")
|
||||
|
||||
# ── 端口白名单 ────────────────────────────────────────────────────────
|
||||
|
||||
def test_port_80_allowed(self):
|
||||
"""端口 80 应该被允许."""
|
||||
# 80端口是默认HTTP端口,不显式指定也可以
|
||||
result = validate_url_safety("http://example.com:80/test", purpose="test")
|
||||
self.assertIn("example.com", result)
|
||||
|
||||
def test_port_443_allowed(self):
|
||||
"""端口 443 应该被允许."""
|
||||
result = validate_url_safety("https://example.com:443/test", purpose="test")
|
||||
self.assertIn("example.com", result)
|
||||
|
||||
def test_port_8080_rejected(self):
|
||||
"""非标准端口 8080 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http://example.com:8080/test", purpose="test")
|
||||
|
||||
def test_port_22_rejected(self):
|
||||
"""SSH 端口 22 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http://example.com:22/test", purpose="test")
|
||||
|
||||
# ── SSRF: 直接 IP 访问 ───────────────────────────────────────────────
|
||||
|
||||
def test_loopback_ip_rejected(self):
|
||||
"""回环地址 127.0.0.1 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http://127.0.0.1/test", purpose="test")
|
||||
|
||||
def test_private_ip_192_rejected(self):
|
||||
"""内网地址 192.168.x.x 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http://192.168.1.1/test", purpose="test")
|
||||
|
||||
def test_private_ip_10_rejected(self):
|
||||
"""内网地址 10.x.x.x 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http://10.0.0.1/test", purpose="test")
|
||||
|
||||
def test_private_ip_172_rejected(self):
|
||||
"""内网地址 172.16.x.x 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http://172.16.0.1/test", purpose="test")
|
||||
|
||||
def test_unspecified_ip_rejected(self):
|
||||
"""未指定地址 0.0.0.0 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http://0.0.0.0/test", purpose="test")
|
||||
|
||||
def test_ipv6_loopback_rejected(self):
|
||||
"""IPv6 回环 ::1 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http://[::1]/test", purpose="test")
|
||||
|
||||
def test_ipv6_link_local_rejected(self):
|
||||
"""IPv6 链路本地地址应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http://[fe80::1]/test", purpose="test")
|
||||
|
||||
# ── SSRF: 内网主机名 ─────────────────────────────────────────────────
|
||||
|
||||
def test_localhost_rejected(self):
|
||||
"""localhost 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http://localhost/test", purpose="test")
|
||||
|
||||
def test_local_domain_rejected(self):
|
||||
""".local 域名应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http://printer.local/test", purpose="test")
|
||||
|
||||
def test_internal_domain_rejected(self):
|
||||
""".internal 域名应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http://db.internal/test", purpose="test")
|
||||
|
||||
# ── URL 格式校验 ─────────────────────────────────────────────────────
|
||||
|
||||
def test_empty_url_rejected(self):
|
||||
"""空 URL 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("", purpose="test")
|
||||
|
||||
def test_none_url_rejected(self):
|
||||
"""None URL 应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety(None, purpose="test") # type: ignore
|
||||
|
||||
def test_url_too_long_rejected(self):
|
||||
"""超长 URL 应该被拒绝."""
|
||||
long_url = "https://example.com/" + "a" * 3000
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety(long_url, purpose="test")
|
||||
|
||||
def test_no_hostname_rejected(self):
|
||||
"""缺少主机名应该被拒绝."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
validate_url_safety("http:///test", purpose="test")
|
||||
|
||||
# ── is_url_safe 便捷函数 ─────────────────────────────────────────────
|
||||
|
||||
def test_is_url_safe_true(self):
|
||||
"""安全 URL 应该返回 True."""
|
||||
self.assertTrue(is_url_safe("https://example.com/test", purpose="test"))
|
||||
|
||||
def test_is_url_safe_false(self):
|
||||
"""不安全 URL 应该返回 False."""
|
||||
self.assertFalse(is_url_safe("http://127.0.0.1/test", purpose="test"))
|
||||
|
||||
def test_is_url_safe_empty(self):
|
||||
"""空 URL 应该返回 False."""
|
||||
self.assertFalse(is_url_safe("", purpose="test"))
|
||||
# ── validate_url_safety 基础校验 ─────────────────────────────────────────────
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
class TestValidateUrlSafetyBasics:
|
||||
"""URL 安全校验基础测试"""
|
||||
|
||||
def test_valid_http_url(self):
|
||||
url = "http://example.com/file.mp4"
|
||||
result = validate_url_safety(url)
|
||||
assert result == url
|
||||
|
||||
def test_valid_https_url(self):
|
||||
url = "https://example.com/file.mp4"
|
||||
result = validate_url_safety(url)
|
||||
assert result == url
|
||||
|
||||
def test_empty_url_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="为空"):
|
||||
validate_url_safety("")
|
||||
|
||||
def test_none_url_raises(self):
|
||||
with pytest.raises(UrlSecurityError):
|
||||
validate_url_safety(None)
|
||||
|
||||
def test_url_too_long_raises(self):
|
||||
long_url = "https://example.com/" + "a" * 2050
|
||||
with pytest.raises(UrlSecurityError, match="过长"):
|
||||
validate_url_safety(long_url)
|
||||
|
||||
def test_url_at_max_length_ok(self):
|
||||
base = "https://example.com/"
|
||||
pad = "a" * (MAX_URL_LENGTH - len(base))
|
||||
url = base + pad
|
||||
assert len(url) <= MAX_URL_LENGTH
|
||||
result = validate_url_safety(url)
|
||||
assert result == url
|
||||
|
||||
def test_invalid_scheme_ftp_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="scheme"):
|
||||
validate_url_safety("ftp://example.com/file")
|
||||
|
||||
def test_invalid_scheme_file_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="scheme"):
|
||||
validate_url_safety("file:///etc/passwd")
|
||||
|
||||
def test_invalid_scheme_data_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="scheme"):
|
||||
validate_url_safety("data:text/html,<script>")
|
||||
|
||||
def test_missing_scheme_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="scheme"):
|
||||
validate_url_safety("example.com/file")
|
||||
|
||||
def test_missing_hostname_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="主机名"):
|
||||
validate_url_safety("http:///path")
|
||||
|
||||
def test_uppercase_scheme_normalized(self):
|
||||
"""HTTP/HTTPS 大写也能通过"""
|
||||
url = "HTTPS://example.com/file"
|
||||
# scheme 检查用 lower 比较
|
||||
result = validate_url_safety(url)
|
||||
assert result == url
|
||||
|
||||
def test_default_port_80_ok(self):
|
||||
url = "http://example.com:80/file"
|
||||
result = validate_url_safety(url)
|
||||
assert result == url
|
||||
|
||||
def test_default_port_443_ok(self):
|
||||
url = "https://example.com:443/file"
|
||||
result = validate_url_safety(url)
|
||||
assert result == url
|
||||
|
||||
def test_non_standard_port_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="端口"):
|
||||
validate_url_safety("http://example.com:8080/file")
|
||||
|
||||
def test_port_22_ssh_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="端口"):
|
||||
validate_url_safety("http://example.com:22/file")
|
||||
|
||||
def test_port_3306_mysql_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="端口"):
|
||||
validate_url_safety("http://example.com:3306/file")
|
||||
|
||||
|
||||
class TestSafeDownload(unittest.TestCase):
|
||||
"""安全下载函数测试."""
|
||||
# ── 内网主机名 / SSRF 防护 ───────────────────────────────────────────────────
|
||||
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.temp_dir, ignore_errors=True)
|
||||
class TestInternalHostnameProtection:
|
||||
"""内网主机名防护测试"""
|
||||
|
||||
def test_safe_download_file_rejects_ssrf(self):
|
||||
"""SSRF 风险 URL 应该被拒绝下载."""
|
||||
dest = os.path.join(self.temp_dir, "test.bin")
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
safe_download_file("http://127.0.0.1/test", dest, purpose="test")
|
||||
def test_localhost_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="内部主机名"):
|
||||
validate_url_safety("http://localhost/file")
|
||||
|
||||
def test_safe_download_bytes_rejects_ssrf(self):
|
||||
"""SSRF 风险 URL 应该被拒绝下载(bytes 版本)."""
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
safe_download_bytes("http://localhost/test", purpose="test")
|
||||
def test_localhost_mixed_case_raises(self):
|
||||
with pytest.raises(UrlSecurityError):
|
||||
validate_url_safety("http://LocalHost/file")
|
||||
|
||||
def test_safe_download_file_size_limit(self):
|
||||
"""超过大小限制应该被拒绝."""
|
||||
# 用 mock server 测试太大的 content-length
|
||||
dest = os.path.join(self.temp_dir, "test.bin")
|
||||
# 直接验证参数:max_size=0 时任何下载都应超限
|
||||
# (这里用一个可访问的 URL 并设置极小的限制)
|
||||
# 为避免依赖外部网络,这里只测试函数参数传递
|
||||
with unittest.mock.patch("urllib.request.build_opener") as mock_opener:
|
||||
mock_resp = unittest.mock.MagicMock()
|
||||
mock_resp.headers = {"Content-Length": "1000"}
|
||||
mock_resp.read.return_value = b""
|
||||
mock_opener.return_value.open.return_value = mock_resp
|
||||
# 设置 max_size=500,content-length=1000 应被拒绝
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
safe_download_file(
|
||||
"https://example.com/test",
|
||||
def test_localhost_localdomain_raises(self):
|
||||
with pytest.raises(UrlSecurityError):
|
||||
_check_internal_hostnames("localhost.localdomain")
|
||||
|
||||
def test_metadata_hostname_raises(self):
|
||||
with pytest.raises(UrlSecurityError):
|
||||
_check_internal_hostnames("metadata")
|
||||
|
||||
def test_metadata_google_internal_raises(self):
|
||||
with pytest.raises(UrlSecurityError):
|
||||
_check_internal_hostnames("metadata.google.internal")
|
||||
|
||||
def test_dot_local_domain_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="内网域名"):
|
||||
validate_url_safety("http://myservice.local/file")
|
||||
|
||||
def test_dot_internal_domain_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="内网域名"):
|
||||
validate_url_safety("http://myservice.internal/file")
|
||||
|
||||
def test_dot_localdomain_raises(self):
|
||||
with pytest.raises(UrlSecurityError):
|
||||
_check_internal_hostnames("server.localdomain")
|
||||
|
||||
def test_loopback_ip_127_0_0_1_raises(self):
|
||||
with pytest.raises(UrlSecurityError, match="直接 IP|回环"):
|
||||
validate_url_safety("http://127.0.0.1/file")
|
||||
|
||||
def test_metadata_ip_169_254_raises(self):
|
||||
"""云元数据服务 IP"""
|
||||
with pytest.raises(UrlSecurityError):
|
||||
validate_url_safety("http://169.254.169.254/latest/meta-data/")
|
||||
|
||||
def test_private_ip_10_raises(self):
|
||||
with pytest.raises(UrlSecurityError):
|
||||
validate_url_safety("http://10.0.0.1/file")
|
||||
|
||||
def test_private_ip_172_16_raises(self):
|
||||
with pytest.raises(UrlSecurityError):
|
||||
validate_url_safety("http://172.16.0.1/file")
|
||||
|
||||
def test_private_ip_192_168_raises(self):
|
||||
with pytest.raises(UrlSecurityError):
|
||||
validate_url_safety("http://192.168.1.1/file")
|
||||
|
||||
def test_unspecified_ip_0_0_0_0_raises(self):
|
||||
with pytest.raises(UrlSecurityError):
|
||||
validate_url_safety("http://0.0.0.0/file")
|
||||
|
||||
def test_ipv6_loopback_raises(self):
|
||||
with pytest.raises(UrlSecurityError):
|
||||
validate_url_safety("http://[::1]/file")
|
||||
|
||||
def test_public_ip_ok(self):
|
||||
"""公网IP在ALLOW_DIRECT_IP默认关闭时应被拦截"""
|
||||
# 默认 ALLOW_DIRECT_IP = false
|
||||
with pytest.raises(UrlSecurityError, match="直接 IP"):
|
||||
validate_url_safety("http://8.8.8.8/file")
|
||||
|
||||
|
||||
# ── 可信域名白名单 ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestTrustedDomains:
|
||||
"""可信域名白名单测试"""
|
||||
|
||||
def test_is_trusted_domain_exact_match(self):
|
||||
with patch("packages.shared.url_security.TRUSTED_DOMAINS", {"example.com", "cdn.example.org"}):
|
||||
# 重新加载模块以应用环境变量不太现实,直接测函数
|
||||
# 直接改全局状态再还原
|
||||
import packages.shared.url_security as mod
|
||||
from packages.shared.url_security import _is_trusted_domain
|
||||
|
||||
original = mod.TRUSTED_DOMAINS
|
||||
mod.TRUSTED_DOMAINS = {"example.com", "cdn.example.org"}
|
||||
try:
|
||||
assert _is_trusted_domain("example.com") is True
|
||||
assert _is_trusted_domain("cdn.example.org") is True
|
||||
finally:
|
||||
mod.TRUSTED_DOMAINS = original
|
||||
|
||||
def test_is_trusted_domain_subdomain(self):
|
||||
import packages.shared.url_security as mod
|
||||
|
||||
original = mod.TRUSTED_DOMAINS
|
||||
mod.TRUSTED_DOMAINS = {"example.com"}
|
||||
try:
|
||||
assert mod._is_trusted_domain("sub.example.com") is True
|
||||
assert mod._is_trusted_domain("a.b.example.com") is True
|
||||
finally:
|
||||
mod.TRUSTED_DOMAINS = original
|
||||
|
||||
def test_is_trusted_domain_no_match(self):
|
||||
import packages.shared.url_security as mod
|
||||
|
||||
original = mod.TRUSTED_DOMAINS
|
||||
mod.TRUSTED_DOMAINS = {"example.com"}
|
||||
try:
|
||||
assert mod._is_trusted_domain("other.com") is False
|
||||
assert mod._is_trusted_domain("notexample.com") is False
|
||||
finally:
|
||||
mod.TRUSTED_DOMAINS = original
|
||||
|
||||
def test_validate_with_trusted_domains_restricted(self):
|
||||
"""白名单非空时,不在白名单中的域名被拒"""
|
||||
import packages.shared.url_security as mod
|
||||
|
||||
original = mod.TRUSTED_DOMAINS
|
||||
mod.TRUSTED_DOMAINS = {"trusted.com"}
|
||||
try:
|
||||
# 不在白名单中 - 在 _is_trusted_domain 检查时就被拒,不走 DNS
|
||||
with pytest.raises(UrlSecurityError, match="白名单"):
|
||||
validate_url_safety("https://untrusted.com/file")
|
||||
|
||||
# 在白名单中 - 需要 mock DNS 解析避免实际网络请求
|
||||
with patch("packages.shared.url_security._check_ssrf_domain"):
|
||||
result = validate_url_safety("https://trusted.com/file")
|
||||
assert result == "https://trusted.com/file"
|
||||
# 子域名
|
||||
result = validate_url_safety("https://sub.trusted.com/file")
|
||||
assert result == "https://sub.trusted.com/file"
|
||||
finally:
|
||||
mod.TRUSTED_DOMAINS = original
|
||||
|
||||
|
||||
# ── is_url_safe 便捷函数 ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestIsUrlSafe:
|
||||
"""is_url_safe 便捷函数测试"""
|
||||
|
||||
def test_safe_url_returns_true(self):
|
||||
assert is_url_safe("https://example.com/file") is True
|
||||
|
||||
def test_unsafe_url_returns_false(self):
|
||||
assert is_url_safe("http://localhost/file") is False
|
||||
|
||||
def test_empty_url_returns_false(self):
|
||||
assert is_url_safe("") is False
|
||||
|
||||
def test_invalid_scheme_returns_false(self):
|
||||
assert is_url_safe("ftp://example.com/file") is False
|
||||
|
||||
|
||||
# ── UrlSecurityError 异常类 ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestUrlSecurityError:
|
||||
"""UrlSecurityError 异常类测试"""
|
||||
|
||||
def test_is_value_error_subclass(self):
|
||||
assert issubclass(UrlSecurityError, ValueError)
|
||||
|
||||
def test_error_message(self):
|
||||
err = UrlSecurityError("test message")
|
||||
assert str(err) == "test message"
|
||||
|
||||
|
||||
# ── NoRedirectHandler ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestNoRedirectHandler:
|
||||
"""NoRedirectHandler 测试"""
|
||||
|
||||
def test_redirect_request_returns_none(self):
|
||||
handler = NoRedirectHandler()
|
||||
result = handler.redirect_request(
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
302,
|
||||
"Found",
|
||||
{"Location": "http://other.com"},
|
||||
"http://other.com",
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
# ── 魔数校验 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestMagicNumberValidation:
|
||||
"""文件魔数校验测试"""
|
||||
|
||||
def test_valid_png(self, tmp_path):
|
||||
f = tmp_path / "test.png"
|
||||
# PNG 文件头: 89 50 4E 47 0D 0A 1A 0A
|
||||
f.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 100)
|
||||
# 不抛异常 = 通过
|
||||
_validate_magic_number(str(f), ALLOWED_IMAGE_MIME_TYPES)
|
||||
|
||||
def test_valid_jpeg(self, tmp_path):
|
||||
f = tmp_path / "test.jpg"
|
||||
# JPEG 文件头: FF D8 FF
|
||||
f.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 100)
|
||||
_validate_magic_number(str(f), ALLOWED_IMAGE_MIME_TYPES)
|
||||
|
||||
def test_valid_gif87a(self, tmp_path):
|
||||
f = tmp_path / "test.gif"
|
||||
f.write_bytes(b"GIF87a" + b"\x00" * 100)
|
||||
_validate_magic_number(str(f), ALLOWED_IMAGE_MIME_TYPES)
|
||||
|
||||
def test_valid_gif89a(self, tmp_path):
|
||||
f = tmp_path / "test.gif"
|
||||
f.write_bytes(b"GIF89a" + b"\x00" * 100)
|
||||
_validate_magic_number(str(f), ALLOWED_IMAGE_MIME_TYPES)
|
||||
|
||||
def test_valid_webp(self, tmp_path):
|
||||
f = tmp_path / "test.webp"
|
||||
# RIFF....WEBP
|
||||
data = bytearray(b"RIFF")
|
||||
data += b"\x00\x00\x00\x00" # size placeholder
|
||||
data += b"WEBP"
|
||||
data += b"\x00" * 100
|
||||
f.write_bytes(bytes(data))
|
||||
_validate_magic_number(str(f), ALLOWED_IMAGE_MIME_TYPES)
|
||||
|
||||
def test_valid_bmp(self, tmp_path):
|
||||
f = tmp_path / "test.bmp"
|
||||
f.write_bytes(b"BM" + b"\x00" * 100)
|
||||
_validate_magic_number(str(f), ALLOWED_IMAGE_MIME_TYPES)
|
||||
|
||||
def test_valid_wav(self, tmp_path):
|
||||
f = tmp_path / "test.wav"
|
||||
# RIFF....WAVE
|
||||
data = bytearray(b"RIFF")
|
||||
data += b"\x00\x00\x00\x00"
|
||||
data += b"WAVE"
|
||||
data += b"\x00" * 100
|
||||
f.write_bytes(bytes(data))
|
||||
_validate_magic_number(str(f), ALLOWED_AUDIO_MIME_TYPES)
|
||||
|
||||
def test_valid_mp3_id3(self, tmp_path):
|
||||
f = tmp_path / "test.mp3"
|
||||
f.write_bytes(b"ID3\x03\x00\x00\x00\x00\x00\x00" + b"\x00" * 100)
|
||||
_validate_magic_number(str(f), ALLOWED_AUDIO_MIME_TYPES)
|
||||
|
||||
def test_valid_mp3_adts(self, tmp_path):
|
||||
f = tmp_path / "test.mp3"
|
||||
f.write_bytes(b"\xff\xfb\x90\x00" + b"\x00" * 100)
|
||||
_validate_magic_number(str(f), ALLOWED_AUDIO_MIME_TYPES)
|
||||
|
||||
def test_valid_ogg(self, tmp_path):
|
||||
f = tmp_path / "test.ogg"
|
||||
f.write_bytes(b"OggS\x00\x02\x00\x00" + b"\x00" * 100)
|
||||
_validate_magic_number(str(f), ALLOWED_AUDIO_MIME_TYPES)
|
||||
|
||||
def test_valid_flac(self, tmp_path):
|
||||
f = tmp_path / "test.flac"
|
||||
f.write_bytes(b"fLaC" + b"\x00" * 100)
|
||||
_validate_magic_number(str(f), ALLOWED_AUDIO_MIME_TYPES)
|
||||
|
||||
def test_invalid_file_content_raises(self, tmp_path):
|
||||
f = tmp_path / "test.bin"
|
||||
f.write_bytes(b"this is not an image file at all")
|
||||
with pytest.raises(UrlSecurityError, match="魔数"):
|
||||
_validate_magic_number(str(f), ALLOWED_IMAGE_MIME_TYPES)
|
||||
|
||||
def test_empty_file_raises(self, tmp_path):
|
||||
f = tmp_path / "empty.bin"
|
||||
f.write_bytes(b"")
|
||||
with pytest.raises(UrlSecurityError, match="为空"):
|
||||
_validate_magic_number(str(f), ALLOWED_IMAGE_MIME_TYPES)
|
||||
|
||||
def test_nonexistent_file_raises(self, tmp_path):
|
||||
with pytest.raises(UrlSecurityError, match="读取文件头失败"):
|
||||
_validate_magic_number(str(tmp_path / "no_such_file"), ALLOWED_IMAGE_MIME_TYPES)
|
||||
|
||||
def test_no_allowed_mime_types_skips(self, tmp_path):
|
||||
"""allowed_mime_types 为空时跳过校验"""
|
||||
f = tmp_path / "test.bin"
|
||||
f.write_bytes(b"random data here")
|
||||
# 不抛异常
|
||||
_validate_magic_number(str(f), set())
|
||||
|
||||
def test_unknown_mime_types_skips(self, tmp_path):
|
||||
"""没有已知魔数的 MIME 类型跳过校验"""
|
||||
f = tmp_path / "test.bin"
|
||||
f.write_bytes(b"random data")
|
||||
_validate_magic_number(str(f), {"application/x-unknown-type"})
|
||||
|
||||
|
||||
# ── safe_download_file (mock 网络) ───────────────────────────────────────────
|
||||
|
||||
|
||||
class TestSafeDownloadFile:
|
||||
"""safe_download_file 下载测试(mock 网络)"""
|
||||
|
||||
def test_download_success(self, tmp_path):
|
||||
test_content = b"Hello, this is test file content!"
|
||||
dest = str(tmp_path / "output.bin")
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.headers = {"Content-Type": "application/octet-stream"}
|
||||
mock_resp.read.side_effect = [test_content, b""]
|
||||
|
||||
with patch("packages.shared.url_security.NoRedirectHandler") as mock_handler_cls:
|
||||
mock_handler = MagicMock()
|
||||
mock_handler_cls.return_value = mock_handler
|
||||
|
||||
mock_opener = MagicMock()
|
||||
mock_opener.open.return_value = mock_resp
|
||||
|
||||
with patch("urllib.request.build_opener", return_value=mock_opener):
|
||||
size = safe_download_file(
|
||||
"https://example.com/test.bin",
|
||||
dest,
|
||||
purpose="test",
|
||||
max_size=500,
|
||||
)
|
||||
|
||||
def test_safe_download_file_mime_rejected(self):
|
||||
"""不允许的 MIME 类型应该被拒绝."""
|
||||
dest = os.path.join(self.temp_dir, "test.bin")
|
||||
with unittest.mock.patch("urllib.request.build_opener") as mock_opener:
|
||||
mock_resp = unittest.mock.MagicMock()
|
||||
mock_resp.headers = {"Content-Type": "text/html"}
|
||||
mock_resp.read.return_value = b""
|
||||
mock_opener.return_value.open.return_value = mock_resp
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
safe_download_file(
|
||||
"https://example.com/test.mp3",
|
||||
dest,
|
||||
purpose="test",
|
||||
allowed_mime_types=ALLOWED_AUDIO_MIME_TYPES,
|
||||
)
|
||||
assert size == len(test_content)
|
||||
with open(dest, "rb") as f:
|
||||
assert f.read() == test_content
|
||||
|
||||
def test_download_with_mime_check_passes(self, tmp_path):
|
||||
# PNG 文件
|
||||
test_content = b"\x89PNG\r\n\x1a\n" + b"\x00" * 200
|
||||
dest = str(tmp_path / "test.png")
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.headers = {"Content-Type": "image/png"}
|
||||
mock_resp.read.side_effect = [test_content, b""]
|
||||
|
||||
with patch("urllib.request.build_opener") as mock_build:
|
||||
mock_opener = MagicMock()
|
||||
mock_opener.open.return_value = mock_resp
|
||||
mock_build.return_value = mock_opener
|
||||
|
||||
def test_safe_download_file_mime_allowed(self):
|
||||
"""允许的 MIME 类型应该通过."""
|
||||
dest = os.path.join(self.temp_dir, "test.mp3")
|
||||
with unittest.mock.patch("urllib.request.build_opener") as mock_opener:
|
||||
mock_resp = unittest.mock.MagicMock()
|
||||
mock_resp.headers = {"Content-Type": "audio/mpeg"}
|
||||
mock_resp.read.side_effect = [b"ID3audio_data", b""]
|
||||
mock_resp.geturl.return_value = "https://example.com/test.mp3"
|
||||
mock_opener.return_value.open.return_value = mock_resp
|
||||
size = safe_download_file(
|
||||
"https://example.com/test.mp3",
|
||||
"https://example.com/test.png",
|
||||
dest,
|
||||
purpose="test",
|
||||
allowed_mime_types=ALLOWED_AUDIO_MIME_TYPES,
|
||||
allowed_mime_types={"image/png", "image/jpeg"},
|
||||
)
|
||||
self.assertEqual(size, 13)
|
||||
self.assertTrue(os.path.exists(dest))
|
||||
|
||||
def test_safe_download_file_stream_size_limit(self):
|
||||
"""流式下载时超过大小限制应该中断."""
|
||||
dest = os.path.join(self.temp_dir, "test.bin")
|
||||
with unittest.mock.patch("urllib.request.build_opener") as mock_opener:
|
||||
mock_resp = unittest.mock.MagicMock()
|
||||
mock_resp.headers = {}
|
||||
# 每次返回 100 字节,max_size=500,第 6 次读取就超限
|
||||
mock_resp.read.side_effect = lambda n: b"x" * n if n < 1000 else b"x" * 100
|
||||
# 改成返回固定 100 字节,直到第 N 次后返回空
|
||||
call_count = [0]
|
||||
assert size == len(test_content)
|
||||
|
||||
def mock_read(size):
|
||||
call_count[0] += 1
|
||||
if call_count[0] > 10:
|
||||
return b""
|
||||
return b"x" * 100
|
||||
def test_download_mime_type_rejected(self, tmp_path):
|
||||
test_content = b"GIF89a" + b"\x00" * 50
|
||||
dest = str(tmp_path / "test.gif")
|
||||
|
||||
mock_resp.read = mock_read
|
||||
mock_opener.return_value.open.return_value = mock_resp
|
||||
with self.assertRaises(UrlSecurityError):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.headers = {"Content-Type": "image/gif"}
|
||||
mock_resp.read.side_effect = [test_content, b""]
|
||||
|
||||
with patch("urllib.request.build_opener") as mock_build:
|
||||
mock_opener = MagicMock()
|
||||
mock_opener.open.return_value = mock_resp
|
||||
mock_build.return_value = mock_opener
|
||||
|
||||
with pytest.raises(UrlSecurityError, match="Content-Type"):
|
||||
safe_download_file(
|
||||
"https://example.com/test",
|
||||
"https://example.com/test.gif",
|
||||
dest,
|
||||
purpose="test",
|
||||
max_size=500, # 500 字节上限
|
||||
allowed_mime_types={"image/png"},
|
||||
)
|
||||
|
||||
def test_safe_download_bytes_returns_content(self):
|
||||
"""safe_download_bytes 应该返回文件内容."""
|
||||
test_data = b"ID3hello world test audio"
|
||||
with unittest.mock.patch("urllib.request.build_opener") as mock_opener:
|
||||
mock_resp = unittest.mock.MagicMock()
|
||||
mock_resp.headers = {"Content-Type": "audio/mpeg"}
|
||||
call_count = [0]
|
||||
def test_download_size_limit_exceeded(self, tmp_path):
|
||||
"""流式下载时超过大小限制被中断(无 Content-Length header)"""
|
||||
dest = str(tmp_path / "big.bin")
|
||||
chunk = b"x" * 1024 # 1KB chunks
|
||||
|
||||
def mock_read(size):
|
||||
call_count[0] += 1
|
||||
if call_count[0] > 1:
|
||||
return b""
|
||||
return test_data
|
||||
mock_resp = MagicMock()
|
||||
# 没有 Content-Length header,走流式检查
|
||||
mock_resp.headers = {"Content-Type": "application/octet-stream"}
|
||||
# 模拟多次读取,超过 5KB 限制(6个chunk = 6KB)
|
||||
mock_resp.read.side_effect = [chunk] * 6 + [b""]
|
||||
|
||||
with patch("urllib.request.build_opener") as mock_build:
|
||||
mock_opener = MagicMock()
|
||||
mock_opener.open.return_value = mock_resp
|
||||
mock_build.return_value = mock_opener
|
||||
|
||||
with pytest.raises(UrlSecurityError, match="超过大小限制"):
|
||||
safe_download_file(
|
||||
"https://example.com/big.bin",
|
||||
dest,
|
||||
purpose="test",
|
||||
max_size=5000, # 5KB limit
|
||||
)
|
||||
|
||||
def test_download_content_length_too_large(self, tmp_path):
|
||||
dest = str(tmp_path / "big.bin")
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.headers = {"Content-Type": "application/octet-stream", "Content-Length": "1000000"}
|
||||
mock_resp.read.side_effect = [b"data"]
|
||||
|
||||
with patch("urllib.request.build_opener") as mock_build:
|
||||
mock_opener = MagicMock()
|
||||
mock_opener.open.return_value = mock_resp
|
||||
mock_build.return_value = mock_opener
|
||||
|
||||
with pytest.raises(UrlSecurityError, match="文件过大"):
|
||||
safe_download_file(
|
||||
"https://example.com/big.bin",
|
||||
dest,
|
||||
purpose="test",
|
||||
max_size=500000,
|
||||
)
|
||||
|
||||
def test_download_localhost_rejected(self, tmp_path):
|
||||
"""内网 URL 在下载前就被拒"""
|
||||
dest = str(tmp_path / "out.bin")
|
||||
with pytest.raises(UrlSecurityError):
|
||||
safe_download_file("http://localhost/file", dest)
|
||||
|
||||
def test_download_invalid_scheme_rejected(self, tmp_path):
|
||||
dest = str(tmp_path / "out.bin")
|
||||
with pytest.raises(UrlSecurityError, match="scheme"):
|
||||
safe_download_file("ftp://example.com/file", dest)
|
||||
|
||||
|
||||
# ── safe_download_bytes ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestSafeDownloadBytes:
|
||||
"""safe_download_bytes 测试"""
|
||||
|
||||
def test_download_returns_bytes(self, tmp_path):
|
||||
test_content = b"hello bytes download test"
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.headers = {"Content-Type": "application/octet-stream"}
|
||||
mock_resp.read.side_effect = [test_content, b""]
|
||||
|
||||
with patch("urllib.request.build_opener") as mock_build:
|
||||
mock_opener = MagicMock()
|
||||
mock_opener.open.return_value = mock_resp
|
||||
mock_build.return_value = mock_opener
|
||||
|
||||
mock_resp.read = mock_read
|
||||
mock_opener.return_value.open.return_value = mock_resp
|
||||
result = safe_download_bytes(
|
||||
"https://example.com/test.mp3",
|
||||
"https://example.com/test.bin",
|
||||
purpose="test",
|
||||
allowed_mime_types=ALLOWED_AUDIO_MIME_TYPES,
|
||||
)
|
||||
self.assertEqual(result, test_data)
|
||||
|
||||
assert result == test_content
|
||||
assert isinstance(result, bytes)
|
||||
|
||||
def test_download_unsafe_url_raises(self):
|
||||
with pytest.raises(UrlSecurityError):
|
||||
safe_download_bytes("http://127.0.0.1/secret")
|
||||
|
||||
|
||||
# ── 常量导出验证 ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestConstants:
|
||||
"""模块常量验证"""
|
||||
|
||||
def test_allowed_schemes(self):
|
||||
assert "http" in ALLOWED_SCHEMES
|
||||
assert "https" in ALLOWED_SCHEMES
|
||||
assert len(ALLOWED_SCHEMES) == 2
|
||||
|
||||
def test_allowed_ports(self):
|
||||
assert 80 in ALLOWED_PORTS
|
||||
assert 443 in ALLOWED_PORTS
|
||||
assert len(ALLOWED_PORTS) == 2
|
||||
|
||||
def test_max_url_length(self):
|
||||
assert MAX_URL_LENGTH == 2048
|
||||
|
||||
Executable
+282
@@ -0,0 +1,282 @@
|
||||
"""
|
||||
验证工具与schema守卫单元测试
|
||||
|
||||
覆盖:
|
||||
- validate_phone / normalize_phone / validate_email
|
||||
- schema_guard: normalize_environment / assert_auto_create_schema_allowed
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from packages.adapters.sqlalchemy_impl.schema_guard import (
|
||||
BLOCKED_AUTO_CREATE_ENVIRONMENTS,
|
||||
assert_auto_create_schema_allowed,
|
||||
normalize_environment,
|
||||
)
|
||||
from packages.application.auth.verification_code_service import (
|
||||
normalize_phone,
|
||||
validate_email,
|
||||
validate_phone,
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# validate_phone
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestValidatePhone:
|
||||
"""validate_phone 手机号校验"""
|
||||
|
||||
def test_valid_11digit_mainland(self):
|
||||
ok, msg = validate_phone("13812345678")
|
||||
assert ok is True
|
||||
assert msg == ""
|
||||
|
||||
def test_valid_with_plus86_prefix(self):
|
||||
ok, msg = validate_phone("+8613812345678")
|
||||
assert ok is True
|
||||
assert msg == ""
|
||||
|
||||
def test_valid_13x_prefix(self):
|
||||
ok, _ = validate_phone("13012345678")
|
||||
assert ok is True
|
||||
|
||||
def test_valid_19x_prefix(self):
|
||||
ok, _ = validate_phone("19912345678")
|
||||
assert ok is True
|
||||
|
||||
def test_valid_17x_prefix(self):
|
||||
ok, _ = validate_phone("17612345678")
|
||||
assert ok is True
|
||||
|
||||
def test_empty_returns_false(self):
|
||||
ok, msg = validate_phone("")
|
||||
assert ok is False
|
||||
assert "不能为空" in msg
|
||||
|
||||
def test_whitespace_stripped_empty(self):
|
||||
ok, msg = validate_phone(" ")
|
||||
assert ok is False
|
||||
assert "不能为空" in msg
|
||||
|
||||
def test_too_short_returns_false(self):
|
||||
ok, msg = validate_phone("138123456") # 9位
|
||||
assert ok is False
|
||||
assert "格式不正确" in msg
|
||||
|
||||
def test_too_long_returns_false(self):
|
||||
ok, msg = validate_phone("138123456789") # 12位
|
||||
assert ok is False
|
||||
assert "格式不正确" in msg
|
||||
|
||||
def test_invalid_prefix_12x(self):
|
||||
ok, _ = validate_phone("12312345678")
|
||||
assert ok is False
|
||||
|
||||
def test_invalid_prefix_11x(self):
|
||||
ok, _ = validate_phone("11012345678")
|
||||
assert ok is False
|
||||
|
||||
def test_contains_letters(self):
|
||||
ok, _ = validate_phone("138abc45678")
|
||||
assert ok is False
|
||||
|
||||
def test_contains_spaces_in_middle(self):
|
||||
ok, _ = validate_phone("138 1234 5678")
|
||||
assert ok is False
|
||||
|
||||
def test_whitespace_stripped_valid(self):
|
||||
ok, _ = validate_phone(" 13812345678 ")
|
||||
assert ok is True
|
||||
|
||||
def test_plus86_with_spaces(self):
|
||||
ok, _ = validate_phone(" +8613812345678 ")
|
||||
assert ok is True
|
||||
|
||||
|
||||
# ============================================================
|
||||
# normalize_phone
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestNormalizePhone:
|
||||
"""normalize_phone 手机号标准化"""
|
||||
|
||||
def test_plain_number_unchanged(self):
|
||||
assert normalize_phone("13812345678") == "13812345678"
|
||||
|
||||
def test_plus86_prefix_removed(self):
|
||||
assert normalize_phone("+8613812345678") == "13812345678"
|
||||
|
||||
def test_whitespace_stripped(self):
|
||||
assert normalize_phone(" 13812345678 ") == "13812345678"
|
||||
|
||||
def test_plus86_with_spaces(self):
|
||||
assert normalize_phone(" +8613900001111 ") == "13900001111"
|
||||
|
||||
def test_empty_string(self):
|
||||
assert normalize_phone("") == ""
|
||||
|
||||
def test_only_plus86(self):
|
||||
assert normalize_phone("+86") == ""
|
||||
|
||||
|
||||
# ============================================================
|
||||
# validate_email
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestValidateEmail:
|
||||
"""validate_email 邮箱校验"""
|
||||
|
||||
def test_valid_simple(self):
|
||||
ok, msg = validate_email("user@example.com")
|
||||
assert ok is True
|
||||
assert msg == ""
|
||||
|
||||
def test_valid_with_subdomain(self):
|
||||
ok, _ = validate_email("user@mail.example.com")
|
||||
assert ok is True
|
||||
|
||||
def test_valid_with_plus_suffix(self):
|
||||
ok, _ = validate_email("user+tag@example.com")
|
||||
assert ok is True
|
||||
|
||||
def test_valid_with_dots_in_local(self):
|
||||
ok, _ = validate_email("user.name@example.com")
|
||||
assert ok is True
|
||||
|
||||
def test_valid_with_underscore(self):
|
||||
ok, _ = validate_email("user_name@example.com")
|
||||
assert ok is True
|
||||
|
||||
def test_valid_with_hyphen(self):
|
||||
ok, _ = validate_email("user-name@example.com")
|
||||
assert ok is True
|
||||
|
||||
def test_valid_uppercase(self):
|
||||
ok, _ = validate_email("User.Name@Example.COM")
|
||||
assert ok is True
|
||||
|
||||
def test_empty_returns_false(self):
|
||||
ok, msg = validate_email("")
|
||||
assert ok is False
|
||||
assert "不能为空" in msg
|
||||
|
||||
def test_whitespace_only_returns_false(self):
|
||||
ok, msg = validate_email(" ")
|
||||
assert ok is False
|
||||
assert "不能为空" in msg
|
||||
|
||||
def test_no_at_sign(self):
|
||||
ok, _ = validate_email("userexample.com")
|
||||
assert ok is False
|
||||
|
||||
def test_no_domain(self):
|
||||
ok, _ = validate_email("user@")
|
||||
assert ok is False
|
||||
|
||||
def test_no_local_part(self):
|
||||
ok, _ = validate_email("@example.com")
|
||||
assert ok is False
|
||||
|
||||
def test_no_dot_in_domain(self):
|
||||
ok, _ = validate_email("user@example")
|
||||
assert ok is False
|
||||
|
||||
def test_single_letter_tld(self):
|
||||
ok, _ = validate_email("user@example.c")
|
||||
assert ok is False
|
||||
|
||||
def test_whitespace_stripped(self):
|
||||
ok, _ = validate_email(" user@example.com ")
|
||||
assert ok is True
|
||||
|
||||
def test_special_chars_percent(self):
|
||||
ok, _ = validate_email("user%name@example.com")
|
||||
assert ok is True
|
||||
|
||||
|
||||
# ============================================================
|
||||
# normalize_environment
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestNormalizeEnvironment:
|
||||
"""normalize_environment 环境名标准化"""
|
||||
|
||||
def test_development(self):
|
||||
assert normalize_environment("development") == "development"
|
||||
|
||||
def test_production(self):
|
||||
assert normalize_environment("production") == "production"
|
||||
|
||||
def test_staging(self):
|
||||
assert normalize_environment("staging") == "staging"
|
||||
|
||||
def test_none_defaults_to_development(self):
|
||||
assert normalize_environment(None) == "development"
|
||||
|
||||
def test_empty_defaults_to_development(self):
|
||||
assert normalize_environment("") == "development"
|
||||
|
||||
def test_uppercase_lowered(self):
|
||||
assert normalize_environment("PRODUCTION") == "production"
|
||||
|
||||
def test_mixed_case(self):
|
||||
assert normalize_environment("Staging") == "staging"
|
||||
|
||||
def test_whitespace_stripped(self):
|
||||
assert normalize_environment(" production ") == "production"
|
||||
|
||||
def test_custom_env_name(self):
|
||||
assert normalize_environment("custom-env") == "custom-env"
|
||||
|
||||
|
||||
# ============================================================
|
||||
# assert_auto_create_schema_allowed
|
||||
# ============================================================
|
||||
|
||||
|
||||
class TestAssertAutoCreateSchemaAllowed:
|
||||
"""assert_auto_create_schema_allowed 安全守卫"""
|
||||
|
||||
def test_development_enabled_ok(self):
|
||||
# 不抛异常
|
||||
assert_auto_create_schema_allowed("development", enabled=True)
|
||||
|
||||
def test_development_disabled_ok(self):
|
||||
assert_auto_create_schema_allowed("development", enabled=False)
|
||||
|
||||
def test_staging_enabled_raises(self):
|
||||
with pytest.raises(RuntimeError, match="forbidden"):
|
||||
assert_auto_create_schema_allowed("staging", enabled=True)
|
||||
|
||||
def test_production_enabled_raises(self):
|
||||
with pytest.raises(RuntimeError, match="forbidden"):
|
||||
assert_auto_create_schema_allowed("production", enabled=True)
|
||||
|
||||
def test_staging_disabled_ok(self):
|
||||
assert_auto_create_schema_allowed("staging", enabled=False)
|
||||
|
||||
def test_production_disabled_ok(self):
|
||||
assert_auto_create_schema_allowed("production", enabled=False)
|
||||
|
||||
def test_none_env_enabled_ok(self):
|
||||
# None → development → 允许
|
||||
assert_auto_create_schema_allowed(None, enabled=True)
|
||||
|
||||
def test_uppercase_still_blocked(self):
|
||||
with pytest.raises(RuntimeError):
|
||||
assert_auto_create_schema_allowed("PRODUCTION", enabled=True)
|
||||
|
||||
def test_whitespace_staging_blocked(self):
|
||||
with pytest.raises(RuntimeError):
|
||||
assert_auto_create_schema_allowed(" staging ", enabled=True)
|
||||
|
||||
def test_custom_env_enabled_ok(self):
|
||||
assert_auto_create_schema_allowed("test-env", enabled=True)
|
||||
|
||||
def test_blocked_envs_constant(self):
|
||||
assert "staging" in BLOCKED_AUTO_CREATE_ENVIRONMENTS
|
||||
assert "production" in BLOCKED_AUTO_CREATE_ENVIRONMENTS
|
||||
Reference in New Issue
Block a user