From 6eac0b2cf2cffdf615dba02e944c796ae749ca7d Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Fri, 24 Jul 2026 10:36:29 +0800 Subject: [PATCH] =?UTF-8?q?chore(ci):=20=E5=90=8C=E6=AD=A5main=E5=88=86?= =?UTF-8?q?=E6=94=AFCI=E9=85=8D=E7=BD=AE=E4=B8=8Escripts/ci=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=20-=20=E4=B8=8Edevelop=E5=AF=B9=E9=BD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 同步内容： 1. CI流水线配置（ci-pipeline.yml）与develop对齐 2. PR构建脚本docker_build_only.sh增加buildx→docker build回退 3. pre-build步骤worker基础镜像构建增加buildx回退 4. 单元测试脚本全量覆盖率改为仅报告不阻塞 5. diff-cover依赖加入requirements-dev.txt 6. worker base builder/runtime Dockerfile同步 7. test_config_oss.py clear=False→clear=True修复OSS污染 8. Frontend Lint增加prettier依赖 --- .gitea/workflows/auto-approve.yml | 165 -- .gitea/workflows/auto-merge.yml | 131 -- .gitea/workflows/ci-cd.yml | 657 -------- .gitea/workflows/ci-failure-monitor.yml | 78 + .gitea/workflows/ci-health-daily.yml | 103 ++ .gitea/workflows/ci-pipeline.yml | 1573 +++++++++++++++++++ .gitea/workflows/ci-trigger-monitor.yml | 52 + .gitea/workflows/code-review.yml | 79 + .gitea/workflows/daily-check.yml | 718 +++++++++ .gitea/workflows/pr-auto-scan.yml | 56 + .gitea/workflows/pr-automation.yml | 113 ++ .gitea/workflows/preview-cleanup.yml | 207 +++ .gitea/workflows/preview-deploy.yml | 296 ++++ .gitea/workflows/tests.yml | 167 -- .gitea/workflows/worker-base-image.yml | 103 ++ apps/web/package-lock.json | 434 +---- apps/web/package.json | 3 +- infra/docker/worker-base-builder.Dockerfile | 43 + infra/docker/worker-base-runtime.Dockerfile | 17 + requirements-dev.txt | 1 + scripts/ci/acr_cleanup.py | 426 +++++ scripts/ci/auto_approve.sh | 156 ++ scripts/ci/auto_fix_formatting.py | 397 +++++ scripts/ci/auto_merge.sh | 149 ++ scripts/ci/chatops/__init__.py | 19 + scripts/ci/chatops/ci_query.py | 296 ++++ scripts/ci/chatops/ci_trigger.py | 169 ++ scripts/ci/chatops/config.py | 55 + scripts/ci/chatops/feishu_notify.py | 390 +++++ scripts/ci/chatops/gitea_client.py | 243 +++ scripts/ci/chatops/webhook_server.py | 446 ++++++ scripts/ci/check_migration_chain.py | 174 ++ scripts/ci/ci_dashboard.py | 973 ++++++++++++ scripts/ci/ci_failure_diagnosis.py | 447 ++++++ scripts/ci/ci_health_check.py | 298 ++++ scripts/ci/ci_health_report.py | 251 +++ scripts/ci/ci_repeated_failure_detector.py | 417 +++++ scripts/ci/ci_trace_report.py | 375 +++++ scripts/ci/docker_build_only.sh | 82 + scripts/ci/docker_build_push.sh | 106 ++ scripts/ci/generate_ci_dashboard.sh | 97 ++ scripts/ci/mypy_check.sh | 48 + scripts/ci/pr_auto_scan.py | 408 +++++ scripts/ci/preview_comment.py | 54 + scripts/ci/preview_init_server.sh | 264 ++++ scripts/ci/preview_nginx.conf.template | 226 +++ scripts/ci/run_integration_tests.sh | 342 ++++ scripts/ci/run_unit_tests.sh | 157 ++ scripts/ci/run_validate.sh | 653 ++++++++ scripts/ci/runner_monitor/__init__.py | 17 + scripts/ci/runner_monitor/alert_manager.py | 492 ++++++ scripts/ci/runner_monitor/config.py | 87 + scripts/ci/runner_monitor/runner_metrics.py | 82 + scripts/ci/runner_monitor/runner_status.py | 320 ++++ scripts/ci/select_unit_tests.py | 226 +++ scripts/ci/step_checkout.sh | 44 + scripts/ci/step_frontend_install.sh | 18 + scripts/ci/step_frontend_run.sh | 9 + scripts/ci/step_install_ffmpeg.sh | 24 + scripts/ci/step_timer_end.sh | 14 + scripts/ci/step_timer_start.sh | 6 + scripts/ci/validate_code_quality.sh | 234 +++ scripts/ci/validate_migration.sh | 183 +++ scripts/ci/validate_mypy.sh | 10 + scripts/ci/vitest_incremental.sh | 70 + tests/conftest.py | 39 + tests/unit/test_config_oss.py | 4 +- 67 files changed, 13455 insertions(+), 1538 deletions(-) delete mode 100644 .gitea/workflows/auto-approve.yml delete mode 100644 .gitea/workflows/auto-merge.yml delete mode 100755 .gitea/workflows/ci-cd.yml create mode 100644 .gitea/workflows/ci-failure-monitor.yml create mode 100644 .gitea/workflows/ci-health-daily.yml create mode 100755 .gitea/workflows/ci-pipeline.yml create mode 100644 .gitea/workflows/ci-trigger-monitor.yml create mode 100644 .gitea/workflows/code-review.yml create mode 100644 .gitea/workflows/daily-check.yml create mode 100644 .gitea/workflows/pr-auto-scan.yml create mode 100755 .gitea/workflows/pr-automation.yml create mode 100755 .gitea/workflows/preview-cleanup.yml create mode 100755 .gitea/workflows/preview-deploy.yml delete mode 100755 .gitea/workflows/tests.yml create mode 100644 .gitea/workflows/worker-base-image.yml create mode 100644 infra/docker/worker-base-builder.Dockerfile create mode 100644 infra/docker/worker-base-runtime.Dockerfile create mode 100644 scripts/ci/acr_cleanup.py create mode 100644 scripts/ci/auto_approve.sh create mode 100755 scripts/ci/auto_fix_formatting.py create mode 100644 scripts/ci/auto_merge.sh create mode 100755 scripts/ci/chatops/__init__.py create mode 100755 scripts/ci/chatops/ci_query.py create mode 100755 scripts/ci/chatops/ci_trigger.py create mode 100755 scripts/ci/chatops/config.py create mode 100755 scripts/ci/chatops/feishu_notify.py create mode 100755 scripts/ci/chatops/gitea_client.py create mode 100755 scripts/ci/chatops/webhook_server.py create mode 100755 scripts/ci/check_migration_chain.py create mode 100644 scripts/ci/ci_dashboard.py create mode 100644 scripts/ci/ci_failure_diagnosis.py create mode 100644 scripts/ci/ci_health_check.py create mode 100644 scripts/ci/ci_health_report.py create mode 100644 scripts/ci/ci_repeated_failure_detector.py create mode 100755 scripts/ci/ci_trace_report.py create mode 100755 scripts/ci/docker_build_only.sh create mode 100755 scripts/ci/docker_build_push.sh create mode 100644 scripts/ci/generate_ci_dashboard.sh create mode 100755 scripts/ci/mypy_check.sh create mode 100644 scripts/ci/pr_auto_scan.py create mode 100755 scripts/ci/preview_comment.py create mode 100755 scripts/ci/preview_init_server.sh create mode 100644 scripts/ci/preview_nginx.conf.template create mode 100755 scripts/ci/run_integration_tests.sh create mode 100755 scripts/ci/run_unit_tests.sh create mode 100755 scripts/ci/run_validate.sh create mode 100755 scripts/ci/runner_monitor/__init__.py create mode 100644 scripts/ci/runner_monitor/alert_manager.py create mode 100755 scripts/ci/runner_monitor/config.py create mode 100755 scripts/ci/runner_monitor/runner_metrics.py create mode 100755 scripts/ci/runner_monitor/runner_status.py create mode 100644 scripts/ci/select_unit_tests.py create mode 100755 scripts/ci/step_checkout.sh create mode 100755 scripts/ci/step_frontend_install.sh create mode 100755 scripts/ci/step_frontend_run.sh create mode 100755 scripts/ci/step_install_ffmpeg.sh create mode 100755 scripts/ci/step_timer_end.sh create mode 100755 scripts/ci/step_timer_start.sh create mode 100644 scripts/ci/validate_code_quality.sh create mode 100644 scripts/ci/validate_migration.sh create mode 100644 scripts/ci/validate_mypy.sh create mode 100755 scripts/ci/vitest_incremental.sh diff --git a/.gitea/workflows/auto-approve.yml b/.gitea/workflows/auto-approve.yml deleted file mode 100644 index 0c6fe36de..000000000 --- a/.gitea/workflows/auto-approve.yml +++ /dev/null @@ -1,165 +0,0 @@ -name: Auto Approve CI PRs - -on: - pull_request: - types: [synchronize, opened, ready_for_review] - -jobs: - auto-approve: - name: Auto Approve on CI Green - runs-on: ci-l1 - if: github.event_name == 'pull_request' && !github.event.pull_request.draft - timeout-minutes: 20 - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Auto approve when CI passes - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - REVIEW_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - set -eu - - echo "PR #${PR_NUMBER} - 检查CI状态并自动审批" - - # 检查是否纯前端改动 - API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300" - FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin)]") - FRONTEND_COUNT=$(echo "$FILES" | grep -c '^apps/web/' || true) - BACKEND_COUNT=$(echo "$FILES" | grep -cv '^apps/web/' || true) - TOTAL=$(echo "$FILES" | grep -cv '^$' || true) - echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT})" - - if [ "$BACKEND_COUNT" = "0" ] && [ "$FRONTEND_COUNT" -gt "0" ]; then - SKIP_BACKEND=true - echo "✅ 纯前端改动，只检查Frontend Lint" - else - SKIP_BACKEND=false - echo "🔧 包含后端/公共变更，检查全部CI" - fi - - # 定义需要检查的context - # 根据目标分支决定检查哪些门禁 - TARGET_BRANCH="${GITHUB_BASE_REF}" - echo "目标分支: ${TARGET_BRANCH}" - - if [ "$SKIP_BACKEND" = "true" ]; then - CONTEXTS=("CI/CD Pipeline / Frontend Lint (pull_request)") - elif [ "$TARGET_BRANCH" = "main" ]; then - # main分支只检查required statuses: Validate + Frontend Lint - # 不检查Tests/test（不是required门禁） - CONTEXTS=( - "CI/CD Pipeline / Validate Code Quality And Tests (pull_request)" - "CI/CD Pipeline / Frontend Lint (pull_request)" - "Tests / test (pull_request)" - ) - else - CONTEXTS=( - "CI/CD Pipeline / Validate Code Quality And Tests (pull_request)" - "CI/CD Pipeline / Unit Tests (pull_request)" - "CI/CD Pipeline / Frontend Lint (pull_request)" - ) - fi - - echo "需要通过的CI检查: ${#CONTEXTS[@]} 项" - for ctx in "${CONTEXTS[@]}"; do - echo " - $ctx" - done - echo - - # 轮询等待，最多20分钟（120次x10秒） - for attempt in $(seq 1 120); do - ALL_SUCCESS=true - ANY_FAILED=false - - echo "--- 第${attempt}次检查 ($(date '+%H:%M:%S')) ---" - - # 调用辅助脚本检查每个context状态 - for ctx in "${CONTEXTS[@]}"; do - STATE=$(python3 scripts/check_ci_status.py "$GITHUB_TOKEN" "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$ctx") - echo " $ctx: $STATE" - - if [ "$STATE" != "success" ]; then - ALL_SUCCESS=false - fi - if [ "$STATE" = "failure" ] || [ "$STATE" = "error" ]; then - ANY_FAILED=true - fi - done - - if [ "$ALL_SUCCESS" = "true" ]; then - echo - echo "✅ 所有CI检查通过，自动审批 PR #${PR_NUMBER}" - - # 检查是否已有审批（任何用户的APPROVED都算，避免重复审批） - EXISTING=$(curl -s -H "Authorization: token ${REVIEW_TOKEN}" \ - "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" \ - | python3 -c "import sys,json; reviews=json.load(sys.stdin); print('yes' if any(r.get('state')=='APPROVED' for r in reviews) else 'no')") - - if [ "$EXISTING" = "yes" ]; then - echo "ℹ️ PR #${PR_NUMBER} 已有审批，跳过" - exit 0 - fi - - # 第一步：创建PENDING review（Gitea API需要先创建再提交） - echo "创建review..." - REVIEW_CREATE=$(curl -s -X POST \ - -H "Authorization: token ${REVIEW_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{"event": "PENDING", "body": "CI全绿，自动审批通过。"}' \ - "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews") - - REVIEW_ID=$(echo "$REVIEW_CREATE" | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))") - REVIEW_STATE=$(echo "$REVIEW_CREATE" | python3 -c "import sys,json; print(json.load(sys.stdin).get('state',''))") - echo "创建结果: id=$REVIEW_ID state=$REVIEW_STATE" - - if [ -z "$REVIEW_ID" ]; then - echo "❌ 创建review失败" - echo "$REVIEW_CREATE" - exit 1 - fi - - # 如果已经是APPROVED就不用再submit了（兼容不同Gitea版本） - if [ "$REVIEW_STATE" = "APPROVED" ]; then - echo "✅ 自动审批成功（直接创建为APPROVED）" - exit 0 - fi - - # 第二步：submit review为APPROVED - echo "提交review审批..." - SUBMIT_CODE=$(curl -s -o /tmp/submit_resp.json -w "%{http_code}" \ - -X POST \ - -H "Authorization: token ${REVIEW_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{"event": "APPROVED", "body": "CI全绿，自动审批通过。"}' \ - "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews/${REVIEW_ID}") - - echo "提交API HTTP状态: $SUBMIT_CODE" - cat /tmp/submit_resp.json 2>/dev/null || true - echo - - if [ "$SUBMIT_CODE" = "200" ] || [ "$SUBMIT_CODE" = "201" ]; then - FINAL_STATE=$(python3 -c "import json; print(json.load(open('/tmp/submit_resp.json')).get('state',''))" 2>/dev/null || echo "?") - echo "✅ 自动审批成功 (state: $FINAL_STATE)" - exit 0 - else - echo "❌ 提交审批失败" - exit 1 - fi - fi - - if [ "$ANY_FAILED" = "true" ]; then - echo - echo "❌ CI检查有失败项，不自动审批" - exit 0 - fi - - sleep 10 - done - - echo - echo "⏰ 等待超时（20分钟），CI尚未全部完成" - exit 0 diff --git a/.gitea/workflows/auto-merge.yml b/.gitea/workflows/auto-merge.yml deleted file mode 100644 index 1304e5efd..000000000 --- a/.gitea/workflows/auto-merge.yml +++ /dev/null @@ -1,131 +0,0 @@ -name: Auto Merge PRs (main) - -on: - pull_request: - types: [synchronize, opened, ready_for_review, review_requested] - -jobs: - auto-merge: - name: Auto Merge on CI Green + Approved (main) - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' && !github.event.pull_request.draft && github.event.pull_request.base.ref == 'main' - timeout-minutes: 30 - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Auto merge when CI passes and approved - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - MERGE_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} - BASE_REF: ${{ github.event.pull_request.base.ref }} - run: | - set -eu - - echo "PR #${PR_NUMBER} - 检查CI状态+审批并自动合并到${BASE_REF}" - echo - - # 只合main分支 - if [ "$BASE_REF" != "main" ]; then - echo "Skip: 目标分支不是main" - exit 0 - fi - - # main分支门禁：Validate + Frontend Lint - CONTEXTS=( - "CI/CD Pipeline / Validate Code Quality And Tests (pull_request)" - "Tests / test (pull_request)" - "CI/CD Pipeline / Frontend Lint (pull_request)" - ) - echo "检查门禁: ${#CONTEXTS[@]} 项" - echo - - # 轮询等待，最多30分钟（180次x10秒） - for attempt in $(seq 1 180); do - ALL_SUCCESS=true - ANY_FAILED=false - - echo "--- 第${attempt}次检查 ($(date '+%H:%M:%S')) ---" - - # 检查CI状态 - for ctx in "${CONTEXTS[@]}"; do - STATE=$(python3 scripts/check_ci_status.py "$GITHUB_TOKEN" "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$ctx") - echo " CI: ${ctx##*/}: $STATE" - if [ "$STATE" != "success" ]; then - ALL_SUCCESS=false - fi - if [ "$STATE" = "failure" ] || [ "$STATE" = "error" ]; then - ANY_FAILED=true - fi - done - - # 检查审批状态 - APPROVAL_RESULT=$(python3 scripts/check_pr_approval.py "$MERGE_TOKEN" "$GITHUB_REPOSITORY" "$PR_NUMBER" 1) - echo " 审批: $APPROVAL_RESULT" - HAS_APPROVAL=false - if echo "$APPROVAL_RESULT" | grep -q '^approved'; then - HAS_APPROVAL=true - fi - - # 全部满足 → 合并 - if [ "$ALL_SUCCESS" = "true" ] && [ "$HAS_APPROVAL" = "true" ]; then - echo - echo "CI全绿 + 审批通过，执行自动合并" - - # 幂等检查：PR是否还是open - PR_STATE=$(curl -s -H "Authorization: token ${MERGE_TOKEN}" \ - "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" \ - | python3 -c "import sys,json; print(json.load(sys.stdin).get('state',''))") - - if [ "$PR_STATE" != "open" ]; then - echo "PR状态为 ${PR_STATE}，无需合并" - exit 0 - fi - - # 执行merge（main分支用merge，保留历史） - HTTP_CODE=$(curl -s -o /tmp/merge_resp.json -w "%{http_code}" \ - -X POST \ - -H "Authorization: token ${MERGE_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{"do":"merge","merge_title_field":"","merge_message_field":"","delete_branch_after_merge":true,"force_merge":false}' \ - "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/merge") - - echo "合并API HTTP状态: $HTTP_CODE" - - if [ "$HTTP_CODE" = "200" ]; then - echo "自动合并成功" - exit 0 - elif [ "$HTTP_CODE" = "405" ]; then - echo "合并失败（405），可能有冲突或门禁未通过" - curl -s -X POST \ - -H "Authorization: token ${MERGE_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{"body": "Auto merge failed: PR may have conflicts or unresolved checks. Please review manually."}' \ - "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" > /dev/null 2>&1 || true - exit 0 - else - echo "自动合并失败 (HTTP $HTTP_CODE)" - cat /tmp/merge_resp.json 2>/dev/null || true - curl -s -X POST \ - -H "Authorization: token ${MERGE_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "{\"body\": \"Auto merge failed (HTTP ${HTTP_CODE}), please check manually.\"}" \ - "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" > /dev/null 2>&1 || true - exit 1 - fi - fi - - if [ "$ANY_FAILED" = "true" ]; then - echo - echo "CI有失败项，不自动合并" - exit 0 - fi - - sleep 10 - done - - echo - echo "等待超时（30分钟）" - exit 0 diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml deleted file mode 100755 index 3bcc81d9d..000000000 --- a/.gitea/workflows/ci-cd.yml +++ /dev/null @@ -1,657 +0,0 @@ -name: CI/CD Pipeline - -on: - push: - branches: - - main - - develop - - 'feature/**' - - 'bugfix/**' - - 'hotfix/**' - - 'release/**' - tags: - - 'v*' - pull_request: - branches: - - main - - develop - -permissions: - contents: read - -jobs: - validate: - name: Validate Code Quality And Tests - runs-on: ubuntu-22.04 - - env: - DATABASE_URL: postgresql+psycopg://postgres:postgres@127.0.0.1:5433/xiaoxia_saas - USE_IN_MEMORY_DB: "false" - - steps: - - name: Checkout code - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - set -eu - python3 - <<'PY' - import io, os, tarfile, time, urllib.request, urllib.error - url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" - request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - last_err = None - for attempt in range(5): - try: - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() - break - except urllib.error.HTTPError as e: - last_err = e - if e.code >= 500 and attempt < 4: - wait = 2 ** attempt - print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - except Exception as e: - last_err = e - if attempt < 4: - wait = 2 ** attempt - print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - else: - raise last_err - with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: - root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' - for member in tar.getmembers(): - name = member.name - if name == root_prefix[:-1]: - continue - if name.startswith(root_prefix): - member.name = name[len(root_prefix):] - if member.name: - tar.extract(member, '.') - PY - - - name: Verify CI environment - shell: sh - run: | - set -eu - python --version - python3 -m pip --version - echo "CI environment is ready" - - - name: Install dependencies - shell: sh - run: | - set -eu - python3 -m pip install -q -r requirements-base.txt - python3 -m pip install -q -r requirements.txt - python3 -m pip install -q -r requirements-dev.txt - python3 -m black --version - python3 -m isort --version-number - python3 -m flake8 --version - bandit --version - pytest --version - - - name: Run code quality checks - shell: sh - run: | - set -eu - python3 -m compileall -q alembic apps packages tests scripts - python3 -m black --check --fast alembic apps packages tests scripts - python3 -m isort --check-only alembic apps packages tests scripts - python3 -m flake8 apps packages tests --count --statistics - - - name: Run security scan - shell: sh - run: | - set -eu - bandit -r apps packages -q -ll - - - name: Validate release scripts syntax - shell: sh - run: | - set -eu - bash -n scripts/backup_postgres.sh - bash -n scripts/restore_postgres_plan.sh - bash -n scripts/init_production_env.sh - - - name: Validate Alembic migrations - shell: sh - run: | - set -eu - python3 -m alembic upgrade head --sql > /tmp/alembic-upgrade.sql - test -s /tmp/alembic-upgrade.sql - grep -q "Running upgrade" /tmp/alembic-upgrade.sql - python3 scripts/check_schema_metadata.py - - - name: Start Redis for unit tests - shell: sh - run: | - set -eu - docker rm -f ci-redis-validate 2>/dev/null || true - docker run -d --name ci-redis-validate \ - -p 6379:6379 \ - --health-cmd "redis-cli ping" \ - --health-interval 2s \ - --health-timeout 2s \ - --health-retries 15 \ - redis:7-alpine - for i in $(seq 1 20); do - if docker inspect --format='{{.State.Health.Status}}' ci-redis-validate 2>/dev/null | grep -q healthy; then - echo "Redis is ready" - break - fi - echo "Waiting for Redis... ($i/20)" - sleep 1 - done - docker inspect --format='{{.State.Health.Status}}' ci-redis-validate | grep -q healthy - - name: Run unit tests - shell: sh - env: - USE_IN_MEMORY_DB: "true" - run: | - set -eu - PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/unit -q - - - name: Start PostgreSQL for integration tests - shell: sh - run: | - set -eu - docker rm -f ci-pg-validate 2>/dev/null || true - docker run -d --name ci-pg-validate \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=xiaoxia_saas \ - -p 5433:5432 \ - --health-cmd "pg_isready -U postgres" \ - --health-interval 5s \ - --health-timeout 5s \ - --health-retries 12 \ - postgres:16 - for i in $(seq 1 30); do - if docker inspect --format='{{.State.Health.Status}}' ci-pg-validate 2>/dev/null | grep -q healthy; then - echo "PostgreSQL is ready" - break - fi - echo "Waiting for PostgreSQL... ($i/30)" - sleep 2 - done - docker inspect --format='{{.State.Health.Status}}' ci-pg-validate | grep -q healthy - - - name: Apply migrations for integration tests - shell: sh - run: | - set -eu - PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head - - - name: Run integration tests - shell: sh - run: | - set -eu - PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration -q --timeout=60 -x - - - name: Cleanup PostgreSQL - if: always() - shell: sh - run: | - docker rm -f ci-pg-validate 2>/dev/null || true - echo "PostgreSQL container cleaned up" - - - name: Build summary - if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main' - shell: sh - run: | - set -eu - echo "Build completed successfully!" - echo "Branch: ${GITHUB_REF_NAME}" - echo "Commit: ${GITHUB_SHA}" - - frontend-lint: - name: Frontend Lint - runs-on: ubuntu-22.04 - - steps: - - name: Checkout code - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - set -eu - archive_url="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/archive/${GITHUB_SHA}.tar.gz" - for i in 1 2 3 4 5; do - if wget --header="Authorization: token ${GITHUB_TOKEN}" -O /tmp/repo.tar.gz "$archive_url" 2>&1; then - break - fi - if [ "$i" -lt 5 ]; then - wait=$((2 ** i)) - echo "Checkout failed (attempt $i/5), retrying in ${wait}s..." - sleep "$wait" - else - echo "Checkout failed after 5 attempts" - exit 1 - fi - done - tar -xzf /tmp/repo.tar.gz --strip-components=1 -C . - rm -f /tmp/repo.tar.gz - - - name: Install dependencies - shell: sh - run: | - set -eu - docker run --rm \ - -v "$PWD:/workspace" \ - -w /workspace/apps/web \ - docker.m.daocloud.io/library/node:20 \ - sh -lc 'npm ci' - - - name: Run ESLint - shell: sh - run: | - set -eu - docker run --rm \ - -v "$PWD:/workspace" \ - -w /workspace/apps/web \ - docker.m.daocloud.io/library/node:20 \ - sh -lc 'npx eslint src --ext .ts,.tsx --max-warnings 50' - - - name: Run TypeScript type check - shell: sh - run: | - set -eu - docker run --rm \ - -v "$PWD:/workspace" \ - -w /workspace/apps/web \ - docker.m.daocloud.io/library/node:20 \ - sh -lc 'npx tsc --noEmit' - - - name: Run Prettier check - shell: sh - run: | - set -eu - docker run --rm \ - -v "$PWD:/workspace" \ - -w /workspace/apps/web \ - docker.m.daocloud.io/library/node:20 \ - sh -lc 'npx prettier --check "src/**/*.{ts,tsx,md}"' - - - name: Run Vitest tests - shell: sh - run: | - set -eu - docker run --rm \ - -v "$PWD:/workspace" \ - -w /workspace/apps/web \ - docker.m.daocloud.io/library/node:20 \ - sh -lc 'npx vitest run src/test' - deploy-staging: - name: Build & Push Staging (Watchtower auto-deploy) - runs-on: saas - needs: [validate, frontend-lint] - - if: github.ref_name == 'main' || github.ref_name == 'develop' || startsWith(github.ref_name, 'feature/') - - steps: - - name: Checkout code - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - set -eu - python3 - <<'INNERPY' - import io, os, tarfile, time, urllib.request, urllib.error - url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" - request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - last_err = None - for attempt in range(5): - try: - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() - break - except urllib.error.HTTPError as e: - last_err = e - if e.code >= 500 and attempt < 4: - wait = 2 ** attempt - print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - except Exception as e: - last_err = e - if attempt < 4: - wait = 2 ** attempt - print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - else: - raise last_err - with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: - root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' - for member in tar.getmembers(): - name = member.name - if name == root_prefix[:-1]: - continue - if name.startswith(root_prefix): - member.name = name[len(root_prefix):] - if member.name: - tar.extract(member, '.') - INNERPY - - - name: Build and push all images to Gitea Registry - shell: sh - env: - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - run: | - set -eu - chmod +x scripts/build_release_images.sh - ALLOW_SHARED_PRODUCTION_BUILD_HOST=true REGISTRY_TOKEN="${REGISTRY_TOKEN}" \ - scripts/build_release_images.sh "${GITHUB_SHA}" - - - name: Tag and push :staging images (Watchtower auto-update) - shell: sh - env: - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - run: | - set -eu - REGISTRY="git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas" - if [ -n "${REGISTRY_TOKEN:-}" ]; then - printf '%s' "${REGISTRY_TOKEN}" | docker login git.xiaoxiajianji.com -u xiaoxia --password-stdin 2>/dev/null - fi - for svc in api worker web; do - docker tag "${REGISTRY}/xiaoxia-saas-${svc}:${GITHUB_SHA}" "${REGISTRY}/xiaoxia-saas-${svc}:staging" - docker push "${REGISTRY}/xiaoxia-saas-${svc}:staging" - done - echo "All :staging images pushed. Watchtower will auto-deploy within 60s." - - - name: Wait for Watchtower update + smoke test - shell: sh - run: | - set -eu - echo "Waiting 90s for Watchtower to detect new image and restart containers..." - sleep 90 - - echo "--- Smoke test 1: Health check ---" - for i in $(seq 1 12); do - HEALTH=$(curl -sf --max-time 10 https://staging-api.xiaoxiajianji.com/health) && break - echo " Attempt $i/12: not ready yet, waiting 5s..." - sleep 5 - done - if [ -z "$HEALTH" ]; then - echo "FAIL: health endpoint unreachable after 60s" - exit 1 - fi - echo "Health OK: $HEALTH" - - echo "--- Smoke test 2: Login API (expect 401) ---" - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 -X POST \ - https://staging-api.xiaoxiajianji.com/api/v1/auth/login \ - -H "Content-Type: application/json" \ - -d '{"email":"smoke@test.com","password":"wrong"}') - if [ "$HTTP_CODE" != "401" ] && [ "$HTTP_CODE" != "422" ]; then - echo "FAIL: login returned HTTP $HTTP_CODE (expected 401 or 422)" - exit 1 - fi - echo "Login API OK: HTTP $HTTP_CODE" - - echo "--- Smoke test 3: API docs endpoint ---" - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 https://staging-api.xiaoxiajianji.com/docs) - if [ "$HTTP_CODE" != "200" ]; then - echo "FAIL: /docs returned HTTP $HTTP_CODE (expected 200)" - exit 1 - fi - echo "Docs endpoint OK: HTTP $HTTP_CODE" - - echo "--- Smoke test 4: Web frontend ---" - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 https://staging.xiaoxiajianji.com/) - if [ "$HTTP_CODE" != "200" ]; then - echo "FAIL: web frontend returned HTTP $HTTP_CODE (expected 200)" - exit 1 - fi - echo "Web frontend OK: HTTP $HTTP_CODE" - - echo "" - echo "=== All smoke tests passed! ===" - echo "Branch: ${GITHUB_REF_NAME}" - echo "Commit: ${GITHUB_SHA}" - - - staging-e2e: - name: Staging E2E Tests - runs-on: saas - if: github.ref_name == 'develop' || github.ref_name == 'main' - needs: deploy-staging - - steps: - - name: Install SSH client - shell: sh - run: | - set -eu - apt-get update -qq && apt-get install -y -qq openssh-client >/dev/null 2>&1 - echo "openssh-client installed" - - - name: Run Playwright E2E on staging server - shell: sh - env: - STAGING_SSH_HOST: ${{ secrets.STAGING_SSH_HOST }} - STAGING_SSH_USER: ${{ secrets.STAGING_SSH_USER }} - STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }} - run: | - set -eu - staging_host="${STAGING_SSH_HOST:-47.98.113.167}" - staging_user="${STAGING_SSH_USER:-root}" - if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then - key_path="/root/.ssh/xiaoxia_runtime_builder" - elif [ -n "${STAGING_SSH_KEY:-}" ]; then - key_path="$HOME/.ssh/id_ed25519" - printf '%s\n' "$STAGING_SSH_KEY" > "$key_path" - chmod 600 "$key_path" - else - echo "ERROR: No SSH key available" - exit 1 - fi - ssh-keyscan -H "$staging_host" >> ~/.ssh/known_hosts - - # 在业务服务器上跑 Playwright E2E（用 host 网络访问 staging 3001/8000 端口） - ssh -i "$key_path" "$staging_user@$staging_host" ' - cd /var/lib/xiaoxia-saas-staging/repo - docker run --rm \ - -e E2E_BASE_URL=http://127.0.0.1:3001 \ - -e E2E_API_BASE=http://127.0.0.1:8000/api/v1 \ - -e E2E_BROWSER_CHANNEL=chromium \ - -v "$PWD:/workspace" \ - -w /workspace/apps/web \ - --network host \ - mcr.microsoft.com/playwright:v1.45.0-jammy \ - sh -lc "npm ci && npx playwright test --reporter=line --project=chromium" - ' - - build-production-runtime-images: - name: Build Production Runtime Images - runs-on: saas - needs: [validate, frontend-lint] - - if: startsWith(github.ref, 'refs/tags/v') - - steps: - - name: Checkout code - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - set -eu - python3 - <<'PY' - import io, os, tarfile, time, urllib.request, urllib.error - url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" - request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - last_err = None - for attempt in range(5): - try: - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() - break - except urllib.error.HTTPError as e: - last_err = e - if e.code >= 500 and attempt < 4: - wait = 2 ** attempt - print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - except Exception as e: - last_err = e - if attempt < 4: - wait = 2 ** attempt - print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - else: - raise last_err - with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: - root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' - for member in tar.getmembers(): - name = member.name - if name == root_prefix[:-1]: - continue - if name.startswith(root_prefix): - member.name = name[len(root_prefix):] - if member.name: - tar.extract(member, '.') - PY - - - name: Build and push all images (api + worker + web, with buildx cache) - shell: sh - env: - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - run: | - set -eu - chmod +x scripts/build_release_images.sh - REGISTRY_TOKEN="${REGISTRY_TOKEN}" scripts/build_release_images.sh "${GITHUB_REF_NAME}" - - - name: Cleanup old Docker images - if: always() - shell: sh - run: | - set -eu - if [ -f scripts/cleanup_old_images.sh ]; then - chmod +x scripts/cleanup_old_images.sh - scripts/cleanup_old_images.sh - else - echo "Cleanup script not found, doing basic prune..." - docker image prune -f 2>/dev/null || true - fi - echo "Disk usage after cleanup:" - df -h / | tail -1 - - deploy-production: - name: Deploy Production - runs-on: saas - if: startsWith(github.ref, 'refs/tags/v') - needs: build-production-runtime-images - - steps: - - name: Install SSH client - shell: sh - run: | - set -eu - apt-get update -qq && apt-get install -y -qq openssh-client >/dev/null 2>&1 - echo "openssh-client installed" - - - name: Deploy production over SSH (Registry pull) - shell: sh - env: - PRODUCTION_SSH_HOST: ${{ secrets.PRODUCTION_SSH_HOST }} - PRODUCTION_SSH_USER: ${{ secrets.PRODUCTION_SSH_USER }} - PRODUCTION_SSH_KEY: ${{ secrets.PRODUCTION_SSH_KEY }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - run: | - set -eu - production_host="${PRODUCTION_SSH_HOST:-47.98.113.167}" - production_user="${PRODUCTION_SSH_USER:-root}" - mkdir -p ~/.ssh - if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then - key_path="/root/.ssh/xiaoxia_runtime_builder" - elif [ -n "${PRODUCTION_SSH_KEY:-}" ]; then - key_path="$HOME/.ssh/id_ed25519" - printf '%s\n' "$PRODUCTION_SSH_KEY" > "$key_path" - chmod 600 "$key_path" - else - echo "ERROR: No SSH key available" - exit 1 - fi - ssh-keyscan -H "$production_host" >> ~/.ssh/known_hosts - - # Registry 方式部署脚本（base64 编码避免转义问题） - DEPLOY_B64="IyEvYmluL3NoCnNldCAtZXUKCiMgPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KIyBQcm9kdWN0aW9uIOmDqOe9suiEmuacrCAtIFJlZ2lzdHJ5IOaWueW8jwojIOeUqOazle+8mklNQUdFX1RBRz08dmVyc2lvbj4gUkVHSVNUUllfVE9LRU49PHRva2VuPiBzaCBkZXBsb3ktcHJvZHVjdGlvbi1yZWdpc3RyeS5zaAojID09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgpJTUFHRV9UQUc9IiR7SU1BR0VfVEFHOi19IgpSRUdJU1RSWT0iJHtSRUdJU1RSWTotZ2l0LnhpYW94aWFqaWFuamkuY29tL3hpYW94aWEveGlhb3hpYS1zYWFzfSIKUkVHSVNUUllfVVNFUj0iJHtSRUdJU1RSWV9VU0VSOi14aWFveGlhfSIKUkVHSVNUUllfVE9LRU49IiR7UkVHSVNUUllfVE9LRU46LX0iCgpFTlZfRklMRT0iJHtFTlZfRklMRTotL3Zhci9saWIveGlhb3hpYS1zYWFzLXByb2R1Y3Rpb24vLmVudn0iCkdFTkVSQVRFRF9ESVI9IiR7R0VORVJBVEVEX0RJUjotL3Zhci9saWIveGlhb3hpYS1zYWFzLXByb2R1Y3Rpb24vZ2VuZXJhdGVkfSIKTEVHQUNZX0FTU0VUU19ESVI9IiR7TEVHQUNZX0FTU0VUU19ESVI6LS92YXIvbGliL3hpYW94aWEtc2Fhcy1wcm9kdWN0aW9uL2xlZ2FjeS1hc3NldHN9IgoKaWYgWyAteiAiJElNQUdFX1RBRyIgXTsgdGhlbgogIGVjaG8gIkVSUk9SOiBJTUFHRV9UQUcgaXMgcmVxdWlyZWQiCiAgZXhpdCAxCmZpCgp0ZXN0IC1mICIkRU5WX0ZJTEUiCm1rZGlyIC1wICIkR0VORVJBVEVEX0RJUiIKbWtkaXIgLXAgIiRMRUdBQ1lfQVNTRVRTX0RJUiIKCiMgLS0tLSDnmbvlvZUgUmVnaXN0cnkgLS0tLQppZiBbIC1uICIkUkVHSVNUUllfVE9LRU4iIF07IHRoZW4KICBlY2hvICJMb2dnaW5nIGluIHRvIHJlZ2lzdHJ5OiAkUkVHSVNUUlkiCiAgUkVHSVNUUllfSE9TVD0kKGVjaG8gIiRSRUdJU1RSWSIgfCBjdXQgLWQvIC1mMSkKICBwcmludGYgJXMgIiRSRUdJU1RSWV9UT0tFTiIgfCBkb2NrZXIgbG9naW4gIiRSRUdJU1RSWV9IT1NUIiAtdSAiJFJFR0lTVFJZX1VTRVIiIC0tcGFzc3dvcmQtc3RkaW4gMj4vZGV2L251bGwgfHwgewogICAgZWNobyAiV0FSTjogZG9ja2VyIGxvZ2luIGZhaWxlZCwgd2lsbCB0cnkgdG8gcHVsbCBhbnl3YXkiCiAgfQpmaQoKIyAtLS0tIFB1bGwg5LiJ6ZWc5YOPIC0tLS0KUkVHSVNUUllfQVBJPSIke1JFR0lTVFJZfS94aWFveGlhLXNhYXMtYXBpOiR7SU1BR0VfVEFHfSIKUkVHSVNUUllfV09SS0VSPSIke1JFR0lTVFJZfS94aWFveGlhLXNhYXMtd29ya2VyOiR7SU1BR0VfVEFHfSIKUkVHSVNUUllfV0VCPSIke1JFR0lTVFJZfS94aWFveGlhLXNhYXMtd2ViOiR7SU1BR0VfVEFHfSIKCkxPQ0FMX0FQST0ieGlhb3hpYS1zYWFzLWFwaToke0lNQUdFX1RBR30iCkxPQ0FMX1dPUktFUj0ieGlhb3hpYS1zYWFzLXdvcmtlcjoke0lNQUdFX1RBR30iCkxPQ0FMX1dFQj0ieGlhb3hpYS1zYWFzLXdlYjoke0lNQUdFX1RBR30iCgplY2hvICJQdWxsaW5nIEFQSSBpbWFnZS4uLiIKZG9ja2VyIHB1bGwgIiRSRUdJU1RSWV9BUEkiCmVjaG8gIlB1bGxpbmcgV29ya2VyIGltYWdlLi4uIgpkb2NrZXIgcHVsbCAiJFJFR0lTVFJZX1dPUktFUiIKZWNobyAiUHVsbGluZyBXZWIgaW1hZ2UuLi4iCmRvY2tlciBwdWxsICIkUkVHSVNUUllfV0VCIgoKIyAtLS0tIFJlLXRhZyDmiJDmnKzlnLDlkI0gLS0tLQpkb2NrZXIgdGFnICIkUkVHSVNUUllfQVBJIiAiJExPQ0FMX0FQSSIKZG9ja2VyIHRhZyAiJFJFR0lTVFJZX1dPUktFUiIgIiRMT0NBTF9XT1JLRVIiCmRvY2tlciB0YWcgIiRSRUdJU1RSWV9XRUIiICIkTE9DQUxfV0VCIgplY2hvICJBbGwgaW1hZ2VzIHB1bGxlZCBhbmQgdGFnZ2VkLiIKCiMgLS0tLSDlpIfku73ml6fniYggYXNzZXRz77yI6YOo572y5pyf6Ze057yT5a2Y55So5oi35LiNIDQwNO+8iSAtLS0tCmVjaG8gIkJhY2tpbmcgdXAgbGVnYWN5IGFzc2V0cyBmcm9tIGN1cnJlbnQgd2ViIGNvbnRhaW5lci4uLiIKaWYgZG9ja2VyIGluc3BlY3QgeGlhb3hpYS13ZWItcHJvZHVjdGlvbiA+L2Rldi9udWxsIDI+JjE7IHRoZW4KICBfdG1wZGlyPSIvdG1wL2xlZ2FjeS1hc3NldHMtJCQiCiAgcm0gLXJmICIkX3RtcGRpciIKICBta2RpciAtcCAiJF90bXBkaXIiCiAgZG9ja2VyIGNwIHhpYW94aWEtd2ViLXByb2R1Y3Rpb246L3Vzci9zaGFyZS9uZ2lueC9odG1sL2Fzc2V0cy8uICIkX3RtcGRpci8iIDI+L2Rldi9udWxsIHx8IHRydWUKICAjIOWQiOW5tuWIsCBMRUdBQ1lfQVNTRVRTX0RJUu+8iOS/neeVmeaJgOacieWOhuWPsueJiOacrOeahCBhc3NldHPvvIkKICBpZiBbIC1kICIkX3RtcGRpciIgXSAmJiBbICIkKGxzIC1BICIkX3RtcGRpciIgMj4vZGV2L251bGwpIiBdOyB0aGVuCiAgICBjcCAtYW4gIiRfdG1wZGlyIi8uICIkTEVHQUNZX0FTU0VUU19ESVIiLyAyPi9kZXYvbnVsbCB8fCB0cnVlCiAgICBlY2hvICJMZWdhY3kgYXNzZXRzIGJhY2tlZCB1cDogJChscyAiJF90bXBkaXIiIHwgd2MgLWwpIGZpbGVzIgogIGZpCiAgcm0gLXJmICIkX3RtcGRpciIKZWxzZQogIGVjaG8gIk5vIGV4aXN0aW5nIHdlYiBjb250YWluZXIsIHNraXBwaW5nIGxlZ2FjeSBhc3NldHMgYmFja3VwIgpmaQoKIyDmuIXnkIbotoXov4cgNyDlpKnnmoTml6cgYXNzZXRzIOaWh+S7tu+8iOmBv+WFjeaXoOmZkOWinumVv++8iQppZiBbIC1kICIkTEVHQUNZX0FTU0VUU19ESVIiIF07IHRoZW4KICBmaW5kICIkTEVHQUNZX0FTU0VUU19ESVIiIC10eXBlIGYgLW10aW1lICs3IC1kZWxldGUgMj4vZGV2L251bGwgfHwgdHJ1ZQogIGVjaG8gIkxlZ2FjeSBhc3NldHMgY2xlYW51cCBkb25lIChyZXRhaW4gNyBkYXlzKSIKZmkKCiMgLS0tLSDnoa7kv53ln7rnoYDorr7mlr3lrrnlmajlnKjov5DooYwgLS0tLQplY2hvICJDaGVja2luZyBpbmZyYXN0cnVjdHVyZSBjb250YWluZXJzLi4uIgpmb3IgYyBpbiB4aWFveGlhLXBvc3RncmVzLXByb2R1Y3Rpb24geGlhb3hpYS1yZWRpcy1wcm9kdWN0aW9uOyBkbwogIGlmICEgZG9ja2VyIGluc3BlY3QgIiRjIiA+L2Rldi9udWxsIDI+JjE7IHRoZW4KICAgIGVjaG8gIkVSUk9SOiBSZXF1aXJlZCBjb250YWluZXIgbm90IGZvdW5kOiAkYyIKICAgIGV4aXQgMQogIGZpCiAgc3RhdGU9JChkb2NrZXIgaW5zcGVjdCAtZiAne3suU3RhdGUuU3RhdHVzfX0nICIkYyIpCiAgaWYgWyAiJHN0YXRlIiAhPSAicnVubmluZyIgXTsgdGhlbgogICAgZWNobyAiRVJST1I6IENvbnRhaW5lciBub3QgcnVubmluZzogJGMgKCRzdGF0ZSkiCiAgICBleGl0IDEKICBmaQpkb25lCgojIC0tLS0g56Gu5L+d55Sf5Lqn572R57uc5a2Y5ZyoIC0tLS0KZG9ja2VyIG5ldHdvcmsgY3JlYXRlIHhpYW94aWEtbmV0LXByb2R1Y3Rpb24gMj4vZGV2L251bGwgfHwgdHJ1ZQoKIyAtLS0tIOaJp+ihjOaVsOaNruW6kyBNaWdyYXRpb24gLS0tLQplY2hvICJSdW5uaW5nIGRhdGFiYXNlIG1pZ3JhdGlvbnMuLi4iCmRvY2tlciBydW4gLS1ybSBcCiAgLS1lbnYtZmlsZSAiJEVOVl9GSUxFIiBcCiAgLS1uZXR3b3JrIHhpYW94aWEtbmV0LXByb2R1Y3Rpb24gXAogIC1lIEFQUF9FTlY9cHJvZHVjdGlvbiBcCiAgIiRMT0NBTF9BUEkiIHNoIC1jICJjZCAvYXBwICYmIGFsZW1iaWMgdXBncmFkZSBoZWFkIgplY2hvICJNaWdyYXRpb25zIGNvbXBsZXRlZC4iCgojIC0tLS0g5YGc5q2i5pen5a655ZmoIC0tLS0KZWNobyAiU3RvcHBpbmcgb2xkIGNvbnRhaW5lcnMuLi4iCmRvY2tlciBybSAtZiB4aWFveGlhLWFwaS1wcm9kdWN0aW9uIDI+L2Rldi9udWxsIHx8IHRydWUKZG9ja2VyIHJtIC1mIHhpYW94aWEtd29ya2VyLXByb2R1Y3Rpb24gMj4vZGV2L251bGwgfHwgdHJ1ZQpkb2NrZXIgcm0gLWYgeGlhb3hpYS13ZWItcHJvZHVjdGlvbiAyPi9kZXYvbnVsbCB8fCB0cnVlCgojIC0tLS0g5pel5b+X6YWN572u77yI5omA5pyJ5a655Zmo5YWx55So77yJIC0tLS0KTE9HX09QVFM9Ii0tbG9nLWRyaXZlciBqc29uLWZpbGUgLS1sb2ctb3B0IG1heC1zaXplPTUwbSAtLWxvZy1vcHQgbWF4LWZpbGU9MyIKCiMgLS0tLSDlkK/liqggQVBJIC0tLS0KZWNobyAiU3RhcnRpbmcgQVBJIGNvbnRhaW5lci4uLiIKZG9ja2VyIHJ1biAtZCBcCiAgLS1uYW1lIHhpYW94aWEtYXBpLXByb2R1Y3Rpb24gXAogIC0tZW52LWZpbGUgIiRFTlZfRklMRSIgXAogIC0tbmV0d29yayB4aWFveGlhLW5ldC1wcm9kdWN0aW9uIFwKICAtcCAxMjcuMC4wLjE6ODAwMTo4MDAwIFwKICAtZSBBUFBfRU5WPXByb2R1Y3Rpb24gXAogIC1lIEFQUF9WRVJTSU9OPSIkSU1BR0VfVEFHIiBcCiAgLWUgR0VORVJBVEVEX0ZJTEVTX0RJUj0vYXBwL2dlbmVyYXRlZCBcCiAgLWUgR0VORVJBVEVEX0ZJTEVTX1VSTF9QUkVGSVg9L2dlbmVyYXRlZC1maWxlcyBcCiAgLWUgUFVCTElDX0FQSV9CQVNFX1VSTD1odHRwczovL2FwaS54aWFveGlhamlhbmppLmNvbSBcCiAgLXYgIiRHRU5FUkFURURfRElSOi9hcHAvZ2VuZXJhdGVkIiBcCiAgLS1yZXN0YXJ0IHVubGVzcy1zdG9wcGVkIFwKICAtLWNwdXMgMiBcCiAgLS1tZW1vcnkgMmcgXAogIC0taGVhbHRoLWNtZCAicHl0aG9uIC1jIFwiaW1wb3J0IHVybGxpYi5yZXF1ZXN0OyB1cmxsaWIucmVxdWVzdC51cmxvcGVuKCdodHRwOi8vbG9jYWxob3N0OjgwMDAvaGVhbHRoJywgdGltZW91dD01KVwiIiBcCiAgLS1oZWFsdGgtaW50ZXJ2YWwgMzBzIFwKICAtLWhlYWx0aC10aW1lb3V0IDEwcyBcCiAgLS1oZWFsdGgtcmV0cmllcyAzIFwKICAtLWhlYWx0aC1zdGFydC1wZXJpb2QgNDBzIFwKICAkTE9HX09QVFMgXAogICIkTE9DQUxfQVBJIgoKIyAtLS0tIOWQr+WKqCBXb3JrZXIgLS0tLQplY2hvICJTdGFydGluZyBXb3JrZXIgY29udGFpbmVyLi4uIgpkb2NrZXIgcnVuIC1kIFwKICAtLW5hbWUgeGlhb3hpYS13b3JrZXItcHJvZHVjdGlvbiBcCiAgLS1lbnYtZmlsZSAiJEVOVl9GSUxFIiBcCiAgLS1uZXR3b3JrIHhpYW94aWEtbmV0LXByb2R1Y3Rpb24gXAogIC1lIEFQUF9FTlY9cHJvZHVjdGlvbiBcCiAgLWUgQVBQX1ZFUlNJT049IiRJTUFHRV9UQUciIFwKICAtZSBXT1JLRVJfQ09OQ1VSUkVOQ1k9MSBcCiAgLWUgV09SS0VSX01BWF9UQVNLU19QRVJfQ0hJTEQ9MTAwIFwKICAtZSBHRU5FUkFURURfRklMRVNfRElSPS9hcHAvZ2VuZXJhdGVkIFwKICAtZSBHRU5FUkFURURfRklMRVNfVVJMX1BSRUZJWD0vZ2VuZXJhdGVkLWZpbGVzIFwKICAtZSBQVUJMSUNfQVBJX0JBU0VfVVJMPWh0dHBzOi8vYXBpLnhpYW94aWFqaWFuamkuY29tIFwKICAtdiAiJEdFTkVSQVRFRF9ESVI6L2FwcC9nZW5lcmF0ZWQiIFwKICAtLXJlc3RhcnQgdW5sZXNzLXN0b3BwZWQgXAogIC0tY3B1cyAyIFwKICAtLW1lbW9yeSAyZyBcCiAgLS1oZWFsdGgtY21kICJzaCAtYyBcImdyZXAgLXEgY2VsZXJ5IC9wcm9jLzEvY21kbGluZSB8fCBleGl0IDFcIiIgXAogIC0taGVhbHRoLWludGVydmFsIDMwcyBcCiAgLS1oZWFsdGgtdGltZW91dCAxMHMgXAogIC0taGVhbHRoLXJldHJpZXMgMyBcCiAgLS1oZWFsdGgtc3RhcnQtcGVyaW9kIDMwcyBcCiAgJExPR19PUFRTIFwKICAiJExPQ0FMX1dPUktFUiIKCiMgLS0tLSDlkK/liqggV2ViIC0tLS0KIyBMZWdhY3kgYXNzZXRzIOaMgui9veWIsCAvdXNyL3NoYXJlL25naW54L2h0bWwvYXNzZXRzLWxlZ2FjeS9hc3NldHMvCiMgbmdpbngg6YWN572u5LitIGFzc2V0cyBsb2NhdGlvbiDmnIkgZmFsbGJhY2sg6YC76L6RCkxFR0FDWV9WT0xVTUU9IiIKaWYgWyAtZCAiJExFR0FDWV9BU1NFVFNfRElSIiBdICYmIFsgIiQobHMgLUEgIiRMRUdBQ1lfQVNTRVRTX0RJUiIgMj4vZGV2L251bGwpIiBdOyB0aGVuCiAgTEVHQUNZX1ZPTFVNRT0iLXYgJHtMRUdBQ1lfQVNTRVRTX0RJUn06L3Vzci9zaGFyZS9uZ2lueC9odG1sL2Fzc2V0cy1sZWdhY3kvYXNzZXRzOnJvIgogIGVjaG8gIldlYiBjb250YWluZXI6IGxlZ2FjeSBhc3NldHMgbW91bnRlZCAoZmFsbGJhY2spIgplbHNlCiAgZWNobyAiV2ViIGNvbnRhaW5lcjogbm8gbGVnYWN5IGFzc2V0cyB0byBtb3VudCIKZmkKCmVjaG8gIlN0YXJ0aW5nIFdlYiBjb250YWluZXIuLi4iCmRvY2tlciBydW4gLWQgXAogIC0tbmFtZSB4aWFveGlhLXdlYi1wcm9kdWN0aW9uIFwKICAtLW5ldHdvcmsgeGlhb3hpYS1uZXQtcHJvZHVjdGlvbiBcCiAgLXAgMTI3LjAuMC4xOjMwMDI6ODAgXAogIC0tcmVzdGFydCB1bmxlc3Mtc3RvcHBlZCBcCiAgLS1jcHVzIDAuNSBcCiAgLS1tZW1vcnkgNTEybSBcCiAgJExFR0FDWV9WT0xVTUUgXAogIC0taGVhbHRoLWNtZCAid2dldCAtLXNwaWRlciAtcSBodHRwOi8vMTI3LjAuMC4xOjgwIiBcCiAgLS1oZWFsdGgtaW50ZXJ2YWwgMzBzIFwKICAtLWhlYWx0aC10aW1lb3V0IDVzIFwKICAtLWhlYWx0aC1yZXRyaWVzIDMgXAogICRMT0dfT1BUUyBcCiAgIiRMT0NBTF9XRUIiCgojIC0tLS0g562J5b6FIEFQSSDlgaXlurcgLS0tLQplY2hvICJXYWl0aW5nIGZvciBBUEkgdG8gYmVjb21lIGhlYWx0aHkuLi4iCmk9MAp3aGlsZSBbICIkaSIgLWx0IDQwIF07IGRvCiAgaWYgY3VybCAtc2YgLS1tYXgtdGltZSA1IGh0dHA6Ly8xMjcuMC4wLjE6ODAwMS9oZWFsdGggPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgICBlY2hvICJBUEkgaXMgaGVhbHRoeSEiCiAgICBicmVhawogIGZpCiAgaT0kKChpICsgMSkpCiAgZWNobyAiICBXYWl0aW5nLi4uICgkaS80MCkiCiAgc2xlZXAgMwpkb25lCgppZiBbICIkaSIgLWdlIDQwIF07IHRoZW4KICBlY2hvICJFUlJPUjogQVBJIGRpZCBub3QgYmVjb21lIGhlYWx0aHkgd2l0aGluIDEyMHMiCiAgZG9ja2VyIGxvZ3MgLS10YWlsIDUwIHhpYW94aWEtYXBpLXByb2R1Y3Rpb24KICBleGl0IDEKZmkKCiMgLS0tLSDnrYnlvoUgV2ViIOWBpeW6tyAtLS0tCmVjaG8gIldhaXRpbmcgZm9yIFdlYiB0byBiZWNvbWUgaGVhbHRoeS4uLiIKaT0wCndoaWxlIFsgIiRpIiAtbHQgMTUgXTsgZG8KICBpZiBjdXJsIC1zZiAtLW1heC10aW1lIDUgaHR0cDovLzEyNy4wLjAuMTozMDAyLyA+L2Rldi9udWxsIDI+JjE7IHRoZW4KICAgIGVjaG8gIldlYiBpcyBoZWFsdGh5ISIKICAgIGJyZWFrCiAgZmkKICBpPSQoKGkgKyAxKSkKICBlY2hvICIgIFdhaXRpbmcuLi4gKCRpLzE1KSIKICBzbGVlcCAyCmRvbmUKCmlmIFsgIiRpIiAtZ2UgMTUgXTsgdGhlbgogIGVjaG8gIkVSUk9SOiBXZWIgZGlkIG5vdCBiZWNvbWUgaGVhbHRoeSB3aXRoaW4gMzBzIgogIGRvY2tlciBsb2dzIC0tdGFpbCAzMCB4aWFveGlhLXdlYi1wcm9kdWN0aW9uCiAgZXhpdCAxCmZpCgojIC0tLS0g5riF55CG5pen6ZWc5YOPIC0tLS0KZWNobyAiQ2xlYW5pbmcgdXAgb2xkIGltYWdlcy4uLiIKZG9ja2VyIGltYWdlIHBydW5lIC1hZiAtLWZpbHRlciAidW50aWw9MTY4aCIgMj4vZGV2L251bGwgfHwgdHJ1ZQpkb2NrZXIgYnVpbGRlciBwcnVuZSAtYWYgLS1maWx0ZXIgInVudGlsPTE2OGgiIDI+L2Rldi9udWxsIHx8IHRydWUKCmVjaG8gIiIKZWNobyAiPT09IFByb2R1Y3Rpb24gZGVwbG95bWVudCBjb21wbGV0ZSA9PT0iCmVjaG8gIkFQSTogICAgICBodHRwOi8vMTI3LjAuMC4xOjgwMDEiCmVjaG8gIldlYjogICAgICBodHRwOi8vMTI3LjAuMC4xOjMwMDIiCmVjaG8gIlZlcnNpb246ICAkSU1BR0VfVEFHIgpkb2NrZXIgcHMgLS1mb3JtYXQgInRhYmxlIHt7Lk5hbWVzfX1cdHt7LlN0YXR1c319XHR7ey5JbWFnZX19IiB8IGdyZXAgcHJvZHVjdGlvbgo=" - - echo "$DEPLOY_B64" | base64 -d | ssh -i "$key_path" "$production_user@$production_host" "IMAGE_TAG='${GITHUB_REF_NAME}' REGISTRY_TOKEN='${REGISTRY_TOKEN}' sh" - - production-e2e: - name: Production Browser E2E - runs-on: saas - if: startsWith(github.ref, 'refs/tags/v') - needs: deploy-production - - steps: - - name: Checkout code - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - set -eu - python3 - <<'PY' - import io, os, tarfile, time, urllib.request, urllib.error - url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" - request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - # Retry up to 5 times with backoff for transient 5xx errors - last_err = None - for attempt in range(5): - try: - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() - break - except urllib.error.HTTPError as e: - last_err = e - if e.code >= 500 and attempt < 4: - wait = 2 ** attempt - print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - except Exception as e: - last_err = e - if attempt < 4: - wait = 2 ** attempt - print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - else: - raise last_err - with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: - root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' - for member in tar.getmembers(): - name = member.name - if name == root_prefix[:-1]: - continue - if name.startswith(root_prefix): - member.name = name[len(root_prefix):] - if member.name: - tar.extract(member, '.') - PY - - - name: Run production browser E2E - shell: sh - run: | - set -eu - docker run --rm \ - -e E2E_BASE_URL=https://saas.xiaoxiajianji.com \ - -e E2E_API_BASE=https://api.xiaoxiajianji.com/api/v1 \ - -e E2E_BROWSER_CHANNEL=chromium \ - -v "$PWD:/workspace" \ - -w /workspace/apps/web \ - mcr.microsoft.com/playwright:v1.45.0-jammy \ - sh -lc 'npm ci && npx playwright test --reporter=line --project=chromium e2e/core-upload.spec.ts e2e/core-generation.spec.ts e2e/core-titles.spec.ts' diff --git a/.gitea/workflows/ci-failure-monitor.yml b/.gitea/workflows/ci-failure-monitor.yml new file mode 100644 index 000000000..659769723 --- /dev/null +++ b/.gitea/workflows/ci-failure-monitor.yml @@ -0,0 +1,78 @@ +name: CI Failure Monitor + +on: + schedule: + - cron: '0 */6 * * *' # 每6小时检查一次 + workflow_dispatch: + inputs: + days: + description: '统计最近N天的失败' + required: false + default: '7' + fail_threshold: + description: '失败次数阈值' + required: false + default: '3' + fail_rate_threshold: + description: '失败率阈值(%)' + required: false + default: '30' + +permissions: + contents: read + +jobs: + monitor: + name: CI重复失败检测 + runs-on: ci-l2 + timeout-minutes: 10 + + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" \ + | bash + + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + + - name: Run failure detection + shell: sh + env: + GITEA_API_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }} + GITEA_URL: https://git.xiaoxiajianji.com + GITEA_REPO: xiaoxia/xiaoxia-saas + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + FAIL_CHECK_DAYS: ${{ inputs.days || 7 }} + FAIL_THRESHOLD: ${{ inputs.fail_threshold || 3 }} + FAIL_RATE_THRESHOLD: ${{ inputs.fail_rate_threshold || 30 }} + run: | + set +e + python3 scripts/ci/ci_repeated_failure_detector.py + EXIT_CODE=$? + echo "检测完成，退出码: $EXIT_CODE" + # 0=无异常, 1=有警告, 2=有严重问题 + # 监控脚本永远不fail，避免告警风暴 + exit 0 + + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true \ No newline at end of file diff --git a/.gitea/workflows/ci-health-daily.yml b/.gitea/workflows/ci-health-daily.yml new file mode 100644 index 000000000..26a02ab8a --- /dev/null +++ b/.gitea/workflows/ci-health-daily.yml @@ -0,0 +1,103 @@ +name: CI Health Daily Report +on: + schedule: + - cron: '0 1 * * *' # UTC 01:00 = 北京时间 09:00 + workflow_dispatch: +permissions: + contents: read +jobs: + ci-health-report: + name: CI健康度每日巡检 + runs-on: saas + timeout-minutes: 15 + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -eu + python3 - <<'PY' + import io, os, tarfile, time, urllib.request, urllib.error + url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" + request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' + for member in tar.getmembers(): + name = member.name + if name == root_prefix[:-1]: + continue + if name.startswith(root_prefix): + member.name = name[len(root_prefix):] + if member.name: + tar.extract(member, '.') + PY + + - name: Generate CI Dashboard HTML + shell: sh + env: + GITEA_TOKEN: ${{ github.token }} + run: | + set +e + echo "=== 生成 CI 健康度 HTML 看板 ===" + echo "时间: $(date '+%Y-%m-%d %H:%M:%S')" + echo "" + python3 scripts/ci/ci_dashboard.py --days 7 --html --html-output ci_dashboard.html + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ] && [ -f ci_dashboard.html ]; then + HTML_SIZE=$(wc -c < ci_dashboard.html) + echo "" + echo "✅ HTML 看板生成成功 (${HTML_SIZE} bytes)" + echo "路径: $(pwd)/ci_dashboard.html" + # 输出文件内容前几行，方便在 Actions 日志中确认 + echo "" + echo "--- 看板预览 (前 5 行) ---" + head -5 ci_dashboard.html + echo "...（完整内容见产物文件）" + else + echo "❌ HTML 看板生成失败 (exit code: $EXIT_CODE)" + fi + echo "" + # 永远成功，看板生成失败不影响主流程 + exit 0 + + - name: Run CI health check and report + shell: sh + env: + GITEA_TOKEN: ${{ github.token }} + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + echo "=== CI健康度每日巡检 ===" + echo "时间: $(date '+%Y-%m-%d %H:%M:%S')" + echo "" + python3 scripts/ci/ci_health_report.py --limit 30 + EXIT_CODE=$? + echo "" + echo "巡检完成 (exit code: $EXIT_CODE)" + # 永远成功，不影响CI状态（通知失败不应该标红） + exit 0 diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml new file mode 100755 index 000000000..1556e3e48 --- /dev/null +++ b/.gitea/workflows/ci-pipeline.yml @@ -0,0 +1,1573 @@ +name: CI/CD Pipeline +on: + push: + branches: + - main + - develop + tags: + - v* + pull_request: + branches: + - main + - develop + schedule: + - cron: '0 19 * * *' # UTC 19:00 = 北京时间凌晨3:00，每日全量CI回归 + workflow_dispatch: + inputs: + reason: + description: "触发原因" + required: false + default: "手动触发 - CI漏触发补跑" +permissions: + contents: read +concurrency: + group: ci-pipeline-${{ gitea.event_name }}-${{ gitea.ref }} + # PR事件取消进行中的旧run，push事件不取消（确保完整CI跑完） + cancel-in-progress: ${{ gitea.event_name == 'pull_request' }} +jobs: + check-frontend-only: + name: Check if frontend-only change + runs-on: ci-l2 + if: github.event_name == 'pull_request' + outputs: + skip_backend: ${{ steps.check.outputs.skip_backend }} + skip_frontend: ${{ steps.check.outputs.skip_frontend }} + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Check changed files + id: check + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -eu + PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') + API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300" + FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin)]") + FRONTEND_COUNT=$(echo "$FILES" | grep -c '^apps/web/' || true) + BACKEND_COUNT=$(echo "$FILES" | grep -cv '^apps/web/' || true) + TOTAL=$(echo "$FILES" | grep -cv '^$' || true) + echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT})" + if [ "$BACKEND_COUNT" = "0" ] && [ "$FRONTEND_COUNT" -gt "0" ]; then + echo "skip_backend=true" >> $GITHUB_OUTPUT + echo "skip_frontend=false" >> $GITHUB_OUTPUT + echo "✅ 纯前端改动，跳过后端检查" + elif [ "$FRONTEND_COUNT" = "0" ] && [ "$BACKEND_COUNT" -gt "0" ]; then + echo "skip_backend=false" >> $GITHUB_OUTPUT + echo "skip_frontend=true" >> $GITHUB_OUTPUT + echo "🔧 纯后端改动，跳过前端检查" + else + echo "skip_backend=false" >> $GITHUB_OUTPUT + echo "skip_frontend=false" >> $GITHUB_OUTPUT + echo "🔧 包含全栈变更，运行完整CI" + fi + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + validate-code-quality: + name: Validate - Code Quality + runs-on: ci-l2 + timeout-minutes: 8 + permissions: + contents: write + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Install dependencies + shell: sh + run: | + set -eu + for i in 1 2 3; do + python3 -m pip install -q -r requirements-base.txt && break + echo "pip install requirements-base.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + for i in 1 2 3; do + python3 -m pip install -q -r requirements.txt && break + echo "pip install requirements.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + for i in 1 2 3; do + python3 -m pip install -q -r requirements-dev.txt && break + echo "pip install requirements-dev.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + for i in 1 2 3; do + python3 -m pip install --no-binary :all: black==26.5.1 isort==8.0.1 && break + echo "pip install black/isort 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + - name: Run code quality and security checks + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: bash scripts/ci/validate_code_quality.sh + - name: Auto-fix formatting (black + isort) + if: failure() + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + REVIEW_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }} + run: python3 scripts/ci/auto_fix_formatting.py + - name: CI failure notification + if: failure() + shell: sh + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }} + run: | + set +e + FAILED_JOB="Validate - Code Quality" python3 scripts/ci_notify_failure.py + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Validate - Code Quality" python3 scripts/ci_notify.py + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + validate-type-check: + name: Validate - Type Check (mypy) + runs-on: ci-l2 + timeout-minutes: 8 + permissions: + contents: read + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Install dependencies + shell: sh + run: | + set -eu + for i in 1 2 3; do + python3 -m pip install -q -r requirements-base.txt && break + echo "pip install requirements-base.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + for i in 1 2 3; do + python3 -m pip install -q -r requirements.txt && break + echo "pip install requirements.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + for i in 1 2 3; do + python3 -m pip install -q -r requirements-dev.txt && break + echo "pip install requirements-dev.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + - name: Run mypy type check + shell: bash + run: bash scripts/ci/validate_mypy.sh + - name: CI failure notification + if: failure() + shell: sh + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }} + run: | + set +e + FAILED_JOB="Validate - Type Check (mypy)" python3 scripts/ci_notify_failure.py + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Validate - Type Check (mypy)" python3 scripts/ci_notify.py + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + validate-migration: + name: Validate - Migration (alembic) + runs-on: ci-l2 + timeout-minutes: 8 + permissions: + contents: read + env: + DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas + USE_IN_MEMORY_DB: 'false' + CI_USE_SHARED_PG: 'true' + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Install dependencies + shell: sh + run: | + set -eu + for i in 1 2 3; do + python3 -m pip install -q -r requirements-base.txt && break + echo "pip install requirements-base.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + for i in 1 2 3; do + python3 -m pip install -q -r requirements.txt && break + echo "pip install requirements.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + for i in 1 2 3; do + python3 -m pip install -q -r requirements-dev.txt && break + echo "pip install requirements-dev.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + - name: Run alembic migration validation + shell: bash + run: bash scripts/ci/validate_migration.sh + - name: CI failure notification + if: failure() + shell: sh + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }} + run: | + set +e + FAILED_JOB="Validate - Migration (alembic)" python3 scripts/ci_notify_failure.py + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Validate - Migration (alembic)" python3 scripts/ci_notify.py + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + unit-tests: + needs: check-frontend-only + if: always() && needs.check-frontend-only.outputs.skip_backend != 'true' + name: Unit Tests + runs-on: ci-l2 + timeout-minutes: 8 + env: + USE_IN_MEMORY_DB: 'true' + OSS_ACCESS_KEY_ID: placeholder + OSS_ACCESS_KEY_SECRET: placeholder + OSS_BUCKET_NAME: xiaoxia-autocut + OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Install ffmpeg + shell: sh + run: bash scripts/ci/step_install_ffmpeg.sh + - name: Run unit tests with coverage + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: bash scripts/ci/run_unit_tests.sh + - name: CI failure notification + if: failure() + shell: sh + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }} + run: | + set +e + FAILED_JOB="Unit Tests" python3 scripts/ci_notify_failure.py + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Unit Tests" python3 scripts/ci_notify.py + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + integration-tests: + name: Integration Tests + runs-on: ci-l2 + timeout-minutes: 30 + if: always() && needs.check-frontend-only.outputs.skip_backend != 'true' + needs: + - check-frontend-only + - validate-code-quality + - validate-type-check + - validate-migration + env: + DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas + USE_IN_MEMORY_DB: 'false' + CI_USE_SHARED_PG: 'true' + OSS_ACCESS_KEY_ID: placeholder + OSS_ACCESS_KEY_SECRET: placeholder + OSS_BUCKET_NAME: xiaoxia-autocut + OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Run integration tests + shell: bash + run: bash scripts/ci/run_integration_tests.sh + - name: CI failure notification + if: failure() + shell: sh + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }} + run: | + set +e + FAILED_JOB="Integration Tests" python3 scripts/ci_notify_failure.py + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Integration Tests" python3 scripts/ci_notify.py + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + frontend-lint: + name: Frontend Lint + runs-on: ci-l2 + timeout-minutes: 10 + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Install frontend dependencies (with retry) + shell: sh + run: | + set -eu + # npm install 带重试（网络不稳定时自动重试） + for i in 1 2 3; do + bash scripts/ci/step_frontend_install.sh && break + echo "前端依赖安装失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + - name: Run ESLint + shell: sh + run: bash scripts/ci/step_frontend_run.sh "npx --no-install eslint src --ext .ts,.tsx --max-warnings 0" + - name: Run TypeScript type check + shell: sh + run: bash scripts/ci/step_frontend_run.sh "npx --no-install tsc --noEmit" + - name: Run Prettier check + shell: sh + run: bash scripts/ci/step_frontend_run.sh "npx --no-install prettier --check \"src/**/*.{ts,tsx,md}\"" + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Frontend Lint" python3 scripts/ci_notify.py + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + frontend-unit-test: + name: Frontend Unit Tests + runs-on: ci-l2 + timeout-minutes: 15 + needs: check-frontend-only + if: always() && needs.check-frontend-only.outputs.skip_frontend != 'true' + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Install frontend dependencies (vitest only, with retry) + shell: sh + run: | + set -eu + for i in 1 2 3; do + bash scripts/ci/step_frontend_install.sh vitest && break + echo "前端依赖安装失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + - name: Run Vitest (incremental for PRs, full for main branches) + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: bash scripts/ci/vitest_incremental.sh + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Frontend Unit Tests" python3 scripts/ci_notify.py + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + + build-pr: + name: PR Build ${{ matrix.service_display }} Image + runs-on: runtime-builder + timeout-minutes: ${{ matrix.timeout }} + if: github.event_name == 'pull_request' + strategy: + fail-fast: false + matrix: + include: + - service: api + service_display: API + dockerfile: infra/docker/api.Dockerfile + image_name: xiaoxia-saas-api + cache_name: api-cache + timeout: 30 + - service: worker + service_display: Worker + dockerfile: infra/docker/worker.Dockerfile + image_name: xiaoxia-saas-worker + cache_name: worker-cache + timeout: 40 + - service: web + service_display: Web + dockerfile: infra/docker/web.Dockerfile + image_name: xiaoxia-saas-web + cache_name: web-cache + timeout: 30 + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Docker login to Registry (for cache read) + shell: sh + env: + ACR_USERNAME: ${{ secrets.ACR_USERNAME }} + ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} + GITEA_REGISTRY_USER: xiaoxia + GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + set -eu + for i in 1 2 3; do + echo "Docker login attempt $i/3" + if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then + echo "Docker login successful" + break + fi + echo "Docker login failed ($i/3), retrying in 5s..." + sleep 5 + done + - name: Pre-build worker base images (fallback if not exist) + if: matrix.service == 'worker' + id: prebuild + shell: sh + run: | + set -eu + REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas" + BASE_BUILDER="${REGISTRY}/worker-base-builder:latest" + BASE_RUNTIME="${REGISTRY}/worker-base-runtime:latest" + + # 尝试拉取基础镜像 + echo "检查基础镜像..." + if docker pull "$BASE_BUILDER" 2>/dev/null && docker pull "$BASE_RUNTIME" 2>/dev/null; then + echo "基础镜像已存在，使用远程镜像" + echo "fallback=false" >> $GITHUB_OUTPUT + else + echo "基础镜像不存在，本地构建（fallback模式）..." + + # 尝试用buildx构建，失败则回退到普通docker build（DooD模式下buildx builder偶发崩溃） + BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}" + BUILDX_AVAILABLE=true + if ! docker buildx create --use --name "$BUILDER_NAME" --driver docker-container > /dev/null 2>&1; then + BUILDX_AVAILABLE=false + fi + if [ "$BUILDX_AVAILABLE" = true ] && ! docker buildx inspect --bootstrap > /dev/null 2>&1; then + BUILDX_AVAILABLE=false + docker buildx rm "$BUILDER_NAME" > /dev/null 2>&1 || true + fi + + build_base() { + local df="$1" + local tag="$2" + local name="$3" + if [ "$BUILDX_AVAILABLE" = true ]; then + echo "构建 $name（buildx）..." + if docker buildx build --load -f "$df" -t "$tag" . > /dev/null 2>&1; then + echo "$name 构建成功" + return 0 + fi + echo "buildx失败，回退到普通docker build" + BUILDX_AVAILABLE=false + docker buildx rm "$BUILDER_NAME" > /dev/null 2>&1 || true + fi + echo "构建 $name（docker build）..." + docker build -f "$df" -t "$tag" . + } + + build_base infra/docker/worker-base-builder.Dockerfile "$BASE_BUILDER" "worker-base-builder" + build_base infra/docker/worker-base-runtime.Dockerfile "$BASE_RUNTIME" "worker-base-runtime" + + echo "fallback=true" >> $GITHUB_OUTPUT + echo "基础镜像本地构建完成" + fi + + - name: Build PR image (verify only, no push) + shell: sh + run: | + set -eu + REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji" + IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:pr-${GITHUB_SHA}" + CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:develop" + + EXTRA_BUILD_ARGS="APP_VERSION=\"${GITHUB_SHA}\"" + if [ "${{ matrix.service }}" = "web" ]; then + EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf" + fi + + # Worker fallback模式：基础镜像本地已构建，用普通docker build绕过buildx + if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.fallback }}" = "true" ]; then + echo "Fallback模式：用普通docker build（基础镜像本地已构建）" + BUILD_ARG_STR="" + for arg in $EXTRA_BUILD_ARGS; do + BUILD_ARG_STR="$BUILD_ARG_STR --build-arg $arg" + done + docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR . + echo "Fallback PR Build successful" + exit 0 + fi + + NO_CACHE_FLAG="" + for i in 1 2 3; do + echo "PR Build attempt $i/3" + if bash scripts/ci/docker_build_only.sh $NO_CACHE_FLAG ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS; then + echo "PR Build successful" + break + fi + echo "PR Build failed (attempt $i/3)" + [ $i -eq 3 ] && exit 1 + sleep 10 + if [ $i -eq 2 ]; then + NO_CACHE_FLAG="--no-cache" + echo "Next retry with --no-cache" + fi + done + echo + echo "${{ matrix.service_display }} PR build verified: ${IMAGE_TAG}" + - name: Cleanup buildx builder + if: always() + shell: sh + run: | + BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}" + docker buildx rm "$BUILDER_NAME" 2>/dev/null || true + docker buildx prune -f 2>/dev/null || true + echo "Builder cleanup done" + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="PR Build ${{ matrix.service_display }} Image" python3 scripts/ci_notify.py + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + build-staging: + name: Build Staging ${{ matrix.service_display }} Image + runs-on: runtime-builder + timeout-minutes: ${{ matrix.timeout }} + if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop') + strategy: + fail-fast: false + matrix: + include: + - service: api + service_display: API + dockerfile: infra/docker/api.Dockerfile + image_name: xiaoxia-saas-api + cache_name: api-cache + timeout: 30 + - service: worker + service_display: Worker + dockerfile: infra/docker/worker.Dockerfile + image_name: xiaoxia-saas-worker + cache_name: worker-cache + timeout: 40 + - service: web + service_display: Web + dockerfile: infra/docker/web.Dockerfile + image_name: xiaoxia-saas-web + cache_name: web-cache + timeout: 30 + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Docker login to Registry + shell: sh + env: + ACR_USERNAME: ${{ secrets.ACR_USERNAME }} + ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} + GITEA_REGISTRY_USER: xiaoxia + GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + set -eu + # Docker login 带重试（网络波动时自动重试） + for i in 1 2 3; do + echo "=== Docker login 尝试 $i/3 ===" + if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then + echo "✅ Docker login successful" + break + fi + echo "❌ Docker login 失败（尝试 $i/3），5s 后重试..." + sleep 5 + done + - name: Setup cache strategy + shell: sh + run: | + set -eu + if [ "${GITHUB_REF_NAME}" = "develop" ] || [ "${GITHUB_REF_NAME}" = "main" ]; then + echo "CACHE_MODE=read-write" >> $GITHUB_ENV + echo "Cache mode: read-write (will push cache)" + else + echo "CACHE_MODE=read-only" >> $GITHUB_ENV + echo "Cache mode: read-only" + fi + + - name: Setup buildx builder + shell: sh + run: | + set -eu + if ! docker buildx inspect ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} > /dev/null 2>&1; then + docker buildx create --use --name ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} --driver docker-container + echo "Created ci-builder (docker-container driver)" + else + docker buildx use ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} + echo "Using existing ci-builder" + fi + docker buildx inspect --bootstrap + + - name: Build and push ${{ matrix.service_display }} image (with retry) + shell: sh + run: | + set -eu + REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji" + IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:${GITHUB_SHA}" + CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:${GITHUB_REF_NAME}" + + EXTRA_BUILD_ARGS="APP_VERSION=\"${GITHUB_SHA}\"" + if [ "${{ matrix.service }}" = "web" ]; then + EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf" + fi + + # Docker build 带重试：失败自动重试2次，第2次重试加--no-cache + NO_CACHE_FLAG="" + for i in 1 2 3; do + echo "=== Docker build 尝试 $i/3 ===" + if bash scripts/ci/docker_build_push.sh $NO_CACHE_FLAG ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS; then + echo "✅ Docker build 成功" + break + fi + echo "❌ Docker build 失败（尝试 $i/3）" + [ $i -eq 3 ] && exit 1 + sleep 10 + # 第2次重试使用 --no-cache + if [ $i -eq 2 ]; then + NO_CACHE_FLAG="--no-cache" + echo "下次重试将使用 --no-cache" + fi + done + + echo + echo "${{ matrix.service_display }} image pushed: ${IMAGE_TAG}" + - name: Cleanup buildx builder + if: always() + shell: sh + run: | + docker buildx rm ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} 2>/dev/null || true + docker buildx rm ci-builder 2>/dev/null || true + docker buildx prune -f 2>/dev/null || true + echo "Builder cleanup done" + + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Build Staging ${{ matrix.service_display }} Image" python3 scripts/ci_notify.py + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + deploy-staging: + name: Deploy Staging (Watchtower auto-deploy) + runs-on: runtime-builder + timeout-minutes: 15 + concurrency: + group: deploy-staging-${{ gitea.ref }} + cancel-in-progress: false + needs: + - build-staging + if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop') + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Notify job start + continue-on-error: true + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=start JOB_NAME="Deploy Staging" python3 scripts/ci_notify.py + - name: Docker login to Registry + shell: sh + env: + ACR_USERNAME: ${{ secrets.ACR_USERNAME }} + ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} + GITEA_REGISTRY_USER: xiaoxia + GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + set -eu + # Docker login 带重试（网络波动时自动重试） + for i in 1 2 3; do + echo "=== Docker login 尝试 $i/3 ===" + if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then + echo "✅ Docker login successful" + break + fi + echo "❌ Docker login 失败（尝试 $i/3），5s 后重试..." + sleep 5 + done + - name: Install SSH client + if: success() + shell: sh + run: | + set -eu + apt-get update -qq && apt-get install -y -qq openssh-client >/dev/null 2>&1 + echo "openssh-client installed" + + - name: Deploy staging over SSH (Registry pull) + if: success() + shell: sh + env: + STAGING_SSH_HOST: ${{ secrets.STAGING_SSH_HOST }} + STAGING_SSH_USER: ${{ secrets.STAGING_SSH_USER }} + STAGING_SSH_PORT: ${{ secrets.STAGING_SSH_PORT }} + STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }} + ACR_USERNAME: ${{ secrets.ACR_USERNAME }} + ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} + run: | + set -eux + staging_host="${STAGING_SSH_HOST:-47.98.113.167}" + staging_user="${STAGING_SSH_USER:-root}" + staging_port="${STAGING_SSH_PORT:-22222}" + echo "Host: $staging_host" + echo "Port: $staging_port" + + mkdir -p ~/.ssh + + key_path="" + if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then + key_path="/root/.ssh/xiaoxia_runtime_builder" + echo "Using key: $key_path (builder key)" + elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then + key_path="$HOME/.ssh/xiaoxia_runtime_builder" + echo "Using key: $key_path (home key)" + elif [ -n "${STAGING_SSH_KEY:-}" ]; then + key_path="$HOME/.ssh/id_ed25519" + printf '%s\n' "$STAGING_SSH_KEY" > "$key_path" + chmod 600 "$key_path" + echo "Using key from STAGING_SSH_KEY secret" + else + echo "ERROR: No SSH key available" + ls -la ~/.ssh/ 2>/dev/null || true + ls -la /root/.ssh/ 2>/dev/null || true + exit 1 + fi + + ssh-keyscan -p "$staging_port" -H "$staging_host" >> ~/.ssh/known_hosts 2>/dev/null + echo "SSH keyscan done" + + ssh -p "$staging_port" -i "$key_path" -o StrictHostKeyChecking=no "${staging_user}@${staging_host}" "echo SSH_CONNECTION_OK && hostname" + echo "SSH connection verified" + + # 通过环境变量传递凭证，避免命令行引号转义问题 + cat scripts/ci_staging_deploy.sh | ssh -p "$staging_port" -i "$key_path" -o StrictHostKeyChecking=no "${staging_user}@${staging_host}" "IMAGE_TAG=${GITHUB_SHA} ACR_USERNAME=${ACR_USERNAME} ACR_PASSWORD=${ACR_PASSWORD} sh" + + - name: Staging health check + auto rollback + if: success() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + STAGING_SSH_HOST: ${{ secrets.STAGING_SSH_HOST }} + STAGING_SSH_USER: ${{ secrets.STAGING_SSH_USER }} + STAGING_SSH_PORT: ${{ secrets.STAGING_SSH_PORT }} + STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }} + ACR_USERNAME: ${{ secrets.ACR_USERNAME }} + ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} + run: | + set -eu + echo "==========================================" + echo " Staging 健康检查（Watchtower 模式）" + echo "==========================================" + echo "" + bash scripts/ci_staging_healthcheck.sh + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on success + continue-on-error: true + if: success() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=success JOB_NAME="Deploy Staging" python3 scripts/ci_notify.py + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Deploy Staging" python3 scripts/ci_notify.py + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + staging-e2e: + name: Staging E2E Tests + runs-on: runtime-builder + timeout-minutes: 15 + if: github.ref_name == 'develop' || github.ref_name == 'main' + needs: deploy-staging + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Run Playwright E2E on staging + shell: sh + run: | + set -eu + # DooD模式下不能用-v挂载（宿主机路径与CI容器路径不一致） + # 改用 docker create + docker cp 方式把代码拷进容器 + CONTAINER_NAME="staging-e2e-$$" + docker create --name "$CONTAINER_NAME" --ipc=host \ + -e E2E_BASE_URL=https://staging.xiaoxiajianji.com \ + -e E2E_API_BASE=https://staging-api.xiaoxiajianji.com/api/v1 \ + -e E2E_BROWSER_CHANNEL=chromium \ + -e PLAYWRIGHT_HEADLESS=1 \ + -w /workspace/apps/web \ + git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy \ + sh -lc "npm ci && npx playwright test --reporter=line --project=chromium e2e/auth.spec.ts e2e/auth-guard.spec.ts e2e/core-upload.spec.ts e2e/core-generation.spec.ts" + docker cp apps "$CONTAINER_NAME:/workspace/" + docker cp package-lock.json "$CONTAINER_NAME:/workspace/" 2>/dev/null || true + docker start -a "$CONTAINER_NAME" + EXIT_CODE=$(docker wait "$CONTAINER_NAME") + docker rm "$CONTAINER_NAME" 2>/dev/null || true + exit $EXIT_CODE + + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Staging E2E Tests" python3 scripts/ci_notify.py + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + staging-api-tests: + name: Staging API Integration Tests + runs-on: runtime-builder + timeout-minutes: 10 + if: github.ref_name == 'develop' || github.ref_name == 'main' + needs: deploy-staging + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Run API integration tests on staging + shell: sh + run: | + set -eu + # DooD模式下不能用-v挂载（宿主机路径与CI容器路径不一致） + # 改用 docker create + docker cp 方式把代码拷进容器 + CONTAINER_NAME="staging-api-tests-$$" + docker create --name "$CONTAINER_NAME" \ + -e E2E_BASE_URL=https://staging.xiaoxiajianji.com \ + -e E2E_API_BASE=https://staging-api.xiaoxiajianji.com/api/v1 \ + -w /workspace/apps/web \ + git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy \ + sh -lc 'npm ci && npx playwright test --reporter=line e2e/test_auth.spec.ts e2e/test_asset.spec.ts e2e/test_project.spec.ts' + docker cp apps "$CONTAINER_NAME:/workspace/" + docker cp package-lock.json "$CONTAINER_NAME:/workspace/" 2>/dev/null || true + docker start -a "$CONTAINER_NAME" + EXIT_CODE=$(docker wait "$CONTAINER_NAME") + docker rm "$CONTAINER_NAME" 2>/dev/null || true + exit $EXIT_CODE + + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Staging API Integration Tests" python3 scripts/ci_notify.py + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + build-production: + name: Build Production ${{ matrix.service_display }} Image + runs-on: runtime-builder + timeout-minutes: ${{ matrix.timeout }} + needs: + if: startsWith(github.ref, 'refs/tags/v') + strategy: + fail-fast: false + matrix: + include: + - service: api + service_display: API + dockerfile: infra/docker/api.Dockerfile + image_name: xiaoxia-saas-api + cache_name: api-cache + timeout: 30 + - service: worker + service_display: Worker + dockerfile: infra/docker/worker.Dockerfile + image_name: xiaoxia-saas-worker + cache_name: worker-cache + timeout: 40 + - service: web + service_display: Web + dockerfile: infra/docker/web.Dockerfile + image_name: xiaoxia-saas-web + cache_name: web-cache + timeout: 30 + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Docker login to Registry + shell: sh + env: + ACR_USERNAME: ${{ secrets.ACR_USERNAME }} + ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} + GITEA_REGISTRY_USER: xiaoxia + GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + set -eu + # Docker login 带重试（网络波动时自动重试） + for i in 1 2 3; do + echo "=== Docker login 尝试 $i/3 ===" + if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then + echo "✅ Docker login successful" + break + fi + echo "❌ Docker login 失败（尝试 $i/3），5s 后重试..." + sleep 5 + done + - name: Setup cache strategy + shell: sh + run: | + set -eu + echo "CACHE_MODE=read-only" >> $GITHUB_ENV + echo "Cache mode: read-only (production build uses cached layers)" + + - name: Setup buildx builder + shell: sh + run: | + set -eu + if ! docker buildx inspect ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} > /dev/null 2>&1; then + docker buildx create --use --name ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} --driver docker-container + echo "Created ci-builder" + else + docker buildx use ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} + echo "Using existing ci-builder" + fi + docker buildx inspect --bootstrap + + - name: Build and push production ${{ matrix.service_display }} image (with retry) + shell: sh + run: | + set -eu + REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji" + IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:${GITHUB_REF_NAME}" + CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:main" + + EXTRA_BUILD_ARGS="APP_VERSION=\"${GITHUB_REF_NAME}\"" + if [ "${{ matrix.service }}" = "web" ]; then + EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-production.conf" + fi + + # Docker build 带重试：失败自动重试2次，第2次重试加--no-cache + NO_CACHE_FLAG="" + for i in 1 2 3; do + echo "=== Docker build 尝试 $i/3 ===" + if bash scripts/ci/docker_build_push.sh $NO_CACHE_FLAG ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS; then + echo "✅ Docker build 成功" + break + fi + echo "❌ Docker build 失败（尝试 $i/3）" + [ $i -eq 3 ] && exit 1 + sleep 10 + # 第2次重试使用 --no-cache + if [ $i -eq 2 ]; then + NO_CACHE_FLAG="--no-cache" + echo "下次重试将使用 --no-cache" + fi + done + + echo + echo "${{ matrix.service_display }} production image pushed: ${IMAGE_TAG}" + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Build Production ${{ matrix.service_display }} Image" python3 scripts/ci_notify.py + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + deploy-production: + name: Deploy Production + runs-on: runtime-builder + timeout-minutes: 30 + concurrency: + group: deploy-production-${{ gitea.ref }} + cancel-in-progress: false + if: startsWith(github.ref, 'refs/tags/v') + needs: + - build-production + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Notify job start + continue-on-error: true + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=start JOB_NAME="Deploy Production" python3 scripts/ci_notify.py + - name: Install SSH client + shell: sh + run: | + set -eu + apt-get update -qq && apt-get install -y -qq openssh-client >/dev/null 2>&1 + echo "openssh-client installed" + - name: Deploy production over SSH (Registry pull) + if: success() + shell: sh + env: + PRODUCTION_SSH_HOST: ${{ secrets.PRODUCTION_SSH_HOST }} + PRODUCTION_SSH_USER: ${{ secrets.PRODUCTION_SSH_USER }} + PRODUCTION_SSH_PORT: ${{ secrets.PRODUCTION_SSH_PORT }} + PRODUCTION_SSH_KEY: ${{ secrets.PRODUCTION_SSH_KEY }} + ACR_USERNAME: ${{ secrets.ACR_USERNAME }} + ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} + run: | + set -eux + production_host="${PRODUCTION_SSH_HOST:-47.98.113.167}" + production_user="${PRODUCTION_SSH_USER:-root}" + production_port="${PRODUCTION_SSH_PORT:-22222}" + echo "Host: $production_host" + echo "Port: $production_port" + + mkdir -p ~/.ssh + + key_path="" + if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then + key_path="/root/.ssh/xiaoxia_runtime_builder" + echo "Using key: $key_path (builder key)" + elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then + key_path="$HOME/.ssh/xiaoxia_runtime_builder" + echo "Using key: $key_path (home key)" + elif [ -n "${PRODUCTION_SSH_KEY:-}" ]; then + key_path="$HOME/.ssh/id_ed25519" + printf '%s\n' "$PRODUCTION_SSH_KEY" > "$key_path" + chmod 600 "$key_path" + echo "Using key from PRODUCTION_SSH_KEY secret" + else + echo "ERROR: No SSH key available" + ls -la ~/.ssh/ 2>/dev/null || true + ls -la /root/.ssh/ 2>/dev/null || true + exit 1 + fi + + ssh-keyscan -p "$production_port" -H "$production_host" >> ~/.ssh/known_hosts 2>/dev/null + echo "SSH keyscan done" + + ssh -p "$production_port" -i "$key_path" -o StrictHostKeyChecking=no "${production_user}@${production_host}" "echo SSH_CONNECTION_OK && hostname" + echo "SSH connection verified" + + # 通过环境变量传递凭证，避免命令行引号转义问题 + cat scripts/ci_production_deploy.sh | ssh -p "$production_port" -i "$key_path" -o StrictHostKeyChecking=no "${production_user}@${production_host}" "IMAGE_TAG=${GITHUB_REF_NAME} ACR_USERNAME=${ACR_USERNAME} ACR_PASSWORD=${ACR_PASSWORD} sh" + + - name: Production health check + auto rollback + if: success() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + PRODUCTION_SSH_HOST: ${{ secrets.PRODUCTION_SSH_HOST }} + PRODUCTION_SSH_USER: ${{ secrets.PRODUCTION_SSH_USER }} + PRODUCTION_SSH_PORT: ${{ secrets.PRODUCTION_SSH_PORT }} + PRODUCTION_SSH_KEY: ${{ secrets.PRODUCTION_SSH_KEY }} + ACR_USERNAME: ${{ secrets.ACR_USERNAME }} + ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} + run: | + set -eu + echo "==========================================" + echo " Production 健康检查 + 自动回滚" + echo "==========================================" + echo "" + bash scripts/ci_production_healthcheck.sh + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on success + continue-on-error: true + if: success() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=success JOB_NAME="Deploy Production" python3 scripts/ci_notify.py + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Deploy Production" python3 scripts/ci_notify.py + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + production-e2e: + name: Production Browser E2E + runs-on: runtime-builder + timeout-minutes: 15 + if: startsWith(github.ref, 'refs/tags/v') + needs: deploy-production + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Run production browser E2E + shell: sh + run: | + set -eu + docker run --rm --ipc=host \ + -e E2E_BASE_URL=https://saas.xiaoxiajianji.com \ + -e E2E_API_BASE=https://api.xiaoxiajianji.com/api/v1 \ + -e E2E_BROWSER_CHANNEL=chromium \ + -e PLAYWRIGHT_HEADLESS=1 \ + -v "$PWD:/workspace" \ + -w /workspace/apps/web \ + git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy \ + sh -lc 'npm ci && npx playwright test --reporter=line --project=chromium e2e/auth.spec.ts e2e/auth-guard.spec.ts e2e/core-upload.spec.ts e2e/core-generation.spec.ts e2e/core-titles.spec.ts' + + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Production Browser E2E" python3 scripts/ci_notify.py + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + acr-cleanup: + name: ACR Image Cleanup + runs-on: runtime-builder + timeout-minutes: 10 + needs: + - deploy-staging + if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop') + env: + ACR_REGISTRY: xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com + ACR_NAMESPACE: xiaoxiakeji + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Run ACR cleanup + shell: sh + env: + ACR_USERNAME: ${{ secrets.ACR_USERNAME }} + ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} + PROTECTED_TAG: ${{ github.sha }} + run: | + set -eu + python3 scripts/ci/acr_cleanup.py \ + --keep 20 \ + --pr-days 7 \ + --execute + + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="ACR Image Cleanup" python3 scripts/ci_notify.py + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true \ No newline at end of file diff --git a/.gitea/workflows/ci-trigger-monitor.yml b/.gitea/workflows/ci-trigger-monitor.yml new file mode 100644 index 000000000..4a0292353 --- /dev/null +++ b/.gitea/workflows/ci-trigger-monitor.yml @@ -0,0 +1,52 @@ +name: CI Trigger Monitor + +on: + schedule: + - cron: '*/5 * * * *' # 每5分钟检查一次 + workflow_dispatch: + inputs: + stale_threshold: + description: 'CI未触发告警阈值（分钟）' + required: false + default: '5' + +permissions: + contents: read + +jobs: + monitor: + name: Monitor CI Trigger Reliability + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout code + uses: actions/checkout@v3 + # 网络波动自动重试2次 + retry: + max_attempts: 2 + retry_on: error + + - name: Check CI trigger status for all open PRs + env: + GITEA_API_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }} + GITEA_URL: https://git.xiaoxiajianji.com + GITEA_REPO: xiaoxia/xiaoxia-saas + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + STALE_THRESHOLD_MIN: ${{ inputs.stale_threshold || 5 }} + run: | + set +e + python3 scripts/ci_trigger_monitor.py + # 监控脚本永远不fail，避免告警风暴 + exit 0 + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + diff --git a/.gitea/workflows/code-review.yml b/.gitea/workflows/code-review.yml new file mode 100644 index 000000000..4a0000285 --- /dev/null +++ b/.gitea/workflows/code-review.yml @@ -0,0 +1,79 @@ +name: AI Code Review + +on: + pull_request: + types: + - opened + - synchronize + - reopened + +# 同一个 PR 只跑一个 review，新的取消旧的 +concurrency: + group: code-review-${{ gitea.repository }}-${{ gitea.event.pull_request.number }} + cancel-in-progress: true + +jobs: + code-review: + name: AI Code Review + runs-on: ubuntu-latest + # 跳过草稿 PR + if: ${{ !gitea.event.pull_request.draft }} + + steps: + # actions/checkout 由 runner 在宿主机层面处理，不受容器网络影响 + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + # 网络波动自动重试2次 + retry: + max_attempts: 2 + retry_on: error + + - name: Install dependencies + run: | + # 确保 python3-pip 可用（兼容不同基础镜像） + if ! python3 -m pip --version >/dev/null 2>&1; then + apt-get update -qq && apt-get install -y -qq python3-pip python3-venv >/dev/null 2>&1 + fi + # 部分镜像 ensurepip 方式兜底 + if ! python3 -m pip --version >/dev/null 2>&1; then + python3 -m ensurepip --upgrade 2>/dev/null || curl -sS https://bootstrap.pypa.io/get-pip.py | python3 + fi + python3 -m pip install --upgrade pip + python3 -m pip install requests + + - name: Run AI Code Review + env: + # Gitea 配置（自动从运行环境获取） + GITEA_API_URL: ${{ gitea.server_url }} + GITEA_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }} + REPO_NAME: ${{ gitea.repository }} + PR_NUMBER: ${{ gitea.event.pull_request.number }} + # LLM 提供商: coze (扣子原生Bot) / openai (OpenAI兼容) + LLM_PROVIDER: "coze" + # 扣子模式配置（默认国内站 api.coze.cn） + LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + COZE_BOT_ID: ${{ secrets.COZE_BOT_ID }} + LLM_MODEL: ${{ secrets.LLM_MODEL }} + # 可选参数 + MAX_DIFF_CHARS: "30000" + LLM_TIMEOUT: "120" + run: | + python3 scripts/ci_code_review.py + # 审查脚本异常不影响 CI 通过 + continue-on-error: true + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + diff --git a/.gitea/workflows/daily-check.yml b/.gitea/workflows/daily-check.yml new file mode 100644 index 000000000..b352213db --- /dev/null +++ b/.gitea/workflows/daily-check.yml @@ -0,0 +1,718 @@ +name: Daily Health Check + +on: + schedule: + - cron: '0 19 * * *' # UTC 19:00 = 北京时间凌晨 3:00 + workflow_dispatch: + +permissions: + contents: read + +jobs: + # ── 1. 生产环境冒烟测试 ───────────────────────────────────────────── + production-smoke: + name: Production Smoke Test + runs-on: saas + timeout-minutes: 8 + outputs: + report: ${{ steps.smoke.outputs.report }} + + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -eu + python3 - <<'PY' + import io, os, tarfile, time, urllib.request, urllib.error + url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" + request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' + for member in tar.getmembers(): + name = member.name + if name == root_prefix[:-1]: + continue + if name.startswith(root_prefix): + member.name = name[len(root_prefix):] + if member.name: + tar.extract(member, '.') + PY + + - name: Production health check & smoke test + id: smoke + shell: sh + env: + SMOKE_ENV: production + EXISTING_TOKEN: ${{ secrets.PROD_E2E_TOKEN }} + MODULES: health,assets,generation,subscription,nginx + run: | + set +e + START_TIME=$(date +%s) + chmod +x tests/e2e/api_smoke_test.sh + BASE_URL="https://api.xiaoxiajianji.com" \ + WEB_URL="https://saas.xiaoxiajianji.com" \ + SMOKE_ENV="${SMOKE_ENV}" \ + EXISTING_TOKEN="${EXISTING_TOKEN}" \ + MODULES="${MODULES}" \ + CLEANUP_ENABLED=0 \ + PERF_CHECK_ENABLED=1 \ + PERF_WARN_THRESHOLD_MS=500 \ + PERF_FAIL_THRESHOLD_MS=5000 \ + bash tests/e2e/api_smoke_test.sh 2>&1 | tee /tmp/prod-smoke.log + SMOKE_EXIT=${PIPESTATUS[0]} + END_TIME=$(date +%s) + ELAPSED=$((END_TIME - START_TIME)) + + echo "" + echo "========== 生产冒烟测试报告 ==========" + echo "环境: https://api.xiaoxiajianji.com" + echo "耗时: ${ELAPSED}s" + # 提取通过/失败数 + grep "测试完成:" /tmp/prod-smoke.log || true + if [ "$SMOKE_EXIT" -eq 0 ]; then + echo "结果: PASS" + echo "report=PASS" >> "${GITHUB_OUTPUT}" + else + echo "结果: FAIL" + grep "失败用例:" /tmp/prod-smoke.log || true + echo "report=FAIL" >> "${GITHUB_OUTPUT}" + fi + echo "======================================" + exit $SMOKE_EXIT + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + # ── 2. Staging API 集成测试 ───────────────────────────────────────── + staging-api-tests: + name: Staging API Integration Tests + runs-on: saas + timeout-minutes: 10 + outputs: + report: ${{ steps.smoke.outputs.report }} + + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -eu + python3 - <<'PY' + import io, os, tarfile, time, urllib.request, urllib.error + url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" + request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' + for member in tar.getmembers(): + name = member.name + if name == root_prefix[:-1]: + continue + if name.startswith(root_prefix): + member.name = name[len(root_prefix):] + if member.name: + tar.extract(member, '.') + PY + + - name: Run API smoke test on staging + id: smoke + shell: sh + run: | + set +e + START_TIME=$(date +%s) + chmod +x tests/e2e/api_smoke_test.sh + docker run --rm \ + -e BASE_URL=https://staging-api.xiaoxiajianji.com \ + -e WEB_URL=https://staging.xiaoxiajianji.com \ + -e TEST_USER=18314979086@163.com \ + -e TEST_PASSWORD=Ying1234 \ + -e CLEANUP_ENABLED=1 \ + -e PERF_CHECK_ENABLED=1 \ + -e PERF_WARN_THRESHOLD_MS=500 \ + -e PERF_FAIL_THRESHOLD_MS=3000 \ + -v "$PWD:/workspace" \ + -w /workspace \ + git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy \ + bash tests/e2e/api_smoke_test.sh 2>&1 | tee /tmp/staging-api-smoke.log + SMOKE_EXIT=${PIPESTATUS[0]} + END_TIME=$(date +%s) + ELAPSED=$((END_TIME - START_TIME)) + + echo "" + echo "========== Staging API 冒烟测试报告 ==========" + echo "环境: https://staging-api.xiaoxiajianji.com" + echo "耗时: ${ELAPSED}s" + grep "测试完成:" /tmp/staging-api-smoke.log || true + if [ "$SMOKE_EXIT" -eq 0 ]; then + echo "结果: PASS" + echo "api_report=PASS" >> "${GITHUB_OUTPUT}" + else + echo "结果: FAIL" + grep "失败用例:" /tmp/staging-api-smoke.log || true + echo "api_report=FAIL" >> "${GITHUB_OUTPUT}" + fi + echo "==============================================" + exit $SMOKE_EXIT + + - name: Run Staging API Integration Tests (Playwright) + id: e2e_api + shell: sh + run: | + set +e + START_TIME=$(date +%s) + docker run --rm \ + -e E2E_BASE_URL=https://staging.xiaoxiajianji.com \ + -e E2E_API_BASE=https://staging-api.xiaoxiajianji.com/api/v1 \ + -v "$PWD:/workspace" \ + -w /workspace/apps/web \ + git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy \ + sh -lc "npm ci && npx playwright test --reporter=line e2e/test_auth.spec.ts e2e/test_asset.spec.ts e2e/test_project.spec.ts" 2>&1 | tee /tmp/staging-api-e2e.log + EXIT_CODE=${PIPESTATUS[0]} + END_TIME=$(date +%s) + ELAPSED=$((END_TIME - START_TIME)) + + echo "" + echo "========== Staging API 集成测试报告 ==========" + echo "环境: https://staging-api.xiaoxiajianji.com" + echo "耗时: ${ELAPSED}s" + grep -E "passed|failed|timed out" /tmp/staging-api-e2e.log || true + if [ "$EXIT_CODE" -eq 0 ]; then + echo "结果: PASS" + echo "int_report=PASS" >> "${GITHUB_OUTPUT}" + else + echo "结果: FAIL" + echo "int_report=FAIL" >> "${GITHUB_OUTPUT}" + fi + echo "==============================================" + exit $EXIT_CODE + + - name: Set report output + id: report + shell: sh + run: | + if [ "${{ steps.smoke.outputs.api_report }}" = "PASS" ] && [ "${{ steps.e2e_api.outputs.int_report }}" = "PASS" ]; then + echo "report=PASS" >> "${GITHUB_OUTPUT}" + else + echo "report=FAIL" >> "${GITHUB_OUTPUT}" + fi + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + # ── 3. Staging 浏览器 E2E ────────────────────────────────────────── + staging-e2e: + name: Staging Browser E2E + runs-on: saas + timeout-minutes: 15 + outputs: + report: ${{ steps.smoke.outputs.report }} + + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -eu + python3 - <<'PY' + import io, os, tarfile, time, urllib.request, urllib.error + url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" + request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' + for member in tar.getmembers(): + name = member.name + if name == root_prefix[:-1]: + continue + if name.startswith(root_prefix): + member.name = name[len(root_prefix):] + if member.name: + tar.extract(member, '.') + PY + + - name: Run Playwright E2E on staging + id: e2e + shell: sh + run: | + set +e + START_TIME=$(date +%s) + docker run --rm --ipc=host \ + -e E2E_BASE_URL=https://staging.xiaoxiajianji.com \ + -e E2E_API_BASE=https://staging-api.xiaoxiajianji.com/api/v1 \ + -e E2E_BROWSER_CHANNEL=chromium \ + -e PLAYWRIGHT_HEADLESS=1 \ + -v "$PWD:/workspace" \ + -w /workspace/apps/web \ + git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy \ + sh -lc 'npm ci && npx playwright test --reporter=line --project=chromium e2e/auth.spec.ts e2e/auth-guard.spec.ts e2e/core-upload.spec.ts e2e/core-generation.spec.ts e2e/core-titles.spec.ts' 2>&1 | tee /tmp/staging-e2e.log + EXIT_CODE=${PIPESTATUS[0]} + END_TIME=$(date +%s) + ELAPSED=$((END_TIME - START_TIME)) + + echo "" + echo "========== Staging E2E 测试报告 ==========" + echo "环境: https://staging.xiaoxiajianji.com" + echo "耗时: ${ELAPSED}s" + grep -E "passed|failed|timed out" /tmp/staging-e2e.log || true + if [ "$EXIT_CODE" -eq 0 ]; then + echo "结果: PASS" + echo "report=PASS" >> "${GITHUB_OUTPUT}" + else + echo "结果: FAIL" + echo "report=FAIL" >> "${GITHUB_OUTPUT}" + fi + echo "==========================================" + exit $EXIT_CODE + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + # ── 4. 性能基线巡检 ──────────────────────────────────────────────── + performance-check: + name: Performance Baseline Check + runs-on: saas + timeout-minutes: 8 + outputs: + report: ${{ steps.report.outputs.report }} + + steps: + - name: Run performance baseline checks + id: perf + shell: sh + run: | + set +e + START_TIME=$(date +%s) + echo "==========================================" + echo " 性能基线巡检 - Staging API" + echo " 目标: https://staging-api.xiaoxiajianji.com" + echo "==========================================" + echo "" + + TOTAL=0 + PASS=0 + FAIL=0 + WARN=0 + WARN_LIST="" + FAIL_LIST="" + + # 核心接口配置: 名称|路径|方法|阈值(ms)|失败阈值(ms) + # 核心接口(core): 500ms + # 普通接口(normal): 1000ms + # 重操作接口(heavy): 3000ms + ENDPOINTS=" + 登录|/api/v1/auth/login|POST|500|3000 + 获取当前用户|/api/v1/auth/me|GET|500|3000 + 项目列表|/api/v1/projects|GET|500|3000 + 素材列表|/api/v1/assets|GET|500|3000 + 模板列表|/api/v1/templates|GET|500|3000 + 剪辑计划列表|/api/v1/edit-plans|GET|500|3000 + 生成任务列表|/api/v1/generation/tasks|GET|500|3000 + 订阅信息|/api/v1/subscription/current|GET|500|3000 + 音色列表|/api/v1/voices|GET|1000|5000 + 健康检查|/health|GET|200|1000 + " + + # 先登录获取 token + echo "--- 准备: 获取测试 Token ---" + AUTH_RESP=$(curl -s -w "\n%{http_code}" -X POST \ + -H "Content-Type: application/json" \ + -d '{"email":"18314979086@163.com","password":"Ying1234"}' \ + "https://staging-api.xiaoxiajianji.com/api/v1/auth/login" \ + --max-time 10 2>&1) + AUTH_CODE=$(echo "$AUTH_RESP" | tail -1) + AUTH_BODY=$(echo "$AUTH_RESP" | sed '$d') + + if [ "$AUTH_CODE" = "200" ]; then + TOKEN=$(echo "$AUTH_BODY" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('access_token',''))" 2>/dev/null) + if [ -n "$TOKEN" ]; then + echo "Token 获取成功" + else + echo "Token 解析失败，部分接口可能无法测试" + TOKEN="" + fi + else + echo "登录失败 (HTTP $AUTH_CODE)，部分接口将跳过鉴权测试" + TOKEN="" + fi + + echo "" + echo "--- 开始性能测试 ---" + echo "" + + echo "$ENDPOINTS" | while IFS='|' read -r name path method warn_ms fail_ms; do + [ -z "$name" ] && continue + TOTAL=$((TOTAL + 1)) + + # 构建 curl 命令 + CURL_ARGS="-s -o /dev/null -w '%{http_code} %{time_total}' --max-time 30" + if [ "$method" = "POST" ]; then + CURL_ARGS="$CURL_ARGS -X POST -H 'Content-Type: application/json' -d '{\"email\":\"18314979086@163.com\",\"password\":\"Ying1234\"}'" + fi + if [ -n "$TOKEN" ] && [ "$name" != "健康检查" ]; then + CURL_ARGS="$CURL_ARGS -H 'Authorization: Bearer $TOKEN'" + fi + + # 执行请求 + RESP=$(eval curl $CURL_ARGS "https://staging-api.xiaoxiajianji.com${path}" 2>&1) + HTTP_CODE=$(echo "$RESP" | awk '{print $1}') + TIME_TOTAL=$(echo "$RESP" | awk '{print $2}') + ELAPSED_MS=$(python3 -c "print(int(float('${TIME_TOTAL:-0}') * 1000))" 2>/dev/null || echo "0") + + if [ "$HTTP_CODE" -ge 500 ] 2>/dev/null; then + FAIL=$((FAIL + 1)) + FAIL_LIST="$FAIL_LIST\n ❌ $name - HTTP $HTTP_CODE (${ELAPSED_MS}ms)" + echo "❌ $name - HTTP $HTTP_CODE - ${ELAPSED_MS}ms (FAIL)" + elif [ "$ELAPSED_MS" -ge "$fail_ms" ] 2>/dev/null; then + FAIL=$((FAIL + 1)) + FAIL_LIST="$FAIL_LIST\n ❌ $name - ${ELAPSED_MS}ms > ${fail_ms}ms" + echo "❌ $name - ${ELAPSED_MS}ms > ${fail_ms}ms (FAIL)" + elif [ "$ELAPSED_MS" -ge "$warn_ms" ] 2>/dev/null; then + WARN=$((WARN + 1)) + WARN_LIST="$WARN_LIST\n ⚠️ $name - ${ELAPSED_MS}ms > ${warn_ms}ms" + echo "⚠️ $name - ${ELAPSED_MS}ms (WARN, threshold: ${warn_ms}ms)" + PASS=$((PASS + 1)) + else + PASS=$((PASS + 1)) + echo "✅ $name - ${ELAPSED_MS}ms (OK, threshold: ${warn_ms}ms)" + fi + done + + # 由于 while 在子 shell 中执行，用文件传递结果 + # 重新跑一次用文件计数方式 + echo "" + echo "--- 汇总性能数据 ---" + + END_TIME=$(date +%s) + ELAPSED=$((END_TIME - START_TIME)) + + echo "" + echo "========== 性能基线巡检报告 ==========" + echo "环境: https://staging-api.xiaoxiajianji.com" + echo "耗时: ${ELAPSED}s" + echo "======================================" + + - name: Generate performance report + id: report + shell: sh + run: | + set +e + echo "" + echo "==========================================" + echo " 性能基线巡检 - 详细报告" + echo "==========================================" + + TOTAL=0 + PASS=0 + FAIL=0 + WARN=0 + RESULTS="" + START_TIME=$(date +%s) + + # 先登录获取 token + AUTH_RESP=$(curl -s -w "\n%{http_code}" -X POST \ + -H "Content-Type: application/json" \ + -d '{"email":"18314979086@163.com","password":"Ying1234"}' \ + "https://staging-api.xiaoxiajianji.com/api/v1/auth/login" \ + --max-time 10 2>&1) + AUTH_CODE=$(echo "$AUTH_RESP" | tail -1) + AUTH_BODY=$(echo "$AUTH_RESP" | sed '$d') + TOKEN="" + if [ "$AUTH_CODE" = "200" ]; then + TOKEN=$(echo "$AUTH_BODY" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('access_token',''))" 2>/dev/null || echo "") + fi + + run_perf_test() { + local name="$1" path="$2" method="$3" warn_ms="$4" fail_ms="$5" + TOTAL=$((TOTAL + 1)) + + local CURL_ARGS="-s -o /dev/null -w '%{http_code} %{time_total}' --max-time 30" + if [ "$method" = "POST" ]; then + CURL_ARGS="$CURL_ARGS -X POST -H 'Content-Type: application/json' -d '{\"email\":\"18314979086@163.com\",\"password\":\"Ying1234\"}'" + fi + if [ -n "$TOKEN" ] && [ "$name" != "健康检查" ]; then + CURL_ARGS="$CURL_ARGS -H 'Authorization: Bearer $TOKEN'" + fi + + local RESP=$(eval curl $CURL_ARGS "https://staging-api.xiaoxiajianji.com${path}" 2>&1) + local HTTP_CODE=$(echo "$RESP" | awk '{print $1}') + local TIME_TOTAL=$(echo "$RESP" | awk '{print $2}') + local ELAPSED_MS=$(python3 -c "print(int(float('${TIME_TOTAL:-0}') * 1000))" 2>/dev/null || echo "0") + + if echo "$HTTP_CODE" | grep -q "^[5]"; then + FAIL=$((FAIL + 1)) + RESULTS="$RESULTS\n ❌ $name - HTTP $HTTP_CODE (${ELAPSED_MS}ms)" + echo "❌ $name - HTTP $HTTP_CODE - ${ELAPSED_MS}ms [FAIL]" + return 1 + elif [ "$ELAPSED_MS" -ge "$fail_ms" ] 2>/dev/null; then + FAIL=$((FAIL + 1)) + RESULTS="$RESULTS\n ❌ $name - ${ELAPSED_MS}ms > ${fail_ms}ms [FAIL]" + echo "❌ $name - ${ELAPSED_MS}ms > ${fail_ms}ms [FAIL]" + return 1 + elif [ "$ELAPSED_MS" -ge "$warn_ms" ] 2>/dev/null; then + WARN=$((WARN + 1)) + PASS=$((PASS + 1)) + RESULTS="$RESULTS\n ⚠️ $name - ${ELAPSED_MS}ms (阈值: ${warn_ms}ms) [WARN]" + echo "⚠️ $name - ${ELAPSED_MS}ms > 阈值 ${warn_ms}ms [WARN]" + return 0 + else + PASS=$((PASS + 1)) + RESULTS="$RESULTS\n ✅ $name - ${ELAPSED_MS}ms (阈值: ${warn_ms}ms) [OK]" + echo "✅ $name - ${ELAPSED_MS}ms (阈值: ${warn_ms}ms) [OK]" + return 0 + fi + } + + echo "" + echo "=== 核心接口 (阈值: 500ms / 3000ms) ===" + run_perf_test "登录" "/api/v1/auth/login" "POST" 500 3000 || true + run_perf_test "获取当前用户" "/api/v1/auth/me" "GET" 500 3000 || true + run_perf_test "项目列表" "/api/v1/projects" "GET" 500 3000 || true + run_perf_test "素材列表" "/api/v1/assets" "GET" 500 3000 || true + run_perf_test "模板列表" "/api/v1/templates" "GET" 500 3000 || true + run_perf_test "剪辑计划列表" "/api/v1/edit-plans" "GET" 500 3000 || true + run_perf_test "生成任务列表" "/api/v1/generation/tasks" "GET" 500 3000 || true + run_perf_test "订阅信息" "/api/v1/subscription/current" "GET" 500 3000 || true + + echo "" + echo "=== 普通接口 (阈值: 1000ms / 5000ms) ===" + run_perf_test "音色列表" "/api/v1/voices" "GET" 1000 5000 || true + + echo "" + echo "=== 基础接口 (阈值: 200ms / 1000ms) ===" + run_perf_test "健康检查" "/health" "GET" 200 1000 || true + + END_TIME=$(date +%s) + ELAPSED=$((END_TIME - START_TIME)) + + echo "" + echo "========== 性能基线巡检报告 ==========" + echo "环境: https://staging-api.xiaoxiajianji.com" + echo "总接口: ${TOTAL}" + echo "通过: ${PASS}" + echo "失败: ${FAIL}" + echo "警告: ${WARN}" + echo "耗时: ${ELAPSED}s" + echo "======================================" + + # 写入结果文件供 report job 使用 + echo "${TOTAL}" > /tmp/perf_total + echo "${PASS}" > /tmp/perf_pass + echo "${FAIL}" > /tmp/perf_fail + echo "${WARN}" > /tmp/perf_warn + echo "${ELAPSED}" > /tmp/perf_elapsed + + if [ "$FAIL" -gt 0 ]; then + echo "report=FAIL" >> "${GITHUB_OUTPUT}" + echo "perf_detail=fail:${FAIL}:warn:${WARN}" >> "${GITHUB_OUTPUT}" + exit 1 + else + echo "report=PASS" >> "${GITHUB_OUTPUT}" + if [ "$WARN" -gt 0 ]; then + echo "perf_detail=pass:warn:${WARN}" >> "${GITHUB_OUTPUT}" + else + echo "perf_detail=pass" >> "${GITHUB_OUTPUT}" + fi + exit 0 + fi + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + # ── 5. 每日巡检汇总报告 ──────────────────────────────────────────── + daily-report: + name: Daily Check Report + runs-on: saas + timeout-minutes: 2 + if: always() + needs: + - production-smoke + - staging-api-tests + - staging-e2e + - performance-check + + steps: + - name: Print summary report + shell: sh + run: | + echo "" + echo "╔══════════════════════════════════════════════════════╗" + echo "║ 每日巡检报告 ║" + echo "╠══════════════════════════════════════════════════════╣" + + # 获取各 job 状态 + PROD_STATUS="${{ needs.production-smoke.result }}" + STAGING_API_STATUS="${{ needs.staging-api-tests.result }}" + STAGING_E2E_STATUS="${{ needs.staging-e2e.result }}" + PERF_STATUS="${{ needs.performance-check.result }}" + + format_result() { + if [ "$1" = "success" ]; then + echo "✅ PASS" + elif [ "$1" = "failure" ]; then + echo "❌ FAIL" + elif [ "$1" = "skipped" ]; then + echo "⏭️ SKIP" + else + echo "❓ UNKNOWN ($1)" + fi + } + + echo "║" + echo "║ 生产冒烟测试: $(format_result "$PROD_STATUS")" + echo "║ Staging API: $(format_result "$STAGING_API_STATUS")" + echo "║ Staging E2E: $(format_result "$STAGING_E2E_STATUS")" + echo "║ 性能基线巡检: $(format_result "$PERF_STATUS")" + echo "║" + echo "║ 巡检时间: $(date '+%Y-%m-%d %H:%M:%S UTC')" + echo "║" + + # 判断整体状态 + ALL_PASS=true + FAILED_ITEMS="" + for status_name in "$PROD_STATUS:生产冒烟" "$STAGING_API_STATUS:Staging API" "$STAGING_E2E_STATUS:Staging E2E" "$PERF_STATUS:性能基线"; do + STATUS=$(echo "$status_name" | cut -d: -f1) + NAME=$(echo "$status_name" | cut -d: -f2) + if [ "$STATUS" != "success" ] && [ "$STATUS" != "skipped" ]; then + ALL_PASS=false + FAILED_ITEMS="$FAILED_ITEMS $NAME" + fi + done + + echo "╠══════════════════════════════════════════════════════╣" + if [ "$ALL_PASS" = "true" ]; then + echo "║ 整体状态: ✅ 全部通过 ║" + else + echo "║ 整体状态: ❌ 存在失败 ║" + echo "║ 失败项: ${FAILED_ITEMS} ║" + fi + echo "╚══════════════════════════════════════════════════════╝" + echo "" + + # 如果有失败项，以非零退出码结束（方便 Gitea 标记流水线失败） + if [ "$ALL_PASS" = "false" ]; then + echo "⚠️ 部分巡检项失败，请检查上方日志获取详细信息。" + # 不 exit 1，因为我们用了 always()，保持 report job 成功， + # 但其他失败的 job 已经让整体流水线标记为失败 + fi + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + diff --git a/.gitea/workflows/pr-auto-scan.yml b/.gitea/workflows/pr-auto-scan.yml new file mode 100644 index 000000000..8621ce69e --- /dev/null +++ b/.gitea/workflows/pr-auto-scan.yml @@ -0,0 +1,56 @@ +name: PR Auto Scan +# 定时扫描所有open PR，对CI全绿的触发审批/合并 +# 作为短作业模式的兜底，防止事件驱动遗漏 +on: + schedule: + - cron: "*/5 * * * *" # 每5分钟扫描一次 + workflow_dispatch: + +permissions: + contents: read + +jobs: + auto-scan: + name: Auto Scan Open PRs + runs-on: ci-check + timeout-minutes: 5 + if: github.repository == 'xiaoxia/xiaoxia-saas' + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/pr_auto_scan.py?ref=develop" -o /tmp/pr_auto_scan.py + python3 /tmp/pr_auto_scan.py --help > /dev/null 2>&1 || { + # fallback: checkout + echo "使用checkout方式" + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=develop" | bash + } + + - name: Scan and auto process PRs + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + REVIEW_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }} + MERGE_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }} + run: | + set -eu + echo "=== 扫描所有open PR并自动处理 ===" + echo "时间: $(date)" + echo + + python3 /tmp/pr_auto_scan.py --token "$REVIEW_TOKEN" --repo "$GITHUB_REPOSITORY" --base develop --approve --merge --dry-run false + + echo "" + echo "✅ 扫描完成" + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "" || true diff --git a/.gitea/workflows/pr-automation.yml b/.gitea/workflows/pr-automation.yml new file mode 100755 index 000000000..663576e21 --- /dev/null +++ b/.gitea/workflows/pr-automation.yml @@ -0,0 +1,113 @@ +name: PR Automation + +on: + pull_request: + types: [synchronize, opened, ready_for_review, review_requested] + workflow_dispatch: + +permissions: + contents: read + +jobs: + auto-approve: + name: Auto Approve on CI Green + runs-on: ci-check + if: github.event_name == 'pull_request' && !github.event.pull_request.draft + timeout-minutes: 3 # 长等待模式：等CI全绿后自动合并，不遗漏任何PR + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + + - name: "🔍 脚本语法自检" + shell: bash + run: | + ERROR=0 + for f in scripts/ci/*.sh; do [ -f "$f" ] && bash -n "$f" 2>&1 || ERROR=$((ERROR+1)); done + for f in scripts/ci/*.py; do [ -f "$f" ] && python3 -m py_compile "$f" 2>&1 || ERROR=$((ERROR+1)); done + if [ "$ERROR" -ne 0 ]; then echo "❌ 语法自检失败 ($ERROR个)"; exit 1; fi + echo "✅ 脚本语法自检通过" + + - name: Auto approve when CI passes + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + REVIEW_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + bash scripts/ci/auto_approve.sh + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + auto-merge: + name: Auto Merge on CI Green + Approved + runs-on: ci-check + if: github.event_name == 'pull_request' && !github.event.pull_request.draft && github.event.pull_request.base.ref == 'develop' + timeout-minutes: 45 # 长等待模式：等CI全绿后自动合并，不遗漏任何PR + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + + - name: "🔍 脚本语法自检（防止脚本bug导致所有PR挂掉）" + shell: bash + run: | + echo "=== CI脚本语法自检 ===" + ERROR=0 + for f in scripts/ci/*.sh; do + [ -f "$f" ] || continue + if ! bash -n "$f" 2>&1; then + echo "FAIL: $f" + ERROR=1 + fi + done + for f in scripts/ci/*.py; do + [ -f "$f" ] || continue + if ! python3 -m py_compile "$f" 2>&1; then + echo "FAIL: $f" + ERROR=1 + fi + done + if [ "$ERROR" -ne 0 ]; then + echo "❌ 脚本语法自检失败" + exit 1 + fi + echo "✅ 所有CI脚本语法自检通过" + + - name: Auto merge when CI passes and approved + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + MERGE_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + bash scripts/ci/auto_merge.sh + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true diff --git a/.gitea/workflows/preview-cleanup.yml b/.gitea/workflows/preview-cleanup.yml new file mode 100755 index 000000000..a1977ab08 --- /dev/null +++ b/.gitea/workflows/preview-cleanup.yml @@ -0,0 +1,207 @@ +name: Preview Cleanup +on: + pull_request: + types: + - closed + branches: + - main + - develop +permissions: + contents: read + pull-requests: write +jobs: + cleanup-preview: + name: Cleanup Preview Environment + runs-on: runtime-builder + timeout-minutes: 10 + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -eu + python3 - <<'PY' + import io, os, tarfile, time, urllib.request, urllib.error + url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" + request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' + for member in tar.getmembers(): + name = member.name + if name == root_prefix[:-1]: + continue + if name.startswith(root_prefix): + member.name = name[len(root_prefix):] + if member.name: + tar.extract(member, '.') + PY + + - name: Extract PR number + shell: sh + run: | + set -eu + # 优先从event payload中读取（兼容所有PR事件类型） + if [ -n "${GITHUB_EVENT_PATH:-}" ] && [ -f "$GITHUB_EVENT_PATH" ]; then + PR_NUMBER=$(python3 -c "import json,sys; print(json.load(sys.stdin).get('number',''))" < "$GITHUB_EVENT_PATH") + fi + # fallback: 从GITHUB_REF中提取 + if [ -z "${PR_NUMBER:-}" ]; then + PR_NUMBER=$(echo "$GITHUB_REF" | sed -n 's|refs/pull/\([0-9]*\)/.*|\1|p') + fi + # 再fallback: 兼容纯数字ref + if [ -z "${PR_NUMBER:-}" ] || ! echo "$PR_NUMBER" | grep -qE '^[0-9]+$'; then + echo "WARNING: Could not extract PR number cleanly, using raw ref suffix" + PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') + fi + echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV + echo "PR number: $PR_NUMBER" + echo "Preview dir: /var/www/preview/pr-${PR_NUMBER}" + + - name: Install SSH client + shell: sh + run: | + set -eu + # 先检查是否已存在ssh + if command -v ssh >/dev/null 2>&1 && command -v ssh-keyscan >/dev/null 2>&1; then + echo "SSH client already available: $(ssh -V 2>&1)" + exit 0 + fi + # 尝试多种包管理器安装 + if command -v apk >/dev/null 2>&1; then + apk add --no-cache openssh-client >/dev/null 2>&1 + echo "openssh-client installed via apk" + elif command -v apt-get >/dev/null 2>&1; then + apt-get update -qq && apt-get install -y -qq openssh-client >/dev/null 2>&1 + echo "openssh-client installed via apt-get" + elif command -v yum >/dev/null 2>&1; then + yum install -y openssh-clients >/dev/null 2>&1 + echo "openssh-client installed via yum" + elif command -v dnf >/dev/null 2>&1; then + dnf install -y openssh-clients >/dev/null 2>&1 + echo "openssh-client installed via dnf" + else + echo "ERROR: No package manager found and ssh not pre-installed" + which ssh 2>/dev/null || echo " ssh: not found" + which ssh-keyscan 2>/dev/null || echo " ssh-keyscan: not found" + exit 1 + fi + + - name: Remove preview directory from server + shell: sh + env: + PREVIEW_SSH_HOST: ${{ secrets.PREVIEW_SSH_HOST }} + PREVIEW_SSH_USER: ${{ secrets.PREVIEW_SSH_USER }} + PREVIEW_SSH_PORT: ${{ secrets.PREVIEW_SSH_PORT }} + PREVIEW_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }} + run: | + set -eux + preview_host="${PREVIEW_SSH_HOST:-172.30.18.197}" + preview_user="${PREVIEW_SSH_USER:-deploy}" + preview_port="${PREVIEW_SSH_PORT:-22222}" + preview_dir="/var/www/preview/pr-${PR_NUMBER}" + + mkdir -p ~/.ssh + + # 查找可用的SSH密钥（优先用 secret 里专门为 preview 配置的 key） + key_path="" + if [ -n "${PREVIEW_SSH_KEY:-}" ]; then + key_path="$HOME/.ssh/id_ed25519" + printf '%s\n' "$PREVIEW_SSH_KEY" > "$key_path" + chmod 600 "$key_path" + echo "Using key from PREVIEW_SSH_KEY secret" + elif [ -f /root/.ssh/xiaoxia_runtime_builder ]; then + key_path="/root/.ssh/xiaoxia_runtime_builder" + echo "Using key: $key_path (builder key)" + elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then + key_path="$HOME/.ssh/xiaoxia_runtime_builder" + echo "Using key: $key_path (home key)" + else + echo "ERROR: No SSH key available" + ls -la ~/.ssh/ 2>/dev/null || true + ls -la /root/.ssh/ 2>/dev/null || true + exit 1 + fi + + ssh-keyscan -p "$preview_port" -H "$preview_host" >> ~/.ssh/known_hosts 2>/dev/null + echo "SSH keyscan done" + + # 测试SSH连接 + ssh -p "$preview_port" -i "$key_path" -o StrictHostKeyChecking=no "${preview_user}@${preview_host}" "echo SSH_CONNECTION_OK && hostname" + echo "SSH connection verified" + + # 检查目录是否存在 + DIR_EXISTS=$(ssh -p "$preview_port" -i "$key_path" -o StrictHostKeyChecking=no "${preview_user}@${preview_host}" \ + "if [ -d '${preview_dir}' ]; then echo 'yes'; else echo 'no'; fi") + + if [ "$DIR_EXISTS" = "yes" ]; then + echo "Removing preview directory: ${preview_dir}" + ssh -p "$preview_port" -i "$key_path" -o StrictHostKeyChecking=no "${preview_user}@${preview_host}" \ + "rm -rf ${preview_dir} && echo 'Preview directory removed successfully'" + echo "Cleanup completed: ${preview_dir}" + else + echo "Preview directory does not exist: ${preview_dir}, nothing to clean up" + fi + + - name: Comment cleanup notice on PR + if: success() + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -eu + # 从event payload读取PR号（最可靠） + if [ -n "${GITHUB_EVENT_PATH:-}" ] && [ -f "$GITHUB_EVENT_PATH" ]; then + PR_NUMBER=$(python3 -c "import json,sys; print(json.load(sys.stdin).get('number',''))" < "$GITHUB_EVENT_PATH") + else + PR_NUMBER=$(echo "$GITHUB_REF" | sed -n 's|refs/pull/\([0-9]*\)/.*|\1|p') + fi + export PR_NUMBER + + COMMENT_BODY=$(python3 scripts/ci/preview_comment.py cleanup) + + API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" + curl -s -X POST \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$COMMENT_BODY" \ + "$API_URL" \ + > /dev/null + echo "Cleanup comment posted" + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + diff --git a/.gitea/workflows/preview-deploy.yml b/.gitea/workflows/preview-deploy.yml new file mode 100755 index 000000000..22cd780ab --- /dev/null +++ b/.gitea/workflows/preview-deploy.yml @@ -0,0 +1,296 @@ +name: Preview Deploy +on: + pull_request: + types: + - opened + - synchronize + - reopened + branches: + - main + - develop + workflow_dispatch: + inputs: + reason: + description: "触发原因" + required: false + default: "手动触发 - 预览环境补跑" +permissions: + contents: read + pull-requests: write +concurrency: + group: preview-deploy-${{ gitea.ref }} + cancel-in-progress: true +jobs: + deploy-preview: + name: Deploy Preview Environment + runs-on: runtime-builder + timeout-minutes: 20 + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -eu + python3 - <<'PY' + import io, os, tarfile, time, urllib.request, urllib.error + url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" + request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' + for member in tar.getmembers(): + name = member.name + if name == root_prefix[:-1]: + continue + if name.startswith(root_prefix): + member.name = name[len(root_prefix):] + if member.name: + tar.extract(member, '.') + PY + + - name: Record job start time + shell: sh + run: | + set -eu + echo "JOB_START_TIME=$(date +%s)" >> $GITHUB_ENV + echo "Job started at $(date)" + + - name: Extract PR number + shell: sh + run: | + set -eu + PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') + echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV + echo "PR number: $PR_NUMBER" + echo "PREVIEW_URL=https://pr-${PR_NUMBER}.preview.xiaoxiajianji.com" >> $GITHUB_ENV + echo "Preview URL: https://pr-${PR_NUMBER}.preview.xiaoxiajianji.com" + + - name: Build frontend + shell: sh + run: | + set -eu + NPM_CACHE_VOLUME="xiaoxia-npm-cache" + if ! docker volume inspect "$NPM_CACHE_VOLUME" >/dev/null 2>&1; then + docker volume create "$NPM_CACHE_VOLUME" >/dev/null + echo "Created npm cache volume: $NPM_CACHE_VOLUME" + fi + + docker run --rm \ + -v "$PWD:/workspace" \ + -v "$NPM_CACHE_VOLUME:/workspace/apps/web/node_modules" \ + -w /workspace/apps/web \ + -e VITE_API_URL=https://staging-api.xiaoxiajianji.com \ + docker.m.daocloud.io/library/node:20 \ + sh -lc ' + PACKAGE_LOCK_HASH=$(md5sum package-lock.json 2>/dev/null | cut -d" " -f1) + CACHE_HASH_FILE="node_modules/.package-lock-hash" + CACHE_VALID=false + if [ -f "$CACHE_HASH_FILE" ] && [ "$(cat "$CACHE_HASH_FILE")" = "$PACKAGE_LOCK_HASH" ] && [ -x "node_modules/.bin/vite" ] && [ -x "node_modules/.bin/tsc" ]; then + CACHE_VALID=true + echo "Cache hit: dependencies valid, skipping npm ci" + fi + if [ "$CACHE_VALID" = "false" ]; then + echo "Cache miss or invalid: running npm ci..." + if ! npm ci --include=dev; then + echo "npm ci failed, cleaning node_modules and retrying..." + rm -rf node_modules + mkdir -p node_modules + npm ci --include=dev + fi + echo "$PACKAGE_LOCK_HASH" > "$CACHE_HASH_FILE" + echo "Dependencies installed, cache updated" + fi + echo "Running TypeScript check..." + npx --no-install tsc + echo "Running Vite build..." + npx --no-install vite build + echo "Build completed successfully" + ls -la dist/ + ' + + - name: Install SSH client and rsync + shell: sh + run: | + set -eu + if command -v apk >/dev/null 2>&1; then + apk add --no-cache openssh-client rsync >/dev/null 2>&1 + elif command -v apt-get >/dev/null 2>&1; then + apt-get update -qq && apt-get install -y -qq openssh-client rsync >/dev/null 2>&1 + elif command -v yum >/dev/null 2>&1; then + yum install -y openssh-clients rsync >/dev/null 2>&1 + else + echo "ERROR: No package manager found" + exit 1 + fi + echo "openssh-client and rsync installed" + + - name: Deploy preview to server + shell: sh + env: + PREVIEW_SSH_HOST: ${{ secrets.PREVIEW_SSH_HOST }} + PREVIEW_SSH_USER: ${{ secrets.PREVIEW_SSH_USER }} + PREVIEW_SSH_PORT: ${{ secrets.PREVIEW_SSH_PORT }} + PREVIEW_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }} + run: | + set -eux + preview_host="${PREVIEW_SSH_HOST:-47.98.113.167}" + preview_user="${PREVIEW_SSH_USER:-root}" + preview_port="${PREVIEW_SSH_PORT:-22222}" + preview_dir="/var/www/preview/pr-${PR_NUMBER}" + + mkdir -p ~/.ssh + + # 查找可用的SSH密钥（优先用 secret 里专门为 preview 配置的 key） + key_path="" + if [ -n "${PREVIEW_SSH_KEY:-}" ]; then + key_path="$HOME/.ssh/id_ed25519" + printf '%s\n' "$PREVIEW_SSH_KEY" > "$key_path" + chmod 600 "$key_path" + echo "Using key from PREVIEW_SSH_KEY secret" + elif [ -f /root/.ssh/xiaoxia_runtime_builder ]; then + key_path="/root/.ssh/xiaoxia_runtime_builder" + echo "Using key: $key_path (builder key)" + elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then + key_path="$HOME/.ssh/xiaoxia_runtime_builder" + echo "Using key: $key_path (home key)" + else + echo "ERROR: No SSH key available" + ls -la ~/.ssh/ 2>/dev/null || true + ls -la /root/.ssh/ 2>/dev/null || true + exit 1 + fi + + # SSH密钥完整性自检 + if ! ssh-keygen -y -f "$key_path" > /dev/null 2>&1; then + echo "ERROR: SSH密钥损坏（private key contents do not match public）" + echo "请检查 PREVIEW_SSH_KEY secret 中的私钥是否完整正确" + echo "私钥文件大小: $(wc -c < "$key_path") 字节" + head -2 "$key_path" + exit 1 + fi + echo "SSH key integrity check passed" + + ssh-keyscan -p "$preview_port" -H "$preview_host" >> ~/.ssh/known_hosts 2>/dev/null + echo "SSH keyscan done" + + # 测试SSH连接 + ssh -p "$preview_port" -i "$key_path" -o StrictHostKeyChecking=no "${preview_user}@${preview_host}" "echo SSH_CONNECTION_OK && hostname" + echo "SSH connection verified" + + # 创建预览目录并上传文件 + ssh -p "$preview_port" -i "$key_path" -o StrictHostKeyChecking=no "${preview_user}@${preview_host}" \ + "mkdir -p ${preview_dir} && echo 'Preview directory created: ${preview_dir}'" + + # 使用rsync上传dist目录内容 + rsync -avz --delete -e "ssh -p ${preview_port} -i ${key_path} -o StrictHostKeyChecking=no" \ + apps/web/dist/ \ + "${preview_user}@${preview_host}:${preview_dir}/" + + echo "Preview deployed to: ${preview_dir}" + echo "Preview URL: https://pr-${PR_NUMBER}.preview.xiaoxiajianji.com" + + - name: Comment preview link on PR + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -eu + PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') + PREVIEW_URL="https://pr-${PR_NUMBER}.preview.xiaoxiajianji.com" + export PR_NUMBER PREVIEW_URL + + COMMENT_BODY=$(python3 scripts/ci/preview_comment.py deploy) + + API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" + + EXISTING_COMMENT_ID=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c " + import sys, json + try: + for c in json.load(sys.stdin): + if '预览环境已部署' in c.get('body', ''): + print(c['id']) + break + except Exception: + pass + ") + + if [ -n "$EXISTING_COMMENT_ID" ]; then + curl -s -X PATCH \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$COMMENT_BODY" \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/issues/comments/${EXISTING_COMMENT_ID}" \ + > /dev/null + echo "Comment updated" + else + curl -s -X POST \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$COMMENT_BODY" \ + "$API_URL" \ + > /dev/null + echo "Comment posted" + fi + + - name: Job duration summary + if: always() + shell: sh + run: | + set +eu + if [ -n "$JOB_START_TIME" ]; then + END_TIME=$(date +%s) + DURATION=$((END_TIME - JOB_START_TIME)) + MINS=$((DURATION / 60)) + SECS=$((DURATION % 60)) + echo "JOB_DURATION_SECONDS=$DURATION" >> $GITHUB_ENV + echo "=== Job Duration: ${MINS}m${SECS}s ===" + else + echo "JOB_DURATION_SECONDS=0" >> $GITHUB_ENV + echo "=== Job Duration: unknown ===" + fi + + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Deploy Preview Environment" python3 scripts/ci_notify.py + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml deleted file mode 100755 index 773129aca..000000000 --- a/.gitea/workflows/tests.yml +++ /dev/null @@ -1,167 +0,0 @@ -name: Tests - -on: - pull_request: - branches: [ main ] - -jobs: - test: - runs-on: runtime-builder - - steps: - - name: Checkout code - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - set -eu - python3 - <<'PY' - import io - import os - import tarfile - import time - import urllib.error - import urllib.request - - url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" - request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - # Retry up to 5 times with backoff for transient 5xx errors - last_err = None - for attempt in range(5): - try: - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() - break - except urllib.error.HTTPError as e: - last_err = e - if e.code >= 500 and attempt < 4: - wait = 2 ** attempt - print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - except Exception as e: - last_err = e - if attempt < 4: - wait = 2 ** attempt - print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - else: - raise last_err - - with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: - root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' - for member in tar.getmembers(): - name = member.name - if name == root_prefix[:-1]: - continue - if name.startswith(root_prefix): - member.name = name[len(root_prefix):] - if member.name: - tar.extract(member, '.') - PY - - - name: Show Python version - shell: sh - run: | - set -eu - python --version - python -m pip --version - - - name: Install dependencies - shell: sh - run: | - set -eu - python -m pip install --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - python -m pip install -r requirements.txt -r requirements-dev.txt -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - - - name: Run unit tests - shell: sh - run: | - set -eu - PYTHONPATH="$PWD/apps/api:$PWD" python -m pytest tests/unit -q - - - name: Run integration tests - shell: sh - run: | - set -eu - PYTHONPATH="$PWD/apps/api:$PWD" python -m pytest tests/integration -q --timeout=60 -x - - lint: - runs-on: runtime-builder - - steps: - - name: Checkout code - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - set -eu - python3 - <<'PY' - import io - import os - import tarfile - import time - import urllib.error - import urllib.request - - url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" - request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - # Retry up to 5 times with backoff for transient 5xx errors - last_err = None - for attempt in range(5): - try: - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() - break - except urllib.error.HTTPError as e: - last_err = e - if e.code >= 500 and attempt < 4: - wait = 2 ** attempt - print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - except Exception as e: - last_err = e - if attempt < 4: - wait = 2 ** attempt - print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - else: - raise last_err - - with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: - root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' - for member in tar.getmembers(): - name = member.name - if name == root_prefix[:-1]: - continue - if name.startswith(root_prefix): - member.name = name[len(root_prefix):] - if member.name: - tar.extract(member, '.') - PY - - - name: Install dependencies - shell: sh - run: | - set -eu - python -m pip install --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - python -m pip install -r requirements.txt -r requirements-dev.txt -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - - - name: Run Black (check only) - shell: sh - run: | - set -eu - python -m black --check alembic apps packages tests scripts - - - name: Run Flake8 - shell: sh - run: | - set -eu - python -m flake8 apps packages tests --count --statistics diff --git a/.gitea/workflows/worker-base-image.yml b/.gitea/workflows/worker-base-image.yml new file mode 100644 index 000000000..c07286aa8 --- /dev/null +++ b/.gitea/workflows/worker-base-image.yml @@ -0,0 +1,103 @@ +name: Worker Base Image Build + +on: + push: + branches: + - develop + - main + paths: + - 'requirements-base.txt' + - 'requirements-worker.txt' + - 'infra/docker/worker-base-builder.Dockerfile' + - 'infra/docker/worker-base-runtime.Dockerfile' + workflow_dispatch: # 支持手动触发 + +jobs: + build-worker-base: + name: Build Worker Base Images + runs-on: runtime-builder + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - name: builder + dockerfile: infra/docker/worker-base-builder.Dockerfile + image_name: worker-base-builder + cache_name: worker-base-builder-cache + - name: runtime + dockerfile: infra/docker/worker-base-runtime.Dockerfile + image_name: worker-base-runtime + cache_name: worker-base-runtime-cache + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" | bash + + - name: Docker login to Registry + shell: sh + env: + ACR_USERNAME: ${{ secrets.ACR_USERNAME }} + ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} + GITEA_REGISTRY_USER: xiaoxia + GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + set -eu + for i in 1 2 3; do + echo "=== Docker login 尝试 $i/3 ===" + if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then + echo "✅ Docker login successful" + break + fi + echo "❌ Docker login 失败（尝试 $i/3），5s 后重试..." + sleep 5 + done + + - name: Setup buildx builder + shell: sh + run: | + set -eu + BUILDER_NAME="ci-builder-${GITHUB_RUN_ID}-${{ matrix.name }}" + if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then + docker buildx create --use --name "$BUILDER_NAME" --driver docker-container + echo "Created $BUILDER_NAME" + else + docker buildx use "$BUILDER_NAME" + echo "Using existing $BUILDER_NAME" + fi + docker buildx inspect --bootstrap + + - name: Build and push base image + shell: sh + run: | + set -eu + REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji" + IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:latest" + SAFE_REF_NAME=$(echo "${GITHUB_REF_NAME}" | tr '/' '-') + CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:${SAFE_REF_NAME}" + + echo "=== Building ${{ matrix.name }} base image ===" + echo "Image: ${IMAGE_TAG}" + echo "Cache: ${CACHE_REF}" + + # 用通用构建脚本 + bash scripts/ci/docker_build_push.sh ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" + + # 同时推送到 Gitea Packages 作为备份（可选） + GITEA_IMAGE="git.xiaoxiajianji.com/xiaoxia-saas/${{ matrix.image_name }}:latest" + docker tag "${IMAGE_TAG}" "${GITEA_IMAGE}" + docker push "${GITEA_IMAGE}" || echo "Gitea Packages push failed (non-fatal)" + + echo "" + echo "✅ ${{ matrix.name }} base image built and pushed" + + - name: Cleanup buildx builder + if: always() + shell: sh + run: | + docker buildx rm "ci-builder-${GITHUB_RUN_ID}-${{ matrix.name }}" 2>/dev/null || true + docker buildx prune -f 2>/dev/null || true + echo "Builder cleanup done" diff --git a/apps/web/package-lock.json b/apps/web/package-lock.json index 430331e1c..18ed5b7e4 100644 --- a/apps/web/package-lock.json +++ b/apps/web/package-lock.json @@ -14,10 +14,7 @@ "axios": "^1.7.2", "react": "^18.3.1", "react-dom": "^18.3.1", - "react-hook-form": "^7.52.0", "react-router-dom": "^6.24.0", - "recharts": "^3.8.1", - "zod": "^3.23.8", "zustand": "^4.5.2" }, "devDependencies": { @@ -36,6 +33,7 @@ "eslint-plugin-react-hooks": "^4.6.2", "eslint-plugin-react-refresh": "^0.4.7", "jsdom": "^24.1.0", + "prettier": "^3.9.5", "typescript": "^5.5.3", "vite": "^5.3.1", "vitest": "^1.6.0" @@ -1415,42 +1413,6 @@ "react-dom": ">=16.9.0" } }, - "node_modules/@reduxjs/toolkit": { - "version": "2.12.0", - "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.12.0.tgz", - "integrity": "sha512-KiT+RzZbp6mQET+Mg+h2c97+9j1sNflUxQkIHI7Yuzf6Peu+OYpmkn6nbHWmLLWj+1ZODUJFwGZ7gx3L9R9EOw==", - "license": "MIT", - "dependencies": { - "@standard-schema/spec": "^1.0.0", - "@standard-schema/utils": "^0.3.0", - "immer": "^11.0.0", - "redux": "^5.0.1", - "redux-thunk": "^3.1.0", - "reselect": "^5.1.0" - }, - "peerDependencies": { - "react": "^16.9.0 || ^17.0.0 || ^18 || ^19", - "react-redux": "^7.2.1 || ^8.1.3 || ^9.0.0" - }, - "peerDependenciesMeta": { - "react": { - "optional": true - }, - "react-redux": { - "optional": true - } - } - }, - "node_modules/@reduxjs/toolkit/node_modules/immer": { - "version": "11.1.8", - "resolved": "https://registry.npmjs.org/immer/-/immer-11.1.8.tgz", - "integrity": "sha512-/tbkHMW7y10Lx6i1crLjD4/OhNkRG+Fo7byZHtah0547nIeXYcpIXaUh0IAQY6gO5459qpGGYapcEOHtFXkIuA==", - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/immer" - } - }, "node_modules/@remix-run/router": { "version": "1.23.3", "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.3.tgz", @@ -1824,18 +1786,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@standard-schema/spec": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", - "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", - "license": "MIT" - }, - "node_modules/@standard-schema/utils": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@standard-schema/utils/-/utils-0.3.0.tgz", - "integrity": "sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g==", - "license": "MIT" - }, "node_modules/@tanstack/query-core": { "version": "5.101.0", "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.101.0.tgz", @@ -2005,69 +1955,6 @@ "@babel/types": "^7.28.2" } }, - "node_modules/@types/d3-array": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz", - "integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==", - "license": "MIT" - }, - "node_modules/@types/d3-color": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", - "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", - "license": "MIT" - }, - "node_modules/@types/d3-ease": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", - "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", - "license": "MIT" - }, - "node_modules/@types/d3-interpolate": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", - "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", - "license": "MIT", - "dependencies": { - "@types/d3-color": "*" - } - }, - "node_modules/@types/d3-path": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz", - "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", - "license": "MIT" - }, - "node_modules/@types/d3-scale": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz", - "integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==", - "license": "MIT", - "dependencies": { - "@types/d3-time": "*" - } - }, - "node_modules/@types/d3-shape": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.8.tgz", - "integrity": "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==", - "license": "MIT", - "dependencies": { - "@types/d3-path": "*" - } - }, - "node_modules/@types/d3-time": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz", - "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", - "license": "MIT" - }, - "node_modules/@types/d3-timer": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", - "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", - "license": "MIT" - }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -2113,12 +2000,6 @@ "@types/react": "^18.0.0" } }, - "node_modules/@types/use-sync-external-store": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.6.tgz", - "integrity": "sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg==", - "license": "MIT" - }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "7.18.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.18.0.tgz", @@ -2932,15 +2813,6 @@ "integrity": "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==", "license": "MIT" }, - "node_modules/clsx": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", - "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -3058,127 +2930,6 @@ "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", "license": "MIT" }, - "node_modules/d3-array": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", - "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", - "license": "ISC", - "dependencies": { - "internmap": "1 - 2" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-color": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", - "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-ease": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", - "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", - "license": "BSD-3-Clause", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-format": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz", - "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-interpolate": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", - "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", - "license": "ISC", - "dependencies": { - "d3-color": "1 - 3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-path": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", - "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-scale": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", - "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", - "license": "ISC", - "dependencies": { - "d3-array": "2.10.0 - 3", - "d3-format": "1 - 3", - "d3-interpolate": "1.2.0 - 3", - "d3-time": "2.1.1 - 3", - "d3-time-format": "2 - 4" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-shape": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", - "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", - "license": "ISC", - "dependencies": { - "d3-path": "^3.1.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-time": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", - "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", - "license": "ISC", - "dependencies": { - "d3-array": "2 - 3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-time-format": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", - "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", - "license": "ISC", - "dependencies": { - "d3-time": "1 - 3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-timer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", - "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, "node_modules/data-urls": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", @@ -3223,12 +2974,6 @@ "dev": true, "license": "MIT" }, - "node_modules/decimal.js-light": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz", - "integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==", - "license": "MIT" - }, "node_modules/deep-eql": { "version": "4.1.4", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.4.tgz", @@ -3391,16 +3136,6 @@ "node": ">= 0.4" } }, - "node_modules/es-toolkit": { - "version": "1.47.1", - "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.47.1.tgz", - "integrity": "sha512-5RAqEwf4P4E17p+W75KLOWw/nOvKZzSQpxM32IpI2KZLaVonjTrZ0Ai5ghMaVI9eKC2p8eoQgcBdkEDgzFk6+Q==", - "license": "MIT", - "workspaces": [ - "docs", - "benchmarks" - ] - }, "node_modules/esbuild": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", @@ -3671,12 +3406,6 @@ "node": ">=0.10.0" } }, - "node_modules/eventemitter3": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", - "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", - "license": "MIT" - }, "node_modules/execa": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", @@ -4223,6 +3952,8 @@ "resolved": "https://registry.npmjs.org/immer/-/immer-10.2.0.tgz", "integrity": "sha512-d/+XTN3zfODyjr89gM3mPq1WNX2B8pYsu7eORitdwyA2sBubnTl3laYlBk4sXY5FUa5qTZGBDPJICVbvqzjlbw==", "license": "MIT", + "optional": true, + "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/immer" @@ -4284,15 +4015,6 @@ "dev": true, "license": "ISC" }, - "node_modules/internmap": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", - "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -5107,6 +4829,22 @@ "node": ">= 0.8.0" } }, + "node_modules/prettier": { + "version": "3.9.6", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.6.tgz", + "integrity": "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, "node_modules/pretty-format": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", @@ -5842,52 +5580,6 @@ "react": "^18.3.1" } }, - "node_modules/react-hook-form": { - "version": "7.79.0", - "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.79.0.tgz", - "integrity": "sha512-mhYp/MTmXvzYX6AJcJVko0rktoIhhmRnEouObj4wF5i/tCttgJvnp1+9wRkpITZjDTqpo4IOSJqu0dBlPlV/Lw==", - "license": "MIT", - "engines": { - "node": ">=18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/react-hook-form" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17 || ^18 || ^19" - } - }, - "node_modules/react-is": { - "version": "19.2.7", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.2.7.tgz", - "integrity": "sha512-kZFnouyVv7eP/Phmrlo9FK+zcAdriZJvzxXHF1Sl1P377WSGe2G/JxVolhTrB/jeV47lKImhNUsijjHAAbcl/A==", - "license": "MIT", - "peer": true - }, - "node_modules/react-redux": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.3.0.tgz", - "integrity": "sha512-KQopgqFo/p/fgmAs5qz6p5RWaNAzq40WAu7fJIXnQpYxFPbJYtsJPWvGeF2rOBaY/kEuV77AVsX8TsQzKm+A/g==", - "license": "MIT", - "dependencies": { - "@types/use-sync-external-store": "^0.0.6", - "use-sync-external-store": "^1.4.0" - }, - "peerDependencies": { - "@types/react": "^18.2.25 || ^19", - "react": "^18.0 || ^19", - "redux": "^5.0.0" - }, - "peerDependenciesMeta": { - "@types/react": { - "optional": true - }, - "redux": { - "optional": true - } - } - }, "node_modules/react-refresh": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", @@ -5930,36 +5622,6 @@ "react-dom": ">=16.8" } }, - "node_modules/recharts": { - "version": "3.8.1", - "resolved": "https://registry.npmjs.org/recharts/-/recharts-3.8.1.tgz", - "integrity": "sha512-mwzmO1s9sFL0TduUpwndxCUNoXsBw3u3E/0+A+cLcrSfQitSG62L32N69GhqUrrT5qKcAE3pCGVINC6pqkBBQg==", - "license": "MIT", - "workspaces": [ - "www" - ], - "dependencies": { - "@reduxjs/toolkit": "^1.9.0 || 2.x.x", - "clsx": "^2.1.1", - "decimal.js-light": "^2.5.1", - "es-toolkit": "^1.39.3", - "eventemitter3": "^5.0.1", - "immer": "^10.1.1", - "react-redux": "8.x.x || 9.x.x", - "reselect": "5.1.1", - "tiny-invariant": "^1.3.3", - "use-sync-external-store": "^1.2.2", - "victory-vendor": "^37.0.2" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "react-is": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" - } - }, "node_modules/redent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", @@ -5974,21 +5636,6 @@ "node": ">=8" } }, - "node_modules/redux": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", - "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", - "license": "MIT" - }, - "node_modules/redux-thunk": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-3.1.0.tgz", - "integrity": "sha512-NW2r5T6ksUKXCabzhL9z+h206HQw/NJkcLm1GPImRQ8IzfXwRGqjVhKJGauHirT0DAuyy6hjdnMZaRoAcy0Klw==", - "license": "MIT", - "peerDependencies": { - "redux": "^5.0.0" - } - }, "node_modules/requires-port": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", @@ -5996,12 +5643,6 @@ "dev": true, "license": "MIT" }, - "node_modules/reselect": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/reselect/-/reselect-5.1.1.tgz", - "integrity": "sha512-K/BG6eIky/SBpzfHZv/dd+9JBFiS4SWV7FIujVyJRux6e45+73RaUHXLmIR1f7WOMaQ0U1km6qwklRQxpJJY0w==", - "license": "MIT" - }, "node_modules/resize-observer-polyfill": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz", @@ -6385,12 +6026,6 @@ "node": ">=12.22" } }, - "node_modules/tiny-invariant": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", - "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==", - "license": "MIT" - }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -6624,28 +6259,6 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, - "node_modules/victory-vendor": { - "version": "37.3.6", - "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-37.3.6.tgz", - "integrity": "sha512-SbPDPdDBYp+5MJHhBCAyI7wKM3d5ivekigc2Dk2s7pgbZ9wIgIBYGVw4zGHBml/qTFbexrofXW6Gu4noGxrOwQ==", - "license": "MIT AND ISC", - "dependencies": { - "@types/d3-array": "^3.0.3", - "@types/d3-ease": "^3.0.0", - "@types/d3-interpolate": "^3.0.1", - "@types/d3-scale": "^4.0.2", - "@types/d3-shape": "^3.1.0", - "@types/d3-time": "^3.0.0", - "@types/d3-timer": "^3.0.0", - "d3-array": "^3.1.6", - "d3-ease": "^3.0.1", - "d3-interpolate": "^3.0.1", - "d3-scale": "^4.0.2", - "d3-shape": "^3.1.0", - "d3-time": "^3.0.0", - "d3-timer": "^3.0.1" - } - }, "node_modules/vite": { "version": "5.4.21", "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", @@ -6980,15 +6593,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, "node_modules/zustand": { "version": "4.5.7", "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.7.tgz", diff --git a/apps/web/package.json b/apps/web/package.json index 8fe1514e2..55c181ab9 100755 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -44,6 +44,7 @@ "jsdom": "^24.1.0", "typescript": "^5.5.3", "vite": "^5.3.1", - "vitest": "^1.6.0" + "vitest": "^1.6.0", + "prettier": "^3.9.5" } } diff --git a/infra/docker/worker-base-builder.Dockerfile b/infra/docker/worker-base-builder.Dockerfile new file mode 100644 index 000000000..4a1bdb1d3 --- /dev/null +++ b/infra/docker/worker-base-builder.Dockerfile @@ -0,0 +1,43 @@ +# ============================================================ +# Worker Builder 基础镜像 +# 预编译：编译工具 + 基础依赖 + Worker大包 +# 当 requirements-base.txt 或 requirements-worker.txt 变更时重新构建 +# 业务构建从此镜像开始，只需要安装业务依赖，节省15+分钟 +# ============================================================ + +FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim + +# 使用阿里云镜像加速 +RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \ + sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true + +# 安装编译工具 +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + g++ \ + python3-dev \ + binutils \ + && rm -rf /var/lib/apt/lists/* + +# 创建 venv +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +WORKDIR /tmp + +# 基础依赖（变化极少） +COPY requirements-base.txt /tmp/requirements-base.txt +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \ + pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \ + -r /tmp/requirements-base.txt \ + && rm /tmp/requirements-base.txt + +# Worker 大包（变化少） +COPY requirements-worker.txt /tmp/requirements-worker.txt +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \ + pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \ + -r /tmp/requirements-worker.txt \ + && rm /tmp/requirements-worker.txt + +# 预先做一次 strip（基础层瘦身，业务层增量） +RUN find /opt/venv -name "*.so" -type f -exec strip --strip-all {} \; 2>/dev/null || true diff --git a/infra/docker/worker-base-runtime.Dockerfile b/infra/docker/worker-base-runtime.Dockerfile new file mode 100644 index 000000000..4f83780ef --- /dev/null +++ b/infra/docker/worker-base-runtime.Dockerfile @@ -0,0 +1,17 @@ +# ============================================================ +# Worker Runtime 基础镜像 +# 预安装：ffmpeg + 运行时依赖 +# 变化极少，业务构建从此镜像开始 +# ============================================================ + +FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim + +# 使用阿里云镜像加速 +RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \ + sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true + +# 运行时依赖：ffmpeg + opencv需要的libglib +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* diff --git a/requirements-dev.txt b/requirements-dev.txt index 4aafad283..918e52da3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,3 +11,4 @@ pytest==8.3.3 pytest-asyncio==0.24.0 pytest-cov==6.0.0 pytest-timeout==2.3.1 +diff-cover==8.0.3 diff --git a/scripts/ci/acr_cleanup.py b/scripts/ci/acr_cleanup.py new file mode 100644 index 000000000..a705f618e --- /dev/null +++ b/scripts/ci/acr_cleanup.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python3 +""" +ACR 镜像清理脚本 +策略： +- 版本tag (v*): 永久保留 +- 固定tag (latest, main, develop, master): 永久保留 +- 缓存镜像 (*-cache): 永久保留 +- PR预览tag (pr-*): 保留 N 天（默认7天） +- 普通commit hash tag: 保留最近 N 个（默认20），老的删除 + +使用方式： + python3 acr_cleanup.py --dry-run # 预览，不实际删除 + python3 acr_cleanup.py --execute # 实际执行删除 + python3 acr_cleanup.py --keep 20 --execute # 保留最近20个 +""" + +import argparse +import base64 +import json +import os +import sys +import urllib.error +import urllib.request +from datetime import datetime, timedelta, timezone + +# 配置 +REGISTRY = os.environ.get("ACR_REGISTRY", "xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com") +AUTH_URL = "https://dockerauth.cn-hangzhou.aliyuncs.com/auth" +SERVICE = os.environ.get("ACR_SERVICE", "registry.aliyuncs.com:cn-hangzhou:china:cri-fvec8o9q4mmxrkaa") +NAMESPACE = os.environ.get("ACR_NAMESPACE", "xiaoxiakeji") +USERNAME = os.environ.get("ACR_USERNAME", "") +PASSWORD = os.environ.get("ACR_PASSWORD", "") + +REPOS = [ + "xiaoxia-saas-api", + "xiaoxia-saas-worker", + "xiaoxia-saas-web", + "api-cache", + "worker-cache", + "web-cache", +] + +# 缓存镜像仓库（所有tag永久保留） +CACHE_REPOS = {"api-cache", "worker-cache", "web-cache"} + +# OCI / Docker manifest types +ACCEPT_INDEX = "application/vnd.oci.image.index.v1+json" +ACCEPT_MANIFEST_OCI = "application/vnd.oci.image.manifest.v1+json" +ACCEPT_MANIFEST_V2 = "application/vnd.docker.distribution.manifest.v2+json" + + +def get_token(repo, action="pull"): + """获取仓库访问token""" + scope = "repository:" + NAMESPACE + "/" + repo + ":" + action + token_url = AUTH_URL + "?service=" + SERVICE + "&scope=" + scope + req = urllib.request.Request(token_url) + req.add_header("Authorization", "Basic " + base64.b64encode((USERNAME + ":" + PASSWORD).encode()).decode()) + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read()) + return data.get("token", "") + + +def get_tags(repo, token): + """获取仓库所有tag""" + url = "https://" + REGISTRY + "/v2/" + NAMESPACE + "/" + repo + "/tags/list?n=1000" + req = urllib.request.Request(url) + req.add_header("Authorization", "Bearer " + token) + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read()) + return data.get("tags", []) or [] + + +def http_get_json(url, token, accept_header): + """带Authorization的GET请求，返回(json_data, headers)""" + req = urllib.request.Request(url) + req.add_header("Authorization", "Bearer " + token) + req.add_header("Accept", accept_header) + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()), resp.headers + + +def get_manifest_info(repo, tag, token): + """ + 获取tag的manifest信息，支持OCI index和普通manifest两种格式。 + 返回: {digest, created, media_type} + - digest: 顶层manifest的digest（用于删除） + - created: 镜像创建时间 + """ + url = "https://" + REGISTRY + "/v2/" + NAMESPACE + "/" + repo + "/manifests/" + tag + result = {"digest": "", "created": "", "media_type": "", "error": ""} + + # 先尝试 OCI index 格式（ACR多用这种） + try: + data, headers = http_get_json(url, token, ACCEPT_INDEX) + top_digest = headers.get("Docker-Content-Digest", "") + result["digest"] = top_digest + result["media_type"] = data.get("mediaType", ACCEPT_INDEX) + + # OCI index：找amd64的manifest，再取config blob + manifests = data.get("manifests", []) + amd64_manifest = None + for m in manifests: + arch = m.get("platform", {}).get("architecture", "") + if arch == "amd64": + amd64_manifest = m + break + # 没有amd64就用第一个 + if not amd64_manifest and manifests: + amd64_manifest = manifests[0] + + if amd64_manifest: + inner_digest = amd64_manifest["digest"] + inner_url = "https://" + REGISTRY + "/v2/" + NAMESPACE + "/" + repo + "/manifests/" + inner_digest + try: + inner_data, _ = http_get_json(inner_url, token, ACCEPT_MANIFEST_OCI) + except Exception: + # 退而求其次用v2格式 + inner_data, _ = http_get_json(inner_url, token, ACCEPT_MANIFEST_V2) + + config_digest = inner_data.get("config", {}).get("digest", "") + if config_digest: + blob_url = "https://" + REGISTRY + "/v2/" + NAMESPACE + "/" + repo + "/blobs/" + config_digest + try: + blob_data, _ = http_get_json(blob_url, token, "application/json") + result["created"] = blob_data.get("created", "") + except Exception: + pass + return result + except urllib.error.HTTPError: + pass + + # 再尝试普通 OCI manifest 格式 + try: + data, headers = http_get_json(url, token, ACCEPT_MANIFEST_OCI) + result["digest"] = headers.get("Docker-Content-Digest", "") + result["media_type"] = data.get("mediaType", ACCEPT_MANIFEST_OCI) + config_digest = data.get("config", {}).get("digest", "") + if config_digest: + blob_url = "https://" + REGISTRY + "/v2/" + NAMESPACE + "/" + repo + "/blobs/" + config_digest + try: + blob_data, _ = http_get_json(blob_url, token, "application/json") + result["created"] = blob_data.get("created", "") + except Exception: + pass + return result + except urllib.error.HTTPError: + pass + + # 最后试 Docker v2 格式 + try: + data, headers = http_get_json(url, token, ACCEPT_MANIFEST_V2) + result["digest"] = headers.get("Docker-Content-Digest", "") + result["media_type"] = data.get("mediaType", ACCEPT_MANIFEST_V2) + config_digest = data.get("config", {}).get("digest", "") + if config_digest: + blob_url = "https://" + REGISTRY + "/v2/" + NAMESPACE + "/" + repo + "/blobs/" + config_digest + try: + blob_data, _ = http_get_json(blob_url, token, "application/json") + result["created"] = blob_data.get("created", "") + except Exception: + pass + return result + except urllib.error.HTTPError as e: + result["error"] = "HTTP " + str(e.code) + " " + e.read().decode()[:200] + + return result + + +def delete_manifest(repo, digest, token): + """按digest删除manifest（会级联删除所有指向它的tag）""" + url = "https://" + REGISTRY + "/v2/" + NAMESPACE + "/" + repo + "/manifests/" + digest + req = urllib.request.Request(url, method="DELETE") + req.add_header("Authorization", "Bearer " + token) + req.add_header("Accept", ACCEPT_INDEX) + req.add_header("Accept", ACCEPT_MANIFEST_OCI) + req.add_header("Accept", ACCEPT_MANIFEST_V2) + try: + with urllib.request.urlopen(req) as resp: + return True, resp.status + except urllib.error.HTTPError as e: + return False, str(e.code) + " " + e.read().decode()[:200] + + +def parse_time(created_str): + """解析ISO时间字符串""" + if not created_str: + return datetime.min.replace(tzinfo=timezone.utc) + try: + if created_str.endswith("Z"): + created_str = created_str[:-1] + "+00:00" + return datetime.fromisoformat(created_str) + except Exception: + return datetime.min.replace(tzinfo=timezone.utc) + + +def is_version_tag(tag): + """判断是否是版本tag (v1.2.3, v0.1.0-alpha等)""" + return tag.startswith("v") and len(tag) > 1 and tag[1].isdigit() + + +def is_fixed_tag(tag): + """判断是否是固定tag""" + return tag in ("latest", "main", "develop", "master", "dev", "stable") + + +def is_pr_tag(tag): + """判断是否是PR预览tag""" + return tag.startswith("pr-") + + +def cleanup_repo(repo, keep_count, pr_days, dry_run): + """清理单个仓库""" + print("=" * 60) + print("仓库:", repo) + print("=" * 60) + + # 缓存仓库不清理 + if repo in CACHE_REPOS: + token_pull = get_token(repo, "pull") + tags = get_tags(repo, token_pull) + print(" 缓存仓库，跳过清理 (共", len(tags), "个tag)") + return len(tags), 0 + + token_pull = get_token(repo, "pull") + tags = get_tags(repo, token_pull) + print(" 总tag数:", len(tags)) + + # 分类 + version_tags = [] + fixed_tags = [] + pr_tags_list = [] + commit_tags = [] + + for tag in tags: + if is_version_tag(tag): + version_tags.append(tag) + elif is_fixed_tag(tag): + fixed_tags.append(tag) + elif is_pr_tag(tag): + pr_tags_list.append(tag) + else: + commit_tags.append(tag) + + print(" 版本tag (v*):", len(version_tags), "-> 永久保留") + print(" 固定tag:", len(fixed_tags), "-> 永久保留") + print(" PR预览tag (pr-*):", len(pr_tags_list), "-> 保留", pr_days, "天") + print(" Commit hash tag:", len(commit_tags), "-> 保留最近", keep_count, "个") + + # 获取所有commit tag的创建时间 + print() + print(" 获取commit tag创建时间...") + tag_info_list = [] + errors = 0 + for i, tag in enumerate(commit_tags): + info = get_manifest_info(repo, tag, token_pull) + if info["error"] or not info["digest"]: + errors += 1 + # 取不到信息的tag，放到最后（最旧处理），但标记一下 + tag_info_list.append({"tag": tag, "digest": info["digest"], "created": "", "error": info.get("error", "")}) + else: + tag_info_list.append({"tag": tag, "digest": info["digest"], "created": info["created"], "error": ""}) + if (i + 1) % 20 == 0: + print(" 已获取", i + 1, "/", len(commit_tags), "...") + + if errors: + print(" 注意:", errors, "个tag获取manifest失败") + + # 按时间倒序排序（空时间放最后） + tag_info_list.sort(key=lambda x: parse_time(x["created"]), reverse=True) + + # 确定要删除的commit tag + to_delete = [] + if len(tag_info_list) > keep_count: + to_delete = tag_info_list[keep_count:] + print(" 保留前", keep_count, "个commit tag，删除", len(to_delete), "个") + # 打印保留范围 + kept = tag_info_list[:keep_count] + valid_kept = [t for t in kept if t["created"]] + if valid_kept: + print(" 最早保留:", valid_kept[-1]["tag"][:12], "(" + valid_kept[-1]["created"][:10] + ")") + # 保护当前构建的tag（通过PROTECTED_TAG环境变量传入，如GITHUB_SHA） + protected_tag = os.environ.get("PROTECTED_TAG", "").strip() + if protected_tag: + before = len(to_delete) + to_delete = [t for t in to_delete if not t["tag"].startswith(protected_tag)] + removed = before - len(to_delete) + if removed > 0: + print(f" 保护当前构建tag: {protected_tag[:12]} (跳过{removed}个)") + + to_del_valid = [t for t in to_delete if t["digest"]] + print(" 可删除(有digest):", len(to_del_valid), "个") + else: + print(" commit tag数量不足", keep_count, "，无需清理") + + # PR tag按时间清理 + pr_to_delete = [] + if pr_tags_list: + cutoff = datetime.now(timezone.utc) - timedelta(days=pr_days) + print() + print(" 检查PR预览tag（超过", pr_days, "天删除）...") + for tag in pr_tags_list: + info = get_manifest_info(repo, tag, token_pull) + created = parse_time(info["created"]) + if created < cutoff: + pr_to_delete.append({"tag": tag, "digest": info["digest"], "created": info["created"]}) + print(" PR tag将删除:", len(pr_to_delete), "个") + + all_to_delete = [t for t in to_delete if t["digest"]] + [t for t in pr_to_delete if t["digest"]] + + if not all_to_delete: + print() + print(" 无需删除任何tag") + return len(tags), 0 + + # 执行删除 + print() + if dry_run: + print(" [DRY RUN] 将删除", len(all_to_delete), "个tag（预览模式，不实际删除）") + # 去重digest + unique_digests = set(t["digest"] for t in all_to_delete if t["digest"]) + print(" 去重后唯一digest数:", len(unique_digests)) + for item in all_to_delete[:5]: + created_str = item.get("created", "")[:10] or "未知" + print(" -", item["tag"][:20], "(" + created_str + ")") + if len(all_to_delete) > 5: + print(" ... 还有", len(all_to_delete) - 5, "个") + return len(tags), len(unique_digests) + + token_delete = get_token(repo, "delete") + deleted = 0 + failed = 0 + # 按digest去重，避免重复删除同一镜像 + seen_digests = set() + unique_delete = [] + for item in all_to_delete: + if item["digest"] and item["digest"] not in seen_digests: + seen_digests.add(item["digest"]) + unique_delete.append(item) + + print(" 开始删除", len(unique_delete), "个唯一manifest...") + for item in unique_delete: + success, result = delete_manifest(repo, item["digest"], token_delete) + if success: + deleted += 1 + print(" 已删除:", item["tag"][:20]) + else: + failed += 1 + print(" 删除失败:", item["tag"][:20], "-", result) + + print() + print(" 删除完成: 成功", deleted, "个，失败", failed, "个") + return len(tags), deleted + + +def main(): + parser = argparse.ArgumentParser(description="ACR镜像清理工具") + parser.add_argument("--keep", type=int, default=20, help="保留最近N个commit hash tag（默认20）") + parser.add_argument("--pr-days", type=int, default=7, help="PR预览tag保留天数（默认7天）") + parser.add_argument("--dry-run", action="store_true", help="预览模式，不实际删除") + parser.add_argument("--execute", action="store_true", help="实际执行删除") + parser.add_argument("--repo", type=str, default="", help="只清理指定仓库") + args = parser.parse_args() + + # 必须指定 --dry-run 或 --execute + if not args.dry_run and not args.execute: + print("请指定 --dry-run（预览）或 --execute（执行）") + print() + print("示例:") + print(" python3 acr_cleanup.py --dry-run # 预览清理效果") + print(" python3 acr_cleanup.py --execute # 实际执行清理") + print(" python3 acr_cleanup.py --keep 20 --execute # 保留最近20个") + sys.exit(1) + + # 凭证检查 + global USERNAME, PASSWORD + if not USERNAME or not PASSWORD: + # 尝试从docker config读取 + try: + docker_config_path = os.path.expanduser("~/.docker/config.json") + with open(docker_config_path) as f: + config = json.load(f) + auth = config.get("auths", {}).get(REGISTRY, {}).get("auth", "") + if auth: + creds = base64.b64decode(auth).decode().strip() + USERNAME, PASSWORD = creds.split(":", 1) + except Exception: + pass + + if not USERNAME or not PASSWORD: + print("错误: 缺少ACR凭证，请设置 ACR_USERNAME 和 ACR_PASSWORD 环境变量") + print("或确保已执行 docker login", REGISTRY) + sys.exit(1) + + dry_run = args.dry_run or not args.execute + mode = "预览模式" if dry_run else "执行模式" + print("ACR 镜像清理工具 -", mode) + print("Registry:", REGISTRY) + print("Namespace:", NAMESPACE) + print("保留commit tag数:", args.keep) + print("PR预览保留天数:", args.pr_days) + print() + + repos_to_clean = REPOS + if args.repo: + repos_to_clean = [args.repo] + + total_deleted = 0 + total_tags = 0 + for repo in repos_to_clean: + count, deleted = cleanup_repo(repo, args.keep, args.pr_days, dry_run) + total_tags += count + total_deleted += deleted + + print() + print("=" * 60) + print("清理完成") + print(" 总tag数:", total_tags) + if dry_run: + print(" 预览将删除(去重后):", total_deleted, "个manifest") + else: + print(" 已删除:", total_deleted, "个manifest") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/auto_approve.sh b/scripts/ci/auto_approve.sh new file mode 100644 index 000000000..5736898f1 --- /dev/null +++ b/scripts/ci/auto_approve.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# 自动审批：CI全绿后自动approve PR +# 环境变量：GITHUB_TOKEN, REVIEW_TOKEN, PR_NUMBER, PR_HEAD_SHA, GITHUB_API_URL, GITHUB_REPOSITORY +set -eu + +set -eu + +echo "PR #${PR_NUMBER} - 检查CI状态并自动审批" + +# 检查是否纯前端改动 +API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300" +FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin)]") +FRONTEND_COUNT=$(echo "$FILES" | grep -c '^apps/web/' || true) +BACKEND_COUNT=$(echo "$FILES" | grep -cv '^apps/web/' || true) +TOTAL=$(echo "$FILES" | grep -cv '^$' || true) +echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT})" + +if [ "$BACKEND_COUNT" = "0" ] && [ "$FRONTEND_COUNT" -gt "0" ]; then + SKIP_BACKEND=true + echo "✅ 纯前端改动，只检查Frontend Lint" +else + SKIP_BACKEND=false + echo "🔧 包含后端/公共变更，检查全部CI" +fi + +# 定义需要检查的context +if [ "$SKIP_BACKEND" = "true" ]; then + CONTEXTS=("CI/CD Pipeline / Frontend Lint (pull_request)") +else + CONTEXTS=( + "CI/CD Pipeline / Validate - Code Quality (pull_request)" + "CI/CD Pipeline / Validate - Type Check (mypy) (pull_request)" + "CI/CD Pipeline / Validate - Migration (alembic) (pull_request)" + "CI/CD Pipeline / Frontend Lint (pull_request)" + "CI/CD Pipeline / Unit Tests (pull_request)" + "CI/CD Pipeline / Frontend Unit Tests (pull_request)" + "CI/CD Pipeline / PR Build API Image (pull_request)" + "CI/CD Pipeline / PR Build Web Image (pull_request)" + "CI/CD Pipeline / PR Build Worker Image (pull_request)" + ) +fi + +echo "需要通过的CI检查: ${#CONTEXTS[@]} 项（与分支保护required门禁一致）" +for ctx in "${CONTEXTS[@]}"; do + echo " - $ctx" +done +echo + +# 初始等待30秒，给CI启动写status的时间 +echo "等待30秒让CI启动..." +sleep 30 + +# 轮询等待，最多20分钟（120次x10秒） +for attempt in $(seq 1 12); do # 短作业模式：最多等2分钟(12次x10秒)，不满足就退出等下次触发 + ALL_SUCCESS=true + ANY_FAILED=false + ANY_PENDING=false + + echo "--- 第${attempt}次检查 ($(date '+%H:%M:%S')) ---" + + # 调用辅助脚本检查每个context状态 + for ctx in "${CONTEXTS[@]}"; do + STATE=$(python3 scripts/check_ci_status.py "$GITHUB_TOKEN" "$GITHUB_REPOSITORY" "$PR_HEAD_SHA" "$ctx") + echo " $ctx: $STATE" + + if [ "$STATE" != "success" ]; then + ALL_SUCCESS=false + fi + if [ "$STATE" = "failure" ] || [ "$STATE" = "error" ]; then + ANY_FAILED=true + fi + if [ "$STATE" = "pending" ] || [ "$STATE" = "null" ]; then + ANY_PENDING=true + fi + done + + if [ "$ALL_SUCCESS" = "true" ]; then + echo + echo "✅ 所有CI检查通过，自动审批 PR #${PR_NUMBER}" + + # 检查是否已有审批 + EXISTING=$(curl -s -H "Authorization: token ${REVIEW_TOKEN}" \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" \ + | python3 -c "import sys,json; reviews=json.load(sys.stdin); print('yes' if any(r.get('state')=='APPROVED' for r in reviews) else 'no')") + + if [ "$EXISTING" = "yes" ]; then + echo "ℹ️ PR #${PR_NUMBER} 已有审批，跳过" + exit 0 + fi + + # 第一步：创建PENDING review + echo "创建review..." + REVIEW_CREATE=$(curl -s -X POST \ + -H "Authorization: token ${REVIEW_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"event": "PENDING", "body": "CI全绿，自动审批通过。"}' \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews") + + REVIEW_ID=$(echo "$REVIEW_CREATE" | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))") + REVIEW_STATE=$(echo "$REVIEW_CREATE" | python3 -c "import sys,json; print(json.load(sys.stdin).get('state',''))") + echo "创建结果: id=$REVIEW_ID state=$REVIEW_STATE" + + if [ -z "$REVIEW_ID" ]; then + echo "❌ 创建review失败" + echo "$REVIEW_CREATE" + exit 1 + fi + + if [ "$REVIEW_STATE" = "APPROVED" ]; then + echo "✅ 自动审批成功（直接创建为APPROVED）" + exit 0 + fi + + # 第二步：submit review为APPROVED + echo "提交review审批..." + SUBMIT_CODE=$(curl -s -o /tmp/submit_resp.json -w "%{http_code}" \ + -X POST \ + -H "Authorization: token ${REVIEW_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"event": "APPROVED", "body": "CI全绿，自动审批通过。"}' \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews/${REVIEW_ID}") + + echo "提交API HTTP状态: $SUBMIT_CODE" + cat /tmp/submit_resp.json 2>/dev/null || true + echo + + if [ "$SUBMIT_CODE" = "200" ] || [ "$SUBMIT_CODE" = "201" ]; then + FINAL_STATE=$(python3 -c "import json; print(json.load(open('/tmp/submit_resp.json')).get('state',''))" 2>/dev/null || echo "?") + echo "✅ 自动审批成功 (state: $FINAL_STATE)" + exit 0 + else + echo "❌ 提交审批失败" + exit 1 + fi + fi + + # 还有CI在跑 → 继续等 + if [ "$ANY_PENDING" = "true" ]; then + echo "⏳ CI仍在运行中（第${attempt}/12次），超时后将退出等待下次触发..." + sleep 10 + continue + fi + + # 所有CI都跑完了但有失败 → 退出 + if [ "$ANY_FAILED" = "true" ]; then + echo + echo "❌ CI检查有失败项，不自动审批" + exit 0 + fi + + sleep 10 +done + +echo +echo "⏰ 快速检查超时（2分钟），CI尚未完成，退出等待下次触发（workflow_run事件或5分钟定时扫描）" +exit 0 diff --git a/scripts/ci/auto_fix_formatting.py b/scripts/ci/auto_fix_formatting.py new file mode 100755 index 000000000..f4e493e3d --- /dev/null +++ b/scripts/ci/auto_fix_formatting.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python3 +"""CI中自动修复代码格式（Python: black + isort | Frontend: prettier），并推送回原分支。 + +- PR事件：自动修复并push回PR源分支（Agent提交的PR自动修，人提交的仅诊断） +- Push事件（develop/main）：自动修复并push回原分支，保持主干格式永远正确 +当code quality检查因格式问题失败时触发。 +""" + +import json +import os +import subprocess +import sys +import time +import urllib.request + + +def run(cmd, check=True, capture=True, cwd=None): + """运行shell命令""" + result = subprocess.run(cmd, shell=True, capture_output=capture, text=True, cwd=cwd) + if check and result.returncode != 0: + print(f"命令失败: {cmd}", file=sys.stderr) + if result.stderr: + print(result.stderr, file=sys.stderr) + sys.exit(1) + return result + + +def ensure_git_repo(api_url, repo, token, pr_number): + """确保当前目录是git仓库，并切换到PR源分支。 + + checkout脚本用tarball方式下载代码（PR merge后的commit），没有.git目录。 + 这里自动初始化git仓库，fetch PR源分支并强制checkout， + 使工作区变为PR源分支的代码，确保后续格式化修复基于源分支。 + """ + if os.path.exists(".git"): + return + + print("检测到tarball checkout（无.git目录），自动初始化git仓库...") + + # 构造带认证的远端URL + server_url = api_url.rsplit("/api/v1", 1)[0] + remote_url = f"{server_url.replace('https://', f'https://x-access-token:{token}@')}/{repo}.git" + + # 获取PR的源分支 + pr_api_url = f"{api_url}/repos/{repo}/pulls/{pr_number}" + req_obj = urllib.request.Request(pr_api_url, headers={"Authorization": f"token {token}"}) + with urllib.request.urlopen(req_obj) as resp: + pr = json.loads(resp.read()) + head_branch = pr["head"]["ref"] + + print(f"PR源分支: {head_branch}") + + # 初始化git + run("git init -q") + run(f"git remote add origin {remote_url}") + run('git config user.name "CI Bot"') + run('git config user.email "ci-bot@xiaoxiajianji.com"') + + # fetch源分支（浅克隆，只要最新commit） + print("fetch源分支...") + run(f"git fetch --depth=1 origin {head_branch}") + + # 强制checkout到源分支（覆盖tarball内容） + # tarball是merge后的commit，源分支才是我们要修改并推送的目标 + print("切换到源分支...") + run(f"git checkout -f -B {head_branch} FETCH_HEAD") + + result = run("git status --porcelain") + if result.stdout.strip(): + n = len(result.stdout.strip().splitlines()) + print(f"⚠️ 工作区有 {n} 个未追踪文件") + else: + print("✅ git仓库就绪，工作区clean") + + return head_branch + + +def ensure_git_repo_for_push(api_url, repo, token, branch_name): + """push事件下确保git仓库可用，并切换到目标分支。 + + checkout脚本用tarball方式下载代码，没有.git目录。 + 这里自动初始化git仓库，fetch目标分支并checkout。 + """ + if os.path.exists(".git"): + # 已有git，确认在正确分支 + result = run("git rev-parse --abbrev-ref HEAD", check=False) + if result.stdout.strip() == branch_name: + return + # 不在目标分支，切换 + run(f"git checkout {branch_name}", check=False) + return + + print(f"检测到tarball checkout（无.git目录），初始化git仓库（push模式，分支: {branch_name}）...") + + server_url = api_url.rsplit("/api/v1", 1)[0] + remote_url = f"{server_url.replace('https://', f'https://x-access-token:{token}@')}/{repo}.git" + + run("git init -q") + run(f"git remote add origin {remote_url}") + run('git config user.name "CI Bot"') + run('git config user.email "ci-bot@xiaoxiajianji.com"') + + print(f"fetch {branch_name} 分支...") + run(f"git fetch --depth=1 origin {branch_name}") + + print(f"切换到 {branch_name} 分支...") + run(f"git checkout -f -B {branch_name} FETCH_HEAD") + + result = run("git status --porcelain") + if result.stdout.strip(): + n = len(result.stdout.strip().splitlines()) + print(f"⚠️ 工作区有 {n} 个未追踪文件") + else: + print("✅ git仓库就绪，工作区clean") + + +def get_changed_files(pr_number, api_url, token): + """获取PR中变更的文件列表""" + url = f"{api_url}/pulls/{pr_number}/files?limit=100" + req = urllib.request.Request(url, headers={"Authorization": f"token {token}"}) + with urllib.request.urlopen(req) as resp: + files = json.loads(resp.read()) + return [f["filename"] for f in files if f["status"] != "removed"] + + +def get_pr_head_branch(pr_number, api_url, token): + """获取PR的来源分支名""" + url = f"{api_url}/pulls/{pr_number}" + req = urllib.request.Request(url, headers={"Authorization": f"token {token}"}) + with urllib.request.urlopen(req) as resp: + pr = json.loads(resp.read()) + return pr["head"]["ref"] + + +def fix_python(target_py_files, scan_mode): + """修复 Python 文件格式 (black + isort)""" + if not target_py_files: + print("没有需要修复的 Python 文件，跳过") + return + + target_str = " ".join(target_py_files) + print() + print("--- black 格式化 ---") + result = run(f"python3 -m black {target_str}", check=False) + print(result.stdout[-500:] if result.stdout else "") + if result.returncode != 0: + print("black执行失败，但继续尝试isort", file=sys.stderr) + + print() + print("--- isort 排序 ---") + result = run(f"python3 -m isort {target_str}", check=False) + print(result.stdout[-500:] if result.stdout else "") + if result.returncode != 0: + print("isort执行失败", file=sys.stderr) + + +def fix_frontend(target_fe_files, scan_mode, repo_root): + """修复前端文件格式 (prettier)""" + if not target_fe_files: + print("没有需要修复的前端文件，跳过") + return + + # 检查 prettier 是否可用 + web_dir = os.path.join(repo_root, "apps", "web") + prettier_bin = os.path.join(web_dir, "node_modules", ".bin", "prettier") + + if not os.path.exists(prettier_bin): + print() + print("--- 安装前端依赖 (prettier) ---") + result = run("npm install --no-audit --no-fund --prefer-offline", check=False, cwd=web_dir) + if result.returncode != 0: + print("npm install 失败，跳过 prettier 修复", file=sys.stderr) + return + print("依赖安装完成") + + if not os.path.exists(prettier_bin): + print("prettier 仍不可用，跳过", file=sys.stderr) + return + + print() + print("--- prettier 格式化 ---") + + if scan_mode == "incremental": + # 增量模式：只格式化变更的前端文件 + target_str = " ".join(target_fe_files) + cmd = f"{prettier_bin} --write {target_str}" + else: + # 全量模式：格式化整个前端目录 + cmd = f"{prettier_bin} --write ." + + result = run(cmd, check=False, cwd=web_dir if scan_mode != "incremental" else repo_root) + print(result.stdout[-800:] if result.stdout else "") + if result.stderr: + print(result.stderr[-500:], file=sys.stderr) + + +def main(): + event_name = os.environ.get("GITHUB_EVENT_NAME", "") + github_ref = os.environ.get("GITHUB_REF", "") + api_url = os.environ.get("GITHUB_API_URL", "") + repo = os.environ.get("GITHUB_REPOSITORY", "") + token = os.environ.get("REVIEW_TOKEN", "") or os.environ.get("GITHUB_TOKEN", "") + scan_mode = os.environ.get("SCAN_MODE", "full") + changed_files_env = os.environ.get("CHANGED_FILES", "") + + if not token: + print("缺少REVIEW_TOKEN或GITHUB_TOKEN，无法推送修复", file=sys.stderr) + sys.exit(1) + + repo_root = os.getcwd() + + # ====== Push事件处理（develop/main等受保护分支） ====== + if event_name == "push": + # 从 refs/heads/xxx 提取分支名 + if not github_ref.startswith("refs/heads/"): + print(f"push事件但refs格式异常: {github_ref}，跳过") + return + branch_name = github_ref.replace("refs/heads/", "") + + # 只在受保护分支（develop/main）上自动修复并推送 + protected_branches = {"develop", "main", "master"} + if branch_name not in protected_branches: + print(f"push事件，分支 {branch_name} 不是受保护分支，跳过自动修复") + return + + print("=== Push事件：检测到格式问题，自动修复并推送回原分支 ===") + print(f"分支: {branch_name}") + print(f"扫描模式: {scan_mode}") + + # 初始化git仓库 + ensure_git_repo_for_push(api_url, repo, token, branch_name) + head_branch = branch_name + fix_mode = "auto_fix_and_push" + + # ====== PR事件处理 ====== + elif event_name == "pull_request": + pr_number = github_ref.split("/")[2] if github_ref.startswith("refs/pull/") else "" + if not pr_number: + print("无法获取PR号，跳过自动修复") + return + + # 获取PR作者信息，判断是人还是Agent提交的 + pr_info_url = f"{api_url}/repos/{repo}/pulls/{pr_number}" + req_pr = urllib.request.Request(pr_info_url, headers={"Authorization": f"token {token}"}) + with urllib.request.urlopen(req_pr) as resp: + pr_info = json.loads(resp.read()) + pr_author = pr_info.get("user", {}).get("login", "") + print(f"PR作者: {pr_author}") + + # 判断是否为Agent提交的PR + agent_authors = {"actions", "auto-approve-bot", "gitea-actions"} + is_agent_pr = pr_author in agent_authors or "bot" in pr_author.lower() + + if is_agent_pr: + print(f"检测到Agent提交的PR（作者: {pr_author}），将自动修复并推送") + fix_mode = "auto_fix_and_push" + else: + print(f"检测到人提交的PR（作者: {pr_author}），仅诊断不自动修改") + print("（如需自动修复，请用Agent账号提交PR，或手动运行格式化脚本）") + fix_mode = "diagnose_only" + + print("=== 检测到代码格式问题，尝试自动修复 ===") + print(f"PR #{pr_number}") + print(f"扫描模式: {scan_mode}") + + # 确保git仓库可用（tarball checkout模式下自动初始化） + head_branch = ensure_git_repo(api_url, repo, token, pr_number) + + # ====== 其他事件跳过 ====== + else: + print(f"事件 {event_name} 不支持自动修复，跳过") + return + + # 前端文件扩展名 + fe_extensions = ( + ".ts", + ".tsx", + ".js", + ".jsx", + ".css", + ".scss", + ".less", + ".json", + ".html", + ".md", + ".yaml", + ".yml", + ) + py_extensions = (".py",) + + # 确定要修复的文件范围 + if scan_mode == "incremental" and changed_files_env: + all_changed = changed_files_env.split() + target_py_files = [f for f in all_changed if f.endswith(py_extensions)] + target_fe_files = [f for f in all_changed if f.endswith(fe_extensions)] + print(f"增量模式: {len(target_py_files)} 个Python文件, {len(target_fe_files)} 个前端文件") + else: + target_py_files = ["alembic", "apps", "packages", "tests", "scripts"] + target_fe_files = ["apps/web"] + print("全量模式，修复所有文件") + + # Python 格式化 + fix_python(target_py_files, scan_mode) + + # 前端格式化 + if scan_mode != "incremental": + fix_frontend(["apps/web"], scan_mode, repo_root) + else: + fix_frontend(target_fe_files, scan_mode, repo_root) + + # 检查是否有改动 + result = run("git status --porcelain") + if not result.stdout.strip(): + print() + print("没有需要提交的格式改动") + return + + # 诊断模式：只报告问题，不修改不推送 + if fix_mode == "diagnose_only": + print() + print("=" * 50) + print("📋 格式问题诊断报告（人提交的PR，仅诊断不自动修复）") + print("=" * 50) + print() + print("以下文件存在格式问题，建议手动修复：") + for line in result.stdout.strip().split("\n"): + print(f" {line}") + print() + print("修复方式：") + print(" 后端(Python): 运行 black + isort") + print(" 前端: 运行 prettier --write") + print(" 或使用 scripts/agent-commit.sh 提交（自动格式化）") + print() + print("=" * 50) + # 以非0状态码退出，让CI继续报失败（因为问题没修） + sys.exit(1) + + print() + print("变更文件:") + for line in result.stdout.strip().split("\n"): + print(f" {line}") + + # 提交修复 + run("git add -A") + run('git commit -m "style: auto-format with black + isort + prettier"') + + # 推送（head_branch已从ensure_git_repo获取） + print(f"\nPR来源分支: {head_branch}") + print("推送格式修复到远端...") + + # 推送前先 rebase 拉取远端最新，避免快进冲突 + # 最多重试 3 次：rebase → push，失败则重新拉取再试 + max_retries = 3 + push_success = False + last_error = "" + + for attempt in range(1, max_retries + 1): + print(f" 尝试 {attempt}/{max_retries}: 拉取最新代码并推送...") + + # 先拉取远端最新 commit 并 rebase + fetch_result = run(f"git fetch origin {head_branch}", check=False) + if fetch_result.returncode != 0: + last_error = f"git fetch 失败: {fetch_result.stderr.strip()}" + print(f" {last_error}") + time.sleep(2) + continue + + rebase_result = run(f"git rebase origin/{head_branch}", check=False) + if rebase_result.returncode != 0: + last_error = f"git rebase 失败，中止并重置: {rebase_result.stderr.strip()[:200]}" + print(f" {last_error}") + run("git rebase --abort", check=False) + # rebase 失败通常是冲突，重试没用，直接跳出 + break + + # 推送 + push_result = run(f'git push origin "HEAD:{head_branch}"', check=False) + if push_result.returncode == 0: + push_success = True + break + + last_error = push_result.stderr.strip() or push_result.stdout.strip() + print(f" push 失败: {last_error[:200]}") + time.sleep(3) + + if not push_success: + print(f"\n❌ 推送失败（已重试 {max_retries} 次）", file=sys.stderr) + print(f"最后错误: {last_error}", file=sys.stderr) + sys.exit(1) + + print() + print("✅ 格式已自动修复并推送回分支") + print("新的commit会重新触发CI检查") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/auto_merge.sh b/scripts/ci/auto_merge.sh new file mode 100644 index 000000000..d251b9820 --- /dev/null +++ b/scripts/ci/auto_merge.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# 自动合并：CI全绿+已审批后自动squash merge PR到develop +# 环境变量：GITHUB_TOKEN, MERGE_TOKEN, PR_NUMBER, PR_HEAD_SHA, BASE_REF, GITHUB_API_URL, GITHUB_REPOSITORY +set -eu + +set -eu + +echo "PR #${PR_NUMBER} - 检查CI状态+审批并自动合并到${BASE_REF}" +echo + +# 只合develop分支 +if [ "$BASE_REF" != "develop" ]; then + echo "Skip: 目标分支不是develop" + exit 0 +fi + +# 判断是否纯前端改动 +FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300" \ + | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin)]") +TOTAL=$(echo "$FILES" | grep -cv '^$' || true) +FRONTEND_COUNT=$(echo "$FILES" | grep -c '^apps/web/' || true) +BACKEND_COUNT=$((TOTAL - FRONTEND_COUNT)) +echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT})" + +if [ "$BACKEND_COUNT" = "0" ] && [ "$FRONTEND_COUNT" -gt "0" ]; then + CONTEXTS=("CI/CD Pipeline / Frontend Lint (pull_request)") + echo "纯前端改动，只检查Frontend Lint" +else + CONTEXTS=( + "CI/CD Pipeline / Validate - Code Quality (pull_request)" + "CI/CD Pipeline / Validate - Type Check (mypy) (pull_request)" + "CI/CD Pipeline / Validate - Migration (alembic) (pull_request)" + "CI/CD Pipeline / Frontend Lint (pull_request)" + "CI/CD Pipeline / Unit Tests (pull_request)" + "CI/CD Pipeline / Frontend Unit Tests (pull_request)" + "CI/CD Pipeline / PR Build API Image (pull_request)" + "CI/CD Pipeline / PR Build Web Image (pull_request)" + "CI/CD Pipeline / PR Build Worker Image (pull_request)" + ) + echo "检查required门禁（与分支保护一致）" +fi +echo + +# 初始等待30秒，给CI启动写status的时间 +echo "等待30秒让CI启动..." +sleep 30 + +# 405连续计数器 +MERGE_405_COUNT=0 +MAX_405_RETRIES=10 + +# 轮询等待，最多30分钟（180次x10秒） +for attempt in $(seq 1 90); do # 最多等45分钟(90次x30秒)，确保等得到Worker构建完成 + ALL_SUCCESS=true + ANY_FAILED=false + ANY_PENDING=false + + echo "--- 第${attempt}次检查 ($(date '+%H:%M:%S')) ---" + + # 检查CI状态 + for ctx in "${CONTEXTS[@]}"; do + STATE=$(python3 scripts/check_ci_status.py "$GITHUB_TOKEN" "$GITHUB_REPOSITORY" "$PR_HEAD_SHA" "$ctx") + echo " CI: ${ctx##*/}: $STATE" + if [ "$STATE" != "success" ]; then + ALL_SUCCESS=false + fi + if [ "$STATE" = "failure" ] || [ "$STATE" = "error" ]; then + ANY_FAILED=true + fi + if [ "$STATE" = "pending" ]; then + ANY_PENDING=true + fi + done + + + # CI全绿 → 合并 + if [ "$ALL_SUCCESS" = "true" ]; then + echo + echo "CI全绿，执行自动合并" + echo "等待60秒冷却，给Gitea内部状态同步时间..." + sleep 60 + + # 幂等检查：PR是否还是open + PR_STATE=$(curl -s -H "Authorization: token ${MERGE_TOKEN}" \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" \ + | python3 -c "import sys,json; print(json.load(sys.stdin).get('state',''))") + + if [ "$PR_STATE" != "open" ]; then + echo "PR状态为 ${PR_STATE}，无需合并" + exit 0 + fi + + # 执行squash merge + HTTP_CODE=$(curl -s -o /tmp/merge_resp.json -w "%{http_code}" \ + -X POST \ + -H "Authorization: token ${MERGE_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"do":"squash","merge_title_field":"","merge_message_field":"","delete_branch_after_merge":true,"force_merge":false}' \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/merge") + + echo "合并API HTTP状态: $HTTP_CODE" + + if [ "$HTTP_CODE" = "200" ]; then + echo "自动合并成功" + exit 0 + elif [ "$HTTP_CODE" = "405" ]; then + MERGE_405_COUNT=$((MERGE_405_COUNT + 1)) + echo "⚠️ 合并返回405（第${MERGE_405_COUNT}次），可能CI状态尚未同步或有未解决的门禁，继续等待重试..." + cat /tmp/merge_resp.json 2>/dev/null || true + echo + if [ "$MERGE_405_COUNT" -ge "$MAX_405_RETRIES" ]; then + echo "⚠️ 连续${MAX_405_RETRIES}次合并返回405，放弃自动合并（需人工确认，非代码问题）" + curl -s -X POST \ + -H "Authorization: token ${MERGE_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"body": "Auto merge skipped after multiple 405 errors: PR may have conflicts or unresolved checks. Please review manually. This is not a CI failure."}' \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" > /dev/null 2>&1 || true + exit 0 + fi + sleep 30 + continue + else + echo "自动合并失败 (HTTP $HTTP_CODE)" + cat /tmp/merge_resp.json 2>/dev/null || true + curl -s -X POST \ + -H "Authorization: token ${MERGE_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{\"body\": \"Auto merge failed (HTTP ${HTTP_CODE}), please check manually.\"}" \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" > /dev/null 2>&1 || true + exit 1 + fi + else + # 本轮不满足合并条件，重置405计数器 + MERGE_405_COUNT=0 + fi + + if [ "$ANY_FAILED" = "true" ]; then + echo + echo "CI有失败项，不自动合并" + exit 0 + fi + + sleep 30 +done + +echo +echo "快速检查超时（3分钟），CI尚未全绿或无审批，退出等待下次触发" +exit 0 diff --git a/scripts/ci/chatops/__init__.py b/scripts/ci/chatops/__init__.py new file mode 100755 index 000000000..ab54c0fa0 --- /dev/null +++ b/scripts/ci/chatops/__init__.py @@ -0,0 +1,19 @@ +"""CI ChatOps 工具包 - 飞书机器人对接 Gitea Actions + +模块: + config - 配置管理（环境变量） + gitea_client - Gitea API 客户端封装 + feishu_notify - 飞书通知（失败/恢复/E2E摘要） + ci_query - CI 状态查询 + ci_trigger - CI 重跑触发 + webhook_server - Gitea webhook 接收服务（FastAPI） +""" + +__all__ = [ + "config", + "gitea_client", + "feishu_notify", + "ci_query", + "ci_trigger", + "webhook_server", +] diff --git a/scripts/ci/chatops/ci_query.py b/scripts/ci/chatops/ci_query.py new file mode 100755 index 000000000..d4a9c0a34 --- /dev/null +++ b/scripts/ci/chatops/ci_query.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +""" +CI 状态查询模块 - 查询 run 列表、某分支/某 PR 的 CI 状态、失败详情 + +支持查询类型: + - list_runs: 列出最近的 workflow runs + - branch_status: 某分支最新 CI 状态 + - pr_status: 某 PR 的 CI 状态 + - failure_detail: 某次 run 的失败详情 + +用法: + python3 scripts/ci/chatops/ci_query.py --branch develop + python3 scripts/ci/chatops/ci_query.py --pr 123 + python3 scripts/ci/chatops/ci_query.py --run-id 456 --detail + +设计: + - 与飞书机器人 /ci status 命令对接 + - 返回结构化数据，上层负责格式化输出 +""" + +import argparse +import sys + +from . import config +from .gitea_client import GiteaClient + + +class CIQuery: + """CI 状态查询器""" + + def __init__(self, gitea_client=None): + self.gitea = gitea_client or GiteaClient() + + # ── 查询方法 ────────────────────────────────────── + + def get_branch_status(self, branch, limit=5): + """获取指定分支最新的 CI 状态 + + Returns: + dict: {branch, latest_run, recent_runs, overall_status} + """ + runs, total = self.gitea.list_runs(branch=branch, limit=limit) + if not runs: + return { + "branch": branch, + "latest_run": None, + "recent_runs": [], + "overall_status": "no_runs", + "total_count": total, + } + + latest = runs[0] + overall = self._derive_overall_status(runs) + + return { + "branch": branch, + "latest_run": latest, + "recent_runs": runs, + "overall_status": overall, + "total_count": total, + } + + def get_pr_status(self, pr_number): + """获取指定 PR 的 CI 状态 + + Returns: + dict: {pr_number, pr_title, runs, overall_status} + """ + pr = self.gitea.get_pr(pr_number) + if not pr: + return { + "pr_number": pr_number, + "pr_title": "未知", + "runs": [], + "overall_status": "pr_not_found", + } + + pr_title = pr.get("title", "") + runs = self.gitea.get_pr_ci_runs(pr_number, limit=10) + overall = self._derive_overall_status(runs) if runs else "no_runs" + + return { + "pr_number": pr_number, + "pr_title": pr_title, + "runs": runs, + "overall_status": overall, + "head_sha": pr.get("head", {}).get("sha", ""), + } + + def get_failure_detail(self, run_id): + """获取某次 run 的失败详情 + + Returns: + dict: {run_info, failed_jobs, summary} + """ + run = self.gitea.get_run(run_id) + if not run: + return {"run_info": None, "failed_jobs": [], "summary": "Run not found"} + + failed_jobs = self.gitea.get_failed_jobs_summary(run_id, max_lines_per_job=30) + + summary_parts = [] + for job in failed_jobs: + step = f"（步骤: {job['failed_step']}）" if job["failed_step"] else "" + summary_parts.append(f"• {job['name']}{step}") + + summary = "\n".join(summary_parts) if summary_parts else "无失败 job（可能还在运行中）" + + return { + "run_info": run, + "failed_jobs": failed_jobs, + "summary": summary, + "total_jobs": len(self.gitea.get_run_jobs(run_id)), + } + + def list_recent_runs(self, status=None, branch=None, limit=10): + """列出最近的 runs""" + runs, total = self.gitea.list_runs(status=status, branch=branch, limit=limit) + return {"runs": runs, "total_count": total} + + # ── 辅助方法 ──────────────────────────────────── + + @staticmethod + def _derive_overall_status(runs): + """根据最近 runs 推导整体状态 + + Returns: + success: 最近一次成功 + failing: 最近一次失败（连续失败） + flaky: 有失败有成功（最近一次失败 + running: 有正在运行的 + unknown: 未知 + """ + if not runs: + return "no_runs" + + # 检查是否有运行中的 + running = [r for r in runs if r.get("status") != "completed"] + if running: + return "running" + + # 看最近一次 + latest = runs[0] + latest_conclusion = latest.get("conclusion", "unknown") + + if latest_conclusion == "success": + return "success" + + if latest_conclusion == "failure": + # 检查是否连续失败 + consecutive_failures = 0 + for r in runs: + if r.get("conclusion") == "failure": + consecutive_failures += 1 + else: + break + + # 看之前有没有成功 + has_success = any(r.get("conclusion") == "success" for r in runs) + + if has_success: + return "flaky" + return "failing" + + return "unknown" + + # ── 格式化输出 ──────────────────────────────────── + + @staticmethod + def format_branch_status(status_data): + """格式化分支状态为人类可读文本""" + branch = status_data["branch"] + latest = status_data["latest_run"] + overall = status_data["overall_status"] + + status_emoji = { + "success": "✅", + "failing": "🔴", + "flaky": "🟡", + "running": "🔄", + "no_runs": "⚪", + "unknown": "❓", + }.get(overall, "❓") + + lines = [f"**CI 状态：{branch} 分支**", f"整体状态: {status_emoji} {overall}"] + + if latest: + name = latest.get("name", "Unknown") + conclusion = latest.get("conclusion", latest.get("status", "unknown")) + run_id = latest.get("id", "") + created = latest.get("created_at", "")[:16].replace("T", " ") + run_url = f"{config.GITEA_URL}/{config.GITEA_REPO}/actions/runs/{run_id}" + lines.append(f"最新: [{name} #{run_id}]({run_url}) - {conclusion} ({created})") + + recent = status_data["recent_runs"] + if len(recent) > 1: + lines.append(f"\n最近 {len(recent)} 次:") + for r in recent[:5]: + c = r.get("conclusion", r.get("status", "?")) + emoji = {"success": "✅", "failure": "❌", "skipped": "⏭️"}.get(c, "🔄") + lines.append(f" {emoji} #{r.get('id', '?')} {r.get('name', '?')[:30]} - {c}") + + return "\n".join(lines) + + @staticmethod + def format_pr_status(status_data): + """格式化 PR 状态为人类可读文本""" + pr_num = status_data["pr_number"] + pr_title = status_data["pr_title"] + overall = status_data["overall_status"] + + status_emoji = { + "success": "✅", + "failing": "🔴", + "flaky": "🟡", + "running": "🔄", + "no_runs": "⚪", + "pr_not_found": "❓", + "unknown": "❓", + }.get(overall, "❓") + + pr_url = f"{config.GITEA_URL}/{config.GITEA_REPO}/pulls/{pr_num}" + lines = [ + f"**CI 状态：PR #{pr_num}**", + f"标题: [{pr_title}]({pr_url})", + f"状态: {status_emoji} {overall}", + ] + + runs = status_data["runs"] + if runs: + lines.append(f"\nCI Runs ({len(runs)}):") + for r in runs[:5]: + c = r.get("conclusion", r.get("status", "?")) + emoji = {"success": "✅", "failure": "❌", "skipped": "⏭️"}.get(c, "🔄") + run_id = r.get("id", "?") + run_url = f"{config.GITEA_URL}/{config.GITEA_REPO}/actions/runs/{run_id}" + lines.append(f" {emoji} [{r.get('name', '?')[:30]} #{run_id}]({run_url}) - {c}") + + return "\n".join(lines) + + +# ── CLI 入口 ────────────────────────────────────────── + + +def main(): + parser = argparse.ArgumentParser(description="CI 状态查询") + parser.add_argument("--branch", help="查询指定分支的 CI 状态") + parser.add_argument("--pr", type=int, help="查询指定 PR 的 CI 状态") + parser.add_argument("--run-id", help="查询指定 run 的详情") + parser.add_argument("--detail", action="store_true", help="显示失败详情") + parser.add_argument("--limit", type=int, default=5, help="返回数量限制") + parser.add_argument("--status", help="按状态过滤: success/failure/running") + + args = parser.parse_args() + + query = CIQuery() + + if args.run_id: + if args.detail: + result = query.get_failure_detail(args.run_id) + print(f"Run #{args.run_id} 失败详情:") + print(result["summary"]) + if result["failed_jobs"]: + print("\n详细日志尾部:") + for job in result["failed_jobs"]: + print(f"\n--- {job['name']} ---") + print(job["log_tail"][:500] if job["log_tail"] else "无日志") + else: + run = query.gitea.get_run(args.run_id) + if run: + print(f"Run #{args.run_id}: {run.get('name')} - {run.get('conclusion', run.get('status'))}") + print(f"分支: {run.get('head_branch', '?')}") + print(f"触发: {run.get('event', '?')}") + else: + print(f"Run {args.run_id} 不存在") + elif args.pr: + result = query.get_pr_status(args.pr) + print(CIQuery.format_pr_status(result)) + elif args.branch: + result = query.get_branch_status(args.branch, limit=args.limit) + print(CIQuery.format_branch_status(result)) + elif args.status: + result = query.list_recent_runs(status=args.status, limit=args.limit) + for r in result["runs"]: + print( + f"#{r.get('id')} {r.get('name')[:40]} - {r.get('conclusion', r.get('status'))} ({r.get('head_branch', '?')})" + ) + else: + parser.print_help() + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/chatops/ci_trigger.py b/scripts/ci/chatops/ci_trigger.py new file mode 100755 index 000000000..4ad7f1271 --- /dev/null +++ b/scripts/ci/chatops/ci_trigger.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +""" +CI 触发模块 - 重新运行失败 job、重跑整个 workflow、取消 run + +支持操作: + - rerun_failed: 重跑失败的 jobs + - rerun_all: 重跑整个 workflow + - cancel: 取消运行中的 run + +用法: + python3 scripts/ci/chatops/ci_trigger.py --run-id 456 --action rerun_failed + python3 scripts/ci/chatops/ci_trigger.py --run-id 456 --action rerun_all + python3 scripts/ci/chatops/ci_trigger.py --run-id 456 --action cancel + +设计: + - 与飞书机器人 /ci rerun 命令对接 + - 操作前自动校验 run 状态，避免无效操作 +""" + +import argparse +import sys + +from . import config +from .gitea_client import GiteaClient + + +class CITrigger: + """CI 操作触发器""" + + def __init__(self, gitea_client=None): + self.gitea = gitea_client or GiteaClient() + + # ── 触发操作 ───────────────────────────────────── + + def rerun_failed(self, run_id): + """重跑失败的 jobs + + Returns: + dict: {success, message, new_run_id?} + """ + run = self.gitea.get_run(run_id) + if not run: + return {"success": False, "message": f"Run {run_id} 不存在"} + + status = run.get("status", "") + if status != "completed": + return { + "success": False, + "message": f"Run {run_id} 当前状态为 {status}，仅 completed 状态才能重跑", + } + + result = self.gitea.rerun_failed_jobs(run_id) + if result is None: + return {"success": False, "message": "重跑请求失败"} + + # Gitea rerun 后返回的 run id 通常不变（复用原 run） + return { + "success": True, + "message": f"已触发重跑失败 jobs: Run #{run_id}", + "run_id": run_id, + "run_url": f"{config.GITEA_URL}/{config.GITEA_REPO}/actions/runs/{run_id}", + } + + def rerun_all(self, run_id): + """重跑整个 workflow run + + Returns: + dict: {success, message, run_id, run_url} + """ + run = self.gitea.get_run(run_id) + if not run: + return {"success": False, "message": f"Run {run_id} 不存在"} + + status = run.get("status", "") + if status == "running" or status == "pending": + return { + "success": False, + "message": f"Run {run_id} 正在运行中，无需重跑", + } + + result = self.gitea.rerun_run(run_id) + if result is None: + return {"success": False, "message": "重跑请求失败"} + + return { + "success": True, + "message": f"已触发完整重跑: Run #{run_id}", + "run_id": run_id, + "run_url": f"{config.GITEA_URL}/{config.GITEA_REPO}/actions/runs/{run_id}", + } + + def cancel_run(self, run_id): + """取消运行中的 run + + Returns: + dict: {success, message} + """ + run = self.gitea.get_run(run_id) + if not run: + return {"success": False, "message": f"Run {run_id} 不存在"} + + status = run.get("status", "") + if status == "completed": + return { + "success": False, + "message": f"Run {run_id} 已完成，无需取消", + } + + result = self.gitea.cancel_run(run_id) + if result is None: + return {"success": False, "message": "取消请求失败"} + + return { + "success": True, + "message": f"已取消 Run #{run_id}", + "run_id": run_id, + } + + def rerun_latest_failed(self, branch="develop", workflow_id=None): + """重跑指定分支最近一次失败的 run + + 用于快速恢复场景，不需要先查 run_id + """ + runs, _ = self.gitea.list_runs(branch=branch, workflow_id=workflow_id, status="failure", limit=5) + if not runs: + return {"success": False, "message": f"{branch} 分支没有失败的 run"} + + latest = runs[0] + run_id = latest.get("id") + return self.rerun_failed(run_id) + + +# ── CLI 入口 ────────────────────────────────────────── + + +def main(): + parser = argparse.ArgumentParser(description="CI 触发操作") + parser.add_argument("--run-id", required=True, help="Workflow Run ID") + parser.add_argument( + "--action", + required=True, + choices=["rerun_failed", "rerun_all", "cancel"], + help="操作类型", + ) + + args = parser.parse_args() + + trigger = CITrigger() + + if args.action == "rerun_failed": + result = trigger.rerun_failed(args.run_id) + elif args.action == "rerun_all": + result = trigger.rerun_all(args.run_id) + elif args.action == "cancel": + result = trigger.cancel_run(args.run_id) + else: + print(f"未知操作: {args.action}") + return 1 + + status = "✅" if result["success"] else "❌" + print(f"{status} {result['message']}") + if result.get("run_url"): + print(f" {result['run_url']}") + + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/chatops/config.py b/scripts/ci/chatops/config.py new file mode 100755 index 000000000..008bb7f6d --- /dev/null +++ b/scripts/ci/chatops/config.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +""" +ChatOps 配置管理 - 统一从环境变量读取配置，不硬编码任何敏感信息 + +环境变量: + GITEA_URL Gitea 地址 (默认 https://git.xiaoxiajianji.com) + GITEA_REPO 仓库路径 (默认 xiaoxia/xiaoxia-saas) + GITEA_TOKEN Gitea API Token (优先使用) + GITEA_USERNAME Gitea 用户名 (密码认证时) + GITEA_PASSWORD Gitea 密码 (密码认证时) + FEISHU_WEBHOOK_URL 飞书自定义机器人 webhook 地址 + FEISHU_APP_ID 飞书应用 App ID (应用机器人模式，预留) + FEISHU_APP_SECRET 飞书应用 App Secret (应用机器人模式，预留) + CHATOPS_NOTIFY_BRANCHES 触发通知的分支，逗号分隔 (默认 main,develop) + CHATOPS_WEBHOOK_PORT webhook 服务监听端口 (默认 8090) + CHATOPS_WEBHOOK_SECRET Gitea webhook 密钥 (校验签名，可选) +""" + +import os + +# ── Gitea 配置 ──────────────────────────────────────── +GITEA_URL = os.environ.get("GITEA_URL", "https://git.xiaoxiajianji.com").rstrip("/") +GITEA_REPO = os.environ.get("GITEA_REPO", "xiaoxia/xiaoxia-saas") +GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "") +GITEA_USERNAME = os.environ.get("GITEA_USERNAME", "") +GITEA_PASSWORD = os.environ.get("GITEA_PASSWORD", "") + +# ── 飞书配置 ────────────────────────────────────────── +FEISHU_WEBHOOK_URL = os.environ.get("FEISHU_WEBHOOK_URL", "") +FEISHU_APP_ID = os.environ.get("FEISHU_APP_ID", "") +FEISHU_APP_SECRET = os.environ.get("FEISHU_APP_SECRET", "") + +# ── 通知配置 ────────────────────────────────────────── +NOTIFY_BRANCHES = [b.strip() for b in os.environ.get("CHATOPS_NOTIFY_BRANCHES", "main,develop").split(",") if b.strip()] + +# ── Webhook 服务配置 ────────────────────────────────── +WEBHOOK_PORT = int(os.environ.get("CHATOPS_WEBHOOK_PORT", "8090")) +WEBHOOK_SECRET = os.environ.get("CHATOPS_WEBHOOK_SECRET", "") + +# ── 常量 ────────────────────────────────────────────── +PAGE_LIMIT = 50 # Gitea API 每页最大数量 + + +def has_gitea_auth() -> bool: + """检查是否配置了 Gitea 认证信息""" + if GITEA_TOKEN: + return True + if GITEA_USERNAME and GITEA_PASSWORD: + return True + return False + + +def has_feishu_webhook() -> bool: + """检查是否配置了飞书 webhook""" + return bool(FEISHU_WEBHOOK_URL) diff --git a/scripts/ci/chatops/feishu_notify.py b/scripts/ci/chatops/feishu_notify.py new file mode 100755 index 000000000..fefc48dca --- /dev/null +++ b/scripts/ci/chatops/feishu_notify.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +""" +飞书通知模块 - CI 关键事件推送 + +支持通知类型: + - branch_failure: main/develop 分支 CI 失败 + - branch_recovery: main/develop 分支 CI 从失败恢复（绿色恢复） + - e2e_failure: E2E 测试失败摘要 + - pr_failure: PR CI 失败（可选） + +用法: + # 命令行直接调用（供 CI workflow 使用） + python3 -m scripts.ci.chatops.feishu_notify --mode failure --run-id 12345 + python3 scripts/ci/chatops/feishu_notify.py --mode recovery --run-id 12345 + + # Python 模块调用 + from scripts.ci.chatops.feishu_notify import FeishuNotifier + notifier = FeishuNotifier() + notifier.notify_branch_failure(run_id=12345, branch="develop") + +设计原则: + 1. 通知失败永远不阻断主流程（返回 0） + 2. 卡片信息丰富，一键跳转 Gitea 详情页 + 3. 失败通知包含错误摘要，不用点进去就能判断严重程度 +""" + +import argparse +import json +import sys +import urllib.request +from datetime import datetime + +from . import config +from .gitea_client import GiteaClient + + +class FeishuNotifier: + """飞书通知发送器""" + + def __init__(self, webhook_url=None, gitea_client=None): + self.webhook_url = webhook_url or config.FEISHU_WEBHOOK_URL + self.gitea = gitea_client or GiteaClient() + + def _send_card(self, card_payload): + """发送飞书卡片消息 + + Returns: + True 表示发送成功（飞书返回 code=0） + """ + if not self.webhook_url: + print("[INFO] 未配置 FEISHU_WEBHOOK_URL，跳过飞书通知") + return False + + payload = {"msg_type": "interactive", "card": card_payload} + data = json.dumps(payload).encode("utf-8") + req = urllib.request.Request( + self.webhook_url, + data=data, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + resp_body = resp.read().decode("utf-8") + result = json.loads(resp_body) + if result.get("code", 0) != 0: + print( + f"[WARN] 飞书通知返回错误: {result.get('msg', resp_body)}", + file=sys.stderr, + ) + return False + return True + except Exception as e: + print(f"[WARN] 飞书通知发送失败: {e}", file=sys.stderr) + return False + + # ── 通知模板 ────────────────────────────────────── + + def _run_url(self, run_id): + return f"{config.GITEA_URL}/{config.GITEA_REPO}/actions/runs/{run_id}" + + def _pr_url(self, pr_number): + return f"{config.GITEA_URL}/{config.GITEA_REPO}/pulls/{pr_number}" + + def notify_branch_failure(self, run_id, branch, run_data=None): + """main/develop 分支 CI 失败通知 + + 包含: 失败 job 列表、错误摘要、一键重跑链接 + """ + run = run_data or self.gitea.get_run(run_id) + if not run: + print(f"[WARN] 无法获取 run {run_id} 详情", file=sys.stderr) + return False + + workflow_name = run.get("name", "Unknown Workflow") + commit_msg = run.get("head_commit", {}).get("message", "未知").splitlines()[0][:60] + commit_sha = run.get("head_sha", "")[:8] + actor = ( + run.get("trigger_event", {}).get("actor", {}).get("login", "unknown") + if isinstance(run.get("trigger_event"), dict) + else run.get("actor", "unknown") + ) + run_url = self._run_url(run_id) + + # 获取失败 job 摘要 + failed_jobs = self.gitea.get_failed_jobs_summary(run_id, max_lines_per_job=15) + + # 构建失败摘要 + failure_summary = "" + if failed_jobs: + job_lines = [] + for job in failed_jobs[:3]: # 最多显示 3 个 + step_info = f"（{job['failed_step']}）" if job["failed_step"] else "" + job_lines.append(f"• **{job['name']}**{step_info}") + if job["log_tail"]: + # 取最后 3 行日志 + tail_lines = job["log_tail"].strip().splitlines()[-3:] + for line in tail_lines: + clean_line = line.strip()[:100] + if clean_line: + job_lines.append(f" `{clean_line}`") + failure_summary = "\n".join(job_lines) + else: + failure_summary = "（获取失败详情中，点击查看日志）" + + # 字段 + fields = [ + {"is_short": True, "text": {"tag": "lark_md", "content": f"**分支**\n{branch}"}}, + {"is_short": True, "text": {"tag": "lark_md", "content": f"**Workflow**\n{workflow_name}"}}, + {"is_short": True, "text": {"tag": "lark_md", "content": f"**提交**\n`{commit_sha}`"}}, + {"is_short": True, "text": {"tag": "lark_md", "content": f"**触发者**\n{actor}"}}, + {"is_short": False, "text": {"tag": "lark_md", "content": f"**提交信息**\n{commit_msg}"}}, + {"is_short": False, "text": {"tag": "lark_md", "content": f"**失败详情**\n{failure_summary}"}}, + ] + + card = { + "header": { + "title": { + "tag": "plain_text", + "content": f"❌ CI告警：{branch} 分支构建失败", + }, + "status": "red", + }, + "elements": [ + {"tag": "div", "fields": fields}, + { + "tag": "action", + "actions": [ + { + "tag": "button", + "text": {"tag": "plain_text", "content": "查看失败日志"}, + "url": run_url, + "type": "danger", + }, + { + "tag": "button", + "text": {"tag": "plain_text", "content": "重跑失败Job"}, + "url": f"{run_url}/rerun-failed-jobs", + "type": "default", + }, + ], + }, + ], + } + + result = self._send_card(card) + print(f"[INFO] 分支失败通知已发送: {branch} run={run_id}") + return result + + def notify_branch_recovery(self, run_id, branch, previous_failure_run_id=None): + """分支 CI 恢复通知（从失败变成功）""" + run = self.gitea.get_run(run_id) + if not run: + print(f"[WARN] 无法获取 run {run_id} 详情", file=sys.stderr) + return False + + workflow_name = run.get("name", "Unknown Workflow") + commit_sha = run.get("head_sha", "")[:8] + run_url = self._run_url(run_id) + + # 计算恢复耗时 + duration_text = "已恢复" + if previous_failure_run_id: + prev_run = self.gitea.get_run(previous_failure_run_id) + if prev_run: + # 简单计算两个 run 的时间差 + prev_time = prev_run.get("created_at", "") + cur_time = run.get("created_at", "") + if prev_time and cur_time: + try: + t1 = datetime.fromisoformat(prev_time.replace("Z", "+00:00")) + t2 = datetime.fromisoformat(cur_time.replace("Z", "+00:00")) + diff = (t2 - t1).total_seconds() / 60 + duration_text = f"故障时长约 {diff:.0f} 分钟" + except Exception: + pass + + fields = [ + {"is_short": True, "text": {"tag": "lark_md", "content": f"**分支**\n{branch}"}}, + {"is_short": True, "text": {"tag": "lark_md", "content": f"**Workflow**\n{workflow_name}"}}, + {"is_short": True, "text": {"tag": "lark_md", "content": f"**提交**\n`{commit_sha}`"}}, + {"is_short": True, "text": {"tag": "lark_md", "content": "**状态**\n✅ 已恢复"}}, + {"is_short": False, "text": {"tag": "lark_md", "content": f"**说明**\n{duration_text}"}}, + ] + + card = { + "header": { + "title": { + "tag": "plain_text", + "content": f"✅ CI通知：{branch} 分支构建已恢复", + }, + "status": "green", + }, + "elements": [ + {"tag": "div", "fields": fields}, + { + "tag": "action", + "actions": [ + { + "tag": "button", + "text": {"tag": "plain_text", "content": "查看详情"}, + "url": run_url, + "type": "primary", + } + ], + }, + ], + } + + result = self._send_card(card) + print(f"[INFO] 分支恢复通知已发送: {branch} run={run_id}") + return result + + def notify_e2e_failure(self, run_id, branch="develop", pr_number=None): + """E2E 测试失败摘要通知""" + failed_jobs = self.gitea.get_failed_jobs_summary(run_id, max_lines_per_job=50) + e2e_jobs = [j for j in failed_jobs if "e2e" in j["name"].lower() or "test" in j["name"].lower()] + + if not e2e_jobs: + # 没有明确的 e2e job，取所有失败的 + e2e_jobs = failed_jobs + + run_url = self._run_url(run_id) + title_suffix = f"PR #{pr_number}" if pr_number else f"{branch} 分支" + + # 构建失败用例摘要 + case_summary = "" + for job in e2e_jobs[:3]: + case_summary += f"**{job['name']}**\n" + if job["log_tail"]: + # 尝试提取 FAIL 行 + fail_lines = [ + line.strip() + for line in job["log_tail"].splitlines() + if "FAIL" in line or "fail" in line.lower() or "✗" in line or "●" in line + ][:5] + if fail_lines: + for line in fail_lines: + case_summary += f" • {line[:120]}\n" + else: + tail = job["log_tail"].strip().splitlines()[-5:] + for line in tail: + case_summary += f" `{line.strip()[:100]}`\n" + case_summary += "\n" + + if not case_summary: + case_summary = "（点击查看完整测试报告）" + + fields = [ + {"is_short": True, "text": {"tag": "lark_md", "content": f"**来源**\n{title_suffix}"}}, + {"is_short": True, "text": {"tag": "lark_md", "content": f"**失败Job数**\n{len(e2e_jobs)}"}}, + {"is_short": False, "text": {"tag": "lark_md", "content": f"**失败摘要**\n{case_summary}"}}, + ] + + card = { + "header": { + "title": { + "tag": "plain_text", + "content": f"🧪 CI告警：E2E 测试失败 - {title_suffix}", + }, + "status": "orange", + }, + "elements": [ + {"tag": "div", "fields": fields}, + { + "tag": "action", + "actions": [ + { + "tag": "button", + "text": {"tag": "plain_text", "content": "查看完整报告"}, + "url": run_url, + "type": "danger", + } + ], + }, + ], + } + + result = self._send_card(card) + print(f"[INFO] E2E失败通知已发送: run={run_id}") + return result + + def notify_pr_failure(self, run_id, pr_number, pr_title=""): + """PR CI 失败通知（轻量版，可选开启）""" + run_url = self._run_url(run_id) + pr_url = self._pr_url(pr_number) + + failed_jobs = self.gitea.get_failed_jobs_summary(run_id, max_lines_per_job=10) + failure_names = [j["name"] for j in failed_jobs[:3]] + failure_text = "、".join(failure_names) if failure_names else "未知" + + fields = [ + { + "is_short": False, + "text": {"tag": "lark_md", "content": f"**PR**\n[#{pr_number} {pr_title[:50]}]({pr_url})"}, + }, + {"is_short": False, "text": {"tag": "lark_md", "content": f"**失败任务**\n{failure_text}"}}, + ] + + card = { + "header": { + "title": { + "tag": "plain_text", + "content": f"⚠️ CI通知：PR #{pr_number} 构建失败", + }, + "status": "yellow", + }, + "elements": [ + {"tag": "div", "fields": fields}, + { + "tag": "action", + "actions": [ + { + "tag": "button", + "text": {"tag": "plain_text", "content": "查看日志"}, + "url": run_url, + "type": "default", + }, + { + "tag": "button", + "text": {"tag": "plain_text", "content": "查看PR"}, + "url": pr_url, + "type": "default", + }, + ], + }, + ], + } + + result = self._send_card(card) + print(f"[INFO] PR失败通知已发送: PR #{pr_number} run={run_id}") + return result + + +# ── CLI 入口 ────────────────────────────────────────── + + +def main(): + parser = argparse.ArgumentParser(description="飞书 CI 通知") + parser.add_argument( + "--mode", + required=True, + choices=["failure", "recovery", "e2e_failure", "pr_failure"], + help="通知模式", + ) + parser.add_argument("--run-id", required=True, help="Workflow Run ID") + parser.add_argument("--branch", default="develop", help="分支名") + parser.add_argument("--pr-number", type=int, help="PR 编号") + parser.add_argument("--pr-title", default="", help="PR 标题") + parser.add_argument("--prev-run-id", help="上一个失败的 run ID（恢复通知用）") + parser.add_argument("--webhook", help="飞书 webhook URL（覆盖环境变量）") + + args = parser.parse_args() + + notifier = FeishuNotifier(webhook_url=args.webhook) + + if args.mode == "failure": + notifier.notify_branch_failure(args.run_id, args.branch) + elif args.mode == "recovery": + notifier.notify_branch_recovery(args.run_id, args.branch, previous_failure_run_id=args.prev_run_id) + elif args.mode == "e2e_failure": + notifier.notify_e2e_failure(args.run_id, branch=args.branch, pr_number=args.pr_number) + elif args.mode == "pr_failure": + notifier.notify_pr_failure(args.run_id, args.pr_number, pr_title=args.pr_title) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/chatops/gitea_client.py b/scripts/ci/chatops/gitea_client.py new file mode 100755 index 000000000..49f10ac68 --- /dev/null +++ b/scripts/ci/chatops/gitea_client.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +""" +Gitea API 客户端封装 - Actions + PR + Webhook 相关接口 + +基于 urllib 实现，无第三方依赖，与 ci_dashboard.py 风格一致。 +支持 token 和 basic auth 两种认证方式。 +""" + +import base64 +import json +import sys +import urllib.error +import urllib.request + +from . import config + + +class GiteaClient: + """Gitea API 客户端""" + + def __init__( + self, + base_url=None, + repo=None, + token=None, + username=None, + password=None, + ): + self.base_url = (base_url or config.GITEA_URL).rstrip("/") + self.repo = repo or config.GITEA_REPO + self.token = token or config.GITEA_TOKEN + self.username = username or config.GITEA_USERNAME + self.password = password or config.GITEA_PASSWORD + self.api_base = f"{self.base_url}/api/v1/repos/{self.repo}" + + def _request(self, path, method="GET", data=None): + """通用 HTTP 请求 + + Args: + path: API 路径（相对于 /api/v1/repos/{repo}/） + method: HTTP 方法 + data: 请求体（dict 或 bytes） + + Returns: + 解析后的 JSON 数据，失败返回 None + """ + url = f"{self.api_base}/{path}" + body = None + if data is not None: + if isinstance(data, (dict, list)): + body = json.dumps(data).encode("utf-8") + else: + body = data if isinstance(data, bytes) else str(data).encode() + + req = urllib.request.Request(url, data=body, method=method) + req.add_header("Content-Type", "application/json") + + if self.token: + req.add_header("Authorization", f"token {self.token}") + elif self.username and self.password: + auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode() + req.add_header("Authorization", f"Basic {auth}") + + try: + with urllib.request.urlopen(req, timeout=30) as resp: + resp_body = resp.read().decode() + if not resp_body: + return {} + return json.loads(resp_body) + except urllib.error.HTTPError as e: + err_body = "" + try: + err_body = e.read().decode() + except Exception: + pass + print( + f"[WARN] HTTP {e.code}: {url} - {err_body[:200]}", + file=sys.stderr, + ) + return None + except Exception as e: + print(f"[WARN] 请求失败 {url}: {e}", file=sys.stderr) + return None + + # ── Actions: Workflow Runs ──────────────────────── + + def list_runs( + self, + status=None, + branch=None, + event=None, + workflow_id=None, + page=1, + limit=config.PAGE_LIMIT, + ): + """获取 workflow runs 列表 + + Returns: + (runs列表, 总数) + """ + params = [] + if status: + params.append(f"status={status}") + if branch: + params.append(f"branch={branch}") + if event: + params.append(f"event={event}") + if workflow_id: + params.append(f"workflow_id={workflow_id}") + params.append(f"page={page}") + params.append(f"limit={limit}") + path = f"actions/runs?{'&'.join(params)}" + data = self._request(path) + if not data: + return [], 0 + runs = data.get("workflow_runs", []) + total = data.get("total_count", 0) + return runs, total + + def get_run(self, run_id): + """获取单个 run 详情""" + return self._request(f"actions/runs/{run_id}") + + def get_run_jobs(self, run_id): + """获取 run 的 jobs 列表""" + data = self._request(f"actions/runs/{run_id}/jobs") + if not data: + return [] + return data.get("jobs", []) + + def get_job_log(self, run_id, job_id): + """获取 job 日志（纯文本）""" + url = f"{self.api_base}/actions/runs/{run_id}/jobs/{job_id}/logs" + req = urllib.request.Request(url) + if self.token: + req.add_header("Authorization", f"token {self.token}") + elif self.username and self.password: + auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode() + req.add_header("Authorization", f"Basic {auth}") + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return resp.read().decode("utf-8", errors="replace") + except Exception as e: + print(f"[WARN] 获取日志失败 job={job_id}: {e}", file=sys.stderr) + return "" + + def rerun_run(self, run_id): + """重新运行整个 workflow run""" + return self._request(f"actions/runs/{run_id}/rerun", method="POST") + + def rerun_failed_jobs(self, run_id): + """重新运行失败的 jobs""" + return self._request(f"actions/runs/{run_id}/rerun-failed-jobs", method="POST") + + def cancel_run(self, run_id): + """取消 run""" + return self._request(f"actions/runs/{run_id}/cancel", method="POST") + + # ── Actions: Workflows ──────────────────────────── + + def list_workflows(self): + """获取 workflow 列表""" + data = self._request("actions/workflows") + if not data: + return [] + return data.get("workflows", []) + + def get_workflow(self, workflow_id): + """获取单个 workflow 详情""" + return self._request(f"actions/workflows/{workflow_id}") + + # ── Pull Requests ───────────────────────────────── + + def get_pr(self, pr_number): + """获取 PR 详情""" + return self._request(f"pulls/{pr_number}") + + def get_pr_ci_runs(self, pr_number, limit=20): + """获取 PR 关联的 CI runs（通过 head_sha 查询）""" + pr = self.get_pr(pr_number) + if not pr: + return [] + head_sha = pr.get("head", {}).get("sha", "") + if not head_sha: + return [] + # 用 head_sha 过滤 runs + runs, _ = self.list_runs(limit=limit) + return [r for r in runs if r.get("head_sha", "") == head_sha] + + # ── 便捷方法 ────────────────────────────────────── + + def get_latest_run(self, branch, workflow_id=None, status=None): + """获取指定分支最新的 run""" + runs, _ = self.list_runs(branch=branch, workflow_id=workflow_id, status=status, limit=5) + return runs[0] if runs else None + + def get_failed_jobs_summary(self, run_id, max_lines_per_job=30): + """获取失败 job 的摘要信息（用于通知） + + Returns: + list[dict]: 每个失败 job 的 {name, conclusion, failed_step, log_tail} + """ + jobs = self.get_run_jobs(run_id) + if not jobs: + return [] + + failed = [j for j in jobs if j.get("status") == "completed" and j.get("conclusion") == "failure"] + if not failed: + # 运行中的也返回，方便定位 + failed = [j for j in jobs if j.get("status") != "completed"] + + result = [] + for job in failed[:5]: # 最多取 5 个失败 job + job_id = job.get("id", "") + name = job.get("name", "Unknown") + conclusion = job.get("conclusion", job.get("status", "unknown")) + + # 找失败的 step + failed_step = "" + steps = job.get("steps", []) + for step in steps: + if step.get("conclusion") == "failure": + failed_step = step.get("name", "") + break + + # 取日志尾部 + log_tail = "" + if job_id: + log = self.get_job_log(run_id, job_id) + if log: + lines = log.strip().splitlines() + log_tail = "\n".join(lines[-max_lines_per_job:]) + + result.append( + { + "name": name, + "conclusion": conclusion, + "failed_step": failed_step, + "log_tail": log_tail, + "job_id": job_id, + } + ) + return result diff --git a/scripts/ci/chatops/webhook_server.py b/scripts/ci/chatops/webhook_server.py new file mode 100755 index 000000000..0cbcc5d14 --- /dev/null +++ b/scripts/ci/chatops/webhook_server.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +""" +Gitea Webhook 接收服务 - FastAPI 实现 + +功能: + 1. 接收 Gitea Actions webhook 事件，触发飞书通知 + 2. 接收飞书机器人回调消息，处理 /ci 交互命令 + 3. 维护简单的状态缓存，检测分支恢复等状态变化 + +部署: + 部署到构建服务器，监听 8090 端口（可配置） + Gitea webhook 指向: http://:8090/webhook/gitea + 飞书消息回调指向: http://:8090/webhook/feishu + +依赖: + fastapi + uvicorn（可选，未安装时仅模块可用，服务不可启动） + +注意: + 本文件为第一版骨架，通知逻辑已实现，飞书交互命令待后续完善。 +""" + +import hashlib +import hmac +import json +import sys +import threading +import time +from typing import Optional + +from . import config +from .gitea_client import GiteaClient + +# FastAPI 是可选依赖，未安装时仅导出类不启动服务 +try: + from fastapi import FastAPI, Header, HTTPException, Request + from fastapi.responses import JSONResponse + + FASTAPI_AVAILABLE = True +except ImportError: + FASTAPI_AVAILABLE = False + FastAPI = None # type: ignore + + +# ── 状态缓存 ────────────────────────────────────────── + + +class StateCache: + """简单的内存状态缓存，用于检测状态变化 + + 记录每个分支最后一次 run 的状态，用于判断: + - 是否从失败变成功（恢复通知） + - 是否连续失败（避免重复告警） + """ + + def __init__(self, max_entries=100): + self._cache = {} # {branch: {last_status, last_run_id, last_notified_failure}} + self._lock = threading.Lock() + self._max = max_entries + + def get(self, key): + with self._lock: + return self._cache.get(key) + + def set(self, key, value): + with self._lock: + self._cache[key] = value + # 简单的淘汰策略 + if len(self._cache) > self._max: + oldest_key = next(iter(self._cache)) + del self._cache[oldest_key] + + def check_and_update(self, branch, run_id, conclusion): + """检查状态变化并更新缓存 + + Returns: + dict: {is_new_failure, is_recovery, previous_status, previous_run_id} + """ + prev = self.get(branch) or {} + prev_status = prev.get("last_status", "unknown") + prev_run_id = prev.get("last_run_id") + + is_new_failure = False + is_recovery = False + + if conclusion == "failure" and prev_status != "failure": + is_new_failure = True + if conclusion == "success" and prev_status == "failure": + is_recovery = True + + self.set( + branch, + { + "last_status": conclusion, + "last_run_id": run_id, + "last_updated": time.time(), + "last_notified_failure": run_id if is_new_failure else prev.get("last_notified_failure"), + }, + ) + + return { + "is_new_failure": is_new_failure, + "is_recovery": is_recovery, + "previous_status": prev_status, + "previous_run_id": prev_run_id, + } + + +state_cache = StateCache() + + +# ── Gitea Webhook 处理 ──────────────────────────────── + + +def verify_gitea_signature(payload: bytes, signature: str) -> bool: + """校验 Gitea webhook 签名（X-Gitea-Signature） + + Gitea 使用 HMAC-SHA256 签名，格式: sha256=xxx + """ + if not config.WEBHOOK_SECRET: + return True # 未配置密钥则跳过校验 + + if not signature: + return False + + try: + algo, sig_hex = signature.split("=", 1) + if algo != "sha256": + return False + expected = hmac.new(config.WEBHOOK_SECRET.encode(), payload, hashlib.sha256).hexdigest() + return hmac.compare_digest(expected, sig_hex) + except Exception: + return False + + +def handle_gitea_webhook(payload: dict, event_type: str) -> dict: + """处理 Gitea webhook 事件 + + Args: + payload: webhook 请求体 + event_type: X-Gitea-Event 头 + + Returns: + dict: {handled, notifications_sent, message} + """ + if event_type != "create" and event_type != "push": + # 我们主要关心 push 和 actions 事件 + # Gitea Actions 的 webhook 事件类型可能是 "push" 或专门的 actions 事件 + pass + + # 尝试提取 run 信息 + run_info = _extract_run_info(payload) + if not run_info: + return {"handled": False, "notifications_sent": 0, "message": "非 CI 事件，跳过"} + + branch = run_info["branch"] + run_id = run_info["run_id"] + status = run_info["status"] + conclusion = run_info.get("conclusion", "") + + # 只处理已完成的 run + if status != "completed": + return {"handled": True, "notifications_sent": 0, "message": f"Run {run_id} 仍在运行中 ({status})"} + + # 检查是否在通知分支列表中 + if branch not in config.NOTIFY_BRANCHES: + return { + "handled": True, + "notifications_sent": 0, + "message": f"分支 {branch} 不在通知列表中", + } + + # 检查状态变化 + change_info = state_cache.check_and_update(branch, run_id, conclusion) + notifications = 0 + + # 延迟导入，避免循环依赖 + from .feishu_notify import FeishuNotifier + + notifier = FeishuNotifier() + + if conclusion == "failure" and change_info["is_new_failure"]: + # 新失败 → 发失败通知 + notifier.notify_branch_failure(run_id, branch) + notifications += 1 + + # 检查是否是 E2E 失败 + gitea = GiteaClient() + failed_jobs = gitea.get_failed_jobs_summary(run_id) + has_e2e = any("e2e" in j["name"].lower() for j in failed_jobs) + if has_e2e: + notifier.notify_e2e_failure(run_id, branch=branch) + notifications += 1 + + elif conclusion == "success" and change_info["is_recovery"]: + # 从失败恢复 → 发恢复通知 + prev_run_id = change_info.get("previous_run_id") + notifier.notify_branch_recovery(run_id, branch, previous_failure_run_id=prev_run_id) + notifications += 1 + + return { + "handled": True, + "notifications_sent": notifications, + "message": f"分支 {branch} run {run_id} {conclusion}", + } + + +def _extract_run_info(payload: dict) -> Optional[dict]: + """从 webhook payload 中提取 run 信息 + + Gitea Actions webhook 的 payload 结构可能不同，这里做兼容处理。 + 如果 payload 不是 run 事件，返回 None。 + """ + # 尝试多种可能的结构 + if "workflow_run" in payload: + wr = payload["workflow_run"] + return { + "run_id": wr.get("id"), + "branch": wr.get("head_branch", ""), + "status": wr.get("status", ""), + "conclusion": wr.get("conclusion", ""), + "name": wr.get("name", ""), + } + + if "action" in payload and "pull_request" in payload: + # PR 事件，暂不处理 + return None + + if "ref" in payload and "head_commit" in payload: + # push 事件，不是 run 事件 + return None + + return None + + +# ── 飞书消息处理 ────────────────────────────────────── + + +def handle_feishu_message(payload: dict) -> dict: + """处理飞书机器人回调消息 + + 支持命令: + /ci status [branch] - 查询分支 CI 状态 + /ci rerun - 重跑失败的 jobs + /ci help - 帮助 + + 注意: 第一版骨架，仅解析命令，实际执行逻辑待完善。 + """ + # 飞书消息回调格式 + header = payload.get("header", {}) + event_type = header.get("event_type", "") + + if event_type == "url_verification": + # 飞书 URL 验证 + return {"challenge": payload.get("challenge", "")} + + if event_type != "im.message.receive_v1": + return {"handled": False, "message": f"非消息事件: {event_type}"} + + event = payload.get("event", {}) + message = event.get("message", {}) + content_str = message.get("content", "{}") + + try: + content = json.loads(content_str) + except json.JSONDecodeError: + content = {} + + text = content.get("text", "") + if not text: + return {"handled": False, "message": "空消息"} + + # 解析命令 + text = text.strip() + if not text.startswith("/ci"): + return {"handled": False, "message": "非 CI 命令"} + + parts = text.split() + if len(parts) < 2: + return _help_response() + + cmd = parts[1].lower() + + if cmd == "status": + branch = parts[2] if len(parts) > 2 else "develop" + return _handle_status_command(branch) + + elif cmd == "rerun": + if len(parts) < 3: + return {"text": "用法: /ci rerun 或 /ci rerun latest [branch]"} + arg = parts[2] + if arg == "latest": + branch = parts[3] if len(parts) > 3 else "develop" + return _handle_rerun_latest(branch) + return _handle_rerun_command(arg) + + elif cmd == "help": + return _help_response() + + else: + return {"text": f"未知命令: {cmd}\n输入 /ci help 查看帮助"} + + +def _handle_status_command(branch: str) -> dict: + """处理 /ci status 命令""" + from .ci_query import CIQuery + + query = CIQuery() + result = query.get_branch_status(branch) + reply = CIQuery.format_branch_status(result) + return {"text": reply} + + +def _handle_rerun_command(run_id: str) -> dict: + """处理 /ci rerun 命令""" + from .ci_trigger import CITrigger + + trigger = CITrigger() + try: + result = trigger.rerun_failed(int(run_id)) + except ValueError: + return {"text": f"无效的 run id: {run_id}"} + + if result["success"]: + return {"text": f"✅ {result['message']}\n{result.get('run_url', '')}"} + return {"text": f"❌ {result['message']}"} + + +def _handle_rerun_latest(branch: str) -> dict: + """处理 /ci rerun latest 命令""" + from .ci_trigger import CITrigger + + trigger = CITrigger() + result = trigger.rerun_latest_failed(branch=branch) + + if result["success"]: + return {"text": f"✅ {result['message']}\n{result.get('run_url', '')}"} + return {"text": f"❌ {result['message']}"} + + +def _help_response() -> dict: + """返回帮助信息""" + help_text = """**CI ChatOps 命令帮助** + +`/ci status [branch]` 查询分支 CI 状态（默认 develop） +`/ci rerun ` 重跑指定 run 的失败 jobs +`/ci rerun latest [branch]` 重跑分支最近一次失败的 run +`/ci help` 显示此帮助 + +**环境变量配置:** + `GITEA_TOKEN` / `GITEA_USERNAME + GITEA_PASSWORD` + `FEISHU_WEBHOOK_URL` + `CHATOPS_NOTIFY_BRANCHES=main,develop` +""" + return {"text": help_text} + + +# ── FastAPI 应用 ────────────────────────────────────── + + +def create_app(): + """创建 FastAPI 应用 + + 如果 FastAPI 未安装，返回 None + """ + if not FASTAPI_AVAILABLE: + print( + "[WARN] FastAPI 未安装，无法启动 webhook 服务。" " 请运行: pip install fastapi uvicorn", + file=sys.stderr, + ) + return None + + app = FastAPI(title="CI ChatOps Webhook", version="0.1.0") + + @app.post("/webhook/gitea") + async def gitea_webhook( + request: Request, + x_gitea_event: str = Header(default=""), + x_gitea_signature: str = Header(default=""), + ): + body = await request.body() + + # 签名校验 + if not verify_gitea_signature(body, x_gitea_signature): + raise HTTPException(status_code=401, detail="Invalid signature") + + try: + payload = json.loads(body.decode()) + except json.JSONDecodeError as e: + raise HTTPException(status_code=400, detail="Invalid JSON") from e + + result = handle_gitea_webhook(payload, x_gitea_event) + return JSONResponse(content=result) + + @app.post("/webhook/feishu") + async def feishu_webhook(request: Request): + body = await request.body() + try: + payload = json.loads(body.decode()) + except json.JSONDecodeError as e: + raise HTTPException(status_code=400, detail="Invalid JSON") from e + + result = handle_feishu_message(payload) + return JSONResponse(content=result) + + @app.get("/health") + async def health(): + return {"status": "ok", "service": "ci-chatops"} + + return app + + +# ── CLI 入口 ────────────────────────────────────────── + + +def main(): + """启动 webhook 服务""" + import argparse + + parser = argparse.ArgumentParser(description="CI ChatOps Webhook 服务") + parser.add_argument("--port", type=int, default=config.WEBHOOK_PORT, help="监听端口") + parser.add_argument("--host", default="0.0.0.0", help="监听地址") + args = parser.parse_args() + + app = create_app() + if not app: + print("[ERROR] FastAPI 不可用，请先安装: pip install fastapi uvicorn") + return 1 + + try: + import uvicorn + except ImportError: + print("[ERROR] uvicorn 未安装，请先安装: pip install uvicorn") + return 1 + + print(f"[INFO] CI ChatOps Webhook 服务启动: http://{args.host}:{args.port}") + print("[INFO] Gitea webhook: POST /webhook/gitea") + print("[INFO] 飞书 webhook: POST /webhook/feishu") + print("[INFO] 健康检查: GET /health") + print(f"[INFO] 通知分支: {', '.join(config.NOTIFY_BRANCHES)}") + + uvicorn.run(app, host=args.host, port=args.port) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/check_migration_chain.py b/scripts/ci/check_migration_chain.py new file mode 100755 index 000000000..aec324d9b --- /dev/null +++ b/scripts/ci/check_migration_chain.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +""" +检查 Alembic migration 编号连续性。 + +扫描 alembic/versions/ 下所有 migration 文件，提取 revision 和 down_revision， +验证整条链是否完整——每个 down_revision（除了 baseline 的 None）都必须对应一个存在的 revision。 + +支持两种格式: + revision: str = "001" # 旧格式（带类型注解） + revision = "038_error_retry" # 新格式（带描述后缀） + +匹配策略：提取 revision 名称的数字前缀（如 "001"、"038"）作为唯一标识进行匹配， +兼容纯数字编号和"数字_描述"两种命名风格。 + +用法: + python3 scripts/ci/check_migration_chain.py [alembic_versions_dir] + +默认目录: alembic/versions/ + +退出码: + 0 - 链完整 + 1 - 有断链或其他错误 +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +# 匹配 revision / down_revision，支持带类型注解和不带类型注解两种格式 +# revision: str = "xxx" 或 revision = "xxx" +REV_PATTERN = re.compile( + r'^\s*revision\s*(?::\s*str\s*)?=\s*["\']([^"\']+)["\']', + re.MULTILINE, +) +DOWN_PATTERN = re.compile( + r'^\s*down_revision\s*(?::\s*(?:Union\[str,\s*None\]|str\s*\|\s*None|None|str)\s*)?=\s*(["\']([^"\']+)["\']|None)', + re.MULTILINE, +) + +# 提取 revision 名称的数字前缀，如 "001" 或 "038_error_retry" → "038" +NUM_PREFIX_PATTERN = re.compile(r"^(\d+)") + + +def num_prefix(name: str) -> str: + """提取 revision 名称的数字前缀。""" + m = NUM_PREFIX_PATTERN.match(name) + return m.group(1) if m else name + + +def extract_migration_info(filepath: Path) -> tuple[str, str | None]: + """从 migration 文件中提取 revision 和 down_revision（返回完整名称）。""" + content = filepath.read_text(encoding="utf-8") + + rev_match = REV_PATTERN.search(content) + down_match = DOWN_PATTERN.search(content) + + if not rev_match: + raise ValueError(f"{filepath.name}: 未找到 revision 定义") + + revision = rev_match.group(1) + + if not down_match: + raise ValueError(f"{filepath.name}: 未找到 down_revision 定义") + + # down_match group(2) 是引号内的值，如果是 None 则 group(2) 为 None + down_revision = down_match.group(2) + + return revision, down_revision + + +def check_chain(versions_dir: Path) -> list[str]: + """检查 migration 链是否完整，返回错误列表。""" + errors: list[str] = [] + + if not versions_dir.is_dir(): + return [f"目录不存在: {versions_dir}"] + + py_files = sorted(versions_dir.glob("*.py")) + if not py_files: + return [f"目录下没有 migration 文件: {versions_dir}"] + + # 收集所有 revision（用数字前缀做唯一标识） + revisions_by_num: dict[str, str] = {} # 数字前缀 -> 完整 revision 名 + revision_files: dict[str, str] = {} # 数字前缀 -> 文件名 + down_revisions: list[tuple[str, str | None]] = [] # (文件名, down_revision 数字前缀或None) + + for f in py_files: + if f.name.startswith("__"): + continue + try: + rev, down = extract_migration_info(f) + except ValueError as e: + errors.append(str(e)) + continue + + rev_num = num_prefix(rev) + + if rev_num in revisions_by_num: + errors.append( + f"编号重复: 编号 {rev_num} 同时出现在 " + f"{f.name} (revision={rev}) 和 {revision_files[rev_num]} (revision={revisions_by_num[rev_num]})" + ) + else: + revisions_by_num[rev_num] = rev + revision_files[rev_num] = f.name + + down_num = num_prefix(down) if down else None + down_revisions.append((f.name, down_num)) + + if errors: + return errors + + # 检查每个 down_revision 是否存在 + baselines = 0 + for filename, down_num in down_revisions: + if down_num is None: + baselines += 1 + continue + + if down_num not in revisions_by_num: + errors.append( + f"断链: {filename} 的 down_revision 指向编号 '{down_num}'，但没有任何 migration 的 revision 是这个编号" + ) + + if baselines == 0: + errors.append("没有找到 baseline migration（down_revision = None 的文件）") + elif baselines > 1: + errors.append(f"发现 {baselines} 个 baseline migration，通常只能有 1 个") + + # 额外检查：数字编号是否连续（只对能提取出数字的） + if revisions_by_num and not errors: + nums = sorted(int(n) for n in revisions_by_num if n.isdigit()) + if nums: + expected = list(range(nums[0], nums[-1] + 1)) + missing = [n for n in expected if n not in nums] + if missing: + missing_str = ", ".join(f"{n:03d}" for n in missing) + errors.append(f"编号不连续: 缺少编号 {missing_str}") + + return errors + + +def main() -> int: + if len(sys.argv) > 1: + versions_dir = Path(sys.argv[1]) + else: + versions_dir = Path("alembic/versions") + + print(f"检查 migration 编号连续性: {versions_dir}") + print() + + errors = check_chain(versions_dir) + + py_files = [f for f in versions_dir.glob("*.py") if not f.name.startswith("__")] + + if errors: + print(f"❌ Migration 链有问题（共 {len(py_files)} 个文件，{len(errors)} 个错误）:") + for e in errors: + print(f" - {e}") + print() + print("请修复后再提交。常见原因：") + print(" 1. 新 migration 的 down_revision 编号写错了") + print(" 2. 多个 PR 同时加 migration，编号冲突") + print(" 3. 合并代码时漏了某个 migration 文件") + return 1 + + print(f"✅ Migration 链完整，共 {len(py_files)} 个版本") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/ci_dashboard.py b/scripts/ci/ci_dashboard.py new file mode 100644 index 000000000..3c7c1f6f8 --- /dev/null +++ b/scripts/ci/ci_dashboard.py @@ -0,0 +1,973 @@ +#!/usr/bin/env python3 +""" +CI 可观测性看板 - 从 Gitea Actions API 拉取数据并生成 Markdown/HTML 日报 +用法: + python3 scripts/ci/ci_dashboard.py --days 7 + python3 scripts/ci/ci_dashboard.py --days 30 --output ci_report.md + python3 scripts/ci/ci_dashboard.py --workflow ci-cd.yml --days 7 + python3 scripts/ci/ci_dashboard.py --days 7 --html --html-output dashboard.html +环境变量: + GITEA_URL Gitea 地址 (默认 https://git.xiaoxiajianji.com) + GITEA_REPO 仓库 (默认 xiaoxia/xiaoxia-saas) + GITEA_TOKEN API Token (优先) 或 GITEA_USERNAME + GITEA_PASSWORD +""" + +import argparse +import base64 +import json +import math +import os +import statistics +import sys +import urllib.error +import urllib.request +from collections import defaultdict +from datetime import datetime, timedelta, timezone + +# ── 配置 ────────────────────────────────────────────── +DEFAULT_GITEA_URL = "https://git.xiaoxiajianji.com" +DEFAULT_REPO = "xiaoxia/xiaoxia-saas" +DEFAULT_DAYS = 7 +PAGE_LIMIT = 50 # 每页数量，最大50 + + +# ── API 封装 ───────────────────────────────────────── +class GiteaActions: + def __init__(self, base_url, repo, token=None, username=None, password=None): + self.base_url = base_url.rstrip("/") + self.repo = repo + self.token = token + self.username = username + self.password = password + self.api_base = f"{self.base_url}/api/v1/repos/{self.repo}/actions" + + def _request(self, path): + url = f"{self.api_base}/{path}" + req = urllib.request.Request(url) + if self.token: + req.add_header("Authorization", f"token {self.token}") + elif self.username and self.password: + auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode() + req.add_header("Authorization", f"Basic {auth}") + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + print(f"[WARN] HTTP {e.code}: {url}", file=sys.stderr) + return None + except Exception as e: + print(f"[WARN] 请求失败 {url}: {e}", file=sys.stderr) + return None + + def list_runs(self, status=None, branch=None, event=None, page=1, limit=PAGE_LIMIT): + """获取 workflow runs 列表""" + params = [] + if status: + params.append(f"status={status}") + if branch: + params.append(f"branch={branch}") + if event: + params.append(f"event={event}") + params.append(f"page={page}") + params.append(f"limit={limit}") + path = f"runs?{'&'.join(params)}" + data = self._request(path) + if not data: + return [], 0 + runs = data.get("workflow_runs", []) + total = data.get("total_count", 0) + return runs, total + + def get_run_jobs(self, run_id): + """获取 run 的所有 job""" + data = self._request(f"runs/{run_id}/jobs") + if not data: + return [] + return data.get("jobs", []) + + def list_workflows(self): + """获取所有 workflow""" + data = self._request("workflows") + if not data: + return [] + return data.get("workflows", []) + + +# ── 工具函数 ───────────────────────────────────────── +def parse_datetime(s): + """解析 ISO 格式时间字符串""" + if not s or s.startswith("1970") or s.startswith("0001"): + return None + try: + if s.endswith("Z"): + s = s[:-1] + "+00:00" + return datetime.fromisoformat(s) + except Exception: + return None + + +def to_shanghai(dt): + """转换为上海时区""" + if dt is None: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone(timedelta(hours=8))) + + +def duration_seconds(start_str, end_str): + """计算耗时（秒）""" + start = parse_datetime(start_str) + end = parse_datetime(end_str) + if not start or not end: + return None + return (end - start).total_seconds() + + +def fmt_duration(seconds): + """格式化耗时显示""" + if seconds is None: + return "N/A" + seconds = int(seconds) + if seconds < 60: + return f"{seconds}s" + mins, secs = divmod(seconds, 60) + if mins < 60: + return f"{mins}m{secs:02d}s" + hours, mins = divmod(mins, 60) + return f"{hours}h{mins:02d}m" + + +def percentile(sorted_values, p): + """计算百分位数""" + if not sorted_values: + return None + k = (len(sorted_values) - 1) * (p / 100) + f = math.floor(k) + c = math.ceil(k) + if f == c: + return sorted_values[int(k)] + return sorted_values[f] * (c - k) + sorted_values[c] * (k - f) + + +def classify_failure(job_name, step_name=None): + """根据失败的 job/step 名称分类失败原因""" + name = f"{job_name} {step_name or ''}".lower() + if any(k in name for k in ["lint", "ruff", "flake8", "eslint", "prettier", "black", "mypy"]): + return "代码质量 / Lint" + if any(k in name for k in ["unit test", "pytest", "vitest", "jest"]): + return "单元测试失败" + if any(k in name for k in ["integration", "e2e"]): + return "集成测试 / E2E" + if any(k in name for k in ["build", "compile", "docker", "image"]): + return "构建失败" + if any(k in name for k in ["deploy", "preview", "release"]): + return "部署失败" + if any(k in name for k in ["setup", "checkout", "cache", "install", "deps"]): + return "环境 / 依赖" + if any(k in name for k in ["migrate", "migration", "schema"]): + return "数据库迁移" + return "其他" + + +# ── 数据收集 ───────────────────────────────────────── +def fetch_runs_in_range(ga, start_date, end_date, workflow_filter=None): + """拉取指定日期范围内的所有 completed runs""" + all_runs = [] + page = 1 + print(f"[INFO] 拉取 {start_date} ~ {end_date} 的 CI runs...", file=sys.stderr) + while True: + runs, total = ga.list_runs(status="completed", page=page, limit=PAGE_LIMIT) + if not runs: + break + if workflow_filter: + runs = [r for r in runs if workflow_filter in r.get("path", "")] + in_range = [] + out_range_old = False + for run in runs: + started = to_shanghai(parse_datetime(run.get("started_at"))) + if not started: + continue + run_date = started.date() + if start_date <= run_date <= end_date: + in_range.append(run) + elif run_date < start_date: + out_range_old = True + all_runs.extend(in_range) + print( + f"[INFO] 第 {page} 页: {len(runs)} 条, 范围内 {len(in_range)} 条, 累计 {len(all_runs)} 条", file=sys.stderr + ) + if out_range_old or len(runs) < PAGE_LIMIT: + break + page += 1 + if page > 100: + print("[WARN] 超过100页，停止拉取", file=sys.stderr) + break + print(f"[INFO] 共获取 {len(all_runs)} 条 run 数据", file=sys.stderr) + return all_runs + + +def enrich_with_jobs(ga, runs, max_failures=50): + """为 runs 补充 job 详情（失败原因分析 + runner 统计） + 失败 run 按时间倒序取最近 N 个（避免 API 调用过多）， + 成功 run 采样用于 runner 分布统计。 + """ + # 失败 run 取最近 N 个 + failure_runs = [r for r in runs if r.get("conclusion") != "success"] + failure_runs = failure_runs[:max_failures] # 已经是时间倒序 + print(f"[INFO] 为最近 {len(failure_runs)} 个失败 run 拉取 job 详情...", file=sys.stderr) + for i, run in enumerate(failure_runs): + jobs = ga.get_run_jobs(run["id"]) + run["_jobs"] = jobs + if (i + 1) % 10 == 0: + print(f"[INFO] 已处理 {i+1}/{len(failure_runs)}", file=sys.stderr) + # 成功 run 采样用于 runner 分布 + success_runs = [r for r in runs if r.get("conclusion") == "success"] + sample_size = min(50, len(success_runs)) + if sample_size > 0: + sampled = success_runs[:: max(1, len(success_runs) // sample_size)] + print(f"[INFO] 采样 {len(sampled)} 个成功 run 用于 runner 统计...", file=sys.stderr) + for run in sampled: + if "_jobs" not in run: + jobs = ga.get_run_jobs(run["id"]) + run["_jobs"] = jobs + return runs + + +# ── 统计分析 ───────────────────────────────────────── +def analyze_runs(runs): + """对 runs 做全面统计分析""" + if not runs: + return {} + + # 基础统计 + total = len(runs) + success = sum(1 for r in runs if r.get("conclusion") == "success") + failure = sum(1 for r in runs if r.get("conclusion") == "failure") + cancelled = sum(1 for r in runs if r.get("conclusion") == "cancelled") + other = total - success - failure - cancelled + success_rate = (success / total * 100) if total > 0 else 0 + + # 耗时统计 + durations = [] + for r in runs: + d = duration_seconds(r.get("started_at"), r.get("completed_at")) + if d and d > 0: + durations.append(d) + durations.sort() + avg_dur = statistics.mean(durations) if durations else None + median_dur = percentile(durations, 50) + p95_dur = percentile(durations, 95) + + # 按日期统计 + daily_stats = defaultdict(lambda: {"total": 0, "success": 0, "failure": 0, "durations": []}) + for r in runs: + started = to_shanghai(parse_datetime(r.get("started_at"))) + if not started: + continue + day = started.date().isoformat() + daily_stats[day]["total"] += 1 + if r.get("conclusion") == "success": + daily_stats[day]["success"] += 1 + elif r.get("conclusion") == "failure": + daily_stats[day]["failure"] += 1 + d = duration_seconds(r.get("started_at"), r.get("completed_at")) + if d and d > 0: + daily_stats[day]["durations"].append(d) + + # 按 workflow 统计 + wf_stats = defaultdict(lambda: {"total": 0, "success": 0, "failure": 0, "durations": []}) + for r in runs: + path = r.get("path", "") + wf_name = path.split("@")[0] if "@" in path else path + wf_stats[wf_name]["total"] += 1 + if r.get("conclusion") == "success": + wf_stats[wf_name]["success"] += 1 + elif r.get("conclusion") == "failure": + wf_stats[wf_name]["failure"] += 1 + d = duration_seconds(r.get("started_at"), r.get("completed_at")) + if d and d > 0: + wf_stats[wf_name]["durations"].append(d) + + # 按触发事件统计 + event_stats = defaultdict(lambda: {"total": 0, "success": 0, "failure": 0}) + for r in runs: + evt = r.get("event", "unknown") + event_stats[evt]["total"] += 1 + if r.get("conclusion") == "success": + event_stats[evt]["success"] += 1 + elif r.get("conclusion") == "failure": + event_stats[evt]["failure"] += 1 + + # 失败原因 + runner + job 耗时（需要 _jobs 数据） + failure_categories = defaultdict(int) + failed_jobs_by_name = defaultdict(int) + job_success_stats = defaultdict(lambda: {"total": 0, "success": 0, "failure": 0}) + runner_stats = defaultdict(lambda: {"jobs": 0, "success": 0, "failure": 0, "durations": []}) + job_time_stats = defaultdict(list) + infra_failures = 0 + business_failures = 0 + other_failures_count = 0 + + # 基础设施关键词（与 ci_health_check.py 保持一致的分类逻辑） + infra_job_keywords = ["checkout", "build", "deploy", "cleanup", "setup", "cache", "install", "docker"] + business_job_keywords = [ + "unit test", + "pytest", + "vitest", + "jest", + "lint", + "eslint", + "prettier", + "integration", + "e2e", + "validate", + "code quality", + "mypy", + "ruff", + "flake8", + ] + + for r in runs: + jobs = r.get("_jobs", []) + if not jobs: + continue + for job in jobs: + runner = job.get("runner_name", "unknown") + conclusion = job.get("conclusion", "unknown") + job_name = job.get("name", "unknown") + job_name_lower = job_name.lower() + + runner_stats[runner]["jobs"] += 1 + job_success_stats[job_name]["total"] += 1 + if conclusion == "success": + runner_stats[runner]["success"] += 1 + job_success_stats[job_name]["success"] += 1 + elif conclusion == "failure": + runner_stats[runner]["failure"] += 1 + job_success_stats[job_name]["failure"] += 1 + + jd = duration_seconds(job.get("started_at"), job.get("completed_at")) + if jd and jd > 0: + runner_stats[runner]["durations"].append(jd) + job_time_stats[job_name].append(jd) + + if conclusion == "failure": + failed_jobs_by_name[job_name] += 1 + failed_step = None + for step in job.get("steps", []): + if step.get("conclusion") == "failure": + failed_step = step.get("name") + break + category = classify_failure(job_name, failed_step) + failure_categories[category] += 1 + + # 基础设施 vs 业务代码分类 + is_infra = any(k in job_name_lower for k in infra_job_keywords) and not any( + k in job_name_lower for k in business_job_keywords + ) + is_business = any(k in job_name_lower for k in business_job_keywords) + if is_infra: + infra_failures += 1 + elif is_business: + business_failures += 1 + else: + other_failures_count += 1 + + return { + "total": total, + "success": success, + "failure": failure, + "cancelled": cancelled, + "other": other, + "success_rate": success_rate, + "avg_duration": avg_dur, + "median_duration": median_dur, + "p95_duration": p95_dur, + "durations": durations, + "daily_stats": dict(sorted(daily_stats.items())), + "workflow_stats": dict(wf_stats), + "event_stats": dict(event_stats), + "failure_categories": dict(failure_categories), + "failed_jobs_top": dict(sorted(failed_jobs_by_name.items(), key=lambda x: -x[1])[:15]), + "runner_stats": dict(runner_stats), + "job_time_stats": dict(job_time_stats), + "job_success_stats": dict(job_success_stats), + "infra_failures": infra_failures, + "business_failures": business_failures, + "other_failures_combined": other_failures_count, + } + + +# ── Markdown 报表生成 ──────────────────────────────── +def generate_markdown(stats, start_date, end_date, repo): + """生成 Markdown 格式的日报""" + lines = [] + lines.append("# CI 运行状态看板") + lines.append("") + lines.append(f"> 统计周期: **{start_date} ~ {end_date}**") + lines.append(f"> 仓库: `{repo}`") + lines.append(f"> 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + lines.append("") + + # 概览 + lines.append("## 📊 整体概览") + lines.append("") + lines.append("| 指标 | 数值 |") + lines.append("|------|------|") + lines.append(f"| 总构建次数 | **{stats['total']}** |") + lines.append(f"| ✅ 成功 | {stats['success']} |") + lines.append(f"| ❌ 失败 | {stats['failure']} |") + lines.append(f"| ⏹️ 取消 | {stats['cancelled']} |") + lines.append(f"| 📈 成功率 | **{stats['success_rate']:.1f}%** |") + lines.append(f"| ⏱️ 平均耗时 | {fmt_duration(stats['avg_duration'])} |") + lines.append(f"| ⏱️ P50 耗时 | {fmt_duration(stats['median_duration'])} |") + lines.append(f"| ⏱️ P95 耗时 | {fmt_duration(stats['p95_duration'])} |") + lines.append("") + + # 每日趋势 + lines.append("## 📈 每日趋势") + lines.append("") + lines.append("| 日期 | 总次数 | 成功 | 失败 | 成功率 | 平均耗时 | P95 耗时 |") + lines.append("|------|--------|------|------|--------|----------|----------|") + for day, s in stats["daily_stats"].items(): + rate = (s["success"] / s["total"] * 100) if s["total"] > 0 else 0 + durations = sorted(s["durations"]) + avg = statistics.mean(durations) if durations else None + p95 = percentile(durations, 95) if durations else None + lines.append( + f"| {day} | {s['total']} | {s['success']} | {s['failure']} | {rate:.1f}% | {fmt_duration(avg)} | {fmt_duration(p95)} |" + ) + lines.append("") + + # 成功率趋势图 + lines.append("### 成功率趋势图") + lines.append("") + lines.append("```") + max_bar = 40 + days = list(stats["daily_stats"].keys()) + if len(days) > 14: + days = days[-14:] + for day in days: + s = stats["daily_stats"][day] + rate = (s["success"] / s["total"] * 100) if s["total"] > 0 else 0 + bar_len = int(rate / 100 * max_bar) + bar = "█" * bar_len + "░" * (max_bar - bar_len) + lines.append(f"{day} {bar} {rate:5.1f}% ({s['total']}次)") + lines.append("```") + lines.append("") + + # 按 Workflow 统计 + lines.append("## 🧩 各 Workflow 统计") + lines.append("") + wf_sorted = sorted(stats["workflow_stats"].items(), key=lambda x: -x[1]["total"]) + lines.append("| Workflow | 次数 | 成功 | 失败 | 成功率 | 平均耗时 | P95 耗时 |") + lines.append("|----------|------|------|------|--------|----------|----------|") + for wf, s in wf_sorted: + rate = (s["success"] / s["total"] * 100) if s["total"] > 0 else 0 + durations = sorted(s["durations"]) + avg = statistics.mean(durations) if durations else None + p95 = percentile(durations, 95) if durations else None + wf_short = wf.split("/")[-1] if "/" in wf else wf + lines.append( + f"| `{wf_short}` | {s['total']} | {s['success']} | {s['failure']} | {rate:.1f}% | {fmt_duration(avg)} | {fmt_duration(p95)} |" + ) + lines.append("") + + # 失败原因分析 + if stats["failure_categories"]: + lines.append("## ❌ 失败原因分析") + lines.append("") + lines.append("> ⚠️ 基于最近 N 个失败 run 采样分析，用于趋势参考") + lines.append("") + lines.append("### 按分类统计") + lines.append("") + total_failures = sum(stats["failure_categories"].values()) + fc_sorted = sorted(stats["failure_categories"].items(), key=lambda x: -x[1]) + lines.append("| 分类 | 次数 | 占比 |") + lines.append("|------|------|------|") + for cat, cnt in fc_sorted: + pct = (cnt / total_failures * 100) if total_failures > 0 else 0 + lines.append(f"| {cat} | {cnt} | {pct:.1f}% |") + lines.append("") + + lines.append("### Top 失败 Job") + lines.append("") + lines.append("| Job 名称 | 失败次数 |") + lines.append("|----------|----------|") + for job, cnt in stats["failed_jobs_top"].items(): + lines.append(f"| `{job}` | {cnt} |") + lines.append("") + + # Runner 利用率 + if stats["runner_stats"]: + lines.append("## 🏃 Runner 利用率") + lines.append("") + runner_sorted = sorted(stats["runner_stats"].items(), key=lambda x: -x[1]["jobs"]) + lines.append("| Runner | Job 数 | 成功 | 失败 | 成功率 | 平均耗时 |") + lines.append("|--------|--------|------|------|--------|----------|") + for runner, s in runner_sorted: + rate = (s["success"] / s["jobs"] * 100) if s["jobs"] > 0 else 0 + avg = statistics.mean(s["durations"]) if s["durations"] else None + lines.append( + f"| `{runner}` | {s['jobs']} | {s['success']} | {s['failure']} | {rate:.1f}% | {fmt_duration(avg)} |" + ) + lines.append("") + + # Job 耗时排行 + if stats["job_time_stats"]: + lines.append("## ⏱️ Job 耗时排行 (Top 20 by P95)") + lines.append("") + job_stats = [] + for name, durs in stats["job_time_stats"].items(): + if not durs: + continue + durs_sorted = sorted(durs) + job_stats.append( + { + "name": name, + "count": len(durs_sorted), + "avg": statistics.mean(durs_sorted), + "p50": percentile(durs_sorted, 50), + "p95": percentile(durs_sorted, 95), + } + ) + job_stats.sort(key=lambda x: -x["p95"]) + top_n = min(20, len(job_stats)) + lines.append("| Job 名称 | 次数 | 平均 | P50 | P95 |") + lines.append("|----------|------|------|-----|-----|") + for j in job_stats[:top_n]: + lines.append( + f"| `{j['name']}` | {j['count']} | {fmt_duration(j['avg'])} | {fmt_duration(j['p50'])} | {fmt_duration(j['p95'])} |" + ) + lines.append("") + + # 触发事件分布 + lines.append("## 📋 触发事件分布") + lines.append("") + evt_sorted = sorted(stats["event_stats"].items(), key=lambda x: -x[1]["total"]) + lines.append("| 事件类型 | 次数 | 成功 | 失败 | 成功率 |") + lines.append("|----------|------|------|------|--------|") + for evt, s in evt_sorted: + rate = (s["success"] / s["total"] * 100) if s["total"] > 0 else 0 + lines.append(f"| `{evt}` | {s['total']} | {s['success']} | {s['failure']} | {rate:.1f}% |") + lines.append("") + + return "\n".join(lines) + + +# ── HTML 看板生成 ──────────────────────────────────── +def generate_html(stats, start_date, end_date, repo): + """生成 HTML 格式的可视化看板（内嵌 ECharts）""" + # 准备图表数据 + + # 1. 每日成功率趋势 + daily_dates = list(stats["daily_stats"].keys()) + daily_success_rates = [] + daily_run_counts = [] + for day in daily_dates: + s = stats["daily_stats"][day] + rate = (s["success"] / s["total"] * 100) if s["total"] > 0 else 0 + daily_success_rates.append(round(rate, 1)) + daily_run_counts.append(s["total"]) + + # 2. 各 Workflow 耗时对比 + wf_sorted = sorted(stats["workflow_stats"].items(), key=lambda x: -x[1]["total"]) + wf_names = [] + wf_avg_durations = [] + for wf, s in wf_sorted: + wf_short = wf.split("/")[-1] if "/" in wf else wf + wf_names.append(wf_short) + avg = statistics.mean(s["durations"]) if s["durations"] else 0 + wf_avg_durations.append(round(avg / 60, 1)) # 转为分钟 + + # 3. 失败原因分布（饼图数据 - 基础设施 vs 业务 vs 其他） + total_infra_biz = stats["infra_failures"] + stats["business_failures"] + stats["other_failures_combined"] + infra_rate = (stats["infra_failures"] / total_infra_biz * 100) if total_infra_biz > 0 else 0 + failure_pie_data = [ + {"value": stats["infra_failures"], "name": "基础设施问题"}, + {"value": stats["business_failures"], "name": "业务代码问题"}, + {"value": stats["other_failures_combined"], "name": "其他"}, + ] + + # 4. 各 Job 成功率排行（横向柱状图，取成功率最低的 Top 15） + job_stats_list = [] + for name, s in stats["job_success_stats"].items(): + if s["total"] >= 3: # 至少有3次才统计 + rate = (s["success"] / s["total"] * 100) if s["total"] > 0 else 0 + job_stats_list.append( + { + "name": name, + "rate": round(rate, 1), + "total": s["total"], + "success": s["success"], + } + ) + job_stats_list.sort(key=lambda x: x["rate"]) + job_stats_list = job_stats_list[:15] # 取成功率最低的15个 + job_names = [j["name"] for j in job_stats_list] + job_rates = [j["rate"] for j in job_stats_list] + + # 核心指标 + total_runs = stats["total"] + success_rate = round(stats["success_rate"], 1) + avg_dur_min = round(stats["avg_duration"] / 60, 1) if stats["avg_duration"] else 0 + infra_fail_rate = round(infra_rate, 1) + + now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + # 序列化数据为 JSON（供 JS 使用） + data_json = json.dumps( + { + "daily_dates": daily_dates, + "daily_success_rates": daily_success_rates, + "daily_run_counts": daily_run_counts, + "wf_names": wf_names, + "wf_avg_durations": wf_avg_durations, + "failure_pie_data": failure_pie_data, + "job_names": job_names, + "job_rates": job_rates, + }, + ensure_ascii=False, + ) + + # HTML 模板（注意：不使用 f-string，避免与 CSS/JS 的大括号冲突） + html_parts = [] + html_parts.append("") + html_parts.append('') + html_parts.append("") + html_parts.append(' ') + html_parts.append(' ') + html_parts.append(f" CI 健康度看板 - {repo}") + html_parts.append(' ') + html_parts.append(" ") + html_parts.append("") + html_parts.append("") + html_parts.append('
') + html_parts.append('
') + html_parts.append("

📊 CI 健康度看板

") + html_parts.append(f'
仓库: {repo}
') + html_parts.append(f'
统计周期: {start_date} ~ {end_date} | 生成时间: {now_str}
') + html_parts.append("
") + html_parts.append('
') + html_parts.append('
') + html_parts.append('
总成功率
') + html_parts.append(f'
{success_rate}%
') + html_parts.append("
") + html_parts.append('
') + html_parts.append('
总 Run 数
') + html_parts.append(f'
{total_runs}次
') + html_parts.append("
") + html_parts.append('
') + html_parts.append('
平均耗时
') + html_parts.append(f'
{avg_dur_min}分钟
') + html_parts.append("
") + html_parts.append('
') + html_parts.append('
基础设施故障率
') + html_parts.append(f'
{infra_fail_rate}%
') + html_parts.append("
") + html_parts.append("
") + html_parts.append('
') + html_parts.append('
') + html_parts.append("

📈 CI 成功率趋势

") + html_parts.append('
') + html_parts.append("
") + html_parts.append("
") + html_parts.append('
') + html_parts.append('
') + html_parts.append("

⏱️ 各 Workflow 平均耗时

") + html_parts.append('
') + html_parts.append("
") + html_parts.append('
') + html_parts.append("

❌ 失败原因分布

") + html_parts.append('
') + html_parts.append("
") + html_parts.append("
") + html_parts.append('
') + html_parts.append('
') + html_parts.append("

📋 各 Job 成功率排行（最低 15 名）

") + html_parts.append('
') + html_parts.append("
") + html_parts.append("
") + html_parts.append('
') + html_parts.append('
') + html_parts.append("

📊 每日 Run 数量趋势

") + html_parts.append('
') + html_parts.append("
") + html_parts.append("
") + html_parts.append('
') + html_parts.append(" 由 ci_dashboard.py 自动生成 | ECharts 可视化") + html_parts.append("
") + html_parts.append("
") + html_parts.append(" ") + html_parts.append("") + html_parts.append("") + + return "\n".join(html_parts) + + +# ── 主函数 ─────────────────────────────────────────── +def main(): + parser = argparse.ArgumentParser(description="CI 可观测性看板 - 生成 Gitea Actions 运行状态报表") + parser.add_argument("--days", type=int, default=DEFAULT_DAYS, help=f"统计最近 N 天 (默认 {DEFAULT_DAYS})") + parser.add_argument("--output", "-o", type=str, help="输出文件路径 (默认输出到 stdout)") + parser.add_argument("--workflow", type=str, help="只统计指定 workflow (如 ci-cd.yml)") + parser.add_argument("--gitea-url", type=str, default=os.environ.get("GITEA_URL", DEFAULT_GITEA_URL)) + parser.add_argument("--repo", type=str, default=os.environ.get("GITEA_REPO", DEFAULT_REPO)) + parser.add_argument("--token", type=str, default=os.environ.get("GITEA_TOKEN")) + parser.add_argument("--username", type=str, default=os.environ.get("GITEA_USERNAME")) + parser.add_argument("--password", type=str, default=os.environ.get("GITEA_PASSWORD")) + parser.add_argument("--no-job-detail", action="store_true", help="不拉取 job 详情") + parser.add_argument("--max-failures", type=int, default=50, help="最多分析多少个失败 run 的 job 详情 (默认 50)") + + # HTML 输出相关参数 + parser.add_argument("--html", action="store_true", help="生成 HTML 可视化看板") + parser.add_argument("--html-output", type=str, help="HTML 输出文件路径 (默认 ci_dashboard.html)") + + args = parser.parse_args() + + ga = GiteaActions( + base_url=args.gitea_url, + repo=args.repo, + token=args.token, + username=args.username, + password=args.password, + ) + + end_date = datetime.now().date() + start_date = end_date - timedelta(days=args.days - 1) + + runs = fetch_runs_in_range(ga, start_date, end_date, args.workflow) + if not runs: + print("[ERROR] 未获取到任何数据", file=sys.stderr) + sys.exit(1) + + if not args.no_job_detail: + runs = enrich_with_jobs(ga, runs, max_failures=args.max_failures) + + stats = analyze_runs(runs) + + # HTML 模式 + if args.html: + html = generate_html(stats, start_date, end_date, args.repo) + html_output = args.html_output or args.output or "ci_dashboard.html" + with open(html_output, "w", encoding="utf-8") as f: + f.write(html) + print(f"[INFO] HTML 看板已保存到 {html_output}", file=sys.stderr) + return + + # 默认 Markdown 模式（向后兼容） + md = generate_markdown(stats, start_date, end_date, args.repo) + if args.output: + with open(args.output, "w", encoding="utf-8") as f: + f.write(md) + print(f"[INFO] 报表已保存到 {args.output}", file=sys.stderr) + else: + print(md) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/ci_failure_diagnosis.py b/scripts/ci/ci_failure_diagnosis.py new file mode 100644 index 000000000..5244484b2 --- /dev/null +++ b/scripts/ci/ci_failure_diagnosis.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +"""CI失败诊断增强脚本：自动分类失败原因 + 提取关键错误 + 给出修复建议。 +# Trigger CI after auto-format fix + +支持的失败类型： +1. Lint/格式问题 (ruff/black/eslint/prettier) +2. 单元测试失败 +3. Docker构建失败 +4. 依赖安装失败 (pip/npm) +5. 超时 +6. 缓存问题 +7. 数据库/迁移问题 +8. 网络问题 +9. 其他 + +用法： + python3 scripts/ci/ci_failure_diagnosis.py [--job-name "Job Name"] [--log-file /path/to/log] + +如果不传--log-file，会尝试从Gitea API获取失败job的日志。 +""" + +import json +import os +import re +import sys +import urllib.request +from dataclasses import dataclass, field +from typing import List, Optional + + +@dataclass +class FailureDiagnosis: + """失败诊断结果""" + + category: str # 失败分类 + category_cn: str # 中文分类名 + severity: str # 严重程度: high / medium / low + summary: str # 一句话摘要 + error_lines: List[str] = field(default_factory=list) # 关键错误行 + suggestions: List[str] = field(default_factory=list) # 修复建议 + auto_fixable: bool = False # 是否可以自动修复 + related_docs: str = "" # 相关文档链接 + + +# ============================================================ +# 失败模式定义 +# ============================================================ + +FAILURE_PATTERNS = [ + # ===== Lint / 格式问题 ===== + { + "pattern": r"(ruff|black|isort)\b.*(error|failed|Error)", + "category": "lint_python", + "category_cn": "Python代码质量检查", + "severity": "low", + "summary_contains": ["ruff", "black", "isort"], + "suggestions": [ + "本地运行 `black . && isort . && ruff check --fix .` 自动修复", + "使用 `scripts/agent-commit.sh` 提交（自动格式化）", + "如确认无误，可加 `# noqa: xxx` 忽略特定规则", + ], + "auto_fixable": True, + }, + { + "pattern": r"ESLint|prettier|eslint", + "category": "lint_frontend", + "category_cn": "前端代码检查", + "severity": "low", + "summary_contains": ["eslint", "prettier"], + "suggestions": [ + "本地运行 `cd apps/web && npm run lint:fix` 自动修复", + "Prettier问题: `cd apps/web && npx prettier --write .`", + ], + "auto_fixable": True, + }, + { + "pattern": r"F\d{3}|E\d{3}|W\d{3}.*ruff|ruff.*F\d{3}", + "category": "lint_python", + "category_cn": "Python代码质量检查", + "severity": "low", + "suggestions": [ + "F401: 删除未使用的import", + "F841: 删除未使用的变量或加下划线前缀", + "E501: 行超长，加 `# noqa: E501`", + "F811: 删重复import", + "运行 `ruff check --fix .` 自动修复大部分问题", + ], + "auto_fixable": True, + }, + # ===== 单元测试失败 ===== + { + "pattern": r"FAILED|assert.*Error|AssertionError", + "category": "unit_test", + "category_cn": "单元测试失败", + "severity": "high", + "suggestions": [ + "检查相关测试文件，确认是代码问题还是测试用例问题", + "本地运行对应测试：`pytest path/to/test.py -v`", + "如测试依赖外部服务，检查mock是否正确", + ], + "auto_fixable": False, + }, + { + "pattern": r"pytest.*failed|\d+ failed.*\d+ passed", + "category": "unit_test", + "category_cn": "单元测试失败", + "severity": "high", + "suggestions": [ + "查看上方日志中的FAILED测试用例", + "检查失败断言的期望值 vs 实际值", + "新代码影响了现有测试行为，确认是预期内变更吗？", + ], + "auto_fixable": False, + }, + # ===== Docker 构建失败 ===== + { + "pattern": r"Dockerfile.*not found|docker build.*failed|ERROR: failed to solve", + "category": "docker_build", + "category_cn": "Docker构建失败", + "severity": "high", + "suggestions": [ + "检查Dockerfile语法是否正确", + "检查引用的基础镜像是否存在", + "本地运行 `docker build -f path/to/Dockerfile .` 复现", + ], + "auto_fixable": False, + }, + { + "pattern": r"manifest.*not found|no such image|image.*not found", + "category": "docker_build", + "category_cn": "镜像不存在", + "severity": "medium", + "suggestions": [ + "检查基础镜像名称和tag是否正确", + "确认镜像仓库可访问，登录是否有效", + "如为新基础镜像，需先手动构建一次基础镜像", + ], + "auto_fixable": False, + }, + { + "pattern": r"ETXTBSY|text file busy", + "category": "docker_build", + "category_cn": "文件锁冲突（ETXTBSY）", + "severity": "low", + "summary": "esbuild并发构建冲突，重试即可", + "suggestions": ["偶发问题，点击Rerun重新运行即可", "如频繁出现，检查是否有多个job并发写入同一文件"], + "auto_fixable": True, + }, + # ===== 依赖安装失败 ===== + { + "pattern": r"pip install.*error|Could not find a version|No matching distribution", + "category": "dependency", + "category_cn": "pip依赖安装失败", + "severity": "medium", + "suggestions": [ + "检查requirements.txt中的版本号是否正确", + "如为新版本刚发布，可能源还没同步，稍后重试", + "检查网络连接，可尝试切换pip镜像源", + ], + "auto_fixable": False, + }, + { + "pattern": r"npm.*ERR|npm install.*failed|E404|ECONNREFUSED.*npm", + "category": "dependency", + "category_cn": "npm依赖安装失败", + "severity": "medium", + "suggestions": [ + "检查package.json中的版本号是否存在", + "网络问题：检查npm registry是否可访问", + "国内网络建议配置npmmirror镜像源", + ], + "auto_fixable": False, + }, + { + "pattern": r"Connection refused|timed out|network.*unreachable", + "category": "network", + "category_cn": "网络问题", + "severity": "medium", + "summary": "网络连接失败，可能是源站问题或DNS问题", + "suggestions": [ + "点击Rerun重试，网络问题通常是临时的", + "如持续失败，检查对应服务是否正常", + "检查Runner网络配置", + ], + "auto_fixable": True, + }, + # ===== 超时 ===== + { + "pattern": r"timeout|timed out|exceeded.*time limit|job.*cancelled.*timeout", + "category": "timeout", + "category_cn": "执行超时", + "severity": "medium", + "suggestions": [ + "如首次出现：重试一次，可能是临时性能波动", + "频繁出现：检查构建是否变慢了，最近是否加了新依赖", + "可适当增加timeout-minutes配置", + ], + "auto_fixable": False, + }, + # ===== 数据库/迁移 ===== + { + "pattern": r"alembic.*error|migration.*failed|relation.*does not exist|column.*does not exist", + "category": "migration", + "category_cn": "数据库迁移失败", + "severity": "high", + "suggestions": [ + "检查迁移脚本是否正确，down_revision是否对", + "确认数据库中是否有脏数据或残留表", + "迁移脚本合并冲突时，重新生成迁移文件", + ], + "auto_fixable": False, + }, + # ===== 缓存问题 ===== + { + "pattern": r"cache.*corrupt|cache.*invalid|snapshot.*not found|failed to compute cache key", + "category": "cache", + "category_cn": "缓存损坏", + "severity": "low", + "suggestions": ["构建系统会自动清理损坏缓存并重试，通常无需干预", "如持续失败，手动清理Runner上的缓存目录"], + "auto_fixable": True, + }, + # ===== Checkout 失败 ===== + { + "pattern": r"Could not resolve host|fatal:.*repository|SSL.*problem", + "category": "checkout", + "category_cn": "代码拉取失败", + "severity": "low", + "suggestions": ["临时网络问题，点击Rerun重试", "如持续失败，检查Gitea服务状态"], + "auto_fixable": True, + }, +] + + +def analyze_log(log_text: str, job_name: str = "") -> FailureDiagnosis: + """分析日志，返回诊断结果""" + + lines = log_text.strip().split("\n") + + # 收集所有匹配的模式 + matched = [] + error_lines = [] + + for line in lines: + line_stripped = line.strip() + # 收集ERROR/FAILED/Failed等错误行（最多20行） + if re.search(r"(ERROR|FAILED|Error|error:|FAIL:|Traceback)", line_stripped): + if len(error_lines) < 20: + error_lines.append(line_stripped) + + for pattern_info in FAILURE_PATTERNS: + if re.search(pattern_info["pattern"], line_stripped, re.IGNORECASE): + matched.append(pattern_info) + break # 一行只匹配一个模式 + + if not matched: + # 未识别的失败类型 + return FailureDiagnosis( + category="unknown", + category_cn="未知错误", + severity="medium", + summary="未识别的失败类型，需要人工查看日志", + error_lines=error_lines[:10], + suggestions=[ + "点击'查看失败日志'查看完整日志", + "如为偶发问题，可先重试一次", + "常见原因：环境问题、配置问题、新增逻辑引入的bug", + ], + auto_fixable=False, + ) + + # 选最严重、最具体的那个 + severity_order = {"high": 3, "medium": 2, "low": 1} + matched.sort(key=lambda x: severity_order.get(x["severity"], 0), reverse=True) + best_match = matched[0] + + # 生成摘要 + if "summary" in best_match: + summary = best_match["summary"] + else: + summary = f"{best_match['category_cn']}检查失败" + if job_name: + summary = f"[{job_name}] {summary}" + + # 从error_lines中过滤出与该分类相关的 + relevant_errors = error_lines[:10] + + return FailureDiagnosis( + category=best_match["category"], + category_cn=best_match["category_cn"], + severity=best_match["severity"], + summary=summary, + error_lines=relevant_errors, + suggestions=best_match["suggestions"], + auto_fixable=best_match.get("auto_fixable", False), + ) + + +def fetch_failed_job_log(run_id: str, job_id: str, token: str, repo: str) -> Optional[str]: + """从Gitea API获取失败job的日志""" + api_base = f"https://git.xiaoxiajianji.com/api/v1/repos/{repo}" + + # 尝试获取job的日志 + url = f"{api_base}/actions/runs/{run_id}/jobs/{job_id}/log" + req = urllib.request.Request(url) + req.add_header("Authorization", f"token {token}") + + try: + with urllib.request.urlopen(req, timeout=15) as resp: + return resp.read().decode("utf-8", errors="replace") + except Exception as e: + print(f"获取日志失败: {e}", file=sys.stderr) + return None + + +def format_diagnosis_markdown(d: FailureDiagnosis, job_name: str = "", run_url: str = "") -> str: + """将诊断结果格式化为飞书卡片markdown""" + + severity_emoji = {"high": "🔴", "medium": "🟡", "low": "🟢"} + emoji = severity_emoji.get(d.severity, "⚪") + + lines = [] + lines.append(f"**分类**: {emoji} {d.category_cn}") + lines.append(f"**问题**: {d.summary}") + + if d.error_lines: + lines.append("") + lines.append("**关键错误行**:") + for err in d.error_lines[:5]: + # 截断过长的行 + if len(err) > 150: + err = err[:147] + "..." + lines.append(f" `{err}`") + + lines.append("") + lines.append("**修复建议**:") + for i, s in enumerate(d.suggestions[:5], 1): + lines.append(f" {i}. {s}") + + if d.auto_fixable: + lines.append("") + lines.append("💡 **可自动修复**：如格式问题，可尝试点击Rerun让auto-fix自动处理") + + if run_url: + lines.append("") + lines.append(f"[查看完整日志]({run_url})") + + return "\n".join(lines) + + +def main(): + job_name = os.environ.get("FAILED_JOB", "") + run_id = os.environ.get("GITHUB_RUN_ID", "") + repo = os.environ.get("GITHUB_REPOSITORY", "xiaoxia/xiaoxia-saas") + token = os.environ.get("GITHUB_TOKEN", "") + + # 1. 尝试获取日志 + log_text = "" + + # 优先从环境变量或文件读取 + log_file = os.environ.get("CI_LOG_FILE", "") + if log_file and os.path.exists(log_file): + with open(log_file) as f: + log_text = f.read() + elif run_id and token: + # 尝试从API获取（需要job_id，这里简化处理） + pass + + # 如果没有日志，用job_name做粗略分类 + if not log_text: + # 基于job名做初始判断 + if any(k in job_name.lower() for k in ["validate", "lint", "quality"]): + d = FailureDiagnosis( + category="lint_general", + category_cn="代码质量检查", + severity="low", + summary=f"{job_name} 检查失败（日志不可用，基于job名初步诊断）", + suggestions=["点击查看日志获取具体错误信息", "格式类问题通常可自动修复"], + auto_fixable=True, + ) + elif "build" in job_name.lower(): + d = FailureDiagnosis( + category="build_general", + category_cn="构建失败", + severity="high", + summary=f"{job_name} 构建失败（日志不可用）", + suggestions=["点击查看日志获取具体构建错误", "常见原因：Dockerfile错误、依赖安装失败、网络问题"], + auto_fixable=False, + ) + elif "test" in job_name.lower(): + d = FailureDiagnosis( + category="test_general", + category_cn="测试失败", + severity="high", + summary=f"{job_name} 测试失败（日志不可用）", + suggestions=["点击查看日志获取具体失败的测试用例", "检查最近代码改动是否影响了测试"], + auto_fixable=False, + ) + elif "deploy" in job_name.lower(): + d = FailureDiagnosis( + category="deploy_general", + category_cn="部署失败", + severity="high", + summary=f"{job_name} 部署失败（日志不可用）", + suggestions=["检查目标服务器状态和网络", "检查镜像是否正确推送", "查看服务器上的容器日志"], + auto_fixable=False, + ) + else: + d = FailureDiagnosis( + category="unknown", + category_cn="未知错误", + severity="medium", + summary=f"{job_name} 失败", + suggestions=["点击查看日志获取详细信息"], + auto_fixable=False, + ) + else: + d = analyze_log(log_text, job_name) + + # 输出诊断结果 + run_url = f"https://git.xiaoxiajianji.com/{repo}/actions/runs/{run_id}" if run_id else "" + + print("=" * 60) + print(" CI 失败诊断报告") + print("=" * 60) + print() + print(format_diagnosis_markdown(d, job_name, run_url)) + print() + print("=" * 60) + + # 将诊断结果写入文件（供通知脚本读取） + output_file = os.environ.get("DIAGNOSIS_OUTPUT", "/tmp/ci_diagnosis.json") + result = { + "category": d.category, + "category_cn": d.category_cn, + "severity": d.severity, + "summary": d.summary, + "error_lines": d.error_lines, + "suggestions": d.suggestions, + "auto_fixable": d.auto_fixable, + } + with open(output_file, "w") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + print(f"\n诊断结果已保存到: {output_file}") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/ci_health_check.py b/scripts/ci/ci_health_check.py new file mode 100644 index 000000000..cc7523e60 --- /dev/null +++ b/scripts/ci/ci_health_check.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +""" +CI 健康度快速检查脚本 +- 统计最近 N 条 run 的成功率（按 workflow 分类） +- 列出失败的 run 和失败的 job/step +- 区分基础设施问题 vs 业务代码问题 +- 输出简洁的健康度报告 + +用法: + python3 scripts/ci/ci_health_check.py [--limit 20] [--workflow ci-pipeline.yml] [--json] + +环境变量: + GITEA_TOKEN API token（必需） + GITEA_API_URL Gitea API 地址，默认 https://git.xiaoxiajianji.com/api/v1 + GITEA_REPO 仓库，默认 xiaoxia/xiaoxia-saas +""" + +import argparse +import json +import os +import sys +import urllib.request +from datetime import datetime, timedelta, timezone + +# ---- 基础设施问题关键词（命中即判定为基础设施问题）---- +INFRA_KEYWORDS = [ + # 网络/连接 + "Couldn't connect to server", + "Connection refused", + "Connection reset", + "Connection timed out", + "Failed to connect to", + "network is unreachable", + "TLS handshake timeout", + "SSL certificate problem", + # 容器/Runner + "No such container", + "container already exists", + "docker: not found", + "no space left on device", + "out of memory", + "OOMKilled", + "pull access denied", + "manifest unknown", + "Error response from daemon", + "runner", + "runner is not online", + "no matching runners", + # Checkout/Git + "Could not resolve host", + "fatal: unable to access", + "The remote end hung up unexpectedly", + "early EOF", + "index-pack failed", + "git fetch", + "checkout failed", + "ETXTBSY", + "text file busy", + # 镜像/环境 + "No module named pip", + "pip: not found", + "command not found: python", + "python3: not found", + "node: not found", + "npm: not found", + "exec format error", + "standard_init_linux.go", + # 系统/资源 + "Input/output error", + "device or resource busy", + "No space left on device", + "Disk full", + # 鉴权/配置 + "401 Unauthorized", + "403 Forbidden", + "404 Not Found", + "identity_sign: private key", + "Permission denied", +] + + +def api_get(path: str) -> dict: + base = os.environ.get("GITEA_API_URL", "https://git.xiaoxiajianji.com/api/v1") + repo = os.environ.get("GITEA_REPO", "xiaoxia/xiaoxia-saas") + token = os.environ.get("GITEA_TOKEN", "") + url = f"{base}/repos/{repo}/{path}" + req = urllib.request.Request(url, headers={"Authorization": f"token {token}"}) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode()) + + +def get_run_jobs(run_id: int) -> list: + return api_get(f"actions/runs/{run_id}/jobs").get("jobs", []) + + +def get_job_log(job_id: int) -> str: + try: + return api_get(f"actions/jobs/{job_id}/logs") + except Exception: + return "" + + +def classify_failure(job: dict) -> str: + """判断失败原因类型: infra / business / unknown""" + name = job.get("name", "") + # 仅根据 job 名称做初步分类（更精确需读日志，但代价高） + infra_jobs = ["Checkout", "Build", "Deploy", "Cleanup"] + business_jobs = [ + "Unit Tests", + "Integration Tests", + "Frontend Lint", + "Frontend Unit Tests", + "Staging E2E", + "E2E", + "Validate Code Quality", + ] + name_lower = name.lower() + if ( + any(k.lower() in name_lower for k in infra_jobs) + and "Test" not in name + and "Lint" not in name + and "Validate" not in name + ): + return "infra" + if any(k.lower() in name_lower for k in business_jobs): + return "business" + return "unknown" + + +def analyze_with_log(job_id: int) -> str: + """通过日志关键词精确分类""" + log = get_job_log(job_id) + log_lower = log.lower() + for kw in INFRA_KEYWORDS: + if kw.lower() in log_lower: + return "infra" + return "business" + + +def fmt_time(t: str) -> str: + if not t or t.startswith("1970"): + return "-" + try: + dt = datetime.fromisoformat(t.replace("Z", "+00:00")) + bj = dt.astimezone(timezone(timedelta(hours=8))) + return bj.strftime("%m-%d %H:%M") + except Exception: + return t[:16] + + +def main(): + parser = argparse.ArgumentParser(description="CI 健康度快速检查") + parser.add_argument("--limit", type=int, default=20, help="最近多少条 run") + parser.add_argument("--workflow", type=str, default="", help="只看某个 workflow") + parser.add_argument("--json", action="store_true", help="JSON 输出") + parser.add_argument("--deep", action="store_true", help="深度检查（读日志，较慢）") + args = parser.parse_args() + + if not os.environ.get("GITEA_TOKEN"): + print("错误: 请设置 GITEA_TOKEN 环境变量", file=sys.stderr) + sys.exit(1) + + # 1. 获取最近 run + runs = api_get(f"actions/runs?limit={args.limit}").get("workflow_runs", []) + if args.workflow: + runs = [r for r in runs if args.workflow in r.get("path", "")] + + if not runs: + print("没有找到匹配的 run") + return + + # 按 workflow 分组统计 + wf_stats = {} + failed_runs = [] + + for r in runs: + path = r.get("path", "unknown") + # 提取 workflow 文件名，兼容各种 path 格式 + if ".yml" in path or ".yaml" in path: + # ci-pipeline.yml@refs/heads/develop -> ci-pipeline.yml + wf = path.split("@")[0].split("/")[-1] + else: + wf = path.split("/")[-1] if "/" in path else path + if wf not in wf_stats: + wf_stats[wf] = {"total": 0, "success": 0, "failure": 0, "cancelled": 0, "others": 0} + wf_stats[wf]["total"] += 1 + status = r.get("status", "") + conc = r.get("conclusion", "") + if status != "completed": + wf_stats[wf]["others"] += 1 + continue + if conc == "success": + wf_stats[wf]["success"] += 1 + elif conc == "failure": + wf_stats[wf]["failure"] += 1 + failed_runs.append(r) + elif conc == "cancelled": + wf_stats[wf]["cancelled"] += 1 + else: + wf_stats[wf]["others"] += 1 + + # 2. 失败 run 详情 + failed_details = [] + for r in failed_runs[:10]: # 最多看10个失败的 + jobs = get_run_jobs(r["id"]) + failed_jobs = [j for j in jobs if j.get("conclusion") == "failure"] + job_infos = [] + for j in failed_jobs: + cat = classify_failure(j) + if args.deep and cat == "unknown": + cat = analyze_with_log(j["id"]) + # 找失败的 step + failed_steps = [] + for step in j.get("steps", []): + if step.get("conclusion") == "failure": + failed_steps.append(step.get("name", "?")) + job_infos.append( + { + "name": j.get("name", ""), + "category": cat, + "failed_steps": failed_steps, + "runner": j.get("runner_name", ""), + } + ) + failed_details.append( + { + "id": r["id"], + "title": r.get("display_title", ""), + "branch": r.get("head_branch", ""), + "time": fmt_time(r.get("updated_at", "")), + "jobs": job_infos, + } + ) + + # 3. 输出 + if args.json: + result = {"workflows": wf_stats, "failed_runs": failed_details} + print(json.dumps(result, ensure_ascii=False, indent=2)) + return + + # 文本报告 + print("=" * 60) + print(" CI 健康度报告") + print("=" * 60) + print(f"统计范围: 最近 {len(runs)} 条 run") + print(f"时间: {datetime.now(timezone(timedelta(hours=8))).strftime('%Y-%m-%d %H:%M:%S')}") + print() + + print("📊 各 Workflow 成功率:") + print("-" * 60) + for wf, s in sorted(wf_stats.items()): + total = s["total"] + succ = s["success"] + rate = (succ / total * 100) if total > 0 else 0 + bar = "█" * int(rate / 5) + "░" * (20 - int(rate / 5)) + icon = "🟢" if rate >= 90 else ("🟡" if rate >= 70 else "🔴") + print(f" {icon} {wf:35s} {rate:5.1f}% {bar} ({succ}/{total})") + if s["failure"]: + print(f" 失败: {s['failure']} 取消: {s['cancelled']} 进行中: {s['others']}") + + if failed_details: + print() + print("❌ 失败详情:") + print("-" * 60) + for d in failed_details: + print(f" #{d['id']} [{d['time']}] {d['title'][:45]}") + print(f" 分支: {d['branch']}") + for j in d["jobs"]: + cat_icon = "🏗️" if j["category"] == "infra" else ("🐛" if j["category"] == "business" else "❓") + steps = ", ".join(j["failed_steps"][:3]) if j["failed_steps"] else "未知" + print(f" {cat_icon} {j['name'][:30]:30s} 失败步骤: {steps}") + if j["runner"]: + print(f" runner: {j['runner']}") + else: + print() + print("✅ 最近没有失败的 run") + + # 总结 + total_all = sum(s["total"] for s in wf_stats.values()) + succ_all = sum(s["success"] for s in wf_stats.values()) + fail_all = sum(s["failure"] for s in wf_stats.values()) + infra_fail = sum(1 for d in failed_details for j in d["jobs"] if j["category"] == "infra") + biz_fail = sum(1 for d in failed_details for j in d["jobs"] if j["category"] == "business") + rate_all = (succ_all / total_all * 100) if total_all > 0 else 0 + print() + print("=" * 60) + print(f" 总结: 总成功率 {rate_all:.1f}% ({succ_all}/{total_all})") + if fail_all > 0: + print(f" 失败job分类: 基础设施 {infra_fail} 个 | 业务代码 {biz_fail} 个") + if infra_fail > biz_fail: + print(" ⚠️ 主要是基础设施问题，建议优先排查 CI 环境") + else: + print(" 💡 主要是业务代码问题，建议关注业务侧修复") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/ci_health_report.py b/scripts/ci/ci_health_report.py new file mode 100644 index 000000000..12b52bfb4 --- /dev/null +++ b/scripts/ci/ci_health_report.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +""" +CI健康度每日巡检报告脚本 +- 调用ci_health_check.py获取数据 +- 有失败时生成飞书卡片通知并发送 +- 无失败时静默退出（不打扰） +- 用于每日定时巡检 + +用法: + python3 scripts/ci/ci_health_report.py [--limit 30] [--dry-run] + +环境变量: + GITEA_TOKEN API token（必需） + CI_NOTIFY_WEBHOOK 飞书webhook地址（必需，用于发报告） + GITEA_API_URL Gitea API 地址 + GITEA_REPO 仓库 +""" + +import argparse +import json +import os +import subprocess +import sys +import urllib.request +from datetime import datetime, timedelta, timezone + + +def run_health_check(limit: int) -> dict: + """调用ci_health_check.py获取JSON结果""" + script_dir = os.path.dirname(os.path.abspath(__file__)) + cmd = [ + sys.executable, + os.path.join(script_dir, "ci_health_check.py"), + "--json", + "--limit", + str(limit), + ] + env = os.environ.copy() + # 确保GITEA_TOKEN传递 + if not env.get("GITEA_TOKEN") and env.get("GITHUB_TOKEN"): + env["GITEA_TOKEN"] = env["GITHUB_TOKEN"] + + result = subprocess.run(cmd, capture_output=True, text=True, env=env) + if result.returncode != 0: + print(f"health check failed: {result.stderr}") + return {"workflows": {}, "failed_runs": []} + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + print(f"failed to parse health check output: {result.stdout[:200]}") + return {"workflows": {}, "failed_runs": []} + + +def build_feishu_card(data: dict) -> dict: + """构建飞书卡片消息""" + wf_stats = data.get("workflows", {}) + failed_runs = data.get("failed_runs", []) + + # 统计数据 + total_all = sum(s["total"] for s in wf_stats.values()) + succ_all = sum(s["success"] for s in wf_stats.values()) + fail_all = sum(s["failure"] for s in wf_stats.values()) + rate_all = (succ_all / total_all * 100) if total_all > 0 else 0 + + # 失败分类 + infra_fail = 0 + biz_fail = 0 + unknown_fail = 0 + for run in failed_runs: + for job in run.get("jobs", []): + cat = job.get("category", "unknown") + if cat == "infra": + infra_fail += 1 + elif cat == "business": + biz_fail += 1 + else: + unknown_fail += 1 + + now = datetime.now(timezone(timedelta(hours=8))).strftime("%Y-%m-%d %H:%M") + + # 各workflow成功率行 + wf_lines = [] + for wf, s in sorted(wf_stats.items()): + total = s["total"] + succ = s["success"] + fail = s["failure"] + rate = (succ / total * 100) if total > 0 else 0 + icon = "🟢" if rate >= 90 else ("🟡" if rate >= 70 else "🔴") + wf_name = ( + wf.replace("ci-pipeline.yml", "CI Pipeline") + .replace("code-review.yml", "Code Review") + .replace("daily-check.yml", "Daily Check") + .replace("preview-deploy.yml", "Preview Deploy") + ) + wf_lines.append(f"{icon} **{wf_name}**: {rate:.0f}% ({succ}/{total}，失败{fail})") + + # 失败详情（最多显示5条） + fail_detail_lines = [] + for i, run in enumerate(failed_runs[:5]): + run_id = run["id"] + title = run.get("title", "")[:35] + branch = run.get("branch", "") + jobs_str = ", ".join(j["name"][:15] for j in run.get("jobs", [])[:3]) + fail_detail_lines.append(f"• **#{run_id}** {title}\n 分支: {branch} | 失败: {jobs_str}") + + if len(failed_runs) > 5: + fail_detail_lines.append(f"... 还有 {len(failed_runs) - 5} 条失败记录") + + # 整体状态 + if fail_all == 0: + status_text = "✅ 全部通过" + status_color = "green" + elif infra_fail > biz_fail: + status_text = "⚠️ 基础设施问题为主" + status_color = "yellow" + else: + status_text = "🔴 存在业务失败" + status_color = "red" + + card = { + "config": {"wide_screen_mode": True}, + "header": { + "title": {"tag": "plain_text", "content": f"CI告警 - 每日健康度巡检 ({now})"}, + "template": status_color, + }, + "elements": [ + { + "tag": "div", + "text": { + "tag": "lark_md", + "content": f"**统计范围**: 最近 {total_all} 条 run\n**整体状态**: {status_text}\n**总成功率**: {rate_all:.1f}% ({succ_all}/{total_all})", + }, + }, + {"tag": "hr"}, + { + "tag": "div", + "text": { + "tag": "lark_md", + "content": "**📊 各Workflow成功率**\n" + "\n".join(wf_lines) if wf_lines else "暂无数据", + }, + }, + ], + } + + # 失败分类统计 + if fail_all > 0: + card["elements"].append({"tag": "hr"}) + card["elements"].append( + { + "tag": "div", + "text": { + "tag": "lark_md", + "content": f"**失败原因分类**\n🏗️ 基础设施: {infra_fail} 个\n🐛 业务代码: {biz_fail} 个\n❓ 待确认: {unknown_fail} 个", + }, + } + ) + + # 失败详情 + if fail_detail_lines: + card["elements"].append({"tag": "hr"}) + card["elements"].append( + { + "tag": "div", + "text": { + "tag": "lark_md", + "content": "**❌ 失败详情**\n" + "\n\n".join(fail_detail_lines), + }, + } + ) + + # 查看更多 + card["elements"].append({"tag": "hr"}) + base_url = os.environ.get("GITEA_BASE_URL", "https://git.xiaoxiajianji.com") + repo = os.environ.get("GITEA_REPO", "xiaoxia/xiaoxia-saas") + card["elements"].append( + { + "tag": "action", + "actions": [ + { + "tag": "button", + "text": {"tag": "plain_text", "content": "查看CI面板"}, + "type": "primary", + "url": f"{base_url}/{repo}/actions", + } + ], + } + ) + + return {"msg_type": "interactive", "card": card} + + +def send_feishu(webhook: str, payload: dict) -> bool: + """发送飞书webhook""" + data = json.dumps(payload).encode() + req = urllib.request.Request( + webhook, + data=data, + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + result = json.loads(resp.read().decode()) + return result.get("code", -1) == 0 or result.get("StatusCode", -1) == 0 + except Exception as e: + print(f"send feishu failed: {e}") + return False + + +def main(): + parser = argparse.ArgumentParser(description="CI健康度每日巡检报告") + parser.add_argument("--limit", type=int, default=30, help="统计最近N条run") + parser.add_argument("--dry-run", action="store_true", help="只打印不发送") + parser.add_argument("--always-notify", action="store_true", help="即使全部通过也发送通知") + args = parser.parse_args() + + webhook = os.environ.get("CI_NOTIFY_WEBHOOK", "") + if not webhook and not args.dry_run: + print("未配置 CI_NOTIFY_WEBHOOK，跳过通知") + # 还是执行健康检查输出到日志，方便排查 + data = run_health_check(args.limit) + print(f"health check done: {len(data.get('failed_runs', []))} failed") + return 0 + + # 执行健康检查 + data = run_health_check(args.limit) + failed_count = len(data.get("failed_runs", [])) + + # 无失败且不强制通知 → 静默退出 + if failed_count == 0 and not args.always_notify: + print("✅ 全部通过，静默退出") + return 0 + + # 构建并发送卡片 + card = build_feishu_card(data) + + if args.dry_run: + print(json.dumps(card, ensure_ascii=False, indent=2)) + return 0 + + success = send_feishu(webhook, card) + if success: + print(f"📤 已发送健康度报告，失败 {failed_count} 条") + else: + print("❌ 发送飞书通知失败") + + # 通知失败不阻断流程 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/ci_repeated_failure_detector.py b/scripts/ci/ci_repeated_failure_detector.py new file mode 100644 index 000000000..5a4c8b556 --- /dev/null +++ b/scripts/ci/ci_repeated_failure_detector.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +""" +CI重复失败检测脚本 +- 扫描最近N天的CI失败 +- 按job名称分组统计失败率 +- 识别高失败率job（系统性故障） +- 飞书通知告警 +""" + +import json +import os +import sys +import time +import urllib.error +import urllib.request +from collections import defaultdict +from datetime import datetime, timedelta, timezone + + +def get_env(name, default=None, required=False): + val = os.environ.get(name, default) + if required and not val: + print(f"❌ 缺少环境变量: {name}") + sys.exit(1) + return val + + +GITEA_URL = get_env("GITEA_URL", "https://git.xiaoxiajianji.com") +GITEA_TOKEN = get_env("GITEA_API_TOKEN", required=False) or get_env("GITHUB_TOKEN", "") +REPO = get_env("GITEA_REPO", "xiaoxia/xiaoxia-saas") +DAYS = int(get_env("FAIL_CHECK_DAYS", "7")) +FAIL_THRESHOLD = int(get_env("FAIL_THRESHOLD", 3)) # 失败次数阈值 +FAIL_RATE_THRESHOLD = float(get_env("FAIL_RATE_THRESHOLD", "30")) # 失败率阈值% +CONSECUTIVE_FAIL_THRESHOLD = int(get_env("CONSECUTIVE_FAIL_THRESHOLD", "3")) # 连续失败阈值 +WEBHOOK = get_env("CI_NOTIFY_WEBHOOK", "") + + +def api_get(path): + """调用Gitea API""" + url = f"{GITEA_URL}/api/v1{path}" + req = urllib.request.Request(url) + if GITEA_TOKEN: + req.add_header("Authorization", f"token {GITEA_TOKEN}") + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read()) + except urllib.error.HTTPError as e: + print(f" HTTP {e.code}: {path}") + return None + except Exception as e: + print(f" 错误: {e}") + return None + + +def fetch_recent_runs(days=7, per_page=50, max_pages=10): + """获取最近N天的runs""" + since = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() + all_runs = [] + + for page in range(1, max_pages + 1): + path = f"/repos/{REPO}/actions/runs?page={page}&limit={per_page}" + data = api_get(path) + if not data: + break + + runs = data.get("workflow_runs", data.get("runs", [])) + if not runs: + break + + # 检查时间范围（Gitea用started_at，格式2026-07-22T10:58:10+08:00） + oldest = None + for r in runs: + started = r.get("started_at", r.get("created_at", "")) + if started and started >= since: + all_runs.append(r) + else: + oldest = started + + if oldest and oldest < since: + break + + if len(runs) < per_page: + break + + return all_runs + + +def fetch_run_jobs(run_id): + """获取run的所有jobs""" + path = f"/repos/{REPO}/actions/runs/{run_id}/jobs" + data = api_get(path) + if not data: + return [] + return data.get("jobs", []) + + +def analyze_failures(runs): + """ + 分析失败情况 + + 返回: + - job_stats: {job_name: {total, success, failure, skipped, failure_rate, failures: [...]}} + - consecutive_failures: {job_name: current_streak, max_streak, last_status} + """ + job_stats = defaultdict( + lambda: { + "total": 0, + "success": 0, + "failure": 0, + "error": 0, + "skipped": 0, + "cancelled": 0, + "failures": [], + } + ) + + # 按时间正序排列（旧→新）用于连续失败计算 + sorted_runs = sorted(runs, key=lambda r: r.get("started_at", r.get("created_at", ""))) + + # 连续失败跟踪 {job_name: streak} + consecutive = defaultdict(lambda: {"current": 0, "max": 0, "last_run": None}) + + for run in sorted_runs: + run_id = run.get("id") + run_status = run.get("status", "") + run_conclusion = run.get("conclusion", "") + run_started = run.get("started_at", run.get("created_at", "")) + event = run.get("event", "") + + # 只统计pull_request和push事件的CI + if event not in ("pull_request", "push"): + continue + + jobs = fetch_run_jobs(run_id) + + for job in jobs: + name = job.get("name", "") + status = job.get("status", "") + conclusion = job.get("conclusion", "") + + # 跳过非CI核心job（如AI Code Review、Preview等） + skip_prefixes = ("AI Code Review", "Preview", "PR Automation", "Auto") + if any(name.startswith(p) for p in skip_prefixes): + continue + + stats = job_stats[name] + stats["total"] += 1 + + if conclusion == "success": + stats["success"] += 1 + consecutive[name]["current"] = 0 + elif conclusion == "failure": + stats["failure"] += 1 + stats["failures"].append( + { + "run_id": run_id, + "time": run_started, + "event": event, + } + ) + consecutive[name]["current"] += 1 + if consecutive[name]["current"] > consecutive[name]["max"]: + consecutive[name]["max"] = consecutive[name]["current"] + consecutive[name]["last_run"] = run_id + elif conclusion == "error": + stats["error"] += 1 + # error也算失败的一种 + consecutive[name]["current"] += 1 + if consecutive[name]["current"] > consecutive[name]["max"]: + consecutive[name]["max"] = consecutive[name]["current"] + elif conclusion == "skipped": + stats["skipped"] += 1 + # skipped不算也不打断连续失败 + elif conclusion == "cancelled": + stats["cancelled"] += 1 + # cancelled不算失败也不打断 + + # 计算失败率 + for name, stats in job_stats.items(): + total_actual = stats["total"] - stats["skipped"] - stats["cancelled"] + if total_actual > 0: + stats["failure_rate"] = round((stats["failure"] + stats["error"]) / total_actual * 100, 1) + else: + stats["failure_rate"] = 0.0 + + return dict(job_stats), dict(consecutive) + + +def find_high_failures(job_stats, consecutive): + """ + 找出高风险job + + 告警级别： + - critical: 连续失败 >= CONSECUTIVE_FAIL_THRESHOLD，或 失败率>=50%且失败次数>=5 + - warning: 失败率>=FAIL_RATE_THRESHOLD且失败次数>=FAIL_THRESHOLD + - info: 失败次数>=2 + """ + critical = [] + warning = [] + info = [] + + for name, stats in job_stats.items(): + fail_count = stats["failure"] + stats["error"] + rate = stats["failure_rate"] + streak = consecutive.get(name, {}).get("current", 0) + max_streak = consecutive.get(name, {}).get("max", 0) + + issue = { + "name": name, + "fail_count": fail_count, + "total": stats["total"], + "failure_rate": rate, + "current_streak": streak, + "max_streak": max_streak, + "recent_failures": stats["failures"][-5:], # 最近5次 + } + + if streak >= CONSECUTIVE_FAIL_THRESHOLD or (rate >= 50 and fail_count >= 5): + critical.append(issue) + elif rate >= FAIL_RATE_THRESHOLD and fail_count >= FAIL_THRESHOLD: + warning.append(issue) + elif fail_count >= 2: + info.append(issue) + + # 按失败次数倒序 + critical.sort(key=lambda x: x["fail_count"], reverse=True) + warning.sort(key=lambda x: x["fail_count"], reverse=True) + info.sort(key=lambda x: x["fail_count"], reverse=True) + + return critical, warning, info + + +def generate_report(critical, warning, info, days, total_runs): + """生成Markdown报告""" + lines = [] + lines.append("# CI重复失败检测报告") + lines.append("") + lines.append(f"**统计周期**: 最近{days}天") + lines.append(f"**扫描Runs**: {total_runs}个") + lines.append(f"**生成时间**: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}") + lines.append("") + + lines.append(f"## 概览") + lines.append("") + lines.append(f"| 级别 | 数量 |") + lines.append(f"|------|------|") + lines.append(f"| 🔴 严重 (连续失败≥{CONSECUTIVE_FAIL_THRESHOLD}次 或 失败率≥50%) | {len(critical)} |") + lines.append(f"| 🟡 警告 (失败率≥{FAIL_RATE_THRESHOLD}% 且 失败≥{FAIL_THRESHOLD}次) | {len(warning)} |") + lines.append(f"| 🔵 关注 (失败≥2次) | {len(info)} |") + lines.append("") + + if critical: + lines.append("## 🔴 严重问题") + lines.append("") + for item in critical: + lines.append(f"### {item['name']}") + lines.append("") + lines.append(f"- 失败次数: **{item['fail_count']}** / {item['total']} 次运行") + lines.append(f"- 失败率: **{item['failure_rate']}%**") + lines.append(f"- 当前连续失败: **{item['current_streak']}** 次 (历史最高: {item['max_streak']} 次)") + lines.append("") + if item["recent_failures"]: + lines.append("最近失败:") + lines.append("") + for f in item["recent_failures"]: + lines.append(f"- [{f['time'][:16]}] run #{f['run_id']} ({f['event']})") + lines.append("") + + if warning: + lines.append("## 🟡 警告") + lines.append("") + for item in warning: + lines.append( + f"- **{item['name']}**: {item['fail_count']}次失败 / {item['total']}次运行 ({item['failure_rate']}%)" + ) + lines.append("") + + if info: + lines.append("## 🔵 关注列表") + lines.append("") + lines.append("| Job名称 | 失败次数 | 总次数 | 失败率 | 当前连续 |") + lines.append("|---------|----------|--------|--------|----------|") + for item in info[:20]: # 最多显示20个 + lines.append( + f"| {item['name']} | {item['fail_count']} | {item['total']} | {item['failure_rate']}% | {item['current_streak']} |" + ) + lines.append("") + + return "\n".join(lines) + + +def send_feishu_notification(critical, warning, info, days): + """发送飞书通知""" + if not WEBHOOK: + print(" ⚠️ 未配置WEBHOOK，跳过飞书通知") + return False + + total_issues = len(critical) + len(warning) + len(info) + if total_issues == 0: + print(" ✅ 无异常，不发送通知") + return True + + level = "🔴 严重告警" if critical else "🟡 警告" if warning else "🔵 关注" + + title = f"CI重复失败检测 - {level}" + text = f"统计周期: 最近{days}天\n\n" + + if critical: + text += "【严重问题】\n" + for item in critical[:5]: + text += f"• {item['name']}\n" + text += f" 失败 {item['fail_count']}/{item['total']} ({item['failure_rate']}%) 连续{item['current_streak']}次\n" + if len(critical) > 5: + text += f" ...还有{len(critical)-5}个\n" + text += "\n" + + if warning: + text += "【警告】\n" + for item in warning[:5]: + text += f"• {item['name']}: {item['fail_count']}次失败 ({item['failure_rate']}%)\n" + if len(warning) > 5: + text += f" ...还有{len(warning)-5}个\n" + text += "\n" + + if info and not critical and not warning: + text += "【关注列表】\n" + for item in info[:10]: + text += f"• {item['name']}: {item['fail_count']}次失败\n" + text += "\n" + + text += f"共发现 {total_issues} 个异常job" + + payload = {"msg_type": "text", "content": {"text": f"{title}\n\n{text}"}} + + data = json.dumps(payload).encode() + req = urllib.request.Request(WEBHOOK, data=data, headers={"Content-Type": "application/json"}) + + try: + with urllib.request.urlopen(req, timeout=10) as resp: + result = json.loads(resp.read()) + if result.get("code") == 0 or result.get("StatusCode") == 0: + print(" ✅ 飞书通知已发送") + return True + else: + print(f" ⚠️ 飞书返回: {result}") + return False + except Exception as e: + print(f" ❌ 飞书通知失败: {e}") + return False + + +def main(): + print(f"=== CI重复失败检测 ===") + print(f"统计周期: 最近{DAYS}天") + print(f"仓库: {REPO}") + print() + + print("1. 获取最近的Runs...") + runs = fetch_recent_runs(days=DAYS) + print(f" 找到 {len(runs)} 个runs") + + if not runs: + print("⚠️ 没有找到runs，退出") + return + + print() + print("2. 分析job失败情况（可能需要点时间）...") + job_stats, consecutive = analyze_failures(runs) + print(f" 共统计 {len(job_stats)} 个job") + + print() + print("3. 识别高风险job...") + critical, warning, info = find_high_failures(job_stats, consecutive) + print(f" 🔴 严重: {len(critical)}") + print(f" 🟡 警告: {len(warning)}") + print(f" 🔵 关注: {len(info)}") + + print() + print("4. 生成报告...") + report = generate_report(critical, warning, info, DAYS, len(runs)) + + # 保存报告 + report_path = os.environ.get("REPORT_PATH", f"/tmp/ci_failure_report_{int(time.time())}.md") + with open(report_path, "w") as f: + f.write(report) + print(f" 报告已保存: {report_path}") + + # 打印摘要 + print() + print("=== 摘要 ===") + if critical: + print("🔴 严重问题:") + for item in critical[:5]: + print( + f" {item['name']}: {item['fail_count']}次失败, {item['failure_rate']}%, 连续{item['current_streak']}次" + ) + if warning: + print("🟡 警告:") + for item in warning[:5]: + print(f" {item['name']}: {item['fail_count']}次失败, {item['failure_rate']}%") + + print() + print("5. 发送飞书通知...") + send_feishu_notification(critical, warning, info, DAYS) + + print() + print("✅ 检测完成") + + # 有严重问题时退出码非零，方便workflow标记 + if critical: + sys.exit(2) + elif warning: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/ci_trace_report.py b/scripts/ci/ci_trace_report.py new file mode 100755 index 000000000..eb53f1422 --- /dev/null +++ b/scripts/ci/ci_trace_report.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +""" +CI Trace Report Script - Reports CI Trace data to AgentLoop from Gitea Actions workflows. + +Usage in CI workflow jobs: + - At start: python3 scripts/ci/ci_trace_report.py --status running + - At end: python3 scripts/ci/ci_trace_report.py --status ok --start-time $CI_TRACE_START_TIME + +Environment variables (built-in Gitea Actions): + GITEA_REPOSITORY / GITHUB_REPOSITORY - repository (owner/repo) + GITEA_WORKFLOW / GITHUB_WORKFLOW - workflow name + GITEA_JOB / GITHUB_JOB - job ID + GITEA_SHA / GITHUB_SHA - commit SHA + GITEA_REF_NAME / GITHUB_REF_NAME - branch name + GITEA_RUN_ID / GITHUB_RUN_ID - run ID + GITEA_ACTOR / GITHUB_ACTOR - trigger actor + GITEA_EVENT_NAME / GITHUB_EVENT_NAME - event type + PR_NUMBER / GITEA_PR_NUMBER - PR number (if PR triggered) + +AgentLoop configuration (injected via Secrets): + AGENTLOOP_LICENSE_KEY - LicenseKey (required) + AGENTLOOP_ENDPOINT - Trace endpoint (optional, has default) + AGENTLOOP_PROJECT - SLS Project name (optional) + AGENTLOOP_WORKSPACE - CMS Workspace name (optional) +""" + +import argparse +import json +import os +import sys +import time +import urllib.error +import urllib.request +import uuid + +# ========== Default Configuration ========== +DEFAULT_ENDPOINT = "https://proj-xtrace-495e81719a1fd9a2c5fd671eefafbe-cn-hangzhou.cn-hangzhou.log.aliyuncs.com/apm/trace/opentelemetry/v1/traces" +DEFAULT_PROJECT = "proj-xtrace-495e81719a1fd9a2c5fd671eefafbe-cn-hangzhou" +DEFAULT_WORKSPACE = "agentloop-13b8d6efb7fde6e9b193eb982ade68e2" + + +# ========== OTLP Protobuf Manual Encoding ========== + + +def _encode_varint(value): + result = bytearray() + while value > 0x7F: + result.append((value & 0x7F) | 0x80) + value >>= 7 + result.append(value & 0x7F) + return bytes(result) + + +def _encode_tag(field_number, wire_type): + return _encode_varint((field_number << 3) | wire_type) + + +def _encode_string_field(field_number, value): + value_bytes = value.encode("utf-8") + return _encode_tag(field_number, 2) + _encode_varint(len(value_bytes)) + value_bytes + + +def _encode_bytes_field(field_number, value_bytes): + return _encode_tag(field_number, 2) + _encode_varint(len(value_bytes)) + value_bytes + + +def _encode_int_field(field_number, value): + return _encode_tag(field_number, 0) + _encode_varint(value & 0xFFFFFFFFFFFFFFFF) + + +def _encode_message_field(field_number, message_bytes): + return _encode_tag(field_number, 2) + _encode_varint(len(message_bytes)) + message_bytes + + +def _encode_key_value(key, value_str): + any_value = _encode_string_field(1, value_str) + return _encode_string_field(1, key) + _encode_message_field(2, any_value) + + +def _encode_status(status_code, status_msg=""): + data = _encode_int_field(1, status_code) + if status_msg: + data += _encode_string_field(2, status_msg) + return data + + +def _encode_span( + trace_id_bytes, + span_id_bytes, + parent_span_id_bytes, + name, + start_time_unix_nano, + end_time_unix_nano, + span_kind, + attributes, + status_code, + status_msg="", +): + data = b"" + data += _encode_bytes_field(1, trace_id_bytes) + data += _encode_bytes_field(2, span_id_bytes) + if parent_span_id_bytes: + data += _encode_bytes_field(3, parent_span_id_bytes) + data += _encode_string_field(4, name) + data += _encode_int_field(5, span_kind) + data += _encode_int_field(6, start_time_unix_nano) + data += _encode_int_field(7, end_time_unix_nano) + for key, value in attributes.items(): + kv = _encode_key_value(key, str(value)) + data += _encode_message_field(9, kv) + status = _encode_status(status_code, status_msg) + data += _encode_message_field(12, status) + return data + + +def _encode_resource_spans(service_name, scope_spans_bytes): + svc_kv = _encode_key_value("service.name", service_name) + resource = _encode_message_field(1, svc_kv) + data = _encode_message_field(1, resource) + data += _encode_message_field(2, scope_spans_bytes) + return data + + +def _encode_scope_spans(scope_name, spans_bytes_list): + scope = _encode_string_field(1, scope_name) + data = _encode_message_field(1, scope) + for span_bytes in spans_bytes_list: + data += _encode_message_field(2, span_bytes) + return data + + +def _encode_traces_data(resource_spans_bytes_list): + data = b"" + for rs_bytes in resource_spans_bytes_list: + data += _encode_message_field(1, rs_bytes) + return data + + +# ========== Helper Functions ========== + + +def _gen_trace_id(): + return uuid.uuid4().bytes + + +def _gen_span_id(): + return uuid.uuid4().bytes[:8] + + +def _env(name, default=""): + """Get env var with GITEA_/GITHUB_ prefix fallback.""" + val = os.getenv(name, "") + if val: + return val + if name.startswith("GITEA_"): + alt = "GITHUB_" + name[6:] + return os.getenv(alt, default) + if name.startswith("GITHUB_"): + alt = "GITEA_" + name[7:] + return os.getenv(alt, default) + return default + + +def _get_pr_number(): + """Get PR number from environment or event file.""" + pr = os.getenv("PR_NUMBER", "") or os.getenv("GITEA_PR_NUMBER", "") + if pr: + return pr + + event_path = os.getenv("GITHUB_EVENT_PATH", "") or os.getenv("GITEA_EVENT_PATH", "") + if event_path and os.path.isfile(event_path): + try: + with open(event_path, "r") as f: + event = json.load(f) + if "pull_request" in event and "number" in event["pull_request"]: + return str(event["pull_request"]["number"]) + except Exception: + pass + + return "" + + +def _get_ci_attributes(): + """Collect attributes from CI environment variables.""" + attrs = { + "ci.repo": _env("GITEA_REPOSITORY") or _env("GITHUB_REPOSITORY") or "unknown", + "ci.workflow": _env("GITEA_WORKFLOW") or _env("GITHUB_WORKFLOW") or "unknown", + "ci.job": _env("GITEA_JOB") or _env("GITHUB_JOB") or "unknown", + "ci.commit_sha": _env("GITEA_SHA") or _env("GITHUB_SHA") or "unknown", + "ci.branch": _env("GITEA_REF_NAME") or _env("GITHUB_REF_NAME") or "unknown", + "ci.run_id": _env("GITEA_RUN_ID") or _env("GITHUB_RUN_ID") or "unknown", + "ci.actor": _env("GITEA_ACTOR") or _env("GITHUB_ACTOR") or "unknown", + "ci.event": _env("GITEA_EVENT_NAME") or _env("GITHUB_EVENT_NAME") or "unknown", + } + pr = _get_pr_number() + if pr: + attrs["ci.pr_number"] = pr + return attrs + + +# ========== Trace Building & Reporting ========== + + +def build_trace(service_name, trace_name, status, duration_ms, attributes=None): + """Build an OTLP trace payload (protobuf bytes). No external dependencies.""" + trace_id = _gen_trace_id() + end_time = int(time.time() * 1e9) + start_time = end_time - int(duration_ms * 1e6) + status_code = 1 if status in ("ok", "running") else 2 + status_msg = "" if status in ("ok", "running") else "Job failed" + + main_attrs = { + "agent.trace_name": trace_name, + "agent.service": service_name, + "ci.trace_status": status, + } + if attributes: + main_attrs.update(attributes) + + main_span = _encode_span( + trace_id_bytes=trace_id, + span_id_bytes=_gen_span_id(), + parent_span_id_bytes=b"", + name=trace_name, + start_time_unix_nano=start_time, + end_time_unix_nano=end_time, + span_kind=1, + attributes=main_attrs, + status_code=status_code, + status_msg=status_msg, + ) + + scope_spans = _encode_scope_spans("ci-trace", [main_span]) + resource_spans = _encode_resource_spans(service_name, scope_spans) + return _encode_traces_data([resource_spans]) + + +def report_ci_trace( + service_name, + trace_name, + status="ok", + duration_ms=1000, + endpoint=None, + license_key=None, + project=None, + workspace=None, + extra_attributes=None, +): + """ + Report CI Trace data. Returns (success: bool, message: str). + Never raises exceptions; returns False on failure. + """ + try: + endpoint = endpoint or os.getenv("AGENTLOOP_ENDPOINT", DEFAULT_ENDPOINT) + license_key = license_key or os.getenv("AGENTLOOP_LICENSE_KEY", "") + project = project or os.getenv("AGENTLOOP_PROJECT", DEFAULT_PROJECT) + workspace = workspace or os.getenv("AGENTLOOP_WORKSPACE", DEFAULT_WORKSPACE) + + if not license_key: + return False, "[Trace] skipped: AGENTLOOP_LICENSE_KEY not configured" + + attrs = _get_ci_attributes() + if extra_attributes: + attrs.update(extra_attributes) + + payload = build_trace( + service_name=service_name, + trace_name=trace_name, + status=status, + duration_ms=duration_ms, + attributes=attrs, + ) + + headers = { + "Content-Type": "application/x-protobuf", + "x-arms-license-key": license_key, + "x-arms-project": project, + "x-cms-workspace": workspace, + } + + req = urllib.request.Request(endpoint, data=payload, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=10) as resp: + status_code = resp.status + resp_body = resp.read().decode("utf-8", errors="replace") + except urllib.error.HTTPError as e: + status_code = e.code + resp_body = e.read().decode("utf-8", errors="replace") + + if status_code in (200, 202): + return True, (f"[Trace] success: {service_name} / {trace_name} " f"({status}, {duration_ms}ms)") + else: + return False, (f"[Trace] failed: HTTP {status_code} - {resp_body[:200]}") + except Exception as e: + return False, f"[Trace] error: {type(e).__name__}: {str(e)}" + + +def main(): + parser = argparse.ArgumentParser(description="CI AgentLoop Trace Reporter") + parser.add_argument( + "--service", + dest="service_name", + default=os.getenv("TRACE_SERVICE", ""), + help="Service name (also via TRACE_SERVICE env)", + ) + parser.add_argument( + "--name", + dest="trace_name", + default=os.getenv("TRACE_NAME", ""), + help="Trace name (also via TRACE_NAME env)", + ) + parser.add_argument( + "--status", + default=os.getenv("TRACE_STATUS", "ok"), + choices=["ok", "error", "running"], + help="Status: ok / error / running (default ok)", + ) + parser.add_argument( + "--start-time", + dest="start_time", + default=os.getenv("TRACE_START_TIME", ""), + help="Start timestamp (seconds) for duration calculation", + ) + parser.add_argument( + "--duration-ms", + dest="duration_ms", + type=int, + default=0, + help="Direct duration in ms; takes precedence over --start-time", + ) + parser.add_argument("--attrs", default="", help="Extra attributes (JSON string)") + + args = parser.parse_args() + + if not args.service_name: + print("[Trace] skipped: no service specified (--service or TRACE_SERVICE)") + sys.exit(0) + + duration_ms = args.duration_ms + if duration_ms <= 0 and args.start_time: + try: + start_ts = float(args.start_time) + duration_ms = int((time.time() - start_ts) * 1000) + except (ValueError, TypeError): + duration_ms = 1000 + if duration_ms <= 0: + duration_ms = 1000 + + extra_attrs = {} + if args.attrs: + try: + extra_attrs = json.loads(args.attrs) + except json.JSONDecodeError: + pass + + trace_name = args.trace_name + if not trace_name: + wf = _env("GITEA_WORKFLOW") or _env("GITHUB_WORKFLOW") or "CI" + job = _env("GITEA_JOB") or _env("GITHUB_JOB") or "job" + trace_name = f"{wf} / {job}" + + success, msg = report_ci_trace( + service_name=args.service_name, + trace_name=trace_name, + status=args.status, + duration_ms=duration_ms, + extra_attributes=extra_attrs, + ) + + print(msg) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/docker_build_only.sh b/scripts/ci/docker_build_only.sh new file mode 100755 index 000000000..7eda422ab --- /dev/null +++ b/scripts/ci/docker_build_only.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# PR构建专用：只构建不输出，验证Dockerfile能否正常构建 +# 优先用buildx + 远程缓存，失败自动回退到普通docker build（DooD模式下buildx builder偶发崩溃） +set -eu + +NO_CACHE_FLAG="" +if [ "$1" = "--no-cache" ]; then + NO_CACHE_FLAG="--no-cache" + shift +fi + +DOCKERFILE="$1" +IMAGE_TAG="$2" +CACHE_REF="$3" +shift 3 +BUILD_ARGS="" +for arg in "$@"; do + BUILD_ARGS="$BUILD_ARGS --build-arg $arg" +done + +BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}" + +echo "=== PR Build: buildx + remote cache (attempt 1) ===" +echo "Dockerfile: ${DOCKERFILE}" +echo "Image tag: ${IMAGE_TAG}" +echo "" + +# --- 尝试 buildx docker-container driver --- +if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then + docker buildx create --use --name "$BUILDER_NAME" --driver docker-container 2>/dev/null || true +else + docker buildx use "$BUILDER_NAME" 2>/dev/null || true +fi +docker buildx inspect --bootstrap > /dev/null 2>&1 || true + +set +e +docker buildx build \ + $NO_CACHE_FLAG \ + $BUILD_ARGS \ + --cache-from "type=registry,ref=${CACHE_REF}" \ + -f "${DOCKERFILE}" \ + -t "${IMAGE_TAG}" \ + --load \ + . +BUILDX_EXIT=$? +set -e + +if [ $BUILDX_EXIT -eq 0 ]; then + echo "" + echo "PR build OK (buildx): ${IMAGE_TAG}" + exit 0 +fi + +echo "" +echo "⚠️ buildx build失败，回退到普通docker build" +echo " 原因：buildx builder在DooD模式下偶发不稳定（graceful_stop / buildkitd.sock）" +echo "" + +# 清理 buildx builder +docker buildx rm "$BUILDER_NAME" 2>/dev/null || true + +# --- 回退：普通 docker build --- +# 注意：普通docker build不支持远程缓存，但更稳定 +set +e +docker build \ + $NO_CACHE_FLAG \ + $BUILD_ARGS \ + -f "${DOCKERFILE}" \ + -t "${IMAGE_TAG}" \ + . +DOCKER_EXIT=$? +set -e + +if [ $DOCKER_EXIT -eq 0 ]; then + echo "" + echo "PR build OK (fallback docker build): ${IMAGE_TAG}" + exit 0 +fi + +echo "" +echo "❌ PR build failed (both buildx and docker build)" +exit 1 diff --git a/scripts/ci/docker_build_push.sh b/scripts/ci/docker_build_push.sh new file mode 100755 index 000000000..f3ba4bb80 --- /dev/null +++ b/scripts/ci/docker_build_push.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# 通用Docker镜像构建+推送脚本（local cache为主 + registry cache共享） +# 用法: docker_build_push.sh [--no-cache] [build_arg...] +set -eu + +NO_CACHE_FLAG="" +if [ "$1" = "--no-cache" ]; then + NO_CACHE_FLAG="--no-cache" + shift + echo "模式: --no-cache (不使用缓存，全新构建)" +fi + +DOCKERFILE="$1" +IMAGE_TAG="$2" +CACHE_REF="$3" +shift 3 +BUILD_ARGS="" +for arg in "$@"; do + BUILD_ARGS="$BUILD_ARGS --build-arg $arg" +done + +if ! docker buildx inspect ci-builder > /dev/null 2>&1; then + docker buildx create --use --name ci-builder --driver docker-container + echo "Created ci-builder" +else + docker buildx use ci-builder + echo "Using existing ci-builder" +fi +docker buildx inspect --bootstrap + +# 从cache_ref中提取缓存名称（如 api-cache:develop -> api-cache-develop） +CACHE_NAME=$(echo "$CACHE_REF" | tr '/' '_' | tr ':' '-') +LOCAL_CACHE_DIR="/tmp/buildx-cache/${CACHE_NAME}" + +mkdir -p "$LOCAL_CACHE_DIR" + +# 缓存源：local优先（带自动修复），registry兜底读写 +# 本地缓存损坏时自动清理后重试，避免snapshot not found导致构建全挂 +build_with_cache_retry() { + local attempt=1 + local max_attempts=2 + while [ $attempt -le $max_attempts ]; do + local build_output + local exit_code + set +e + build_output=$(docker buildx build \ + $NO_CACHE_FLAG \ + $BUILD_ARGS \ + --cache-from "type=local,src=${LOCAL_CACHE_DIR}" \ + --cache-from "type=registry,ref=${CACHE_REF}" \ + --cache-to "type=local,dest=${LOCAL_CACHE_DIR},mode=max" \ + --cache-to "type=registry,ref=${CACHE_REF},mode=max,ignore-error=true" \ + -f "${DOCKERFILE}" \ + -t "${IMAGE_TAG}" \ + --push \ + . 2>&1) + exit_code=$? + set -e + if [ $exit_code -eq 0 ]; then + echo "$build_output" + return 0 + fi + # 检测到缓存损坏类错误，清掉本地缓存重试 + if echo "$build_output" | grep -qE "parent snapshot.*not found|snapshot.*does not exist|cache.*corrupt|failed to compute cache key"; then + echo "$build_output" + echo "" + echo "⚠️ Local cache appears corrupted, cleaning up and retrying (attempt $attempt/$max_attempts)..." + rm -rf "${LOCAL_CACHE_DIR}" + mkdir -p "${LOCAL_CACHE_DIR}" + # 清理buildx builder的内部snapshot状态 + docker buildx prune -f -a >/dev/null 2>&1 || true + attempt=$((attempt + 1)) + else + # 非缓存类错误，直接输出并返回 + echo "$build_output" + return $exit_code + fi + done + # 重试完还是失败，不用本地缓存最后试一次（只从registry读） + echo "⚠️ All cached attempts failed, building without local cache..." + docker buildx build \ + $NO_CACHE_FLAG \ + $BUILD_ARGS \ + --cache-from "type=registry,ref=${CACHE_REF}" \ + --cache-to "type=local,dest=${LOCAL_CACHE_DIR},mode=max" \ + --cache-to "type=registry,ref=${CACHE_REF},mode=max,ignore-error=true" \ + -f "${DOCKERFILE}" \ + -t "${IMAGE_TAG}" \ + --push \ + . +} + +echo "=== Step 1: Build & push image (local cache + registry cache, with auto-repair) ===" +echo "Local cache: ${LOCAL_CACHE_DIR}" +echo "Registry cache: ${CACHE_REF}" +echo "" + +build_with_cache_retry + +echo "" +echo "Image pushed: ${IMAGE_TAG}" +echo "Local cache updated" +echo "Registry cache updated (if supported)" + +echo "" +echo "Build completed: ${IMAGE_TAG}" diff --git a/scripts/ci/generate_ci_dashboard.sh b/scripts/ci/generate_ci_dashboard.sh new file mode 100644 index 000000000..fcd7abf8f --- /dev/null +++ b/scripts/ci/generate_ci_dashboard.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# +# CI 健康度看板一键生成脚本 +# - 从 Gitea Actions API 拉取数据 +# - 生成 HTML 可视化看板 +# - 输出文件路径 +# +# 用法: +# bash scripts/ci/generate_ci_dashboard.sh [--days 7] [--output ci_dashboard.html] +# +# 环境变量: +# GITEA_TOKEN API Token（必需） +# GITEA_URL Gitea 地址（可选，默认 https://git.xiaoxiajianji.com） +# GITEA_REPO 仓库（可选，默认 xiaoxia/xiaoxia-saas） +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +# 默认参数 +DAYS=7 +OUTPUT="ci_dashboard.html" + +# 解析参数 +while [[ $# -gt 0 ]]; do + case "$1" in + --days) + DAYS="$2" + shift 2 + ;; + --output|-o) + OUTPUT="$2" + shift 2 + ;; + --help|-h) + echo "用法: bash scripts/ci/generate_ci_dashboard.sh [--days 7] [--output ci_dashboard.html]" + echo "" + echo "选项:" + echo " --days N 统计最近 N 天 (默认 7)" + echo " --output PATH HTML 输出路径 (默认 ci_dashboard.html)" + echo " --help 显示帮助" + echo "" + echo "环境变量:" + echo " GITEA_TOKEN API Token（必需）" + echo " GITEA_URL Gitea 地址" + echo " GITEA_REPO 仓库" + exit 0 + ;; + *) + echo "未知参数: $1" + exit 1 + ;; + esac +done + +# 检查 Python +if ! command -v python3 &> /dev/null; then + echo "[ERROR] 未找到 python3，请先安装 Python 3" + exit 1 +fi + +# 检查 Token +if [[ -z "${GITEA_TOKEN:-}" ]]; then + echo "[ERROR] 请设置 GITEA_TOKEN 环境变量" + exit 1 +fi + +echo "========================================" +echo " CI 健康度看板生成器" +echo "========================================" +echo "" +echo "统计天数: ${DAYS} 天" +echo "输出文件: ${OUTPUT}" +echo "" + +# 生成 HTML 看板 +echo "[INFO] 正在拉取数据并生成看板..." +python3 "${SCRIPT_DIR}/ci_dashboard.py" \ + --days "${DAYS}" \ + --html \ + --html-output "${OUTPUT}" + +echo "" +echo "========================================" +echo " ✅ 看板生成完成！" +echo "========================================" +echo "" +echo "文件路径: $(realpath "${OUTPUT}")" +echo "" + +# 如果在 macOS 上，尝试打开 +if [[ "$(uname)" == "Darwin" ]]; then + echo "[INFO] 正在打开浏览器..." + open "${OUTPUT}" +fi diff --git a/scripts/ci/mypy_check.sh b/scripts/ci/mypy_check.sh new file mode 100755 index 000000000..01e7c97ea --- /dev/null +++ b/scripts/ci/mypy_check.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# mypy增量扫描脚本 - CI中调用 +# 环境变量: SCAN_MODE, CHANGED_PY_FILES + +set -e + +echo "=== Installing mypy ===" +python3 -m pip install -q mypy +mypy --version +echo "" +echo "=== Running mypy type check (hard gate mode) ===" +echo "告警模式，阻断CI" +echo "" + +MYPY_COMMON_ARGS="--ignore-missing-imports --no-site-packages --no-strict-optional --explicit-package-bases --exclude tests/|test_|migrations/|alembic/ --no-error-summary --incremental --cache-dir .mypy_cache" + +EXIT_CODE=0 + +if [ "$SCAN_MODE" = "incremental" ] && [ -n "$CHANGED_PY_FILES" ]; then + echo "=== Incremental mypy scan (PR mode) ===" + echo "Changed files: $(echo $CHANGED_PY_FILES | wc -w) files" + MYPY_FILES="" + for f in $CHANGED_PY_FILES; do + case "$f" in + apps/*|packages/*) + MYPY_FILES="$MYPY_FILES $f" + ;; + esac + done + if [ -n "$MYPY_FILES" ]; then + echo "Checking: $MYPY_FILES" + mypy $MYPY_FILES $MYPY_COMMON_ARGS 2>&1 | head -80 || EXIT_CODE=$? + else + echo "No mypy-checkable files changed, skipping" + fi +else + echo "=== Full mypy scan ===" + mypy apps/api/app packages $MYPY_COMMON_ARGS 2>&1 | head -60 || EXIT_CODE=$? +fi + +echo "" +if [ "$EXIT_CODE" != "0" ]; then + echo "mypy 发现类型问题（告警模式，阻断）" + echo "建议后续逐步修复" +else + echo "mypy 类型检查通过" +fi + diff --git a/scripts/ci/pr_auto_scan.py b/scripts/ci/pr_auto_scan.py new file mode 100644 index 000000000..f17753d27 --- /dev/null +++ b/scripts/ci/pr_auto_scan.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python3 +""" +PR自动扫描器：扫描所有open PR，对CI全绿的进行自动审批/合并 +作为短作业模式的兜底机制，每5分钟运行一次 + +新增：AI审查联动 - AI代码审查发现严重问题时，不自动审批 +""" + +import argparse +import json +import os +import re +import sys +import time +import urllib.error +import urllib.request + + +def api_request(token, repo, endpoint, method="GET", data=None): + """Gitea API请求""" + url = f"https://git.xiaoxiajianji.com/api/v1/repos/{repo}/{endpoint}" + headers = {"Authorization": f"token {token}", "Content-Type": "application/json"} + body = json.dumps(data).encode() if data else None + req = urllib.request.Request(url, data=body, headers=headers, method=method) + + # 跳过SSL验证 + import ssl + + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + try: + resp = urllib.request.urlopen(req, context=ctx) + return json.loads(resp.read().decode()), resp.status + except urllib.error.HTTPError as e: + return json.loads(e.read().decode()) if e.read() else {"error": str(e)}, e.code + + +def get_open_prs(token, repo, base="develop"): + """获取所有open的PR""" + prs = [] + page = 1 + while True: + data, code = api_request(token, repo, f"pulls?state=open&base={base}&sort=recentupdate&per_page=50&page={page}") + if code != 200 or not isinstance(data, list) or len(data) == 0: + break + prs.extend(data) + if len(data) < 50: + break + page += 1 + return prs + + +def get_commit_status(token, repo, sha): + """获取commit的CI状态汇总""" + data, code = api_request(token, repo, f"commits/{sha}/status") + if code != 200: + return {}, "error" + return data, data.get("state", "unknown") + + +def check_required_contexts(token, repo, sha, contexts): + """检查指定的context是否都通过""" + data, _ = get_commit_status(token, repo, sha) + statuses = {s["context"]: s["status"] for s in data.get("statuses", [])} + + all_success = True + any_pending = False + any_failed = False + + for ctx in contexts: + state = statuses.get(ctx, "pending") + if state != "success": + all_success = False + if state == "pending": + any_pending = True + if state in ("failure", "error"): + any_failed = True + + return all_success, any_pending, any_failed, statuses + + +def get_pr_files(token, repo, pr_number): + """获取PR变更文件""" + files = [] + page = 1 + while True: + data, code = api_request(token, repo, f"pulls/{pr_number}/files?per_page=300&page={page}") + if code != 200 or not isinstance(data, list) or len(data) == 0: + break + files.extend(data) + if len(data) < 300: + break + page += 1 + return [f["filename"] for f in files] + + +def is_frontend_only(files): + """判断是否纯前端改动""" + if not files: + return False + frontend_count = sum(1 for f in files if f.startswith("apps/web/")) + backend_count = len(files) - frontend_count + return backend_count == 0 and frontend_count > 0 + + +def has_approval(token, repo, pr_number): + """检查PR是否已有审批""" + reviews, code = api_request(token, repo, f"pulls/{pr_number}/reviews") + if code != 200: + return False + return any(r.get("state") == "APPROVED" for r in reviews if isinstance(r, dict)) + + +def get_ai_review_result(token, repo, pr_number): + """ + 检查AI代码审查结果，返回 (has_critical, review_body) + has_critical: 是否有严重问题（需修改的问题 > 0） + review_body: 最新的AI审查评论文本 + """ + # AI审查评论标记 + AI_REVIEW_MARKER = "AI_CODE_REVIEW_AUTO_COMMENT" + + comments, code = api_request(token, repo, f"issues/{pr_number}/comments") + if code != 200: + return False, None + + # 找最新的AI审查评论 + ai_comments = [c for c in comments if isinstance(c, dict) and AI_REVIEW_MARKER in c.get("body", "")] + + if not ai_comments: + return False, None + + # 按时间排序，取最新的 + latest = max(ai_comments, key=lambda c: c.get("created_at", "")) + body = latest.get("body", "") + + # 解析严重问题数量 + # 匹配 "严重问题数量：X 个" 或 "需修改的问题（严重）" 下的列表 + critical_count = 0 + + # 方式1：直接匹配数字 + match = re.search(r"严重问题数量[：:]\s*(\d+)\s*个", body) + if match: + critical_count = int(match.group(1)) + else: + # 方式2：数 "需修改的问题" 章节下的条目数 + critical_section = re.search( + r"###\s*[❌⚠️].*?(?:需修改|问题).*?\n(.*?)(?=\n###|\Z)", + body, + re.DOTALL, + ) + if critical_section: + section_text = critical_section.group(1) + # 数编号条目 1. 2. 3. + items = re.findall(r"^\d+\.\s+\*\*", section_text, re.MULTILINE) + critical_count = len(items) + + has_critical = critical_count > 0 + return has_critical, body + + +def approve_pr(token, repo, pr_number, reason="CI全绿，自动审批通过。"): + """审批PR""" + # 创建review + data, code = api_request( + token, + repo, + f"pulls/{pr_number}/reviews", + method="POST", + data={"event": "PENDING", "body": reason}, + ) + + if code not in (200, 201): + return False, f"创建review失败: HTTP {code}" + + review_id = data.get("id") + if data.get("state") == "APPROVED": + return True, "直接创建APPROVED成功" + + if not review_id: + return False, "未获取到review ID" + + # submit为APPROVED + data2, code2 = api_request( + token, + repo, + f"pulls/{pr_number}/reviews/{review_id}/events", + method="POST", + data={"event": "APPROVED", "body": reason}, + ) + + if code2 in (200, 201): + return True, "审批提交成功" + else: + # 尝试另一个端点 + data3, code3 = api_request( + token, + repo, + f"pulls/{pr_number}/reviews/{review_id}", + method="POST", + data={"event": "APPROVED", "body": reason}, + ) + if code3 in (200, 201): + return True, "审批提交成功(备用端点)" + return False, f"审批提交失败: HTTP {code2}/{code3}" + + +def add_pr_label(token, repo, pr_number, label): + """给PR添加标签""" + data, code = api_request( + token, + repo, + f"issues/{pr_number}/labels", + method="POST", + data={"labels": [label]}, + ) + return code in (200, 201) + + +def merge_pr(token, repo, pr_number): + """合并PR（squash merge）""" + # 等待几秒让状态同步 + time.sleep(30) + + # 检查PR状态 + pr_data, code = api_request(token, repo, f"pulls/{pr_number}") + if code != 200: + return False, f"获取PR状态失败: HTTP {code}" + if pr_data.get("state") != "open": + return False, f"PR状态不是open: {pr_data.get('state')}" + + # 执行squash merge + data, code = api_request( + token, + repo, + f"pulls/{pr_number}/merge", + method="POST", + data={ + "do": "squash", + "merge_title_field": "", + "merge_message_field": "", + "delete_branch_after_merge": True, + "force_merge": False, + }, + ) + + if code == 200: + return True, "合并成功" + elif code == 405: + return False, "合并返回405（门禁未满足或冲突）" + else: + return False, f"合并失败: HTTP {code}" + + +def main(): + parser = argparse.ArgumentParser(description="PR自动扫描器") + parser.add_argument("--token", required=True, help="Gitea API token") + parser.add_argument("--repo", default="xiaoxia/xiaoxia-saas", help="仓库") + parser.add_argument("--base", default="develop", help="目标分支") + parser.add_argument("--approve", action="store_true", help="执行自动审批") + parser.add_argument("--merge", action="store_true", help="执行自动合并") + parser.add_argument("--dry-run", default="false", help="试运行模式") + parser.add_argument("--max-prs", type=int, default=20, help="最多处理的PR数") + parser.add_argument("--skip-ai-review", action="store_true", help="跳过AI审查检查（强制审批）") + + args = parser.parse_args() + + dry_run = args.dry_run.lower() == "true" + + # required contexts（与分支保护一致） + REQUIRED_CONTEXTS_FULL = [ + "CI/CD Pipeline / Validate - Code Quality (pull_request)", + "CI/CD Pipeline / Validate - Type Check (mypy) (pull_request)", + "CI/CD Pipeline / Validate - Migration (alembic) (pull_request)", + "CI/CD Pipeline / Frontend Lint (pull_request)", + "CI/CD Pipeline / PR Build API Image (pull_request)", + "CI/CD Pipeline / PR Build Worker Image (pull_request)", + "CI/CD Pipeline / PR Build Web Image (pull_request)", + ] + REQUIRED_CONTEXTS_APPROVE = [ + "CI/CD Pipeline / Validate - Code Quality (pull_request)", + "CI/CD Pipeline / Validate - Type Check (mypy) (pull_request)", + "CI/CD Pipeline / Validate - Migration (alembic) (pull_request)", + "CI/CD Pipeline / Frontend Lint (pull_request)", + ] + FRONTEND_ONLY_CONTEXT = [ + "CI/CD Pipeline / Frontend Lint (pull_request)", + ] + + # 获取所有open PR + print(f"获取 {args.base} 分支的open PR...") + prs = get_open_prs(args.token, args.repo, args.base) + print(f"找到 {len(prs)} 个open PR") + + approved_count = 0 + merged_count = 0 + skipped_count = 0 + ai_blocked_count = 0 + + for pr in prs[: args.max_prs]: + pr_num = pr["number"] + pr_title = pr["title"] + head_sha = pr["head"]["sha"] + base_ref = pr.get("base", {}).get("re", "") + + # 跳过draft + if pr.get("draft"): + print(f"\n⏭️ #{pr_num} {pr_title[:50]} - draft，跳过") + skipped_count += 1 + continue + + # 跳过目标分支不对的 + if base_ref != args.base: + skipped_count += 1 + continue + + print(f"\n--- #{pr_num} {pr_title[:60]} ---") + + # 判断是否纯前端 + files = get_pr_files(args.token, args.repo, pr_num) + frontend_only = is_frontend_only(files) + + if frontend_only: + approve_contexts = FRONTEND_ONLY_CONTEXT + merge_contexts = FRONTEND_ONLY_CONTEXT + print(f" 类型: 纯前端改动 ({len(files)}个文件)") + else: + approve_contexts = REQUIRED_CONTEXTS_APPROVE + merge_contexts = REQUIRED_CONTEXTS_FULL + print(f" 类型: 全栈/后端改动 ({len(files)}个文件)") + + # 检查审批用的CI状态 + all_ok, pending, failed, _ = check_required_contexts(args.token, args.repo, head_sha, approve_contexts) + + # === AI审查检查 === + ai_has_critical = False + if not args.skip_ai_review and all_ok and not failed and args.approve: + ai_has_critical, ai_body = get_ai_review_result(args.token, args.repo, pr_num) + if ai_has_critical: + print(" ⚠️ AI审查发现严重问题，阻止自动审批") + ai_blocked_count += 1 + # 给PR打标签便于人工识别 + if not dry_run: + add_pr_label(args.token, args.repo, pr_num, "ai-review/需修改") + + # === 自动审批 === + if args.approve and all_ok and not failed and not ai_has_critical: + if has_approval(args.token, args.repo, pr_num): + print(" ✅ 已有审批，跳过") + else: + if dry_run: + print(" 🎯 [DRY-RUN] 将自动审批") + else: + print(" 🎯 执行自动审批...") + ok, msg = approve_pr(args.token, args.repo, pr_num) + if ok: + print(f" ✅ 审批成功: {msg}") + approved_count += 1 + else: + print(f" ❌ 审批失败: {msg}") + elif ai_has_critical: + print(" 🚫 AI审查阻止审批（人工可手动审批覆盖）") + elif failed: + print(" ❌ CI有失败项，跳过审批") + elif pending: + print(" ⏳ CI仍在运行，跳过") + + # === 自动合并 === + if args.merge: + # 检查合并用的CI状态 + merge_ok, merge_pending, merge_failed, _ = check_required_contexts( + args.token, args.repo, head_sha, merge_contexts + ) + + # 检查审批 + approved = has_approval(args.token, args.repo, pr_num) + + if merge_ok and approved and not merge_failed: + if dry_run: + print(" 🎯 [DRY-RUN] 将自动合并") + else: + print(" 🎯 执行自动合并...") + ok, msg = merge_pr(args.token, args.repo, pr_num) + if ok: + print(f" ✅ 合并成功: {msg}") + merged_count += 1 + else: + print(f" ⚠️ 合并失败: {msg}") + elif merge_pending: + print(" ⏳ 合并条件未满足: CI运行中") + elif merge_failed: + print(" ❌ 合并条件未满足: CI有失败") + elif not approved: + print(" ⏳ 合并条件未满足: 无审批") + + print("\n=== 扫描结果 ===") + print(f" 处理PR数: {min(len(prs), args.max_prs)}") + print(f" 自动审批: {approved_count} 个") + print(f" 自动合并: {merged_count} 个") + print(f" AI审查阻止: {ai_blocked_count} 个") + print(f" 跳过: {skipped_count} 个") + print(" 模式: {'DRY-RUN' if dry_run else '正式执行'}") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/preview_comment.py b/scripts/ci/preview_comment.py new file mode 100755 index 000000000..cdf35d07a --- /dev/null +++ b/scripts/ci/preview_comment.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""生成预览环境PR评论内容""" + +import json +import os +import sys + + +def generate_deploy_comment(pr_number, preview_url): + """生成部署成功的评论内容""" + return f"""🚀 **预览环境已部署** + +| 项目 | 详情 | +|------|------| +| PR号 | #{pr_number} | +| 预览链接 | [{preview_url}]({preview_url}) | +| API环境 | staging | + +> 💡 预览环境使用 staging API 数据，请勿在预览环境中操作重要数据。 +> +> 🔄 每次提交新代码后预览环境会自动更新。 +> +> 🗑️ PR 关闭或合并后，预览环境会自动清理。 +""" + + +def generate_cleanup_comment(pr_number): + """生成清理完成的评论内容""" + return f"""🗑️ **预览环境已清理** + +PR #{pr_number} 已关闭或合并，对应的预览环境已被清理。 + +> 如有需要，可以重新打开 PR 来重新生成预览环境。 +""" + + +def main(): + mode = sys.argv[1] if len(sys.argv) > 1 else "deploy" + pr_number = os.environ.get("PR_NUMBER", "") + preview_url = os.environ.get("PREVIEW_URL", "") + + if mode == "deploy": + body = generate_deploy_comment(pr_number, preview_url) + elif mode == "cleanup": + body = generate_cleanup_comment(pr_number) + else: + print(f"Unknown mode: {mode}", file=sys.stderr) + sys.exit(1) + + print(json.dumps({"body": body})) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/preview_init_server.sh b/scripts/ci/preview_init_server.sh new file mode 100755 index 000000000..2091882e0 --- /dev/null +++ b/scripts/ci/preview_init_server.sh @@ -0,0 +1,264 @@ +#!/bin/bash +# ============================================================ +# 预览环境服务器初始化脚本 +# 用途：在业务服务器上创建预览环境所需的目录和配置 +# 使用方式：bash scripts/ci/preview_init_server.sh +# ============================================================ + +set -eu + +PREVIEW_ROOT="/var/www/preview" +NGINX_CONF_PATH="/etc/nginx/conf.d/preview.conf" +DOMAIN="xiaoxiajianji.com" +STAGING_API="https://staging-api.xiaoxiajianji.com" + +echo "==========================================" +echo " 预览环境服务器初始化" +echo "==========================================" +echo "" + +# 1. 创建预览根目录 +echo "[1/4] 创建预览根目录..." +if [ -d "$PREVIEW_ROOT" ]; then + echo " 目录已存在: $PREVIEW_ROOT" +else + mkdir -p "$PREVIEW_ROOT" + echo " 已创建: $PREVIEW_ROOT" +fi +chown -R root:root "$PREVIEW_ROOT" +chmod -R 755 "$PREVIEW_ROOT" +echo "" + +# 2. 创建测试页面（验证Nginx配置用） +echo "[2/4] 创建测试页面..." +TEST_DIR="${PREVIEW_ROOT}/pr-demo" +mkdir -p "$TEST_DIR" +cat > "$TEST_DIR/index.html" <<'EOF' + + + + + + 预览环境测试页 + + + +
+
✅
+

预览环境配置成功！

+

如果你能看到这个页面，说明 Nginx 预览环境配置正确。

+

当前站点通过子域名 pr-demo.preview 路由到 /var/www/preview/pr-demo/ 目录。

+
+ + +EOF +echo " 测试页面已创建: $TEST_DIR/index.html" +echo "" + +# 3. 检查Nginx是否安装 +echo "[3/4] 检查Nginx环境..." +if command -v nginx > /dev/null 2>&1; then + echo " Nginx 已安装: $(nginx -v 2>&1)" + NGINX_INSTALLED=true +else + echo " ⚠️ Nginx 未安装，请先安装 Nginx" + NGINX_INSTALLED=false +fi +echo "" + +# 4. 输出Nginx配置建议 +echo "[4/4] Nginx 配置建议" +echo "" +echo "----------------------------------------" +echo " 请将以下配置保存到: $NGINX_CONF_PATH" +echo " 或复制到 Nginx 配置目录中" +echo "----------------------------------------" +echo "" + +cat <<'NGINX_CONF' +# ============================================================ +# 预览环境 Nginx 配置 +# 支持 *.preview.xiaoxiajianji.com 通配符子域名 +# ============================================================ + +# 从子域名中提取 PR 号（如 pr-123.preview -> pr-123） +map $host $preview_pr { + default ""; + ~^(?pr-\d+)\.preview\.xiaoxiajianji\.com$ $pr; +} + +# HTTP 服务器（80端口） +server { + listen 80; + server_name *.preview.xiaoxiajianji.com; + + # 根目录根据子域名动态映射 + root /var/www/preview/$preview_pr; + + # 索引文件 + index index.html; + + # 字符集 + charset utf-8; + + # 访问日志 + access_log /var/log/nginx/preview_access.log; + error_log /var/log/nginx/preview_error.log warn; + + # 如果子域名格式不正确，返回404 + if ($preview_pr = "") { + return 404; + } + + # 如果预览目录不存在，返回404 + if (!-d $document_root) { + return 404; + } + + # API 反向代理到 staging 环境 + location /api/ { + proxy_pass https://staging-api.xiaoxiajianji.com/api/; + proxy_http_version 1.1; + proxy_set_header Host staging-api.xiaoxiajianji.com; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # 超时设置 + proxy_connect_timeout 30s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + + # 缓冲设置 + proxy_buffering on; + proxy_buffer_size 4k; + proxy_buffers 8 4k; + + # WebSocket 支持（如需要） + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection "upgrade"; + } + + # 静态资源缓存 + location /assets/ { + expires 7d; + add_header Cache-Control "public, max-age=604800, immutable"; + try_files $uri =404; + } + + # SPA 路由支持 + location / { + try_files $uri $uri/ /index.html; + } + + # 安全相关响应头 + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-XSS-Protection "1; mode=block" always; + + # 禁止隐藏文件访问 + location ~ /\. { + deny all; + access_log off; + log_not_found off; + } +} + +# HTTPS 服务器（443端口） +# 注意：需要先配置 SSL 证书 +# 建议使用 certbot 或手动配置证书 +# +# server { +# listen 443 ssl http2; +# server_name *.preview.xiaoxiajianji.com; +# +# # SSL 证书配置（请替换为实际证书路径） +# ssl_certificate /path/to/fullchain.pem; +# ssl_certificate_key /path/to/privkey.pem; +# +# # SSL 安全配置 +# ssl_protocols TLSv1.2 TLSv1.3; +# ssl_ciphers HIGH:!aNULL:!MD5; +# ssl_prefer_server_ciphers on; +# ssl_session_cache shared:SSL:10m; +# ssl_session_timeout 10m; +# +# # 其余配置与 HTTP 相同 +# root /var/www/preview/$preview_pr; +# index index.html; +# charset utf-8; +# +# access_log /var/log/nginx/preview_ssl_access.log; +# error_log /var/log/nginx/preview_ssl_error.log warn; +# +# if ($preview_pr = "") { +# return 404; +# } +# +# if (!-d $document_root) { +# return 404; +# } +# +# location /api/ { +# proxy_pass https://staging-api.xiaoxiajianji.com/api/; +# proxy_http_version 1.1; +# proxy_set_header Host staging-api.xiaoxiajianji.com; +# proxy_set_header X-Real-IP $remote_addr; +# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +# proxy_set_header X-Forwarded-Proto $scheme; +# proxy_set_header X-Forwarded-Host $host; +# proxy_connect_timeout 30s; +# proxy_send_timeout 60s; +# proxy_read_timeout 60s; +# } +# +# location /assets/ { +# expires 7d; +# add_header Cache-Control "public, max-age=604800, immutable"; +# try_files $uri =404; +# } +# +# location / { +# try_files $uri $uri/ /index.html; +# } +# +# add_header X-Frame-Options "SAMEORIGIN" always; +# add_header X-Content-Type-Options "nosniff" always; +# add_header X-XSS-Protection "1; mode=block" always; +# +# location ~ /\. { +# deny all; +# access_log off; +# log_not_found off; +# } +# } +NGINX_CONF + +echo "" +echo "----------------------------------------" +echo " 配置完成后的操作步骤：" +echo "----------------------------------------" +echo "" +echo "1. 将上面的 Nginx 配置保存到合适的位置（如 /etc/nginx/conf.d/preview.conf）" +echo "2. 测试配置: nginx -t" +echo "3. 重载配置: nginx -s reload" +echo "4. 配置 DNS 解析: 将 *.preview.xiaoxiajianji.com 指向服务器 IP" +echo "5. 配置 SSL 证书（推荐使用 Let's Encrypt 通配符证书）" +echo "" +echo "测试方式：" +echo " 访问 http://pr-demo.preview.xiaoxiajianji.com 验证配置" +echo "" +echo "==========================================" +echo " 初始化完成" +echo "==========================================" diff --git a/scripts/ci/preview_nginx.conf.template b/scripts/ci/preview_nginx.conf.template new file mode 100644 index 000000000..2b420eda5 --- /dev/null +++ b/scripts/ci/preview_nginx.conf.template @@ -0,0 +1,226 @@ +# ============================================================ +# 预览环境 Nginx 配置模板 +# 支持 *.preview.xiaoxiajianji.com 通配符子域名 +# +# 使用方法： +# 1. 将本文件复制到 Nginx 配置目录（如 /etc/nginx/conf.d/preview.conf） +# 2. 根据实际情况修改域名和 API 地址 +# 3. 运行 nginx -t 测试配置 +# 4. 运行 nginx -s reload 重载配置 +# +# 前置条件： +# - DNS 已配置 *.preview.xiaoxiajianji.com 指向本服务器 +# - 预览根目录已创建：/var/www/preview/ +# - 每个 PR 的静态文件放在 /var/www/preview/pr-{N}/ 下 +# ============================================================ + +# ---- 变量定义 ---- +# 从子域名中提取 PR 号（如 pr-123.preview -> pr-123） +map $host $preview_pr { + default ""; + ~^(?pr-\d+)\.preview\.xiaoxiajianji\.com$ $pr; +} + +# ---- HTTP 服务器（80端口） ---- +server { + listen 80; + server_name *.preview.xiaoxiajianji.com; + + # 根目录根据子域名动态映射 + root /var/www/preview/$preview_pr; + + # 索引文件 + index index.html; + + # 字符集 + charset utf-8; + + # 访问日志 + access_log /var/log/nginx/preview_access.log; + error_log /var/log/nginx/preview_error.log warn; + + # 如果子域名格式不正确，返回404 + if ($preview_pr = "") { + return 404; + } + + # 如果预览目录不存在，返回404 + if (!-d $document_root) { + return 404; + } + + # ---- API 反向代理到 staging 环境 ---- + location /api/ { + proxy_pass https://staging-api.xiaoxiajianji.com/api/; + proxy_http_version 1.1; + + # 请求头设置 + proxy_set_header Host staging-api.xiaoxiajianji.com; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # 超时设置 + proxy_connect_timeout 30s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + + # 缓冲设置 + proxy_buffering on; + proxy_buffer_size 4k; + proxy_buffers 8 4k; + + # 重定向跟随 + proxy_redirect off; + + # WebSocket 支持（如需要，取消注释） + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection "upgrade"; + } + + # ---- 生成文件代理（如需要） ---- + # location /generated-files/ { + # proxy_pass https://staging-api.xiaoxiajianji.com/generated-files/; + # proxy_http_version 1.1; + # proxy_set_header Host staging-api.xiaoxiajianji.com; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + # } + + # ---- 静态资源缓存 ---- + location /assets/ { + expires 7d; + add_header Cache-Control "public, max-age=604800, immutable"; + try_files $uri =404; + } + + # ---- SPA 路由支持 ---- + location / { + try_files $uri $uri/ /index.html; + } + + # ---- 安全相关响应头 ---- + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-XSS-Protection "1; mode=block" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + # ---- 禁止隐藏文件访问 ---- + location ~ /\. { + deny all; + access_log off; + log_not_found off; + } + + # ---- 禁止敏感文件访问 ---- + location ~* \.(env|log|sql|bak|swp|tmp|zip|tar|gz)$ { + deny all; + access_log off; + log_not_found off; + } +} + +# ============================================================ +# HTTPS 服务器配置（可选，需要 SSL 证书） +# +# 推荐使用 Let's Encrypt 通配符证书： +# certbot certonly --dns-xxx -d "*.preview.xiaoxiajianji.com" +# +# 启用方法：取消下方注释，并修改证书路径 +# ============================================================ +# +# server { +# listen 443 ssl http2; +# server_name *.preview.xiaoxiajianji.com; +# +# # SSL 证书配置 +# ssl_certificate /etc/letsencrypt/live/preview.xiaoxiajianji.com/fullchain.pem; +# ssl_certificate_key /etc/letsencrypt/live/preview.xiaoxiajianji.com/privkey.pem; +# +# # SSL 安全配置 +# ssl_protocols TLSv1.2 TLSv1.3; +# ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; +# ssl_prefer_server_ciphers off; +# ssl_session_cache shared:SSL:10m; +# ssl_session_timeout 10m; +# ssl_session_tickets off; +# +# # OCSP Stapling +# ssl_stapling on; +# ssl_stapling_verify on; +# +# # 根目录根据子域名动态映射 +# root /var/www/preview/$preview_pr; +# +# # 索引文件 +# index index.html; +# +# # 字符集 +# charset utf-8; +# +# # 访问日志 +# access_log /var/log/nginx/preview_ssl_access.log; +# error_log /var/log/nginx/preview_ssl_error.log warn; +# +# # 如果子域名格式不正确，返回404 +# if ($preview_pr = "") { +# return 404; +# } +# +# # 如果预览目录不存在，返回404 +# if (!-d $document_root) { +# return 404; +# } +# +# # API 反向代理到 staging 环境 +# location /api/ { +# proxy_pass https://staging-api.xiaoxiajianji.com/api/; +# proxy_http_version 1.1; +# proxy_set_header Host staging-api.xiaoxiajianji.com; +# proxy_set_header X-Real-IP $remote_addr; +# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +# proxy_set_header X-Forwarded-Proto $scheme; +# proxy_set_header X-Forwarded-Host $host; +# proxy_connect_timeout 30s; +# proxy_send_timeout 60s; +# proxy_read_timeout 60s; +# proxy_buffering on; +# proxy_buffer_size 4k; +# proxy_buffers 8 4k; +# } +# +# # 静态资源缓存 +# location /assets/ { +# expires 7d; +# add_header Cache-Control "public, max-age=604800, immutable"; +# try_files $uri =404; +# } +# +# # SPA 路由支持 +# location / { +# try_files $uri $uri/ /index.html; +# } +# +# # 安全相关响应头 +# add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; +# add_header X-Frame-Options "SAMEORIGIN" always; +# add_header X-Content-Type-Options "nosniff" always; +# add_header X-XSS-Protection "1; mode=block" always; +# add_header Referrer-Policy "strict-origin-when-cross-origin" always; +# +# # 禁止隐藏文件访问 +# location ~ /\. { +# deny all; +# access_log off; +# log_not_found off; +# } +# +# # 禁止敏感文件访问 +# location ~* \.(env|log|sql|bak|swp|tmp|zip|tar|gz)$ { +# deny all; +# access_log off; +# log_not_found off; +# } +# } diff --git a/scripts/ci/run_integration_tests.sh b/scripts/ci/run_integration_tests.sh new file mode 100755 index 000000000..49aa82ad8 --- /dev/null +++ b/scripts/ci/run_integration_tests.sh @@ -0,0 +1,342 @@ +#!/bin/bash +# CI Integration Tests Job 主脚本 +# 包含：依赖安装、ffmpeg安装、Redis启动、PG启动、迁移、测试、清理、覆盖率 +# 支持 pytest-xdist 并行执行：每个 worker 使用独立数据库，预期加速 2-4 倍 +set -eu + +echo "=== CI Integration Tests 开始 ===" + +# --- 安装依赖 --- +echo "" +echo "=== 安装 Python 依赖 ===" +# pip install 带重试（网络不稳定时自动重试） +for i in 1 2 3; do + python3 -m pip install -q -r requirements-base.txt && break + echo "pip install requirements-base.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 +done +for i in 1 2 3; do + python3 -m pip install -q -r requirements.txt && break + echo "pip install requirements.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 +done +for i in 1 2 3; do + python3 -m pip install -q -r requirements-dev.txt && break + echo "pip install requirements-dev.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 +done +for i in 1 2 3; do + python3 -m pip install -q pytest-rerunfailures pytest-xdist && break + echo "pip install pytest-rerunfailures/pytest-xdist 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 +done +pytest --version +echo "pytest-xdist: $(python3 -c "import xdist; print(xdist.__version__)" 2>/dev/null || echo 'not installed')" + +# --- 安装 ffmpeg --- +echo "" +echo "=== 安装 ffmpeg ===" +bash scripts/ci/step_install_ffmpeg.sh + +# --- DooD模式检测：确定宿主机访问地址 --- +# DooD模式下，docker run启动的容器跑在宿主机Docker上 +# 需要用宿主机IP访问映射端口 +# 检测策略：host.docker.internal -> docker0桥接IP -> 容器IP直连 -> 默认网关 -> 127.0.0.1 +detect_docker_host() { + local test_port="${1:-5432}" + + # 候选IP列表 + local candidates=() + + # 1. host.docker.internal（runner配置了--add-host时可用） + if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then + candidates+=("host.docker.internal") + fi + + # 2. docker0 桥接网关 (172.17.0.1) + candidates+=("172.17.0.1") + + # 3. 默认网关（容器网络的网关即宿主机） + local gw="" + gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1) + if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then + candidates+=("$gw") + fi + + # 4. 宿主机可能的IP：容器同网段的.1或.254 + local my_ip="" + my_ip=$(hostname -I 2>/dev/null | awk '{print $1}') + if [ -n "$my_ip" ]; then + # 尝试同网段的常见宿主机IP + local subnet=$(echo "$my_ip" | cut -d. -f1-3) + candidates+=("${subnet}.1") + candidates+=("${subnet}.254") + fi + + # 5. 127.0.0.1 最后尝试 + candidates+=("127.0.0.1") + + # 测试每个候选IP + for candidate in "${candidates[@]}"; do + if python3 -c " +import socket +s = socket.socket() +s.settimeout(2) +try: + s.connect(('$candidate', $test_port)) + s.close() + print('ok') +except: + pass +" 2>/dev/null | grep -q ok; then + echo "$candidate" + return 0 + fi + done + + # 都失败则返回127.0.0.1 + echo "127.0.0.1" + return 1 +} + +# 获取宿主机IP（先尝试用共享PG端口5433测试，再回退到其他端口） +if [ -S /var/run/docker.sock ]; then + # 先用共享PG端口5433探测 + DOCKER_HOST_IP=$(detect_docker_host 5433) + if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then + # 如果共享PG端口探测失败，说明不在DooD或共享PG不可用，再试其他端口 + DOCKER_HOST_IP=$(detect_docker_host 22) + fi + echo "检测到DooD模式（/var/run/docker.sock已挂载），宿主机地址: $DOCKER_HOST_IP" +else + DOCKER_HOST_IP="127.0.0.1" + echo "非DooD模式，使用 127.0.0.1" +fi +PG_HOST="$DOCKER_HOST_IP" +REDIS_HOST="$DOCKER_HOST_IP" +echo "PG host: $PG_HOST, Redis host: $REDIS_HOST" + +# --- 指数退避TCP连接检查函数 --- +# 用法: wait_tcp_ready host port max_attempts +wait_tcp_ready() { + local host="$1" + local port="$2" + local max_attempts="${3:-5}" + local delay=1 + local attempt=1 + while [ "$attempt" -le "$max_attempts" ]; do + if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then + return 0 + fi + echo "TCP连接尝试 $attempt/$max_attempts 失败，${delay}s后重试..." + sleep "$delay" + delay=$((delay * 2)) + attempt=$((attempt + 1)) + done + return 1 +} + +# --- 启动 Redis --- +echo "" +echo "=== 启动 Redis ===" +REDIS_CONTAINER="ci-redis-${GITHUB_RUN_ID:-$$}" +docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true +docker run -d --name "$REDIS_CONTAINER" \ + -P \ + --health-cmd "redis-cli ping" \ + --health-interval 2s \ + --health-timeout 2s \ + --health-retries 10 \ + redis:7-alpine +REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | cut -d: -f2) +echo "Redis port: $REDIS_PORT" +export REDIS_URL="redis://${REDIS_HOST}:${REDIS_PORT}/0" + +# 等待容器健康 +for i in $(seq 1 15); do + if docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" 2>/dev/null | grep -q healthy; then + echo "Redis container is ready on port $REDIS_PORT" + break + fi + echo "Waiting for Redis container health... ($i/15)" + sleep 2 +done +docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" | grep -q healthy + +# TCP连通性检查（指数退避） +echo "验证Redis TCP连通性 ($REDIS_HOST:$REDIS_PORT)..." +wait_tcp_ready "$REDIS_HOST" "$REDIS_PORT" 5 +echo "TCP connectivity to Redis confirmed on port $REDIS_PORT" + +# --- 启动/连接 PostgreSQL --- +echo "" +echo "=== 准备 PostgreSQL ===" +USE_SHARED_PG="${CI_USE_SHARED_PG:-false}" +CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}" + +if [ "$USE_SHARED_PG" = "true" ]; then + # 使用常驻共享PG实例 + echo "使用常驻共享PG实例（CI_USE_SHARED_PG=true）" + SHARED_PG_HOST="$PG_HOST" + SHARED_PG_PORT="5433" + SHARED_PG_USER="postgres" + SHARED_PG_PASSWORD="ci_pg_2026!" + + echo "等待共享PG连接就绪..." + wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5 + + # 创建主数据库（xdist 模式下各 worker 会创建自己的数据库，主库作为 fallback） + echo "创建主测试数据库: $CI_DB_NAME" + PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c " +import psycopg2 +conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres') +conn.autocommit = True +cur = conn.cursor() +cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)') +cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"') +cur.close() +conn.close() +" + export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}" + echo "✅ 共享PG数据库已创建: $CI_DB_NAME" + PG_CONTAINER="" +else + # 使用临时PG容器 + echo "使用临时PG容器模式" + PG_CONTAINER="ci-pg-${GITHUB_RUN_ID:-$$}" + docker rm -f "$PG_CONTAINER" 2>/dev/null || true + docker run -d --name "$PG_CONTAINER" \ + --shm-size=256m \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=xiaoxia_saas \ + -P \ + --health-cmd "pg_isready -U postgres" \ + --health-interval 5s \ + --health-timeout 5s \ + --health-retries 12 \ + postgres:16 + PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2) + echo "PostgreSQL port: $PG_PORT" + export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas" + + # 等待容器健康 + for i in $(seq 1 30); do + if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then + echo "PostgreSQL container is ready on port $PG_PORT" + break + fi + echo "Waiting for PostgreSQL container health... ($i/30)" + sleep 2 + done + docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy + + # TCP连通性检查（指数退避） + echo "验证PostgreSQL TCP连通性 ($PG_HOST:$PG_PORT)..." + wait_tcp_ready "$PG_HOST" "$PG_PORT" 5 + echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT" +fi + +# --- 执行迁移（主数据库，xdist worker 会各自创建自己的库并迁移） --- +echo "" +echo "=== 执行 Alembic 迁移（主数据库） ===" +PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head +echo "✅ 迁移完成" + +# --- 运行集成测试（pytest-xdist 并行） --- +echo "" +echo "=== 运行集成测试（pytest-xdist 并行模式） ===" +echo "CPU 核数: $(nproc 2>/dev/null || echo 'unknown')" + +# 集成测试使用 pytest-xdist 并行加速（coverage 由单元测试负责，并行模式下 coverage 不稳定） +# -n auto: 自动使用 CPU 核数（DooD模式下加--maxprocesses=4防止OOM +# --dist loadfile: 同一测试文件分配到同一 worker（共享 fixture 更高效） +# --maxfail=1: 遇到失败停止调度新测试（并行模式下等价于 -x） +PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration \ + -q --timeout=60 --maxfail=1 --reruns 3 --reruns-delay 5 \ + -m "not performance" \ + -n auto --maxprocesses=4 --dist loadfile \ + -p no:cacheprovider + +echo "✅ 集成测试通过" + +# --- API 性能基线测试（仅告警，串行执行） --- +echo "" +echo "=== API 性能基线测试（仅告警） ===" +set +e +PERF_OUTPUT=$(mktemp) +# 性能测试单独串行运行（不参与并行，避免资源竞争影响测量结果） +PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration/test_api_performance.py \ + -v --timeout=120 -p no:cacheprovider 2>&1 | tee "$PERF_OUTPUT" \ + --reruns 3 \ + --reruns-delay=10 +echo "" +echo "=== 性能测试摘要 ===" +grep "PERF_STATS:" "$PERF_OUTPUT" || echo "PERF_STATS: 未找到统计数据" +grep "PERF_RESULT:" "$PERF_OUTPUT" || echo "PERF_RESULT: 未找到详细结果" +TOTAL=$(grep -c "PERF_RESULT:" "$PERF_OUTPUT" || echo 0) +PASSED=$(grep "PERF_RESULT: PASS" "$PERF_OUTPUT" | wc -l) +FAILED=$(grep "PERF_RESULT: FAIL" "$PERF_OUTPUT" | wc -l) +echo "" +echo "性能测试结果: $PASSED/$TOTAL 通过, $FAILED 未达标" +if [ "$FAILED" -gt 0 ]; then + echo "" + echo "⚠️ 警告: $FAILED 个接口性能未达标" +fi +rm -f "$PERF_OUTPUT" +set -e + +# --- 清理 --- +echo "" +echo "=== 清理 ===" +if [ "$USE_SHARED_PG" = "true" ]; then + # 清理共享PG上的测试数据库（主库 + 可能残留的 worker 库） + echo "清理共享PG测试数据库..." + + # 清理所有以 CI_DB_NAME 开头的数据库（主库 + worker 库） + PGPASSWORD="${SHARED_PG_PASSWORD}" python3 -c " +import psycopg2 +conn = psycopg2.connect(host='${SHARED_PG_HOST}', port=${SHARED_PG_PORT}, user='${SHARED_PG_USER}', password='${SHARED_PG_PASSWORD}', dbname='postgres') +conn.autocommit = True +cur = conn.cursor() + +# 查找所有需要清理的数据库（主库 + worker 库） +cur.execute(\"SELECT datname FROM pg_database WHERE datname LIKE '$CI_DB_NAME%'\") +dbs = [row[0] for row in cur.fetchall()] + +for db in dbs: + try: + # 强制断开所有连接 + cur.execute(f\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{db}' AND pid <> pg_backend_pid()\") + cur.execute(f'DROP DATABASE IF EXISTS \"{db}\" WITH (FORCE)') + print(f' 已清理: {db}') + except Exception as e: + print(f' 警告: 清理 {db} 失败: {e}') + +cur.close() +conn.close() +" 2>/dev/null || echo "WARN: 数据库清理失败（可能已被清理）" + echo "✅ 共享PG数据库已清理" +else + # 清理临时PG容器 + docker rm -f "$PG_CONTAINER" 2>/dev/null || true + echo "✅ PG容器已清理" +fi + +# 清理Redis容器 +docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true +echo "✅ Redis容器已清理" + +# --- 覆盖率汇总 --- +echo "" +echo "=== 覆盖率汇总 ===" +set +e +python3 scripts/ci_coverage_summary.py +set -e + +echo "" +echo "=== CI Integration Tests 全部通过 ✅ ===" diff --git a/scripts/ci/run_unit_tests.sh b/scripts/ci/run_unit_tests.sh new file mode 100755 index 000000000..ae3dc2e4b --- /dev/null +++ b/scripts/ci/run_unit_tests.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# CI Unit Tests Job 主脚本 +# 包含：依赖安装、增量测试选择、覆盖率测试、diff覆盖率门禁 +set -eu + +JOB_NAME="${1:-Unit Tests}" + +echo "=== CI Unit Tests 开始 ===" + +# --- 配置 pip 国内源（加速下载，减少网络失败）--- +python3 -m pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ +python3 -m pip config set global.timeout 120 +python3 -m pip config set global.retries 5 + +# --- 安装依赖 --- +echo "" +echo "=== 安装 Python 依赖 ===" +# pip install 带重试（网络不稳定时自动重试） +for i in 1 2 3; do + python3 -m pip install -q -r requirements-base.txt && break + echo "pip install requirements-base.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 +done +for i in 1 2 3; do + python3 -m pip install -q -r requirements.txt && break + echo "pip install requirements.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 +done +for i in 1 2 3; do + python3 -m pip install -q -r requirements-worker.txt && break + echo "pip install requirements-worker.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 +done +for i in 1 2 3; do + python3 -m pip install -q -r requirements-dev.txt && break + echo "pip install requirements-dev.txt 失败，重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 +done +pytest --version + +# 双保险：确保numpy已安装 +python3 -m pip install -q numpy==1.26.4 || true + +# --- 增量测试选择（仅PR） --- +UNIT_TEST_MODE="full" +SELECTED_TEST_FILES="tests/unit" + +if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_TOKEN:-}" ]; then + echo "" + echo "=== 增量测试选择 ===" + PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') + API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300" + CHANGED_FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin) if f['status'] != 'removed']") + echo "改动文件数: $(echo "$CHANGED_FILES" | grep -c . || echo 0)" + set +e + CHANGED_FILES="$CHANGED_FILES" \ + SELECTED_TESTS_OUTPUT=/tmp/selected_tests.txt \ + python3 scripts/ci/select_unit_tests.py + SELECT_EXIT=$? + set -e + if [ $SELECT_EXIT -eq 0 ]; then + UNIT_TEST_MODE="incremental" + TEST_FILES=$(cat /tmp/selected_tests.txt | tr '\n' ' ') + SELECTED_TEST_FILES="$TEST_FILES" + echo "增量模式: $(cat /tmp/selected_tests.txt | wc -l) 个测试文件" + else + echo "全量模式" + fi +fi + +# --- 运行单元测试 + 覆盖率 --- +echo "" +echo "=== 运行单元测试 (模式: $UNIT_TEST_MODE) ===" + +if [ "$UNIT_TEST_MODE" = "incremental" ]; then + echo "=== 增量测试模式 ===" + PYTHONPATH="$PWD/apps/api:$PWD/apps/worker:$PWD/packages:$PWD" python3 -m coverage run \ + --source=apps/api/app,apps/worker/worker_app,packages \ + --omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \ + --branch \ + -m pytest $SELECTED_TEST_FILES -q + python3 -m coverage report --show-missing + python3 -m coverage xml -o coverage.xml + python3 -m coverage report --fail-under=10 > /dev/null || true +else + PYTHONPATH="$PWD/apps/api:$PWD/apps/worker:$PWD/packages:$PWD" python3 -m coverage run \ + --source=apps/api/app,apps/worker/worker_app,packages \ + --omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \ + --branch \ + -m pytest tests/unit -q + python3 -m coverage report --show-missing + python3 -m coverage xml -o coverage.xml + python3 -m coverage report --fail-under=65 > /dev/null || true # 全量覆盖率仅作参考，不阻塞合并 +fi + +# --- Diff 覆盖率检查（仅PR） --- +if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_TOKEN:-}" ]; then + echo "" + echo "=== Diff 覆盖率检查 ===" + BASE_BRANCH="${GITHUB_BASE_REF:-develop}" + echo "Base branch: $BASE_BRANCH" + + PR_CODE_DIR="/tmp/pr-code-$$" + mkdir -p "$PR_CODE_DIR" + # 备份PR代码（含coverage.xml，diff-cover需要用到 + find . -maxdepth 1 -mindepth 1 ! -name 'diff_coverage.html' -exec cp -r {} "$PR_CODE_DIR/" \; + rm -rf .git + git init > /dev/null 2>&1 + git remote add origin https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas.git > /dev/null 2>&1 + git config user.email "ci@local" + git config user.name "CI" + git fetch origin "$BASE_BRANCH" --depth=200 + # 先清理工作目录，避免未跟踪文件导致checkout失败 + find . -mindepth 1 -maxdepth 1 ! -name '.git' -exec rm -rf {} + + git checkout -b ci-pr-branch "origin/$BASE_BRANCH" > /dev/null 2>&1 + # 清除base分支源码，用PR代码覆盖 + find . -mindepth 1 -maxdepth 1 ! -name '.git' -exec rm -rf {} + + cp -r "$PR_CODE_DIR"/. . + rm -rf "$PR_CODE_DIR" + git add -A > /dev/null 2>&1 + git commit -m "ci-tmp" > /dev/null 2>&1 + + if [ "$UNIT_TEST_MODE" = "incremental" ]; then + THRESHOLD=40 + echo "增量测试模式，增量覆盖率门槛: ${THRESHOLD}%" + else + THRESHOLD=60 + echo "全量测试模式，增量覆盖率门槛: ${THRESHOLD}%" + fi + + set +e + python3 -m diff_cover.diff_cover_tool coverage.xml \ + --compare-branch="origin/$BASE_BRANCH" \ + --fail-under=$THRESHOLD \ + --html-report diff_coverage.html \ + 2>&1 + DIFF_EXIT=$? + set -e + if [ $DIFF_EXIT -ne 0 ]; then + echo "" + echo "❌ 增量覆盖率未达到门槛 (${THRESHOLD}%)" + echo " 请为改动的代码添加单元测试后再提交" + echo "" + echo "=== 覆盖率报告 ===" + python3 -m diff_cover.diff_cover_tool coverage.xml \ + --compare-branch="origin/$BASE_BRANCH" 2>&1 | tail -30 + exit 1 + fi + echo "✅ 增量覆盖率达标" +fi + +echo "" +echo "=== CI Unit Tests 全部通过 ✅ ===" diff --git a/scripts/ci/run_validate.sh b/scripts/ci/run_validate.sh new file mode 100755 index 000000000..050a82b94 --- /dev/null +++ b/scripts/ci/run_validate.sh @@ -0,0 +1,653 @@ +#!/bin/bash +# CI Validate Job 主脚本：并行化代码质量检查 +# 将 8 项检查分为 2 组并行执行，预计耗时从 ~1.8min 降至 ~1min +# +# 并行分组： +# Group A（独立并行）: +# A1: Secret detection (detect-secrets) +# A2: Code quality checks (black/isort/ruff/compileall) +# A3: Mypy type check +# A4: Advisory checks (bandit + pip-audit + vulture + release scripts syntax) +# Group B（PG 依赖，独立并行）: +# B1: Alembic migrations validation（需要 PG） +# +# 所有子任务同时启动，最后汇总结果。 +set -eu + +echo "=== CI Validate: 并行化代码质量检查 ===" +echo "" + +# ============================================================ +# 配置 +# ============================================================ +LOG_DIR="/tmp/validate_logs" +rm -rf "$LOG_DIR" +mkdir -p "$LOG_DIR" + +# 子任务结果文件（每个记录 exit code） +RESULT_FILE="$LOG_DIR/results.json" +echo '{}' > "$RESULT_FILE" + +# ============================================================ +# 工具函数 +# ============================================================ + +# 记录子任务结果 +# 用法: record_result +record_result() { + local name="$1" + local exit_code="$2" + local blocking="$3" # "yes" or "no" + # 写入独立文件，避免并发写 JSON 冲突 + echo "${exit_code}" > "$LOG_DIR/exit_${name}" + echo "${blocking}" > "$LOG_DIR/blocking_${name}" +} + +# ============================================================ +# 子任务定义（每个子任务输出写入独立日志文件） +# ============================================================ + +# --- A1: Secret detection --- +task_secret_detection() { + local log="$LOG_DIR/task_secret_detection.log" + exec > "$log" 2>&1 + set +e + + echo "=== [A1] Secret detection (detect-secrets) ===" + python3 -m pip install -q detect-secrets + detect-secrets --version + + detect-secrets scan \ + --all-files \ + --exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \ + --exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \ + --exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \ + --disable-plugin Base64HighEntropyString \ + --disable-plugin HexHighEntropyString \ + --disable-plugin BasicAuthDetector \ + --disable-plugin KeywordDetector \ + --disable-plugin IPPublicDetector \ + > /tmp/secrets-scan.json 2>&1 + + FOUND=$(python3 -c " +import json +try: + with open('/tmp/secrets-scan.json') as f: + data = json.load(f) + results = data.get('results', {}) + total = sum(len(v) for v in results.values()) + print(total) +except Exception: + print('error') +") + + echo "Secrets detected: $FOUND" + local exit_code=0 + if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then + echo "" + echo "=== Secret details ===" + python3 -c " +import json +with open('/tmp/secrets-scan.json') as f: + data = json.load(f) +for fpath, items in data.get('results', {}).items(): + for item in items: + line = item.get('line_number', '?') + stype = item.get('type', '?') + hashed = item.get('hashed_secret', '')[:16] + print(f' {fpath}:{line} [{stype}] {hashed}...') +" + echo "" + echo "ERROR: Potential secrets detected in code!" + exit_code=1 + else + echo "✅ Secret scan passed" + fi + + record_result "secret_detection" "$exit_code" "yes" + exit $exit_code +} + +# --- A2: Code quality checks --- +task_code_quality() { + local log="$LOG_DIR/task_code_quality.log" + exec > "$log" 2>&1 + set +e + + echo "=== [A2] Code quality checks (black/isort/ruff/compileall) ===" + + # --- 增量/全量模式判断 --- + local SCAN_MODE="full" + local CHANGED_PY_FILES="" + + if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then + PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') + API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100" + RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL") + HTTP_CODE=$(echo "$RESPONSE" | tail -n1) + BODY=$(echo "$RESPONSE" | sed '$d') + if [ "$HTTP_CODE" = "200" ]; then + CHANGED_PY_FILES=$(echo "$BODY" | python3 -c " +import json, sys +try: + files = json.load(sys.stdin) + py_files = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] != 'removed'] + print(' '.join(py_files)) +except Exception: + print('') +") + if [ -n "$CHANGED_PY_FILES" ]; then + SCAN_MODE="incremental" + echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed" + else + SCAN_MODE="skip_py" + echo "No Python files changed in this PR" + fi + else + echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan" + fi + else + echo "Full scan mode (not a PR event)" + fi + + local exit_code=0 + + if [ "$SCAN_MODE" = "incremental" ]; then + # 防御性过滤：磁盘上不存在的文件（已删除文件）不参与检查 + local EXISTING_PY_FILES="" + for f in $CHANGED_PY_FILES; do + if [ -f "$f" ]; then + if [ -z "$EXISTING_PY_FILES" ]; then + EXISTING_PY_FILES="$f" + else + EXISTING_PY_FILES="$EXISTING_PY_FILES $f" + fi + fi + done + CHANGED_PY_FILES="$EXISTING_PY_FILES" + + python3 -m compileall -q $CHANGED_PY_FILES || exit_code=$? + if [ $exit_code -eq 0 ]; then + python3 -m black --check --fast $CHANGED_PY_FILES || exit_code=$? + fi + if [ $exit_code -eq 0 ]; then + python3 -m isort --check-only $CHANGED_PY_FILES || exit_code=$? + fi + if [ $exit_code -eq 0 ]; then + local RUFF_FILES + RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | grep -v '^$' | xargs) + if [ -n "$RUFF_FILES" ]; then + python3 -m ruff check $RUFF_FILES --statistics || exit_code=$? + else + echo "No ruff-checkable files changed, skipping" + fi + fi + elif [ "$SCAN_MODE" = "skip_py" ]; then + echo "No Python files changed - skipping Python lint checks" + else + echo "Full scan mode" + python3 -m compileall -q alembic apps packages tests scripts || exit_code=$? + if [ $exit_code -eq 0 ]; then + python3 -m black --check --fast alembic apps packages tests scripts || exit_code=$? + fi + if [ $exit_code -eq 0 ]; then + python3 -m isort --check-only alembic apps packages tests scripts || exit_code=$? + fi + if [ $exit_code -eq 0 ]; then + python3 -m ruff check apps packages tests --statistics || exit_code=$? + fi + fi + + if [ $exit_code -eq 0 ]; then + echo "✅ Code quality checks passed" + else + echo "❌ Code quality checks FAILED" + fi + + record_result "code_quality" "$exit_code" "yes" + exit $exit_code +} + +# --- A3: Mypy type check --- +task_mypy() { + local log="$LOG_DIR/task_mypy.log" + exec > "$log" 2>&1 + set +e + + echo "=== [A3] Type check (mypy) ===" + bash scripts/ci/mypy_check.sh + local exit_code=$? + + if [ $exit_code -eq 0 ]; then + echo "✅ Mypy type check passed" + else + echo "❌ Mypy type check FAILED" + fi + + record_result "mypy" "$exit_code" "yes" + exit $exit_code +} + +# --- A4: Advisory checks (bandit + pip-audit + vulture + release scripts syntax) --- +task_advisory() { + local log="$LOG_DIR/task_advisory.log" + exec > "$log" 2>&1 + set +e + + # --- Bandit 安全扫描（仅告警） --- + echo "=== [A4a] Security scan (bandit, advisory only) ===" + bandit -r apps packages -q -ll + local BANDIT_EXIT=$? + if [ "$BANDIT_EXIT" -ne 0 ]; then + echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)" + else + echo "✅ Bandit security scan passed" + fi + + # --- Pip-audit 依赖漏洞扫描（仅告警） --- + echo "" + echo "=== [A4b] Python dependency vulnerability scan (pip-audit, advisory only) ===" + python3 -m pip install -q pip-audit + pip-audit --version + for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do + if [ -f "$req_file" ]; then + echo "--- Scanning $req_file ---" + pip-audit -r "$req_file" --desc on 2>&1 | head -40 || true + echo "" + fi + done + echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)" + + # --- Vulture 死代码检测（仅告警） --- + echo "" + echo "=== [A4c] Dead code detection (vulture, advisory only) ===" + python3 -m pip install -q vulture + vulture --version + echo "告警模式，不阻断CI。置信度>=90%建议尽快确认。" + echo "" + vulture apps packages scripts \ + --exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \ + --min-confidence 70 \ + 2>&1 | sort -t'(' -k2 -rn | head -80 + echo "" + echo "=== vulture scan summary ===" + echo "发现潜在死代码（可能包含框架装饰器注册的函数，为误报）" + echo "建议：定期人工审查高置信度（>=90%）条目" + + # --- Release 脚本语法校验（不阻断） --- + echo "" + echo "=== [A4d] Release scripts syntax validation ===" + local syntax_exit=0 + bash -n scripts/backup_postgres.sh || syntax_exit=$? + bash -n scripts/restore_postgres_plan.sh || syntax_exit=$? + bash -n scripts/init_production_env.sh || syntax_exit=$? + if [ $syntax_exit -eq 0 ]; then + echo "✅ Release scripts syntax OK" + else + echo "⚠️ Release scripts have syntax issues (advisory)" + fi + + # Advisory checks never block + record_result "advisory" 0 "no" + exit 0 +} + +# --- B1: Alembic migrations validation (needs PG) --- +task_alembic() { + local log="$LOG_DIR/task_alembic.log" + exec > "$log" 2>&1 + set +e + + echo "=== [B1] Alembic migrations validation ===" + + # --- DooD模式检测：确定宿主机访问地址 --- + detect_docker_host() { + local test_port="${1:-5432}" + local candidates=() + + # 1. host.docker.internal + if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then + candidates+=("host.docker.internal") + fi + + # 2. docker0 桥接网关 (172.17.0.1) + candidates+=("172.17.0.1") + + # 3. 默认网关 + local gw="" + gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1) + if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then + candidates+=("$gw") + fi + + # 4. 宿主机可能的IP + local my_ip="" + my_ip=$(hostname -I 2>/dev/null | awk '{print $1}') + if [ -n "$my_ip" ]; then + local subnet + subnet=$(echo "$my_ip" | cut -d. -f1-3) + candidates+=("${subnet}.1") + candidates+=("${subnet}.254") + fi + + # 5. 127.0.0.1 + candidates+=("127.0.0.1") + + for candidate in "${candidates[@]}"; do + if python3 -c " +import socket +s = socket.socket() +s.settimeout(2) +try: + s.connect(('$candidate', $test_port)) + s.close() + print('ok') +except: + pass +" 2>/dev/null | grep -q ok; then + echo "$candidate" + return 0 + fi + done + + echo "127.0.0.1" + return 1 + } + + # 指数退避TCP连接检查函数 + wait_tcp_ready() { + local host="$1" + local port="$2" + local max_attempts="${3:-5}" + local delay=1 + local attempt=1 + while [ "$attempt" -le "$max_attempts" ]; do + if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then + return 0 + fi + echo "TCP连接尝试 $attempt/$max_attempts 失败，${delay}s后重试..." + sleep "$delay" + delay=$((delay * 2)) + attempt=$((attempt + 1)) + done + return 1 + } + + # 获取宿主机IP + local PG_HOST + if [ -S /var/run/docker.sock ]; then + PG_HOST=$(detect_docker_host 5433) + if [ "$PG_HOST" = "127.0.0.1" ]; then + PG_HOST=$(detect_docker_host 22) + fi + echo "检测到DooD模式（/var/run/docker.sock已挂载），宿主机地址: $PG_HOST" + else + PG_HOST="127.0.0.1" + echo "非DooD模式，使用 127.0.0.1" + fi + echo "PG host: $PG_HOST" + + local USE_SHARED_PG="${CI_USE_SHARED_PG:-false}" + local exit_code=0 + + if [ "$USE_SHARED_PG" = "true" ]; then + # 使用常驻共享PG实例 + echo "使用常驻共享PG实例（CI_USE_SHARED_PG=true）" + local SHARED_PG_HOST="$PG_HOST" + local SHARED_PG_PORT="5433" + local SHARED_PG_USER="postgres" + local SHARED_PG_PASSWORD="ci_pg_2026!" + local CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}" + + echo "等待共享PG连接就绪..." + wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5 + + echo "创建测试数据库: $CI_DB_NAME" + PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c " +import psycopg2 +conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres') +conn.autocommit = True +cur = conn.cursor() +cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)') +cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"') +cur.close() +conn.close() +" || exit_code=$? + + if [ $exit_code -eq 0 ]; then + export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}" + echo "✅ 共享PG数据库已创建: $CI_DB_NAME" + + PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head || exit_code=$? + if [ $exit_code -eq 0 ]; then + echo "✅ Alembic migrations applied successfully" + fi + + # 清理数据库 + echo "清理测试数据库: $CI_DB_NAME" + PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c " +import psycopg2 +conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres') +conn.autocommit = True +cur = conn.cursor() +cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)') +cur.close() +conn.close() +" 2>/dev/null || echo "WARN: 数据库清理失败（可能已被清理）" + echo "✅ 共享PG数据库已清理" + fi + + else + # 使用临时PG容器 + echo "使用临时PG容器模式" + local PG_CONTAINER="ci-pg-validate-${GITHUB_RUN_ID:-$$}" + docker rm -f "$PG_CONTAINER" 2>/dev/null || true + docker run -d --name "$PG_CONTAINER" \ + --shm-size=256m \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=xiaoxia_saas \ + -P \ + --health-cmd "pg_isready -U postgres" \ + --health-interval 3s \ + --health-timeout 3s \ + --health-retries 20 \ + postgres:16-alpine || exit_code=$? + + if [ $exit_code -eq 0 ]; then + local PG_PORT + PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2) + echo "PostgreSQL port: $PG_PORT" + export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas" + + # 等待容器健康 + local i + for i in $(seq 1 30); do + if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then + echo "PostgreSQL container is healthy on port $PG_PORT" + break + fi + echo "Waiting for PostgreSQL container health... ($i/30)" + sleep 2 + done + + if ! docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then + echo "❌ PostgreSQL container failed health check" + exit_code=1 + else + # TCP连通性检查 + echo "验证TCP连通性 ($PG_HOST:$PG_PORT)..." + if wait_tcp_ready "$PG_HOST" "$PG_PORT" 5; then + echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT" + + # 执行迁移 + PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head || exit_code=$? + if [ $exit_code -eq 0 ]; then + echo "✅ Alembic migrations applied successfully" + fi + else + echo "❌ TCP connectivity to PostgreSQL failed" + exit_code=1 + fi + fi + + # 清理 + docker rm -f "$PG_CONTAINER" 2>/dev/null || true + fi + fi + + if [ $exit_code -eq 0 ]; then + echo "✅ Alembic migrations validation passed" + else + echo "❌ Alembic migrations validation FAILED" + fi + + record_result "alembic" "$exit_code" "yes" + exit $exit_code +} + +# ============================================================ +# 主流程：并行启动所有子任务 +# ============================================================ + +echo "启动并行检查（5 个子任务同时运行）..." +echo "" + +# 记录开始时间 +START_TIME=$(date +%s) + +# 启动所有子任务（后台运行） +task_secret_detection & +PID_A1=$! + +task_code_quality & +PID_A2=$! + +task_mypy & +PID_A3=$! + +task_advisory & +PID_A4=$! + +task_alembic & +PID_B1=$! + +echo "子任务 PID: A1=$PID_A1 A2=$PID_A2 A3=$PID_A3 A4=$PID_A4 B1=$PID_B1" +echo "" + +# 等待所有后台任务完成（不因单个失败而中断） +# 使用 set +e 临时取消 errexit +set +e +wait $PID_A1; EXIT_A1=$? +wait $PID_A2; EXIT_A2=$? +wait $PID_A3; EXIT_A3=$? +wait $PID_A4; EXIT_A4=$? +wait $PID_B1; EXIT_B1=$? +set -e + +# 计算耗时 +END_TIME=$(date +%s) +ELAPSED=$((END_TIME - START_TIME)) + +# ============================================================ +# 结果汇总 +# ============================================================ + +echo "" +echo "============================================" +echo " CI Validate 结果汇总（耗时 ${ELAPSED}s）" +echo "============================================" +echo "" + +# 定义任务信息：名称 | PID | 退出码 | 描述 | 是否阻断 +declare -A TASK_DESC +TASK_DESC[A1]="Secret detection" +TASK_DESC[A2]="Code quality (black/isort/ruff)" +TASK_DESC[A3]="Mypy type check" +TASK_DESC[A4]="Advisory (bandit/pip-audit/vulture/syntax)" +TASK_DESC[B1]="Alembic migrations" + +declare -A TASK_PID +TASK_PID[A1]=$PID_A1 +TASK_PID[A2]=$PID_A2 +TASK_PID[A3]=$PID_A3 +TASK_PID[A4]=$PID_A4 +TASK_PID[B1]=$PID_B1 + +declare -A TASK_EXIT +TASK_EXIT[A1]=$EXIT_A1 +TASK_EXIT[A2]=$EXIT_A2 +TASK_EXIT[A3]=$EXIT_A3 +TASK_EXIT[A4]=$EXIT_A4 +TASK_EXIT[B1]=$EXIT_B1 + +declare -A TASK_LOG +TASK_LOG[A1]="task_secret_detection" +TASK_LOG[A2]="task_code_quality" +TASK_LOG[A3]="task_mypy" +TASK_LOG[A4]="task_advisory" +TASK_LOG[B1]="task_alembic" + +declare -A TASK_BLOCKING +TASK_BLOCKING[A1]="yes" +TASK_BLOCKING[A2]="yes" +TASK_BLOCKING[A3]="yes" +TASK_BLOCKING[A4]="no" +TASK_BLOCKING[B1]="yes" + +OVERALL_EXIT=0 +FAILED_TASKS=() + +# 按固定顺序打印摘要 +for task_id in A1 A2 A3 A4 B1; do + local_exit=${TASK_EXIT[$task_id]} + local_desc=${TASK_DESC[$task_id]} + local_blocking=${TASK_BLOCKING[$task_id]} + + if [ "$local_exit" -eq 0 ]; then + echo " ✅ $task_id: $local_desc — PASSED" + else + if [ "$local_blocking" = "yes" ]; then + echo " ❌ $task_id: $local_desc — FAILED (blocking)" + OVERALL_EXIT=1 + FAILED_TASKS+=("$task_id") + else + echo " ⚠️ $task_id: $local_desc — FAILED (advisory, not blocking)" + # Advisory tasks don't cause overall failure + if [ "$local_blocking" = "no" ]; then + echo " → 告警类检查，不阻断流水线" + fi + fi + fi +done + +echo "" + +# 打印失败任务的完整日志 +if [ ${#FAILED_TASKS[@]} -gt 0 ]; then + echo "============================================" + echo " 失败任务详细日志" + echo "============================================" + for task_id in "${FAILED_TASKS[@]}"; do + local_log="${TASK_LOG[$task_id]}" + local_desc="${TASK_DESC[$task_id]}" + echo "" + echo "--- $task_id: $local_desc ---" + if [ -f "$LOG_DIR/${local_log}.log" ]; then + cat "$LOG_DIR/${local_log}.log" + else + echo "(日志文件不存在)" + fi + echo "" + done +fi + +# 最终结论 +echo "" +if [ $OVERALL_EXIT -eq 0 ]; then + echo "=== CI Validate: 所有检查通过 ✅ （并行耗时 ${ELAPSED}s） ===" +else + echo "=== CI Validate: 存在阻断性检查失败 ❌ （并行耗时 ${ELAPSED}s） ===" +fi + +exit $OVERALL_EXIT diff --git a/scripts/ci/runner_monitor/__init__.py b/scripts/ci/runner_monitor/__init__.py new file mode 100755 index 000000000..0675ca821 --- /dev/null +++ b/scripts/ci/runner_monitor/__init__.py @@ -0,0 +1,17 @@ +"""Runner 监控告警工具包 + +模块: + config - 配置管理（阈值、检测间隔等） + runner_status - Runner 在线状态巡检（Gitea API） + runner_metrics - 系统指标采集（SSH，后补） + alert_manager - 告警调度（阈值判断+去重+飞书通知） + snapshot - Runner 状态快照生成 +""" + +__all__ = [ + "config", + "runner_status", + "runner_metrics", + "alert_manager", + "snapshot", +] diff --git a/scripts/ci/runner_monitor/alert_manager.py b/scripts/ci/runner_monitor/alert_manager.py new file mode 100644 index 000000000..e0f3b119e --- /dev/null +++ b/scripts/ci/runner_monitor/alert_manager.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python3 +""" +告警调度器 - 阈值判断 + 去重 + 飞书通知 + +功能: + 1. 从 runner_status 和 runner_metrics 获取数据 + 2. 根据阈值判断是否触发告警 + 3. 告警去重（同一问题 30 分钟内只报一次） + 4. 飞书卡片通知（复用 chatops FeishuNotifier） + 5. 生成状态快照 JSON（供看板用） + +告警规则: + P1（严重）: + - Runner 离线超过 5 分钟 + - 磁盘使用率 > 90% + + P2（警告）: + - 磁盘使用率 > 85% + - 内存使用率 > 90% 持续 5 分钟 + - CI 队列积压 > 10 个 pending 超过 10 分钟 + +用法: + python3 scripts/ci/runner_monitor/alert_manager.py --check + python3 scripts/ci/runner_monitor/alert_manager.py --daemon # 持续运行 + python3 scripts/ci/runner_monitor/alert_manager.py --snapshot +""" + +import argparse +import json +import os +import sys +import time +from datetime import datetime, timezone + +# 复用 chatops 的飞书通知 +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_CI_DIR = os.path.dirname(_SCRIPT_DIR) +if _CI_DIR not in sys.path: + sys.path.insert(0, _CI_DIR) + +from runner_monitor import config # noqa: E402 +from runner_monitor.runner_metrics import RunnerMetricsCollector # noqa: E402 +from runner_monitor.runner_status import RunnerStatusChecker # noqa: E402 + + +class Alert: + """单条告警""" + + def __init__(self, alert_id, level, title, description, details=None, source="runner_monitor"): + self.alert_id = alert_id # 唯一标识，用于去重 + self.level = level # P1 / P2 / INFO + self.title = title + self.description = description + self.details = details or {} + self.source = source + self.timestamp = datetime.now(timezone.utc).isoformat() + + def to_dict(self): + return { + "alert_id": self.alert_id, + "level": self.level, + "title": self.title, + "description": self.description, + "details": self.details, + "source": self.source, + "timestamp": self.timestamp, + } + + +class AlertManager: + """告警调度器""" + + def __init__( + self, + status_checker=None, + metrics_collector=None, + dedupe_window=None, + ): + self.status_checker = status_checker or RunnerStatusChecker() + self.metrics = metrics_collector or RunnerMetricsCollector() + self.dedupe_window = dedupe_window or config.DEDUPE_WINDOW + + # 告警历史: {alert_id: last_triggered_timestamp} + self._alert_history = {} + # 内存持续超阈值记录: {host: first_detected_timestamp} + self._mem_high_since = {} + + # ── 告警检测 ────────────────────────────────────── + + def detect_alerts(self): + """执行所有检测规则，返回触发的告警列表 + + Returns: + list[Alert]: 新触发的告警（已去重） + """ + all_alerts = [] + + # 1. Runner 离线检测 + all_alerts.extend(self._check_runner_offline()) + + # 2. 队列积压检测 + all_alerts.extend(self._check_queue_backlog()) + + # 3. 系统指标检测（SSH，可能为空） + all_alerts.extend(self._check_system_metrics()) + + # 去重过滤 + new_alerts = [a for a in all_alerts if self._should_alert(a)] + + # 更新告警历史 + for alert in new_alerts: + self._alert_history[alert.alert_id] = time.time() + + return new_alerts + + def _check_runner_offline(self): + """检测离线 runner""" + offline = self.status_checker.get_offline_runners(offline_minutes=config.RUNNER_OFFLINE_MINUTES) + alerts = [] + + for runner in offline: + name = runner.get("name", "unknown") + runner_id = runner.get("id", "?") + alert_id = f"runner_offline_{runner_id}" + + # Gitea API 没有心跳时间，status != online 就告警（P1） + alerts.append( + Alert( + alert_id=alert_id, + level=config.P1, + title=f"Runner 离线: {name}", + description=( + f"Runner **{name}** (ID: {runner_id}) 状态为 " + f"{runner.get('status', 'unknown')}，已离线\n" + f"标签: {', '.join(label.get('name') for label in runner.get('labels', [])[:5])}" + ), + details={ + "runner_id": runner_id, + "runner_name": name, + "status": runner.get("status"), + "labels": [label.get("name") for label in runner.get("labels", [])], + }, + ) + ) + + return alerts + + def _check_queue_backlog(self): + """检测队列积压""" + backlog = self.status_checker.get_queue_backlog( + pending_threshold=config.QUEUE_PENDING_COUNT, + duration_minutes=config.QUEUE_PENDING_MINUTES, + ) + + if not backlog["is_backlogged"]: + return [] + + count = backlog["pending_count"] + age = backlog["oldest_pending_minutes"] + alert_id = f"queue_backlog_{int(age // 30)}" # 每30分钟一个新告警id + + return [ + Alert( + alert_id=alert_id, + level=config.P2, + title="CI 队列积压", + description=( + f"当前有 **{count}** 个 pending run，最老的已等待 **{age:.0f} 分钟**\n" + f"阈值: >{config.QUEUE_PENDING_COUNT}个 且 超过{config.QUEUE_PENDING_MINUTES}分钟" + ), + details={ + "pending_count": count, + "oldest_pending_minutes": age, + }, + ) + ] + + def _check_system_metrics(self): + """检测系统指标（磁盘/内存/CPU）""" + metrics_list = self.metrics.collect_all() + if not metrics_list: + return [] + + alerts = [] + now = time.time() + + for m in metrics_list: + host = m.get("host", "unknown") + if m.get("status") != "ok": + continue + + # 磁盘告警 + disk_pct = m.get("disk_percent", 0) + if disk_pct and disk_pct >= config.DISK_CRIT_PERCENT: + alerts.append( + Alert( + alert_id=f"disk_crit_{host}", + level=config.P1, + title=f"磁盘使用率严重过高: {host}", + description=( + f"服务器 **{host}** 磁盘使用率 **{disk_pct:.1f}%** (P1阈值: {config.DISK_CRIT_PERCENT}%)\n" + f"已用: {m.get('disk_used_gb', '?')}G / {m.get('disk_total_gb', '?')}G" + ), + details={"host": host, "disk_percent": disk_pct}, + ) + ) + elif disk_pct and disk_pct >= config.DISK_WARN_PERCENT: + alerts.append( + Alert( + alert_id=f"disk_warn_{host}", + level=config.P2, + title=f"磁盘使用率过高: {host}", + description=( + f"服务器 **{host}** 磁盘使用率 **{disk_pct:.1f}%** (P2阈值: {config.DISK_WARN_PERCENT}%)\n" + f"已用: {m.get('disk_used_gb', '?')}G / {m.get('disk_total_gb', '?')}G" + ), + details={"host": host, "disk_percent": disk_pct}, + ) + ) + + # 内存告警（持续 N 分钟） + mem_pct = m.get("mem_percent", 0) + mem_key = f"mem_high_{host}" + if mem_pct and mem_pct >= config.MEM_WARN_PERCENT: + if mem_key not in self._mem_high_since: + self._mem_high_since[mem_key] = now + else: + duration_min = (now - self._mem_high_since[mem_key]) / 60 + if duration_min >= config.MEM_DURATION_MINUTES: + alerts.append( + Alert( + alert_id=f"mem_warn_{host}", + level=config.P2, + title=f"内存使用率持续过高: {host}", + description=( + f"服务器 **{host}** 内存使用率 **{mem_pct:.1f}%**，" + f"已持续 **{duration_min:.0f} 分钟**\n" + f"阈值: {config.MEM_WARN_PERCENT}% 持续 {config.MEM_DURATION_MINUTES} 分钟" + ), + details={"host": host, "mem_percent": mem_pct, "duration_min": duration_min}, + ) + ) + else: + # 恢复了，清除记录 + self._mem_high_since.pop(mem_key, None) + + return alerts + + # ── 去重 ────────────────────────────────────────── + + def _should_alert(self, alert): + """判断是否应该发送告警（去重 + 等级开关）""" + # 等级开关 + if alert.level == config.P1 and not config.P1_ENABLED: + return False + if alert.level == config.P2 and not config.P2_ENABLED: + return False + + # 去重窗口 + last = self._alert_history.get(alert.alert_id, 0) + if time.time() - last < self.dedupe_window: + return False + + return True + + # ── 通知 ────────────────────────────────────────── + + def send_alerts(self, alerts): + """发送告警到飞书 + + 复用 chatops 的 FeishuNotifier，这里直接构造卡片。 + 不依赖 FeishuNotifier 实例方法，因为告警卡片格式不同。 + """ + if not alerts: + return 0 + + # 延迟导入 + # 直接用 urllib 发，走同一个 webhook + import urllib.request + + from chatops.feishu_notify import FeishuNotifier # noqa: F401 + + webhook_url = config.__dict__.get("FEISHU_WEBHOOK_URL", "") + if not webhook_url: + # 从 chatops config 拿 + from chatops import config as chatops_config + + webhook_url = chatops_config.FEISHU_WEBHOOK_URL + + if not webhook_url: + print("[WARN] 未配置飞书 webhook，跳过告警通知") + return 0 + + sent = 0 + for alert in alerts: + card = self._build_alert_card(alert) + payload = json.dumps({"msg_type": "interactive", "card": card}).encode("utf-8") + req = urllib.request.Request( + webhook_url, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + body = resp.read().decode() + result = json.loads(body) + if result.get("code", 0) == 0: + sent += 1 + print(f"[INFO] 告警已发送: [{alert.level}] {alert.title}") + else: + print(f"[WARN] 告警发送失败: {result.get('msg', body)}", file=sys.stderr) + except Exception as e: + print(f"[WARN] 告警发送异常: {e}", file=sys.stderr) + + return sent + + @staticmethod + def _build_alert_card(alert): + """构建飞书告警卡片""" + color = config.LEVEL_COLOR.get(alert.level, "blue") + emoji = config.LEVEL_EMOJI.get(alert.level, "ℹ️") + + fields = [ + { + "is_short": True, + "text": {"tag": "lark_md", "content": f"**等级**\n{alert.level}"}, + }, + { + "is_short": True, + "text": {"tag": "lark_md", "content": f"**来源**\n{alert.source}"}, + }, + { + "is_short": False, + "text": {"tag": "lark_md", "content": f"**详情**\n{alert.description}"}, + }, + ] + + return { + "header": { + "title": {"tag": "plain_text", "content": f"{emoji} Runner监控告警: {alert.title}"}, + "status": color, + }, + "elements": [ + {"tag": "div", "fields": fields}, + { + "tag": "note", + "elements": [ + { + "tag": "plain_text", + "content": f"告警ID: {alert.alert_id} | {alert.timestamp[:19].replace('T', ' ')}", + } + ], + }, + ], + } + + # ── 快照 ────────────────────────────────────────── + + def generate_snapshot(self, alerts=None): + """生成完整的监控快照 + + Returns: + dict: 快照数据 + """ + status_result = self.status_checker.run_full_check() + metrics = self.metrics.collect_all() + + if alerts is None: + alerts = self.detect_alerts() + + snapshot = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "runner_summary": status_result["runner_summary"], + "offline_runners": status_result["offline_runners"], + "queue_backlog": status_result["queue_backlog"], + "system_metrics": metrics, + "active_alerts": [a.to_dict() for a in alerts], + "alert_history_count": len(self._alert_history), + } + + return snapshot + + def save_snapshot(self, output_dir=None): + """保存快照到文件""" + from runner_monitor.runner_status import RunnerStatusChecker as RSC + + snapshot = self.generate_snapshot() + + if output_dir is None: + output_dir = config.OUTPUT_DIR + + os.makedirs(output_dir, exist_ok=True) + ts = time.strftime("%Y%m%d_%H%M%S") + filepath = os.path.join(output_dir, f"monitor_snapshot_{ts}.json") + + with open(filepath, "w", encoding="utf-8") as f: + json.dump(snapshot, f, indent=2, ensure_ascii=False) + + # 清理旧快照 + RSC._cleanup_old_snapshots(output_dir, keep=24) + + return filepath + + # ── 单次检查 ────────────────────────────────────── + + def run_once(self): + """执行一次完整检查 + 告警 + 快照 + + Returns: + dict: {alerts_count, sent_count, snapshot_path} + """ + alerts = self.detect_alerts() + sent = self.send_alerts(alerts) + snapshot_path = self.save_snapshot() + + return { + "alerts_detected": len(alerts), + "alerts_sent": sent, + "snapshot_path": snapshot_path, + "alerts": [a.to_dict() for a in alerts], + } + + +# ── CLI 入口 ────────────────────────────────────────── + + +def main(): + parser = argparse.ArgumentParser(description="Runner 监控告警调度器") + parser.add_argument("--check", action="store_true", help="执行一次检查") + parser.add_argument("--snapshot", action="store_true", help="生成快照") + parser.add_argument("--daemon", action="store_true", help="持续运行模式") + parser.add_argument("--dry-run", action="store_true", help="只检测不发通知") + parser.add_argument("--interval", type=int, help="检测间隔（秒），覆盖环境变量") + + args = parser.parse_args() + + if args.interval: + config.CHECK_INTERVAL = args.interval + + manager = AlertManager() + + if args.daemon: + print(f"[INFO] Runner 监控告警服务启动，检测间隔 {config.CHECK_INTERVAL} 秒") + print(f"[INFO] P1告警: {'开启' if config.P1_ENABLED else '关闭'}") + print(f"[INFO] P2告警: {'开启' if config.P2_ENABLED else '关闭'}") + print(f"[INFO] 去重窗口: {config.DEDUPE_WINDOW} 秒") + + while True: + try: + result = ( + manager.run_once() + if not args.dry_run + else { + "alerts_detected": len(manager.detect_alerts()), + "alerts_sent": 0, + } + ) + now = time.strftime("%Y-%m-%d %H:%M:%S") + print( + f"[{now}] 检测完成 - " + f"发现 {result['alerts_detected']} 个告警, " + f"发送 {result['alerts_sent']} 条通知" + ) + except Exception as e: + print(f"[ERROR] 检测异常: {e}", file=sys.stderr) + + time.sleep(config.CHECK_INTERVAL) + + elif args.snapshot: + path = manager.save_snapshot() + print(f"快照已保存: {path}") + + elif args.check or args.dry_run: + if args.dry_run: + alerts = manager.detect_alerts() + print(f"检测到 {len(alerts)} 个告警（dry-run，不发送）:") + for a in alerts: + print(f" [{a.level}] {a.title}") + print(f" {a.description[:100]}") + else: + result = manager.run_once() + print(json.dumps(result, indent=2, ensure_ascii=False)) + else: + parser.print_help() + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/runner_monitor/config.py b/scripts/ci/runner_monitor/config.py new file mode 100755 index 000000000..4c041a613 --- /dev/null +++ b/scripts/ci/runner_monitor/config.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +""" +Runner 监控告警配置 - 全部走环境变量，不硬编码 + +环境变量: + GITEA_URL / GITEA_REPO / GITEA_TOKEN / GITEA_USERNAME / GITEA_PASSWORD + （复用 chatops 的 Gitea 配置） + + FEISHU_WEBHOOK_URL + 飞书 webhook 地址（复用 chatops） + + ALERT_RUNNER_OFFLINE_MINUTES + Runner 离线超过多少分钟触发告警，默认 5 分钟（P1） + + ALERT_DISK_WARN_PERCENT 磁盘告警阈值 P2，默认 85 + ALERT_DISK_CRIT_PERCENT 磁盘告警阈值 P1，默认 90 + + ALERT_MEM_WARN_PERCENT 内存告警阈值 P2，默认 90 + ALERT_MEM_DURATION_MINUTES 内存持续超阈值多久告警，默认 5 分钟 + + ALERT_QUEUE_PENDING_COUNT CI 队列积压数量阈值，默认 10 + ALERT_QUEUE_PENDING_MINUTES CI 队列积压持续时间阈值（分钟），默认 10 + + ALERT_CHECK_INTERVAL 检测间隔（秒），默认 60 + ALERT_DEDUPE_WINDOW 同一告警去重窗口（秒），默认 1800（30分钟） + + ALERT_P1_ENABLED P1 告警开关，默认 true + ALERT_P2_ENABLED P2 告警开关，默认 true + + RUNNER_MONITOR_OUTPUT_DIR 状态快照输出目录，默认 scripts/ci/runner_monitor/snapshots +""" + +import os + +# ── Runner 离线告警 ─────────────────────────────────── +RUNNER_OFFLINE_MINUTES = int(os.environ.get("ALERT_RUNNER_OFFLINE_MINUTES", "5")) + +# ── 磁盘告警 ────────────────────────────────────────── +DISK_WARN_PERCENT = int(os.environ.get("ALERT_DISK_WARN_PERCENT", "85")) +DISK_CRIT_PERCENT = int(os.environ.get("ALERT_DISK_CRIT_PERCENT", "90")) + +# ── 内存告警 ────────────────────────────────────────── +MEM_WARN_PERCENT = int(os.environ.get("ALERT_MEM_WARN_PERCENT", "90")) +MEM_DURATION_MINUTES = int(os.environ.get("ALERT_MEM_DURATION_MINUTES", "5")) + +# ── 队列积压告警 ────────────────────────────────────── +QUEUE_PENDING_COUNT = int(os.environ.get("ALERT_QUEUE_PENDING_COUNT", "10")) +QUEUE_PENDING_MINUTES = int(os.environ.get("ALERT_QUEUE_PENDING_MINUTES", "10")) + +# ── 检测与去重 ──────────────────────────────────────── +CHECK_INTERVAL = int(os.environ.get("ALERT_CHECK_INTERVAL", "60")) +DEDUPE_WINDOW = int(os.environ.get("ALERT_DEDUPE_WINDOW", "1800")) + +# ── 告警等级开关 ────────────────────────────────────── +P1_ENABLED = os.environ.get("ALERT_P1_ENABLED", "true").lower() == "true" +P2_ENABLED = os.environ.get("ALERT_P2_ENABLED", "true").lower() == "true" + +# ── 输出目录 ────────────────────────────────────────── +OUTPUT_DIR = os.environ.get( + "RUNNER_MONITOR_OUTPUT_DIR", + "scripts/ci/runner_monitor/snapshots", +) + +# ── SSH 配置（后补） ───────────────────────────────── +# SSH 主机列表，格式: user@host:port,user@host2:port +SSH_HOSTS = [h.strip() for h in os.environ.get("RUNNER_SSH_HOSTS", "").split(",") if h.strip()] +SSH_KEY_PATH = os.environ.get("RUNNER_SSH_KEY_PATH", "") +SSH_USER = os.environ.get("RUNNER_SSH_USER", "root") + + +# ── 告警等级常量 ────────────────────────────────────── +P1 = "P1" +P2 = "P2" +INFO = "INFO" + +# 等级对应飞书卡片颜色 +LEVEL_COLOR = { + P1: "red", + P2: "orange", + INFO: "blue", +} + +LEVEL_EMOJI = { + P1: "🔥", + P2: "⚠️", + INFO: "ℹ️", +} diff --git a/scripts/ci/runner_monitor/runner_metrics.py b/scripts/ci/runner_monitor/runner_metrics.py new file mode 100755 index 000000000..226056773 --- /dev/null +++ b/scripts/ci/runner_monitor/runner_metrics.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +""" +Runner 系统指标采集 - CPU/内存/磁盘（通过 SSH 连接构建服务器） + +⚠️ 第一版：骨架 + 接口定义，SSH 实装后续补充 +原因：跨机器 SSH 需要密钥管理和网络权限，先把监控框架搭好。 + +接口约定（与 alert_manager 对接）: + metrics = RunnerMetricsCollector().collect_all() + # 返回: [{"host": "...", "cpu_percent": 75.2, "mem_percent": 80.1, "disk_percent": 65.0, + # "disk_total_gb": 500, "disk_used_gb": 325, "status": "ok"}, ...] + +当 SSH 不可用时，返回空列表，不影响其他监控功能。 +""" + +from runner_monitor import config + + +class RunnerMetricsCollector: + """Runner 系统指标采集器 + + 第一版：返回空数据（SSH 实装待后续迭代） + 接口已定义好，alert_manager 直接消费。 + """ + + def __init__(self, ssh_hosts=None, ssh_key_path=None, ssh_user=None): + self.ssh_hosts = ssh_hosts or config.SSH_HOSTS + self.ssh_key_path = ssh_key_path or config.SSH_KEY_PATH + self.ssh_user = ssh_user or config.SSH_USER + + def collect_all(self): + """采集所有 runner 的系统指标 + + Returns: + list[dict]: 每台机器的指标数据 + """ + if not self.ssh_hosts: + # 没有配置 SSH 主机，返回空列表 + return [] + + results = [] + for host in self.ssh_hosts: + try: + metrics = self._collect_one(host) + results.append(metrics) + except Exception as e: + results.append( + { + "host": host, + "status": "error", + "error": str(e), + } + ) + + return results + + def _collect_one(self, host): + """采集单台机器的指标（SSH 实装待后续） + + 当前直接返回 not_available 状态。 + 后续实现方案：用 paramiko 或 subprocess + ssh 命令执行： + - top / mpstat 取 CPU + - free 取内存 + - df -h 取磁盘 + """ + return { + "host": host, + "status": "not_available", + "cpu_percent": None, + "mem_percent": None, + "disk_percent": None, + "disk_total_gb": None, + "disk_used_gb": None, + "note": "SSH metrics collection not implemented yet", + } + + # ── 便捷方法 ────────────────────────────────────── + + @staticmethod + def is_available(): + """是否有可用的指标采集（SSH 已配置）""" + return bool(config.SSH_HOSTS and config.SSH_KEY_PATH) diff --git a/scripts/ci/runner_monitor/runner_status.py b/scripts/ci/runner_monitor/runner_status.py new file mode 100755 index 000000000..358aececa --- /dev/null +++ b/scripts/ci/runner_monitor/runner_status.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +""" +Runner 状态巡检 - 调 Gitea API 查 runner 列表 + 状态 + 队列积压 + +功能: + - 获取所有 runner 的在线状态、忙闲状态 + - 检测离线/禁用 runner + - 检测 CI 队列积压（pending 数量 + 持续时间） + - 生成 runner 状态快照 + +说明: + Gitea Actions API 直接返回的 runner 信息不包含心跳时间， + 因此"离线超过N分钟"的判断通过以下方式近似: + 1. status != "online" 的 runner 直接判定为离线 + 2. busy=true 且长时间无 job 完成的 runner 标记为疑似挂起（待增强） + 3. 通过 pending job 数量和时长判断队列积压 + +用法: + python3 scripts/ci/runner_monitor/runner_status.py --check + python3 scripts/ci/runner_monitor/runner_status.py --snapshot + python3 scripts/ci/runner_monitor/runner_status.py --list +""" + +import argparse +import json +import os +import sys +import time +from datetime import datetime, timezone + +# 复用 chatops 的 GiteaClient +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_CI_DIR = os.path.dirname(_SCRIPT_DIR) +if _CI_DIR not in sys.path: + sys.path.insert(0, _CI_DIR) + +from chatops.gitea_client import GiteaClient # noqa: E402 + + +class RunnerStatusChecker: + """Runner 状态巡检器""" + + def __init__(self, gitea_client=None): + self.gitea = gitea_client or GiteaClient() + + # ── Runner 列表 ────────────────────────────────── + + def get_runners(self): + """获取仓库所有 runner 列表 + + Returns: + list[dict]: runner 列表 + """ + # 直接调用 Gitea Actions runners API + data = self.gitea._request("actions/runners") + if not data: + return [] + return data.get("runners", []) + + def get_runner_summary(self): + """获取 runner 汇总信息 + + Returns: + dict: {total, online, offline, busy, disabled, runners} + """ + runners = self.get_runners() + if not runners: + return { + "total": 0, + "online": 0, + "offline": 0, + "busy": 0, + "disabled": 0, + "runners": [], + } + + online = sum(1 for r in runners if r.get("status") == "online" and not r.get("disabled")) + offline = sum(1 for r in runners if r.get("status") != "online" and not r.get("disabled")) + busy = sum(1 for r in runners if r.get("busy")) + disabled = sum(1 for r in runners if r.get("disabled")) + + return { + "total": len(runners), + "online": online, + "offline": offline, + "busy": busy, + "disabled": disabled, + "runners": runners, + } + + def get_offline_runners(self, offline_minutes=5): + """获取离线的 runner 列表 + + 由于 Gitea API 不返回心跳时间，status != online 即视为离线。 + offline_minutes 参数保留用于后续 SSH 心跳检测增强。 + + Returns: + list[dict]: 离线 runner 列表 + """ + runners = self.get_runners() + if not runners: + return [] + + offline = [r for r in runners if not r.get("disabled") and r.get("status") != "online"] + # 补充离线时长字段（暂时用 None，后续增强） + for r in offline: + r["offline_minutes"] = None + r["offline_reason"] = f"status={r.get('status', 'unknown')}" + + return offline + + # ── 队列积压检测 ────────────────────────────────── + + def get_pending_runs(self): + """获取 pending / queued 状态的 workflow runs + + Returns: + list[dict]: pending run 列表 + """ + # 尝试多种状态名（Gitea 可能用 queued / pending / waiting） + pending = [] + for status in ["queued", "pending", "waiting"]: + runs, _ = self.gitea.list_runs(status=status, limit=50) + pending.extend(runs) + + # 去重 + seen = set() + unique = [] + for r in pending: + rid = r.get("id") + if rid and rid not in seen: + seen.add(rid) + unique.append(r) + + return unique + + def get_queue_backlog(self, pending_threshold=10, duration_minutes=10): + """检测队列积压 + + Args: + pending_threshold: pending 数量阈值 + duration_minutes: 持续时间阈值（分钟） + + Returns: + dict: {is_backlogged, pending_count, oldest_pending_minutes, pending_runs} + """ + pending = self.get_pending_runs() + if not pending: + return { + "is_backlogged": False, + "pending_count": 0, + "oldest_pending_minutes": 0, + "pending_runs": [], + } + + now = datetime.now(timezone.utc) + oldest_minutes = 0 + for r in pending: + created = r.get("created_at", "") + if not created: + continue + try: + t = datetime.fromisoformat(created.replace("Z", "+00:00")) + age = (now - t).total_seconds() / 60 + oldest_minutes = max(oldest_minutes, age) + except Exception: + pass + + is_backlogged = len(pending) >= pending_threshold and oldest_minutes >= duration_minutes + + return { + "is_backlogged": is_backlogged, + "pending_count": len(pending), + "oldest_pending_minutes": round(oldest_minutes, 1), + "pending_runs": pending, + } + + # ── 综合巡检 ────────────────────────────────────── + + def run_full_check(self, offline_minutes=5, pending_threshold=10, pending_duration=10): + """执行完整的 runner 巡检 + + Returns: + dict: 巡检结果 + """ + summary = self.get_runner_summary() + offline_runners = self.get_offline_runners(offline_minutes=offline_minutes) + backlog = self.get_queue_backlog( + pending_threshold=pending_threshold, + duration_minutes=pending_duration, + ) + + return { + "timestamp": datetime.now(timezone.utc).isoformat(), + "runner_summary": { + "total": summary["total"], + "online": summary["online"], + "offline": summary["offline"], + "busy": summary["busy"], + "disabled": summary["disabled"], + }, + "offline_runners": [ + { + "id": r.get("id"), + "name": r.get("name"), + "status": r.get("status"), + "busy": r.get("busy"), + "labels": [label.get("name") for label in r.get("labels", [])], + "offline_minutes": r.get("offline_minutes"), + "offline_reason": r.get("offline_reason"), + } + for r in offline_runners + ], + "queue_backlog": { + "is_backlogged": backlog["is_backlogged"], + "pending_count": backlog["pending_count"], + "oldest_pending_minutes": backlog["oldest_pending_minutes"], + }, + "issues_found": len(offline_runners) > 0 or backlog["is_backlogged"], + } + + # ── 快照输出 ────────────────────────────────────── + + def save_snapshot(self, output_dir=None, data=None): + """保存状态快照为 JSON 文件 + + Returns: + str: 快照文件路径 + """ + if data is None: + data = self.run_full_check() + if output_dir is None: + from runner_monitor import config + + output_dir = config.OUTPUT_DIR + + os.makedirs(output_dir, exist_ok=True) + ts = time.strftime("%Y%m%d_%H%M%S") + filename = f"runner_snapshot_{ts}.json" + filepath = os.path.join(output_dir, filename) + + with open(filepath, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + # 清理旧快照（保留最近 24 个） + self._cleanup_old_snapshots(output_dir, keep=24) + + return filepath + + @staticmethod + def _cleanup_old_snapshots(directory, keep=24): + """清理旧快照文件""" + try: + files = sorted( + [f for f in os.listdir(directory) if f.startswith("runner_snapshot_")], + reverse=True, + ) + for old in files[keep:]: + os.remove(os.path.join(directory, old)) + except OSError: + pass + + +# ── CLI 入口 ────────────────────────────────────────── + + +def main(): + parser = argparse.ArgumentParser(description="Runner 状态巡检") + parser.add_argument("--list", action="store_true", help="列出所有 runner") + parser.add_argument("--check", action="store_true", help="执行完整巡检") + parser.add_argument("--snapshot", action="store_true", help="生成快照 JSON") + parser.add_argument("--pending", action="store_true", help="查看 pending 队列") + parser.add_argument("--output-dir", help="快照输出目录") + + args = parser.parse_args() + + checker = RunnerStatusChecker() + + if args.list: + summary = checker.get_runner_summary() + print( + f"Runner 总览: {summary['online']}/{summary['total']} 在线, " + f"{summary['busy']} 忙碌, {summary['offline']} 离线, " + f"{summary['disabled']} 禁用" + ) + print() + for r in summary["runners"]: + status_icon = "🟢" if r.get("status") == "online" else "🔴" + if r.get("disabled"): + status_icon = "⚪" + busy_icon = "⚡" if r.get("busy") else " " + labels = ", ".join(label.get("name") for label in r.get("labels", [])[:4]) + print(f" {status_icon}{busy_icon} {r['name']:<30} {r.get('status', '?'):<10} labels: {labels}") + + elif args.pending: + pending = checker.get_pending_runs() + print(f"Pending runs: {len(pending)}") + for r in pending[:10]: + print( + f" #{r.get('id')} {r.get('name', '?')} - {r.get('status', '?')} " + f"({r.get('head_branch', '?')}) created: {r.get('created_at', '?')[:16]}" + ) + + elif args.check: + result = checker.run_full_check() + print(json.dumps(result, indent=2, ensure_ascii=False)) + + elif args.snapshot: + path = checker.save_snapshot(output_dir=args.output_dir) + print(f"快照已保存: {path}") + + else: + parser.print_help() + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/select_unit_tests.py b/scripts/ci/select_unit_tests.py new file mode 100644 index 000000000..90f3d50ee --- /dev/null +++ b/scripts/ci/select_unit_tests.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +""" +根据PR改动文件选择需要运行的单元测试文件。 + +映射规则： +1. 改了tests/unit下的测试文件 -> 直接跑这些测试 +2. 改了apps/api/app/api/routes/xxx.py -> 匹配 test_*xxx*.py +3. 改了apps/api/app/services/xxx.py -> 匹配 test_*xxx*.py +4. 改了apps/worker/.../xxx.py -> 匹配 test_*xxx*.py +5. 改了apps/worker/video_processing/xxx_engine.py -> 匹配 test_*xxx*.py +6. 改了packages/.../xxx.py -> 匹配 test_*xxx*.py +7. 改了公共核心模块(core/middleware/schemas/config/db/auth/dependencies) -> 全量 +8. 改了依赖文件(requirements*.txt, pyproject.toml) -> 全量 +9. 改了alembic/migrations -> 全量 +10. 匹配不到测试的改动 -> 全量兜底 +""" + +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TESTS_DIR = ROOT / "tests" / "unit" + +# 触发全量的文件模式（公共核心/基础设施） +FULL_RUN_PATTERNS = [ + "apps/api/app/core/", + "apps/api/app/middleware/", + "apps/api/app/schemas/", + "apps/api/app/config.py", + "apps/api/app/db.py", + "apps/api/app/auth.py", + "apps/api/app/dependencies.py", + "packages/shared/", + "alembic/", + "migrations/", + "requirements-base.txt", + "requirements.txt", + "requirements-dev.txt", + "pyproject.toml", + "setup.cfg", + ".gitea/workflows/", + "scripts/ci/", + "tests/conftest.py", +] + +# 目录到测试文件关键词的映射（模糊匹配） +DIR_KEYWORD_MAP = { + "apps/api/app/api/routes/": "", # 用文件名匹配 + "apps/api/app/services/": "", # 用文件名匹配 + "apps/worker/worker_app/tasks/": "", + "apps/worker/video_processing/": "", + "apps/worker/services/": "", + "packages/application/": "", + "packages/adapters/": "", +} + + +def get_changed_files(): + """获取改动文件列表（从环境变量或git diff）。""" + # 优先从环境变量读取（CI中传入） + changed_env = os.environ.get("CHANGED_FILES", "") + if changed_env: + return [f.strip() for f in changed_env.split("\n") if f.strip()] + + # 回退到git diff（本地调试用） + try: + result = subprocess.run( + ["git", "diff", "--name-only", "origin/develop...HEAD"], + capture_output=True, + text=True, + cwd=ROOT, + ) + if result.returncode == 0: + return [f.strip() for f in result.stdout.split("\n") if f.strip()] + except Exception: + pass + + return [] + + +def should_full_run(files): + """检查是否需要全量运行。""" + for f in files: + for pattern in FULL_RUN_PATTERNS: + if f.startswith(pattern) or f == pattern: + print(f"[full-run] 触发全量: {f} 匹配 {pattern}") + return True + return False + + +def extract_module_name(filepath): + """从文件路径提取模块名（用于匹配测试文件）。""" + # 去掉扩展名 + name = Path(filepath).stem + + # 特殊映射 + special_mappings = { + # 路由文件 + "edit_plans_adjustments": "edit_plan_adjustments", + "edit_plans_ai": "edit_plan", + "edit_plans_clips_batch": "edit_plan", + "edit_plans_cover": "edit_plan_cover", + "edit_plans_export": "edit_plan_export", + "edit_plans_filter": "edit_plan_filter", + "edit_plans_generation": "edit_plan_generation", + "edit_plans_transitions": "edit_plan_transitions", + "asset_libraries": "asset_library", + "classification_jobs": "classification", + "chunked_upload": "chunked_upload", + "form_upload": "form_upload", + # 服务文件 + "edit_template_service": "edit_template_service", + "edit_plan_service": "edit_plan_service", + "unified_render_service": "unified_render", + "job_service": "job_service", + "auto_clip_service": "auto_clip", + "cosyvoice_service": "cosyvoice", + "video_compose_service": "video_compose", + "email_service": "email_service", + } + + return special_mappings.get(name, name) + + +def find_matching_tests(keyword, all_test_files): + """模糊匹配测试文件。""" + keyword_lower = keyword.lower().replace("_", "") + matches = [] + for tf in all_test_files: + tf_name = Path(tf).stem.lower().replace("_", "") + if keyword_lower in tf_name or tf_name in keyword_lower: + matches.append(tf) + return matches + + +def get_all_test_files(): + """获取所有单元测试文件。""" + if not TESTS_DIR.exists(): + return [] + return sorted(str(f.relative_to(ROOT)) for f in TESTS_DIR.glob("test_*.py")) + + +def select_tests(changed_files): + """主函数：选择要运行的测试文件。""" + all_tests = get_all_test_files() + + if not changed_files: + print("[info] 未找到改动文件，全量运行") + return all_tests, "full (no changes detected)" + + if should_full_run(changed_files): + return all_tests, "full (core/common files changed)" + + selected = set() + test_file_changes = [] + source_file_changes = [] + + for f in changed_files: + # 测试文件本身改动（仅保留仍存在的文件，删除的测试文件不加入运行列表） + if f.startswith("tests/unit/test_") and f.endswith(".py"): + if (ROOT / f).exists(): + test_file_changes.append(f) + selected.add(f) + else: + print(f"[skip-deleted] 测试文件已删除，跳过: {f}") + # 源码文件改动 + elif f.endswith(".py"): + source_file_changes.append(f) + module_name = extract_module_name(f) + matches = find_matching_tests(module_name, all_tests) + if matches: + for m in matches: + selected.add(m) + print(f"[map] {f} -> {len(matches)} 个测试: {[Path(m).name for m in matches]}") + else: + print(f"[nomatch] {f} (module: {module_name}) 未找到匹配的测试文件") + + if not selected: + print("[info] 未匹配到任何测试文件，全量运行兜底") + return all_tests, "full (no matching tests)" + + return sorted(selected), f"incremental ({len(selected)} test files)" + + +def main(): + changed_files = get_changed_files() + print(f"=== 改动文件 ({len(changed_files)} 个) ===") + for f in changed_files[:20]: + print(f" {f}") + if len(changed_files) > 20: + print(f" ... 还有 {len(changed_files) - 20} 个") + print() + + selected, mode = select_tests(changed_files) + + print() + print(f"=== 运行模式: {mode} ===") + print(f"=== 选中测试文件: {len(selected)} 个 ===") + for t in selected[:20]: + print(f" {t}") + if len(selected) > 20: + print(f" ... 还有 {len(selected) - 20} 个") + + # 输出结果文件（供CI后续步骤使用） + output_file = os.environ.get("SELECTED_TESTS_OUTPUT", "") + if output_file: + with open(output_file, "w") as f: + for t in selected: + f.write(t + "\n") + print(f"\n已写入到: {output_file}") + + # 设置环境变量标记 + gh_output = os.environ.get("GITHUB_OUTPUT", "") + if gh_output: + with open(gh_output, "a") as f: + f.write(f"test_count={len(selected)}\n") + f.write("mode=" + ("incremental" if "incremental" in mode else "full") + "\n") + + # 退出码：0=增量, 1=全量（供CI判断） + sys.exit(0 if "incremental" in mode else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/step_checkout.sh b/scripts/ci/step_checkout.sh new file mode 100755 index 000000000..cfe0de57f --- /dev/null +++ b/scripts/ci/step_checkout.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# CI 公共步骤：Checkout 代码（带重试） +# 用法：直接 source 或调用，需要 GITHUB_TOKEN 环境变量 +set -eu + +python3 - <<'PY' +import io, os, tarfile, time, urllib.request, urllib.error +url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" +request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) +last_err = None +for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise +else: + raise last_err +with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' + for member in tar.getmembers(): + name = member.name + if name == root_prefix[:-1]: + continue + if name.startswith(root_prefix): + member.name = name[len(root_prefix):] + if member.name: + tar.extract(member, '.') +PY diff --git a/scripts/ci/step_frontend_install.sh b/scripts/ci/step_frontend_install.sh new file mode 100755 index 000000000..8d1729b02 --- /dev/null +++ b/scripts/ci/step_frontend_install.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# CI 公共步骤：前端依赖安装 +# 直接在 CI 容器内运行（CI 镜像已包含 Node.js），无需 Docker 嵌套 +set -eu + +MODE="${1:-full}" + +echo "=== 前端依赖安装开始 (模式: $MODE) ===" + +cd apps/web + +# 配置国内镜像源加速 +npm config set registry https://registry.npmmirror.com + +# 安装依赖 +npm ci --no-audit --no-fund + +echo "=== 前端依赖安装完成 ===" diff --git a/scripts/ci/step_frontend_run.sh b/scripts/ci/step_frontend_run.sh new file mode 100755 index 000000000..377c7a44f --- /dev/null +++ b/scripts/ci/step_frontend_run.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# CI 公共步骤：前端命令执行 +# 直接在 CI 容器内运行（CI 镜像已包含 Node.js + pnpm），无需 Docker 嵌套 +set -eu + +CMD="${1:-echo 'no command'}" + +cd apps/web +sh -lc "$CMD" diff --git a/scripts/ci/step_install_ffmpeg.sh b/scripts/ci/step_install_ffmpeg.sh new file mode 100755 index 000000000..51e376cd7 --- /dev/null +++ b/scripts/ci/step_install_ffmpeg.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# CI 公共步骤：安装 ffmpeg +set +e +if command -v ffmpeg > /dev/null 2>&1; then + echo "ffmpeg already installed: $(ffmpeg -version | head -1)" + exit 0 +fi +if command -v apt-get > /dev/null 2>&1; then + apt-get update -qq && apt-get install -y -qq ffmpeg +elif command -v yum > /dev/null 2>&1; then + yum install -y -q epel-release 2>/dev/null + yum install -y -q ffmpeg 2>/dev/null + if [ $? -ne 0 ] && command -v dnf > /dev/null 2>&1; then + dnf install -y -q --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfusion-free-release-$(rpm -E %rhel).noarch.rpm 2>/dev/null + dnf install -y -q ffmpeg 2>/dev/null + fi +elif command -v dnf > /dev/null 2>&1; then + dnf install -y -q ffmpeg 2>/dev/null +fi +if command -v ffmpeg > /dev/null 2>&1; then + echo "ffmpeg installed successfully: $(ffmpeg -version | head -1)" +else + echo "Warning: ffmpeg installation failed or not available, some tests may be skipped" +fi diff --git a/scripts/ci/step_timer_end.sh b/scripts/ci/step_timer_end.sh new file mode 100755 index 000000000..7a99c4722 --- /dev/null +++ b/scripts/ci/step_timer_end.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# CI 公共步骤：Job 结束计时统计 +set +eu +if [ -n "$JOB_START_TIME" ]; then + END_TIME=$(date +%s) + DURATION=$((END_TIME - JOB_START_TIME)) + MINS=$((DURATION / 60)) + SECS=$((DURATION % 60)) + echo "JOB_DURATION_SECONDS=$DURATION" >> $GITHUB_ENV + echo "=== Job Duration: ${MINS}m${SECS}s ===" +else + echo "JOB_DURATION_SECONDS=0" >> $GITHUB_ENV + echo "=== Job Duration: unknown ===" +fi diff --git a/scripts/ci/step_timer_start.sh b/scripts/ci/step_timer_start.sh new file mode 100755 index 000000000..b53accb82 --- /dev/null +++ b/scripts/ci/step_timer_start.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# CI 公共步骤：Job 开始计时 +echo "JOB_START_TIME=$(date +%s)" >> $GITHUB_ENV +echo "Job started at $(date)" +# trigger CI run for PR validation +# trigger CI - worker dood fallback fix test \ No newline at end of file diff --git a/scripts/ci/validate_code_quality.sh b/scripts/ci/validate_code_quality.sh new file mode 100644 index 000000000..ce67490eb --- /dev/null +++ b/scripts/ci/validate_code_quality.sh @@ -0,0 +1,234 @@ +#!/bin/bash +# CI Validate: 代码质量与安全扫描（并行Job 1/3） +# 包含：密钥扫描、格式检查、安全扫描、依赖漏洞、死代码检测、脚本语法校验 +set -eu + +echo "=== CI Validate: 代码质量与安全扫描 ===" + +# --- 密钥检测 --- +echo "" +echo "=== [1/6] Secret detection (detect-secrets) ===" +python3 -m pip install -q detect-secrets +detect-secrets --version + +detect-secrets scan \ + --all-files \ + --exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \ + --exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \ + --exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \ + --disable-plugin Base64HighEntropyString \ + --disable-plugin HexHighEntropyString \ + --disable-plugin BasicAuthDetector \ + --disable-plugin KeywordDetector \ + --disable-plugin IPPublicDetector \ + > /tmp/secrets-scan.json 2>&1 + +FOUND=$(python3 -c " +import json +try: + with open('/tmp/secrets-scan.json') as f: + data = json.load(f) + results = data.get('results', {}) + total = sum(len(v) for v in results.values()) + print(total) +except Exception: + print('error') +") + +echo "Secrets detected: $FOUND" +if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then + echo "" + echo "=== Secret details ===" + python3 -c " +import json +with open('/tmp/secrets-scan.json') as f: + data = json.load(f) +for fpath, items in data.get('results', {}).items(): + for item in items: + line = item.get('line_number', '?') + stype = item.get('type', '?') + hashed = item.get('hashed_secret', '')[:16] + print(f' {fpath}:{line} [{stype}] {hashed}...') +" + echo "" + echo "ERROR: Potential secrets detected in code!" + exit 1 +fi +echo "✅ Secret scan passed" + +# --- 增量/全量模式判断 --- +echo "" +echo "=== [2/6] Code quality checks ===" +SCAN_MODE="full" +CHANGED_PY_FILES="" + +if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then + PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') + API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100" + set +e + RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL") + HTTP_CODE=$(echo "$RESPONSE" | tail -n1) + BODY=$(echo "$RESPONSE" | sed '$d') + set -e + if [ "$HTTP_CODE" = "200" ]; then + CHANGED_PY_FILES=$(echo "$BODY" | python3 -c " +import json, sys +try: + files = json.load(sys.stdin) + py_files = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] != 'removed'] + print(' '.join(py_files)) +except Exception: + print('') +") + # 新增文件(added)强制全量检查，防止增量漏检 + ADDED_PY_FILES=$(echo "$BODY" | python3 -c " +import json, sys +try: + files = json.load(sys.stdin) + added = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] == 'added'] + print(' '.join(added)) +except Exception: + print('') +") + MODIFIED_PY_FILES=$(echo "$BODY" | python3 -c " +import json, sys +try: + files = json.load(sys.stdin) + modified = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] not in ('removed', 'added')] + print(' '.join(modified)) +except Exception: + print('') +") + if [ -n "$CHANGED_PY_FILES" ]; then + SCAN_MODE="incremental" + echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed" + else + SCAN_MODE="skip_py" + echo "No Python files changed in this PR" + fi + else + echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan" + fi +else + echo "Full scan mode (not a PR event)" +fi + +if [ "$SCAN_MODE" = "incremental" ]; then + # 防御性过滤 + EXISTING_PY_FILES="" + for f in $CHANGED_PY_FILES; do + if [ -f "$f" ]; then + if [ -z "$EXISTING_PY_FILES" ]; then + EXISTING_PY_FILES="$f" + else + EXISTING_PY_FILES="$EXISTING_PY_FILES $f" + fi + fi + done + CHANGED_PY_FILES="$EXISTING_PY_FILES" + + python3 -m compileall -q $CHANGED_PY_FILES + python3 -m black --check --fast $CHANGED_PY_FILES + python3 -m isort --check-only $CHANGED_PY_FILES + RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | grep -v '^$' | xargs) + if [ -n "$RUFF_FILES" ]; then + python3 -m ruff check $RUFF_FILES --statistics + else + echo "No ruff-checkable files changed, skipping" + fi +elif [ "$SCAN_MODE" = "skip_py" ]; then + echo "No Python files changed - skipping Python lint checks" +else + echo "Full scan mode" + python3 -m compileall -q alembic apps packages tests scripts + python3 -m black --check --fast alembic apps packages tests scripts + python3 -m isort --check-only alembic apps packages tests scripts + python3 -m ruff check apps packages tests --statistics +fi +echo "✅ Code quality checks passed" + +# --- Bandit 安全扫描（仅告警） --- +echo "" +echo "=== [3/6] Security scan (bandit, advisory only) ===" +set +e +bandit -r apps packages -q -ll +BANDIT_EXIT=$? +set -e +if [ "$BANDIT_EXIT" -ne 0 ]; then + echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)" +else + echo "✅ Bandit security scan passed" +fi + +# --- Pip-audit 依赖漏洞扫描（仅告警） --- +echo "" +echo "=== [4/6] Python dependency vulnerability scan (pip-audit, advisory only) ===" +python3 -m pip install -q pip-audit +pip-audit --version +EXIT_CODE=0 +for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do + if [ -f "$req_file" ]; then + echo "--- Scanning $req_file ---" + pip-audit -r "$req_file" --desc on 2>&1 | head -40 || EXIT_CODE=$? + echo "" + fi +done +echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)" + +# --- Vulture 死代码检测（仅告警） --- +echo "" +echo "=== [5/6] Dead code detection (vulture, advisory only) ===" +set +e +python3 -m pip install -q vulture +vulture --version +echo "告警模式，不阻断CI。置信度>=90%建议尽快确认。" +echo "" +vulture apps packages scripts \ + --exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \ + --min-confidence 70 \ + 2>&1 | sort -t'(' -k2 -rn | head -80 +echo "" +echo "=== vulture scan summary ===" +echo "发现潜在死代码（可能包含框架装饰器注册的函数，为误报）" +echo "建议：定期人工审查高置信度（>=90%）条目" +set -e + +# --- CI脚本语法校验 --- +echo "" +echo "=== [6/6] CI & shell scripts syntax validation ===" +SYNTAX_ERROR=0 +# 检查所有 CI shell 脚本 +for script in scripts/ci/*.sh; do + if [ -f "$script" ]; then + if ! bash -n "$script" 2>&1; then + echo "❌ 语法错误: $script" + SYNTAX_ERROR=1 + fi + fi +done +# 检查所有 CI Python 脚本语法 +for script in scripts/ci/*.py; do + if [ -f "$script" ]; then + if ! python3 -m py_compile "$script" 2>&1; then + echo "❌ Python语法错误: $script" + SYNTAX_ERROR=1 + fi + fi +done +# 检查 .gitea/workflows 下的脚本（如果有） +for script in .gitea/workflows/*.sh; do + if [ -f "$script" ]; then + if ! bash -n "$script" 2>&1; then + echo "❌ 语法错误: $script" + SYNTAX_ERROR=1 + fi + fi +done +if [ "$SYNTAX_ERROR" -ne 0 ]; then + echo "❌ CI脚本语法校验失败，见上方错误" + exit 1 +fi +echo "✅ All CI scripts syntax OK" + +echo "" +echo "=== CI Validate: 代码质量与安全扫描 全部通过 ✅ ===" diff --git a/scripts/ci/validate_migration.sh b/scripts/ci/validate_migration.sh new file mode 100644 index 000000000..48243dd3d --- /dev/null +++ b/scripts/ci/validate_migration.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# CI Validate: Alembic迁移验证（并行Job 3/3） +# 需要PostgreSQL数据库 +set -eu + +echo "=== CI Validate: Alembic迁移验证 ===" + +# --- DooD模式检测：确定宿主机访问地址 --- +detect_docker_host() { + local test_port="${1:-5432}" + + local candidates=() + + # 1. host.docker.internal + if python3 -c "import socket; socket.gethostbyname('host.docker.internal')" 2>/dev/null; then + candidates+=("host.docker.internal") + fi + + # 2. docker0 桥接网关 + candidates+=("172.17.0.1") + + # 3. 默认网关 + local gw="" + gw=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1) + if [ -n "$gw" ] && [ "$gw" != "127.0.0.1" ]; then + candidates+=("$gw") + fi + + # 4. 宿主机同网段的.1或.254 + local my_ip="" + my_ip=$(hostname -I 2>/dev/null | awk '{print $1}') + if [ -n "$my_ip" ]; then + local subnet=$(echo "$my_ip" | cut -d. -f1-3) + candidates+=("${subnet}.1") + candidates+=("${subnet}.254") + fi + + # 5. 127.0.0.1 最后尝试 + candidates+=("127.0.0.1") + + for candidate in "${candidates[@]}"; do + if python3 -c " +import socket +s = socket.socket() +s.settimeout(2) +try: + s.connect(('$candidate', $test_port)) + s.close() + print('ok') +except: + pass +" 2>/dev/null | grep -q ok; then + echo "$candidate" + return 0 + fi + done + + echo "127.0.0.1" + return 1 +} + +# 获取宿主机IP +if [ -S /var/run/docker.sock ]; then + DOCKER_HOST_IP=$(detect_docker_host 5433) + if [ "$DOCKER_HOST_IP" = "127.0.0.1" ]; then + DOCKER_HOST_IP=$(detect_docker_host 22) + fi + echo "检测到DooD模式，宿主机地址: $DOCKER_HOST_IP" +else + DOCKER_HOST_IP="127.0.0.1" + echo "非DooD模式，使用 127.0.0.1" +fi +PG_HOST="$DOCKER_HOST_IP" +echo "PG host: $PG_HOST" + +# 指数退避TCP连接检查 +wait_tcp_ready() { + local host="$1" + local port="$2" + local max_attempts="${3:-5}" + local delay=1 + local attempt=1 + while [ "$attempt" -le "$max_attempts" ]; do + if python3 -c "import socket; s=socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then + return 0 + fi + echo "TCP连接尝试 $attempt/$max_attempts 失败，${delay}s后重试..." + sleep "$delay" + delay=$((delay * 2)) + attempt=$((attempt + 1)) + done + return 1 +} + +USE_SHARED_PG="${CI_USE_SHARED_PG:-false}" + +if [ "$USE_SHARED_PG" = "true" ]; then + # 使用常驻共享PG实例 + echo "使用常驻共享PG实例（CI_USE_SHARED_PG=true）" + SHARED_PG_HOST="$PG_HOST" + SHARED_PG_PORT="5433" + SHARED_PG_USER="postgres" + SHARED_PG_PASSWORD="ci_pg_2026!" + CI_DB_NAME="ci_run_${GITHUB_RUN_ID:-$$}" + + echo "等待共享PG连接就绪..." + wait_tcp_ready "$SHARED_PG_HOST" "$SHARED_PG_PORT" 5 + + echo "创建测试数据库: $CI_DB_NAME" + PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c " +import psycopg2 +conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres') +conn.autocommit = True +cur = conn.cursor() +cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)') +cur.execute(f'CREATE DATABASE \"$CI_DB_NAME\"') +cur.close() +conn.close() +" + export DATABASE_URL="postgresql+psycopg://${SHARED_PG_USER}:${SHARED_PG_PASSWORD}@${SHARED_PG_HOST}:${SHARED_PG_PORT}/${CI_DB_NAME}" + echo "✅ 共享PG数据库已创建: $CI_DB_NAME" + + # 执行迁移 + PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head + echo "✅ Alembic migrations applied successfully" + + # 清理数据库 + echo "清理测试数据库: $CI_DB_NAME" + PGPASSWORD="$SHARED_PG_PASSWORD" python3 -c " +import psycopg2 +conn = psycopg2.connect(host='$SHARED_PG_HOST', port=$SHARED_PG_PORT, user='$SHARED_PG_USER', password='$SHARED_PG_PASSWORD', dbname='postgres') +conn.autocommit = True +cur = conn.cursor() +cur.execute(f'DROP DATABASE IF EXISTS \"$CI_DB_NAME\" WITH (FORCE)') +cur.close() +conn.close() +" 2>/dev/null || echo "WARN: 数据库清理失败" + echo "✅ 共享PG数据库已清理" +else + # 使用临时PG容器（默认模式） + echo "使用临时PG容器模式" + PG_CONTAINER=ci-pg-validate-migration-${GITHUB_RUN_ID:-$$} + docker rm -f "$PG_CONTAINER" 2>/dev/null || true + docker run -d --name "$PG_CONTAINER" \ + --shm-size=256m \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=xiaoxia_saas \ + -P \ + --health-cmd "pg_isready -U postgres" \ + --health-interval 3s \ + --health-timeout 3s \ + --health-retries 20 \ + postgres:16-alpine + PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2) + echo "PostgreSQL port: $PG_PORT" + export DATABASE_URL="postgresql+psycopg://postgres:postgres@${PG_HOST}:${PG_PORT}/xiaoxia_saas" + + # 等待容器健康 + for i in $(seq 1 30); do + if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then + echo "PostgreSQL container is healthy on port $PG_PORT" + break + fi + echo "Waiting for PostgreSQL container health... ($i/30)" + sleep 2 + done + docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy + + # TCP连通性检查 + echo "验证TCP连通性 ($PG_HOST:$PG_PORT)..." + wait_tcp_ready "$PG_HOST" "$PG_PORT" 5 + echo "TCP connectivity to PostgreSQL confirmed on port $PG_PORT" + + # 执行迁移 + PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head + echo "✅ Alembic migrations applied successfully" + + docker rm -f "$PG_CONTAINER" 2>/dev/null || true +fi + +echo "" +echo "=== CI Validate: Alembic迁移验证 通过 ✅ ===" diff --git a/scripts/ci/validate_mypy.sh b/scripts/ci/validate_mypy.sh new file mode 100644 index 000000000..645776828 --- /dev/null +++ b/scripts/ci/validate_mypy.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# CI Validate: Mypy类型检查（并行Job 2/3） +set -eu + +echo "=== CI Validate: Mypy类型检查 ===" + +bash scripts/ci/mypy_check.sh + +echo "" +echo "=== CI Validate: Mypy类型检查 通过 ✅ ===" diff --git a/scripts/ci/vitest_incremental.sh b/scripts/ci/vitest_incremental.sh new file mode 100755 index 000000000..7c5e73c59 --- /dev/null +++ b/scripts/ci/vitest_incremental.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Vitest 增量执行脚本（在Docker Node容器中运行） +# PR模式下只跑与改动文件相关的测试，大幅节省时间 +set -eu + +# 如果不是PR事件，直接全量跑 +if [ "${GITHUB_EVENT_NAME:-}" != "pull_request" ]; then + echo "非PR模式，全量执行Vitest" + bash scripts/ci/step_frontend_run.sh "npx vitest run --coverage" + exit $? +fi + +# 获取PR改动的文件列表 +PR_NUMBER=$(echo "${GITHUB_REF:-}" | sed 's|refs/pull/||; s|/.*||') +if [ -z "$PR_NUMBER" ]; then + echo "无法获取PR编号，全量执行Vitest" + bash scripts/ci/step_frontend_run.sh "npx vitest run --coverage" + exit $? +fi + +API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100" +CHANGED_FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c " +import json, sys +try: + files = json.load(sys.stdin) + web_files = [] + for f in files: + fname = f['filename'] + if fname.startswith('apps/web/src/') and fname.endswith(('.ts', '.tsx', '.js', '.jsx')) and f['status'] != 'removed': + web_files.append(fname.replace('apps/web/', '')) + print(' '.join(web_files)) +except Exception as e: + print('') +") + +if [ -z "$CHANGED_FILES" ]; then + echo "PR未改动前端源码文件，跳过Vitest" + exit 0 +fi + +FILE_COUNT=$(echo "$CHANGED_FILES" | wc -w) +echo "PR改动了 $FILE_COUNT 个前端文件" + +# 如果改动文件太多（超过30个），全量跑更可靠 +if [ "$FILE_COUNT" -gt 30 ]; then + echo "改动文件较多，全量执行Vitest" + bash scripts/ci/step_frontend_run.sh "npx vitest run --coverage" + exit $? +fi + +echo "" +echo "=== 增量执行 Vitest（只跑相关测试）===" +echo "相关源文件: $CHANGED_FILES" +echo "" + +# 在Docker Node容器中执行增量测试 +set +e +bash scripts/ci/step_frontend_run.sh "npx vitest run related $CHANGED_FILES" +VITEST_EXIT=$? +set -e + +if [ "$VITEST_EXIT" -eq 0 ]; then + echo "" + echo "✅ 增量测试通过" + exit 0 +else + echo "" + echo "❌ 增量测试失败" + exit $VITEST_EXIT +fi diff --git a/tests/conftest.py b/tests/conftest.py index 1e9afca79..e2155d134 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,3 +11,42 @@ if str(ROOT) not in sys.path: # 必须在任何 app 模块导入之前设置，否则 pydantic Settings 验证失败 os.environ.setdefault("JWT_SECRET_KEY", "test-secret-key-for-all-tests") os.environ.setdefault("USE_IN_MEMORY_DB", "True") + + +# ── Celery 全局 mock ────────────────────────────────────────────────────── +# CI 环境没有 Redis，所有 Celery 异步任务都 mock 掉，避免连接超时报错 +# 集成测试只测 API 层逻辑（参数校验、权限、DB 操作），异步任务由 worker 单测覆盖 + +from unittest.mock import MagicMock + + +def _mock_celery_task(): + """全局 mock Celery 任务的 delay/apply_async/send_task 方法。""" + from celery import Celery, Task + + def _mock_delay(self, *args, **kwargs): + mock_result = MagicMock() + mock_result.id = "mock-task-id" + mock_result.state = "PENDING" + mock_result.ready.return_value = False + mock_result.get.return_value = None + return mock_result + + def _mock_apply_async(self, *args, **kwargs): + return _mock_delay(self, *args, **kwargs) + + def _mock_send_task(self, name, *args, **kwargs): + mock_result = MagicMock() + mock_result.id = f"mock-{name}" + mock_result.state = "PENDING" + mock_result.ready.return_value = False + mock_result.get.return_value = None + return mock_result + + Task.delay = _mock_delay + Task.apply_async = _mock_apply_async + Celery.send_task = _mock_send_task + + +# 在任何 app 模块导入之前就 patch 掉 +_mock_celery_task() diff --git a/tests/unit/test_config_oss.py b/tests/unit/test_config_oss.py index 077c84178..dffdb676d 100644 --- a/tests/unit/test_config_oss.py +++ b/tests/unit/test_config_oss.py @@ -40,7 +40,7 @@ def _fresh_settings(**env_overrides: dict[str, str]): "JWT_SECRET_KEY": "unit-test-secret-key-12345", **env_overrides, } - with patch.dict(os.environ, env, clear=False): + with patch.dict(os.environ, env, clear=True): Settings = _load_settings_class() return Settings() @@ -192,7 +192,7 @@ class TestOSSConfigAliases: "JWT_SECRET_KEY": "unit-test-secret-key-12345", "MAX_UPLOAD_SIZE_MB": "3000", } - with patch.dict(os.environ, env, clear=False): + with patch.dict(os.environ, env, clear=True): os.environ.pop("OSS_DIRECT_UPLOAD_MAX_MB", None) Settings = _load_settings_class() settings = Settings()