diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml index 0e3a48ca3..9f1a2c5a3 100755 --- a/.gitea/workflows/ci-pipeline.yml +++ b/.gitea/workflows/ci-pipeline.yml @@ -121,12 +121,12 @@ jobs: [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) curl -sfH "Authorization: token ${GITHUB_TOKEN:-$GITEA_TOKEN}" -o /tmp/_ci_trace.py "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/ci_trace_report.py?ref=${GITHUB_SHA}" 2>/dev/null && python3 /tmp/_ci_trace.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true - validate-code-quality: + validate-style: needs: dedupe-check if: always() && needs.dedupe-check.outputs.skip_tests != 'true' - name: Validate - Code Quality + name: Validate - Style runs-on: ci-l2 - timeout-minutes: 8 + timeout-minutes: 6 env: PIP_CACHE_DIR: /root/.cache/pip PIP_NO_CACHE_DIR: '' @@ -152,9 +152,9 @@ jobs: uses: actions/cache@v4 with: path: /root/.cache/pip - key: ${{ runner.os }}-pip-codequality-${{ hashFiles('requirements*.txt') }} + key: ${{ runner.os }}-pip-style-${{ hashFiles('requirements*.txt') }} restore-keys: | - ${{ runner.os }}-pip-codequality- + ${{ runner.os }}-pip-style- ${{ runner.os }}-pip- - name: Install dependencies shell: sh @@ -178,17 +178,9 @@ jobs: [ $i -eq 3 ] && exit 1 sleep 5 done - for i in 1 2 3; do - python3 -m pip install --no-binary :all: black==26.5.1 isort==8.0.1 && break - echo "pip install black/isort 失败,重试 $i/3..." - [ $i -eq 3 ] && exit 1 - sleep 5 - done - - name: Run code quality and security checks + - name: Run style checks shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - run: bash scripts/ci/validate_code_quality.sh + run: bash scripts/ci/validate_style.sh - name: Auto-fix formatting (black + isort) if: failure() shell: sh @@ -204,7 +196,7 @@ jobs: CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }} run: | set +e - FAILED_JOB="Validate - Code Quality" python3 scripts/ci_notify_failure.py + FAILED_JOB="Validate - Style" python3 scripts/ci_notify_failure.py - name: Job duration summary if: always() shell: sh @@ -217,7 +209,7 @@ jobs: CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} run: | set +e - NOTIFY_MODE=failure JOB_NAME="Validate - Code Quality" python3 scripts/ci_notify.py + NOTIFY_MODE=failure JOB_NAME="Validate - Style" python3 scripts/ci_notify.py - name: Report CI trace if: always() shell: sh @@ -230,12 +222,16 @@ jobs: [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true - validate-type-check: + + validate-security: needs: dedupe-check if: always() && needs.dedupe-check.outputs.skip_tests != 'true' - name: Validate - Type Check (mypy) + name: Validate - Security runs-on: ci-l2 timeout-minutes: 8 + env: + PIP_CACHE_DIR: /root/.cache/pip + PIP_NO_CACHE_DIR: '' permissions: contents: read steps: @@ -254,6 +250,114 @@ jobs: - name: Record job start time shell: sh run: bash scripts/ci/step_timer_start.sh + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: /root/.cache/pip + key: ${{ runner.os }}-pip-security-${{ hashFiles('requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip-security- + ${{ runner.os }}-pip- + - name: Install dependencies + shell: sh + run: | + set -eu + for i in 1 2 3; do + python3 -m pip install -q -r requirements-base.txt && break + echo "pip install requirements-base.txt 失败,重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + for i in 1 2 3; do + python3 -m pip install -q -r requirements.txt && break + echo "pip install requirements.txt 失败,重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + for i in 1 2 3; do + python3 -m pip install -q -r requirements-dev.txt && break + echo "pip install requirements-dev.txt 失败,重试 $i/3..." + [ $i -eq 3 ] && exit 1 + sleep 5 + done + - name: Run security checks + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: bash scripts/ci/validate_security.sh + - name: CI failure notification + if: failure() + shell: sh + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }} + run: | + set +e + FAILED_JOB="Validate - Security" python3 scripts/ci_notify_failure.py + - name: Job duration summary + if: always() + shell: sh + run: bash scripts/ci/step_timer_end.sh + - name: Notify on failure + continue-on-error: true + if: failure() + shell: sh + env: + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + run: | + set +e + NOTIFY_MODE=failure JOB_NAME="Validate - Security" python3 scripts/ci_notify.py + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ ${{ job.status }} = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + + validate-python: + needs: dedupe-check + if: always() && needs.dedupe-check.outputs.skip_tests != 'true' + name: Validate - Python (mypy + alembic) + runs-on: ci-l2 + timeout-minutes: 10 + env: + PIP_CACHE_DIR: /root/.cache/pip + PIP_NO_CACHE_DIR: '' + DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas + USE_IN_MEMORY_DB: 'false' + CI_USE_SHARED_PG: 'true' + permissions: + contents: read + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sfH "Authorization: token $GITHUB_TOKEN" -o /tmp/_ci_checkout.sh "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" && bash /tmp/_ci_checkout.sh + - name: Zombie run selfcheck + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + run: bash scripts/ci/ci_run_selfcheck.sh + - name: Record job start time + shell: sh + run: bash scripts/ci/step_timer_start.sh + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: /root/.cache/pip + key: ${{ runner.os }}-pip-python-${{ hashFiles('requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip-python- + ${{ runner.os }}-pip- - name: Install dependencies shell: sh run: | @@ -279,6 +383,9 @@ jobs: - name: Run mypy type check shell: bash run: bash scripts/ci/validate_mypy.sh + - name: Run alembic migration validation + shell: bash + run: bash scripts/ci/validate_migration.sh - name: CI failure notification if: failure() shell: sh @@ -287,7 +394,7 @@ jobs: CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }} run: | set +e - FAILED_JOB="Validate - Type Check (mypy)" python3 scripts/ci_notify_failure.py + FAILED_JOB="Validate - Python (mypy + alembic)" python3 scripts/ci_notify_failure.py - name: Job duration summary if: always() shell: sh @@ -300,7 +407,7 @@ jobs: CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} run: | set +e - NOTIFY_MODE=failure JOB_NAME="Validate - Type Check (mypy)" python3 scripts/ci_notify.py + NOTIFY_MODE=failure JOB_NAME="Validate - Python (mypy + alembic)" python3 scripts/ci_notify.py - name: Report CI trace if: always() shell: sh @@ -313,92 +420,6 @@ jobs: [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true - validate-migration: - needs: dedupe-check - if: always() && needs.dedupe-check.outputs.skip_tests != 'true' - name: Validate - Migration (alembic) - runs-on: ci-l2 - timeout-minutes: 8 - permissions: - contents: read - env: - DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas - USE_IN_MEMORY_DB: 'false' - CI_USE_SHARED_PG: 'true' - steps: - - name: Checkout code - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - curl -sfH "Authorization: token $GITHUB_TOKEN" -o /tmp/_ci_checkout.sh "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" && bash /tmp/_ci_checkout.sh - - name: Zombie run selfcheck - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - GITHUB_RUN_ID: ${{ github.run_id }} - run: bash scripts/ci/ci_run_selfcheck.sh - - name: Record job start time - shell: sh - run: bash scripts/ci/step_timer_start.sh - - name: Install dependencies - shell: sh - run: | - set -eu - for i in 1 2 3; do - python3 -m pip install -q -r requirements-base.txt && break - echo "pip install requirements-base.txt 失败,重试 $i/3..." - [ $i -eq 3 ] && exit 1 - sleep 5 - done - for i in 1 2 3; do - python3 -m pip install -q -r requirements.txt && break - echo "pip install requirements.txt 失败,重试 $i/3..." - [ $i -eq 3 ] && exit 1 - sleep 5 - done - for i in 1 2 3; do - python3 -m pip install -q -r requirements-dev.txt && break - echo "pip install requirements-dev.txt 失败,重试 $i/3..." - [ $i -eq 3 ] && exit 1 - sleep 5 - done - - name: Run alembic migration validation - shell: bash - run: bash scripts/ci/validate_migration.sh - - name: CI failure notification - if: failure() - shell: sh - env: - GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} - CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }} - run: | - set +e - FAILED_JOB="Validate - Migration (alembic)" python3 scripts/ci_notify_failure.py - - name: Job duration summary - if: always() - shell: sh - run: bash scripts/ci/step_timer_end.sh - - name: Notify on failure - continue-on-error: true - if: failure() - shell: sh - env: - CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} - run: | - set +e - NOTIFY_MODE=failure JOB_NAME="Validate - Migration (alembic)" python3 scripts/ci_notify.py - - name: Report CI trace - if: always() - shell: sh - env: - AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} - run: | - STATUS="ok" - [ ${{ job.status }} = "success" ] || STATUS="error" - START_TIME="" - [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) - python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true unit-tests: needs: [check-frontend-only, dedupe-check] @@ -1384,8 +1405,9 @@ jobs: runs-on: runtime-builder timeout-minutes: ${{ matrix.timeout }} needs: - - validate-code-quality - - validate-type-check + - validate-style + - validate-security + - validate-python - unit-tests - frontend-lint - frontend-unit-test @@ -1871,9 +1893,9 @@ jobs: if: always() && github.event_name == 'pull_request' needs: - check-frontend-only - - validate-code-quality - - validate-type-check - - validate-migration + - validate-style + - validate-security + - validate-python - unit-tests - integration-tests - frontend-lint @@ -1889,9 +1911,9 @@ jobs: PR_NUMBER: ${{ github.event.pull_request.number }} PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} RESULT_CHECK_FRONTEND: ${{ needs.check-frontend-only.result }} - RESULT_CODE_QUALITY: ${{ needs.validate-code-quality.result }} - RESULT_TYPE_CHECK: ${{ needs.validate-type-check.result }} - RESULT_MIGRATION: ${{ needs.validate-migration.result }} + RESULT_STYLE: ${{ needs.validate-style.result }} + RESULT_SECURITY: ${{ needs.validate-security.result }} + RESULT_PYTHON: ${{ needs.validate-python.result }} RESULT_UNIT_TESTS: ${{ needs.unit-tests.result }} RESULT_INTEGRATION: ${{ needs.integration-tests.result }} RESULT_FRONTEND_LINT: ${{ needs.frontend-lint.result }} @@ -1903,9 +1925,9 @@ jobs: echo "" echo "各job结果:" echo " check-frontend-only: $RESULT_CHECK_FRONTEND" - echo " validate-code-quality: $RESULT_CODE_QUALITY" - echo " validate-type-check: $RESULT_TYPE_CHECK" - echo " validate-migration: $RESULT_MIGRATION" + echo " validate-style: $RESULT_STYLE" + echo " validate-security: $RESULT_SECURITY" + echo " validate-python: $RESULT_PYTHON" echo " unit-tests: $RESULT_UNIT_TESTS" echo " integration-tests: $RESULT_INTEGRATION" echo " frontend-lint: $RESULT_FRONTEND_LINT" @@ -1945,9 +1967,9 @@ jobs: # 必填检查项(根据PR类型决定) # 通用检查(所有PR都必须过) REQUIRED_GENERAL=( - "validate-code-quality:$RESULT_CODE_QUALITY" - "validate-type-check:$RESULT_TYPE_CHECK" - "validate-migration:$RESULT_MIGRATION" + "validate-style:$RESULT_STYLE" + "validate-security:$RESULT_SECURITY" + "validate-python:$RESULT_PYTHON" "frontend-lint:$RESULT_FRONTEND_LINT" "build-pr:$RESULT_BUILD_PR" "ai-code-review:$AI_REVIEW_STATUS" diff --git a/scripts/ci/validate_code_quality.sh b/scripts/ci/validate_security.sh old mode 100644 new mode 100755 similarity index 61% rename from scripts/ci/validate_code_quality.sh rename to scripts/ci/validate_security.sh index 874c9c6cf..b08904327 --- a/scripts/ci/validate_code_quality.sh +++ b/scripts/ci/validate_security.sh @@ -1,13 +1,13 @@ #!/bin/bash -# CI Validate: 代码质量与安全扫描(并行Job 1/3) -# 包含:密钥扫描、格式检查、安全扫描、依赖漏洞、死代码检测、脚本语法校验 +# CI Validate: 安全扫描(validate-security) +# 包含:密钥扫描、bandit 安全扫描(仅告警)、pip-audit 依赖漏洞(仅告警)、CI 脚本语法校验 set -eu -echo "=== CI Validate: 代码质量与安全扫描 ===" +echo "=== CI Validate: 安全扫描 ===" # --- 密钥检测 --- echo "" -echo "=== [1/6] Secret detection (detect-secrets) ===" +echo "=== [1/4] Secret detection (detect-secrets) ===" python3 -m pip install -q detect-secrets detect-secrets --version @@ -56,23 +56,9 @@ for fpath, items in data.get('results', {}).items(): fi echo "✅ Secret scan passed" -# --- 代码质量检查(全量,PR 和 push 统一标准)--- -# 历史:PR 侧用增量检查以加速,但会导致 push 侧全量检查失败时 PR 侧感知不到 -# 现在统一全量检查,确保 CI 真正保护主分支(black/isort/ruff 全量仅多几十秒) +# --- Bandit 安全扫描(仅告警)--- echo "" -echo "=== [2/6] Code quality checks (full scan) ===" -SCAN_MODE="full" -echo "Full scan mode" -python3 -m compileall -q alembic apps packages tests scripts -python3 -m black --check --fast alembic apps packages tests scripts -python3 -m isort --check-only alembic apps packages tests scripts -python3 -m ruff check apps packages tests --statistics - -echo "✅ Code quality checks passed" - -# --- Bandit 安全扫描(仅告警) --- -echo "" -echo "=== [3/6] Security scan (bandit, advisory only) ===" +echo "=== [2/4] Security scan (bandit, advisory only) ===" set +e bandit -r apps packages -q -ll BANDIT_EXIT=$? @@ -83,9 +69,9 @@ else echo "✅ Bandit security scan passed" fi -# --- Pip-audit 依赖漏洞扫描(仅告警) --- +# --- Pip-audit 依赖漏洞扫描(仅告警)--- echo "" -echo "=== [4/6] Python dependency vulnerability scan (pip-audit, advisory only) ===" +echo "=== [3/4] Python dependency vulnerability scan (pip-audit, advisory only) ===" python3 -m pip install -q pip-audit pip-audit --version EXIT_CODE=0 @@ -98,27 +84,9 @@ for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do done echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)" -# --- Vulture 死代码检测(仅告警) --- -echo "" -echo "=== [5/6] Dead code detection (vulture, advisory only) ===" -set +e -python3 -m pip install -q vulture -vulture --version -echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。" -echo "" -vulture apps packages scripts \ - --exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \ - --min-confidence 70 \ - 2>&1 | sort -t'(' -k2 -rn | head -80 -echo "" -echo "=== vulture scan summary ===" -echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)" -echo "建议:定期人工审查高置信度(>=90%)条目" -set -e - # --- CI脚本语法校验 --- echo "" -echo "=== [6/6] CI & shell scripts syntax validation ===" +echo "=== [4/4] CI & shell scripts syntax validation ===" SYNTAX_ERROR=0 # 检查所有 CI shell 脚本 for script in scripts/ci/*.sh; do @@ -154,4 +122,4 @@ fi echo "✅ All CI scripts syntax OK" echo "" -echo "=== CI Validate: 代码质量与安全扫描 全部通过 ✅ ===" +echo "=== CI Validate: 安全扫描 全部通过 ✅ ===" diff --git a/scripts/ci/validate_style.sh b/scripts/ci/validate_style.sh new file mode 100755 index 000000000..c87ec7f3c --- /dev/null +++ b/scripts/ci/validate_style.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# CI Validate: 代码风格检查(validate-style) +# 包含:Python 字节码编译、black 格式、isort 排序、ruff lint、vulture 死代码(仅告警) +set -eu + +echo "=== CI Validate: 代码风格检查 ===" + +# --- Python 字节码编译 --- +echo "" +echo "=== [1/3] Python bytecode compilation ===" +python3 -m compileall -q alembic apps packages tests scripts +echo "✅ Bytecode compilation passed" + +# --- 代码格式检查(全量)--- +echo "" +echo "=== [2/3] Code formatting (black + isort + ruff) ===" +echo "Full scan mode" +python3 -m black --check --fast alembic apps packages tests scripts +python3 -m isort --check-only alembic apps packages tests scripts +python3 -m ruff check apps packages tests --statistics +echo "✅ Code formatting checks passed" + +# --- Vulture 死代码检测(仅告警)--- +echo "" +echo "=== [3/3] Dead code detection (vulture, advisory only) ===" +set +e +python3 -m pip install -q vulture +vulture --version +echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。" +echo "" +vulture apps packages scripts \ + --exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \ + --min-confidence 70 \ + 2>&1 | sort -t'(' -k2 -rn | head -80 +echo "" +echo "=== vulture scan summary ===" +echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)" +echo "建议:定期人工审查高置信度(>=90%)条目" +set -e + +echo "" +echo "=== CI Validate: 代码风格检查 全部通过 ✅ ==="