#!/bin/bash # CI Validate: 代码质量与安全扫描(并行Job 1/3) # 包含:密钥扫描、格式检查、安全扫描、依赖漏洞、死代码检测、脚本语法校验 set -eu echo "=== CI Validate: 代码质量与安全扫描 ===" # --- 密钥检测 --- echo "" echo "=== [1/6] Secret detection (detect-secrets) ===" python3 -m pip install -q detect-secrets detect-secrets --version detect-secrets scan \ --all-files \ --exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \ --exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \ --exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \ --disable-plugin Base64HighEntropyString \ --disable-plugin HexHighEntropyString \ --disable-plugin BasicAuthDetector \ --disable-plugin KeywordDetector \ --disable-plugin IPPublicDetector \ > /tmp/secrets-scan.json 2>&1 FOUND=$(python3 -c " import json try: with open('/tmp/secrets-scan.json') as f: data = json.load(f) results = data.get('results', {}) total = sum(len(v) for v in results.values()) print(total) except Exception: print('error') ") echo "Secrets detected: $FOUND" if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then echo "" echo "=== Secret details ===" python3 -c " import json with open('/tmp/secrets-scan.json') as f: data = json.load(f) for fpath, items in data.get('results', {}).items(): for item in items: line = item.get('line_number', '?') stype = item.get('type', '?') hashed = item.get('hashed_secret', '')[:16] print(f' {fpath}:{line} [{stype}] {hashed}...') " echo "" echo "ERROR: Potential secrets detected in code!" exit 1 fi echo "✅ Secret scan passed" # --- 代码质量检查(全量,PR 和 push 统一标准)--- # 历史:PR 侧用增量检查以加速,但会导致 push 侧全量检查失败时 PR 侧感知不到 # 现在统一全量检查,确保 CI 真正保护主分支(black/isort/ruff 全量仅多几十秒) echo "" echo "=== [2/6] Code quality checks (full scan) ===" SCAN_MODE="full" echo "Full scan mode" python3 -m compileall -q alembic apps packages tests scripts python3 -m black --check --fast alembic apps packages tests scripts python3 -m isort --check-only alembic apps packages tests scripts python3 -m ruff check apps packages tests --statistics echo "✅ Code quality checks passed" # --- Bandit 安全扫描(仅告警) --- echo "" echo "=== [3/6] Security scan (bandit, advisory only) ===" set +e bandit -r apps packages -q -ll BANDIT_EXIT=$? set -e if [ "$BANDIT_EXIT" -ne 0 ]; then echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)" else echo "✅ Bandit security scan passed" fi # --- Pip-audit 依赖漏洞扫描(仅告警) --- echo "" echo "=== [4/6] Python dependency vulnerability scan (pip-audit, advisory only) ===" python3 -m pip install -q pip-audit pip-audit --version EXIT_CODE=0 for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do if [ -f "$req_file" ]; then echo "--- Scanning $req_file ---" pip-audit -r "$req_file" --desc on 2>&1 | head -40 || EXIT_CODE=$? echo "" fi done echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)" # --- Vulture 死代码检测(仅告警) --- echo "" echo "=== [5/6] Dead code detection (vulture, advisory only) ===" set +e python3 -m pip install -q vulture vulture --version echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。" echo "" vulture apps packages scripts \ --exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \ --min-confidence 70 \ 2>&1 | sort -t'(' -k2 -rn | head -80 echo "" echo "=== vulture scan summary ===" echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)" echo "建议:定期人工审查高置信度(>=90%)条目" set -e # --- CI脚本语法校验 --- echo "" echo "=== [6/6] CI & shell scripts syntax validation ===" SYNTAX_ERROR=0 # 检查所有 CI shell 脚本 for script in scripts/ci/*.sh; do if [ -f "$script" ]; then if ! bash -n "$script" 2>&1; then echo "❌ 语法错误: $script" SYNTAX_ERROR=1 fi fi done # 检查所有 CI Python 脚本语法 for script in scripts/ci/*.py; do if [ -f "$script" ]; then if ! python3 -m py_compile "$script" 2>&1; then echo "❌ Python语法错误: $script" SYNTAX_ERROR=1 fi fi done # 检查 .gitea/workflows 下的脚本(如果有) for script in .gitea/workflows/*.sh; do if [ -f "$script" ]; then if ! bash -n "$script" 2>&1; then echo "❌ 语法错误: $script" SYNTAX_ERROR=1 fi fi done if [ "$SYNTAX_ERROR" -ne 0 ]; then echo "❌ CI脚本语法校验失败,见上方错误" exit 1 fi echo "✅ All CI scripts syntax OK" echo "" echo "=== CI Validate: 代码质量与安全扫描 全部通过 ✅ ==="