diff --git a/scripts/ci/validate_code_quality.sh b/scripts/ci/validate_code_quality.sh new file mode 100644 index 000000000..22b83f294 --- /dev/null +++ b/scripts/ci/validate_code_quality.sh @@ -0,0 +1,186 @@ +#!/bin/bash +# CI Validate: 代码质量与安全扫描(并行Job 1/3) +# 包含:密钥扫描、格式检查、安全扫描、依赖漏洞、死代码检测、脚本语法校验 +set -eu + +echo "=== CI Validate: 代码质量与安全扫描 ===" + +# --- 密钥检测 --- +echo "" +echo "=== [1/6] Secret detection (detect-secrets) ===" +python3 -m pip install -q detect-secrets +detect-secrets --version + +detect-secrets scan \ + --all-files \ + --exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \ + --exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' \ + --exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' \ + --disable-plugin Base64HighEntropyString \ + --disable-plugin HexHighEntropyString \ + --disable-plugin BasicAuthDetector \ + --disable-plugin KeywordDetector \ + --disable-plugin IPPublicDetector \ + > /tmp/secrets-scan.json 2>&1 + +FOUND=$(python3 -c " +import json +try: + with open('/tmp/secrets-scan.json') as f: + data = json.load(f) + results = data.get('results', {}) + total = sum(len(v) for v in results.values()) + print(total) +except Exception: + print('error') +") + +echo "Secrets detected: $FOUND" +if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then + echo "" + echo "=== Secret details ===" + python3 -c " +import json +with open('/tmp/secrets-scan.json') as f: + data = json.load(f) +for fpath, items in data.get('results', {}).items(): + for item in items: + line = item.get('line_number', '?') + stype = item.get('type', '?') + hashed = item.get('hashed_secret', '')[:16] + print(f' {fpath}:{line} [{stype}] {hashed}...') +" + echo "" + echo "ERROR: Potential secrets detected in code!" + exit 1 +fi +echo "✅ Secret scan passed" + +# --- 增量/全量模式判断 --- +echo "" +echo "=== [2/6] Code quality checks ===" +SCAN_MODE="full" +CHANGED_PY_FILES="" + +if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_REF_NAME:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then + PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') + API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100" + set +e + RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL") + HTTP_CODE=$(echo "$RESPONSE" | tail -n1) + BODY=$(echo "$RESPONSE" | sed '$d') + set -e + if [ "$HTTP_CODE" = "200" ]; then + CHANGED_PY_FILES=$(echo "$BODY" | python3 -c " +import json, sys +try: + files = json.load(sys.stdin) + py_files = [f['filename'] for f in files if f['filename'].endswith('.py') and f['status'] != 'removed'] + print(' '.join(py_files)) +except Exception: + print('') +") + if [ -n "$CHANGED_PY_FILES" ]; then + SCAN_MODE="incremental" + echo "Incremental mode: $(echo "$CHANGED_PY_FILES" | wc -w) Python files changed" + else + SCAN_MODE="skip_py" + echo "No Python files changed in this PR" + fi + else + echo "WARN: API returned HTTP $HTTP_CODE, falling back to full scan" + fi +else + echo "Full scan mode (not a PR event)" +fi + +if [ "$SCAN_MODE" = "incremental" ]; then + # 防御性过滤 + EXISTING_PY_FILES="" + for f in $CHANGED_PY_FILES; do + if [ -f "$f" ]; then + if [ -z "$EXISTING_PY_FILES" ]; then + EXISTING_PY_FILES="$f" + else + EXISTING_PY_FILES="$EXISTING_PY_FILES $f" + fi + fi + done + CHANGED_PY_FILES="$EXISTING_PY_FILES" + + python3 -m compileall -q $CHANGED_PY_FILES + python3 -m black --check --fast $CHANGED_PY_FILES + python3 -m isort --check-only $CHANGED_PY_FILES + RUFF_FILES=$(echo "$CHANGED_PY_FILES" | tr ' ' '\n' | grep -v '^scripts/' | grep -v '^$' | xargs) + if [ -n "$RUFF_FILES" ]; then + python3 -m ruff check $RUFF_FILES --statistics + else + echo "No ruff-checkable files changed, skipping" + fi +elif [ "$SCAN_MODE" = "skip_py" ]; then + echo "No Python files changed - skipping Python lint checks" +else + echo "Full scan mode" + python3 -m compileall -q alembic apps packages tests scripts + python3 -m black --check --fast alembic apps packages tests scripts + python3 -m isort --check-only alembic apps packages tests scripts + python3 -m ruff check apps packages tests --statistics +fi +echo "✅ Code quality checks passed" + +# --- Bandit 安全扫描(仅告警) --- +echo "" +echo "=== [3/6] Security scan (bandit, advisory only) ===" +set +e +bandit -r apps packages -q -ll +BANDIT_EXIT=$? +set -e +if [ "$BANDIT_EXIT" -ne 0 ]; then + echo "⚠️ Bandit found security issues (advisory mode - not blocking CI)" +else + echo "✅ Bandit security scan passed" +fi + +# --- Pip-audit 依赖漏洞扫描(仅告警) --- +echo "" +echo "=== [4/6] Python dependency vulnerability scan (pip-audit, advisory only) ===" +python3 -m pip install -q pip-audit +pip-audit --version +EXIT_CODE=0 +for req_file in requirements.txt requirements-base.txt requirements-dev.txt; do + if [ -f "$req_file" ]; then + echo "--- Scanning $req_file ---" + pip-audit -r "$req_file" --desc on 2>&1 | head -40 || EXIT_CODE=$? + echo "" + fi +done +echo "pip-audit scan completed (advisory mode - warnings only, not blocking CI)" + +# --- Vulture 死代码检测(仅告警) --- +echo "" +echo "=== [5/6] Dead code detection (vulture, advisory only) ===" +set +e +python3 -m pip install -q vulture +vulture --version +echo "告警模式,不阻断CI。置信度>=90%建议尽快确认。" +echo "" +vulture apps packages scripts \ + --exclude "tests,test,migrations,.gitea,docs,node_modules,site-packages,*/test_*.py,*/conftest.py" \ + --min-confidence 70 \ + 2>&1 | sort -t'(' -k2 -rn | head -80 +echo "" +echo "=== vulture scan summary ===" +echo "发现潜在死代码(可能包含框架装饰器注册的函数,为误报)" +echo "建议:定期人工审查高置信度(>=90%)条目" +set -e + +# --- Release 脚本语法校验 --- +echo "" +echo "=== [6/6] Release scripts syntax validation ===" +bash -n scripts/backup_postgres.sh +bash -n scripts/restore_postgres_plan.sh +bash -n scripts/init_production_env.sh +echo "✅ Release scripts syntax OK" + +echo "" +echo "=== CI Validate: 代码质量与安全扫描 全部通过 ✅ ==="