From 1daf8d946aeb632e4fecbfe9f0fa21eb3cc3662a Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Thu, 16 Jul 2026 13:48:24 +0800 Subject: [PATCH] =?UTF-8?q?fix(ci):=20Code=20Quality=E7=B2=BE=E7=AE=80?= =?UTF-8?q?=E4=B8=BA3=E6=AD=A5=EF=BC=8C=E5=BD=BB=E5=BA=95=E8=A7=A3?= =?UTF-8?q?=E5=86=B3act-runner=E4=B8=8A=E6=8A=A5=E8=B6=85=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将Code Quality从11步合并为3步（Checkout/All checks/Notify），与Unit Tests对齐。之前DB Migrations(10步)能正常上报但Code Quality(9步)不行，根因是act-runner v0.2.6长连接不稳，步骤越少越可靠。 --- .gitea/workflows/ci-cd.yml | 157 ++++++++++++++++++------------------- 1 file changed, 77 insertions(+), 80 deletions(-) diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 845a95b8b..c871d2690 100755 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -44,97 +44,95 @@ jobs: shell: sh env: GITHUB_TOKEN: ${{ github.token }} - run: "set -eu\npython3 - <<'PY'\nimport io, os, tarfile, time, urllib.request, urllib.error\nurl = f\"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz\"\nrequest = urllib.request.Request(url, headers={\"Authorization\": f\"token {os.environ['GITHUB_TOKEN']}\"})\nlast_err = None\nfor attempt in range(5):\n try:\n with urllib.request.urlopen(request, timeout=120) as response:\n archive = response.read()\n break\n except urllib.error.HTTPError as e:\n last_err = e\n if e.code >= 500 and attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...\")\n time.sleep(wait)\n continue\n raise\n except Exception as e:\n last_err = e\n if attempt < 4:\n wait = 2 ** attempt\n print(f\"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...\"\ - )\n time.sleep(wait)\n continue\n raise\nelse:\n raise last_err\nwith tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:\n root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'\n for member in tar.getmembers():\n name = member.name\n if name == root_prefix[:-1]:\n continue\n if name.startswith(root_prefix):\n member.name = name[len(root_prefix):]\n if member.name:\n tar.extract(member, '.')\nPY\n" - - name: Install dependencies + run: | + set -eu + python3 - <<'PY' + import io, os, tarfile, time, urllib.request, urllib.error + url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" + request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' + for member in tar.getmembers(): + name = member.name + if name == root_prefix[:-1]: + continue + if name.startswith(root_prefix): + member.name = name[len(root_prefix):] + if member.name: + tar.extract(member, '.') + PY + + - name: Run all code quality checks shell: sh - run: 'set -eu - + run: | + set -eu + echo "=== Installing dependencies ===" python3 -m pip install -q -r requirements-base.txt - python3 -m pip install -q -r requirements.txt - python3 -m pip install -q -r requirements-dev.txt - python3 -m black --version + echo "" + echo "=== 1/5 Secret detection ===" + python3 -m pip install -q detect-secrets + detect-secrets scan --all-files --exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' --exclude-files '\.(md|rst|txt|lock|example|sample|min\.js|min\.css|spec\.ts|test\.ts|test\.py)$' --exclude-files '(package-lock|yarn\.lock|poetry\.lock|Pipfile\.lock)$' --disable-plugin Base64HighEntropyString --disable-plugin HexHighEntropyString --disable-plugin BasicAuthDetector --disable-plugin KeywordDetector --disable-plugin IPPublicDetector 2>&1 | tee /tmp/secrets-scan.json + FOUND=$(python3 -c "import json; d=json.load(open('/tmp/secrets-scan.json')); print(sum(len(v) for v in d.get('results',{}).values()))" 2>/dev/null || echo error) + if [ "$FOUND" != "0" ] && [ "$FOUND" != "error" ]; then + echo "❌ Secrets detected: $FOUND" + exit 1 + fi + echo "✅ Secret scan passed" - python3 -m isort --version-number + echo "" + echo "=== 2/5 Code quality (full scan) ===" + python3 -m compileall -q alembic apps packages tests scripts + python3 -m black --check --fast alembic apps packages tests scripts + python3 -m isort --check-only alembic apps packages tests scripts + python3 -m ruff check apps packages tests --statistics + echo "✅ Code quality passed" - python3 -m ruff --version - - bandit --version - - pytest --version - - ' - - name: Secret detection (detect-secrets) - shell: sh - run: "set -eu\necho \"=== Installing detect-secrets ===\"\npython3 -m pip install -q detect-secrets\ndetect-secrets --version\necho \"\"\necho \"=== Running secret scan ===\"\ndetect-secrets scan \\\n --all-files \\\n --exclude-files '(^|/)(tests|test|e2e|__tests__|spec|docs|node_modules|site-packages|migrations|alembic|.gitea|.git|.pytest_cache|.next|dist|build)/' \\\n --exclude-files '\\.(md|rst|txt|lock|example|sample|min\\.js|min\\.css|spec\\.ts|test\\.ts|test\\.py)$' \\\n --exclude-files '(package-lock|yarn\\.lock|poetry\\.lock|Pipfile\\.lock)$' \\\n --disable-plugin Base64HighEntropyString \\\n --disable-plugin HexHighEntropyString \\\n --disable-plugin BasicAuthDetector \\\n --disable-plugin KeywordDetector \\\n --disable-plugin IPPublicDetector \\\n 2>&1 | tee /tmp/secrets-scan.json\n\nFOUND=$(python3 -c \"\nimport json\ntry:\n with open('/tmp/secrets-scan.json') as f:\n data = json.load(f)\n results = data.get('results', {})\n total = sum(len(v) for\ - \ v in results.values())\n print(total)\nexcept Exception:\n print('error')\n\")\necho \"\"\necho \"Secrets detected: $FOUND\"\nif [ \"$FOUND\" != \"0\" ] && [ \"$FOUND\" != \"error\" ]; then\n echo \"\"\n echo \"=== Secret details ===\"\n python3 -c \"\nimport json\nwith open('/tmp/secrets-scan.json') as f:\n data = json.load(f)\nfor fpath, items in data.get('results', {}).items():\n for item in items:\n line = item.get('line_number', '?')\n stype = item.get('type', '?')\n hashed = item.get('hashed_secret', '')[:16]\n print(f' {fpath}:{line} [{stype}] {hashed}...')\n\"\n echo \"\"\n echo \"ERROR: Potential secrets detected in code!\"\n echo \"If these are false positives, add exclusions in the CI workflow.\"\n exit 1\nfi\necho \"Secret scan completed - no secrets detected\"\n" - - name: Calculate changed Python files (incremental scan) - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - run: "set -eu\nSCAN_MODE=\"full\"\nCHANGED_PY_FILES=\"\"\n\nif [ \"${GITHUB_EVENT_NAME:-}\" = \"pull_request\" ] && [ -n \"${GITHUB_REF_NAME:-}\" ]; then\n echo \"PR mode (#${GITHUB_REF_NAME}) - fetching changed files from API\"\n\n PR_NUMBER=$(echo \"$GITHUB_REF\" | sed 's|refs/pull/||; s|/.*||')\n API_URL=\"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=100\"\n\n set +e\n RESPONSE=$(curl -s -w \"\\n%{http_code}\" -H \"Authorization: token ${GITHUB_TOKEN}\" \"${API_URL}\")\n HTTP_CODE=$(echo \"$RESPONSE\" | tail -n1)\n BODY=$(echo \"$RESPONSE\" | sed '$d')\n set -e\n\n if [ \"$HTTP_CODE\" = \"200\" ]; then\n CHANGED_PY_FILES=$(echo \"$BODY\" | python3 -c \"\nimport json, sys\ntry:\n files = json.load(sys.stdin)\n py_files = [f['filename'] for f in files\n if f['filename'].endswith('.py') and f['status'] != 'removed']\n print(' '.join(py_files))\nexcept Exception:\n print('')\n\")\n if [ -n \"$CHANGED_PY_FILES\" ]; then\n SCAN_MODE=\"incremental\"\n FILE_COUNT=$(echo \"$CHANGED_PY_FILES\" | wc -w)\n echo \"Changed Python files: ${FILE_COUNT}\"\n echo \"$CHANGED_PY_FILES\" | tr ' ' '\\n' | grep -v '^$'\n else\n SCAN_MODE=\"skip_py\"\n echo \"No Python files changed in this PR\"\n fi\n else\n echo \"WARN: API returned HTTP $HTTP_CODE, falling back to full scan\"\n fi\nelse\n echo \"Full scan mode (not a PR event)\"\nfi\n\necho \"SCAN_MODE=$SCAN_MODE\" >> $GITHUB_ENV\necho \"CHANGED_PY_FILES=$CHANGED_PY_FILES\" >> $GITHUB_ENV\n" - - name: Run code quality checks - shell: sh - run: "set -eu\n\nif [ \"$SCAN_MODE\" = \"incremental\" ]; then\n echo \"=== Incremental scan mode ===\"\n\n python3 -m compileall -q $CHANGED_PY_FILES\n\n python3 -m black --check --fast $CHANGED_PY_FILES\n\n python3 -m isort --check-only $CHANGED_PY_FILES\n\n RUFF_FILES=$(echo \"$CHANGED_PY_FILES\" | tr ' ' '\\n' | grep -v '^scripts/' | tr '\\n' ' ')\n if [ -n \"$RUFF_FILES\" ]; then\n python3 -m ruff check $RUFF_FILES --statistics\n else\n echo \"No ruff-checkable files changed, skipping\"\n fi\n\nelif [ \"$SCAN_MODE\" = \"skip_py\" ]; then\n echo \"No Python files changed - skipping Python lint checks\"\n\nelse\n echo \"=== Full scan mode ===\"\n\n python3 -m compileall -q alembic apps packages tests scripts\n\n python3 -m black --check --fast alembic apps packages tests scripts\n\n python3 -m isort --check-only alembic apps packages tests scripts\n\n python3 -m ruff check apps packages tests --statistics\nfi\n" - - name: Type check (mypy, hard gate) - - shell: sh - run: "bash scripts/ci/mypy_check.sh" - - name: Run security scan (bandit) - shell: sh - run: 'set -eu + echo "" + echo "=== 3/5 Type check (mypy) ===" + bash scripts/ci/mypy_check.sh + echo "✅ Type check passed" + echo "" + echo "=== 4/5 Security scan (bandit) ===" bandit -r apps packages -q -ll + echo "✅ Security scan passed" - ' - - name: Validate release scripts syntax - shell: sh - run: 'set -eu - + echo "" + echo "=== 5/5 Release scripts syntax ===" bash -n scripts/backup_postgres.sh - bash -n scripts/restore_postgres_plan.sh - bash -n scripts/init_production_env.sh + echo "✅ Release scripts syntax OK" - ' - - name: Report status to Gitea - if: success() - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - run: "set -eu\n - echo \"Reporting success status to Gitea...\"\n - STATE=success\n - CONTEXT=\"CI/CD Pipeline / Validate - Code Quality\"\n - API_URL=\"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}\"\n - set +e\n - curl -s -X POST \"$API_URL\" \\\n - -H \"Authorization: token ${GITHUB_TOKEN}\" \\\n - -H \"Content-Type: application/json\" \\\n - -d \"{\\\\\"state\\\\\":\\\\\"$STATE\\\\\",\\\\\"context\\\\\":\\\\\"$CONTEXT\\\\\",\\\\\"description\\\\\":\\\\\"Manual report\\\\\"}\"\n - echo \"Status reported.\"\n - " - - - name: Report failure status to Gitea - if: failure() - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - run: "set +eu\n - echo \"Reporting failure status to Gitea...\"\n - STATE=failure\n - CONTEXT=\"CI/CD Pipeline / Validate - Code Quality\"\n - API_URL=\"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}\"\n - curl -s -X POST \"$API_URL\" \\\n - -H \"Authorization: token ${GITHUB_TOKEN}\" \\\n - -H \"Content-Type: application/json\" \\\n - -d \"{\\\\\"state\\\\\":\\\\\"$STATE\\\\\",\\\\\"context\\\\\":\\\\\"$CONTEXT\\\\\",\\\\\"description\\\\\":\\\\\"Manual report\\\\\"}\"\n - echo \"Failure status reported.\"\n - " + echo "" + echo "🎉 All code quality checks passed!" - name: Notify on failure continue-on-error: true @@ -142,11 +140,10 @@ jobs: shell: sh env: CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} - run: 'set +e - + run: | + set +e NOTIFY_MODE=failure JOB_NAME="Validate - Code Quality" python3 scripts/ci_notify.py - ' validate-db-migrations: name: Validate - DB Migrations