diff --git a/.gitea/workflows/auto-merge.yml b/.gitea/workflows/auto-merge.yml deleted file mode 100644 index e2e924cdb..000000000 --- a/.gitea/workflows/auto-merge.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Auto Merge PRs - -on: - schedule: - - cron: '0 */6 * * *' - workflow_dispatch: - -jobs: - auto-merge: - runs-on: saas - timeout-minutes: 10 - steps: - - name: Checkout code - shell: sh - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - set -eu - python3 - <<'PY' - import io, os, tarfile, time, urllib.request, urllib.error - url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/{os.environ['GITHUB_SHA']}.tar.gz" - request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - last_err = None - for attempt in range(5): - try: - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() - break - except urllib.error.HTTPError as e: - last_err = e - if e.code >= 500 and attempt < 4: - wait = 2 ** attempt - print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - except Exception as e: - last_err = e - if attempt < 4: - wait = 2 ** attempt - print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - else: - raise last_err - with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: - top_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' - for member in tar.getmembers(): - name = member.name - if name == top_prefix[:-1]: - continue - if name.startswith(top_prefix): - member.name = name[len(top_prefix):] - if member.name: - tar.extract(member, '.') - PY - - - name: Auto merge develop PRs - run: | - bash scripts/auto_merge_prs.sh develop - - - name: Auto merge main PRs (release only) - run: | - bash scripts/auto_merge_prs.sh main diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index a689a30d8..3e875526e 100755 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -22,7 +22,7 @@ permissions: jobs: validate: name: Validate Code Quality And Tests - runs-on: ubuntu-22.04 + runs-on: host timeout-minutes: 10 env: @@ -80,7 +80,7 @@ jobs: shell: sh run: | set -eu - python --version + python3 --version python3 -m pip --version echo "CI environment is ready" @@ -220,6 +220,119 @@ jobs: python3 -m coverage xml -o coverage.xml python3 -m coverage report --fail-under=60 > /dev/null + - name: Build summary + if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main' + shell: sh + run: | + set -eu + echo "Build completed successfully!" + echo "Branch: ${GITHUB_REF_NAME}" + echo "Commit: ${GITHUB_SHA}" + # 输出最终覆盖率 + python3 scripts/ci_coverage_summary.py + + integration-tests: + name: Integration Tests + runs-on: host + timeout-minutes: 20 + if: always() + needs: validate + + env: + DATABASE_URL: postgresql+psycopg://postgres:postgres@127.0.0.1:5432/xiaoxia_saas + USE_IN_MEMORY_DB: "false" + + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -eu + python3 - <<'PY' + import io, os, tarfile, time, urllib.request, urllib.error + url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" + request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' + for member in tar.getmembers(): + name = member.name + if name == root_prefix[:-1]: + continue + if name.startswith(root_prefix): + member.name = name[len(root_prefix):] + if member.name: + tar.extract(member, '.') + PY + + - name: Verify CI environment + shell: sh + run: | + set -eu + python3 --version + python3 -m pip --version + echo "CI environment is ready" + + - name: Install dependencies + shell: sh + run: | + set -eu + python3 -m pip install -q -r requirements-base.txt + python3 -m pip install -q -r requirements.txt + python3 -m pip install -q -r requirements-dev.txt + pytest --version + + - name: Start Redis + shell: sh + run: | + set -eu + REDIS_CONTAINER="ci-redis-${GITHUB_RUN_ID:-$$}" + echo "REDIS_CONTAINER=$REDIS_CONTAINER" >> "$GITHUB_ENV" + docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true + docker run -d --name "$REDIS_CONTAINER" \ + -P \ + --health-cmd "redis-cli ping" \ + --health-interval 2s \ + --health-timeout 2s \ + --health-retries 10 \ + redis:7-alpine + REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | cut -d: -f2) + echo "Redis port: $REDIS_PORT" + echo "REDIS_URL=redis://127.0.0.1:$REDIS_PORT/0" >> "$GITHUB_ENV" + for i in $(seq 1 15); do + if docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" 2>/dev/null | grep -q healthy; then + echo "Redis is ready on port $REDIS_PORT" + break + fi + echo "Waiting for Redis... ($i/15)" + sleep 2 + done + docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" | grep -q healthy + - name: Start PostgreSQL for integration tests shell: sh run: | @@ -270,7 +383,7 @@ jobs: -m pytest tests/integration -q --timeout=60 -x --reruns 2 --reruns-delay 1 -m "not performance" python3 -m coverage report --show-missing python3 -m coverage xml -o coverage.xml - python3 -m coverage report --fail-under=65 > /dev/null + python3 -m coverage report --fail-under=40 > /dev/null # 集成测试覆盖率门槛较低,核心目标是功能验证 - name: Run API performance baseline tests shell: sh @@ -314,16 +427,20 @@ jobs: exit 0 - - name: Cleanup PostgreSQL + - name: Cleanup PostgreSQL & Redis if: always() shell: sh run: | docker rm -f "${PG_CONTAINER:-ci-pg-validate}" 2>/dev/null || true + docker rm -f "${REDIS_CONTAINER:-ci-redis-int}" 2>/dev/null || true echo "PostgreSQL container cleaned up" + echo "Redis container cleaned up" - name: Coverage summary if: always() shell: sh + env: + COVERAGE_THRESHOLD: "40" run: | set +e echo "=== 覆盖率汇总 ===" @@ -336,20 +453,18 @@ jobs: echo "=== CI 失败通知 ===" FAILED_JOB="Validate Code Quality And Tests" python3 scripts/ci_notify_failure.py - - name: Build summary - if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main' + - name: Notify CI failure - Integration Tests + if: failure() shell: sh run: | - set -eu - echo "Build completed successfully!" - echo "Branch: ${GITHUB_REF_NAME}" - echo "Commit: ${GITHUB_SHA}" - # 输出最终覆盖率 - python3 scripts/ci_coverage_summary.py + set +e + echo "=== CI 失败通知 ===" + FAILED_JOB="Integration Tests" python3 scripts/ci_notify_failure.py + frontend-lint: name: Frontend Lint - runs-on: ubuntu-22.04 + runs-on: host timeout-minutes: 10 steps: diff --git a/.gitea/workflows/test-ssh-secret.yml b/.gitea/workflows/test-ssh-secret.yml deleted file mode 100644 index c457a7aec..000000000 --- a/.gitea/workflows/test-ssh-secret.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Test SSH Secret -on: - push: - branches: [develop] - paths: - - '.gitea/workflows/test-ssh-secret.yml' - -jobs: - test-ssh: - runs-on: ubuntu-22.04 - steps: - - name: Install SSH client - run: | - which ssh || (apt-get update && apt-get install -y openssh-client) - ssh -V - - - name: Debug environment - run: | - echo "=== Environment ===" - echo "Runner hostname: $(hostname)" - echo "Runner IP: $(hostname -i || echo 'unknown')" - echo "Current user: $(whoami)" - echo "=== Secrets check ===" - if [ -n "$STAGING_SSH_HOST" ]; then - echo "STAGING_SSH_HOST: [SET] value_length=${#STAGING_SSH_HOST}" - else - echo "STAGING_SSH_HOST: [EMPTY]" - fi - if [ -n "$STAGING_SSH_USER" ]; then - echo "STAGING_SSH_USER: [SET] value_length=${#STAGING_SSH_USER}" - else - echo "STAGING_SSH_USER: [EMPTY]" - fi - if [ -n "$STAGING_SSH_KEY" ]; then - echo "STAGING_SSH_KEY: [SET] value_length=${#STAGING_SSH_KEY}" - else - echo "STAGING_SSH_KEY: [EMPTY]" - fi - env: - STAGING_SSH_HOST: ${{ secrets.STAGING_SSH_HOST }} - STAGING_SSH_USER: ${{ secrets.STAGING_SSH_USER }} - STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }} - - - name: Setup SSH key - run: | - mkdir -p ~/.ssh - chmod 700 ~/.ssh - echo "$STAGING_SSH_KEY" > ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 - ssh-keygen -y -f ~/.ssh/id_ed25519 > ~/.ssh/id_ed25519.pub 2>/dev/null || echo "No public key generated" - echo "=== SSH Key fingerprint ===" - ssh-keygen -lf ~/.ssh/id_ed25519 || echo "Key fingerprint failed" - env: - STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }} - - - name: Test SSH connection - run: | - echo "Attempting SSH connection to $STAGING_SSH_HOST..." - ssh -i ~/.ssh/id_ed25519 \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o ConnectTimeout=10 \ - -o BatchMode=yes \ - -v \ - $STAGING_SSH_USER@$STAGING_SSH_HOST "echo 'SSH_CONNECTION_SUCCESS' && hostname && whoami" - echo "=== SSH Test Complete ===" - env: - STAGING_SSH_HOST: ${{ secrets.STAGING_SSH_HOST }} - STAGING_SSH_USER: ${{ secrets.STAGING_SSH_USER }} diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml deleted file mode 100755 index f8ffeb555..000000000 --- a/.gitea/workflows/tests.yml +++ /dev/null @@ -1,163 +0,0 @@ -name: Tests - -on: - pull_request: - branches: [ main ] - -jobs: - test: - runs-on: runtime-builder - - steps: - - name: Checkout code - shell: sh - run: | - set -eu - python - <<'PY' - import io - import os - import tarfile - import time - import urllib.error - import urllib.request - - url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" - request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - # Retry up to 5 times with backoff for transient 5xx errors - last_err = None - for attempt in range(5): - try: - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() - break - except urllib.error.HTTPError as e: - last_err = e - if e.code >= 500 and attempt < 4: - wait = 2 ** attempt - print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - except Exception as e: - last_err = e - if attempt < 4: - wait = 2 ** attempt - print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - else: - raise last_err - - with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: - root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' - for member in tar.getmembers(): - name = member.name - if name == root_prefix[:-1]: - continue - if name.startswith(root_prefix): - member.name = name[len(root_prefix):] - if member.name: - tar.extract(member, '.') - PY - - - name: Show Python version - shell: sh - run: | - set -eu - python --version - python -m pip --version - - - name: Install dependencies - shell: sh - run: | - set -eu - python -m pip install --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - python -m pip install -r requirements.txt -r requirements-dev.txt -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - - - name: Run unit tests - shell: sh - run: | - set -eu - PYTHONPATH="$PWD/apps/api:$PWD" python -m pytest tests/unit -q - - - name: Run integration tests - shell: sh - run: | - set -eu - PYTHONPATH="$PWD/apps/api:$PWD" python -m pytest tests/integration -q --timeout=60 -x - - lint: - runs-on: runtime-builder - - steps: - - name: Checkout code - shell: sh - run: | - set -eu - python - <<'PY' - import io - import os - import tarfile - import time - import urllib.error - import urllib.request - - url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" - request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - # Retry up to 5 times with backoff for transient 5xx errors - last_err = None - for attempt in range(5): - try: - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() - break - except urllib.error.HTTPError as e: - last_err = e - if e.code >= 500 and attempt < 4: - wait = 2 ** attempt - print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - except Exception as e: - last_err = e - if attempt < 4: - wait = 2 ** attempt - print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") - time.sleep(wait) - continue - raise - else: - raise last_err - - with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: - root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' - for member in tar.getmembers(): - name = member.name - if name == root_prefix[:-1]: - continue - if name.startswith(root_prefix): - member.name = name[len(root_prefix):] - if member.name: - tar.extract(member, '.') - PY - - - name: Install dependencies - shell: sh - run: | - set -eu - python -m pip install --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - python -m pip install -r requirements.txt -r requirements-dev.txt -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - - - name: Run Black (check only) - shell: sh - run: | - set -eu - python -m black --check alembic apps packages tests scripts - - - name: Run Flake8 - shell: sh - run: | - set -eu - python -m flake8 apps packages tests --count --statistics diff --git a/scripts/ci_coverage_summary.py b/scripts/ci_coverage_summary.py index 24c2ae6a3..f0de830ba 100755 --- a/scripts/ci_coverage_summary.py +++ b/scripts/ci_coverage_summary.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 """解析 coverage.xml 并输出覆盖率汇总。""" +import os import sys import xml.etree.ElementTree as ET -THRESHOLD = 65 # 行覆盖率门槛,百分比 +THRESHOLD = int(os.environ.get("COVERAGE_THRESHOLD", 65)) # 行覆盖率门槛,百分比,可通过环境变量覆盖 def main() -> int: