name: CI/CD Pipeline

on:
  push:
    branches:
      - main
      - develop
      - 'feature/**'
      - 'bugfix/**'
      - 'hotfix/**'
      - 'release/**'
    tags:
      - 'v*'
  pull_request:
    branches:
      - main
      - develop

permissions:
  contents: read

jobs:
  validate:
    name: Validate Code Quality And Tests
    runs-on: host
    timeout-minutes: 10

    env:
      DATABASE_URL: postgresql+psycopg://postgres:postgres@127.0.0.1:5432/xiaoxia_saas
      USE_IN_MEMORY_DB: "false"

    steps:
      - name: Checkout code
        shell: sh
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          set -eu
          python3 - <<'PY'
          import io, os, tarfile, time, urllib.request, urllib.error
          url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
          request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
          last_err = None
          for attempt in range(5):
              try:
                  with urllib.request.urlopen(request, timeout=120) as response:
                      archive = response.read()
                  break
              except urllib.error.HTTPError as e:
                  last_err = e
                  if e.code >= 500 and attempt < 4:
                      wait = 2 ** attempt
                      print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
                      time.sleep(wait)
                      continue
                  raise
              except Exception as e:
                  last_err = e
                  if attempt < 4:
                      wait = 2 ** attempt
                      print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
                      time.sleep(wait)
                      continue
                  raise
          else:
              raise last_err
          with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
              root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
              for member in tar.getmembers():
                  name = member.name
                  if name == root_prefix[:-1]:
                      continue
                  if name.startswith(root_prefix):
                      member.name = name[len(root_prefix):]
                  if member.name:
                      tar.extract(member, '.')
          PY

      - name: Verify CI environment
        shell: sh
        run: |
          set -eu
          python3 --version
          python3 -m pip --version
          echo "CI environment is ready"

      - name: Install dependencies
        shell: sh
        run: |
          set -eu
          python3 -m pip install -q -r requirements-base.txt
          python3 -m pip install -q -r requirements.txt
          python3 -m pip install -q -r requirements-dev.txt
          python3 -m black --version
          python3 -m isort --version-number
          python3 -m flake8 --version
          bandit --version
          pytest --version

      - name: Run code quality checks
        shell: sh
        run: |
          set -eu
          python3 -m compileall -q alembic apps packages tests scripts
          python3 -m black --check --fast alembic apps packages tests scripts
          python3 -m isort --check-only alembic apps packages tests scripts
          python3 -m flake8 apps packages tests --count --statistics

      - name: Run security scan
        shell: sh
        run: |
          set -eu
          bandit -r apps packages -q -ll

      - name: Validate release scripts syntax
        shell: sh
        run: |
          set -eu
          bash -n scripts/backup_postgres.sh
          bash -n scripts/restore_postgres_plan.sh
          bash -n scripts/init_production_env.sh

      - name: Validate Alembic migrations
        shell: sh
        run: |
          set -eu
          python3 -m alembic upgrade head --sql > /tmp/alembic-upgrade.sql
          test -s /tmp/alembic-upgrade.sql
          grep -q "Running upgrade" /tmp/alembic-upgrade.sql
          python3 scripts/check_schema_metadata.py

      - name: Prepare git for migration diff
        shell: sh
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          set -eu
          if command -v git >/dev/null 2>&1; then
            git init -q
            git config user.email "ci@localhost"
            git config user.name "CI"
            git add .
            git commit -q -m "current"
            REPO_URL="https://x-access-token:${GITHUB_TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
            git remote add origin "$REPO_URL"
            git fetch origin main --depth=1 -q 2>/dev/null || echo "WARN: cannot fetch main, will check all migrations"
          else
            echo "WARN: git not available, will check all migrations"
          fi

      - name: Check migration safety
        shell: sh
        run: |
          set -eu
          if git rev-parse origin/main >/dev/null 2>&1; then
            python3 scripts/check_migration_safety.py --allow-medium-risk --diff-against origin/main
          else
            python3 scripts/check_migration_safety.py --allow-medium-risk
          fi

  unit-tests:
    name: Unit Tests
    runs-on: host
    timeout-minutes: 8

    env:
      USE_IN_MEMORY_DB: "true"

    steps:
      - name: Checkout code
        shell: sh
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          set -eu
          python3 - <<'PY'
          import io, os, tarfile, time, urllib.request, urllib.error
          url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
          request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
          last_err = None
          for attempt in range(5):
              try:
                  with urllib.request.urlopen(request, timeout=120) as response:
                      archive = response.read()
                  break
              except urllib.error.HTTPError as e:
                  last_err = e
                  if e.code >= 500 and attempt < 4:
                      wait = 2 ** attempt
                      print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
                      time.sleep(wait)
                      continue
                  raise
              except Exception as e:
                  last_err = e
                  if attempt < 4:
                      wait = 2 ** attempt
                      print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
                      time.sleep(wait)
                      continue
                  raise
          else:
              raise last_err
          with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
              root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
              for member in tar.getmembers():
                  name = member.name
                  if name == root_prefix[:-1]:
                      continue
                  if name.startswith(root_prefix):
                      member.name = name[len(root_prefix):]
                  if member.name:
                      tar.extract(member, '.')
          PY

      - name: Install dependencies
        shell: sh
        run: |
          set -eu
          python3 -m pip install -q -r requirements-base.txt
          python3 -m pip install -q -r requirements.txt
          python3 -m pip install -q -r requirements-dev.txt
          pytest --version

      - name: Run unit tests with coverage
        shell: sh
        run: |
          set -eu
          PYTHONPATH="$PWD/apps/api:$PWD" python3 -m coverage run \
            --source=apps/api/app,packages \
            --omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \
            --branch \
            -m pytest tests/unit -q
          python3 -m coverage report --show-missing
          python3 -m coverage xml -o coverage.xml
          python3 -m coverage report --fail-under=60 > /dev/null

      - name: CI failure notification
        if: failure()
        shell: sh
        env:
          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
          CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
        run: |
          set +e
          FAILED_JOB="Unit Tests" python3 scripts/ci_notify_failure.py


  integration-tests:
    name: Integration Tests
    runs-on: host
    timeout-minutes: 20
    if: always()
    needs: validate

    env:
      DATABASE_URL: postgresql+psycopg://postgres:postgres@127.0.0.1:5432/xiaoxia_saas
      USE_IN_MEMORY_DB: "false"

    steps:
      - name: Checkout code
        shell: sh
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          set -eu
          python3 - <<'PY'
          import io, os, tarfile, time, urllib.request, urllib.error
          url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
          request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
          last_err = None
          for attempt in range(5):
              try:
                  with urllib.request.urlopen(request, timeout=120) as response:
                      archive = response.read()
                  break
              except urllib.error.HTTPError as e:
                  last_err = e
                  if e.code >= 500 and attempt < 4:
                      wait = 2 ** attempt
                      print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
                      time.sleep(wait)
                      continue
                  raise
              except Exception as e:
                  last_err = e
                  if attempt < 4:
                      wait = 2 ** attempt
                      print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
                      time.sleep(wait)
                      continue
                  raise
          else:
              raise last_err
          with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
              root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
              for member in tar.getmembers():
                  name = member.name
                  if name == root_prefix[:-1]:
                      continue
                  if name.startswith(root_prefix):
                      member.name = name[len(root_prefix):]
                  if member.name:
                      tar.extract(member, '.')
          PY

      - name: Verify CI environment
        shell: sh
        run: |
          set -eu
          python3 --version
          python3 -m pip --version
          echo "CI environment is ready"

      - name: Install dependencies
        shell: sh
        run: |
          set -eu
          python3 -m pip install -q -r requirements-base.txt
          python3 -m pip install -q -r requirements.txt
          python3 -m pip install -q -r requirements-dev.txt
          pytest --version

      - name: Start Redis
        shell: sh
        run: |
          set -eu
          REDIS_CONTAINER="ci-redis-${GITHUB_RUN_ID:-$$}"
          echo "REDIS_CONTAINER=$REDIS_CONTAINER" >> "$GITHUB_ENV"
          docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
          docker run -d --name "$REDIS_CONTAINER" \
            -P \
            --health-cmd "redis-cli ping" \
            --health-interval 2s \
            --health-timeout 2s \
            --health-retries 10 \
            redis:7-alpine
          REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | cut -d: -f2)
          echo "Redis port: $REDIS_PORT"
          echo "REDIS_URL=redis://127.0.0.1:$REDIS_PORT/0" >> "$GITHUB_ENV"
          for i in $(seq 1 15); do
            if docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" 2>/dev/null | grep -q healthy; then
              echo "Redis is ready on port $REDIS_PORT"
              break
            fi
            echo "Waiting for Redis... ($i/15)"
            sleep 2
          done
          docker inspect --format='{{.State.Health.Status}}' "$REDIS_CONTAINER" | grep -q healthy

      - name: Start PostgreSQL for integration tests
        shell: sh
        run: |
          set -eu
          PG_CONTAINER="ci-pg-${GITHUB_RUN_ID:-$$}"
          echo "PG_CONTAINER=$PG_CONTAINER" >> "$GITHUB_ENV"
          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
          docker run -d --name "$PG_CONTAINER" \
            --shm-size=256m \
            -e POSTGRES_USER=postgres \
            -e POSTGRES_PASSWORD=postgres \
            -e POSTGRES_DB=xiaoxia_saas \
            -P \
            --health-cmd "pg_isready -U postgres" \
            --health-interval 5s \
            --health-timeout 5s \
            --health-retries 12 \
            postgres:16
          # 获取随机映射的端口
          PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | cut -d: -f2)
          echo "PostgreSQL port: $PG_PORT"
          echo "DATABASE_URL=postgresql+psycopg://postgres:postgres@127.0.0.1:$PG_PORT/xiaoxia_saas" >> "$GITHUB_ENV"
          for i in $(seq 1 30); do
            if docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" 2>/dev/null | grep -q healthy; then
              echo "PostgreSQL is ready on port $PG_PORT"
              break
            fi
            echo "Waiting for PostgreSQL... ($i/30)"
            sleep 2
          done
          docker inspect --format='{{.State.Health.Status}}' "$PG_CONTAINER" | grep -q healthy

      - name: Apply migrations for integration tests
        shell: sh
        run: |
          set -eu
          PYTHONPATH="$PWD/apps/api:$PWD" python3 -m alembic upgrade head

      - name: Run integration tests
        shell: sh
        run: |
          set -eu
          pip install -q pytest-rerunfailures
          PYTHONPATH="$PWD/apps/api:$PWD" python3 -m coverage run --append \
            --source=apps/api/app,packages \
            --omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \
            --branch \
            -m pytest tests/integration -q --timeout=60 -x --reruns 2 --reruns-delay 1 -m "not performance"
          python3 -m coverage report --show-missing
          python3 -m coverage xml -o coverage.xml
          python3 -m coverage report --fail-under=40 > /dev/null  # 集成测试覆盖率门槛较低，核心目标是功能验证

      - name: Run API performance baseline tests
        shell: sh
        continue-on-error: true
        run: |
          set +e
          echo "=== API 性能基线测试 ==="
          PERF_OUTPUT=$(mktemp)
          PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration/test_api_performance.py \
            -v --timeout=120 -p no:cacheprovider 2>&1 | tee "$PERF_OUTPUT"
          PERF_EXIT=$?
          
          # 提取性能统计
          echo ""
          echo "=== 性能测试摘要 ==="
          grep "PERF_STATS:" "$PERF_OUTPUT" || echo "PERF_STATS: 未找到统计数据"
          grep "PERF_RESULT:" "$PERF_OUTPUT" || echo "PERF_RESULT: 未找到详细结果"
          
          # 统计通过率
          TOTAL=$(grep -c "PERF_RESULT:" "$PERF_OUTPUT" || echo 0)
          PASSED=$(grep "PERF_RESULT: PASS" "$PERF_OUTPUT" | wc -l)
          FAILED=$(grep "PERF_RESULT: FAIL" "$PERF_OUTPUT" | wc -l)
          
          echo ""
          echo "性能测试结果: $PASSED/$TOTAL 通过, $FAILED 未达标"
          
          if [ "$FAILED" -gt 0 ]; then
            echo ""
            echo "⚠️  警告: $FAILED 个接口性能未达标，请关注以下接口:"
            grep "PERF_RESULT: FAIL" "$PERF_OUTPUT" | while read line; do
              echo "   $line"
            done
            echo ""
            echo "性能测试失败不阻塞主流水线，但建议尽快优化。"
          else
            echo "✅ 所有接口性能达标！"
          fi
          
          rm -f "$PERF_OUTPUT"
          # 始终返回 0，不阻塞流水线
          exit 0


      - name: Cleanup PostgreSQL & Redis
        if: always()
        shell: sh
        run: |
          docker rm -f "${PG_CONTAINER:-ci-pg-validate}" 2>/dev/null || true
          docker rm -f "${REDIS_CONTAINER:-ci-redis-int}" 2>/dev/null || true
          echo "PostgreSQL container cleaned up"
          echo "Redis container cleaned up"

      - name: Coverage summary
        if: always()
        shell: sh
        env:
          COVERAGE_THRESHOLD: "40"
        run: |
          set +e
          echo "=== 覆盖率汇总 ==="
          python3 scripts/ci_coverage_summary.py
      - name: Notify CI failure
        if: failure()
        shell: sh
        run: |
          set +e
          echo "=== CI 失败通知 ==="
          FAILED_JOB="Validate Code Quality And Tests" python3 scripts/ci_notify_failure.py

      - name: Notify CI failure - Integration Tests
        if: failure()
        shell: sh
        run: |
          set +e
          echo "=== CI 失败通知 ==="
          FAILED_JOB="Integration Tests" python3 scripts/ci_notify_failure.py


  frontend-lint:
    name: Frontend Lint
    runs-on: host
    timeout-minutes: 10

    steps:
      - name: Checkout code
        shell: sh
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          set -eu
          python3 - <<'PY'
          import io, os, tarfile, time, urllib.request, urllib.error
          url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
          request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
          last_err = None
          for attempt in range(5):
              try:
                  with urllib.request.urlopen(request, timeout=120) as response:
                      archive = response.read()
                  break
              except urllib.error.HTTPError as e:
                  last_err = e
                  if e.code >= 500 and attempt < 4:
                      wait = 2 ** attempt
                      print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
                      time.sleep(wait)
                      continue
                  raise
              except Exception as e:
                  last_err = e
                  if attempt < 4:
                      wait = 2 ** attempt
                      print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
                      time.sleep(wait)
                      continue
                  raise
          else:
              raise last_err
          with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
              root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
              for member in tar.getmembers():
                  name = member.name
                  if name == root_prefix[:-1]:
                      continue
                  if name.startswith(root_prefix):
                      member.name = name[len(root_prefix):]
                  if member.name:
                      tar.extract(member, '.')
          PY

      - name: Install dependencies
        shell: sh
        run: |
          set -eu
          docker run --rm \
            -v "$PWD:/workspace" \
            -w /workspace/apps/web \
            docker.m.daocloud.io/library/node:20 \
            sh -lc 'npm ci'

      - name: Run ESLint
        shell: sh
        run: |
          set -eu
          docker run --rm \
            -v "$PWD:/workspace" \
            -w /workspace/apps/web \
            docker.m.daocloud.io/library/node:20 \
            sh -lc 'npx eslint src --ext .ts,.tsx --max-warnings 50'

      - name: Run TypeScript type check
        shell: sh
        run: |
          set -eu
          docker run --rm \
            -v "$PWD:/workspace" \
            -w /workspace/apps/web \
            docker.m.daocloud.io/library/node:20 \
            sh -lc 'npx tsc --noEmit'

      - name: Run Prettier check
        shell: sh
        run: |
          set -eu
          docker run --rm \
            -v "$PWD:/workspace" \
            -w /workspace/apps/web \
            docker.m.daocloud.io/library/node:20 \
            sh -lc 'npx prettier --check "src/**/*.{ts,tsx,md}"'

      - name: Run Vitest tests
        shell: sh
        run: |
          set -eu
          docker run --rm \
            -v "$PWD:/workspace" \
            -w /workspace/apps/web \
            docker.m.daocloud.io/library/node:20 \
            sh -lc 'npx vitest run src/test'

      - name: Notify CI failure
        if: failure()
        shell: sh
        run: |
          set +e
          echo "=== CI 失败通知 ==="
          FAILED_JOB="Frontend Lint" python3 scripts/ci_notify_failure.py

  deploy-staging:
    name: Build & Push Staging (Watchtower auto-deploy)
    runs-on: saas
    timeout-minutes: 30
    needs: [validate, frontend-lint]

    if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop')

    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        set -eu
        python3 - <<'INNERPY'
        import io, os, tarfile, time, urllib.request, urllib.error
        url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
        request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
        last_err = None
        for attempt in range(5):
            try:
                with urllib.request.urlopen(request, timeout=120) as response:
                    archive = response.read()
                break
            except urllib.error.HTTPError as e:
                last_err = e
                if e.code >= 500 and attempt < 4:
                    wait = 2 ** attempt
                    print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
                    time.sleep(wait)
                    continue
                raise
            except Exception as e:
                last_err = e
                if attempt < 4:
                    wait = 2 ** attempt
                    print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
                    time.sleep(wait)
                    continue
        else:
            raise last_err
        with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
            root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
            for member in tar.getmembers():
                name = member.name
                if name == root_prefix[:-1]:
                    continue
                if name.startswith(root_prefix):
                    member.name = name[len(root_prefix):]
                if member.name:
                    tar.extract(member, '.')
        INNERPY

    - name: Build and push all images to Gitea Registry
      shell: sh
      env:
        REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
      run: |
        set -eu
        chmod +x scripts/build_release_images.sh
        ALLOW_SHARED_PRODUCTION_BUILD_HOST=true REGISTRY_TOKEN="${REGISTRY_TOKEN}" \
          scripts/build_release_images.sh "${GITHUB_SHA}" staging

    - name: Tag and push :staging images (Watchtower auto-update)
      shell: sh
      env:
        REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
      run: |
        set -eu
        REGISTRY="git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas"
        if [ -n "${REGISTRY_TOKEN:-}" ]; then
          printf '%s' "${REGISTRY_TOKEN}" | docker login git.xiaoxiajianji.com -u xiaoxia --password-stdin 2>/dev/null
        fi
        for svc in api worker web; do
          docker tag "${REGISTRY}/xiaoxia-saas-${svc}:${GITHUB_SHA}" "${REGISTRY}/xiaoxia-saas-${svc}:staging"
          docker push "${REGISTRY}/xiaoxia-saas-${svc}:staging"
        done
        echo "All :staging images pushed. Watchtower will auto-deploy within 60s."

    - name: Wait for Watchtower update + smoke test
      shell: sh
      run: |
        set -eu
        echo "Waiting 90s for Watchtower to detect new image and restart containers..."
        sleep 90

        echo "--- Smoke test 1: Health check ---"
        for i in $(seq 1 12); do
          HEALTH=$(curl -sf --max-time 10 https://staging-api.xiaoxiajianji.com/health) && break
          echo "  Attempt $i/12: not ready yet, waiting 5s..."
          sleep 5
        done
        if [ -z "$HEALTH" ]; then
          echo "FAIL: health endpoint unreachable after 60s"
          exit 1
        fi
        echo "Health OK: $HEALTH"

        echo "--- Smoke test 2: Login API (expect 401) ---"
        HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 -X POST \
          https://staging-api.xiaoxiajianji.com/api/v1/auth/login \
          -H "Content-Type: application/json" \
          -d '{"email":"smoke@test.com","password":"wrong"}')
        if [ "$HTTP_CODE" != "401" ] && [ "$HTTP_CODE" != "422" ]; then
          echo "FAIL: login returned HTTP $HTTP_CODE (expected 401 or 422)"
          exit 1
        fi
        echo "Login API OK: HTTP $HTTP_CODE"

        echo "--- Smoke test 3: API docs endpoint ---"
        HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 https://staging-api.xiaoxiajianji.com/docs)
        if [ "$HTTP_CODE" != "200" ]; then
          echo "FAIL: /docs returned HTTP $HTTP_CODE (expected 200)"
          exit 1
        fi
        echo "Docs endpoint OK: HTTP $HTTP_CODE"

        echo "--- Smoke test 4: Web frontend ---"
        HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 https://staging.xiaoxiajianji.com/)
        if [ "$HTTP_CODE" != "200" ]; then
          echo "FAIL: web frontend returned HTTP $HTTP_CODE (expected 200)"
          exit 1
        fi
        echo "Web frontend OK: HTTP $HTTP_CODE"

        echo ""
        echo "=== All smoke tests passed! ==="
        echo "Branch: ${GITHUB_REF_NAME}"
        echo "Commit: ${GITHUB_SHA}"

    - name: Notify CI success
      if: success()
      shell: sh
      run: |
        set +e
        echo "=== CI 成功通知 ==="
        SUCCESS_JOB="Staging部署成功" python3 scripts/ci_notify_success.py

    - name: Notify CI failure
      if: failure()
      shell: sh
      run: |
        set +e
        echo "=== CI 失败通知 ==="
        FAILED_JOB="Build & Push Staging (Watchtower auto-deploy)" python3 scripts/ci_notify_failure.py



  staging-e2e:
    name: Staging E2E Tests
    runs-on: saas
    timeout-minutes: 15
    if: github.ref_name == 'develop' || github.ref_name == 'main'
    needs: deploy-staging

    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        set -eu
        python3 - <<'PY'
        import io, os, tarfile, time, urllib.request, urllib.error
        url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
        request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
        last_err = None
        for attempt in range(5):
            try:
                with urllib.request.urlopen(request, timeout=120) as response:
                    archive = response.read()
                break
            except urllib.error.HTTPError as e:
                last_err = e
                if e.code >= 500 and attempt < 4:
                    wait = 2 ** attempt
                    print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
                    time.sleep(wait)
                    continue
                raise
            except Exception as e:
                last_err = e
                if attempt < 4:
                    wait = 2 ** attempt
                    print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
                    time.sleep(wait)
                    continue
                raise
        else:
            raise last_err
        with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
            root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
            for member in tar.getmembers():
                name = member.name
                if name == root_prefix[:-1]:
                    continue
                if name.startswith(root_prefix):
                    member.name = name[len(root_prefix):]
                if member.name:
                    tar.extract(member, '.')
        PY
    - name: Run Playwright E2E on staging
      shell: sh
      run: |
        set -eu
        docker run --rm --ipc=host \
          -e E2E_BASE_URL=https://staging.xiaoxiajianji.com \
          -e E2E_API_BASE=https://staging-api.xiaoxiajianji.com/api/v1 \
          -e E2E_BROWSER_CHANNEL=chromium \
          -e PLAYWRIGHT_HEADLESS=1 \
          -v "$PWD:/workspace" \
          -w /workspace/apps/web \
          git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy \
          sh -lc "npm ci && npx playwright test --reporter=line --project=chromium e2e/auth.spec.ts e2e/auth-guard.spec.ts e2e/core-upload.spec.ts e2e/core-generation.spec.ts"

    - name: Notify CI failure
      if: failure()
      shell: sh
      run: |
        set +e
        echo "=== CI 失败通知 ==="
        FAILED_JOB="Staging E2E Tests" python3 scripts/ci_notify_failure.py


  staging-api-tests:
    name: Staging API Integration Tests
    runs-on: saas
    timeout-minutes: 10
    if: github.ref_name == 'develop' || github.ref_name == 'main'
    needs: deploy-staging

    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        set -eu
        python3 - <<'PY'
        import io, os, tarfile, time, urllib.request, urllib.error
        url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
        request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
        last_err = None
        for attempt in range(5):
            try:
                with urllib.request.urlopen(request, timeout=120) as response:
                    archive = response.read()
                break
            except urllib.error.HTTPError as e:
                last_err = e
                if e.code >= 500 and attempt < 4:
                    wait = 2 ** attempt
                    print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
                    time.sleep(wait)
                    continue
                raise
            except Exception as e:
                last_err = e
                if attempt < 4:
                    wait = 2 ** attempt
                    print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
                    time.sleep(wait)
                    continue
                raise
        else:
            raise last_err
        with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
            root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
            for member in tar.getmembers():
                name = member.name
                if name == root_prefix[:-1]:
                    continue
                if name.startswith(root_prefix):
                    member.name = name[len(root_prefix):]
                if member.name:
                    tar.extract(member, '.')
        PY
    - name: Run API integration tests on staging
      shell: sh
      run: |
        set -eu
        docker run --rm \
          -e E2E_BASE_URL=https://staging.xiaoxiajianji.com \
          -e E2E_API_BASE=https://staging-api.xiaoxiajianji.com/api/v1 \
          -v "$PWD:/workspace" \
          -w /workspace/apps/web \
          git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy \
          sh -lc 'npm ci && npx playwright test --reporter=line e2e/test_auth.spec.ts e2e/test_asset.spec.ts e2e/test_project.spec.ts'

    - name: Notify CI failure
      if: failure()
      shell: sh
      run: |
        set +e
        echo "=== CI 失败通知 ==="
        FAILED_JOB="Staging API Integration Tests" python3 scripts/ci_notify_failure.py



  build-production-runtime-images:
    name: Build Production Runtime Images
    runs-on: saas
    timeout-minutes: 30
    needs: [validate, frontend-lint]

    if: startsWith(github.ref, 'refs/tags/v')

    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        set -eu
        python3 - <<'PY'
        import io, os, tarfile, time, urllib.request, urllib.error
        url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
        request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
        last_err = None
        for attempt in range(5):
            try:
                with urllib.request.urlopen(request, timeout=120) as response:
                    archive = response.read()
                break
            except urllib.error.HTTPError as e:
                last_err = e
                if e.code >= 500 and attempt < 4:
                    wait = 2 ** attempt
                    print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
                    time.sleep(wait)
                    continue
                raise
            except Exception as e:
                last_err = e
                if attempt < 4:
                    wait = 2 ** attempt
                    print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
                    time.sleep(wait)
                    continue
                raise
        else:
            raise last_err
        with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
            root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
            for member in tar.getmembers():
                name = member.name
                if name == root_prefix[:-1]:
                    continue
                if name.startswith(root_prefix):
                    member.name = name[len(root_prefix):]
                if member.name:
                    tar.extract(member, '.')
        PY

    - name: Build and push all images (api + worker + web, with buildx cache)
      shell: sh
      env:
        REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
      run: |
        set -eu
        chmod +x scripts/build_release_images.sh
        REGISTRY_TOKEN="${REGISTRY_TOKEN}" scripts/build_release_images.sh "${GITHUB_REF_NAME}"

    - name: Cleanup old Docker images
      if: always()
      shell: sh
      run: |
        set -eu
        if [ -f scripts/cleanup_old_images.sh ]; then
          chmod +x scripts/cleanup_old_images.sh
          scripts/cleanup_old_images.sh
        else
          echo "Cleanup script not found, doing basic prune..."
          docker image prune -f 2>/dev/null || true
        fi
        echo "Disk usage after cleanup:"
        df -h / | tail -1

    - name: Notify CI failure
      if: failure()
      shell: sh
      run: |
        set +e
        echo "=== CI 失败通知 ==="
        FAILED_JOB="Build Production Runtime Images" python3 scripts/ci_notify_failure.py


  deploy-production:
    name: Deploy Production
    runs-on: saas
    timeout-minutes: 20
    if: startsWith(github.ref, 'refs/tags/v')
    needs: build-production-runtime-images

    steps:
    - name: Install SSH client
      shell: sh
      run: |
        set -eu
        apt-get update -qq && apt-get install -y -qq openssh-client >/dev/null 2>&1
        echo "openssh-client installed"

    - name: Deploy production over SSH (Registry pull)
      shell: sh
      env:
        PRODUCTION_SSH_HOST: ${{ secrets.PRODUCTION_SSH_HOST }}
        PRODUCTION_SSH_USER: ${{ secrets.PRODUCTION_SSH_USER }}
        PRODUCTION_SSH_KEY: ${{ secrets.PRODUCTION_SSH_KEY }}
        REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
      run: |
        set -eux
        production_host="${PRODUCTION_SSH_HOST:-47.98.113.167}"
        production_user="${PRODUCTION_SSH_USER:-root}"
        echo "Host: $production_host"
        echo "User: $production_user"
        echo "Home: $HOME"
        echo "Whoami: $(whoami)"

        mkdir -p ~/.ssh

        # 查找可用的SSH密钥
        key_path=""
        if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
          key_path="/root/.ssh/xiaoxia_runtime_builder"
          echo "Using key: $key_path (builder key)"
        elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then
          key_path="$HOME/.ssh/xiaoxia_runtime_builder"
          echo "Using key: $key_path (home key)"
        elif [ -n "${PRODUCTION_SSH_KEY:-}" ]; then
          key_path="$HOME/.ssh/id_ed25519"
          printf '%s\n' "$PRODUCTION_SSH_KEY" > "$key_path"
          chmod 600 "$key_path"
          echo "Using key from PRODUCTION_SSH_KEY secret"
        else
          echo "ERROR: No SSH key available"
          ls -la ~/.ssh/ 2>/dev/null || true
          ls -la /root/.ssh/ 2>/dev/null || true
          exit 1
        fi

        ssh-keyscan -p 22222 -H "$production_host" >> ~/.ssh/known_hosts 2>/dev/null
        echo "SSH keyscan done"

        # 测试SSH连接
        ssh -p 22222 -i "$key_path" -o StrictHostKeyChecking=no "${production_user}@${production_host}" "echo SSH_CONNECTION_OK && hostname"
        echo "SSH connection verified"

        # Registry 方式部署脚本（base64 编码避免转义问题）
        DEPLOY_B64="[encoded data, 10184 characters]"

        echo "$DEPLOY_B64" | base64 -d | ssh -p 22222 -i "$key_path" "$production_user@$production_host" "IMAGE_TAG='${GITHUB_REF_NAME}' REGISTRY_TOKEN='${REGISTRY_TOKEN}' sh"

    - name: Notify CI success
      if: success()
      shell: sh
      run: |
        set +e
        echo "=== CI 成功通知 ==="
        SUCCESS_JOB="生产部署成功" python3 scripts/ci_notify_success.py

    - name: Notify CI failure
      if: failure()
      shell: sh
      run: |
        set +e
        echo "=== CI 失败通知 ==="
        FAILED_JOB="Deploy Production" python3 scripts/ci_notify_failure.py


  production-e2e:
    name: Production Browser E2E
    runs-on: saas
    timeout-minutes: 15
    if: startsWith(github.ref, 'refs/tags/v')
    needs: deploy-production

    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        set -eu
        python3 - <<'PY'
        import io, os, tarfile, time, urllib.request, urllib.error
        url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
        request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
        # Retry up to 5 times with backoff for transient 5xx errors
        last_err = None
        for attempt in range(5):
            try:
                with urllib.request.urlopen(request, timeout=120) as response:
                    archive = response.read()
                break
            except urllib.error.HTTPError as e:
                last_err = e
                if e.code >= 500 and attempt < 4:
                    wait = 2 ** attempt
                    print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
                    time.sleep(wait)
                    continue
                raise
            except Exception as e:
                last_err = e
                if attempt < 4:
                    wait = 2 ** attempt
                    print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
                    time.sleep(wait)
                    continue
                raise
        else:
            raise last_err
        with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
            root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
            for member in tar.getmembers():
                name = member.name
                if name == root_prefix[:-1]:
                    continue
                if name.startswith(root_prefix):
                    member.name = name[len(root_prefix):]
                if member.name:
                    tar.extract(member, '.')
        PY

    - name: Run production browser E2E
      shell: sh
      run: |
        set -eu
        docker run --rm --ipc=host \
          -e E2E_BASE_URL=https://saas.xiaoxiajianji.com \
          -e E2E_API_BASE=https://api.xiaoxiajianji.com/api/v1 \
          -e E2E_BROWSER_CHANNEL=chromium \
          -e PLAYWRIGHT_HEADLESS=1 \
          -v "$PWD:/workspace" \
          -w /workspace/apps/web \
          git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy \
          sh -lc 'npm ci && npx playwright test --reporter=line --project=chromium e2e/auth.spec.ts e2e/auth-guard.spec.ts e2e/core-upload.spec.ts e2e/core-generation.spec.ts e2e/core-titles.spec.ts'

    - name: Notify CI failure
      if: failure()
      shell: sh
      run: |
        set +e
        echo "=== CI 失败通知 ==="
        FAILED_JOB="Production Browser E2E" python3 scripts/ci_notify_failure.py


