name: CI/CD Pipeline
on:
  push:
    branches:
    - main
    - develop
    tags:
    - v*
  pull_request:
    branches:
    - main
    - develop
  schedule:
    - cron: '0 19 * * *'  # UTC 19:00 = 北京时间凌晨3:00，每日全量CI回归
  workflow_dispatch:
    inputs:
      reason:
        description: "触发原因"
        required: false
        default: "手动触发 - CI漏触发补跑"
permissions:
  contents: read
concurrency:
  group: ci-pipeline-${{ gitea.event_name }}-${{ gitea.ref }}
  # PR事件取消进行中的旧run，push事件不取消（确保完整CI跑完）
  cancel-in-progress: ${{ gitea.event_name == 'pull_request' }}
jobs:
  check-frontend-only:
    name: Check if frontend-only change
    runs-on: ci-l2
    if: github.event_name == 'pull_request'
    outputs:
      skip_backend: ${{ steps.check.outputs.skip_backend }}
      skip_frontend: ${{ steps.check.outputs.skip_frontend }}
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Check changed files
      id: check
      shell: bash
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        set -eu
        PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||')
        API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300"
        FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin)]")
        FRONTEND_COUNT=$(echo "$FILES" | grep -c '^apps/web/' || true)
        BACKEND_COUNT=$(echo "$FILES" | grep -cv '^apps/web/' || true)
        TOTAL=$(echo "$FILES" | grep -cv '^$' || true)
        echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT})"
        if [ "$BACKEND_COUNT" = "0" ] && [ "$FRONTEND_COUNT" -gt "0" ]; then
          echo "skip_backend=true" >> $GITHUB_OUTPUT
          echo "skip_frontend=false" >> $GITHUB_OUTPUT
          echo "✅ 纯前端改动，跳过后端检查"
        elif [ "$FRONTEND_COUNT" = "0" ] && [ "$BACKEND_COUNT" -gt "0" ]; then
          echo "skip_backend=false" >> $GITHUB_OUTPUT
          echo "skip_frontend=true" >> $GITHUB_OUTPUT
          echo "🔧 纯后端改动，跳过前端检查"
        else
          echo "skip_backend=false" >> $GITHUB_OUTPUT
          echo "skip_frontend=false" >> $GITHUB_OUTPUT
          echo "🔧 包含全栈变更，运行完整CI"
        fi

    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  validate-code-quality:
    name: Validate - Code Quality
    runs-on: ci-l2
    timeout-minutes: 8
    permissions:
      contents: write
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Install dependencies
      shell: sh
      run: |
        set -eu
        for i in 1 2 3; do
          python3 -m pip install -q -r requirements-base.txt && break
          echo "pip install requirements-base.txt 失败，重试 $i/3..."
          [ $i -eq 3 ] && exit 1
          sleep 5
        done
        for i in 1 2 3; do
          python3 -m pip install -q -r requirements.txt && break
          echo "pip install requirements.txt 失败，重试 $i/3..."
          [ $i -eq 3 ] && exit 1
          sleep 5
        done
        for i in 1 2 3; do
          python3 -m pip install -q -r requirements-dev.txt && break
          echo "pip install requirements-dev.txt 失败，重试 $i/3..."
          [ $i -eq 3 ] && exit 1
          sleep 5
        done
        for i in 1 2 3; do
          python3 -m pip install --no-binary :all: black==26.5.1 isort==8.0.1 && break
          echo "pip install black/isort 失败，重试 $i/3..."
          [ $i -eq 3 ] && exit 1
          sleep 5
        done
    - name: Run code quality and security checks
      shell: bash
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: bash scripts/ci/validate_code_quality.sh
    - name: Auto-fix formatting (black + isort)
      if: failure()
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
        REVIEW_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
      run: python3 scripts/ci/auto_fix_formatting.py
    - name: CI failure notification
      if: failure()
      shell: sh
      env:
        GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
        CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
      run: |
        set +e
        FAILED_JOB="Validate - Code Quality" python3 scripts/ci_notify_failure.py
    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Validate - Code Quality" python3 scripts/ci_notify.py
    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  validate-type-check:
    name: Validate - Type Check (mypy)
    runs-on: ci-l2
    timeout-minutes: 8
    permissions:
      contents: read
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Install dependencies
      shell: sh
      run: |
        set -eu
        for i in 1 2 3; do
          python3 -m pip install -q -r requirements-base.txt && break
          echo "pip install requirements-base.txt 失败，重试 $i/3..."
          [ $i -eq 3 ] && exit 1
          sleep 5
        done
        for i in 1 2 3; do
          python3 -m pip install -q -r requirements.txt && break
          echo "pip install requirements.txt 失败，重试 $i/3..."
          [ $i -eq 3 ] && exit 1
          sleep 5
        done
        for i in 1 2 3; do
          python3 -m pip install -q -r requirements-dev.txt && break
          echo "pip install requirements-dev.txt 失败，重试 $i/3..."
          [ $i -eq 3 ] && exit 1
          sleep 5
        done
    - name: Run mypy type check
      shell: bash
      run: bash scripts/ci/validate_mypy.sh
    - name: CI failure notification
      if: failure()
      shell: sh
      env:
        GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
        CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
      run: |
        set +e
        FAILED_JOB="Validate - Type Check (mypy)" python3 scripts/ci_notify_failure.py
    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Validate - Type Check (mypy)" python3 scripts/ci_notify.py
    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  validate-migration:
    name: Validate - Migration (alembic)
    runs-on: ci-l2
    timeout-minutes: 8
    permissions:
      contents: read
    env:
      DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas
      USE_IN_MEMORY_DB: 'false'
      CI_USE_SHARED_PG: 'true'
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Install dependencies
      shell: sh
      run: |
        set -eu
        for i in 1 2 3; do
          python3 -m pip install -q -r requirements-base.txt && break
          echo "pip install requirements-base.txt 失败，重试 $i/3..."
          [ $i -eq 3 ] && exit 1
          sleep 5
        done
        for i in 1 2 3; do
          python3 -m pip install -q -r requirements.txt && break
          echo "pip install requirements.txt 失败，重试 $i/3..."
          [ $i -eq 3 ] && exit 1
          sleep 5
        done
        for i in 1 2 3; do
          python3 -m pip install -q -r requirements-dev.txt && break
          echo "pip install requirements-dev.txt 失败，重试 $i/3..."
          [ $i -eq 3 ] && exit 1
          sleep 5
        done
    - name: Run alembic migration validation
      shell: bash
      run: bash scripts/ci/validate_migration.sh
    - name: CI failure notification
      if: failure()
      shell: sh
      env:
        GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
        CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
      run: |
        set +e
        FAILED_JOB="Validate - Migration (alembic)" python3 scripts/ci_notify_failure.py
    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Validate - Migration (alembic)" python3 scripts/ci_notify.py
    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  unit-tests:
    needs: check-frontend-only
    if: always() && needs.check-frontend-only.outputs.skip_backend != 'true'
    name: Unit Tests
    runs-on: ci-l2
    timeout-minutes: 8
    env:
      USE_IN_MEMORY_DB: 'true'
      OSS_ACCESS_KEY_ID: placeholder
      OSS_ACCESS_KEY_SECRET: placeholder
      OSS_BUCKET_NAME: xiaoxia-autocut
      OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Install ffmpeg
      shell: sh
      run: bash scripts/ci/step_install_ffmpeg.sh
    - name: Run unit tests with coverage
      shell: bash
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: bash scripts/ci/run_unit_tests.sh
    - name: CI failure notification
      if: failure()
      shell: sh
      env:
        GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
        CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
      run: |
        set +e
        FAILED_JOB="Unit Tests" python3 scripts/ci_notify_failure.py
    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Unit Tests" python3 scripts/ci_notify.py

    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  integration-tests:
    name: Integration Tests
    runs-on: ci-l2
    timeout-minutes: 30
    if: always() && needs.check-frontend-only.outputs.skip_backend != 'true'
    needs:
      - check-frontend-only
      - validate-code-quality
      - validate-type-check
      - validate-migration
    env:
      DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas
      USE_IN_MEMORY_DB: 'false'
      CI_USE_SHARED_PG: 'true'
      OSS_ACCESS_KEY_ID: placeholder
      OSS_ACCESS_KEY_SECRET: placeholder
      OSS_BUCKET_NAME: xiaoxia-autocut
      OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Run integration tests
      shell: bash
      run: bash scripts/ci/run_integration_tests.sh
    - name: CI failure notification
      if: failure()
      shell: sh
      env:
        GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
        CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
      run: |
        set +e
        FAILED_JOB="Integration Tests" python3 scripts/ci_notify_failure.py
    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Integration Tests" python3 scripts/ci_notify.py

    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  frontend-lint:
    name: Frontend Lint
    runs-on: ci-l2
    timeout-minutes: 10
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Install frontend dependencies (with retry)
      shell: sh
      run: |
        set -eu
        # npm install 带重试（网络不稳定时自动重试）
        for i in 1 2 3; do
          bash scripts/ci/step_frontend_install.sh && break
          echo "前端依赖安装失败，重试 $i/3..."
          [ $i -eq 3 ] && exit 1
          sleep 5
        done
    - name: Run ESLint
      shell: sh
      run: bash scripts/ci/step_frontend_run.sh "npx --no-install eslint src --ext .ts,.tsx --max-warnings 0"
    - name: Run TypeScript type check
      shell: sh
      run: bash scripts/ci/step_frontend_run.sh "npx --no-install tsc --noEmit"
    - name: Run Prettier check
      shell: sh
      run: bash scripts/ci/step_frontend_run.sh "npx --no-install prettier --check \"src/**/*.{ts,tsx,md}\""
    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Frontend Lint" python3 scripts/ci_notify.py

    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  frontend-unit-test:
    name: Frontend Unit Tests
    runs-on: ci-l2
    timeout-minutes: 15
    needs: check-frontend-only
    if: always() && needs.check-frontend-only.outputs.skip_frontend != 'true'
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Install frontend dependencies (vitest only, with retry)
      shell: sh
      run: |
        set -eu
        for i in 1 2 3; do
          bash scripts/ci/step_frontend_install.sh vitest && break
          echo "前端依赖安装失败，重试 $i/3..."
          [ $i -eq 3 ] && exit 1
          sleep 5
        done
    - name: Run Vitest (incremental for PRs, full for main branches)
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: bash scripts/ci/vitest_incremental.sh
    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Frontend Unit Tests" python3 scripts/ci_notify.py

    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true


  build-pr:
    name: PR Build ${{ matrix.service_display }} Image
    runs-on: runtime-builder
    timeout-minutes: ${{ matrix.timeout }}
    if: github.event_name == 'pull_request'
    strategy:
      fail-fast: false
      matrix:
        include:
        - service: api
          service_display: API
          dockerfile: infra/docker/api.Dockerfile
          image_name: xiaoxia-saas-api
          cache_name: api-cache
          timeout: 30
        - service: worker
          service_display: Worker
          dockerfile: infra/docker/worker.Dockerfile
          image_name: xiaoxia-saas-worker
          cache_name: worker-cache
          timeout: 40
        - service: web
          service_display: Web
          dockerfile: infra/docker/web.Dockerfile
          image_name: xiaoxia-saas-web
          cache_name: web-cache
          timeout: 30
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Docker login to Registry (for cache read)
      shell: sh
      env:
        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
        GITEA_REGISTRY_USER: xiaoxia
        GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
      run: |
        set -eu
        for i in 1 2 3; do
          echo "Docker login attempt $i/3"
          if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin              && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then
            echo "Docker login successful"
            break
          fi
          echo "Docker login failed ($i/3), retrying in 5s..."
          sleep 5
        done
    - name: Pre-build worker base images (fallback if not exist)
      if: matrix.service == 'worker'
      id: prebuild
      shell: sh
      run: |
        set -eu
        REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas"
        BASE_BUILDER="${REGISTRY}/worker-base-builder:latest"
        BASE_RUNTIME="${REGISTRY}/worker-base-runtime:latest"
        
        # 尝试拉取基础镜像
        echo "检查基础镜像..."
        if docker pull "$BASE_BUILDER" 2>/dev/null && docker pull "$BASE_RUNTIME" 2>/dev/null; then
          echo "基础镜像已存在，使用远程镜像"
          echo "fallback=false" >> $GITHUB_OUTPUT
        else
          echo "基础镜像不存在，本地构建（fallback模式）..."
          
          # 尝试用buildx构建，失败则回退到普通docker build（DooD模式下buildx builder偶发崩溃）
          BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}"
          BUILDX_AVAILABLE=true
          if ! docker buildx create --use --name "$BUILDER_NAME" --driver docker-container > /dev/null 2>&1; then
            BUILDX_AVAILABLE=false
          fi
          if [ "$BUILDX_AVAILABLE" = true ] && ! docker buildx inspect --bootstrap > /dev/null 2>&1; then
            BUILDX_AVAILABLE=false
            docker buildx rm "$BUILDER_NAME" > /dev/null 2>&1 || true
          fi
          
          build_base() {
            local df="$1"
            local tag="$2"
            local name="$3"
            if [ "$BUILDX_AVAILABLE" = true ]; then
              echo "构建 $name（buildx）..."
              if docker buildx build --load -f "$df" -t "$tag" . > /dev/null 2>&1; then
                echo "$name 构建成功"
                return 0
              fi
              echo "buildx失败，回退到普通docker build"
              BUILDX_AVAILABLE=false
              docker buildx rm "$BUILDER_NAME" > /dev/null 2>&1 || true
            fi
            echo "构建 $name（docker build）..."
            docker build -f "$df" -t "$tag" .
          }
          
          build_base infra/docker/worker-base-builder.Dockerfile "$BASE_BUILDER" "worker-base-builder"
          build_base infra/docker/worker-base-runtime.Dockerfile "$BASE_RUNTIME" "worker-base-runtime"
          
          echo "fallback=true" >> $GITHUB_OUTPUT
          echo "基础镜像本地构建完成"
        fi

    - name: Build PR image (verify only, no push)
      shell: sh
      run: |
        set -eu
        REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
        IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:pr-${GITHUB_SHA}"
        CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:develop"

        EXTRA_BUILD_ARGS="APP_VERSION=\"${GITHUB_SHA}\""
        if [ "${{ matrix.service }}" = "web" ]; then
          EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
        fi

        # Worker fallback模式：基础镜像本地已构建，用普通docker build绕过buildx
        if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.fallback }}" = "true" ]; then
          echo "Fallback模式：用普通docker build（基础镜像本地已构建）"
          BUILD_ARG_STR=""
          for arg in $EXTRA_BUILD_ARGS; do
            BUILD_ARG_STR="$BUILD_ARG_STR --build-arg $arg"
          done
          docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR .
          echo "Fallback PR Build successful"
          exit 0
        fi

        NO_CACHE_FLAG=""
        for i in 1 2 3; do
          echo "PR Build attempt $i/3"
          if bash scripts/ci/docker_build_only.sh $NO_CACHE_FLAG ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS; then
            echo "PR Build successful"
            break
          fi
          echo "PR Build failed (attempt $i/3)"
          [ $i -eq 3 ] && exit 1
          sleep 10
          if [ $i -eq 2 ]; then
            NO_CACHE_FLAG="--no-cache"
            echo "Next retry with --no-cache"
          fi
        done
        echo
        echo "${{ matrix.service_display }} PR build verified: ${IMAGE_TAG}"
    - name: Cleanup buildx builder
      if: always()
      shell: sh
      run: |
        BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}"
        docker buildx rm "$BUILDER_NAME" 2>/dev/null || true
        docker buildx prune -f 2>/dev/null || true
        echo "Builder cleanup done"
    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="PR Build ${{ matrix.service_display }} Image" python3 scripts/ci_notify.py
    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  build-staging:
    name: Build Staging ${{ matrix.service_display }} Image
    runs-on: runtime-builder
    timeout-minutes: ${{ matrix.timeout }}
    if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop')
    strategy:
      fail-fast: false
      matrix:
        include:
        - service: api
          service_display: API
          dockerfile: infra/docker/api.Dockerfile
          image_name: xiaoxia-saas-api
          cache_name: api-cache
          timeout: 30
        - service: worker
          service_display: Worker
          dockerfile: infra/docker/worker.Dockerfile
          image_name: xiaoxia-saas-worker
          cache_name: worker-cache
          timeout: 40
        - service: web
          service_display: Web
          dockerfile: infra/docker/web.Dockerfile
          image_name: xiaoxia-saas-web
          cache_name: web-cache
          timeout: 30
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Docker login to Registry
      shell: sh
      env:
        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
        GITEA_REGISTRY_USER: xiaoxia
        GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
      run: |
        set -eu
        # Docker login 带重试（网络波动时自动重试）
        for i in 1 2 3; do
          echo "=== Docker login 尝试 $i/3 ==="
          if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin              && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then
            echo "✅ Docker login successful"
            break
          fi
          echo "❌ Docker login 失败（尝试 $i/3），5s 后重试..."
          sleep 5
        done
    - name: Setup cache strategy
      shell: sh
      run: |
        set -eu
        if [ "${GITHUB_REF_NAME}" = "develop" ] || [ "${GITHUB_REF_NAME}" = "main" ]; then
          echo "CACHE_MODE=read-write" >> $GITHUB_ENV
          echo "Cache mode: read-write (will push cache)"
        else
          echo "CACHE_MODE=read-only" >> $GITHUB_ENV
          echo "Cache mode: read-only"
        fi

    - name: Setup buildx builder
      shell: sh
      run: |
        set -eu
        if ! docker buildx inspect ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} > /dev/null 2>&1; then
          docker buildx create --use --name ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} --driver docker-container
          echo "Created ci-builder (docker-container driver)"
        else
          docker buildx use ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }}
          echo "Using existing ci-builder"
        fi
        docker buildx inspect --bootstrap

    - name: Build and push ${{ matrix.service_display }} image (with retry)
      shell: sh
      run: |
        set -eu
        REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
        IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:${GITHUB_SHA}"
        CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:${GITHUB_REF_NAME}"

        EXTRA_BUILD_ARGS="APP_VERSION=\"${GITHUB_SHA}\""
        if [ "${{ matrix.service }}" = "web" ]; then
          EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
        fi

        # Docker build 带重试：失败自动重试2次，第2次重试加--no-cache
        NO_CACHE_FLAG=""
        for i in 1 2 3; do
          echo "=== Docker build 尝试 $i/3 ==="
          if bash scripts/ci/docker_build_push.sh $NO_CACHE_FLAG ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS; then
            echo "✅ Docker build 成功"
            break
          fi
          echo "❌ Docker build 失败（尝试 $i/3）"
          [ $i -eq 3 ] && exit 1
          sleep 10
          # 第2次重试使用 --no-cache
          if [ $i -eq 2 ]; then
            NO_CACHE_FLAG="--no-cache"
            echo "下次重试将使用 --no-cache"
          fi
        done

        echo
        echo "${{ matrix.service_display }} image pushed: ${IMAGE_TAG}"
    - name: Cleanup buildx builder
      if: always()
      shell: sh
      run: |
        docker buildx rm ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} 2>/dev/null || true
        docker buildx rm ci-builder 2>/dev/null || true
        docker buildx prune -f 2>/dev/null || true
        echo "Builder cleanup done"

    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Build Staging ${{ matrix.service_display }} Image" python3 scripts/ci_notify.py

    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  deploy-staging:
    name: Deploy Staging (Watchtower auto-deploy)
    runs-on: runtime-builder
    timeout-minutes: 15
    concurrency:
      group: deploy-staging-${{ gitea.ref }}
      cancel-in-progress: false
    needs:
    - build-staging
    if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop')
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Notify job start
      continue-on-error: true
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=start JOB_NAME="Deploy Staging" python3 scripts/ci_notify.py
    - name: Docker login to Registry
      shell: sh
      env:
        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
        GITEA_REGISTRY_USER: xiaoxia
        GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
      run: |
        set -eu
        # Docker login 带重试（网络波动时自动重试）
        for i in 1 2 3; do
          echo "=== Docker login 尝试 $i/3 ==="
          if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin              && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then
            echo "✅ Docker login successful"
            break
          fi
          echo "❌ Docker login 失败（尝试 $i/3），5s 后重试..."
          sleep 5
        done
    - name: Install SSH client
      if: success()
      shell: sh
      run: |
        set -eu
        apt-get update -qq && apt-get install -y -qq openssh-client >/dev/null 2>&1
        echo "openssh-client installed"

    - name: Deploy staging over SSH (Registry pull)
      if: success()
      shell: sh
      env:
        STAGING_SSH_HOST: ${{ secrets.STAGING_SSH_HOST }}
        STAGING_SSH_USER: ${{ secrets.STAGING_SSH_USER }}
        STAGING_SSH_PORT: ${{ secrets.STAGING_SSH_PORT }}
        STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
      run: |
        set -eux
        staging_host="${STAGING_SSH_HOST:-47.98.113.167}"
        staging_user="${STAGING_SSH_USER:-root}"
        staging_port="${STAGING_SSH_PORT:-22222}"
        echo "Host: $staging_host"
        echo "Port: $staging_port"

        mkdir -p ~/.ssh

        key_path=""
        if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
          key_path="/root/.ssh/xiaoxia_runtime_builder"
          echo "Using key: $key_path (builder key)"
        elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then
          key_path="$HOME/.ssh/xiaoxia_runtime_builder"
          echo "Using key: $key_path (home key)"
        elif [ -n "${STAGING_SSH_KEY:-}" ]; then
          key_path="$HOME/.ssh/id_ed25519"
          printf '%s\n' "$STAGING_SSH_KEY" > "$key_path"
          chmod 600 "$key_path"
          echo "Using key from STAGING_SSH_KEY secret"
        else
          echo "ERROR: No SSH key available"
          ls -la ~/.ssh/ 2>/dev/null || true
          ls -la /root/.ssh/ 2>/dev/null || true
          exit 1
        fi

        ssh-keyscan -p "$staging_port" -H "$staging_host" >> ~/.ssh/known_hosts 2>/dev/null
        echo "SSH keyscan done"

        ssh -p "$staging_port" -i "$key_path" -o StrictHostKeyChecking=no "${staging_user}@${staging_host}" "echo SSH_CONNECTION_OK && hostname"
        echo "SSH connection verified"

        # 通过环境变量传递凭证，避免命令行引号转义问题
        cat scripts/ci_staging_deploy.sh | ssh -p "$staging_port" -i "$key_path" -o StrictHostKeyChecking=no "${staging_user}@${staging_host}" "IMAGE_TAG=${GITHUB_SHA} ACR_USERNAME=${ACR_USERNAME} ACR_PASSWORD=${ACR_PASSWORD} sh"

    - name: Staging health check + auto rollback
      if: success()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
        STAGING_SSH_HOST: ${{ secrets.STAGING_SSH_HOST }}
        STAGING_SSH_USER: ${{ secrets.STAGING_SSH_USER }}
        STAGING_SSH_PORT: ${{ secrets.STAGING_SSH_PORT }}
        STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
      run: |
        set -eu
        echo "=========================================="
        echo "  Staging 健康检查（Watchtower 模式）"
        echo "=========================================="
        echo ""
        bash scripts/ci_staging_healthcheck.sh
    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on success
      continue-on-error: true
      if: success()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=success JOB_NAME="Deploy Staging" python3 scripts/ci_notify.py
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Deploy Staging" python3 scripts/ci_notify.py

    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  staging-e2e:
    name: Staging E2E Tests
    runs-on: runtime-builder
    timeout-minutes: 15
    if: github.ref_name == 'develop' || github.ref_name == 'main'
    needs: deploy-staging
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Run Playwright E2E on staging
      shell: bash
      run: |
        set -eu
        # DooD模式下不能用-v挂载（宿主机路径与CI容器路径不一致）
        # 改用 docker create + docker cp 方式把代码拷进容器
        CONTAINER_NAME="staging-e2e-$$"
        docker create --name "$CONTAINER_NAME" --ipc=host \
          -e E2E_BASE_URL=https://staging.xiaoxiajianji.com \
          -e E2E_API_BASE=https://staging-api.xiaoxiajianji.com/api/v1 \
          -e E2E_BROWSER_CHANNEL=chromium \
          -e PLAYWRIGHT_HEADLESS=1 \
          -w /workspace/apps/web \
          git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy \
          sh -lc "npm ci && npx playwright test --reporter=line --project=chromium e2e/auth.spec.ts e2e/auth-guard.spec.ts e2e/core-upload.spec.ts e2e/core-generation.spec.ts"
        docker cp apps "$CONTAINER_NAME:/workspace/"
        docker cp package-lock.json "$CONTAINER_NAME:/workspace/" 2>/dev/null || true
        docker start -a "$CONTAINER_NAME"
        EXIT_CODE=$(docker wait "$CONTAINER_NAME")
        docker rm "$CONTAINER_NAME" 2>/dev/null || true
        exit $EXIT_CODE

    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Staging E2E Tests" python3 scripts/ci_notify.py

    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  staging-api-tests:
    name: Staging API Integration Tests
    runs-on: runtime-builder
    timeout-minutes: 10
    if: github.ref_name == 'develop' || github.ref_name == 'main'
    needs: deploy-staging
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Run API integration tests on staging
      shell: bash
      run: |
        set -eu
        # DooD模式下不能用-v挂载（宿主机路径与CI容器路径不一致）
        # 改用 docker create + docker cp 方式把代码拷进容器
        CONTAINER_NAME="staging-api-tests-$$"
        docker create --name "$CONTAINER_NAME" \
          -e E2E_BASE_URL=https://staging.xiaoxiajianji.com \
          -e E2E_API_BASE=https://staging-api.xiaoxiajianji.com/api/v1 \
          -w /workspace/apps/web \
          git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy \
          sh -lc 'npm ci && npx playwright test --reporter=line e2e/test_auth.spec.ts e2e/test_asset.spec.ts e2e/test_project.spec.ts'
        docker cp apps "$CONTAINER_NAME:/workspace/"
        docker cp package-lock.json "$CONTAINER_NAME:/workspace/" 2>/dev/null || true
        docker start -a "$CONTAINER_NAME"
        EXIT_CODE=$(docker wait "$CONTAINER_NAME")
        docker rm "$CONTAINER_NAME" 2>/dev/null || true
        exit $EXIT_CODE

    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Staging API Integration Tests" python3 scripts/ci_notify.py

    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  build-production:
    name: Build Production ${{ matrix.service_display }} Image
    runs-on: runtime-builder
    timeout-minutes: ${{ matrix.timeout }}
    needs:
    if: startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'push' && github.ref_name == 'main')
    strategy:
      fail-fast: false
      matrix:
        include:
        - service: api
          service_display: API
          dockerfile: infra/docker/api.Dockerfile
          image_name: xiaoxia-saas-api
          cache_name: api-cache
          timeout: 30
        - service: worker
          service_display: Worker
          dockerfile: infra/docker/worker.Dockerfile
          image_name: xiaoxia-saas-worker
          cache_name: worker-cache
          timeout: 40
        - service: web
          service_display: Web
          dockerfile: infra/docker/web.Dockerfile
          image_name: xiaoxia-saas-web
          cache_name: web-cache
          timeout: 30
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Docker login to Registry
      shell: sh
      env:
        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
        GITEA_REGISTRY_USER: xiaoxia
        GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
      run: |
        set -eu
        # Docker login 带重试（网络波动时自动重试）
        for i in 1 2 3; do
          echo "=== Docker login 尝试 $i/3 ==="
          if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin              && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then
            echo "✅ Docker login successful"
            break
          fi
          echo "❌ Docker login 失败（尝试 $i/3），5s 后重试..."
          sleep 5
        done
    - name: Setup cache strategy
      shell: sh
      run: |
        set -eu
        echo "CACHE_MODE=read-only" >> $GITHUB_ENV
        echo "Cache mode: read-only (production build uses cached layers)"

    - name: Setup buildx builder
      shell: sh
      run: |
        set -eu
        if ! docker buildx inspect ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} > /dev/null 2>&1; then
          docker buildx create --use --name ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} --driver docker-container
          echo "Created ci-builder"
        else
          docker buildx use ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }}
          echo "Using existing ci-builder"
        fi
        docker buildx inspect --bootstrap

    - name: Build and push production ${{ matrix.service_display }} image (with retry)
      shell: sh
      run: |
        set -eu
        REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
        # 根据ref类型设置镜像标签：tag用版本号，分支用分支名+sha
        if [[ "$GITHUB_REF" == refs/tags/* ]]; then
          TAG_NAME="${GITHUB_REF_NAME}"
        else
          TAG_NAME="${GITHUB_REF_NAME}-${GITHUB_SHA::8}"
        fi
        IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:${TAG_NAME}"
        CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:main"

        EXTRA_BUILD_ARGS="APP_VERSION=\"${TAG_NAME}\""
        if [ "${{ matrix.service }}" = "web" ]; then
          EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-production.conf"
        fi

        # Docker build 带重试：失败自动重试2次，第2次重试加--no-cache
        NO_CACHE_FLAG=""
        for i in 1 2 3; do
          echo "=== Docker build 尝试 $i/3 ==="
          if bash scripts/ci/docker_build_push.sh $NO_CACHE_FLAG ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS; then
            echo "✅ Docker build 成功"
            break
          fi
          echo "❌ Docker build 失败（尝试 $i/3）"
          [ $i -eq 3 ] && exit 1
          sleep 10
          # 第2次重试使用 --no-cache
          if [ $i -eq 2 ]; then
            NO_CACHE_FLAG="--no-cache"
            echo "下次重试将使用 --no-cache"
          fi
        done

        echo
        echo "${{ matrix.service_display }} production image pushed: ${IMAGE_TAG}"
    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Build Production ${{ matrix.service_display }} Image" python3 scripts/ci_notify.py

    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  deploy-production:
    name: Deploy Production
    runs-on: runtime-builder
    timeout-minutes: 30
    concurrency:
      group: deploy-production-${{ gitea.ref }}
      cancel-in-progress: false
    if: startsWith(github.ref, 'refs/tags/v')
    needs:
    - build-production
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Notify job start
      continue-on-error: true
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=start JOB_NAME="Deploy Production" python3 scripts/ci_notify.py
    - name: Install SSH client
      shell: sh
      run: |
        set -eu
        apt-get update -qq && apt-get install -y -qq openssh-client >/dev/null 2>&1
        echo "openssh-client installed"
    - name: Deploy production over SSH (Registry pull)
      if: success()
      shell: sh
      env:
        PRODUCTION_SSH_HOST: ${{ secrets.PRODUCTION_SSH_HOST }}
        PRODUCTION_SSH_USER: ${{ secrets.PRODUCTION_SSH_USER }}
        PRODUCTION_SSH_PORT: ${{ secrets.PRODUCTION_SSH_PORT }}
        PRODUCTION_SSH_KEY: ${{ secrets.PRODUCTION_SSH_KEY }}
        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
      run: |
        set -eux
        production_host="${PRODUCTION_SSH_HOST:-47.98.113.167}"
        production_user="${PRODUCTION_SSH_USER:-root}"
        production_port="${PRODUCTION_SSH_PORT:-22222}"
        echo "Host: $production_host"
        echo "Port: $production_port"

        mkdir -p ~/.ssh

        key_path=""
        if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
          key_path="/root/.ssh/xiaoxia_runtime_builder"
          echo "Using key: $key_path (builder key)"
        elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then
          key_path="$HOME/.ssh/xiaoxia_runtime_builder"
          echo "Using key: $key_path (home key)"
        elif [ -n "${PRODUCTION_SSH_KEY:-}" ]; then
          key_path="$HOME/.ssh/id_ed25519"
          printf '%s\n' "$PRODUCTION_SSH_KEY" > "$key_path"
          chmod 600 "$key_path"
          echo "Using key from PRODUCTION_SSH_KEY secret"
        else
          echo "ERROR: No SSH key available"
          ls -la ~/.ssh/ 2>/dev/null || true
          ls -la /root/.ssh/ 2>/dev/null || true
          exit 1
        fi

        ssh-keyscan -p "$production_port" -H "$production_host" >> ~/.ssh/known_hosts 2>/dev/null
        echo "SSH keyscan done"

        ssh -p "$production_port" -i "$key_path" -o StrictHostKeyChecking=no "${production_user}@${production_host}" "echo SSH_CONNECTION_OK && hostname"
        echo "SSH connection verified"

        # 通过环境变量传递凭证，避免命令行引号转义问题
        cat scripts/ci_production_deploy.sh | ssh -p "$production_port" -i "$key_path" -o StrictHostKeyChecking=no "${production_user}@${production_host}" "IMAGE_TAG=${GITHUB_REF_NAME} ACR_USERNAME=${ACR_USERNAME} ACR_PASSWORD=${ACR_PASSWORD} sh"

    - name: Production health check + auto rollback
      if: success()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
        PRODUCTION_SSH_HOST: ${{ secrets.PRODUCTION_SSH_HOST }}
        PRODUCTION_SSH_USER: ${{ secrets.PRODUCTION_SSH_USER }}
        PRODUCTION_SSH_PORT: ${{ secrets.PRODUCTION_SSH_PORT }}
        PRODUCTION_SSH_KEY: ${{ secrets.PRODUCTION_SSH_KEY }}
        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
      run: |
        set -eu
        echo "=========================================="
        echo "  Production 健康检查 + 自动回滚"
        echo "=========================================="
        echo ""
        bash scripts/ci_production_healthcheck.sh
    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on success
      continue-on-error: true
      if: success()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=success JOB_NAME="Deploy Production" python3 scripts/ci_notify.py
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Deploy Production" python3 scripts/ci_notify.py

    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  production-e2e:
    name: Production Browser E2E
    runs-on: runtime-builder
    timeout-minutes: 15
    if: startsWith(github.ref, 'refs/tags/v')
    needs: deploy-production
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Run production browser E2E
      shell: sh
      run: |
        set -eu
        docker run --rm --ipc=host \
          -e E2E_BASE_URL=https://saas.xiaoxiajianji.com \
          -e E2E_API_BASE=https://api.xiaoxiajianji.com/api/v1 \
          -e E2E_BROWSER_CHANNEL=chromium \
          -e PLAYWRIGHT_HEADLESS=1 \
          -v "$PWD:/workspace" \
          -w /workspace/apps/web \
          git.xiaoxiajianji.com/xiaoxia/base/playwright:v1.45.0-jammy \
          sh -lc 'npm ci && npx playwright test --reporter=line --project=chromium e2e/auth.spec.ts e2e/auth-guard.spec.ts e2e/core-upload.spec.ts e2e/core-generation.spec.ts e2e/core-titles.spec.ts'

    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Production Browser E2E" python3 scripts/ci_notify.py

    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  acr-cleanup:
    name: ACR Image Cleanup
    runs-on: runtime-builder
    timeout-minutes: 10
    needs:
    - deploy-staging
    if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop')
    env:
      ACR_REGISTRY: xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com
      ACR_NAMESPACE: xiaoxiakeji
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Run ACR cleanup
      shell: sh
      env:
        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
        PROTECTED_TAG: ${{ github.sha }}
      run: |
        set -eu
        python3 scripts/ci/acr_cleanup.py \
          --keep 20 \
          --pr-days 7 \
          --execute

    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="ACR Image Cleanup" python3 scripts/ci_notify.py
    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true

  canary-release:
    name: Canary Release to Production
    runs-on: runtime-builder
    timeout-minutes: 120
    concurrency:
      group: canary-release-production
      cancel-in-progress: false
    if: github.event_name == 'push' && github.ref_name == 'main'
    needs:
    - build-production
    - staging-api-tests
    steps:
    - name: Checkout code
      shell: sh
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
    - name: Record job start time
      shell: sh
      run: bash scripts/ci/step_timer_start.sh
    - name: Notify canary release start
      continue-on-error: true
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=start JOB_NAME="Canary Release" python3 scripts/ci_notify.py
    - name: Install SSH client
      shell: sh
      run: |
        set -eu
        apt-get update -qq && apt-get install -y -qq openssh-client curl >/dev/null 2>&1
        echo "openssh-client installed"
    - name: Run canary release
      shell: bash
      env:
        PRODUCTION_SSH_HOST: ${{ secrets.PRODUCTION_SSH_HOST }}
        PRODUCTION_SSH_USER: ${{ secrets.PRODUCTION_SSH_USER }}
        PRODUCTION_SSH_PORT: ${{ secrets.PRODUCTION_SSH_PORT }}
        PRODUCTION_SSH_KEY: ${{ secrets.PRODUCTION_SSH_KEY }}
        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set -eu
        IMAGE_TAG="main-${GITHUB_SHA::8}"
        export IMAGE_TAG
        echo "Canary release version: $IMAGE_TAG"
        bash scripts/ci/canary_release.sh
    - name: Job duration summary
      if: always()
      shell: sh
      run: bash scripts/ci/step_timer_end.sh
    - name: Notify on success
      continue-on-error: true
      if: success()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=success JOB_NAME="Canary Release" python3 scripts/ci_notify.py
    - name: Notify on failure
      continue-on-error: true
      if: failure()
      shell: sh
      env:
        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
      run: |
        set +e
        NOTIFY_MODE=failure JOB_NAME="Canary Release" python3 scripts/ci_notify.py
    - name: Report CI trace
      if: always()
      shell: sh
      env:
        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
      run: |
        STATUS="ok"
        [ ${{ job.status }} = "success" ] || STATUS="error"
        START_TIME=""
        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
