chore: sync ci-pipeline and validate scripts from develop

.gitea/workflows/ci-pipeline.yml

@@ -172,6 +172,250 @@ jobs:
         [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
         python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
 
+
+  validate-code-quality:
+    name: Validate - Code Quality
+    runs-on: ci-l2
+    timeout-minutes: 8
+    permissions:
+      contents: write
+    steps:
+    - name: Checkout code
+      shell: sh
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
+    - name: Record job start time
+      shell: sh
+      run: bash scripts/ci/step_timer_start.sh
+    - name: Install dependencies
+      shell: sh
+      run: |
+        set -eu
+        for i in 1 2 3; do
+          python3 -m pip install -q -r requirements-base.txt && break
+          echo "pip install requirements-base.txt 失败，重试 $i/3..."
+          [ $i -eq 3 ] && exit 1
+          sleep 5
+        done
+        for i in 1 2 3; do
+          python3 -m pip install -q -r requirements.txt && break
+          echo "pip install requirements.txt 失败，重试 $i/3..."
+          [ $i -eq 3 ] && exit 1
+          sleep 5
+        done
+        for i in 1 2 3; do
+          python3 -m pip install -q -r requirements-dev.txt && break
+          echo "pip install requirements-dev.txt 失败，重试 $i/3..."
+          [ $i -eq 3 ] && exit 1
+          sleep 5
+        done
+        for i in 1 2 3; do
+          python3 -m pip install --no-binary :all: black==26.5.1 isort==8.0.1 && break
+          echo "pip install black/isort 失败，重试 $i/3..."
+          [ $i -eq 3 ] && exit 1
+          sleep 5
+        done
+    - name: Run code quality and security checks
+      shell: bash
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: bash scripts/ci/validate_code_quality.sh
+    - name: Auto-fix formatting (black + isort)
+      if: failure()
+      shell: sh
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: python3 scripts/ci/auto_fix_formatting.py
+    - name: CI failure notification
+      if: failure()
+      shell: sh
+      env:
+        GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
+      run: |
+        set +e
+        FAILED_JOB="Validate - Code Quality" python3 scripts/ci_notify_failure.py
+    - name: Job duration summary
+      if: always()
+      shell: sh
+      run: bash scripts/ci/step_timer_end.sh
+    - name: Notify on failure
+      continue-on-error: true
+      if: failure()
+      shell: sh
+      env:
+        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
+      run: |
+        set +e
+        NOTIFY_MODE=failure JOB_NAME="Validate - Code Quality" python3 scripts/ci_notify.py
+    - name: Report CI trace
+      if: always()
+      shell: sh
+      env:
+        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
+      run: |
+        STATUS="ok"
+        [ ${{ job.status }} = "success" ] || STATUS="error"
+        START_TIME=""
+        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
+        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
+
+  validate-type-check:
+    name: Validate - Type Check (mypy)
+    runs-on: ci-l2
+    timeout-minutes: 8
+    permissions:
+      contents: read
+    steps:
+    - name: Checkout code
+      shell: sh
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
+    - name: Record job start time
+      shell: sh
+      run: bash scripts/ci/step_timer_start.sh
+    - name: Install dependencies
+      shell: sh
+      run: |
+        set -eu
+        for i in 1 2 3; do
+          python3 -m pip install -q -r requirements-base.txt && break
+          echo "pip install requirements-base.txt 失败，重试 $i/3..."
+          [ $i -eq 3 ] && exit 1
+          sleep 5
+        done
+        for i in 1 2 3; do
+          python3 -m pip install -q -r requirements.txt && break
+          echo "pip install requirements.txt 失败，重试 $i/3..."
+          [ $i -eq 3 ] && exit 1
+          sleep 5
+        done
+        for i in 1 2 3; do
+          python3 -m pip install -q -r requirements-dev.txt && break
+          echo "pip install requirements-dev.txt 失败，重试 $i/3..."
+          [ $i -eq 3 ] && exit 1
+          sleep 5
+        done
+    - name: Run mypy type check
+      shell: bash
+      run: bash scripts/ci/validate_mypy.sh
+    - name: CI failure notification
+      if: failure()
+      shell: sh
+      env:
+        GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
+      run: |
+        set +e
+        FAILED_JOB="Validate - Type Check (mypy)" python3 scripts/ci_notify_failure.py
+    - name: Job duration summary
+      if: always()
+      shell: sh
+      run: bash scripts/ci/step_timer_end.sh
+    - name: Notify on failure
+      continue-on-error: true
+      if: failure()
+      shell: sh
+      env:
+        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
+      run: |
+        set +e
+        NOTIFY_MODE=failure JOB_NAME="Validate - Type Check (mypy)" python3 scripts/ci_notify.py
+    - name: Report CI trace
+      if: always()
+      shell: sh
+      env:
+        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
+      run: |
+        STATUS="ok"
+        [ ${{ job.status }} = "success" ] || STATUS="error"
+        START_TIME=""
+        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
+        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
+
+  validate-migration:
+    name: Validate - Migration (alembic)
+    runs-on: ci-l2
+    timeout-minutes: 8
+    permissions:
+      contents: read
+    env:
+      DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas
+      USE_IN_MEMORY_DB: 'false'
+      CI_USE_SHARED_PG: 'true'
+    steps:
+    - name: Checkout code
+      shell: sh
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        curl -sH "Authorization: token $GITHUB_TOKEN"           "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}"           | bash
+    - name: Record job start time
+      shell: sh
+      run: bash scripts/ci/step_timer_start.sh
+    - name: Install dependencies
+      shell: sh
+      run: |
+        set -eu
+        for i in 1 2 3; do
+          python3 -m pip install -q -r requirements-base.txt && break
+          echo "pip install requirements-base.txt 失败，重试 $i/3..."
+          [ $i -eq 3 ] && exit 1
+          sleep 5
+        done
+        for i in 1 2 3; do
+          python3 -m pip install -q -r requirements.txt && break
+          echo "pip install requirements.txt 失败，重试 $i/3..."
+          [ $i -eq 3 ] && exit 1
+          sleep 5
+        done
+        for i in 1 2 3; do
+          python3 -m pip install -q -r requirements-dev.txt && break
+          echo "pip install requirements-dev.txt 失败，重试 $i/3..."
+          [ $i -eq 3 ] && exit 1
+          sleep 5
+        done
+    - name: Run alembic migration validation
+      shell: bash
+      run: bash scripts/ci/validate_migration.sh
+    - name: CI failure notification
+      if: failure()
+      shell: sh
+      env:
+        GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        CI_WEBHOOK_URL: ${{ secrets.CI_WEBHOOK_URL }}
+      run: |
+        set +e
+        FAILED_JOB="Validate - Migration (alembic)" python3 scripts/ci_notify_failure.py
+    - name: Job duration summary
+      if: always()
+      shell: sh
+      run: bash scripts/ci/step_timer_end.sh
+    - name: Notify on failure
+      continue-on-error: true
+      if: failure()
+      shell: sh
+      env:
+        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
+      run: |
+        set +e
+        NOTIFY_MODE=failure JOB_NAME="Validate - Migration (alembic)" python3 scripts/ci_notify.py
+    - name: Report CI trace
+      if: always()
+      shell: sh
+      env:
+        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
+      run: |
+        STATUS="ok"
+        [ ${{ job.status }} = "success" ] || STATUS="error"
+        START_TIME=""
+        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
+        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
+
   unit-tests:
     needs: check-frontend-only
     if: always() && needs.check-frontend-only.outputs.skip_backend != 'true'
@@ -387,9 +631,11 @@ jobs:
           [ $i -eq 3 ] && exit 1
           sleep 5
         done
-    - name: Run Vitest with coverage
+    - name: Run Vitest (incremental for PRs, full for main branches)
       shell: sh
-      run: bash scripts/ci/step_frontend_run.sh "npx --no-install vitest run --coverage"
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: bash scripts/ci/vitest_incremental.sh
     - name: Job duration summary
       if: always()
       shell: sh
@@ -472,6 +718,36 @@ jobs:
           echo "Docker login failed ($i/3), retrying in 5s..."
           sleep 5
         done
+    - name: Pre-build worker base images (fallback if not exist)
+      if: matrix.service == 'worker'
+      id: prebuild
+      shell: sh
+      run: |
+        set -eu
+        REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas"
+        BASE_BUILDER="${REGISTRY}/worker-base-builder:latest"
+        BASE_RUNTIME="${REGISTRY}/worker-base-runtime:latest"
+        
+        # 尝试拉取基础镜像
+        echo "检查基础镜像..."
+        if docker pull "$BASE_BUILDER" 2>/dev/null && docker pull "$BASE_RUNTIME" 2>/dev/null; then
+          echo "基础镜像已存在，使用远程镜像"
+          echo "fallback=false" >> $GITHUB_OUTPUT
+        else
+          echo "基础镜像不存在，本地构建（fallback模式）..."
+          
+          # 构建builder基础镜像
+          echo "构建 worker-base-builder..."
+          docker build -f infra/docker/worker-base-builder.Dockerfile -t "$BASE_BUILDER" .
+          
+          # 构建runtime基础镜像
+          echo "构建 worker-base-runtime..."
+          docker build -f infra/docker/worker-base-runtime.Dockerfile -t "$BASE_RUNTIME" .
+          
+          echo "fallback=true" >> $GITHUB_OUTPUT
+          echo "基础镜像本地构建完成"
+        fi
+
     - name: Build PR image (verify only, no push)
       shell: sh
       run: |
@@ -485,6 +761,18 @@ jobs:
           EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
         fi
 
+        # Worker fallback模式：基础镜像本地已构建，用普通docker build绕过buildx
+        if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.fallback }}" = "true" ]; then
+          echo "Fallback模式：用普通docker build（基础镜像本地已构建）"
+          BUILD_ARG_STR=""
+          for arg in $EXTRA_BUILD_ARGS; do
+            BUILD_ARG_STR="$BUILD_ARG_STR --build-arg $arg"
+          done
+          docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR .
+          echo "Fallback PR Build successful"
+          exit 0
+        fi
+
         NO_CACHE_FLAG=""
         for i in 1 2 3; do
           echo "PR Build attempt $i/3"