From a64037881aba894d237a746d0b055989e95534cc Mon Sep 17 00:00:00 2001
From: xiaoxia <dev@xiaoxiajianji.com>
Date: Sat, 29 Aug 2026 11:28:12 +0800
Subject: [PATCH 1/8] =?UTF-8?q?perf(ci):=20=E6=B5=81=E6=B0=B4=E7=BA=BF?=
 =?UTF-8?q?=E6=8F=90=E9=80=9F=20-=20=E8=B7=AF=E5=BE=84=E8=BF=87=E6=BB=A4/?=
 =?UTF-8?q?=E6=8C=81=E4=B9=85builder=E7=BC=93=E5=AD=98/=E4=BE=9D=E8=B5=96?=
 =?UTF-8?q?=E7=BC=93=E5=AD=98=E5=8D=B7/=E9=83=A8=E7=BD=B2job=E8=BF=81?=
 =?UTF-8?q?=E7=A7=BBci-l2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

1. build-staging 路径过滤：纯前端改动只构建Web，纯后端只构建API/Worker；
   未重建镜像通过 retag-staging-skipped job 将分支tag retag为新SHA推送，Watchtower链路不变
2. deploy-staging、acr-cleanup runs-on 从 runtime-builder 改为 ci-l2，释放构建槽
3. buildx 改为宿主机持久 builder(ci-builder-persist)+buildkit命名卷缓存，
   job结束不再prune，Worker镜像同样接入buildx缓存
4. validate/unit-test/frontend-test 的 pip/npm 依赖通过宿主机命名卷持久化缓存
   （runner config 挂载 ci-pip-cache/xiaoxia-npm-cache/act-toolcache）
5. 不影响生产构建/发版/canary 相关 job
---
 .gitea/workflows/ci-pipeline.yml        | 231 +++++++++++++++++-------
 scripts/ci/ci_push_paths.sh             |  77 ++++++++
 scripts/ci/docker_build_only.sh         |  21 ++-
 scripts/ci/docker_build_push.sh         |  92 +++++-----
 scripts/ci/ensure_persistent_builder.sh |  25 +++
 scripts/ci/retag_skipped_image.sh       |  47 +++++
 6 files changed, 375 insertions(+), 118 deletions(-)
 create mode 100755 scripts/ci/ci_push_paths.sh
 create mode 100755 scripts/ci/ensure_persistent_builder.sh
 create mode 100755 scripts/ci/retag_skipped_image.sh

diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml
index a8955e571..8ba048574 100755
--- a/.gitea/workflows/ci-pipeline.yml
+++ b/.gitea/workflows/ci-pipeline.yml
@@ -696,14 +696,14 @@ jobs:
         done
         echo
         echo "${{ matrix.service_display }} PR build verified: ${IMAGE_TAG}"
-    - name: Cleanup buildx builder
+    - name: Builder cache note
       if: always()
       shell: sh
       run: |
-        BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}"
-        docker buildx rm "$BUILDER_NAME" 2>/dev/null || true
-        docker buildx prune -f 2>/dev/null || true
-        echo "Builder cleanup done"
+        # 持久 builder (ci-builder-persist) 跨 job 共享，不删除不 prune；
+        # 残留容器/卷由宿主机 /usr/local/bin/ci-docker-cleanup.sh 兜底清理
+        docker buildx ls | head -5
+        echo "Persistent builder kept warm for next job"
     - name: Job duration summary
       if: always()
       shell: sh
@@ -729,11 +729,49 @@ jobs:
         [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
         python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
 
+  check-push-paths:
+    name: Check push changed paths
+    runs-on: ci-l2
+    if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop')
+    outputs:
+      skip_backend: ${{ steps.check.outputs.skip_backend }}
+      skip_frontend: ${{ steps.check.outputs.skip_frontend }}
+    steps:
+    - name: Checkout code
+      shell: sh
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        curl -sfH "Authorization: token $GITHUB_TOKEN" -o /tmp/_ci_checkout.sh "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" && bash /tmp/_ci_checkout.sh
+    - name: Check changed paths
+      id: check
+      shell: bash
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        bash scripts/ci/ci_push_paths.sh
+    - name: Report CI trace
+      if: always()
+      shell: sh
+      env:
+        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
+      run: |
+        STATUS="ok"
+        [ ${{ job.status }} = "success" ] || STATUS="error"
+        START_TIME=""
+        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
+        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
+
   build-staging:
     name: Build Staging ${{ matrix.service_display }} Image
     runs-on: runtime-builder
     timeout-minutes: ${{ matrix.timeout }}
-    if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop')
+    needs: check-push-paths
+    if: |
+      github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop') && (
+        (matrix.service == 'web' && needs.check-push-paths.outputs.skip_frontend != 'true') ||
+        (matrix.service != 'web' && needs.check-push-paths.outputs.skip_backend != 'true')
+      )
     strategy:
       fail-fast: false
       matrix:
@@ -797,23 +835,16 @@ jobs:
           echo "Cache mode: read-only"
         fi
 
-    - name: Setup buildx builder
-      if: matrix.service != 'worker'
+    - name: Ensure persistent buildx builder
       shell: sh
-      run: |
-        set -eu
-        if ! docker buildx inspect ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} > /dev/null 2>&1; then
-          docker buildx create --use --name ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} --driver docker-container
-          echo "Created ci-builder (docker-container driver)"
-        else
-          docker buildx use ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }}
-          echo "Using existing ci-builder"
-        fi
-        docker buildx inspect --bootstrap
+      run: bash scripts/ci/ensure_persistent_builder.sh
 
-    - name: Pre-build worker base image (fallback if not exist)
+    - name: Pre-pull worker base image (fallback build if not exist)
       if: matrix.service == 'worker'
       shell: sh
+      env:
+        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
+        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
       run: |
         set -eu
         REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
@@ -821,11 +852,13 @@ jobs:
 
         echo "检查 Worker 基础镜像..."
         if docker pull "$BASE_IMAGE" 2>/dev/null; then
-          echo "✅ 基础镜像已存在"
+          echo "✅ 基础镜像已存在（buildkit 可直接命中）"
         else
-          echo "⚠️ 基础镜像不存在，本地构建（fallback）..."
-          docker build -f infra/docker/worker-base.Dockerfile -t "$BASE_IMAGE" .
-          echo "✅ Worker 基础镜像本地构建完成"
+          echo "⚠️ 基础镜像不存在，用持久 builder 构建并推送（fallback）..."
+          docker buildx build --builder ci-builder-persist \
+            -f infra/docker/worker-base.Dockerfile \
+            -t "$BASE_IMAGE" --push .
+          echo "✅ Worker 基础镜像构建推送完成"
         fi
 
     - name: Build and push ${{ matrix.service_display }} image
@@ -833,48 +866,34 @@ jobs:
       run: |
         set -eu
         REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
-        IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:${GITHUB_SHA}"
+        IMAGE_FULL="${REGISTRY}/${{ matrix.image_name }}"
+        IMAGE_TAG="${IMAGE_FULL}:${GITHUB_SHA}"
+        # 同时推分支 tag，作为未重建镜像 retag 的稳定来源
+        BRANCH_TAG="${IMAGE_FULL}:${GITHUB_REF_NAME}"
+        CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:${GITHUB_REF_NAME}"
 
-        if [ "${{ matrix.service }}" = "worker" ]; then
-          # Worker: plain docker build（基础镜像已预装全部依赖，无需 buildx）
-          echo "=== Worker: plain docker build ==="
-          docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" --build-arg APP_VERSION="${GITHUB_SHA}" .
-          docker push "${IMAGE_TAG}"
-          echo "✅ Worker image pushed: ${IMAGE_TAG}"
-        else
-          # API/Web: buildx with registry cache
-          CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:${GITHUB_REF_NAME}"
-          EXTRA_BUILD_ARGS="APP_VERSION=\"${GITHUB_SHA}\""
-          if [ "${{ matrix.service }}" = "web" ]; then
-            EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
-          fi
-
-          NO_CACHE_FLAG=""
-          for i in 1 2 3; do
-            echo "=== Docker build 尝试 $i/3 ==="
-            if bash scripts/ci/docker_build_push.sh $NO_CACHE_FLAG ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS; then
-              echo "✅ Docker build 成功"
-              break
-            fi
-            echo "❌ Docker build 失败（尝试 $i/3）"
-            [ $i -eq 3 ] && exit 1
-            sleep 10
-            if [ $i -eq 2 ]; then
-              NO_CACHE_FLAG="--no-cache"
-              echo "下次重试将使用 --no-cache"
-            fi
-          done
-
-          echo "${{ matrix.service_display }} image pushed: ${IMAGE_TAG}"
+        EXTRA_BUILD_ARGS="APP_VERSION=\"${GITHUB_SHA}\""
+        if [ "${{ matrix.service }}" = "web" ]; then
+          EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
         fi
-    - name: Cleanup buildx builder
-      if: matrix.service != 'worker' && always()
-      shell: sh
-      run: |
-        docker buildx rm ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} 2>/dev/null || true
-        docker buildx rm ci-builder 2>/dev/null || true
-        docker buildx prune -f 2>/dev/null || true
-        echo "Builder cleanup done"
+
+        NO_CACHE_FLAG=""
+        for i in 1 2 3; do
+          echo "=== Docker build 尝试 $i/3 (${{ matrix.service_display }}) ==="
+          if EXTRA_TAGS="$BRANCH_TAG" bash scripts/ci/docker_build_push.sh $NO_CACHE_FLAG ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS; then
+            echo "✅ Docker build 成功"
+            break
+          fi
+          echo "❌ Docker build 失败（尝试 $i/3）"
+          [ $i -eq 3 ] && exit 1
+          sleep 10
+          if [ $i -eq 2 ]; then
+            NO_CACHE_FLAG="--no-cache"
+            echo "下次重试将使用 --no-cache"
+          fi
+        done
+
+        echo "${{ matrix.service_display }} image pushed: ${IMAGE_TAG} (+ ${BRANCH_TAG})"
 
     - name: Job duration summary
       if: always()
@@ -902,15 +921,97 @@ jobs:
         [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
         python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
 
+  retag-staging-skipped:
+    name: Retag skipped Staging ${{ matrix.service_display }} Image
+    runs-on: runtime-builder
+    timeout-minutes: 10
+    needs:
+    - check-push-paths
+    - build-staging
+    if: |
+      github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop') && (
+        (matrix.service == 'web' && needs.check-push-paths.outputs.skip_frontend == 'true') ||
+        (matrix.service != 'web' && needs.check-push-paths.outputs.skip_backend == 'true')
+      )
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - service: api
+          service_display: API
+          image_name: xiaoxia-saas-api
+        - service: worker
+          service_display: Worker
+          image_name: xiaoxia-saas-worker
+        - service: web
+          service_display: Web
+          image_name: xiaoxia-saas-web
+    steps:
+    - name: Checkout code
+      shell: sh
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        curl -sfH "Authorization: token $GITHUB_TOKEN" -o /tmp/_ci_checkout.sh "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" && bash /tmp/_ci_checkout.sh
+    - name: Docker login to Registry
+      shell: sh
+      env:
+        ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
+        ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
+        GITEA_REGISTRY_USER: xiaoxia
+        GITEA_REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
+      run: |
+        set -eu
+        for i in 1 2 3; do
+          if printf '%s' "${ACR_PASSWORD}" | docker login xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com -u "${ACR_USERNAME}" --password-stdin \
+             && docker login git.xiaoxiajianji.com -u "${GITEA_REGISTRY_USER}" -p "${GITEA_REGISTRY_TOKEN}"; then
+            echo "✅ Docker login successful"
+            break
+          fi
+          echo "Docker login 失败（$i/3），5s 后重试..."
+          sleep 5
+        done
+    - name: Retag previous branch image to new SHA
+      shell: sh
+      run: |
+        set -eu
+        REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
+        bash scripts/ci/retag_skipped_image.sh \
+          "${REGISTRY}/${{ matrix.image_name }}" \
+          "${GITHUB_SHA}" \
+          "${GITHUB_REF_NAME}"
+    - name: Notify on failure
+      continue-on-error: true
+      if: failure()
+      shell: sh
+      env:
+        CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
+      run: |
+        set +e
+        NOTIFY_MODE=failure JOB_NAME="Retag Staging ${{ matrix.service_display }} Image" python3 scripts/ci_notify.py
+    - name: Report CI trace
+      if: always()
+      shell: sh
+      env:
+        AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }}
+      run: |
+        STATUS="ok"
+        [ ${{ job.status }} = "success" ] || STATUS="error"
+        START_TIME=""
+        [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
+        python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
+
   deploy-staging:
     name: Deploy Staging (Watchtower auto-deploy)
-    runs-on: runtime-builder
+    runs-on: ci-l2
     timeout-minutes: 15
     concurrency:
       group: deploy-staging-${{ gitea.ref }}
       cancel-in-progress: false
     needs:
+    - check-push-paths
     - build-staging
+    - retag-staging-skipped
     if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop')
     steps:
     - name: Checkout code
@@ -1491,7 +1592,7 @@ jobs:
 
   acr-cleanup:
     name: ACR Image Cleanup
-    runs-on: runtime-builder
+    runs-on: ci-l2
     timeout-minutes: 10
     needs:
     - deploy-staging
diff --git a/scripts/ci/ci_push_paths.sh b/scripts/ci/ci_push_paths.sh
new file mode 100755
index 000000000..0c7dbdfc7
--- /dev/null
+++ b/scripts/ci/ci_push_paths.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+# CI 公共步骤：检测 push(develop/main) 事件的改动范围
+# 输出 skip_backend / skip_frontend（复用 PR check 的语义）
+# - 纯前端改动(仅 apps/web/): skip_backend=true
+# - 纯后端改动(不含 apps/web/): skip_frontend=true
+# - 全栈 / 无法判断: 两者都 false（走全量，安全兜底）
+# 需要环境变量: GITHUB_TOKEN, GITHUB_API_URL, GITHUB_REPOSITORY, GITHUB_SHA
+set -eu
+
+OUTPUT="${GITHUB_OUTPUT:-/dev/stdout}"
+
+before="${GITHUB_EVENT_BEFORE:-}"
+after="${GITHUB_SHA:-}"
+repo="${GITHUB_REPOSITORY:-}"
+base="${GITHUB_API_URL:-}"
+
+# Gitea Actions 中 push 事件的前一个 SHA 在 event payload 的 before 字段
+if [ -z "$before" ] && [ -n "${GITHUB_EVENT_PATH:-}" ] && [ -f "$GITHUB_EVENT_PATH" ]; then
+  before=$(python3 -c "
+import json,sys
+try:
+    d=json.load(open('${GITHUB_EVENT_PATH}'))
+    print(d.get('before','') or '')
+except Exception:
+    print('')
+")
+fi
+
+echo "改动范围检测: before=${before:-<empty>} after=${after}"
+
+FILES=""
+if [ -n "$before" ] && [ "$before" != "0000000000000000000000000000000000000000" ]; then
+  # compare API: {base}...{head}
+  API_URL="${base}/repos/${repo}/compare/${before}...${after}?per_page=300"
+  for attempt in 1 2 3; do
+    FILES=$(curl -s --max-time 30 -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" \
+      | python3 -c "
+import json,sys
+try:
+    d=json.load(sys.stdin)
+    for f in d.get('files', []):
+        print(f.get('filename',''))
+except Exception:
+    pass
+")
+    [ -n "$FILES" ] && break
+    echo "compare API 无返回，重试 $attempt/3..."
+    sleep 3
+  done
+fi
+
+if [ -z "$FILES" ]; then
+  echo "⚠️ 无法获取改动文件列表（新分支/API异常），保守起见走全量构建"
+  echo "skip_backend=false" >> "$OUTPUT"
+  echo "skip_frontend=false" >> "$OUTPUT"
+  exit 0
+fi
+
+TOTAL=$(printf '%s\n' "$FILES" | grep -c . || true)
+FRONTEND_COUNT=$(printf '%s\n' "$FILES" | grep -c '^apps/web/' || true)
+BACKEND_COUNT=$(python3 -c "print($TOTAL - $FRONTEND_COUNT)")
+
+echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT})"
+
+if [ "$BACKEND_COUNT" = "0" ] && [ "$FRONTEND_COUNT" -gt "0" ]; then
+  echo "skip_backend=true" >> "$OUTPUT"
+  echo "skip_frontend=false" >> "$OUTPUT"
+  echo "✅ 纯前端改动，跳过后端镜像构建"
+elif [ "$FRONTEND_COUNT" = "0" ] && [ "$BACKEND_COUNT" -gt "0" ]; then
+  echo "skip_backend=false" >> "$OUTPUT"
+  echo "skip_frontend=true" >> "$OUTPUT"
+  echo "🔧 纯后端改动，跳过 Web 镜像构建"
+else
+  echo "skip_backend=false" >> "$OUTPUT"
+  echo "skip_frontend=false" >> "$OUTPUT"
+  echo "🔧 包含全栈/公共变更，三个镜像全部构建"
+fi
diff --git a/scripts/ci/docker_build_only.sh b/scripts/ci/docker_build_only.sh
index d0aaa18d7..db97b681d 100755
--- a/scripts/ci/docker_build_only.sh
+++ b/scripts/ci/docker_build_only.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 # PR构建专用：只构建不输出，验证Dockerfile能否正常构建
-# 无本地缓存（12个runner不共享，反而添乱），只用ACR远程缓存
+# 缓存：复用宿主机持久 builder (ci-builder-persist) 的层缓存 + ACR registry 缓存兜底
+# 无状态：build-only 不推送，job 结束无需清理（builder 为共享持久资源）
 set -eu
 
 NO_CACHE_FLAG=""
@@ -18,17 +19,21 @@ for arg in "$@"; do
   BUILD_ARGS="$BUILD_ARGS --build-arg $arg"
 done
 
-BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}"
+BUILDER_NAME="ci-builder-persist"
 if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then
-  docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
-else
-  docker buildx use "$BUILDER_NAME"
+  echo "持久 builder 不存在，创建中..."
+  docker buildx create --name "$BUILDER_NAME" --driver docker-container \
+    --driver-opt network=host \
+    --buildkitd-flags "--allow-insecure-entitlement network.host" \
+    --platform linux/amd64
 fi
-docker buildx inspect --bootstrap
+docker buildx use "$BUILDER_NAME"
+docker buildx inspect "$BUILDER_NAME" --bootstrap
 
-echo "=== PR Build: build only, no output, remote cache only ==="
+echo "=== PR Build: build only, no push (persistent builder cache) ==="
 echo "Dockerfile: ${DOCKERFILE}"
-echo "Image tag: ${IMAGE_TAG}"
+echo "Image tag:  ${IMAGE_TAG}"
+echo "Builder:    ${BUILDER_NAME}"
 echo ""
 
 docker buildx build \
diff --git a/scripts/ci/docker_build_push.sh b/scripts/ci/docker_build_push.sh
index bff4325ca..e2fb79061 100755
--- a/scripts/ci/docker_build_push.sh
+++ b/scripts/ci/docker_build_push.sh
@@ -1,10 +1,19 @@
 #!/bin/bash
-# 通用Docker镜像构建+推送脚本（local cache为主 + registry cache共享）
+# 通用Docker镜像构建+推送脚本
+# 缓存策略（2026-08 起）：
+#   - buildx 使用宿主机持久 builder (ci-builder-persist)，层缓存保存在
+#     buildkit 容器/命名卷中，跨 job 共享、job 结束不清理
+#   - registry cache 仅作为冷启动兜底读取
+#   - 额外 tag（如分支 tag :develop）通过 EXTRA_TAGS 环境变量传入，随构建一并推送
 # 用法: docker_build_push.sh [--no-cache] <Dockerfile> <image_tag> <cache_ref> [build_arg...]
+# 环境变量:
+#   EXTRA_TAGS  空格分隔的额外 tag（完整 image:tag 引用），可选
 set -eu
 
 # 单次 build 超时时间（秒），防止 docker buildx build 无限挂起
 BUILD_TIMEOUT=1500
+# 持久 builder 名（宿主机级，所有 CI job 共享；由 ensure_persistent_builder.sh 维护）
+BUILDER_NAME="ci-builder-persist"
 
 NO_CACHE_FLAG=""
 if [ "$1" = "--no-cache" ]; then
@@ -22,23 +31,27 @@ for arg in "$@"; do
   BUILD_ARGS="$BUILD_ARGS --build-arg $arg"
 done
 
-if ! docker buildx inspect ci-builder > /dev/null 2>&1; then
-  docker buildx create --use --name ci-builder --driver docker-container
-  echo "Created ci-builder"
-else
-  docker buildx use ci-builder
-  echo "Using existing ci-builder"
+# 确保持久 builder 存在并使用（幂等）
+if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then
+  echo "持久 builder 不存在，创建中..."
+  docker buildx create --name "$BUILDER_NAME" --driver docker-container \
+    --driver-opt network=host \
+    --buildkitd-flags "--allow-insecure-entitlement network.host" \
+    --platform linux/amd64
 fi
-docker buildx inspect --bootstrap
+docker buildx use "$BUILDER_NAME"
+docker buildx inspect "$BUILDER_NAME" --bootstrap
 
-# 从cache_ref中提取缓存名称（如 api-cache:develop -> api-cache-develop）
-CACHE_NAME=$(echo "$CACHE_REF" | tr '/' '_' | tr ':' '-')
-LOCAL_CACHE_DIR="/tmp/buildx-cache/${CACHE_NAME}"
+# 组装额外 tag 参数
+EXTRA_TAG_FLAGS=""
+EXTRA_TAG_LIST=""
+if [ -n "${EXTRA_TAGS:-}" ]; then
+  for t in $EXTRA_TAGS; do
+    EXTRA_TAG_FLAGS="$EXTRA_TAG_FLAGS -t $t"
+    EXTRA_TAG_LIST="$EXTRA_TAG_LIST $t"
+  done
+fi
 
-mkdir -p "$LOCAL_CACHE_DIR"
-
-# 缓存源：local优先（带自动修复），registry兜底读写
-# 本地缓存损坏时自动清理后重试，避免snapshot not found导致构建全挂
 build_with_cache_retry() {
   local attempt=1
   local max_attempts=2
@@ -49,12 +62,10 @@ build_with_cache_retry() {
     build_output=$(timeout ${BUILD_TIMEOUT} docker buildx build \
       $NO_CACHE_FLAG \
       $BUILD_ARGS \
-      --cache-from "type=local,src=${LOCAL_CACHE_DIR}" \
       --cache-from "type=registry,ref=${CACHE_REF}" \
-      --cache-to "type=local,dest=${LOCAL_CACHE_DIR},mode=max" \
-      --cache-to "type=registry,ref=${CACHE_REF},mode=max,ignore-error=true" \
       -f "${DOCKERFILE}" \
       -t "${IMAGE_TAG}" \
+      $EXTRA_TAG_FLAGS \
       --push \
       . 2>&1)
     exit_code=$?
@@ -69,48 +80,39 @@ build_with_cache_retry() {
       echo "$build_output" | tail -20
       return $exit_code
     fi
-    # 检测到缓存损坏类错误，清掉本地缓存重试
-    if echo "$build_output" | grep -qE "parent snapshot.*not found|snapshot.*does not exist|cache.*corrupt|failed to compute cache key"; then
+    # 检测到缓存/快照损坏类错误，重建 builder 后重试
+    if echo "$build_output" | grep -qE "parent snapshot.*not found|snapshot.*does not exist|cache.*corrupt|failed to compute cache key|no such file or directory.*cache"; then
       echo "$build_output"
       echo ""
-      echo "⚠️  Local cache appears corrupted, cleaning up and retrying (attempt $attempt/$max_attempts)..."
-      rm -rf "${LOCAL_CACHE_DIR}"
-      mkdir -p "${LOCAL_CACHE_DIR}"
-      # 清理buildx builder的内部snapshot状态
-      docker buildx prune -f -a > /dev/null 2>&1 || true
+      echo "⚠️  builder 缓存异常，重建持久 builder 后重试 (attempt $attempt/$max_attempts)..."
+      docker buildx rm "$BUILDER_NAME" >/dev/null 2>&1 || true
+      docker buildx create --name "$BUILDER_NAME" --driver docker-container \
+        --driver-opt network=host \
+        --buildkitd-flags "--allow-insecure-entitlement network.host" \
+        --platform linux/amd64
+      docker buildx use "$BUILDER_NAME"
+      docker buildx inspect "$BUILDER_NAME" --bootstrap
       attempt=$((attempt + 1))
     else
-      # 非缓存类错误，直接输出并返回
       echo "$build_output"
       return $exit_code
     fi
   done
-  # 重试完还是失败，不用本地缓存最后试一次（只从registry读）
-  echo "⚠️  All cached attempts failed, building without local cache..."
-  timeout ${BUILD_TIMEOUT} docker buildx build \
-    $NO_CACHE_FLAG \
-    $BUILD_ARGS \
-    --cache-from "type=registry,ref=${CACHE_REF}" \
-    --cache-to "type=local,dest=${LOCAL_CACHE_DIR},mode=max" \
-    --cache-to "type=registry,ref=${CACHE_REF},mode=max,ignore-error=true" \
-    -f "${DOCKERFILE}" \
-    -t "${IMAGE_TAG}" \
-    --push \
-    .
+  return 1
 }
 
-echo "=== Step 1: Build & push image (local cache + registry cache, with auto-repair) ==="
-echo "Local cache: ${LOCAL_CACHE_DIR}"
-echo "Registry cache: ${CACHE_REF}"
-echo "Build timeout: ${BUILD_TIMEOUT}s"
+echo "=== Build & push image (persistent builder cache) ==="
+echo "Builder:      ${BUILDER_NAME} (persistent)"
+echo "Registry cache(from): ${CACHE_REF}"
+echo "Image tag:    ${IMAGE_TAG}"
+[ -n "$EXTRA_TAG_LIST" ] && echo "Extra tags:   ${EXTRA_TAG_LIST}"
+echo "Timeout:      ${BUILD_TIMEOUT}s"
 echo ""
 
 build_with_cache_retry
 
 echo ""
 echo "Image pushed: ${IMAGE_TAG}"
-echo "Local cache updated"
-echo "Registry cache updated (if supported)"
-
+[ -n "$EXTRA_TAG_LIST" ] && echo "Also pushed:   ${EXTRA_TAG_LIST}"
 echo ""
 echo "Build completed: ${IMAGE_TAG}"
diff --git a/scripts/ci/ensure_persistent_builder.sh b/scripts/ci/ensure_persistent_builder.sh
new file mode 100755
index 000000000..928dcbadf
--- /dev/null
+++ b/scripts/ci/ensure_persistent_builder.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# CI 公共步骤：确保宿主机持久 buildx builder 存在（DooD 模式下所有 job 共享）
+# - builder 名固定: ci-builder-persist
+# - docker-container driver, host 网络
+# - 层缓存保存在 buildkit 容器及其 _state 命名卷中，job 结束不清理
+# - 宿主机 ci-docker-cleanup.sh 已豁免该 builder
+# 用法: bash scripts/ci/ensure_persistent_builder.sh
+set -eu
+
+BUILDER="ci-builder-persist"
+
+if ! docker buildx inspect "$BUILDER" >/dev/null 2>&1; then
+  echo "=== 创建持久 buildx builder: $BUILDER ==="
+  docker buildx create --name "$BUILDER" --driver docker-container \
+    --driver-opt network=host \
+    --buildkitd-flags "--allow-insecure-entitlement network.host" \
+    --platform linux/amd64
+else
+  echo "=== 复用持久 buildx builder: $BUILDER ==="
+fi
+
+docker buildx use "$BUILDER"
+docker buildx inspect "$BUILDER" --bootstrap
+echo "✅ builder ready"
+docker buildx ls | head -5
diff --git a/scripts/ci/retag_skipped_image.sh b/scripts/ci/retag_skipped_image.sh
new file mode 100755
index 000000000..4f7102e73
--- /dev/null
+++ b/scripts/ci/retag_skipped_image.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+# CI 步骤：未重建的镜像，把 registry 上一个分支 tag 复制为新 SHA tag
+# 保证 deploy-staging 的 Watchtower 链路三个镜像都有新 SHA 可拉
+# 用法: bash scripts/ci/retag_skipped_image.sh <image_full_name> <new_sha> <branch>
+# 例:   bash scripts/ci/retag_skipped_image.sh xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/xiaoxia-saas-web <sha> develop
+set -eu
+
+IMAGE="$1"
+NEW_TAG="$2"
+BRANCH="${3:-develop}"
+
+NEW_REF="${IMAGE}:${NEW_TAG}"
+
+echo "=== 复用已有镜像（本次未重建）: $IMAGE ==="
+echo "目标 tag: $NEW_TAG"
+
+# 源 tag 候选（按优先级）
+CANDIDATES=()
+# 1. 分支 tag（构建 job 每次成功都会推）
+CANDIDATES+=("$BRANCH")
+# 2. 本 push 的前一个 commit SHA（compare 事件）
+if [ -n "${GITHUB_EVENT_BEFORE:-}" ] && [ "${GITHUB_EVENT_BEFORE}" != "0000000000000000000000000000000000000000" ]; then
+  CANDIDATES+=("${GITHUB_EVENT_BEFORE}")
+fi
+# 3. registry 上最新的 sha 形式 tag（通过 ACR tags API 兜底，不需要额外认证则跳过）
+
+SRC_TAG=""
+for cand in "${CANDIDATES[@]}"; do
+  echo "尝试拉取 ${IMAGE}:${cand} ..."
+  if docker pull "${IMAGE}:${cand}" >/dev/null 2>&1; then
+    SRC_TAG="$cand"
+    break
+  fi
+done
+
+if [ -z "$SRC_TAG" ]; then
+  echo "❌ 找不到可复用的源镜像（已尝试: ${CANDIDATES[*]}）"
+  echo "   请检查该镜像是否曾成功构建推送，或临时改用全量构建。"
+  exit 1
+fi
+
+echo "✅ 源镜像: ${IMAGE}:${SRC_TAG}"
+docker tag "${IMAGE}:${SRC_TAG}" "${NEW_REF}"
+
+# 推新 SHA tag；分支 tag 若指向的就是源 digest 则无需重复，失败可忽略
+docker push "${NEW_REF}"
+echo "✅ retag 推送完成: ${NEW_REF} (from ${SRC_TAG})"
-- 
2.54.0


From a441b760d4768817799f27e244b9c54997060a9b Mon Sep 17 00:00:00 2001
From: xiaoxia <dev@xiaoxiajianji.com>
Date: Sat, 29 Aug 2026 12:43:37 +0800
Subject: [PATCH 2/8] =?UTF-8?q?perf(ci):=20validate/unit-tests/frontend-un?=
 =?UTF-8?q?it-test=20=E4=B8=89=E4=B8=AA=20job=20=E5=A2=9E=E5=8A=A0=20actio?=
 =?UTF-8?q?ns/cache@v4=20=E4=BE=9D=E8=B5=96=E7=BC=93=E5=AD=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

act_runner v0.2.13 不支持 runner config 自定义 volume 挂载（container.volumes 不在 schema，
container.options -v 实测不生效），改用 Gitea Actions 标准 actions/cache@v4：
- validate-code-quality / unit-tests: 缓存 /root/.cache/pip，key 按 requirements*.txt hash
- frontend-unit-test: 缓存 /root/.npm，key 按 apps/web/package-lock.json hash
runner 已启用内置 cache server。
---
 .gitea/workflows/ci-pipeline.yml | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml
index 8ba048574..9a9c8e810 100755
--- a/.gitea/workflows/ci-pipeline.yml
+++ b/.gitea/workflows/ci-pipeline.yml
@@ -94,6 +94,14 @@ jobs:
     - name: Record job start time
       shell: sh
       run: bash scripts/ci/step_timer_start.sh
+    - name: Cache pip dependencies
+      uses: actions/cache@v4
+      with:
+        path: /root/.cache/pip
+        key: ${{ runner.os }}-pip-codequality-${{ hashFiles('requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-codequality-
+          ${{ runner.os }}-pip-
     - name: Install dependencies
       shell: sh
       run: |
@@ -347,6 +355,14 @@ jobs:
     - name: Install ffmpeg
       shell: sh
       run: bash scripts/ci/step_install_ffmpeg.sh
+    - name: Cache pip dependencies
+      uses: actions/cache@v4
+      with:
+        path: /root/.cache/pip
+        key: ${{ runner.os }}-pip-unittests-${{ hashFiles('requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-unittests-
+          ${{ runner.os }}-pip-
     - name: Run unit tests with coverage
       shell: bash
       env:
@@ -531,6 +547,13 @@ jobs:
     - name: Record job start time
       shell: sh
       run: bash scripts/ci/step_timer_start.sh
+    - name: Cache npm dependencies
+      uses: actions/cache@v4
+      with:
+        path: /root/.npm
+        key: ${{ runner.os }}-npm-vitest-${{ hashFiles('apps/web/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-npm-
     - name: Install frontend dependencies (vitest only, with retry)
       shell: sh
       run: |
@@ -1918,4 +1941,4 @@ jobs:
           [ "${{ steps.gate.outputs.gate_result }}" = "success" ] || STATUS="error"
           START_TIME=""
           [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time)
-          python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
\ No newline at end of file
+          python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true
-- 
2.54.0


From 88cd5db2994d7a315916e6b95514d15048fc22fa Mon Sep 17 00:00:00 2001
From: xiaoxia <dev@xiaoxiajianji.com>
Date: Sat, 29 Aug 2026 13:03:51 +0800
Subject: [PATCH 3/8] =?UTF-8?q?fix(ci):=20=E5=BC=BA=E5=88=B6=E5=90=AF?=
 =?UTF-8?q?=E7=94=A8=20pip=20=E7=BC=93=E5=AD=98=E7=9B=AE=E5=BD=95=EF=BC=88?=
 =?UTF-8?q?=E9=95=9C=E5=83=8F=20PIP=5FNO=5FCACHE=5FDIR=3D0=20=E8=A2=AB?=
 =?UTF-8?q?=E5=BD=93=E4=BD=9C=E7=9C=9F=E5=80=BC=E7=A6=81=E7=94=A8=E7=BC=93?=
 =?UTF-8?q?=E5=AD=98=EF=BC=89?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ci-base 镜像 ENV PIP_NO_CACHE_DIR=0 中字符串 "0" 被 pip 解析为真值，
导致 /root/.cache/pip 从不写入，actions/cache 报 Path Validation Error。
在 validate-code-quality / unit-tests 两个 pip job 的 env 中显式覆盖：
PIP_CACHE_DIR=/root/.cache/pip + PIP_NO_CACHE_DIR=''。
---
 .gitea/workflows/ci-pipeline.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml
index 9a9c8e810..85a6bbb08 100755
--- a/.gitea/workflows/ci-pipeline.yml
+++ b/.gitea/workflows/ci-pipeline.yml
@@ -82,6 +82,9 @@ jobs:
     name: Validate - Code Quality
     runs-on: ci-l2
     timeout-minutes: 8
+    env:
+      PIP_CACHE_DIR: /root/.cache/pip
+      PIP_NO_CACHE_DIR: ''
     permissions:
       contents: write
     steps:
@@ -337,6 +340,8 @@ jobs:
     runs-on: ci-l2
     timeout-minutes: 8
     env:
+      PIP_CACHE_DIR: /root/.cache/pip
+      PIP_NO_CACHE_DIR: ''
       USE_IN_MEMORY_DB: 'true'
       OSS_ACCESS_KEY_ID: placeholder
       OSS_ACCESS_KEY_SECRET: placeholder
-- 
2.54.0


From 3a41c9b5982e9410ab3e5c6b3a219298213168e5 Mon Sep 17 00:00:00 2001
From: xiaoxia <dev@xiaoxiajianji.com>
Date: Sat, 29 Aug 2026 13:25:28 +0800
Subject: [PATCH 4/8] =?UTF-8?q?perf(ci):=20Worker=20PR=20=E6=9E=84?=
 =?UTF-8?q?=E5=BB=BA=E4=B9=9F=E6=8E=A5=E5=85=A5=E6=8C=81=E4=B9=85=20builde?=
 =?UTF-8?q?r=EF=BC=88=E7=A7=BB=E9=99=A4=E6=99=AE=E9=80=9A=20docker=20build?=
 =?UTF-8?q?=20=E6=8D=B7=E5=BE=84=EF=BC=89?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

原 Worker PR 构建走 plain docker build，无 buildx 层缓存；
现与 API/Web 统一走 docker_build_only.sh (ci-builder-persist)。
---
 .gitea/workflows/ci-pipeline.yml | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml
index 85a6bbb08..61c99b293 100755
--- a/.gitea/workflows/ci-pipeline.yml
+++ b/.gitea/workflows/ci-pipeline.yml
@@ -695,18 +695,7 @@ jobs:
           EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
         fi
 
-        # Worker: 始终用普通docker build（基础镜像已预装全部依赖，无需buildx）
-        if [ "${{ matrix.service }}" = "worker" ]; then
-          echo "Worker: 使用普通docker build"
-          BUILD_ARG_STR=""
-          for arg in $EXTRA_BUILD_ARGS; do
-            BUILD_ARG_STR="$BUILD_ARG_STR --build-arg $arg"
-          done
-          docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR .
-          echo "PR Build successful (worker, no buildx)"
-          exit 0
-        fi
-
+        # Worker 与 API/Web 统一走持久 builder（ci-builder-persist），共享宿主机层缓存
         NO_CACHE_FLAG=""
         for i in 1 2 3; do
           echo "PR Build attempt $i/3"
-- 
2.54.0


From 77d582f13a1b02cb4b2f4762fe7534e28dde9875 Mon Sep 17 00:00:00 2001
From: xiaoxia <dev@xiaoxiajianji.com>
Date: Sat, 29 Aug 2026 14:18:42 +0800
Subject: [PATCH 5/8] =?UTF-8?q?fix(ci):=20ensure=5Fpersistent=5Fbuilder=20?=
 =?UTF-8?q?=E5=B9=B6=E5=8F=91=E5=AE=89=E5=85=A8=20-=20create=20=E5=A4=B1?=
 =?UTF-8?q?=E8=B4=A5=E6=97=B6=E8=8B=A5=E5=B7=B2=E8=A2=AB=E5=B9=B6=E5=8F=91?=
 =?UTF-8?q?=20job=20=E5=88=9B=E5=BB=BA=E5=88=99=E5=A4=8D=E7=94=A8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/ci/ensure_persistent_builder.sh | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/scripts/ci/ensure_persistent_builder.sh b/scripts/ci/ensure_persistent_builder.sh
index 928dcbadf..0f3c45bdc 100755
--- a/scripts/ci/ensure_persistent_builder.sh
+++ b/scripts/ci/ensure_persistent_builder.sh
@@ -11,10 +11,21 @@ BUILDER="ci-builder-persist"
 
 if ! docker buildx inspect "$BUILDER" >/dev/null 2>&1; then
   echo "=== 创建持久 buildx builder: $BUILDER ==="
-  docker buildx create --name "$BUILDER" --driver docker-container \
-    --driver-opt network=host \
-    --buildkitd-flags "--allow-insecure-entitlement network.host" \
-    --platform linux/amd64
+  # 并发安全：matrix 多个 job 可能同时检测到 builder 不存在，只有一个 create 成功；
+  # 其余 job 的 create 会因 "builder already exists" 失败（set -e 下会退出）。
+  # 用 create || inspect 兜底：create 失败时若 builder 实际已被别的 job 创建，直接复用。
+  if ! docker buildx create --name "$BUILDER" --driver docker-container \
+      --driver-opt network=host \
+      --buildkitd-flags "--allow-insecure-entitlement network.host" \
+      --platform linux/amd64 2>/tmp/_buildx_create.err; then
+    if docker buildx inspect "$BUILDER" >/dev/null 2>&1; then
+      echo "=== builder 已被并发任务创建，复用: $BUILDER ==="
+    else
+      echo "❌ builder 创建失败且不存在:"
+      cat /tmp/_buildx_create.err
+      exit 1
+    fi
+  fi
 else
   echo "=== 复用持久 buildx builder: $BUILDER ==="
 fi
-- 
2.54.0


From acc21ee22fc2b2a7459f85f3dc46558998f248e7 Mon Sep 17 00:00:00 2001
From: xiaoxia <dev@xiaoxiajianji.com>
Date: Sat, 29 Aug 2026 14:18:42 +0800
Subject: [PATCH 6/8] =?UTF-8?q?fix(ci):=20docker=5Fbuild=5Fpush=20?=
 =?UTF-8?q?=E7=BC=93=E5=AD=98=E6=8D=9F=E5=9D=8F=E9=87=8D=E5=BB=BA=E6=94=B9?=
 =?UTF-8?q?=E4=B8=BA=20flock=20=E4=B8=B2=E8=A1=8C=20+=20=E5=81=A5=E5=BA=B7?=
 =?UTF-8?q?=E5=BA=A6=E5=A4=8D=E6=9F=A5=EF=BC=8C=E9=81=BF=E5=85=8D=E5=B9=B6?=
 =?UTF-8?q?=E5=8F=91=E4=BA=92=E5=88=A0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/ci/docker_build_push.sh | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/scripts/ci/docker_build_push.sh b/scripts/ci/docker_build_push.sh
index e2fb79061..26fbc8bb5 100755
--- a/scripts/ci/docker_build_push.sh
+++ b/scripts/ci/docker_build_push.sh
@@ -85,13 +85,29 @@ build_with_cache_retry() {
       echo "$build_output"
       echo ""
       echo "⚠️  builder 缓存异常，重建持久 builder 后重试 (attempt $attempt/$max_attempts)..."
-      docker buildx rm "$BUILDER_NAME" >/dev/null 2>&1 || true
-      docker buildx create --name "$BUILDER_NAME" --driver docker-container \
-        --driver-opt network=host \
-        --buildkitd-flags "--allow-insecure-entitlement network.host" \
-        --platform linux/amd64
-      docker buildx use "$BUILDER_NAME"
-      docker buildx inspect "$BUILDER_NAME" --bootstrap
+      # 共享 builder 的重建必须串行：ci-builder-persist 被所有 build job 共用，
+      # 若 job A 正在构建、job B 检测到损坏直接 rm，会把 A 正在用的 buildkit 杀掉。
+      # 用 flock 串行化重建；拿到锁后再次检查 builder 健康度，已被别的 job 重建则直接复用。
+      LOCK_FILE="/tmp/ci-builder-persist-rebuild.lock"
+      exec 9>"$LOCK_FILE"
+      echo "🔒 等待重建锁（最多 120s）..."
+      if flock -w 120 9; then
+        if docker buildx inspect "$BUILDER_NAME" --bootstrap >/dev/null 2>&1; then
+          echo "✅ builder 已被其他并发 job 重建/恢复，直接复用"
+        else
+          echo "🔨 锁内重建持久 builder..."
+          docker buildx rm "$BUILDER_NAME" >/dev/null 2>&1 || true
+          docker buildx create --name "$BUILDER_NAME" --driver docker-container \
+            --driver-opt network=host \
+            --buildkitd-flags "--allow-insecure-entitlement network.host" \
+            --platform linux/amd64
+          docker buildx use "$BUILDER_NAME"
+          docker buildx inspect "$BUILDER_NAME" --bootstrap
+        fi
+      else
+        echo "⚠️ 等待重建锁超时，直接重试 build（失败将重试/--no-cache）"
+        docker buildx use "$BUILDER_NAME" 2>/dev/null || true
+      fi
       attempt=$((attempt + 1))
     else
       echo "$build_output"
-- 
2.54.0


From 36e170ae04f96592e096741fa83103e9a9b45339 Mon Sep 17 00:00:00 2001
From: xiaoxia <dev@xiaoxiajianji.com>
Date: Sat, 29 Aug 2026 14:18:43 +0800
Subject: [PATCH 7/8] =?UTF-8?q?fix(ci):=20docker=5Fbuild=5Fonly=20(PR?=
 =?UTF-8?q?=E6=9E=84=E5=BB=BA)=20=E5=90=8C=E6=AD=A5=E5=A2=9E=E5=8A=A0?=
 =?UTF-8?q?=E7=BC=93=E5=AD=98=E6=8D=9F=E5=9D=8F=E4=B8=B2=E8=A1=8C=E9=87=8D?=
 =?UTF-8?q?=E5=BB=BA=E5=AE=B9=E9=94=99?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/ci/docker_build_only.sh | 45 ++++++++++++++++++++++++++++-----
 1 file changed, 38 insertions(+), 7 deletions(-)

diff --git a/scripts/ci/docker_build_only.sh b/scripts/ci/docker_build_only.sh
index db97b681d..aed388ba9 100755
--- a/scripts/ci/docker_build_only.sh
+++ b/scripts/ci/docker_build_only.sh
@@ -36,13 +36,44 @@ echo "Image tag:  ${IMAGE_TAG}"
 echo "Builder:    ${BUILDER_NAME}"
 echo ""
 
-docker buildx build \
-  $NO_CACHE_FLAG \
-  $BUILD_ARGS \
-  --cache-from "type=registry,ref=${CACHE_REF}" \
-  -f "${DOCKERFILE}" \
-  -t "${IMAGE_TAG}" \
-  .
+run_build() {
+  docker buildx build \
+    $NO_CACHE_FLAG \
+    $BUILD_ARGS \
+    --cache-from "type=registry,ref=${CACHE_REF}" \
+    -f "${DOCKERFILE}" \
+    -t "${IMAGE_TAG}" \
+    .
+}
+
+# PR 构建同样容错：检测到 builder 缓存损坏时，用 flock 串行重建共享 builder 后重试一次
+if ! build_output=$(run_build 2>&1); then
+  if echo "$build_output" | grep -qE "parent snapshot.*not found|snapshot.*does not exist|cache.*corrupt|failed to compute cache key|no such file or directory.*cache"; then
+    echo "$build_output"
+    echo "⚠️  builder 缓存异常，串行重建持久 builder 后重试..."
+    LOCK_FILE="/tmp/ci-builder-persist-rebuild.lock"
+    exec 9>"$LOCK_FILE"
+    flock -w 120 9 || echo "⚠️ 等待重建锁超时，直接重试 build"
+    if ! docker buildx inspect "$BUILDER_NAME" --bootstrap >/dev/null 2>&1; then
+      echo "🔨 锁内重建持久 builder..."
+      docker buildx rm "$BUILDER_NAME" >/dev/null 2>&1 || true
+      docker buildx create --name "$BUILDER_NAME" --driver docker-container \
+        --driver-opt network=host \
+        --buildkitd-flags "--allow-insecure-entitlement network.host" \
+        --platform linux/amd64
+      docker buildx use "$BUILDER_NAME"
+      docker buildx inspect "$BUILDER_NAME" --bootstrap
+    else
+      echo "✅ builder 已被其他并发 job 重建/恢复，直接复用"
+    fi
+    run_build
+  else
+    echo "$build_output"
+    exit 1
+  fi
+else
+  echo "$build_output"
+fi
 
 echo ""
 echo "PR build OK (build only, no output): ${IMAGE_TAG}"
-- 
2.54.0


From b290b15eb34577f9eaa76ff44e5917f59b4d0aac Mon Sep 17 00:00:00 2001
From: xiaoxia <dev@xiaoxiajianji.com>
Date: Sat, 29 Aug 2026 14:18:44 +0800
Subject: [PATCH 8/8] =?UTF-8?q?fix(ci):=20retag=5Fskipped=5Fimage=20?=
 =?UTF-8?q?=E5=80=99=E9=80=89=E6=BA=90=20docker=20pull=20=E5=A2=9E?=
 =?UTF-8?q?=E5=8A=A0=203=20=E6=AC=A1=E9=87=8D=E8=AF=95=EF=BC=8C=E7=BD=91?=
 =?UTF-8?q?=E7=BB=9C=E6=8A=96=E5=8A=A8=E5=AE=B9=E9=94=99?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/ci/retag_skipped_image.sh | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scripts/ci/retag_skipped_image.sh b/scripts/ci/retag_skipped_image.sh
index 4f7102e73..d2b39f2c8 100755
--- a/scripts/ci/retag_skipped_image.sh
+++ b/scripts/ci/retag_skipped_image.sh
@@ -27,7 +27,17 @@ fi
 SRC_TAG=""
 for cand in "${CANDIDATES[@]}"; do
   echo "尝试拉取 ${IMAGE}:${cand} ..."
-  if docker pull "${IMAGE}:${cand}" >/dev/null 2>&1; then
+  # 网络抖动容错：每个候选源最多重试 3 次
+  pull_ok=""
+  for try in 1 2 3; do
+    if docker pull "${IMAGE}:${cand}" >/dev/null 2>&1; then
+      pull_ok="yes"
+      break
+    fi
+    echo "  拉取失败（第 $try/3 次），2s 后重试..."
+    sleep 2
+  done
+  if [ -n "$pull_ok" ]; then
     SRC_TAG="$cand"
     break
   fi
-- 
2.54.0