From 688d32fe8a8fa995a410390e1be08e6e34f127df Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Mon, 27 Jul 2026 16:18:06 +0800 Subject: [PATCH] =?UTF-8?q?fix(ci):=20=E6=B7=BB=E5=8A=A0ci-gate=E6=B1=87?= =?UTF-8?q?=E6=80=BBjob=EF=BC=8C=E8=A7=A3=E5=86=B3=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E5=90=88=E5=B9=B6405=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/ci-pipeline.yml | 358 +++++++++++++++++++------------ 1 file changed, 223 insertions(+), 135 deletions(-) diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml index e834fb87d..4d4e5b212 100755 --- a/.gitea/workflows/ci-pipeline.yml +++ b/.gitea/workflows/ci-pipeline.yml @@ -24,17 +24,6 @@ concurrency: group: ci-pipeline-${{ gitea.event_name }}-${{ gitea.ref }} # PR事件取消进行中的旧run，push事件不取消（确保完整CI跑完） cancel-in-progress: ${{ gitea.event_name == 'pull_request' }} -env: - CI_PG_HOST: host.docker.internal - CI_LOCAL_PG_PORT: "5432" - CI_PG_USER: postgres - CI_PG_PASSWORD: postgres - CI_PG_DB: xiaoxia_saas - CI_SHARED_PG_PORT: "5433" - CI_SHARED_PG_USER: postgres - CI_SHARED_PG_PASSWORD: ci_pg_2026! - CI_DEFAULT_DB: xiaoxia_saas - jobs: check-frontend-only: name: Check if frontend-only change @@ -58,56 +47,24 @@ jobs: run: | set -eu PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') - - # 分页获取所有变更文件（修复>300文件时漏判） - ALL_FILES="" - PAGE=1 - while true; do - API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300&page=${PAGE}" - PAGE_FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "import sys,json; files=json.load(sys.stdin); [print(f['filename']) for f in files]; sys.exit(0 if len(files)==300 else 1)" 2>/dev/null || true) - PAGE_COUNT=$(echo "$PAGE_FILES" | wc -l) - if [ "$PAGE_COUNT" -lt 300 ]; then HAS_MORE=1; else HAS_MORE=0; fi - ALL_FILES="${ALL_FILES}${PAGE_FILES}"$'\n' - if [ "$HAS_MORE" != "0" ]; then - break - fi - PAGE=$((PAGE + 1)) - done - - FRONTEND_COUNT=$(echo "$ALL_FILES" | grep -c '^apps/web/' || true) - TOTAL=$(echo "$ALL_FILES" | grep -cv '^$' || true) - BACKEND_COUNT=$(( TOTAL - FRONTEND_COUNT )) - - # 基础设施文件：改了就强制全量CI（不跳过任何检查） - INFRA_COUNT=$(echo "$ALL_FILES" | grep -cE '^(infra/|Dockerfile|\.gitea/workflows/|scripts/ci/|docker/)' || true) - - echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT}, 基础设施: ${INFRA_COUNT})" - - # 判定是否纯前端/纯后端 - PURE_FRONTEND=false - PURE_BACKEND=false - if [ "$BACKEND_COUNT" = "0" ] && [ "$FRONTEND_COUNT" -gt "0" ] && [ "$INFRA_COUNT" = "0" ]; then - PURE_FRONTEND=true - elif [ "$FRONTEND_COUNT" = "0" ] && [ "$BACKEND_COUNT" -gt "0" ] && [ "$INFRA_COUNT" = "0" ]; then - PURE_BACKEND=true - fi - - if [ "$PURE_FRONTEND" = "true" ]; then + API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300" + FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin)]") + FRONTEND_COUNT=$(echo "$FILES" | grep -c '^apps/web/' || true) + BACKEND_COUNT=$(echo "$FILES" | grep -cv '^apps/web/' || true) + TOTAL=$(echo "$FILES" | grep -cv '^$' || true) + echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT})" + if [ "$BACKEND_COUNT" = "0" ] && [ "$FRONTEND_COUNT" -gt "0" ]; then echo "skip_backend=true" >> $GITHUB_OUTPUT echo "skip_frontend=false" >> $GITHUB_OUTPUT echo "✅ 纯前端改动，跳过后端检查" - elif [ "$PURE_BACKEND" = "true" ]; then + elif [ "$FRONTEND_COUNT" = "0" ] && [ "$BACKEND_COUNT" -gt "0" ]; then echo "skip_backend=false" >> $GITHUB_OUTPUT echo "skip_frontend=true" >> $GITHUB_OUTPUT echo "🔧 纯后端改动，跳过前端检查" else echo "skip_backend=false" >> $GITHUB_OUTPUT echo "skip_frontend=false" >> $GITHUB_OUTPUT - if [ "$INFRA_COUNT" -gt "0" ]; then - echo "🏗️ 包含基础设施变更，强制运行完整CI" - else - echo "🔧 包含全栈变更，运行完整CI" - fi + echo "🔧 包含全栈变更，运行完整CI" fi - name: Report CI trace @@ -294,7 +251,7 @@ jobs: permissions: contents: read env: - DATABASE_URL: postgresql+psycopg://${{ env.CI_PG_USER }}:${{ env.CI_PG_PASSWORD }}@${{ env.CI_PG_HOST }}:${{ env.CI_LOCAL_PG_PORT }}/${{ env.CI_PG_DB }} + DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas USE_IN_MEMORY_DB: 'false' CI_USE_SHARED_PG: 'true' steps: @@ -378,7 +335,6 @@ jobs: OSS_ACCESS_KEY_SECRET: placeholder OSS_BUCKET_NAME: xiaoxia-autocut OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com - JWT_SECRET_KEY: test-jwt-secret-for-ci-only-2026 steps: - name: Checkout code shell: sh @@ -443,14 +399,13 @@ jobs: - validate-type-check - validate-migration env: - DATABASE_URL: postgresql+psycopg://${{ env.CI_PG_USER }}:${{ env.CI_PG_PASSWORD }}@${{ env.CI_PG_HOST }}:${{ env.CI_LOCAL_PG_PORT }}/${{ env.CI_PG_DB }} + DATABASE_URL: postgresql+psycopg://postgres:postgres@host.docker.internal:5432/xiaoxia_saas USE_IN_MEMORY_DB: 'false' CI_USE_SHARED_PG: 'true' OSS_ACCESS_KEY_ID: placeholder OSS_ACCESS_KEY_SECRET: placeholder OSS_BUCKET_NAME: xiaoxia-autocut OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com - JWT_SECRET_KEY: test-jwt-secret-for-ci-only-2026 steps: - name: Checkout code shell: sh @@ -672,83 +627,60 @@ jobs: echo "Docker login failed ($i/3), retrying in 5s..." sleep 5 done - - name: Pre-build worker base images (3-level cache) + - name: Pre-build worker base images (fallback if not exist) if: matrix.service == 'worker' id: prebuild - shell: bash + shell: sh run: | set -eu - GITEA_REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas" - ACR_REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji" - GITEA_BUILDER="${GITEA_REGISTRY}/worker-base-builder:latest" - GITEA_RUNTIME="${GITEA_REGISTRY}/worker-base-runtime:latest" - ACR_BUILDER="${ACR_REGISTRY}/worker-base-builder:latest" - ACR_RUNTIME="${ACR_REGISTRY}/worker-base-runtime:latest" - - # L1: 本地daemon缓存（DooD模式8runner共享宿主机daemon） - echo "=== L1 本地缓存 ===" - if docker image inspect "$ACR_BUILDER" > /dev/null 2>&1 \ - && docker image inspect "$ACR_RUNTIME" > /dev/null 2>&1; then - echo "本地缓存命中" - echo "has_local_base=true" >> $GITHUB_OUTPUT - exit 0 - fi - echo "本地无缓存" - - # L2: Gitea registry缓存（内网快） - echo "=== L2 Registry拉取 ===" - if docker pull "$GITEA_BUILDER" 2>/dev/null && docker pull "$GITEA_RUNTIME" 2>/dev/null; then - echo "Registry拉取成功，重tag供Dockerfile使用" - docker tag "$GITEA_BUILDER" "$ACR_BUILDER" - docker tag "$GITEA_RUNTIME" "$ACR_RUNTIME" - echo "has_local_base=true" >> $GITHUB_OUTPUT - exit 0 - fi - echo "Registry无缓存，需本地构建" - - # L3: 本地构建 - echo "=== L3 本地构建 ===" - BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}" - if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then - docker buildx create --use --name "$BUILDER_NAME" --driver docker-container + REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas" + BASE_BUILDER="${REGISTRY}/worker-base-builder:latest" + BASE_RUNTIME="${REGISTRY}/worker-base-runtime:latest" + + # 尝试拉取基础镜像 + echo "检查基础镜像..." + if docker pull "$BASE_BUILDER" 2>/dev/null && docker pull "$BASE_RUNTIME" 2>/dev/null; then + echo "基础镜像已存在，使用远程镜像" + echo "fallback=false" >> $GITHUB_OUTPUT else - docker buildx use "$BUILDER_NAME" + echo "基础镜像不存在，本地构建（fallback模式）..." + + # 尝试用buildx构建，失败则回退到普通docker build（DooD模式下buildx builder偶发崩溃） + BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}" + BUILDX_AVAILABLE=true + if ! docker buildx create --use --name "$BUILDER_NAME" --driver docker-container > /dev/null 2>&1; then + BUILDX_AVAILABLE=false + fi + if [ "$BUILDX_AVAILABLE" = true ] && ! docker buildx inspect --bootstrap > /dev/null 2>&1; then + BUILDX_AVAILABLE=false + docker buildx rm "$BUILDER_NAME" > /dev/null 2>&1 || true + fi + + build_base() { + local df="$1" + local tag="$2" + local name="$3" + if [ "$BUILDX_AVAILABLE" = true ]; then + echo "构建 $name（buildx）..." + if docker buildx build --load -f "$df" -t "$tag" . > /dev/null 2>&1; then + echo "$name 构建成功" + return 0 + fi + echo "buildx失败，回退到普通docker build" + BUILDX_AVAILABLE=false + docker buildx rm "$BUILDER_NAME" > /dev/null 2>&1 || true + fi + echo "构建 $name（docker build）..." + docker build -f "$df" -t "$tag" . + } + + build_base infra/docker/worker-base-builder.Dockerfile "$BASE_BUILDER" "worker-base-builder" + build_base infra/docker/worker-base-runtime.Dockerfile "$BASE_RUNTIME" "worker-base-runtime" + + echo "fallback=true" >> $GITHUB_OUTPUT + echo "基础镜像本地构建完成" fi - docker buildx inspect --bootstrap > /dev/null 2>&1 - echo "构建 worker-base-builder..." - for attempt in 1 2 3; do - if docker buildx build --load -f infra/docker/worker-base-builder.Dockerfile -t "$ACR_BUILDER" .; then - echo "worker-base-builder 构建成功" - break - fi - echo "worker-base-builder 失败，重试 $attempt/3..." - docker buildx rm "$BUILDER_NAME" 2>/dev/null || true - docker buildx create --use --name "$BUILDER_NAME" --driver docker-container - sleep 3 - done - - echo "构建 worker-base-runtime..." - for attempt in 1 2 3; do - if docker buildx build --load -f infra/docker/worker-base-runtime.Dockerfile -t "$ACR_RUNTIME" .; then - echo "worker-base-runtime 构建成功" - break - fi - echo "worker-base-runtime 失败，重试 $attempt/3..." - docker buildx rm "$BUILDER_NAME" 2>/dev/null || true - docker buildx create --use --name "$BUILDER_NAME" --driver docker-container - sleep 3 - done - - # 推送到Gitea registry供后续复用 - echo "=== 推送缓存到Registry ===" - docker tag "$ACR_BUILDER" "$GITEA_BUILDER" - docker tag "$ACR_RUNTIME" "$GITEA_RUNTIME" - docker push "$GITEA_BUILDER" 2>/dev/null || echo "push builder失败（不影响）" - docker push "$GITEA_RUNTIME" 2>/dev/null || echo "push runtime失败（不影响）" - - echo "has_local_base=true" >> $GITHUB_OUTPUT - echo "基础镜像构建完成" - name: Build PR image (verify only, no push) shell: sh run: | @@ -762,15 +694,15 @@ jobs: EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf" fi - # Worker有本地base镜像时：用BuildKit直接构建（快，无需起buildx容器） - if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.has_local_base }}" = "true" ]; then - echo "本地base镜像已就绪，BuildKit快速构建" + # Worker fallback模式：基础镜像本地已构建，用普通docker build绕过buildx + if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.fallback }}" = "true" ]; then + echo "Fallback模式：用普通docker build（基础镜像本地已构建）" BUILD_ARG_STR="" for arg in $EXTRA_BUILD_ARGS; do BUILD_ARG_STR="$BUILD_ARG_STR --build-arg $arg" done - DOCKER_BUILDKIT=1 docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR . - echo "快速构建成功" + docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR . + echo "Fallback PR Build successful" exit 0 fi @@ -1151,8 +1083,7 @@ jobs: set -eu # DooD模式下不能用-v挂载（宿主机路径与CI容器路径不一致） # 改用 docker create + docker cp 方式把代码拷进容器 - CONTAINER_NAME="staging-e2e-${GITHUB_SHA::8}" - docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + CONTAINER_NAME="staging-e2e-$$" docker create --name "$CONTAINER_NAME" --ipc=host \ -e E2E_BASE_URL=https://staging.xiaoxiajianji.com \ -e E2E_API_BASE=https://staging-api.xiaoxiajianji.com/api/v1 \ @@ -1216,8 +1147,7 @@ jobs: set -eu # DooD模式下不能用-v挂载（宿主机路径与CI容器路径不一致） # 改用 docker create + docker cp 方式把代码拷进容器 - CONTAINER_NAME="staging-api-tests-${GITHUB_SHA::8}" - docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + CONTAINER_NAME="staging-api-tests-$$" docker create --name "$CONTAINER_NAME" \ -e E2E_BASE_URL=https://staging.xiaoxiajianji.com \ -e E2E_API_BASE=https://staging-api.xiaoxiajianji.com/api/v1 \ @@ -1620,6 +1550,7 @@ jobs: set -eu python3 scripts/ci/acr_cleanup.py \ --keep 20 \ + --pr-days 7 \ --execute - name: Job duration summary @@ -1731,3 +1662,160 @@ jobs: START_TIME="" [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true + + ci-gate: + name: CI Gate + runs-on: ci-l2 + if: always() && github.event_name == 'pull_request' + needs: + - check-frontend-only + - validate-code-quality + - validate-type-check + - validate-migration + - unit-tests + - integration-tests + - frontend-lint + - frontend-unit-test + - build-pr + timeout-minutes: 3 + steps: + - name: Checkout code + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + curl -sH "Authorization: token $GITHUB_TOKEN" \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/raw/scripts/ci/step_checkout.sh?ref=${GITHUB_SHA}" \ + | bash + + - name: Evaluate CI Gate + id: gate + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + RESULT_CHECK_FRONTEND: ${{ needs.check-frontend-only.result }} + RESULT_CODE_QUALITY: ${{ needs.validate-code-quality.result }} + RESULT_TYPE_CHECK: ${{ needs.validate-type-check.result }} + RESULT_MIGRATION: ${{ needs.validate-migration.result }} + RESULT_UNIT_TESTS: ${{ needs.unit-tests.result }} + RESULT_INTEGRATION: ${{ needs.integration-tests.result }} + RESULT_FRONTEND_LINT: ${{ needs.frontend-lint.result }} + RESULT_FRONTEND_UNIT: ${{ needs.frontend-unit-test.result }} + RESULT_BUILD_PR: ${{ needs.build-pr.result }} + run: | + set -eu + echo "=== CI Gate 评估 ===" + echo "" + echo "各job结果:" + echo " check-frontend-only: $RESULT_CHECK_FRONTEND" + echo " validate-code-quality: $RESULT_CODE_QUALITY" + echo " validate-type-check: $RESULT_TYPE_CHECK" + echo " validate-migration: $RESULT_MIGRATION" + echo " unit-tests: $RESULT_UNIT_TESTS" + echo " integration-tests: $RESULT_INTEGRATION" + echo " frontend-lint: $RESULT_FRONTEND_LINT" + echo " frontend-unit-test: $RESULT_FRONTEND_UNIT" + echo " build-pr: $RESULT_BUILD_PR" + echo "" + + # 判断PR类型 + SKIP_BACKEND="${{ needs.check-frontend-only.outputs.skip_backend }}" + SKIP_FRONTEND="${{ needs.check-frontend-only.outputs.skip_frontend }}" + echo "PR类型: skip_backend=$SKIP_BACKEND, skip_frontend=$SKIP_FRONTEND" + + # 必填检查项（根据PR类型决定） + # 通用检查（所有PR都必须过） + REQUIRED_GENERAL=( + "validate-code-quality:$RESULT_CODE_QUALITY" + "validate-type-check:$RESULT_TYPE_CHECK" + "validate-migration:$RESULT_MIGRATION" + "frontend-lint:$RESULT_FRONTEND_LINT" + "build-pr:$RESULT_BUILD_PR" + ) + + # 后端检查 + REQUIRED_BACKEND=( + "unit-tests:$RESULT_UNIT_TESTS" + ) + + # 前端检查 + REQUIRED_FRONTEND=( + "frontend-unit-test:$RESULT_FRONTEND_UNIT" + ) + + ALL_PASSED=true + FAILED_ITEMS=() + + check_job() { + local name=$1 + local result=$2 + if [ "$result" = "success" ]; then + echo " ✅ $name: success" + elif [ "$result" = "skipped" ]; then + echo " ⏭️ $name: skipped（跳过，不影响）" + else + echo " ❌ $name: $result" + ALL_PASSED=false + FAILED_ITEMS+=("$name=$result") + fi + } + + echo "" + echo "=== 通用检查（所有PR必填）===" + for item in "${REQUIRED_GENERAL[@]}"; do + name="${item%%:*}" + result="${item##*:}" + check_job "$name" "$result" + done + + if [ "$SKIP_BACKEND" != "true" ]; then + echo "" + echo "=== 后端检查 ===" + for item in "${REQUIRED_BACKEND[@]}"; do + name="${item%%:*}" + result="${item##*:}" + check_job "$name" "$result" + done + else + echo "" + echo "=== 后端检查（纯前端PR，跳过）===" + fi + + if [ "$SKIP_FRONTEND" != "true" ]; then + echo "" + echo "=== 前端检查 ===" + for item in "${REQUIRED_FRONTEND[@]}"; do + name="${item%%:*}" + result="${item##*:}" + check_job "$name" "$result" + done + else + echo "" + echo "=== 前端检查（纯后端PR，跳过）===" + fi + + echo "" + if [ "$ALL_PASSED" = "true" ]; then + echo "✅ CI Gate: PASSED" + echo "gate_result=success" >> $GITHUB_OUTPUT + exit 0 + else + echo "❌ CI Gate: FAILED" + echo "失败项: ${FAILED_ITEMS[*]}" + echo "gate_result=failure" >> $GITHUB_OUTPUT + exit 0 + fi + + - name: Report CI trace + if: always() + shell: sh + env: + AGENTLOOP_LICENSE_KEY: ${{ secrets.AGENTLOOP_LICENSE_KEY }} + run: | + STATUS="ok" + [ "${{ steps.gate.outputs.gate_result }}" = "success" ] || STATUS="error" + START_TIME="" + [ -f /tmp/ci_job_start_time ] && START_TIME=$(cat /tmp/ci_job_start_time) + python3 scripts/ci/ci_trace_report.py --service xiaoxia-saas-ci --status $STATUS --start-time "$START_TIME" || true