From 824222de873f9a6e4d10b8efeb9b39e2441ae4cf Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Fri, 24 Jul 2026 16:57:52 +0800 Subject: [PATCH 1/4] =?UTF-8?q?perf(ci):=20Worker=E5=9F=BA=E7=A1=80?= =?UTF-8?q?=E9=95=9C=E5=83=8F=E4=B8=89=E7=BA=A7=E7=BC=93=E5=AD=98=E7=AD=96?= =?UTF-8?q?=E7=95=A5=E4=BC=98=E5=8C=96\n\n-=20L1=20=E6=9C=AC=E5=9C=B0daemo?= =?UTF-8?q?n=E7=BC=93=E5=AD=98=EF=BC=9ADooD=E6=A8=A1=E5=BC=8F8runner?= =?UTF-8?q?=E5=85=B1=E4=BA=AB=E5=AE=BF=E4=B8=BB=E6=9C=BAdaemon=EF=BC=8C?= =?UTF-8?q?=E5=91=BD=E4=B8=AD=E5=8D=B3=E7=A7=92=E8=BF=87\n-=20L2=20Registr?= =?UTF-8?q?y=E7=BC=93=E5=AD=98=EF=BC=9A=E4=BB=8EGitea=20registry=E6=8B=89?= =?UTF-8?q?=E5=8F=96=E5=90=8E=E9=87=8Dtag=E4=BE=9BDockerfile=E4=BD=BF?= =?UTF-8?q?=E7=94=A8\n-=20L3=20=E6=9C=AC=E5=9C=B0=E6=9E=84=E5=BB=BA?= =?UTF-8?q?=EF=BC=9A=E6=9E=84=E5=BB=BA=E6=88=90=E5=8A=9F=E5=90=8E=E6=8E=A8?= =?UTF-8?q?=E9=80=81=E5=9B=9ERegistry=E4=BE=9B=E5=90=8E=E7=BB=AD=E5=A4=8D?= =?UTF-8?q?=E7=94=A8\n-=20=E4=BF=AE=E5=A4=8Dpre-build=E6=9E=84=E5=BB=BA?= =?UTF-8?q?=E9=95=9C=E5=83=8Ftag=E4=B8=8EDockerfile=20FROM=E4=B8=8D?= =?UTF-8?q?=E4=B8=80=E8=87=B4=E7=9A=84bug\n-=20=E6=99=AE=E9=80=9Adocker=20?= =?UTF-8?q?build=E6=94=B9=E7=94=A8DOCKER=5FBUILDKIT=3D1=E5=8A=A0=E9=80=9F\?= =?UTF-8?q?n-=20Worker=20Dockerfile=E5=B1=82=E9=A1=BA=E5=BA=8F=E4=BC=98?= =?UTF-8?q?=E5=8C=96=EF=BC=8C=E5=8F=98=E5=8C=96=E5=B0=91=E7=9A=84=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E6=94=BE=E5=89=8D=E9=9D=A2\n-=20=E7=A7=BB=E9=99=A4bui?= =?UTF-8?q?lder=E9=98=B6=E6=AE=B5=E5=86=97=E4=BD=99=E7=9A=84=E5=85=A8?= =?UTF-8?q?=E9=87=8Fstrip=EF=BC=88base=E9=95=9C=E5=83=8F=E5=B7=B2strip?= =?UTF-8?q?=E8=BF=87=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/ci-pipeline.yml | 136 +++++++++++++++++-------------- infra/docker/worker.Dockerfile | 45 +++++----- 2 files changed, 100 insertions(+), 81 deletions(-) diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml index 329be86cd..6c2624485 100755 --- a/.gitea/workflows/ci-pipeline.yml +++ b/.gitea/workflows/ci-pipeline.yml @@ -638,65 +638,83 @@ jobs: echo "Docker login failed ($i/3), retrying in 5s..." sleep 5 done - - name: Pre-build worker base images (fallback if not exist) + - name: Pre-build worker base images (3-level cache) if: matrix.service == 'worker' id: prebuild - shell: sh + shell: bash run: | set -eu - REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas" - BASE_BUILDER="${REGISTRY}/worker-base-builder:latest" - BASE_RUNTIME="${REGISTRY}/worker-base-runtime:latest" - - # 尝试拉取基础镜像 - echo "检查基础镜像..." - if docker pull "$BASE_BUILDER" 2>/dev/null && docker pull "$BASE_RUNTIME" 2>/dev/null; then - echo "基础镜像已存在，使用远程镜像" - echo "fallback=false" >> $GITHUB_OUTPUT - else - echo "基础镜像不存在，本地构建（fallback模式）..." - - # 构建builder基础镜像 - echo "构建 worker-base-builder..." - # 用buildx docker-container驱动构建（兼容DooD模式：普通docker build看不到容器内文件） - BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}" - if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then - docker buildx create --use --name "$BUILDER_NAME" --driver docker-container - else - docker buildx use "$BUILDER_NAME" - fi - docker buildx inspect --bootstrap > /dev/null 2>&1 - - # 构建builder基础镜像（带重试，buildx容器偶发不稳定） - echo "构建 worker-base-builder..." - for attempt in 1 2 3; do - if docker buildx build --load -f infra/docker/worker-base-builder.Dockerfile -t "$BASE_BUILDER" .; then - echo "worker-base-builder 构建成功" - break - fi - echo "worker-base-builder 构建失败，重试 $attempt/3..." - docker buildx rm "$BUILDER_NAME" 2>/dev/null || true - docker buildx create --use --name "$BUILDER_NAME" --driver docker-container - sleep 3 - done - - # 构建runtime基础镜像 - echo "构建 worker-base-runtime..." - for attempt in 1 2 3; do - if docker buildx build --load -f infra/docker/worker-base-runtime.Dockerfile -t "$BASE_RUNTIME" .; then - echo "worker-base-runtime 构建成功" - break - fi - echo "worker-base-runtime 构建失败，重试 $attempt/3..." - docker buildx rm "$BUILDER_NAME" 2>/dev/null || true - docker buildx create --use --name "$BUILDER_NAME" --driver docker-container - sleep 3 - done - - echo "fallback=true" >> $GITHUB_OUTPUT - echo "基础镜像本地构建完成" - fi + GITEA_REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas" + ACR_REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji" + GITEA_BUILDER="${GITEA_REGISTRY}/worker-base-builder:latest" + GITEA_RUNTIME="${GITEA_REGISTRY}/worker-base-runtime:latest" + ACR_BUILDER="${ACR_REGISTRY}/worker-base-builder:latest" + ACR_RUNTIME="${ACR_REGISTRY}/worker-base-runtime:latest" + # L1: 本地daemon缓存（DooD模式8runner共享宿主机daemon） + echo "=== L1 本地缓存 ===" + if docker image inspect "$ACR_BUILDER" > /dev/null 2>&1 \ + && docker image inspect "$ACR_RUNTIME" > /dev/null 2>&1; then + echo "本地缓存命中" + echo "has_local_base=true" >> $GITHUB_OUTPUT + exit 0 + fi + echo "本地无缓存" + + # L2: Gitea registry缓存（内网快） + echo "=== L2 Registry拉取 ===" + if docker pull "$GITEA_BUILDER" 2>/dev/null && docker pull "$GITEA_RUNTIME" 2>/dev/null; then + echo "Registry拉取成功，重tag供Dockerfile使用" + docker tag "$GITEA_BUILDER" "$ACR_BUILDER" + docker tag "$GITEA_RUNTIME" "$ACR_RUNTIME" + echo "has_local_base=true" >> $GITHUB_OUTPUT + exit 0 + fi + echo "Registry无缓存，需本地构建" + + # L3: 本地构建 + echo "=== L3 本地构建 ===" + BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}" + if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then + docker buildx create --use --name "$BUILDER_NAME" --driver docker-container + else + docker buildx use "$BUILDER_NAME" + fi + docker buildx inspect --bootstrap > /dev/null 2>&1 + + echo "构建 worker-base-builder..." + for attempt in 1 2 3; do + if docker buildx build --load -f infra/docker/worker-base-builder.Dockerfile -t "$ACR_BUILDER" .; then + echo "worker-base-builder 构建成功" + break + fi + echo "worker-base-builder 失败，重试 $attempt/3..." + docker buildx rm "$BUILDER_NAME" 2>/dev/null || true + docker buildx create --use --name "$BUILDER_NAME" --driver docker-container + sleep 3 + done + + echo "构建 worker-base-runtime..." + for attempt in 1 2 3; do + if docker buildx build --load -f infra/docker/worker-base-runtime.Dockerfile -t "$ACR_RUNTIME" .; then + echo "worker-base-runtime 构建成功" + break + fi + echo "worker-base-runtime 失败，重试 $attempt/3..." + docker buildx rm "$BUILDER_NAME" 2>/dev/null || true + docker buildx create --use --name "$BUILDER_NAME" --driver docker-container + sleep 3 + done + + # 推送到Gitea registry供后续复用 + echo "=== 推送缓存到Registry ===" + docker tag "$ACR_BUILDER" "$GITEA_BUILDER" + docker tag "$ACR_RUNTIME" "$GITEA_RUNTIME" + docker push "$GITEA_BUILDER" 2>/dev/null || echo "push builder失败（不影响）" + docker push "$GITEA_RUNTIME" 2>/dev/null || echo "push runtime失败（不影响）" + + echo "has_local_base=true" >> $GITHUB_OUTPUT + echo "基础镜像构建完成" - name: Build PR image (verify only, no push) shell: sh run: | @@ -710,15 +728,15 @@ jobs: EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf" fi - # Worker fallback模式：基础镜像本地已构建，用普通docker build绕过buildx - if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.fallback }}" = "true" ]; then - echo "Fallback模式：用普通docker build（基础镜像本地已构建）" + # Worker有本地base镜像时：用BuildKit直接构建（快，无需起buildx容器） + if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.has_local_base }}" = "true" ]; then + echo "本地base镜像已就绪，BuildKit快速构建" BUILD_ARG_STR="" for arg in $EXTRA_BUILD_ARGS; do BUILD_ARG_STR="$BUILD_ARG_STR --build-arg $arg" done - docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR . - echo "Fallback PR Build successful" + DOCKER_BUILDKIT=1 docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR . + echo "快速构建成功" exit 0 fi diff --git a/infra/docker/worker.Dockerfile b/infra/docker/worker.Dockerfile index beb15a399..7805b9d79 100755 --- a/infra/docker/worker.Dockerfile +++ b/infra/docker/worker.Dockerfile @@ -2,7 +2,6 @@ # Worker Dockerfile - 分层缓存优化版 # 优化：基础依赖 + Worker大包预构建为基础镜像，业务构建仅叠加业务依赖 # 基础镜像：worker-base-builder / worker-base-runtime -# 预计节省：依赖不变时构建时间从23min降至5min以内 # ============================================================ # ==================== Builder 阶段 ==================== @@ -24,8 +23,7 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \ -r /tmp/requirements.txt \ && rm /tmp/requirements.txt -# ---- 增量瘦身（只处理新增的业务依赖）---- -RUN find /opt/venv -name "*.so" -type f -exec strip --strip-all {} \; 2>/dev/null || true +# ---- 增量瘦身（清理新增业务依赖的冗余文件）---- RUN find /opt/venv -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null; \ find /opt/venv -name "*.pyc" -delete 2>/dev/null || true @@ -41,30 +39,33 @@ ARG APP_VERSION=dev # 从 builder 复制 Python 虚拟环境 COPY --from=builder /opt/venv /opt/venv -# 设置工作目录 -WORKDIR /app - -# 复制应用代码 -COPY apps/worker/ /app/apps/worker/ -COPY apps/api/app/config.py /app/apps/api/app/config.py -COPY apps/api/app/core/ /app/apps/api/app/core/ -COPY packages/ /app/packages/ -COPY alembic.ini /app/alembic.ini -COPY migrations/ /app/migrations/ - -# 复制 Worker 启动脚本 -COPY infra/docker/entrypoint-worker.sh /usr/local/bin/entrypoint-worker.sh -RUN chmod +x /usr/local/bin/entrypoint-worker.sh - -# 设置 Python 路径 +# 设置 Python 环境变量 ENV PATH="/opt/venv/bin:$PATH" ENV PYTHONPATH=/app:/app/packages ENV PYTHONUNBUFFERED=1 ENV APP_VERSION=$APP_VERSION -# 创建非 root 用户运行 Worker -RUN groupadd -r celery && useradd -r -g celery -d /app -s /sbin/nologin celery \ - && mkdir -p /app/generated && chown celery:celery /app/generated +# 创建非 root 用户（极少变化，放最前） +RUN groupadd -r celery \ + && useradd -r -g celery -d /app -s /sbin/nologin celery \ + && mkdir -p /app/generated \ + && chown celery:celery /app/generated + +WORKDIR /app + +# 复制文件按变化频率从低到高排序，最大化层缓存命中 +COPY alembic.ini /app/alembic.ini +COPY migrations/ /app/migrations/ +COPY packages/ /app/packages/ +COPY apps/api/app/config.py /app/apps/api/app/config.py +COPY apps/api/app/core/ /app/apps/api/app/core/ + +# 复制 Worker 启动脚本 +COPY infra/docker/entrypoint-worker.sh /usr/local/bin/entrypoint-worker.sh +RUN chmod +x /usr/local/bin/entrypoint-worker.sh + +# 业务代码（变化最频繁，放最后） +COPY apps/worker/ /app/apps/worker/ USER celery -- 2.54.0 From a1ba05d8692c5ca818456420825c54a785c7d543 Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Fri, 24 Jul 2026 17:38:16 +0800 Subject: [PATCH 2/4] =?UTF-8?q?fix(ci):=20=E8=A1=A5=E5=85=85unit/integrati?= =?UTF-8?q?on=20tests=E8=A1=A5=E5=85=85JWT=5FSECRET=5FKEY=E7=8E=AF?= =?UTF-8?q?=E5=A2=83=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/ci-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml index 6c2624485..b0b3d7eb3 100755 --- a/.gitea/workflows/ci-pipeline.yml +++ b/.gitea/workflows/ci-pipeline.yml @@ -346,6 +346,7 @@ jobs: OSS_ACCESS_KEY_SECRET: placeholder OSS_BUCKET_NAME: xiaoxia-autocut OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com + JWT_SECRET_KEY: test-jwt-secret-for-ci-only-2026 steps: - name: Checkout code shell: sh @@ -417,6 +418,7 @@ jobs: OSS_ACCESS_KEY_SECRET: placeholder OSS_BUCKET_NAME: xiaoxia-autocut OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com + JWT_SECRET_KEY: test-jwt-secret-for-ci-only-2026 steps: - name: Checkout code shell: sh -- 2.54.0 From 38617515eed6bb7713520db1214cc4450948758f Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Fri, 24 Jul 2026 17:53:20 +0800 Subject: [PATCH 3/4] =?UTF-8?q?fix(ci):=20unit=20tests=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E5=85=9C=E5=BA=95JWT=5FSECRET=5FKEY?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/ci/run_unit_tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/ci/run_unit_tests.sh b/scripts/ci/run_unit_tests.sh index f6613c8c1..d1804b723 100755 --- a/scripts/ci/run_unit_tests.sh +++ b/scripts/ci/run_unit_tests.sh @@ -3,6 +3,9 @@ # 包含：依赖安装、增量测试选择、覆盖率测试、diff覆盖率门禁 set -eu +# 测试环境必须的密钥变量 +export JWT_SECRET_KEY=${JWT_SECRET_KEY:-test-jwt-secret-for-ci-only-2026} + JOB_NAME="${1:-Unit Tests}" echo "=== CI Unit Tests 开始 ===" -- 2.54.0 From 5b42ff0e962d75cedc8e5a67ac0cf7ed3efe8f45 Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Fri, 24 Jul 2026 18:49:32 +0800 Subject: [PATCH 4/4] =?UTF-8?q?style:=20black=E6=A0=BC=E5=BC=8F=E5=8C=96te?= =?UTF-8?q?st=5Fapi=5Fsettings.py=E6=A0=BC=E5=BC=8F=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/unit/test_api_settings.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_api_settings.py b/tests/unit/test_api_settings.py index f7689fd31..1b4c776f0 100755 --- a/tests/unit/test_api_settings.py +++ b/tests/unit/test_api_settings.py @@ -12,11 +12,23 @@ from packages.config.base import SharedSettings, get_cached_settings, reload_set def _reset_cache(): """每个测试前清空配置缓存，避免单例污染.""" reload_settings_cache() + # 保存关键环境变量（避免其他测试模块的全局污染） + _saved_env = {} + for key in ["JWT_SECRET_KEY", "DATABASE_URL", "USE_IN_MEMORY_DB", "APP_ENV"]: + _saved_env[key] = os.environ.get(key) # 设置必要的环境变量，避免 JWT 校验失败 os.environ["JWT_SECRET_KEY"] = "test-secret-key-for-unit-tests-only-12345" + # 清除可能被其他模块污染的变量，确保默认值测试准确 + for key in ["DATABASE_URL", "APP_ENV"]: + os.environ.pop(key, None) yield reload_settings_cache() - os.environ.pop("JWT_SECRET_KEY", None) + # 恢复所有保存的环境变量，避免污染其他测试模块 + for key, val in _saved_env.items(): + if val is None: + os.environ.pop(key, None) + else: + os.environ[key] = val class TestSharedSettingsDefaults: -- 2.54.0