diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml index 329be86cd..b0b3d7eb3 100755 --- a/.gitea/workflows/ci-pipeline.yml +++ b/.gitea/workflows/ci-pipeline.yml @@ -346,6 +346,7 @@ jobs: OSS_ACCESS_KEY_SECRET: placeholder OSS_BUCKET_NAME: xiaoxia-autocut OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com + JWT_SECRET_KEY: test-jwt-secret-for-ci-only-2026 steps: - name: Checkout code shell: sh @@ -417,6 +418,7 @@ jobs: OSS_ACCESS_KEY_SECRET: placeholder OSS_BUCKET_NAME: xiaoxia-autocut OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com + JWT_SECRET_KEY: test-jwt-secret-for-ci-only-2026 steps: - name: Checkout code shell: sh @@ -638,65 +640,83 @@ jobs: echo "Docker login failed ($i/3), retrying in 5s..." sleep 5 done - - name: Pre-build worker base images (fallback if not exist) + - name: Pre-build worker base images (3-level cache) if: matrix.service == 'worker' id: prebuild - shell: sh + shell: bash run: | set -eu - REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas" - BASE_BUILDER="${REGISTRY}/worker-base-builder:latest" - BASE_RUNTIME="${REGISTRY}/worker-base-runtime:latest" - - # 尝试拉取基础镜像 - echo "检查基础镜像..." - if docker pull "$BASE_BUILDER" 2>/dev/null && docker pull "$BASE_RUNTIME" 2>/dev/null; then - echo "基础镜像已存在,使用远程镜像" - echo "fallback=false" >> $GITHUB_OUTPUT - else - echo "基础镜像不存在,本地构建(fallback模式)..." - - # 构建builder基础镜像 - echo "构建 worker-base-builder..." - # 用buildx docker-container驱动构建(兼容DooD模式:普通docker build看不到容器内文件) - BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}" - if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then - docker buildx create --use --name "$BUILDER_NAME" --driver docker-container - else - docker buildx use "$BUILDER_NAME" - fi - docker buildx inspect --bootstrap > /dev/null 2>&1 - - # 构建builder基础镜像(带重试,buildx容器偶发不稳定) - echo "构建 worker-base-builder..." - for attempt in 1 2 3; do - if docker buildx build --load -f infra/docker/worker-base-builder.Dockerfile -t "$BASE_BUILDER" .; then - echo "worker-base-builder 构建成功" - break - fi - echo "worker-base-builder 构建失败,重试 $attempt/3..." - docker buildx rm "$BUILDER_NAME" 2>/dev/null || true - docker buildx create --use --name "$BUILDER_NAME" --driver docker-container - sleep 3 - done - - # 构建runtime基础镜像 - echo "构建 worker-base-runtime..." - for attempt in 1 2 3; do - if docker buildx build --load -f infra/docker/worker-base-runtime.Dockerfile -t "$BASE_RUNTIME" .; then - echo "worker-base-runtime 构建成功" - break - fi - echo "worker-base-runtime 构建失败,重试 $attempt/3..." - docker buildx rm "$BUILDER_NAME" 2>/dev/null || true - docker buildx create --use --name "$BUILDER_NAME" --driver docker-container - sleep 3 - done - - echo "fallback=true" >> $GITHUB_OUTPUT - echo "基础镜像本地构建完成" - fi + GITEA_REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas" + ACR_REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji" + GITEA_BUILDER="${GITEA_REGISTRY}/worker-base-builder:latest" + GITEA_RUNTIME="${GITEA_REGISTRY}/worker-base-runtime:latest" + ACR_BUILDER="${ACR_REGISTRY}/worker-base-builder:latest" + ACR_RUNTIME="${ACR_REGISTRY}/worker-base-runtime:latest" + # L1: 本地daemon缓存(DooD模式8runner共享宿主机daemon) + echo "=== L1 本地缓存 ===" + if docker image inspect "$ACR_BUILDER" > /dev/null 2>&1 \ + && docker image inspect "$ACR_RUNTIME" > /dev/null 2>&1; then + echo "本地缓存命中" + echo "has_local_base=true" >> $GITHUB_OUTPUT + exit 0 + fi + echo "本地无缓存" + + # L2: Gitea registry缓存(内网快) + echo "=== L2 Registry拉取 ===" + if docker pull "$GITEA_BUILDER" 2>/dev/null && docker pull "$GITEA_RUNTIME" 2>/dev/null; then + echo "Registry拉取成功,重tag供Dockerfile使用" + docker tag "$GITEA_BUILDER" "$ACR_BUILDER" + docker tag "$GITEA_RUNTIME" "$ACR_RUNTIME" + echo "has_local_base=true" >> $GITHUB_OUTPUT + exit 0 + fi + echo "Registry无缓存,需本地构建" + + # L3: 本地构建 + echo "=== L3 本地构建 ===" + BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}" + if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then + docker buildx create --use --name "$BUILDER_NAME" --driver docker-container + else + docker buildx use "$BUILDER_NAME" + fi + docker buildx inspect --bootstrap > /dev/null 2>&1 + + echo "构建 worker-base-builder..." + for attempt in 1 2 3; do + if docker buildx build --load -f infra/docker/worker-base-builder.Dockerfile -t "$ACR_BUILDER" .; then + echo "worker-base-builder 构建成功" + break + fi + echo "worker-base-builder 失败,重试 $attempt/3..." + docker buildx rm "$BUILDER_NAME" 2>/dev/null || true + docker buildx create --use --name "$BUILDER_NAME" --driver docker-container + sleep 3 + done + + echo "构建 worker-base-runtime..." + for attempt in 1 2 3; do + if docker buildx build --load -f infra/docker/worker-base-runtime.Dockerfile -t "$ACR_RUNTIME" .; then + echo "worker-base-runtime 构建成功" + break + fi + echo "worker-base-runtime 失败,重试 $attempt/3..." + docker buildx rm "$BUILDER_NAME" 2>/dev/null || true + docker buildx create --use --name "$BUILDER_NAME" --driver docker-container + sleep 3 + done + + # 推送到Gitea registry供后续复用 + echo "=== 推送缓存到Registry ===" + docker tag "$ACR_BUILDER" "$GITEA_BUILDER" + docker tag "$ACR_RUNTIME" "$GITEA_RUNTIME" + docker push "$GITEA_BUILDER" 2>/dev/null || echo "push builder失败(不影响)" + docker push "$GITEA_RUNTIME" 2>/dev/null || echo "push runtime失败(不影响)" + + echo "has_local_base=true" >> $GITHUB_OUTPUT + echo "基础镜像构建完成" - name: Build PR image (verify only, no push) shell: sh run: | @@ -710,15 +730,15 @@ jobs: EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf" fi - # Worker fallback模式:基础镜像本地已构建,用普通docker build绕过buildx - if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.fallback }}" = "true" ]; then - echo "Fallback模式:用普通docker build(基础镜像本地已构建)" + # Worker有本地base镜像时:用BuildKit直接构建(快,无需起buildx容器) + if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.has_local_base }}" = "true" ]; then + echo "本地base镜像已就绪,BuildKit快速构建" BUILD_ARG_STR="" for arg in $EXTRA_BUILD_ARGS; do BUILD_ARG_STR="$BUILD_ARG_STR --build-arg $arg" done - docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR . - echo "Fallback PR Build successful" + DOCKER_BUILDKIT=1 docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR . + echo "快速构建成功" exit 0 fi diff --git a/infra/docker/worker.Dockerfile b/infra/docker/worker.Dockerfile index beb15a399..7805b9d79 100755 --- a/infra/docker/worker.Dockerfile +++ b/infra/docker/worker.Dockerfile @@ -2,7 +2,6 @@ # Worker Dockerfile - 分层缓存优化版 # 优化:基础依赖 + Worker大包预构建为基础镜像,业务构建仅叠加业务依赖 # 基础镜像:worker-base-builder / worker-base-runtime -# 预计节省:依赖不变时构建时间从23min降至5min以内 # ============================================================ # ==================== Builder 阶段 ==================== @@ -24,8 +23,7 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \ -r /tmp/requirements.txt \ && rm /tmp/requirements.txt -# ---- 增量瘦身(只处理新增的业务依赖)---- -RUN find /opt/venv -name "*.so" -type f -exec strip --strip-all {} \; 2>/dev/null || true +# ---- 增量瘦身(清理新增业务依赖的冗余文件)---- RUN find /opt/venv -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null; \ find /opt/venv -name "*.pyc" -delete 2>/dev/null || true @@ -41,30 +39,33 @@ ARG APP_VERSION=dev # 从 builder 复制 Python 虚拟环境 COPY --from=builder /opt/venv /opt/venv -# 设置工作目录 -WORKDIR /app - -# 复制应用代码 -COPY apps/worker/ /app/apps/worker/ -COPY apps/api/app/config.py /app/apps/api/app/config.py -COPY apps/api/app/core/ /app/apps/api/app/core/ -COPY packages/ /app/packages/ -COPY alembic.ini /app/alembic.ini -COPY migrations/ /app/migrations/ - -# 复制 Worker 启动脚本 -COPY infra/docker/entrypoint-worker.sh /usr/local/bin/entrypoint-worker.sh -RUN chmod +x /usr/local/bin/entrypoint-worker.sh - -# 设置 Python 路径 +# 设置 Python 环境变量 ENV PATH="/opt/venv/bin:$PATH" ENV PYTHONPATH=/app:/app/packages ENV PYTHONUNBUFFERED=1 ENV APP_VERSION=$APP_VERSION -# 创建非 root 用户运行 Worker -RUN groupadd -r celery && useradd -r -g celery -d /app -s /sbin/nologin celery \ - && mkdir -p /app/generated && chown celery:celery /app/generated +# 创建非 root 用户(极少变化,放最前) +RUN groupadd -r celery \ + && useradd -r -g celery -d /app -s /sbin/nologin celery \ + && mkdir -p /app/generated \ + && chown celery:celery /app/generated + +WORKDIR /app + +# 复制文件按变化频率从低到高排序,最大化层缓存命中 +COPY alembic.ini /app/alembic.ini +COPY migrations/ /app/migrations/ +COPY packages/ /app/packages/ +COPY apps/api/app/config.py /app/apps/api/app/config.py +COPY apps/api/app/core/ /app/apps/api/app/core/ + +# 复制 Worker 启动脚本 +COPY infra/docker/entrypoint-worker.sh /usr/local/bin/entrypoint-worker.sh +RUN chmod +x /usr/local/bin/entrypoint-worker.sh + +# 业务代码(变化最频繁,放最后) +COPY apps/worker/ /app/apps/worker/ USER celery diff --git a/scripts/ci/run_unit_tests.sh b/scripts/ci/run_unit_tests.sh index f6613c8c1..d1804b723 100755 --- a/scripts/ci/run_unit_tests.sh +++ b/scripts/ci/run_unit_tests.sh @@ -3,6 +3,9 @@ # 包含:依赖安装、增量测试选择、覆盖率测试、diff覆盖率门禁 set -eu +# 测试环境必须的密钥变量 +export JWT_SECRET_KEY=${JWT_SECRET_KEY:-test-jwt-secret-for-ci-only-2026} + JOB_NAME="${1:-Unit Tests}" echo "=== CI Unit Tests 开始 ===" diff --git a/tests/unit/test_api_settings.py b/tests/unit/test_api_settings.py index f7689fd31..1b4c776f0 100755 --- a/tests/unit/test_api_settings.py +++ b/tests/unit/test_api_settings.py @@ -12,11 +12,23 @@ from packages.config.base import SharedSettings, get_cached_settings, reload_set def _reset_cache(): """每个测试前清空配置缓存,避免单例污染.""" reload_settings_cache() + # 保存关键环境变量(避免其他测试模块的全局污染) + _saved_env = {} + for key in ["JWT_SECRET_KEY", "DATABASE_URL", "USE_IN_MEMORY_DB", "APP_ENV"]: + _saved_env[key] = os.environ.get(key) # 设置必要的环境变量,避免 JWT 校验失败 os.environ["JWT_SECRET_KEY"] = "test-secret-key-for-unit-tests-only-12345" + # 清除可能被其他模块污染的变量,确保默认值测试准确 + for key in ["DATABASE_URL", "APP_ENV"]: + os.environ.pop(key, None) yield reload_settings_cache() - os.environ.pop("JWT_SECRET_KEY", None) + # 恢复所有保存的环境变量,避免污染其他测试模块 + for key, val in _saved_env.items(): + if val is None: + os.environ.pop(key, None) + else: + os.environ[key] = val class TestSharedSettingsDefaults: