Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 xiaoxia
|
5b42ff0e96 | ||
|
xiaoxia
|
38617515ee | ||
|
xiaoxia
|
a1ba05d869 | ||
|
xiaoxia
|
824222de87 |
@@ -346,6 +346,7 @@ jobs:
|
||||
OSS_ACCESS_KEY_SECRET: placeholder
|
||||
OSS_BUCKET_NAME: xiaoxia-autocut
|
||||
OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com
|
||||
JWT_SECRET_KEY: test-jwt-secret-for-ci-only-2026
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
@@ -417,6 +418,7 @@ jobs:
|
||||
OSS_ACCESS_KEY_SECRET: placeholder
|
||||
OSS_BUCKET_NAME: xiaoxia-autocut
|
||||
OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com
|
||||
JWT_SECRET_KEY: test-jwt-secret-for-ci-only-2026
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
@@ -638,65 +640,83 @@ jobs:
|
||||
echo "Docker login failed ($i/3), retrying in 5s..."
|
||||
sleep 5
|
||||
done
|
||||
- name: Pre-build worker base images (fallback if not exist)
|
||||
- name: Pre-build worker base images (3-level cache)
|
||||
if: matrix.service == 'worker'
|
||||
id: prebuild
|
||||
shell: sh
|
||||
shell: bash
|
||||
run: |
|
||||
set -eu
|
||||
REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas"
|
||||
BASE_BUILDER="${REGISTRY}/worker-base-builder:latest"
|
||||
BASE_RUNTIME="${REGISTRY}/worker-base-runtime:latest"
|
||||
|
||||
# 尝试拉取基础镜像
|
||||
echo "检查基础镜像..."
|
||||
if docker pull "$BASE_BUILDER" 2>/dev/null && docker pull "$BASE_RUNTIME" 2>/dev/null; then
|
||||
echo "基础镜像已存在，使用远程镜像"
|
||||
echo "fallback=false" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "基础镜像不存在，本地构建（fallback模式）..."
|
||||
|
||||
# 构建builder基础镜像
|
||||
echo "构建 worker-base-builder..."
|
||||
# 用buildx docker-container驱动构建（兼容DooD模式：普通docker build看不到容器内文件）
|
||||
BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}"
|
||||
if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
else
|
||||
docker buildx use "$BUILDER_NAME"
|
||||
fi
|
||||
docker buildx inspect --bootstrap > /dev/null 2>&1
|
||||
|
||||
# 构建builder基础镜像（带重试，buildx容器偶发不稳定）
|
||||
echo "构建 worker-base-builder..."
|
||||
for attempt in 1 2 3; do
|
||||
if docker buildx build --load -f infra/docker/worker-base-builder.Dockerfile -t "$BASE_BUILDER" .; then
|
||||
echo "worker-base-builder 构建成功"
|
||||
break
|
||||
fi
|
||||
echo "worker-base-builder 构建失败，重试 $attempt/3..."
|
||||
docker buildx rm "$BUILDER_NAME" 2>/dev/null || true
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# 构建runtime基础镜像
|
||||
echo "构建 worker-base-runtime..."
|
||||
for attempt in 1 2 3; do
|
||||
if docker buildx build --load -f infra/docker/worker-base-runtime.Dockerfile -t "$BASE_RUNTIME" .; then
|
||||
echo "worker-base-runtime 构建成功"
|
||||
break
|
||||
fi
|
||||
echo "worker-base-runtime 构建失败，重试 $attempt/3..."
|
||||
docker buildx rm "$BUILDER_NAME" 2>/dev/null || true
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
sleep 3
|
||||
done
|
||||
|
||||
echo "fallback=true" >> $GITHUB_OUTPUT
|
||||
echo "基础镜像本地构建完成"
|
||||
fi
|
||||
GITEA_REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas"
|
||||
ACR_REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
|
||||
GITEA_BUILDER="${GITEA_REGISTRY}/worker-base-builder:latest"
|
||||
GITEA_RUNTIME="${GITEA_REGISTRY}/worker-base-runtime:latest"
|
||||
ACR_BUILDER="${ACR_REGISTRY}/worker-base-builder:latest"
|
||||
ACR_RUNTIME="${ACR_REGISTRY}/worker-base-runtime:latest"
|
||||
|
||||
# L1: 本地daemon缓存（DooD模式8runner共享宿主机daemon）
|
||||
echo "=== L1 本地缓存 ==="
|
||||
if docker image inspect "$ACR_BUILDER" > /dev/null 2>&1 \
|
||||
&& docker image inspect "$ACR_RUNTIME" > /dev/null 2>&1; then
|
||||
echo "本地缓存命中"
|
||||
echo "has_local_base=true" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
echo "本地无缓存"
|
||||
|
||||
# L2: Gitea registry缓存（内网快）
|
||||
echo "=== L2 Registry拉取 ==="
|
||||
if docker pull "$GITEA_BUILDER" 2>/dev/null && docker pull "$GITEA_RUNTIME" 2>/dev/null; then
|
||||
echo "Registry拉取成功，重tag供Dockerfile使用"
|
||||
docker tag "$GITEA_BUILDER" "$ACR_BUILDER"
|
||||
docker tag "$GITEA_RUNTIME" "$ACR_RUNTIME"
|
||||
echo "has_local_base=true" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
echo "Registry无缓存，需本地构建"
|
||||
|
||||
# L3: 本地构建
|
||||
echo "=== L3 本地构建 ==="
|
||||
BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}"
|
||||
if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
else
|
||||
docker buildx use "$BUILDER_NAME"
|
||||
fi
|
||||
docker buildx inspect --bootstrap > /dev/null 2>&1
|
||||
|
||||
echo "构建 worker-base-builder..."
|
||||
for attempt in 1 2 3; do
|
||||
if docker buildx build --load -f infra/docker/worker-base-builder.Dockerfile -t "$ACR_BUILDER" .; then
|
||||
echo "worker-base-builder 构建成功"
|
||||
break
|
||||
fi
|
||||
echo "worker-base-builder 失败，重试 $attempt/3..."
|
||||
docker buildx rm "$BUILDER_NAME" 2>/dev/null || true
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
sleep 3
|
||||
done
|
||||
|
||||
echo "构建 worker-base-runtime..."
|
||||
for attempt in 1 2 3; do
|
||||
if docker buildx build --load -f infra/docker/worker-base-runtime.Dockerfile -t "$ACR_RUNTIME" .; then
|
||||
echo "worker-base-runtime 构建成功"
|
||||
break
|
||||
fi
|
||||
echo "worker-base-runtime 失败，重试 $attempt/3..."
|
||||
docker buildx rm "$BUILDER_NAME" 2>/dev/null || true
|
||||
docker buildx create --use --name "$BUILDER_NAME" --driver docker-container
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# 推送到Gitea registry供后续复用
|
||||
echo "=== 推送缓存到Registry ==="
|
||||
docker tag "$ACR_BUILDER" "$GITEA_BUILDER"
|
||||
docker tag "$ACR_RUNTIME" "$GITEA_RUNTIME"
|
||||
docker push "$GITEA_BUILDER" 2>/dev/null || echo "push builder失败（不影响）"
|
||||
docker push "$GITEA_RUNTIME" 2>/dev/null || echo "push runtime失败（不影响）"
|
||||
|
||||
echo "has_local_base=true" >> $GITHUB_OUTPUT
|
||||
echo "基础镜像构建完成"
|
||||
- name: Build PR image (verify only, no push)
|
||||
shell: sh
|
||||
run: |
|
||||
@@ -710,15 +730,15 @@ jobs:
|
||||
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
|
||||
fi
|
||||
|
||||
# Worker fallback模式：基础镜像本地已构建，用普通docker build绕过buildx
|
||||
if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.fallback }}" = "true" ]; then
|
||||
echo "Fallback模式：用普通docker build（基础镜像本地已构建）"
|
||||
# Worker有本地base镜像时：用BuildKit直接构建（快，无需起buildx容器）
|
||||
if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.has_local_base }}" = "true" ]; then
|
||||
echo "本地base镜像已就绪，BuildKit快速构建"
|
||||
BUILD_ARG_STR=""
|
||||
for arg in $EXTRA_BUILD_ARGS; do
|
||||
BUILD_ARG_STR="$BUILD_ARG_STR --build-arg $arg"
|
||||
done
|
||||
docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR .
|
||||
echo "Fallback PR Build successful"
|
||||
DOCKER_BUILDKIT=1 docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR .
|
||||
echo "快速构建成功"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# Worker Dockerfile - 分层缓存优化版
|
||||
# 优化：基础依赖 + Worker大包预构建为基础镜像，业务构建仅叠加业务依赖
|
||||
# 基础镜像：worker-base-builder / worker-base-runtime
|
||||
# 预计节省：依赖不变时构建时间从23min降至5min以内
|
||||
# ============================================================
|
||||
|
||||
# ==================== Builder 阶段 ====================
|
||||
@@ -24,8 +23,7 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
|
||||
-r /tmp/requirements.txt \
|
||||
&& rm /tmp/requirements.txt
|
||||
|
||||
# ---- 增量瘦身（只处理新增的业务依赖）----
|
||||
RUN find /opt/venv -name "*.so" -type f -exec strip --strip-all {} \; 2>/dev/null || true
|
||||
# ---- 增量瘦身（清理新增业务依赖的冗余文件）----
|
||||
RUN find /opt/venv -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null; \
|
||||
find /opt/venv -name "*.pyc" -delete 2>/dev/null || true
|
||||
|
||||
@@ -41,30 +39,33 @@ ARG APP_VERSION=dev
|
||||
# 从 builder 复制 Python 虚拟环境
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
|
||||
# 设置工作目录
|
||||
WORKDIR /app
|
||||
|
||||
# 复制应用代码
|
||||
COPY apps/worker/ /app/apps/worker/
|
||||
COPY apps/api/app/config.py /app/apps/api/app/config.py
|
||||
COPY apps/api/app/core/ /app/apps/api/app/core/
|
||||
COPY packages/ /app/packages/
|
||||
COPY alembic.ini /app/alembic.ini
|
||||
COPY migrations/ /app/migrations/
|
||||
|
||||
# 复制 Worker 启动脚本
|
||||
COPY infra/docker/entrypoint-worker.sh /usr/local/bin/entrypoint-worker.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint-worker.sh
|
||||
|
||||
# 设置 Python 路径
|
||||
# 设置 Python 环境变量
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
ENV PYTHONPATH=/app:/app/packages
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV APP_VERSION=$APP_VERSION
|
||||
|
||||
# 创建非 root 用户运行 Worker
|
||||
RUN groupadd -r celery && useradd -r -g celery -d /app -s /sbin/nologin celery \
|
||||
&& mkdir -p /app/generated && chown celery:celery /app/generated
|
||||
# 创建非 root 用户（极少变化，放最前）
|
||||
RUN groupadd -r celery \
|
||||
&& useradd -r -g celery -d /app -s /sbin/nologin celery \
|
||||
&& mkdir -p /app/generated \
|
||||
&& chown celery:celery /app/generated
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 复制文件按变化频率从低到高排序，最大化层缓存命中
|
||||
COPY alembic.ini /app/alembic.ini
|
||||
COPY migrations/ /app/migrations/
|
||||
COPY packages/ /app/packages/
|
||||
COPY apps/api/app/config.py /app/apps/api/app/config.py
|
||||
COPY apps/api/app/core/ /app/apps/api/app/core/
|
||||
|
||||
# 复制 Worker 启动脚本
|
||||
COPY infra/docker/entrypoint-worker.sh /usr/local/bin/entrypoint-worker.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint-worker.sh
|
||||
|
||||
# 业务代码（变化最频繁，放最后）
|
||||
COPY apps/worker/ /app/apps/worker/
|
||||
|
||||
USER celery
|
||||
|
||||
|
||||
@@ -3,6 +3,9 @@
|
||||
# 包含：依赖安装、增量测试选择、覆盖率测试、diff覆盖率门禁
|
||||
set -eu
|
||||
|
||||
# 测试环境必须的密钥变量
|
||||
export JWT_SECRET_KEY=${JWT_SECRET_KEY:-test-jwt-secret-for-ci-only-2026}
|
||||
|
||||
JOB_NAME="${1:-Unit Tests}"
|
||||
|
||||
echo "=== CI Unit Tests 开始 ==="
|
||||
|
||||
@@ -12,11 +12,23 @@ from packages.config.base import SharedSettings, get_cached_settings, reload_set
|
||||
def _reset_cache():
|
||||
"""每个测试前清空配置缓存，避免单例污染."""
|
||||
reload_settings_cache()
|
||||
# 保存关键环境变量（避免其他测试模块的全局污染）
|
||||
_saved_env = {}
|
||||
for key in ["JWT_SECRET_KEY", "DATABASE_URL", "USE_IN_MEMORY_DB", "APP_ENV"]:
|
||||
_saved_env[key] = os.environ.get(key)
|
||||
# 设置必要的环境变量，避免 JWT 校验失败
|
||||
os.environ["JWT_SECRET_KEY"] = "test-secret-key-for-unit-tests-only-12345"
|
||||
# 清除可能被其他模块污染的变量，确保默认值测试准确
|
||||
for key in ["DATABASE_URL", "APP_ENV"]:
|
||||
os.environ.pop(key, None)
|
||||
yield
|
||||
reload_settings_cache()
|
||||
os.environ.pop("JWT_SECRET_KEY", None)
|
||||
# 恢复所有保存的环境变量，避免污染其他测试模块
|
||||
for key, val in _saved_env.items():
|
||||
if val is None:
|
||||
os.environ.pop(key, None)
|
||||
else:
|
||||
os.environ[key] = val
|
||||
|
||||
|
||||
class TestSharedSettingsDefaults:
|
||||
|
||||
Reference in New Issue
Block a user