perf(worker): optimize worker image build with unified base image #1430

Merged
xiaoxia merged 1 commits from perf/worker-base-image into develop 2026-08-18 23:04:43 +08:00
4 changed files with 155 additions and 158 deletions
+64 -73
View File
@@ -633,58 +633,25 @@ jobs:
echo "Docker login failed ($i/3), retrying in 5s..."
sleep 5
done
- name: Pre-build worker base images (fallback if not exist)
- name: Pre-build worker base image (fallback if not exist)
if: matrix.service == 'worker'
id: prebuild
shell: sh
run: |
set -eu
REGISTRY="git.xiaoxiajianji.com/xiaoxia-saas"
BASE_BUILDER="${REGISTRY}/worker-base-builder:latest"
BASE_RUNTIME="${REGISTRY}/worker-base-runtime:latest"
REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
BASE_IMAGE="${REGISTRY}/saas-worker-base:latest"
# 尝试拉取基础镜像
echo "检查基础镜像..."
if docker pull "$BASE_BUILDER" 2>/dev/null && docker pull "$BASE_RUNTIME" 2>/dev/null; then
echo "基础镜像已存在,使用远程镜像"
echo "检查 Worker 基础镜像..."
if docker pull "$BASE_IMAGE" 2>/dev/null; then
echo "基础镜像已存在"
echo "fallback=false" >> $GITHUB_OUTPUT
else
echo "基础镜像不存在,本地构建(fallback模式)..."
# 尝试用buildx构建,失败则回退到普通docker buildDooD模式下buildx builder偶发崩溃)
BUILDER_NAME="ci-pr-builder-${GITHUB_RUN_ID:-local}"
BUILDX_AVAILABLE=true
if ! docker buildx create --use --name "$BUILDER_NAME" --driver docker-container > /dev/null 2>&1; then
BUILDX_AVAILABLE=false
fi
if [ "$BUILDX_AVAILABLE" = true ] && ! docker buildx inspect --bootstrap > /dev/null 2>&1; then
BUILDX_AVAILABLE=false
docker buildx rm "$BUILDER_NAME" > /dev/null 2>&1 || true
fi
build_base() {
local df="$1"
local tag="$2"
local name="$3"
if [ "$BUILDX_AVAILABLE" = true ]; then
echo "构建 $namebuildx..."
if docker buildx build --load -f "$df" -t "$tag" . > /dev/null 2>&1; then
echo "$name 构建成功"
return 0
fi
echo "buildx失败,回退到普通docker build"
BUILDX_AVAILABLE=false
docker buildx rm "$BUILDER_NAME" > /dev/null 2>&1 || true
fi
echo "构建 $namedocker build..."
docker build -f "$df" -t "$tag" .
}
build_base infra/docker/worker-base-builder.Dockerfile "$BASE_BUILDER" "worker-base-builder"
build_base infra/docker/worker-base-runtime.Dockerfile "$BASE_RUNTIME" "worker-base-runtime"
echo "⚠️ 基础镜像不存在,本地构建(fallback模式)..."
docker build -f infra/docker/worker-base.Dockerfile -t "$BASE_IMAGE" .
echo "fallback=true" >> $GITHUB_OUTPUT
echo "基础镜像本地构建完成"
echo "✅ Worker 基础镜像本地构建完成"
fi
- name: Build PR image (verify only, no push)
@@ -700,15 +667,15 @@ jobs:
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
fi
# Worker fallback模式:基础镜像本地已构建,用普通docker build绕过buildx
if [ "${{ matrix.service }}" = "worker" ] && [ "${{ steps.prebuild.outputs.fallback }}" = "true" ]; then
echo "Fallback模式:用普通docker build(基础镜像本地已构建)"
# Worker: 始终用普通docker build(基础镜像已预装全部依赖,无需buildx
if [ "${{ matrix.service }}" = "worker" ]; then
echo "Worker: 使用普通docker build"
BUILD_ARG_STR=""
for arg in $EXTRA_BUILD_ARGS; do
BUILD_ARG_STR="$BUILD_ARG_STR --build-arg $arg"
done
docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" $BUILD_ARG_STR .
echo "Fallback PR Build successful"
echo "PR Build successful (worker, no buildx)"
exit 0
fi
@@ -831,6 +798,7 @@ jobs:
fi
- name: Setup buildx builder
if: matrix.service != 'worker'
shell: sh
run: |
set -eu
@@ -843,41 +811,64 @@ jobs:
fi
docker buildx inspect --bootstrap
- name: Build and push ${{ matrix.service_display }} image (with retry)
- name: Pre-build worker base image (fallback if not exist)
if: matrix.service == 'worker'
shell: sh
run: |
set -eu
REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
BASE_IMAGE="${REGISTRY}/saas-worker-base:latest"
echo "检查 Worker 基础镜像..."
if docker pull "$BASE_IMAGE" 2>/dev/null; then
echo "✅ 基础镜像已存在"
else
echo "⚠️ 基础镜像不存在,本地构建(fallback..."
docker build -f infra/docker/worker-base.Dockerfile -t "$BASE_IMAGE" .
echo "✅ Worker 基础镜像本地构建完成"
fi
- name: Build and push ${{ matrix.service_display }} image
shell: sh
run: |
set -eu
REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:${GITHUB_SHA}"
CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:${GITHUB_REF_NAME}"
EXTRA_BUILD_ARGS="APP_VERSION=\"${GITHUB_SHA}\""
if [ "${{ matrix.service }}" = "web" ]; then
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
if [ "${{ matrix.service }}" = "worker" ]; then
# Worker: plain docker build(基础镜像已预装全部依赖,无需 buildx)
echo "=== Worker: plain docker build ==="
docker build -f ${{ matrix.dockerfile }} -t "${IMAGE_TAG}" --build-arg APP_VERSION="${GITHUB_SHA}" .
docker push "${IMAGE_TAG}"
echo "✅ Worker image pushed: ${IMAGE_TAG}"
else
# API/Web: buildx with registry cache
CACHE_REF="${REGISTRY}/${{ matrix.cache_name }}:${GITHUB_REF_NAME}"
EXTRA_BUILD_ARGS="APP_VERSION=\"${GITHUB_SHA}\""
if [ "${{ matrix.service }}" = "web" ]; then
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS NGINX_CONF=infra/docker/nginx-staging.conf"
fi
NO_CACHE_FLAG=""
for i in 1 2 3; do
echo "=== Docker build 尝试 $i/3 ==="
if bash scripts/ci/docker_build_push.sh $NO_CACHE_FLAG ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS; then
echo "✅ Docker build 成功"
break
fi
echo "❌ Docker build 失败(尝试 $i/3"
[ $i -eq 3 ] && exit 1
sleep 10
if [ $i -eq 2 ]; then
NO_CACHE_FLAG="--no-cache"
echo "下次重试将使用 --no-cache"
fi
done
echo "${{ matrix.service_display }} image pushed: ${IMAGE_TAG}"
fi
# Docker build 带重试:失败自动重试2次,第2次重试加--no-cache
NO_CACHE_FLAG=""
for i in 1 2 3; do
echo "=== Docker build 尝试 $i/3 ==="
if bash scripts/ci/docker_build_push.sh $NO_CACHE_FLAG ${{ matrix.dockerfile }} "${IMAGE_TAG}" "${CACHE_REF}" $EXTRA_BUILD_ARGS; then
echo "✅ Docker build 成功"
break
fi
echo "❌ Docker build 失败(尝试 $i/3"
[ $i -eq 3 ] && exit 1
sleep 10
# 第2次重试使用 --no-cache
if [ $i -eq 2 ]; then
NO_CACHE_FLAG="--no-cache"
echo "下次重试将使用 --no-cache"
fi
done
echo
echo "${{ matrix.service_display }} image pushed: ${IMAGE_TAG}"
- name: Cleanup buildx builder
if: always()
if: matrix.service != 'worker' && always()
shell: sh
run: |
docker buildx rm ci-builder-${GITHUB_RUN_ID}-${GITHUB_JOB}-${{ matrix.cache_name }} 2>/dev/null || true
+27 -36
View File
@@ -7,26 +7,16 @@ on:
- main
paths:
- 'requirements-base.txt'
- 'requirements.txt'
- 'requirements-worker.txt'
- 'infra/docker/worker-base-builder.Dockerfile'
- 'infra/docker/worker-base-runtime.Dockerfile'
workflow_dispatch: # 支持手动触发
- 'infra/docker/worker-base.Dockerfile'
workflow_dispatch:
jobs:
build-worker-base:
name: Build Worker Base Images
name: Build Worker Base Image
runs-on: runtime-builder
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- name: builder
dockerfile: infra/docker/worker-base-builder.Dockerfile
image_name: worker-base-builder
- name: runtime
dockerfile: infra/docker/worker-base-runtime.Dockerfile
image_name: worker-base-runtime
timeout-minutes: 45
steps:
- name: Checkout code
shell: sh
@@ -57,39 +47,40 @@ jobs:
sleep 5
done
- name: Build and push base image
- name: Build and push Worker base image
shell: sh
run: |
set -eu
REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
IMAGE_TAG="${REGISTRY}/${{ matrix.image_name }}:latest"
ACR_IMAGE="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/saas-worker-base:latest"
GITEA_IMAGE="git.xiaoxiajianji.com/xiaoxia-saas/saas-worker-base:latest"
echo "=== Building ${{ matrix.name }} base image ==="
echo "Dockerfile: ${{ matrix.dockerfile }}"
echo "Image: ${IMAGE_TAG}"
echo "=== Building Worker base image ==="
# 使用普通 docker build(单平台不需要 buildx
docker build \
-f "${{ matrix.dockerfile }}" \
-t "${IMAGE_TAG}" \
-f infra/docker/worker-base.Dockerfile \
-t "${ACR_IMAGE}" \
.
docker push "${IMAGE_TAG}"
# 同时推送到 Gitea Packages 作为备份
GITEA_IMAGE="git.xiaoxiajianji.com/xiaoxia-saas/${{ matrix.image_name }}:latest"
docker tag "${IMAGE_TAG}" "${GITEA_IMAGE}"
docker push "${GITEA_IMAGE}" || echo "Gitea Packages push failed (non-fatal)"
echo ""
echo "✅ ${{ matrix.name }} base image built and pushed"
echo "✅ Image built successfully"
- name: Cleanup local images
# 推送到 ACR
echo "=== Pushing to ACR ==="
docker push "${ACR_IMAGE}"
echo "✅ Pushed to ACR"
# 打标签并推送到 Gitea Packages 作为备份
echo "=== Pushing to Gitea Packages ==="
docker tag "${ACR_IMAGE}" "${GITEA_IMAGE}"
docker push "${GITEA_IMAGE}" || echo "⚠️ Gitea Packages push failed (non-fatal)"
echo "✅ Gitea backup push completed"
- name: Cleanup
if: always()
shell: sh
run: |
REGISTRY="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji"
docker rmi "${REGISTRY}/${{ matrix.image_name }}:latest" 2>/dev/null || true
docker rmi "git.xiaoxiajianji.com/xiaoxia-saas/${{ matrix.image_name }}:latest" 2>/dev/null || true
ACR_IMAGE="xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/saas-worker-base:latest"
docker rmi "${ACR_IMAGE}" 2>/dev/null || true
docker image prune -f 2>/dev/null || true
echo "Image cleanup done"
echo "Cleanup done"
+49
View File
@@ -0,0 +1,49 @@
# ============================================================
# Worker 统一基础镜像(预构建)
# 预装系统依赖 + Python 全部依赖 + CJK 字体
# 业务构建从此镜像开始,只需 COPY 业务代码,构建时间 < 5 分钟
# 当 requirements-*.txt 变更时重新构建
# ============================================================
FROM git.xiaoxiajianji.com/xiaoxia/base/python:3.12-slim
# 使用阿里云镜像加速
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
# 预装系统依赖(编译工具 + 运行时 + CJK 字体用于 ASS 字幕渲染)
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc \
g++ \
python3-dev \
binutils \
ffmpeg \
libglib2.0-0 \
fonts-noto-cjk \
&& fc-cache -fv \
&& rm -rf /var/lib/apt/lists/*
# 创建虚拟环境
RUN python -m venv /opt/venv
ENV PATH="/opt/venv/bin:$PATH"
WORKDIR /tmp
# 预装全部 Python 依赖(基础 + Worker 大包 + 业务依赖)
COPY requirements-base.txt requirements.txt requirements-worker.txt ./
RUN pip install --no-cache-dir \
-i https://mirrors.aliyun.com/pypi/simple/ \
--trusted-host mirrors.aliyun.com \
-r requirements-base.txt \
-r requirements-worker.txt \
-r requirements.txt
# 虚拟环境瘦身
RUN find /opt/venv -name "*.so" -type f -exec strip --strip-all {} \; 2>/dev/null || true
RUN find /opt/venv -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null; \
find /opt/venv -name "*.pyc" -delete 2>/dev/null || true
# 清理临时文件
RUN rm -f /tmp/requirements-base.txt /tmp/requirements.txt /tmp/requirements-worker.txt
ENV PYTHONPATH=/app:/app/packages
+15 -49
View File
@@ -1,51 +1,16 @@
# ============================================================
# Worker Dockerfile - 分层缓存优化版
# 优化:基础依赖 + Worker大包预构建基础镜像,业务构建仅叠加业务依赖
# 基础镜像worker-base-builder / worker-base-runtime
# Worker Dockerfile - 极简化
# 预构建统一基础镜像开始,仅叠加业务代码
# 基础镜像包含:系统依赖 + 全部 Python 依赖 + CJK 字体
# 构建时间目标:< 5 分钟
# ============================================================
# ==================== Builder 阶段 ====================
# 从预构建的builder基础镜像开始,已经包含:
# - 编译工具 (gcc/g++/python3-dev/binutils)
# - requirements-base.txt 全部依赖
# - requirements-worker.txt 全部依赖 (numpy/scipy/opencv)
# - 预strip的.so文件
FROM xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/worker-base-builder:latest AS builder
FROM xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/saas-worker-base:latest
ENV PATH="/opt/venv/bin:$PATH"
WORKDIR /tmp
# ---- 安装业务依赖(变化频繁,单独一层)----
COPY requirements.txt /tmp/requirements.txt
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com \
-r /tmp/requirements.txt \
&& rm /tmp/requirements.txt
# ---- 增量瘦身(清理新增业务依赖的冗余文件)----
RUN find /opt/venv -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null; \
find /opt/venv -name "*.pyc" -delete 2>/dev/null || true
# ==================== Runtime 阶段 ====================
# 从预构建的runtime基础镜像开始,已经包含:
# - ffmpeg
# - libglib2.0-0
FROM xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji/worker-base-runtime:latest AS runtime
# 构建参数:版本号
# 构建参数:版本号(CI 传入 commit hash
ARG APP_VERSION=dev
# 从 builder 复制 Python 虚拟环境
COPY --from=builder /opt/venv /opt/venv
# 设置 Python 环境变量
ENV PATH="/opt/venv/bin:$PATH"
ENV PYTHONPATH=/app:/app/packages
ENV PYTHONUNBUFFERED=1
ENV APP_VERSION=$APP_VERSION
# 创建非 root 用户(极少变化,放最前)
# 创建非 root 用户
RUN groupadd -r celery \
&& useradd -r -g celery -d /app -s /sbin/nologin celery \
&& mkdir -p /app/generated \
@@ -53,25 +18,26 @@ RUN groupadd -r celery \
WORKDIR /app
# 复制文件按变化频率从低到高排序,最大化层缓存命中
# 设置 Python 环境变量
ENV PATH="/opt/venv/bin:$PATH"
ENV PYTHONPATH=/app:/app/packages
ENV PYTHONUNBUFFERED=1
ENV APP_VERSION=$APP_VERSION
# 复制文件(按变化频率从低到高排序,最大化层缓存命中)
COPY alembic.ini /app/alembic.ini
COPY migrations/ /app/migrations/
COPY packages/ /app/packages/
COPY apps/api/app/config.py /app/apps/api/app/config.py
COPY apps/api/app/core/ /app/apps/api/app/core/
# 复制 Worker 启动脚本
# Worker 启动脚本
COPY infra/docker/entrypoint-worker.sh /usr/local/bin/entrypoint-worker.sh
RUN chmod +x /usr/local/bin/entrypoint-worker.sh
# 业务代码(变化最频繁,放最后)
COPY apps/worker/ /app/apps/worker/
# ---- Install CJK fonts for ASS subtitle rendering ----
RUN apt-get update && apt-get install -y --no-install-recommends fonts-noto-cjk && fc-cache -fv && rm -rf /var/lib/apt/lists/*
USER celery
# Worker 入口点
WORKDIR /app/apps/worker
CMD ["/usr/local/bin/entrypoint-worker.sh"]