From bcb1663de4358cdee6b9614dbd703f4d52b00fe7 Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Fri, 3 Jul 2026 18:12:36 +0800 Subject: [PATCH] =?UTF-8?q?ci:=20=E5=90=AF=E7=94=A8BuildKit=E5=88=86?= =?UTF-8?q?=E5=B8=83=E5=BC=8F=E7=BC=93=E5=AD=98+Gitea=20Registry,=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E6=9E=84=E5=BB=BA=E9=80=9F=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 拆分 requirements.txt: 稳定大包放 requirements-base.txt（numpy/scipy/opencv/Pillow/sqlalchemy 等），业务依赖留 requirements.txt - 优化 Dockerfile: 分两层安装依赖，基础层缓存命中率高 - CI 改用 docker buildx build，缓存存 Gitea Container Registry - 启用 Gitea Packages/Container Registry（git.xiaoxiajianji.com） - 镜像仍推内网 Registry（172.30.18.198:5000），缓存独立存 Gitea Registry --- .gitea/workflows/deploy.yml | 26 ++++++++++++++++--- infra/docker/api.Dockerfile | 17 +++++++----- infra/docker/worker.Dockerfile | 15 +++++++---- requirements-base.txt | 39 ++++++++++++++++++++++++++++ requirements.txt | 47 ++++++---------------------------- 5 files changed, 89 insertions(+), 55 deletions(-) mode change 100644 => 100755 infra/docker/api.Dockerfile mode change 100644 => 100755 infra/docker/worker.Dockerfile create mode 100755 requirements-base.txt mode change 100644 => 100755 requirements.txt diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index b12bb3548..172667a8e 100755 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -64,19 +64,37 @@ jobs: - name: Build and push staging API/Worker images shell: sh + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} run: | set -eu REGISTRY="172.30.18.198:5000" - docker build --pull=false \ + CACHE_REGISTRY="git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas" + CACHE_BRANCH="${GITHUB_REF_NAME//\//-}" + + # 登录 Gitea Registry（用于构建缓存） + echo "${GITHUB_TOKEN}" | docker login git.xiaoxiajianji.com -u xiaoxia --password-stdin + + # 确保 buildx builder 可用 + docker buildx use default || docker buildx create --use --name default-builder 2>/dev/null || true + + # 构建 API（带分布式缓存，缓存存 Gitea Registry，镜像推内网 Registry） + docker buildx build \ + --cache-from "type=registry,ref=${CACHE_REGISTRY}/api-cache:${CACHE_BRANCH}" \ + --cache-to "type=registry,ref=${CACHE_REGISTRY}/api-cache:${CACHE_BRANCH},mode=max" \ -f infra/docker/api.Dockerfile \ -t "${REGISTRY}/xiaoxia-saas-api:dev" \ + --push \ . - docker build --pull=false \ + + # 构建 Worker（带分布式缓存） + docker buildx build \ + --cache-from "type=registry,ref=${CACHE_REGISTRY}/worker-cache:${CACHE_BRANCH}" \ + --cache-to "type=registry,ref=${CACHE_REGISTRY}/worker-cache:${CACHE_BRANCH},mode=max" \ -f infra/docker/worker.Dockerfile \ -t "${REGISTRY}/xiaoxia-saas-worker:dev" \ + --push \ . - docker push "${REGISTRY}/xiaoxia-saas-api:dev" - docker push "${REGISTRY}/xiaoxia-saas-worker:dev" - name: Package staging release artifact shell: sh diff --git a/infra/docker/api.Dockerfile b/infra/docker/api.Dockerfile old mode 100644 new mode 100755 index bb249bb53..06b338a38 --- a/infra/docker/api.Dockerfile +++ b/infra/docker/api.Dockerfile @@ -1,6 +1,6 @@ # ============================================================ # API Dockerfile - 专门用于 FastAPI 应用 -# 优化：仅包含 API 所需的依赖 +# 优化：依赖分层缓存 + 多阶段构建基础层 # ============================================================ # 基础镜像：Python 3.12 @@ -18,14 +18,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # 设置工作目录 WORKDIR /app -# 复制 requirements.txt（排除 worker 专用依赖） -# API 需要 psycopg2/sqlalchemy 用于数据库连接 -# 注意：opencv、scipy 等是 worker 专用依赖，不在 API 中安装 +# ---- 依赖分层：基础依赖（变化少，缓存命中率高）---- +COPY requirements-base.txt /tmp/requirements-base.txt + +RUN python -m venv /opt/venv \ + && /opt/venv/bin/pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements-base.txt \ + && rm /tmp/requirements-base.txt + +# ---- 依赖分层：业务依赖（变化频繁）---- COPY requirements.txt /tmp/requirements.txt -# 创建虚拟环境并安装依赖 -RUN python -m venv /opt/venv \ - && /opt/venv/bin/pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements.txt \ +RUN /opt/venv/bin/pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements.txt \ && rm /tmp/requirements.txt # 复制应用代码 diff --git a/infra/docker/worker.Dockerfile b/infra/docker/worker.Dockerfile old mode 100644 new mode 100755 index 560fcf030..6eaa9e63b --- a/infra/docker/worker.Dockerfile +++ b/infra/docker/worker.Dockerfile @@ -1,6 +1,6 @@ # ============================================================ # Worker Dockerfile - 专门用于 Celery Worker -# 优化：仅包含 worker 任务所需的依赖，减小镜像体积 +# 优化：依赖分层缓存，基础大包和业务依赖分开 # ============================================================ # 基础镜像：Python 3.12 + ffmpeg @@ -21,12 +21,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # 设置工作目录 WORKDIR /app -# 安装 Python 依赖（优化顺序以利用 Docker 缓存） -# 先安装无变化的依赖 +# ---- 依赖分层：基础依赖（变化少，缓存命中率高）---- +COPY requirements-base.txt /tmp/requirements-base.txt + +RUN pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements-base.txt \ + && rm /tmp/requirements-base.txt + +# ---- 依赖分层：业务依赖（变化频繁）---- COPY requirements.txt /tmp/requirements.txt -# 安装 Python 包 -RUN pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements.txt +RUN pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements.txt \ + && rm /tmp/requirements.txt # 复制应用代码 COPY apps/worker/ /app/apps/worker/ diff --git a/requirements-base.txt b/requirements-base.txt new file mode 100755 index 000000000..64b22cf16 --- /dev/null +++ b/requirements-base.txt @@ -0,0 +1,39 @@ +# 小虾 SaaS - 基础依赖（稳定、变化少，用于 Docker 缓存分层） +# 修改此文件会触发完整重新构建，请谨慎修改 + +# 数据库（基础层） +psycopg2-binary==2.9.10 +psycopg[binary]>=3.2.2 +sqlalchemy==2.0.35 +alembic==1.13.3 + +# Web 框架核心 +fastapi==0.115.0 +uvicorn[standard]==0.32.0 +pydantic==2.9.0 + +# 认证核心 +pyjwt==2.9.0 +bcrypt==4.2.0 + +# Redis +redis==5.2.0 + +# 任务队列 +celery==5.4.0 + +# 视频处理（大包，变化极少） +numpy>=1.24.0 +scipy>=1.10.0 +opencv-python-headless>=4.8.0 +Pillow==10.4.0 +ffmpeg-python==0.2.0 + +# 对象存储 +oss2==2.18.4 + +# HTTP 客户端 +httpx==0.27.2 + +# Prometheus monitoring +prometheus-client==0.21.1 diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 index ae0d9cdc0..cda4fe188 --- a/requirements.txt +++ b/requirements.txt @@ -1,49 +1,18 @@ -# 小虾 SaaS - Python 依赖 +# 小虾 SaaS - 业务依赖（变化频繁，放在 requirements-base 之上） +# 基础依赖（大包）请放在 requirements-base.txt 中 -# Web 框架 -fastapi==0.115.0 -uvicorn[standard]==0.32.0 -pydantic==2.9.0 -email-validator==2.2.0 +# 验证 & 设置 pydantic-settings==2.6.0 - -# 数据库 -psycopg2-binary==2.9.10 -psycopg[binary]>=3.2.2 -sqlalchemy==2.0.35 -alembic==1.13.3 - -# 认证 -pyjwt==2.9.0 -bcrypt==4.2.0 +email-validator==2.2.0 python-multipart==0.0.32 -# Redis -redis==5.2.0 - -# HTTP 客户端 -httpx==0.27.2 +# 邮件 +aiosmtplib==3.0.2 # 测试 pytest==8.3.3 pytest-asyncio==0.24.0 pytest-cov==6.0.0 -# 视频处理 -ffmpeg-python==0.2.0 -Pillow==10.4.0 - -# 对象存储（阿里云 OSS） -oss2==2.18.4 - -# 任务队列 -celery==5.4.0 - -# 数据分析（用于视频质量评估） -numpy>=1.24.0 -scipy>=1.10.0 - -opencv-python-headless>=4.8.0 - -# Prometheus monitoring -prometheus-client==0.21.1 +# 工具 +python-dotenv==1.0.1