From 548be6aced7a68ae235106165f63d42e22a2d910 Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Mon, 28 Sep 2026 02:23:45 +0800 Subject: [PATCH 1/2] fix(ci): worker compose deploy - remove curl fetch, add --env-file, use existing compose.yml --- scripts/ci_staging_deploy.sh | 50 ++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/scripts/ci_staging_deploy.sh b/scripts/ci_staging_deploy.sh index 82f791d2b..dc8831358 100755 --- a/scripts/ci_staging_deploy.sh +++ b/scripts/ci_staging_deploy.sh @@ -52,6 +52,8 @@ COOKIES_FILE="${COOKIES_FILE:-/var/lib/xiaoxia-saas-staging/configs/douyin_cooki INFRA_DOCKER_DIR="${INFRA_DOCKER_DIR:-/var/lib/xiaoxia-saas-staging/infra/docker}" COMPOSE_PROJECT="${COMPOSE_PROJECT:-xiaoxia-staging}" COMPOSE_ENV_VALUE="${COMPOSE_ENV_VALUE:-staging}" +# COMPOSE_SYNC: CI workflow 已通过 scp 把 infra/docker/compose.yml 上传到服务器时设为 0 跳过同步 +COMPOSE_SYNC="${COMPOSE_SYNC:-1}" SKIP_MIGRATION="${SKIP_MIGRATION:-false}" SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}" @@ -123,26 +125,24 @@ server { NGINX_EOF echo "✅ Nginx config written: $NGINX_CONF_FILE" -# ── 同步 infra/docker/compose.yml（单一事实来源）── -COMPOSE_FILE_URL="${COMPOSE_FILE_URL:-https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/raw/branch/${IMAGE_TAG}/infra/docker/compose.yml}" -echo "Syncing infra/docker/compose.yml from repo (tag=$IMAGE_TAG)..." -COMPOSE_SYNC_OK=0 -if command -v curl >/dev/null 2>&1; then - if retry_cmd 3 5 curl -sf --max-time 15 -o "$INFRA_DOCKER_DIR/compose.yml" "$COMPOSE_FILE_URL"; then - echo "✅ compose.yml written: $INFRA_DOCKER_DIR/compose.yml ($(wc -l < "$INFRA_DOCKER_DIR/compose.yml") lines)" - COMPOSE_SYNC_OK=1 - else - echo "WARN: failed to fetch compose.yml from repo, will use existing server copy" - fi -else - echo "WARN: curl not available, skipping compose.yml sync" -fi -if [ "$COMPOSE_SYNC_OK" -ne 1 ] && [ ! -f "$INFRA_DOCKER_DIR/compose.yml" ]; then - echo "ERROR: no compose.yml available (sync failed and no existing copy)" +# ── 确认 infra/docker/compose.yml 存在 ── +# CI workflow 在执行本脚本前已通过 scp 把 infra/docker/compose.yml 上传到 $INFRA_DOCKER_DIR +# （workflow 里做：scp infra/docker/compose.yml :$INFRA_DOCKER_DIR/compose.yml）。 +# 这里只做存在性检查 + nginx 软链；不再 curl 私有仓库（SSH 环境无 Gitea token）。 +COMPOSE_FILE_PATH="$INFRA_DOCKER_DIR/compose.yml" +if [ ! -f "$COMPOSE_FILE_PATH" ]; then + echo "ERROR: $COMPOSE_FILE_PATH 不存在。CI workflow 应先 scp infra/docker/compose.yml 到服务器" exit 1 fi +echo "✅ compose.yml ready: $COMPOSE_FILE_PATH ($(wc -l < "$COMPOSE_FILE_PATH") lines)" ln -sf "$NGINX_CONF_FILE" "$INFRA_DOCKER_DIR/nginx-${COMPOSE_ENV_VALUE}.conf" 2>/dev/null || true +# 封装 docker compose 调用：统一 --env-file（compose 默认只读取 project 目录下的 .env， +# 我们的 .env 在 $INFRA_DOCKER_DIR/../../.env，必须显式传入才能读到 GENERATED_FILES_HOST_DIR 等变量） +compose() { + (cd "$INFRA_DOCKER_DIR" && docker compose --env-file "$ENV_FILE" -p "$COMPOSE_PROJECT" "$@") +} + echo "===========================================" echo " Staging 部署 - $IMAGE_TAG" echo "===========================================" @@ -182,11 +182,8 @@ rollback() { docker rm -f xiaoxia-api-staging xiaoxia-web-staging 2>/dev/null || true if [ -n "$PREV_WORKER_IMAGE" ]; then echo "Rolling back Worker to: $PREV_WORKER_IMAGE (via compose)" - (cd "$INFRA_DOCKER_DIR" && \ - ENV="$COMPOSE_ENV_VALUE" \ - WORKER_IMAGE="$PREV_WORKER_IMAGE" \ - docker compose -p "$COMPOSE_PROJECT" up -d worker) 2>&1 || \ - echo "WARN: compose rollback for worker failed, fallback to docker run" + compose up -d --no-deps worker 2>&1 || echo "WARN: compose rollback failed, fallback to docker run" + # 镜像通过 env 注入：compose 默认读 WORKER_IMAGE（未设则用 :dev），这里用临时 env 覆盖 if ! docker inspect xiaoxia-worker-staging >/dev/null 2>&1; then echo "Fallback: docker run previous worker image" docker run -d \ @@ -518,14 +515,11 @@ docker run -d \ PID_API_START=$! # ── Worker: 通过 compose 启动（单一事实来源）── +# compose.yml 定义：三进程(beat+generation+transcode)、独立并发、BEAT_ENABLED、 +# healthcheck 匹配 'celery.*worker'（不把 beat 算活）、资源限制 4C/8G。 +# WORKER_IMAGE 通过环境变量覆盖镜像 tag（compose.yml 默认 :dev）。 echo "Starting worker via docker compose (from $INFRA_DOCKER_DIR)..." -( - cd "$INFRA_DOCKER_DIR" && \ - ENV="$COMPOSE_ENV_VALUE" \ - WORKER_IMAGE="$REGISTRY_WORKER" \ - APP_VERSION="$IMAGE_TAG" \ - docker compose -p "$COMPOSE_PROJECT" up -d worker -) & +WORKER_IMAGE="$REGISTRY_WORKER" APP_VERSION="$IMAGE_TAG" compose up -d --no-deps worker & PID_WORKER_START=$! # ── Web: 暂保留 docker run（TODO: 后续收敛到 compose）── From 6de6971e7ba74fd9f0ef1080d4d7655930c4ad27 Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Mon, 28 Sep 2026 02:23:49 +0800 Subject: [PATCH 2/2] fix(ci): scp compose.yml to staging host before running deploy script --- .gitea/workflows/ci-pipeline.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml index 20878d205..55bba5d05 100755 --- a/.gitea/workflows/ci-pipeline.yml +++ b/.gitea/workflows/ci-pipeline.yml @@ -1302,8 +1302,16 @@ jobs: "${staging_user}@${staging_host}:/var/lib/xiaoxia-saas-staging/configs/douyin_cookies.txt" echo "✅ Douyin cookies uploaded" + # 上传 infra/docker 配置到服务器（compose 单一事实来源） + echo "Uploading infra/docker configs to staging server..." + ssh -p "$staging_port" -i "$key_path" -o StrictHostKeyChecking=no "${staging_user}@${staging_host}" \ + "mkdir -p /var/lib/xiaoxia-saas-staging/infra/docker" + scp -P "$staging_port" -i "$key_path" -o StrictHostKeyChecking=no infra/docker/compose.yml \ + "${staging_user}@${staging_host}:/var/lib/xiaoxia-saas-staging/infra/docker/compose.yml" + echo "✅ infra/docker/compose.yml uploaded" + # 通过环境变量传递凭证，避免命令行引号转义问题 - cat scripts/ci_staging_deploy.sh | ssh -p "$staging_port" -i "$key_path" -o StrictHostKeyChecking=no "${staging_user}@${staging_host}" "IMAGE_TAG=${GITHUB_SHA} ACR_USERNAME=${ACR_USERNAME} ACR_PASSWORD=${ACR_PASSWORD} sh" + cat scripts/ci_staging_deploy.sh | ssh -p "$staging_port" -i "$key_path" -o StrictHostKeyChecking=no "${staging_user}@${staging_host}" "IMAGE_TAG=${GITHUB_SHA} ACR_USERNAME=${ACR_USERNAME} ACR_PASSWORD=${ACR_PASSWORD} COMPOSE_SYNC=0 sh" # 清理 CI runner 上的渲染文件 rm -f .env.rendered