diff --git a/scripts/ci_staging_deploy.sh b/scripts/ci_staging_deploy.sh index 7a704e822..82f791d2b 100755 --- a/scripts/ci_staging_deploy.sh +++ b/scripts/ci_staging_deploy.sh @@ -1,6 +1,7 @@ #!/bin/sh # =========================================== # Staging 部署脚本（SSH 模式，并行优化版） +# worker 已收敛到 infra/docker/compose.yml 单一事实来源；api/web 暂保留 docker run。 # =========================================== set -eu @@ -48,6 +49,9 @@ GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-staging/generated}" LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-staging/legacy-assets}" NGINX_CONF_FILE="${NGINX_CONF_FILE:-/var/lib/xiaoxia-saas-staging/nginx-staging.conf}" COOKIES_FILE="${COOKIES_FILE:-/var/lib/xiaoxia-saas-staging/configs/douyin_cookies.txt}" +INFRA_DOCKER_DIR="${INFRA_DOCKER_DIR:-/var/lib/xiaoxia-saas-staging/infra/docker}" +COMPOSE_PROJECT="${COMPOSE_PROJECT:-xiaoxia-staging}" +COMPOSE_ENV_VALUE="${COMPOSE_ENV_VALUE:-staging}" SKIP_MIGRATION="${SKIP_MIGRATION:-false}" SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}" @@ -57,8 +61,6 @@ if [ -z "$IMAGE_TAG" ]; then exit 1 fi -# .env 文件由 CI 从模板 + Secrets 渲染后通过 SCP 上传到服务器 -# 如果文件不存在，说明 CI 渲染步骤失败或未执行 if [ ! -f "$ENV_FILE" ]; then echo "ERROR: $ENV_FILE 不存在。CI 应先在 render_env 步骤渲染并上传此文件" exit 1 @@ -67,7 +69,7 @@ echo "✅ .env file found: $ENV_FILE ($(wc -l < "$ENV_FILE") lines)" mkdir -p "$GENERATED_DIR" mkdir -p "$LEGACY_ASSETS_DIR" mkdir -p "$(dirname "$COOKIES_FILE")" -# 抖音 cookies 文件：CI workflow 已通过 scp 上传；如果不存在（非 CI 环境）则创建占位 +mkdir -p "$INFRA_DOCKER_DIR" if [ ! -f "$COOKIES_FILE" ] || [ "$(wc -c < "$COOKIES_FILE" 2>/dev/null || echo 0)" -lt 200 ]; then printf '# Netscape HTTP Cookie File\n# 抖音 cookies 占位（CI 应通过 scp 上传真实 cookies）\n' > "$COOKIES_FILE" echo "WARNING: Douyin cookies not found or too small at $COOKIES_FILE (extraction will 503)" @@ -76,7 +78,6 @@ else fi # ── 写入 Staging Nginx 配置 ── -# 运行时覆盖 nginx 配置，确保 upstream 指向正确的 staging 网络 echo "Writing staging nginx config..." cat > "$NGINX_CONF_FILE" << 'NGINX_EOF' server { @@ -122,8 +123,28 @@ server { NGINX_EOF echo "✅ Nginx config written: $NGINX_CONF_FILE" +# ── 同步 infra/docker/compose.yml（单一事实来源）── +COMPOSE_FILE_URL="${COMPOSE_FILE_URL:-https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/raw/branch/${IMAGE_TAG}/infra/docker/compose.yml}" +echo "Syncing infra/docker/compose.yml from repo (tag=$IMAGE_TAG)..." +COMPOSE_SYNC_OK=0 +if command -v curl >/dev/null 2>&1; then + if retry_cmd 3 5 curl -sf --max-time 15 -o "$INFRA_DOCKER_DIR/compose.yml" "$COMPOSE_FILE_URL"; then + echo "✅ compose.yml written: $INFRA_DOCKER_DIR/compose.yml ($(wc -l < "$INFRA_DOCKER_DIR/compose.yml") lines)" + COMPOSE_SYNC_OK=1 + else + echo "WARN: failed to fetch compose.yml from repo, will use existing server copy" + fi +else + echo "WARN: curl not available, skipping compose.yml sync" +fi +if [ "$COMPOSE_SYNC_OK" -ne 1 ] && [ ! -f "$INFRA_DOCKER_DIR/compose.yml" ]; then + echo "ERROR: no compose.yml available (sync failed and no existing copy)" + exit 1 +fi +ln -sf "$NGINX_CONF_FILE" "$INFRA_DOCKER_DIR/nginx-${COMPOSE_ENV_VALUE}.conf" 2>/dev/null || true + echo "===========================================" -echo " Staging 部署 - $IMAGE_TAG (并行优化版)" +echo " Staging 部署 - $IMAGE_TAG" echo "===========================================" echo "Recording current image versions for rollback..." @@ -144,6 +165,7 @@ for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do fi done +# ── 回滚函数 ── rollback() { echo "" echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" @@ -157,9 +179,40 @@ rollback() { fi echo "Stopping new containers..." - docker rm -f xiaoxia-api-staging 2>/dev/null || true - docker rm -f xiaoxia-worker-staging 2>/dev/null || true - docker rm -f xiaoxia-web-staging 2>/dev/null || true + docker rm -f xiaoxia-api-staging xiaoxia-web-staging 2>/dev/null || true + if [ -n "$PREV_WORKER_IMAGE" ]; then + echo "Rolling back Worker to: $PREV_WORKER_IMAGE (via compose)" + (cd "$INFRA_DOCKER_DIR" && \ + ENV="$COMPOSE_ENV_VALUE" \ + WORKER_IMAGE="$PREV_WORKER_IMAGE" \ + docker compose -p "$COMPOSE_PROJECT" up -d worker) 2>&1 || \ + echo "WARN: compose rollback for worker failed, fallback to docker run" + if ! docker inspect xiaoxia-worker-staging >/dev/null 2>&1; then + echo "Fallback: docker run previous worker image" + docker run -d \ + --name xiaoxia-worker-staging \ + --env-file "$ENV_FILE" \ + --network "xiaoxia-net-${COMPOSE_ENV_VALUE}" \ + -e APP_ENV="$COMPOSE_ENV_VALUE" \ + -e APP_VERSION="$(echo "$PREV_WORKER_IMAGE" | grep -oE '[^:]+$')" \ + -e WORKER_MAX_TASKS_PER_CHILD=100 \ + -e GENERATION_CONCURRENCY="${GENERATION_CONCURRENCY:-2}" \ + -e TRANSCODE_CONCURRENCY="${TRANSCODE_CONCURRENCY:-2}" \ + -e BEAT_ENABLED=1 \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "grep -q 'celery.*worker' /proc/[0-9]*/cmdline 2>/dev/null || exit 1" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 40s \ + --log-driver json-file --log-opt max-size=50m --log-opt max-file=3 \ + "$PREV_WORKER_IMAGE" || true + fi + fi LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" @@ -168,10 +221,10 @@ rollback() { docker run -d \ --name xiaoxia-api-staging \ --env-file "$ENV_FILE" \ - --network xiaoxia-net-staging \ + --network "xiaoxia-net-${COMPOSE_ENV_VALUE}" \ -p 127.0.0.1:8000:8000 \ - -e APP_ENV=staging \ - -e APP_VERSION="$(echo $PREV_API_IMAGE | grep -oE '[^:]+$')" \ + -e APP_ENV="$COMPOSE_ENV_VALUE" \ + -e APP_VERSION="$(echo "$PREV_API_IMAGE" | grep -oE '[^:]+$')" \ -e GENERATED_FILES_DIR=/app/generated \ -e GENERATED_FILES_URL_PREFIX=/generated-files \ -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ @@ -183,31 +236,7 @@ rollback() { --health-retries 3 \ --health-start-period 40s \ $LOG_OPTS \ - "$PREV_API_IMAGE" & - fi - - if [ -n "$PREV_WORKER_IMAGE" ]; then - echo "Rolling back Worker to: $PREV_WORKER_IMAGE" - docker run -d \ - --name xiaoxia-worker-staging \ - --env-file "$ENV_FILE" \ - --network xiaoxia-net-staging \ - -e APP_ENV=staging \ - -e APP_VERSION="$(echo $PREV_WORKER_IMAGE | grep -oE '[^:]+$')" \ - -e WORKER_CONCURRENCY=1 \ - -e WORKER_MAX_TASKS_PER_CHILD=100 \ - -e GENERATED_FILES_DIR=/app/generated \ - -e GENERATED_FILES_URL_PREFIX=/generated-files \ - -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ - -v "$GENERATED_DIR:/app/generated" \ - --restart unless-stopped \ - --health-cmd "grep -lq celery /proc/[0-9]*/cmdline 2>/dev/null || exit 1" \ - --health-interval 30s \ - --health-timeout 10s \ - --health-retries 3 \ - --health-start-period 30s \ - $LOG_OPTS \ - "$PREV_WORKER_IMAGE" & + "$PREV_API_IMAGE" || true fi if [ -n "$PREV_WEB_IMAGE" ]; then @@ -218,7 +247,7 @@ rollback() { fi docker run -d \ --name xiaoxia-web-staging \ - --network xiaoxia-net-staging \ + --network "xiaoxia-net-${COMPOSE_ENV_VALUE}" \ -p 127.0.0.1:3001:80 \ --restart unless-stopped \ $LEGACY_VOLUME \ @@ -228,27 +257,25 @@ rollback() { --health-timeout 5s \ --health-retries 3 \ $LOG_OPTS \ - "$PREV_WEB_IMAGE" & + "$PREV_WEB_IMAGE" || true fi - wait + sleep 3 - if [ -n "$PREV_API_IMAGE" ]; then - echo "Waiting for rolled-back API to become healthy..." - i=0 - while [ "$i" -lt 40 ]; do - if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then - echo "Rolled-back API is healthy!" - break - fi - i=$((i + 1)) - echo " Waiting... ($i/40)" - sleep 3 - done - if [ "$i" -ge 40 ]; then - echo "WARN: Rolled-back API did not become healthy within 120s" - docker logs --tail 30 xiaoxia-api-staging + echo "Waiting for rolled-back API to become healthy..." + i=0 + while [ "$i" -lt 40 ]; do + if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then + echo "Rolled-back API is healthy!" + break fi + i=$((i + 1)) + echo " Waiting... ($i/40)" + sleep 3 + done + if [ "$i" -ge 40 ]; then + echo "WARN: Rolled-back API did not become healthy within 120s" + docker logs --tail 30 xiaoxia-api-staging 2>/dev/null || true fi echo "" @@ -260,7 +287,7 @@ rollback() { echo "Previous Web: ${PREV_WEB_IMAGE:-none}" echo "" echo "部署失败，已自动回滚到上一版本" - docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging + docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging || true exit 1 } @@ -272,10 +299,6 @@ if [ -n "$REGISTRY_TOKEN" ]; then retry_docker_login fi -# ---- 并行 Pull 三个镜像 ---- -# 注意：这里必须使用 IMAGE_TAG（commit SHA）做确定性部署，不要改成 :dev。 -# :dev 是 floating tag，可能被并发构建覆盖，导致部署版本不可重现、回滚混乱。 -# Watchtower 可监听 :dev 做非关键路径的自动同步；正式部署/回滚一律锚定 SHA。 REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" @@ -304,7 +327,6 @@ for svc in api worker web; do elif grep -qE "Digest:|Status: Downloaded" "$PULL_LOG_DIR/$svc.log" 2>/dev/null; then echo " OK $svc" else - # 检查docker pull返回值不直接，用镜像是否存在来判断 img_var="REGISTRY_$(echo $svc | tr '[:lower:]' '[:upper:]')" img_val=$(eval echo "\$$img_var") if docker image inspect "$img_val" >/dev/null 2>&1; then @@ -327,7 +349,7 @@ fi echo "All images pulled." -# ====== 镜像内容校验（CI 加固 - 防止静默部署损坏/过期镜像） ====== +# ====== 镜像内容校验 ====== echo "" echo "==========================================" echo " 镜像内容校验" @@ -336,7 +358,6 @@ echo "==========================================" VERIFY_FAILED=0 DEPLOY_MANIFEST="${GENERATED_DIR}/deploy-manifest.json" -# 读取上次部署的 manifest（用于对比） PREV_MANIFEST="" if [ -f "$DEPLOY_MANIFEST" ]; then PREV_MANIFEST=$(cat "$DEPLOY_MANIFEST") @@ -348,14 +369,12 @@ for svc in api worker web; do img_var="REGISTRY_$(echo $svc | tr '[:lower:]' '[:upper:]')" img_val=$(eval echo "\$$img_var") - # 1. 检查镜像是否存在 if ! docker image inspect "$img_val" >/dev/null 2>&1; then echo " ❌ $svc: 镜像不存在 ($img_val)" VERIFY_FAILED=$((VERIFY_FAILED + 1)) continue fi - # 2. 检查 layers 有效性 LAYER_COUNT=$(docker inspect --format='{{len .RootFS.Layers}}' "$img_val" 2>/dev/null || echo "0") if [ "$LAYER_COUNT" -eq 0 ]; then echo " ❌ $svc: 镜像无有效 layers ($img_val)" @@ -363,14 +382,12 @@ for svc in api worker web; do continue fi - # 3. 获取 digest 和创建时间 IMG_ID=$(docker inspect --format='{{.Id}}' "$img_val") IMG_CREATED=$(docker inspect --format='{{.Created}}' "$img_val") IMG_SIZE=$(docker inspect --format='{{.Size}}' "$img_val") echo " ✅ $svc: ${LAYER_COUNT} layers, size=${IMG_SIZE}, created=${IMG_CREATED}" echo " id: $IMG_ID" - # 4. 对比上次部署 CHANGED="unchanged" if [ -n "$PREV_MANIFEST" ]; then PREV_ID=$(echo "$PREV_MANIFEST" | grep "\"${svc}_id\"" | sed 's/.*: *"\(.*\)".*/\1/' 2>/dev/null || echo "") @@ -398,7 +415,6 @@ if [ "$VERIFY_FAILED" -gt 0 ]; then exit 1 fi -# 写入新 manifest cat > "$DEPLOY_MANIFEST" </dev/null || true +docker network create "xiaoxia-net-${COMPOSE_ENV_VALUE}" 2>/dev/null || true if [ "$SKIP_MIGRATION" != "true" ]; then echo "Running database migrations..." docker run --rm \ --env-file "$ENV_FILE" \ - --network xiaoxia-net-staging \ - -e APP_ENV=staging \ + --network "xiaoxia-net-${COMPOSE_ENV_VALUE}" \ + -e APP_ENV="$COMPOSE_ENV_VALUE" \ "$REGISTRY_API" sh -c "cd /app && alembic upgrade head" || { echo "ERROR: Database migration failed" exit 1 @@ -462,29 +478,28 @@ else fi echo "Stopping old containers..." -# 优雅关闭：先 stop（发 SIGTERM，等待），再 rm -# Worker 需要更长时间（视频任务最长可能5分钟） -docker stop -t 120 xiaoxia-worker-staging 2>/dev/null || true -docker stop -t 30 xiaoxia-api-staging 2>/dev/null || true docker stop -t 10 xiaoxia-web-staging 2>/dev/null || true -docker rm xiaoxia-worker-staging xiaoxia-api-staging xiaoxia-web-staging 2>/dev/null || true +docker stop -t 30 xiaoxia-api-staging 2>/dev/null || true +docker stop -t 120 xiaoxia-worker-staging 2>/dev/null || true +docker rm -f xiaoxia-api-staging xiaoxia-web-staging 2>/dev/null || true +docker rm -f xiaoxia-worker-staging 2>/dev/null || true LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" -# ---- 并行启动三个容器 ---- -echo "Starting all containers (parallel)..." +echo "Starting all containers..." LEGACY_VOLUME="" if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" fi +# ── API: 暂保留 docker run（TODO: 后续收敛到 compose）── docker run -d \ --name xiaoxia-api-staging \ --env-file "$ENV_FILE" \ - --network xiaoxia-net-staging \ + --network "xiaoxia-net-${COMPOSE_ENV_VALUE}" \ -p 127.0.0.1:8000:8000 \ - -e APP_ENV=staging \ + -e APP_ENV="$COMPOSE_ENV_VALUE" \ -e APP_VERSION="$IMAGE_TAG" \ -e GENERATED_FILES_DIR=/app/generated \ -e GENERATED_FILES_URL_PREFIX=/generated-files \ @@ -502,31 +517,21 @@ docker run -d \ "$REGISTRY_API" & PID_API_START=$! -docker run -d \ - --name xiaoxia-worker-staging \ - --env-file "$ENV_FILE" \ - --network xiaoxia-net-staging \ - -e APP_ENV=staging \ - -e APP_VERSION="$IMAGE_TAG" \ - -e WORKER_CONCURRENCY=1 \ - -e WORKER_MAX_TASKS_PER_CHILD=100 \ - -e GENERATED_FILES_DIR=/app/generated \ - -e GENERATED_FILES_URL_PREFIX=/generated-files \ - -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ - -v "$GENERATED_DIR:/app/generated" \ - --restart unless-stopped \ - --health-cmd "grep -lq celery /proc/[0-9]*/cmdline 2>/dev/null || exit 1" \ - --health-interval 30s \ - --health-timeout 10s \ - --health-retries 3 \ - --health-start-period 30s \ - $LOG_OPTS \ - "$REGISTRY_WORKER" & +# ── Worker: 通过 compose 启动（单一事实来源）── +echo "Starting worker via docker compose (from $INFRA_DOCKER_DIR)..." +( + cd "$INFRA_DOCKER_DIR" && \ + ENV="$COMPOSE_ENV_VALUE" \ + WORKER_IMAGE="$REGISTRY_WORKER" \ + APP_VERSION="$IMAGE_TAG" \ + docker compose -p "$COMPOSE_PROJECT" up -d worker +) & PID_WORKER_START=$! +# ── Web: 暂保留 docker run（TODO: 后续收敛到 compose）── docker run -d \ --name xiaoxia-web-staging \ - --network xiaoxia-net-staging \ + --network "xiaoxia-net-${COMPOSE_ENV_VALUE}" \ -p 127.0.0.1:3001:80 \ --restart unless-stopped \ $LEGACY_VOLUME \ @@ -563,9 +568,8 @@ if [ "$START_FAILED" -gt 0 ]; then rollback fi -# ---- 并行等待 API 和 Web 健康 ---- echo "" -echo "Waiting for API + Web health (parallel)..." +echo "Waiting for all services health (parallel)..." HEALTH_LOG_DIR="/tmp/staging-health-$$" mkdir -p "$HEALTH_LOG_DIR" @@ -600,36 +604,60 @@ PID_API_HEALTH=$! ) > "$HEALTH_LOG_DIR/web.log" 2>&1 & PID_WEB_HEALTH=$! +( + i=0 + while [ "$i" -lt 20 ]; do + hc=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' xiaoxia-worker-staging 2>/dev/null || echo "missing") + if [ "$hc" = "healthy" ]; then + echo "Worker healthy after $((i * 3))s" + exit 0 + fi + if [ "$hc" = "unhealthy" ]; then + echo "Worker UNHEALTHY after $((i * 3))s" + docker logs --tail 30 xiaoxia-worker-staging 2>/dev/null || true + exit 1 + fi + i=$((i + 1)) + sleep 3 + done + echo "Worker health unknown after 60s (last: $hc)" + exit 1 +) > "$HEALTH_LOG_DIR/worker.log" 2>&1 & +PID_WORKER_HEALTH=$! + set +e wait $PID_API_HEALTH API_EXIT=$? wait $PID_WEB_HEALTH WEB_EXIT=$? +wait $PID_WORKER_HEALTH +WORKER_EXIT=$? set -e echo "" echo "健康检查结果:" -API_OK=0 -WEB_OK=0 if [ "$API_EXIT" -eq 0 ]; then - echo " OK API: $(cat "$HEALTH_LOG_DIR/api.log")" - API_OK=1 + echo " OK API: $(cat "$HEALTH_LOG_DIR/api.log")" else - echo " FAIL API: 120s未就绪" - docker logs --tail 50 xiaoxia-api-staging + echo " FAIL API: 120s未就绪" + docker logs --tail 50 xiaoxia-api-staging 2>/dev/null || true fi - if [ "$WEB_EXIT" -eq 0 ]; then - echo " OK Web: $(cat "$HEALTH_LOG_DIR/web.log")" - WEB_OK=1 + echo " OK Web: $(cat "$HEALTH_LOG_DIR/web.log")" else - echo " FAIL Web: 30s未就绪" - docker logs --tail 30 xiaoxia-web-staging + echo " FAIL Web: 30s未就绪" + docker logs --tail 30 xiaoxia-web-staging 2>/dev/null || true +fi +if [ "$WORKER_EXIT" -eq 0 ]; then + echo " OK Worker: $(cat "$HEALTH_LOG_DIR/worker.log")" +else + echo " FAIL Worker: $(cat "$HEALTH_LOG_DIR/worker.log")" + docker logs --tail 50 xiaoxia-worker-staging 2>/dev/null || true fi rm -rf "$HEALTH_LOG_DIR" -if [ "$API_OK" -eq 0 ] || [ "$WEB_OK" -eq 0 ]; then +if [ "$API_EXIT" -ne 0 ] || [ "$WEB_EXIT" -ne 0 ] || [ "$WORKER_EXIT" -ne 0 ]; then echo "" echo "ERROR: 健康检查失败" rollback @@ -640,8 +668,9 @@ docker image prune -af --filter "until=168h" 2>/dev/null || true docker builder prune -af --filter "until=168h" 2>/dev/null || true echo "" -echo "=== Staging deployment complete (并行优化版) ===" +echo "=== Staging deployment complete ===" echo "API: http://127.0.0.1:8000" echo "Web: http://127.0.0.1:3001" +echo "Worker: managed by docker compose (project=$COMPOSE_PROJECT)" echo "Version: $IMAGE_TAG" docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging