fix(ci): worker deploy via compose to eliminate env/HC drift
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (pull_request) Successful in 1s
CI/CD Pipeline / Check push changed paths (pull_request) Has been skipped
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 1s
CI/CD Pipeline / Frontend Lint (pull_request) Has been skipped
CI/CD Pipeline / Frontend Unit Tests (pull_request) Has been skipped
CI/CD Pipeline / PR Build Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging API Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging Web Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Has been skipped
CI/CD Pipeline / Staging E2E Tests (pull_request) Has been skipped
CI/CD Pipeline / Staging API Integration Tests (pull_request) Has been skipped
CI/CD Pipeline / ACR Image Cleanup (pull_request) Has been skipped
CI/CD Pipeline / PR Build Worker Image (pull_request) Successful in 47s
CI/CD Pipeline / PR Build API Image (pull_request) Successful in 50s
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 1m18s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 2m36s
PR Automation / Auto Approve on CI Green (pull_request) Successful in 2m59s
CI/CD Pipeline / Validate - Style (pull_request) Successful in 5m15s
CI/CD Pipeline / Validate - Python (mypy + alembic) (pull_request) Successful in 5m38s
AI Code Review / AI Code Review (pull_request) Successful in 6m38s
CI/CD Pipeline / Validate - Security (pull_request) Successful in 6m38s
CI/CD Pipeline / Unit Tests (pull_request) Successful in 9m9s
CI/CD Pipeline / Build Production API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production Worker Image (pull_request) Has been skipped
CI/CD Pipeline / CI Gate (pull_request) Successful in 1s
CI/CD Pipeline / Canary Release to Production (pull_request) Has been skipped
CI/CD Pipeline / Production Browser E2E (pull_request) Has been skipped
CI/CD Pipeline / Deploy Production (pull_request) Has been skipped
PR Automation / Auto Merge on CI Green + Approved (pull_request) Successful in 6m47s
ACR Cleanup / ACR Image Cleanup (pull_request_target) Successful in 4m0s
Preview Cleanup / Cleanup Preview Environment (pull_request) Successful in 4m4s
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (pull_request) Successful in 1s
CI/CD Pipeline / Check push changed paths (pull_request) Has been skipped
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 1s
CI/CD Pipeline / Frontend Lint (pull_request) Has been skipped
CI/CD Pipeline / Frontend Unit Tests (pull_request) Has been skipped
CI/CD Pipeline / PR Build Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging API Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging Web Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Has been skipped
CI/CD Pipeline / Staging E2E Tests (pull_request) Has been skipped
CI/CD Pipeline / Staging API Integration Tests (pull_request) Has been skipped
CI/CD Pipeline / ACR Image Cleanup (pull_request) Has been skipped
CI/CD Pipeline / PR Build Worker Image (pull_request) Successful in 47s
CI/CD Pipeline / PR Build API Image (pull_request) Successful in 50s
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 1m18s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 2m36s
PR Automation / Auto Approve on CI Green (pull_request) Successful in 2m59s
CI/CD Pipeline / Validate - Style (pull_request) Successful in 5m15s
CI/CD Pipeline / Validate - Python (mypy + alembic) (pull_request) Successful in 5m38s
AI Code Review / AI Code Review (pull_request) Successful in 6m38s
CI/CD Pipeline / Validate - Security (pull_request) Successful in 6m38s
CI/CD Pipeline / Unit Tests (pull_request) Successful in 9m9s
CI/CD Pipeline / Build Production API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production Worker Image (pull_request) Has been skipped
CI/CD Pipeline / CI Gate (pull_request) Successful in 1s
CI/CD Pipeline / Canary Release to Production (pull_request) Has been skipped
CI/CD Pipeline / Production Browser E2E (pull_request) Has been skipped
CI/CD Pipeline / Deploy Production (pull_request) Has been skipped
PR Automation / Auto Merge on CI Green + Approved (pull_request) Successful in 6m47s
ACR Cleanup / ACR Image Cleanup (pull_request_target) Successful in 4m0s
Preview Cleanup / Cleanup Preview Environment (pull_request) Successful in 4m4s
This commit is contained in:
+144
-115
@@ -1,6 +1,7 @@
|
||||
#!/bin/sh
|
||||
# ===========================================
|
||||
# Staging 部署脚本(SSH 模式,并行优化版)
|
||||
# worker 已收敛到 infra/docker/compose.yml 单一事实来源;api/web 暂保留 docker run。
|
||||
# ===========================================
|
||||
set -eu
|
||||
|
||||
@@ -48,6 +49,9 @@ GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-staging/generated}"
|
||||
LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-staging/legacy-assets}"
|
||||
NGINX_CONF_FILE="${NGINX_CONF_FILE:-/var/lib/xiaoxia-saas-staging/nginx-staging.conf}"
|
||||
COOKIES_FILE="${COOKIES_FILE:-/var/lib/xiaoxia-saas-staging/configs/douyin_cookies.txt}"
|
||||
INFRA_DOCKER_DIR="${INFRA_DOCKER_DIR:-/var/lib/xiaoxia-saas-staging/infra/docker}"
|
||||
COMPOSE_PROJECT="${COMPOSE_PROJECT:-xiaoxia-staging}"
|
||||
COMPOSE_ENV_VALUE="${COMPOSE_ENV_VALUE:-staging}"
|
||||
|
||||
SKIP_MIGRATION="${SKIP_MIGRATION:-false}"
|
||||
SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}"
|
||||
@@ -57,8 +61,6 @@ if [ -z "$IMAGE_TAG" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# .env 文件由 CI 从模板 + Secrets 渲染后通过 SCP 上传到服务器
|
||||
# 如果文件不存在,说明 CI 渲染步骤失败或未执行
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "ERROR: $ENV_FILE 不存在。CI 应先在 render_env 步骤渲染并上传此文件"
|
||||
exit 1
|
||||
@@ -67,7 +69,7 @@ echo "✅ .env file found: $ENV_FILE ($(wc -l < "$ENV_FILE") lines)"
|
||||
mkdir -p "$GENERATED_DIR"
|
||||
mkdir -p "$LEGACY_ASSETS_DIR"
|
||||
mkdir -p "$(dirname "$COOKIES_FILE")"
|
||||
# 抖音 cookies 文件:CI workflow 已通过 scp 上传;如果不存在(非 CI 环境)则创建占位
|
||||
mkdir -p "$INFRA_DOCKER_DIR"
|
||||
if [ ! -f "$COOKIES_FILE" ] || [ "$(wc -c < "$COOKIES_FILE" 2>/dev/null || echo 0)" -lt 200 ]; then
|
||||
printf '# Netscape HTTP Cookie File\n# 抖音 cookies 占位(CI 应通过 scp 上传真实 cookies)\n' > "$COOKIES_FILE"
|
||||
echo "WARNING: Douyin cookies not found or too small at $COOKIES_FILE (extraction will 503)"
|
||||
@@ -76,7 +78,6 @@ else
|
||||
fi
|
||||
|
||||
# ── 写入 Staging Nginx 配置 ──
|
||||
# 运行时覆盖 nginx 配置,确保 upstream 指向正确的 staging 网络
|
||||
echo "Writing staging nginx config..."
|
||||
cat > "$NGINX_CONF_FILE" << 'NGINX_EOF'
|
||||
server {
|
||||
@@ -122,8 +123,28 @@ server {
|
||||
NGINX_EOF
|
||||
echo "✅ Nginx config written: $NGINX_CONF_FILE"
|
||||
|
||||
# ── 同步 infra/docker/compose.yml(单一事实来源)──
|
||||
COMPOSE_FILE_URL="${COMPOSE_FILE_URL:-https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/raw/branch/${IMAGE_TAG}/infra/docker/compose.yml}"
|
||||
echo "Syncing infra/docker/compose.yml from repo (tag=$IMAGE_TAG)..."
|
||||
COMPOSE_SYNC_OK=0
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
if retry_cmd 3 5 curl -sf --max-time 15 -o "$INFRA_DOCKER_DIR/compose.yml" "$COMPOSE_FILE_URL"; then
|
||||
echo "✅ compose.yml written: $INFRA_DOCKER_DIR/compose.yml ($(wc -l < "$INFRA_DOCKER_DIR/compose.yml") lines)"
|
||||
COMPOSE_SYNC_OK=1
|
||||
else
|
||||
echo "WARN: failed to fetch compose.yml from repo, will use existing server copy"
|
||||
fi
|
||||
else
|
||||
echo "WARN: curl not available, skipping compose.yml sync"
|
||||
fi
|
||||
if [ "$COMPOSE_SYNC_OK" -ne 1 ] && [ ! -f "$INFRA_DOCKER_DIR/compose.yml" ]; then
|
||||
echo "ERROR: no compose.yml available (sync failed and no existing copy)"
|
||||
exit 1
|
||||
fi
|
||||
ln -sf "$NGINX_CONF_FILE" "$INFRA_DOCKER_DIR/nginx-${COMPOSE_ENV_VALUE}.conf" 2>/dev/null || true
|
||||
|
||||
echo "==========================================="
|
||||
echo " Staging 部署 - $IMAGE_TAG (并行优化版)"
|
||||
echo " Staging 部署 - $IMAGE_TAG"
|
||||
echo "==========================================="
|
||||
|
||||
echo "Recording current image versions for rollback..."
|
||||
@@ -144,6 +165,7 @@ for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do
|
||||
fi
|
||||
done
|
||||
|
||||
# ── 回滚函数 ──
|
||||
rollback() {
|
||||
echo ""
|
||||
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||
@@ -157,9 +179,40 @@ rollback() {
|
||||
fi
|
||||
|
||||
echo "Stopping new containers..."
|
||||
docker rm -f xiaoxia-api-staging 2>/dev/null || true
|
||||
docker rm -f xiaoxia-worker-staging 2>/dev/null || true
|
||||
docker rm -f xiaoxia-web-staging 2>/dev/null || true
|
||||
docker rm -f xiaoxia-api-staging xiaoxia-web-staging 2>/dev/null || true
|
||||
if [ -n "$PREV_WORKER_IMAGE" ]; then
|
||||
echo "Rolling back Worker to: $PREV_WORKER_IMAGE (via compose)"
|
||||
(cd "$INFRA_DOCKER_DIR" && \
|
||||
ENV="$COMPOSE_ENV_VALUE" \
|
||||
WORKER_IMAGE="$PREV_WORKER_IMAGE" \
|
||||
docker compose -p "$COMPOSE_PROJECT" up -d worker) 2>&1 || \
|
||||
echo "WARN: compose rollback for worker failed, fallback to docker run"
|
||||
if ! docker inspect xiaoxia-worker-staging >/dev/null 2>&1; then
|
||||
echo "Fallback: docker run previous worker image"
|
||||
docker run -d \
|
||||
--name xiaoxia-worker-staging \
|
||||
--env-file "$ENV_FILE" \
|
||||
--network "xiaoxia-net-${COMPOSE_ENV_VALUE}" \
|
||||
-e APP_ENV="$COMPOSE_ENV_VALUE" \
|
||||
-e APP_VERSION="$(echo "$PREV_WORKER_IMAGE" | grep -oE '[^:]+$')" \
|
||||
-e WORKER_MAX_TASKS_PER_CHILD=100 \
|
||||
-e GENERATION_CONCURRENCY="${GENERATION_CONCURRENCY:-2}" \
|
||||
-e TRANSCODE_CONCURRENCY="${TRANSCODE_CONCURRENCY:-2}" \
|
||||
-e BEAT_ENABLED=1 \
|
||||
-e GENERATED_FILES_DIR=/app/generated \
|
||||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
||||
-e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \
|
||||
-v "$GENERATED_DIR:/app/generated" \
|
||||
--restart unless-stopped \
|
||||
--health-cmd "grep -q 'celery.*worker' /proc/[0-9]*/cmdline 2>/dev/null || exit 1" \
|
||||
--health-interval 30s \
|
||||
--health-timeout 10s \
|
||||
--health-retries 3 \
|
||||
--health-start-period 40s \
|
||||
--log-driver json-file --log-opt max-size=50m --log-opt max-file=3 \
|
||||
"$PREV_WORKER_IMAGE" || true
|
||||
fi
|
||||
fi
|
||||
|
||||
LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
|
||||
|
||||
@@ -168,10 +221,10 @@ rollback() {
|
||||
docker run -d \
|
||||
--name xiaoxia-api-staging \
|
||||
--env-file "$ENV_FILE" \
|
||||
--network xiaoxia-net-staging \
|
||||
--network "xiaoxia-net-${COMPOSE_ENV_VALUE}" \
|
||||
-p 127.0.0.1:8000:8000 \
|
||||
-e APP_ENV=staging \
|
||||
-e APP_VERSION="$(echo $PREV_API_IMAGE | grep -oE '[^:]+$')" \
|
||||
-e APP_ENV="$COMPOSE_ENV_VALUE" \
|
||||
-e APP_VERSION="$(echo "$PREV_API_IMAGE" | grep -oE '[^:]+$')" \
|
||||
-e GENERATED_FILES_DIR=/app/generated \
|
||||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
||||
-e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \
|
||||
@@ -183,31 +236,7 @@ rollback() {
|
||||
--health-retries 3 \
|
||||
--health-start-period 40s \
|
||||
$LOG_OPTS \
|
||||
"$PREV_API_IMAGE" &
|
||||
fi
|
||||
|
||||
if [ -n "$PREV_WORKER_IMAGE" ]; then
|
||||
echo "Rolling back Worker to: $PREV_WORKER_IMAGE"
|
||||
docker run -d \
|
||||
--name xiaoxia-worker-staging \
|
||||
--env-file "$ENV_FILE" \
|
||||
--network xiaoxia-net-staging \
|
||||
-e APP_ENV=staging \
|
||||
-e APP_VERSION="$(echo $PREV_WORKER_IMAGE | grep -oE '[^:]+$')" \
|
||||
-e WORKER_CONCURRENCY=1 \
|
||||
-e WORKER_MAX_TASKS_PER_CHILD=100 \
|
||||
-e GENERATED_FILES_DIR=/app/generated \
|
||||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
||||
-e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \
|
||||
-v "$GENERATED_DIR:/app/generated" \
|
||||
--restart unless-stopped \
|
||||
--health-cmd "grep -lq celery /proc/[0-9]*/cmdline 2>/dev/null || exit 1" \
|
||||
--health-interval 30s \
|
||||
--health-timeout 10s \
|
||||
--health-retries 3 \
|
||||
--health-start-period 30s \
|
||||
$LOG_OPTS \
|
||||
"$PREV_WORKER_IMAGE" &
|
||||
"$PREV_API_IMAGE" || true
|
||||
fi
|
||||
|
||||
if [ -n "$PREV_WEB_IMAGE" ]; then
|
||||
@@ -218,7 +247,7 @@ rollback() {
|
||||
fi
|
||||
docker run -d \
|
||||
--name xiaoxia-web-staging \
|
||||
--network xiaoxia-net-staging \
|
||||
--network "xiaoxia-net-${COMPOSE_ENV_VALUE}" \
|
||||
-p 127.0.0.1:3001:80 \
|
||||
--restart unless-stopped \
|
||||
$LEGACY_VOLUME \
|
||||
@@ -228,27 +257,25 @@ rollback() {
|
||||
--health-timeout 5s \
|
||||
--health-retries 3 \
|
||||
$LOG_OPTS \
|
||||
"$PREV_WEB_IMAGE" &
|
||||
"$PREV_WEB_IMAGE" || true
|
||||
fi
|
||||
|
||||
wait
|
||||
sleep 3
|
||||
|
||||
if [ -n "$PREV_API_IMAGE" ]; then
|
||||
echo "Waiting for rolled-back API to become healthy..."
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then
|
||||
echo "Rolled-back API is healthy!"
|
||||
break
|
||||
fi
|
||||
i=$((i + 1))
|
||||
echo " Waiting... ($i/40)"
|
||||
sleep 3
|
||||
done
|
||||
if [ "$i" -ge 40 ]; then
|
||||
echo "WARN: Rolled-back API did not become healthy within 120s"
|
||||
docker logs --tail 30 xiaoxia-api-staging
|
||||
echo "Waiting for rolled-back API to become healthy..."
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then
|
||||
echo "Rolled-back API is healthy!"
|
||||
break
|
||||
fi
|
||||
i=$((i + 1))
|
||||
echo " Waiting... ($i/40)"
|
||||
sleep 3
|
||||
done
|
||||
if [ "$i" -ge 40 ]; then
|
||||
echo "WARN: Rolled-back API did not become healthy within 120s"
|
||||
docker logs --tail 30 xiaoxia-api-staging 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo ""
|
||||
@@ -260,7 +287,7 @@ rollback() {
|
||||
echo "Previous Web: ${PREV_WEB_IMAGE:-none}"
|
||||
echo ""
|
||||
echo "部署失败,已自动回滚到上一版本"
|
||||
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging
|
||||
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -272,10 +299,6 @@ if [ -n "$REGISTRY_TOKEN" ]; then
|
||||
retry_docker_login
|
||||
fi
|
||||
|
||||
# ---- 并行 Pull 三个镜像 ----
|
||||
# 注意:这里必须使用 IMAGE_TAG(commit SHA)做确定性部署,不要改成 :dev。
|
||||
# :dev 是 floating tag,可能被并发构建覆盖,导致部署版本不可重现、回滚混乱。
|
||||
# Watchtower 可监听 :dev 做非关键路径的自动同步;正式部署/回滚一律锚定 SHA。
|
||||
REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}"
|
||||
REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}"
|
||||
REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}"
|
||||
@@ -304,7 +327,6 @@ for svc in api worker web; do
|
||||
elif grep -qE "Digest:|Status: Downloaded" "$PULL_LOG_DIR/$svc.log" 2>/dev/null; then
|
||||
echo " OK $svc"
|
||||
else
|
||||
# 检查docker pull返回值不直接,用镜像是否存在来判断
|
||||
img_var="REGISTRY_$(echo $svc | tr '[:lower:]' '[:upper:]')"
|
||||
img_val=$(eval echo "\$$img_var")
|
||||
if docker image inspect "$img_val" >/dev/null 2>&1; then
|
||||
@@ -327,7 +349,7 @@ fi
|
||||
|
||||
echo "All images pulled."
|
||||
|
||||
# ====== 镜像内容校验(CI 加固 - 防止静默部署损坏/过期镜像) ======
|
||||
# ====== 镜像内容校验 ======
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo " 镜像内容校验"
|
||||
@@ -336,7 +358,6 @@ echo "=========================================="
|
||||
VERIFY_FAILED=0
|
||||
DEPLOY_MANIFEST="${GENERATED_DIR}/deploy-manifest.json"
|
||||
|
||||
# 读取上次部署的 manifest(用于对比)
|
||||
PREV_MANIFEST=""
|
||||
if [ -f "$DEPLOY_MANIFEST" ]; then
|
||||
PREV_MANIFEST=$(cat "$DEPLOY_MANIFEST")
|
||||
@@ -348,14 +369,12 @@ for svc in api worker web; do
|
||||
img_var="REGISTRY_$(echo $svc | tr '[:lower:]' '[:upper:]')"
|
||||
img_val=$(eval echo "\$$img_var")
|
||||
|
||||
# 1. 检查镜像是否存在
|
||||
if ! docker image inspect "$img_val" >/dev/null 2>&1; then
|
||||
echo " ❌ $svc: 镜像不存在 ($img_val)"
|
||||
VERIFY_FAILED=$((VERIFY_FAILED + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
# 2. 检查 layers 有效性
|
||||
LAYER_COUNT=$(docker inspect --format='{{len .RootFS.Layers}}' "$img_val" 2>/dev/null || echo "0")
|
||||
if [ "$LAYER_COUNT" -eq 0 ]; then
|
||||
echo " ❌ $svc: 镜像无有效 layers ($img_val)"
|
||||
@@ -363,14 +382,12 @@ for svc in api worker web; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# 3. 获取 digest 和创建时间
|
||||
IMG_ID=$(docker inspect --format='{{.Id}}' "$img_val")
|
||||
IMG_CREATED=$(docker inspect --format='{{.Created}}' "$img_val")
|
||||
IMG_SIZE=$(docker inspect --format='{{.Size}}' "$img_val")
|
||||
echo " ✅ $svc: ${LAYER_COUNT} layers, size=${IMG_SIZE}, created=${IMG_CREATED}"
|
||||
echo " id: $IMG_ID"
|
||||
|
||||
# 4. 对比上次部署
|
||||
CHANGED="unchanged"
|
||||
if [ -n "$PREV_MANIFEST" ]; then
|
||||
PREV_ID=$(echo "$PREV_MANIFEST" | grep "\"${svc}_id\"" | sed 's/.*: *"\(.*\)".*/\1/' 2>/dev/null || echo "")
|
||||
@@ -398,7 +415,6 @@ if [ "$VERIFY_FAILED" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 写入新 manifest
|
||||
cat > "$DEPLOY_MANIFEST" <<MANIFEST_EOF
|
||||
{
|
||||
"deployed_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
@@ -444,14 +460,14 @@ for c in xiaoxia-postgres-staging xiaoxia-redis-staging; do
|
||||
fi
|
||||
done
|
||||
|
||||
docker network create xiaoxia-net-staging 2>/dev/null || true
|
||||
docker network create "xiaoxia-net-${COMPOSE_ENV_VALUE}" 2>/dev/null || true
|
||||
|
||||
if [ "$SKIP_MIGRATION" != "true" ]; then
|
||||
echo "Running database migrations..."
|
||||
docker run --rm \
|
||||
--env-file "$ENV_FILE" \
|
||||
--network xiaoxia-net-staging \
|
||||
-e APP_ENV=staging \
|
||||
--network "xiaoxia-net-${COMPOSE_ENV_VALUE}" \
|
||||
-e APP_ENV="$COMPOSE_ENV_VALUE" \
|
||||
"$REGISTRY_API" sh -c "cd /app && alembic upgrade head" || {
|
||||
echo "ERROR: Database migration failed"
|
||||
exit 1
|
||||
@@ -462,29 +478,28 @@ else
|
||||
fi
|
||||
|
||||
echo "Stopping old containers..."
|
||||
# 优雅关闭:先 stop(发 SIGTERM,等待),再 rm
|
||||
# Worker 需要更长时间(视频任务最长可能5分钟)
|
||||
docker stop -t 120 xiaoxia-worker-staging 2>/dev/null || true
|
||||
docker stop -t 30 xiaoxia-api-staging 2>/dev/null || true
|
||||
docker stop -t 10 xiaoxia-web-staging 2>/dev/null || true
|
||||
docker rm xiaoxia-worker-staging xiaoxia-api-staging xiaoxia-web-staging 2>/dev/null || true
|
||||
docker stop -t 30 xiaoxia-api-staging 2>/dev/null || true
|
||||
docker stop -t 120 xiaoxia-worker-staging 2>/dev/null || true
|
||||
docker rm -f xiaoxia-api-staging xiaoxia-web-staging 2>/dev/null || true
|
||||
docker rm -f xiaoxia-worker-staging 2>/dev/null || true
|
||||
|
||||
LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
|
||||
|
||||
# ---- 并行启动三个容器 ----
|
||||
echo "Starting all containers (parallel)..."
|
||||
echo "Starting all containers..."
|
||||
|
||||
LEGACY_VOLUME=""
|
||||
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
|
||||
LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
|
||||
fi
|
||||
|
||||
# ── API: 暂保留 docker run(TODO: 后续收敛到 compose)──
|
||||
docker run -d \
|
||||
--name xiaoxia-api-staging \
|
||||
--env-file "$ENV_FILE" \
|
||||
--network xiaoxia-net-staging \
|
||||
--network "xiaoxia-net-${COMPOSE_ENV_VALUE}" \
|
||||
-p 127.0.0.1:8000:8000 \
|
||||
-e APP_ENV=staging \
|
||||
-e APP_ENV="$COMPOSE_ENV_VALUE" \
|
||||
-e APP_VERSION="$IMAGE_TAG" \
|
||||
-e GENERATED_FILES_DIR=/app/generated \
|
||||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
||||
@@ -502,31 +517,21 @@ docker run -d \
|
||||
"$REGISTRY_API" &
|
||||
PID_API_START=$!
|
||||
|
||||
docker run -d \
|
||||
--name xiaoxia-worker-staging \
|
||||
--env-file "$ENV_FILE" \
|
||||
--network xiaoxia-net-staging \
|
||||
-e APP_ENV=staging \
|
||||
-e APP_VERSION="$IMAGE_TAG" \
|
||||
-e WORKER_CONCURRENCY=1 \
|
||||
-e WORKER_MAX_TASKS_PER_CHILD=100 \
|
||||
-e GENERATED_FILES_DIR=/app/generated \
|
||||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
||||
-e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \
|
||||
-v "$GENERATED_DIR:/app/generated" \
|
||||
--restart unless-stopped \
|
||||
--health-cmd "grep -lq celery /proc/[0-9]*/cmdline 2>/dev/null || exit 1" \
|
||||
--health-interval 30s \
|
||||
--health-timeout 10s \
|
||||
--health-retries 3 \
|
||||
--health-start-period 30s \
|
||||
$LOG_OPTS \
|
||||
"$REGISTRY_WORKER" &
|
||||
# ── Worker: 通过 compose 启动(单一事实来源)──
|
||||
echo "Starting worker via docker compose (from $INFRA_DOCKER_DIR)..."
|
||||
(
|
||||
cd "$INFRA_DOCKER_DIR" && \
|
||||
ENV="$COMPOSE_ENV_VALUE" \
|
||||
WORKER_IMAGE="$REGISTRY_WORKER" \
|
||||
APP_VERSION="$IMAGE_TAG" \
|
||||
docker compose -p "$COMPOSE_PROJECT" up -d worker
|
||||
) &
|
||||
PID_WORKER_START=$!
|
||||
|
||||
# ── Web: 暂保留 docker run(TODO: 后续收敛到 compose)──
|
||||
docker run -d \
|
||||
--name xiaoxia-web-staging \
|
||||
--network xiaoxia-net-staging \
|
||||
--network "xiaoxia-net-${COMPOSE_ENV_VALUE}" \
|
||||
-p 127.0.0.1:3001:80 \
|
||||
--restart unless-stopped \
|
||||
$LEGACY_VOLUME \
|
||||
@@ -563,9 +568,8 @@ if [ "$START_FAILED" -gt 0 ]; then
|
||||
rollback
|
||||
fi
|
||||
|
||||
# ---- 并行等待 API 和 Web 健康 ----
|
||||
echo ""
|
||||
echo "Waiting for API + Web health (parallel)..."
|
||||
echo "Waiting for all services health (parallel)..."
|
||||
|
||||
HEALTH_LOG_DIR="/tmp/staging-health-$$"
|
||||
mkdir -p "$HEALTH_LOG_DIR"
|
||||
@@ -600,36 +604,60 @@ PID_API_HEALTH=$!
|
||||
) > "$HEALTH_LOG_DIR/web.log" 2>&1 &
|
||||
PID_WEB_HEALTH=$!
|
||||
|
||||
(
|
||||
i=0
|
||||
while [ "$i" -lt 20 ]; do
|
||||
hc=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' xiaoxia-worker-staging 2>/dev/null || echo "missing")
|
||||
if [ "$hc" = "healthy" ]; then
|
||||
echo "Worker healthy after $((i * 3))s"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$hc" = "unhealthy" ]; then
|
||||
echo "Worker UNHEALTHY after $((i * 3))s"
|
||||
docker logs --tail 30 xiaoxia-worker-staging 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 3
|
||||
done
|
||||
echo "Worker health unknown after 60s (last: $hc)"
|
||||
exit 1
|
||||
) > "$HEALTH_LOG_DIR/worker.log" 2>&1 &
|
||||
PID_WORKER_HEALTH=$!
|
||||
|
||||
set +e
|
||||
wait $PID_API_HEALTH
|
||||
API_EXIT=$?
|
||||
wait $PID_WEB_HEALTH
|
||||
WEB_EXIT=$?
|
||||
wait $PID_WORKER_HEALTH
|
||||
WORKER_EXIT=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
echo "健康检查结果:"
|
||||
API_OK=0
|
||||
WEB_OK=0
|
||||
if [ "$API_EXIT" -eq 0 ]; then
|
||||
echo " OK API: $(cat "$HEALTH_LOG_DIR/api.log")"
|
||||
API_OK=1
|
||||
echo " OK API: $(cat "$HEALTH_LOG_DIR/api.log")"
|
||||
else
|
||||
echo " FAIL API: 120s未就绪"
|
||||
docker logs --tail 50 xiaoxia-api-staging
|
||||
echo " FAIL API: 120s未就绪"
|
||||
docker logs --tail 50 xiaoxia-api-staging 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [ "$WEB_EXIT" -eq 0 ]; then
|
||||
echo " OK Web: $(cat "$HEALTH_LOG_DIR/web.log")"
|
||||
WEB_OK=1
|
||||
echo " OK Web: $(cat "$HEALTH_LOG_DIR/web.log")"
|
||||
else
|
||||
echo " FAIL Web: 30s未就绪"
|
||||
docker logs --tail 30 xiaoxia-web-staging
|
||||
echo " FAIL Web: 30s未就绪"
|
||||
docker logs --tail 30 xiaoxia-web-staging 2>/dev/null || true
|
||||
fi
|
||||
if [ "$WORKER_EXIT" -eq 0 ]; then
|
||||
echo " OK Worker: $(cat "$HEALTH_LOG_DIR/worker.log")"
|
||||
else
|
||||
echo " FAIL Worker: $(cat "$HEALTH_LOG_DIR/worker.log")"
|
||||
docker logs --tail 50 xiaoxia-worker-staging 2>/dev/null || true
|
||||
fi
|
||||
|
||||
rm -rf "$HEALTH_LOG_DIR"
|
||||
|
||||
if [ "$API_OK" -eq 0 ] || [ "$WEB_OK" -eq 0 ]; then
|
||||
if [ "$API_EXIT" -ne 0 ] || [ "$WEB_EXIT" -ne 0 ] || [ "$WORKER_EXIT" -ne 0 ]; then
|
||||
echo ""
|
||||
echo "ERROR: 健康检查失败"
|
||||
rollback
|
||||
@@ -640,8 +668,9 @@ docker image prune -af --filter "until=168h" 2>/dev/null || true
|
||||
docker builder prune -af --filter "until=168h" 2>/dev/null || true
|
||||
|
||||
echo ""
|
||||
echo "=== Staging deployment complete (并行优化版) ==="
|
||||
echo "=== Staging deployment complete ==="
|
||||
echo "API: http://127.0.0.1:8000"
|
||||
echo "Web: http://127.0.0.1:3001"
|
||||
echo "Worker: managed by docker compose (project=$COMPOSE_PROJECT)"
|
||||
echo "Version: $IMAGE_TAG"
|
||||
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging
|
||||
|
||||
Reference in New Issue
Block a user