CI Bot
e4ac398397
根因：#1714 队列隔离后 entrypoint 启动两个 celery worker 子进程， PID 1 变为 bash。旧健康检查依赖 pgrep（容器内不存在）或 /proc/1/cmdline（PID 1 是 bash 不是 celery），永远返回失败。 此前靠 cron 脚本 fix-worker-healthcheck.sh 每 30 分钟擦屁股。 修复：统一改为扫描 /proc/[0-9]*/cmdline 查找 celery 进程， 覆盖所有部署/回滚/CI 脚本（10 处替换）。已移除 cron 补丁。
242 lines
7.6 KiB
Bash
Executable File
242 lines
7.6 KiB
Bash
Executable File
#!/bin/sh
|
||
set -eu
|
||
|
||
# ============================================
|
||
# Production 部署脚本 - Registry 方式
|
||
# 用法：IMAGE_TAG=<version> REGISTRY_TOKEN=<token> sh deploy-production-registry.sh
|
||
# ============================================
|
||
|
||
IMAGE_TAG="${IMAGE_TAG:-}"
|
||
REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}"
|
||
REGISTRY_USER="${REGISTRY_USER:-xiaoxia}"
|
||
REGISTRY_TOKEN="${REGISTRY_TOKEN:-}"
|
||
|
||
ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-production/.env}"
|
||
GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-production/generated}"
|
||
LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-production/legacy-assets}"
|
||
|
||
if [ -z "$IMAGE_TAG" ]; then
|
||
echo "ERROR: IMAGE_TAG is required"
|
||
exit 1
|
||
fi
|
||
|
||
test -f "$ENV_FILE"
|
||
mkdir -p "$GENERATED_DIR"
|
||
mkdir -p "$LEGACY_ASSETS_DIR"
|
||
|
||
# ---- 登录 Registry ----
|
||
if [ -n "$REGISTRY_TOKEN" ]; then
|
||
echo "Logging in to registry: $REGISTRY"
|
||
REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1)
|
||
printf %s "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin 2>/dev/null || {
|
||
echo "WARN: docker login failed, will try to pull anyway"
|
||
}
|
||
fi
|
||
|
||
# ---- Pull 三镜像 ----
|
||
REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}"
|
||
REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}"
|
||
REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}"
|
||
|
||
LOCAL_API="xiaoxia-saas-api:${IMAGE_TAG}"
|
||
LOCAL_WORKER="xiaoxia-saas-worker:${IMAGE_TAG}"
|
||
LOCAL_WEB="xiaoxia-saas-web:${IMAGE_TAG}"
|
||
|
||
echo "Pulling API image..."
|
||
docker pull "$REGISTRY_API"
|
||
echo "Pulling Worker image..."
|
||
docker pull "$REGISTRY_WORKER"
|
||
echo "Pulling Web image..."
|
||
docker pull "$REGISTRY_WEB"
|
||
|
||
# ---- Re-tag 成本地名 ----
|
||
docker tag "$REGISTRY_API" "$LOCAL_API"
|
||
docker tag "$REGISTRY_WORKER" "$LOCAL_WORKER"
|
||
docker tag "$REGISTRY_WEB" "$LOCAL_WEB"
|
||
echo "All images pulled and tagged."
|
||
|
||
# ---- 备份旧版 assets（部署期间缓存用户不 404） ----
|
||
echo "Backing up legacy assets from current web container..."
|
||
if docker inspect xiaoxia-web-production >/dev/null 2>&1; then
|
||
_tmpdir="/tmp/legacy-assets-$$"
|
||
rm -rf "$_tmpdir"
|
||
mkdir -p "$_tmpdir"
|
||
docker cp xiaoxia-web-production:/usr/share/nginx/html/assets/. "$_tmpdir/" 2>/dev/null || true
|
||
# 合并到 LEGACY_ASSETS_DIR（保留所有历史版本的 assets）
|
||
if [ -d "$_tmpdir" ] && [ "$(ls -A "$_tmpdir" 2>/dev/null)" ]; then
|
||
cp -an "$_tmpdir"/. "$LEGACY_ASSETS_DIR"/ 2>/dev/null || true
|
||
echo "Legacy assets backed up: $(ls "$_tmpdir" | wc -l) files"
|
||
fi
|
||
rm -rf "$_tmpdir"
|
||
else
|
||
echo "No existing web container, skipping legacy assets backup"
|
||
fi
|
||
|
||
# 清理超过 7 天的旧 assets 文件（避免无限增长）
|
||
if [ -d "$LEGACY_ASSETS_DIR" ]; then
|
||
find "$LEGACY_ASSETS_DIR" -type f -mtime +7 -delete 2>/dev/null || true
|
||
echo "Legacy assets cleanup done (retain 7 days)"
|
||
fi
|
||
|
||
# ---- 确保基础设施容器在运行 ----
|
||
echo "Checking infrastructure containers..."
|
||
for c in xiaoxia-postgres-production xiaoxia-redis-production; do
|
||
if ! docker inspect "$c" >/dev/null 2>&1; then
|
||
echo "ERROR: Required container not found: $c"
|
||
exit 1
|
||
fi
|
||
state=$(docker inspect -f '{{.State.Status}}' "$c")
|
||
if [ "$state" != "running" ]; then
|
||
echo "ERROR: Container not running: $c ($state)"
|
||
exit 1
|
||
fi
|
||
done
|
||
|
||
# ---- 确保生产网络存在 ----
|
||
docker network create xiaoxia-net-production 2>/dev/null || true
|
||
|
||
# ---- 执行数据库 Migration ----
|
||
echo "Running database migrations..."
|
||
docker run --rm \
|
||
--env-file "$ENV_FILE" \
|
||
--network xiaoxia-net-production \
|
||
-e APP_ENV=production \
|
||
"$LOCAL_API" sh -c "cd /app && alembic upgrade head"
|
||
echo "Migrations completed."
|
||
|
||
# ---- 停止旧容器 ----
|
||
echo "Stopping old containers..."
|
||
docker rm -f xiaoxia-api-production 2>/dev/null || true
|
||
docker rm -f xiaoxia-worker-production 2>/dev/null || true
|
||
docker rm -f xiaoxia-web-production 2>/dev/null || true
|
||
|
||
# ---- 日志配置（所有容器共用） ----
|
||
LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3"
|
||
|
||
# ---- 启动 API ----
|
||
echo "Starting API container..."
|
||
docker run -d \
|
||
--name xiaoxia-api-production \
|
||
--env-file "$ENV_FILE" \
|
||
--network xiaoxia-net-production \
|
||
-p 127.0.0.1:8001:8000 \
|
||
-e APP_ENV=production \
|
||
-e APP_VERSION="$IMAGE_TAG" \
|
||
-e GENERATED_FILES_DIR=/app/generated \
|
||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
||
-e PUBLIC_API_BASE_URL=https://api.xiaoxiajianji.com \
|
||
-v "$GENERATED_DIR:/app/generated" \
|
||
--restart unless-stopped \
|
||
--cpus 2 \
|
||
--memory 2g \
|
||
--health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \
|
||
--health-interval 30s \
|
||
--health-timeout 10s \
|
||
--health-retries 3 \
|
||
--health-start-period 40s \
|
||
$LOG_OPTS \
|
||
"$LOCAL_API"
|
||
|
||
# ---- 启动 Worker ----
|
||
echo "Starting Worker container..."
|
||
docker run -d \
|
||
--name xiaoxia-worker-production \
|
||
--env-file "$ENV_FILE" \
|
||
--network xiaoxia-net-production \
|
||
-e APP_ENV=production \
|
||
-e APP_VERSION="$IMAGE_TAG" \
|
||
-e WORKER_CONCURRENCY="${WORKER_CONCURRENCY:-4}" \
|
||
-e GENERATION_CONCURRENCY="${GENERATION_CONCURRENCY:-2}" \
|
||
-e WORKER_MAX_TASKS_PER_CHILD=100 \
|
||
-e GENERATED_FILES_DIR=/app/generated \
|
||
-e GENERATED_FILES_URL_PREFIX=/generated-files \
|
||
-e PUBLIC_API_BASE_URL=https://api.xiaoxiajianji.com \
|
||
-v "$GENERATED_DIR:/app/generated" \
|
||
--restart unless-stopped \
|
||
--cpus 2 \
|
||
--memory 2g \
|
||
--health-cmd "sh -c \"for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1\"" \
|
||
--health-interval 30s \
|
||
--health-timeout 10s \
|
||
--health-retries 3 \
|
||
--health-start-period 30s \
|
||
$LOG_OPTS \
|
||
"$LOCAL_WORKER"
|
||
|
||
# ---- 启动 Web ----
|
||
# Legacy assets 挂载到 /usr/share/nginx/html/assets-legacy/assets/
|
||
# nginx 配置中 assets location 有 fallback 逻辑
|
||
LEGACY_VOLUME=""
|
||
if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then
|
||
LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro"
|
||
echo "Web container: legacy assets mounted (fallback)"
|
||
else
|
||
echo "Web container: no legacy assets to mount"
|
||
fi
|
||
|
||
echo "Starting Web container..."
|
||
docker run -d \
|
||
--name xiaoxia-web-production \
|
||
--network xiaoxia-net-production \
|
||
-p 127.0.0.1:3002:80 \
|
||
--restart unless-stopped \
|
||
--cpus 0.5 \
|
||
--memory 512m \
|
||
$LEGACY_VOLUME \
|
||
--health-cmd "wget --spider -q http://127.0.0.1:80" \
|
||
--health-interval 30s \
|
||
--health-timeout 5s \
|
||
--health-retries 3 \
|
||
$LOG_OPTS \
|
||
"$LOCAL_WEB"
|
||
|
||
# ---- 等待 API 健康 ----
|
||
echo "Waiting for API to become healthy..."
|
||
i=0
|
||
while [ "$i" -lt 40 ]; do
|
||
if curl -sf --max-time 5 http://127.0.0.1:8001/health >/dev/null 2>&1; then
|
||
echo "API is healthy!"
|
||
break
|
||
fi
|
||
i=$((i + 1))
|
||
echo " Waiting... ($i/40)"
|
||
sleep 3
|
||
done
|
||
|
||
if [ "$i" -ge 40 ]; then
|
||
echo "ERROR: API did not become healthy within 120s"
|
||
docker logs --tail 50 xiaoxia-api-production
|
||
exit 1
|
||
fi
|
||
|
||
# ---- 等待 Web 健康 ----
|
||
echo "Waiting for Web to become healthy..."
|
||
i=0
|
||
while [ "$i" -lt 15 ]; do
|
||
if curl -sf --max-time 5 http://127.0.0.1:3002/ >/dev/null 2>&1; then
|
||
echo "Web is healthy!"
|
||
break
|
||
fi
|
||
i=$((i + 1))
|
||
echo " Waiting... ($i/15)"
|
||
sleep 2
|
||
done
|
||
|
||
if [ "$i" -ge 15 ]; then
|
||
echo "ERROR: Web did not become healthy within 30s"
|
||
docker logs --tail 30 xiaoxia-web-production
|
||
exit 1
|
||
fi
|
||
|
||
# ---- 清理旧镜像 ----
|
||
echo "Cleaning up old images..."
|
||
docker image prune -af --filter "until=168h" 2>/dev/null || true
|
||
docker builder prune -af --filter "until=168h" 2>/dev/null || true
|
||
|
||
echo ""
|
||
echo "=== Production deployment complete ==="
|
||
echo "API: http://127.0.0.1:8001"
|
||
echo "Web: http://127.0.0.1:3002"
|
||
echo "Version: $IMAGE_TAG"
|
||
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep production
|