diff --git a/scripts/ci_staging_deploy.sh b/scripts/ci_staging_deploy.sh index 80a0337b4..71444545f 100755 --- a/scripts/ci_staging_deploy.sh +++ b/scripts/ci_staging_deploy.sh @@ -1,22 +1,9 @@ #!/bin/sh # =========================================== -# Staging 部署脚本(SSH 模式,支持自动回滚) +# Staging 部署脚本(SSH 模式,并行优化版) # =========================================== -# 通过 SSH 在 staging 服务器上执行 -# -# 环境变量: -# IMAGE_TAG - 镜像版本 tag(如 commit SHA 或分支名) -# REGISTRY_TOKEN - Registry 访问令牌 -# REGISTRY - Registry 地址(默认 xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji) -# REGISTRY_USER - Registry 用户名(默认 xiaoxia) -# ENV_FILE - 环境变量文件路径 -# GENERATED_DIR - 生成文件目录 -# SKIP_MIGRATION - 跳过数据库迁移(true/false,默认 false) -# SKIP_ROLLBACK - 失败时跳过自动回滚(true/false,默认 false) - set -eu -# ---- 重试工具函数 ---- retry_cmd() { local max_attempts=$1 local backoff=$2 @@ -73,10 +60,9 @@ mkdir -p "$GENERATED_DIR" mkdir -p "$LEGACY_ASSETS_DIR" echo "===========================================" -echo " Staging 部署 - $IMAGE_TAG" +echo " Staging 部署 - $IMAGE_TAG (并行优化版)" echo "===========================================" -# ---- 记录当前运行的镜像版本(用于回滚) ---- echo "Recording current image versions for rollback..." PREV_API_IMAGE="" PREV_WORKER_IMAGE="" @@ -95,7 +81,6 @@ for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do fi done -# ---- 回滚函数 ---- rollback() { echo "" echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" @@ -108,7 +93,6 @@ rollback() { exit 1 fi - # 停止当前(失败的)新容器 echo "Stopping new containers..." docker rm -f xiaoxia-api-staging 2>/dev/null || true docker rm -f xiaoxia-worker-staging 2>/dev/null || true @@ -116,7 +100,6 @@ rollback() { LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" - # 恢复 API if [ -n "$PREV_API_IMAGE" ]; then echo "Rolling back API to: $PREV_API_IMAGE" docker run -d \ @@ -137,12 +120,9 @@ rollback() { --health-retries 3 \ --health-start-period 40s \ $LOG_OPTS \ - "$PREV_API_IMAGE" - else - echo "No previous API image to roll back to" + "$PREV_API_IMAGE" & fi - # 恢复 Worker if [ -n "$PREV_WORKER_IMAGE" ]; then echo "Rolling back Worker to: $PREV_WORKER_IMAGE" docker run -d \ @@ -164,12 +144,9 @@ rollback() { --health-retries 3 \ --health-start-period 30s \ $LOG_OPTS \ - "$PREV_WORKER_IMAGE" - else - echo "No previous Worker image to roll back to" + "$PREV_WORKER_IMAGE" & fi - # 恢复 Web if [ -n "$PREV_WEB_IMAGE" ]; then echo "Rolling back Web to: $PREV_WEB_IMAGE" LEGACY_VOLUME="" @@ -187,12 +164,11 @@ rollback() { --health-timeout 5s \ --health-retries 3 \ $LOG_OPTS \ - "$PREV_WEB_IMAGE" - else - echo "No previous Web image to roll back to" + "$PREV_WEB_IMAGE" & fi - # 等待 API 回滚后恢复健康 + wait + if [ -n "$PREV_API_IMAGE" ]; then echo "Waiting for rolled-back API to become healthy..." i=0 @@ -224,7 +200,6 @@ rollback() { exit 1 } -# ---- 登录 Registry ---- if [ -n "$REGISTRY_TOKEN" ]; then echo "==========================================" echo " Login to Registry (with retries)" @@ -233,28 +208,64 @@ if [ -n "$REGISTRY_TOKEN" ]; then retry_docker_login fi -# ---- Pull 新版本镜像 ---- +# ---- 并行 Pull 三个镜像 ---- REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" echo "==========================================" -echo " Pull images (with retries)" +echo " Pull images (parallel, up to 3 retries each)" echo "==========================================" -retry_docker_pull "$REGISTRY_API" -retry_docker_pull "$REGISTRY_WORKER" -retry_docker_pull "$REGISTRY_WEB" +PULL_LOG_DIR="/tmp/staging-pull-$$" +mkdir -p "$PULL_LOG_DIR" + +retry_docker_pull "$REGISTRY_API" > "$PULL_LOG_DIR/api.log" 2>&1 & +PID_API=$! +retry_docker_pull "$REGISTRY_WORKER" > "$PULL_LOG_DIR/worker.log" 2>&1 & +PID_WORKER=$! +retry_docker_pull "$REGISTRY_WEB" > "$PULL_LOG_DIR/web.log" 2>&1 & +PID_WEB=$! + +wait $PID_API $PID_WORKER $PID_WEB + +echo "" +echo "Pull 结果:" +PULL_FAILED=0 +for svc in api worker web; do + if tail -1 "$PULL_LOG_DIR/$svc.log" 2>/dev/null | grep -qE "Status:|Downloaded|already exists|is up to date"; then + echo " OK $svc" + elif grep -qE "Digest:|Status: Downloaded" "$PULL_LOG_DIR/$svc.log" 2>/dev/null; then + echo " OK $svc" + else + # 检查docker pull返回值不直接,用镜像是否存在来判断 + img_var="REGISTRY_$(echo $svc | tr '[:lower:]' '[:upper:]')" + img_val=$(eval echo "\$$img_var") + if docker image inspect "$img_val" >/dev/null 2>&1; then + echo " OK $svc" + else + echo " FAIL $svc" + tail -5 "$PULL_LOG_DIR/$svc.log" 2>/dev/null || true + PULL_FAILED=$((PULL_FAILED + 1)) + fi + fi +done + +rm -rf "$PULL_LOG_DIR" + +if [ "$PULL_FAILED" -gt 0 ]; then + echo "" + echo "ERROR: $PULL_FAILED 个镜像 pull 失败" + exit 1 +fi echo "All images pulled." -# ---- 备份 legacy assets ---- echo "Backing up legacy assets from current web container..." if docker inspect xiaoxia-web-staging >/dev/null 2>&1; then _tmpdir="/tmp/legacy-assets-$$" rm -rf "$_tmpdir" mkdir -p "$_tmpdir" docker cp xiaoxia-web-staging:/usr/share/nginx/html/assets/. "$_tmpdir/" 2>/dev/null || true - # 只有目录非空才拷贝,避免覆盖有内容的 legacy assets if [ -d "$_tmpdir" ] && [ "$(ls -A "$_tmpdir" 2>/dev/null)" ]; then cp -an "$_tmpdir"/. "$LEGACY_ASSETS_DIR"/ 2>/dev/null || true echo "Legacy assets backed up: $(ls "$_tmpdir" | wc -l) files" @@ -264,13 +275,11 @@ else echo "No existing web container, skipping legacy assets backup" fi -# 清理 7 天前的 legacy assets if [ -d "$LEGACY_ASSETS_DIR" ]; then find "$LEGACY_ASSETS_DIR" -type f -mtime +7 -delete 2>/dev/null || true echo "Legacy assets cleanup done (retain 7 days)" fi -# ---- 检查基础设施容器 ---- echo "Checking infrastructure containers..." for c in xiaoxia-postgres-staging xiaoxia-redis-staging; do if ! docker inspect "$c" >/dev/null 2>&1; then @@ -284,10 +293,8 @@ for c in xiaoxia-postgres-staging xiaoxia-redis-staging; do fi done -# ---- 创建网络(不存在则创建) ---- docker network create xiaoxia-net-staging 2>/dev/null || true -# ---- 数据库迁移 ---- if [ "$SKIP_MIGRATION" != "true" ]; then echo "Running database migrations..." docker run --rm \ @@ -296,8 +303,6 @@ if [ "$SKIP_MIGRATION" != "true" ]; then -e APP_ENV=staging \ "$REGISTRY_API" sh -c "cd /app && alembic upgrade head" || { echo "ERROR: Database migration failed" - echo "Note: Migration failures are NOT automatically rolled back (data safety)" - echo "Please manually check and fix the migration, then redeploy" exit 1 } echo "Migrations completed." @@ -305,7 +310,6 @@ else echo "Skipping migrations (SKIP_MIGRATION=true)" fi -# ---- 停止旧容器 ---- echo "Stopping old containers..." docker rm -f xiaoxia-api-staging 2>/dev/null || true docker rm -f xiaoxia-worker-staging 2>/dev/null || true @@ -313,8 +317,14 @@ docker rm -f xiaoxia-web-staging 2>/dev/null || true LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" -# ---- 启动 API ---- -echo "Starting API container..." +# ---- 并行启动三个容器 ---- +echo "Starting all containers (parallel)..." + +LEGACY_VOLUME="" +if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then + LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" +fi + docker run -d \ --name xiaoxia-api-staging \ --env-file "$ENV_FILE" \ @@ -333,10 +343,9 @@ docker run -d \ --health-retries 3 \ --health-start-period 40s \ $LOG_OPTS \ - "$REGISTRY_API" || rollback + "$REGISTRY_API" & +PID_API_START=$! -# ---- 启动 Worker ---- -echo "Starting Worker container..." docker run -d \ --name xiaoxia-worker-staging \ --env-file "$ENV_FILE" \ @@ -356,18 +365,9 @@ docker run -d \ --health-retries 3 \ --health-start-period 30s \ $LOG_OPTS \ - "$REGISTRY_WORKER" || rollback + "$REGISTRY_WORKER" & +PID_WORKER_START=$! -# ---- 启动 Web ---- -LEGACY_VOLUME="" -if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then - LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" - echo "Web container: legacy assets mounted (fallback)" -else - echo "Web container: no legacy assets to mount" -fi - -echo "Starting Web container..." docker run -d \ --name xiaoxia-web-staging \ --network xiaoxia-net-staging \ @@ -379,53 +379,111 @@ docker run -d \ --health-timeout 5s \ --health-retries 3 \ $LOG_OPTS \ - "$REGISTRY_WEB" || rollback + "$REGISTRY_WEB" & +PID_WEB_START=$! -# ---- 等待 API 健康 ---- -echo "Waiting for API to become healthy..." -i=0 -while [ "$i" -lt 40 ]; do - if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then - echo "API is healthy!" - break +wait $PID_API_START $PID_WORKER_START $PID_WEB_START + +START_FAILED=0 +for c in xiaoxia-api-staging xiaoxia-worker-staging xiaoxia-web-staging; do + if ! docker inspect "$c" >/dev/null 2>&1; then + echo " FAIL $c: not created" + START_FAILED=$((START_FAILED + 1)) + else + state=$(docker inspect -f '{{.State.Status}}' "$c") + if [ "$state" = "running" ] || [ "$state" = "starting" ]; then + echo " OK $c: $state" + else + echo " FAIL $c: $state" + docker logs --tail 20 "$c" 2>/dev/null || true + START_FAILED=$((START_FAILED + 1)) + fi fi - i=$((i + 1)) - echo " Waiting... ($i/40)" - sleep 3 done -if [ "$i" -ge 40 ]; then - echo "ERROR: API did not become healthy within 120s" +if [ "$START_FAILED" -gt 0 ]; then + echo "ERROR: $START_FAILED 个容器启动失败" + rollback +fi + +# ---- 并行等待 API 和 Web 健康 ---- +echo "" +echo "Waiting for API + Web health (parallel)..." + +HEALTH_LOG_DIR="/tmp/staging-health-$$" +mkdir -p "$HEALTH_LOG_DIR" + +( + i=0 + while [ "$i" -lt 40 ]; do + if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then + echo "API healthy after $((i * 3))s" + exit 0 + fi + i=$((i + 1)) + sleep 3 + done + echo "API FAILED after 120s" + exit 1 +) > "$HEALTH_LOG_DIR/api.log" 2>&1 & +PID_API_HEALTH=$! + +( + i=0 + while [ "$i" -lt 15 ]; do + if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then + echo "Web healthy after $((i * 2))s" + exit 0 + fi + i=$((i + 1)) + sleep 2 + done + echo "Web FAILED after 30s" + exit 1 +) > "$HEALTH_LOG_DIR/web.log" 2>&1 & +PID_WEB_HEALTH=$! + +set +e +wait $PID_API_HEALTH +API_EXIT=$? +wait $PID_WEB_HEALTH +WEB_EXIT=$? +set -e + +echo "" +echo "健康检查结果:" +API_OK=0 +WEB_OK=0 +if [ "$API_EXIT" -eq 0 ]; then + echo " OK API: $(cat "$HEALTH_LOG_DIR/api.log")" + API_OK=1 +else + echo " FAIL API: 120s未就绪" docker logs --tail 50 xiaoxia-api-staging - rollback fi -# ---- 等待 Web 健康 ---- -echo "Waiting for Web to become healthy..." -i=0 -while [ "$i" -lt 15 ]; do - if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then - echo "Web is healthy!" - break - fi - i=$((i + 1)) - echo " Waiting... ($i/15)" - sleep 2 -done - -if [ "$i" -ge 15 ]; then - echo "ERROR: Web did not become healthy within 30s" +if [ "$WEB_EXIT" -eq 0 ]; then + echo " OK Web: $(cat "$HEALTH_LOG_DIR/web.log")" + WEB_OK=1 +else + echo " FAIL Web: 30s未就绪" docker logs --tail 30 xiaoxia-web-staging +fi + +rm -rf "$HEALTH_LOG_DIR" + +if [ "$API_OK" -eq 0 ] || [ "$WEB_OK" -eq 0 ]; then + echo "" + echo "ERROR: 健康检查失败" rollback fi -# ---- 清理旧镜像 ---- echo "Cleaning up old images..." docker image prune -af --filter "until=168h" 2>/dev/null || true docker builder prune -af --filter "until=168h" 2>/dev/null || true echo "" -echo "=== Staging deployment complete ===" +echo "=== Staging deployment complete (并行优化版) ===" echo "API: http://127.0.0.1:8000" echo "Web: http://127.0.0.1:3001" echo "Version: $IMAGE_TAG"