From f2cea1831e1370f1ebe15800c839033e30a064db Mon Sep 17 00:00:00 2001 From: XiaoXia Bot Date: Wed, 15 Jul 2026 00:52:18 +0800 Subject: [PATCH] =?UTF-8?q?feat(ci):=20Staging=E9=83=A8=E7=BD=B2=E4=BB=8EW?= =?UTF-8?q?atchtower=E6=94=B9=E4=B8=BASSH=20push=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/ci-cd.yml | 26 ++- scripts/ci_staging_deploy.sh | 250 +++++++++++++++++++++ scripts/ci_staging_healthcheck.sh | 353 +++++++++++++++++++++--------- 3 files changed, 520 insertions(+), 109 deletions(-) create mode 100755 scripts/ci_staging_deploy.sh diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 66f060c0f..06fec5f07 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -620,7 +620,7 @@ jobs: CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} run: 'set +e - NOTIFY_MODE=start JOB_NAME="Deploy Staging (Watchtower auto-deploy)" python3 scripts/ci_notify.py + NOTIFY_MODE=start JOB_NAME="Deploy Staging" python3 scripts/ci_notify.py ' - name: Docker login to Registry @@ -634,14 +634,30 @@ jobs: echo "Docker login successful" ' - - name: Tag and push :staging images (Watchtower auto-update) + - name: Install SSH client + if: success() shell: sh - run: "set -eu\nREGISTRY=\"git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas\"\n\nfor svc in api worker web; do\n echo \"Pulling ${REGISTRY}/xiaoxia-saas-${svc}:${GITHUB_SHA} ...\"\n docker pull \"${REGISTRY}/xiaoxia-saas-${svc}:${GITHUB_SHA}\"\n docker tag \"${REGISTRY}/xiaoxia-saas-${svc}:${GITHUB_SHA}\" \"${REGISTRY}/xiaoxia-saas-${svc}:staging\"\n docker push \"${REGISTRY}/xiaoxia-saas-${svc}:staging\"\n echo \"$svc :staging tagged and pushed\"\ndone\necho \"All :staging images pushed. Watchtower will auto-deploy within 60s.\"\n" + run: "set -eu\napt-get update -qq \u0026\u0026 apt-get install -y -qq openssh-client \u003e/dev/null 2\u003e\u00261\necho \"openssh-client installed\"\n" + + - name: Deploy staging over SSH (Registry pull) + if: success() + shell: sh + env: + STAGING_SSH_HOST: ${{ secrets.STAGING_SSH_HOST }} + STAGING_SSH_USER: ${{ secrets.STAGING_SSH_USER }} + STAGING_SSH_PORT: ${{ secrets.STAGING_SSH_PORT }} + STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }} + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: "set -eux\nstaging_host=\"${STAGING_SSH_HOST:-47.98.113.167}\"\nstaging_user=\"${STAGING_SSH_USER:-root}\"\nstaging_port=\"${STAGING_SSH_PORT:-22222}\"\necho \"Host: $staging_host\"\necho \"Port: $staging_port\"\n\nmkdir -p ~/.ssh\n\n# 查找可用的SSH密钥\nkey_path=\"\"\nif [ -f /root/.ssh/xiaoxia_runtime_builder ]; then\n key_path=\"/root/.ssh/xiaoxia_runtime_builder\"\n echo \"Using key: $key_path (builder key)\"\nelif [ -f \"$HOME/.ssh/xiaoxia_runtime_builder\" ]; then\n key_path=\"$HOME/.ssh/xiaoxia_runtime_builder\"\n echo \"Using key: $key_path (home key)\"\nelif [ -n \"${STAGING_SSH_KEY:-\" ]; then\n key_path=\"$HOME/.ssh/id_ed25519\"\n printf \'%s\\n\' \"$STAGING_SSH_KEY\" \u003e \"$key_path\"\n chmod 600 \"$key_path\"\n echo \"Using key from STAGING_SSH_KEY secret\"\nelse\n echo \"ERROR: No SSH key available\"\n ls -la ~/.ssh/ 2\u003e/dev/null || true\n ls -la /root/.ssh/ 2\u003e/dev/null || true\n exit 1\nfi\n\nssh-keyscan -p \"$staging_port\" -H \"$staging_host\" \u003e\u003e ~/.ssh/known_hosts 2\u003e/dev/null\necho \"SSH keyscan done\"\n\n# 测试SSH连接\nssh -p \"$staging_port\" -i \"$key_path\" -o StrictHostKeyChecking=no \"${staging_user}@${staging_host}\" \"echo SSH_CONNECTION_OK \u0026\u0026 hostname\"\necho \"SSH connection verified\"\n\n# 读取部署脚本并通过SSH执行\nbash scripts/ci_staging_deploy.sh | ssh -p \"$staging_port\" -i \"$key_path\" -o StrictHostKeyChecking=no \"${staging_user}@${staging_host}\" \"IMAGE_TAG=\'${GITHUB_SHA}\' REGISTRY_TOKEN=\'${REGISTRY_TOKEN}\' sh\"\n" - name: Staging health check + auto rollback if: success() shell: sh env: CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + STAGING_SSH_HOST: ${{ secrets.STAGING_SSH_HOST }} + STAGING_SSH_USER: ${{ secrets.STAGING_SSH_USER }} + STAGING_SSH_PORT: ${{ secrets.STAGING_SSH_PORT }} + STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }} REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} run: 'set -eu @@ -669,7 +685,7 @@ jobs: CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} run: 'set +e - NOTIFY_MODE=success JOB_NAME="Deploy Staging (Watchtower auto-deploy)" python3 scripts/ci_notify.py + NOTIFY_MODE=success JOB_NAME="Deploy Staging" python3 scripts/ci_notify.py ' - name: Notify on failure @@ -680,7 +696,7 @@ jobs: CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} run: 'set +e - NOTIFY_MODE=failure JOB_NAME="Deploy Staging (Watchtower auto-deploy)" python3 scripts/ci_notify.py + NOTIFY_MODE=failure JOB_NAME="Deploy Staging" python3 scripts/ci_notify.py ' staging-e2e: diff --git a/scripts/ci_staging_deploy.sh b/scripts/ci_staging_deploy.sh new file mode 100755 index 000000000..32088fad5 --- /dev/null +++ b/scripts/ci_staging_deploy.sh @@ -0,0 +1,250 @@ +#!/bin/sh +# =========================================== +# Staging 部署脚本（SSH 模式） +# =========================================== +# 通过 SSH 在 staging 服务器上执行 +# +# 环境变量: +# IMAGE_TAG - 镜像版本 tag（如 commit SHA 或分支名） +# REGISTRY_TOKEN - Registry 访问令牌 +# REGISTRY - Registry 地址（默认 git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas） +# REGISTRY_USER - Registry 用户名（默认 xiaoxia） +# ENV_FILE - 环境变量文件路径 +# GENERATED_DIR - 生成文件目录 +# SKIP_MIGRATION - 跳过数据库迁移（true/false，默认 false） + +set -eu + +IMAGE_TAG="${IMAGE_TAG:-}" +REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}" +REGISTRY_USER="${REGISTRY_USER:-xiaoxia}" +REGISTRY_TOKEN="${REGISTRY_TOKEN:-}" + +ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-staging/.env}" +GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-staging/generated}" +LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-staging/legacy-assets}" + +SKIP_MIGRATION="${SKIP_MIGRATION:-false}" + +if [ -z "$IMAGE_TAG" ]; then + echo "ERROR: IMAGE_TAG is required" + exit 1 +fi + +test -f "$ENV_FILE" +mkdir -p "$GENERATED_DIR" +mkdir -p "$LEGACY_ASSETS_DIR" + +echo "==========================================" +echo " Staging 部署 - $IMAGE_TAG" +echo "==========================================" + +# ---- 登录 Registry ---- +if [ -n "$REGISTRY_TOKEN" ]; then + echo "Logging in to registry: $REGISTRY" + REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1) + printf %s "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin 2>/dev/null || { + echo "WARN: docker login failed, will try to pull anyway" + } +fi + +# ---- Pull 新版本镜像 ---- +REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" +REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" +REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" + +LOCAL_API="xiaoxia-saas-api:${IMAGE_TAG}" +LOCAL_WORKER="xiaoxia-saas-worker:${IMAGE_TAG}" +LOCAL_WEB="xiaoxia-saas-web:${IMAGE_TAG}" + +echo "Pulling API image..." +docker pull "$REGISTRY_API" +echo "Pulling Worker image..." +docker pull "$REGISTRY_WORKER" +echo "Pulling Web image..." +docker pull "$REGISTRY_WEB" + +# Re-tag 成本地名 +docker tag "$REGISTRY_API" "$LOCAL_API" +docker tag "$REGISTRY_WORKER" "$LOCAL_WORKER" +docker tag "$REGISTRY_WEB" "$LOCAL_WEB" +echo "All images pulled and tagged." + +# ---- 备份 legacy assets ---- +echo "Backing up legacy assets from current web container..." +if docker inspect xiaoxia-web-staging >/dev/null 2>&1; then + _tmpdir="/tmp/legacy-assets-$$" + rm -rf "$_tmpdir" + mkdir -p "$_tmpdir" + docker cp xiaoxia-web-staging:/usr/share/nginx/html/assets/. "$_tmpdir/" 2>/dev/null || true + # 只有目录非空才拷贝，避免覆盖有内容的 legacy assets + if [ -d "$_tmpdir" ] && [ "$(ls -A "$_tmpdir" 2>/dev/null)" ]; then + cp -an "$_tmpdir"/. "$LEGACY_ASSETS_DIR"/ 2>/dev/null || true + echo "Legacy assets backed up: $(ls "$_tmpdir" | wc -l) files" + fi + rm -rf "$_tmpdir" +else + echo "No existing web container, skipping legacy assets backup" +fi + +# 清理 7 天前的 legacy assets +if [ -d "$LEGACY_ASSETS_DIR" ]; then + find "$LEGACY_ASSETS_DIR" -type f -mtime +7 -delete 2>/dev/null || true + echo "Legacy assets cleanup done (retain 7 days)" +fi + +# ---- 检查基础设施容器 ---- +echo "Checking infrastructure containers..." +for c in xiaoxia-postgres-staging xiaoxia-redis-staging; do + if ! docker inspect "$c" >/dev/null 2>&1; then + echo "ERROR: Required container not found: $c" + exit 1 + fi + state=$(docker inspect -f '{{.State.Status}}' "$c") + if [ "$state" != "running" ]; then + echo "ERROR: Container not running: $c ($state)" + exit 1 + fi +done + +# ---- 创建网络（不存在则创建） ---- +docker network create xiaoxia-net-staging 2>/dev/null || true + +# ---- 数据库迁移 ---- +if [ "$SKIP_MIGRATION" != "true" ]; then + echo "Running database migrations..." + docker run --rm \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -e APP_ENV=staging \ + "$LOCAL_API" sh -c "cd /app && alembic upgrade head" + echo "Migrations completed." +else + echo "Skipping migrations (SKIP_MIGRATION=true)" +fi + +# ---- 停止旧容器 ---- +echo "Stopping old containers..." +docker rm -f xiaoxia-api-staging 2>/dev/null || true +docker rm -f xiaoxia-worker-staging 2>/dev/null || true +docker rm -f xiaoxia-web-staging 2>/dev/null || true + +LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" + +# ---- 启动 API ---- +echo "Starting API container..." +docker run -d \ + --name xiaoxia-api-staging \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -p 127.0.0.1:8000:8000 \ + -e APP_ENV=staging \ + -e APP_VERSION="$IMAGE_TAG" \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 40s \ + $LOG_OPTS \ + "$LOCAL_API" + +# ---- 启动 Worker ---- +echo "Starting Worker container..." +docker run -d \ + --name xiaoxia-worker-staging \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -e APP_ENV=staging \ + -e APP_VERSION="$IMAGE_TAG" \ + -e WORKER_CONCURRENCY=1 \ + -e WORKER_MAX_TASKS_PER_CHILD=100 \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 30s \ + $LOG_OPTS \ + "$LOCAL_WORKER" + +# ---- 启动 Web ---- +LEGACY_VOLUME="" +if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then + LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" + echo "Web container: legacy assets mounted (fallback)" +else + echo "Web container: no legacy assets to mount" +fi + +echo "Starting Web container..." +docker run -d \ + --name xiaoxia-web-staging \ + --network xiaoxia-net-staging \ + -p 127.0.0.1:3001:80 \ + --restart unless-stopped \ + $LEGACY_VOLUME \ + --health-cmd "wget --spider -q http://127.0.0.1:80" \ + --health-interval 30s \ + --health-timeout 5s \ + --health-retries 3 \ + $LOG_OPTS \ + "$LOCAL_WEB" + +# ---- 等待 API 健康 ---- +echo "Waiting for API to become healthy..." +i=0 +while [ "$i" -lt 40 ]; do + if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then + echo "API is healthy!" + break + fi + i=$((i + 1)) + echo " Waiting... ($i/40)" + sleep 3 +done + +if [ "$i" -ge 40 ]; then + echo "ERROR: API did not become healthy within 120s" + docker logs --tail 50 xiaoxia-api-staging + exit 1 +fi + +# ---- 等待 Web 健康 ---- +echo "Waiting for Web to become healthy..." +i=0 +while [ "$i" -lt 15 ]; do + if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then + echo "Web is healthy!" + break + fi + i=$((i + 1)) + echo " Waiting... ($i/15)" + sleep 2 +done + +if [ "$i" -ge 15 ]; then + echo "ERROR: Web did not become healthy within 30s" + docker logs --tail 30 xiaoxia-web-staging + exit 1 +fi + +# ---- 清理旧镜像 ---- +echo "Cleaning up old images..." +docker image prune -af --filter "until=168h" 2>/dev/null || true +docker builder prune -af --filter "until=168h" 2>/dev/null || true + +echo "" +echo "=== Staging deployment complete ===" +echo "API: http://127.0.0.1:8000" +echo "Web: http://127.0.0.1:3001" +echo "Version: $IMAGE_TAG" +docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging diff --git a/scripts/ci_staging_healthcheck.sh b/scripts/ci_staging_healthcheck.sh index 9373132e9..f8609f956 100644 --- a/scripts/ci_staging_healthcheck.sh +++ b/scripts/ci_staging_healthcheck.sh @@ -1,10 +1,10 @@ #!/bin/bash # =========================================== -# CI Staging 健康检查 + 自动回滚脚本（Watchtower 模式） +# CI Staging 健康检查 + 自动回滚脚本（SSH 部署模式） # =========================================== # # 在 CI Runner 上执行，通过公网 URL 检查 Staging 部署健康状态。 -# 不健康则自动回滚（把 :staging tag 指回旧版本镜像，Watchtower 检测到后自动回滚）。 +# 不健康则通过 SSH 自动回滚到上一个版本的镜像。 # # 用法: # ./ci_staging_healthcheck.sh @@ -13,14 +13,15 @@ # STAGING_API_URL - Staging API 地址 (默认 https://staging-api.xiaoxiajianji.com) # STAGING_WEB_URL - Staging Web 地址 (默认 https://staging.xiaoxiajianji.com) # HEALTH_CHECK_TIMEOUT - 健康检查总超时秒数 (默认 120) -# WATCHTOWER_WAIT - 等待 Watchtower 检测更新的秒数 (默认 90) # SKIP_ROLLBACK - 失败时不自动回滚 (true/false, 默认 false) # SKIP_NOTIFY - 跳过通知 (true/false, 默认 false) # CI_NOTIFY_WEBHOOK - 通知 Webhook URL # -# REGISTRY_TOKEN - Gitea Registry Token（回滚时需要） -# REGISTRY_HOST - Registry 地址 (默认 git.xiaoxiajianji.com) -# REGISTRY_REPO - 仓库路径 (默认 xiaoxia/xiaoxia-saas) +# STAGING_SSH_HOST - Staging 服务器 SSH 地址 (默认 47.98.113.167) +# STAGING_SSH_USER - SSH 用户名 (默认 root) +# STAGING_SSH_PORT - SSH 端口 (默认 22222) +# STAGING_SSH_KEY - SSH 私钥内容 +# REGISTRY_TOKEN - Registry Token（回滚时拉取旧镜像需要） # # GITHUB_SHA - 当前 commit SHA # GITHUB_REF_NAME - 分支名 @@ -36,12 +37,15 @@ SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" STAGING_API_URL="${STAGING_API_URL:-https://staging-api.xiaoxiajianji.com}" STAGING_WEB_URL="${STAGING_WEB_URL:-https://staging.xiaoxiajianji.com}" HEALTH_CHECK_TIMEOUT="${HEALTH_CHECK_TIMEOUT:-120}" -WATCHTOWER_WAIT="${WATCHTOWER_WAIT:-90}" SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}" SKIP_NOTIFY="${SKIP_NOTIFY:-false}" -REGISTRY_HOST="${REGISTRY_HOST:-git.xiaoxiajianji.com}" -REGISTRY_REPO="${REGISTRY_REPO:-xiaoxia/xiaoxia-saas}" +STAGING_SSH_HOST="${STAGING_SSH_HOST:-47.98.113.167}" +STAGING_SSH_USER="${STAGING_SSH_USER:-root}" +STAGING_SSH_PORT="${STAGING_SSH_PORT:-22222}" + +REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}" +REGISTRY_USER="${REGISTRY_USER:-xiaoxia}" # 颜色 RED='\033[0;31m' @@ -56,59 +60,85 @@ log_error() { echo -e "${RED}[ERROR]${NC} $1"; } log_step() { echo -e "${BLUE}[STEP]${NC} $1"; } # =========================================== -# 1. 记录部署前 :staging 镜像 digest（用于回滚） +# SSH 工具函数 # =========================================== +SSH_KEY_PATH="" + +setup_ssh() { + # 查找或创建 SSH 密钥 + if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then + SSH_KEY_PATH="/root/.ssh/xiaoxia_runtime_builder" + elif [ -f "$HOME/.ssh/xiaoxia_runtime_builder" ]; then + SSH_KEY_PATH="$HOME/.ssh/xiaoxia_runtime_builder" + elif [ -n "${STAGING_SSH_KEY:-}" ]; then + SSH_KEY_PATH="$HOME/.ssh/staging_deploy_key" + mkdir -p "$HOME/.ssh" + printf '%s\n' "$STAGING_SSH_KEY" > "$SSH_KEY_PATH" + chmod 600 "$SSH_KEY_PATH" + else + log_error "没有可用的 SSH 密钥" + return 1 + fi + + ssh-keyscan -p "$STAGING_SSH_PORT" -H "$STAGING_SSH_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true + log_info "SSH 已配置: ${STAGING_SSH_USER}@${STAGING_SSH_HOST}:${STAGING_SSH_PORT}" +} + +run_ssh() { + local cmd="$1" + ssh -p "$STAGING_SSH_PORT" -i "$SSH_KEY_PATH" -o StrictHostKeyChecking=no \ + "${STAGING_SSH_USER}@${STAGING_SSH_HOST}" "$cmd" +} + +# =========================================== +# 1. 记录部署前各服务的镜像版本（用于回滚） +# =========================================== +ROLLBACK_API_TAG="" +ROLLBACK_WORKER_TAG="" +ROLLBACK_WEB_TAG="" + save_rollback_target() { - log_step "记录当前 :staging 镜像 digest（回滚目标）..." + log_step "记录当前 staging 各服务镜像版本（回滚目标）..." - # 登录 Registry - if [ -n "${REGISTRY_TOKEN:-}" ]; then - printf '%s' "${REGISTRY_TOKEN}" | docker login "${REGISTRY_HOST}" -u xiaoxia --password-stdin 2>/dev/null || true - fi + # 通过 SSH 获取当前运行的容器镜像 + local api_image worker_image web_image + api_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-api-staging 2>/dev/null || echo ''") + worker_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-worker-staging 2>/dev/null || echo ''") + web_image=$(run_ssh "docker inspect --format '{{.Config.Image}}' xiaoxia-web-staging 2>/dev/null || echo ''") - ROLLBACK_IMAGES="" + # 提取 tag（镜像名是 xiaoxia-saas-api:abc123 或 git.xiaoxiajianji.com/.../xiaoxia-saas-api:staging 格式） + ROLLBACK_API_TAG=$(echo "$api_image" | sed 's/.*://' || echo "") + ROLLBACK_WORKER_TAG=$(echo "$worker_image" | sed 's/.*://' || echo "") + ROLLBACK_WEB_TAG=$(echo "$web_image" | sed 's/.*://' || echo "") - for svc in api worker web; do - image="${REGISTRY_HOST}/${REGISTRY_REPO}/xiaoxia-saas-${svc}:staging" - # pull 当前 :staging 镜像获取 digest - if docker pull "$image" 2>/dev/null; then - digest=$(docker inspect -f '{{index .RepoDigests 0}}' "$image" 2>/dev/null || echo "") - if [ -n "$digest" ]; then - ROLLBACK_IMAGES="${ROLLBACK_IMAGES}${svc}:${digest} " - log_info " $svc: $digest" - else - log_warn " $svc: 无法获取 digest，该服务将不参与回滚" - fi + log_info " API: ${ROLLBACK_API_TAG:-未知}" + log_info " Worker: ${ROLLBACK_WORKER_TAG:-未知}" + log_info " Web: ${ROLLBACK_WEB_TAG:-未知}" + + # 验证三个服务版本是否一致 + if [ -n "$ROLLBACK_API_TAG" ] && [ -n "$ROLLBACK_WORKER_TAG" ] && [ -n "$ROLLBACK_WEB_TAG" ]; then + if [ "$ROLLBACK_API_TAG" = "$ROLLBACK_WORKER_TAG" ] && [ "$ROLLBACK_API_TAG" = "$ROLLBACK_WEB_TAG" ]; then + log_info " ✅ 三个服务版本一致: $ROLLBACK_API_TAG" + export ROLLBACK_TAG="$ROLLBACK_API_TAG" else - log_warn " $svc: 拉取 :staging 镜像失败，该服务将不参与回滚" + log_warn " ⚠️ 三个服务版本不一致，回滚时将分别使用各自版本" + export ROLLBACK_API_TAG ROLLBACK_WORKER_TAG ROLLBACK_WEB_TAG + export ROLLBACK_TAG_MIXED="true" fi - done - - if [ -z "$ROLLBACK_IMAGES" ]; then - log_warn "没有获取到任何可回滚镜像，回滚功能将不可用" + else + log_warn " ⚠️ 未能获取全部服务版本，回滚功能可能受限" fi - - export ROLLBACK_IMAGES } # =========================================== -# 2. 等待 Watchtower 更新 -# =========================================== -wait_for_watchtower() { - log_step "等待 Watchtower 检测新镜像并更新（${WATCHTOWER_WAIT}s）..." - sleep "$WATCHTOWER_WAIT" - log_info "等待结束，开始健康检查" -} - -# =========================================== -# 3. 健康检查 +# 2. 健康检查（公网视角） # =========================================== health_check() { local timeout="$HEALTH_CHECK_TIMEOUT" local start_time start_time=$(date +%s) - log_step "健康检查（超时 ${timeout}s）..." + log_step "公网健康检查（超时 ${timeout}s）..." log_info " API: ${STAGING_API_URL}/health" log_info " Web: ${STAGING_WEB_URL}/" @@ -171,69 +201,183 @@ health_check() { } # =========================================== -# 4. 执行回滚（把 :staging tag 改回旧版本） +# 3. 执行回滚（SSH 重新部署旧版本） # =========================================== do_rollback() { - log_step "执行回滚：将 :staging tag 指回旧版本镜像..." + log_step "执行回滚：通过 SSH 重新部署旧版本镜像..." - if [ -z "${ROLLBACK_IMAGES:-}" ]; then - log_error "没有可回滚的镜像记录，无法自动回滚" + local rollback_tag="${ROLLBACK_TAG:-}" + if [ -z "$rollback_tag" ] && [ "${ROLLBACK_TAG_MIXED:-}" != "true" ]; then + log_error "没有可回滚的版本记录，无法自动回滚" return 1 fi - local rollback_ok=true + # 如果版本不一致，用 API 的版本作为回滚目标 + if [ -z "$rollback_tag" ]; then + rollback_tag="$ROLLBACK_API_TAG" + fi - for entry in $ROLLBACK_IMAGES; do - svc="${entry%%:*}" - digest="${entry#*:}" - # digest 是 host/repo/image@sha256:xxx 格式，entry 里只取了第一部分，需要重新组合 - # 实际上 ROLLBACK_IMAGES 存的是 svc:repo/image@sha256:xxx - # 让我们重新从完整digest中提取 - image_digest="${entry#*:}:${entry#*:*:}" - # 不对，格式是 svc:REGISTRY_HOST/REGISTRY_REPO/xiaoxia-saas-svc@sha256:xxx - # 让我重新组织一下 - : - done + if [ -z "$rollback_tag" ]; then + log_error "无法确定回滚版本" + return 1 + fi - # 上面的解析有问题，重新来 - # ROLLBACK_IMAGES 格式: "api:digest1 worker:digest2 web:digest3" - # 但digest本身带冒号(@sha256:xxx)，所以分隔有问题 - # 改用数组方式 - log_info "开始回滚各服务..." + log_info "回滚目标版本: $rollback_tag" - local IFS=' ' - for pair in $ROLLBACK_IMAGES; do - svc="${pair%%:*}" - # 剩余部分是完整的 digest 字符串（含 @sha256:xxx） - digest="${pair#*:}" + # 构建回滚脚本（直接部署旧版本镜像，不跑 migration） + local rollback_script=$(cat << 'ROLLBACK_EOF' +#!/bin/sh +set -eu - image_name="${REGISTRY_HOST}/${REGISTRY_REPO}/xiaoxia-saas-${svc}" +IMAGE_TAG="$1" +REGISTRY_TOKEN="$2" +REGISTRY="${REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}" +REGISTRY_USER="${REGISTRY_USER:-xiaoxia}" - log_info " 回滚 $svc 到: $digest" +ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-staging/.env}" +GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-staging/generated}" +LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-staging/legacy-assets}" - # 通过 digest 拉取旧镜像 - if docker pull "$digest" 2>/dev/null; then - # 重新打 :staging tag - docker tag "$digest" "${image_name}:staging" - docker push "${image_name}:staging" - log_info " ✅ $svc 已回滚" - else - log_error " ❌ $svc 回滚失败：无法拉取旧镜像" - rollback_ok=false - fi - done +echo "=== Rollback to $IMAGE_TAG ===" - if [ "$rollback_ok" = true ]; then - log_info "所有服务回滚命令执行完成，Watchtower 将在 ~60s 内检测到并更新" +# 登录 Registry +if [ -n "$REGISTRY_TOKEN" ]; then + REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1) + printf %s "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin 2>/dev/null || true +fi + +# Pull 旧版本镜像 +LOCAL_API="xiaoxia-saas-api:${IMAGE_TAG}" +LOCAL_WORKER="xiaoxia-saas-worker:${IMAGE_TAG}" +LOCAL_WEB="xiaoxia-saas-web:${IMAGE_TAG}" + +docker pull "${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" +docker pull "${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" +docker pull "${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" + +docker tag "${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" "$LOCAL_API" +docker tag "${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" "$LOCAL_WORKER" +docker tag "${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" "$LOCAL_WEB" + +echo "Rollback images pulled." + +# 停止当前容器（回滚不跑 migration，避免数据问题） +docker rm -f xiaoxia-api-staging 2>/dev/null || true +docker rm -f xiaoxia-worker-staging 2>/dev/null || true +docker rm -f xiaoxia-web-staging 2>/dev/null || true + +LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" + +# 启动 API（回滚不跑 migration） +echo "Starting API (rollback)..." +docker run -d \ + --name xiaoxia-api-staging \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -p 127.0.0.1:8000:8000 \ + -e APP_ENV=staging \ + -e APP_VERSION="$IMAGE_TAG" \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 40s \ + $LOG_OPTS \ + "$LOCAL_API" + +# 启动 Worker +echo "Starting Worker (rollback)..." +docker run -d \ + --name xiaoxia-worker-staging \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-staging \ + -e APP_ENV=staging \ + -e APP_VERSION="$IMAGE_TAG" \ + -e WORKER_CONCURRENCY=1 \ + -e WORKER_MAX_TASKS_PER_CHILD=100 \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 30s \ + $LOG_OPTS \ + "$LOCAL_WORKER" + +# 启动 Web +LEGACY_VOLUME="" +if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then + LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" +fi + +echo "Starting Web (rollback)..." +docker run -d \ + --name xiaoxia-web-staging \ + --network xiaoxia-net-staging \ + -p 127.0.0.1:3001:80 \ + --restart unless-stopped \ + $LEGACY_VOLUME \ + --health-cmd "wget --spider -q http://127.0.0.1:80" \ + --health-interval 30s \ + --health-timeout 5s \ + --health-retries 3 \ + $LOG_OPTS \ + "$LOCAL_WEB" + +# 等待 API 健康 +echo "Waiting for API (rollback)..." +i=0 +while [ "$i" -lt 40 ]; do + if curl -sf --max-time 5 http://127.0.0.1:8000/health >/dev/null 2>&1; then + echo "API healthy (rollback)." + break + fi + i=$((i + 1)) + sleep 3 +done + +# 等待 Web 健康 +echo "Waiting for Web (rollback)..." +i=0 +while [ "$i" -lt 15 ]; do + if curl -sf --max-time 5 http://127.0.0.1:3001/ >/dev/null 2>&1; then + echo "Web healthy (rollback)." + break + fi + i=$((i + 1)) + sleep 2 +done + +echo "=== Rollback complete: $IMAGE_TAG ===" +docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep staging +ROLLBACK_EOF +) + + # 将脚本 base64 编码后通过 SSH 执行 + local script_b64 + script_b64=$(echo "$rollback_script" | base64 -w 0) + + log_info "在 staging 服务器上执行回滚脚本..." + if run_ssh "echo '$script_b64' | base64 -d | sh -s -- '$rollback_tag' '${REGISTRY_TOKEN:-}'" 2>&1; then + log_info "✅ 回滚命令执行完成" return 0 else - log_error "部分服务回滚失败" + log_error "❌ 回滚命令执行失败" return 1 fi } # =========================================== -# 5. 发送通知 +# 4. 发送通知 # =========================================== send_notification() { local status="$1" # success / failure / rollback @@ -250,12 +394,12 @@ send_notification() { return 0 fi - # 调用通知脚本 if [ -f "$SCRIPT_DIR/deploy_notify.py" ]; then python3 "$SCRIPT_DIR/deploy_notify.py" \ --status "$status" \ --detail "$detail" \ --webhook "$webhook" \ + --env staging \ 2>/dev/null || log_warn "通知发送失败（非致命）" else log_warn "找不到 deploy_notify.py，跳过通知" @@ -267,41 +411,42 @@ send_notification() { # =========================================== main() { echo "" - echo "==========================================" - echo " CI Staging 健康检查 + 自动回滚" - echo "==========================================" + echo "===========================================" + echo " CI Staging 健康检查 + 自动回滚（SSH模式）" + echo "===========================================" echo "" local deploy_status="success" local deploy_detail="" - # 1. 记录部署前状态（回滚目标） - save_rollback_target + # 1. 设置 SSH + if ! setup_ssh; then + log_error "SSH 配置失败，无法执行回滚" + fi - # 2. 等待 Watchtower 更新 - wait_for_watchtower + # 2. 记录部署前状态（回滚目标） + save_rollback_target || true - # 3. 健康检查 + # 3. 健康检查（公网视角） if ! health_check; then log_error "健康检查失败" deploy_status="failure" - deploy_detail="健康检查超时，部署后服务未正常启动" + deploy_detail="公网健康检查超时，部署后服务未正常响应" # 自动回滚 if [ "${SKIP_ROLLBACK:-false}" != "true" ]; then log_warn "开始自动回滚..." if do_rollback; then deploy_status="rollback" - deploy_detail="健康检查失败，已自动回滚到上一版本" + deploy_detail="健康检查失败，已自动回滚到上一版本 (${ROLLBACK_TAG:-未知})" - # 回滚后再检查一下回滚后的状态 - log_info "等待 Watchtower 应用回滚（90s）..." - sleep 90 + # 回滚后再检查一下公网状态 + log_info "回滚完成，重新检查公网健康状态..." if health_check; then log_info "✅ 回滚后服务已恢复" deploy_detail="${deploy_detail}，回滚后服务已恢复" else - log_error "⚠️ 回滚后健康检查仍未通过，请手动排查" + log_error "⚠️ 回滚后健康检查仍未通过，请手动排查" deploy_detail="${deploy_detail}，但回滚后仍未恢复，请紧急排查" fi else @@ -321,7 +466,7 @@ main() { log_info " ✅ Staging 部署成功！" log_info "==================================" - deploy_detail="部署成功，所有健康检查通过" + deploy_detail="部署成功，所有健康检查通过 (${GITHUB_SHA:-未知版本})" send_notification "success" "$deploy_detail" } -- 2.54.0