From e4ac398397e01681fa9c32baedba31709f9c29b6 Mon Sep 17 00:00:00 2001 From: CI Bot Date: Tue, 8 Sep 2026 19:07:37 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20Worker=20=E5=81=A5=E5=BA=B7=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=E5=BD=BB=E5=BA=95=E4=BF=AE=E5=A4=8D=20-=20=E7=A7=BB?= =?UTF-8?q?=E9=99=A4=E4=B8=8D=E5=8F=AF=E7=94=A8=E7=9A=84=20pgrep=20?= =?UTF-8?q?=E5=92=8C=20PID1=20=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 根因：#1714 队列隔离后 entrypoint 启动两个 celery worker 子进程， PID 1 变为 bash。旧健康检查依赖 pgrep（容器内不存在）或 /proc/1/cmdline（PID 1 是 bash 不是 celery），永远返回失败。 此前靠 cron 脚本 fix-worker-healthcheck.sh 每 30 分钟擦屁股。 修复：统一改为扫描 /proc/[0-9]*/cmdline 查找 celery 进程， 覆盖所有部署/回滚/CI 脚本（10 处替换）。已移除 cron 补丁。 --- infra/docker/compose.yml | 2 +- infra/docker/deploy-production-registry.sh | 2 +- infra/docker/deploy-staging-registry.sh | 2 +- scripts/ci_production_deploy.sh | 4 ++-- scripts/ci_production_healthcheck.sh | 2 +- scripts/ci_staging_deploy.sh | 4 ++-- scripts/ci_staging_healthcheck.sh | 2 +- scripts/rollback_production.sh | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/infra/docker/compose.yml b/infra/docker/compose.yml index 154e9fad1..b0ce86443 100755 --- a/infra/docker/compose.yml +++ b/infra/docker/compose.yml @@ -130,7 +130,7 @@ services: # 健康检查配置 # 注：celery inspect ping 依赖 broker 连接，在容器内不可靠，改用进程检查 healthcheck: - test: ["CMD-SHELL", "pgrep -f 'celery.*worker' | head -n1 >/dev/null 2>&1 || grep -q celery /proc/1/cmdline || exit 1"] + test: ["CMD-SHELL", "for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1"] interval: 30s timeout: 10s retries: 3 diff --git a/infra/docker/deploy-production-registry.sh b/infra/docker/deploy-production-registry.sh index 664e71f6e..e9a4a6a27 100755 --- a/infra/docker/deploy-production-registry.sh +++ b/infra/docker/deploy-production-registry.sh @@ -155,7 +155,7 @@ docker run -d \ --restart unless-stopped \ --cpus 2 \ --memory 2g \ - --health-cmd "sh -c \"pgrep -f 'celery.*worker' >/dev/null 2>&1 || grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-cmd "sh -c \"for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \ diff --git a/infra/docker/deploy-staging-registry.sh b/infra/docker/deploy-staging-registry.sh index b444b9ad9..e7494c9b3 100755 --- a/infra/docker/deploy-staging-registry.sh +++ b/infra/docker/deploy-staging-registry.sh @@ -116,7 +116,7 @@ docker run -d \ -v "$GENERATED_DIR:/app/generated" \ --restart unless-stopped \ --label com.centurylinklabs.watchtower.enable=true \ - --health-cmd "sh -c \"pgrep -f 'celery.*worker' >/dev/null 2>&1 || grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-cmd "sh -c \"for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \ diff --git a/scripts/ci_production_deploy.sh b/scripts/ci_production_deploy.sh index 10c4e0f7c..c9a592892 100644 --- a/scripts/ci_production_deploy.sh +++ b/scripts/ci_production_deploy.sh @@ -209,7 +209,7 @@ rollback() { --restart unless-stopped \ --cpus 2 \ --memory 2g \ - --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-cmd "sh -c \"for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \ @@ -408,7 +408,7 @@ docker run -d \ --restart unless-stopped \ --cpus 2 \ --memory 2g \ - --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-cmd "sh -c \"for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \ diff --git a/scripts/ci_production_healthcheck.sh b/scripts/ci_production_healthcheck.sh index 72abd9f26..3b311904c 100644 --- a/scripts/ci_production_healthcheck.sh +++ b/scripts/ci_production_healthcheck.sh @@ -310,7 +310,7 @@ docker run -d \ --restart unless-stopped \ --cpus 2 \ --memory 2g \ - --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-cmd "sh -c \"for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \ diff --git a/scripts/ci_staging_deploy.sh b/scripts/ci_staging_deploy.sh index aee1c9132..19e2d7386 100755 --- a/scripts/ci_staging_deploy.sh +++ b/scripts/ci_staging_deploy.sh @@ -192,7 +192,7 @@ rollback() { -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ -v "$GENERATED_DIR:/app/generated" \ --restart unless-stopped \ - --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-cmd "sh -c \"for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \ @@ -501,7 +501,7 @@ docker run -d \ -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ -v "$GENERATED_DIR:/app/generated" \ --restart unless-stopped \ - --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-cmd "sh -c \"for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \ diff --git a/scripts/ci_staging_healthcheck.sh b/scripts/ci_staging_healthcheck.sh index 79c30fa80..2954c5cfa 100644 --- a/scripts/ci_staging_healthcheck.sh +++ b/scripts/ci_staging_healthcheck.sh @@ -305,7 +305,7 @@ docker run -d \ -e PUBLIC_API_BASE_URL=https://staging-api.xiaoxiajianji.com \ -v "$GENERATED_DIR:/app/generated" \ --restart unless-stopped \ - --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-cmd "sh -c \"for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \ diff --git a/scripts/rollback_production.sh b/scripts/rollback_production.sh index ceba80954..4e4d3f59c 100755 --- a/scripts/rollback_production.sh +++ b/scripts/rollback_production.sh @@ -201,7 +201,7 @@ docker run -d \ --restart unless-stopped \ --cpus 2 \ --memory 2g \ - --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-cmd "sh -c \"for pid in /proc/[0-9]*/cmdline; do if grep -ql celery \"$pid\" 2>/dev/null; then exit 0; fi; done; exit 1\"" \ --health-interval 30s \ --health-timeout 10s \ --health-retries 3 \