From ca108bb14e7a1f84d74091605d0fbee2aa842cdb Mon Sep 17 00:00:00 2001 From: xiaoxia-agent Date: Fri, 11 Sep 2026 09:18:18 +0800 Subject: [PATCH] debug(lipsync): add SSH diag workflow for tts_processing stuck issue [skip ci-test] --- .gitea/workflows/debug-lipsync-diag.yml | 43 +++++++++ infra/scripts/server-lipsync-diag.sh | 117 ++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 .gitea/workflows/debug-lipsync-diag.yml create mode 100644 infra/scripts/server-lipsync-diag.sh diff --git a/.gitea/workflows/debug-lipsync-diag.yml b/.gitea/workflows/debug-lipsync-diag.yml new file mode 100644 index 000000000..025228e7c --- /dev/null +++ b/.gitea/workflows/debug-lipsync-diag.yml @@ -0,0 +1,43 @@ +name: Debug Lipsync SSH Diag +on: + workflow_dispatch: + inputs: + reason: + description: "debug reason" + required: false + default: "lipsync tts_processing stuck diagnosis" + push: + branches: + - debug/lipsync-ssh-diag +permissions: + contents: read +jobs: + ssh-diag: + name: SSH Staging Diagnostics + runs-on: runtime-builder + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup SSH + shell: bash + env: + STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }} + run: | + set -euo pipefail + which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client) + mkdir -p ~/.ssh && chmod 700 ~/.ssh + printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + H=47.98.113.167; P=22222 + ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null + ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H "echo SSH_OK; hostname; date" + - name: Upload diag script and run + shell: bash + run: | + set -euo pipefail + H=47.98.113.167; P=22222 + scp -P $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no \ + infra/scripts/server-lipsync-diag.sh root@$H:/tmp/server-lipsync-diag.sh + ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H \ + "bash /tmp/server-lipsync-diag.sh 2>&1" diff --git a/infra/scripts/server-lipsync-diag.sh b/infra/scripts/server-lipsync-diag.sh new file mode 100644 index 000000000..9093e1059 --- /dev/null +++ b/infra/scripts/server-lipsync-diag.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# Lipsync staging 诊断脚本（在staging服务器本机执行） +set +e +echo "#########################################################" +echo "# Lipsync TTS async 诊断报告" +echo "# Host: $(hostname)" +echo "# Date: $(date)" +echo "#########################################################" + +echo "" +echo "=== 1. docker ps (worker/api/redis) ===" +docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.CreatedAt}}' | grep -E "worker|api|redis|nginx|NAMES" + +W=$(docker ps --format '{{.Names}}' | grep -E 'worker' | head -1) +A=$(docker ps --format '{{.Names}}' | grep -E 'api' | grep -v 'web' | head -1) +R=$(docker ps --format '{{.Names}}' | grep -E 'redis' | head -1) + +echo "" +echo "=== Worker container: $W ===" +echo "=== API container: $A ===" +echo "=== Redis container: $R ===" + +echo "" +echo "=== 2. Worker container image + created time ===" +docker inspect "$W" --format 'Image={{.Config.Image}} Created={{.Created}}' +docker inspect "$A" --format 'Image={{.Config.Image}} Created={{.Created}}' + +echo "" +echo "=== 3. Worker 启动日志：celery ready / registered / lipsync 相关 ===" +docker logs --tail=500 "$W" 2>&1 | grep -iE "lipsync|tts_synth|synthesize|celery@|ready|registered|register|error|traceback|import|critical|not registered|Consumer|mingle|tasks" | tail -120 + +echo "" +echo "=== 4. celery -A inspect registered ===" +docker exec "$W" celery -A worker_app.celery_app inspect registered 2>&1 | tail -80 + +echo "" +echo "=== 5. celery -A inspect active/reserved ===" +echo "--- active ---" +docker exec "$W" celery -A worker_app.celery_app inspect active 2>&1 | tail -30 +echo "--- reserved ---" +docker exec "$W" celery -A worker_app.celery_app inspect reserved 2>&1 | tail -30 + +echo "" +echo "=== 6. Worker 内 Python: import lipsync_tts task，看绑定 ===" +docker exec "$W" python -c " +from worker_app.celery_app import celery_app +print('worker celery_app.main:', celery_app.main) +print('worker celery_app.conf.imports:') +for m in celery_app.conf.imports: + print(' ', m) +import app.tasks.lipsync_tts as m +t = m.tts_synthesize_and_submit +print('task name:', t.name) +print('task.app.main:', t.app.main) +print('task.app is worker celery_app:', t.app is celery_app) +print('task.name in worker celery_app.tasks:', t.name in celery_app.tasks) +" 2>&1 + +echo "" +echo "=== 7. API 容器日志：lipsync/celery/apply_async 相关错误 ===" +docker logs --tail=800 "$A" 2>&1 | grep -iE "lipsync|tts_|celery|apply_async|NotRegistered|traceback|error.*task" | tail -120 + +echo "" +echo "=== 8. API 内 Python: import task 看绑定 + apply_async 试投递（不消费） ===" +docker exec "$A" python -c " +import traceback +try: + from app.tasks.lipsync_tts import tts_synthesize_and_submit + from app.core.celery_app import celery_app as api_app + print('api celery_app.main:', api_app.main) + print('task name:', tts_synthesize_and_submit.name) + print('task.app.main:', tts_synthesize_and_submit.app.main) + print('task.app is api_app:', tts_synthesize_and_submit.app is api_app) + print('task.name in api_app.tasks:', tts_synthesize_and_submit.name in api_app.tasks) + # 测试 send_task 是否能路由（不发送真任务） + print('api_app tasks includes lipsync_tts.synthesize_and_submit:', 'lipsync_tts.synthesize_and_submit' in api_app.tasks) +except Exception as e: + traceback.print_exc() +" 2>&1 + +echo "" +echo "=== 9. Redis 队列长度 ===" +for q in celery generation transcode; do + len=$(docker exec "$R" redis-cli LLEN $q 2>&1) + echo " $q length: $len" +done + +echo "" +echo "=== 10. Redis celery 队列 peek（最多5条） ===" +docker exec "$R" redis-cli LRANGE celery 0 4 2>&1 | head -40 + +echo "" +echo "=== 11. Worker 关键文件内容校验 ===" +echo "--- worker_app/celery_app.py imports 段 ---" +docker exec "$W" grep -n "lipsync\|imports\s*=\|app.tasks\|apps.api" /app/apps/worker/worker_app/celery_app.py 2>&1 +echo "" +echo "--- lipsync_tts.py 前20行（应该是 shared_task） ---" +docker exec "$W" head -25 /app/apps/api/app/tasks/lipsync_tts.py 2>&1 + +echo "" +echo "=== 12. API lipsync_service.py apply_async 上下文 ===" +docker exec "$A" grep -n -B3 -A10 "apply_async\|tts_synthesize" /app/apps/api/app/services/lipsync_service.py 2>&1 | head -80 + +echo "" +echo "=== 13. Worker /app 目录结构（app/tasks） ===" +docker exec "$W" ls -la /app/apps/api/app/tasks/ 2>&1 | head -30 +echo "" +echo "--- /app/apps/api/app/__init__.py? ---" +docker exec "$W" ls -la /app/apps/api/app/__init__.py /app/apps/api/app/core/__init__.py /app/apps/api/app/api/__init__.py 2>&1 +echo "" +echo "--- PYTHONPATH inside worker ---" +docker exec "$W" env | grep PYTHONPATH + +echo "" +echo "#########################################################" +echo "# 诊断完成" +echo "#########################################################"