debug(lipsync): add SSH diag workflow for tts_processing stuck issue [skip ci-test]

2026-09-11 09:18:18 +08:00
parent 29ca51da9c
commit ca108bb14e
2 changed files with 160 additions and 0 deletions
@@ -0,0 +1,43 @@
+name: Debug Lipsync SSH Diag
+on:
+  workflow_dispatch:
+    inputs:
+      reason:
+        description: "debug reason"
+        required: false
+        default: "lipsync tts_processing stuck diagnosis"
+  push:
+    branches:
+      - debug/lipsync-ssh-diag
+permissions:
+  contents: read
+jobs:
+  ssh-diag:
+    name: SSH Staging Diagnostics
+    runs-on: runtime-builder
+    timeout-minutes: 15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup SSH
+        shell: bash
+        env:
+          STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
+        run: |
+          set -euo pipefail
+          which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          H=47.98.113.167; P=22222
+          ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null
+          ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H "echo SSH_OK; hostname; date"
+      - name: Upload diag script and run
+        shell: bash
+        run: |
+          set -euo pipefail
+          H=47.98.113.167; P=22222
+          scp -P $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no \
+            infra/scripts/server-lipsync-diag.sh root@$H:/tmp/server-lipsync-diag.sh
+          ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H \
+            "bash /tmp/server-lipsync-diag.sh 2>&1"
@@ -0,0 +1,117 @@
+#!/bin/bash
+# Lipsync staging 诊断脚本（在staging服务器本机执行）
+set +e
+echo "#########################################################"
+echo "#  Lipsync TTS async 诊断报告"
+echo "#  Host: $(hostname)"
+echo "#  Date: $(date)"
+echo "#########################################################"
+
+echo ""
+echo "=== 1. docker ps (worker/api/redis) ==="
+docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.CreatedAt}}' | grep -E "worker|api|redis|nginx|NAMES"
+
+W=$(docker ps --format '{{.Names}}' | grep -E 'worker' | head -1)
+A=$(docker ps --format '{{.Names}}' | grep -E 'api' | grep -v 'web' | head -1)
+R=$(docker ps --format '{{.Names}}' | grep -E 'redis' | head -1)
+
+echo ""
+echo "=== Worker container: $W ==="
+echo "=== API container: $A ==="
+echo "=== Redis container: $R ==="
+
+echo ""
+echo "=== 2. Worker container image + created time ==="
+docker inspect "$W" --format 'Image={{.Config.Image}} Created={{.Created}}'
+docker inspect "$A" --format 'Image={{.Config.Image}} Created={{.Created}}'
+
+echo ""
+echo "=== 3. Worker 启动日志：celery ready / registered / lipsync 相关 ==="
+docker logs --tail=500 "$W" 2>&1 | grep -iE "lipsync|tts_synth|synthesize|celery@|ready|registered|register|error|traceback|import|critical|not registered|Consumer|mingle|tasks" | tail -120
+
+echo ""
+echo "=== 4. celery -A inspect registered ==="
+docker exec "$W" celery -A worker_app.celery_app inspect registered 2>&1 | tail -80
+
+echo ""
+echo "=== 5. celery -A inspect active/reserved ==="
+echo "--- active ---"
+docker exec "$W" celery -A worker_app.celery_app inspect active 2>&1 | tail -30
+echo "--- reserved ---"
+docker exec "$W" celery -A worker_app.celery_app inspect reserved 2>&1 | tail -30
+
+echo ""
+echo "=== 6. Worker 内 Python: import lipsync_tts task，看绑定 ==="
+docker exec "$W" python -c "
+from worker_app.celery_app import celery_app
+print('worker celery_app.main:', celery_app.main)
+print('worker celery_app.conf.imports:')
+for m in celery_app.conf.imports:
+    print('  ', m)
+import app.tasks.lipsync_tts as m
+t = m.tts_synthesize_and_submit
+print('task name:', t.name)
+print('task.app.main:', t.app.main)
+print('task.app is worker celery_app:', t.app is celery_app)
+print('task.name in worker celery_app.tasks:', t.name in celery_app.tasks)
+" 2>&1
+
+echo ""
+echo "=== 7. API 容器日志：lipsync/celery/apply_async 相关错误 ==="
+docker logs --tail=800 "$A" 2>&1 | grep -iE "lipsync|tts_|celery|apply_async|NotRegistered|traceback|error.*task" | tail -120
+
+echo ""
+echo "=== 8. API 内 Python: import task 看绑定 + apply_async 试投递（不消费） ==="
+docker exec "$A" python -c "
+import traceback
+try:
+    from app.tasks.lipsync_tts import tts_synthesize_and_submit
+    from app.core.celery_app import celery_app as api_app
+    print('api celery_app.main:', api_app.main)
+    print('task name:', tts_synthesize_and_submit.name)
+    print('task.app.main:', tts_synthesize_and_submit.app.main)
+    print('task.app is api_app:', tts_synthesize_and_submit.app is api_app)
+    print('task.name in api_app.tasks:', tts_synthesize_and_submit.name in api_app.tasks)
+    # 测试 send_task 是否能路由（不发送真任务）
+    print('api_app tasks includes lipsync_tts.synthesize_and_submit:', 'lipsync_tts.synthesize_and_submit' in api_app.tasks)
+except Exception as e:
+    traceback.print_exc()
+" 2>&1
+
+echo ""
+echo "=== 9. Redis 队列长度 ==="
+for q in celery generation transcode; do
+    len=$(docker exec "$R" redis-cli LLEN $q 2>&1)
+    echo "  $q length: $len"
+done
+
+echo ""
+echo "=== 10. Redis celery 队列 peek（最多5条） ==="
+docker exec "$R" redis-cli LRANGE celery 0 4 2>&1 | head -40
+
+echo ""
+echo "=== 11. Worker 关键文件内容校验 ==="
+echo "--- worker_app/celery_app.py imports 段 ---"
+docker exec "$W" grep -n "lipsync\|imports\s*=\|app.tasks\|apps.api" /app/apps/worker/worker_app/celery_app.py 2>&1
+echo ""
+echo "--- lipsync_tts.py 前20行（应该是 shared_task） ---"
+docker exec "$W" head -25 /app/apps/api/app/tasks/lipsync_tts.py 2>&1
+
+echo ""
+echo "=== 12. API lipsync_service.py apply_async 上下文 ==="
+docker exec "$A" grep -n -B3 -A10 "apply_async\|tts_synthesize" /app/apps/api/app/services/lipsync_service.py 2>&1 | head -80
+
+echo ""
+echo "=== 13. Worker /app 目录结构（app/tasks） ==="
+docker exec "$W" ls -la /app/apps/api/app/tasks/ 2>&1 | head -30
+echo ""
+echo "--- /app/apps/api/app/__init__.py? ---"
+docker exec "$W" ls -la /app/apps/api/app/__init__.py /app/apps/api/app/core/__init__.py /app/apps/api/app/api/__init__.py 2>&1
+echo ""
+echo "--- PYTHONPATH inside worker ---"
+docker exec "$W" env | grep PYTHONPATH
+
+echo ""
+echo "#########################################################"
+echo "#  诊断完成"
+echo "#########################################################"