Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 72aa5009be | |||
| 9c0d4b136f | |||
| 387514c111 | |||
| 76cdb15c6b |
@@ -1,37 +0,0 @@
|
||||
name: "Debug: Web container crash diag"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'debug/web-crash-diag'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
web-diag:
|
||||
name: Web Crash Diagnostics
|
||||
runs-on: runtime-builder
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup SSH
|
||||
shell: bash
|
||||
env:
|
||||
STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
|
||||
mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||
printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
|
||||
chmod 600 ~/.ssh/id_rsa
|
||||
H=47.98.113.167; P=22222
|
||||
ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null
|
||||
ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H "echo SSH_OK; hostname; date"
|
||||
- name: Run web diag
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
H=47.98.113.167; P=22222
|
||||
scp -P $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no \
|
||||
infra/scripts/server-web-diag.sh root@$H:/tmp/server-web-diag.sh
|
||||
ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H \
|
||||
"bash /tmp/server-web-diag.sh 2>&1"
|
||||
@@ -211,12 +211,18 @@ class LipsyncService:
|
||||
normalize_emotion(emotion),
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Celery 任务提交失败,TTS 任务已创建但未触发执行: %s",
|
||||
except Exception as exc:
|
||||
# 投递失败时立即把 job 标成 failed 并写入 error_message,
|
||||
# 前端轮询时能直接看到失败原因,不会无限卡在 tts_processing。
|
||||
logger.exception(
|
||||
"Celery 任务提交失败,TTS 任务已创建但未触发执行: job_id=%s err=%s",
|
||||
job_id,
|
||||
exc_info=True,
|
||||
exc,
|
||||
)
|
||||
job.status = "failed"
|
||||
job.error_message = f"Celery 任务投递失败: {exc}"
|
||||
job.error_code = "AsyncDispatchFailed"
|
||||
job.updated_at = datetime.now(timezone.utc)
|
||||
else:
|
||||
# 2b. 直接音频模式:同步签名并提交 MediaKit
|
||||
video_url = self._sign_media_url(video_url)
|
||||
|
||||
@@ -76,11 +76,21 @@ def tts_synthesize_and_submit(
|
||||
from app.services.mediakit_client import MediaKitError, get_mediakit_client
|
||||
from sqlalchemy.orm import Session as DBSession
|
||||
|
||||
from packages.adapters.sqlalchemy_impl.database import SessionLocal
|
||||
from packages.adapters.sqlalchemy_impl.models import LipsyncJobModel
|
||||
from packages.application.cosyvoice_service import CosyVoiceError, CosyVoiceService
|
||||
from packages.shared.url_security import safe_download_bytes
|
||||
|
||||
# SessionLocal 获取:
|
||||
# - API 容器:app.db.SessionLocal(环境变量完整,导入即建引擎)
|
||||
# - Worker 容器:worker_app.db.SessionLocal(Worker 自己的 settings 初始化引擎)
|
||||
# API 侧没有 worker_app 模块 → ImportError 直接回退;
|
||||
# Worker 侧 app.db 会因缺少 API 专有环境变量抛 pydantic ValidationError,
|
||||
# 此时也要回退到 worker_app.db。
|
||||
try:
|
||||
from worker_app.db import SessionLocal # type: ignore
|
||||
except Exception: # noqa: BLE001
|
||||
from app.db import SessionLocal # type: ignore
|
||||
|
||||
db: DBSession = SessionLocal()
|
||||
try:
|
||||
job = (
|
||||
|
||||
@@ -2,17 +2,37 @@
|
||||
# Select nginx config based on APP_ENV (staging/production).
|
||||
# Both configs are baked into the image at well-known paths.
|
||||
# nginx reads config only at startup, so symlink before exec.
|
||||
#
|
||||
# 兼容两种部署方式:
|
||||
# 1) 镜像自包含模式(local dev / 无外部挂载):基础镜像自带 default.conf 普通文件,
|
||||
# entrypoint 按 APP_ENV 先 rm 再 ln -s 指向烤入的环境配置。
|
||||
# 注意:alpine busybox ln -sf 在 target 为已存在普通文件时行为不稳定,
|
||||
# 必须先 rm 再 ln 才能正确替换。
|
||||
# 2) 外部 bind-mount 模式(staging / production CI 部署):CI 脚本通过
|
||||
# -v /path/nginx-<env>.conf:/etc/nginx/conf.d/default.conf:ro 把环境
|
||||
# 配置以只读方式挂载进来。此时 default.conf 已经是正确的环境配置,
|
||||
# 且为只读挂载点:rm 会报 "Read-only file system" 失败。
|
||||
# entrypoint 应识别此情况并跳过 rm/ln,直接 exec nginx。
|
||||
set -e
|
||||
|
||||
NGINX_CONF_DIR="/etc/nginx/conf.d"
|
||||
DEFAULT_CONF="$NGINX_CONF_DIR/default.conf"
|
||||
|
||||
case "${APP_ENV:-production}" in
|
||||
staging)
|
||||
ln -sf /etc/nginx/nginx-staging.conf "$NGINX_CONF_DIR/default.conf"
|
||||
;;
|
||||
*)
|
||||
ln -sf /etc/nginx/nginx-production.conf "$NGINX_CONF_DIR/default.conf"
|
||||
;;
|
||||
esac
|
||||
# 判断 default.conf 是否可被 rm 替换(镜像自包含普通文件 → 可删;
|
||||
# 外部只读 bind mount → rm 失败)。rm 失败时视为外部已提供正确配置,跳过。
|
||||
if rm -f "$DEFAULT_CONF" 2>/dev/null; then
|
||||
# 镜像自包含:按 APP_ENV 建立正确 symlink
|
||||
case "${APP_ENV:-production}" in
|
||||
staging)
|
||||
ln -s /etc/nginx/nginx-staging.conf "$DEFAULT_CONF"
|
||||
;;
|
||||
*)
|
||||
ln -s /etc/nginx/nginx-production.conf "$DEFAULT_CONF"
|
||||
;;
|
||||
esac
|
||||
else
|
||||
# 外部 bind-mount(如 CI staging/prod 部署):配置已挂好,什么都不做。
|
||||
echo "[nginx-entrypoint] default.conf is externally mounted (read-only), skipping config symlink."
|
||||
fi
|
||||
|
||||
exec nginx -g "daemon off;"
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
#!/bin/bash
|
||||
set +e
|
||||
echo "=== 1. Check base nginx image conf.d ==="
|
||||
docker run --rm --entrypoint sh git.xiaoxiajianji.com/xiaoxia/base/nginx:alpine -c \
|
||||
"ls -la /etc/nginx/conf.d/ 2>&1; echo '---'; file /etc/nginx/conf.d/default.conf 2>&1; echo '---'; ls -la /etc/nginx/nginx*.conf 2>&1"
|
||||
echo ""
|
||||
echo "=== 2. Check staging web image (currently running) ==="
|
||||
WEB_IMG=$(docker inspect xiaoxia-web-staging --format '{{.Config.Image}}' 2>/dev/null)
|
||||
echo "Staging web image: $WEB_IMG"
|
||||
docker run --rm --entrypoint sh "$WEB_IMG" -c \
|
||||
"ls -la /etc/nginx/conf.d/ 2>&1; echo '---'; cat /docker-entrypoint.sh 2>&1; echo '---'; ls -la /etc/nginx/nginx-*.conf 2>&1"
|
||||
echo ""
|
||||
echo "=== 3. Run entrypoint with APP_ENV=staging to reproduce ==="
|
||||
docker run --rm -e APP_ENV=staging --entrypoint sh "$WEB_IMG" -c \
|
||||
"set -x; sh -x /docker-entrypoint.sh 2>&1 | head -30"
|
||||
echo ""
|
||||
echo "=== 4. What if we rm -f default.conf first then ln? ==="
|
||||
docker run --rm -e APP_ENV=staging --entrypoint sh "$WEB_IMG" -c \
|
||||
"rm -f /etc/nginx/conf.d/default.conf && ln -sf /etc/nginx/nginx-staging.conf /etc/nginx/conf.d/default.conf && ls -la /etc/nginx/conf.d/default.conf && echo OK"
|
||||
echo ""
|
||||
echo "=== 5. Check running web logs for crash reason ==="
|
||||
docker logs xiaoxia-web-staging 2>&1 | tail -30
|
||||
echo ""
|
||||
echo "=== Done ==="
|
||||
@@ -0,0 +1 @@
|
||||
# xiaoxia-saas shared packages namespace
|
||||
@@ -0,0 +1 @@
|
||||
# adapter implementations namespace
|
||||
@@ -143,8 +143,8 @@ class TestCreateJobAsyncTTS:
|
||||
assert args[2] == "v-1" # voice_id
|
||||
assert args[3] == "测试文本" # script_text
|
||||
|
||||
def test_tts_mode_celery_dispatch_failure_still_creates_job(self):
|
||||
"""Celery dispatch 失败时,job 记录已创建,状态保持 tts_processing."""
|
||||
def test_tts_mode_celery_dispatch_failure_marks_job_failed(self):
|
||||
"""Celery dispatch 失败时,job 标为 failed 并写入 error_message,前端轮询能直接看到错误."""
|
||||
svc, client, cosy = _make_service_with_mocks()
|
||||
|
||||
with patch("app.services.lipsync_service.tts_synthesize_and_submit") as mock_task:
|
||||
@@ -157,9 +157,11 @@ class TestCreateJobAsyncTTS:
|
||||
script_text="测试文本",
|
||||
)
|
||||
|
||||
# job 已创建
|
||||
# job 已创建且状态标为 failed
|
||||
assert job is not None
|
||||
assert job.status == "tts_processing"
|
||||
assert job.status == "failed"
|
||||
assert "Celery 任务投递失败" in job.error_message
|
||||
assert job.error_code == "AsyncDispatchFailed"
|
||||
# MediaKit 未被调用
|
||||
client.submit_lipsync.assert_not_called()
|
||||
|
||||
|
||||
@@ -73,12 +73,16 @@ def _apply_all_patches(
|
||||
lipsync_tts.run() 在函数体内部懒 import 多个模块,通过 sys.modules 注入
|
||||
伪造包路径避免真实导入;对存在的模块用 patch() 替换返回值/side_effect。
|
||||
"""
|
||||
fake_db_mod = ModuleType("packages.adapters.sqlalchemy_impl.database")
|
||||
# SessionLocal 通过懒探测获取(Worker 用 worker_app.db,API 用 app.db),
|
||||
# 测试环境里两个模块都能被真实导入,必须同时 mock 保证用的是 fake session。
|
||||
fake_app_db = ModuleType("app.db")
|
||||
fake_worker_db = ModuleType("worker_app.db")
|
||||
session, factory = _build_session(job)
|
||||
fake_db_mod.SessionLocal = factory
|
||||
fake_app_db.SessionLocal = factory
|
||||
fake_worker_db.SessionLocal = factory
|
||||
|
||||
patches = [
|
||||
patch.dict(sys.modules, {"packages.adapters.sqlalchemy_impl.database": fake_db_mod}),
|
||||
patch.dict(sys.modules, {"app.db": fake_app_db, "worker_app.db": fake_worker_db}),
|
||||
patch(
|
||||
"app.tasks.lipsync_tts._sign_media_url",
|
||||
side_effect=lambda url: url + "?signed" if url else url,
|
||||
@@ -289,9 +293,11 @@ class TestTtsSynthesizeAndSubmit:
|
||||
from app.tasks.lipsync_tts import tts_synthesize_and_submit
|
||||
|
||||
job = _make_fake_job()
|
||||
fake_db_mod = ModuleType("packages.adapters.sqlalchemy_impl.database")
|
||||
fake_app_db = ModuleType("app.db")
|
||||
fake_worker_db = ModuleType("worker_app.db")
|
||||
session, factory = _build_session(job)
|
||||
fake_db_mod.SessionLocal = factory
|
||||
fake_app_db.SessionLocal = factory
|
||||
fake_worker_db.SessionLocal = factory
|
||||
|
||||
# CosyVoiceService 在 __init__ 抛 RuntimeError(非 CosyVoiceError/ValueError)
|
||||
fake_cosy_mod = ModuleType("packages.application.cosyvoice_service")
|
||||
@@ -309,7 +315,8 @@ class TestTtsSynthesizeAndSubmit:
|
||||
with patch.dict(
|
||||
sys.modules,
|
||||
{
|
||||
"packages.adapters.sqlalchemy_impl.database": fake_db_mod,
|
||||
"app.db": fake_app_db,
|
||||
"worker_app.db": fake_worker_db,
|
||||
"packages.application.cosyvoice_service": fake_cosy_mod,
|
||||
},
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user