Compare commits

...

6 Commits

Author SHA1 Message Date
xiaoxia 4fcedfd5a9 debug: fix YAML, use base64 script transfer
Debug: Web container v2 (mount conflict) / web-diag (push) Successful in 22s
2026-09-11 11:09:13 +08:00
xiaoxia cc335673a9 debug: add ssh diag on debug branch 2026-09-11 11:06:16 +08:00
xiaoxia 029267cb46 debug: SSH diagnose web container v2 2026-09-11 10:55:04 +08:00
xiaoxia 9c0d4b136f chore: trigger redeploy after nginx entrypoint fix
CI/CD Pipeline / Check if frontend-only change (push) Has been skipped
CI/CD Pipeline / PR Build API Image (push) Has been skipped
CI/CD Pipeline / PR Build Web Image (push) Has been skipped
CI/CD Pipeline / PR Build Worker Image (push) Has been skipped
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (push) Successful in 1s
CI/CD Pipeline / Check push changed paths (push) Successful in 4s
CI/CD Pipeline / Frontend Lint (push) Has been skipped
CI/CD Pipeline / Build Staging Web Image (push) Successful in 17s
CI/CD Pipeline / Build Staging Worker Image (push) Successful in 21s
CI/CD Pipeline / Build Staging API Image (push) Successful in 58s
CI/CD Pipeline / Retag skipped Staging API Image (push) Has been skipped
CI/CD Pipeline / Retag skipped Staging Web Image (push) Has been skipped
CI/CD Pipeline / Retag skipped Staging Worker Image (push) Has been skipped
CI/CD Pipeline / Integration Tests (push) Successful in 1m27s
CI/CD Pipeline / Validate - Python (mypy + alembic) (push) Successful in 1m45s
CI/CD Pipeline / Validate - Style (push) Successful in 1m59s
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (push) Failing after 1m58s
CI/CD Pipeline / Staging E2E Tests (push) Has been skipped
CI/CD Pipeline / Staging API Integration Tests (push) Has been skipped
CI/CD Pipeline / ACR Image Cleanup (push) Has been skipped
CI/CD Pipeline / Frontend Unit Tests (push) Successful in 3m34s
CI/CD Pipeline / Validate - Security (push) Successful in 6m3s
CI/CD Pipeline / Unit Tests (push) Successful in 6m22s
CI/CD Pipeline / Build Production Web Image (push) Has been skipped
CI/CD Pipeline / Build Production API Image (push) Has been skipped
CI/CD Pipeline / CI Gate (push) Has been skipped
CI/CD Pipeline / Build Production Worker Image (push) Has been skipped
CI/CD Pipeline / Deploy Production (push) Has been skipped
CI/CD Pipeline / Canary Release to Production (push) Has been skipped
CI/CD Pipeline / Production Browser E2E (push) Has been skipped
2026-09-11 10:49:25 +08:00
xiaoxia 387514c111 fix(web, P0): nginx entrypoint ln -sf 替换已有文件失败导致web容器无限重启,阻塞staging部署 (#1855)
CI/CD Pipeline / Check if frontend-only change (push) Has been skipped
CI/CD Pipeline / PR Build API Image (push) Has been skipped
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (push) Successful in 1s
CI/CD Pipeline / PR Build Web Image (push) Has been skipped
CI/CD Pipeline / PR Build Worker Image (push) Has been skipped
CI/CD Pipeline / Check push changed paths (push) Successful in 4s
CI/CD Pipeline / Frontend Lint (push) Has been skipped
CI/CD Pipeline / Build Staging Worker Image (push) Successful in 24s
CI/CD Pipeline / Build Staging API Image (push) Successful in 36s
CI/CD Pipeline / Build Staging Web Image (push) Successful in 1m12s
CI/CD Pipeline / Retag skipped Staging API Image (push) Has been skipped
CI/CD Pipeline / Retag skipped Staging Web Image (push) Has been skipped
CI/CD Pipeline / Retag skipped Staging Worker Image (push) Has been skipped
CI/CD Pipeline / Validate - Python (mypy + alembic) (push) Successful in 1m54s
CI/CD Pipeline / Integration Tests (push) Successful in 1m54s
CI/CD Pipeline / Validate - Style (push) Successful in 2m32s
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (push) Failing after 2m9s
CI/CD Pipeline / Staging E2E Tests (push) Has been skipped
CI/CD Pipeline / ACR Image Cleanup (push) Has been skipped
CI/CD Pipeline / Staging API Integration Tests (push) Has been skipped
CI/CD Pipeline / Frontend Unit Tests (push) Successful in 4m33s
CI/CD Pipeline / Validate - Security (push) Successful in 6m46s
CI/CD Pipeline / Unit Tests (push) Successful in 7m32s
CI/CD Pipeline / Build Production API Image (push) Has been skipped
CI/CD Pipeline / Build Production Web Image (push) Has been skipped
CI/CD Pipeline / Build Production Worker Image (push) Has been skipped
CI/CD Pipeline / Deploy Production (push) Has been skipped
CI/CD Pipeline / CI Gate (push) Has been skipped
CI/CD Pipeline / Production Browser E2E (push) Has been skipped
CI/CD Pipeline / Canary Release to Production (push) Has been skipped
Co-authored-by: xiaoxia <dev@xiaoxiajianji.com>
Co-committed-by: xiaoxia <dev@xiaoxiajianji.com>
2026-09-11 10:35:21 +08:00
xiaoxia 76cdb15c6b fix(lipsync, P0): Worker运行时ModuleNotFoundError导致任务卡在tts_processing (#1854)
CI/CD Pipeline / Check if frontend-only change (push) Has been skipped
CI/CD Pipeline / PR Build API Image (push) Has been skipped
CI/CD Pipeline / PR Build Web Image (push) Has been skipped
CI/CD Pipeline / PR Build Worker Image (push) Has been skipped
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (push) Successful in 1s
CI/CD Pipeline / Check push changed paths (push) Successful in 4s
CI/CD Pipeline / Frontend Lint (push) Has been skipped
CI/CD Pipeline / Build Staging Worker Image (push) Successful in 26s
CI/CD Pipeline / Build Staging API Image (push) Successful in 28s
CI/CD Pipeline / Build Staging Web Image (push) Successful in 1m27s
CI/CD Pipeline / Retag skipped Staging API Image (push) Has been skipped
CI/CD Pipeline / Retag skipped Staging Web Image (push) Has been skipped
CI/CD Pipeline / Retag skipped Staging Worker Image (push) Has been skipped
CI/CD Pipeline / Validate - Python (mypy + alembic) (push) Successful in 1m47s
CI/CD Pipeline / Integration Tests (push) Successful in 1m58s
CI/CD Pipeline / Validate - Style (push) Successful in 2m32s
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (push) Failing after 2m0s
CI/CD Pipeline / Staging E2E Tests (push) Has been skipped
CI/CD Pipeline / Staging API Integration Tests (push) Has been skipped
CI/CD Pipeline / ACR Image Cleanup (push) Has been skipped
CI/CD Pipeline / Frontend Unit Tests (push) Successful in 4m47s
CI/CD Pipeline / Validate - Security (push) Successful in 5m40s
CI/CD Pipeline / Unit Tests (push) Successful in 7m43s
CI/CD Pipeline / Build Production API Image (push) Has been skipped
CI/CD Pipeline / Build Production Web Image (push) Has been skipped
CI/CD Pipeline / Build Production Worker Image (push) Has been skipped
CI/CD Pipeline / CI Gate (push) Has been skipped
CI/CD Pipeline / Deploy Production (push) Has been skipped
CI/CD Pipeline / Production Browser E2E (push) Has been skipped
CI/CD Pipeline / Canary Release to Production (push) Has been skipped
Co-authored-by: xiaoxia <dev@xiaoxiajianji.com>
Co-committed-by: xiaoxia <dev@xiaoxiajianji.com>
2026-09-11 09:52:05 +08:00
8 changed files with 79 additions and 17 deletions
+30
View File
@@ -0,0 +1,30 @@
name: "Debug: Web container v2 (mount conflict)"
on:
push:
branches: [debug/web-crash-v2]
workflow_dispatch:
jobs:
web-diag:
runs-on: runtime-builder
timeout-minutes: 10
steps:
- name: Setup SSH
shell: bash
env:
STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
run: |
set -eux
which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
mkdir -p ~/.ssh && chmod 700 ~/.ssh
printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
H=47.98.113.167; P=22222
ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null
- name: Run diag via base64
shell: bash
run: |
set -x
echo 'IyEvYmluL2Jhc2gKc2V0IC14CmVjaG8gIj09PSBDdXJyZW50IHN0YWdpbmcgY29udGFpbmVycyA9PT0iCmRvY2tlciBwcyAtYSAtLWZpbHRlciBuYW1lPXhpYW94aWEtKi1zdGFnaW5nIC0tZm9ybWF0ICJ0YWJsZSB7ey5OYW1lc319XHR7ey5TdGF0dXN9fVx0e3suSW1hZ2V9fSIKZWNobyAiIgplY2hvICI9PT0gV2ViIGNvbnRhaW5lciBsb2dzIChjdXJyZW50L3JvbGxlZC1iYWNrKSA9PT0iCmRvY2tlciBsb2dzIHhpYW94aWEtd2ViLXN0YWdpbmcgMj4mMSB8IHRhaWwgLTQwCmVjaG8gIiIKZWNobyAiPT09IFdlYiBpbnNwZWN0OiBlbnYgJiBtb3VudHMgPT09Igpkb2NrZXIgaW5zcGVjdCB4aWFveGlhLXdlYi1zdGFnaW5nIC0tZm9ybWF0ICdFbnRyeXBvaW50OiB7ey5Db25maWcuRW50cnlwb2ludH19IENtZDoge3suQ29uZmlnLkNtZH19Jwpkb2NrZXIgaW5zcGVjdCB4aWFveGlhLXdlYi1zdGFnaW5nIC0tZm9ybWF0ICd7e3JhbmdlIC5Db25maWcuRW52fX17ey59fXt7IlxuIn19e3tlbmR9fScgfCBncmVwIC1FICJBUFBfRU5WfFZFUlNJT04iCmVjaG8gIk1vdW50czoiCmRvY2tlciBpbnNwZWN0IHhpYW94aWEtd2ViLXN0YWdpbmcgLS1mb3JtYXQgJ3t7cmFuZ2UgLk1vdW50c319e3suVHlwZX19IHt7LlNvdXJjZX19IC0+IHt7LkRlc3RpbmF0aW9ufX0gKHJ3PXt7LlJXfX0pe3siXG4ifX17e2VuZH19JwplY2hvICIiCmVjaG8gIj09PSBSZXByb2R1Y2U6IHJtIG9uIHJlYWQtb25seSBiaW5kIG1vdW50IChpbWFnZSAzODc1MTRjKSA9PT0iCmRvY2tlciBydW4gLS1ybSAtLW5hbWUgbmdpbngtcm8tdGVzdCBcCiAgLXYgL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvbmdpbngtc3RhZ2luZy5jb25mOi9ldGMvbmdpbngvY29uZi5kL2RlZmF1bHQuY29uZjpybyBcCiAgZ2l0LnhpYW94aWFqaWFuamkuY29tL3hpYW94aWEveGlhb3hpYS1zYWFzL3hpYW94aWEtc2Fhcy13ZWI6Mzg3NTE0YyBcCiAgL2Jpbi9zaCAtYyAnc2V0IC14OyBlY2hvIEJFRk9SRTsgbHMgLWxhIC9ldGMvbmdpbngvY29uZi5kLzsgcm0gLWYgL2V0Yy9uZ2lueC9jb25mLmQvZGVmYXVsdC5jb25mOyBlY2hvICJybSBleGl0PSQ/IjsgZWNobyBBRlRFUjsgbHMgLWxhIC9ldGMvbmdpbngvY29uZi5kLycgMj4mMQplY2hvICIiCmVjaG8gIj09PSBMb2NhbCB3ZWIgaW1hZ2VzIGF2YWlsYWJsZSA9PT0iCmRvY2tlciBpbWFnZXMgfCBncmVwIHhpYW94aWEtc2Fhcy13ZWIgfCBoZWFkIC01Cg==' | base64 -d > /tmp/diag.sh
chmod +x /tmp/diag.sh
cat /tmp/diag.sh
ssh -p 22222 -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@47.98.113.167 'bash -s' < /tmp/diag.sh
+10 -4
View File
@@ -211,12 +211,18 @@ class LipsyncService:
normalize_emotion(emotion),
)
)
except Exception:
logger.warning(
"Celery 任务提交失败,TTS 任务已创建但未触发执行: %s",
except Exception as exc:
# 投递失败时立即把 job 标成 failed 并写入 error_message
# 前端轮询时能直接看到失败原因,不会无限卡在 tts_processing。
logger.exception(
"Celery 任务提交失败,TTS 任务已创建但未触发执行: job_id=%s err=%s",
job_id,
exc_info=True,
exc,
)
job.status = "failed"
job.error_message = f"Celery 任务投递失败: {exc}"
job.error_code = "AsyncDispatchFailed"
job.updated_at = datetime.now(timezone.utc)
else:
# 2b. 直接音频模式:同步签名并提交 MediaKit
video_url = self._sign_media_url(video_url)
+11 -1
View File
@@ -76,11 +76,21 @@ def tts_synthesize_and_submit(
from app.services.mediakit_client import MediaKitError, get_mediakit_client
from sqlalchemy.orm import Session as DBSession
from packages.adapters.sqlalchemy_impl.database import SessionLocal
from packages.adapters.sqlalchemy_impl.models import LipsyncJobModel
from packages.application.cosyvoice_service import CosyVoiceError, CosyVoiceService
from packages.shared.url_security import safe_download_bytes
# SessionLocal 获取:
# - API 容器:app.db.SessionLocal(环境变量完整,导入即建引擎)
# - Worker 容器:worker_app.db.SessionLocalWorker 自己的 settings 初始化引擎)
# API 侧没有 worker_app 模块 → ImportError 直接回退;
# Worker 侧 app.db 会因缺少 API 专有环境变量抛 pydantic ValidationError
# 此时也要回退到 worker_app.db。
try:
from worker_app.db import SessionLocal # type: ignore
except Exception: # noqa: BLE001
from app.db import SessionLocal # type: ignore
db: DBSession = SessionLocal()
try:
job = (
+7 -2
View File
@@ -2,16 +2,21 @@
# Select nginx config based on APP_ENV (staging/production).
# Both configs are baked into the image at well-known paths.
# nginx reads config only at startup, so symlink before exec.
# 注意:基础镜像 /etc/nginx/conf.d/default.conf 是普通文件(非 symlink/目录),
# alpine busybox ln -sf 在 target 已存在且为普通文件时行为不稳定(会尝试在
# target 目录下建子链接),必须先 rm 再 ln 才能正确替换。
set -e
NGINX_CONF_DIR="/etc/nginx/conf.d"
case "${APP_ENV:-production}" in
staging)
ln -sf /etc/nginx/nginx-staging.conf "$NGINX_CONF_DIR/default.conf"
rm -f "$NGINX_CONF_DIR/default.conf"
ln -s /etc/nginx/nginx-staging.conf "$NGINX_CONF_DIR/default.conf"
;;
*)
ln -sf /etc/nginx/nginx-production.conf "$NGINX_CONF_DIR/default.conf"
rm -f "$NGINX_CONF_DIR/default.conf"
ln -s /etc/nginx/nginx-production.conf "$NGINX_CONF_DIR/default.conf"
;;
esac
+1
View File
@@ -0,0 +1 @@
# xiaoxia-saas shared packages namespace
+1
View File
@@ -0,0 +1 @@
# adapter implementations namespace
@@ -143,8 +143,8 @@ class TestCreateJobAsyncTTS:
assert args[2] == "v-1" # voice_id
assert args[3] == "测试文本" # script_text
def test_tts_mode_celery_dispatch_failure_still_creates_job(self):
"""Celery dispatch 失败时,job 记录已创建,状态保持 tts_processing."""
def test_tts_mode_celery_dispatch_failure_marks_job_failed(self):
"""Celery dispatch 失败时,job 标为 failed 并写入 error_message,前端轮询能直接看到错误."""
svc, client, cosy = _make_service_with_mocks()
with patch("app.services.lipsync_service.tts_synthesize_and_submit") as mock_task:
@@ -157,9 +157,11 @@ class TestCreateJobAsyncTTS:
script_text="测试文本",
)
# job 已创建
# job 已创建且状态标为 failed
assert job is not None
assert job.status == "tts_processing"
assert job.status == "failed"
assert "Celery 任务投递失败" in job.error_message
assert job.error_code == "AsyncDispatchFailed"
# MediaKit 未被调用
client.submit_lipsync.assert_not_called()
+13 -6
View File
@@ -73,12 +73,16 @@ def _apply_all_patches(
lipsync_tts.run() 在函数体内部懒 import 多个模块,通过 sys.modules 注入
伪造包路径避免真实导入;对存在的模块用 patch() 替换返回值/side_effect。
"""
fake_db_mod = ModuleType("packages.adapters.sqlalchemy_impl.database")
# SessionLocal 通过懒探测获取(Worker 用 worker_app.dbAPI 用 app.db),
# 测试环境里两个模块都能被真实导入,必须同时 mock 保证用的是 fake session。
fake_app_db = ModuleType("app.db")
fake_worker_db = ModuleType("worker_app.db")
session, factory = _build_session(job)
fake_db_mod.SessionLocal = factory
fake_app_db.SessionLocal = factory
fake_worker_db.SessionLocal = factory
patches = [
patch.dict(sys.modules, {"packages.adapters.sqlalchemy_impl.database": fake_db_mod}),
patch.dict(sys.modules, {"app.db": fake_app_db, "worker_app.db": fake_worker_db}),
patch(
"app.tasks.lipsync_tts._sign_media_url",
side_effect=lambda url: url + "?signed" if url else url,
@@ -289,9 +293,11 @@ class TestTtsSynthesizeAndSubmit:
from app.tasks.lipsync_tts import tts_synthesize_and_submit
job = _make_fake_job()
fake_db_mod = ModuleType("packages.adapters.sqlalchemy_impl.database")
fake_app_db = ModuleType("app.db")
fake_worker_db = ModuleType("worker_app.db")
session, factory = _build_session(job)
fake_db_mod.SessionLocal = factory
fake_app_db.SessionLocal = factory
fake_worker_db.SessionLocal = factory
# CosyVoiceService 在 __init__ 抛 RuntimeError(非 CosyVoiceError/ValueError
fake_cosy_mod = ModuleType("packages.application.cosyvoice_service")
@@ -309,7 +315,8 @@ class TestTtsSynthesizeAndSubmit:
with patch.dict(
sys.modules,
{
"packages.adapters.sqlalchemy_impl.database": fake_db_mod,
"app.db": fake_app_db,
"worker_app.db": fake_worker_db,
"packages.application.cosyvoice_service": fake_cosy_mod,
},
):