fix(backend): #1867 对口型卡running三层防御修复(rebase到最新develop) (#1916)
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (push) Successful in 2s
CI/CD Pipeline / Check push changed paths (push) Successful in 12s
CI/CD Pipeline / Build Staging API Image (push) Successful in 3m25s
CI/CD Pipeline / Integration Tests (push) Successful in 4m20s
CI/CD Pipeline / Build Staging Worker Image (push) Successful in 4m17s
CI/CD Pipeline / Validate - Python (mypy + alembic) (push) Successful in 4m34s
CI/CD Pipeline / Build Staging Web Image (push) Successful in 4m28s
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (push) Successful in 1m2s
CI/CD Pipeline / Validate - Style (push) Successful in 6m31s
CI/CD Pipeline / Frontend Unit Tests (push) Successful in 7m13s
CI/CD Pipeline / ACR Image Cleanup (push) Successful in 1m51s
CI/CD Pipeline / Staging API Integration Tests (push) Successful in 3m20s
CI/CD Pipeline / Staging E2E Tests (push) Failing after 3m23s
CI/CD Pipeline / Validate - Security (push) Successful in 10m16s
CI/CD Pipeline / Unit Tests (push) Successful in 10m35s
CI/CD Pipeline / Production Browser E2E (push) Has been skipped
CI/CD Pipeline / CI Gate (push) Failing after 35h37m35s
CI/CD Pipeline / Build Production API Image (push) Failing after 35h37m35s
CI/CD Pipeline / PR Build API Image (push) Failing after 35h48m10s
CI/CD Pipeline / Build Production Worker Image (push) Failing after 35h37m2s
CI/CD Pipeline / PR Build Worker Image (push) Failing after 35h47m37s
CI/CD Pipeline / PR Build Web Image (push) Failing after 35h47m37s
CI/CD Pipeline / Retag skipped Staging Worker Image (push) Failing after 35h42m57s
CI/CD Pipeline / Canary Release to Production (push) Failing after 35h36m58s
CI/CD Pipeline / Build Production Web Image (push) Failing after 35h37m2s
CI/CD Pipeline / Frontend Lint (push) Failing after 35h47m37s
CI/CD Pipeline / Deploy Production (push) Failing after 35h36m58s
CI/CD Pipeline / Retag skipped Staging Web Image (push) Failing after 35h42m57s
CI/CD Pipeline / Check if frontend-only change (push) Failing after 35h47m42s
CI/CD Pipeline / Retag skipped Staging API Image (push) Failing after 36h18m28s

Co-authored-by: xiaoxia <dev@xiaoxiajianji.com>
Co-committed-by: xiaoxia <dev@xiaoxiajianji.com>
This commit was merged in pull request #1916.
This commit is contained in:
2026-09-15 03:35:30 +08:00
committed by auto-approve-bot
parent 5d4e07d4f4
commit 695a491c5d
5 changed files with 633 additions and 43 deletions
+17 -1
View File
@@ -193,7 +193,23 @@ def get_lipsync_job(
raise HTTPException(status_code=404, detail="任务不存在")
if job.status not in ("completed", "failed"):
background.add_task(svc.refresh_job_status, job_id, current_user.user.id)
# 三层防御 ①:如果距上次更新超过 30 秒,同步刷新一次(避免 background task
# 静默失败导致前端永远看到 running);否则挂后台异步刷新(避免阻塞轮询)。
from datetime import datetime as _dt
from datetime import timezone as _tz
_now = _dt.now(_tz.utc)
_stale = job.updated_at is None or (_now - job.updated_at).total_seconds() > 30
if _stale:
try:
refreshed = svc.refresh_job_status(job_id, current_user.user.id)
if refreshed is not None:
job = refreshed
except Exception as exc: # noqa: BLE001
logger.error("同步刷新对口型状态失败 job_id=%s err=%s", job_id, exc, exc_info=True)
background.add_task(svc.refresh_job_status, job_id, current_user.user.id)
else:
background.add_task(svc.refresh_job_status, job_id, current_user.user.id)
return job
+43 -30
View File
@@ -489,39 +489,52 @@ class LipsyncService:
mk_status = status_data.get("status", STATUS_RUNNING)
logger.info("MediaKit 对口型状态 [%s]: %s", job_id, mk_status)
if mk_status == STATUS_COMPLETED:
result = status_data.get("result", {})
job.status = STATUS_COMPLETED
temp_url = result.get("video_url", "")
job.output_video_url = temp_url
job.output_duration = result.get("duration", 0.0)
job.completed_at = datetime.now(timezone.utc)
try:
if mk_status == STATUS_COMPLETED:
result = status_data.get("result", {})
job.status = STATUS_COMPLETED
temp_url = result.get("video_url", "")
job.output_video_url = temp_url
job.output_duration = result.get("duration", 0.0)
job.completed_at = datetime.now(timezone.utc)
job.updated_at = datetime.now(timezone.utc)
self.db.commit()
# 异步转存自家 OSS
try:
from app.tasks.lipsync_tts import persist_output_video_task
persist_output_video_task.apply_async(args=(job_id, user_id, temp_url))
except Exception as exc:
logger.warning(
"提交输出视频异步转存任务失败,保留临时 URL: job_id=%s err=%s",
job_id,
exc,
)
elif mk_status == STATUS_FAILED:
error = status_data.get("error", {})
job.status = "failed"
job.error_message = error.get("message", "任务执行失败")
job.error_code = error.get("code", "TaskFailed")
job.completed_at = datetime.now(timezone.utc)
else:
# 中间状态(running/processing/queued 等)同步到 DB,避免前端永远卡在 submitted
if isinstance(mk_status, str) and mk_status:
job.status = mk_status
job.updated_at = datetime.now(timezone.utc)
self.db.commit()
# 异步转存自家 OSS
except Exception as exc: # noqa: BLE001 - DB 提交失败必须记录日志并重试,否则后台任务静默失败
logger.error(
"refresh_job_status 提交 DB 失败 job_id=%s mk_status=%s err=%s",
job_id,
mk_status,
exc,
exc_info=True,
)
try:
from app.tasks.lipsync_tts import persist_output_video_task
persist_output_video_task.apply_async(args=(job_id, user_id, temp_url))
except Exception as exc:
logger.warning(
"提交输出视频异步转存任务失败,保留临时 URL: job_id=%s err=%s",
job_id,
exc,
)
self.db.refresh(job)
return job
elif mk_status == STATUS_FAILED:
error = status_data.get("error", {})
job.status = "failed"
job.error_message = error.get("message", "任务执行失败")
job.error_code = error.get("code", "TaskFailed")
job.completed_at = datetime.now(timezone.utc)
else:
if isinstance(mk_status, str) and mk_status:
job.status = mk_status
job.updated_at = datetime.now(timezone.utc)
self.db.commit()
self.db.rollback()
except Exception:
pass
# DB commit 失败不 raise,返回当前 job 对象让下次轮询再试
self.db.refresh(job)
return job
+105
View File
@@ -282,6 +282,18 @@ def tts_synthesize_and_submit(
job.error_code = exc.code
logger.error("[lipsync_tts] 提交 MediaKit 失败: job_id=%s err=%s", job_id, exc)
# 三层防御 ③:链式触发 Celery 兜底轮询——MediaKit 提交成功后由 worker
# 主动拉取状态到终态,不依赖前端轮询触发的 FastAPI background task
# (background task 可能静默失败导致永久卡 running)。
if job.status == "submitted" and job.mediakit_task_id:
try:
poll_mediakit_status.apply_async(
kwargs={"job_id": job_id, "user_id": user_id},
countdown=10, # 10 秒后开始轮询,给 MediaKit 一点处理时间
)
except Exception as exc: # noqa: BLE001
logger.warning("[lipsync_tts] 提交兜底轮询任务失败(不影响主流程): job_id=%s err=%s", job_id, exc)
db.commit()
except Exception:
@@ -300,6 +312,99 @@ def tts_synthesize_and_submit(
db.close()
@shared_task(
bind=True,
name="lipsync_tts.poll_mediakit_status",
max_retries=60, # 最多轮询 60 次
default_retry_delay=10, # 每次间隔 10 秒(总兜底时长 10 分钟)
)
def poll_mediakit_status(self, job_id: str, user_id: str):
"""Celery 兜底轮询:TTS 提交 MediaKit 后,由 worker 主动拉取状态直到终态。
不依赖前端轮询,避免 background task 静默失败导致任务永久卡 running/submitted。
"""
from sqlalchemy.orm import Session as DBSession
from packages.adapters.sqlalchemy_impl.models import LipsyncJobModel
try:
from worker_app.db import SessionLocal # type: ignore
except Exception: # noqa: BLE001
from app.db import SessionLocal # type: ignore
db: DBSession = SessionLocal()
try:
job = db.query(LipsyncJobModel).filter(LipsyncJobModel.id == job_id, LipsyncJobModel.user_id == user_id).first()
if job is None:
logger.warning("[lipsync_poll] Job not found: job_id=%s", job_id)
return
# 已终态,不需要再轮询
if job.status in ("completed", "failed", "cancelled"):
return
if not job.mediakit_task_id:
logger.warning("[lipsync_poll] Job has no mediakit_task_id: job_id=%s status=%s", job_id, job.status)
return
from app.services.lipsync_service import STATUS_COMPLETED as _SC
from app.services.lipsync_service import STATUS_FAILED as _SF
from app.services.lipsync_service import LipsyncService
from app.services.mediakit_client import MediaKitError, get_mediakit_client
client = get_mediakit_client()
try:
status_data = client.get_task_status(job.mediakit_task_id)
except MediaKitError as exc:
logger.warning("[lipsync_poll] 拉取 MediaKit 状态失败,将重试: job_id=%s err=%s", job_id, exc)
raise self.retry(exc=exc) from exc
mk_status = status_data.get("status", "running")
if mk_status in ("succeeded", _SC):
svc = LipsyncService(db)
result = status_data.get("result", {})
job.status = "completed"
output_url = result.get("video_url", "")
try:
job.output_video_url = svc._persist_output_video(output_url, job_id, user_id)
except Exception as exc: # noqa: BLE001
logger.warning("[lipsync_poll] 转存 OSS 失败,保留临时 URL: job_id=%s err=%s", job_id, exc)
job.output_video_url = output_url
job.output_duration = result.get("duration", 0.0)
job.completed_at = datetime.now(timezone.utc)
job.updated_at = datetime.now(timezone.utc)
db.commit()
logger.info("[lipsync_poll] 任务完成: job_id=%s", job_id)
elif mk_status in ("failed", "error", _SF):
error = status_data.get("error", {})
job.status = "failed"
job.error_message = error.get("message", "任务执行失败")
job.error_code = error.get("code", "TaskFailed")
job.completed_at = datetime.now(timezone.utc)
job.updated_at = datetime.now(timezone.utc)
db.commit()
logger.info("[lipsync_poll] 任务失败: job_id=%s err=%s", job_id, job.error_message)
else:
# 中间状态,更新时间戳,继续重试
job.updated_at = datetime.now(timezone.utc)
if isinstance(mk_status, str) and mk_status:
job.status = mk_status
db.commit()
logger.debug("[lipsync_poll] 任务仍在 %s,继续轮询: job_id=%s", mk_status, job_id)
raise self.retry()
except Exception as exc:
logger.exception("[lipsync_poll] 未预期异常: job_id=%s", job_id)
try:
db.rollback()
except Exception:
pass
raise self.retry(exc=exc) from exc
finally:
db.close()
@shared_task(
name="lipsync_tts.persist_output_video",
max_retries=2,
+120 -12
View File
@@ -227,9 +227,7 @@ class TestLipsyncServiceUnit:
mock_repo = MagicMock()
mock_repo.get.return_value = None # 预置音色,原样返回 voice_id
with patch(
"app.services.lipsync_service.tts_synthesize_and_submit"
) as mock_task:
with patch("app.services.lipsync_service.tts_synthesize_and_submit") as mock_task:
mock_task.apply_async.return_value = MagicMock(id="celery-task-123")
svc = LipsyncService(
@@ -275,9 +273,7 @@ class TestLipsyncServiceUnit:
voice_clone_repo=mock_repo,
)
with patch(
"app.services.lipsync_service.tts_synthesize_and_submit"
) as mock_task:
with patch("app.services.lipsync_service.tts_synthesize_and_submit") as mock_task:
mock_task.apply_async.return_value = MagicMock(id="celery-task-456")
job = svc.create_job(
@@ -444,9 +440,7 @@ class TestLipsyncServiceUnit:
mock_repo = MagicMock()
mock_repo.get.return_value = None
with patch(
"app.services.lipsync_service.tts_synthesize_and_submit"
) as mock_task:
with patch("app.services.lipsync_service.tts_synthesize_and_submit") as mock_task:
mock_task.apply_async.return_value = MagicMock(id="celery-task-789")
svc = LipsyncService(
@@ -568,9 +562,7 @@ class TestErrorHandling:
voice_clone_repo=mock_repo,
)
with patch(
"app.services.lipsync_service.tts_synthesize_and_submit"
) as mock_task:
with patch("app.services.lipsync_service.tts_synthesize_and_submit") as mock_task:
mock_task.apply_async.return_value = MagicMock(id="celery-task-789")
# TTS 模式下 create_job 不再同步失败
@@ -718,3 +710,119 @@ class TestSignMediaUrl403Fix:
kw = mock_mediakit.submit_lipsync.call_args.kwargs
assert kw["video_url"] == signed_video
assert kw["audio_url"] == signed_audio
class TestLipsyncRouteStaleRefresh:
"""#1867 三层防御 ①:路由层 stale>30s 同步刷新 / 新鲜走后台 / 异常 fallback 单测.
直接调用路由函数 get_lipsync_job(不经过 TestClient/HTTP 栈),mock 掉
current_user、svc、background 三个依赖,覆盖 stale/sync/exception/terminal/404 五个分支。
"""
def _call(self, job, svc, background=None):
from app.api.routes.lipsync import get_lipsync_job
user = MagicMock()
user.user.id = "user-1"
bg = background if background is not None else MagicMock()
return get_lipsync_job(job_id="job-1", background=bg, current_user=user, svc=svc)
def test_stale_job_triggers_sync_refresh(self, mock_mediakit, mock_cosyvoice):
"""updated_at 超过 30 秒 → 同步调用 svc.refresh_job_status,不走 background."""
from datetime import datetime, timedelta, timezone
from app.services.lipsync_service import LipsyncService
mock_job = _make_mock_job(status="submitted")
mock_job.updated_at = datetime.now(timezone.utc) - timedelta(seconds=45)
refreshed_job = _make_mock_job(status="completed", output_video_url="https://cdn/out.mp4", output_duration=12.0)
mock_db = MagicMock()
mock_query = MagicMock()
mock_filter = MagicMock()
mock_filter.first.return_value = mock_job
mock_query.filter.return_value = mock_filter
mock_db.query.return_value = mock_query
svc = LipsyncService(mock_db, client=mock_mediakit, cosyvoice_service=mock_cosyvoice)
svc.get_job = MagicMock(return_value=mock_job)
svc.refresh_job_status = MagicMock(return_value=refreshed_job)
bg = MagicMock()
result = self._call(mock_job, svc, bg)
svc.refresh_job_status.assert_called_once_with("job-1", "user-1")
bg.add_task.assert_not_called()
assert result is refreshed_job
def test_fresh_job_uses_background_refresh(self, mock_mediakit, mock_cosyvoice):
"""updated_at 在 30 秒内 → 直接挂后台刷新,不阻塞."""
from datetime import datetime, timezone
from app.services.lipsync_service import LipsyncService
mock_job = _make_mock_job(status="submitted")
mock_job.updated_at = datetime.now(timezone.utc)
mock_db = MagicMock()
svc = LipsyncService(mock_db, client=mock_mediakit, cosyvoice_service=mock_cosyvoice)
svc.get_job = MagicMock(return_value=mock_job)
svc.refresh_job_status = MagicMock()
bg = MagicMock()
result = self._call(mock_job, svc, bg)
svc.refresh_job_status.assert_not_called()
bg.add_task.assert_called_once()
args, _ = bg.add_task.call_args
assert args[0] == svc.refresh_job_status
assert args[1:] == ("job-1", "user-1")
assert result is mock_job
def test_completed_job_skips_refresh(self, mock_mediakit, mock_cosyvoice):
"""终态 completed → 不刷新,直接返回."""
from app.services.lipsync_service import LipsyncService
mock_job = _make_mock_job(status="completed", output_video_url="https://cdn/out.mp4", output_duration=12.0)
mock_db = MagicMock()
svc = LipsyncService(mock_db, client=mock_mediakit, cosyvoice_service=mock_cosyvoice)
svc.get_job = MagicMock(return_value=mock_job)
svc.refresh_job_status = MagicMock()
bg = MagicMock()
result = self._call(mock_job, svc, bg)
svc.refresh_job_status.assert_not_called()
bg.add_task.assert_not_called()
assert result is mock_job
def test_sync_refresh_exception_falls_back_to_background(self, mock_mediakit, mock_cosyvoice):
"""同步刷新抛异常 → 捕获后 fallback 到 background.add_task,接口仍正常返回 job."""
from datetime import datetime, timedelta, timezone
from app.services.lipsync_service import LipsyncService
mock_job = _make_mock_job(status="submitted")
mock_job.updated_at = datetime.now(timezone.utc) - timedelta(seconds=60)
mock_db = MagicMock()
svc = LipsyncService(mock_db, client=mock_mediakit, cosyvoice_service=mock_cosyvoice)
svc.get_job = MagicMock(return_value=mock_job)
svc.refresh_job_status = MagicMock(side_effect=ConnectionError("db down"))
bg = MagicMock()
result = self._call(mock_job, svc, bg)
svc.refresh_job_status.assert_called_once_with("job-1", "user-1")
bg.add_task.assert_called_once()
args, _ = bg.add_task.call_args
assert args[0] == svc.refresh_job_status
assert args[1:] == ("job-1", "user-1")
assert result is mock_job
def test_job_not_found_raises_404(self, mock_mediakit, mock_cosyvoice):
"""svc.get_job 返回 None → 抛 HTTPException(404)."""
import pytest
from app.services.lipsync_service import LipsyncService
from fastapi import HTTPException
mock_db = MagicMock()
svc = LipsyncService(mock_db, client=mock_mediakit, cosyvoice_service=mock_cosyvoice)
svc.get_job = MagicMock(return_value=None)
bg = MagicMock()
with pytest.raises(HTTPException) as exc_info:
self._call(None, svc, bg)
assert exc_info.value.status_code == 404
+348
View File
@@ -626,3 +626,351 @@ class TestLipsyncServiceRefreshCompletedAsyncPersist:
assert result.status == "completed"
assert result.output_video_url == "https://temp.mk/out2.mp4"
fake_persist_task.apply_async.assert_called_once()
class TestRefreshJobStatusDbRollback:
"""#1867 三层防御 ②:refresh_job_status DB commit 失败时 rollback 不 raise."""
def test_refresh_db_commit_failure_rollback_and_return(self):
"""commit 抛异常 → rollback,不 raise,返回当前 job 对象让下次轮询重试."""
from app.services.lipsync_service import LipsyncService
mock_job = MagicMock()
mock_job.id = "job-rollback"
mock_job.user_id = "u"
mock_job.mediakit_task_id = "mk"
mock_job.status = "submitted"
mock_job.output_video_url = ""
mock_job.output_duration = 0.0
mock_db = MagicMock()
mock_query = MagicMock()
mock_filter = MagicMock()
mock_filter.first.return_value = mock_job
mock_query.filter.return_value = mock_filter
mock_db.query.return_value = mock_query
mock_db.commit.side_effect = Exception("DB connection lost")
mock_client = MagicMock()
mock_client.get_task_status.return_value = {
"status": "completed",
"result": {"video_url": "https://temp.mk/out.mp4", "duration": 5.0},
}
# refresh_job_status 内部懒 import persist_output_video_task
# 用 patch 替换模块属性,create=True 因为原模块里它是 shared_task,patch 后是 MagicMock
with patch("app.tasks.lipsync_tts.persist_output_video_task", MagicMock(), create=True):
svc = LipsyncService(mock_db, client=mock_client, cosyvoice_service=MagicMock())
result_job = svc.refresh_job_status("job-rollback", "u")
assert result_job is mock_job
mock_db.rollback.assert_called()
def test_refresh_intermediate_status_commits(self):
"""中间状态(running/processing 等)→ 更新 status 和 updated_at,正常 commit."""
from app.services.lipsync_service import LipsyncService
mock_job = MagicMock()
mock_job.id = "job-mid"
mock_job.user_id = "u"
mock_job.mediakit_task_id = "mk"
mock_job.status = "submitted"
mock_job.output_video_url = ""
mock_job.output_duration = 0.0
mock_db = MagicMock()
mock_query = MagicMock()
mock_filter = MagicMock()
mock_filter.first.return_value = mock_job
mock_query.filter.return_value = mock_filter
mock_db.query.return_value = mock_query
mock_client = MagicMock()
mock_client.get_task_status.return_value = {"status": "processing"}
svc = LipsyncService(mock_db, client=mock_client, cosyvoice_service=MagicMock())
result_job = svc.refresh_job_status("job-mid", "u")
assert result_job is mock_job
assert mock_job.status == "processing"
mock_db.commit.assert_called()
mock_db.rollback.assert_not_called()
def test_refresh_failed_commits_error_info(self):
"""failed 状态 → 写入 error_message/error_code/completed_at,commit."""
from app.services.lipsync_service import LipsyncService
mock_job = MagicMock()
mock_job.id = "job-fail"
mock_job.user_id = "u"
mock_job.mediakit_task_id = "mk"
mock_job.status = "submitted"
mock_job.output_video_url = ""
mock_job.output_duration = 0.0
mock_db = MagicMock()
mock_query = MagicMock()
mock_filter = MagicMock()
mock_filter.first.return_value = mock_job
mock_query.filter.return_value = mock_filter
mock_db.query.return_value = mock_query
mock_client = MagicMock()
mock_client.get_task_status.return_value = {
"status": "failed",
"error": {"code": "BadAudio", "message": "音频损坏"},
}
svc = LipsyncService(mock_db, client=mock_client, cosyvoice_service=MagicMock())
result_job = svc.refresh_job_status("job-fail", "u")
assert result_job.status == "failed"
assert mock_job.error_message == "音频损坏"
assert mock_job.error_code == "BadAudio"
mock_db.commit.assert_called()
class TestPollMediakitStatus:
"""#1867 三层防御 ③:Celery 兜底轮询任务单元测试."""
def _make_session(self, job):
mock_db = MagicMock()
mock_query = MagicMock()
mock_filter = MagicMock()
mock_filter.first.return_value = job
mock_query.filter.return_value = mock_filter
mock_db.query.return_value = mock_query
return mock_db
def _patch_sessions(self, mock_db):
"""用 patch.dict 注入 fake SessionLocal 到 worker_app.db 和 app.db."""
from types import ModuleType
fake_app_db = ModuleType("app.db")
fake_worker_db = ModuleType("worker_app.db")
fake_sl = MagicMock(return_value=mock_db)
fake_app_db.SessionLocal = fake_sl
fake_worker_db.SessionLocal = fake_sl
fake_worker_pkg = ModuleType("worker_app")
return patch.dict(
sys.modules,
{
"worker_app": fake_worker_pkg,
"worker_app.db": fake_worker_db,
"app.db": fake_app_db,
},
)
def _run(self, mock_db, job_id="j", user_id="u1", fake_self=None):
from app.tasks.lipsync_tts import poll_mediakit_status
self_ref = fake_self if fake_self is not None else MagicMock()
real = poll_mediakit_status._get_current_object()
return real.run.__func__(self_ref, job_id=job_id, user_id=user_id), self_ref
def test_poll_job_not_found_returns(self):
mock_db = self._make_session(None)
with self._patch_sessions(mock_db):
self._run(mock_db, job_id="missing")
mock_db.close.assert_called_once()
def test_poll_terminal_completed_skips_mediakit(self):
mock_job = MagicMock(status="completed", mediakit_task_id="mk-1")
mock_db = self._make_session(mock_job)
with self._patch_sessions(mock_db), patch("app.services.mediakit_client.get_mediakit_client") as fg:
self._run(mock_db)
fg.assert_not_called()
mock_db.close.assert_called_once()
def test_poll_terminal_failed_skips(self):
mock_job = MagicMock(status="failed", mediakit_task_id="mk-1")
mock_db = self._make_session(mock_job)
with self._patch_sessions(mock_db), patch("app.services.mediakit_client.get_mediakit_client") as fg:
self._run(mock_db, job_id="j1b")
fg.assert_not_called()
def test_poll_terminal_cancelled_skips(self):
mock_job = MagicMock(status="cancelled", mediakit_task_id="mk-1")
mock_db = self._make_session(mock_job)
with self._patch_sessions(mock_db), patch("app.services.mediakit_client.get_mediakit_client") as fg:
self._run(mock_db, job_id="j1c")
fg.assert_not_called()
def test_poll_no_mk_task_id_returns(self):
mock_job = MagicMock(status="submitted", mediakit_task_id=None)
mock_db = self._make_session(mock_job)
with self._patch_sessions(mock_db):
self._run(mock_db, job_id="j6")
mock_db.close.assert_called_once()
def test_poll_succeeded_marks_completed(self):
mock_job = MagicMock(status="submitted", mediakit_task_id="mk-2", output_video_url="", output_duration=0.0)
mock_db = self._make_session(mock_job)
mock_client = MagicMock()
mock_client.get_task_status.return_value = {
"status": "succeeded",
"result": {"video_url": "https://temp/x.mp4", "duration": 12.0},
}
with (
self._patch_sessions(mock_db),
patch("app.services.mediakit_client.get_mediakit_client", return_value=mock_client),
patch("app.services.lipsync_service.LipsyncService") as MS,
):
svc = MS.return_value
svc._persist_output_video.return_value = "https://oss/x.mp4"
self._run(mock_db, job_id="j2")
assert mock_job.status == "completed"
assert mock_job.output_video_url == "https://oss/x.mp4"
assert mock_job.output_duration == 12.0
svc._persist_output_video.assert_called_once_with("https://temp/x.mp4", "j2", "u1")
mock_db.commit.assert_called()
def test_poll_succeeded_persist_fail_fallback(self):
mock_job = MagicMock(status="submitted", mediakit_task_id="mk-3", output_video_url="", output_duration=0.0)
mock_db = self._make_session(mock_job)
mock_client = MagicMock()
mock_client.get_task_status.return_value = {
"status": "succeeded",
"result": {"video_url": "https://temp/y.mp4", "duration": 8.0},
}
with (
self._patch_sessions(mock_db),
patch("app.services.mediakit_client.get_mediakit_client", return_value=mock_client),
patch("app.services.lipsync_service.LipsyncService") as MS,
):
svc = MS.return_value
svc._persist_output_video.side_effect = Exception("OSS down")
self._run(mock_db, job_id="j3")
assert mock_job.status == "completed"
assert mock_job.output_video_url == "https://temp/y.mp4"
def test_poll_failed_marks_failed(self):
mock_job = MagicMock(status="running", mediakit_task_id="mk-4", error_message="", error_code="")
mock_db = self._make_session(mock_job)
mock_client = MagicMock()
mock_client.get_task_status.return_value = {
"status": "failed",
"error": {"code": "TaskFailed", "message": "音频错误"},
}
with (
self._patch_sessions(mock_db),
patch("app.services.mediakit_client.get_mediakit_client", return_value=mock_client),
):
self._run(mock_db, job_id="j4")
assert mock_job.status == "failed"
assert mock_job.error_message == "音频错误"
assert mock_job.error_code == "TaskFailed"
def test_poll_error_status_marks_failed(self):
mock_job = MagicMock(status="submitted", mediakit_task_id="mk-4b", error_message="", error_code="")
mock_db = self._make_session(mock_job)
mock_client = MagicMock()
mock_client.get_task_status.return_value = {
"status": "error",
"error": {"code": "Internal", "message": "内部错误"},
}
with (
self._patch_sessions(mock_db),
patch("app.services.mediakit_client.get_mediakit_client", return_value=mock_client),
):
self._run(mock_db, job_id="j4b")
assert mock_job.status == "failed"
assert mock_job.error_message == "内部错误"
def test_poll_running_triggers_retry(self):
mock_job = MagicMock(status="submitted", mediakit_task_id="mk-5")
mock_db = self._make_session(mock_job)
mock_client = MagicMock()
mock_client.get_task_status.return_value = {"status": "running"}
class _Retry(Exception):
pass
fake_self = MagicMock()
fake_self.retry.side_effect = _Retry()
with (
self._patch_sessions(mock_db),
patch("app.services.mediakit_client.get_mediakit_client", return_value=mock_client),
patch("app.services.lipsync_service.LipsyncService"),
):
try:
self._run(mock_db, job_id="j5", fake_self=fake_self)
assert False, "expected retry"
except _Retry:
pass
assert mock_job.updated_at is not None
assert mock_job.status == "running"
assert fake_self.retry.call_count >= 1
class TestTtsSubmitDispatchesPoll:
"""#1867 三层防御 ③:tts_synthesize_and_submit 成功后链式触发 poll."""
def _run_tts(self, job, mk_client, fake_poll):
"""运行 tts_synthesize_and_submit.run,直接在用户函数 __globals__ 替换 poll_mediakit_status.
pytest --import-mode=importlib 下,patch("app.tasks.lipsync_tts.xxx") 拿到的模块对象
可能与 celery autoretry 包装下用户函数的 __globals__ 不一致(之前的 Poll 测试通过
patch.dict(sys.modules, ...) 注入 fake 模块,进一步加剧了模块命名空间不一致)。
直接 patch _orig_run.__func__.__globals__ 是最稳的方式。
"""
from app.tasks.lipsync_tts import tts_synthesize_and_submit
session, patches = _apply_all_patches(job=job, mk_client=mk_client)
for p in patches:
p.__enter__()
# 定位 autoretry 包装下的真实用户函数
real_tts = tts_synthesize_and_submit._get_current_object()
fn = real_tts._orig_run
while hasattr(fn, "__func__"):
fn = fn.__func__
g = fn.__globals__
orig_poll = g.get("poll_mediakit_status")
g["poll_mediakit_status"] = fake_poll
try:
tts_synthesize_and_submit.run("job-1", "user-1", "v1", "你好", 1.0, "")
finally:
if orig_poll is not None:
g["poll_mediakit_status"] = orig_poll
else:
g.pop("poll_mediakit_status", None)
for p in reversed(patches):
p.__exit__(None, None, None)
return session
def test_submit_success_dispatches_poll(self):
"""提交 MediaKit 成功后链式触发 poll_mediakit_status.apply_async (countdown=10)."""
job = _make_fake_job()
job.emotion = ""
mk_client = MagicMock()
mk_client.submit_lipsync.return_value = {"success": True, "task_id": "mk-poll-1"}
fake_poll = MagicMock()
self._run_tts(job, mk_client, fake_poll)
assert job.status == "submitted"
assert job.mediakit_task_id == "mk-poll-1"
fake_poll.apply_async.assert_called_once()
kwargs = fake_poll.apply_async.call_args.kwargs
assert kwargs["kwargs"] == {"job_id": "job-1", "user_id": "user-1"}
assert kwargs["countdown"] == 10
def test_submit_failure_no_poll(self):
from app.services.mediakit_client import MediaKitError
job = _make_fake_job()
job.emotion = ""
mk_client = MagicMock()
mk_client.submit_lipsync.side_effect = MediaKitError("bad", code="X")
fake_poll = MagicMock()
self._run_tts(job, mk_client, fake_poll)
assert job.status == "failed"
fake_poll.apply_async.assert_not_called()
def test_poll_dispatch_fail_does_not_break(self):
job = _make_fake_job()
job.emotion = ""
mk_client = MagicMock()
mk_client.submit_lipsync.return_value = {"success": True, "task_id": "mk-poll-2"}
fake_poll = MagicMock()
fake_poll.apply_async.side_effect = ConnectionError("celery down")
session = self._run_tts(job, mk_client, fake_poll)
assert job.status == "submitted"
assert job.mediakit_task_id == "mk-poll-2"
session.commit.assert_called()