From a27ed596b411045e3dba4f772cd2458aef34cf52 Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Wed, 9 Sep 2026 19:13:03 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat(ai-avatar):=20TTS=E7=9B=B4=E7=94=9F?= =?UTF-8?q?=E5=AF=B9=E5=8F=A3=E5=9E=8B=20+=20=E8=AF=AD=E9=80=9F/=E6=83=85?= =?UTF-8?q?=E7=BB=AA=E9=80=8F=E4=BC=A0=20+=20MediaKit=E6=99=BA=E8=83=BD?= =?UTF-8?q?=E5=B0=81=E9=9D=A2=20+=20=E5=A5=91=E7=BA=A6=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 对口型支持 TTS 直生模式：POST /lipsync/jobs 传 voice_id+script_text(+speed/emotion)， 后端内部调 CosyVoice 合成音频→转存OSS→提交 MediaKit；保留 audio_url 直接音频模式 - cosyvoice_service: 新增 emotion 参数 + normalize_emotion（中文 自然/兴奋/沉稳/亲切 映射 natural/excited/calm/friendly），空值不透传 - TTS 链路 speed/emotion 全链路透传：schema→use_case(metadata)→workflow(单段/分段/重合成) →submit_synthesize_task(rate/emotion)；preview 即时试听同步 - lipsync refresh: 中间状态(running/processing)同步DB；completed 输出视频转存自家OSS防过期 - 修复 lipsync/ai-avatar 路由 current_user.id → current_user.user.id（AuthenticatedUser 无 .id） - 新增 POST /ai-avatar/render/smart-cover 独立封面接口：复用 MediaKit extract_frames + cover_frame_scorer 评分选最佳帧（非 FFmpeg 首帧），渲染管线封面同样优先智能选帧 - 迁移 073: lipsync_jobs 增加 voice_id/script_text/speed/emotion 列，audio_url 改可空 - docs/ai-avatar-api-contract-1822.md: 前后端接口契约 + title_config 字段清单 - 新增 11 个单测（情绪归一化/payload透传/TTS直生/中间状态/智能封面），相关 82 测试全绿 Refs #1797 #1822 --- .../versions/073_add_lipsync_tts_fields.py | 45 +++ apps/api/app/api/routes/ai_avatar_render.py | 44 ++- apps/api/app/api/routes/lipsync.py | 36 ++- apps/api/app/api/routes/tts.py | 11 +- apps/api/app/schemas/ai_avatar_render.py | 15 + apps/api/app/schemas/lipsync.py | 82 ++++-- apps/api/app/schemas/tts.py | 2 + .../app/services/ai_avatar_cover_service.py | 162 +++++++++++ .../app/services/ai_avatar_render_service.py | 19 +- apps/api/app/services/lipsync_service.py | 171 +++++++++++- docs/ai-avatar-api-contract-1822.md | 146 ++++++++++ packages/adapters/sqlalchemy_impl/models.py | 8 +- packages/application/cosyvoice_service.py | 56 +++- packages/application/tts_job/workflow.py | 14 + .../test_ai_avatar_emotion_tts_lipsync.py | 261 ++++++++++++++++++ 15 files changed, 1011 insertions(+), 61 deletions(-) create mode 100644 alembic/versions/073_add_lipsync_tts_fields.py create mode 100644 apps/api/app/services/ai_avatar_cover_service.py create mode 100644 docs/ai-avatar-api-contract-1822.md create mode 100644 tests/unit/test_ai_avatar_emotion_tts_lipsync.py diff --git a/alembic/versions/073_add_lipsync_tts_fields.py b/alembic/versions/073_add_lipsync_tts_fields.py new file mode 100644 index 000000000..0edc08e71 --- /dev/null +++ b/alembic/versions/073_add_lipsync_tts_fields.py @@ -0,0 +1,45 @@ +"""lipsync_jobs 增加 TTS 直生字段（voice_id/script_text/speed/emotion） + +Revision ID: 073_add_lipsync_tts_fields +Revises: 072_add_ai_avatar_render +Create Date: 2026-09-09 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "073_add_lipsync_tts_fields" +down_revision = "072_add_ai_avatar_render" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # 对口型支持「传音色 + 文案直接生成」：后端内部先 TTS 合成音频再提交对口型 + op.add_column( + "lipsync_jobs", + sa.Column("voice_id", sa.String(200), nullable=False, server_default=""), + ) + op.add_column( + "lipsync_jobs", + sa.Column("script_text", sa.Text(), nullable=False, server_default=""), + ) + op.add_column( + "lipsync_jobs", + sa.Column("speed", sa.Float(), nullable=False, server_default=sa.text("1.0")), + ) + op.add_column( + "lipsync_jobs", + sa.Column("emotion", sa.String(20), nullable=False, server_default=""), + ) + # audio_url 改为可空：直生模式下音频由后端 TTS 合成后回填 + op.alter_column("lipsync_jobs", "audio_url", existing_type=sa.Text(), nullable=True) + + +def downgrade() -> None: + op.alter_column("lipsync_jobs", "audio_url", existing_type=sa.Text(), nullable=False) + op.drop_column("lipsync_jobs", "emotion") + op.drop_column("lipsync_jobs", "speed") + op.drop_column("lipsync_jobs", "script_text") + op.drop_column("lipsync_jobs", "voice_id") diff --git a/apps/api/app/api/routes/ai_avatar_render.py b/apps/api/app/api/routes/ai_avatar_render.py index 38b7d91e5..02a381846 100644 --- a/apps/api/app/api/routes/ai_avatar_render.py +++ b/apps/api/app/api/routes/ai_avatar_render.py @@ -17,7 +17,10 @@ from app.dependencies import get_db_session from app.schemas.ai_avatar_render import ( AiAvatarRenderJobResponse, CreateAiAvatarRenderRequest, + SmartCoverRequest, + SmartCoverResponse, ) +from app.services.ai_avatar_cover_service import generate_smart_cover from app.services.ai_avatar_render_service import ( AiAvatarRenderError, AiAvatarRenderService, @@ -49,7 +52,7 @@ def create_render_job( """ try: job = svc.create_render_job( - user_id=current_user.id, + user_id=current_user.user.id, lipsync_job_id=body.lipsync_job_id, script_id=body.script_id, b_roll_segments=[s.model_dump() for s in body.b_roll_segments], @@ -94,7 +97,7 @@ def list_render_jobs( ): """获取 AI 数字人渲染任务列表.""" items, total = svc.list_render_jobs( - user_id=current_user.id, + user_id=current_user.user.id, project_id=project_id, status=status, offset=offset, @@ -118,7 +121,7 @@ def get_render_job( svc: AiAvatarRenderService = Depends(_get_service), ): """获取渲染任务详情.""" - job = svc.get_render_job(job_id, current_user.id) + job = svc.get_render_job(job_id, current_user.user.id) if job is None: raise HTTPException(status_code=404, detail="渲染任务不存在") return job @@ -134,7 +137,7 @@ def cancel_render_job( svc: AiAvatarRenderService = Depends(_get_service), ): """取消渲染任务（仅 pending 状态可取消）.""" - job = svc.cancel_render_job(job_id, current_user.id) + job = svc.cancel_render_job(job_id, current_user.user.id) if job is None: raise HTTPException(status_code=404, detail="渲染任务不存在") if job.status != "cancelled": @@ -155,7 +158,7 @@ def retry_render_job( svc: AiAvatarRenderService = Depends(_get_service), ): """重试失败的渲染任务.""" - job = svc.retry_render_job(job_id, current_user.id) + job = svc.retry_render_job(job_id, current_user.user.id) if job is None: raise HTTPException(status_code=404, detail="渲染任务不存在") if job.status != "pending": @@ -173,3 +176,34 @@ def retry_render_job( logger.warning("Celery 任务提交失败，重试任务已重置但未触发执行: %s", job.id) return job + + + +# ── POST /smart-cover — 智能获取封面（MediaKit 抽帧 + 评分选帧）──────── + + +@router.post("/smart-cover", response_model=SmartCoverResponse) +def generate_avatar_smart_cover( + body: SmartCoverRequest, + current_user: AuthenticatedUser = Depends(get_current_user), +) -> SmartCoverResponse: + """智能获取数字人视频封面. + + 复用智能剪辑的 MediaKit 抽帧 + 质量评分选最佳帧逻辑（非 FFmpeg 简单截帧）， + 并将选中帧转存到自家 OSS，返回非临时的封面公网 URL。 + + 前端「智能获取封面」按钮可直接调用本接口；不依赖渲染任务完成。 + """ + video_url = (body.video_url or "").strip() + if not video_url.startswith(("http://", "https://")): + raise HTTPException(status_code=400, detail="video_url 必须是合法的 HTTP/HTTPS URL") + + cover_url = generate_smart_cover(video_url, max_frames=body.max_frames) + if not cover_url: + return SmartCoverResponse( + cover_url="", + status="fallback_failed", + message="智能抽帧失败（MediaKit 不可用或抽帧异常），请稍后重试", + ) + logger.info("智能封面生成成功: user=%s", current_user.user.id) + return SmartCoverResponse(cover_url=cover_url, status="completed") diff --git a/apps/api/app/api/routes/lipsync.py b/apps/api/app/api/routes/lipsync.py index 9e478bd43..0dfa3a05b 100644 --- a/apps/api/app/api/routes/lipsync.py +++ b/apps/api/app/api/routes/lipsync.py @@ -13,7 +13,7 @@ from __future__ import annotations import logging from app.auth import AuthenticatedUser, get_current_user -from app.dependencies import get_db_session +from app.dependencies import get_db_session, get_voice_clone_profile_repository from app.schemas.lipsync import CreateLipsyncJobRequest, LipsyncJobResponse from app.services.lipsync_service import LipsyncService from app.services.mediakit_client import MediaKitError @@ -25,8 +25,11 @@ logger = logging.getLogger(__name__) router = APIRouter() -def _get_service(db: Session = Depends(get_db_session)) -> LipsyncService: - return LipsyncService(db) +def _get_service( + db: Session = Depends(get_db_session), + voice_clone_repo=Depends(get_voice_clone_profile_repository), +) -> LipsyncService: + return LipsyncService(db, voice_clone_repo=voice_clone_repo) # ── POST /jobs — 提交对口型任务 ─────────────────────────────────────────── @@ -44,20 +47,31 @@ def create_lipsync_job( """ try: job = svc.create_job( - user_id=current_user.id, + user_id=current_user.user.id, video_url=body.video_url, audio_url=body.audio_url, + voice_id=body.voice_id, + script_text=body.script_text, + speed=body.speed, + emotion=body.emotion, enable_video_loop=body.enable_video_loop, project_id=body.project_id, ) except MediaKitError as exc: - # 创建失败（job 已记录 error），返回 502 + # TTS 合成失败 / 音色无权访问 → 400/403；MediaKit 提交失败 → 502 + status_code = 502 + if exc.code in ("VoiceForbidden",): + status_code = 403 + elif exc.code in ("InvalidInput", "TTSInvalidParam", "VoiceNotReady"): + status_code = 400 + elif exc.code == "TTSSynthesisFailed": + status_code = 502 raise HTTPException( - status_code=502, + status_code=status_code, detail={ "code": exc.code, "message": str(exc), - "request_id": exc.request_id, + "request_id": getattr(exc, "request_id", ""), }, ) from exc @@ -78,7 +92,7 @@ def list_lipsync_jobs( ): """获取对口型任务列表.""" items, total = svc.list_jobs( - user_id=current_user.id, + user_id=current_user.user.id, project_id=project_id, status=status, offset=offset, @@ -102,7 +116,7 @@ def get_lipsync_job( svc: LipsyncService = Depends(_get_service), ): """获取对口型任务详情.""" - job = svc.get_job(job_id, current_user.id) + job = svc.get_job(job_id, current_user.user.id) if job is None: raise HTTPException(status_code=404, detail="任务不存在") return job @@ -118,7 +132,7 @@ def refresh_lipsync_job( svc: LipsyncService = Depends(_get_service), ): """从 MediaKit 拉取最新状态并更新.""" - job = svc.refresh_job_status(job_id, current_user.id) + job = svc.refresh_job_status(job_id, current_user.user.id) if job is None: raise HTTPException(status_code=404, detail="任务不存在") return job @@ -134,7 +148,7 @@ def cancel_lipsync_job( svc: LipsyncService = Depends(_get_service), ): """取消对口型任务（仅 pending/submitted 状态可取消）.""" - job = svc.cancel_job(job_id, current_user.id) + job = svc.cancel_job(job_id, current_user.user.id) if job is None: raise HTTPException(status_code=404, detail="任务不存在") if job.status != "cancelled": diff --git a/apps/api/app/api/routes/tts.py b/apps/api/app/api/routes/tts.py index ec000a5b8..400b012e2 100755 --- a/apps/api/app/api/routes/tts.py +++ b/apps/api/app/api/routes/tts.py @@ -173,6 +173,14 @@ def synthesize( # job.voice_id 统一存解析后的 CosyVoice voice_id actual_voice_id = resolved_profile.voice_id + # 语速/情绪等合成参数随 metadata 落库，workflow 提交 CosyVoice 时读取透传 + synthesis_meta = { + "speed": request.speed, + "emotion": request.emotion or "", + } + if request.metadata_: + synthesis_meta.update(request.metadata_) + use_case = CreateTTSJobUseCase(repository) job = use_case.execute( user_id=user_id, @@ -180,7 +188,7 @@ def synthesize( voice_id=actual_voice_id, voice_model=request.voice_model, voice_clone_profile_id=voice_clone_profile_id, - metadata=request.metadata_, + metadata=synthesis_meta, ) # 提交 CosyVoice 合成任务 @@ -567,6 +575,7 @@ def preview_tts( text=request.text, voice_id=actual_voice_id, speed=request.speed, + emotion=request.emotion, ) except CosyVoiceError as e: raise HTTPException( diff --git a/apps/api/app/schemas/ai_avatar_render.py b/apps/api/app/schemas/ai_avatar_render.py index 2284bdfa2..b240a9d9d 100644 --- a/apps/api/app/schemas/ai_avatar_render.py +++ b/apps/api/app/schemas/ai_avatar_render.py @@ -109,3 +109,18 @@ class AiAvatarRenderProgressResponse(BaseModel): output_cover_url: str output_duration: float error_message: str + + +class SmartCoverRequest(BaseModel): + """智能封面请求 — MediaKit 抽帧 + 质量评分选最佳帧.""" + + video_url: str = Field(..., description="数字人视频 URL（对口型/渲染成片）") + max_frames: int = Field(5, ge=1, le=10, description="抽帧数量（默认 5）") + + +class SmartCoverResponse(BaseModel): + """智能封面响应.""" + + cover_url: str = Field("", description="封面图公网 URL（OSS，非临时）；失败为空") + status: str = Field("completed", description="completed / fallback_failed") + message: str = Field("", description="失败原因（如有）") diff --git a/apps/api/app/schemas/lipsync.py b/apps/api/app/schemas/lipsync.py index 2429b6977..0836f3fdc 100644 --- a/apps/api/app/schemas/lipsync.py +++ b/apps/api/app/schemas/lipsync.py @@ -1,11 +1,17 @@ -"""对口型 API Schema 定义 — #1796.""" +"""对口型 API Schema 定义 — #1796 / #1822. + +支持两种输入模式（二选一）： +1. TTS 直生模式（推荐）：传 voice_id + script_text（+ speed/emotion）， + 后端内部先调 CosyVoice 合成音频，再提交 MediaKit 对口型。 +2. 直接音频模式：传 video_url + audio_url（音频已由调用方准备好）。 +""" from __future__ import annotations from datetime import datetime from typing import Optional -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, model_validator class LipsyncJobResponse(BaseModel): @@ -17,6 +23,10 @@ class LipsyncJobResponse(BaseModel): video_url: str audio_url: str enable_video_loop: bool + voice_id: str = "" + script_text: str = "" + speed: float = 1.0 + emotion: str = "" mediakit_task_id: str status: str output_video_url: str @@ -33,38 +43,56 @@ class LipsyncJobResponse(BaseModel): class CreateLipsyncJobRequest(BaseModel): - """创建对口型任务请求.""" + """创建对口型任务请求. + + 两种模式： + - TTS 直生：voice_id + script_text 必填；video_url 必填（人物视频）； + audio_url 留空（后端合成）。 + - 直接音频：video_url + audio_url 必填。 + """ video_url: str = Field(..., description="人物视频 URL（MP4，≤30min，单人真人）") - audio_url: str = Field(..., description="驱动音频 URL（mp3/aac/wav/m4a/flac）") + + # 模式 2：直接音频 + audio_url: str = Field("", description="驱动音频 URL（mp3/aac/wav/m4a/flac）；直生模式留空") + + # 模式 1：TTS 直生 + voice_id: str = Field("", description="音色 ID（预置音色或克隆音色 profile UUID）") + script_text: str = Field("", description="要合成的文案（直生模式必填）") + speed: float = Field(1.0, ge=0.5, le=2.0, description="语速（0.5-2.0），默认 1.0") + emotion: str = Field("", description="情绪（natural/excited/calm/friendly 或中文 自然/兴奋/沉稳/亲切）") + enable_video_loop: bool = Field(False, description="音频长于视频时是否循环画面") project_id: str = Field("", description="项目 ID（可选）") - @field_validator("video_url") - @classmethod - def validate_video_url(cls, v: str) -> str: - v = v.strip() - if not v: + @model_validator(mode="after") + def _validate_input_mode(self) -> "CreateLipsyncJobRequest": + video = (self.video_url or "").strip() + if not video: raise ValueError("video_url 不能为空") - if not v.startswith(("http://", "https://")): + if not video.startswith(("http://", "https://")): raise ValueError("video_url 必须是 HTTP/HTTPS URL") - # 仅支持 MP4 - lower = v.lower().split("?")[0] + lower = video.lower().split("?")[0] if not lower.endswith(".mp4"): raise ValueError("video_url 仅支持 MP4 格式") - return v - @field_validator("audio_url") - @classmethod - def validate_audio_url(cls, v: str) -> str: - v = v.strip() - if not v: - raise ValueError("audio_url 不能为空") - if not v.startswith(("http://", "https://")): - raise ValueError("audio_url 必须是 HTTP/HTTPS URL") - # 支持的音频格式 - lower = v.lower().split("?")[0] - allowed_exts = (".mp3", ".aac", ".wav", ".m4a", ".flac") - if not any(lower.endswith(ext) for ext in allowed_exts): - raise ValueError(f"audio_url 格式不支持，仅支持: {', '.join(allowed_exts)}") - return v + has_audio = bool((self.audio_url or "").strip()) + has_tts = bool((self.voice_id or "").strip()) and bool((self.script_text or "").strip()) + + if not has_audio and not has_tts: + raise ValueError( + "必须提供驱动音频：要么传 audio_url（直接音频模式），" + "要么同时传 voice_id + script_text（TTS 直生模式）" + ) + + if has_audio: + au = self.audio_url.strip() + if not au.startswith(("http://", "https://")): + raise ValueError("audio_url 必须是 HTTP/HTTPS URL") + au_lower = au.lower().split("?")[0] + allowed = (".mp3", ".aac", ".wav", ".m4a", ".flac") + if not any(au_lower.endswith(ext) for ext in allowed): + raise ValueError(f"audio_url 格式不支持，仅支持: {', '.join(allowed)}") + self.audio_url = au + + return self diff --git a/apps/api/app/schemas/tts.py b/apps/api/app/schemas/tts.py index 1e6b47513..535f03ba2 100644 --- a/apps/api/app/schemas/tts.py +++ b/apps/api/app/schemas/tts.py @@ -16,6 +16,7 @@ class TTSSynthesizeRequest(BaseModel): output_name: str = Field("", description="输出文件名") language: str = Field("zh-CN", description="语言") speed: float = Field(1.0, ge=0.5, le=2.0, description="语速") + emotion: str = Field("", description="情绪（natural/excited/calm/friendly，或中文 自然/兴奋/沉稳/亲切）") voice_model: str = Field("", description="语音模型名称") voice_clone_profile_id: str = Field("", description="关联的音色克隆档案 ID") format: str = Field("mp3", description="输出格式（mp3/wav/pcm）") @@ -109,6 +110,7 @@ class TTSPreviewRequest(BaseModel): text: str = Field(..., min_length=1, max_length=200, description="合成文本，限制 200 字") voice_id: str = Field(..., min_length=1, description="音色 ID") speed: float = Field(1.0, ge=0.5, le=2.0, description="语速") + emotion: str = Field("", description="情绪（natural/excited/calm/friendly，或中文）") pitch: float = Field(1.0, ge=0.5, le=2.0, description="音调（预留，当前未使用）") diff --git a/apps/api/app/services/ai_avatar_cover_service.py b/apps/api/app/services/ai_avatar_cover_service.py new file mode 100644 index 000000000..df0bd40aa --- /dev/null +++ b/apps/api/app/services/ai_avatar_cover_service.py @@ -0,0 +1,162 @@ +"""AI 数字人封面服务 — 复用智能剪辑的 MediaKit 抽帧 + 质量评分选最佳帧. + +与 generation_cover.py 的智能选帧能力对齐（不再用 FFmpeg 简单截帧）： +1. MediaKit extract_frames 抽取多帧（默认 5 帧，SpecifiedFrames 策略） +2. cover_frame_scorer.score_frames 按清晰度/亮度/色彩评分选最佳 +3. 下载最佳帧并转存 OSS，返回公网封面 URL + +降级：MediaKit 不可用或抽帧失败时返回空字符串，由调用方决定回退策略。 +""" + +from __future__ import annotations + +import logging +import tempfile +import uuid +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + + +def select_best_cover_frame(video_url: str, *, max_frames: int = 5) -> str: + """从视频抽取多帧并评分选最佳帧，返回最佳帧的临时 URL. + + Args: + video_url: 可公网访问的视频 URL + max_frames: 抽帧数量 + + Returns: + 最佳帧图片 URL；失败返回空字符串 + """ + if not video_url: + return "" + try: + from packages.shared.cover_frame_scorer import score_frames + from packages.shared.mediakit_client import get_mediakit_client + + mk = get_mediakit_client() + if not mk.is_available: + logger.warning("[数字人封面] MediaKit 未配置，无法智能抽帧") + return "" + + snapshots = mk.extract_frames( + video_url=video_url, + strategy="SpecifiedFrames", + max_frames=max_frames, + poll_interval=2.0, + max_poll_attempts=5, + max_retries=0, + ) + if not snapshots: + logger.warning("[数字人封面] MediaKit 未返回帧: %s", video_url[:80]) + return "" + + if len(snapshots) == 1: + return snapshots[0].get("image_url") or snapshots[0].get("url") or "" + + # 下载各帧评分 + import httpx + + candidates = [] + for snap in snapshots: + url = snap.get("image_url") or snap.get("url") or "" + if not url: + continue + tmp_path: Optional[str] = None + try: + resp = httpx.get(url, timeout=15, follow_redirects=True) + resp.raise_for_status() + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp: + tmp.write(resp.content) + tmp_path = tmp.name + candidates.append({"image_path": tmp_path, "url": url}) + except Exception: + candidates.append({"image_path": None, "url": url, "score": 0.0}) + + if not candidates: + return snapshots[0].get("image_url") or snapshots[0].get("url") or "" + + scored = score_frames(candidates) + best = scored[0] if scored else None + best_url = best.get("url", "") if best else "" + + # 清理临时文件 + for c in candidates: + p = c.get("image_path") + if p: + try: + Path(p).unlink(missing_ok=True) + except Exception: + pass + + logger.info( + "[数字人封面] 智能选帧完成: candidates=%d best_score=%s", + len(candidates), + best.get("score") if best else "n/a", + ) + return best_url + + except Exception: + logger.warning("[数字人封面] 智能选帧失败", exc_info=True) + return "" + + +def persist_cover_to_oss(frame_url: str, *, job_id: str = "", prefix: str = "ai-avatar/covers") -> str: + """下载帧图并转存到 OSS，返回公网封面 URL. + + Args: + frame_url: MediaKit 返回的临时帧图 URL + job_id: 关联任务 ID（用于 OSS key 命名） + prefix: OSS key 前缀 + + Returns: + OSS 公网 URL；失败回退原始 frame_url + """ + if not frame_url: + return "" + tmp_path: Optional[str] = None + try: + import httpx + + resp = httpx.get(frame_url, timeout=30, follow_redirects=True) + resp.raise_for_status() + if not resp.content: + return frame_url + + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp: + tmp.write(resp.content) + tmp_path = tmp.name + + from packages.shared.storage import get_shared_storage_service + + storage = get_shared_storage_service() + token = job_id or uuid.uuid4().hex[:12] + cover_key = f"{prefix}/{token}/cover_{uuid.uuid4().hex[:8]}.jpg" + public_url = storage.upload_file( + file_or_path=tmp_path, + storage_key=cover_key, + content_type="image/jpeg", + ) + logger.info("[数字人封面] 封面已转存 OSS: key=%s", cover_key) + return public_url or frame_url + except Exception: + logger.warning("[数字人封面] 封面转存 OSS 失败，返回原始 URL", exc_info=True) + return frame_url + finally: + if tmp_path: + try: + Path(tmp_path).unlink(missing_ok=True) + except Exception: + pass + + +def generate_smart_cover(video_url: str, *, job_id: str = "", max_frames: int = 5) -> str: + """一站式：MediaKit 智能抽帧选最佳 → 转存 OSS，返回封面公网 URL. + + 供独立封面接口与渲染管线复用。失败返回空字符串。 + """ + best_frame = select_best_cover_frame(video_url, max_frames=max_frames) + if not best_frame: + return "" + return persist_cover_to_oss(best_frame, job_id=job_id) diff --git a/apps/api/app/services/ai_avatar_render_service.py b/apps/api/app/services/ai_avatar_render_service.py index d13af622d..b24bbc203 100644 --- a/apps/api/app/services/ai_avatar_render_service.py +++ b/apps/api/app/services/ai_avatar_render_service.py @@ -288,7 +288,24 @@ class AiAvatarRenderService: output_video_url = self._upload_to_oss(output_video_path, f"ai-avatar/{job_id}/output.mp4") job.output_video_url = output_video_url - if cover_path: + # 封面：优先复用智能剪辑的 MediaKit 抽帧 + 质量评分选最佳帧； + # MediaKit 不可用时回退到 FFmpeg 已按 cover_config 抽取的 cover_path + smart_cover_url = "" + if output_video_url: + try: + from app.services.ai_avatar_cover_service import ( + generate_smart_cover, + ) + + smart_cover_url = generate_smart_cover( + output_video_url, job_id=job_id, max_frames=5 + ) + except Exception: + logger.warning("智能封面(MediaKit)失败，回退 FFmpeg 封面 job_id=%s", job_id, exc_info=True) + + if smart_cover_url: + job.output_cover_url = smart_cover_url + elif cover_path: output_cover_url = self._upload_to_oss(cover_path, f"ai-avatar/{job_id}/cover.jpg") job.output_cover_url = output_cover_url diff --git a/apps/api/app/services/lipsync_service.py b/apps/api/app/services/lipsync_service.py index 3f7dd142e..3b8a02b5d 100644 --- a/apps/api/app/services/lipsync_service.py +++ b/apps/api/app/services/lipsync_service.py @@ -14,6 +14,8 @@ import uuid from datetime import datetime, timezone from typing import Optional +import io + from app.services.mediakit_client import ( STATUS_COMPLETED, STATUS_FAILED, @@ -25,6 +27,9 @@ from app.services.mediakit_client import ( from sqlalchemy.orm import Session from packages.adapters.sqlalchemy_impl.models import LipsyncJobModel +from packages.application.cosyvoice_service import CosyVoiceError, normalize_emotion +from packages.shared.storage import get_shared_storage_service +from packages.shared.url_security import ALLOWED_AUDIO_MIME_TYPES, safe_download_bytes logger = logging.getLogger(__name__) @@ -32,9 +37,104 @@ logger = logging.getLogger(__name__) class LipsyncService: """对口型任务 Service.""" - def __init__(self, db: Session, client: Optional[MediaKitClient] = None): + def __init__( + self, + db: Session, + client: Optional[MediaKitClient] = None, + cosyvoice_service=None, + voice_clone_repo=None, + ): self.db = db self.client = client or get_mediakit_client() + self._cosyvoice = cosyvoice_service + self._voice_clone_repo = voice_clone_repo + + def _get_cosyvoice(self): + """延迟获取 CosyVoiceService（与 tts 路由一致，含 OSS 预签名配置）.""" + if self._cosyvoice is None: + from app.dependencies import get_cosyvoice_service + + self._cosyvoice = get_cosyvoice_service() + return self._cosyvoice + + def _resolve_voice_id(self, voice_id: str, user_id: str) -> str: + """将克隆音色 profile UUID 解析为 CosyVoice voice_id。 + + 与 /tts/synthesize 保持一致：命中 profile → 校验归属 → 返回其 voice_id； + 未命中（预置音色 ID 或克隆 CosyVoice voice_id）原样返回。 + """ + if not voice_id: + return "" + if self._voice_clone_repo is None: + try: + from app.dependencies import get_voice_clone_profile_repository + + self._voice_clone_repo = get_voice_clone_profile_repository(self.db) + except Exception: + return voice_id + try: + profile = self._voice_clone_repo.get(voice_id) + except Exception: + return voice_id + if profile is None: + return voice_id + if getattr(profile, "user_id", "") != user_id: + raise MediaKitError("无权访问该音色", code="VoiceForbidden") + if not getattr(profile, "voice_id", ""): + raise MediaKitError("音色克隆尚未完成，请稍后再试", code="VoiceNotReady") + return profile.voice_id + + def _synthesize_and_persist_audio( + self, + *, + user_id: str, + job_id: str, + voice_id: str, + script_text: str, + speed: float, + emotion: str, + ) -> str: + """TTS 直生：调 CosyVoice 合成音频并转存 OSS，返回可公网访问的音频 URL. + + Raises: + MediaKitError: 合成失败 + """ + actual_voice_id = self._resolve_voice_id(voice_id, user_id) + cosyvoice = self._get_cosyvoice() + try: + result = cosyvoice.submit_synthesize_task( + text=script_text, + voice_id=actual_voice_id, + speed=speed, + emotion=normalize_emotion(emotion), + ) + except CosyVoiceError as exc: + raise MediaKitError(f"TTS 合成失败: {exc}", code="TTSSynthesisFailed") from exc + except ValueError as exc: + raise MediaKitError(f"TTS 参数错误: {exc}", code="TTSInvalidParam") from exc + + temp_url = result.get("audio_url", "") + if not temp_url: + raise MediaKitError("TTS 未返回音频 URL", code="TTSNoAudio") + + # 转存到自家 OSS，避免临时 URL 过期导致 MediaKit 拉取失败 + try: + audio_data = safe_download_bytes( + temp_url, + purpose="lipsync_tts_audio", + allowed_mime_types=ALLOWED_AUDIO_MIME_TYPES, + timeout=60.0, + ) + storage = get_shared_storage_service() + storage_key = f"lipsync-tts/{user_id}/{job_id}.mp3" + permanent_url = storage.upload_file( + io.BytesIO(audio_data), storage_key, content_type="audio/mpeg" + ) + logger.info("对口型 TTS 音频已转存 OSS: job_id=%s key=%s", job_id, storage_key) + return permanent_url + except Exception as exc: + logger.warning("TTS 音频转存 OSS 失败，回退临时 URL: job_id=%s err=%s", job_id, exc) + return temp_url # ── 创建任务 ────────────────────────────────────────────────────────── @@ -43,15 +143,41 @@ class LipsyncService: *, user_id: str, video_url: str, - audio_url: str, + audio_url: str = "", + voice_id: str = "", + script_text: str = "", + speed: float = 1.0, + emotion: str = "", enable_video_loop: bool = False, project_id: str = "", ) -> LipsyncJobModel: """创建对口型任务并提交到 MediaKit. + 两种输入模式： + - TTS 直生：voice_id + script_text（audio_url 留空），后端先合成音频 + - 直接音频：提供 audio_url + Raises: - MediaKitError: API 调用失败 + MediaKitError: TTS 合成或 MediaKit 提交失败 """ + # 0. TTS 直生模式：先合成音频（在创建 DB 记录之前完成，失败直接抛出） + if not audio_url: + if not (voice_id and script_text): + raise MediaKitError( + "必须提供 audio_url 或 voice_id+script_text", + code="InvalidInput", + ) + # 预合成：用临时 job_id 命名 OSS 对象 + pre_job_id = str(uuid.uuid4()) + audio_url = self._synthesize_and_persist_audio( + user_id=user_id, + job_id=pre_job_id, + voice_id=voice_id, + script_text=script_text, + speed=speed, + emotion=emotion, + ) + # 1. 创建数据库记录 job_id = str(uuid.uuid4()) job = LipsyncJobModel( @@ -61,6 +187,10 @@ class LipsyncService: video_url=video_url, audio_url=audio_url, enable_video_loop=enable_video_loop, + voice_id=voice_id or "", + script_text=script_text or "", + speed=speed, + emotion=normalize_emotion(emotion), status="pending", ) self.db.add(job) @@ -145,11 +275,14 @@ class LipsyncService: return job mk_status = status_data.get("status", STATUS_RUNNING) + logger.info("MediaKit 对口型状态 [%s]: %s", job_id, mk_status) if mk_status == STATUS_COMPLETED: result = status_data.get("result", {}) job.status = STATUS_COMPLETED - job.output_video_url = result.get("video_url", "") + output_url = result.get("video_url", "") + # MediaKit 输出为临时 URL，转存自家 OSS 防止过期（失败则回退临时 URL） + job.output_video_url = self._persist_output_video(output_url, job_id, user_id) job.output_duration = result.get("duration", 0.0) job.completed_at = datetime.now(timezone.utc) elif mk_status == STATUS_FAILED: @@ -158,12 +291,40 @@ class LipsyncService: job.error_message = error.get("message", "任务执行失败") job.error_code = error.get("code", "TaskFailed") job.completed_at = datetime.now(timezone.utc) - # running 状态只更新时间戳 + else: + # 中间状态（running/processing/queued 等）同步到 DB，避免前端永远卡在 submitted + if isinstance(mk_status, str) and mk_status: + job.status = mk_status job.updated_at = datetime.now(timezone.utc) self.db.commit() self.db.refresh(job) return job + def _persist_output_video(self, temp_url: str, job_id: str, user_id: str) -> str: + """将 MediaKit 输出的临时视频 URL 转存到自家 OSS. + + 失败时回退返回原始临时 URL，不影响任务完成。 + """ + if not temp_url: + return "" + try: + import httpx + + with httpx.Client(timeout=180.0, follow_redirects=True) as client: + resp = client.get(temp_url) + resp.raise_for_status() + data = resp.content + storage = get_shared_storage_service() + storage_key = f"lipsync-outputs/{user_id}/{job_id}.mp4" + permanent_url = storage.upload_file( + io.BytesIO(data), storage_key, content_type="video/mp4" + ) + logger.info("对口型输出视频已转存 OSS: job_id=%s key=%s", job_id, storage_key) + return permanent_url or temp_url + except Exception as exc: + logger.warning("对口型输出视频转存 OSS 失败，回退临时 URL: job_id=%s err=%s", job_id, exc) + return temp_url + # ── 取消任务 ────────────────────────────────────────────────────────── def cancel_job(self, job_id: str, user_id: str) -> Optional[LipsyncJobModel]: diff --git a/docs/ai-avatar-api-contract-1822.md b/docs/ai-avatar-api-contract-1822.md new file mode 100644 index 000000000..aac212af2 --- /dev/null +++ b/docs/ai-avatar-api-contract-1822.md @@ -0,0 +1,146 @@ +# AI 数字人前后端接口契约（#1797 / #1822） + +> 分支：`fix/ai-avatar-v3-1797` +> 范围：TTS→对口型链路打通、语速/情绪透传、封面智能选帧、标题字段对齐 +> 本文档为前后端联调的唯一字段口径。 + +--- + +## 1. 对口型创建接口 `POST /api/v1/lipsync/jobs` + +支持两种输入模式，**二选一**： + +### 模式 A（推荐）：TTS 直生 —— 传音色 + 文案，后端内部合成音频 + +前端无需先调 TTS。后端收到请求后：先调 CosyVoice 合成音频 → 转存 OSS → 再提交 MediaKit 对口型。 + +```jsonc +{ + "video_url": "https://oss.../person.mp4", // 必填，人物视频（MP4） + "voice_id": "cosyvoice-v3-flash-99-xxxx", // 必填，音色 ID（预置音色 或 克隆 profile UUID） + "script_text": "省是浙江省，市是永康市……", // 必填，要合成的文案 + "speed": 1.0, // 可选，语速 0.5~2.0，默认 1.0 + "emotion": "excited", // 可选，情绪，见 §3 + "enable_video_loop": false, // 可选，音频长于视频时是否循环画面 + "project_id": "" // 可选 +} +``` + +### 模式 B：直接音频 —— 前端已准备好音频 + +```jsonc +{ + "video_url": "https://oss.../person.mp4", // 必填 + "audio_url": "https://oss.../voice.mp3", // 必填，mp3/aac/wav/m4a/flac + "enable_video_loop": false +} +``` + +### 校验与错误码 + +| 场景 | HTTP | detail.code | +|------|------|-------------| +| 既无 audio_url 又无 voice_id+script_text | 422 | （schema 校验） | +| video_url 非 MP4 / audio_url 格式不支持 | 422 | （schema 校验） | +| 克隆音色不属于当前用户 | 403 | `VoiceForbidden` | +| 克隆音色尚未合成完成 | 400 | `VoiceNotReady` | +| TTS 合成失败（如 CosyVoice 欠费） | 502 | `TTSSynthesisFailed` | +| MediaKit 提交失败 | 502 | `*`（透传 MediaKit code） | + +### 轮询 + +- `GET /api/v1/lipsync/jobs/{id}`：非终态任务先返回 DB 缓存，**后台异步刷新 MediaKit**（不会阻塞轮询）。 +- `status` 流转：`pending` → `submitted` → `running`/`processing`（MediaKit 中间态同步）→ `completed` / `failed`。 +- `completed` 时 `output_video_url` 为**已转存自家 OSS 的非临时 URL**（不会过期）。 +- 前端每 3s 轮询，命中 `completed`/`failed` 即停。 + +--- + +## 2. TTS 合成接口语速/情绪透传 + +- `POST /api/v1/tts/synthesize`（异步任务）与 `POST /api/v1/tts/preview`（即时试听）均新增： + - `speed`：float，0.5~2.0，默认 1.0 → 透传 CosyVoice payload 的 `rate` + - `emotion`：string，见 §3 映射 → 透传 `emotion` +- 透传链路：`route → CreateTTSJobUseCase(metadata) → TTSJobWorkflow.start_synthesis / 分段合成 → CosyVoiceService.submit_synthesize_task(rate/emotion)`。 +- 分段合成（长文案）与失败重合成路径同样透传 speed/emotion。 + +--- + +## 3. 情绪枚举（前后端统一） + +前端把中文选项映射成英文后传后端；后端同时接受中文/英文，非法值忽略（走默认自然）。 + +| 前端选项 | 传参值 | CosyVoice 枚举 | +|---------|--------|---------------| +| 自然 | `natural` | natural | +| 兴奋 | `excited` | excited | +| 沉稳 | `calm` | calm | +| 亲切 | `friendly` | friendly | + +后端 `normalize_emotion()` 也接受中文（自然/兴奋/沉稳/亲切）做兜底映射。 + +--- + +## 4. 智能封面接口 `POST /api/v1/ai-avatar/render/smart-cover` + +独立接口，**不依赖渲染任务**，前端「智能获取封面」按钮直接调用。 + +**请求** +```jsonc +{ + "video_url": "https://oss.../avatar_output.mp4", // 必填，数字人视频 + "max_frames": 5 // 可选，抽帧数量 1~10，默认 5 +} +``` + +**响应** +```jsonc +{ + "cover_url": "https://oss.../ai-avatar/covers/xxx/cover_yy.jpg", // OSS 非临时 URL + "status": "completed", // completed / fallback_failed + "message": "" // 失败原因 +} +``` + +**实现**：复用智能剪辑同款能力 —— MediaKit `extract_frames(SpecifiedFrames)` 抽 5 帧 → `cover_frame_scorer.score_frames`（清晰度+亮度+色彩）评分选最佳 → 转存 OSS。 +**不再使用 FFmpeg 简单首帧**。渲染管线最终封面也优先走该智能选帧，MediaKit 不可用时才回退 FFmpeg。 + +--- + +## 5. 标题配置 `title_config` 字段清单（build_title_drawtext_filter） + +AI 数字人渲染请求 `titles[]` 每项结构，字段名与类型如下： + +| 字段 | 类型 | 必填 | 默认 | 说明 | +|------|------|------|------|------| +| `text` | string | ✅ | — | 标题文本。**多行用 `\n` 分隔**；内部统一把 `/` 替换为 `／`（drawtext 转义保护） | +| `start` | float | ✅ | — | 出现时间（秒） | +| `end` | float | ✅ | — | 消失时间（秒），须 > start | +| `fontSize` | number | ❌ | 48 | 字号（像素），范围建议 12~200 | +| `color` | string | ❌ | `white` | 文字颜色，CSS/FFmpeg 颜色名或 `0xRRGGBB`（如 `red`、`0xFF5733`） | +| `fontPath` | string | ❌ | 内置思源黑体 | 字体文件绝对路径（通常不传，用默认中文字体） | +| `backgroundColor` | string | ❌ | 黑色 | 背景框颜色 | +| `borderColor` | string | ❌ | 白色 | 边框颜色 | +| `borderWidth` | int | ❌ | 2 | 边框宽度（像素） | +| `backgroundOpacity` | float | ❌ | 0.5 | 背景框不透明度 0~1 | +| `frame` | object | ❌ | 居中底部 | 画面内位置，见下 | +| `frame.x` | float | ❌ | `null` | 框中心 X 比例 0~1；null=水平居中 | +| `frame.y` | float | ❌ | `null` | 框中心 Y 比例 0~1；null=底部（0.75） | +| `frame.widthRatio` | float | ❌ | 0.9 | 框宽占画面比例 0~1 | +| `frame.heightRatio` | float | ❌ | 0.2 | 框高占画面比例 0~1 | +| `z_index` | int | ❌ | 0 | 层级（预留，当前标题在 B-roll 之后） | + +**前端注意事项** +- 多行标题直接在 `text` 里写 `\n`，后端按行处理 drawtext 的 `textfile` 换行，无需手工拆滤镜。 +- 颜色建议固定传英文颜色名（`white`/`black`/`red`…）或 `0xRRGGBB`，不要传 `#RRGGBB`（`#` 会被滤镜解析干扰）。 +- 位置不传 frame 时标题在画面居中偏下（y≈0.75），这是常见口播标题位置。 + +--- + +## 6. 前端对接清单 + +1. 对口型：改用**模式 A**（voice_id + script_text + speed + emotion），不要再先调 TTS 拿 audio_url。 +2. 音色 ID：`voice_id` 可直接传克隆音色的 profile UUID，后端会解析为 CosyVoice voice_id（与 /tts 一致）。 +3. 情绪下拉：自然/兴奋/沉稳/亲切 → natural/excited/calm/friendly。 +4. 封面：点「智能获取封面」→ POST `/ai-avatar/render/smart-cover`，用返回的 `cover_url`。 +5. 轮询：识别 `running` 等中间态，不要只认 `submitted`。 diff --git a/packages/adapters/sqlalchemy_impl/models.py b/packages/adapters/sqlalchemy_impl/models.py index 45a671c11..fb3cfe25c 100755 --- a/packages/adapters/sqlalchemy_impl/models.py +++ b/packages/adapters/sqlalchemy_impl/models.py @@ -684,9 +684,15 @@ class LipsyncJobModel(Base): # 输入参数 video_url = Column(Text, nullable=False) - audio_url = Column(Text, nullable=False) + audio_url = Column(Text, nullable=True) # 直生模式（voice_id+script_text）下 TTS 合成后回填 enable_video_loop = Column(Boolean, nullable=False, default=False) + # TTS 直生字段：传音色 + 文案，由后端先合成音频再对口型 + voice_id = Column(String(200), nullable=False, default="") + script_text = Column(Text, nullable=False, default="") + speed = Column(Float, nullable=False, default=1.0) + emotion = Column(String(20), nullable=False, default="") + # MediaKit 任务状态 mediakit_task_id = Column(String(200), nullable=False, default="", index=True) status = Column( diff --git a/packages/application/cosyvoice_service.py b/packages/application/cosyvoice_service.py index 6b645f91c..d47c83570 100755 --- a/packages/application/cosyvoice_service.py +++ b/packages/application/cosyvoice_service.py @@ -25,6 +25,35 @@ from packages.shared.config import get_shared_settings logger = logging.getLogger(__name__) +# CosyVoice 支持的情绪：中文标签 → API 英文值 +EMOTION_MAP = { + "自然": "natural", + "兴奋": "excited", + "沉稳": "calm", + "亲切": "friendly", + "natural": "natural", + "excited": "excited", + "calm": "calm", + "friendly": "friendly", +} +VALID_EMOTIONS = {"natural", "excited", "calm", "friendly"} + + +def normalize_emotion(emotion: str) -> str: + """将前端情绪值归一化为 CosyVoice 英文枚举。 + + 支持中文（自然/兴奋/沉稳/亲切）和英文；非法值返回空串（不传，走默认）。 + """ + if not emotion: + return "" + key = emotion.strip().lower() + mapped = EMOTION_MAP.get(emotion.strip()) or EMOTION_MAP.get(key) + if mapped and mapped in VALID_EMOTIONS: + return mapped + logger.warning("未知的 emotion 值，忽略: %r", emotion) + return "" + + class CosyVoiceError(Exception): """CosyVoice API 调用异常。""" @@ -431,6 +460,7 @@ class CosyVoiceService: format: str = "", speed: float = 1.0, volume: int = 50, + emotion: str = "", ) -> dict: """提交语音合成任务（同步非流式，直接返回结果）. @@ -444,6 +474,7 @@ class CosyVoiceService: format: 输出格式（mp3/wav/pcm），空表示使用配置默认值 speed: 语速（0.5-2.0），1.0 为正常速度 volume: 音量（0-100），默认 50 + emotion: 情绪（natural/excited/calm/friendly），空串不传 Returns: dict: {"audio_url": str, "request_id": str, @@ -463,17 +494,20 @@ class CosyVoiceService: settings = get_shared_settings() - payload = { - "model": self._model, - "input": { - "text": text, - "voice": voice_id, - "format": format or settings.cosyvoice_format, - "sample_rate": sample_rate or settings.cosyvoice_sample_rate, - "rate": speed, - "volume": volume, - }, + input_payload: dict[str, Any] = { + "text": text, + "voice": voice_id, + "format": format or settings.cosyvoice_format, + "sample_rate": sample_rate or settings.cosyvoice_sample_rate, + "rate": speed, + "volume": volume, } + # 情绪：归一化（中文→英文）后透传；空/非法则不传，走 CosyVoice 默认 + norm_emotion = normalize_emotion(emotion) + if norm_emotion: + input_payload["emotion"] = norm_emotion + + payload = {"model": self._model, "input": input_payload} response = self._call_api( method="POST", @@ -517,6 +551,7 @@ class CosyVoiceService: format: str = "", speed: float = 1.0, volume: int = 50, + emotion: str = "", timeout: float = 120.0, ) -> SynthesizeResult: """语音合成（同步非流式）. @@ -548,6 +583,7 @@ class CosyVoiceService: format=format, speed=speed, volume=volume, + emotion=emotion, ) return SynthesizeResult( diff --git a/packages/application/tts_job/workflow.py b/packages/application/tts_job/workflow.py index 58a913126..d26249bc0 100644 --- a/packages/application/tts_job/workflow.py +++ b/packages/application/tts_job/workflow.py @@ -143,11 +143,16 @@ class TTSWorkflowService: return self._start_segment_synthesis(job) try: + _meta = dict(job.metadata) + _speed = float(_meta.get("speed", 1.0) or 1.0) + _emotion = str(_meta.get("emotion", "") or "") submit_result = self.cosyvoice_service.submit_synthesize_task( text=job.input_text, voice_id=job.voice_id, sample_rate=job.sample_rate, format=job.format, + speed=_speed, + emotion=_emotion, ) # 保存 task_id / request_id 到 metadata @@ -283,6 +288,7 @@ class TTSWorkflowService: job_metadata = job.metadata or {} speed = float(job_metadata.get("speed", 1.0)) volume = int(job_metadata.get("volume", 50)) + emotion = str(job_metadata.get("emotion", "") or "") result = self.cosyvoice_service.submit_synthesize_task( text=job.input_text, @@ -291,6 +297,7 @@ class TTSWorkflowService: format=job.format, speed=speed, volume=volume, + emotion=emotion, ) audio_url = result.get("audio_url", "") if not audio_url: @@ -401,6 +408,9 @@ class TTSWorkflowService: """ max_workers = min(len(segments), _MAX_SEGMENT_WORKERS) results: list[dict | None] = [None] * len(segments) + _seg_meta = job.metadata or {} + _seg_speed = float(_seg_meta.get("speed", 1.0) or 1.0) + _seg_emotion = str(_seg_meta.get("emotion", "") or "") with ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_idx = {} @@ -411,6 +421,8 @@ class TTSWorkflowService: voice_id=job.voice_id, sample_rate=job.sample_rate, format=job.format, + speed=_seg_speed, + emotion=_seg_emotion, ) future_to_idx[future] = idx @@ -500,6 +512,7 @@ class TTSWorkflowService: job_metadata = job.metadata or {} speed = float(job_metadata.get("speed", 1.0)) volume = int(job_metadata.get("volume", 50)) + emotion = str(job_metadata.get("emotion", "") or "") # 分段文本（用于缺失段重新合成） segments = split_text(job.input_text, max_chars=_SEGMENT_THRESHOLD) @@ -534,6 +547,7 @@ class TTSWorkflowService: format=job.format, speed=speed, volume=volume, + emotion=emotion, ) future_to_idx[future] = idx diff --git a/tests/unit/test_ai_avatar_emotion_tts_lipsync.py b/tests/unit/test_ai_avatar_emotion_tts_lipsync.py new file mode 100644 index 000000000..26c400961 --- /dev/null +++ b/tests/unit/test_ai_avatar_emotion_tts_lipsync.py @@ -0,0 +1,261 @@ +"""#1822 情绪/语速透传 + 对口型 TTS 直生 + 智能封面 单元测试. + +CI 增量映射: + cosyvoice_service.normalize_emotion / payload emotion + lipsync_service TTS 直生分支（voice_id+script_text） + ai_avatar_cover_service 智能选帧 +""" + +import os +from unittest.mock import MagicMock, patch + +import pytest + +os.environ.setdefault("JWT_SECRET_KEY", "dev-secret-key-for-testing") + + +# ── 情绪归一化 ────────────────────────────────────────────────────────── + +def test_normalize_emotion_english_values(): + from packages.application.cosyvoice_service import normalize_emotion + + assert normalize_emotion("natural") == "natural" + assert normalize_emotion("excited") == "excited" + assert normalize_emotion("calm") == "calm" + assert normalize_emotion("friendly") == "friendly" + # 大小写 / 空白容错 + assert normalize_emotion(" Excited ") == "excited" + + +def test_normalize_emotion_chinese_values(): + from packages.application.cosyvoice_service import normalize_emotion + + assert normalize_emotion("自然") == "natural" + assert normalize_emotion("兴奋") == "excited" + assert normalize_emotion("沉稳") == "calm" + assert normalize_emotion("亲切") == "friendly" + + +def test_normalize_emotion_invalid_returns_empty(): + from packages.application.cosyvoice_service import normalize_emotion + + assert normalize_emotion("") == "" + assert normalize_emotion("angry") == "" + assert normalize_emotion("喜怒哀乐") == "" + + +# ── CosyVoice payload 携带 emotion + rate ────────────────────────────── + +def _make_service_with_captured_client(captured: dict): + """构造 CosyVoiceService，拦截 post 请求体到 captured['json'].""" + import httpx as _httpx + + from packages.application import cosyvoice_service as mod + + mock_client = MagicMock(spec=_httpx.Client) + resp = MagicMock() + resp.status_code = 200 + resp.json.return_value = { + "request_id": "req-1", + "output": {"audio": {"url": "https://tts/a.mp3", "duration": 1.0}}, + } + resp.raise_for_status = MagicMock() + + def fake_request(method, url, headers, json, timeout): + captured["json"] = json + return resp + + mock_client.request.side_effect = fake_request + + with patch.object(mod, "get_shared_settings") as settings_patch: + s = MagicMock() + s.cosyvoice_api_key = "sk-test" + s.cosyvoice_base_url = "https://x/api/v1" + s.cosyvoice_model = "cosyvoice-v3-flash" + s.cosyvoice_clone_model = "voice-enrollment" + s.cosyvoice_format = "mp3" + s.cosyvoice_sample_rate = 22050 + s.cosyvoice_voice = "longxiaochun" + settings_patch.return_value = s + svc = mod.CosyVoiceService(http_client=mock_client) + return svc + + +def test_submit_synthesize_payload_includes_emotion_and_rate(): + captured: dict = {} + svc = _make_service_with_captured_client(captured) + + svc.submit_synthesize_task(text="你好", voice_id="v-1", speed=1.5, emotion="兴奋") + + inp = captured["json"]["input"] + assert inp["emotion"] == "excited" + assert inp["rate"] == 1.5 + + +def test_submit_synthesize_payload_omits_emotion_when_empty(): + captured: dict = {} + svc = _make_service_with_captured_client(captured) + + svc.submit_synthesize_task(text="你好", voice_id="v-1") + + assert "emotion" not in captured["json"]["input"] + + +# ── 对口型 TTS 直生分支 ───────────────────────────────────────────────── + +def _lipsync_service_with_mocks(): + from app.services.lipsync_service import LipsyncService + + db = MagicMock() + client = MagicMock() + client.is_available = True + client.submit_lipsync.return_value = { + "success": True, + "task_id": "mk-1", + "request_id": "req-1", + } + cosy = MagicMock() + cosy.submit_synthesize_task.return_value = { + "audio_url": "https://tts/raw.mp3", + "request_id": "tts-req", + "audio_duration": 3.0, + } + svc = LipsyncService(db, client=client, cosyvoice_service=cosy, voice_clone_repo=MagicMock()) + # _resolve_voice_id 默认原样返回（repo.get 返回 None） + svc._voice_clone_repo.get.return_value = None + return svc, client, cosy + + +def test_create_job_tts_direct_mode_synthesizes_audio(): + svc, client, cosy = _lipsync_service_with_mocks() + + with patch("app.services.lipsync_service.get_shared_storage_service") as storage_patch, \ + patch("app.services.lipsync_service.safe_download_bytes") as dl_patch: + storage = MagicMock() + storage.upload_file.return_value = "https://oss/tts.mp3" + storage_patch.return_value = storage + dl_patch.return_value = b"FAKEAUDIO" + + job = svc.create_job( + user_id="user-1", + video_url="https://oss/person.mp4", + voice_id="cosy-v1", + script_text="你好世界", + speed=1.2, + emotion="兴奋", + ) + + # 调了 TTS 合成，带 speed/emotion + cosy.submit_synthesize_task.assert_called_once() + _, kwargs = cosy.submit_synthesize_task.call_args + assert kwargs["speed"] == 1.2 + assert kwargs["emotion"] == "excited" + assert kwargs["voice_id"] == "cosy-v1" + # MediaKit 用合成后的 OSS 音频 URL 提交 + _, submit_kwargs = client.submit_lipsync.call_args + assert submit_kwargs["audio_url"] == "https://oss/tts.mp3" + assert submit_kwargs["video_url"] == "https://oss/person.mp4" + # DB 记录了 TTS 字段 + assert job.emotion == "excited" + assert job.speed == 1.2 + + +def test_create_job_direct_audio_mode_skips_tts(): + svc, client, cosy = _lipsync_service_with_mocks() + + job = svc.create_job( + user_id="user-1", + video_url="https://oss/person.mp4", + audio_url="https://oss/ready.mp3", + ) + + cosy.submit_synthesize_task.assert_not_called() + _, submit_kwargs = client.submit_lipsync.call_args + assert submit_kwargs["audio_url"] == "https://oss/ready.mp3" + + +def test_create_job_tts_failure_raises(): + from app.services.mediakit_client import MediaKitError + from packages.application.cosyvoice_service import CosyVoiceError + + svc, client, cosy = _lipsync_service_with_mocks() + cosy.submit_synthesize_task.side_effect = CosyVoiceError("Arrearage") + + with pytest.raises(MediaKitError) as exc: + svc.create_job( + user_id="user-1", + video_url="https://oss/person.mp4", + voice_id="v-1", + script_text="文本", + ) + assert exc.value.code == "TTSSynthesisFailed" + # TTS 失败不应提交 MediaKit + client.submit_lipsync.assert_not_called() + + +# ── refresh 同步中间状态 ──────────────────────────────────────────────── + +def test_refresh_syncs_running_status(): + from app.services.lipsync_service import LipsyncService + + db = MagicMock() + client = MagicMock() + client.get_task_status.return_value = {"success": True, "status": "running"} + svc = LipsyncService(db, client=client) + + job = MagicMock() + job.status = "submitted" + job.mediakit_task_id = "mk-1" + job.id = "j-1" + svc.get_job = MagicMock(return_value=job) + + result = svc.refresh_job_status("j-1", "user-1") + assert result.status == "running" + + +# ── 智能封面 ──────────────────────────────────────────────────────────── + +def test_smart_cover_selects_best_frame_and_persists(): + from app.services import ai_avatar_cover_service as cov + + snapshots = [ + {"image_url": "https://mk/f0.jpg"}, + {"image_url": "https://mk/f1.jpg"}, + ] + with patch("packages.shared.mediakit_client.get_mediakit_client") as mk_patch, \ + patch("packages.shared.cover_frame_scorer.score_frames") as score_patch, \ + patch("httpx.get") as http_get, \ + patch("packages.shared.storage.get_shared_storage_service") as storage_patch: + mk = MagicMock() + mk.is_available = True + mk.extract_frames.return_value = snapshots + mk_patch.return_value = mk + # score_frames 把 f1 选为最佳 + score_patch.side_effect = lambda cands: [ + {"url": "https://mk/f1.jpg", "score": 90.0, "image_path": cands[1]["image_path"]}, + {"url": "https://mk/f0.jpg", "score": 60.0, "image_path": cands[0]["image_path"]}, + ] + resp = MagicMock() + resp.content = b"IMGDATA" + resp.raise_for_status = MagicMock() + http_get.return_value = resp + storage = MagicMock() + storage.upload_file.return_value = "https://oss/cover.jpg" + storage_patch.return_value = storage + + url = cov.generate_smart_cover("https://oss/avatar.mp4", job_id="job-1") + + assert url == "https://oss/cover.jpg" + mk.extract_frames.assert_called_once() + score_patch.assert_called_once() + + +def test_smart_cover_returns_empty_when_mediakit_unavailable(): + from app.services import ai_avatar_cover_service as cov + + with patch("packages.shared.mediakit_client.get_mediakit_client") as mk_patch: + mk = MagicMock() + mk.is_available = False + mk_patch.return_value = mk + url = cov.generate_smart_cover("https://oss/avatar.mp4") + assert url == "" -- 2.54.0 From 8052f245ecea8aadee49d3448c01b76798f4ce53 Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Wed, 9 Sep 2026 19:15:46 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20=E5=9C=A8=20worker=20Dockerfile=20?= =?UTF-8?q?=E4=B8=AD=E6=B7=BB=E5=8A=A0=20HEALTHCHECK=20=E6=8C=87=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将健康检查命令直接写入 Docker 镜像，确保 Watchtower 重建容器时使用正确的健康检查。 修复 Worker 容器持续显示 unhealthy 的问题。 --- infra/docker/worker.Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/docker/worker.Dockerfile b/infra/docker/worker.Dockerfile index 74baddcb5..9ca769af3 100755 --- a/infra/docker/worker.Dockerfile +++ b/infra/docker/worker.Dockerfile @@ -38,6 +38,10 @@ RUN chmod +x /usr/local/bin/entrypoint-worker.sh # 业务代码（变化最频繁，放最后） COPY apps/worker/ /app/apps/worker/ +# 健康检查：扫描所有进程的 cmdline 查找 celery 进程 +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD grep -lq celery /proc/[0-9]*/cmdline 2>/dev/null || exit 1 + USER celery WORKDIR /app/apps/worker CMD ["/usr/local/bin/entrypoint-worker.sh"] -- 2.54.0 From 4d09bd630e8a21bdee575b516c799e946fecdcf7 Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Wed, 9 Sep 2026 19:55:01 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix(ai-avatar):=20=E5=A5=91=E7=BA=A6?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E6=A0=87=E9=A2=98=E5=AD=97=E6=AE=B5=E7=BA=A0?= =?UTF-8?q?=E6=AD=A3=20+=20script=5Fid=20=E6=94=B9=E4=B8=BA=E5=8F=AF?= =?UTF-8?q?=E9=80=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 文档 §5/§6：标题是单个 title_config dict（非 titles[] 数组），字段以 build_title_drawtext_filter() 为准：text/content、font_size/size、 font_color/color、position(top/center/bottom/custom)、pos_x/pos_y、bold、 stroke、shadow、enabled；删除不存在的 titles[]/fontSize/frame/start/end 描述 - script_id 改为可选：手动输入文案/TTS 直生场景不关联文案库条目，留空不校验， 避免手动文案用户被 ScriptNotFound 卡住；选了文案库时仍做归属校验 - 迁移 074：ai_avatar_render_jobs.script_id 默认空串 - 新增 schema 单测（script_id 空/空白/title_config 单 dict），94 测试全绿 Refs #1797 #1826 --- ...074_ai_avatar_render_script_id_optional.py | 36 +++++++++ apps/api/app/schemas/ai_avatar_render.py | 10 +-- .../app/services/ai_avatar_render_service.py | 28 +++---- docs/ai-avatar-api-contract-1822.md | 77 ++++++++++++------- packages/adapters/sqlalchemy_impl/models.py | 3 +- .../test_ai_avatar_emotion_tts_lipsync.py | 23 ++++++ 6 files changed, 130 insertions(+), 47 deletions(-) create mode 100644 alembic/versions/074_ai_avatar_render_script_id_optional.py diff --git a/alembic/versions/074_ai_avatar_render_script_id_optional.py b/alembic/versions/074_ai_avatar_render_script_id_optional.py new file mode 100644 index 000000000..ffd2156a0 --- /dev/null +++ b/alembic/versions/074_ai_avatar_render_script_id_optional.py @@ -0,0 +1,36 @@ +"""ai_avatar_render_jobs.script_id 放宽为可空串（手动文案直生场景不关联文案库） + +Revision ID: 074_render_script_id_optional +Revises: 073_add_lipsync_tts_fields +Create Date: 2026-09-09 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "074_render_script_id_optional" +down_revision = "073_add_lipsync_tts_fields" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # 列保持 NOT NULL（空串占位），仅应用层允许不传；这里显式补 server_default 防止历史约束歧义 + with op.batch_alter_table("ai_avatar_render_jobs") as batch: + batch.alter_column( + "script_id", + existing_type=sa.String(length=36), + nullable=False, + server_default="", + ) + + +def downgrade() -> None: + with op.batch_alter_table("ai_avatar_render_jobs") as batch: + batch.alter_column( + "script_id", + existing_type=sa.String(length=36), + nullable=False, + server_default=None, + ) diff --git a/apps/api/app/schemas/ai_avatar_render.py b/apps/api/app/schemas/ai_avatar_render.py index b240a9d9d..88832017d 100644 --- a/apps/api/app/schemas/ai_avatar_render.py +++ b/apps/api/app/schemas/ai_avatar_render.py @@ -50,7 +50,7 @@ class CreateAiAvatarRenderRequest(BaseModel): """创建渲染任务请求.""" lipsync_job_id: str = Field(..., description="对口型任务 ID") - script_id: str = Field(..., description="文案 ID") + script_id: str = Field("", description="文案 ID（选自文案库时传；手动输入文案直生场景可留空）") b_roll_segments: list[BRollSegment] = Field(default_factory=list, description="B-roll 片段列表") title_config: dict[str, Any] = Field(default_factory=dict, description="标题配置") cover_config: dict[str, Any] = Field(default_factory=dict, description="封面配置") @@ -67,10 +67,8 @@ class CreateAiAvatarRenderRequest(BaseModel): @field_validator("script_id") @classmethod def validate_script_id(cls, v: str) -> str: - v = v.strip() - if not v: - raise ValueError("script_id 不能为空") - return v + # script_id 可选：手动输入文案（TTS 直生）场景不关联文案库条目 + return (v or "").strip() class AiAvatarRenderJobResponse(BaseModel): @@ -80,7 +78,7 @@ class AiAvatarRenderJobResponse(BaseModel): user_id: str project_id: str lipsync_job_id: str - script_id: str + script_id: str = "" b_roll_segments: list[dict[str, Any]] title_config: dict[str, Any] cover_config: dict[str, Any] diff --git a/apps/api/app/services/ai_avatar_render_service.py b/apps/api/app/services/ai_avatar_render_service.py index b24bbc203..2baa5cac2 100644 --- a/apps/api/app/services/ai_avatar_render_service.py +++ b/apps/api/app/services/ai_avatar_render_service.py @@ -53,8 +53,8 @@ class AiAvatarRenderService: *, user_id: str, lipsync_job_id: str, - script_id: str, - b_roll_segments: list[dict[str, Any]], + script_id: str = "", + b_roll_segments: list[dict[str, Any]] | None = None, title_config: dict[str, Any], cover_config: dict[str, Any], project_id: str = "", @@ -83,17 +83,19 @@ class AiAvatarRenderService: if not lipsync_job.output_video_url: raise AiAvatarRenderError("对口型任务输出视频 URL 为空", code="LipsyncJobNoOutput") - # 2. 验证文案归属 - script = ( - self.db.query(ScriptModel) - .filter( - ScriptModel.id == script_id, - ScriptModel.user_id == user_id, + # 2. 验证文案归属（仅当选了文案库条目时；手动输入文案直生场景 script_id 可空） + script_id = (script_id or "").strip() + if script_id: + script = ( + self.db.query(ScriptModel) + .filter( + ScriptModel.id == script_id, + ScriptModel.user_id == user_id, + ) + .first() ) - .first() - ) - if script is None: - raise AiAvatarRenderError("文案不存在或无权访问", code="ScriptNotFound") + if script is None: + raise AiAvatarRenderError("文案不存在或无权访问", code="ScriptNotFound") # 3. 创建渲染任务 job_id = str(uuid.uuid4()) @@ -103,7 +105,7 @@ class AiAvatarRenderService: project_id=project_id, lipsync_job_id=lipsync_job_id, script_id=script_id, - b_roll_segments=[s if isinstance(s, dict) else s.model_dump() for s in b_roll_segments], + b_roll_segments=[s if isinstance(s, dict) else s.model_dump() for s in (b_roll_segments or [])], title_config=title_config, cover_config=cover_config, status="pending", diff --git a/docs/ai-avatar-api-contract-1822.md b/docs/ai-avatar-api-contract-1822.md index aac212af2..960efc061 100644 --- a/docs/ai-avatar-api-contract-1822.md +++ b/docs/ai-avatar-api-contract-1822.md @@ -107,40 +107,63 @@ --- -## 5. 标题配置 `title_config` 字段清单（build_title_drawtext_filter） +## 5. 渲染接口 `POST /api/v1/ai-avatar/render` -AI 数字人渲染请求 `titles[]` 每项结构，字段名与类型如下： +```jsonc +{ + "lipsync_job_id": "7c29a3b2-...", // 必填，已 completed 的对口型任务 + "script_id": "", // 可选！见下方说明 + "b_roll_segments": [], // 可选，B-roll 片段 + "title_config": { ... }, // 可选，单个标题配置 dict（见 §6） + "cover_config": { ... }, // 可选，封面配置（建议改用 smart-cover） + "project_id": "" +} +``` -| 字段 | 类型 | 必填 | 默认 | 说明 | -|------|------|------|------|------| -| `text` | string | ✅ | — | 标题文本。**多行用 `\n` 分隔**；内部统一把 `/` 替换为 `／`（drawtext 转义保护） | -| `start` | float | ✅ | — | 出现时间（秒） | -| `end` | float | ✅ | — | 消失时间（秒），须 > start | -| `fontSize` | number | ❌ | 48 | 字号（像素），范围建议 12~200 | -| `color` | string | ❌ | `white` | 文字颜色，CSS/FFmpeg 颜色名或 `0xRRGGBB`（如 `red`、`0xFF5733`） | -| `fontPath` | string | ❌ | 内置思源黑体 | 字体文件绝对路径（通常不传，用默认中文字体） | -| `backgroundColor` | string | ❌ | 黑色 | 背景框颜色 | -| `borderColor` | string | ❌ | 白色 | 边框颜色 | -| `borderWidth` | int | ❌ | 2 | 边框宽度（像素） | -| `backgroundOpacity` | float | ❌ | 0.5 | 背景框不透明度 0~1 | -| `frame` | object | ❌ | 居中底部 | 画面内位置，见下 | -| `frame.x` | float | ❌ | `null` | 框中心 X 比例 0~1；null=水平居中 | -| `frame.y` | float | ❌ | `null` | 框中心 Y 比例 0~1；null=底部（0.75） | -| `frame.widthRatio` | float | ❌ | 0.9 | 框宽占画面比例 0~1 | -| `frame.heightRatio` | float | ❌ | 0.2 | 框高占画面比例 0~1 | -| `z_index` | int | ❌ | 0 | 层级（预留，当前标题在 B-roll 之后） | - -**前端注意事项** -- 多行标题直接在 `text` 里写 `\n`，后端按行处理 drawtext 的 `textfile` 换行，无需手工拆滤镜。 -- 颜色建议固定传英文颜色名（`white`/`black`/`red`…）或 `0xRRGGBB`，不要传 `#RRGGBB`（`#` 会被滤镜解析干扰）。 -- 位置不传 frame 时标题在画面居中偏下（y≈0.75），这是常见口播标题位置。 +**`script_id` 是否必填：可选。** +- 从文案库选了文案时传对应文案 ID（后端做归属校验）。 +- **手动输入文案、走 TTS 直生模式时不传（留空）即可**——渲染管线不依赖文案内容，`script_id` 仅用于归属校验。留空不会卡手动文案用户。 --- -## 6. 前端对接清单 +## 6. 标题配置 `title_config` 字段清单（以 build_title_drawtext_filter 为准） + +渲染请求收的是**单个 `title_config` dict**（不是 `titles[]` 数组），字段与 `packages/domain/video_filter_builder.py` 的 `build_title_drawtext_filter()` 完全对齐： + +| 字段 | 别名 | 类型 | 必填 | 默认 | 说明 | +|------|------|------|------|------|------| +| `text` | `content` | string | ✅ | — | 标题文字；为空或 `enabled=false` 时不渲染标题 | +| `enabled` | — | bool | ❌ | `true` | 是否启用标题；false 跳过 | +| `font` | `font_preset` | string | ❌ | 思源黑体 | 字体名（后端按名字解析字体文件） | +| `font_size` | `size` | int | ❌ | 36 | 字号（像素） | +| `font_color` | `color` | string | ❌ | `#ffffff` | 文字颜色，`#RRGGBB`；后端自动去掉 `#`，也可传 `RRGGBB` 或颜色名 | +| `position` | — | string | ❌ | `top` | 预设位置：`top`(y=50) / `center`(垂直居中) / `bottom`(底部上移50px) / `custom` | +| `pos_x` | — | int/float | ❌ | — | 自定义 X 坐标（像素），仅 `position=custom` 生效 | +| `pos_y` | — | int/float | ❌ | — | 自定义 Y 坐标（像素），仅 `position=custom` 生效 | +| `bold` | — | bool | ❌ | `true` | 粗体（Bold 字体变体，回退 borderw 模拟） | +| `stroke` | — | bool/object | ❌ | — | 描边。`true`=黑描边宽2；object 见下 | +| `stroke.enabled` | — | bool | ❌ | true | 是否描边 | +| `stroke.width` | — | int | ❌ | 2 | 描边宽度 | +| `stroke.color` | — | string | ❌ | `#000000` | 描边颜色 | +| `shadow` | — | bool/object | ❌ | — | 阴影。`true`=黑色阴影偏移2px；object 见下 | +| `shadow.enabled` | — | bool | ❌ | true | 是否阴影 | +| `shadow.color` | — | string | ❌ | `#000000` | 阴影颜色 | +| `shadow.offset_x` | — | int | ❌ | 2 | 阴影 X 偏移 | +| `shadow.offset_y` | — | int | ❌ | 2 | 阴影 Y 偏移 | + +**前端注意事项** +- 标题是**整条成片一个标题**（单个 dict），不是按时间段的标题数组；没有 `start`/`end`/`frame`/`fontSize` 这些字段。 +- 位置用 `position` 四档枚举；自由摆放用 `position="custom"` + `pos_x`/`pos_y`（像素坐标，非比例）。 +- 颜色统一传 `#RRGGBB` 即可，后端会处理 `#`；三档预设位置下标题始终水平居中。 +- `stroke`/`shadow` 传 `true` 用默认样式，或传 object 精细控制颜色/宽度/偏移。 + +--- + +## 7. 前端对接清单 1. 对口型：改用**模式 A**（voice_id + script_text + speed + emotion），不要再先调 TTS 拿 audio_url。 2. 音色 ID：`voice_id` 可直接传克隆音色的 profile UUID，后端会解析为 CosyVoice voice_id（与 /tts 一致）。 3. 情绪下拉：自然/兴奋/沉稳/亲切 → natural/excited/calm/friendly。 4. 封面：点「智能获取封面」→ POST `/ai-avatar/render/smart-cover`，用返回的 `cover_url`。 -5. 轮询：识别 `running` 等中间态，不要只认 `submitted`。 +5. 渲染：手动文案直生场景 `script_id` 留空；标题传**单个** `title_config` dict（字段见 §6）。 +6. 轮询：识别 `running` 等中间态，不要只认 `submitted`。 diff --git a/packages/adapters/sqlalchemy_impl/models.py b/packages/adapters/sqlalchemy_impl/models.py index fb3cfe25c..135c54ad9 100755 --- a/packages/adapters/sqlalchemy_impl/models.py +++ b/packages/adapters/sqlalchemy_impl/models.py @@ -721,7 +721,8 @@ class AiAvatarRenderJob(Base): # 输入参数 lipsync_job_id = Column(String(36), nullable=False) - script_id = Column(String(36), nullable=False) + # 文案 ID 可选：手动输入文案（TTS 直生）场景不关联文案库条目 + script_id = Column(String(36), nullable=False, default="") b_roll_segments = Column(JSON, nullable=False, default=list) # b_roll_segments 格式: [{"script_segment_index": 0, "asset_url": "...", "mode": "fullscreen|pip", "start_time": 5.0, "end_time": 10.0}, ...] title_config = Column(JSON, nullable=False, default=dict) diff --git a/tests/unit/test_ai_avatar_emotion_tts_lipsync.py b/tests/unit/test_ai_avatar_emotion_tts_lipsync.py index 26c400961..f81041d5c 100644 --- a/tests/unit/test_ai_avatar_emotion_tts_lipsync.py +++ b/tests/unit/test_ai_avatar_emotion_tts_lipsync.py @@ -259,3 +259,26 @@ def test_smart_cover_returns_empty_when_mediakit_unavailable(): mk_patch.return_value = mk url = cov.generate_smart_cover("https://oss/avatar.mp4") assert url == "" + + +# ── 渲染 script_id 可选（手动文案直生场景）────────────────────────────── + +def test_render_request_script_id_optional(): + import sys, os + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "apps", "api")) + from app.schemas.ai_avatar_render import CreateAiAvatarRenderRequest + + # 手动文案直生：不传 script_id 也合法 + req = CreateAiAvatarRenderRequest(lipsync_job_id="j-1") + assert req.script_id == "" + + # 空白被 strip + req2 = CreateAiAvatarRenderRequest(lipsync_job_id="j-1", script_id=" ") + assert req2.script_id == "" + + # title_config 是单个 dict + req3 = CreateAiAvatarRenderRequest( + lipsync_job_id="j-1", + title_config={"text": "标题", "position": "top", "font_size": 40}, + ) + assert req3.title_config["position"] == "top" -- 2.54.0