diff --git a/alembic/versions/073_add_lipsync_tts_fields.py b/alembic/versions/073_add_lipsync_tts_fields.py new file mode 100644 index 000000000..0edc08e71 --- /dev/null +++ b/alembic/versions/073_add_lipsync_tts_fields.py @@ -0,0 +1,45 @@ +"""lipsync_jobs 增加 TTS 直生字段（voice_id/script_text/speed/emotion） + +Revision ID: 073_add_lipsync_tts_fields +Revises: 072_add_ai_avatar_render +Create Date: 2026-09-09 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "073_add_lipsync_tts_fields" +down_revision = "072_add_ai_avatar_render" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # 对口型支持「传音色 + 文案直接生成」：后端内部先 TTS 合成音频再提交对口型 + op.add_column( + "lipsync_jobs", + sa.Column("voice_id", sa.String(200), nullable=False, server_default=""), + ) + op.add_column( + "lipsync_jobs", + sa.Column("script_text", sa.Text(), nullable=False, server_default=""), + ) + op.add_column( + "lipsync_jobs", + sa.Column("speed", sa.Float(), nullable=False, server_default=sa.text("1.0")), + ) + op.add_column( + "lipsync_jobs", + sa.Column("emotion", sa.String(20), nullable=False, server_default=""), + ) + # audio_url 改为可空：直生模式下音频由后端 TTS 合成后回填 + op.alter_column("lipsync_jobs", "audio_url", existing_type=sa.Text(), nullable=True) + + +def downgrade() -> None: + op.alter_column("lipsync_jobs", "audio_url", existing_type=sa.Text(), nullable=False) + op.drop_column("lipsync_jobs", "emotion") + op.drop_column("lipsync_jobs", "speed") + op.drop_column("lipsync_jobs", "script_text") + op.drop_column("lipsync_jobs", "voice_id") diff --git a/alembic/versions/074_ai_avatar_render_script_id_optional.py b/alembic/versions/074_ai_avatar_render_script_id_optional.py new file mode 100644 index 000000000..ffd2156a0 --- /dev/null +++ b/alembic/versions/074_ai_avatar_render_script_id_optional.py @@ -0,0 +1,36 @@ +"""ai_avatar_render_jobs.script_id 放宽为可空串（手动文案直生场景不关联文案库） + +Revision ID: 074_render_script_id_optional +Revises: 073_add_lipsync_tts_fields +Create Date: 2026-09-09 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "074_render_script_id_optional" +down_revision = "073_add_lipsync_tts_fields" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # 列保持 NOT NULL（空串占位），仅应用层允许不传；这里显式补 server_default 防止历史约束歧义 + with op.batch_alter_table("ai_avatar_render_jobs") as batch: + batch.alter_column( + "script_id", + existing_type=sa.String(length=36), + nullable=False, + server_default="", + ) + + +def downgrade() -> None: + with op.batch_alter_table("ai_avatar_render_jobs") as batch: + batch.alter_column( + "script_id", + existing_type=sa.String(length=36), + nullable=False, + server_default=None, + ) diff --git a/apps/api/app/api/routes/ai_avatar_render.py b/apps/api/app/api/routes/ai_avatar_render.py index 4c14961c5..02a381846 100644 --- a/apps/api/app/api/routes/ai_avatar_render.py +++ b/apps/api/app/api/routes/ai_avatar_render.py @@ -17,7 +17,10 @@ from app.dependencies import get_db_session from app.schemas.ai_avatar_render import ( AiAvatarRenderJobResponse, CreateAiAvatarRenderRequest, + SmartCoverRequest, + SmartCoverResponse, ) +from app.services.ai_avatar_cover_service import generate_smart_cover from app.services.ai_avatar_render_service import ( AiAvatarRenderError, AiAvatarRenderService, @@ -173,3 +176,34 @@ def retry_render_job( logger.warning("Celery 任务提交失败，重试任务已重置但未触发执行: %s", job.id) return job + + + +# ── POST /smart-cover — 智能获取封面（MediaKit 抽帧 + 评分选帧）──────── + + +@router.post("/smart-cover", response_model=SmartCoverResponse) +def generate_avatar_smart_cover( + body: SmartCoverRequest, + current_user: AuthenticatedUser = Depends(get_current_user), +) -> SmartCoverResponse: + """智能获取数字人视频封面. + + 复用智能剪辑的 MediaKit 抽帧 + 质量评分选最佳帧逻辑（非 FFmpeg 简单截帧）， + 并将选中帧转存到自家 OSS，返回非临时的封面公网 URL。 + + 前端「智能获取封面」按钮可直接调用本接口；不依赖渲染任务完成。 + """ + video_url = (body.video_url or "").strip() + if not video_url.startswith(("http://", "https://")): + raise HTTPException(status_code=400, detail="video_url 必须是合法的 HTTP/HTTPS URL") + + cover_url = generate_smart_cover(video_url, max_frames=body.max_frames) + if not cover_url: + return SmartCoverResponse( + cover_url="", + status="fallback_failed", + message="智能抽帧失败（MediaKit 不可用或抽帧异常），请稍后重试", + ) + logger.info("智能封面生成成功: user=%s", current_user.user.id) + return SmartCoverResponse(cover_url=cover_url, status="completed") diff --git a/apps/api/app/api/routes/lipsync.py b/apps/api/app/api/routes/lipsync.py index d627e3ee0..c0d989dea 100644 --- a/apps/api/app/api/routes/lipsync.py +++ b/apps/api/app/api/routes/lipsync.py @@ -13,7 +13,11 @@ from __future__ import annotations import logging from app.auth import AuthenticatedUser, get_current_user -from app.dependencies import get_cosyvoice_service, get_db_session, get_voice_clone_profile_repository +from app.dependencies import ( + get_cosyvoice_service, + get_db_session, + get_voice_clone_profile_repository, +) from app.schemas.lipsync import CreateLipsyncJobRequest, LipsyncJobResponse from app.services.lipsync_service import LipsyncService from app.services.mediakit_client import MediaKitError @@ -29,35 +33,16 @@ router = APIRouter() def _get_service( db: Session = Depends(get_db_session), + voice_clone_repo=Depends(get_voice_clone_profile_repository), cosyvoice_service: CosyVoiceService = Depends(get_cosyvoice_service), ) -> LipsyncService: - return LipsyncService(db, cosyvoice_service=cosyvoice_service) - - -def _resolve_voice_id( - raw_voice_id: str, - user_id: str, - voice_clone_repo, -) -> str: - """解析 voice_id：支持预设音色 ID 或克隆音色 profile UUID. - - 与 TTS 路由保持一致：命中 profile → 校验归属 → 取 CosyVoice voice_id。 - """ - try: - profile = voice_clone_repo.get(raw_voice_id) - except Exception as exc: - logger.error("查询克隆音色失败: voice_id=%s, error=%s", raw_voice_id, exc) - raise HTTPException( - status_code=400, - detail=f"voice_id 无效: {raw_voice_id}", - ) from exc - if profile is not None: - if profile.user_id != user_id: - raise HTTPException(status_code=403, detail="无权访问该音色") - if not profile.voice_id: - raise HTTPException(status_code=400, detail="音色克隆尚未完成，请稍后再试") - return profile.voice_id - return raw_voice_id + # voice_clone_repo 用于克隆音色 profile 解析；cosyvoice_service 用于 TTS 直生 + # （TTS 合成、音色解析、错误码归一化都在 LipsyncService 内部完成） + return LipsyncService( + db, + cosyvoice_service=cosyvoice_service, + voice_clone_repo=voice_clone_repo, + ) # ── POST /jobs — 提交对口型任务 ─────────────────────────────────────────── @@ -68,22 +53,22 @@ def create_lipsync_job( body: CreateLipsyncJobRequest, current_user: AuthenticatedUser = Depends(get_current_user), svc: LipsyncService = Depends(_get_service), - voice_clone_repo=Depends(get_voice_clone_profile_repository), ): """提交对口型任务. - #1809: 前端传 {voice_id, script_text, video_url}， - 后端内部调 TTS 合成音频，再提交 MediaKit。 + #1809/#1822: 前端传 {video_url, voice_id, script_text, speed?, emotion?}， + 后端内部解析音色、调 TTS 合成音频、转存 OSS，再提交 MediaKit； + 也支持直接传 {video_url, audio_url}。 """ - # 解析 voice_id（支持克隆音色 profile UUID） - actual_voice_id = _resolve_voice_id(body.voice_id, current_user.user.id, voice_clone_repo) - try: job = svc.create_job( user_id=current_user.user.id, video_url=body.video_url, - voice_id=actual_voice_id, + audio_url=body.audio_url, + voice_id=body.voice_id, script_text=body.script_text, + speed=body.speed, + emotion=body.emotion, enable_video_loop=body.enable_video_loop, project_id=body.project_id, ) @@ -97,12 +82,20 @@ def create_lipsync_job( detail={"code": "TTSSynthesisFailed", "message": str(exc)}, ) from exc except MediaKitError as exc: + # TTS 合成失败 / 音色无权访问 → 400/403；MediaKit 提交失败 → 502 + status_code = 502 + if exc.code in ("VoiceForbidden",): + status_code = 403 + elif exc.code in ("InvalidInput", "TTSInvalidParam", "VoiceNotReady"): + status_code = 400 + elif exc.code == "TTSSynthesisFailed": + status_code = 502 raise HTTPException( - status_code=502, + status_code=status_code, detail={ "code": exc.code, "message": str(exc), - "request_id": exc.request_id, + "request_id": getattr(exc, "request_id", ""), }, ) from exc except Exception as exc: @@ -156,23 +149,17 @@ def get_lipsync_job( ): """获取对口型任务详情. - 非终态任务：先返回 DB 缓存，挂后台刷新（下次轮询拿到新状态）。 + 非终态任务：先返回 DB 缓存，挂后台刷新（下次轮询拿到新状态）， + 避免 MediaKit 慢响应阻塞前端轮询。 """ - try: - logger.info(f"get_job debug: job_id={job_id}, user_type={type(current_user).__name__}") - job = svc.get_job(job_id, current_user.user.id) - if job is None: - raise HTTPException(status_code=404, detail="任务不存在") + job = svc.get_job(job_id, current_user.user.id) + if job is None: + raise HTTPException(status_code=404, detail="任务不存在") - if job.status not in ("completed", "failed"): - background.add_task(svc.refresh_job_status, job_id, current_user.user.id) + if job.status not in ("completed", "failed"): + background.add_task(svc.refresh_job_status, job_id, current_user.user.id) - return job - except HTTPException: - raise - except Exception as e: - logger.error(f"get_lipsync_job error: {type(e).__name__}: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=f"调试：{type(e).__name__}: {e}") from None + return job # ── POST /jobs/{job_id}/refresh — 刷新状态 ─────────────────────────────── diff --git a/apps/api/app/api/routes/tts.py b/apps/api/app/api/routes/tts.py index ec000a5b8..400b012e2 100755 --- a/apps/api/app/api/routes/tts.py +++ b/apps/api/app/api/routes/tts.py @@ -173,6 +173,14 @@ def synthesize( # job.voice_id 统一存解析后的 CosyVoice voice_id actual_voice_id = resolved_profile.voice_id + # 语速/情绪等合成参数随 metadata 落库，workflow 提交 CosyVoice 时读取透传 + synthesis_meta = { + "speed": request.speed, + "emotion": request.emotion or "", + } + if request.metadata_: + synthesis_meta.update(request.metadata_) + use_case = CreateTTSJobUseCase(repository) job = use_case.execute( user_id=user_id, @@ -180,7 +188,7 @@ def synthesize( voice_id=actual_voice_id, voice_model=request.voice_model, voice_clone_profile_id=voice_clone_profile_id, - metadata=request.metadata_, + metadata=synthesis_meta, ) # 提交 CosyVoice 合成任务 @@ -567,6 +575,7 @@ def preview_tts( text=request.text, voice_id=actual_voice_id, speed=request.speed, + emotion=request.emotion, ) except CosyVoiceError as e: raise HTTPException( diff --git a/apps/api/app/schemas/ai_avatar_render.py b/apps/api/app/schemas/ai_avatar_render.py index 2284bdfa2..88832017d 100644 --- a/apps/api/app/schemas/ai_avatar_render.py +++ b/apps/api/app/schemas/ai_avatar_render.py @@ -50,7 +50,7 @@ class CreateAiAvatarRenderRequest(BaseModel): """创建渲染任务请求.""" lipsync_job_id: str = Field(..., description="对口型任务 ID") - script_id: str = Field(..., description="文案 ID") + script_id: str = Field("", description="文案 ID（选自文案库时传；手动输入文案直生场景可留空）") b_roll_segments: list[BRollSegment] = Field(default_factory=list, description="B-roll 片段列表") title_config: dict[str, Any] = Field(default_factory=dict, description="标题配置") cover_config: dict[str, Any] = Field(default_factory=dict, description="封面配置") @@ -67,10 +67,8 @@ class CreateAiAvatarRenderRequest(BaseModel): @field_validator("script_id") @classmethod def validate_script_id(cls, v: str) -> str: - v = v.strip() - if not v: - raise ValueError("script_id 不能为空") - return v + # script_id 可选：手动输入文案（TTS 直生）场景不关联文案库条目 + return (v or "").strip() class AiAvatarRenderJobResponse(BaseModel): @@ -80,7 +78,7 @@ class AiAvatarRenderJobResponse(BaseModel): user_id: str project_id: str lipsync_job_id: str - script_id: str + script_id: str = "" b_roll_segments: list[dict[str, Any]] title_config: dict[str, Any] cover_config: dict[str, Any] @@ -109,3 +107,18 @@ class AiAvatarRenderProgressResponse(BaseModel): output_cover_url: str output_duration: float error_message: str + + +class SmartCoverRequest(BaseModel): + """智能封面请求 — MediaKit 抽帧 + 质量评分选最佳帧.""" + + video_url: str = Field(..., description="数字人视频 URL（对口型/渲染成片）") + max_frames: int = Field(5, ge=1, le=10, description="抽帧数量（默认 5）") + + +class SmartCoverResponse(BaseModel): + """智能封面响应.""" + + cover_url: str = Field("", description="封面图公网 URL（OSS，非临时）；失败为空") + status: str = Field("completed", description="completed / fallback_failed") + message: str = Field("", description="失败原因（如有）") diff --git a/apps/api/app/schemas/lipsync.py b/apps/api/app/schemas/lipsync.py index ac216b343..77fe9fa5b 100644 --- a/apps/api/app/schemas/lipsync.py +++ b/apps/api/app/schemas/lipsync.py @@ -1,11 +1,17 @@ -"""对口型 API Schema 定义 — #1796, #1809 参数调整.""" +"""对口型 API Schema 定义 — #1796 / #1809 / #1822. + +支持两种输入模式（二选一）： +1. TTS 直生模式（推荐）：传 voice_id + script_text（+ speed/emotion）， + 后端内部先调 CosyVoice 合成音频，再提交 MediaKit 对口型。 +2. 直接音频模式：传 video_url + audio_url（音频已由调用方准备好）。 +""" from __future__ import annotations from datetime import datetime from typing import Optional -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, model_validator class LipsyncJobResponse(BaseModel): @@ -17,6 +23,10 @@ class LipsyncJobResponse(BaseModel): video_url: str audio_url: str enable_video_loop: bool + voice_id: str = "" + script_text: str = "" + speed: float = 1.0 + emotion: str = "" mediakit_task_id: str status: str output_video_url: str @@ -33,45 +43,58 @@ class LipsyncJobResponse(BaseModel): class CreateLipsyncJobRequest(BaseModel): - """创建对口型任务请求 — #1809. + """创建对口型任务请求. - 前端传 {voice_id, script_text, video_url}， - 后端内部调 TTS 生成 audio_url 再提交 MediaKit。 + 两种模式（二选一）： + - TTS 直生：voice_id + script_text 必填（+ 可选 speed/emotion）；audio_url 留空。 + - 直接音频：video_url + audio_url 必填。 """ video_url: str = Field(..., description="人物视频 URL（MP4，≤30min，单人真人）") - voice_id: str = Field(..., description="音色 ID（预设音色或克隆音色 profile ID）") - script_text: str = Field(..., description="要合成的脚本文本") + + # 模式 2：直接音频 + audio_url: str = Field("", description="驱动音频 URL（mp3/aac/wav/m4a/flac）；直生模式留空") + + # 模式 1：TTS 直生 + voice_id: str = Field("", description="音色 ID（预置音色或克隆音色 profile UUID）") + script_text: str = Field("", description="要合成的文案（直生模式必填，最长 5000 字符）") + speed: float = Field(1.0, ge=0.5, le=2.0, description="语速（0.5-2.0），默认 1.0") + emotion: str = Field("", description="情绪（natural/excited/calm/friendly 或中文 自然/兴奋/沉稳/亲切）") + enable_video_loop: bool = Field(False, description="音频长于视频时是否循环画面") project_id: str = Field("", description="项目 ID（可选）") - @field_validator("video_url") - @classmethod - def validate_video_url(cls, v: str) -> str: - v = v.strip() - if not v: + @model_validator(mode="after") + def _validate_input_mode(self) -> "CreateLipsyncJobRequest": + video = (self.video_url or "").strip() + if not video: raise ValueError("video_url 不能为空") - if not v.startswith(("http://", "https://")): + if not video.startswith(("http://", "https://")): raise ValueError("video_url 必须是 HTTP/HTTPS URL") - lower = v.lower().split("?")[0] + lower = video.lower().split("?")[0] if not lower.endswith(".mp4"): raise ValueError("video_url 仅支持 MP4 格式") - return v - @field_validator("voice_id") - @classmethod - def validate_voice_id(cls, v: str) -> str: - v = v.strip() - if not v: - raise ValueError("voice_id 不能为空") - return v + has_audio = bool((self.audio_url or "").strip()) + has_tts = bool((self.voice_id or "").strip()) and bool((self.script_text or "").strip()) - @field_validator("script_text") - @classmethod - def validate_script_text(cls, v: str) -> str: - v = v.strip() - if not v: - raise ValueError("script_text 不能为空") - if len(v) > 5000: + if not has_audio and not has_tts: + raise ValueError( + "必须提供驱动音频：要么传 audio_url（直接音频模式），" + "要么同时传 voice_id + script_text（TTS 直生模式）" + ) + + if has_tts and len(self.script_text) > 5000: raise ValueError("script_text 最长 5000 字符") - return v + + if has_audio: + au = self.audio_url.strip() + if not au.startswith(("http://", "https://")): + raise ValueError("audio_url 必须是 HTTP/HTTPS URL") + au_lower = au.lower().split("?")[0] + allowed = (".mp3", ".aac", ".wav", ".m4a", ".flac") + if not any(au_lower.endswith(ext) for ext in allowed): + raise ValueError(f"audio_url 格式不支持，仅支持: {', '.join(allowed)}") + self.audio_url = au + + return self diff --git a/apps/api/app/schemas/tts.py b/apps/api/app/schemas/tts.py index 1e6b47513..535f03ba2 100644 --- a/apps/api/app/schemas/tts.py +++ b/apps/api/app/schemas/tts.py @@ -16,6 +16,7 @@ class TTSSynthesizeRequest(BaseModel): output_name: str = Field("", description="输出文件名") language: str = Field("zh-CN", description="语言") speed: float = Field(1.0, ge=0.5, le=2.0, description="语速") + emotion: str = Field("", description="情绪（natural/excited/calm/friendly，或中文 自然/兴奋/沉稳/亲切）") voice_model: str = Field("", description="语音模型名称") voice_clone_profile_id: str = Field("", description="关联的音色克隆档案 ID") format: str = Field("mp3", description="输出格式（mp3/wav/pcm）") @@ -109,6 +110,7 @@ class TTSPreviewRequest(BaseModel): text: str = Field(..., min_length=1, max_length=200, description="合成文本，限制 200 字") voice_id: str = Field(..., min_length=1, description="音色 ID") speed: float = Field(1.0, ge=0.5, le=2.0, description="语速") + emotion: str = Field("", description="情绪（natural/excited/calm/friendly，或中文）") pitch: float = Field(1.0, ge=0.5, le=2.0, description="音调（预留，当前未使用）") diff --git a/apps/api/app/services/ai_avatar_cover_service.py b/apps/api/app/services/ai_avatar_cover_service.py new file mode 100644 index 000000000..df0bd40aa --- /dev/null +++ b/apps/api/app/services/ai_avatar_cover_service.py @@ -0,0 +1,162 @@ +"""AI 数字人封面服务 — 复用智能剪辑的 MediaKit 抽帧 + 质量评分选最佳帧. + +与 generation_cover.py 的智能选帧能力对齐（不再用 FFmpeg 简单截帧）： +1. MediaKit extract_frames 抽取多帧（默认 5 帧，SpecifiedFrames 策略） +2. cover_frame_scorer.score_frames 按清晰度/亮度/色彩评分选最佳 +3. 下载最佳帧并转存 OSS，返回公网封面 URL + +降级：MediaKit 不可用或抽帧失败时返回空字符串，由调用方决定回退策略。 +""" + +from __future__ import annotations + +import logging +import tempfile +import uuid +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + + +def select_best_cover_frame(video_url: str, *, max_frames: int = 5) -> str: + """从视频抽取多帧并评分选最佳帧，返回最佳帧的临时 URL. + + Args: + video_url: 可公网访问的视频 URL + max_frames: 抽帧数量 + + Returns: + 最佳帧图片 URL；失败返回空字符串 + """ + if not video_url: + return "" + try: + from packages.shared.cover_frame_scorer import score_frames + from packages.shared.mediakit_client import get_mediakit_client + + mk = get_mediakit_client() + if not mk.is_available: + logger.warning("[数字人封面] MediaKit 未配置，无法智能抽帧") + return "" + + snapshots = mk.extract_frames( + video_url=video_url, + strategy="SpecifiedFrames", + max_frames=max_frames, + poll_interval=2.0, + max_poll_attempts=5, + max_retries=0, + ) + if not snapshots: + logger.warning("[数字人封面] MediaKit 未返回帧: %s", video_url[:80]) + return "" + + if len(snapshots) == 1: + return snapshots[0].get("image_url") or snapshots[0].get("url") or "" + + # 下载各帧评分 + import httpx + + candidates = [] + for snap in snapshots: + url = snap.get("image_url") or snap.get("url") or "" + if not url: + continue + tmp_path: Optional[str] = None + try: + resp = httpx.get(url, timeout=15, follow_redirects=True) + resp.raise_for_status() + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp: + tmp.write(resp.content) + tmp_path = tmp.name + candidates.append({"image_path": tmp_path, "url": url}) + except Exception: + candidates.append({"image_path": None, "url": url, "score": 0.0}) + + if not candidates: + return snapshots[0].get("image_url") or snapshots[0].get("url") or "" + + scored = score_frames(candidates) + best = scored[0] if scored else None + best_url = best.get("url", "") if best else "" + + # 清理临时文件 + for c in candidates: + p = c.get("image_path") + if p: + try: + Path(p).unlink(missing_ok=True) + except Exception: + pass + + logger.info( + "[数字人封面] 智能选帧完成: candidates=%d best_score=%s", + len(candidates), + best.get("score") if best else "n/a", + ) + return best_url + + except Exception: + logger.warning("[数字人封面] 智能选帧失败", exc_info=True) + return "" + + +def persist_cover_to_oss(frame_url: str, *, job_id: str = "", prefix: str = "ai-avatar/covers") -> str: + """下载帧图并转存到 OSS，返回公网封面 URL. + + Args: + frame_url: MediaKit 返回的临时帧图 URL + job_id: 关联任务 ID（用于 OSS key 命名） + prefix: OSS key 前缀 + + Returns: + OSS 公网 URL；失败回退原始 frame_url + """ + if not frame_url: + return "" + tmp_path: Optional[str] = None + try: + import httpx + + resp = httpx.get(frame_url, timeout=30, follow_redirects=True) + resp.raise_for_status() + if not resp.content: + return frame_url + + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp: + tmp.write(resp.content) + tmp_path = tmp.name + + from packages.shared.storage import get_shared_storage_service + + storage = get_shared_storage_service() + token = job_id or uuid.uuid4().hex[:12] + cover_key = f"{prefix}/{token}/cover_{uuid.uuid4().hex[:8]}.jpg" + public_url = storage.upload_file( + file_or_path=tmp_path, + storage_key=cover_key, + content_type="image/jpeg", + ) + logger.info("[数字人封面] 封面已转存 OSS: key=%s", cover_key) + return public_url or frame_url + except Exception: + logger.warning("[数字人封面] 封面转存 OSS 失败，返回原始 URL", exc_info=True) + return frame_url + finally: + if tmp_path: + try: + Path(tmp_path).unlink(missing_ok=True) + except Exception: + pass + + +def generate_smart_cover(video_url: str, *, job_id: str = "", max_frames: int = 5) -> str: + """一站式：MediaKit 智能抽帧选最佳 → 转存 OSS，返回封面公网 URL. + + 供独立封面接口与渲染管线复用。失败返回空字符串。 + """ + best_frame = select_best_cover_frame(video_url, max_frames=max_frames) + if not best_frame: + return "" + return persist_cover_to_oss(best_frame, job_id=job_id) diff --git a/apps/api/app/services/ai_avatar_render_service.py b/apps/api/app/services/ai_avatar_render_service.py index d13af622d..21266333b 100644 --- a/apps/api/app/services/ai_avatar_render_service.py +++ b/apps/api/app/services/ai_avatar_render_service.py @@ -53,8 +53,8 @@ class AiAvatarRenderService: *, user_id: str, lipsync_job_id: str, - script_id: str, - b_roll_segments: list[dict[str, Any]], + script_id: str = "", + b_roll_segments: list[dict[str, Any]] | None = None, title_config: dict[str, Any], cover_config: dict[str, Any], project_id: str = "", @@ -83,17 +83,19 @@ class AiAvatarRenderService: if not lipsync_job.output_video_url: raise AiAvatarRenderError("对口型任务输出视频 URL 为空", code="LipsyncJobNoOutput") - # 2. 验证文案归属 - script = ( - self.db.query(ScriptModel) - .filter( - ScriptModel.id == script_id, - ScriptModel.user_id == user_id, + # 2. 验证文案归属（仅当选了文案库条目时；手动输入文案直生场景 script_id 可空） + script_id = (script_id or "").strip() + if script_id: + script = ( + self.db.query(ScriptModel) + .filter( + ScriptModel.id == script_id, + ScriptModel.user_id == user_id, + ) + .first() ) - .first() - ) - if script is None: - raise AiAvatarRenderError("文案不存在或无权访问", code="ScriptNotFound") + if script is None: + raise AiAvatarRenderError("文案不存在或无权访问", code="ScriptNotFound") # 3. 创建渲染任务 job_id = str(uuid.uuid4()) @@ -103,7 +105,7 @@ class AiAvatarRenderService: project_id=project_id, lipsync_job_id=lipsync_job_id, script_id=script_id, - b_roll_segments=[s if isinstance(s, dict) else s.model_dump() for s in b_roll_segments], + b_roll_segments=[s if isinstance(s, dict) else s.model_dump() for s in (b_roll_segments or [])], title_config=title_config, cover_config=cover_config, status="pending", @@ -288,7 +290,22 @@ class AiAvatarRenderService: output_video_url = self._upload_to_oss(output_video_path, f"ai-avatar/{job_id}/output.mp4") job.output_video_url = output_video_url - if cover_path: + # 封面：优先复用智能剪辑的 MediaKit 抽帧 + 质量评分选最佳帧； + # MediaKit 不可用时回退到 FFmpeg 已按 cover_config 抽取的 cover_path + smart_cover_url = "" + if output_video_url: + try: + from app.services.ai_avatar_cover_service import ( + generate_smart_cover, + ) + + smart_cover_url = generate_smart_cover(output_video_url, job_id=job_id, max_frames=5) + except Exception: + logger.warning("智能封面(MediaKit)失败，回退 FFmpeg 封面 job_id=%s", job_id, exc_info=True) + + if smart_cover_url: + job.output_cover_url = smart_cover_url + elif cover_path: output_cover_url = self._upload_to_oss(cover_path, f"ai-avatar/{job_id}/cover.jpg") job.output_cover_url = output_cover_url diff --git a/apps/api/app/services/lipsync_service.py b/apps/api/app/services/lipsync_service.py index 257d71ed1..36fa4b20c 100644 --- a/apps/api/app/services/lipsync_service.py +++ b/apps/api/app/services/lipsync_service.py @@ -1,15 +1,16 @@ """对口型 Service — #1796 MediaKit 对口型业务逻辑, #1809 参数调整. 职责： -- 创建/查询/取消对口型任务 -- 调用 TTS 合成音频（#1809：前端不再传 audio_url） +- 创建/查询对口型任务 +- 双输入模式：TTS 直生（voice_id + script_text，内部先合成音频转存 OSS）或直接音频（audio_url） - 调用 MediaKit 客户端提交异步任务 -- 轮询更新任务状态 +- 轮询更新任务状态（中间状态同步 DB，成片转存自家 OSS） - 用户隔离（每个用户只能操作自己的任务） """ from __future__ import annotations +import io import logging import uuid from datetime import datetime, timezone @@ -26,7 +27,9 @@ from app.services.mediakit_client import ( from sqlalchemy.orm import Session from packages.adapters.sqlalchemy_impl.models import LipsyncJobModel -from packages.application.cosyvoice_service import CosyVoiceError, CosyVoiceService +from packages.application.cosyvoice_service import CosyVoiceError, normalize_emotion +from packages.shared.storage import get_shared_storage_service +from packages.shared.url_security import ALLOWED_AUDIO_MIME_TYPES, safe_download_bytes logger = logging.getLogger(__name__) @@ -38,19 +41,98 @@ class LipsyncService: self, db: Session, client: Optional[MediaKitClient] = None, - cosyvoice_service: Optional[CosyVoiceService] = None, + cosyvoice_service=None, + voice_clone_repo=None, ): self.db = db self.client = client or get_mediakit_client() - self._cosyvoice_service = cosyvoice_service + self._cosyvoice = cosyvoice_service + self._voice_clone_repo = voice_clone_repo - @property - def cosyvoice_service(self) -> CosyVoiceService: - if self._cosyvoice_service is None: + def _get_cosyvoice(self): + """延迟获取 CosyVoiceService（与 tts 路由一致，含 OSS 预签名配置）.""" + if self._cosyvoice is None: from app.dependencies import get_cosyvoice_service - self._cosyvoice_service = get_cosyvoice_service() - return self._cosyvoice_service + self._cosyvoice = get_cosyvoice_service() + return self._cosyvoice + + def _resolve_voice_id(self, voice_id: str, user_id: str) -> str: + """将克隆音色 profile UUID 解析为 CosyVoice voice_id。 + + 与 /tts/synthesize 保持一致：命中 profile → 校验归属 → 返回其 voice_id； + 未命中（预置音色 ID 或克隆 CosyVoice voice_id）原样返回。 + """ + if not voice_id: + return "" + if self._voice_clone_repo is None: + try: + from app.dependencies import get_voice_clone_profile_repository + + self._voice_clone_repo = get_voice_clone_profile_repository(self.db) + except Exception: + return voice_id + try: + profile = self._voice_clone_repo.get(voice_id) + except Exception: + return voice_id + if profile is None: + return voice_id + if getattr(profile, "user_id", "") != user_id: + raise MediaKitError("无权访问该音色", code="VoiceForbidden") + if not getattr(profile, "voice_id", ""): + raise MediaKitError("音色克隆尚未完成，请稍后再试", code="VoiceNotReady") + return profile.voice_id + + def _synthesize_and_persist_audio( + self, + *, + user_id: str, + job_id: str, + voice_id: str, + script_text: str, + speed: float, + emotion: str, + ) -> str: + """TTS 直生：调 CosyVoice 合成音频并转存 OSS，返回可公网访问的音频 URL. + + Raises: + MediaKitError: 合成失败 + """ + actual_voice_id = self._resolve_voice_id(voice_id, user_id) + cosyvoice = self._get_cosyvoice() + try: + result = cosyvoice.submit_synthesize_task( + text=script_text, + voice_id=actual_voice_id, + speed=speed, + emotion=normalize_emotion(emotion), + ) + except CosyVoiceError as exc: + raise MediaKitError(f"TTS 合成失败: {exc}", code="TTSSynthesisFailed") from exc + except ValueError as exc: + raise MediaKitError(f"TTS 参数错误: {exc}", code="TTSInvalidParam") from exc + + temp_url = result.get("audio_url", "") + if not temp_url: + raise MediaKitError("TTS 未返回音频 URL", code="TTSNoAudio") + + # 转存到自家 OSS，避免临时 URL 过期导致 MediaKit 拉取失败 + try: + audio_data = safe_download_bytes( + temp_url, + purpose="lipsync_tts_audio", + allowed_mime_types=ALLOWED_AUDIO_MIME_TYPES, + timeout=60.0, + ) + storage = get_shared_storage_service() + storage_key = f"lipsync-tts/{user_id}/{job_id}.mp3" + permanent_url = storage.upload_file(io.BytesIO(audio_data), storage_key, content_type="audio/mpeg") + logger.info("对口型 TTS 音频已转存 OSS: job_id=%s key=%s", job_id, storage_key) + return permanent_url + except Exception as exc: + logger.warning("TTS 音频转存 OSS 失败，回退临时 URL: job_id=%s err=%s", job_id, exc) + return temp_url # ── 创建任务 ────────────────────────────────────────────────────────── @@ -59,47 +141,42 @@ class LipsyncService: *, user_id: str, video_url: str, - voice_id: str, - script_text: str, + audio_url: str = "", + voice_id: str = "", + script_text: str = "", + speed: float = 1.0, + emotion: str = "", enable_video_loop: bool = False, project_id: str = "", ) -> LipsyncJobModel: """创建对口型任务并提交到 MediaKit. - #1809: 内部调 TTS 合成音频，不再由前端传 audio_url。 + 两种输入模式： + - TTS 直生：voice_id + script_text（audio_url 留空），后端先合成音频 + - 直接音频：提供 audio_url Raises: - CosyVoiceError: TTS 合成失败 - MediaKitError: API 调用失败 + MediaKitError: TTS 合成或 MediaKit 提交失败 """ - # 1. 调 TTS 合成音频 - try: - tts_result = self.cosyvoice_service.synthesize_speech( - text=script_text, - voice_id=voice_id, - ) - audio_url = tts_result.audio_url - except CosyVoiceError as exc: - logger.error("TTS 合成失败: voice_id=%s, error=%s", voice_id, exc) - # 创建失败记录 - job_id = str(uuid.uuid4()) - job = LipsyncJobModel( - id=job_id, + # 0. TTS 直生模式：先合成音频（在创建 DB 记录之前完成，失败直接抛出） + if not audio_url: + if not (voice_id and script_text): + raise MediaKitError( + "必须提供 audio_url 或 voice_id+script_text", + code="InvalidInput", + ) + # 预合成：用临时 job_id 命名 OSS 对象 + pre_job_id = str(uuid.uuid4()) + audio_url = self._synthesize_and_persist_audio( user_id=user_id, - project_id=project_id, - video_url=video_url, - audio_url="", - enable_video_loop=enable_video_loop, - status="failed", - error_message=f"TTS 合成失败: {exc}", - error_code="TTSSynthesisFailed", + job_id=pre_job_id, + voice_id=voice_id, + script_text=script_text, + speed=speed, + emotion=emotion, ) - self.db.add(job) - self.db.commit() - self.db.refresh(job) - raise - # 2. 创建数据库记录 + # 1. 创建数据库记录 job_id = str(uuid.uuid4()) job = LipsyncJobModel( id=job_id, @@ -108,6 +185,10 @@ class LipsyncService: video_url=video_url, audio_url=audio_url, enable_video_loop=enable_video_loop, + voice_id=voice_id or "", + script_text=script_text or "", + speed=speed, + emotion=normalize_emotion(emotion), status="pending", ) self.db.add(job) @@ -192,12 +273,14 @@ class LipsyncService: return job mk_status = status_data.get("status", STATUS_RUNNING) - logger.info(f"MediaKit status for {job_id}: {mk_status}, data={status_data}") + logger.info("MediaKit 对口型状态 [%s]: %s", job_id, mk_status) if mk_status == STATUS_COMPLETED: result = status_data.get("result", {}) job.status = STATUS_COMPLETED - job.output_video_url = result.get("video_url", "") + output_url = result.get("video_url", "") + # MediaKit 输出为临时 URL，转存自家 OSS 防止过期（失败则回退临时 URL） + job.output_video_url = self._persist_output_video(output_url, job_id, user_id) job.output_duration = result.get("duration", 0.0) job.completed_at = datetime.now(timezone.utc) elif mk_status == STATUS_FAILED: @@ -207,13 +290,37 @@ class LipsyncService: job.error_code = error.get("code", "TaskFailed") job.completed_at = datetime.now(timezone.utc) else: - # 中间状态（running/processing/queued 等）同步到 DB - job.status = mk_status + # 中间状态（running/processing/queued 等）同步到 DB，避免前端永远卡在 submitted + if isinstance(mk_status, str) and mk_status: + job.status = mk_status job.updated_at = datetime.now(timezone.utc) self.db.commit() self.db.refresh(job) return job + def _persist_output_video(self, temp_url: str, job_id: str, user_id: str) -> str: + """将 MediaKit 输出的临时视频 URL 转存到自家 OSS. + + 失败时回退返回原始临时 URL，不影响任务完成。 + """ + if not temp_url: + return "" + try: + import httpx + + with httpx.Client(timeout=180.0, follow_redirects=True) as client: + resp = client.get(temp_url) + resp.raise_for_status() + data = resp.content + storage = get_shared_storage_service() + storage_key = f"lipsync-outputs/{user_id}/{job_id}.mp4" + permanent_url = storage.upload_file(io.BytesIO(data), storage_key, content_type="video/mp4") + logger.info("对口型输出视频已转存 OSS: job_id=%s key=%s", job_id, storage_key) + return permanent_url or temp_url + except Exception as exc: + logger.warning("对口型输出视频转存 OSS 失败，回退临时 URL: job_id=%s err=%s", job_id, exc) + return temp_url + # ── 取消任务 ────────────────────────────────────────────────────────── def cancel_job(self, job_id: str, user_id: str) -> Optional[LipsyncJobModel]: diff --git a/apps/web/src/pages/ai-avatar/AiAvatarPage.tsx b/apps/web/src/pages/ai-avatar/AiAvatarPage.tsx index 0dd14fe60..14a43197a 100644 --- a/apps/web/src/pages/ai-avatar/AiAvatarPage.tsx +++ b/apps/web/src/pages/ai-avatar/AiAvatarPage.tsx @@ -19,7 +19,13 @@ import { createLipsyncJob, getLipsyncJob, submitRender, + generateSmartCover, } from "./api/aiAvatar" +import { + normalizeEmotion, + buildTitleConfigPayload, + buildCoverConfigPayload, +} from "./utils/contract" /** 面板折叠状态 */ type PanelKey = "video" | "voice" | "script" | "title" | "cover" @@ -40,6 +46,8 @@ const AiAvatarPage: React.FC = () => { "generating", ) const [lipsyncErrorMessage, setLipsyncErrorMessage] = useState("") + /* ── 智能封面加载态 ── */ + const [smartCoverLoading, setSmartCoverLoading] = useState(false) /* ── 对口型轮询 ── */ const lipsyncTimerRef = useRef | null>(null) @@ -87,12 +95,13 @@ const AiAvatarPage: React.FC = () => { message.error("获取出镜视频播放地址失败，请重新选择素材") return } - // ② voice_id（预设/克隆 UUID 均由后端内部调 TTS）+ script_text + video_url + // ② 模式A TTS直生：video_url + voice_id + script_text，语速/情绪英文枚举透传（#1822） const payload = { voice_id: voice.voice_id, script_text: state.scriptText, video_url: videoUrl, - emotion: state.emotion, // 传递情绪参数 + speed: state.speed, // 语速 0.5~2.0 + emotion: normalizeEmotion(state.emotion), // natural/excited/calm/friendly } console.log("[对口型] createLipsyncJob 请求:", payload) const job = await createLipsyncJob(payload) @@ -135,7 +144,7 @@ const AiAvatarPage: React.FC = () => { message.error(err instanceof Error ? err.message : "对口型任务提交失败，请重试") } // eslint-disable-next-line react-hooks/exhaustive-deps - }, [state.selectedVideo, state.selectedVoice, state.scriptText]) + }, [state.selectedVideo, state.selectedVoice, state.scriptText, state.speed, state.emotion]) // 取消对口型生成 const handleCancelLipsync = useCallback(() => { @@ -168,8 +177,10 @@ const AiAvatarPage: React.FC = () => { lipsync_job_id: state.lipsyncJob.id, script_id: state.script?.id, b_roll_segments: state.bRollSegments as never, - title_config: state.titleConfig as unknown as Record, - cover_config: state.coverConfig as unknown as Record, + // 字段映射：build_title_drawtext_filter 真实口径 text/font_size/font_color/position/... + title_config: buildTitleConfigPayload(state.titleConfig), + // cover_config：智能封面 cover_url + 截帧 timestamp + cover_config: buildCoverConfigPayload(state.coverConfig, state.coverConfig.smart_cover_url), resolution: state.resolution, }) message.success("渲染任务已提交，可在视频管理中查看进度") @@ -189,6 +200,37 @@ const AiAvatarPage: React.FC = () => { state.resolution, ]) + /* ── 智能封面：调后端 MediaKit 选帧接口（#1822） ── */ + const handleSmartCover = useCallback(async () => { + // 基于对口型成片抽帧，必须先完成对口型 + const videoUrl = state.lipsyncJob?.output_video_url + if (state.lipsyncJob?.status !== "completed" || !videoUrl) { + message.warning("请先生成对口型视频，完成后再智能获取封面") + return + } + setSmartCoverLoading(true) + try { + const res = await generateSmartCover(videoUrl, 5) + if (res.cover_url) { + state.setCoverConfig((prev) => ({ + ...prev, + mode: "auto_frame", + smart_cover_url: res.cover_url, + thumbnail_url: res.cover_url, + })) + message.success("智能封面已生成") + } else { + message.error(res.message || "智能封面生成失败，请稍后重试") + } + } catch (err) { + console.error("智能封面生成失败:", err) + message.error(err instanceof Error ? err.message : "智能封面生成失败，请重试") + } finally { + setSmartCoverLoading(false) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [state.lipsyncJob]) + /* ── 配置汇总 ── */ const summary = { videoName: state.selectedVideo?.name || null, @@ -287,6 +329,9 @@ const AiAvatarPage: React.FC = () => { onCoverConfigChange={(partial) => state.setCoverConfig((prev) => ({ ...prev, ...partial })) } + onSmartCover={handleSmartCover} + smartCoverLoading={smartCoverLoading} + canSmartCover={state.lipsyncJob?.status === "completed"} resolution={state.resolution} onResolutionChange={state.setResolution} isGenerating={state.isGenerating} diff --git a/apps/web/src/pages/ai-avatar/api/aiAvatar.ts b/apps/web/src/pages/ai-avatar/api/aiAvatar.ts index c65666bb4..87d544150 100644 --- a/apps/web/src/pages/ai-avatar/api/aiAvatar.ts +++ b/apps/web/src/pages/ai-avatar/api/aiAvatar.ts @@ -1,5 +1,5 @@ /** - * AI数字人 — API 封装 + * AI数字人 — API 封装（#1822 契约对齐） */ import apiClient from "@/api/client" import type { Script, LipsyncJob, RenderJob, BRollSegment } from "../types" @@ -28,18 +28,26 @@ export const deleteScript = async (id: string): Promise => { await apiClient.delete(`/scripts/${id}`) } -/* ── 素材单查（用于拿到 file_url 传给对口型等新接口） ── */ +/* ── 素材单查（拿到 file_url 作为对口型的 video_url） ── */ export const getAssetById = async (id: string): Promise<{ file_url?: string; id: string }> => { const response = await apiClient.get<{ file_url?: string; id: string }>(`/assets/${id}`) return response.data } -/* ── 对口型 ── */ +/* ── 对口型（模式A：TTS 直生，后端内部合成音频；不要先调 TTS 拿 audio_url） ── */ export const createLipsyncJob = async (data: { - voice_id: string - script_text: string + /** 人物视频 URL（MP4）；由素材 id 经 getAssetById 拿 file_url，禁止传 video_asset_id */ video_url: string - emotion?: string // 情绪参数：natural/excited/calm/friendly + /** 音色 ID（预置音色 或 克隆音色 profile UUID，后端会解析） */ + voice_id: string + /** 要合成的文案（手动输入或文案库内容） */ + script_text: string + /** 语速 0.5~2.0，默认 1.0 */ + speed?: number + /** 情绪英文枚举：natural/excited/calm/friendly */ + emotion?: string + enable_video_loop?: boolean + project_id?: string }): Promise => { const response = await apiClient.post("/lipsync/jobs", data) return response.data @@ -50,6 +58,18 @@ export const getLipsyncJob = async (id: string): Promise => { return response.data } +/* ── 智能封面（MediaKit 抽帧 + 质量评分选最佳帧，独立于渲染任务） ── */ +export const generateSmartCover = async ( + video_url: string, + max_frames = 5, +): Promise<{ cover_url: string; status: string; message: string }> => { + const response = await apiClient.post<{ cover_url: string; status: string; message: string }>( + "/ai-avatar/render/smart-cover", + { video_url, max_frames }, + ) + return response.data +} + /* ── 渲染 ── */ export const submitRender = async (data: { lipsync_job_id: string diff --git a/apps/web/src/pages/ai-avatar/components/PanelCoverAndGenerate.tsx b/apps/web/src/pages/ai-avatar/components/PanelCoverAndGenerate.tsx index e5c7d00ea..ea9961802 100644 --- a/apps/web/src/pages/ai-avatar/components/PanelCoverAndGenerate.tsx +++ b/apps/web/src/pages/ai-avatar/components/PanelCoverAndGenerate.tsx @@ -17,6 +17,10 @@ interface PanelCoverAndGenerateProps { onResolutionChange: (r: string) => void isGenerating: boolean onGenerate: () => void + /** 智能获取封面（MediaKit 选帧） */ + onSmartCover: () => void + smartCoverLoading: boolean + canSmartCover: boolean /** 配置汇总信息 */ summary: { videoName: string | null @@ -50,6 +54,9 @@ const PanelCoverAndGenerate: React.FC = ({ onResolutionChange, isGenerating, onGenerate, + onSmartCover, + smartCoverLoading, + canSmartCover, summary, }) => { const uploadInputRef = useRef(null) @@ -69,9 +76,10 @@ const PanelCoverAndGenerate: React.FC = ({ e.target.value = "" } - /** 从视频截取（使用配置的帧时间，默认首帧） */ - const handleCaptureFromVideo = () => { + /** 智能获取封面（调后端 MediaKit 抽帧评分选最佳帧，#1822） */ + const handleSmartCover = () => { onCoverConfigChange({ mode: "auto_frame" }) + onSmartCover() } const lipsync = summary.lipsyncStatus ? LIPSYNC_STATUS_LABEL[summary.lipsyncStatus] : null @@ -93,9 +101,11 @@ const PanelCoverAndGenerate: React.FC = ({