xiaoxia-saas/apps/api/app/tasks/lipsync_tts.py

"""AI 数字人对口型 TTS 异步任务 — 将 TTS 合成从 HTTP 请求移至 Celery 后台执行.

优化目标：将 create_job 的 API 响应时间从 6~35s 降到 <1s。
任务流程：
1. 创建新 DB session，加载 job 记录
2. 调用 CosyVoice 合成音频
3. 下载音频并转存到自家 OSS
4. 更新 job 的 audio_url
5. 签名 URL 并提交到 MediaKit
6. 更新 job 状态为 submitted
7. 异常时标记 job 为 failed

注意：使用 @shared_task 而非绑定到某个 celery_app 实例，
确保任务能被 Worker 侧 celery_app 正确注册，同时 API 侧 send_task/apply_async 仍可正常调用。
"""

import io
import logging
from datetime import datetime, timezone
from urllib.parse import urlparse

from celery import shared_task

logger = logging.getLogger(__name__)

# MediaKit 预签名 URL 有效期（7天，秒），与 LipsyncService._sign_media_url 保持一致
_MEDIAKIT_URL_TTL_SECONDS = 7 * 24 * 3600


def _sign_media_url(url: str) -> str:
    """对自家 OSS 私有桶 URL 重签长有效期预签名.

    - 自家 OSS URL → 重签 7 天有效期
    - 外部临时 URL → 原样透传
    - 任何异常降级原样返回，不阻断主流程
    """
    if not url:
        return url
    try:
        from packages.shared.storage import get_shared_storage_service

        storage = get_shared_storage_service()
        public_base = getattr(storage, "public_url", "")
        if not isinstance(public_base, str) or not public_base:
            return url
        own_host = urlparse(public_base).netloc.lower()
        host = urlparse(url).netloc.lower()
        if not own_host or host != own_host:
            return url
        signed = storage.get_download_url(url, expires_seconds=_MEDIAKIT_URL_TTL_SECONDS)
        return signed or url
    except Exception as exc:  # noqa: BLE001
        logger.warning("[lipsync_tts] URL 重签失败，原样返回: url_prefix=%s err=%s", url[:80], exc)
        return url


def _split_script_into_sentences(script_text: str) -> list[str]:
    """按句号/问号/感叹号/分号/换行分句（与前端 splitScriptIntoSentences 一致）."""
    import re

    text = (script_text or "").strip()
    if not text:
        return []
    parts = re.split(r"[。！？!?；;\n\r]+", text)
    return [p.strip() for p in parts if p.strip()]


def _compute_sentence_timings(audio_data: bytes, script_text: str, total_duration: float) -> list[dict]:
    """基于 TTS 音频的静音检测，精确计算每句文案的起止时间.

    使用 ffmpeg silencedetect 检测静音段，将静音点与句子边界对齐。
    比字数比例估算准确得多。

    Args:
        audio_data: TTS 音频二进制数据（MP3）
        script_text: 文案全文
        total_duration: 音频总时长（秒）

    Returns:
        list[{"index": int, "text": str, "start_time": float, "end_time": float}]
    """
    import re
    import subprocess
    import tempfile

    sentences = _split_script_into_sentences(script_text)
    if not sentences:
        return []

    # 写入临时音频文件
    with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as tmp:
        tmp.write(audio_data)
        tmp_path = tmp.name

    try:
        # 用 ffmpeg silencedetect 检测静音段
        result = subprocess.run(
            [
                "ffmpeg",
                "-i",
                tmp_path,
                "-af",
                "silencedetect=noise=-25dB:d=0.3",
                "-f",
                "null",
                "-",
            ],
            capture_output=True,
            text=True,
            timeout=30,
        )
        stderr = result.stderr or ""

        # 解析静音结束时间点（silence_end: X.XXX）
        silence_ends = []
        for match in re.finditer(r"silence_end:\s*([\d.]+)", stderr):
            t = float(match.group(1))
            if 0 < t < total_duration:
                silence_ends.append(t)

        # 如果没有检测到足够的静音点，降级为字数比例估算
        if len(silence_ends) < len(sentences) - 1:
            logger.warning(
                "[sentence_timings] 静音点不足(%d < %d)，降级为字数比例估算",
                len(silence_ends),
                len(sentences) - 1,
            )
            return _estimate_sentence_timings_by_chars(sentences, total_duration)

        # 贪心匹配：N-1 个句子边界对应 N-1 个静音点
        # 按时间均匀分布期望值，选择最近的静音点
        n_boundaries = len(sentences) - 1
        boundaries = []
        used_indices = set()

        for i in range(n_boundaries):
            # 期望的边界位置（按句子数量均匀分布）
            expected_pos = (i + 1) / len(sentences) * total_duration
            # 找最近的未使用静音点
            best_idx = None
            best_dist = float("inf")
            for j, t in enumerate(silence_ends):
                if j in used_indices:
                    continue
                dist = abs(t - expected_pos)
                if dist < best_dist:
                    best_dist = dist
                    best_idx = j
            if best_idx is not None:
                used_indices.add(best_idx)
                boundaries.append(silence_ends[best_idx])

        boundaries.sort()

        # 构建 sentence_timings
        timings = []
        prev_end = 0.0
        for i, sent in enumerate(sentences):
            start = prev_end
            end = boundaries[i] if i < len(boundaries) else total_duration
            timings.append(
                {
                    "index": i,
                    "text": sent,
                    "start_time": round(start, 2),
                    "end_time": round(end, 2),
                }
            )
            prev_end = end

        return timings

    except Exception as exc:
        logger.warning("[sentence_timings] 静音检测异常，降级为字数比例估算: %s", exc)
        return _estimate_sentence_timings_by_chars(sentences, total_duration)
    finally:
        import os

        try:
            os.unlink(tmp_path)
        except Exception:
            pass


def _estimate_sentence_timings_by_chars(sentences: list[str], total_duration: float) -> list[dict]:
    """降级方案：按字数比例估算句子时间（与原前端逻辑一致）."""
    if not sentences or total_duration <= 0:
        return []
    total_chars = sum(len(s.replace(r"\s", "")) for s in sentences)
    if total_chars == 0:
        return []

    timings = []
    acc = 0
    for i, sent in enumerate(sentences):
        chars = len(sent.replace(r"\s", ""))
        start = (acc / total_chars) * total_duration
        end = ((acc + chars) / total_chars) * total_duration
        timings.append(
            {
                "index": i,
                "text": sent,
                "start_time": round(start, 2),
                "end_time": round(end, 2),
            }
        )
        acc += chars
    return timings


@shared_task(
    bind=True,
    name="lipsync_tts.synthesize_and_submit",
    max_retries=2,
    default_retry_delay=30,
)
def tts_synthesize_and_submit(
    self,
    job_id: str,
    user_id: str,
    voice_id: str,
    script_text: str,
    speed: float,
    emotion: str,
):
    """异步执行 TTS 合成 + OSS 转存 + MediaKit 提交.

    在 Celery worker 中运行，不阻塞 HTTP 请求。
    """
    from app.services.mediakit_client import MediaKitError, get_mediakit_client
    from sqlalchemy.orm import Session as DBSession

    from packages.adapters.sqlalchemy_impl.models import LipsyncJobModel
    from packages.application.cosyvoice_service import CosyVoiceError, CosyVoiceService
    from packages.shared.url_security import safe_download_bytes

    # SessionLocal 获取：
    # - API 容器：app.db.SessionLocal（环境变量完整，导入即建引擎）
    # - Worker 容器：worker_app.db.SessionLocal（Worker 自己的 settings 初始化引擎）
    # API 侧没有 worker_app 模块 → ImportError 直接回退；
    # Worker 侧 app.db 会因缺少 API 专有环境变量抛 pydantic ValidationError，
    # 此时也要回退到 worker_app.db。
    try:
        from worker_app.db import SessionLocal  # type: ignore
    except Exception:  # noqa: BLE001
        from app.db import SessionLocal  # type: ignore

    db: DBSession = SessionLocal()
    try:
        job = (
            db.query(LipsyncJobModel)
            .filter(
                LipsyncJobModel.id == job_id,
                LipsyncJobModel.user_id == user_id,
            )
            .first()
        )

        if job is None:
            logger.error("[lipsync_tts] Job not found: job_id=%s", job_id)
            return

        # 已取消的任务不再处理
        if job.status == "cancelled":
            logger.info("[lipsync_tts] Job already cancelled, skipping: job_id=%s", job_id)
            return

        # 1. TTS 合成
        try:
            cosyvoice = CosyVoiceService()
            result = cosyvoice.submit_synthesize_task(
                text=script_text,
                voice_id=voice_id,
                speed=speed,
                emotion=emotion,
            )
        except CosyVoiceError as exc:
            logger.error("[lipsync_tts] TTS 合成失败: job_id=%s err=%s", job_id, exc)
            job.status = "failed"
            job.error_message = f"TTS 合成失败: {exc}"
            job.error_code = "TTSSynthesisFailed"
            job.updated_at = datetime.now(timezone.utc)
            db.commit()
            return
        except ValueError as exc:
            logger.error("[lipsync_tts] TTS 参数错误: job_id=%s err=%s", job_id, exc)
            job.status = "failed"
            job.error_message = f"TTS 参数错误: {exc}"
            job.error_code = "TTSInvalidParam"
            job.updated_at = datetime.now(timezone.utc)
            db.commit()
            return

        temp_url = result.get("audio_url", "")
        if not temp_url:
            logger.error("[lipsync_tts] TTS 未返回音频 URL: job_id=%s", job_id)
            job.status = "failed"
            job.error_message = "TTS 未返回音频 URL"
            job.error_code = "TTSNoAudio"
            job.updated_at = datetime.now(timezone.utc)
            db.commit()
            return

        # 2. 下载并转存到自家 OSS
        try:
            audio_data = safe_download_bytes(
                temp_url,
                purpose="lipsync_tts_audio",
                allowed_mime_types={
                    "audio/mpeg",
                    "audio/mp3",
                    "audio/wav",
                    "audio/x-wav",  # CosyVoice 部分接口返回 audio/x-wav，与 audio/wav 等价（RIFF/WAVE）
                    "audio/mp4",
                    "audio/x-m4a",
                },
                timeout=60.0,
            )
            from packages.shared.storage import get_shared_storage_service

            storage = get_shared_storage_service()
            storage_key = f"lipsync-tts/{user_id}/{job_id}.mp3"
            permanent_url = storage.upload_file(io.BytesIO(audio_data), storage_key, content_type="audio/mpeg")
            logger.info("[lipsync_tts] TTS 音频已转存 OSS: job_id=%s key=%s", job_id, storage_key)
            job.audio_url = permanent_url
        except Exception as exc:
            logger.warning(
                "[lipsync_tts] TTS 音频转存 OSS 失败，回退临时 URL: job_id=%s err=%s",
                job_id,
                exc,
            )
            job.audio_url = temp_url

        db.commit()

        # 2.5 计算精确句子时间戳（基于 TTS 音频静音检测）
        # 直接复用步骤 2 已下载到内存的 audio_data，避免重新从 OSS 下载（私有桶未签名会失败）
        import os as _os

        _st_tmp_path = None
        try:
            import subprocess as _sp
            import tempfile as _tmpf

            if not audio_data:
                logger.warning("[lipsync_tts] 无音频数据，跳过句子时间戳计算: job_id=%s", job_id)
            else:
                # 写入临时文件供 ffprobe/ffmpeg 使用
                with _tmpf.NamedTemporaryFile(suffix=".mp3", delete=False) as _atmp:
                    _atmp.write(audio_data)
                    _st_tmp_path = _atmp.name

                # ffprobe 获取音频时长
                _probe_result = _sp.run(
                    [
                        "ffprobe",
                        "-v",
                        "error",
                        "-show_entries",
                        "format=duration",
                        "-of",
                        "default=noprint_wrappers=1:nokey=1",
                        _st_tmp_path,
                    ],
                    capture_output=True,
                    text=True,
                    timeout=10,
                )
                _audio_duration = float(_probe_result.stdout.strip()) if _probe_result.stdout.strip() else 0.0
                logger.info(
                    "[lipsync_tts] 音频时长探测: job_id=%s duration=%.2f probe_stdout=%s probe_stderr=%s",
                    job_id,
                    _audio_duration,
                    _probe_result.stdout.strip()[:50],
                    _probe_result.stderr.strip()[:100] if _probe_result.stderr else "",
                )

                if _audio_duration > 0:
                    _timings = _compute_sentence_timings(audio_data, script_text, _audio_duration)
                    if _timings:
                        job.sentence_timings = _timings
                        logger.info(
                            "[lipsync_tts] 句子时间戳已计算: job_id=%s sentences=%d duration=%.1f",
                            job_id,
                            len(_timings),
                            _audio_duration,
                        )
                    else:
                        logger.warning("[lipsync_tts] 句子时间戳计算返回空结果: job_id=%s", job_id)
                else:
                    logger.warning(
                        "[lipsync_tts] ffprobe 未获取到有效时长，跳过句子时间戳: job_id=%s stdout=%s stderr=%s",
                        job_id,
                        _probe_result.stdout.strip()[:100],
                        _probe_result.stderr.strip()[:200] if _probe_result.stderr else "",
                    )
                db.commit()
        except Exception as _st_err:
            logger.warning(
                "[lipsync_tts] 句子时间戳计算失败（不影响主流程）: job_id=%s err=%s", job_id, _st_err, exc_info=True
            )
        finally:
            if _st_tmp_path:
                try:
                    _os.unlink(_st_tmp_path)
                except Exception:
                    pass

        # 3. 签名 URL 并提交到 MediaKit（复用模块内 _sign_media_url，避免对 LipsyncService 的耦合）
        audio_url = _sign_media_url(job.audio_url)
        video_url = _sign_media_url(job.video_url)

        client = get_mediakit_client()
        try:
            mk_result = client.submit_lipsync(
                video_url=video_url,
                audio_url=audio_url,
                enable_video_loop=job.enable_video_loop,
                client_token=job_id,
            )
            job.mediakit_task_id = mk_result["task_id"]
            job.status = "submitted"
            job.submitted_at = datetime.now(timezone.utc)
            logger.info(
                "[lipsync_tts] 已提交 MediaKit: job_id=%s task_id=%s",
                job_id,
                mk_result["task_id"],
            )
        except MediaKitError as exc:
            job.status = "failed"
            job.error_message = str(exc)
            job.error_code = exc.code
            logger.error("[lipsync_tts] 提交 MediaKit 失败: job_id=%s err=%s", job_id, exc)

        db.commit()

    except Exception:
        logger.exception("[lipsync_tts] 未预期的异常: job_id=%s", job_id)
        try:
            job = db.query(LipsyncJobModel).filter(LipsyncJobModel.id == job_id).first()
            if job and job.status not in ("cancelled", "failed", "completed"):
                job.status = "failed"
                job.error_message = "TTS 异步任务执行异常"
                job.error_code = "AsyncTaskError"
                job.updated_at = datetime.now(timezone.utc)
                db.commit()
        except Exception:
            logger.exception("[lipsync_tts] 回写失败状态时异常: job_id=%s", job_id)
    finally:
        db.close()