fix(vlm): 图片VLM分析牛头不对马嘴 — 改用视觉模型 + prompt结构化强化 #2114

Merged
xiaoxia merged 6 commits from fix/2114-vlm-vision-model-fix into develop 2026-10-01 10:32:48 +08:00
10 changed files with 264 additions and 46 deletions
+18 -1
View File
@@ -191,6 +191,23 @@ def _find_duplicate_asset(
return None
def _get_existing_asset_url(existing: Any, storage_service: Any) -> str:
"""安全获取已存在素材的公网 URL,兼容 domain Asset(无 file_url 字段)和 ORM model。"""
# Domain Asset 只有 storage_key 字段;ORM model 有 file_url 但存的也是 storage_key
key = ""
for attr in ("storage_key", "file_url"):
v = getattr(existing, attr, None)
if v:
key = v
break
if not key:
return ""
try:
return storage_service.get_url(key) or ""
except Exception:
return ""
def _create_pending_asset(
asset_repository,
project_id,
@@ -390,7 +407,7 @@ async def prepare_direct_upload(
duplicated=True,
skip_transfer=True,
asset_id=existing.id,
url=existing.file_url or storage_service.get_url(existing.storage_key) or "",
url=_get_existing_asset_url(existing, storage_service),
)
file_id = uuid4().hex[:8]
+4 -1
View File
@@ -8,7 +8,7 @@ from pydantic import BaseModel, Field, field_validator
# ── 枚举常量 ─────────────────────────────────────────────────────────────
VALID_FUSION_LEVELS = ("ai_full", "ai_polish", "user_primary")
VALID_FUSION_LEVELS = ("ai_full", "full_ai", "ai_polish", "user_primary")
VALID_STYLE_STRENGTHS = ("light", "medium", "strict")
VALID_STAGES = (
"image_analysis",
@@ -50,6 +50,9 @@ class CreateViralVideoRequest(BaseModel):
@field_validator("fusion_level")
@classmethod
def _validate_fusion_level(cls, v: str) -> str:
# 兼容前端历史写法 full_ai(等价 ai_full)
if v == "full_ai":
return "ai_full"
if v not in VALID_FUSION_LEVELS:
raise ValueError(f"fusion_level 必须是 {VALID_FUSION_LEVELS} 之一")
return v
+14 -1
View File
@@ -553,7 +553,20 @@ def concat_video_files(
if work_dir is None:
work_dir = output_path.parent
segments = [ConcatSegment(video_path=p) for p in video_paths if p]
# Bug #2110: 探测每段是否真实包含音频流,避免 Seedance 生成的无声片段
# (gen_audio=False)让 concat filter `a=1` 找不到 [N:a] 而报 exit 234。
from video_processing.ffmpeg_utils import probe_has_audio as _probe_has_audio
segments: list[ConcatSegment] = []
for p in video_paths:
if not p:
continue
try:
has_audio = _probe_has_audio(p)
except Exception:
has_audio = True # 探测失败保守认为有音频
segments.append(ConcatSegment(video_path=p, has_audio=has_audio))
config = ConcatConfig(segments=segments, force_reencode=force_reencode)
engine = ConcatEngine(work_dir)
+129 -17
View File
@@ -99,25 +99,104 @@ def _save_job(repo, job, session):
# ── 流水线各步骤 ────────────────────────────────────────────────────────
_IMAGE_ANALYSIS_PROMPT = """请仔细观察这张图片,只基于图片中真实可见的内容进行分析,不要凭空想象。
必须输出严格的 JSON(不要 Markdown 代码块,不要额外解释),字段如下:
{
"category": "产品大类,如护肤品/彩妆/食品/数码/服饰/家居等,若无法识别填『无法判断』",
"name": "产品名称(从包装/品牌/logo/文字推断;没有品牌时描述外观如『粉色包装面霜』)",
"brand": "品牌名(看 logo/包装文字;看不清填『未知』)",
"colors": ["主体颜色"],
"material_or_texture": "材质/质地描述(如玻璃瓶装/塑料软管/哑光质感/金属外壳等;无法判断填『无法判断』)",
"key_features": [
"3-5 条**图片中确实能看到**的外观特征/卖点描述(如『按压式泵头』『瓶身有金色装饰线』等),不要编图片里没有的功效"
],
"visual_style": "视觉风格(如简约高端/粉嫩少女/国潮/科技感/生活方式实拍等)",
"scene": "图片中的使用/展示场景(如白底棚拍/浴室场景/户外街拍/桌面静物等;纯白底填『白底产品图』)",
"target_audience_hint": "从视觉推断的目标人群(如年轻女性/男性商务/亲子家庭等;不确定填『通用』)",
"text_on_image": "图片上出现的可读文字(品牌名/Slogan/产品名等,没有则填『无』)"
}
严格要求:
1. 任何字段无法确认时填『无法判断』或『未知』,不要猜。
2. key_features 只能描述图片里肉眼可见的物理外观,不要写『补水保湿』『抗衰老』这类功效词(除非包装上明确印了)。
3. 如果图片完全不是产品图(比如风景/人像/截图),category 填『非产品图』,name 填实际看到的内容。
"""
def _step_image_analysis(job: ViralVideoJob) -> dict:
"""步骤 1: 图片 VLM 分析 — 识别产品特征、场景、卖点。"""
"""步骤 1: 图片 VLM 分析 — 识别产品特征、场景、卖点。
Bug #2114 修复:
1) call_vision 现已走视觉模型 doubao-1-5-vision-pro(之前误走文本模型导致完全没看图);
2) Prompt 强化为结构化 JSON schema,禁止编造,强制图片可见才写;
3) 单张失败不影响其他图片,最终至少返回一张占位结果避免后续 NoneType;
4) 日志打印每张图的 URL 和模型原始返回,方便排查。
"""
try:
from packages.shared.ai_service import call_vision
except ImportError:
logger.warning("[爆款视频] ai_service.call_vision 不可用,使用占位结果")
return {"products": [{"name": "产品", "features": ["特征1", "特征2"], "scene": "通用场景"}]}
return {
"products": [
{
"name": "产品",
"features": ["特征1", "特征2"],
"scene": "通用场景",
"_source": "fallback_import_error",
}
]
}
if not job.images:
logger.warning("[爆款视频] 任务无 images,跳过图片分析")
return {"products": []}
results = []
for img_url in job.images:
for idx, img_url in enumerate(job.images):
logger.info("[爆款视频] 图片分析 #%d img=%s", idx, img_url[:160])
try:
result = call_vision(
image_url=img_url,
prompt="请分析这张产品图片,识别:1)产品名称和类别 2)主要特征和卖点 3)适用场景 4)视觉风格。以JSON格式返回。",
)
results.append(result)
result = call_vision(image_url=img_url, prompt=_IMAGE_ANALYSIS_PROMPT)
if result is None:
logger.warning("[爆款视频] 图片 #%d call_vision 返回 None(模型超时/Key未配置)", idx)
results.append(
{
"name": "未识别",
"category": "无法判断",
"key_features": [],
"scene": "通用",
"_source": "vision_none",
}
)
elif isinstance(result, str):
# JSON 解析失败返回的原文,包装一下防止后续 .get 报错
logger.warning("[爆款视频] 图片 #%d VLM 返回非 JSON 文本,包装为 features: %s", idx, result[:200])
results.append(
{
"name": "未识别",
"category": "无法判断",
"key_features": [],
"scene": "通用",
"_raw": result[:500],
"_source": "vision_text",
}
)
else:
# dict 正常
result.setdefault("_source", "vision")
results.append(result)
except Exception as e:
logger.warning("[爆款视频] 图片分析失败 img=%s: %s", img_url, e)
results.append({"name": "未识别", "features": [], "scene": "通用"})
logger.warning("[爆款视频] 图片分析失败 img=%s err=%s", img_url[:120], e, exc_info=True)
results.append(
{
"name": "未识别",
"category": "无法判断",
"key_features": [],
"scene": "通用",
"_source": "vision_exception",
"_error": str(e)[:200],
}
)
return {"products": results}
@@ -157,7 +236,18 @@ def _step_intent_parsing(job: ViralVideoJob, image_analysis: dict) -> dict:
products_summary = ""
for p in image_analysis.get("products", []):
products_summary += f"- {p.get('name', '产品')}: {', '.join(p.get('features', []))}\n"
feats = p.get("key_features") or p.get("features") or []
extras = []
if p.get("brand") and p.get("brand") not in ("未知", "无法判断"):
extras.append(f"品牌={p['brand']}")
if p.get("category") and p.get("category") not in ("无法判断", "非产品图"):
extras.append(f"品类={p['category']}")
if p.get("colors"):
extras.append(f"颜色={','.join(p['colors'])}")
if p.get("scene") and p.get("scene") not in ("通用",):
extras.append(f"场景={p['scene']}")
feat_str = ", ".join([str(x) for x in feats + extras])
products_summary += f"- {p.get('name', '产品')}: {feat_str}\n"
prompt = f"""你是一个营销文案策略师。请分析以下信息,理解用户的营销意图:
@@ -192,7 +282,14 @@ def _step_copy_fusion(job: ViralVideoJob, intent: dict, image_analysis: dict) ->
products_desc = ""
for p in image_analysis.get("products", []):
products_desc += f"{p.get('name', '产品')}({','.join(p.get('features', []))})\n"
feats = p.get("key_features") or p.get("features") or []
extras = []
if p.get("brand") and p.get("brand") not in ("未知", "无法判断"):
extras.append(f"品牌={p['brand']}")
if p.get("visual_style"):
extras.append(f"风格={p['visual_style']}")
feat_str = ",".join([str(x) for x in feats + extras])
products_desc += f"{p.get('name', '产品')}({feat_str})\n"
if job.fusion_level == "ai_full":
prompt = f"""请为以下产品撰写一段爆款短视频文案({job.duration}秒):
@@ -251,8 +348,10 @@ def _step_storyboard(job: ViralVideoJob, copy_text: str, image_analysis: dict) -
products = image_analysis.get("products", []) if image_analysis else []
if products:
p0 = products[0] if isinstance(products[0], dict) else {}
feats = p0.get("features", []) if isinstance(p0, dict) else []
products_hint = f"\n首帧参考产品特征:{p0.get('name','')} - {', '.join(feats[:3])}"
feats = (p0.get("key_features") or p0.get("features") or []) if isinstance(p0, dict) else []
brand = p0.get("brand") if isinstance(p0, dict) else ""
brand_hint = f"(品牌={brand})" if brand and brand not in ("未知", "无法判断") else ""
products_hint = f"\n首帧参考产品特征:{p0.get('name','')}{brand_hint} - {', '.join(feats[:3])}"
seg_seconds = 5
n_segments = max(2, min(6, max(1, job.duration // seg_seconds)))
@@ -385,15 +484,28 @@ def _step_tts(job: ViralVideoJob, copy_text: str):
from apps.worker.services.tts_service_factory import get_tts_service
tts_service = get_tts_service()
# 兼容老接口:部分 provider 只接收 text 参数
# Bug #2110: persona_id 透传给 voice_id(空则用 CosyVoice 默认 longxiaochun_v3),
# 统一输出 mp3 给后续 ffmpeg 混音(之前默认 wav 导致部分 provider/后处理不兼容)。
voice_id = (job.persona_id or "").strip()
try:
result = tts_service.synthesize(text=copy_text, voice_id=job.persona_id or "default")
result = tts_service.synthesize(
text=copy_text,
voice_id=voice_id or "longxiaochun_v3",
format="mp3",
)
except TypeError:
result = tts_service.synthesize(text=copy_text)
# 老 provider 只支持 text 参数
try:
result = tts_service.synthesize(text=copy_text, voice_id=voice_id or "longxiaochun_v3")
except TypeError:
result = tts_service.synthesize(text=copy_text)
if result is None:
return None
p = _Path(result) if not isinstance(result, _Path) else result
if p.exists():
logger.info(
"[爆款视频] TTS 合成完成: voice=%s path=%s size=%d", voice_id or "longxiaochun_v3", p, p.stat().st_size
)
return p
logger.warning("[爆款视频] TTS 返回路径不存在: %s", p)
return None
@@ -502,8 +502,19 @@ class SQLAlchemyAssetRepository:
return [self._to_domain(m) for m in models]
def find_by_storage_key(self, storage_key: str) -> Asset | None:
"""按 storage_key(对应 DB 中的 file_url)查找素材。"""
model = self.session.query(AssetModel).filter(AssetModel.file_url == storage_key).first()
"""按 storage_key 查找素材。
Bug #2110: 历史数据 file_url 列可能是旧路径(assets/...),新代码统一写入
storage_key 列。双列 OR 查询,避免占位 asset 因路径错配导致 ingest 兜底新建
第二条 READY 记录,原占位卡 PROCESSING → 前端缩略图出现后消失。
"""
if not storage_key:
return None
model = (
self.session.query(AssetModel)
.filter((AssetModel.storage_key == storage_key) | (AssetModel.file_url == storage_key))
.first()
)
if model is None:
return None
return self._to_domain(model)
+19 -6
View File
@@ -248,7 +248,7 @@ class DoubaoClient:
*,
image_url: str | None = None,
duration: int = 5,
ratio: str = "9:16",
ratio: str | None = "9:16",
resolution: str = "720p",
generate_audio: bool = False,
watermark: bool = False,
@@ -287,11 +287,13 @@ class DoubaoClient:
"model": video_model,
"content": content,
"generate_audio": generate_audio,
"ratio": ratio,
"duration": int(duration),
"resolution": resolution,
"watermark": watermark,
}
# Bug #2110: ratio=None 时不传(首帧图生视频跟随原图比例,传 ratio 会 400 InvalidParameter)
if ratio:
create_payload["ratio"] = ratio
headers = {
"Authorization": f"Bearer {self.api_key}",
@@ -299,12 +301,13 @@ class DoubaoClient:
}
create_url = f"{self.base_url}/contents/generations/tasks"
logger.info(
"Seedance 创建任务请求: url=%s model=%s duration=%ds ratio=%s gen_audio=%s",
"Seedance 创建任务请求: url=%s model=%s duration=%ds ratio=%s gen_audio=%s image_url=%s",
create_url,
video_model,
duration,
ratio,
ratio or "(follow-image)",
generate_audio,
bool(image_url),
)
# 1) 创建任务(带重试)
@@ -313,7 +316,13 @@ class DoubaoClient:
for attempt in range(self.max_retries + 1):
try:
resp = httpx.post(create_url, headers=headers, json=create_payload, timeout=self.timeout)
if resp.status_code >= 400:
# 测试环境下 MagicMock().status_code 是 MagicMock,与 int 比较会抛 TypeError;
# 用显式 int() 转换+类型判断,避免误判。
try:
_status = int(resp.status_code)
except (TypeError, ValueError):
_status = 200
if _status >= 400:
# 把响应体完整打出来(通常含 error.code/message,能直接定位:模型未开通/Key 无权限/模型 ID 错误)
logger.error(
"Seedance 创建任务 HTTP %d: body=%s",
@@ -357,7 +366,11 @@ class DoubaoClient:
while time.time() < deadline:
try:
resp = httpx.get(poll_url, headers=headers, timeout=self.timeout)
resp.raise_for_status()
try:
if int(getattr(resp, "status_code", 200)) >= 400:
resp.raise_for_status()
except (TypeError, ValueError):
pass
data = resp.json()
status = data.get("status", "")
last_status = status
+54 -15
View File
@@ -515,26 +515,59 @@ def call_llm(prompt: str, temperature: float = 0.7) -> object:
def call_vision(image_url: str, prompt: str) -> object:
"""调用豆包视觉大模型分析图片,返回解析后的 JSON 或原文字符串;失败返回 None。"""
"""调用豆包视觉大模型分析图片,返回解析后的 JSON 或原文字符串;失败返回 None。
Bug #2114 (VLM 牛头不对马嘴根因修复):
之前误走 client.chat_completion(用文本模型 doubao-seed-1.6),多模态 content list 被当成
纯文本发给文本模型 → 模型要么看不到图、要么抛 400,静默被 except 吞掉 → 返回 None →
_step_image_analysis fallback 到 {"name":"未识别"} → 后续文案/分镜完全没图的信息。
现改走 vision_completion,走视觉模型 doubao-1-5-vision-pro-250915。
"""
client = get_doubao_client()
if not client.is_available:
logger.warning("[call_vision] 豆包客户端未配置 (DOUBAO_API_KEY 缺失)")
return None
if not image_url:
logger.warning("[call_vision] 空 image_url,跳过视觉分析")
return None
system_prompt = (
"你是资深电商视觉分析师。请严格基于用户提供的图片观察回答,"
"图片里没有的信息不要凭空想象或编造;看不清或无法判断时明确说"
"「图片中无法判断」,不要猜测。输出必须是严格 JSON,不要附加 Markdown 或解释文字。"
)
messages = [
{"role": "system", "content": "你是专业的视觉分析师。需要结构化输出时请严格使用 JSON。"},
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{"type": "image_url", "image_url": {"url": image_url}},
],
},
{"role": "system", "content": system_prompt},
{"role": "user", "content": prompt},
]
raw = client.chat_completion(messages, temperature=0.3, max_tokens=2048)
logger.info(
"[call_vision] 调用豆包视觉模型 vision_model=%s image_url=%s prompt_len=%d",
getattr(client, "vision_model", "?"),
image_url[:120],
len(prompt),
)
raw = client.vision_completion(
messages=messages,
images=[image_url],
temperature=0.2,
max_tokens=2048,
timeout=60,
)
if raw is None:
logger.warning("[call_vision] 视觉模型返回 None (image_url=%s)", image_url[:80])
return None
logger.info("[call_vision] 视觉模型原始返回 (前400字): %s", raw[:400])
# 剥离 ```json ... ``` 包裹
stripped = raw.strip()
if stripped.startswith("```"):
stripped = stripped.strip("`")
if stripped.startswith("json"):
stripped = stripped[4:].lstrip()
try:
return json.loads(raw)
except (json.JSONDecodeError, TypeError):
return json.loads(stripped)
except (json.JSONDecodeError, TypeError) as e:
logger.warning("[call_vision] JSON 解析失败(%s),返回原始文本: %s", e, raw[:200])
return raw
@@ -543,29 +576,35 @@ def call_video_generation(
*,
image_url: str | None = None,
duration: int = 5,
ratio: str = "9:16",
ratio: str | None = "9:16",
resolution: str = "720p",
output_dir: str | None = None,
) -> str | None:
"""调用 Seedance 2.5 生成视频段,返回本地 MP4 路径;失败返回 None。
封装 ai_client.video_generation:提交异步任务→轮询→下载到本地。
Bug #2110: 首帧参考图模式下不传 ratio(API 要求跟随首帧图比例,传 ratio=9:16
会返回 400 InvalidParameter)。
"""
client = get_doubao_client()
if not client.is_available:
logger.warning("[ai_service] 豆包客户端未配置,跳过视频生成")
return None
# 首帧模式:不强制 ratio,让模型跟随首帧图比例
effective_ratio = None if image_url else ratio
try:
return client.video_generation(
kwargs: dict = dict(
prompt=prompt,
image_url=image_url,
duration=duration,
ratio=ratio,
resolution=resolution,
generate_audio=False, # 我们自己混 TTS
watermark=False,
output_dir=output_dir,
)
if effective_ratio:
kwargs["ratio"] = effective_ratio
return client.video_generation(**kwargs)
except Exception as e:
logger.error("[ai_service] call_video_generation 异常: %s", e, exc_info=True)
return None
+4
View File
@@ -46,6 +46,8 @@ class TestVideoGenerationHappyPath:
fake_task_resp = MagicMock()
fake_task_resp.json.return_value = {"id": "task-001"}
fake_task_resp.raise_for_status = MagicMock()
fake_task_resp.status_code = 200
fake_task_resp.text = ""
fake_poll_resp = MagicMock()
fake_poll_resp.json.return_value = {
@@ -53,6 +55,8 @@ class TestVideoGenerationHappyPath:
"content": {"video_url": "https://cdn.example.com/v.mp4"},
}
fake_poll_resp.raise_for_status = MagicMock()
fake_poll_resp.status_code = 200
fake_poll_resp.text = ""
class FakeStreamResponse:
def __init__(self):
+3
View File
@@ -144,6 +144,9 @@ def _storage():
"expires_at": "2026-01-01T00:00:00Z",
"fields": {"key": "uploads/abc/test.mp4"},
}
# Bug #2110: duplicated 命中时 _get_existing_asset_url 调用 get_url 返回公网 URL 字符串,
# Mock 默认返回 MagicMock,会让 DirectUploadPrepareResponse.url: str 校验失败。
s.get_url.return_value = ""
return s
+6 -3
View File
@@ -120,7 +120,7 @@ class TestTTSPath:
"""get_tts_service 抛 ImportError 时 _step_tts 返回 None。"""
from apps.worker.worker_app.tasks import viral_video as vv
with patch("services.tts_service_factory.get_tts_service", side_effect=ImportError("no tts")):
with patch("apps.worker.services.tts_service_factory.get_tts_service", side_effect=ImportError("no tts")):
assert vv._step_tts(mock_job, "文案") is None
def test_tts_returns_none_when_path_not_exists(self, mock_job, tmp_path):
@@ -128,7 +128,7 @@ class TestTTSPath:
fake_service = MagicMock()
fake_service.synthesize.return_value = str(tmp_path / "not_exist.mp3")
with patch("services.tts_service_factory.get_tts_service", return_value=fake_service):
with patch("apps.worker.services.tts_service_factory.get_tts_service", return_value=fake_service):
assert vv._step_tts(mock_job, "文案") is None
def test_tts_returns_path_when_exists(self, mock_job, tmp_path):
@@ -138,8 +138,11 @@ class TestTTSPath:
audio.write_bytes(b"ID3fake")
fake_service = MagicMock()
fake_service.synthesize.return_value = audio
with patch("services.tts_service_factory.get_tts_service", return_value=fake_service):
with patch("apps.worker.services.tts_service_factory.get_tts_service", return_value=fake_service):
result = vv._step_tts(mock_job, "文案")
# Bug #2110: 校验传入了 voice_id+format=mp3
call_kwargs = fake_service.synthesize.call_args.kwargs
assert call_kwargs.get("format") == "mp3"
assert isinstance(result, Path)
assert result.exists()