diff --git a/tests/render_compare/README.md b/tests/render_compare/README.md new file mode 100644 index 000000000..4e8464504 --- /dev/null +++ b/tests/render_compare/README.md @@ -0,0 +1,136 @@ +# 灰度对比测试工具 + +用于统一渲染引擎灰度发布期间的新旧引擎对比验证。 + +## 能力 + +- **像素对比**:基于 FFmpeg SSIM + PSNR 双指标,评估视频画质差异 +- **音频对比**:基于差值音频 RMS,评估音频波形差异 +- **批量对比**:10个预设场景覆盖 P0/P1/P2 优先级 +- **HTML 报告**:可视化对比结果,包含画质、音频、性能三维度 +- **两种切换方式**:支持 engine 参数直传 或 Feature Flag 白名单切换 + +## 目录结构 + +``` +tests/render_compare/ +├── __init__.py # 包导出 +├── README.md # 本文档 +├── video_diff.py # 视频像素对比(SSIM + PSNR) +├── audio_diff.py # 音频对比(差值 RMS) +├── scenarios.py # 预定义对比场景(10个) +└── runner.py # 批量对比执行器 + HTML 报告生成 +``` + +## 快速开始 + +### 环境要求 + +- FFmpeg 4.4+(需带 ssim 和 psnr 滤镜) +- Python 3.10+ +- httpx(API 调用) + +### 配置环境变量 + +```bash +export STAGING_API_URL=https://api.staging.example.com +export STAGING_API_KEY=your_api_key +export STAGING_INTERNAL_API_KEY=your_internal_key # 可选,Feature Flag 模式需要 +``` + +### 运行对比 + +```bash +# 运行所有 P0 场景(最核心的5个) +python -m tests.render_compare.runner --priority P0 --output ./report/ + +# 运行 P0 + P1 场景 +python -m tests.render_compare.runner --priority P1 --output ./report/ + +# 只跑指定场景 +python -m tests.render_compare.runner --scenarios simple_pass_through,subtitle_rendering + +# 使用 Feature Flag 方式切换引擎(需要 internal key) +python -m tests.render_compare.runner --priority P0 --flag-mode + +# 自定义阈值 +python -m tests.render_compare.runner --priority P0 --ssim-threshold 0.95 --psnr-threshold 30 +``` + +## 对比场景 + +| ID | 名称 | 优先级 | 验证点 | +|----|------|--------|--------| +| simple_pass_through | 简单直通 | P0 | 直通优化路径正确性 | +| multi_clip_transition | 多clip转场 | P0 | 转场效果 + concat | +| subtitle_rendering | 字幕渲染 | P0 | ASS字幕渲染 | +| independent_audio_track | 独立音频轨 | P0 | 音频混音(amix) | +| no_audio_video | 无音轨视频 | P0 | 无音轨防御逻辑 | +| picture_in_picture | 画中画 | P1 | overlay 图层 | +| multi_layer_mix | 多图层混合 | P1 | 多图层复杂场景 | +| image_background | 图片背景 | P1 | background 层 + 无音频 | +| long_video_stress | 长视频压力 | P2 | 多clip性能 | +| vertical_portrait | 竖屏9:16 | P2 | scale 策略(铺满裁剪) | + +## 验收标准(建议) + +### 视频质量 +- **平均 SSIM >= 0.90**:通过(有微小差异但视觉可接受) +- **平均 SSIM >= 0.95**:优秀(视觉几乎无差异) +- **平均 PSNR >= 25 dB**:通过 +- **分辨率一致 + 时长差 < 0.1s**:通过 + +### 音频质量 +- **相似度 >= 0.85**:通过 +- **采样率/声道数一致**:通过 + +### 性能 +- **平均性能差异在 ±10% 以内**:可接受 +- **直通场景新引擎更快**(预期 +30%) + +## API 约定 + +Runner 默认假设渲染 API 支持以下接口: + +### 提交任务 +``` +POST /api/v1/render/compose +Authorization: Bearer {api_key} +Body: { ...plan_payload, "engine": "legacy" | "unified" } +Response: { "task_id": "xxx" } +``` + +### 查询状态 +``` +GET /api/v1/tasks/{task_id} +Response: { "status": "completed", "output_url": "...", "duration_sec": 5.2 } +``` + +### Feature Flag(flag-mode) +``` +PUT /api/v1/internal/feature-flags/render_engine +X-API-Key: {internal_key} +Body: { "enabled": true, "percentage": 100 } +``` + +如果你的 API 接口不同,请修改 `StagingAPI` 类中的对应方法。 + +## 故障排查 + +### 对比失败定位指南 + +1. **像素差异大(SSIM < 0.90)** + - 检查分辨率是否一致 + - 检查帧率是否一致 + - 用 `save_diff_frame` 生成差异帧可视化 + - 检查转场效果(slideup/slidedown 是新引擎独有) + +2. **音频不一致** + - 检查音频编码参数(码率、采样率) + - 检查主音频源优先级(main > broll) + - 用 ffprobe 对比两视频音频流参数 + +3. **渲染失败** + - 检查日志:`[unified-render] render failed` + - 检查素材是否完整下载 + - 检查 FFmpeg 命令是否正确 diff --git a/tests/render_compare/__init__.py b/tests/render_compare/__init__.py new file mode 100644 index 000000000..188f7dce8 --- /dev/null +++ b/tests/render_compare/__init__.py @@ -0,0 +1,26 @@ +"""灰度对比测试工具包. + +用于新旧渲染引擎的批量对比测试,包含: +- video_diff: 视频像素对比(SSIM + PSNR) +- audio_diff: 音频对比(差值 RMS) +- scenarios: 预定义对比场景 +- runner: 批量对比执行器 + HTML 报告 +""" + +from .audio_diff import AudioDiffResult, compute_audio_diff, extract_audio, probe_duration, probe_has_audio +from .scenarios import SCENARIOS, CompareScenario, get_scenarios_by_priority +from .video_diff import VideoDiffResult, compute_video_diff, save_diff_frame + +__all__ = [ + "VideoDiffResult", + "compute_video_diff", + "save_diff_frame", + "AudioDiffResult", + "compute_audio_diff", + "extract_audio", + "probe_has_audio", + "probe_duration", + "SCENARIOS", + "CompareScenario", + "get_scenarios_by_priority", +] diff --git a/tests/render_compare/audio_diff.py b/tests/render_compare/audio_diff.py new file mode 100644 index 000000000..d9e5013da --- /dev/null +++ b/tests/render_compare/audio_diff.py @@ -0,0 +1,322 @@ +"""音频对比工具 — 基于 FFmpeg 的音频质量对比. + +使用以下指标评估两段音频的相似度: +1. 波形差异(RMS 差值) +2. 频谱相似度(FFT 分帧比较) +3. 时长差异 + +对比方式: +- 直接对两个音频做 `ametadata=select='gt(scene\\,0.3)'` 过于复杂 +- 简化方案:用 `amerge` + `astats` 计算差值音频的 RMS + +更精确的方案(已实现): +- 将两轨音频做差(amix=0:weights='1 -1' → 实际上用 pan 更简单) +- 对差值音频做 astats,获取差值的 RMS、峰值等指标 +""" + +from __future__ import annotations + +import json +import re +import shutil +import subprocess # nosec B404 +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +FFMPEG_BIN: str = shutil.which("ffmpeg") or "ffmpeg" +FFPROBE_BIN: str = shutil.which("ffprobe") or "ffprobe" + + +@dataclass +class AudioDiffResult: + """音频对比结果.""" + + audio_a: str + audio_b: str + duration_a: float + duration_b: float + duration_diff: float + sample_rate_match: bool + channels_match: bool + diff_rms_db: float # 差值音频的 RMS(dB,越低越相似) + diff_peak_db: float # 差值音频的峰值(dB,越低越相似) + similarity_score: float # 综合相似度评分 [0, 1],1 = 完全一致 + passed: bool + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def probe_duration(file_path: str) -> float: + """探测文件时长(秒),失败返回 0.""" + try: + result = subprocess.run( # nosec B603 + [ + FFPROBE_BIN, + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + str(file_path), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10, + ) + return round(float(result.stdout.strip()), 3) + except Exception: + return 0.0 + + +def probe_has_audio(file_path: str | Path) -> bool: + """探测文件是否包含音频流.""" + try: + result = subprocess.run( # nosec B603 + [ + FFPROBE_BIN, + "-v", + "error", + "-select_streams", + "a:0", + "-show_entries", + "stream=codec_type", + "-of", + "default=noprint_wrappers=1:nokey=1", + str(file_path), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10, + ) + return result.stdout.strip() == "audio" + except Exception: + return False # 探测失败保守返回 False,避免误判有音频 + + +def compute_audio_diff( + audio_a: str | Path, + audio_b: str | Path, + *, + similarity_threshold: float = 0.90, + duration_tolerance: float = 0.1, +) -> AudioDiffResult: + """计算两段音频的差异. + + 方案:用 pan 滤镜将两轨相减,对差值音频做 astats 分析。 + + Args: + audio_a: 音频A(基线) + audio_b: 音频B(对比) + similarity_threshold: 相似度合格阈值 + duration_tolerance: 时长容忍度(秒) + + Returns: + AudioDiffResult 对比结果 + """ + dur_a = probe_duration(str(audio_a)) + dur_b = probe_duration(str(audio_b)) + duration_diff = abs(dur_a - dur_b) + + # 获取音频元信息 + info_a = _probe_audio_info(str(audio_a)) + info_b = _probe_audio_info(str(audio_b)) + + sample_rate_match = info_a["sample_rate"] == info_b["sample_rate"] + channels_match = info_a["channels"] == info_b["channels"] + + # 相减后分析差值 + # 取较短时长做对比 + min_dur = min(dur_a, dur_b) + if min_dur <= 0: + return AudioDiffResult( + audio_a=str(audio_a), + audio_b=str(audio_b), + duration_a=dur_a, + duration_b=dur_b, + duration_diff=duration_diff, + sample_rate_match=sample_rate_match, + channels_match=channels_match, + diff_rms_db=-999.0, + diff_peak_db=-999.0, + similarity_score=0.0, + passed=False, + ) + + # 做差值音频:a - b + # 注意:amix 会自动按输入数归一化音量(除以N), + # 所以 a + (-1)*b 经过 amix=inputs=2 后整体音量会减半(-6dB)。 + # 加 volume=2 补偿回来,确保差值 RMS 反映真实差异幅度。 + command = [ + FFMPEG_BIN, + "-i", + str(audio_a), + "-i", + str(audio_b), + "-filter_complex", + # 第2轨反相 → amix混合 → volume=2补偿amix的自动缩放 + "[1:a]volume=-1[inv];[0:a][inv]amix=inputs=2:duration=shortest:dropout_transition=0,volume=2[diff]", + "-map", + "[diff]", + "-f", + "null", + "-af", + "astats=metadata=1:reset=0", + "-", + ] + + try: + result = subprocess.run( # nosec B603 + command, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=120, + ) + stderr = result.stderr or "" + except subprocess.CalledProcessError as e: + # 如果音频格式不兼容,返回失败 + return AudioDiffResult( + audio_a=str(audio_a), + audio_b=str(audio_b), + duration_a=dur_a, + duration_b=dur_b, + duration_diff=duration_diff, + sample_rate_match=sample_rate_match, + channels_match=channels_match, + diff_rms_db=999.0, + diff_peak_db=999.0, + similarity_score=0.0, + passed=False, + ) + + diff_rms_db, diff_peak_db = _parse_astats(stderr) + + # 相似度评分:基于差值 RMS + # 差值 RMS -60dB → 相似度 ~1.0(几乎无声差) + # 差值 RMS -20dB → 相似度 ~0.5(有明显差异) + # 差值 RMS 0dB → 相似度 ~0.0(完全相反) + if diff_rms_db <= -60: + similarity_score = 1.0 + elif diff_rms_db >= 0: + similarity_score = 0.0 + else: + # 线性映射:-60dB → 1.0, 0dB → 0.0 + similarity_score = max(0.0, min(1.0, 1.0 + diff_rms_db / 60.0)) + + passed = ( + duration_diff <= duration_tolerance + and sample_rate_match + and channels_match + and similarity_score >= similarity_threshold + ) + + return AudioDiffResult( + audio_a=str(audio_a), + audio_b=str(audio_b), + duration_a=round(dur_a, 3), + duration_b=round(dur_b, 3), + duration_diff=round(duration_diff, 3), + sample_rate_match=sample_rate_match, + channels_match=channels_match, + diff_rms_db=round(diff_rms_db, 2), + diff_peak_db=round(diff_peak_db, 2), + similarity_score=round(similarity_score, 4), + passed=passed, + ) + + +def _probe_audio_info(file_path: str) -> dict[str, int]: + """探测音频元信息.""" + try: + result = subprocess.run( # nosec B603 + [ + FFPROBE_BIN, + "-v", + "error", + "-select_streams", + "a:0", + "-show_entries", + "stream=sample_rate,channels", + "-of", + "json", + file_path, + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10, + ) + + info = json.loads(result.stdout) + stream = info.get("streams", [{}])[0] + return { + "sample_rate": int(stream.get("sample_rate", 44100)), + "channels": int(stream.get("channels", 2)), + } + except Exception: + return {"sample_rate": 0, "channels": 0} + + +def _parse_astats(stderr: str) -> tuple[float, float]: + """从 astats 输出中解析 RMS 和峰值. + + astats 输出格式(在 stderr 中): + [Parsed_astats_1 @ 0x...] Channel: 1 + [Parsed_astats_1 @ 0x...] ... + [Parsed_astats_1 @ 0x...] Overall + [Parsed_astats_1 @ 0x...] DC offset: 0.000000 + [Parsed_astats_1 @ 0x...] Min level: -0.123456 + [Parsed_astats_1 @ 0x...] Max level: 0.789012 + [Parsed_astats_1 @ 0x...] Peak level dB: -2.01 + [Parsed_astats_1 @ 0x...] RMS level dB: -10.56 + ... + """ + lines = stderr.split("\n") + rms_db = -999.0 + peak_db = -999.0 + + for line in lines: + # 找 Overall 部分的统计(双声道时取整体值) + rms_match = re.search(r"RMS level dB:\s*(-?\d+\.?\d*)", line) + peak_match = re.search(r"Peak level dB:\s*(-?\d+\.?\d*)", line) + if rms_match: + rms_db = float(rms_match.group(1)) + if peak_match: + peak_db = float(peak_match.group(1)) + + return rms_db, peak_db + + +def extract_audio(video_path: str | Path, output_path: str | Path) -> Path: + """从视频中提取音频(AAC 格式). + + Args: + video_path: 视频文件路径 + output_path: 输出音频路径 + + Returns: + 输出音频文件路径 + """ + command = [ + FFMPEG_BIN, + "-y", + "-i", + str(video_path), + "-vn", + "-acodec", + "aac", + "-b:a", + "128k", + str(output_path), + ] + subprocess.run(command, check=True, capture_output=True, timeout=120) # nosec B603 + return Path(output_path) diff --git a/tests/render_compare/runner.py b/tests/render_compare/runner.py new file mode 100644 index 000000000..393908382 --- /dev/null +++ b/tests/render_compare/runner.py @@ -0,0 +1,628 @@ +"""灰度对比测试 Runner — 新旧引擎批量对比 + 报告生成. + +使用方法: + # 配置环境变量 + export STAGING_API_URL=https://api.staging.example.com + export STAGING_API_KEY=your_key + + # 运行全部 P0 场景 + python -m tests.render_compare.runner --priority P0 --output ./report/ + + # 只跑指定场景 + python -m tests.render_compare.runner --scenario simple_pass_through,subtitle_rendering + +对比流程: +1. 对每个场景,分别提交到 legacy 和 unified 引擎(通过 Feature Flag 白名单/百分比控制) + - 方式A:通过内部 API 临时切换 flag(需要 admin key) + - 方式B:提交任务时指定 engine 参数(如果 API 支持) +2. 等待任务完成,下载输出视频 +3. 像素对比(SSIM + PSNR)+ 音频对比(差值RMS) +4. 生成 HTML 对比报告 + +注意:默认假设 API 支持 `engine` 参数来指定渲染引擎。 +如果不支持,需要先通过内部 API 切换 Feature Flag,然后提交任务。 +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any + +import httpx + +# 确保项目根目录在 path 中 +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from .audio_diff import AudioDiffResult, compute_audio_diff +from .scenarios import SCENARIOS, CompareScenario, get_scenarios_by_priority +from .video_diff import VideoDiffResult, compute_video_diff + + +@dataclass +class ScenarioResult: + """单个场景的对比结果.""" + + scenario: CompareScenario + legacy_task_id: str = "" + unified_task_id: str = "" + legacy_video_path: str = "" + unified_video_path: str = "" + legacy_duration_sec: float = 0.0 + unified_duration_sec: float = 0.0 + video_diff: VideoDiffResult | None = None + audio_diff: AudioDiffResult | None = None + legacy_success: bool = False + unified_success: bool = False + error: str = "" + + @property + def passed(self) -> bool: + if not (self.legacy_success and self.unified_success): + return False + if self.video_diff and not self.video_diff.passed: + return False + if self.audio_diff and not self.audio_diff.passed: + return False + return True + + +class StagingAPI: + """Staging 环境 API 客户端.""" + + def __init__(self, base_url: str, api_key: str, internal_api_key: str = ""): + self.base_url = base_url.rstrip("/") + self.api_key = api_key + self.internal_api_key = internal_api_key + self.client = httpx.Client(timeout=30.0) + + def _headers(self, internal: bool = False) -> dict[str, str]: + headers = {"Authorization": f"Bearer {self.api_key}"} + if internal and self.internal_api_key: + headers["X-API-Key"] = self.internal_api_key + return headers + + def submit_render_task(self, plan_payload: dict[str, Any], engine: str = "") -> str: + """提交渲染任务,返回 task_id. + + Args: + plan_payload: EditPlan payload + engine: 可选,指定引擎("legacy" / "unified") + + Returns: + task_id + """ + url = f"{self.base_url}/api/v1/render/compose" + payload = dict(plan_payload) + if engine: + payload["engine"] = engine + resp = self.client.post(url, json=payload, headers=self._headers()) + resp.raise_for_status() + data = resp.json() + return data.get("task_id") or data.get("id", "") + + def get_task_status(self, task_id: str) -> dict[str, Any]: + """获取任务状态.""" + url = f"{self.base_url}/api/v1/tasks/{task_id}" + resp = self.client.get(url, headers=self._headers()) + resp.raise_for_status() + return resp.json() + + def wait_for_task(self, task_id: str, timeout: float = 300.0, poll_interval: float = 3.0) -> dict[str, Any]: + """等待任务完成. + + Returns: + 最终任务状态 + + Raises: + TimeoutError: 超时 + """ + start = time.time() + while time.time() - start < timeout: + status = self.get_task_status(task_id) + state = status.get("status", "") + if state in ("completed", "success", "done", "failed", "error"): + return status + time.sleep(poll_interval) + raise TimeoutError(f"Task {task_id} timed out after {timeout}s") + + def set_feature_flag(self, flag_name: str, enabled: bool, percentage: int = 0, whitelist: list[str] | None = None): + """通过内部 API 设置 Feature Flag. + + 用于不支持 engine 参数的场景,切换全局灰度比例。 + """ + if not self.internal_api_key: + raise ValueError("internal_api_key is required for feature flag operations") + url = f"{self.base_url}/api/v1/internal/feature-flags/{flag_name}" + body: dict[str, Any] = {"enabled": enabled, "percentage": percentage} + if whitelist is not None: + body["whitelist"] = whitelist + resp = self.client.put(url, json=body, headers=self._headers(internal=True)) + resp.raise_for_status() + return resp.json() + + def get_feature_flag(self, flag_name: str) -> dict[str, Any]: + """获取 Feature Flag 配置.""" + if not self.internal_api_key: + raise ValueError("internal_api_key is required") + url = f"{self.base_url}/api/v1/internal/feature-flags/{flag_name}" + resp = self.client.get(url, headers=self._headers(internal=True)) + resp.raise_for_status() + return resp.json() + + def download_video(self, video_url: str, output_path: str | Path) -> Path: + """下载视频文件.""" + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + with self.client.stream("GET", video_url, timeout=60.0) as resp: + resp.raise_for_status() + with open(output_path, "wb") as f: + for chunk in resp.iter_bytes(): + f.write(chunk) + return output_path + + +class CompareRunner: + """新旧引擎对比 Runner.""" + + # 全局默认阈值(唯一真实来源,所有入口统一引用) + DEFAULT_SSIM_THRESHOLD: float = 0.95 + DEFAULT_PSNR_THRESHOLD: float = 28.0 + DEFAULT_AUDIO_SIMILARITY_THRESHOLD: float = 0.90 + DEFAULT_DURATION_TOLERANCE: float = 0.1 + DEFAULT_TASK_TIMEOUT: float = 300.0 + + def __init__( + self, + api: StagingAPI, + output_dir: Path, + *, + ssim_threshold: float | None = None, + psnr_threshold: float | None = None, + audio_similarity_threshold: float | None = None, + task_timeout: float | None = None, + flag_mode: bool = False, # 是否使用 Feature Flag 方式切换引擎 + duration_tolerance: float | None = None, + ): + self.api = api + self.output_dir = output_dir + self.ssim_threshold = ssim_threshold if ssim_threshold is not None else self.DEFAULT_SSIM_THRESHOLD + self.psnr_threshold = psnr_threshold if psnr_threshold is not None else self.DEFAULT_PSNR_THRESHOLD + self.audio_similarity_threshold = ( + audio_similarity_threshold + if audio_similarity_threshold is not None + else self.DEFAULT_AUDIO_SIMILARITY_THRESHOLD + ) + self.duration_tolerance = ( + duration_tolerance if duration_tolerance is not None else self.DEFAULT_DURATION_TOLERANCE + ) + self.task_timeout = task_timeout if task_timeout is not None else self.DEFAULT_TASK_TIMEOUT + self.flag_mode = flag_mode + self.results: list[ScenarioResult] = [] + # flag_mode 下保存原始配置,测试结束后恢复(防污染线上) + self._original_flag_config: dict[str, Any] | None = None + + def run_scenario(self, scenario: CompareScenario) -> ScenarioResult: + """运行单个场景对比.""" + print(f"\n{'='*60}") + print(f"[{scenario.priority}] {scenario.id}: {scenario.name}") + print(f" {scenario.description}") + + result = ScenarioResult(scenario=scenario) + scenario_dir = self.output_dir / scenario.id + scenario_dir.mkdir(parents=True, exist_ok=True) + + try: + # 1. 提交两个引擎的任务 + legacy_task_id = self._submit_with_engine(scenario, "legacy") + unified_task_id = self._submit_with_engine(scenario, "unified") + result.legacy_task_id = legacy_task_id + result.unified_task_id = unified_task_id + print(f" legacy task: {legacy_task_id}") + print(f" unified task: {unified_task_id}") + + # 2. 等待完成 + print(" waiting for legacy...", end="", flush=True) + legacy_status = self.api.wait_for_task(legacy_task_id, timeout=self.task_timeout) + result.legacy_success = legacy_status.get("status") in ("completed", "success", "done") + legacy_video_url = legacy_status.get("output_url", "") or legacy_status.get("video_url", "") + print(f" {'✅' if result.legacy_success else '❌'} ({legacy_status.get('duration_sec', '?')}s)") + + print(" waiting for unified...", end="", flush=True) + unified_status = self.api.wait_for_task(unified_task_id, timeout=self.task_timeout) + result.unified_success = unified_status.get("status") in ("completed", "success", "done") + unified_video_url = unified_status.get("output_url", "") or unified_status.get("video_url", "") + print(f" {'✅' if result.unified_success else '❌'} ({unified_status.get('duration_sec', '?')}s)") + + result.legacy_duration_sec = float(legacy_status.get("duration_sec", 0)) + result.unified_duration_sec = float(unified_status.get("duration_sec", 0)) + + if not (result.legacy_success and result.unified_success): + result.error = f"Legacy success={result.legacy_success}, Unified success={result.unified_success}" + print(" ⚠️ 任务未全部成功,跳过对比") + return result + + # 3. 下载视频 + print(" downloading...", end="", flush=True) + legacy_path = self.api.download_video(legacy_video_url, scenario_dir / "legacy.mp4") + unified_path = self.api.download_video(unified_video_url, scenario_dir / "unified.mp4") + result.legacy_video_path = str(legacy_path) + result.unified_video_path = str(unified_path) + print(" ✅") + + # 4. 像素对比 + print(" computing video diff...", end="", flush=True) + result.video_diff = compute_video_diff( + legacy_path, + unified_path, + ssim_threshold=self.ssim_threshold, + psnr_threshold=self.psnr_threshold, + duration_tolerance=self.duration_tolerance, + ) + print( + f" SSIM={result.video_diff.avg_ssim:.4f} PSNR={result.video_diff.avg_psnr:.2f}dB {'✅' if result.video_diff.passed else '❌'}" + ) + + # 5. 音频对比(仅当都有音频时) + from .audio_diff import probe_has_audio + + legacy_has_audio = probe_has_audio(legacy_path) + unified_has_audio = probe_has_audio(unified_path) + + if legacy_has_audio and unified_has_audio: + print(" computing audio diff...", end="", flush=True) + result.audio_diff = compute_audio_diff( + legacy_path, + unified_path, + similarity_threshold=self.audio_similarity_threshold, + ) + print( + f" similarity={result.audio_diff.similarity_score:.4f} {'✅' if result.audio_diff.passed else '❌'}" + ) + elif legacy_has_audio != unified_has_audio: + result.error = f"音频不一致: legacy_has_audio={legacy_has_audio}, unified_has_audio={unified_has_audio}" + print(f" ⚠️ 音频不一致: legacy={legacy_has_audio}, unified={unified_has_audio}") + else: + print(" audio: both silent (skip)") + + except Exception as e: + result.error = str(e) + print(f" ❌ 错误: {e}") + + self.results.append(result) + return result + + def _submit_with_engine(self, scenario: CompareScenario, engine: str) -> str: + """提交指定引擎的任务. + + 如果 flag_mode=True,通过 Feature Flag 切换,否则通过 engine 参数。 + """ + if self.flag_mode: + # 先设置 flag(用白名单方式,确保只有当前测试用户命中) + percentage = 0 if engine == "legacy" else 100 + self.api.set_feature_flag("render_engine", enabled=True, percentage=percentage) + time.sleep(1) # 给 worker 一点时间刷新配置 + return self.api.submit_render_task(scenario.plan_payload) + else: + return self.api.submit_render_task(scenario.plan_payload, engine=engine) + + def run_all(self, scenarios: list[CompareScenario]) -> list[ScenarioResult]: + """运行所有场景. + + flag_mode=True 时,测试开始前保存原始 Feature Flag 配置, + 结束后(无论成功失败)自动恢复,避免污染线上环境。 + """ + print(f"\n灰度对比测试开始 - {len(scenarios)} 个场景") + print(f"输出目录: {self.output_dir}") + print(f"视频阈值: SSIM>={self.ssim_threshold}, PSNR>={self.psnr_threshold}dB") + print(f"音频阈值: similarity>={self.audio_similarity_threshold}") + + # flag_mode:保存原始配置,测试结束后恢复(防污染) + if self.flag_mode: + try: + self._original_flag_config = self.api.get_feature_flag("render_engine") + print(f" [flag_mode] 已保存原始配置: {self._original_flag_config}") + except Exception as e: + print(f" ⚠️ [flag_mode] 保存原始配置失败: {e}") + print(" 为避免污染线上,将中止测试。请检查 internal_api_key 配置。") + return self.results + + try: + for i, scenario in enumerate(scenarios): + print(f"\n进度: {i+1}/{len(scenarios)}") + self.run_scenario(scenario) + finally: + # 始终恢复原始 flag 配置 + if self.flag_mode and self._original_flag_config: + try: + orig = self._original_flag_config + self.api.set_feature_flag( + "render_engine", + enabled=orig.get("enabled", False), + percentage=orig.get("percentage", 0), + whitelist=orig.get("whitelist"), + ) + print("\n[flag_mode] ✅ 已恢复原始 Feature Flag 配置") + except Exception as e: + print(f"\n[flag_mode] ❌ 恢复 Feature Flag 失败: {e}") + print(" 请手动检查并恢复 render_engine flag 配置!") + + return self.results + + def summary(self) -> dict[str, Any]: + """生成汇总统计.""" + total = len(self.results) + passed = sum(1 for r in self.results if r.passed) + failed = total - passed + + # 性能对比 + perf_diffs = [] + for r in self.results: + if r.legacy_success and r.unified_success and r.legacy_duration_sec > 0: + diff_pct = (r.unified_duration_sec - r.legacy_duration_sec) / r.legacy_duration_sec * 100 + perf_diffs.append(diff_pct) + avg_perf_diff = sum(perf_diffs) / len(perf_diffs) if perf_diffs else 0.0 + + return { + "total": total, + "passed": passed, + "failed": failed, + "pass_rate": f"{passed/total*100:.1f}%" if total > 0 else "0%", + "avg_perf_diff_pct": round(avg_perf_diff, 2), + "scenarios": [self._result_to_dict(r) for r in self.results], + "timestamp": datetime.now().isoformat(), + "ssim_threshold": self.ssim_threshold, + "psnr_threshold": self.psnr_threshold, + "audio_threshold": self.audio_similarity_threshold, + } + + def _result_to_dict(self, r: ScenarioResult) -> dict[str, Any]: + return { + "id": r.scenario.id, + "name": r.scenario.name, + "priority": r.scenario.priority, + "passed": r.passed, + "legacy_success": r.legacy_success, + "unified_success": r.unified_success, + "legacy_duration_sec": r.legacy_duration_sec, + "unified_duration_sec": r.unified_duration_sec, + "video_diff": r.video_diff.to_dict() if r.video_diff else None, + "audio_diff": r.audio_diff.to_dict() if r.audio_diff else None, + "error": r.error, + } + + +def generate_html_report(summary: dict[str, Any], output_path: Path): + """生成 HTML 对比报告.""" + scenarios = summary["scenarios"] + + # 按通过/失败分组 + passed_list = [s for s in scenarios if s["passed"]] + failed_list = [s for s in scenarios if not s["passed"]] + + # 构建场景卡片 + scenario_cards = "" + for s in scenarios: + status_class = "pass" if s["passed"] else "fail" + status_text = "✅ 通过" if s["passed"] else "❌ 失败" + + vdiff = s.get("video_diff") or {} + adiff = s.get("audio_diff") or {} + + video_info = "" + if vdiff: + video_info = f""" +
无数据
'} +无音频或跳过
'} +无数据
'} +