"""Issue #1702 — 查重率恒为 0% 修复：单测. 覆盖验收要求： 1. 同源不同裁剪的两个视频能检出非 0 相似度（指纹中心裁剪绕开降重 + 阈值校准） 2. 局部片段复用（B 结尾 2s ≈ A 中间 2s）能检出 3. 异源视频不误报（相似度接近 0） 4. N=1 现有流程不回归 5. P1 确定性 bug：时长预过滤单位 /1000、直方图归一化、temporal_coverage 量纲、阈值比较统一 6. P0：±1 邻接对齐、短视频自适应连续门槛 7. P2：0 匹配也要落日志 """ from __future__ import annotations import logging import sys from pathlib import Path from unittest.mock import MagicMock, patch import pytest sys.modules.setdefault("cv2", MagicMock()) ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "apps" / "worker")) sys.path.insert(0, str(ROOT / "packages")) from video_processing.dedup import ( # noqa: E402 PHASH_THRESHOLD, SEGMENT_MATCH_THRESHOLD, FingerprintChunk, VideoDeduplicator, VideoFingerprint, find_duplicate_segments, ) # ── helpers ──────────────────────────────────────────────────── def _h(d: int) -> str: """64-bit phash with exactly d bits set vs zero hash.""" bits = ["0"] * 64 for i in range(d): bits[i] = "1" return f"{int(''.join(bits), 2):016x}" def _chunk(phash: str, t0: float, t1: float): return FingerprintChunk( start_time_ms=int(t0 * 1000), end_time_ms=int(t1 * 1000), phash_binary=phash, color_histogram=[], frame_count=1, ) def _fingerprint(phashes, duration, chunks=None, md5="fp-md5-x"): return VideoFingerprint( md5=md5, keyframe_phashes=list(phashes), color_histograms=[], duration=duration, resolution=(1280, 720), chunks=chunks or [], ) def _video(vid, phashes, duration=10.0, project_id="proj1"): from packages.domain import GeneratedVideo return GeneratedVideo( id=vid, project_id=project_id, generation_task_id=f"task-{vid}", name=f"video-{vid}.mp4", file_url=f"https://example.com/{vid}.mp4", file_size=1000, duration=duration, width=1280, height=720, fps=25.0, video_fingerprint={"md5": f"md5-{vid}", "keyframe_phashes": list(phashes)}, ) def _rate(deduplicator, fp, videos, session=None): session_magic = MagicMock() # 分片表无数据 -> 回退 JSON keyframe_phashes session_magic.query.return_value.filter.return_value.order_by.return_value.all.return_value = [] with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo: repo = MockRepo.return_value repo.list_by_project.return_value = videos repo.list_by_user.return_value = videos return deduplicator.compute_duplicate_rate(fp, "proj1", "new-vid", session_magic, scope="project") def _check(deduplicator, fp, videos, scope="project", **kw): session_magic = MagicMock() session_magic.query.return_value.filter.return_value.order_by.return_value.all.return_value = [] with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo: repo = MockRepo.return_value repo.list_by_project.return_value = videos repo.list_by_user.return_value = videos return deduplicator.check_duplicate(fp, "proj1", session_magic, scope=scope, **kw) # ── P0-1/P0-2: 同源不同裁剪（距离 6~10）检出非 0 ────────────── class TestSameSourceDifferentCrop: """同源成片：random_edge_crop 后 pHash 距离 6~10，应检出非 0 相似度。""" def test_same_source_high_similarity_detected(self): ddp = VideoDeduplicator() # 新视频 5 个分片，每个 phash 与已有视频对应分片距离 6（< 阈值） base = [_h(0) for _ in range(5)] new = [_h(6) for _ in range(5)] existing = _video("v-old", base, duration=11.0) chunks = [_chunk(h, i * 2.2, (i + 1) * 2.2) for i, h in enumerate(new)] fp = _fingerprint(new, 11.0, chunks=chunks) result = _rate(ddp, fp, [existing], MagicMock()) assert result["duplicate_rate"] > 0 assert result["visual_similarity"] > 0 def test_same_source_distance_at_threshold_still_detected(self): """距离正好等于阈值（<=）也要算匹配——阈值比较统一为 <=。""" assert PHASH_THRESHOLD <= 16, "阈值应经真实数据校准保持在能检出同源裁剪/降重对的范围（#1702 二次校准为 16）" ddp = VideoDeduplicator() base = [_h(0) for _ in range(6)] new = [_h(PHASH_THRESHOLD) for _ in range(6)] existing = _video("v-old", base, duration=12.0) chunks = [_chunk(h, i * 2, (i + 1) * 2) for i, h in enumerate(new)] fp = _fingerprint(new, 12.0, chunks=chunks) result = _rate(ddp, fp, [existing], MagicMock()) assert result["duplicate_rate"] > 0 # ── P0-2: 局部片段复用（B 结尾 2s ≈ A 中间 2s） ──────────────── class TestPartialReuse: def test_partial_reuse_tail_overlap_detected(self): """新视频 6 片，最后 2 片命中已有视频中间 2 片（距离 4），其余不匹配。 旧逻辑 frame_match_rate=2/6≈0.33（<0.3 硬跳过边界）+ MIN_CONSECUTIVE=5 导致完全检不出；新逻辑 coverage 为主指标 + 自适应门槛应检出。 """ ddp = VideoDeduplicator() # 已有 8 片：索引 3、4 是被复用的镜头 old = [_h(20 + i) for i in range(8)] # 新视频 6 片：最后 2 片对应 old[3], old[4]，距离 4；其余距离 30 new = [_h(50 + i) for i in range(4)] + [_h(4)] * 2 # 让 new[4] 与 old[3] 距离 4、new[5] 与 old[4] 距离 4（构造近似） new[4] = f"{int('1' * 4 + '0' * 60, 2):016x}" new[5] = f"{int('1' * 4 + '0' * 60, 2):016x}" old[3] = _h(0) old[4] = _h(0) existing = _video("v-old", old, duration=16.0) chunks = [_chunk(h, i * 2, (i + 1) * 2) for i, h in enumerate(new)] fp = _fingerprint(new, 12.0, chunks=chunks) result = _rate(ddp, fp, [existing], MagicMock()) # 局部复用：duplicate_rate 必须非 0 assert result["duplicate_rate"] > 0 def test_short_video_adaptive_consecutive_threshold(self): """11s/5 片短视频：MIN_CONSECUTIVE 自适应 min(5, max(2, 5//2))=2， 2 片连续命中即报片段（旧值 5 让短视频永远无法报片段）。""" q = [ FingerprintChunk(0, 2000, "f" * 16, []), FingerprintChunk(2000, 4000, "0" * 16, []), FingerprintChunk(4000, 6000, f"{int('11110000', 2):016x}", []), ] t = [ FingerprintChunk(0, 2000, "f" * 16, []), FingerprintChunk(2000, 4000, "0" * 16, []), FingerprintChunk(4000, 6000, "e" * 16, []), ] # 3 片视频自适应门槛 = min(5, max(2, 3//2)) = 2 segs = find_duplicate_segments(q, t) assert len(segs) >= 1 # ── P0-3: ±1 邻接窗口对齐 ───────────────────────────────────── class TestNeighborAlignment: def test_neighbor_window_absorbs_boundary_jitter(self): """切点错位导致目标索引偏移 ±1 时，连续匹配不应被中断。""" q = [FingerprintChunk(i * 1000, (i + 1) * 1000, f"{i:016x}", []) for i in range(4)] # 目标：前 3 片与 q 相同，但第 3 片最佳匹配偏移 +1（t[4]），t[3] 是无关内容 t_hashes = [f"{i:016x}" for i in range(3)] + ["f" * 16, f"{3:016x}"] t = [FingerprintChunk(i * 1000, (i + 1) * 1000, h, []) for i, h in enumerate(t_hashes)] segs = find_duplicate_segments(q, t) # q[0],q[1] 精确匹配 t[0],t[1]；q[2]->t[2]；q[3]->t[4]（步进 2，窗口 ±1 内） assert len(segs) >= 1 assert segs[0].query_end_ms >= 3000 # ── P0-5 / 验收：异源不误报 ─────────────────────────────────── class TestDifferentSourceNoFalsePositive: def test_unrelated_videos_near_zero(self): ddp = VideoDeduplicator() # 异源：所有分片距离 >= 20 old = [_h(40 + i * 3 % 20) for i in range(6)] new = [_h(0 + i) for i in range(6)] existing = _video("v-old", old, duration=12.0) chunks = [_chunk(h, i * 2, (i + 1) * 2) for i, h in enumerate(new)] fp = _fingerprint(new, 12.0, chunks=chunks) result = _rate(ddp, fp, [existing], MagicMock()) assert result["duplicate_rate"] == 0 assert result["visual_similarity"] < 0.7 assert result["match_count"] == 0 def test_check_duplicate_returns_none_for_unrelated(self): ddp = VideoDeduplicator() old = [_h(40 + i) for i in range(6)] new = [_h(i) for i in range(6)] existing = _video("v-old", old, duration=12.0) fp = _fingerprint(new, 12.0) result = _check(ddp, fp, [existing]) assert result is None # ── N=1 不回归 ──────────────────────────────────────────────── class TestSingleChunkNoRegression: def test_single_chunk_identical_detected(self): ddp = VideoDeduplicator() h = _h(2) existing = _video("v-old", [h], duration=3.0) chunks = [_chunk(h, 0, 3000)] fp = _fingerprint([h], 3.0, chunks=chunks) result = _rate(ddp, fp, [existing], MagicMock()) assert result["duplicate_rate"] > 0 def test_single_chunk_md5_exact_match(self): ddp = VideoDeduplicator() existing = _video("v-old", [_h(0)], duration=3.0) existing.video_fingerprint["md5"] = "same" fp = _fingerprint([_h(0)], 3.0, md5="same") result = _check(ddp, fp, [existing]) assert result is not None assert result["reason"] == "exact_md5_match" # ── P1-6: 时长预过滤单位 bug ────────────────────────────────── class TestDurationPrefilterUnit: def test_user_scope_skips_duration_prefilter(self): """Issue #1702: scope=user 跨项目查重不做 ±15% 时长预过滤。 旧逻辑 duration/1000 单位 bug 先修成秒，但 ±15% 窗口与局部片段复用 根本矛盾——复用片段的两个视频时长必然不同（证据视频 20s vs 11s 差 42%）， 窗口内找不到对方导致 is_duplicate 恒 False。最终口径：scope=user 全量 遍历同用户视频（与 compute_duplicate_rate 一致），不传 duration_min/max。 """ ddp = VideoDeduplicator() fp = _fingerprint([_h(0)], 13.5) session_magic = MagicMock() session_magic.query.return_value.filter.return_value.order_by.return_value.all.return_value = [] with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo: repo = MockRepo.return_value repo.list_by_user.return_value = [] ddp.check_duplicate(fp, "proj1", session_magic, scope="user", user_id="u1", duration_sec=fp.duration) args, kwargs = repo.list_by_user.call_args # 全量查询：不带任何时长过滤参数（局部复用必须跨时长比较） assert "duration_min" not in kwargs assert "duration_max" not in kwargs assert args == ("u1",) or args == () # ── P1-7: 颜色直方图归一化 ──────────────────────────────────── class TestHistogramNormalization: def test_bhattacharyya_coefficient_in_unit_range(self): """Bhattacharyya 系数必须在 [0,1]（旧 L2 + 3 通道拼接算出 ~14.9）。""" # 3 通道拼接、每通道概率分布（Σ=1） hist_a = [0.5, 0.5] + [0.0] * 94 + [0.5, 0.5] + [0.0] * 94 + [0.5, 0.5] + [0.0] * 94 # 长度裁剪到 96（3 通道 × 32 bins） hist_a = ([0.5, 0.5] + [0.0] * 30) * 3 hist_b = ([0.5, 0.5] + [0.0] * 30) * 3 coeff = VideoDeduplicator._bhattacharyya_coefficient(hist_a, hist_b) assert 0.0 <= coeff <= 1.0 assert coeff > 0.99 # 完全相同 -> 1.0 def test_bhattacharyya_disjoint_hist_low(self): hist_a = ([1.0] + [0.0] * 31) * 3 hist_b = ([0.0] * 31 + [1.0]) * 3 coeff = VideoDeduplicator._bhattacharyya_coefficient(hist_a, hist_b) assert coeff < 0.05 # ── P1-8: temporal_coverage 量纲 ────────────────────────────── class TestTemporalCoverageUnits: def test_coverage_uses_milliseconds(self): """命中片段 6s / 视频 12s -> coverage=0.5；旧 bug 把 duration（秒）当毫秒， covered_ms(6000)/duration(12) = 500 -> min(1.0)=1.0 误判 100% 覆盖。""" ddp = VideoDeduplicator() old = [_h(0) for _ in range(6)] new = [_h(0) for _ in range(3)] + [_h(30) for _ in range(3)] existing = _video("v-old", old, duration=12.0) # 新视频 12s，前 6s（3 片）与 old 相同 chunks = [_chunk(h, i * 2, (i + 1) * 2) for i, h in enumerate(new)] fp = _fingerprint(new, 12.0, chunks=chunks) result = _rate(ddp, fp, [existing], MagicMock()) # coverage 应约 0.5（3 片 × 2s = 6s / 12s），duplicate_rate ≈ (0.5*0.4 + 0.5*0.6)*100 = 50 assert 30 < result["duplicate_rate"] < 70 # ── P1-9: 阈值比较统一 ──────────────────────────────────────── class TestThresholdConsistency: def test_frame_and_segment_thresholds_same_source(self): assert SEGMENT_MATCH_THRESHOLD == PHASH_THRESHOLD assert VideoDeduplicator.PHASH_THRESHOLD == PHASH_THRESHOLD # ── P2: 0 匹配也要有日志痕迹 ────────────────────────────────── class TestZeroMatchLogging: def test_no_match_emits_info_log(self, caplog): ddp = VideoDeduplicator() old = [_h(40 + i) for i in range(5)] existing = _video("v-old", old, duration=10.0) fp = _fingerprint([_h(i) for i in range(5)], 10.0) with caplog.at_level(logging.INFO, logger="video_processing.dedup"): result = _check(ddp, fp, [existing]) assert result is None assert any("no match" in r.message for r in caplog.records) # ── recompute 任务下载路径（#1702 连带修复：旧硬编码 key 404） ───── class TestRecomputeDownloadPath: """recompute-dedup 走 check_duplicate_task，需要从 OSS 重新下载成片。 旧代码硬编码 projects/{pid}/generated/{vid}/{vid}.mp4（从不存在）， 真实 key 在 file_url：generated/projects/{pid}/tasks/{tid}/rendered_*.mp4。 """ def test_task_downloads_from_file_url(self): import inspect import video_processing.dedup as dedup_mod source = inspect.getsource(dedup_mod.check_duplicate_task) # 下载 key 必须来自 video.file_url assert 'getattr(video, "file_url"' in source or "video.file_url" in source # 旧的硬编码 key 只能作为回退存在，不能是主路径 assert "falling back to legacy key" in source # download_file 接收的是派生 key 而非硬编码 f-string assert "storage_service.download_file(download_key" in source assert '/generated/{generated_video_id}/{generated_video_id}.mp4"' not in source.replace( 'download_key = f"projects/{video.project_id}/generated/{generated_video_id}/{generated_video_id}.mp4"', "", ) # ── check_duplicate 排除自身（#1702 连带修复：recompute 自匹配） ───── class TestCheckDuplicateExcludesSelf: def test_exclude_video_id_skips_self_match(self): """recompute 时当前视频已在候选列表：自匹配距离 0 分会让 duplicate_of 指向自己。exclude_video_id 必须跳过自身，返回真实的其他匹配或 None。 """ ddp = VideoDeduplicator() h = _h(0) # 候选列表里同时放「自己」（完全相同）和一个异源视频 self_video = _video("v-self", [h], duration=10.0) other_video = _video("v-other", [_h(40 + i) for i in range(3)], duration=10.0) fp = _fingerprint([h], 10.0) session = MagicMock() session.query.return_value.filter.return_value.order_by.return_value.all.return_value = [] # 不传 exclude → 自匹配命中（错误行为复现） with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo: MockRepo.return_value.list_by_project.return_value = [self_video, other_video] result = ddp.check_duplicate(fp, "proj1", session) assert result is not None and result["duplicate_of"] == "v-self" # 传 exclude_video_id → 跳过自己，异源不匹配 → None with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo: MockRepo.return_value.list_by_project.return_value = [self_video, other_video] result = ddp.check_duplicate(fp, "proj1", session, exclude_video_id="v-self") assert result is None # 排除自己后，真实同源其他视频仍能检出 real_dup = _video("v-real", [h], duration=10.0) with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo: MockRepo.return_value.list_by_project.return_value = [self_video, real_dup] result = ddp.check_duplicate(fp, "proj1", session, exclude_video_id="v-self") assert result is not None and result["duplicate_of"] == "v-real" # ── 阈值 16 二次校准 + 时序抖动对齐（#1702 第二轮真实数据校准） ────── class TestThreshold16Calibration: """二次校准：staging 15 个真实成片实测——同源降重对中位数距离 14、 <=16 命中 8/11=0.73；异源 13 个候选每帧全局最近邻最小距离 18、<=16 命中全 0。阈值 16 检出同源且异源零误报（>=2bit 安全裕度）。""" def test_threshold_calibrated_to_16(self): assert PHASH_THRESHOLD == 16 @staticmethod def _variant(phash: str, d: int) -> str: """在 phash 基础上翻转恰好 d 个低位 bit → 与原哈希汉明距离恰为 d。""" v = int(phash, 16) for b in range(d): v ^= 1 << b return f"{v:016x}" def test_distance_18_unrelated_not_matched(self): """距离 18（异源实测最小最近邻距离）不判匹配，距离 16 判匹配。""" ddp = VideoDeduplicator() # 多样化 base（相邻帧各不相同，避免黑屏过滤器） base = [_h(i + 4) for i in range(8)] near = [self._variant(h, 16) for h in base] # 同源降重：每帧距离恰 16 far = [self._variant(h, 18) for h in base] # 异源边界：每帧距离恰 18 fp_near = _fingerprint(near, 8.0, chunks=[_chunk(h, i, i + 1) for i, h in enumerate(near)]) fp_far = _fingerprint(far, 8.0, chunks=[_chunk(h, i, i + 1) for i, h in enumerate(far)]) r_near = _rate(ddp, fp_near, [_video("v-base", base, duration=8.0)]) r_far = _rate(ddp, fp_far, [_video("v-base", base, duration=8.0)]) assert r_near["duplicate_rate"] > 0, "距离16的同源降重对必须检出" assert r_far["duplicate_rate"] == 0.0, "距离18的异源对不得误报" assert r_far["match_count"] == 0 def test_deduped_pair_frame_match_rate_over_threshold(self): """真实场景比例：11 帧中 8 帧距离 <=16（0.73 >= 0.7）， 其余 3 帧异源距离（>=18）——frame_match_rate 必须过 0.7 门槛。""" ddp = VideoDeduplicator() base = [_h(i + 4) for i in range(11)] near = [self._variant(h, 14) for h in base[:8]] # 中位数 14 的同源降重帧 # 异源帧用完全不同前缀（与 base 距离 >=30） far = [_h(52 + i) for i in range(3)] query = near + far fp = _fingerprint(query, 11.0, chunks=[_chunk(h, i, i + 1) for i, h in enumerate(query)]) r = _rate(ddp, fp, [_video("v-base", base, duration=11.0)]) # frame_match_rate=8/11=0.73、时序片段覆盖 ~0.73 # → duplicate_rate = 0.4*0.73+0.6*0.73 ≈ 73%（空直方图回退下 fusion=0.6965 # 略低于 is_duplicate 的 0.70 判定阈值，故此处断言查重率而非 match_count； # 真实视频带颜色直方图时 fusion≈0.80，staging A-C 实测 is_duplicate=True） assert r["duplicate_rate"] >= 70.0 class TestTemporalJitterAlignment: """时序对齐允许目标索引正/反向 ±(neighbor_window+1) 抖动。 密集 1s 采样下相邻帧 pHash 接近，全局最近邻会在目标相邻帧间 正负 1 跳变（场景切割/取帧错位/局部倒退）；旧逻辑只允许正向 delta，把同源连续匹配拆碎，min_consecutive 门槛够不上而漏检。 """ def test_backward_jitter_keeps_run_continuous(self): """匹配目标索引序列 0,1,2,1,2,3（含一次 -1 倒退）应保持同一 run。""" from video_processing.dedup import find_duplicate_segments # 构造 target 相邻帧 pHash 相同（距离0），query 帧的最近邻在 # target[1]/target[2] 之间抖动；全部 <= 阈值 t_hash = _h(0) other = _h(40) # target: 帧0-3 相同场景，帧4+ 异源 t_chunks = [_chunk(t_hash, i, i + 1) for i in range(4)] + [_chunk(other, i, i + 1) for i in range(4, 8)] # query 6 帧同场景（最近邻会落到 target 0~3，索引可正可负） q_chunks = [_chunk(t_hash, i, i + 1) for i in range(6)] segments = find_duplicate_segments(q_chunks, t_chunks) assert segments, "含 ±1 时序抖动的连续匹配必须形成片段" # 6 帧匹配 >= min_consecutive(min(5,max(2,6//2))=5)，报为一个片段 assert len(segments) == 1 seg = segments[0] assert seg.query_end_ms - seg.query_start_ms >= 5000 def test_large_backward_jump_breaks_run(self): """目标索引倒退 > neighbor_window+1（如从 5 跳回 0）不属于抖动， 不桥接为同一片段；孤立短匹配 < min_consecutive 不报片段。""" from video_processing.dedup import find_duplicate_segments # 异源段：9-bit 不重叠段（相邻段隔 3 bit），跨段距离 18~24 > 阈值 16 def _bit_seg(start): bits = ["0"] * 64 for b in range(9): bits[start + b] = "1" return f"{int(''.join(bits), 2):016x}" t_hash = _bit_seg(0) # 复用场景：bit 0-8 t_other = [_bit_seg(22 + 4 * i) for i in range(4)] # target 异源段 q_other = [_bit_seg(40 + 4 * i) for i in range(3)] # query 异源段 # target: 帧0 同场景；帧1-4 异源；帧5-6 同场景 t_chunks = ( [_chunk(t_hash, 0, 1)] + [_chunk(t_other[i - 1], i, i + 1) for i in range(1, 5)] + [_chunk(t_hash, i, i + 1) for i in range(5, 7)] ) # query: 帧0 匹配 target[0]；帧1-3 异源（与 target 任何帧距离 >16）；帧4-5 匹配 target[5,6] q_chunks = ( [_chunk(t_hash, 0, 1)] + [_chunk(q_other[i - 1], i, i + 1) for i in range(1, 4)] + [_chunk(t_hash, i, i + 1) for i in range(4, 6)] ) segments = find_duplicate_segments(q_chunks, t_chunks) # 两段各 1、2 帧 < min_consecutive=5 → 不报片段（大跳跃不桥接） assert segments == []