"""Issue #1658: pHash 阈值校准 + 颜色直方图融合 — 单元测试. 在 #1659（动态抽帧+滑动窗口）与 #1660（查重率）已合入 develop 的基础上， 本测试覆盖 #1658 的最小增量改动： 1. PHASH_THRESHOLD 由 10 收紧到 8（核心校准） 2. 融合权重常量 MATCH_RATIO_THRESHOLD / PHASH_WEIGHT / HISTOGRAM_WEIGHT 实际生效 （不再是硬编码魔法数字） 3. VideoDeduplicator._compute_fusion_score 统一融合得分方法： - 无直方图数据时回退中性值 0.5 - DB NULL（None）显式回退空列表，不崩溃 - 全零直方图（全黑视频）为有效数据，参与 Bhattacharyya 计算 - 返回 0~1 原始得分，判重由调用方与 DUPLICATE_THRESHOLD 比较 4. Bhattacharyya 系数对上游异常负值有 sqrt domain 防御 """ from __future__ import annotations import sys from unittest.mock import MagicMock import pytest def _mock_module(**attrs): """Create a mock module with __spec__ to avoid AttributeError.""" m = MagicMock() m.__spec__ = None for k, v in attrs.items(): setattr(m, k, v) return m # ── Module-level setup: mock deps, import dedup, then restore sys.modules ── _SAVED_MODULES_KEYS = set(sys.modules.keys()) _SAVED_MODULES_VALUES = { k: sys.modules.get(k) for k in [ "cv2", "celery", "sqlalchemy", "sqlalchemy.orm", "sqlalchemy.engine", "sqlalchemy.ext", "sqlalchemy.ext.declarative", "worker_app.db", "worker_app.celery_app", "worker_app.core.config", "packages.adapters.sqlalchemy_impl.session", "packages.adapters.sqlalchemy_impl.generated_video_repository", "packages.adapters.sqlalchemy_impl.models", "packages.shared.config", "packages.shared.storage", ] } sys.modules["cv2"] = _mock_module() _mock_celery = MagicMock() _mock_celery.Task = MagicMock _mock_celery.Celery = MagicMock _mock_celery.__spec__ = None sys.modules["celery"] = _mock_celery _mock_sqla = MagicMock() _mock_sqla.__path__ = [] _mock_sqla.__spec__ = None sys.modules["sqlalchemy"] = _mock_sqla _mock_sqla_orm = MagicMock() _mock_sqla_orm.__path__ = [] _mock_sqla_orm.__spec__ = None _mock_sqla_orm.Session = MagicMock sys.modules["sqlalchemy.orm"] = _mock_sqla_orm sys.modules["sqlalchemy.engine"] = _mock_module() sys.modules["sqlalchemy.ext"] = _mock_module() sys.modules["sqlalchemy.ext.declarative"] = _mock_module() sys.modules["worker_app.db"] = _mock_module(SessionLocal=MagicMock()) sys.modules["worker_app.celery_app"] = _mock_module(celery_app=MagicMock()) sys.modules["worker_app.core.config"] = _mock_module(get_settings=MagicMock(return_value=MagicMock())) sys.modules["packages.adapters.sqlalchemy_impl.session"] = _mock_module( Base=MagicMock(), build_engine=MagicMock(), build_session_factory=MagicMock(), ensure_database_exists=MagicMock(), initialize_database=MagicMock(), ) sys.modules["packages.adapters.sqlalchemy_impl.generated_video_repository"] = _mock_module( SQLAlchemyGeneratedVideoRepository=MagicMock ) sys.modules["packages.adapters.sqlalchemy_impl.models"] = _mock_module( VideoFingerprintChunkModel=MagicMock, GeneratedVideoModel=MagicMock, ) sys.modules["packages.shared.config"] = _mock_module(get_shared_settings=MagicMock(return_value=MagicMock())) sys.modules["packages.shared.storage"] = _mock_module() import video_processing.dedup as _dedup_mod # noqa: E402 from video_processing.dedup import ( # noqa: E402 DUPLICATE_THRESHOLD, HISTOGRAM_WEIGHT, MATCH_RATIO_THRESHOLD, PHASH_THRESHOLD, PHASH_WEIGHT, VideoDeduplicator, ) # ── Restore sys.modules immediately after import ── for _key in list(sys.modules.keys()): if _key not in _SAVED_MODULES_KEYS: del sys.modules[_key] for _key, _value in _SAVED_MODULES_VALUES.items(): if _value is not None: sys.modules[_key] = _value elif _key in sys.modules: del sys.modules[_key] del _SAVED_MODULES_KEYS, _SAVED_MODULES_VALUES, _key, _value # ── 测试夹具 ───────────────────────────────────────────────────── _UNIFORM_HIST = [1.0 / 96] * 96 # 归一化均匀直方图，sum=1.0，自相似度≈1.0 _ZERO_HIST = [0.0] * 96 # 全黑视频的全零直方图（有效数据） # ── TestThresholdCalibration：#1658 核心校准 ──────────────────── class TestThresholdCalibration: """pHash 阈值校准（#1658 收紧到 8，#1702 两轮真实数据重校准 12→16）。 #1702 第一轮 staging 离线实验：同帧两次 2-5% 随机裁剪距离 4~10；同源成片 （密集 1s 采样）<=12 命中 4/11、异源成片最小距离 24 → 初定 12。 #1702 第二轮（证据视频 B->A 仍漏检）扩样本到该用户 15 个真实成片实测： 同源降重对中位数距离 14、<=16 命中 8/11=0.73；异源 13 个候选 <=16 命中 全 0、每帧全局最近邻最小距离 18 → 校准为 16（与异源仍有 >=2bit 裕度）。 """ def test_phash_threshold_is_calibrated(self): assert VideoDeduplicator.PHASH_THRESHOLD == PHASH_THRESHOLD == 16 def test_match_ratio_threshold_constant(self): assert MATCH_RATIO_THRESHOLD == 0.7 def test_duplicate_threshold_constant(self): assert DUPLICATE_THRESHOLD == 0.70 def test_fusion_weights(self): assert PHASH_WEIGHT == 0.7 assert HISTOGRAM_WEIGHT == 0.3 def test_threshold_matching_semantics(self): """阈值比较统一为 <=（帧匹配与片段匹配同一口径）。 场景：5 个关键帧距离为 [10, 14, 16, 18, 26]。 - <=16（#1702 二次校准阈值）：3 帧匹配 → 0.6 < 0.7，被帧比例门槛 拦截（异源安全边界：真实数据异源最近邻最小距离 18，<=16 命中 0） - 距离正好 16 的同源降重帧应算匹配（< 与 <= 口径统一） """ distances = [10, 14, 16, 18, 26] matched = sum(1 for d in distances if d <= VideoDeduplicator.PHASH_THRESHOLD) assert matched == 3 assert matched / len(distances) == 0.6 assert matched / len(distances) < MATCH_RATIO_THRESHOLD # 异源安全边界（实测最小距离 18）及以上绝不匹配 assert not any(d <= VideoDeduplicator.PHASH_THRESHOLD for d in (18, 24, 26, 30)) # ── TestComputeFusionScore：统一融合得分方法 ──────────────────── class TestComputeFusionScore: """_compute_fusion_score(median_distance, histograms_a, histograms_b)。""" def test_no_histogram_falls_back_to_neutral_05(self): """双方均无直方图 → hist_similarity 回退 0.5。 d=0: 0.7*1.0 + 0.3*0.5 = 0.85 """ score = VideoDeduplicator._compute_fusion_score(0, [], []) assert score == pytest.approx(0.85, abs=1e-6) def test_none_histograms_treated_as_empty(self): """DB NULL（None）必须显式回退空列表，不得 len(None) 崩溃。""" score_none = VideoDeduplicator._compute_fusion_score(0, [], None) score_empty = VideoDeduplicator._compute_fusion_score(0, [], []) assert score_none == pytest.approx(score_empty, abs=1e-9) assert score_none == pytest.approx(0.85, abs=1e-6) def test_none_histograms_on_query_side_no_crash(self): """查询侧直方图为 None 时同样不崩溃。""" score = VideoDeduplicator._compute_fusion_score(0, None, [_UNIFORM_HIST]) # 查询侧无直方图 → 平均相似度为 0（无 ha 可匹配）→ 0.7*1.0 + 0.3*0 = 0.7 assert score == pytest.approx(0.7, abs=1e-6) def test_identical_uniform_histograms_score_near_1(self): """完全相同的归一化直方图：Bhattacharyya≈1.0 → 融合分≈1.0。""" score = VideoDeduplicator._compute_fusion_score(0, [_UNIFORM_HIST], [_UNIFORM_HIST]) assert score == pytest.approx(1.0, abs=1e-6) def test_all_zero_histogram_is_valid_data(self): """全零直方图（全黑视频）是有效数据，Bhattacharyya=0，不得走 0.5 回退。 若错误地用 `if histograms_b` 之外的 `or []` 把全零列表清空， 会错误回退到 0.5，把全黑视频的相似度抬高 0.15。 d=0 时：正确行为 hist_sim=0 → 0.7*1.0 + 0.3*0 = 0.7； 若全零直方图被错误清空回退 0.5 → 0.85。 """ score = VideoDeduplicator._compute_fusion_score(0, [_ZERO_HIST], [_ZERO_HIST]) assert score == pytest.approx(0.7, abs=1e-6) # 与错误回退值 0.85 明确区分开 assert abs(score - 0.85) > 0.1 # 注：d=0 时 phash 满分 0.7 恰达 DUPLICATE_THRESHOLD，全黑+完全相同 phash 仍判重，符合预期 assert score >= DUPLICATE_THRESHOLD - 1e-9 def test_score_range_within_0_1(self): for d in (0, 8, 16, 32, 64): score = VideoDeduplicator._compute_fusion_score(d, [_UNIFORM_HIST], [_UNIFORM_HIST]) assert 0.0 <= score <= 1.0 def test_formula_matches_weights(self): """得分 = PHASH_WEIGHT * (1 - d/64) + HISTOGRAM_WEIGHT * hist_sim。""" d = 6 # phash_sim = 1 - 6/64 = 0.90625 score = VideoDeduplicator._compute_fusion_score(d, [], []) # hist 回退 0.5 expected = PHASH_WEIGHT * (1 - d / 64) + HISTOGRAM_WEIGHT * 0.5 assert score == pytest.approx(expected, abs=1e-9) # 0.7*0.90625 + 0.15 = 0.634375 + 0.15 = 0.784375 assert score == pytest.approx(0.784375, abs=1e-6) # ── TestBhattacharyyaDefense：负值/异常输入防御 ───────────────── class TestBhattacharyyaDefense: """Bhattacharyya 系数对异常输入的防御。""" def test_negative_values_do_not_raise(self): """上游异常负值不得触发 sqrt domain error（max(0.0, ai*bi) 保护）。""" bad_hist = [-0.01] * 96 # 异常负值 coeff = VideoDeduplicator._bhattacharyya_coefficient(bad_hist, _UNIFORM_HIST) # 负值乘积被钳为 0，系数为 0 而不是抛 ValueError assert coeff == pytest.approx(0.0, abs=1e-9) def test_normal_histograms_coefficient_near_1(self): coeff = VideoDeduplicator._bhattacharyya_coefficient(_UNIFORM_HIST, _UNIFORM_HIST) assert coeff == pytest.approx(1.0, abs=1e-6) def test_disjoint_histograms_coefficient_0(self): """完全不重叠的直方图（前半 vs 后半非零）系数为 0。""" hist_a = [0.0] * 96 hist_b = [0.0] * 96 for i in range(48): hist_a[i] = 1.0 / 48 for i in range(48, 96): hist_b[i] = 1.0 / 48 coeff = VideoDeduplicator._bhattacharyya_coefficient(hist_a, hist_b) assert coeff == pytest.approx(0.0, abs=1e-9) # ── TestHistogramSimilarityEdgeCases ──────────────────────────── class TestHistogramSimilarityEdgeCases: """_compute_histogram_similarity 的边界行为。""" def test_empty_either_side_returns_0(self): assert VideoDeduplicator._compute_histogram_similarity([], [_UNIFORM_HIST]) == 0.0 assert VideoDeduplicator._compute_histogram_similarity([_UNIFORM_HIST], []) == 0.0 def test_best_match_per_histogram(self): """每个查询直方图取与目标集合的最佳匹配，再取平均。""" h1 = _UNIFORM_HIST h2 = [0.0] * 96 h2[0] = 1.0 # 与均匀直方图完全不重叠 # 查询侧两张直方图：h1 最佳匹配≈1.0，h2 最佳匹配≈sqrt(1/96)≈0.102 sim = VideoDeduplicator._compute_histogram_similarity([h1, h2], [h1]) assert sim == pytest.approx((1.0 + (1.0 / 96) ** 0.5) / 2, abs=1e-3)