xiaoxia-saas/tests/unit/test_dedup_v2.py

"""Issue #1659: 动态抽帧 + 滑动窗口时序匹配 单元测试.

覆盖：
- detect_keyframe_timestamps: 关键帧检测（mock cv2）
- find_duplicate_segments: 滑动窗口时序匹配
- DuplicateSegment 数据类
- _bhattacharyya_coefficient / _compute_histogram_similarity
- 帧匹配比例条件 (match_ratio < 0.7 → 跳过)
- 中位数 vs 均值（抵抗异常值）
- 向后兼容（无分片数据时不崩溃）
"""

from __future__ import annotations

import sys
from unittest.mock import MagicMock, patch


def _mock_module(**attrs):
    """Create a mock module with __spec__ to avoid AttributeError."""
    m = MagicMock()
    m.__spec__ = None
    for k, v in attrs.items():
        setattr(m, k, v)
    return m


# ── Module-level setup: mock deps, import dedup, then restore sys.modules ──
_SAVED_MODULES_KEYS = set(sys.modules.keys())
_SAVED_MODULES_VALUES = {
    k: sys.modules.get(k)
    for k in [
        "cv2",
        "celery",
        "sqlalchemy",
        "sqlalchemy.orm",
        "sqlalchemy.engine",
        "sqlalchemy.ext",
        "sqlalchemy.ext.declarative",
        "worker_app.db",
        "worker_app.celery_app",
        "worker_app.core.config",
        "packages.adapters.sqlalchemy_impl.session",
        "packages.adapters.sqlalchemy_impl.generated_video_repository",
        "packages.adapters.sqlalchemy_impl.models",
        "packages.shared.config",
        "packages.shared.storage",
    ]
}

sys.modules["cv2"] = _mock_module()

_mock_celery = MagicMock()
_mock_celery.Task = MagicMock
_mock_celery.Celery = MagicMock
_mock_celery.__spec__ = None
sys.modules["celery"] = _mock_celery

_mock_sqla = MagicMock()
_mock_sqla.__path__ = []
_mock_sqla.__spec__ = None
sys.modules["sqlalchemy"] = _mock_sqla

_mock_sqla_orm = MagicMock()
_mock_sqla_orm.__path__ = []
_mock_sqla_orm.__spec__ = None
_mock_sqla_orm.Session = MagicMock
sys.modules["sqlalchemy.orm"] = _mock_sqla_orm
sys.modules["sqlalchemy.engine"] = _mock_module()
sys.modules["sqlalchemy.ext"] = _mock_module()
sys.modules["sqlalchemy.ext.declarative"] = _mock_module()

sys.modules["worker_app.db"] = _mock_module(SessionLocal=MagicMock())
sys.modules["worker_app.celery_app"] = _mock_module(celery_app=MagicMock())
sys.modules["worker_app.core.config"] = _mock_module(get_settings=MagicMock(return_value=MagicMock()))

sys.modules["packages.adapters.sqlalchemy_impl.session"] = _mock_module(
    Base=MagicMock(),
    build_engine=MagicMock(),
    build_session_factory=MagicMock(),
    ensure_database_exists=MagicMock(),
    initialize_database=MagicMock(),
)
sys.modules["packages.adapters.sqlalchemy_impl.generated_video_repository"] = _mock_module(
    SQLAlchemyGeneratedVideoRepository=MagicMock
)
sys.modules["packages.adapters.sqlalchemy_impl.models"] = _mock_module(
    VideoFingerprintChunkModel=MagicMock,
    GeneratedVideoModel=MagicMock,
)
sys.modules["packages.shared.config"] = _mock_module(get_shared_settings=MagicMock(return_value=MagicMock()))
sys.modules["packages.shared.storage"] = _mock_module()

# Save a reference to the dedup module for use in tests (after sys.modules restore)
import video_processing.dedup as _dedup_mod
from video_processing.dedup import (  # noqa: E402
    DUPLICATE_THRESHOLD,
    HISTOGRAM_WEIGHT,
    LONG_VIDEO_DURATION_THRESHOLD_SEC,
    MATCH_RATIO_THRESHOLD,
    MAX_GAP,
    MAX_KEYFRAMES,
    MIN_CONSECUTIVE_MATCHES,
    MIN_KEYFRAME_INTERVAL_SEC,
    MIN_KEYFRAMES,
    PHASH_THRESHOLD,
    PHASH_WEIGHT,
    SCENE_CHANGE_THRESHOLD,
    SEGMENT_MATCH_THRESHOLD,
    DuplicateSegment,
    FingerprintChunk,
    VideoDeduplicator,
    VideoFingerprint,
    detect_keyframe_timestamps,
    find_duplicate_segments,
    hamming_distance,
)

# ── Restore sys.modules immediately after import ──
for _key in list(sys.modules.keys()):
    if _key not in _SAVED_MODULES_KEYS:
        del sys.modules[_key]
for _key, _value in _SAVED_MODULES_VALUES.items():
    if _value is not None:
        sys.modules[_key] = _value
    elif _key in sys.modules:
        del sys.modules[_key]
del _SAVED_MODULES_KEYS, _SAVED_MODULES_VALUES, _key, _value


# ── Helper ──────────────────────────────────────────────────────


def _make_chunk(start_ms: int, end_ms: int, phash: str, hist: list[float] | None = None) -> FingerprintChunk:
    """创建测试用 FingerprintChunk."""
    return FingerprintChunk(
        start_time_ms=start_ms,
        end_time_ms=end_ms,
        phash_binary=phash,
        color_histogram=hist or [0.1] * 96,
        frame_count=1,
    )


# ── TestDuplicateSegment ────────────────────────────────────────


class TestDuplicateSegment:
    """DuplicateSegment 数据类测试."""

    def test_creation(self):
        """正常创建."""
        seg = DuplicateSegment(
            query_start_ms=1000,
            query_end_ms=5000,
            target_start_ms=2000,
            target_end_ms=6000,
            avg_distance=3.5,
        )
        assert seg.query_start_ms == 1000
        assert seg.avg_distance == 3.5

    def test_fields(self):
        """所有字段可访问."""
        seg = DuplicateSegment(0, 1000, 500, 1500, 2.0)
        assert seg.query_end_ms == 1000
        assert seg.target_start_ms == 500
        assert seg.target_end_ms == 1500


# ── TestDetectKeyframeTimestamps ────────────────────────────────


class TestDetectKeyframeTimestamps:
    """detect_keyframe_timestamps 关键帧检测测试.

    由于 cv2 在单元测试环境中是 mock，这里只测试边界条件。
    完整的视频处理测试在集成测试中进行。
    """

    def test_cannot_open_video_raises(self):
        """无法打开视频时抛出 RuntimeError."""
        cv2_mock = _dedup_mod.cv2
        mock_cap = MagicMock()
        mock_cap.isOpened.return_value = False
        cv2_mock.VideoCapture.return_value = mock_cap

        import pytest

        with pytest.raises(RuntimeError, match="Cannot open video"):
            detect_keyframe_timestamps("/fake/path.mp4")

    def test_zero_duration_returns_empty(self):
        """视频时长为 0 时返回空列表."""
        cv2_mock = _dedup_mod.cv2
        mock_cap = MagicMock()
        mock_cap.isOpened.return_value = True
        # cv2.CAP_PROP_FPS etc. are Mock objects; configure get() to return 0 for frame_count
        mock_cap.get.return_value = 0
        mock_cap.read.return_value = (False, None)
        cv2_mock.VideoCapture.return_value = mock_cap

        result = detect_keyframe_timestamps("/fake/zero.mp4")
        assert result == []

    def test_function_signature(self):
        """验证函数签名和默认参数."""
        import inspect

        sig = inspect.signature(detect_keyframe_timestamps)
        params = sig.parameters
        assert "video_path" in params
        assert "min_interval_sec" in params
        assert "max_frames" in params
        assert "min_frames" in params
        # 默认值
        assert params["min_interval_sec"].default == 1.0
        assert params["max_frames"].default == 30
        assert params["min_frames"].default == 5


# ── TestFindDuplicateSegments ───────────────────────────────────


class TestFindDuplicateSegments:
    """find_duplicate_segments 滑动窗口时序匹配测试."""

    def test_identical_chunks_full_match(self):
        """两组完全相同的 chunks → 整段匹配."""
        chunks_a = [_make_chunk(i * 1000, (i + 1) * 1000, "aaaaaaaaaaaaaaaa") for i in range(10)]
        chunks_b = [_make_chunk(i * 1000, (i + 1) * 1000, "aaaaaaaaaaaaaaaa") for i in range(10)]

        segments = find_duplicate_segments(chunks_a, chunks_b)
        assert len(segments) >= 1
        # 应该覆盖大部分范围
        total_query_range = segments[-1].query_end_ms - segments[0].query_start_ms
        assert total_query_range > 5000  # 至少覆盖 5 秒

    def test_completely_different_chunks(self):
        """两组完全不同的 chunks → 空列表."""
        # 距离都 > 阈值
        chunks_a = [_make_chunk(i * 1000, (i + 1) * 1000, "0000000000000000") for i in range(10)]
        chunks_b = [_make_chunk(i * 1000, (i + 1) * 1000, "ffffffffffffffff") for i in range(10)]

        segments = find_duplicate_segments(chunks_a, chunks_b)
        assert segments == []

    def test_partial_overlap(self):
        """部分重叠 → 只返回重叠段."""
        # 前 5 帧相同，后 5 帧不同
        same_hash = "aaaaaaaaaaaaaaaa"
        diff_hash_a = "0000000000000000"
        diff_hash_b = "ffffffffffffffff"

        chunks_a = [_make_chunk(i * 1000, (i + 1) * 1000, same_hash) for i in range(5)] + [
            _make_chunk(i * 1000, (i + 1) * 1000, diff_hash_a) for i in range(5, 10)
        ]
        chunks_b = [_make_chunk(i * 1000, (i + 1) * 1000, same_hash) for i in range(5)] + [
            _make_chunk(i * 1000, (i + 1) * 1000, diff_hash_b) for i in range(5, 10)
        ]

        segments = find_duplicate_segments(chunks_a, chunks_b)
        # 应该只有前 5 帧的匹配段
        if segments:
            assert segments[0].query_end_ms <= 5000

    def test_min_consecutive_not_met(self):
        """连续 4 帧匹配（< min_consecutive=5）→ 不报重复.

        注意：使用不同的 hash 对，确保后半部分帧距离 > 阈值。
        """
        same_hash = "aaaaaaaaaaaaaaaa"
        # 4 帧匹配，后面 6 帧用与匹配哈希距离 32 的不匹配哈希（> PHASH_THRESHOLD=16）
        nomatch_hash = "cccccccccccccccc"  # hamming(aaaa, cccc)=32
        chunks_a = [_make_chunk(i * 1000, (i + 1) * 1000, same_hash) for i in range(4)] + [
            _make_chunk(i * 1000, (i + 1) * 1000, nomatch_hash) for i in range(4, 10)
        ]
        chunks_b = [_make_chunk(i * 1000, (i + 1) * 1000, same_hash) for i in range(4)] + [
            _make_chunk(i * 1000, (i + 1) * 1000, nomatch_hash) for i in range(4, 10)
        ]
        # hamming(aaaa..., cccc...) = 32 > PHASH_THRESHOLD(16)，后半段不匹配；
        # 前 4 帧匹配 < min_consecutive=5，不形成片段
        segments = find_duplicate_segments(chunks_a, chunks_b)
        assert segments == []

    def test_max_gap_behavior(self):
        """5 帧匹配 + 1 帧间隙 + 3 帧匹配 → 验证 max_gap 行为.

        关键：间隙帧必须在 query 和 target 中使用不同 hash，使其真正不匹配。
        """
        match_hash = "aaaaaaaaaaaaaaaa"
        # 间隙/尾部哈希与 match_hash 及彼此之间汉明距离均 >64 (> PHASH_THRESHOLD=16)，
        # 确保在 ±(neighbor_window+1) 时序抖动对齐窗口内也不会误匹配
        gap_hash_a = "ffffffffffffffff"  # hamming(a,f)=128
        gap_hash_b = "9999999999999999"  # hamming(a,9)=128, hamming(f,9)=128
        tail_hash_a = "7777777777777777"  # hamming(a,7)=192
        tail_hash_b = "1111111111111111"  # hamming(a,1)=192, hamming(7,1)=128

        # 5 帧匹配, 1 帧间隙, 3 帧匹配, 5 帧不匹配
        hashes_a = [match_hash] * 5 + [gap_hash_a] + [match_hash] * 3 + [tail_hash_a] * 5
        hashes_b = [match_hash] * 5 + [gap_hash_b] + [match_hash] * 3 + [tail_hash_b] * 5

        chunks_a = [_make_chunk(i * 1000, (i + 1) * 1000, h) for i, h in enumerate(hashes_a)]
        chunks_b = [_make_chunk(i * 1000, (i + 1) * 1000, h) for i, h in enumerate(hashes_b)]

        # max_gap=2, 所以 1 帧间隙会被合并
        segments = find_duplicate_segments(chunks_a, chunks_b, max_gap=2)
        # 5 match + 1 gap + 3 match = run of 9（间隙被桥接）
        assert len(segments) == 1
        # run 覆盖 indices 0-8（5 match + 1 gap + 3 match），但 gap 帧不计入 match
        # query_start = chunks_a[0].start = 0
        # query_end = chunks_a[8].end = 9000
        assert segments[0].query_start_ms == 0
        assert segments[0].query_end_ms == 9000

    def test_max_gap_exceeded(self):
        """间隙超过 max_gap → 分成两段."""
        match_hash = "aaaaaaaaaaaaaaaa"
        gap_hash_a = "ffffffffffffffff"  # hamming(a,f)=128
        gap_hash_b = "9999999999999999"  # hamming(a,9)=128
        tail_hash_a = "7777777777777777"  # hamming(a,7)=192
        tail_hash_b = "1111111111111111"  # hamming(a,1)=192

        # 5 帧匹配, 3 帧间隙 (> max_gap=2), 5 帧匹配, 5 帧不匹配
        hashes_a = [match_hash] * 5 + [gap_hash_a] * 3 + [match_hash] * 5 + [tail_hash_a] * 5
        hashes_b = [match_hash] * 5 + [gap_hash_b] * 3 + [match_hash] * 5 + [tail_hash_b] * 5

        chunks_a = [_make_chunk(i * 1000, (i + 1) * 1000, h) for i, h in enumerate(hashes_a)]
        chunks_b = [_make_chunk(i * 1000, (i + 1) * 1000, h) for i, h in enumerate(hashes_b)]

        segments = find_duplicate_segments(chunks_a, chunks_b, max_gap=2)
        # 3 帧间隙 > max_gap=2 → 分成两段（每段 5 帧匹配）
        assert len(segments) == 2

    def test_empty_chunks(self):
        """空 chunks 返回空列表."""
        assert find_duplicate_segments([], [_make_chunk(0, 1000, "aa")]) == []
        assert find_duplicate_segments([_make_chunk(0, 1000, "aa")], []) == []
        assert find_duplicate_segments([], []) == []

    def test_dict_chunks_compatibility(self):
        """dict 格式的 chunks 也能正常工作."""
        chunks_a = [
            {"phash_binary": "aaaaaaaaaaaaaaaa", "start_time_ms": i * 1000, "end_time_ms": (i + 1) * 1000}
            for i in range(10)
        ]
        chunks_b = [
            {"phash_binary": "aaaaaaaaaaaaaaaa", "start_time_ms": i * 1000, "end_time_ms": (i + 1) * 1000}
            for i in range(10)
        ]

        segments = find_duplicate_segments(chunks_a, chunks_b)
        assert len(segments) >= 1

    def test_segment_time_ranges(self):
        """返回的 segment 时间范围正确.

        每个 query chunk 匹配到 target 中对应的 chunk（相同 hash），
        确保 target 时间范围正确映射。
        """

        # 给每个 chunk 唯一的 hash（但保证 query[i] == target[i]）
        def _unique_hash(i: int) -> str:
            return format(i, "016x")

        chunks_a = [_make_chunk(i * 2000, (i + 1) * 2000, _unique_hash(i)) for i in range(7)]
        chunks_b = [_make_chunk(i * 2000, (i + 1) * 2000, _unique_hash(i)) for i in range(7)]

        segments = find_duplicate_segments(chunks_a, chunks_b)
        assert len(segments) >= 1
        seg = segments[0]
        assert seg.query_start_ms == 0
        assert seg.query_end_ms == 14000
        # target 应该映射到正确的范围
        assert seg.target_start_ms == 0
        assert seg.target_end_ms == 14000
        assert seg.avg_distance == 0.0  # 完全相同


# ── TestMedianVsMean ────────────────────────────────────────────


class TestMedianVsMean:
    """中位数 vs 均值：验证中位数抵抗异常值."""

    def test_median_resists_outlier(self):
        """距离 [3,3,3,3,30]：均值=8.4，中位数=3.
        中位数 < PHASH_THRESHOLD(10)，均值也 < 10。
        但更极端的：[3,3,3,3,60]：均值=14.4，中位数=3.
        """
        import statistics

        distances = [3, 3, 3, 3, 60]
        assert statistics.median(distances) == 3
        assert sum(distances) / len(distances) == 14.4
        # 中位数 < 10 → 通过阈值
        assert statistics.median(distances) < 10


# ── TestMatchRatioCondition ─────────────────────────────────────


class TestMatchRatioCondition:
    """帧匹配比例条件测试."""

    def test_ratio_below_threshold_skips(self):
        """10 帧中只有 5 帧距离 < 10 → match_ratio=0.5 < 0.7 → 跳过."""
        distances = [3, 5, 7, 8, 9, 15, 20, 25, 30, 40]
        threshold = 10
        matching = sum(1 for d in distances if d < threshold)
        ratio = matching / len(distances)
        assert ratio == 0.5
        assert ratio < 0.7  # 应该被跳过

    def test_ratio_above_threshold_passes(self):
        """10 帧中 8 帧距离 < 10 → match_ratio=0.8 >= 0.7 → 通过."""
        distances = [3, 5, 7, 8, 9, 3, 5, 7, 20, 30]
        threshold = 10
        matching = sum(1 for d in distances if d < threshold)
        ratio = matching / len(distances)
        assert ratio == 0.8
        assert ratio >= 0.7  # 应该通过


# ── TestBhattacharyyaFusion ─────────────────────────────────────


class TestBhattacharyyaFusion:
    """直方图融合逻辑测试."""

    def test_high_phash_high_hist_is_duplicate(self):
        """pHash 高相似 + 直方图高相似 → combined_score 高."""
        phash_similarity = 0.95  # median_distance ≈ 3
        hist_similarity = 0.90
        combined = 0.7 * phash_similarity + 0.3 * hist_similarity
        assert combined > 0.70  # DUPLICATE_THRESHOLD

    def test_high_phash_low_hist_maybe_not(self):
        """pHash 高相似 + 直方图低相似 → combined_score 取决于权重."""
        phash_similarity = 0.85  # median_distance ≈ 10
        hist_similarity = 0.10
        combined = 0.7 * phash_similarity + 0.3 * hist_similarity
        # 0.7 * 0.85 + 0.3 * 0.10 = 0.595 + 0.03 = 0.625 < 0.70
        assert combined < 0.70

    def test_no_histogram_fallback(self):
        """无直方图数据时 hist_similarity 回退到 0.5."""
        phash_similarity = 0.90
        hist_similarity = 0.5  # fallback
        combined = 0.7 * phash_similarity + 0.3 * hist_similarity
        # 0.7 * 0.90 + 0.3 * 0.5 = 0.63 + 0.15 = 0.78 > 0.70
        assert combined > 0.70


# ── TestBackwardCompatibility ───────────────────────────────────


class TestBackwardCompatibility:
    """向后兼容测试."""

    def test_no_chunks_no_crash(self):
        """已有视频无分片数据 → find_duplicate_segments 返回空列表."""
        # 模拟：fingerprint 有 chunks，但 existing 只有 JSON phashes
        query_chunks = [_make_chunk(i * 1000, (i + 1) * 1000, "aaaaaaaaaaaaaaaa") for i in range(10)]
        # 没有 start_time_ms/end_time_ms 的简化 dict
        target_as_dicts = [{"phash_binary": "aaaaaaaaaaaaaaaa"} for _ in range(10)]

        # find_duplicate_segments 需要 start_time_ms/end_time_ms
        # 在没有的情况下应该不崩溃（用默认值）
        # 实际上我们的实现用 _get_start/_get_end 访问，缺 key 会 KeyError
        # 所以 check_duplicate 传入时会补上默认值
        target_with_defaults = [
            {"phash_binary": "aaaaaaaaaaaaaaaa", "start_time_ms": 0, "end_time_ms": 0} for _ in range(10)
        ]
        segments = find_duplicate_segments(query_chunks, target_with_defaults)
        # 不会崩溃
        assert isinstance(segments, list)

    def test_few_chunks_no_crash(self):
        """少量 chunk 不崩溃."""
        chunks_a = [_make_chunk(0, 5000, "aaaaaaaaaaaaaaaa")]
        chunks_b = [{"phash_binary": "aaaaaaaaaaaaaaaa", "start_time_ms": 0, "end_time_ms": 5000}]

        segments = find_duplicate_segments(chunks_a, chunks_b)
        # Issue #1702: 自适应门槛 min(5, max(2, 1//2))=2，1 帧不成段；
        # N=1 的检出由 _evaluate_candidate 匹配帧回退兜底（见 test_dedup_1702）。
        # 这里只要求不崩溃。
        assert isinstance(segments, list)


# ── TestConstants ───────────────────────────────────────────────


class TestConstants:
    """常量值验证 — 使用已在模块顶部导入的常量，避免重新 import."""

    def test_segment_match_threshold(self):
        # Issue #1702 二次校准：阈值经 staging 真实数据两轮回归——
        # 第一轮同源 4/11、异源 min=24 定 12；第二轮扩样本（15 个真实成片）
        # 同源降重对中位数距离 14、<=16 命中 8/11=0.73，异源 13 个候选
        # <=16 命中全 0、最近邻最小距离 18 → 校准为 16。
        assert SEGMENT_MATCH_THRESHOLD == PHASH_THRESHOLD == 16

    def test_min_consecutive_matches(self):
        assert MIN_CONSECUTIVE_MATCHES == 5

    def test_max_gap(self):
        assert MAX_GAP == 2

    def test_scene_change_threshold(self):
        assert SCENE_CHANGE_THRESHOLD == 30

    def test_min_keyframe_interval(self):
        assert MIN_KEYFRAME_INTERVAL_SEC == 1.0

    def test_max_keyframes(self):
        assert MAX_KEYFRAMES == 30

    def test_min_keyframes(self):
        assert MIN_KEYFRAMES == 5

    def test_long_video_threshold(self):
        assert LONG_VIDEO_DURATION_THRESHOLD_SEC == 180

    def test_duplicate_threshold(self):
        assert DUPLICATE_THRESHOLD == 0.70

    def test_phash_weight(self):
        assert PHASH_WEIGHT == 0.7

    def test_histogram_weight(self):
        assert HISTOGRAM_WEIGHT == 0.3

    def test_match_ratio_threshold(self):
        assert MATCH_RATIO_THRESHOLD == 0.7