From 49bdc6fd754ac77c112f42bb61919f7a784e0e68 Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Sat, 5 Sep 2026 10:08:03 +0800 Subject: [PATCH] =?UTF-8?q?fix(dedup):=20scope=3Duser=E8=B7=A8=E9=A1=B9?= =?UTF-8?q?=E7=9B=AE=E6=9F=A5=E9=87=8D=E5=8E=BB=E6=8E=89=C2=B115%=E6=97=B6?= =?UTF-8?q?=E9=95=BF=E9=A2=84=E8=BF=87=E6=BB=A4=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E5=90=8C=E6=BA=90=E4=B8=8D=E5=90=8C=E6=97=B6=E9=95=BF=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E4=BA=92=E7=9B=B8=E4=B8=8D=E5=8F=AF=E8=A7=81=20(Issue?= =?UTF-8?q?=20#1702=E8=BF=9E=E5=B8=A6=E4=BF=AE=E5=A4=8D)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check_duplicate 的 scope=user 分支旧逻辑按 duration_sec*0.85~1.15 缩小候选窗口。局部片段复用场景下两个视频时长必然不同（staging 证据 视频 20s vs 11s，差 42%），±15% 窗口导致 A/B 互相不在候选列表中： compute_duplicate_rate 能算出 27.27%/36.36%，但 is_duplicate 恒 False。 改动： - scope=user 改为 video_repo.list_by_user(user_id) 全量遍历同用户视频， 与 compute_duplicate_rate 口径一致；异源视频由 fusion/temporal_coverage 阈值天然过滤（校准数据：异源最小汉明距离 24，阈值 12） - TestDurationPrefilter 改为断言 scope=user/project 均不传时长过滤参数 - 全量查重相关单测 285 passed --- apps/worker/video_processing/dedup.py | 9 ++-- tests/unit/test_dedup_1702_zero_rate_fix.py | 18 +++++--- tests/unit/test_duplicate_rate_scope.py | 48 +++++++++++++++++---- 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/apps/worker/video_processing/dedup.py b/apps/worker/video_processing/dedup.py index 9b9afc8eb..2b96babb6 100755 --- a/apps/worker/video_processing/dedup.py +++ b/apps/worker/video_processing/dedup.py @@ -809,9 +809,12 @@ class VideoDeduplicator: """ video_repo = SQLAlchemyGeneratedVideoRepository(session) if scope == "user" and user_id: - dur_min = duration_sec * 0.85 if duration_sec > 0 else 0 - dur_max = duration_sec * 1.15 if duration_sec > 0 else 0 - existing_videos = video_repo.list_by_user(user_id, duration_min=dur_min, duration_max=dur_max) + # Issue #1702: 不做 ±15% 时长预过滤。旧逻辑按 duration_sec 缩小候选窗口， + # 但局部片段复用的两个视频时长必然不同（证据视频 20s vs 11s，差 42%）， + # ±15% 窗口让同源视频互相不可见 → is_duplicate 恒 False。 + # 全量遍历同用户视频（与 compute_duplicate_rate 口径一致），异源视频由 + # fusion/temporal_coverage 阈值天然过滤（校准：异源最小汉明距离 24）。 + existing_videos = video_repo.list_by_user(user_id) else: existing_videos = video_repo.list_by_project(project_id) diff --git a/tests/unit/test_dedup_1702_zero_rate_fix.py b/tests/unit/test_dedup_1702_zero_rate_fix.py index 6482fd894..5c914bb7c 100644 --- a/tests/unit/test_dedup_1702_zero_rate_fix.py +++ b/tests/unit/test_dedup_1702_zero_rate_fix.py @@ -269,10 +269,13 @@ class TestSingleChunkNoRegression: class TestDurationPrefilterUnit: - def test_duration_sec_not_divided_by_1000(self): - """fingerprint.duration 单位是秒，传给 check_duplicate 不应再 /1000。 + def test_user_scope_skips_duration_prefilter(self): + """Issue #1702: scope=user 跨项目查重不做 ±15% 时长预过滤。 - 旧 bug：duration/1000 → duration_max≈0.0135s，所有真实视频被过滤。 + 旧逻辑 duration/1000 单位 bug 先修成秒，但 ±15% 窗口与局部片段复用 + 根本矛盾——复用片段的两个视频时长必然不同（证据视频 20s vs 11s 差 42%）， + 窗口内找不到对方导致 is_duplicate 恒 False。最终口径：scope=user 全量 + 遍历同用户视频（与 compute_duplicate_rate 一致），不传 duration_min/max。 """ ddp = VideoDeduplicator() @@ -283,10 +286,11 @@ class TestDurationPrefilterUnit: repo = MockRepo.return_value repo.list_by_user.return_value = [] ddp.check_duplicate(fp, "proj1", session_magic, scope="user", user_id="u1", duration_sec=fp.duration) - _, kwargs = repo.list_by_user.call_args - # ±15% 窗口：13.5s -> [11.475, 15.525] - assert 11.0 < kwargs["duration_min"] < 12.0 - assert 15.0 < kwargs["duration_max"] < 16.0 + args, kwargs = repo.list_by_user.call_args + # 全量查询：不带任何时长过滤参数（局部复用必须跨时长比较） + assert "duration_min" not in kwargs + assert "duration_max" not in kwargs + assert args == ("u1",) or args == () # ── P1-7: 颜色直方图归一化 ──────────────────────────────────── diff --git a/tests/unit/test_duplicate_rate_scope.py b/tests/unit/test_duplicate_rate_scope.py index 4cf6f02ce..6fd9a0d08 100644 --- a/tests/unit/test_duplicate_rate_scope.py +++ b/tests/unit/test_duplicate_rate_scope.py @@ -132,9 +132,14 @@ class TestCheckDuplicateScopeUser: class TestDurationPrefilter: - """test_duration_prefilter：时长 ±15% 过滤.""" + """Issue #1702: scope=user 跨项目查重不做时长预过滤。 - def test_duration_prefilter_passes_correct_range(self): + 局部片段复用的两个视频时长必然不同（证据视频 20s vs 11s，差 42%）， + 旧的 ±15% 窗口会让同源视频互相不可见 → is_duplicate 恒 False。 + 全量遍历同用户视频，异源视频由 fusion/temporal_coverage 阈值天然过滤。 + """ + + def test_user_scope_no_duration_filter(self): from video_processing.dedup import VideoDeduplicator deduplicator = VideoDeduplicator() @@ -153,12 +158,13 @@ class TestDurationPrefilter: duration_sec=30.0, ) - # Should pass duration_min=25.5, duration_max=34.5 (30 ± 15%) + # scope=user 全量遍历：位置参数只传 user_id，kwargs 不含时长过滤 call_args = mock_repo.list_by_user.call_args - assert call_args[1]["duration_min"] == pytest.approx(25.5, abs=0.1) - assert call_args[1]["duration_max"] == pytest.approx(34.5, abs=0.1) + assert call_args[0] == ("user1",) + assert "duration_min" not in call_args[1] + assert "duration_max" not in call_args[1] - def test_no_duration_prefilter_when_zero(self): + def test_user_scope_no_duration_filter_when_zero(self): from video_processing.dedup import VideoDeduplicator deduplicator = VideoDeduplicator() @@ -178,8 +184,34 @@ class TestDurationPrefilter: ) call_args = mock_repo.list_by_user.call_args - assert call_args[1]["duration_min"] == 0 - assert call_args[1]["duration_max"] == 0 + assert "duration_min" not in call_args[1] + assert "duration_max" not in call_args[1] + + def test_project_scope_also_no_duration_filter(self): + """scope=project 走 list_by_project，本来就不做时长过滤。""" + from video_processing.dedup import VideoDeduplicator + + deduplicator = VideoDeduplicator() + fingerprint = _make_fingerprint(duration_ms=30000) + session = MagicMock() + + with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo: + mock_repo = MockRepo.return_value + mock_repo.list_by_project.return_value = [] + deduplicator.check_duplicate( + fingerprint, + "proj1", + session, + scope="project", + user_id="user1", + duration_sec=30.0, + ) + + mock_repo.list_by_project.assert_called_once() + call_args = mock_repo.list_by_project.call_args + assert call_args[0] == ("proj1",) + assert "duration_min" not in call_args[1] + assert "duration_max" not in call_args[1] class TestComputeDuplicateRateFormula: -- 2.54.0