fix(dedup): scope=user跨项目查重不做时长预过滤,同源不同时长视频可互相检出 (#1702) #1708

Merged
auto-approve-bot merged 1 commits from fix/dedup-skip-duration-prefilter-1702 into develop 2026-09-05 10:12:50 +08:00
3 changed files with 57 additions and 18 deletions
+6 -3
View File
@@ -809,9 +809,12 @@ class VideoDeduplicator:
"""
video_repo = SQLAlchemyGeneratedVideoRepository(session)
if scope == "user" and user_id:
dur_min = duration_sec * 0.85 if duration_sec > 0 else 0
dur_max = duration_sec * 1.15 if duration_sec > 0 else 0
existing_videos = video_repo.list_by_user(user_id, duration_min=dur_min, duration_max=dur_max)
# Issue #1702: 不做 ±15% 时长预过滤。旧逻辑按 duration_sec 缩小候选窗口,
# 但局部片段复用的两个视频时长必然不同(证据视频 20s vs 11s,差 42%),
# ±15% 窗口让同源视频互相不可见 → is_duplicate 恒 False。
# 全量遍历同用户视频(与 compute_duplicate_rate 口径一致),异源视频由
# fusion/temporal_coverage 阈值天然过滤(校准:异源最小汉明距离 24)。
existing_videos = video_repo.list_by_user(user_id)
else:
existing_videos = video_repo.list_by_project(project_id)
+11 -7
View File
@@ -269,10 +269,13 @@ class TestSingleChunkNoRegression:
class TestDurationPrefilterUnit:
def test_duration_sec_not_divided_by_1000(self):
"""fingerprint.duration 单位是秒,传给 check_duplicate 不应再 /1000
def test_user_scope_skips_duration_prefilter(self):
"""Issue #1702: scope=user 跨项目查重不做 ±15% 时长预过滤
bugduration/1000 → duration_max≈0.0135s,所有真实视频被过滤。
逻辑 duration/1000 单位 bug 先修成秒,但 ±15% 窗口与局部片段复用
根本矛盾——复用片段的两个视频时长必然不同(证据视频 20s vs 11s 差 42%),
窗口内找不到对方导致 is_duplicate 恒 False。最终口径:scope=user 全量
遍历同用户视频(与 compute_duplicate_rate 一致),不传 duration_min/max。
"""
ddp = VideoDeduplicator()
@@ -283,10 +286,11 @@ class TestDurationPrefilterUnit:
repo = MockRepo.return_value
repo.list_by_user.return_value = []
ddp.check_duplicate(fp, "proj1", session_magic, scope="user", user_id="u1", duration_sec=fp.duration)
_, kwargs = repo.list_by_user.call_args
# ±15% 窗口:13.5s -> [11.475, 15.525]
assert 11.0 < kwargs["duration_min"] < 12.0
assert 15.0 < kwargs["duration_max"] < 16.0
args, kwargs = repo.list_by_user.call_args
# 全量查询:不带任何时长过滤参数(局部复用必须跨时长比较)
assert "duration_min" not in kwargs
assert "duration_max" not in kwargs
assert args == ("u1",) or args == ()
# ── P1-7: 颜色直方图归一化 ────────────────────────────────────
+40 -8
View File
@@ -132,9 +132,14 @@ class TestCheckDuplicateScopeUser:
class TestDurationPrefilter:
"""test_duration_prefilter:时长 ±15% 过滤."""
"""Issue #1702: scope=user 跨项目查重不做时长预过滤
def test_duration_prefilter_passes_correct_range(self):
局部片段复用的两个视频时长必然不同(证据视频 20s vs 11s,差 42%),
旧的 ±15% 窗口会让同源视频互相不可见 → is_duplicate 恒 False。
全量遍历同用户视频,异源视频由 fusion/temporal_coverage 阈值天然过滤。
"""
def test_user_scope_no_duration_filter(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
@@ -153,12 +158,13 @@ class TestDurationPrefilter:
duration_sec=30.0,
)
# Should pass duration_min=25.5, duration_max=34.5 (30 ± 15%)
# scope=user 全量遍历:位置参数只传 user_id,kwargs 不含时长过滤
call_args = mock_repo.list_by_user.call_args
assert call_args[1]["duration_min"] == pytest.approx(25.5, abs=0.1)
assert call_args[1]["duration_max"] == pytest.approx(34.5, abs=0.1)
assert call_args[0] == ("user1",)
assert "duration_min" not in call_args[1]
assert "duration_max" not in call_args[1]
def test_no_duration_prefilter_when_zero(self):
def test_user_scope_no_duration_filter_when_zero(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
@@ -178,8 +184,34 @@ class TestDurationPrefilter:
)
call_args = mock_repo.list_by_user.call_args
assert call_args[1]["duration_min"] == 0
assert call_args[1]["duration_max"] == 0
assert "duration_min" not in call_args[1]
assert "duration_max" not in call_args[1]
def test_project_scope_also_no_duration_filter(self):
"""scope=project 走 list_by_project,本来就不做时长过滤。"""
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
fingerprint = _make_fingerprint(duration_ms=30000)
session = MagicMock()
with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo:
mock_repo = MockRepo.return_value
mock_repo.list_by_project.return_value = []
deduplicator.check_duplicate(
fingerprint,
"proj1",
session,
scope="project",
user_id="user1",
duration_sec=30.0,
)
mock_repo.list_by_project.assert_called_once()
call_args = mock_repo.list_by_project.call_args
assert call_args[0] == ("proj1",)
assert "duration_min" not in call_args[1]
assert "duration_max" not in call_args[1]
class TestComputeDuplicateRateFormula: