fix(dedup): scope=user跨项目查重不做时长预过滤,同源不同时长视频可互相检出 (#1702) #1708
@@ -809,9 +809,12 @@ class VideoDeduplicator:
|
||||
"""
|
||||
video_repo = SQLAlchemyGeneratedVideoRepository(session)
|
||||
if scope == "user" and user_id:
|
||||
dur_min = duration_sec * 0.85 if duration_sec > 0 else 0
|
||||
dur_max = duration_sec * 1.15 if duration_sec > 0 else 0
|
||||
existing_videos = video_repo.list_by_user(user_id, duration_min=dur_min, duration_max=dur_max)
|
||||
# Issue #1702: 不做 ±15% 时长预过滤。旧逻辑按 duration_sec 缩小候选窗口,
|
||||
# 但局部片段复用的两个视频时长必然不同(证据视频 20s vs 11s,差 42%),
|
||||
# ±15% 窗口让同源视频互相不可见 → is_duplicate 恒 False。
|
||||
# 全量遍历同用户视频(与 compute_duplicate_rate 口径一致),异源视频由
|
||||
# fusion/temporal_coverage 阈值天然过滤(校准:异源最小汉明距离 24)。
|
||||
existing_videos = video_repo.list_by_user(user_id)
|
||||
else:
|
||||
existing_videos = video_repo.list_by_project(project_id)
|
||||
|
||||
|
||||
@@ -269,10 +269,13 @@ class TestSingleChunkNoRegression:
|
||||
|
||||
|
||||
class TestDurationPrefilterUnit:
|
||||
def test_duration_sec_not_divided_by_1000(self):
|
||||
"""fingerprint.duration 单位是秒,传给 check_duplicate 不应再 /1000。
|
||||
def test_user_scope_skips_duration_prefilter(self):
|
||||
"""Issue #1702: scope=user 跨项目查重不做 ±15% 时长预过滤。
|
||||
|
||||
旧 bug:duration/1000 → duration_max≈0.0135s,所有真实视频被过滤。
|
||||
旧逻辑 duration/1000 单位 bug 先修成秒,但 ±15% 窗口与局部片段复用
|
||||
根本矛盾——复用片段的两个视频时长必然不同(证据视频 20s vs 11s 差 42%),
|
||||
窗口内找不到对方导致 is_duplicate 恒 False。最终口径:scope=user 全量
|
||||
遍历同用户视频(与 compute_duplicate_rate 一致),不传 duration_min/max。
|
||||
"""
|
||||
|
||||
ddp = VideoDeduplicator()
|
||||
@@ -283,10 +286,11 @@ class TestDurationPrefilterUnit:
|
||||
repo = MockRepo.return_value
|
||||
repo.list_by_user.return_value = []
|
||||
ddp.check_duplicate(fp, "proj1", session_magic, scope="user", user_id="u1", duration_sec=fp.duration)
|
||||
_, kwargs = repo.list_by_user.call_args
|
||||
# ±15% 窗口:13.5s -> [11.475, 15.525]
|
||||
assert 11.0 < kwargs["duration_min"] < 12.0
|
||||
assert 15.0 < kwargs["duration_max"] < 16.0
|
||||
args, kwargs = repo.list_by_user.call_args
|
||||
# 全量查询:不带任何时长过滤参数(局部复用必须跨时长比较)
|
||||
assert "duration_min" not in kwargs
|
||||
assert "duration_max" not in kwargs
|
||||
assert args == ("u1",) or args == ()
|
||||
|
||||
|
||||
# ── P1-7: 颜色直方图归一化 ────────────────────────────────────
|
||||
|
||||
@@ -132,9 +132,14 @@ class TestCheckDuplicateScopeUser:
|
||||
|
||||
|
||||
class TestDurationPrefilter:
|
||||
"""test_duration_prefilter:时长 ±15% 过滤."""
|
||||
"""Issue #1702: scope=user 跨项目查重不做时长预过滤。
|
||||
|
||||
def test_duration_prefilter_passes_correct_range(self):
|
||||
局部片段复用的两个视频时长必然不同(证据视频 20s vs 11s,差 42%),
|
||||
旧的 ±15% 窗口会让同源视频互相不可见 → is_duplicate 恒 False。
|
||||
全量遍历同用户视频,异源视频由 fusion/temporal_coverage 阈值天然过滤。
|
||||
"""
|
||||
|
||||
def test_user_scope_no_duration_filter(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
@@ -153,12 +158,13 @@ class TestDurationPrefilter:
|
||||
duration_sec=30.0,
|
||||
)
|
||||
|
||||
# Should pass duration_min=25.5, duration_max=34.5 (30 ± 15%)
|
||||
# scope=user 全量遍历:位置参数只传 user_id,kwargs 不含时长过滤
|
||||
call_args = mock_repo.list_by_user.call_args
|
||||
assert call_args[1]["duration_min"] == pytest.approx(25.5, abs=0.1)
|
||||
assert call_args[1]["duration_max"] == pytest.approx(34.5, abs=0.1)
|
||||
assert call_args[0] == ("user1",)
|
||||
assert "duration_min" not in call_args[1]
|
||||
assert "duration_max" not in call_args[1]
|
||||
|
||||
def test_no_duration_prefilter_when_zero(self):
|
||||
def test_user_scope_no_duration_filter_when_zero(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
@@ -178,8 +184,34 @@ class TestDurationPrefilter:
|
||||
)
|
||||
|
||||
call_args = mock_repo.list_by_user.call_args
|
||||
assert call_args[1]["duration_min"] == 0
|
||||
assert call_args[1]["duration_max"] == 0
|
||||
assert "duration_min" not in call_args[1]
|
||||
assert "duration_max" not in call_args[1]
|
||||
|
||||
def test_project_scope_also_no_duration_filter(self):
|
||||
"""scope=project 走 list_by_project,本来就不做时长过滤。"""
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = _make_fingerprint(duration_ms=30000)
|
||||
session = MagicMock()
|
||||
|
||||
with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo:
|
||||
mock_repo = MockRepo.return_value
|
||||
mock_repo.list_by_project.return_value = []
|
||||
deduplicator.check_duplicate(
|
||||
fingerprint,
|
||||
"proj1",
|
||||
session,
|
||||
scope="project",
|
||||
user_id="user1",
|
||||
duration_sec=30.0,
|
||||
)
|
||||
|
||||
mock_repo.list_by_project.assert_called_once()
|
||||
call_args = mock_repo.list_by_project.call_args
|
||||
assert call_args[0] == ("proj1",)
|
||||
assert "duration_min" not in call_args[1]
|
||||
assert "duration_max" not in call_args[1]
|
||||
|
||||
|
||||
class TestComputeDuplicateRateFormula:
|
||||
|
||||
Reference in New Issue
Block a user