saas-backend-agent
c6af6dd072
- 新增 Celery task worker.process_duplication_check： OSS 下载上传视频 → 动态抽帧指纹 → 跨项目与用户所有成片比对 (compute_duplicate_rate + find_duplicate_segments) → 更新 DuplicationRecord - 状态流转 pending → processing → completed/failed，失败重试 3 次间隔 60s， 重试耗尽标记 failed，临时文件 finally 清理 - 片段映射：worker 分片 DuplicateSegment(ms/汉明距离) → 领域 DuplicateSegment(秒/0-100 相似度) - DuplicationRecord 实体新增 visual_similarity/match_count（可选 None，向后兼容）， mark_completed()/reset_for_retry() 同步 - migration 065: duplication_records 表加两列 - repository/model/schemas/routes 读写映射与响应透传 - 上传接口与重试接口 enqueue task（此前记录一直 pending 的根因） - 11 个新单测：成功/跳过/未找到/重试/失败标记/临时文件清理/片段映射/schema
380 lines
13 KiB
Python
380 lines
13 KiB
Python
"""#1661 手动查重 worker task 测试：成功/失败/重试/片段映射/schema 字段。"""
|
||
|
||
import sys
|
||
from pathlib import Path
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
# cv2/numpy 在测试环境不可用，提前 mock
|
||
sys.modules.setdefault("cv2", MagicMock())
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
sys.path.insert(0, str(ROOT / "apps" / "api"))
|
||
sys.path.insert(0, str(ROOT / "packages"))
|
||
sys.path.insert(0, str(ROOT / "apps" / "worker"))
|
||
|
||
|
||
def _get_task(mod):
|
||
"""返回 (run_callable, real_task)。
|
||
|
||
- celery task 环境：run 是 bound method（self 已绑定），retry 用 patch.object 打桩
|
||
- 原始函数环境：用一个 mock_self 作为 self
|
||
"""
|
||
task_obj = mod.process_duplication_check
|
||
real = task_obj._get_current_object() if hasattr(task_obj, "_get_current_object") else task_obj
|
||
if hasattr(real, "run") and hasattr(real, "retry"):
|
||
return real.run, real, True # bound
|
||
return real, None, False
|
||
|
||
|
||
def _run(mod, record_id, retries=0):
|
||
"""执行 task，返回 (result_or_None, raised_exc, mock_self_or_None)。"""
|
||
from celery.exceptions import Retry as CeleryRetry
|
||
|
||
func, real_task, bound = _get_task(mod)
|
||
raised = None
|
||
result = None
|
||
if bound:
|
||
mock_retry = MagicMock(side_effect=CeleryRetry("retry"))
|
||
with patch.object(real_task, "retry", mock_retry):
|
||
real_task.request.retries = retries
|
||
real_task.max_retries = 3
|
||
try:
|
||
result = func(record_id)
|
||
except CeleryRetry as e:
|
||
raised = e
|
||
return result, raised, None
|
||
mock_self = MagicMock()
|
||
mock_self.request.retries = retries
|
||
mock_self.max_retries = 3
|
||
mock_self.retry = MagicMock(side_effect=CeleryRetry("retry"))
|
||
try:
|
||
result = func(mock_self, record_id)
|
||
except CeleryRetry as e:
|
||
raised = e
|
||
return result, raised, mock_self
|
||
|
||
|
||
def _make_record(status="pending"):
|
||
from packages.domain.duplication import DuplicationRecord
|
||
|
||
record = DuplicationRecord.create(
|
||
user_id="user-1",
|
||
filename="query.mp4",
|
||
file_size=1024,
|
||
storage_key="duplication/abc/query.mp4",
|
||
)
|
||
if status != "pending":
|
||
record.status = status
|
||
return record
|
||
|
||
|
||
def _make_fingerprint():
|
||
from video_processing.dedup import FingerprintChunk, VideoFingerprint
|
||
|
||
chunks = [
|
||
FingerprintChunk(start_time_ms=0, end_time_ms=2000, phash_binary="0" * 16, color_histogram=[], frame_count=1),
|
||
FingerprintChunk(
|
||
start_time_ms=2000, end_time_ms=4000, phash_binary="1" * 16, color_histogram=[], frame_count=1
|
||
),
|
||
]
|
||
return VideoFingerprint(
|
||
md5="qmd5",
|
||
keyframe_phashes=[c.phash_binary for c in chunks],
|
||
color_histograms=[],
|
||
duration=10000.0,
|
||
resolution=(720, 1280),
|
||
chunks=chunks,
|
||
)
|
||
|
||
|
||
def _patch_common(record, storage=None, dedup=None, session=None):
|
||
from worker_app.tasks import duplication_check as mod
|
||
|
||
fake_repo = MagicMock()
|
||
fake_repo.get.return_value = record
|
||
return [
|
||
patch.object(mod, "SessionLocal", return_value=session or MagicMock()),
|
||
patch.object(mod, "SQLAlchemyDuplicationRecordRepository", return_value=fake_repo),
|
||
patch.object(mod, "get_storage_service", return_value=storage or MagicMock()),
|
||
patch.object(mod, "VideoDeduplicator", return_value=dedup or MagicMock()),
|
||
], fake_repo
|
||
|
||
|
||
class TestProcessDuplicationCheckSuccess:
|
||
def test_success_flow_updates_record(self):
|
||
from worker_app.tasks import duplication_check as mod
|
||
|
||
record = _make_record()
|
||
fake_session = MagicMock()
|
||
fake_storage = MagicMock()
|
||
fake_dedup = MagicMock()
|
||
fake_dedup.compute_fingerprint.return_value = _make_fingerprint()
|
||
fake_dedup.compute_duplicate_rate.return_value = {
|
||
"duplicate_rate": 42.5,
|
||
"visual_similarity": 0.83,
|
||
"match_count": 1,
|
||
}
|
||
patches, fake_repo = _patch_common(record, storage=fake_storage, dedup=fake_dedup, session=fake_session)
|
||
patches.append(patch.object(mod, "_build_domain_segments", return_value=(["SEG"], 1)))
|
||
for p in patches:
|
||
p.start()
|
||
try:
|
||
result, raised, _ = _run(mod, record.id)
|
||
finally:
|
||
for p in patches:
|
||
p.stop()
|
||
|
||
assert raised is None
|
||
assert result["ok"] is True
|
||
assert result["status"] == "completed"
|
||
assert result["duplicate_rate"] == 42.5
|
||
assert result["visual_similarity"] == 0.83
|
||
assert result["match_count"] == 1
|
||
assert result["segments"] == 1
|
||
|
||
assert record.status == "completed"
|
||
assert record.duplicate_rate == 42.5
|
||
assert record.visual_similarity == 0.83
|
||
assert record.match_count == 1
|
||
assert record.duplicate_count == 1
|
||
assert record.segments == ["SEG"]
|
||
|
||
fake_storage.download_file.assert_called_once()
|
||
fake_dedup.compute_fingerprint.assert_called_once()
|
||
_, kwargs = fake_dedup.compute_duplicate_rate.call_args
|
||
assert kwargs["scope"] == "user"
|
||
assert kwargs["user_id"] == "user-1"
|
||
assert kwargs["current_video_id"] is None
|
||
assert fake_repo.update.call_count >= 2
|
||
fake_session.commit.assert_called()
|
||
fake_session.close.assert_called()
|
||
|
||
def test_already_completed_is_skipped(self):
|
||
from worker_app.tasks import duplication_check as mod
|
||
|
||
record = _make_record(status="completed")
|
||
patches, fake_repo = _patch_common(record)
|
||
for p in patches:
|
||
p.start()
|
||
try:
|
||
result, raised, _ = _run(mod, record.id)
|
||
finally:
|
||
for p in patches:
|
||
p.stop()
|
||
assert raised is None
|
||
assert result.get("skipped") is True
|
||
fake_repo.update.assert_not_called()
|
||
|
||
|
||
class TestProcessDuplicationCheckFailure:
|
||
def test_record_not_found_raises(self):
|
||
from worker_app.tasks import duplication_check as mod
|
||
|
||
fake_repo = MagicMock()
|
||
fake_repo.get.return_value = None
|
||
patches = [
|
||
patch.object(mod, "SessionLocal", return_value=MagicMock()),
|
||
patch.object(mod, "SQLAlchemyDuplicationRecordRepository", return_value=fake_repo),
|
||
patch.object(mod, "get_storage_service", return_value=MagicMock()),
|
||
]
|
||
for p in patches:
|
||
p.start()
|
||
try:
|
||
_result, raised, _ = _run(mod, "nope", retries=0)
|
||
finally:
|
||
for p in patches:
|
||
p.stop()
|
||
# 找不到记录触发异常 → retry（第一次）
|
||
assert raised is not None
|
||
|
||
def test_download_failure_retries_then_marks_failed(self):
|
||
from worker_app.tasks import duplication_check as mod
|
||
|
||
# 第一次失败（retries=0）：保持 pending
|
||
record = _make_record()
|
||
fake_storage = MagicMock()
|
||
fake_storage.download_file.side_effect = RuntimeError("oss network down")
|
||
patches, _ = _patch_common(record, storage=fake_storage)
|
||
for p in patches:
|
||
p.start()
|
||
try:
|
||
_, raised, _ = _run(mod, record.id, retries=0)
|
||
finally:
|
||
for p in patches:
|
||
p.stop()
|
||
assert raised is not None
|
||
assert record.status == "processing", "首次失败不应标记 failed（已进入 processing 等待重试）"
|
||
|
||
# 最后一次（retries==max_retries=3）：标记 failed
|
||
record2 = _make_record()
|
||
patches2, fake_repo2 = _patch_common(record2, storage=fake_storage)
|
||
for p in patches2:
|
||
p.start()
|
||
try:
|
||
_run(mod, record2.id, retries=3)
|
||
finally:
|
||
for p in patches2:
|
||
p.stop()
|
||
assert record2.status == "failed"
|
||
assert "查重失败" in record2.error_message
|
||
fake_repo2.update.assert_called()
|
||
|
||
def test_temp_dir_cleaned_after_failure(self):
|
||
import os
|
||
|
||
import tempfile
|
||
|
||
from worker_app.tasks import duplication_check as mod
|
||
|
||
record = _make_record()
|
||
fake_storage = MagicMock()
|
||
fake_storage.download_file.side_effect = RuntimeError("boom")
|
||
|
||
created_dirs = []
|
||
real_mkdtemp = tempfile.mkdtemp
|
||
|
||
def fake_mkdtemp(prefix=None):
|
||
d = real_mkdtemp(prefix=prefix)
|
||
created_dirs.append(d)
|
||
return d
|
||
|
||
patches, _ = _patch_common(record, storage=fake_storage)
|
||
patches.append(patch.object(mod.tempfile, "mkdtemp", fake_mkdtemp))
|
||
for p in patches:
|
||
p.start()
|
||
try:
|
||
_run(mod, record.id, retries=0)
|
||
finally:
|
||
for p in patches:
|
||
p.stop()
|
||
|
||
assert created_dirs, "mkdtemp should have been called"
|
||
assert not os.path.isdir(created_dirs[0]), "temp dir should be removed in finally"
|
||
|
||
|
||
class TestBuildDomainSegments:
|
||
def test_maps_worker_segments_to_domain_with_seconds_and_percent(self):
|
||
from video_processing.dedup import DuplicateSegment as WorkerSegment
|
||
from worker_app.tasks import duplication_check as mod
|
||
|
||
fingerprint = _make_fingerprint()
|
||
|
||
from packages.domain import GeneratedVideo
|
||
|
||
existing = GeneratedVideo(
|
||
id="vid-1",
|
||
project_id="proj-1",
|
||
generation_task_id="t1",
|
||
name="成片A",
|
||
file_url="oss://x",
|
||
file_size=1,
|
||
duration=10.0,
|
||
width=720,
|
||
height=1280,
|
||
fps=30.0,
|
||
video_fingerprint={"md5": "x"},
|
||
)
|
||
fake_video_repo = MagicMock()
|
||
fake_video_repo.list_by_user.return_value = [existing]
|
||
|
||
fake_dedup = MagicMock()
|
||
fake_dedup._get_existing_chunks.return_value = [
|
||
{"phash_binary": "0" * 16, "start_time_ms": 0, "end_time_ms": 2000, "color_histogram": []},
|
||
]
|
||
worker_seg = WorkerSegment(
|
||
query_start_ms=1000,
|
||
query_end_ms=3000,
|
||
target_start_ms=5000,
|
||
target_end_ms=7000,
|
||
avg_distance=6.0,
|
||
)
|
||
|
||
with (
|
||
patch.object(mod, "SQLAlchemyGeneratedVideoRepository", return_value=fake_video_repo),
|
||
patch.object(mod, "find_duplicate_segments", return_value=[worker_seg]),
|
||
):
|
||
segments, dup_count = mod._build_domain_segments(fingerprint, MagicMock(), fake_dedup, "user-1")
|
||
|
||
assert dup_count == 1
|
||
assert len(segments) == 1
|
||
seg = segments[0]
|
||
assert seg.source_start == 1.0
|
||
assert seg.source_end == 3.0
|
||
assert seg.matched_start == 5.0
|
||
assert seg.matched_end == 7.0
|
||
assert seg.matched_video_id == "vid-1"
|
||
assert seg.matched_video_name == "成片A"
|
||
assert abs(seg.similarity - 90.6) < 0.2
|
||
|
||
def test_skips_videos_without_chunks(self):
|
||
from worker_app.tasks import duplication_check as mod
|
||
|
||
fingerprint = _make_fingerprint()
|
||
from packages.domain import GeneratedVideo
|
||
|
||
existing = GeneratedVideo(
|
||
id="vid-2",
|
||
project_id="p",
|
||
generation_task_id="t",
|
||
name="老视频",
|
||
file_url="oss://x",
|
||
file_size=1,
|
||
duration=5.0,
|
||
width=720,
|
||
height=1280,
|
||
fps=30.0,
|
||
video_fingerprint={"md5": "old"},
|
||
)
|
||
fake_video_repo = MagicMock()
|
||
fake_video_repo.list_by_user.return_value = [existing]
|
||
fake_dedup = MagicMock()
|
||
fake_dedup._get_existing_chunks.return_value = []
|
||
|
||
with patch.object(mod, "SQLAlchemyGeneratedVideoRepository", return_value=fake_video_repo):
|
||
segments, dup_count = mod._build_domain_segments(fingerprint, MagicMock(), fake_dedup, "u")
|
||
assert segments == []
|
||
assert dup_count == 0
|
||
|
||
|
||
class TestDuplicationSchemaAndDomainNewFields:
|
||
def test_record_response_includes_new_fields(self):
|
||
from app.schemas.duplication import DuplicationRecordResponse
|
||
|
||
resp = DuplicationRecordResponse(
|
||
id="r1",
|
||
filename="f.mp4",
|
||
file_size=1,
|
||
status="completed",
|
||
duplicate_rate=10.0,
|
||
duplicate_count=1,
|
||
visual_similarity=0.5,
|
||
match_count=2,
|
||
created_at="2026-09-04T00:00:00",
|
||
updated_at="2026-09-04T00:00:00",
|
||
)
|
||
assert resp.visual_similarity == 0.5
|
||
assert resp.match_count == 2
|
||
|
||
def test_record_response_new_fields_default_none(self):
|
||
from app.schemas.duplication import DuplicationRecordResponse
|
||
|
||
resp = DuplicationRecordResponse(id="r1", filename="f.mp4", file_size=1, created_at="x", updated_at="y")
|
||
assert resp.visual_similarity is None
|
||
assert resp.match_count is None
|
||
|
||
def test_domain_mark_completed_accepts_new_fields(self):
|
||
record = _make_record()
|
||
record.mark_completed(33.0, 2, [], visual_similarity=0.77, match_count=3)
|
||
assert record.status == "completed"
|
||
assert record.visual_similarity == 0.77
|
||
assert record.match_count == 3
|
||
|
||
def test_reset_for_retry_clears_new_fields(self):
|
||
record = _make_record()
|
||
record.mark_completed(10.0, 1, [], visual_similarity=0.5, match_count=1)
|
||
record.status = "failed"
|
||
record.reset_for_retry()
|
||
assert record.status == "pending"
|
||
assert record.visual_similarity is None
|
||
assert record.match_count is None
|