style: auto-format with black + isort + prettier [skip ci-format-check]
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (pull_request) Successful in 4s
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 4s
CI/CD Pipeline / Validate - Style (pull_request) Successful in 1m38s
CI/CD Pipeline / PR Build API Image (pull_request) Successful in 7s
AI Code Review / AI Code Review (pull_request) Failing after 2m1s
CI/CD Pipeline / PR Build Worker Image (pull_request) Successful in 12s
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 2m35s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 1m12s
PR Automation / Auto Approve on CI Green (pull_request) Successful in 3m1s
CI/CD Pipeline / Validate - Python (mypy + alembic) (pull_request) Successful in 4m45s
CI/CD Pipeline / Unit Tests (pull_request) Failing after 7m1s
CI/CD Pipeline / Validate - Security (pull_request) Has been cancelled
CI/CD Pipeline / Build Production API Image (pull_request) Has been cancelled
CI/CD Pipeline / Build Production Web Image (pull_request) Has been cancelled
CI/CD Pipeline / Build Production Worker Image (pull_request) Has been cancelled
CI/CD Pipeline / Deploy Production (pull_request) Has been cancelled
CI/CD Pipeline / Production Browser E2E (pull_request) Has been cancelled
CI/CD Pipeline / Canary Release to Production (pull_request) Has been cancelled
CI/CD Pipeline / CI Gate (pull_request) Has been cancelled
PR Automation / Auto Merge on CI Green + Approved (pull_request) Has been cancelled
CI/CD Pipeline / ACR Image Cleanup (pull_request) Failing after 303h50m27s
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Failing after 303h50m37s
CI/CD Pipeline / Staging API Integration Tests (pull_request) Failing after 303h50m28s
CI/CD Pipeline / Staging E2E Tests (pull_request) Failing after 303h50m28s
CI/CD Pipeline / Retag skipped Staging Worker Image (pull_request) Failing after 303h50m38s
CI/CD Pipeline / Retag skipped Staging Web Image (pull_request) Failing after 303h50m38s
CI/CD Pipeline / PR Build Web Image (pull_request) Failing after 303h50m50s
CI/CD Pipeline / Frontend Unit Tests (pull_request) Failing after 303h50m57s
CI/CD Pipeline / Build Staging Worker Image (pull_request) Failing after 303h52m36s
CI/CD Pipeline / Build Staging Web Image (pull_request) Failing after 303h52m37s
CI/CD Pipeline / Build Staging API Image (pull_request) Failing after 303h52m38s
CI/CD Pipeline / Check push changed paths (pull_request) Failing after 303h52m41s
CI/CD Pipeline / Retag skipped Staging API Image (pull_request) Failing after 304h25m3s
CI/CD Pipeline / Frontend Lint (pull_request) Failing after 304h25m22s

This commit is contained in:
CI Bot
2026-09-03 15:32:04 +00:00
parent 28d3c1d33c
commit db11e2dc18
6 changed files with 91 additions and 23 deletions
@@ -4,9 +4,11 @@ Revision ID: 064_match_count_visual_sim
Revises: 063_fingerprint_chunks
Create Date: 2026-09-03
"""
from alembic import op
import sqlalchemy as sa
from alembic import op
revision = "064_match_count_visual_sim"
down_revision = "063_fingerprint_chunks"
branch_labels = None
+11 -4
View File
@@ -979,8 +979,11 @@ def check_duplicate_task(self: Task, generated_video_id: str) -> dict:
# 查重判定(跨项目全局 + 时长预过滤)
duplicate_result = deduplicator.check_duplicate(
fingerprint, video.project_id, session,
scope="user", user_id=video.user_id,
fingerprint,
video.project_id,
session,
scope="user",
user_id=video.user_id,
duration_sec=fingerprint.duration / 1000 if fingerprint.duration else 0,
)
@@ -994,8 +997,12 @@ def check_duplicate_task(self: Task, generated_video_id: str) -> dict:
# 查重率计算(跨项目全局)
rate_result = deduplicator.compute_duplicate_rate(
fingerprint, video.project_id, generated_video_id, session,
scope="user", user_id=video.user_id,
fingerprint,
video.project_id,
generated_video_id,
session,
scope="user",
user_id=video.user_id,
)
video.duplicate_rate = rate_result["duplicate_rate"]
video.match_count = rate_result["match_count"]
+10 -4
View File
@@ -111,8 +111,12 @@ def create_video_record_and_dedup(
# (a) 历史成片查重(跨项目全局 + 时长预过滤)
duration_sec = fingerprint.duration / 1000 if fingerprint.duration else 0
duplicate_result = deduplicator.check_duplicate(
fingerprint, project_id, session,
scope="user", user_id=user_id, duration_sec=duration_sec,
fingerprint,
project_id,
session,
scope="user",
user_id=user_id,
duration_sec=duration_sec,
)
# (b) 批次内查重(仅当有 batch_id 时)
@@ -148,8 +152,10 @@ def create_video_record_and_dedup(
generated_video.visual_similarity = rate_result["visual_similarity"]
logger.info(
"Duplicate rate for %s: %.2f%% (visual_sim=%.3f, matches=%d)",
video_id, rate_result["duplicate_rate"],
rate_result["visual_similarity"], rate_result["match_count"],
video_id,
rate_result["duplicate_rate"],
rate_result["visual_similarity"],
rate_result["match_count"],
)
except Exception as rate_err:
logger.warning("Failed to compute duplicate_rate for %s: %s", video_id, rate_err)
+15 -3
View File
@@ -43,7 +43,11 @@ class TestDedupHelpersUserIdPassthrough:
mock_deduplicator = MagicMock()
mock_deduplicator.compute_fingerprint.return_value = mock_fingerprint
mock_deduplicator.check_duplicate.return_value = None
mock_deduplicator.compute_duplicate_rate.return_value = {"duplicate_rate": 42.5, "visual_similarity": 0.7, "match_count": 2}
mock_deduplicator.compute_duplicate_rate.return_value = {
"duplicate_rate": 42.5,
"visual_similarity": 0.7,
"match_count": 2,
}
with (
patch(
@@ -85,7 +89,11 @@ class TestDedupHelpersUserIdPassthrough:
mock_deduplicator = MagicMock()
mock_deduplicator.compute_fingerprint.return_value = mock_fingerprint
mock_deduplicator.check_duplicate.return_value = None
mock_deduplicator.compute_duplicate_rate.return_value = {"duplicate_rate": 0.0, "visual_similarity": 0.0, "match_count": 0}
mock_deduplicator.compute_duplicate_rate.return_value = {
"duplicate_rate": 0.0,
"visual_similarity": 0.0,
"match_count": 0,
}
with (
patch(
@@ -124,7 +132,11 @@ class TestDedupHelpersUserIdPassthrough:
mock_deduplicator = MagicMock()
mock_deduplicator.compute_fingerprint.return_value = mock_fingerprint
mock_deduplicator.check_duplicate.return_value = None
mock_deduplicator.compute_duplicate_rate.return_value = {"duplicate_rate": 78.5, "visual_similarity": 0.85, "match_count": 3}
mock_deduplicator.compute_duplicate_rate.return_value = {
"duplicate_rate": 78.5,
"visual_similarity": 0.85,
"match_count": 3,
}
with (
patch(
+6 -1
View File
@@ -152,7 +152,12 @@ class TestComputeDuplicateRate:
mock_repo = MockRepo.return_value
mock_repo.list_by_user.return_value = [existing]
rate = deduplicator.compute_duplicate_rate(
fingerprint, "proj1", "vid1", session, scope="user", user_id="user1",
fingerprint,
"proj1",
"vid1",
session,
scope="user",
user_id="user1",
)
# Should use list_by_user and find the match
+46 -10
View File
@@ -16,6 +16,7 @@ sys.path.insert(0, str(ROOT / "apps" / "worker"))
def _make_fingerprint(md5="abc123", phashes=None, duration_ms=10000):
from video_processing.dedup import VideoFingerprint
return VideoFingerprint(
md5=md5,
keyframe_phashes=phashes or ["ff00ff00ff00ff00"],
@@ -27,11 +28,19 @@ def _make_fingerprint(md5="abc123", phashes=None, duration_ms=10000):
def _make_video(vid, fingerprint_dict, project_id="proj1", duration=10.0):
from packages.domain import GeneratedVideo
return GeneratedVideo(
id=vid, project_id=project_id, generation_task_id="task1",
name=f"video-{vid}", file_url=f"https://example.com/{vid}.mp4",
file_size=1000, duration=duration, width=1920, height=1080,
fps=25.0, video_fingerprint=fingerprint_dict,
id=vid,
project_id=project_id,
generation_task_id="task1",
name=f"video-{vid}",
file_url=f"https://example.com/{vid}.mp4",
file_size=1000,
duration=duration,
width=1920,
height=1080,
fps=25.0,
video_fingerprint=fingerprint_dict,
)
@@ -40,6 +49,7 @@ class TestCheckDuplicateScopeProject:
def test_default_scope_queries_by_project(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
fingerprint = _make_fingerprint(md5="unique_md5")
session = MagicMock()
@@ -54,6 +64,7 @@ class TestCheckDuplicateScopeProject:
def test_project_scope_finds_duplicate(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
fingerprint = _make_fingerprint(md5="same_md5")
session = MagicMock()
@@ -75,6 +86,7 @@ class TestCheckDuplicateScopeUser:
def test_user_scope_queries_by_user(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
fingerprint = _make_fingerprint(md5="unique_md5")
session = MagicMock()
@@ -83,7 +95,11 @@ class TestCheckDuplicateScopeUser:
mock_repo = MockRepo.return_value
mock_repo.list_by_user.return_value = []
result = deduplicator.check_duplicate(
fingerprint, "proj1", session, scope="user", user_id="user_123",
fingerprint,
"proj1",
session,
scope="user",
user_id="user_123",
)
mock_repo.list_by_user.assert_called_once()
@@ -91,6 +107,7 @@ class TestCheckDuplicateScopeUser:
def test_user_scope_finds_cross_project_duplicate(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
fingerprint = _make_fingerprint(md5="cross_proj_md5")
session = MagicMock()
@@ -102,7 +119,11 @@ class TestCheckDuplicateScopeUser:
mock_repo = MockRepo.return_value
mock_repo.list_by_user.return_value = [existing]
result = deduplicator.check_duplicate(
fingerprint, "proj1", session, scope="user", user_id="user_123",
fingerprint,
"proj1",
session,
scope="user",
user_id="user_123",
)
assert result is not None
@@ -115,6 +136,7 @@ class TestDurationPrefilter:
def test_duration_prefilter_passes_correct_range(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
fingerprint = _make_fingerprint(duration_ms=30000) # 30s video
session = MagicMock()
@@ -123,8 +145,12 @@ class TestDurationPrefilter:
mock_repo = MockRepo.return_value
mock_repo.list_by_user.return_value = []
deduplicator.check_duplicate(
fingerprint, "proj1", session,
scope="user", user_id="user1", duration_sec=30.0,
fingerprint,
"proj1",
session,
scope="user",
user_id="user1",
duration_sec=30.0,
)
# Should pass duration_min=25.5, duration_max=34.5 (30 ± 15%)
@@ -134,6 +160,7 @@ class TestDurationPrefilter:
def test_no_duration_prefilter_when_zero(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
fingerprint = _make_fingerprint()
session = MagicMock()
@@ -142,8 +169,12 @@ class TestDurationPrefilter:
mock_repo = MockRepo.return_value
mock_repo.list_by_user.return_value = []
deduplicator.check_duplicate(
fingerprint, "proj1", session,
scope="user", user_id="user1", duration_sec=0,
fingerprint,
"proj1",
session,
scope="user",
user_id="user1",
duration_sec=0,
)
call_args = mock_repo.list_by_user.call_args
@@ -156,6 +187,7 @@ class TestComputeDuplicateRateFormula:
def test_formula_with_matching_frames(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
# 10 frames, all identical to existing → frame_match_rate = 1.0
phashes = ["aa00aa00aa00aa00"] * 10
@@ -179,6 +211,7 @@ class TestComputeDuplicateRateFormula:
def test_no_match_returns_zero(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
# Completely different phashes
fingerprint = _make_fingerprint(md5="new", phashes=["ff00ff00ff00ff00"])
@@ -204,6 +237,7 @@ class TestComputeDuplicateRateReturnDict:
def test_return_structure(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
fingerprint = _make_fingerprint()
session = MagicMock()
@@ -227,6 +261,7 @@ class TestBackwardCompat:
def test_default_scope_is_project(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
fingerprint = _make_fingerprint()
session = MagicMock()
@@ -245,6 +280,7 @@ class TestBackwardCompat:
def test_check_duplicate_default_scope_backward_compat(self):
from video_processing.dedup import VideoDeduplicator
deduplicator = VideoDeduplicator()
fingerprint = _make_fingerprint()
session = MagicMock()