style: auto-format with black + isort + prettier [skip ci-format-check]
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (pull_request) Successful in 4s
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 4s
CI/CD Pipeline / Validate - Style (pull_request) Successful in 1m38s
CI/CD Pipeline / PR Build API Image (pull_request) Successful in 7s
AI Code Review / AI Code Review (pull_request) Failing after 2m1s
CI/CD Pipeline / PR Build Worker Image (pull_request) Successful in 12s
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 2m35s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 1m12s
PR Automation / Auto Approve on CI Green (pull_request) Successful in 3m1s
CI/CD Pipeline / Validate - Python (mypy + alembic) (pull_request) Successful in 4m45s
CI/CD Pipeline / Unit Tests (pull_request) Failing after 7m1s
CI/CD Pipeline / Validate - Security (pull_request) Has been cancelled
CI/CD Pipeline / Build Production API Image (pull_request) Has been cancelled
CI/CD Pipeline / Build Production Web Image (pull_request) Has been cancelled
CI/CD Pipeline / Build Production Worker Image (pull_request) Has been cancelled
CI/CD Pipeline / Deploy Production (pull_request) Has been cancelled
CI/CD Pipeline / Production Browser E2E (pull_request) Has been cancelled
CI/CD Pipeline / Canary Release to Production (pull_request) Has been cancelled
CI/CD Pipeline / CI Gate (pull_request) Has been cancelled
PR Automation / Auto Merge on CI Green + Approved (pull_request) Has been cancelled
CI/CD Pipeline / ACR Image Cleanup (pull_request) Failing after 303h50m27s
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Failing after 303h50m37s
CI/CD Pipeline / Staging API Integration Tests (pull_request) Failing after 303h50m28s
CI/CD Pipeline / Staging E2E Tests (pull_request) Failing after 303h50m28s
CI/CD Pipeline / Retag skipped Staging Worker Image (pull_request) Failing after 303h50m38s
CI/CD Pipeline / Retag skipped Staging Web Image (pull_request) Failing after 303h50m38s
CI/CD Pipeline / PR Build Web Image (pull_request) Failing after 303h50m50s
CI/CD Pipeline / Frontend Unit Tests (pull_request) Failing after 303h50m57s
CI/CD Pipeline / Build Staging Worker Image (pull_request) Failing after 303h52m36s
CI/CD Pipeline / Build Staging Web Image (pull_request) Failing after 303h52m37s
CI/CD Pipeline / Build Staging API Image (pull_request) Failing after 303h52m38s
CI/CD Pipeline / Check push changed paths (pull_request) Failing after 303h52m41s
CI/CD Pipeline / Retag skipped Staging API Image (pull_request) Failing after 304h25m3s
CI/CD Pipeline / Frontend Lint (pull_request) Failing after 304h25m22s
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (pull_request) Successful in 4s
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 4s
CI/CD Pipeline / Validate - Style (pull_request) Successful in 1m38s
CI/CD Pipeline / PR Build API Image (pull_request) Successful in 7s
AI Code Review / AI Code Review (pull_request) Failing after 2m1s
CI/CD Pipeline / PR Build Worker Image (pull_request) Successful in 12s
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 2m35s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 1m12s
PR Automation / Auto Approve on CI Green (pull_request) Successful in 3m1s
CI/CD Pipeline / Validate - Python (mypy + alembic) (pull_request) Successful in 4m45s
CI/CD Pipeline / Unit Tests (pull_request) Failing after 7m1s
CI/CD Pipeline / Validate - Security (pull_request) Has been cancelled
CI/CD Pipeline / Build Production API Image (pull_request) Has been cancelled
CI/CD Pipeline / Build Production Web Image (pull_request) Has been cancelled
CI/CD Pipeline / Build Production Worker Image (pull_request) Has been cancelled
CI/CD Pipeline / Deploy Production (pull_request) Has been cancelled
CI/CD Pipeline / Production Browser E2E (pull_request) Has been cancelled
CI/CD Pipeline / Canary Release to Production (pull_request) Has been cancelled
CI/CD Pipeline / CI Gate (pull_request) Has been cancelled
PR Automation / Auto Merge on CI Green + Approved (pull_request) Has been cancelled
CI/CD Pipeline / ACR Image Cleanup (pull_request) Failing after 303h50m27s
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Failing after 303h50m37s
CI/CD Pipeline / Staging API Integration Tests (pull_request) Failing after 303h50m28s
CI/CD Pipeline / Staging E2E Tests (pull_request) Failing after 303h50m28s
CI/CD Pipeline / Retag skipped Staging Worker Image (pull_request) Failing after 303h50m38s
CI/CD Pipeline / Retag skipped Staging Web Image (pull_request) Failing after 303h50m38s
CI/CD Pipeline / PR Build Web Image (pull_request) Failing after 303h50m50s
CI/CD Pipeline / Frontend Unit Tests (pull_request) Failing after 303h50m57s
CI/CD Pipeline / Build Staging Worker Image (pull_request) Failing after 303h52m36s
CI/CD Pipeline / Build Staging Web Image (pull_request) Failing after 303h52m37s
CI/CD Pipeline / Build Staging API Image (pull_request) Failing after 303h52m38s
CI/CD Pipeline / Check push changed paths (pull_request) Failing after 303h52m41s
CI/CD Pipeline / Retag skipped Staging API Image (pull_request) Failing after 304h25m3s
CI/CD Pipeline / Frontend Lint (pull_request) Failing after 304h25m22s
This commit is contained in:
@@ -4,9 +4,11 @@ Revision ID: 064_match_count_visual_sim
|
||||
Revises: 063_fingerprint_chunks
|
||||
Create Date: 2026-09-03
|
||||
"""
|
||||
from alembic import op
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "064_match_count_visual_sim"
|
||||
down_revision = "063_fingerprint_chunks"
|
||||
branch_labels = None
|
||||
|
||||
@@ -979,8 +979,11 @@ def check_duplicate_task(self: Task, generated_video_id: str) -> dict:
|
||||
|
||||
# 查重判定(跨项目全局 + 时长预过滤)
|
||||
duplicate_result = deduplicator.check_duplicate(
|
||||
fingerprint, video.project_id, session,
|
||||
scope="user", user_id=video.user_id,
|
||||
fingerprint,
|
||||
video.project_id,
|
||||
session,
|
||||
scope="user",
|
||||
user_id=video.user_id,
|
||||
duration_sec=fingerprint.duration / 1000 if fingerprint.duration else 0,
|
||||
)
|
||||
|
||||
@@ -994,8 +997,12 @@ def check_duplicate_task(self: Task, generated_video_id: str) -> dict:
|
||||
|
||||
# 查重率计算(跨项目全局)
|
||||
rate_result = deduplicator.compute_duplicate_rate(
|
||||
fingerprint, video.project_id, generated_video_id, session,
|
||||
scope="user", user_id=video.user_id,
|
||||
fingerprint,
|
||||
video.project_id,
|
||||
generated_video_id,
|
||||
session,
|
||||
scope="user",
|
||||
user_id=video.user_id,
|
||||
)
|
||||
video.duplicate_rate = rate_result["duplicate_rate"]
|
||||
video.match_count = rate_result["match_count"]
|
||||
|
||||
@@ -111,8 +111,12 @@ def create_video_record_and_dedup(
|
||||
# (a) 历史成片查重(跨项目全局 + 时长预过滤)
|
||||
duration_sec = fingerprint.duration / 1000 if fingerprint.duration else 0
|
||||
duplicate_result = deduplicator.check_duplicate(
|
||||
fingerprint, project_id, session,
|
||||
scope="user", user_id=user_id, duration_sec=duration_sec,
|
||||
fingerprint,
|
||||
project_id,
|
||||
session,
|
||||
scope="user",
|
||||
user_id=user_id,
|
||||
duration_sec=duration_sec,
|
||||
)
|
||||
|
||||
# (b) 批次内查重(仅当有 batch_id 时)
|
||||
@@ -148,8 +152,10 @@ def create_video_record_and_dedup(
|
||||
generated_video.visual_similarity = rate_result["visual_similarity"]
|
||||
logger.info(
|
||||
"Duplicate rate for %s: %.2f%% (visual_sim=%.3f, matches=%d)",
|
||||
video_id, rate_result["duplicate_rate"],
|
||||
rate_result["visual_similarity"], rate_result["match_count"],
|
||||
video_id,
|
||||
rate_result["duplicate_rate"],
|
||||
rate_result["visual_similarity"],
|
||||
rate_result["match_count"],
|
||||
)
|
||||
except Exception as rate_err:
|
||||
logger.warning("Failed to compute duplicate_rate for %s: %s", video_id, rate_err)
|
||||
|
||||
@@ -43,7 +43,11 @@ class TestDedupHelpersUserIdPassthrough:
|
||||
mock_deduplicator = MagicMock()
|
||||
mock_deduplicator.compute_fingerprint.return_value = mock_fingerprint
|
||||
mock_deduplicator.check_duplicate.return_value = None
|
||||
mock_deduplicator.compute_duplicate_rate.return_value = {"duplicate_rate": 42.5, "visual_similarity": 0.7, "match_count": 2}
|
||||
mock_deduplicator.compute_duplicate_rate.return_value = {
|
||||
"duplicate_rate": 42.5,
|
||||
"visual_similarity": 0.7,
|
||||
"match_count": 2,
|
||||
}
|
||||
|
||||
with (
|
||||
patch(
|
||||
@@ -85,7 +89,11 @@ class TestDedupHelpersUserIdPassthrough:
|
||||
mock_deduplicator = MagicMock()
|
||||
mock_deduplicator.compute_fingerprint.return_value = mock_fingerprint
|
||||
mock_deduplicator.check_duplicate.return_value = None
|
||||
mock_deduplicator.compute_duplicate_rate.return_value = {"duplicate_rate": 0.0, "visual_similarity": 0.0, "match_count": 0}
|
||||
mock_deduplicator.compute_duplicate_rate.return_value = {
|
||||
"duplicate_rate": 0.0,
|
||||
"visual_similarity": 0.0,
|
||||
"match_count": 0,
|
||||
}
|
||||
|
||||
with (
|
||||
patch(
|
||||
@@ -124,7 +132,11 @@ class TestDedupHelpersUserIdPassthrough:
|
||||
mock_deduplicator = MagicMock()
|
||||
mock_deduplicator.compute_fingerprint.return_value = mock_fingerprint
|
||||
mock_deduplicator.check_duplicate.return_value = None
|
||||
mock_deduplicator.compute_duplicate_rate.return_value = {"duplicate_rate": 78.5, "visual_similarity": 0.85, "match_count": 3}
|
||||
mock_deduplicator.compute_duplicate_rate.return_value = {
|
||||
"duplicate_rate": 78.5,
|
||||
"visual_similarity": 0.85,
|
||||
"match_count": 3,
|
||||
}
|
||||
|
||||
with (
|
||||
patch(
|
||||
|
||||
@@ -152,7 +152,12 @@ class TestComputeDuplicateRate:
|
||||
mock_repo = MockRepo.return_value
|
||||
mock_repo.list_by_user.return_value = [existing]
|
||||
rate = deduplicator.compute_duplicate_rate(
|
||||
fingerprint, "proj1", "vid1", session, scope="user", user_id="user1",
|
||||
fingerprint,
|
||||
"proj1",
|
||||
"vid1",
|
||||
session,
|
||||
scope="user",
|
||||
user_id="user1",
|
||||
)
|
||||
|
||||
# Should use list_by_user and find the match
|
||||
|
||||
@@ -16,6 +16,7 @@ sys.path.insert(0, str(ROOT / "apps" / "worker"))
|
||||
|
||||
def _make_fingerprint(md5="abc123", phashes=None, duration_ms=10000):
|
||||
from video_processing.dedup import VideoFingerprint
|
||||
|
||||
return VideoFingerprint(
|
||||
md5=md5,
|
||||
keyframe_phashes=phashes or ["ff00ff00ff00ff00"],
|
||||
@@ -27,11 +28,19 @@ def _make_fingerprint(md5="abc123", phashes=None, duration_ms=10000):
|
||||
|
||||
def _make_video(vid, fingerprint_dict, project_id="proj1", duration=10.0):
|
||||
from packages.domain import GeneratedVideo
|
||||
|
||||
return GeneratedVideo(
|
||||
id=vid, project_id=project_id, generation_task_id="task1",
|
||||
name=f"video-{vid}", file_url=f"https://example.com/{vid}.mp4",
|
||||
file_size=1000, duration=duration, width=1920, height=1080,
|
||||
fps=25.0, video_fingerprint=fingerprint_dict,
|
||||
id=vid,
|
||||
project_id=project_id,
|
||||
generation_task_id="task1",
|
||||
name=f"video-{vid}",
|
||||
file_url=f"https://example.com/{vid}.mp4",
|
||||
file_size=1000,
|
||||
duration=duration,
|
||||
width=1920,
|
||||
height=1080,
|
||||
fps=25.0,
|
||||
video_fingerprint=fingerprint_dict,
|
||||
)
|
||||
|
||||
|
||||
@@ -40,6 +49,7 @@ class TestCheckDuplicateScopeProject:
|
||||
|
||||
def test_default_scope_queries_by_project(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = _make_fingerprint(md5="unique_md5")
|
||||
session = MagicMock()
|
||||
@@ -54,6 +64,7 @@ class TestCheckDuplicateScopeProject:
|
||||
|
||||
def test_project_scope_finds_duplicate(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = _make_fingerprint(md5="same_md5")
|
||||
session = MagicMock()
|
||||
@@ -75,6 +86,7 @@ class TestCheckDuplicateScopeUser:
|
||||
|
||||
def test_user_scope_queries_by_user(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = _make_fingerprint(md5="unique_md5")
|
||||
session = MagicMock()
|
||||
@@ -83,7 +95,11 @@ class TestCheckDuplicateScopeUser:
|
||||
mock_repo = MockRepo.return_value
|
||||
mock_repo.list_by_user.return_value = []
|
||||
result = deduplicator.check_duplicate(
|
||||
fingerprint, "proj1", session, scope="user", user_id="user_123",
|
||||
fingerprint,
|
||||
"proj1",
|
||||
session,
|
||||
scope="user",
|
||||
user_id="user_123",
|
||||
)
|
||||
|
||||
mock_repo.list_by_user.assert_called_once()
|
||||
@@ -91,6 +107,7 @@ class TestCheckDuplicateScopeUser:
|
||||
|
||||
def test_user_scope_finds_cross_project_duplicate(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = _make_fingerprint(md5="cross_proj_md5")
|
||||
session = MagicMock()
|
||||
@@ -102,7 +119,11 @@ class TestCheckDuplicateScopeUser:
|
||||
mock_repo = MockRepo.return_value
|
||||
mock_repo.list_by_user.return_value = [existing]
|
||||
result = deduplicator.check_duplicate(
|
||||
fingerprint, "proj1", session, scope="user", user_id="user_123",
|
||||
fingerprint,
|
||||
"proj1",
|
||||
session,
|
||||
scope="user",
|
||||
user_id="user_123",
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
@@ -115,6 +136,7 @@ class TestDurationPrefilter:
|
||||
|
||||
def test_duration_prefilter_passes_correct_range(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = _make_fingerprint(duration_ms=30000) # 30s video
|
||||
session = MagicMock()
|
||||
@@ -123,8 +145,12 @@ class TestDurationPrefilter:
|
||||
mock_repo = MockRepo.return_value
|
||||
mock_repo.list_by_user.return_value = []
|
||||
deduplicator.check_duplicate(
|
||||
fingerprint, "proj1", session,
|
||||
scope="user", user_id="user1", duration_sec=30.0,
|
||||
fingerprint,
|
||||
"proj1",
|
||||
session,
|
||||
scope="user",
|
||||
user_id="user1",
|
||||
duration_sec=30.0,
|
||||
)
|
||||
|
||||
# Should pass duration_min=25.5, duration_max=34.5 (30 ± 15%)
|
||||
@@ -134,6 +160,7 @@ class TestDurationPrefilter:
|
||||
|
||||
def test_no_duration_prefilter_when_zero(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = _make_fingerprint()
|
||||
session = MagicMock()
|
||||
@@ -142,8 +169,12 @@ class TestDurationPrefilter:
|
||||
mock_repo = MockRepo.return_value
|
||||
mock_repo.list_by_user.return_value = []
|
||||
deduplicator.check_duplicate(
|
||||
fingerprint, "proj1", session,
|
||||
scope="user", user_id="user1", duration_sec=0,
|
||||
fingerprint,
|
||||
"proj1",
|
||||
session,
|
||||
scope="user",
|
||||
user_id="user1",
|
||||
duration_sec=0,
|
||||
)
|
||||
|
||||
call_args = mock_repo.list_by_user.call_args
|
||||
@@ -156,6 +187,7 @@ class TestComputeDuplicateRateFormula:
|
||||
|
||||
def test_formula_with_matching_frames(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
# 10 frames, all identical to existing → frame_match_rate = 1.0
|
||||
phashes = ["aa00aa00aa00aa00"] * 10
|
||||
@@ -179,6 +211,7 @@ class TestComputeDuplicateRateFormula:
|
||||
|
||||
def test_no_match_returns_zero(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
# Completely different phashes
|
||||
fingerprint = _make_fingerprint(md5="new", phashes=["ff00ff00ff00ff00"])
|
||||
@@ -204,6 +237,7 @@ class TestComputeDuplicateRateReturnDict:
|
||||
|
||||
def test_return_structure(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = _make_fingerprint()
|
||||
session = MagicMock()
|
||||
@@ -227,6 +261,7 @@ class TestBackwardCompat:
|
||||
|
||||
def test_default_scope_is_project(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = _make_fingerprint()
|
||||
session = MagicMock()
|
||||
@@ -245,6 +280,7 @@ class TestBackwardCompat:
|
||||
|
||||
def test_check_duplicate_default_scope_backward_compat(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = _make_fingerprint()
|
||||
session = MagicMock()
|
||||
|
||||
Reference in New Issue
Block a user