style: auto-format with black + isort + prettier [skip ci-format-check]
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (pull_request) Successful in 4s
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 4m20s
CI/CD Pipeline / Validate - Type Check (mypy) (pull_request) Successful in 5m3s
CI/CD Pipeline / Validate - Migration (alembic) (pull_request) Successful in 5m11s
CI/CD Pipeline / PR Build API Image (pull_request) Successful in 52s
PR Automation / Auto Merge on CI Green + Approved (pull_request) Successful in 5m21s
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 5m31s
CI/CD Pipeline / PR Build Worker Image (pull_request) Successful in 48s
PR Automation / Auto Approve on CI Green (pull_request) Successful in 7m7s
AI Code Review / AI Code Review (pull_request) Failing after 7m34s
CI/CD Pipeline / Validate - Code Quality (pull_request) Successful in 8m20s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 3m15s
CI/CD Pipeline / Unit Tests (pull_request) Has been cancelled
CI/CD Pipeline / Build Production API Image (pull_request) Has been cancelled
CI/CD Pipeline / Build Production Web Image (pull_request) Has been cancelled
CI/CD Pipeline / Build Production Worker Image (pull_request) Has been cancelled
CI/CD Pipeline / Deploy Production (pull_request) Has been cancelled
CI/CD Pipeline / Production Browser E2E (pull_request) Has been cancelled
CI/CD Pipeline / Canary Release to Production (pull_request) Has been cancelled
CI/CD Pipeline / CI Gate (pull_request) Has been cancelled
CI/CD Pipeline / PR Build Web Image (pull_request) Failing after 383h1m21s
CI/CD Pipeline / Frontend Lint (pull_request) Failing after 383h2m6s
CI/CD Pipeline / ACR Image Cleanup (pull_request) Failing after 383h6m12s
CI/CD Pipeline / Staging E2E Tests (pull_request) Failing after 383h6m16s
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Failing after 383h6m18s
CI/CD Pipeline / Retag skipped Staging Web Image (pull_request) Failing after 383h6m22s
CI/CD Pipeline / Retag skipped Staging API Image (pull_request) Failing after 383h6m24s
CI/CD Pipeline / Build Staging Web Image (pull_request) Failing after 383h6m26s
CI/CD Pipeline / Build Staging API Image (pull_request) Failing after 383h6m26s
CI/CD Pipeline / Check push changed paths (pull_request) Failing after 383h6m28s
CI/CD Pipeline / Frontend Unit Tests (pull_request) Failing after 383h36m16s
CI/CD Pipeline / Staging API Integration Tests (pull_request) Failing after 383h40m26s
CI/CD Pipeline / Retag skipped Staging Worker Image (pull_request) Failing after 383h40m32s
CI/CD Pipeline / Build Staging Worker Image (pull_request) Failing after 383h40m37s
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (pull_request) Successful in 4s
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 4m20s
CI/CD Pipeline / Validate - Type Check (mypy) (pull_request) Successful in 5m3s
CI/CD Pipeline / Validate - Migration (alembic) (pull_request) Successful in 5m11s
CI/CD Pipeline / PR Build API Image (pull_request) Successful in 52s
PR Automation / Auto Merge on CI Green + Approved (pull_request) Successful in 5m21s
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 5m31s
CI/CD Pipeline / PR Build Worker Image (pull_request) Successful in 48s
PR Automation / Auto Approve on CI Green (pull_request) Successful in 7m7s
AI Code Review / AI Code Review (pull_request) Failing after 7m34s
CI/CD Pipeline / Validate - Code Quality (pull_request) Successful in 8m20s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 3m15s
CI/CD Pipeline / Unit Tests (pull_request) Has been cancelled
CI/CD Pipeline / Build Production API Image (pull_request) Has been cancelled
CI/CD Pipeline / Build Production Web Image (pull_request) Has been cancelled
CI/CD Pipeline / Build Production Worker Image (pull_request) Has been cancelled
CI/CD Pipeline / Deploy Production (pull_request) Has been cancelled
CI/CD Pipeline / Production Browser E2E (pull_request) Has been cancelled
CI/CD Pipeline / Canary Release to Production (pull_request) Has been cancelled
CI/CD Pipeline / CI Gate (pull_request) Has been cancelled
CI/CD Pipeline / PR Build Web Image (pull_request) Failing after 383h1m21s
CI/CD Pipeline / Frontend Lint (pull_request) Failing after 383h2m6s
CI/CD Pipeline / ACR Image Cleanup (pull_request) Failing after 383h6m12s
CI/CD Pipeline / Staging E2E Tests (pull_request) Failing after 383h6m16s
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Failing after 383h6m18s
CI/CD Pipeline / Retag skipped Staging Web Image (pull_request) Failing after 383h6m22s
CI/CD Pipeline / Retag skipped Staging API Image (pull_request) Failing after 383h6m24s
CI/CD Pipeline / Build Staging Web Image (pull_request) Failing after 383h6m26s
CI/CD Pipeline / Build Staging API Image (pull_request) Failing after 383h6m26s
CI/CD Pipeline / Check push changed paths (pull_request) Failing after 383h6m28s
CI/CD Pipeline / Frontend Unit Tests (pull_request) Failing after 383h36m16s
CI/CD Pipeline / Staging API Integration Tests (pull_request) Failing after 383h40m26s
CI/CD Pipeline / Retag skipped Staging Worker Image (pull_request) Failing after 383h40m32s
CI/CD Pipeline / Build Staging Worker Image (pull_request) Failing after 383h40m37s
This commit is contained in:
@@ -334,6 +334,7 @@ class VideoDeduplicator:
|
||||
"""
|
||||
# 限制查询最近 100 个视频,避免大项目内存溢出
|
||||
from packages.adapters.sqlalchemy_impl.models import GeneratedVideoModel
|
||||
|
||||
recent_models = (
|
||||
session.query(GeneratedVideoModel)
|
||||
.filter(GeneratedVideoModel.project_id == project_id)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for duplicate_rate computation and API response."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -20,6 +21,7 @@ class TestComputeDuplicateRate:
|
||||
|
||||
def _make_fingerprint(self, md5="abc123", phashes=None):
|
||||
from video_processing.dedup import VideoFingerprint
|
||||
|
||||
return VideoFingerprint(
|
||||
md5=md5,
|
||||
keyframe_phashes=phashes or ["ff00ff00ff00ff00"],
|
||||
@@ -30,6 +32,7 @@ class TestComputeDuplicateRate:
|
||||
|
||||
def _make_existing_video(self, vid, fingerprint_dict):
|
||||
from packages.domain import GeneratedVideo
|
||||
|
||||
return GeneratedVideo(
|
||||
id=vid,
|
||||
project_id="proj1",
|
||||
@@ -46,20 +49,25 @@ class TestComputeDuplicateRate:
|
||||
|
||||
def test_no_existing_videos_returns_zero(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = self._make_fingerprint()
|
||||
session = MagicMock()
|
||||
|
||||
with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo:
|
||||
mock_repo = MockRepo.return_value
|
||||
session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = []
|
||||
session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = (
|
||||
[]
|
||||
)
|
||||
rate = deduplicator.compute_duplicate_rate(fingerprint, "proj1", "vid1", session)
|
||||
|
||||
assert rate == 0.0
|
||||
|
||||
def test_md5_match_returns_100(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
from packages.adapters.sqlalchemy_impl.models import GeneratedVideoModel
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = self._make_fingerprint(md5="exact_match_md5")
|
||||
session = MagicMock()
|
||||
@@ -76,14 +84,18 @@ class TestComputeDuplicateRate:
|
||||
mock_repo = MockRepo.return_value
|
||||
mock_repo._to_domain.return_value = existing
|
||||
# Mock the session.query chain
|
||||
session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [mock_model]
|
||||
session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [
|
||||
mock_model
|
||||
]
|
||||
rate = deduplicator.compute_duplicate_rate(fingerprint, "proj1", "vid1", session)
|
||||
|
||||
assert rate == 100.0
|
||||
|
||||
def test_phash_similarity_computed(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
from packages.adapters.sqlalchemy_impl.models import GeneratedVideoModel
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = self._make_fingerprint(md5="different_md5", phashes=["ff00ff00ff00ff00"])
|
||||
session = MagicMock()
|
||||
@@ -101,7 +113,9 @@ class TestComputeDuplicateRate:
|
||||
with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo:
|
||||
mock_repo = MockRepo.return_value
|
||||
mock_repo._to_domain.return_value = existing
|
||||
session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [mock_model]
|
||||
session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [
|
||||
mock_model
|
||||
]
|
||||
rate = deduplicator.compute_duplicate_rate(fingerprint, "proj1", "vid1", session)
|
||||
|
||||
# hamming distance = 2, similarity = (1 - 2/64) * 100 = 96.875
|
||||
@@ -109,7 +123,9 @@ class TestComputeDuplicateRate:
|
||||
|
||||
def test_excludes_self_video(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
from packages.adapters.sqlalchemy_impl.models import GeneratedVideoModel
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = self._make_fingerprint(md5="same_md5")
|
||||
session = MagicMock()
|
||||
@@ -124,24 +140,24 @@ class TestComputeDuplicateRate:
|
||||
with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo:
|
||||
mock_repo = MockRepo.return_value
|
||||
mock_repo._to_domain.return_value = self_video
|
||||
session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [mock_model]
|
||||
session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [
|
||||
mock_model
|
||||
]
|
||||
rate = deduplicator.compute_duplicate_rate(fingerprint, "proj1", "vid1", session)
|
||||
|
||||
assert rate == 0.0
|
||||
|
||||
def test_takes_max_similarity(self):
|
||||
from video_processing.dedup import VideoDeduplicator
|
||||
|
||||
from packages.adapters.sqlalchemy_impl.models import GeneratedVideoModel
|
||||
|
||||
deduplicator = VideoDeduplicator()
|
||||
fingerprint = self._make_fingerprint(md5="new_md5", phashes=["ff00ff00ff00ff00"])
|
||||
session = MagicMock()
|
||||
|
||||
existing1 = self._make_existing_video(
|
||||
"e1", {"md5": "md5_1", "keyframe_phashes": ["ff00ff00ff00ff0f"]}
|
||||
)
|
||||
existing2 = self._make_existing_video(
|
||||
"e2", {"md5": "md5_2", "keyframe_phashes": ["ff00ff00ff00ff01"]}
|
||||
)
|
||||
existing1 = self._make_existing_video("e1", {"md5": "md5_1", "keyframe_phashes": ["ff00ff00ff00ff0f"]})
|
||||
existing2 = self._make_existing_video("e2", {"md5": "md5_2", "keyframe_phashes": ["ff00ff00ff00ff01"]})
|
||||
mock_model1 = MagicMock(spec=GeneratedVideoModel)
|
||||
mock_model1.id = existing1.id
|
||||
mock_model1.project_id = existing1.project_id
|
||||
@@ -156,7 +172,10 @@ class TestComputeDuplicateRate:
|
||||
with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo:
|
||||
mock_repo = MockRepo.return_value
|
||||
mock_repo._to_domain.side_effect = [existing1, existing2]
|
||||
session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [mock_model1, mock_model2]
|
||||
session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [
|
||||
mock_model1,
|
||||
mock_model2,
|
||||
]
|
||||
rate = deduplicator.compute_duplicate_rate(fingerprint, "proj1", "vid1", session)
|
||||
|
||||
# max similarity: e2 distance=1, (1-1/64)*100 = 98.4375
|
||||
@@ -168,19 +187,35 @@ class TestDuplicateRateAPI:
|
||||
|
||||
def test_video_item_response_has_duplicate_rate(self):
|
||||
from app.schemas.video_center import VideoItemResponse
|
||||
|
||||
resp = VideoItemResponse(
|
||||
id="v1", project_id="p1", generation_task_id="t1",
|
||||
name="test.mp4", file_url="https://example.com/test.mp4",
|
||||
file_size=1000, duration=10.0, width=1920, height=1080, fps=25.0,
|
||||
id="v1",
|
||||
project_id="p1",
|
||||
generation_task_id="t1",
|
||||
name="test.mp4",
|
||||
file_url="https://example.com/test.mp4",
|
||||
file_size=1000,
|
||||
duration=10.0,
|
||||
width=1920,
|
||||
height=1080,
|
||||
fps=25.0,
|
||||
duplicate_rate=75.5,
|
||||
)
|
||||
assert resp.duplicate_rate == 75.5
|
||||
|
||||
def test_video_item_response_duplicate_rate_default_none(self):
|
||||
from app.schemas.video_center import VideoItemResponse
|
||||
|
||||
resp = VideoItemResponse(
|
||||
id="v1", project_id="p1", generation_task_id="t1",
|
||||
name="test.mp4", file_url="https://example.com/test.mp4",
|
||||
file_size=1000, duration=10.0, width=1920, height=1080, fps=25.0,
|
||||
id="v1",
|
||||
project_id="p1",
|
||||
generation_task_id="t1",
|
||||
name="test.mp4",
|
||||
file_url="https://example.com/test.mp4",
|
||||
file_size=1000,
|
||||
duration=10.0,
|
||||
width=1920,
|
||||
height=1080,
|
||||
fps=25.0,
|
||||
)
|
||||
assert resp.duplicate_rate is None
|
||||
|
||||
Reference in New Issue
Block a user