From 61c19c342ea00dd36bef7d0953f4b0241bbfd13c Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Mon, 31 Aug 2026 16:05:50 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20AI=20Code=20Review?= =?UTF-8?q?=20=E9=98=BB=E5=A1=9E=E7=BA=A7=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. assets.py: 修复高频排除回退逻辑 min(limit,1) → limit 2. assets.py: high_use_excluded 改用 set 提升查找效率 3. dedup.py: compute_duplicate_rate 限制查询最近100个视频避免OOM 4. 更新单元测试适配新的查询方式 --- apps/api/app/api/routes/assets.py | 8 +-- apps/worker/video_processing/dedup.py | 11 +++- tests/unit/test_duplicate_rate.py | 89 ++++++++++++++++----------- 3 files changed, 66 insertions(+), 42 deletions(-) diff --git a/apps/api/app/api/routes/assets.py b/apps/api/app/api/routes/assets.py index 933f9aa03..e096f2285 100755 --- a/apps/api/app/api/routes/assets.py +++ b/apps/api/app/api/routes/assets.py @@ -619,7 +619,7 @@ def smart_match_assets( asset_ids=asset_ids, recent_video_count=5, ) - high_use_excluded = [] + high_use_excluded = set() for r in filtered_results: aid = getattr(r.asset, "id", "") count = use_counts.get(aid, 0) @@ -628,12 +628,12 @@ def smart_match_assets( "smart-match 排除高频使用素材: asset_id=%s use_count=%d limit=%d", aid, count, MAX_RECENT_USE_COUNT, ) - high_use_excluded.append(r) + high_use_excluded.add(id(r)) else: pass # 如果排除后不够 limit，放宽到不限制 - remaining = [r for r in filtered_results if r not in high_use_excluded] - if len(remaining) >= min(request.limit, 1): + remaining = [r for r in filtered_results if id(r) not in high_use_excluded] + if len(remaining) >= request.limit: filtered_results = remaining else: logger.info("smart-match 高频排除后素材不足(%d<%d)，保留全部", len(remaining), request.limit) diff --git a/apps/worker/video_processing/dedup.py b/apps/worker/video_processing/dedup.py index 62bf6ab91..dfe5bf115 100755 --- a/apps/worker/video_processing/dedup.py +++ b/apps/worker/video_processing/dedup.py @@ -332,8 +332,17 @@ class VideoDeduplicator: Returns: duplicate_rate: 0~100 的浮点数 """ + # 限制查询最近 100 个视频，避免大项目内存溢出 + from packages.adapters.sqlalchemy_impl.models import GeneratedVideoModel + recent_models = ( + session.query(GeneratedVideoModel) + .filter(GeneratedVideoModel.project_id == project_id) + .order_by(GeneratedVideoModel.generated_at.desc()) + .limit(100) + .all() + ) video_repo = SQLAlchemyGeneratedVideoRepository(session) - existing_videos = video_repo.list_by_project(project_id) + existing_videos = [video_repo._to_domain(m) for m in recent_models] max_similarity = 0.0 for existing in existing_videos: diff --git a/tests/unit/test_duplicate_rate.py b/tests/unit/test_duplicate_rate.py index cb09ef2bc..f7376e23a 100644 --- a/tests/unit/test_duplicate_rate.py +++ b/tests/unit/test_duplicate_rate.py @@ -1,5 +1,4 @@ """Tests for duplicate_rate computation and API response.""" - import json import sys from pathlib import Path @@ -21,7 +20,6 @@ class TestComputeDuplicateRate: def _make_fingerprint(self, md5="abc123", phashes=None): from video_processing.dedup import VideoFingerprint - return VideoFingerprint( md5=md5, keyframe_phashes=phashes or ["ff00ff00ff00ff00"], @@ -32,7 +30,6 @@ class TestComputeDuplicateRate: def _make_existing_video(self, vid, fingerprint_dict): from packages.domain import GeneratedVideo - return GeneratedVideo( id=vid, project_id="proj1", @@ -49,37 +46,44 @@ class TestComputeDuplicateRate: def test_no_existing_videos_returns_zero(self): from video_processing.dedup import VideoDeduplicator - deduplicator = VideoDeduplicator() fingerprint = self._make_fingerprint() session = MagicMock() with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo: mock_repo = MockRepo.return_value - mock_repo.list_by_project.return_value = [] + session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [] rate = deduplicator.compute_duplicate_rate(fingerprint, "proj1", "vid1", session) assert rate == 0.0 def test_md5_match_returns_100(self): from video_processing.dedup import VideoDeduplicator - + from packages.adapters.sqlalchemy_impl.models import GeneratedVideoModel deduplicator = VideoDeduplicator() fingerprint = self._make_fingerprint(md5="exact_match_md5") session = MagicMock() existing = self._make_existing_video("existing1", {"md5": "exact_match_md5", "keyframe_phashes": ["aa"]}) + # Create a mock model with the domain attributes + mock_model = MagicMock(spec=GeneratedVideoModel) + mock_model.id = existing.id + mock_model.project_id = existing.project_id + mock_model.video_fingerprint = existing.video_fingerprint + mock_model.generated_at = "2026-01-01" with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo: mock_repo = MockRepo.return_value - mock_repo.list_by_project.return_value = [existing] + mock_repo._to_domain.return_value = existing + # Mock the session.query chain + session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [mock_model] rate = deduplicator.compute_duplicate_rate(fingerprint, "proj1", "vid1", session) assert rate == 100.0 def test_phash_similarity_computed(self): from video_processing.dedup import VideoDeduplicator - + from packages.adapters.sqlalchemy_impl.models import GeneratedVideoModel deduplicator = VideoDeduplicator() fingerprint = self._make_fingerprint(md5="different_md5", phashes=["ff00ff00ff00ff00"]) session = MagicMock() @@ -88,10 +92,16 @@ class TestComputeDuplicateRate: "existing1", {"md5": "other_md5", "keyframe_phashes": ["ff00ff00ff00ff03"]}, ) + mock_model = MagicMock(spec=GeneratedVideoModel) + mock_model.id = existing.id + mock_model.project_id = existing.project_id + mock_model.video_fingerprint = existing.video_fingerprint + mock_model.generated_at = "2026-01-01" with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo: mock_repo = MockRepo.return_value - mock_repo.list_by_project.return_value = [existing] + mock_repo._to_domain.return_value = existing + session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [mock_model] rate = deduplicator.compute_duplicate_rate(fingerprint, "proj1", "vid1", session) # hamming distance = 2, similarity = (1 - 2/64) * 100 = 96.875 @@ -99,33 +109,54 @@ class TestComputeDuplicateRate: def test_excludes_self_video(self): from video_processing.dedup import VideoDeduplicator - + from packages.adapters.sqlalchemy_impl.models import GeneratedVideoModel deduplicator = VideoDeduplicator() fingerprint = self._make_fingerprint(md5="same_md5") session = MagicMock() self_video = self._make_existing_video("vid1", {"md5": "same_md5", "keyframe_phashes": ["aa"]}) + mock_model = MagicMock(spec=GeneratedVideoModel) + mock_model.id = self_video.id + mock_model.project_id = self_video.project_id + mock_model.video_fingerprint = self_video.video_fingerprint + mock_model.generated_at = "2026-01-01" with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo: mock_repo = MockRepo.return_value - mock_repo.list_by_project.return_value = [self_video] + mock_repo._to_domain.return_value = self_video + session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [mock_model] rate = deduplicator.compute_duplicate_rate(fingerprint, "proj1", "vid1", session) assert rate == 0.0 def test_takes_max_similarity(self): from video_processing.dedup import VideoDeduplicator - + from packages.adapters.sqlalchemy_impl.models import GeneratedVideoModel deduplicator = VideoDeduplicator() fingerprint = self._make_fingerprint(md5="new_md5", phashes=["ff00ff00ff00ff00"]) session = MagicMock() - existing1 = self._make_existing_video("e1", {"md5": "md5_1", "keyframe_phashes": ["ff00ff00ff00ff0f"]}) - existing2 = self._make_existing_video("e2", {"md5": "md5_2", "keyframe_phashes": ["ff00ff00ff00ff01"]}) + existing1 = self._make_existing_video( + "e1", {"md5": "md5_1", "keyframe_phashes": ["ff00ff00ff00ff0f"]} + ) + existing2 = self._make_existing_video( + "e2", {"md5": "md5_2", "keyframe_phashes": ["ff00ff00ff00ff01"]} + ) + mock_model1 = MagicMock(spec=GeneratedVideoModel) + mock_model1.id = existing1.id + mock_model1.project_id = existing1.project_id + mock_model1.video_fingerprint = existing1.video_fingerprint + mock_model1.generated_at = "2026-01-02" + mock_model2 = MagicMock(spec=GeneratedVideoModel) + mock_model2.id = existing2.id + mock_model2.project_id = existing2.project_id + mock_model2.video_fingerprint = existing2.video_fingerprint + mock_model2.generated_at = "2026-01-01" with patch("video_processing.dedup.SQLAlchemyGeneratedVideoRepository") as MockRepo: mock_repo = MockRepo.return_value - mock_repo.list_by_project.return_value = [existing1, existing2] + mock_repo._to_domain.side_effect = [existing1, existing2] + session.query.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [mock_model1, mock_model2] rate = deduplicator.compute_duplicate_rate(fingerprint, "proj1", "vid1", session) # max similarity: e2 distance=1, (1-1/64)*100 = 98.4375 @@ -137,35 +168,19 @@ class TestDuplicateRateAPI: def test_video_item_response_has_duplicate_rate(self): from app.schemas.video_center import VideoItemResponse - resp = VideoItemResponse( - id="v1", - project_id="p1", - generation_task_id="t1", - name="test.mp4", - file_url="https://example.com/test.mp4", - file_size=1000, - duration=10.0, - width=1920, - height=1080, - fps=25.0, + id="v1", project_id="p1", generation_task_id="t1", + name="test.mp4", file_url="https://example.com/test.mp4", + file_size=1000, duration=10.0, width=1920, height=1080, fps=25.0, duplicate_rate=75.5, ) assert resp.duplicate_rate == 75.5 def test_video_item_response_duplicate_rate_default_none(self): from app.schemas.video_center import VideoItemResponse - resp = VideoItemResponse( - id="v1", - project_id="p1", - generation_task_id="t1", - name="test.mp4", - file_url="https://example.com/test.mp4", - file_size=1000, - duration=10.0, - width=1920, - height=1080, - fps=25.0, + id="v1", project_id="p1", generation_task_id="t1", + name="test.mp4", file_url="https://example.com/test.mp4", + file_size=1000, duration=10.0, width=1920, height=1080, fps=25.0, ) assert resp.duplicate_rate is None