style: auto-format with black + isort + prettier [skip ci-format-check]
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 19s
CI/CD Pipeline / PR Build API Image (pull_request) Successful in 29s
CI/CD Pipeline / PR Build Worker Image (pull_request) Successful in 34s
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 1m46s
PR Automation / Auto Approve on CI Green (pull_request) Successful in 3m9s
AI Code Review / AI Code Review (pull_request) Successful in 6m34s
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (pull_request) Failing after 14m3s
PR Automation / Auto Merge on CI Green + Approved (pull_request) Successful in 11m6s
CI/CD Pipeline / Validate - Python (mypy + alembic) (pull_request) Successful in 1m58s
CI/CD Pipeline / Unit Tests (pull_request) Successful in 1m58s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 2m26s
CI/CD Pipeline / Validate - Style (pull_request) Failing after 2m58s
CI/CD Pipeline / Validate - Security (pull_request) Successful in 5m20s
CI/CD Pipeline / CI Gate (pull_request) Failing after 3s
CI/CD Pipeline / Production Browser E2E (pull_request) Has been skipped
CI/CD Pipeline / Canary Release to Production (pull_request) Failing after 301h59m35s
CI/CD Pipeline / Build Production API Image (pull_request) Failing after 301h59m38s
CI/CD Pipeline / Deploy Production (pull_request) Failing after 301h59m35s
CI/CD Pipeline / Build Production Worker Image (pull_request) Failing after 301h59m38s
CI/CD Pipeline / Frontend Unit Tests (pull_request) Failing after 302h4m59s
CI/CD Pipeline / Frontend Lint (pull_request) Failing after 302h4m59s
CI/CD Pipeline / ACR Image Cleanup (pull_request) Failing after 302h17m40s
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Failing after 302h17m46s
CI/CD Pipeline / Staging API Integration Tests (pull_request) Failing after 302h17m41s
CI/CD Pipeline / Retag skipped Staging Worker Image (pull_request) Failing after 302h17m52s
CI/CD Pipeline / Retag skipped Staging API Image (pull_request) Failing after 302h17m55s
CI/CD Pipeline / Build Staging Worker Image (pull_request) Failing after 302h18m1s
CI/CD Pipeline / Build Staging Web Image (pull_request) Failing after 302h18m2s
CI/CD Pipeline / Build Staging API Image (pull_request) Failing after 302h18m5s
CI/CD Pipeline / PR Build Web Image (pull_request) Failing after 302h18m33s
CI/CD Pipeline / Check push changed paths (pull_request) Failing after 302h18m47s
CI/CD Pipeline / Build Production Web Image (pull_request) Failing after 302h34m3s
CI/CD Pipeline / Staging E2E Tests (pull_request) Failing after 302h52m6s
CI/CD Pipeline / Retag skipped Staging Web Image (pull_request) Failing after 302h52m17s

This commit is contained in:
CI Bot
2026-09-03 17:04:19 +00:00
parent 5c48bb67c8
commit 351de916e3
2 changed files with 41 additions and 22 deletions
+10 -17
View File
@@ -179,11 +179,11 @@ class VideoDeduplicator:
"""
# ── Issue #1658: 校准后的常量 ──
PHASH_THRESHOLD = 8 # 从 10 收紧到 8
MATCH_RATIO_THRESHOLD = 0.7 # 至少 70% 帧匹配
DUPLICATE_THRESHOLD = 0.70 # 融合后相似度阈值
PHASH_WEIGHT = 0.7 # pHash 权重
HISTOGRAM_WEIGHT = 0.3 # 直方图权重
PHASH_THRESHOLD = 8 # 从 10 收紧到 8
MATCH_RATIO_THRESHOLD = 0.7 # 至少 70% 帧匹配
DUPLICATE_THRESHOLD = 0.70 # 融合后相似度阈值
PHASH_WEIGHT = 0.7 # pHash 权重
HISTOGRAM_WEIGHT = 0.3 # 直方图权重
def compute_fingerprint(self, video_path: str) -> VideoFingerprint:
"""Compute video fingerprint using MD5, pHash, and color histogram.
@@ -291,9 +291,7 @@ class VideoDeduplicator:
a = hist_a[:min_len]
b = hist_b[:min_len]
# max(0, ...) 防御上游异常负值导致 sqrt domain error
return float(
sum(max(0.0, ai * bi) ** 0.5 for ai, bi in zip(a, b, strict=True))
)
return float(sum(max(0.0, ai * bi) ** 0.5 for ai, bi in zip(a, b, strict=True)))
@staticmethod
def _compute_histogram_similarity(
@@ -376,9 +374,7 @@ class VideoDeduplicator:
# 直方图相似度:任一方无数据时统一返回 0.0(无法判定),避免不对称
if existing_histograms and fingerprint.color_histograms:
hist_similarity = self._compute_histogram_similarity(
fingerprint.color_histograms, existing_histograms
)
hist_similarity = self._compute_histogram_similarity(fingerprint.color_histograms, existing_histograms)
else:
hist_similarity = 0.0
combined_score = self.PHASH_WEIGHT * phash_similarity + self.HISTOGRAM_WEIGHT * hist_similarity
@@ -471,8 +467,7 @@ class VideoDeduplicator:
if chunk_data:
existing_phashes = [c["phash_binary"] for c in chunk_data]
existing_histograms = [
(ch if ch is not None else [])
for ch in (c.get("color_histogram") for c in chunk_data)
(ch if ch is not None else []) for ch in (c.get("color_histogram") for c in chunk_data)
]
else:
# 回退:从 JSON 字段读取(存量旧视频)
@@ -539,8 +534,7 @@ class VideoDeduplicator:
if chunk_data:
existing_phashes = [c["phash_binary"] for c in chunk_data]
existing_histograms = [
(ch if ch is not None else [])
for ch in (c.get("color_histogram") for c in chunk_data)
(ch if ch is not None else []) for ch in (c.get("color_histogram") for c in chunk_data)
]
else:
existing_phashes = ef.get("keyframe_phashes", [])
@@ -632,8 +626,7 @@ class VideoDeduplicator:
if chunk_data:
existing_phashes = [c["phash_binary"] for c in chunk_data]
existing_histograms = [
(ch if ch is not None else [])
for ch in (c.get("color_histogram") for c in chunk_data)
(ch if ch is not None else []) for ch in (c.get("color_histogram") for c in chunk_data)
]
else:
existing_phashes = ef.get("keyframe_phashes", [])
+31 -5
View File
@@ -96,13 +96,14 @@ sys.modules["packages.adapters.sqlalchemy_impl.models"] = _mock_module(
sys.modules["packages.shared.config"] = _mock_module(get_shared_settings=MagicMock(return_value=MagicMock()))
sys.modules["packages.shared.storage"] = _mock_module()
import video_processing.dedup as _dedup_module # noqa: E402
# Import while mocks active
from video_processing.dedup import ( # noqa: E402
VideoDeduplicator,
VideoFingerprint,
hamming_distance,
)
import video_processing.dedup as _dedup_module # noqa: E402
# Restore sys.modules
for _key in list(sys.modules.keys()):
@@ -179,6 +180,7 @@ class TestMedianVsMean:
def test_median_resists_outlier(self):
"""距离 [3, 3, 3, 3, 30]:均值=8.4,中位数=3。"""
import statistics
distances = [3, 3, 3, 3, 30]
mean_val = sum(distances) / len(distances)
median_val = statistics.median(distances)
@@ -620,7 +622,14 @@ class TestComputeDuplicateRate:
with patch.object(_dedup_module, "SQLAlchemyGeneratedVideoRepository", return_value=mock_repo):
with patch.object(d, "_get_existing_chunks", return_value=[]):
# Patch the import inside the method
with patch.dict(sys.modules, {"packages.adapters.sqlalchemy_impl.models": _mock_module(GeneratedVideoModel=mock_generated_model)}):
with patch.dict(
sys.modules,
{
"packages.adapters.sqlalchemy_impl.models": _mock_module(
GeneratedVideoModel=mock_generated_model
)
},
):
# Need to patch session.query to return mock_query
mock_session.query.return_value = mock_query
result = d.compute_duplicate_rate(fp, "proj_1", None, mock_session)
@@ -658,7 +667,14 @@ class TestComputeDuplicateRate:
mock_generated_model = MagicMock()
with patch.object(_dedup_module, "SQLAlchemyGeneratedVideoRepository", return_value=mock_repo):
with patch.object(d, "_get_existing_chunks", return_value=[]):
with patch.dict(sys.modules, {"packages.adapters.sqlalchemy_impl.models": _mock_module(GeneratedVideoModel=mock_generated_model)}):
with patch.dict(
sys.modules,
{
"packages.adapters.sqlalchemy_impl.models": _mock_module(
GeneratedVideoModel=mock_generated_model
)
},
):
result = d.compute_duplicate_rate(fp, "proj_1", None, mock_session)
# 完全相同 → similarity≈1.0 → rate≈100
@@ -698,7 +714,14 @@ class TestComputeDuplicateRate:
mock_generated_model = MagicMock()
with patch.object(_dedup_module, "SQLAlchemyGeneratedVideoRepository", return_value=mock_repo):
with patch.object(d, "_get_existing_chunks", return_value=[]):
with patch.dict(sys.modules, {"packages.adapters.sqlalchemy_impl.models": _mock_module(GeneratedVideoModel=mock_generated_model)}):
with patch.dict(
sys.modules,
{
"packages.adapters.sqlalchemy_impl.models": _mock_module(
GeneratedVideoModel=mock_generated_model
)
},
):
result = d.compute_duplicate_rate(fp, "proj_1", None, mock_session)
# phash 完全不同(0.0) + 直方图完全相同(1.0) → 0.7*0 + 0.3*1 = 0.3 → 30.0
@@ -720,7 +743,10 @@ class TestComputeDuplicateRate:
mock_generated_model = MagicMock()
with patch.object(_dedup_module, "SQLAlchemyGeneratedVideoRepository", return_value=mock_repo):
with patch.dict(sys.modules, {"packages.adapters.sqlalchemy_impl.models": _mock_module(GeneratedVideoModel=mock_generated_model)}):
with patch.dict(
sys.modules,
{"packages.adapters.sqlalchemy_impl.models": _mock_module(GeneratedVideoModel=mock_generated_model)},
):
result = d.compute_duplicate_rate(fp, "proj_1", None, mock_session)
assert result == 0.0