diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 1ee5f0c20..a689a30d8 100755 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -158,15 +158,67 @@ jobs: python3 scripts/check_migration_safety.py --allow-medium-risk fi + - name: Debug coverage paths + shell: sh + run: | + set +e + echo "=== PWD ===" + pwd + echo "=== check source dirs ===" + ls -d apps/api/app packages + echo "=== python import check ===" + python3 - <<'PY' + import sys, os + os.environ["PYTHONPATH"] = f"{os.getcwd()}/apps/api:{os.getcwd()}" + sys.path.insert(0, f"{os.getcwd()}/apps/api") + sys.path.insert(0, os.getcwd()) + print(f"cwd: {os.getcwd()}") + print(f"sys.path[:5]: {sys.path[:5]}") + try: + import app + print(f"app.__file__: {app.__file__}") + except Exception as e: + print(f"import app failed: {e}") + try: + import packages + print(f"packages.__file__: {packages.__file__}") + except Exception as e: + print(f"import packages failed: {e}") + PY + echo "=== coverage debug ===" + python3 - <<'PY' + import os, sys + sys.path.insert(0, f"{os.getcwd()}/apps/api") + sys.path.insert(0, os.getcwd()) + import coverage + cov = coverage.Coverage(source=["apps/api/app", "packages"]) + print(f"source: {cov.config.source}") + for src in cov.config.source or []: + abspath = os.path.abspath(src) + print(f" {src} -> {abspath} exists={os.path.exists(src)}") + if os.path.isdir(src): + pyfiles = [] + for root, dirs, files in os.walk(src): + for f in files: + if f.endswith('.py'): + pyfiles.append(os.path.join(root, f)) + print(f" .py files: {len(pyfiles)}") + PY + - name: Run unit tests shell: sh env: USE_IN_MEMORY_DB: "true" run: | set -eu - PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/unit -q \ - --cov=apps --cov-report=term --cov-report=term-missing --cov-report=xml \ - --cov-fail-under=60 + PYTHONPATH="$PWD/apps/api:$PWD" python3 -m coverage run \ + --source=apps/api/app,packages \ + --omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \ + --branch \ + -m pytest tests/unit -q + python3 -m coverage report --show-missing + python3 -m coverage xml -o coverage.xml + python3 -m coverage report --fail-under=60 > /dev/null - name: Start PostgreSQL for integration tests shell: sh @@ -211,8 +263,14 @@ jobs: run: | set -eu pip install -q pytest-rerunfailures - PYTHONPATH="$PWD/apps/api:$PWD" python3 -m pytest tests/integration -q --timeout=60 -x --reruns 2 --reruns-delay 1 -m "not performance" \ - --cov=apps --cov-append --cov-report=term --cov-report=term-missing --cov-report=xml --cov-fail-under=65 + PYTHONPATH="$PWD/apps/api:$PWD" python3 -m coverage run --append \ + --source=apps/api/app,packages \ + --omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \ + --branch \ + -m pytest tests/integration -q --timeout=60 -x --reruns 2 --reruns-delay 1 -m "not performance" + python3 -m coverage report --show-missing + python3 -m coverage xml -o coverage.xml + python3 -m coverage report --fail-under=65 > /dev/null - name: Run API performance baseline tests shell: sh @@ -269,91 +327,14 @@ jobs: run: | set +e echo "=== 覆盖率汇总 ===" - if [ -f coverage.xml ]; then - python3 -c " -import xml.etree.ElementTree as ET -tree = ET.parse('coverage.xml') -root = tree.getroot() -line_rate = float(root.get('line-rate', 0)) * 100 -branch_rate = float(root.get('branch-rate', 0)) * 100 -lines_covered = int(root.get('lines-covered', 0)) -lines_valid = int(root.get('lines-valid', 0)) -print(f'行覆盖率: {line_rate:.2f}% ({lines_covered}/{lines_valid})') -print(f'分支覆盖率: {branch_rate:.2f}%') -print(f'门槛: 65%') -print(f'状态: {"PASS ✅" if line_rate >= 65 else "FAIL ❌"}') -" - else - echo "coverage.xml 不存在，跳过汇总" - fi - + python3 scripts/ci_coverage_summary.py - name: Notify CI failure if: failure() shell: sh run: | set +e echo "=== CI 失败通知 ===" - - # 收集失败信息 - FAILED_JOB="Validate Code Quality And Tests" - BRANCH="${GITHUB_REF_NAME:-unknown}" - COMMIT="${GITHUB_SHA:0:8}" - ACTOR="${GITHUB_ACTOR:-unknown}" - RUN_ID="${GITHUB_RUN_ID:-unknown}" - REPO="${GITHUB_REPOSITORY:-unknown}" - RUN_URL="https://git.xiaoxiajianji.com/${REPO}/actions/runs/${RUN_ID}" - - # 构造通知消息 - PAYLOAD=$(cat < /dev/null 2>&1 && echo "通知已发送" || echo "通知发送失败" - else - echo "未配置 CI_NOTIFY_WEBHOOK，跳过通知" - echo "如需启用，请在仓库 Settings -> Secrets and variables -> Actions 中添加 CI_NOTIFY_WEBHOOK" - fi + FAILED_JOB="Validate Code Quality And Tests" python3 scripts/ci_notify_failure.py - name: Build summary if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main' @@ -364,15 +345,7 @@ print(f'状态: {"PASS ✅" if line_rate >= 65 else "FAIL ❌"}') echo "Branch: ${GITHUB_REF_NAME}" echo "Commit: ${GITHUB_SHA}" # 输出最终覆盖率 - if [ -f coverage.xml ]; then - python3 -c " -import xml.etree.ElementTree as ET -tree = ET.parse('coverage.xml') -root = tree.getroot() -line_rate = float(root.get('line-rate', 0)) * 100 -print(f'Total coverage: {line_rate:.2f}%') -" - fi + python3 scripts/ci_coverage_summary.py frontend-lint: name: Frontend Lint diff --git a/apps/api/app/api/routes/voice_clones.py b/apps/api/app/api/routes/voice_clones.py old mode 100755 new mode 100644 index 740796283..ccbf28383 --- a/apps/api/app/api/routes/voice_clones.py +++ b/apps/api/app/api/routes/voice_clones.py @@ -6,7 +6,7 @@ import logging from typing import Optional from app.auth import AuthenticatedUser, get_current_user -from app.dependencies import get_audio_url_signer, get_cosyvoice_service, get_voice_clone_profile_repository +from app.dependencies import get_cosyvoice_service, get_voice_clone_profile_repository from app.schemas.voice_clone import ( CreateVoiceCloneRequest, ListVoiceCloneResponse, @@ -37,16 +37,14 @@ logger = logging.getLogger(__name__) router = APIRouter() -def _to_response(profile, sign_url=None) -> VoiceCloneProfileResponse: - source_url = profile.source_audio_url - if sign_url and source_url: - source_url = sign_url(source_url) +def _to_response(profile) -> VoiceCloneProfileResponse: + # source_audio_url 是用户传入的原始 URL（可能是外部地址），不做预签名转换 return VoiceCloneProfileResponse( id=profile.id, user_id=profile.user_id, name=profile.name, description=profile.description, - source_audio_url=source_url, + source_audio_url=profile.source_audio_url, voice_id=profile.voice_id, voice_model=profile.voice_model, language=profile.language, @@ -77,7 +75,6 @@ def create_voice_clone( request: CreateVoiceCloneRequest, authenticated_user: AuthenticatedUser = Depends(get_current_user), workflow: VoiceCloneWorkflowService = Depends(_get_workflow_service), - sign_url=Depends(get_audio_url_signer), ) -> VoiceCloneProfileResponse: """创建音色克隆任务。 @@ -113,7 +110,7 @@ def create_voice_clone( except Exception as inner_e: logger.error(f"Failed to mark profile as failed after dispatch error: {inner_e}") - return _to_response(profile, sign_url) + return _to_response(profile) @router.get("", response_model=ListVoiceCloneResponse) @@ -123,14 +120,13 @@ def list_voice_clones( limit: int = Query(50, ge=1, le=200), authenticated_user: AuthenticatedUser = Depends(get_current_user), repository: SQLAlchemyVoiceCloneProfileRepository = Depends(get_voice_clone_profile_repository), - sign_url=Depends(get_audio_url_signer), ) -> ListVoiceCloneResponse: """获取用户的音色克隆列表。""" user_id = authenticated_user.user.id use_case = ListVoiceClonesUseCase(repository) items, total = use_case.execute(user_id, status=status_filter, skip=skip, limit=limit) return ListVoiceCloneResponse( - items=[_to_response(p, sign_url) for p in items], + items=[_to_response(p) for p in items], total=total, ) @@ -140,7 +136,6 @@ def get_voice_clone( clone_id: str, authenticated_user: AuthenticatedUser = Depends(get_current_user), repository: SQLAlchemyVoiceCloneProfileRepository = Depends(get_voice_clone_profile_repository), - sign_url=Depends(get_audio_url_signer), ) -> VoiceCloneProfileResponse: """获取音色克隆详情。""" user_id = authenticated_user.user.id @@ -149,7 +144,7 @@ def get_voice_clone( profile = use_case.execute(clone_id, user_id) except VoiceCloneNotFoundError: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Voice clone not found") - return _to_response(profile, sign_url) + return _to_response(profile) @router.get("/{clone_id}/status", response_model=VoiceCloneStatusResponse) @@ -198,7 +193,6 @@ def retry_voice_clone( clone_id: str, authenticated_user: AuthenticatedUser = Depends(get_current_user), workflow: VoiceCloneWorkflowService = Depends(_get_workflow_service), - sign_url=Depends(get_audio_url_signer), ) -> VoiceCloneProfileResponse: """重试失败的音色克隆。 @@ -231,4 +225,4 @@ def retry_voice_clone( except Exception as inner_e: logger.error(f"Failed to mark profile as failed after dispatch error: {inner_e}") - return _to_response(profile, sign_url) + return _to_response(profile) diff --git a/apps/api/app/core/task_enqueue.py b/apps/api/app/core/task_enqueue.py index 0b3048d65..a4c5f361b 100755 --- a/apps/api/app/core/task_enqueue.py +++ b/apps/api/app/core/task_enqueue.py @@ -6,8 +6,8 @@ from app.core.celery_app import celery_app logger = logging.getLogger(__name__) # ── 限流阈值常量（全系统统一管理，不要在业务代码里硬编码） ── -USER_PENDING_LIMIT = 3 # 单用户 pending 上限 -GLOBAL_PENDING_LIMIT = 20 # 全局 pending 上限 +USER_PENDING_LIMIT = 3 # 单用户 pending 上限 +GLOBAL_PENDING_LIMIT = 20 # 全局 pending 上限 class UserPendingLimitExceeded(Exception): @@ -71,9 +71,7 @@ def check_queue_limits( user_pending, user_pending_limit, ) - raise UserPendingLimitExceeded( - user_id=user_id, pending_count=user_pending, limit=user_pending_limit - ) + raise UserPendingLimitExceeded(user_id=user_id, pending_count=user_pending, limit=user_pending_limit) def _mark_task_failed_safely( @@ -157,9 +155,7 @@ def safe_enqueue_generation_task( user_pending, user_pending_limit, ) - exc = UserPendingLimitExceeded( - user_id=user_id, pending_count=user_pending, limit=user_pending_limit - ) + exc = UserPendingLimitExceeded(user_id=user_id, pending_count=user_pending, limit=user_pending_limit) _mark_task_failed_safely(task, generation_task_repository, log_prefix, str(exc)) raise exc @@ -201,9 +197,7 @@ def safe_enqueue_generation_task( exc: Exception = GlobalQueueFull(pending_count=global_after, limit=global_pending_limit) else: reason = f"用户 pending 超限(入队后): {user_after}/{user_pending_limit}" - exc = UserPendingLimitExceeded( - user_id=user_id, pending_count=user_after, limit=user_pending_limit - ) + exc = UserPendingLimitExceeded(user_id=user_id, pending_count=user_after, limit=user_pending_limit) logger.warning( "[队列限流] %s, task_id=%s, user_id=%s — 回滚状态为 failed", diff --git a/pyproject.toml b/pyproject.toml old mode 100644 new mode 100755 index 2d06cc424..7a75f45dd --- a/pyproject.toml +++ b/pyproject.toml @@ -39,13 +39,12 @@ extend_skip_glob = [ ] [tool.coverage.run] -source = ["apps", "packages"] +source = ["apps/api/app", "packages"] omit = [ "*/migrations/*", "*/tests/*", "*/test_*.py", "*/site-packages/*", - "*/.cache/*", ] branch = true diff --git a/scripts/ci_coverage_summary.py b/scripts/ci_coverage_summary.py new file mode 100755 index 000000000..24c2ae6a3 --- /dev/null +++ b/scripts/ci_coverage_summary.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""解析 coverage.xml 并输出覆盖率汇总。""" + +import sys +import xml.etree.ElementTree as ET + +THRESHOLD = 65 # 行覆盖率门槛，百分比 + + +def main() -> int: + try: + tree = ET.parse("coverage.xml") + except FileNotFoundError: + print("coverage.xml 不存在，跳过汇总") + return 0 + + root = tree.getroot() + line_rate = float(root.get("line-rate", 0)) * 100 + branch_rate = float(root.get("branch-rate", 0)) * 100 + lines_covered = int(root.get("lines-covered", 0)) + lines_valid = int(root.get("lines-valid", 0)) + + print(f"行覆盖率: {line_rate:.2f}% ({lines_covered}/{lines_valid})") + print(f"分支覆盖率: {branch_rate:.2f}%") + print(f"门槛: {THRESHOLD}%") + status = "PASS ✅" if line_rate >= THRESHOLD else "FAIL ❌" + print(f"状态: {status}") + + return 0 if line_rate >= THRESHOLD else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci_notify_failure.py b/scripts/ci_notify_failure.py new file mode 100755 index 000000000..a8be3de03 --- /dev/null +++ b/scripts/ci_notify_failure.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""发送 CI 失败通知到飞书/项目群 webhook。""" + +import json +import os +import sys +import urllib.request + + +def main() -> int: + webhook = os.environ.get("CI_NOTIFY_WEBHOOK", "") + if not webhook: + print("未配置 CI_NOTIFY_WEBHOOK，跳过通知") + print("如需启用，请在仓库 Settings -> Secrets and variables -> Actions 中添加 CI_NOTIFY_WEBHOOK") + return 0 + + failed_job = os.environ.get("FAILED_JOB", "Unknown Job") + branch = os.environ.get("GITHUB_REF_NAME", "unknown") + commit = os.environ.get("GITHUB_SHA", "unknown")[:8] + actor = os.environ.get("GITHUB_ACTOR", "unknown") + run_id = os.environ.get("GITHUB_RUN_ID", "unknown") + repo = os.environ.get("GITHUB_REPOSITORY", "unknown") + run_url = f"https://git.xiaoxiajianji.com/{repo}/actions/runs/{run_id}" + + payload = { + "msg_type": "interactive", + "card": { + "header": { + "title": { + "tag": "plain_text", + "content": "❌ CI 构建失败", + }, + "status": "red", + }, + "elements": [ + { + "tag": "div", + "text": { + "tag": "lark_md", + "content": ( + f"**任务**: {failed_job}\n" + f"**分支**: {branch}\n" + f"**提交**: {commit}\n" + f"**提交者**: {actor}\n" + f"**Run ID**: {run_id}" + ), + }, + }, + { + "tag": "action", + "actions": [ + { + "tag": "button", + "text": {"tag": "plain_text", "content": "查看失败日志"}, + "url": run_url, + "type": "danger", + } + ], + }, + ], + }, + } + + data = json.dumps(payload).encode("utf-8") + req = urllib.request.Request( + webhook, + data=data, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + resp.read() + print("通知已发送") + except Exception as e: + print(f"通知发送失败: {e}", file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/integration/test_generation_api.py b/tests/integration/test_generation_api.py old mode 100644 new mode 100755 index 867985de4..5a609d7af --- a/tests/integration/test_generation_api.py +++ b/tests/integration/test_generation_api.py @@ -118,6 +118,18 @@ class StubGenerationTaskRepository: def count_by_user(self, user_id: str) -> int: return len([t for t in self._tasks.values() if t.created_by_user_id == user_id]) + def count_pending_by_user(self, user_id: str) -> int: + return len( + [ + t + for t in self._tasks.values() + if t.created_by_user_id == user_id and t.status == GenerationTaskStatus.PENDING + ] + ) + + def count_pending_total(self) -> int: + return len([t for t in self._tasks.values() if t.status == GenerationTaskStatus.PENDING]) + def list_recent_by_user(self, user_id: str, limit: int = 5) -> list[GenerationTask]: items = [t for t in self._tasks.values() if t.created_by_user_id == user_id] items.sort(key=lambda t: t.created_at, reverse=True) diff --git a/tests/integration/test_task_center_api.py b/tests/integration/test_task_center_api.py old mode 100644 new mode 100755 index ee491a64c..bec726186 --- a/tests/integration/test_task_center_api.py +++ b/tests/integration/test_task_center_api.py @@ -83,6 +83,18 @@ class StubGenerationTaskRepository: def count_by_user(self, user_id: str) -> int: return len([t for t in self._tasks.values() if t.created_by_user_id == user_id]) + def count_pending_by_user(self, user_id: str) -> int: + return len( + [ + t + for t in self._tasks.values() + if t.created_by_user_id == user_id and t.status == GenerationTaskStatus.PENDING + ] + ) + + def count_pending_total(self) -> int: + return len([t for t in self._tasks.values() if t.status == GenerationTaskStatus.PENDING]) + def list_recent_by_user(self, user_id: str, limit: int = 5) -> list[GenerationTask]: items = [t for t in self._tasks.values() if t.created_by_user_id == user_id] items.sort(key=lambda t: t.created_at, reverse=True) diff --git a/tests/unit/test_edit_plan_generation_api.py b/tests/unit/test_edit_plan_generation_api.py old mode 100644 new mode 100755 index 37b8601d4..b7ee3c545 --- a/tests/unit/test_edit_plan_generation_api.py +++ b/tests/unit/test_edit_plan_generation_api.py @@ -177,6 +177,18 @@ class StubGenerationTaskRepository: def count_by_user(self, user_id: str) -> int: return len([t for t in self._store.values() if t.created_by_user_id == user_id]) + def count_pending_by_user(self, user_id: str) -> int: + return len( + [ + t + for t in self._store.values() + if t.created_by_user_id == user_id and getattr(t, "status", "") == "pending" + ] + ) + + def count_pending_total(self) -> int: + return len([t for t in self._store.values() if getattr(t, "status", "") == "pending"]) + def list_recent_by_user(self, user_id: str, limit: int = 5) -> list[Any]: items = [t for t in self._store.values() if t.created_by_user_id == user_id] items.sort(key=lambda t: t.created_at, reverse=True) diff --git a/tests/unit/test_edit_plan_service.py b/tests/unit/test_edit_plan_service.py old mode 100644 new mode 100755 index 9388c8a40..eb9be6c2c --- a/tests/unit/test_edit_plan_service.py +++ b/tests/unit/test_edit_plan_service.py @@ -194,6 +194,12 @@ class StubGenerationTaskRepository: self._tasks[task.id] = task return task + def count_pending_by_user(self, user_id: str) -> int: + return 0 + + def count_pending_total(self) -> int: + return 0 + # --------------------------------------------------------------------------- # Service factory diff --git a/tests/unit/test_generation_presigned_url.py b/tests/unit/test_generation_presigned_url.py old mode 100644 new mode 100755 index 12caf3ce7..8a93983d9 --- a/tests/unit/test_generation_presigned_url.py +++ b/tests/unit/test_generation_presigned_url.py @@ -58,6 +58,12 @@ class StubGenerationTaskRepository: def get(self, task_id): return self._tasks.get(task_id) + def count_pending_by_user(self, user_id): + return 0 + + def count_pending_total(self): + return 0 + class StubGeneratedVideoRepository: def __init__(self, videos=None): diff --git a/tests/unit/test_task_queue_limit.py b/tests/unit/test_task_queue_limit.py old mode 100755 new mode 100644 index 6a906df57..dc4e56b41 --- a/tests/unit/test_task_queue_limit.py +++ b/tests/unit/test_task_queue_limit.py @@ -1,8 +1,9 @@ """任务队列限流防护单元测试。""" + from __future__ import annotations -import sys import os +import sys from unittest.mock import MagicMock import pytest @@ -18,7 +19,6 @@ from app.core.task_enqueue import ( safe_enqueue_generation_task, ) - # --------------------------------------------------------------------------- # Mock helpers # ---------------------------------------------------------------------------