diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index d58adc580..4e494df1e 100755 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -262,11 +262,61 @@ jobs: pytest --version ' + - name: Select incremental test files + if: github.event_name == 'pull_request' + shell: sh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set +e + PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||') + API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300" + CHANGED_FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin) if f['status'] != 'removed']") + echo "改动文件数: $(echo "$CHANGED_FILES" | grep -c . || echo 0)" + + CHANGED_FILES="$CHANGED_FILES" \ + SELECTED_TESTS_OUTPUT=/tmp/selected_tests.txt \ + python3 scripts/ci/select_unit_tests.py + SELECT_EXIT=$? + + if [ $SELECT_EXIT -eq 0 ]; then + echo "UNIT_TEST_MODE=incremental" >> $GITHUB_ENV + TEST_FILES=$(cat /tmp/selected_tests.txt | tr '\n' ' ') + echo "SELECTED_TEST_FILES=$TEST_FILES" >> $GITHUB_ENV + echo "增量模式: $(cat /tmp/selected_tests.txt | wc -l) 个测试文件" + else + echo "UNIT_TEST_MODE=full" >> $GITHUB_ENV + echo "SELECTED_TEST_FILES=tests/unit" >> $GITHUB_ENV + echo "全量模式" + fi + - name: Run unit tests with coverage shell: sh - run: "set -eu\nPYTHONPATH=\"$PWD/apps/api:$PWD\" python3 -m coverage run \\\n --source=apps/api/app,packages \\\n --omit=\"*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*\" \\\n --branch \\\n -m pytest tests/unit -q\npython3 -m coverage report --show-missing\npython3 -m coverage xml -o coverage.xml\npython3 -m coverage report --fail-under=65 > /dev/null\n" + run: | + set -eu + if [ "${UNIT_TEST_MODE:-full}" = "incremental" ]; then + echo "=== 增量测试模式 ===" + PYTHONPATH="$PWD/apps/api:$PWD" python3 -m coverage run \ + --source=apps/api/app,packages \ + --omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \ + --branch \ + -m pytest $SELECTED_TEST_FILES -q + python3 -m coverage report --show-missing + python3 -m coverage xml -o coverage.xml + # 增量模式下调低覆盖率门槛（跑的文件少覆盖率自然低，不做强校验） + python3 -m coverage report --fail-under=10 > /dev/null || true + else + PYTHONPATH="$PWD/apps/api:$PWD" python3 -m coverage run \ + --source=apps/api/app,packages \ + --omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \ + --branch \ + -m pytest tests/unit -q + python3 -m coverage report --show-missing + python3 -m coverage xml -o coverage.xml + python3 -m coverage report --fail-under=65 > /dev/null + fi - name: Diff coverage check (增量行覆盖率) - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' && env.HAS_APP_CHANGES == 'true' shell: sh env: GITHUB_TOKEN: ${{ github.token }} @@ -279,12 +329,25 @@ jobs: echo "Base branch: $BASE_BRANCH" # 初始化git (CI tarball checkout没有.git目录) + # 先备份PR代码，再基于base分支建分支，确保HEAD与base有共同祖先 + PR_CODE_DIR="/tmp/pr-code-$$" + mkdir -p "$PR_CODE_DIR" + # 排除隐藏文件（如.env）和后续生成的coverage文件，只备份源码 + find . -maxdepth 1 -mindepth 1 ! -name 'coverage.xml' ! -name 'diff_coverage.html' -exec cp -r {} "$PR_CODE_DIR/" \; + + rm -rf .git git init > /dev/null 2>&1 git remote add origin https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas.git > /dev/null 2>&1 git config user.email "ci@local" git config user.name "CI" # 拉取base分支用于对比 - git fetch origin $BASE_BRANCH --depth=100 + git fetch origin $BASE_BRANCH --depth=200 + # 基于base分支创建当前分支，确保有共同祖先 + git checkout -b ci-pr-branch "origin/$BASE_BRANCH" > /dev/null 2>&1 + # 清除base分支的源码，用PR代码覆盖 + find . -mindepth 1 -maxdepth 1 ! -name '.git' -exec rm -rf {} + + cp -r "$PR_CODE_DIR"/. . + rm -rf "$PR_CODE_DIR" # 提交当前代码 git add -A > /dev/null 2>&1 git commit -m "ci-tmp" > /dev/null 2>&1 @@ -301,7 +364,11 @@ jobs: # 运行diff-cover set +e - python3 -m diff_cover.diff_cover_tool coverage.xml --compare-branch="origin/$BASE_BRANCH" --fail-under=$THRESHOLD --html-report diff_coverage.html 2>&1 + python3 -m diff_cover.diff_cover_tool coverage.xml \ + --compare-branch="origin/$BASE_BRANCH" \ + --fail-under=$THRESHOLD \ + --html-report diff_coverage.html \ + 2>&1 DIFF_EXIT=$? set -e @@ -311,12 +378,12 @@ jobs: echo " 请为改动的代码添加单元测试后再提交" echo "" echo "=== 覆盖率报告 ===" - python3 -m diff_cover.diff_cover_tool coverage.xml --compare-branch="origin/$BASE_BRANCH" 2>&1 | tail -30 + python3 -m diff_cover.diff_cover_tool coverage.xml \ + --compare-branch="origin/$BASE_BRANCH" 2>&1 | tail -30 exit 1 fi echo "✅ 增量覆盖率达标" - - name: CI failure notification if: failure() shell: sh diff --git a/alembic/versions/042_add_storage_key_to_assets.py b/alembic/versions/042_add_storage_key_to_assets.py new file mode 100755 index 000000000..14d5a9c4f --- /dev/null +++ b/alembic/versions/042_add_storage_key_to_assets.py @@ -0,0 +1,29 @@ +"""add storage_key to assets + +Revision ID: 042_storage_key +Revises: 041_result_count +Create Date: 2026-07-17 18:10:00.000000 +""" + +from __future__ import annotations + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "042_storage_key" +down_revision = "041_result_count" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "assets", + sa.Column("storage_key", sa.String(500), nullable=False, server_default=""), + ) + + +def downgrade() -> None: + op.drop_column("assets", "storage_key") diff --git a/apps/worker/video_processing/render_adapter.py b/apps/worker/video_processing/render_adapter.py index 7323e94c8..7bf06ef4e 100755 --- a/apps/worker/video_processing/render_adapter.py +++ b/apps/worker/video_processing/render_adapter.py @@ -366,12 +366,12 @@ class RenderAdapter: failed_clip_ids: list[str] = [] seen_asset_ids: set[str] = set() - # 批量查询素材的 file_url（OSS 存储路径） + # 批量查询素材的 storage_key（OSS 存储路径） clip_asset_ids = [c.asset_id for c in clips if c.asset_id] asset_storage_map: dict[str, str] = {} if clip_asset_ids: assets = self._db.query(AssetModel).filter(AssetModel.id.in_(clip_asset_ids)).all() - asset_storage_map = {a.id: a.file_url for a in assets if a.file_url} + asset_storage_map = {a.id: a.storage_key for a in assets if a.storage_key} for clip in clips: asset_id = clip.asset_id @@ -468,8 +468,8 @@ class RenderAdapter: from packages.adapters.sqlalchemy_impl.models import AssetModel model = self._db.query(AssetModel).filter(AssetModel.id == asset_id).first() - if model and model.file_url: - storage_key = model.file_url + if model and model.storage_key: + storage_key = model.storage_key logger.info("[plan_id=%s] [BGM] 从素材库下载: asset_id=%s", plan_id, asset_id) ok = download_asset(storage_key, bgm_file) if ok and bgm_file.exists() and bgm_file.stat().st_size > 0: diff --git a/apps/worker/worker_app/tasks/edit_plan_generation.py b/apps/worker/worker_app/tasks/edit_plan_generation.py index bba9f5cdd..9fce84cc6 100755 --- a/apps/worker/worker_app/tasks/edit_plan_generation.py +++ b/apps/worker/worker_app/tasks/edit_plan_generation.py @@ -472,14 +472,14 @@ def render_edit_plan(self, plan_id: str) -> dict: rendered_clip_ids: list[str] = [] failed_clip_ids: list[str] = [] - # 预先批量查询所有素材的 storage_key（file_url） + # 预先批量查询所有素材的 storage_key from packages.adapters.sqlalchemy_impl.models import AssetModel clip_asset_ids = [c.asset_id for c in clips if c.asset_id] asset_storage_map: dict[str, str] = {} if clip_asset_ids: assets = db.query(AssetModel).filter(AssetModel.id.in_(clip_asset_ids)).all() - asset_storage_map = {a.id: a.file_url for a in assets if a.file_url} + asset_storage_map = {a.id: a.storage_key for a in assets if a.storage_key} for clip in clips: if not clip.asset_id: diff --git a/apps/worker/worker_app/tasks/generation.py b/apps/worker/worker_app/tasks/generation.py index c425608cd..a9f988d86 100755 --- a/apps/worker/worker_app/tasks/generation.py +++ b/apps/worker/worker_app/tasks/generation.py @@ -537,8 +537,8 @@ def _prepare_bgm_track( session = SessionLocal() try: model = session.query(AssetModel).filter(AssetModel.id == asset_id).first() - if model and model.file_url: - storage_key = model.file_url + if model and model.storage_key: + storage_key = model.storage_key logger.info("[task_id=%s] [BGM] 从素材库下载: asset_id=%s", task_id, asset_id) ok = download_asset(storage_key, bgm_file) if ok and bgm_file.exists() and bgm_file.stat().st_size > 0: diff --git a/docs/schema-metadata-snapshot.json b/docs/schema-metadata-snapshot.json index b81292898..51ac64bce 100644 --- a/docs/schema-metadata-snapshot.json +++ b/docs/schema-metadata-snapshot.json @@ -186,6 +186,14 @@ "type": "VARCHAR(1000)", "unique": false }, + { + "index": false, + "name": "storage_key", + "nullable": false, + "primary_key": false, + "type": "VARCHAR(500)", + "unique": false + }, { "index": false, "name": "thumbnail_url", diff --git a/packages/adapters/sqlalchemy_impl/asset_repository.py b/packages/adapters/sqlalchemy_impl/asset_repository.py index 801679ac3..a36cfa133 100755 --- a/packages/adapters/sqlalchemy_impl/asset_repository.py +++ b/packages/adapters/sqlalchemy_impl/asset_repository.py @@ -72,6 +72,7 @@ class SQLAlchemyAssetRepository: file_type=(asset.mime_type.split("/")[0] if "/" in asset.mime_type else asset.mime_type), file_size=asset.file_size, file_url=asset.storage_key, + storage_key=asset.storage_key, thumbnail_url=asset.thumbnail_url, duration=asset.duration, width=asset.width, @@ -100,6 +101,7 @@ class SQLAlchemyAssetRepository: model.name = asset.name model.file_size = asset.file_size model.file_url = asset.storage_key + model.storage_key = asset.storage_key model.thumbnail_url = asset.thumbnail_url model.duration = asset.duration model.width = asset.width @@ -299,7 +301,7 @@ class SQLAlchemyAssetRepository: project_id=model.project_id, library_id=model.asset_library_id, name=model.name, - storage_key=model.file_url, + storage_key=model.storage_key or model.file_url, mime_type=mime_type, file_size=int(model.file_size or 0), thumbnail_url=model.thumbnail_url, diff --git a/packages/adapters/sqlalchemy_impl/models.py b/packages/adapters/sqlalchemy_impl/models.py index a851eedde..c3d2c0446 100755 --- a/packages/adapters/sqlalchemy_impl/models.py +++ b/packages/adapters/sqlalchemy_impl/models.py @@ -75,6 +75,8 @@ class AssetModel(Base): file_size = Column(Integer, nullable=False) # file_url: 完整可访问的 URL，用于客户端直接访问文件 file_url = Column(String(1000), nullable=False) + # storage_key: OSS 存储键，用于内部下载上传 + storage_key = Column(String(500), nullable=False, default="", server_default="") thumbnail_url = Column(String(1000), nullable=True) duration = Column(Float, nullable=True) width = Column(Float, nullable=True) diff --git a/tests/unit/test_asset_repository_storage_key.py b/tests/unit/test_asset_repository_storage_key.py new file mode 100755 index 000000000..8b144fb9d --- /dev/null +++ b/tests/unit/test_asset_repository_storage_key.py @@ -0,0 +1,92 @@ +"""Unit tests for SQLAlchemyAssetRepository - storage_key field coverage.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "api")) + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from packages.adapters.sqlalchemy_impl.asset_repository import SQLAlchemyAssetRepository +from packages.adapters.sqlalchemy_impl.models import Base +from packages.domain import Asset, AssetStatus + + +def _repository(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine)() + return SQLAlchemyAssetRepository(session) + + +def test_asset_repository_create_preserves_storage_key(): + """create时storage_key字段正确写入数据库。""" + repository = _repository() + asset = Asset.create( + project_id="proj-1", + library_id="lib-1", + name="test_video.mp4", + storage_key="assets/proj-1/test_video.mp4", + mime_type="video/mp4", + file_size=102400, + ) + + repository.create(asset) + saved = repository.get(asset.id) + + assert saved is not None + assert saved.storage_key == "assets/proj-1/test_video.mp4" + + +def test_asset_repository_update_preserves_storage_key(): + """update时storage_key字段正确更新。""" + repository = _repository() + asset = Asset.create( + project_id="proj-1", + library_id="lib-1", + name="test_video.mp4", + storage_key="assets/proj-1/old_key.mp4", + mime_type="video/mp4", + ) + repository.create(asset) + + # 更新storage_key + asset.storage_key = "assets/proj-1/new_key.mp4" + repository.update(asset) + + saved = repository.get(asset.id) + assert saved.storage_key == "assets/proj-1/new_key.mp4" + + +def test_asset_repository_to_domain_fallback_to_file_url(): + """_to_domain中storage_key为空时fallback到file_url（兼容存量数据）。""" + from sqlalchemy.orm import sessionmaker + + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine)() + repository = SQLAlchemyAssetRepository(session) + + # 直接插入一条storage_key为空的记录（模拟存量数据） + from packages.adapters.sqlalchemy_impl.models import AssetModel + + model = AssetModel( + id="asset-legacy-001", + project_id="proj-1", + asset_library_id="lib-1", + name="legacy_asset.mp4", + file_type="video", + file_size=204800, + file_url="legacy/storage/key.mp4", # 老数据file_url存的是storage_key + storage_key="", # 新字段为空 + status="ready", + classification_status="pending", + uploaded_by_user_id="user-1", + ) + session.add(model) + session.commit() + + saved = repository.get("asset-legacy-001") + assert saved is not None + assert saved.storage_key == "legacy/storage/key.mp4" # fallback到file_url diff --git a/tests/unit/test_render_adapter.py b/tests/unit/test_render_adapter.py index e5f2ef79d..13a15fe00 100755 --- a/tests/unit/test_render_adapter.py +++ b/tests/unit/test_render_adapter.py @@ -77,7 +77,7 @@ def _make_adapter( Args: plan: 模拟的剪辑计划 clips: 模拟的片段列表 - asset_url_map: asset_id → file_url 映射，用于 mock assets 表查询 + asset_url_map: asset_id → storage_key 映射，用于 mock assets 表查询 Returns: (adapter, mock_plan_repo, mock_clip_repo) @@ -91,7 +91,7 @@ def _make_adapter( for aid, url in asset_url_map.items(): m = MagicMock() m.id = aid - m.file_url = url + m.storage_key = url mock_assets.append(m) mock_query.all.return_value = mock_assets mock_query.filter.return_value = mock_query @@ -521,17 +521,17 @@ class TestRenderPlan: class TestDownloadAssets: @staticmethod def _make_mock_db(asset_url_map: dict[str, str]): - """构造 mock db，根据 asset_id 返回对应的 AssetModel.file_url。""" + """构造 mock db，根据 asset_id 返回对应的 AssetModel.storage_key。""" mock_db = MagicMock() mock_query = MagicMock() def _fake_filter(query): # 模拟 .filter(AssetModel.id.in_([...])).all() mock_assets = [] - for asset_id, file_url in asset_url_map.items(): + for asset_id, storage_key in asset_url_map.items(): mock_asset = MagicMock() mock_asset.id = asset_id - mock_asset.file_url = file_url + mock_asset.storage_key = storage_key mock_assets.append(mock_asset) mock_query.all.return_value = mock_assets return mock_query @@ -565,7 +565,7 @@ class TestDownloadAssets: assert len(rendered_ids) == 2 assert len(failed_ids) == 0 assert mock_download.call_count == 2 - # 验证传给 download_asset 的是 file_url 而非 asset_id + # 验证传给 download_asset 的是 storage_key 而非 asset_id call_keys = [call[0][0] for call in mock_download.call_args_list] assert "https://bucket.oss.com/videos/key1.mp4" in call_keys assert "https://bucket.oss.com/videos/key2.mp4" in call_keys @@ -670,14 +670,14 @@ class TestDownloadAssets: assert "c2" in rendered_ids @patch("video_processing.render_adapter.download_asset") - def test_asset_without_file_url_skipped(self, mock_download, tmp_path): - """素材在assets表中无file_url时跳过下载，标记为失败。""" - # 构造返回 asset 但 file_url 为空 + def test_asset_without_storage_key_skipped(self, mock_download, tmp_path): + """素材在assets表中无storage_key时跳过下载，标记为失败。""" + # 构造返回 asset 但 storage_key 为空 mock_db = MagicMock() mock_query = MagicMock() mock_asset = MagicMock() mock_asset.id = "asset_no_url" - mock_asset.file_url = "" + mock_asset.storage_key = "" mock_query.all.return_value = [mock_asset] mock_query.filter.return_value = mock_query mock_db.query.return_value = mock_query