xiaoxia-saas/tests/unit/test_ingest_orphan_cleanup_1714.py

"""#1714 上传/转码链路（IngestJob + Asset）孤儿清理测试。

场景：worker 容器重启/进程 OOM 时，已 prefetch 的 transcode celery 消息丢失，
ingest_job 永久卡 pending/processing、asset 永久卡 processing/uploading。
"""

from __future__ import annotations

import os
import sys
from datetime import UTC, datetime, timedelta, timezone
from pathlib import Path

import pytest

os.environ.setdefault("JWT_SECRET_KEY", "unit-test-secret")
os.environ.setdefault("DATABASE_URL", "sqlite:///test_ingest_orphan.db")

sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "worker"))
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "apps" / "api"))
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))

from sqlalchemy import create_engine  # noqa: E402
from sqlalchemy.orm import sessionmaker  # noqa: E402

from packages.adapters.sqlalchemy_impl.models import AssetModel, Base, IngestJobModel  # noqa: E402
from packages.application.ingest_orphan_cleanup import (  # noqa: E402
    cleanup_orphan_processing_assets,
    cleanup_stale_ingest_jobs,
)


@pytest.fixture()
def session():
    engine = create_engine("sqlite:///:memory:")
    Base.metadata.create_all(bind=engine)
    Session = sessionmaker(bind=engine)
    db = Session()
    yield db
    db.close()


def _mk_job(session, *, status="processing", celery_task_id="cel-1", asset_id="a-1", minutes_ago=90):
    now = datetime.now(UTC)
    job = IngestJobModel(
        id=f"job-{minutes_ago}-{status}-{celery_task_id}",
        project_id="p-1",
        library_id="lib-1",
        storage_key="uploads/x/IMG_2285.MOV",
        status=status,
        asset_id=asset_id,
        celery_task_id=celery_task_id,
        created_at=now - timedelta(minutes=minutes_ago),
        updated_at=now - timedelta(minutes=minutes_ago),
    )
    session.add(job)
    session.commit()
    return job


def _mk_asset(session, *, id="a-1", status="processing", minutes_ago=90, file_size=0):
    now = datetime.now(UTC)
    asset = AssetModel(
        id=id,
        project_id="p-1",
        asset_library_id="lib-1",
        name="IMG_2285.MOV",
        file_type="video",
        file_size=file_size,
        file_url="https://example.com/x.mov",
        storage_key="uploads/x/IMG_2285.MOV",
        status=status,
        uploaded_by_user_id="u-1",
        created_at=now - timedelta(minutes=minutes_ago),
        updated_at=now - timedelta(minutes=minutes_ago),
    )
    session.add(asset)
    session.commit()
    return asset


class TestCleanupStaleIngestJobs:
    def test_stale_processing_job_marked_failed_and_asset_to_error(self, session):
        """processing 超 60 分钟 → job failed，关联 processing asset → error。"""
        _mk_asset(session, id="a-1", status="processing")
        _mk_job(session, status="processing", celery_task_id="cel-dead", asset_id="a-1", minutes_ago=90)

        items, asset_ids = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90)

        assert len(items) == 1
        assert items[0] == ("job-90-processing-cel-dead", "cel-dead")
        assert asset_ids == ["a-1"]
        db_job = session.query(IngestJobModel).one()
        assert db_job.status == "failed"
        assert "中断" in db_job.error_message
        db_asset = session.query(AssetModel).one()
        assert db_asset.status == "error"

    def test_stale_pending_job_marked_failed(self, session):
        """pending 超 90 分钟（从未被消费）→ job failed。"""
        _mk_asset(session, id="a-2", status="uploading")
        _mk_job(session, status="pending", celery_task_id="", asset_id="a-2", minutes_ago=120)

        items, asset_ids = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90)

        assert len(items) == 1
        assert items[0][1] == ""  # 无 celery task id
        assert session.query(IngestJobModel).one().status == "failed"
        assert session.query(AssetModel).one().status == "error"

    def test_recent_processing_job_not_touched(self, session):
        """processing 仅 10 分钟（正常转码中）→ 不误杀。"""
        _mk_asset(session, id="a-3", status="processing", minutes_ago=10)
        _mk_job(session, status="processing", celery_task_id="cel-live", asset_id="a-3", minutes_ago=10)

        items, asset_ids = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90)

        assert items == []
        assert asset_ids == []
        assert session.query(IngestJobModel).one().status == "processing"
        assert session.query(AssetModel).one().status == "processing"

    def test_recent_pending_job_not_touched(self, session):
        """pending 仅 30 分钟（队列积压排队中）→ 不误杀。"""
        _mk_job(session, status="pending", asset_id="", minutes_ago=30)

        items, _ = cleanup_stale_ingest_jobs(session, processing_timeout_minutes=60, pending_timeout_minutes=90)

        assert items == []
        assert session.query(IngestJobModel).one().status == "pending"

    def test_terminal_job_not_touched(self, session):
        """已 completed/failed 的 job 不动。"""
        _mk_job(session, status="completed", celery_task_id="", asset_id="", minutes_ago=999)
        _mk_job(session, status="failed", celery_task_id="", asset_id="", minutes_ago=999)

        items, _ = cleanup_stale_ingest_jobs(session)

        assert items == []
        statuses = sorted(j.status for j in session.query(IngestJobModel).all())
        assert statuses == ["completed", "failed"]

    def test_ready_asset_not_demoted(self, session):
        """关联 asset 已是 ready（转码其实成功了，仅 job 回写失败）→ 不降级为 error。"""
        _mk_asset(session, id="a-4", status="ready")
        _mk_job(session, status="processing", celery_task_id="cel-x", asset_id="a-4", minutes_ago=90)

        _, asset_ids = cleanup_stale_ingest_jobs(session)

        assert asset_ids == []  # ready 不动
        assert session.query(AssetModel).one().status == "ready"


class TestCleanupOrphanProcessingAssets:
    def test_orphan_asset_without_job_marked_error(self, session):
        """无 ingest_job 关联、created 超 120 分钟的 processing 占位 → error。"""
        _mk_asset(session, id="orphan-1", status="processing", minutes_ago=150)

        ids = cleanup_orphan_processing_assets(session, timeout_minutes=120)

        assert ids == ["orphan-1"]
        assert session.query(AssetModel).one().status == "error"

    def test_asset_with_active_job_not_touched(self, session):
        """有 processing job 关联的 asset 不由本函数处理（归 cleanup_stale_ingest_jobs）。"""
        _mk_asset(session, id="a-5", status="processing", minutes_ago=150)
        _mk_job(session, status="processing", asset_id="a-5", minutes_ago=150)

        ids = cleanup_orphan_processing_assets(session, timeout_minutes=120)

        assert ids == []
        assert session.query(AssetModel).one().status == "processing"

    def test_recent_orphan_asset_not_touched(self, session):
        """无 job 但才创建 30 分钟 → 可能 complete 刚建、job 派单中，不动。"""
        _mk_asset(session, id="orphan-2", status="processing", minutes_ago=30)

        ids = cleanup_orphan_processing_assets(session, timeout_minutes=120)

        assert ids == []
        assert session.query(AssetModel).one().status == "processing"


class TestRecoverStuckIngestJobsOnStartup:
    def test_stuck_processing_job_requeued(self, session):
        """processing 超 10 分钟 → 重置 pending 并重新 send_task，回写新 celery id。"""
        from types import SimpleNamespace

        from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup

        job = _mk_job(session, status="processing", celery_task_id="old-cel-1", asset_id="a-1", minutes_ago=30)

        sent = []

        def fake_send_task(name, args=None, **kw):
            sent.append((name, args))
            return SimpleNamespace(id="new-cel-9")

        updated_ids = []
        recovered = recover_stuck_ingest_jobs_on_startup(
            session,
            send_task=fake_send_task,
            update_celery_task_id=lambda jid, cid: updated_ids.append((jid, cid)),
            stuck_minutes=10,
        )

        assert recovered == 1
        assert sent == [("worker.ingest_asset", [job.id])]
        refreshed = session.query(IngestJobModel).filter_by(id=job.id).one()
        assert refreshed.status == "pending"
        assert refreshed.celery_task_id == "new-cel-9"
        assert updated_ids == [(job.id, "new-cel-9")]

    def test_recent_processing_job_not_touched(self, session):
        """processing 仅 5 分钟（正常转码中/部署交接窗口）→ 不抢。"""
        from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup

        _mk_job(session, status="processing", celery_task_id="live", asset_id="", minutes_ago=5)

        sent = []
        recovered = recover_stuck_ingest_jobs_on_startup(
            session,
            send_task=lambda *a, **k: sent.append(a),
            stuck_minutes=10,
        )

        assert recovered == 0
        assert sent == []
        assert session.query(IngestJobModel).one().status == "processing"

    def test_lock_not_acquired_skips(self, session):
        """未抢到分布式锁（另一 worker 正在恢复）→ 跳过。"""
        from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup

        _mk_job(session, status="processing", celery_task_id="x", asset_id="", minutes_ago=30)

        recovered = recover_stuck_ingest_jobs_on_startup(
            session,
            send_task=lambda *a, **k: None,
            lock_acquire=lambda: False,
            stuck_minutes=10,
        )

        assert recovered == 0
        assert session.query(IngestJobModel).one().status == "processing"

    def test_pending_and_terminal_not_requeued(self, session):
        """pending/已终态 job 不在恢复范围。"""
        from packages.application.ingest_orphan_cleanup import recover_stuck_ingest_jobs_on_startup

        _mk_job(session, status="pending", celery_task_id="", asset_id="", minutes_ago=60)
        _mk_job(session, status="failed", celery_task_id="", asset_id="", minutes_ago=60)

        recovered = recover_stuck_ingest_jobs_on_startup(
            session,
            send_task=lambda *a, **k: None,
            stuck_minutes=10,
        )

        assert recovered == 0
        statuses = sorted(j.status for j in session.query(IngestJobModel).all())
        assert statuses == ["failed", "pending"]