Files
xiaoxia-saas/packages/domain/entities.py
T
saas-backend-bot df99305dd6
CI/CD Pipeline / Check push changed paths (pull_request) Has been skipped
CI/CD Pipeline / Dedup Check - skip PR tests when covered by push pipeline (pull_request) Successful in 1s
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 2s
CI/CD Pipeline / Frontend Lint (pull_request) Has been skipped
CI/CD Pipeline / Frontend Unit Tests (pull_request) Has been skipped
CI/CD Pipeline / PR Build Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Staging Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging API Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging Web Image (pull_request) Has been skipped
CI/CD Pipeline / Retag skipped Staging Worker Image (pull_request) Has been skipped
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Has been skipped
CI/CD Pipeline / Staging E2E Tests (pull_request) Has been skipped
CI/CD Pipeline / Staging API Integration Tests (pull_request) Has been skipped
CI/CD Pipeline / ACR Image Cleanup (pull_request) Has been skipped
CI/CD Pipeline / PR Build API Image (pull_request) Successful in 29s
CI/CD Pipeline / PR Build Worker Image (pull_request) Successful in 29s
Preview Deploy / Deploy Preview Environment (pull_request) Successful in 49s
CI/CD Pipeline / Validate - Python (mypy + alembic) (pull_request) Successful in 1m39s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 1m44s
CI/CD Pipeline / Validate - Style (pull_request) Successful in 2m19s
AI Code Review / AI Code Review (pull_request) Failing after 2m52s
PR Automation / Auto Approve on CI Green (pull_request) Successful in 2m54s
CI/CD Pipeline / Validate - Security (pull_request) Successful in 4m11s
CI/CD Pipeline / Unit Tests (pull_request) Failing after 6m23s
CI/CD Pipeline / Build Production API Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production Web Image (pull_request) Has been skipped
CI/CD Pipeline / Build Production Worker Image (pull_request) Has been skipped
CI/CD Pipeline / CI Gate (pull_request) Failing after 1s
CI/CD Pipeline / Deploy Production (pull_request) Has been skipped
CI/CD Pipeline / Production Browser E2E (pull_request) Has been skipped
CI/CD Pipeline / Canary Release to Production (pull_request) Has been skipped
PR Automation / Auto Merge on CI Green + Approved (pull_request) Successful in 3m39s
feat(worker): celery 队列隔离 + 孤儿任务消息作废 (#1714)
问题:素材转码与视频生成共用 celery 默认队列、worker 单进程消费,
20+ 转码积压会把用户生成任务堵 40 分钟以上;孤儿清理把任务标 failed
后 Redis 队列消息未作废,消息被重投导致 failed→running 非法转换,
worker 打印 ERROR 后继续产出半成品。

队列隔离:
- 新增 packages/shared/celery_queues.py:generation/transcode/celery
  三队列与 task_routes(generate_video→generation;ingest_asset/
  classify_asset/duplication→transcode),apply_queue_settings()
- worker 入口改双进程:generation worker 独占队列并内嵌 beat
  (prefetch=1, GENERATION_CONCURRENCY 默认 2),transcode worker
  消费 transcode,celery(并发=总-2,最小 1),任一退出则整体终止
- compose/部署脚本/ps1 同步新增 GENERATION_CONCURRENCY 与健康检查

消息作废:
- 新增 packages/shared/celery_orphan_guard.py:终态守卫
  ensure_task_claimable、Redis 队列消息物理清理(JSON 信封解析,
  按业务 id + celery headers.id 双匹配,未命中 rpush 保序)、
  revoke_and_purge(control.revoke + 物理清队列双保险)
- 入队点(生成/上传/分片/重试)send_task 后持久化 celery_task_id
  到 generation_tasks/ingest_jobs(新列,067 迁移,失败仅 warning)
- generate_video/ingest_asset 执行前校验 DB 状态:终态直接 discarded
  不进业务逻辑;mark_processing 返回 False(非法转换)安全中止
- 孤儿/超时清理标 failed 时同时 revoke + 清队列消息
- pending 超时阈值 15→45 分钟,与 running 孤儿(20min)区分

测试:新增 22 个单测(路由表/真实 Redis 消息清理/终态守卫/
非法转换中止/标 failed 后消息不重投/入队持久化),全量
14301 passed;067 迁移隔离 DDL 验证 upgrade/downgrade 通过。
2026-09-05 19:07:47 +08:00

302 lines
9.5 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from __future__ import annotations
import sys
from dataclasses import dataclass, field
from datetime import datetime, timezone
if sys.version_info >= (3, 11):
from enum import StrEnum
else:
from enum import Enum
class StrEnum(str, Enum):
pass
from typing import Any
from uuid import uuid4
# 枚举统一从 classification 模块导入,消除重复定义
from packages.domain.classification import (
AssetLibraryKind,
ClassificationStatus,
IngestJobStatus,
)
@dataclass(slots=True)
class User:
id: str
email: str
display_name: str
username: str = ""
# 认证相关字段
password_hash: str = ""
email_verified: bool = False
email_verification_token: str | None = None
password_reset_token: str | None = None
password_reset_expires_at: datetime | None = None
last_login_at: datetime | None = None
last_login_ip: str | None = None
# 订阅相关字段 (移到 User 级别)
subscription_plan: str = "free" # free, pro, enterprise
subscription_status: str = "active" # active, cancelled, expired
subscription_expires_at: datetime | None = None
# 配额限制 (移到 User 级别)
max_projects: int = 3 # free: 3, pro: unlimited, enterprise: unlimited
max_storage_gb: int = 10 # free: 10, pro: 100, enterprise: 1000
used_storage_gb: float = 0.0
# 管理员标识
is_admin: bool = False
# 微信绑定
wechat_openid: str | None = None
wechat_unionid: str | None = None
# 手机号绑定
phone: str | None = None
phone_verified: bool = False
binding_completed_at: datetime | None = None
created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
@dataclass(slots=True)
class Project:
id: str
owner_user_id: str
name: str
description: str = ""
shared_users: list[str] = field(default_factory=list) # 被共享的用户 ID 列表
created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
@classmethod
def create(cls, owner_user_id: str, name: str, description: str = "") -> "Project":
clean_name = name.strip()
if not clean_name:
raise ValueError("项目名称不能为空")
return cls(
id=uuid4().hex,
owner_user_id=owner_user_id,
name=clean_name,
description=description.strip(),
shared_users=[],
)
def is_owner(self, user_id: str) -> bool:
"""检查用户是否为项目所有者"""
return self.owner_user_id == user_id
def is_shared_with(self, user_id: str) -> bool:
"""检查项目是否共享给指定用户"""
return user_id in self.shared_users
def can_access(self, user_id: str) -> bool:
"""检查用户是否有权限访问项目"""
return self.is_owner(user_id) or self.is_shared_with(user_id)
@dataclass(slots=True)
class AssetLibrary:
id: str
project_id: str
name: str
kind: AssetLibraryKind
asset_count: int = 0
total_size: int = 0
created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
updated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
@classmethod
def create(
cls,
project_id: str,
name: str,
kind: AssetLibraryKind,
) -> "AssetLibrary":
clean_name = name.strip()
if not clean_name:
raise ValueError("素材库名称不能为空")
return cls(
id=uuid4().hex,
project_id=project_id,
name=clean_name,
kind=kind,
asset_count=0,
total_size=0,
)
class AssetStatus(StrEnum):
UPLOADING = "uploading"
READY = "ready"
PROCESSING = "processing"
ERROR = "error"
DELETED = "deleted"
@classmethod
def _missing_(cls, value: object) -> "AssetStatus":
"""兼容历史数据,避免枚举转换失败导致500。
- uploaded → READY(早期版本用 uploaded 表示上传完成)
- 其他未知值 → READY(兜底,不阻塞业务)
"""
if isinstance(value, str):
normalized = value.strip().lower()
if normalized in ("uploaded", "success", "ok", "done", "complete"):
return cls.READY
if normalized in ("upload", "uploading_start", "upload_start"):
return cls.UPLOADING
if normalized in ("failed", "fail", "err"):
return cls.ERROR
if normalized in ("process", "processing", "running", "run"):
return cls.PROCESSING
return cls.READY
@dataclass(slots=True)
class Asset:
id: str
project_id: str
library_id: str
name: str
storage_key: str
mime_type: str
file_size: int = 0
thumbnail_url: str | None = None
duration: float | None = None
width: int | None = None
height: int | None = None
fps: float | None = None
codec: str | None = None
status: AssetStatus = AssetStatus.UPLOADING
classification_status: ClassificationStatus = ClassificationStatus.PENDING
quality_score: float | None = None
uploaded_by_user_id: str = ""
file_hash: str = ""
client_upload_id: str = ""
metadata: dict[str, Any] = field(default_factory=dict)
tag_ids: list[str] = field(default_factory=list)
created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
updated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
@property
def file_type(self) -> str:
"""文件类型(从 mime_type 推导,如 video/audio/image."""
if "/" in self.mime_type:
return self.mime_type.split("/")[0]
return self.mime_type
@classmethod
def create(
cls,
project_id: str,
library_id: str,
name: str,
storage_key: str,
mime_type: str,
metadata: dict[str, Any] | None = None,
*,
file_size: int = 0,
thumbnail_url: str | None = None,
duration: float | None = None,
width: int | None = None,
height: int | None = None,
fps: float | None = None,
codec: str | None = None,
status: AssetStatus = AssetStatus.UPLOADING,
classification_status: ClassificationStatus = ClassificationStatus.PENDING,
quality_score: float | None = None,
uploaded_by_user_id: str = "",
file_hash: str = "",
client_upload_id: str = "",
) -> "Asset":
clean_name = name.strip()
if not clean_name:
raise ValueError("素材名称不能为空")
if not storage_key.strip():
raise ValueError("storage_key 不能为空")
if not mime_type.strip():
raise ValueError("mime_type 不能为空")
return cls(
id=uuid4().hex,
project_id=project_id,
library_id=library_id,
name=clean_name,
storage_key=storage_key.strip(),
mime_type=mime_type.strip(),
file_size=file_size,
thumbnail_url=str(thumbnail_url) if thumbnail_url else None,
duration=duration,
width=width,
height=height,
fps=fps,
codec=codec,
status=status,
classification_status=classification_status,
quality_score=quality_score,
uploaded_by_user_id=uploaded_by_user_id.strip(),
file_hash=file_hash.strip(),
client_upload_id=client_upload_id.strip(),
metadata=metadata or {},
tag_ids=[],
)
def add_tag(self, tag_id: str) -> None:
"""添加标签 ID。空 ID 会被忽略,自动去重。"""
clean_id = tag_id.strip()
if not clean_id:
raise ValueError("标签 ID 不能为空")
if clean_id not in self.tag_ids:
self.tag_ids.append(clean_id)
self.updated_at = datetime.now(timezone.utc)
def remove_tag(self, tag_id: str) -> None:
"""删除标签 ID。如果标签不存在,不报错(幂等性)。"""
clean_id = tag_id.strip()
if clean_id in self.tag_ids:
self.tag_ids.remove(clean_id)
self.updated_at = datetime.now(timezone.utc)
@dataclass(slots=True)
class IngestJob:
id: str
project_id: str
library_id: str
storage_key: str
status: IngestJobStatus = IngestJobStatus.PENDING
error_message: str = ""
result_asset_id: str = ""
file_hash: str = ""
asset_id: str = ""
celery_task_id: str = ""
created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
updated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
@classmethod
def create(
cls,
project_id: str,
library_id: str,
storage_key: str,
file_hash: str = "",
asset_id: str = "",
celery_task_id: str = "",
) -> "IngestJob":
if not project_id.strip():
raise ValueError("project_id 不能为空")
if not library_id.strip():
raise ValueError("library_id 不能为空")
if not storage_key.strip():
raise ValueError("storage_key 不能为空")
return cls(
id=uuid4().hex,
project_id=project_id.strip(),
library_id=library_id.strip(),
storage_key=storage_key.strip(),
file_hash=file_hash.strip(),
asset_id=asset_id.strip(),
celery_task_id=celery_task_id.strip(),
)