xiaoxia/xiaoxia-saas
1
0
Fork 0
Code Issues 24 Pull Requests 5 Actions Packages Projects Releases 7 Wiki Activity

style: normalize python formatting gates

Browse Source
This commit is contained in:
Xiaoxia AI
2026-06-21 06:52:19 +08:00
parent 0809a079c5
commit bfbaddbd9a
129 changed files with 3024 additions and 2485 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/worker/tasks.py
+28 -18
View File
@@ -4,13 +4,22 @@ from datetime import datetime, timezone
from app.config import get_settings
from app.core.storage import get_minio_service
from packages.adapters.sqlalchemy_impl.asset_repository import SQLAlchemyAssetRepository
from packages.adapters.sqlalchemy_impl.generated_video_repository import (
SQLAlchemyGeneratedVideoRepository,
)
from packages.adapters.sqlalchemy_impl.generation_task_repository import (
SQLAlchemyGenerationTaskRepository,
)
from packages.adapters.sqlalchemy_impl.session import (
SessionLocal,
build_session_factory,
)
from packages.domain import GeneratedVideo, GenerationTaskStatus
from .celery_app import celery_app
from .video_processing import VideoProcessor
from packages.adapters.sqlalchemy_impl.session import SessionLocal, build_session_factory
from packages.adapters.sqlalchemy_impl.generation_task_repository import SQLAlchemyGenerationTaskRepository
from packages.adapters.sqlalchemy_impl.generated_video_repository import SQLAlchemyGeneratedVideoRepository
from packages.adapters.sqlalchemy_impl.asset_repository import SQLAlchemyAssetRepository
from packages.domain import GeneratedVideo, GenerationTaskStatus
settings = get_settings()
if SessionLocal is None:
@@ -21,7 +30,7 @@ if SessionLocal is None:
def generate_video(task_id: str) -> dict:
session = SessionLocal()
temp_dir = tempfile.mkdtemp()
try:
task_repo = SQLAlchemyGenerationTaskRepository(session)
video_repo = SQLAlchemyGeneratedVideoRepository(session)
@@ -44,7 +53,7 @@ def generate_video(task_id: str) -> dict:
assets = asset_repo.list_by_library(task.asset_library_id)
if not assets:
raise RuntimeError(f"No assets found in library {task.asset_library_id}")
task.progress = 20.0
task_repo.update(task)
session.commit()
@@ -53,13 +62,13 @@ def generate_video(task_id: str) -> dict:
video_assets = [a for a in assets if a.mime_type.startswith("video/")][:3]
if not video_assets:
raise RuntimeError("No video assets found")
local_paths = []
for i, asset in enumerate(video_assets):
local_path = os.path.join(temp_dir, f"input_{i}.mp4")
storage_service.download_file(asset.storage_key, local_path)
local_paths.append(local_path)
task.progress = 20.0 + (i + 1) * 10.0
task_repo.update(task)
session.commit()
@@ -68,18 +77,18 @@ def generate_video(task_id: str) -> dict:
processor = VideoProcessor(temp_dir=temp_dir)
output_filename = f"{task.id}.mp4"
output_path = os.path.join(temp_dir, output_filename)
task.progress = 50.0
task_repo.update(task)
session.commit()
result = processor.concatenate_videos(
input_paths=local_paths,
output_path=output_path,
resolution=(1920, 1080),
fps=25,
)
task.progress = 80.0
task_repo.update(task)
session.commit()
@@ -87,13 +96,13 @@ def generate_video(task_id: str) -> dict:
# 6. 上传到 MinIO
storage_key = f"workspaces/{task.workspace_id}/projects/{task.project_id}/generated/{task.id}/{output_filename}"
thumbnail_key = f"workspaces/{task.workspace_id}/projects/{task.project_id}/generated/{task.id}/thumbnail.jpg"
storage_service.upload_file(result.output_path, storage_key)
storage_service.upload_file(result.thumbnail_path, thumbnail_key)
file_url = storage_service.get_url(storage_key)
thumbnail_url = storage_service.get_url(thumbnail_key)
task.progress = 90.0
task_repo.update(task)
session.commit()
@@ -130,7 +139,7 @@ def generate_video(task_id: str) -> dict:
"duration": result.duration,
"file_size": result.file_size,
}
except Exception as error:
try:
task_repo = SQLAlchemyGenerationTaskRepository(session)
@@ -143,14 +152,15 @@ def generate_video(task_id: str) -> dict:
session.commit()
except Exception:
pass
return {"ok": False, "task_id": task_id, "error": str(error)}
finally:
session.close()
# 清理临时文件
try:
import shutil
shutil.rmtree(temp_dir, ignore_errors=True)
except:
pass
apps/worker/video_processing/__init__.py
+1
View File
@@ -1,6 +1,7 @@
"""
视频处理模块
"""
from .processor import VideoProcessor, VideoResult
__all__ = ["VideoProcessor", "VideoResult"]
apps/worker/video_processing/processor.py
+33 -33
View File
@@ -1,6 +1,7 @@
"""
视频处理核心类
"""
import os
import tempfile
from dataclasses import dataclass
@@ -13,6 +14,7 @@ import ffmpeg
@dataclass
class VideoResult:
"""视频生成结果"""
output_path: str
thumbnail_path: str
duration: float
@@ -24,16 +26,16 @@ class VideoResult:
class VideoProcessor:
"""视频处理器"""
def __init__(self, temp_dir: str = None):
"""
初始化视频处理器
Args:
temp_dir: 临时文件目录，默认使用系统临时目录
"""
self.temp_dir = temp_dir or tempfile.gettempdir()
def concatenate_videos(
self,
input_paths: List[str],
@@ -43,22 +45,22 @@ class VideoProcessor:
) -> VideoResult:
"""
拼接多个视频
Args:
input_paths: 输入视频路径列表
output_path: 输出视频路径
resolution: 输出分辨率 (width, height)
fps: 输出帧率
Returns:
VideoResult: 生成结果
"""
if not input_paths:
raise ValueError("input_paths cannot be empty")
# 确保输出目录存在
os.makedirs(os.path.dirname(output_path), exist_ok=True)
try:
# 创建临时文件列表
concat_file = os.path.join(self.temp_dir, f"concat_{os.getpid()}.txt")
@@ -66,12 +68,11 @@ class VideoProcessor:
for path in input_paths:
# FFmpeg concat demuxer 格式
f.write(f"file '{os.path.abspath(path)}'\n")
# 使用 FFmpeg 拼接视频
width, height = resolution
(
ffmpeg
.input(concat_file, format="concat", safe=0)
ffmpeg.input(concat_file, format="concat", safe=0)
.output(
output_path,
vcodec="libx264",
@@ -84,28 +85,28 @@ class VideoProcessor:
.overwrite_output()
.run(capture_stdout=True, capture_stderr=True)
)
# 清理临时文件
os.remove(concat_file)
# 获取视频元数据
probe = ffmpeg.probe(output_path)
video_info = next(s for s in probe["streams"] if s["codec_type"] == "video")
duration = float(probe["format"]["duration"])
width = int(video_info["width"])
height = int(video_info["height"])
# 计算帧率
fps_str = video_info.get("r_frame_rate", "25/1")
fps_parts = fps_str.split("/")
fps_value = float(fps_parts[0]) / float(fps_parts[1]) if len(fps_parts) == 2 else float(fps_parts[0])
file_size = os.path.getsize(output_path)
# 生成缩略图
thumbnail_path = self.generate_thumbnail(output_path)
return VideoResult(
output_path=output_path,
thumbnail_path=thumbnail_path,
@@ -115,11 +116,11 @@ class VideoProcessor:
fps=fps_value,
file_size=file_size,
)
except ffmpeg.Error as e:
stderr = e.stderr.decode() if e.stderr else ""
raise RuntimeError(f"FFmpeg error: {stderr}") from e
def generate_thumbnail(
self,
video_path: str,
@@ -128,55 +129,54 @@ class VideoProcessor:
) -> str:
"""
生成视频缩略图
Args:
video_path: 视频文件路径
timestamp: 截图时间点（秒）
output_path: 输出路径，默认为视频路径 + .jpg
Returns:
缩略图路径
"""
if output_path is None:
output_path = f"{os.path.splitext(video_path)[0]}_thumb.jpg"
try:
(
ffmpeg
.input(video_path, ss=timestamp)
ffmpeg.input(video_path, ss=timestamp)
.output(output_path, vframes=1, format="image2", vcodec="mjpeg")
.overwrite_output()
.run(capture_stdout=True, capture_stderr=True)
)
return output_path
except ffmpeg.Error as e:
stderr = e.stderr.decode() if e.stderr else ""
raise RuntimeError(f"FFmpeg thumbnail error: {stderr}") from e
def get_video_info(self, video_path: str) -> dict:
"""
获取视频信息
Args:
video_path: 视频文件路径
Returns:
视频元数据字典
"""
try:
probe = ffmpeg.probe(video_path)
video_info = next(s for s in probe["streams"] if s["codec_type"] == "video")
duration = float(probe["format"]["duration"])
width = int(video_info["width"])
height = int(video_info["height"])
fps_str = video_info.get("r_frame_rate", "25/1")
fps_parts = fps_str.split("/")
fps_value = float(fps_parts[0]) / float(fps_parts[1]) if len(fps_parts) == 2 else float(fps_parts[0])
return {
"duration": duration,
"width": width,
@@ -185,7 +185,7 @@ class VideoProcessor:
"codec": video_info.get("codec_name"),
"bitrate": int(probe["format"].get("bit_rate", 0)),
}
except ffmpeg.Error as e:
stderr = e.stderr.decode() if e.stderr else ""
raise RuntimeError(f"FFmpeg probe error: {stderr}") from e
apps/worker/worker_app/celery_app.py
-2
View File
@@ -1,8 +1,6 @@
from celery import Celery
from worker_app.core.config import get_settings
settings = get_settings()
celery_app = Celery(settings.worker_name)
celery_app.conf.broker_url = settings.broker_url
apps/worker/worker_app/core/config.py
+3 -2
View File
@@ -1,6 +1,7 @@
from pydantic_settings import BaseSettings, SettingsConfigDict
from typing import Optional
import os
from typing import Optional
from pydantic_settings import BaseSettings, SettingsConfigDict
class WorkerSettings(BaseSettings):
apps/worker/worker_app/db.py
+6 -1
View File
@@ -1,5 +1,10 @@
from worker_app.core.config import get_settings
from packages.adapters.sqlalchemy_impl import build_session_factory, ensure_database_exists, initialize_database
from packages.adapters.sqlalchemy_impl import (
build_session_factory,
ensure_database_exists,
initialize_database,
)
settings = get_settings()
ensure_database_exists(settings.database_url)
apps/worker/worker_app/tasks/classification.py
+16 -9
View File
@@ -1,14 +1,21 @@
from worker_app.celery_app import celery_app
from worker_app.db import SessionLocal
from packages.domain import AssetClassification, ClassificationJob, ClassificationJobStatus
from packages.adapters.sqlalchemy_impl.classification_job_repository import SQLAlchemyClassificationJobRepository
from packages.adapters.sqlalchemy_impl.classification_job_repository import (
SQLAlchemyClassificationJobRepository,
)
from packages.domain import (
AssetClassification,
ClassificationJob,
ClassificationJobStatus,
)
@celery_app.task(name="worker.classify_asset")
def classify_asset(job_id: str) -> dict:
"""
Classify asset task.
Steps:
1. Fetch ClassificationJob from repository
2. Fetch Asset from repository
@@ -20,31 +27,31 @@ def classify_asset(job_id: str) -> dict:
session = SessionLocal()
try:
job_repo = SQLAlchemyClassificationJobRepository(session)
job = job_repo.get(job_id)
if job is None:
return {"status": "failed", "error": "job not found"}
try:
# Update job status to PROCESSING
job.status = ClassificationJobStatus.PROCESSING
job_repo.update(job)
session.commit()
# Mock classification (in real implementation: use ML model, vision API, etc.)
# For now, randomly classify based on asset_id hash
asset_id_hash = sum(ord(c) for c in job.asset_id)
classifications = list(AssetClassification)
classification = classifications[asset_id_hash % len(classifications)]
confidence = 0.85
# Update job status to COMPLETED
job.status = ClassificationJobStatus.COMPLETED
job.classification = classification.value
job.confidence = confidence
job_repo.update(job)
session.commit()
return {
"status": "completed",
"job_id": job.id,
@@ -58,7 +65,7 @@ def classify_asset(job_id: str) -> dict:
job.error_message = str(e)
job_repo.update(job)
session.commit()
return {
"status": "failed",
"job_id": job.id,
apps/worker/worker_app/tasks/generation.py
+19 -6
View File
@@ -7,6 +7,8 @@ from pathlib import Path
from urllib.parse import urlparse
import oss2
from worker_app.celery_app import celery_app
from worker_app.db import SessionLocal
from packages.adapters.sqlalchemy_impl import (
SQLAlchemyAssetRepository,
@@ -14,8 +16,6 @@ from packages.adapters.sqlalchemy_impl import (
SQLAlchemyGenerationTaskRepository,
)
from packages.domain import GeneratedVideo, GenerationTaskStatus
from worker_app.celery_app import celery_app
from worker_app.db import SessionLocal
OUTPUT_WIDTH = 1280
OUTPUT_HEIGHT = 720
@@ -176,7 +176,11 @@ def generate_video(task_id: str) -> dict:
task = task_repo.get(task_id)
if task is None:
db.close()
return {"status": "failed", "error": "generation task not found", "task_id": task_id}
return {
"status": "failed",
"error": "generation task not found",
"task_id": task_id,
}
try:
task.status = GenerationTaskStatus.RUNNING
@@ -184,10 +188,14 @@ def generate_video(task_id: str) -> dict:
task.started_at = task.started_at or datetime.now(timezone.utc)
task_repo.update(task)
assets = [asset for asset in asset_repo.list_by_library(task.asset_library_id) if asset.mime_type.startswith("video")]
assets = [
asset for asset in asset_repo.list_by_library(task.asset_library_id) if asset.mime_type.startswith("video")
]
output_name = f"generated-{task.id}.mp4"
storage_key = f"generated/workspaces/{task.workspace_id}/projects/{task.project_id}/tasks/{task.id}/{output_name}"
storage_key = (
f"generated/workspaces/{task.workspace_id}/projects/{task.project_id}/tasks/{task.id}/{output_name}"
)
with tempfile.TemporaryDirectory(prefix="xiaoxia-generation-") as temp_dir:
temp_path = Path(temp_dir)
@@ -235,7 +243,12 @@ def generate_video(task_id: str) -> dict:
task.completed_at = datetime.now(timezone.utc)
task_repo.update(task)
return {"status": "completed", "task_id": task.id, "video_id": video.id, "file_url": file_url}
return {
"status": "completed",
"task_id": task.id,
"video_id": video.id,
"file_url": file_url,
}
except Exception as error:
task.status = GenerationTaskStatus.FAILED
task.error_message = str(error)
apps/worker/worker_app/tasks/ingest.py
+12 -8
View File
@@ -1,16 +1,20 @@
from datetime import datetime, timezone
from packages.adapters.sqlalchemy_impl import SQLAlchemyAssetRepository, SQLAlchemyIngestJobRepository
from packages.domain import Asset, IngestJobStatus
from worker_app.celery_app import celery_app
from worker_app.db import SessionLocal
from packages.adapters.sqlalchemy_impl import (
SQLAlchemyAssetRepository,
SQLAlchemyIngestJobRepository,
)
from packages.domain import Asset, IngestJobStatus
@celery_app.task(name="worker.ingest_asset")
def ingest_asset(job_id: str) -> dict:
"""
Ingest asset task.
Steps:
1. Fetch IngestJob from repository
2. Extract metadata from storage_key (placeholder: mock metadata)
@@ -25,13 +29,13 @@ def ingest_asset(job_id: str) -> dict:
job = job_repo.get(job_id)
if job is None:
return {"status": "failed", "error": "job not found"}
try:
# Update job status to PROCESSING
job.status = IngestJobStatus.PROCESSING
job.updated_at = datetime.now(timezone.utc)
job_repo.update(job)
# Mock metadata extraction (in real implementation: use ffprobe, Pillow, etc.)
mime_type = "video/mp4" if job.storage_key.endswith(".mp4") else "image/jpeg"
metadata = {
@@ -40,10 +44,10 @@ def ingest_asset(job_id: str) -> dict:
"height": 1080,
"size_bytes": 1024000,
}
# Extract filename from storage_key
filename = job.storage_key.split("/")[-1]
# Create Asset
asset = Asset.create(
workspace_id=job.workspace_id,
@@ -55,7 +59,7 @@ def ingest_asset(job_id: str) -> dict:
metadata=metadata,
)
asset_repo.create(asset)
# Update job status to COMPLETED
job.status = IngestJobStatus.COMPLETED
job.result_asset_id = asset.id