From 749ef7f55affd7cae74903158206f37b5ecd3370 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 18:32:05 +0800 Subject: [PATCH 001/112] chore: add production environment configuration system --- .env.example | 86 +++++++++++- .env.production | 85 ++++++++++++ .env.staging | 85 ++++++++++++ .gitignore | 5 +- apps/api/app/core/config.py | 126 +++++++++++++++++- apps/api/app/core/storage.py | 29 ++-- apps/api/main.py | 4 +- docs/CONFIGURATION.md | 247 +++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 9 files changed, 637 insertions(+), 31 deletions(-) create mode 100644 .env.production create mode 100644 .env.staging create mode 100644 docs/CONFIGURATION.md diff --git a/.env.example b/.env.example index 7a76657e6..be61a4a54 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,85 @@ -# Environment example -APP_ENV=development +# ======================= +# 应用基础配置 +# ======================= +APP_ENV=development # development / staging / production APP_NAME=xiaoxia-saas +APP_VERSION=0.1.0 +DEBUG=true + +# ======================= +# API 服务配置 +# ======================= API_HOST=0.0.0.0 API_PORT=8000 +API_PREFIX=/api/v1 + +# ======================= +# Web 前端配置 +# ======================= WEB_PORT=3000 -POSTGRES_URL=postgresql+psycopg://postgres:postgres@localhost:5432/xiaoxia_saas +WEB_URL=http://localhost:3000 + +# ======================= +# 数据库配置 +# ======================= +DATABASE_URL=postgresql+psycopg://postgres:postgres@localhost:5432/xiaoxia_saas +DATABASE_POOL_SIZE=20 +DATABASE_MAX_OVERFLOW=40 +DATABASE_POOL_TIMEOUT=30 +DATABASE_POOL_RECYCLE=3600 + +# ======================= +# Redis 配置 +# ======================= REDIS_URL=redis://localhost:6379/0 -OBJECT_STORAGE_PROVIDER=minio -OBJECT_STORAGE_ENDPOINT=http://localhost:9000 -OBJECT_STORAGE_BUCKET=xiaoxia-saas +REDIS_MAX_CONNECTIONS=50 + +# ======================= +# Celery Worker 配置 +# ======================= +CELERY_BROKER_URL=redis://localhost:6379/0 +CELERY_RESULT_BACKEND=redis://localhost:6379/1 +CELERY_WORKER_CONCURRENCY=4 +CELERY_WORKER_MAX_TASKS_PER_CHILD=1000 + +# ======================= +# MinIO 对象存储配置 +# ======================= +MINIO_ENDPOINT=localhost:9000 +MINIO_ACCESS_KEY=CHANGE_ME +MINIO_SECRET_KEY=CHANGE_ME +MINIO_BUCKET=xiaoxia-assets +MINIO_SECURE=false +MINIO_PUBLIC_URL=http://localhost:9000 + +# ======================= +# 日志配置 +# ======================= +LOG_LEVEL=INFO # DEBUG / INFO / WARNING / ERROR / CRITICAL +LOG_FORMAT=json # json / text +LOG_FILE=/var/log/xiaoxia-saas/app.log + +# ======================= +# CORS 配置 +# ======================= +CORS_ORIGINS=http://localhost:3000,http://localhost:8000 +CORS_ALLOW_CREDENTIALS=true + +# ======================= +# 文件上传限制 +# ======================= +MAX_UPLOAD_SIZE_MB=1000 +ALLOWED_FILE_TYPES=video/mp4,video/quicktime,video/x-msvideo,audio/mpeg,audio/wav,image/jpeg,image/png,image/gif + +# ======================= +# 安全配置 +# ======================= +SECRET_KEY=change-me-in-production-to-a-random-string-at-least-32-chars +ACCESS_TOKEN_EXPIRE_MINUTES=60 +REFRESH_TOKEN_EXPIRE_DAYS=7 + +# ======================= +# 监控与追踪(可选) +# ======================= +# SENTRY_DSN= +# PROMETHEUS_PORT=9090 diff --git a/.env.production b/.env.production new file mode 100644 index 000000000..2d2cd5b41 --- /dev/null +++ b/.env.production @@ -0,0 +1,85 @@ +# ======================= +# 生产环境配置 +# ======================= +APP_ENV=production +APP_NAME=xiaoxia-saas +APP_VERSION=0.1.0 +DEBUG=false + +# ======================= +# API 服务配置 +# ======================= +API_HOST=0.0.0.0 +API_PORT=8000 +API_PREFIX=/api/v1 + +# ======================= +# Web 前端配置 +# ======================= +WEB_PORT=3000 +WEB_URL=https://xiaoxiajianji.com + +# ======================= +# 数据库配置 +# ======================= +DATABASE_URL=postgresql+psycopg://postgres:CHANGE_ME@postgres:5432/xiaoxia_saas_production +DATABASE_POOL_SIZE=50 +DATABASE_MAX_OVERFLOW=100 +DATABASE_POOL_TIMEOUT=30 +DATABASE_POOL_RECYCLE=3600 + +# ======================= +# Redis 配置 +# ======================= +REDIS_URL=redis://redis:6379/0 +REDIS_MAX_CONNECTIONS=100 + +# ======================= +# Celery Worker 配置 +# ======================= +CELERY_BROKER_URL=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/1 +CELERY_WORKER_CONCURRENCY=8 +CELERY_WORKER_MAX_TASKS_PER_CHILD=1000 + +# ======================= +# MinIO 对象存储配置 +# ======================= +MINIO_ENDPOINT=47.98.113.167:9000 +MINIO_ACCESS_KEY=admin +MINIO_SECRET_KEY=xiaoxia2026 +MINIO_BUCKET=xiaoxia-assets +MINIO_SECURE=false +MINIO_PUBLIC_URL=http://47.98.113.167:9000 + +# ======================= +# 日志配置 +# ======================= +LOG_LEVEL=INFO +LOG_FORMAT=json +LOG_FILE=/var/log/xiaoxia-saas/app.log + +# ======================= +# CORS 配置 +# ======================= +CORS_ORIGINS=https://xiaoxiajianji.com,https://api.xiaoxiajianji.com +CORS_ALLOW_CREDENTIALS=true + +# ======================= +# 文件上传限制 +# ======================= +MAX_UPLOAD_SIZE_MB=2000 +ALLOWED_FILE_TYPES=video/mp4,video/quicktime,video/x-msvideo,audio/mpeg,audio/wav,image/jpeg,image/png,image/gif + +# ======================= +# 安全配置 +# ======================= +SECRET_KEY=CHANGE_ME_TO_RANDOM_STRING_AT_LEAST_32_CHARS_IN_PRODUCTION +ACCESS_TOKEN_EXPIRE_MINUTES=60 +REFRESH_TOKEN_EXPIRE_DAYS=7 + +# ======================= +# 监控与追踪(可选) +# ======================= +# SENTRY_DSN= +# PROMETHEUS_PORT=9090 diff --git a/.env.staging b/.env.staging new file mode 100644 index 000000000..d58e723c1 --- /dev/null +++ b/.env.staging @@ -0,0 +1,85 @@ +# ======================= +# Staging 环境配置 +# ======================= +APP_ENV=staging +APP_NAME=xiaoxia-saas +APP_VERSION=0.1.0 +DEBUG=true + +# ======================= +# API 服务配置 +# ======================= +API_HOST=0.0.0.0 +API_PORT=8000 +API_PREFIX=/api/v1 + +# ======================= +# Web 前端配置 +# ======================= +WEB_PORT=3000 +WEB_URL=http://47.98.113.167:3001 + +# ======================= +# 数据库配置 +# ======================= +DATABASE_URL=postgresql+psycopg://postgres:postgres@postgres:5432/xiaoxia_saas_staging +DATABASE_POOL_SIZE=20 +DATABASE_MAX_OVERFLOW=40 +DATABASE_POOL_TIMEOUT=30 +DATABASE_POOL_RECYCLE=3600 + +# ======================= +# Redis 配置 +# ======================= +REDIS_URL=redis://redis:6379/1 +REDIS_MAX_CONNECTIONS=50 + +# ======================= +# Celery Worker 配置 +# ======================= +CELERY_BROKER_URL=redis://redis:6379/1 +CELERY_RESULT_BACKEND=redis://redis:6379/2 +CELERY_WORKER_CONCURRENCY=4 +CELERY_WORKER_MAX_TASKS_PER_CHILD=1000 + +# ======================= +# MinIO 对象存储配置 +# ======================= +MINIO_ENDPOINT=47.98.113.167:9000 +MINIO_ACCESS_KEY=CHANGE_ME_STAGING +MINIO_SECRET_KEY=CHANGE_ME_STAGING +MINIO_BUCKET=xiaoxia-assets +MINIO_SECURE=false +MINIO_PUBLIC_URL=http://47.98.113.167:9000 + +# ======================= +# 日志配置 +# ======================= +LOG_LEVEL=DEBUG +LOG_FORMAT=json +LOG_FILE=/var/log/xiaoxia-saas/staging.log + +# ======================= +# CORS 配置 +# ======================= +CORS_ORIGINS=http://47.98.113.167:3001,http://47.98.113.167:8001 +CORS_ALLOW_CREDENTIALS=true + +# ======================= +# 文件上传限制 +# ======================= +MAX_UPLOAD_SIZE_MB=1000 +ALLOWED_FILE_TYPES=video/mp4,video/quicktime,video/x-msvideo,audio/mpeg,audio/wav,image/jpeg,image/png,image/gif + +# ======================= +# 安全配置 +# ======================= +SECRET_KEY=staging-secret-key-not-for-production +ACCESS_TOKEN_EXPIRE_MINUTES=60 +REFRESH_TOKEN_EXPIRE_DAYS=7 + +# ======================= +# 监控与追踪(可选) +# ======================= +# SENTRY_DSN= +# PROMETHEUS_PORT=9090 diff --git a/.gitignore b/.gitignore index 1e98cdac4..0b3d66442 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,11 @@ ruff_cache/ # Env / secrets .env -.env.* +.env.local +.env.development +.env.production # 生产环境配置不提交(包含敏感信息) !.env.example +!.env.staging # staging 配置可以提交 # OS / editor .DS_Store diff --git a/apps/api/app/core/config.py b/apps/api/app/core/config.py index bf2120b69..c0c31cc90 100644 --- a/apps/api/app/core/config.py +++ b/apps/api/app/core/config.py @@ -1,11 +1,125 @@ -from pydantic import BaseModel +from pydantic_settings import BaseSettings, SettingsConfigDict +from typing import Optional +import os -class AppSettings(BaseModel): - app_name: str = "xiaoxia-saas-api" - app_env: str = "development" - api_prefix: str = "/api" +class AppSettings(BaseSettings): + """应用配置""" + + # 基础配置 + app_name: str = "xiaoxia-saas" + app_env: str = "development" # development / staging / production + app_version: str = "0.1.0" + debug: bool = True + + # API 配置 + api_host: str = "0.0.0.0" + api_port: int = 8000 + api_prefix: str = "/api/v1" + + # 数据库配置 + database_url: str = "postgresql+psycopg://postgres:postgres@localhost:5432/xiaoxia_saas" + database_pool_size: int = 20 + database_max_overflow: int = 40 + database_pool_timeout: int = 30 + database_pool_recycle: int = 3600 + + # Redis 配置 + redis_url: str = "redis://localhost:6379/0" + redis_max_connections: int = 50 + + # Celery 配置 + celery_broker_url: str = "redis://localhost:6379/0" + celery_result_backend: str = "redis://localhost:6379/1" + celery_worker_concurrency: int = 4 + celery_worker_max_tasks_per_child: int = 1000 + + # MinIO 配置 + minio_endpoint: str = "localhost:9000" + minio_access_key: str = "admin" + minio_secret_key: str = "xiaoxia2026" + minio_bucket: str = "xiaoxia-assets" + minio_secure: bool = False + minio_public_url: str = "http://localhost:9000" + + # 日志配置 + log_level: str = "INFO" + log_format: str = "json" # json / text + log_file: Optional[str] = None + + # CORS 配置 + cors_origins: str = "http://localhost:3000,http://localhost:8000" + cors_allow_credentials: bool = True + + # 文件上传限制 + max_upload_size_mb: int = 1000 + allowed_file_types: str = "video/mp4,video/quicktime,video/x-msvideo,audio/mpeg,audio/wav,image/jpeg,image/png,image/gif" + + # 安全配置 + secret_key: str = "change-me-in-production" + access_token_expire_minutes: int = 60 + refresh_token_expire_days: int = 7 + + # 监控配置(可选) + sentry_dsn: Optional[str] = None + prometheus_port: Optional[int] = None + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=False, + extra="ignore", + ) + + @property + def cors_origins_list(self) -> list[str]: + """解析 CORS origins 为列表""" + return [origin.strip() for origin in self.cors_origins.split(",")] + + @property + def allowed_file_types_list(self) -> list[str]: + """解析允许的文件类型为列表""" + return [ft.strip() for ft in self.allowed_file_types.split(",")] + + @property + def is_production(self) -> bool: + """是否为生产环境""" + return self.app_env == "production" + + @property + def is_staging(self) -> bool: + """是否为 staging 环境""" + return self.app_env == "staging" + + @property + def is_development(self) -> bool: + """是否为开发环境""" + return self.app_env == "development" + + +# 全局配置实例 +_settings: Optional[AppSettings] = None def get_settings() -> AppSettings: - return AppSettings() + """获取配置实例(单例模式)""" + global _settings + if _settings is None: + # 根据环境加载不同的 .env 文件 + env = os.getenv("APP_ENV", "development") + env_file = f".env.{env}" if env != "development" else ".env" + + # 如果环境特定的配置文件存在,则使用它 + if os.path.exists(env_file): + _settings = AppSettings(_env_file=env_file) + else: + _settings = AppSettings() + + return _settings + + +def reload_settings(): + """重新加载配置(用于测试)""" + global _settings + _settings = None + return get_settings() diff --git a/apps/api/app/core/storage.py b/apps/api/app/core/storage.py index fe3803d3e..1442bfa83 100644 --- a/apps/api/app/core/storage.py +++ b/apps/api/app/core/storage.py @@ -3,27 +3,24 @@ from minio import Minio from minio.error import S3Error from typing import BinaryIO import os +from app.core.config import get_settings class MinIOService: """MinIO storage service.""" - def __init__( - self, - endpoint: str = "47.98.113.167:9000", - access_key: str = "admin", - secret_key: str = "xiaoxia2026", - bucket_name: str = "xiaoxia-assets", - secure: bool = False, - ): - """Initialize MinIO client.""" + def __init__(self): + """Initialize MinIO client from settings.""" + settings = get_settings() + self.client = Minio( - endpoint, - access_key=access_key, - secret_key=secret_key, - secure=secure, + settings.minio_endpoint, + access_key=settings.minio_access_key, + secret_key=settings.minio_secret_key, + secure=settings.minio_secure, ) - self.bucket_name = bucket_name + self.bucket_name = settings.minio_bucket + self.public_url = settings.minio_public_url self._ensure_bucket() def _ensure_bucket(self): @@ -81,14 +78,14 @@ class MinIOService: ) # Return public URL - return f"http://47.98.113.167:9000/{self.bucket_name}/{storage_key}" + return f"{self.public_url}/{self.bucket_name}/{storage_key}" except S3Error as e: raise Exception(f"Failed to upload file: {e}") def get_url(self, storage_key: str) -> str: """Get public URL for a storage key.""" - return f"http://47.98.113.167:9000/{self.bucket_name}/{storage_key}" + return f"{self.public_url}/{self.bucket_name}/{storage_key}" def delete_file(self, storage_key: str): """Delete file from MinIO.""" diff --git a/apps/api/main.py b/apps/api/main.py index c4dd5014c..55194ed4e 100644 --- a/apps/api/main.py +++ b/apps/api/main.py @@ -52,8 +52,8 @@ def create_app() -> FastAPI: # CORS middleware app.add_middleware( CORSMiddleware, - allow_origins=["*"], # TODO: Configure for production - allow_credentials=True, + allow_origins=settings.cors_origins_list, + allow_credentials=settings.cors_allow_credentials, allow_methods=["*"], allow_headers=["*"], ) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md new file mode 100644 index 000000000..6981badb6 --- /dev/null +++ b/docs/CONFIGURATION.md @@ -0,0 +1,247 @@ +# 环境配置指南 + +## 环境类型 + +小虾 SaaS 支持三种环境: + +- **Development(开发环境)**:本地开发使用 +- **Staging(测试环境)**:用于测试和预发布 +- **Production(生产环境)**:正式生产环境 + +--- + +## 配置文件 + +### 1. `.env.example` - 配置模板 +包含所有可配置项的示例,用于参考和初始化。 + +### 2. `.env` - 开发环境配置(本地) +本地开发时使用,不提交到 Git。 + +### 3. `.env.staging` - Staging 环境配置 +测试环境配置,可以提交到 Git。 + +### 4. `.env.production` - 生产环境配置 +生产环境配置,**敏感信息需要单独管理,不要提交到 Git**。 + +--- + +## 使用方法 + +### 方法 1:使用环境变量指定环境 + +```bash +# 开发环境(默认) +python -m uvicorn apps.api.main:app --reload + +# Staging 环境 +export APP_ENV=staging +python -m uvicorn apps.api.main:app --host 0.0.0.0 --port 8000 + +# 生产环境 +export APP_ENV=production +python -m uvicorn apps.api.main:app --host 0.0.0.0 --port 8000 +``` + +### 方法 2:直接指定配置文件 + +```bash +# 使用 .env.staging +cp .env.staging .env +python -m uvicorn apps.api.main:app + +# 使用 .env.production +cp .env.production .env +python -m uvicorn apps.api.main:app +``` + +--- + +## 配置项说明 + +### 基础配置 +- `APP_ENV`: 环境类型(development / staging / production) +- `APP_NAME`: 应用名称 +- `DEBUG`: 是否启用调试模式 + +### API 配置 +- `API_HOST`: API 监听地址 +- `API_PORT`: API 端口 +- `API_PREFIX`: API 路由前缀(如 /api/v1) + +### 数据库配置 +- `DATABASE_URL`: PostgreSQL 连接字符串 +- `DATABASE_POOL_SIZE`: 连接池大小 +- `DATABASE_MAX_OVERFLOW`: 最大溢出连接数 +- `DATABASE_POOL_TIMEOUT`: 连接池超时时间(秒) +- `DATABASE_POOL_RECYCLE`: 连接回收时间(秒) + +### Redis 配置 +- `REDIS_URL`: Redis 连接字符串 +- `REDIS_MAX_CONNECTIONS`: 最大连接数 + +### Celery 配置 +- `CELERY_BROKER_URL`: Celery broker URL +- `CELERY_RESULT_BACKEND`: Celery result backend URL +- `CELERY_WORKER_CONCURRENCY`: Worker 并发数 +- `CELERY_WORKER_MAX_TASKS_PER_CHILD`: 每个 worker 最大任务数 + +### MinIO 配置 +- `MINIO_ENDPOINT`: MinIO 服务地址 +- `MINIO_ACCESS_KEY`: 访问密钥 +- `MINIO_SECRET_KEY`: 私密密钥 +- `MINIO_BUCKET`: 存储桶名称 +- `MINIO_SECURE`: 是否使用 HTTPS +- `MINIO_PUBLIC_URL`: 公开访问 URL + +### 日志配置 +- `LOG_LEVEL`: 日志级别(DEBUG / INFO / WARNING / ERROR / CRITICAL) +- `LOG_FORMAT`: 日志格式(json / text) +- `LOG_FILE`: 日志文件路径 + +### CORS 配置 +- `CORS_ORIGINS`: 允许的跨域来源(逗号分隔) +- `CORS_ALLOW_CREDENTIALS`: 是否允许携带凭证 + +### 安全配置 +- `SECRET_KEY`: 应用密钥(用于 JWT 签名等) +- `ACCESS_TOKEN_EXPIRE_MINUTES`: 访问令牌过期时间 +- `REFRESH_TOKEN_EXPIRE_DAYS`: 刷新令牌过期天数 + +--- + +## 生产环境最佳实践 + +### 1. 不要硬编码敏感信息 +❌ 不要这样: +```python +DATABASE_URL = "postgresql://user:password@localhost/db" +``` + +✅ 应该这样: +```python +from app.core.config import get_settings +settings = get_settings() +database_url = settings.database_url +``` + +### 2. 使用环境变量覆盖 +生产环境的敏感配置应该通过环境变量注入: + +```bash +export DATABASE_URL="postgresql://user:secure_password@prod-db/db" +export SECRET_KEY="very-long-random-string" +export MINIO_SECRET_KEY="another-secure-key" +``` + +### 3. 使用密钥管理工具 +推荐使用: +- Docker Secrets +- Kubernetes Secrets +- AWS Secrets Manager +- HashiCorp Vault + +### 4. 生产环境配置检查清单 + +- [ ] `DEBUG=false` +- [ ] `SECRET_KEY` 使用随机字符串(至少 32 字符) +- [ ] 数据库使用强密码 +- [ ] MinIO 使用强密码 +- [ ] `CORS_ORIGINS` 只包含信任的域名 +- [ ] 日志级别设置为 `INFO` 或 `WARNING` +- [ ] 数据库连接池配置合理 +- [ ] 启用 HTTPS(`MINIO_SECURE=true`) + +--- + +## Docker 部署配置 + +### docker-compose.yml 中使用环境文件 + +```yaml +services: + api: + image: xiaoxia-saas-api:latest + env_file: + - .env.production + environment: + - DATABASE_URL=${DATABASE_URL} # 从环境变量覆盖 +``` + +### Gitea Actions 部署脚本 + +在 `.gitea/workflows/deploy.yml` 中: + +```yaml +- name: Deploy to production + run: | + # 复制环境配置 + cp .env.production /var/lib/xiaoxia-saas-production/.env + + # 启动服务 + cd /var/lib/xiaoxia-saas-production + docker-compose up -d +``` + +--- + +## 配置验证 + +### 查看当前配置 + +```python +from app.core.config import get_settings + +settings = get_settings() +print(f"Environment: {settings.app_env}") +print(f"Debug mode: {settings.debug}") +print(f"Database: {settings.database_url}") +print(f"Is production: {settings.is_production}") +``` + +### 配置测试脚本 + +```bash +# 测试开发环境配置 +python -c "from app.core.config import get_settings; s=get_settings(); print(s.app_env)" + +# 测试 staging 环境配置 +APP_ENV=staging python -c "from app.core.config import get_settings; s=get_settings(); print(s.app_env)" + +# 测试生产环境配置 +APP_ENV=production python -c "from app.core.config import get_settings; s=get_settings(); print(s.app_env)" +``` + +--- + +## 常见问题 + +### Q: 为什么我的配置没有生效? +A: 检查: +1. 环境变量 `APP_ENV` 是否正确设置 +2. 配置文件 `.env.{环境}` 是否存在 +3. 环境变量名是否正确(大小写不敏感) +4. 是否重启了服务 + +### Q: 如何在 Docker 中使用环境配置? +A: 使用 `env_file` 或 `environment` 指定配置: +```yaml +services: + api: + env_file: .env.production +``` + +### Q: 生产环境如何保护敏感配置? +A: +1. 不要提交 `.env.production` 到 Git +2. 使用 Docker Secrets 或 Kubernetes Secrets +3. 通过 CI/CD 注入环境变量 +4. 使用密钥管理工具(Vault, AWS Secrets Manager 等) + +--- + +## 相关文档 + +- [部署文档](DEPLOYMENT.md) +- [API 文档](API.md) +- [开发指南](DEVELOPMENT.md) diff --git a/requirements.txt b/requirements.txt index d43aa52a9..39df8b3c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ fastapi uvicorn pydantic +pydantic-settings celery redis sqlalchemy -- 2.54.0 From be4b30f4da82e73c5d2f01514b84b0a543f29433 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 18:40:56 +0800 Subject: [PATCH 002/112] chore: wire deployment to environment-based configuration --- .env.production | 4 +- .env.staging | 4 +- .gitea/workflows/deploy.yml | 95 +++++++++----------- apps/api/app/core/database.py | 32 ++++++- apps/api/app/db.py | 8 +- apps/worker/worker_app/core/config.py | 28 +++++- docs/DEPLOYMENT.md | 123 ++++++++++++-------------- infra/docker/compose.yml | 22 +++-- 8 files changed, 176 insertions(+), 140 deletions(-) diff --git a/.env.production b/.env.production index 2d2cd5b41..341ce6ed9 100644 --- a/.env.production +++ b/.env.production @@ -46,8 +46,8 @@ CELERY_WORKER_MAX_TASKS_PER_CHILD=1000 # MinIO 对象存储配置 # ======================= MINIO_ENDPOINT=47.98.113.167:9000 -MINIO_ACCESS_KEY=admin -MINIO_SECRET_KEY=xiaoxia2026 +MINIO_ACCESS_KEY=CHANGE_ME_PRODUCTION +MINIO_SECRET_KEY=CHANGE_ME_PRODUCTION MINIO_BUCKET=xiaoxia-assets MINIO_SECURE=false MINIO_PUBLIC_URL=http://47.98.113.167:9000 diff --git a/.env.staging b/.env.staging index d58e723c1..3775ad9c5 100644 --- a/.env.staging +++ b/.env.staging @@ -22,7 +22,7 @@ WEB_URL=http://47.98.113.167:3001 # ======================= # 数据库配置 # ======================= -DATABASE_URL=postgresql+psycopg://postgres:postgres@postgres:5432/xiaoxia_saas_staging +DATABASE_URL=postgresql+psycopg://postgres:CHANGE_ME_STAGING_DB_PASSWORD@postgres:5432/xiaoxia_saas_staging DATABASE_POOL_SIZE=20 DATABASE_MAX_OVERFLOW=40 DATABASE_POOL_TIMEOUT=30 @@ -74,7 +74,7 @@ ALLOWED_FILE_TYPES=video/mp4,video/quicktime,video/x-msvideo,audio/mpeg,audio/wa # ======================= # 安全配置 # ======================= -SECRET_KEY=staging-secret-key-not-for-production +SECRET_KEY=CHANGE_ME_STAGING_SECRET_KEY ACCESS_TOKEN_EXPIRE_MINUTES=60 REFRESH_TOKEN_EXPIRE_DAYS=7 diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index ccf8c8137..2b44b936f 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -52,26 +52,27 @@ jobs: # Load Docker images docker load < /tmp/api-image.tar.gz docker load < /tmp/worker-image.tar.gz - - # Tag as staging - docker tag xiaoxia-saas-api:${{ github.sha }} xiaoxia-saas-api:staging - docker tag xiaoxia-saas-worker:${{ github.sha }} xiaoxia-saas-worker:staging - - # Deploy to staging directory - cd /var/lib/xiaoxia-saas-staging || mkdir -p /var/lib/xiaoxia-saas-staging - - # Update docker-compose - cat > docker-compose.yml << 'COMPOSE' + + # Prepare staging directory + mkdir -p /var/lib/xiaoxia-saas-staging + cp .env.staging /var/lib/xiaoxia-saas-staging/.env + + # Tag as staging + docker tag xiaoxia-saas-api:${{ github.sha }} xiaoxia-saas-api:staging + docker tag xiaoxia-saas-worker:${{ github.sha }} xiaoxia-saas-worker:staging + + # Deploy with staging env + cd /var/lib/xiaoxia-saas-staging + cat > docker-compose.yml << 'COMPOSE' version: '3.9' services: api: image: xiaoxia-saas-api:staging restart: unless-stopped + env_file: + - ./.env ports: - "8001:8000" - environment: - - DATABASE_URL=postgresql://postgres:postgres@postgres:5432/xiaoxia_saas_staging - - REDIS_URL=redis://redis:6379/1 depends_on: - postgres - redis @@ -79,9 +80,8 @@ services: worker: image: xiaoxia-saas-worker:staging restart: unless-stopped - environment: - - DATABASE_URL=postgresql://postgres:postgres@postgres:5432/xiaoxia_saas_staging - - REDIS_URL=redis://redis:6379/1 + env_file: + - ./.env depends_on: - redis - postgres @@ -92,7 +92,7 @@ services: environment: POSTGRES_DB: xiaoxia_saas_staging POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_PASSWORD: CHANGE_ME_STAGING_DB_PASSWORD volumes: - postgres_staging_data:/var/lib/postgresql/data @@ -103,12 +103,9 @@ services: volumes: postgres_staging_data: COMPOSE - - # Start services - docker-compose up -d - - # Cleanup - rm -f /tmp/api-image.tar.gz /tmp/worker-image.tar.gz + + docker-compose up -d + rm -f /tmp/api-image.tar.gz /tmp/worker-image.tar.gz echo "✅ Staging deployment completed" @@ -126,33 +123,29 @@ COMPOSE # Load Docker images docker load < /tmp/api-image.tar.gz docker load < /tmp/worker-image.tar.gz - - # Tag as production version - docker tag xiaoxia-saas-api:${{ github.sha }} xiaoxia-saas-api:${{ github.ref_name }} - docker tag xiaoxia-saas-worker:${{ github.sha }} xiaoxia-saas-worker:${{ github.ref_name }} - docker tag xiaoxia-saas-api:${{ github.sha }} xiaoxia-saas-api:latest - docker tag xiaoxia-saas-worker:${{ github.sha }} xiaoxia-saas-worker:latest - - # Deploy to production directory - cd /var/lib/xiaoxia-saas-production || mkdir -p /var/lib/xiaoxia-saas-production - - # Backup current version - if [ -f docker-compose.yml ]; then - cp docker-compose.yml docker-compose.yml.backup - fi - - # Update docker-compose - cat > docker-compose.yml << 'COMPOSE' + + # Prepare production directory + mkdir -p /var/lib/xiaoxia-saas-production + cp .env.production /var/lib/xiaoxia-saas-production/.env + + # Tag as production version + docker tag xiaoxia-saas-api:${{ github.sha }} xiaoxia-saas-api:${{ github.ref_name }} + docker tag xiaoxia-saas-worker:${{ github.sha }} xiaoxia-saas-worker:${{ github.ref_name }} + docker tag xiaoxia-saas-api:${{ github.sha }} xiaoxia-saas-api:latest + docker tag xiaoxia-saas-worker:${{ github.sha }} xiaoxia-saas-worker:latest + + # Deploy with production env + cd /var/lib/xiaoxia-saas-production + cat > docker-compose.yml << 'COMPOSE' version: '3.9' services: api: image: xiaoxia-saas-api:latest restart: unless-stopped + env_file: + - ./.env ports: - "8000:8000" - environment: - - DATABASE_URL=postgresql://postgres:postgres@postgres:5432/xiaoxia_saas_production - - REDIS_URL=redis://redis:6379/0 depends_on: - postgres - redis @@ -160,9 +153,8 @@ services: worker: image: xiaoxia-saas-worker:latest restart: unless-stopped - environment: - - DATABASE_URL=postgresql://postgres:postgres@postgres:5432/xiaoxia_saas_production - - REDIS_URL=redis://redis:6379/0 + env_file: + - ./.env depends_on: - redis - postgres @@ -173,7 +165,7 @@ services: environment: POSTGRES_DB: xiaoxia_saas_production POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_PASSWORD: CHANGE_ME_PRODUCTION_DB_PASSWORD volumes: - postgres_production_data:/var/lib/postgresql/data @@ -184,11 +176,8 @@ services: volumes: postgres_production_data: COMPOSE - - # Start services - docker-compose up -d - - # Cleanup - rm -f /tmp/api-image.tar.gz /tmp/worker-image.tar.gz + + docker-compose up -d + rm -f /tmp/api-image.tar.gz /tmp/worker-image.tar.gz echo "✅ Production deployment completed: ${{ github.ref_name }}" diff --git a/apps/api/app/core/database.py b/apps/api/app/core/database.py index 84bf25731..5f6422aa2 100644 --- a/apps/api/app/core/database.py +++ b/apps/api/app/core/database.py @@ -1,9 +1,33 @@ -from pydantic import BaseModel +from pydantic_settings import BaseSettings, SettingsConfigDict +from typing import Optional +import os -class DatabaseSettings(BaseModel): - database_url: str = "postgresql://postgres:postgres@postgres:5432/xiaoxia_saas" +class DatabaseSettings(BaseSettings): + database_url: str = "postgresql+psycopg://postgres:postgres@postgres:5432/xiaoxia_saas" + pool_size: int = 20 + max_overflow: int = 40 + pool_timeout: int = 30 + pool_recycle: int = 3600 + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=False, + extra="ignore", + ) + + +_settings: Optional[DatabaseSettings] = None def get_database_settings() -> DatabaseSettings: - return DatabaseSettings() + global _settings + if _settings is None: + env = os.getenv("APP_ENV", "development") + env_file = f".env.{env}" if env != "development" else ".env" + if os.path.exists(env_file): + _settings = DatabaseSettings(_env_file=env_file) + else: + _settings = DatabaseSettings() + return _settings diff --git a/apps/api/app/db.py b/apps/api/app/db.py index 4e0d843da..32f6cb3d7 100644 --- a/apps/api/app/db.py +++ b/apps/api/app/db.py @@ -4,7 +4,13 @@ from sqlalchemy.orm import sessionmaker, Session from app.core.database import get_database_settings settings = get_database_settings() -engine = create_engine(settings.database_url) +engine = create_engine( + settings.database_url, + pool_size=settings.pool_size, + max_overflow=settings.max_overflow, + pool_timeout=settings.pool_timeout, + pool_recycle=settings.pool_recycle, +) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) diff --git a/apps/worker/worker_app/core/config.py b/apps/worker/worker_app/core/config.py index 789703fff..9b395ae1d 100644 --- a/apps/worker/worker_app/core/config.py +++ b/apps/worker/worker_app/core/config.py @@ -1,11 +1,33 @@ -from pydantic import BaseModel +from pydantic_settings import BaseSettings, SettingsConfigDict +from typing import Optional +import os -class WorkerSettings(BaseModel): +class WorkerSettings(BaseSettings): worker_name: str = "xiaoxia-saas-worker" broker_url: str = "redis://redis:6379/0" result_backend: str = "redis://redis:6379/1" + worker_concurrency: int = 4 + worker_max_tasks_per_child: int = 1000 + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=False, + extra="ignore", + ) + + +_settings: Optional[WorkerSettings] = None def get_settings() -> WorkerSettings: - return WorkerSettings() + global _settings + if _settings is None: + env = os.getenv("APP_ENV", "development") + env_file = f".env.{env}" if env != "development" else ".env" + if os.path.exists(env_file): + _settings = WorkerSettings(_env_file=env_file) + else: + _settings = WorkerSettings() + return _settings diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 12f0a1ac0..fccfcd414 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -1,89 +1,80 @@ -# 部署配置说明 +# 部署说明 -## 部署流程 +## 目标 -### Staging(测试环境) -- **触发条件**:push 到 main 分支 -- **部署地址**:http://47.98.113.167:8001 -- **数据库**:xiaoxia_saas_staging +统一采用环境文件驱动部署: -### Production(生产环境) -- **触发条件**:创建 tag (v*) -- **部署地址**:http://47.98.113.167:8000 -- **数据库**:xiaoxia_saas_production +- `infra/docker/compose.yml` 作为本地/开发 compose +- `.env.staging` 作为测试环境配置 +- `.env.production` 作为生产环境配置 -## 部署步骤 +--- -1. **构建 Docker 镜像** - - API 镜像:xiaoxia-saas-api - - Worker 镜像:xiaoxia-saas-worker +## 环境加载规则 -2. **传输镜像到服务器** - - 通过 SSH 上传到 /tmp/ +### 开发环境 +- 默认读取 `.env` +- 没有 `.env` 时回退到代码默认值 -3. **在服务器上部署** - - Staging: /var/lib/xiaoxia-saas-staging - - Production: /var/lib/xiaoxia-saas-production +### Staging +- 读取 `.env.staging` +- `APP_ENV=staging` +- 端口和数据库与生产隔离 -4. **启动服务** - - docker-compose up -d +### Production +- 读取 `.env.production` +- `APP_ENV=production` +- 必须替换 `CHANGE_ME` 项 -## SSH 密钥配置(必须) +--- -### 方案 A:使用现有 SSH 密钥(推荐) +## Docker Compose 约定 -当前 act_runner 运行在服务器本机,可以直接访问本地文件系统。 +### 开发/本地 +```bash +cd infra/docker +docker-compose up -d +``` -1. **配置 runner 使用本地部署** +### Staging +```bash +cp .env.staging .env +cd infra/docker +docker-compose up -d +``` -修改 deploy.yml,将 SSH 命令改为本地命令: -- 不使用 `ssh root@47.98.113.167` -- 直接执行本地命令 +### Production +```bash +cp .env.production .env +cd infra/docker +docker-compose up -d +``` -2. **或者配置 SSH localhost** +--- + +## Gitea Actions 部署建议 + +部署时建议先放置环境文件,再启动服务: ```bash -# 在服务器上配置 SSH localhost 免密登录 -ssh-keygen -t ed25519 -f ~/.ssh/id_deploy -N "" -cat ~/.ssh/id_deploy.pub >> ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys +cp .env.staging /var/lib/xiaoxia-saas-staging/.env +cd /var/lib/xiaoxia-saas-staging +docker-compose up -d ``` -### 方案 B:使用 Gitea Secrets +生产环境类似: -如果需要远程部署到其他服务器: - -1. 在 Gitea 仓库设置中添加 Secrets: - - `DEPLOY_SSH_KEY` - SSH 私钥 - - `DEPLOY_HOST` - 部署服务器地址 - - `DEPLOY_USER` - SSH 用户名 - -2. 修改 deploy.yml 使用 secrets: -```yaml -- name: Setup SSH - run: | - mkdir -p ~/.ssh - echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/id_deploy - chmod 600 ~/.ssh/id_deploy +```bash +cp .env.production /var/lib/xiaoxia-saas-production/.env +cd /var/lib/xiaoxia-saas-production +docker-compose up -d ``` -## 当前状态 +--- -✅ 部署脚本已完成 -⚠️ SSH 配置待完成 -⚠️ 需要选择方案 A 或 B +## 注意事项 -## 推荐方案 - -**方案 A(本地部署)** - 最简单,因为 runner 和服务在同一台机器上。 - -修改 deploy.yml: -- 去掉 `scp` 和 `ssh` 命令 -- 直接在本地执行 docker 命令 -- 路径直接使用本地路径 - -## 下一步 - -1. 选择部署方案(A 或 B) -2. 完成 SSH 配置 -3. 推送代码测试部署 +- `.env.production` 不要把真实密钥提交进仓库 +- `MINIO_ACCESS_KEY` / `MINIO_SECRET_KEY` 要在部署时注入 +- 数据库密码要通过密钥管理或服务器环境变量替换 +- 生产环境 `DEBUG=false` diff --git a/infra/docker/compose.yml b/infra/docker/compose.yml index b26001f64..802353bd8 100644 --- a/infra/docker/compose.yml +++ b/infra/docker/compose.yml @@ -5,7 +5,7 @@ services: working_dir: /app/apps/web command: sh -c "npm install && npm run dev" ports: - - "3000:3000" + - "${WEB_PORT:-3000}:3000" volumes: - ../..:/app depends_on: @@ -17,9 +17,11 @@ services: context: ../.. dockerfile: infra/docker/api.Dockerfile env_file: - - ../../.env.example + - ../../.env + environment: + APP_ENV: ${APP_ENV:-development} ports: - - "8000:8000" + - "${API_PORT:-8000}:8000" depends_on: - postgres - redis @@ -30,7 +32,9 @@ services: context: ../.. dockerfile: infra/docker/worker.Dockerfile env_file: - - ../../.env.example + - ../../.env + environment: + APP_ENV: ${APP_ENV:-development} depends_on: - redis - postgres @@ -38,18 +42,18 @@ services: postgres: image: postgres:16 environment: - POSTGRES_DB: xiaoxia_saas - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_DB: ${POSTGRES_DB:-xiaoxia_saas} + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} ports: - - "5432:5432" + - "${POSTGRES_PORT:-5432}:5432" volumes: - postgres_data:/var/lib/postgresql/data redis: image: redis:7 ports: - - "6379:6379" + - "${REDIS_PORT:-6379}:6379" volumes: postgres_data: -- 2.54.0 From 823faa4a5264d51f83cb345fa4a867437f96fc7d Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 18:52:33 +0800 Subject: [PATCH 003/112] fix: use domestic pip mirror for docker builds --- infra/docker/api.Dockerfile | 4 +++- infra/docker/worker.Dockerfile | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/infra/docker/api.Dockerfile b/infra/docker/api.Dockerfile index 319ad6bb3..8014494b1 100644 --- a/infra/docker/api.Dockerfile +++ b/infra/docker/api.Dockerfile @@ -1,8 +1,10 @@ FROM python:3.12-slim WORKDIR /app ENV PYTHONPATH=/app +ENV PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple +ENV PIP_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn COPY requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir --default-timeout=120 --retries 10 -r requirements.txt COPY apps/api /app/apps/api COPY packages /app/packages WORKDIR /app/apps/api diff --git a/infra/docker/worker.Dockerfile b/infra/docker/worker.Dockerfile index 1e5be66d1..ca3402c8d 100644 --- a/infra/docker/worker.Dockerfile +++ b/infra/docker/worker.Dockerfile @@ -1,8 +1,10 @@ FROM python:3.12-slim WORKDIR /app ENV PYTHONPATH=/app +ENV PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple +ENV PIP_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn COPY requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir --default-timeout=120 --retries 10 -r requirements.txt COPY apps/worker /app/apps/worker COPY packages /app/packages WORKDIR /app/apps/worker -- 2.54.0 From 35f9ff1d82b5aa8fd55024a81a30f4b5fe650336 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 18:53:37 +0800 Subject: [PATCH 004/112] fix: switch docker pip mirror to aliyun --- infra/docker/api.Dockerfile | 7 ++++--- infra/docker/worker.Dockerfile | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/infra/docker/api.Dockerfile b/infra/docker/api.Dockerfile index 8014494b1..ef7e466e2 100644 --- a/infra/docker/api.Dockerfile +++ b/infra/docker/api.Dockerfile @@ -1,10 +1,11 @@ FROM python:3.12-slim WORKDIR /app ENV PYTHONPATH=/app -ENV PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple -ENV PIP_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn +ENV PIP_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/ +ENV PIP_TRUSTED_HOST=mirrors.aliyun.com COPY requirements.txt ./ -RUN pip install --no-cache-dir --default-timeout=120 --retries 10 -r requirements.txt +RUN python -m pip install --upgrade pip setuptools wheel && \ + pip install --no-cache-dir --default-timeout=120 --retries 10 -r requirements.txt COPY apps/api /app/apps/api COPY packages /app/packages WORKDIR /app/apps/api diff --git a/infra/docker/worker.Dockerfile b/infra/docker/worker.Dockerfile index ca3402c8d..18e0e15b9 100644 --- a/infra/docker/worker.Dockerfile +++ b/infra/docker/worker.Dockerfile @@ -1,10 +1,11 @@ FROM python:3.12-slim WORKDIR /app ENV PYTHONPATH=/app -ENV PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple -ENV PIP_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn +ENV PIP_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/ +ENV PIP_TRUSTED_HOST=mirrors.aliyun.com COPY requirements.txt ./ -RUN pip install --no-cache-dir --default-timeout=120 --retries 10 -r requirements.txt +RUN python -m pip install --upgrade pip setuptools wheel && \ + pip install --no-cache-dir --default-timeout=120 --retries 10 -r requirements.txt COPY apps/worker /app/apps/worker COPY packages /app/packages WORKDIR /app/apps/worker -- 2.54.0 From 6782f79df038c58dc27a75a69afb350464b56394 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 18:57:50 +0800 Subject: [PATCH 005/112] fix: remove stale upload request schema export --- apps/api/app/schemas/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/api/app/schemas/__init__.py b/apps/api/app/schemas/__init__.py index 2a45f7a8a..3bf648c71 100644 --- a/apps/api/app/schemas/__init__.py +++ b/apps/api/app/schemas/__init__.py @@ -5,7 +5,7 @@ from .asset_library import AssetLibraryResponse, CreateAssetLibraryRequest, List from .health import HealthResponse from .ingest_job import IngestJobResponse, SubmitIngestJobRequest from .project import CreateProjectRequest, ListProjectsResponse, ProjectResponse -from .upload import UploadAssetRequest, UploadAssetResponse +from .upload import UploadAssetResponse __all__ = [ "AssetResponse", @@ -20,6 +20,5 @@ __all__ = [ "ListProjectsResponse", "ProjectResponse", "SubmitIngestJobRequest", - "UploadAssetRequest", "UploadAssetResponse", ] -- 2.54.0 From 02000b9889164197a95a99f659e8942f0a9ce3e6 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 18:59:58 +0800 Subject: [PATCH 006/112] fix: decouple api from worker package imports --- apps/api/app/api/routes/ingest_jobs.py | 8 ++++---- apps/api/app/api/routes/upload.py | 6 ++---- apps/api/app/core/celery_app.py | 9 +++++++++ 3 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 apps/api/app/core/celery_app.py diff --git a/apps/api/app/api/routes/ingest_jobs.py b/apps/api/app/api/routes/ingest_jobs.py index 54fab8fb1..0e34ac45d 100644 --- a/apps/api/app/api/routes/ingest_jobs.py +++ b/apps/api/app/api/routes/ingest_jobs.py @@ -1,10 +1,10 @@ from fastapi import APIRouter, Depends +from app.core.celery_app import celery_app from app.dependencies import get_ingest_job_repository from app.schemas.ingest_job import IngestJobResponse, SubmitIngestJobRequest from packages.adapters.in_memory import InMemoryIngestJobRepository from packages.application import SubmitIngestJobCommand, SubmitIngestJobUseCase -from apps.worker.worker_app.tasks.ingest import ingest_asset router = APIRouter() @@ -23,9 +23,9 @@ def submit_ingest_job( storage_key=request.storage_key, ) ) - # Enqueue async worker task - ingest_asset.delay(job.id) - + + celery_app.send_task("worker.ingest_asset", args=[job.id]) + return IngestJobResponse( id=job.id, workspace_id=job.workspace_id, diff --git a/apps/api/app/api/routes/upload.py b/apps/api/app/api/routes/upload.py index 844f718cf..e6ba2d5b2 100644 --- a/apps/api/app/api/routes/upload.py +++ b/apps/api/app/api/routes/upload.py @@ -1,13 +1,12 @@ from fastapi import APIRouter, Depends, UploadFile, File, Form from uuid import uuid4 -from typing import Optional +from app.core.celery_app import celery_app from app.dependencies import get_ingest_job_repository from app.core.storage import get_minio_service, MinIOService from app.schemas.upload import UploadAssetResponse from packages.adapters.in_memory import InMemoryIngestJobRepository from packages.application import SubmitIngestJobCommand, SubmitIngestJobUseCase -from apps.worker.worker_app.tasks.ingest import ingest_asset router = APIRouter() @@ -85,8 +84,7 @@ async def upload_asset( ) ) - # Enqueue async worker task - ingest_asset.delay(job.id) + celery_app.send_task("worker.ingest_asset", args=[job.id]) return UploadAssetResponse( storage_key=storage_key, diff --git a/apps/api/app/core/celery_app.py b/apps/api/app/core/celery_app.py new file mode 100644 index 000000000..6647df637 --- /dev/null +++ b/apps/api/app/core/celery_app.py @@ -0,0 +1,9 @@ +from celery import Celery + +from app.core.config import get_settings + + +settings = get_settings() +celery_app = Celery("xiaoxia-saas-api") +celery_app.conf.broker_url = settings.celery_broker_url +celery_app.conf.result_backend = settings.celery_result_backend -- 2.54.0 From 7f416700de738165d2533b1cc5ffd25e8b1b0ee4 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 19:14:54 +0800 Subject: [PATCH 007/112] fix: register worker tasks explicitly --- apps/worker/worker_app/celery_app.py | 5 +++++ apps/worker/worker_app/tasks/health.py | 17 ----------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/apps/worker/worker_app/celery_app.py b/apps/worker/worker_app/celery_app.py index f5a7e88d0..661dcb962 100644 --- a/apps/worker/worker_app/celery_app.py +++ b/apps/worker/worker_app/celery_app.py @@ -7,3 +7,8 @@ settings = get_settings() celery_app = Celery(settings.worker_name) celery_app.conf.broker_url = settings.broker_url celery_app.conf.result_backend = settings.result_backend +celery_app.conf.imports = ( + "worker_app.tasks.health", + "worker_app.tasks.ingest", + "worker_app.tasks.classification", +) diff --git a/apps/worker/worker_app/tasks/health.py b/apps/worker/worker_app/tasks/health.py index 741fb8079..5162bede1 100644 --- a/apps/worker/worker_app/tasks/health.py +++ b/apps/worker/worker_app/tasks/health.py @@ -4,20 +4,3 @@ from worker_app.celery_app import celery_app @celery_app.task(name="worker.healthcheck") def healthcheck() -> dict: return {"ok": True, "service": "worker"} - - -@celery_app.task(name="worker.ingest_asset") -def ingest_asset(job_id: str, workspace_id: str, project_id: str, library_id: str, storage_key: str) -> dict: - """ - Ingest asset task placeholder. - Real implementation would: - 1. Fetch file from storage_key - 2. Extract metadata (duration, resolution, etc.) - 3. Create Asset entity - 4. Update IngestJob status - """ - return { - "job_id": job_id, - "status": "completed", - "asset_id": "placeholder-asset-id", - } -- 2.54.0 From 99e72cdcba88ed775376c146e6cbb71d500c78cb Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 19:34:47 +0800 Subject: [PATCH 008/112] feat: persist ingest flow in postgres --- apps/api/app/api/routes/asset_libraries.py | 6 +-- apps/api/app/api/routes/assets.py | 6 +-- apps/api/app/api/routes/ingest_jobs.py | 4 +- apps/api/app/api/routes/projects.py | 6 +-- apps/api/app/api/routes/upload.py | 6 +-- apps/api/app/db.py | 13 +++--- apps/api/app/dependencies.py | 34 +++++++------- apps/worker/worker_app/core/config.py | 5 ++ apps/worker/worker_app/db.py | 12 +++++ apps/worker/worker_app/tasks/ingest.py | 29 ++++++------ packages/adapters/sqlalchemy_impl/__init__.py | 17 +++++++ .../asset_library_repository.py | 16 +++---- .../sqlalchemy_impl/asset_repository.py | 23 +++++----- .../sqlalchemy_impl/ingest_job_repository.py | 2 +- .../sqlalchemy_impl/project_repository.py | 14 +++--- packages/adapters/sqlalchemy_impl/session.py | 46 +++++++++++++++++++ 16 files changed, 161 insertions(+), 78 deletions(-) create mode 100644 apps/worker/worker_app/db.py create mode 100644 packages/adapters/sqlalchemy_impl/session.py diff --git a/apps/api/app/api/routes/asset_libraries.py b/apps/api/app/api/routes/asset_libraries.py index f26e6394a..ef1cfbcb4 100644 --- a/apps/api/app/api/routes/asset_libraries.py +++ b/apps/api/app/api/routes/asset_libraries.py @@ -2,7 +2,7 @@ from fastapi import APIRouter, Depends from app.dependencies import get_asset_library_repository from app.schemas.asset_library import AssetLibraryResponse, CreateAssetLibraryRequest, ListAssetLibrariesResponse -from packages.adapters.in_memory import InMemoryAssetLibraryRepository +from packages.adapters.sqlalchemy_impl import SQLAlchemyAssetLibraryRepository from packages.application import CreateAssetLibraryCommand, CreateAssetLibraryUseCase, ListAssetLibrariesUseCase from packages.domain import AssetLibraryKind @@ -13,7 +13,7 @@ router = APIRouter() def list_asset_libraries( project_id: str, kind: str | None = None, - asset_library_repository: InMemoryAssetLibraryRepository = Depends(get_asset_library_repository), + asset_library_repository: SQLAlchemyAssetLibraryRepository = Depends(get_asset_library_repository), ) -> ListAssetLibrariesResponse: use_case = ListAssetLibrariesUseCase(asset_library_repository) parsed_kind = AssetLibraryKind(kind) if kind else None @@ -35,7 +35,7 @@ def list_asset_libraries( @router.post("", response_model=AssetLibraryResponse) def create_asset_library( request: CreateAssetLibraryRequest, - asset_library_repository: InMemoryAssetLibraryRepository = Depends(get_asset_library_repository), + asset_library_repository: SQLAlchemyAssetLibraryRepository = Depends(get_asset_library_repository), ) -> AssetLibraryResponse: use_case = CreateAssetLibraryUseCase(asset_library_repository) item = use_case.execute( diff --git a/apps/api/app/api/routes/assets.py b/apps/api/app/api/routes/assets.py index e222729aa..95a22cd74 100644 --- a/apps/api/app/api/routes/assets.py +++ b/apps/api/app/api/routes/assets.py @@ -2,7 +2,7 @@ from fastapi import APIRouter, Depends from app.dependencies import get_asset_repository from app.schemas.asset import AssetResponse, CreateAssetRequest, ListAssetsResponse -from packages.adapters.in_memory import InMemoryAssetRepository +from packages.adapters.sqlalchemy_impl import SQLAlchemyAssetRepository from packages.application import CreateAssetCommand, CreateAssetUseCase, ListAssetsUseCase router = APIRouter() @@ -11,7 +11,7 @@ router = APIRouter() @router.get("", response_model=ListAssetsResponse) def list_assets( library_id: str, - asset_repository: InMemoryAssetRepository = Depends(get_asset_repository), + asset_repository: SQLAlchemyAssetRepository = Depends(get_asset_repository), ) -> ListAssetsResponse: use_case = ListAssetsUseCase(asset_repository) items = use_case.execute(library_id) @@ -35,7 +35,7 @@ def list_assets( @router.post("", response_model=AssetResponse) def create_asset( request: CreateAssetRequest, - asset_repository: InMemoryAssetRepository = Depends(get_asset_repository), + asset_repository: SQLAlchemyAssetRepository = Depends(get_asset_repository), ) -> AssetResponse: use_case = CreateAssetUseCase(asset_repository) item = use_case.execute( diff --git a/apps/api/app/api/routes/ingest_jobs.py b/apps/api/app/api/routes/ingest_jobs.py index 0e34ac45d..04ad38749 100644 --- a/apps/api/app/api/routes/ingest_jobs.py +++ b/apps/api/app/api/routes/ingest_jobs.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends from app.core.celery_app import celery_app from app.dependencies import get_ingest_job_repository from app.schemas.ingest_job import IngestJobResponse, SubmitIngestJobRequest -from packages.adapters.in_memory import InMemoryIngestJobRepository +from packages.adapters.sqlalchemy_impl import SQLAlchemyIngestJobRepository from packages.application import SubmitIngestJobCommand, SubmitIngestJobUseCase router = APIRouter() @@ -12,7 +12,7 @@ router = APIRouter() @router.post("", response_model=IngestJobResponse) def submit_ingest_job( request: SubmitIngestJobRequest, - ingest_job_repository: InMemoryIngestJobRepository = Depends(get_ingest_job_repository), + ingest_job_repository: SQLAlchemyIngestJobRepository = Depends(get_ingest_job_repository), ) -> IngestJobResponse: use_case = SubmitIngestJobUseCase(ingest_job_repository) job = use_case.execute( diff --git a/apps/api/app/api/routes/projects.py b/apps/api/app/api/routes/projects.py index 8af550475..245f6254d 100644 --- a/apps/api/app/api/routes/projects.py +++ b/apps/api/app/api/routes/projects.py @@ -2,8 +2,8 @@ from fastapi import APIRouter, Depends from app.dependencies import get_project_repository from app.schemas.project import CreateProjectRequest, ListProjectsResponse, ProjectResponse +from packages.adapters.sqlalchemy_impl import SQLAlchemyProjectRepository from packages.application import CreateProjectCommand, CreateProjectUseCase, ListProjectsUseCase -from packages.adapters.in_memory import InMemoryProjectRepository router = APIRouter() @@ -11,7 +11,7 @@ router = APIRouter() @router.get("", response_model=ListProjectsResponse) def list_projects( workspace_id: str, - project_repository: InMemoryProjectRepository = Depends(get_project_repository), + project_repository: SQLAlchemyProjectRepository = Depends(get_project_repository), ) -> ListProjectsResponse: use_case = ListProjectsUseCase(project_repository) projects = use_case.execute(workspace_id) @@ -31,7 +31,7 @@ def list_projects( @router.post("", response_model=ProjectResponse) def create_project( request: CreateProjectRequest, - project_repository: InMemoryProjectRepository = Depends(get_project_repository), + project_repository: SQLAlchemyProjectRepository = Depends(get_project_repository), ) -> ProjectResponse: use_case = CreateProjectUseCase(project_repository) project = use_case.execute( diff --git a/apps/api/app/api/routes/upload.py b/apps/api/app/api/routes/upload.py index e6ba2d5b2..c0a3d2820 100644 --- a/apps/api/app/api/routes/upload.py +++ b/apps/api/app/api/routes/upload.py @@ -3,9 +3,9 @@ from uuid import uuid4 from app.core.celery_app import celery_app from app.dependencies import get_ingest_job_repository -from app.core.storage import get_minio_service, MinIOService +from app.core.storage import MinIOService, get_minio_service from app.schemas.upload import UploadAssetResponse -from packages.adapters.in_memory import InMemoryIngestJobRepository +from packages.adapters.sqlalchemy_impl import SQLAlchemyIngestJobRepository from packages.application import SubmitIngestJobCommand, SubmitIngestJobUseCase router = APIRouter() @@ -17,7 +17,7 @@ async def upload_asset( workspace_id: str = Form(..., description="工作空间 ID"), project_id: str = Form(..., description="项目 ID"), library_id: str = Form(..., description="资产库 ID"), - ingest_job_repository: InMemoryIngestJobRepository = Depends(get_ingest_job_repository), + ingest_job_repository: SQLAlchemyIngestJobRepository = Depends(get_ingest_job_repository), storage_service: MinIOService = Depends(get_minio_service), ) -> UploadAssetResponse: """ diff --git a/apps/api/app/db.py b/apps/api/app/db.py index 32f6cb3d7..93980d99f 100644 --- a/apps/api/app/db.py +++ b/apps/api/app/db.py @@ -1,21 +1,22 @@ -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker, Session +from collections.abc import Generator + +from sqlalchemy.orm import Session from app.core.database import get_database_settings +from packages.adapters.sqlalchemy_impl import build_session_factory, initialize_database settings = get_database_settings() -engine = create_engine( +engine, SessionLocal = build_session_factory( settings.database_url, pool_size=settings.pool_size, max_overflow=settings.max_overflow, pool_timeout=settings.pool_timeout, pool_recycle=settings.pool_recycle, ) -SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +initialize_database(engine) -def get_db() -> Session: - """Dependency for database session.""" +def get_db() -> Generator[Session, None, None]: db = SessionLocal() try: yield db diff --git a/apps/api/app/dependencies.py b/apps/api/app/dependencies.py index 63f583736..dcb94dba2 100644 --- a/apps/api/app/dependencies.py +++ b/apps/api/app/dependencies.py @@ -1,28 +1,26 @@ -from functools import lru_cache +from fastapi import Depends +from sqlalchemy.orm import Session -from packages.adapters.in_memory import ( - InMemoryAssetLibraryRepository, - InMemoryAssetRepository, - InMemoryIngestJobRepository, - InMemoryProjectRepository, +from app.db import get_db +from packages.adapters.sqlalchemy_impl import ( + SQLAlchemyAssetLibraryRepository, + SQLAlchemyAssetRepository, + SQLAlchemyIngestJobRepository, + SQLAlchemyProjectRepository, ) -@lru_cache(maxsize=1) -def get_project_repository() -> InMemoryProjectRepository: - return InMemoryProjectRepository() +def get_project_repository(db: Session = Depends(get_db)) -> SQLAlchemyProjectRepository: + return SQLAlchemyProjectRepository(db) -@lru_cache(maxsize=1) -def get_asset_library_repository() -> InMemoryAssetLibraryRepository: - return InMemoryAssetLibraryRepository() +def get_asset_library_repository(db: Session = Depends(get_db)) -> SQLAlchemyAssetLibraryRepository: + return SQLAlchemyAssetLibraryRepository(db) -@lru_cache(maxsize=1) -def get_asset_repository() -> InMemoryAssetRepository: - return InMemoryAssetRepository() +def get_asset_repository(db: Session = Depends(get_db)) -> SQLAlchemyAssetRepository: + return SQLAlchemyAssetRepository(db) -@lru_cache(maxsize=1) -def get_ingest_job_repository() -> InMemoryIngestJobRepository: - return InMemoryIngestJobRepository() +def get_ingest_job_repository(db: Session = Depends(get_db)) -> SQLAlchemyIngestJobRepository: + return SQLAlchemyIngestJobRepository(db) diff --git a/apps/worker/worker_app/core/config.py b/apps/worker/worker_app/core/config.py index 9b395ae1d..a79244bdd 100644 --- a/apps/worker/worker_app/core/config.py +++ b/apps/worker/worker_app/core/config.py @@ -9,6 +9,11 @@ class WorkerSettings(BaseSettings): result_backend: str = "redis://redis:6379/1" worker_concurrency: int = 4 worker_max_tasks_per_child: int = 1000 + database_url: str = "postgresql+psycopg://postgres:postgres@postgres:5432/xiaoxia_saas" + database_pool_size: int = 20 + database_max_overflow: int = 40 + database_pool_timeout: int = 30 + database_pool_recycle: int = 3600 model_config = SettingsConfigDict( env_file=".env", diff --git a/apps/worker/worker_app/db.py b/apps/worker/worker_app/db.py new file mode 100644 index 000000000..ca58fe311 --- /dev/null +++ b/apps/worker/worker_app/db.py @@ -0,0 +1,12 @@ +from worker_app.core.config import get_settings +from packages.adapters.sqlalchemy_impl import build_session_factory, initialize_database + +settings = get_settings() +engine, SessionLocal = build_session_factory( + settings.database_url, + pool_size=settings.database_pool_size, + max_overflow=settings.database_max_overflow, + pool_timeout=settings.database_pool_timeout, + pool_recycle=settings.database_pool_recycle, +) +initialize_database(engine) diff --git a/apps/worker/worker_app/tasks/ingest.py b/apps/worker/worker_app/tasks/ingest.py index 007f94e5f..a49bf3e22 100644 --- a/apps/worker/worker_app/tasks/ingest.py +++ b/apps/worker/worker_app/tasks/ingest.py @@ -1,11 +1,9 @@ +from datetime import datetime, timezone + +from packages.adapters.sqlalchemy_impl import SQLAlchemyAssetRepository, SQLAlchemyIngestJobRepository +from packages.domain import Asset, IngestJobStatus from worker_app.celery_app import celery_app -from packages.domain import Asset, IngestJob, IngestJobStatus -from packages.adapters.in_memory import InMemoryAssetRepository, InMemoryIngestJobRepository - - -@celery_app.task(name="worker.healthcheck") -def healthcheck() -> dict: - return {"ok": True, "service": "worker"} +from worker_app.db import SessionLocal @celery_app.task(name="worker.ingest_asset") @@ -20,10 +18,10 @@ def ingest_asset(job_id: str) -> dict: 4. Update IngestJob status to COMPLETED 5. Return result """ - # TODO: Replace with real repository injection - job_repo = InMemoryIngestJobRepository() - asset_repo = InMemoryAssetRepository() - + db = SessionLocal() + job_repo = SQLAlchemyIngestJobRepository(db) + asset_repo = SQLAlchemyAssetRepository(db) + job = job_repo.get(job_id) if job is None: return {"status": "failed", "error": "job not found"} @@ -31,6 +29,7 @@ def ingest_asset(job_id: str) -> dict: try: # Update job status to PROCESSING job.status = IngestJobStatus.PROCESSING + job.updated_at = datetime.now(timezone.utc) job_repo.update(job) # Mock metadata extraction (in real implementation: use ffprobe, Pillow, etc.) @@ -60,8 +59,9 @@ def ingest_asset(job_id: str) -> dict: # Update job status to COMPLETED job.status = IngestJobStatus.COMPLETED job.result_asset_id = asset.id + job.updated_at = datetime.now(timezone.utc) job_repo.update(job) - + return { "status": "completed", "job_id": job.id, @@ -71,10 +71,13 @@ def ingest_asset(job_id: str) -> dict: # Update job status to FAILED job.status = IngestJobStatus.FAILED job.error_message = str(e) + job.updated_at = datetime.now(timezone.utc) job_repo.update(job) - + return { "status": "failed", "job_id": job.id, "error": str(e), } + finally: + db.close() diff --git a/packages/adapters/sqlalchemy_impl/__init__.py b/packages/adapters/sqlalchemy_impl/__init__.py index 60e96fbbb..53a79cc54 100644 --- a/packages/adapters/sqlalchemy_impl/__init__.py +++ b/packages/adapters/sqlalchemy_impl/__init__.py @@ -1 +1,18 @@ """SQLAlchemy-based repository implementations.""" + +from .asset_library_repository import SQLAlchemyAssetLibraryRepository +from .asset_repository import SQLAlchemyAssetRepository +from .ingest_job_repository import SQLAlchemyIngestJobRepository +from .project_repository import SQLAlchemyProjectRepository +from .session import Base, build_engine, build_session_factory, initialize_database + +__all__ = [ + "Base", + "SQLAlchemyAssetLibraryRepository", + "SQLAlchemyAssetRepository", + "SQLAlchemyIngestJobRepository", + "SQLAlchemyProjectRepository", + "build_engine", + "build_session_factory", + "initialize_database", +] diff --git a/packages/adapters/sqlalchemy_impl/asset_library_repository.py b/packages/adapters/sqlalchemy_impl/asset_library_repository.py index a4bc0d7a0..c4e25afb0 100644 --- a/packages/adapters/sqlalchemy_impl/asset_library_repository.py +++ b/packages/adapters/sqlalchemy_impl/asset_library_repository.py @@ -1,7 +1,7 @@ from sqlalchemy.orm import Session -from packages.domain import AssetLibrary, AssetLibraryKind from packages.adapters.sqlalchemy_impl.models import AssetLibraryModel +from packages.domain import AssetLibrary, AssetLibraryKind class SQLAlchemyAssetLibraryRepository: @@ -15,14 +15,14 @@ class SQLAlchemyAssetLibraryRepository: models = query.all() return [ AssetLibrary( - id=m.id, - workspace_id=m.workspace_id, - project_id=m.project_id, - name=m.name, - kind=AssetLibraryKind(m.kind), - created_at=m.created_at, + id=model.id, + workspace_id=model.workspace_id, + project_id=model.project_id, + name=model.name, + kind=AssetLibraryKind(model.kind), + created_at=model.created_at, ) - for m in models + for model in models ] def create(self, library: AssetLibrary) -> AssetLibrary: diff --git a/packages/adapters/sqlalchemy_impl/asset_repository.py b/packages/adapters/sqlalchemy_impl/asset_repository.py index f365f7928..e0e6ed0b3 100644 --- a/packages/adapters/sqlalchemy_impl/asset_repository.py +++ b/packages/adapters/sqlalchemy_impl/asset_repository.py @@ -1,8 +1,9 @@ import json + from sqlalchemy.orm import Session -from packages.domain import Asset from packages.adapters.sqlalchemy_impl.models import AssetModel +from packages.domain import Asset class SQLAlchemyAssetRepository: @@ -13,17 +14,17 @@ class SQLAlchemyAssetRepository: models = self.session.query(AssetModel).filter(AssetModel.library_id == library_id).all() return [ Asset( - id=m.id, - workspace_id=m.workspace_id, - project_id=m.project_id, - library_id=m.library_id, - name=m.name, - storage_key=m.storage_key, - mime_type=m.mime_type, - metadata=json.loads(m.metadata_json), - created_at=m.created_at, + id=model.id, + workspace_id=model.workspace_id, + project_id=model.project_id, + library_id=model.library_id, + name=model.name, + storage_key=model.storage_key, + mime_type=model.mime_type, + metadata=json.loads(model.metadata_json), + created_at=model.created_at, ) - for m in models + for model in models ] def create(self, asset: Asset) -> Asset: diff --git a/packages/adapters/sqlalchemy_impl/ingest_job_repository.py b/packages/adapters/sqlalchemy_impl/ingest_job_repository.py index 1ee98409c..141231f48 100644 --- a/packages/adapters/sqlalchemy_impl/ingest_job_repository.py +++ b/packages/adapters/sqlalchemy_impl/ingest_job_repository.py @@ -1,7 +1,7 @@ from sqlalchemy.orm import Session -from packages.domain import IngestJob, IngestJobStatus from packages.adapters.sqlalchemy_impl.models import IngestJobModel +from packages.domain import IngestJob, IngestJobStatus class SQLAlchemyIngestJobRepository: diff --git a/packages/adapters/sqlalchemy_impl/project_repository.py b/packages/adapters/sqlalchemy_impl/project_repository.py index 9a825928b..02c614841 100644 --- a/packages/adapters/sqlalchemy_impl/project_repository.py +++ b/packages/adapters/sqlalchemy_impl/project_repository.py @@ -1,7 +1,7 @@ from sqlalchemy.orm import Session -from packages.domain import Project from packages.adapters.sqlalchemy_impl.models import ProjectModel +from packages.domain import Project class SQLAlchemyProjectRepository: @@ -12,13 +12,13 @@ class SQLAlchemyProjectRepository: models = self.session.query(ProjectModel).filter(ProjectModel.workspace_id == workspace_id).all() return [ Project( - id=m.id, - workspace_id=m.workspace_id, - name=m.name, - description=m.description, - created_at=m.created_at, + id=model.id, + workspace_id=model.workspace_id, + name=model.name, + description=model.description, + created_at=model.created_at, ) - for m in models + for model in models ] def create(self, project: Project) -> Project: diff --git a/packages/adapters/sqlalchemy_impl/session.py b/packages/adapters/sqlalchemy_impl/session.py new file mode 100644 index 000000000..3a8614b8c --- /dev/null +++ b/packages/adapters/sqlalchemy_impl/session.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker + +from packages.adapters.sqlalchemy_impl.models import Base + + +def build_engine( + database_url: str, + *, + pool_size: int = 20, + max_overflow: int = 40, + pool_timeout: int = 30, + pool_recycle: int = 3600, +): + return create_engine( + database_url, + pool_size=pool_size, + max_overflow=max_overflow, + pool_timeout=pool_timeout, + pool_recycle=pool_recycle, + ) + + +def build_session_factory( + database_url: str, + *, + pool_size: int = 20, + max_overflow: int = 40, + pool_timeout: int = 30, + pool_recycle: int = 3600, +): + engine = build_engine( + database_url, + pool_size=pool_size, + max_overflow=max_overflow, + pool_timeout=pool_timeout, + pool_recycle=pool_recycle, + ) + session_factory = sessionmaker(autocommit=False, autoflush=False, bind=engine) + return engine, session_factory + + +def initialize_database(engine) -> None: + Base.metadata.create_all(bind=engine) -- 2.54.0 From e786de7349a37bb09a50abcb008e1f62d8911301 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 19:41:57 +0800 Subject: [PATCH 009/112] fix: serialize schema init for postgres --- apps/api/app/db.py | 3 +- apps/worker/worker_app/db.py | 3 +- packages/adapters/sqlalchemy_impl/__init__.py | 3 +- packages/adapters/sqlalchemy_impl/session.py | 38 +++++++++++++++++-- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/apps/api/app/db.py b/apps/api/app/db.py index 93980d99f..0bd772013 100644 --- a/apps/api/app/db.py +++ b/apps/api/app/db.py @@ -3,9 +3,10 @@ from collections.abc import Generator from sqlalchemy.orm import Session from app.core.database import get_database_settings -from packages.adapters.sqlalchemy_impl import build_session_factory, initialize_database +from packages.adapters.sqlalchemy_impl import build_session_factory, ensure_database_exists, initialize_database settings = get_database_settings() +ensure_database_exists(settings.database_url) engine, SessionLocal = build_session_factory( settings.database_url, pool_size=settings.pool_size, diff --git a/apps/worker/worker_app/db.py b/apps/worker/worker_app/db.py index ca58fe311..78aaee433 100644 --- a/apps/worker/worker_app/db.py +++ b/apps/worker/worker_app/db.py @@ -1,7 +1,8 @@ from worker_app.core.config import get_settings -from packages.adapters.sqlalchemy_impl import build_session_factory, initialize_database +from packages.adapters.sqlalchemy_impl import build_session_factory, ensure_database_exists, initialize_database settings = get_settings() +ensure_database_exists(settings.database_url) engine, SessionLocal = build_session_factory( settings.database_url, pool_size=settings.database_pool_size, diff --git a/packages/adapters/sqlalchemy_impl/__init__.py b/packages/adapters/sqlalchemy_impl/__init__.py index 53a79cc54..fa3c87125 100644 --- a/packages/adapters/sqlalchemy_impl/__init__.py +++ b/packages/adapters/sqlalchemy_impl/__init__.py @@ -4,7 +4,7 @@ from .asset_library_repository import SQLAlchemyAssetLibraryRepository from .asset_repository import SQLAlchemyAssetRepository from .ingest_job_repository import SQLAlchemyIngestJobRepository from .project_repository import SQLAlchemyProjectRepository -from .session import Base, build_engine, build_session_factory, initialize_database +from .session import Base, build_engine, build_session_factory, ensure_database_exists, initialize_database __all__ = [ "Base", @@ -14,5 +14,6 @@ __all__ = [ "SQLAlchemyProjectRepository", "build_engine", "build_session_factory", + "ensure_database_exists", "initialize_database", ] diff --git a/packages/adapters/sqlalchemy_impl/session.py b/packages/adapters/sqlalchemy_impl/session.py index 3a8614b8c..f7707a92f 100644 --- a/packages/adapters/sqlalchemy_impl/session.py +++ b/packages/adapters/sqlalchemy_impl/session.py @@ -1,11 +1,15 @@ from __future__ import annotations -from sqlalchemy import create_engine -from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL, make_url +from sqlalchemy.orm import sessionmaker from packages.adapters.sqlalchemy_impl.models import Base +SCHEMA_INIT_LOCK_ID = 2026061501 + + def build_engine( database_url: str, *, @@ -42,5 +46,33 @@ def build_session_factory( return engine, session_factory +def _build_admin_url(database_url: str) -> URL: + url = make_url(database_url) + return url.set(database="postgres") + + +def ensure_database_exists(database_url: str) -> None: + target_url = make_url(database_url) + admin_engine = create_engine(_build_admin_url(database_url), isolation_level="AUTOCOMMIT") + try: + with admin_engine.connect() as connection: + exists = connection.execute( + text("SELECT 1 FROM pg_database WHERE datname = :database_name"), + {"database_name": target_url.database}, + ).scalar() + if exists: + return + connection.execute(text(f'CREATE DATABASE "{target_url.database}"')) + finally: + admin_engine.dispose() + + def initialize_database(engine) -> None: - Base.metadata.create_all(bind=engine) + with engine.connect() as connection: + connection.execute(text("SELECT pg_advisory_lock(:lock_id)"), {"lock_id": SCHEMA_INIT_LOCK_ID}) + try: + Base.metadata.create_all(bind=connection) + connection.commit() + finally: + connection.execute(text("SELECT pg_advisory_unlock(:lock_id)"), {"lock_id": SCHEMA_INIT_LOCK_ID}) + connection.commit() -- 2.54.0 From 38df02514a8c11063c2943465dd21a0452833ba9 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 19:58:11 +0800 Subject: [PATCH 010/112] chore: wire real host-based gitea deploy --- .gitea/workflows/deploy.yml | 237 +++++++++++++----------------------- docs/CI-CD.md | 137 ++++++++------------- infra/docker/compose.yml | 5 +- 3 files changed, 138 insertions(+), 241 deletions(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 2b44b936f..c562bfcc7 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -7,177 +7,110 @@ on: - 'v*' jobs: - build: - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build API image - run: | - docker build -t xiaoxia-saas-api:${{ github.sha }} \ - -f infra/docker/api.Dockerfile . - - - name: Build Worker image - run: | - docker build -t xiaoxia-saas-worker:${{ github.sha }} \ - -f infra/docker/worker.Dockerfile . - - - name: Save Docker images - run: | - docker save xiaoxia-saas-api:${{ github.sha }} | gzip > api-image.tar.gz - docker save xiaoxia-saas-worker:${{ github.sha }} | gzip > worker-image.tar.gz - - - name: Move images to temp - run: | - mv api-image.tar.gz /tmp/ - mv worker-image.tar.gz /tmp/ - deploy-staging: + name: Deploy Staging runs-on: ubuntu-latest - needs: build if: github.ref == 'refs/heads/main' - + steps: - name: Checkout code uses: actions/checkout@v3 - - - name: Deploy to staging + + - name: Sync code to staging workspace run: | - # Load Docker images - docker load < /tmp/api-image.tar.gz - docker load < /tmp/worker-image.tar.gz - - # Prepare staging directory - mkdir -p /var/lib/xiaoxia-saas-staging - cp .env.staging /var/lib/xiaoxia-saas-staging/.env - - # Tag as staging - docker tag xiaoxia-saas-api:${{ github.sha }} xiaoxia-saas-api:staging - docker tag xiaoxia-saas-worker:${{ github.sha }} xiaoxia-saas-worker:staging - - # Deploy with staging env - cd /var/lib/xiaoxia-saas-staging - cat > docker-compose.yml << 'COMPOSE' -version: '3.9' -services: - api: - image: xiaoxia-saas-api:staging - restart: unless-stopped - env_file: - - ./.env - ports: - - "8001:8000" - depends_on: - - postgres - - redis + mkdir -p /var/lib/xiaoxia-saas-staging/repo + rsync -a --delete --exclude '.git' ./ /var/lib/xiaoxia-saas-staging/repo/ - worker: - image: xiaoxia-saas-worker:staging - restart: unless-stopped - env_file: - - ./.env - depends_on: - - redis - - postgres + - name: Verify staging env file + run: | + test -f /var/lib/xiaoxia-saas-staging/.env - postgres: - image: postgres:16 - restart: unless-stopped - environment: - POSTGRES_DB: xiaoxia_saas_staging - POSTGRES_USER: postgres - POSTGRES_PASSWORD: CHANGE_ME_STAGING_DB_PASSWORD - volumes: - - postgres_staging_data:/var/lib/postgresql/data + - name: Prepare staging env + run: | + cp /var/lib/xiaoxia-saas-staging/.env /var/lib/xiaoxia-saas-staging/repo/.env - redis: - image: redis:7 - restart: unless-stopped + - name: Build staging images + run: | + docker build \ + -t xiaoxia-saas-api:${{ github.sha }} \ + -t xiaoxia-saas-api:staging \ + -f /var/lib/xiaoxia-saas-staging/repo/infra/docker/api.Dockerfile \ + /var/lib/xiaoxia-saas-staging/repo + docker build \ + -t xiaoxia-saas-worker:${{ github.sha }} \ + -t xiaoxia-saas-worker:staging \ + -f /var/lib/xiaoxia-saas-staging/repo/infra/docker/worker.Dockerfile \ + /var/lib/xiaoxia-saas-staging/repo + + - name: Deploy staging containers + run: | + cd /var/lib/xiaoxia-saas-staging/repo/infra/docker + API_IMAGE=xiaoxia-saas-api:staging \ + WORKER_IMAGE=xiaoxia-saas-worker:staging \ + docker compose up -d postgres redis api worker + docker compose ps + + - name: Verify staging health + run: | + for i in $(seq 1 30); do + if curl -fsS http://127.0.0.1:8000/api/v1/health; then + exit 0 + fi + sleep 2 + done + exit 1 -volumes: - postgres_staging_data: -COMPOSE - - docker-compose up -d - rm -f /tmp/api-image.tar.gz /tmp/worker-image.tar.gz - - echo "✅ Staging deployment completed" - deploy-production: + name: Deploy Production runs-on: ubuntu-latest - needs: build if: startsWith(github.ref, 'refs/tags/v') - + steps: - name: Checkout code uses: actions/checkout@v3 - - - name: Deploy to production + + - name: Sync code to production workspace run: | - # Load Docker images - docker load < /tmp/api-image.tar.gz - docker load < /tmp/worker-image.tar.gz - - # Prepare production directory - mkdir -p /var/lib/xiaoxia-saas-production - cp .env.production /var/lib/xiaoxia-saas-production/.env - - # Tag as production version - docker tag xiaoxia-saas-api:${{ github.sha }} xiaoxia-saas-api:${{ github.ref_name }} - docker tag xiaoxia-saas-worker:${{ github.sha }} xiaoxia-saas-worker:${{ github.ref_name }} - docker tag xiaoxia-saas-api:${{ github.sha }} xiaoxia-saas-api:latest - docker tag xiaoxia-saas-worker:${{ github.sha }} xiaoxia-saas-worker:latest - - # Deploy with production env - cd /var/lib/xiaoxia-saas-production - cat > docker-compose.yml << 'COMPOSE' -version: '3.9' -services: - api: - image: xiaoxia-saas-api:latest - restart: unless-stopped - env_file: - - ./.env - ports: - - "8000:8000" - depends_on: - - postgres - - redis + mkdir -p /var/lib/xiaoxia-saas-production/repo + rsync -a --delete --exclude '.git' ./ /var/lib/xiaoxia-saas-production/repo/ - worker: - image: xiaoxia-saas-worker:latest - restart: unless-stopped - env_file: - - ./.env - depends_on: - - redis - - postgres + - name: Verify production env file + run: | + test -f /var/lib/xiaoxia-saas-production/.env - postgres: - image: postgres:16 - restart: unless-stopped - environment: - POSTGRES_DB: xiaoxia_saas_production - POSTGRES_USER: postgres - POSTGRES_PASSWORD: CHANGE_ME_PRODUCTION_DB_PASSWORD - volumes: - - postgres_production_data:/var/lib/postgresql/data + - name: Prepare production env + run: | + cp /var/lib/xiaoxia-saas-production/.env /var/lib/xiaoxia-saas-production/repo/.env - redis: - image: redis:7 - restart: unless-stopped + - name: Build production images + run: | + docker build \ + -t xiaoxia-saas-api:${{ github.sha }} \ + -t xiaoxia-saas-api:${{ github.ref_name }} \ + -t xiaoxia-saas-api:latest \ + -f /var/lib/xiaoxia-saas-production/repo/infra/docker/api.Dockerfile \ + /var/lib/xiaoxia-saas-production/repo + docker build \ + -t xiaoxia-saas-worker:${{ github.sha }} \ + -t xiaoxia-saas-worker:${{ github.ref_name }} \ + -t xiaoxia-saas-worker:latest \ + -f /var/lib/xiaoxia-saas-production/repo/infra/docker/worker.Dockerfile \ + /var/lib/xiaoxia-saas-production/repo -volumes: - postgres_production_data: -COMPOSE - - docker-compose up -d - rm -f /tmp/api-image.tar.gz /tmp/worker-image.tar.gz - - echo "✅ Production deployment completed: ${{ github.ref_name }}" + - name: Deploy production containers + run: | + cd /var/lib/xiaoxia-saas-production/repo/infra/docker + API_IMAGE=xiaoxia-saas-api:latest \ + WORKER_IMAGE=xiaoxia-saas-worker:latest \ + docker compose up -d postgres redis api worker + docker compose ps + + - name: Verify production health + run: | + for i in $(seq 1 30); do + if curl -fsS http://127.0.0.1:8000/api/v1/health; then + exit 0 + fi + sleep 2 + done + exit 1 diff --git a/docs/CI-CD.md b/docs/CI-CD.md index 39f3fdc6f..57b94640e 100644 --- a/docs/CI-CD.md +++ b/docs/CI-CD.md @@ -2,137 +2,102 @@ ## Gitea Actions -本项目使用 Gitea Actions 实现 CI/CD 流水线。 +本项目使用 Gitea Actions + 本机 act_runner 实现 CI/CD。 ### 工作流文件 **1. `.gitea/workflows/tests.yml` - 自动化测试** 触发条件: -- 每次 push 到 main 分支 +- 每次 push 到 `main` - 每次创建 Pull Request 包含任务: - `test` - 运行集成测试,生成覆盖率报告 -- `lint` - 代码质量检查(Black、Flake8、MyPy) +- `lint` - 运行 Black / Flake8 / MyPy **2. `.gitea/workflows/deploy.yml` - 自动化部署** 触发条件: -- push 到 main 分支 → 部署到 staging 环境 -- 创建 tag (v*) → 部署到 production 环境 +- push 到 `main` → 部署到 staging +- push tag `v*` → 部署到 production -当前状态:占位,待补充真实部署脚本 +实际行为: +- runner 在部署主机本机执行 workflow +- workflow 将仓库同步到 `/var/lib/xiaoxia-saas-staging/repo` 或 `/var/lib/xiaoxia-saas-production/repo` +- 读取服务器本地真实 `.env` +- 本机构建 API / Worker 镜像 +- 使用 `infra/docker/compose.yml` 启动 `postgres` / `redis` / `api` / `worker` +- 部署后用 `/api/v1/health` 做健康校验 + +--- + +## 部署目录约定 + +### Staging +- 环境根目录:`/var/lib/xiaoxia-saas-staging` +- 真实环境文件:`/var/lib/xiaoxia-saas-staging/.env` +- Actions 同步代码目录:`/var/lib/xiaoxia-saas-staging/repo` + +### Production +- 环境根目录:`/var/lib/xiaoxia-saas-production` +- 真实环境文件:`/var/lib/xiaoxia-saas-production/.env` +- Actions 同步代码目录:`/var/lib/xiaoxia-saas-production/repo` --- ## 本地验证 -在推送前,可以本地运行验证: +推送前建议先跑: ### 运行测试 ```bash pytest tests/integration/ -v --cov=packages --cov=apps --cov-report=html ``` -### 代码格式化 +### 代码格式化检查 ```bash -black packages/ apps/ tests/ +black --check packages/ apps/ tests/ ``` -### 代码检查 +### 静态检查 ```bash -# Flake8 flake8 packages/ apps/ tests/ --max-line-length=120 --extend-ignore=E203,W503 - -# MyPy mypy packages/ apps/ --ignore-missing-imports ``` -### 排序 imports -```bash -isort packages/ apps/ tests/ -``` - --- -## Gitea Actions 配置 +## 当前已验证结论 -### 前提条件 - -1. Gitea 服务器需要启用 Actions 功能 -2. 需要配置 Actions Runner - -### 启用步骤 - -**在 Gitea 服务器上**: - -1. 编辑 `/etc/gitea/app.ini`(或对应配置文件): -```ini -[actions] -ENABLED = true -``` - -2. 重启 Gitea: -```bash -systemctl restart gitea -``` - -3. 安装 Actions Runner: -```bash -# 下载 act_runner -wget https://dl.gitea.com/act_runner/latest/act_runner-linux-amd64 -chmod +x act_runner-linux-amd64 -mv act_runner-linux-amd64 /usr/local/bin/act_runner - -# 注册 runner -act_runner register --instance http://your-gitea-server --token YOUR_RUNNER_TOKEN - -# 运行 runner -act_runner daemon -``` - -4. 在仓库设置中启用 Actions - ---- - -## 验证 CI/CD - -推送代码后,在 Gitea 仓库页面查看: -- Actions 标签页 -- 查看工作流运行状态 -- 查看测试结果和覆盖率报告 +- Gitea Actions 已启用 +- `act_runner` 已注册并持续运行 +- staging 可手工部署并已完成真实业务闭环验证 +- 当前 CI/CD 的关键目标是让 Gitea push 后自动完成同机部署,而不是只保留占位 YAML --- ## 故障排查 -### Actions 未触发 -- 检查 Gitea 是否启用 Actions -- 检查工作流文件路径是否正确(`.gitea/workflows/`) -- 检查 YAML 语法是否正确 +### Actions 触发了但 checkout 失败 +优先检查 runner 能否从 job 容器访问 Gitea 实例地址。 -### 测试失败 -- 查看 Actions 日志 -- 本地运行 `pytest tests/integration/ -v` -- 检查依赖是否完整 +### Deploy 成功但业务链不通 +优先检查: +- `/var/lib/xiaoxia-saas-staging/.env` +- `MINIO_ENDPOINT` +- `DATABASE_URL` +- worker 日志中的 Celery 任务消费情况 -### Linting 失败 -- 本地运行 `black --check packages/` -- 本地运行 `flake8 packages/` -- 修复后重新推送 - ---- - -## 未来扩展 - -- [ ] 添加性能测试 -- [ ] 添加安全扫描 -- [ ] 添加 Docker 镜像构建 -- [ ] 添加自动化部署脚本 -- [ ] 添加 Slack/钉钉通知 +### 健康检查失败 +查看: +```bash +docker compose ps +docker compose logs api --tail=200 +docker compose logs worker --tail=200 +``` --- **最后更新**: 2026-06-15 -**状态**: 配置完成,待 Gitea Actions 启用 +**状态**: CI/CD 已接入真实主机部署模型,待 push 后持续验证稳定性 diff --git a/infra/docker/compose.yml b/infra/docker/compose.yml index 802353bd8..bcb0f2478 100644 --- a/infra/docker/compose.yml +++ b/infra/docker/compose.yml @@ -1,4 +1,3 @@ -version: '3.9' services: web: image: node:20 @@ -12,7 +11,7 @@ services: - api api: - image: xiaoxia-saas-api:dev + image: ${API_IMAGE:-xiaoxia-saas-api:dev} build: context: ../.. dockerfile: infra/docker/api.Dockerfile @@ -27,7 +26,7 @@ services: - redis worker: - image: xiaoxia-saas-worker:dev + image: ${WORKER_IMAGE:-xiaoxia-saas-worker:dev} build: context: ../.. dockerfile: infra/docker/worker.Dockerfile -- 2.54.0 From 2a7201a2a4cebce0b74efe80afd894ab59ee9348 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 19:59:47 +0800 Subject: [PATCH 011/112] fix: set gitea checkout base url --- .gitea/workflows/deploy.yml | 4 ++++ .gitea/workflows/tests.yml | 4 ++++ docs/CI-CD.md | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index c562bfcc7..160496bc1 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -15,6 +15,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 + with: + github-server-url: https://api.xiaoxiajianji.com/git - name: Sync code to staging workspace run: | @@ -68,6 +70,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 + with: + github-server-url: https://api.xiaoxiajianji.com/git - name: Sync code to production workspace run: | diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index 6164f415a..fc52e0657 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -13,6 +13,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 + with: + github-server-url: https://api.xiaoxiajianji.com/git - name: Set up Python uses: actions/setup-python@v4 @@ -41,6 +43,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 + with: + github-server-url: https://api.xiaoxiajianji.com/git - name: Set up Python uses: actions/setup-python@v4 diff --git a/docs/CI-CD.md b/docs/CI-CD.md index 57b94640e..0e2deeb34 100644 --- a/docs/CI-CD.md +++ b/docs/CI-CD.md @@ -80,7 +80,7 @@ mypy packages/ apps/ --ignore-missing-imports ## 故障排查 ### Actions 触发了但 checkout 失败 -优先检查 runner 能否从 job 容器访问 Gitea 实例地址。 +优先检查 runner 能否从 job 容器访问 Gitea 实例地址;如果 Gitea 挂在 `/git` 这类子路径下,需要给 `actions/checkout` 显式传 `github-server-url`。 ### Deploy 成功但业务链不通 优先检查: -- 2.54.0 From f13c7ab9af0f2d997531e92a9cb8164a92f23fe7 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 20:02:04 +0800 Subject: [PATCH 012/112] fix: use manual checkout for gitea actions --- .gitea/workflows/deploy.yml | 50 ++++++++++++++++++++++++++++-------- .gitea/workflows/tests.yml | 51 ++++++++++++++++++++----------------- docs/CI-CD.md | 2 +- 3 files changed, 69 insertions(+), 34 deletions(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 160496bc1..10d36edb0 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -14,14 +14,29 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 - with: - github-server-url: https://api.xiaoxiajianji.com/git + shell: bash + run: | + set -euo pipefail + repo_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" + auth_header="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 -w0)" + git init . + git remote add origin "$repo_url" + git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" + git checkout --force FETCH_HEAD - name: Sync code to staging workspace + shell: bash run: | - mkdir -p /var/lib/xiaoxia-saas-staging/repo - rsync -a --delete --exclude '.git' ./ /var/lib/xiaoxia-saas-staging/repo/ + set -euo pipefail + python3 - <<'PY' +from pathlib import Path +import shutil +root = Path('/var/lib/xiaoxia-saas-staging/repo') +if root.exists(): + shutil.rmtree(root) +root.mkdir(parents=True, exist_ok=True) +PY + tar --exclude=.git -cf - . | tar -xf - -C /var/lib/xiaoxia-saas-staging/repo - name: Verify staging env file run: | @@ -69,14 +84,29 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 - with: - github-server-url: https://api.xiaoxiajianji.com/git + shell: bash + run: | + set -euo pipefail + repo_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" + auth_header="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 -w0)" + git init . + git remote add origin "$repo_url" + git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" + git checkout --force FETCH_HEAD - name: Sync code to production workspace + shell: bash run: | - mkdir -p /var/lib/xiaoxia-saas-production/repo - rsync -a --delete --exclude '.git' ./ /var/lib/xiaoxia-saas-production/repo/ + set -euo pipefail + python3 - <<'PY' +from pathlib import Path +import shutil +root = Path('/var/lib/xiaoxia-saas-production/repo') +if root.exists(): + shutil.rmtree(root) +root.mkdir(parents=True, exist_ok=True) +PY + tar --exclude=.git -cf - . | tar -xf - -C /var/lib/xiaoxia-saas-production/repo - name: Verify production env file run: | diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index fc52e0657..077faf728 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -9,61 +9,66 @@ on: jobs: test: runs-on: ubuntu-latest - + steps: - name: Checkout code - uses: actions/checkout@v3 - with: - github-server-url: https://api.xiaoxiajianji.com/git - + shell: bash + run: | + set -euo pipefail + repo_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" + auth_header="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 -w0)" + git init . + git remote add origin "$repo_url" + git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" + git checkout --force FETCH_HEAD + - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.12' - + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt - + - name: Run tests run: | pytest tests/integration/ -v --cov=packages --cov=apps --cov-report=xml --cov-report=term - - - name: Upload coverage reports - uses: codecov/codecov-action@v3 - if: always() - with: - file: ./coverage.xml - fail_ci_if_error: false lint: runs-on: ubuntu-latest - + steps: - name: Checkout code - uses: actions/checkout@v3 - with: - github-server-url: https://api.xiaoxiajianji.com/git - + shell: bash + run: | + set -euo pipefail + repo_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" + auth_header="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 -w0)" + git init . + git remote add origin "$repo_url" + git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" + git checkout --force FETCH_HEAD + - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.12' - + - name: Install linting tools run: | python -m pip install --upgrade pip pip install black flake8 mypy - + - name: Run Black (check only) run: | black --check packages/ apps/ tests/ - + - name: Run Flake8 run: | flake8 packages/ apps/ tests/ --max-line-length=120 --extend-ignore=E203,W503 - + - name: Run MyPy run: | mypy packages/ apps/ --ignore-missing-imports diff --git a/docs/CI-CD.md b/docs/CI-CD.md index 0e2deeb34..77b316669 100644 --- a/docs/CI-CD.md +++ b/docs/CI-CD.md @@ -80,7 +80,7 @@ mypy packages/ apps/ --ignore-missing-imports ## 故障排查 ### Actions 触发了但 checkout 失败 -优先检查 runner 能否从 job 容器访问 Gitea 实例地址;如果 Gitea 挂在 `/git` 这类子路径下,需要给 `actions/checkout` 显式传 `github-server-url`。 +优先检查 runner 能否从 job 容器访问 Gitea 实例地址;如果 Gitea 挂在 `/git` 这类子路径下,优先使用手写 `git fetch`,不要依赖 `actions/checkout` 自动拼接仓库地址。 ### Deploy 成功但业务链不通 优先检查: -- 2.54.0 From b9a95eea68047c45c7932d796d8332f245ff225a Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 20:09:59 +0800 Subject: [PATCH 013/112] fix: stabilize gitea workflows --- .gitea/workflows/deploy.yml | 22 ++++++---------------- .gitea/workflows/tests.yml | 32 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 32 deletions(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 10d36edb0..fc4d3cc38 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -28,14 +28,9 @@ jobs: shell: bash run: | set -euo pipefail - python3 - <<'PY' -from pathlib import Path -import shutil -root = Path('/var/lib/xiaoxia-saas-staging/repo') -if root.exists(): - shutil.rmtree(root) -root.mkdir(parents=True, exist_ok=True) -PY + mkdir -p /var/lib/xiaoxia-saas-staging + rm -rf /var/lib/xiaoxia-saas-staging/repo + mkdir -p /var/lib/xiaoxia-saas-staging/repo tar --exclude=.git -cf - . | tar -xf - -C /var/lib/xiaoxia-saas-staging/repo - name: Verify staging env file @@ -98,14 +93,9 @@ PY shell: bash run: | set -euo pipefail - python3 - <<'PY' -from pathlib import Path -import shutil -root = Path('/var/lib/xiaoxia-saas-production/repo') -if root.exists(): - shutil.rmtree(root) -root.mkdir(parents=True, exist_ok=True) -PY + mkdir -p /var/lib/xiaoxia-saas-production + rm -rf /var/lib/xiaoxia-saas-production/repo + mkdir -p /var/lib/xiaoxia-saas-production/repo tar --exclude=.git -cf - . | tar -xf - -C /var/lib/xiaoxia-saas-production/repo - name: Verify production env file diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index 077faf728..178f75a98 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -22,19 +22,19 @@ jobs: git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" git checkout --force FETCH_HEAD - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Show Python runtime + run: | + python3 --version + python3 -m pip --version - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt -r requirements-dev.txt - name: Run tests run: | - pytest tests/integration/ -v --cov=packages --cov=apps --cov-report=xml --cov-report=term + python3 -m pytest tests/integration/ -v --cov=packages --cov=apps --cov-report=xml --cov-report=term lint: runs-on: ubuntu-latest @@ -51,24 +51,24 @@ jobs: git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" git checkout --force FETCH_HEAD - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Show Python runtime + run: | + python3 --version + python3 -m pip --version - name: Install linting tools run: | - python -m pip install --upgrade pip - pip install black flake8 mypy + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt -r requirements-dev.txt - name: Run Black (check only) run: | - black --check packages/ apps/ tests/ + python3 -m black --check packages/ apps/ tests/ - name: Run Flake8 run: | - flake8 packages/ apps/ tests/ --max-line-length=120 --extend-ignore=E203,W503 + python3 -m flake8 packages/ apps/ tests/ --max-line-length=120 --extend-ignore=E203,W503 - name: Run MyPy run: | - mypy packages/ apps/ --ignore-missing-imports + python3 -m mypy packages/ apps/ --ignore-missing-imports -- 2.54.0 From b47cfa549d4b3bdfe541692bea7408a6c0f55b03 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 20:13:12 +0800 Subject: [PATCH 014/112] fix: align workflows with runner containers --- .gitea/workflows/deploy.yml | 166 +++++++++++++++++++++++------------- .gitea/workflows/tests.yml | 16 ++-- 2 files changed, 118 insertions(+), 64 deletions(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index fc4d3cc38..3853be0fd 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -24,53 +24,78 @@ jobs: git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" git checkout --force FETCH_HEAD + - name: Prepare deploy tooling + run: | + apt-get update + apt-get install -y docker.io curl + - name: Sync code to staging workspace shell: bash run: | set -euo pipefail - mkdir -p /var/lib/xiaoxia-saas-staging - rm -rf /var/lib/xiaoxia-saas-staging/repo - mkdir -p /var/lib/xiaoxia-saas-staging/repo - tar --exclude=.git -cf - . | tar -xf - -C /var/lib/xiaoxia-saas-staging/repo + docker run --rm \ + -v /:/host \ + -v "$PWD":/workspace-src \ + alpine:3.20 \ + sh -lc ' + set -eu + mkdir -p /host/var/lib/xiaoxia-saas-staging + rm -rf /host/var/lib/xiaoxia-saas-staging/repo + mkdir -p /host/var/lib/xiaoxia-saas-staging/repo + cd /workspace-src + tar --exclude=.git -cf - . | tar -xf - -C /host/var/lib/xiaoxia-saas-staging/repo + ' - name: Verify staging env file run: | - test -f /var/lib/xiaoxia-saas-staging/.env + docker run --rm -v /:/host alpine:3.20 test -f /host/var/lib/xiaoxia-saas-staging/.env - name: Prepare staging env run: | - cp /var/lib/xiaoxia-saas-staging/.env /var/lib/xiaoxia-saas-staging/repo/.env + docker run --rm -v /:/host alpine:3.20 sh -lc 'cp /host/var/lib/xiaoxia-saas-staging/.env /host/var/lib/xiaoxia-saas-staging/repo/.env' - name: Build staging images run: | - docker build \ - -t xiaoxia-saas-api:${{ github.sha }} \ - -t xiaoxia-saas-api:staging \ - -f /var/lib/xiaoxia-saas-staging/repo/infra/docker/api.Dockerfile \ - /var/lib/xiaoxia-saas-staging/repo - docker build \ - -t xiaoxia-saas-worker:${{ github.sha }} \ - -t xiaoxia-saas-worker:staging \ - -f /var/lib/xiaoxia-saas-staging/repo/infra/docker/worker.Dockerfile \ - /var/lib/xiaoxia-saas-staging/repo + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /:/host \ + docker:27-cli sh -lc ' + docker build \ + -t xiaoxia-saas-api:${{ github.sha }} \ + -t xiaoxia-saas-api:staging \ + -f /host/var/lib/xiaoxia-saas-staging/repo/infra/docker/api.Dockerfile \ + /host/var/lib/xiaoxia-saas-staging/repo && \ + docker build \ + -t xiaoxia-saas-worker:${{ github.sha }} \ + -t xiaoxia-saas-worker:staging \ + -f /host/var/lib/xiaoxia-saas-staging/repo/infra/docker/worker.Dockerfile \ + /host/var/lib/xiaoxia-saas-staging/repo + ' - name: Deploy staging containers run: | - cd /var/lib/xiaoxia-saas-staging/repo/infra/docker - API_IMAGE=xiaoxia-saas-api:staging \ - WORKER_IMAGE=xiaoxia-saas-worker:staging \ - docker compose up -d postgres redis api worker - docker compose ps + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /:/host \ + -w /host/var/lib/xiaoxia-saas-staging/repo/infra/docker \ + -e API_IMAGE=xiaoxia-saas-api:staging \ + -e WORKER_IMAGE=xiaoxia-saas-worker:staging \ + docker:27-cli sh -lc ' + docker compose up -d postgres redis api worker && \ + docker compose ps + ' - name: Verify staging health run: | - for i in $(seq 1 30); do - if curl -fsS http://127.0.0.1:8000/api/v1/health; then - exit 0 - fi - sleep 2 - done - exit 1 + docker run --rm --network host curlimages/curl:8.8.0 sh -lc ' + for i in $(seq 1 30); do + if curl -fsS http://127.0.0.1:8000/api/v1/health; then + exit 0 + fi + sleep 2 + done + exit 1 + ' deploy-production: name: Deploy Production @@ -89,52 +114,77 @@ jobs: git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" git checkout --force FETCH_HEAD + - name: Prepare deploy tooling + run: | + apt-get update + apt-get install -y docker.io curl + - name: Sync code to production workspace shell: bash run: | set -euo pipefail - mkdir -p /var/lib/xiaoxia-saas-production - rm -rf /var/lib/xiaoxia-saas-production/repo - mkdir -p /var/lib/xiaoxia-saas-production/repo - tar --exclude=.git -cf - . | tar -xf - -C /var/lib/xiaoxia-saas-production/repo + docker run --rm \ + -v /:/host \ + -v "$PWD":/workspace-src \ + alpine:3.20 \ + sh -lc ' + set -eu + mkdir -p /host/var/lib/xiaoxia-saas-production + rm -rf /host/var/lib/xiaoxia-saas-production/repo + mkdir -p /host/var/lib/xiaoxia-saas-production/repo + cd /workspace-src + tar --exclude=.git -cf - . | tar -xf - -C /host/var/lib/xiaoxia-saas-production/repo + ' - name: Verify production env file run: | - test -f /var/lib/xiaoxia-saas-production/.env + docker run --rm -v /:/host alpine:3.20 test -f /host/var/lib/xiaoxia-saas-production/.env - name: Prepare production env run: | - cp /var/lib/xiaoxia-saas-production/.env /var/lib/xiaoxia-saas-production/repo/.env + docker run --rm -v /:/host alpine:3.20 sh -lc 'cp /host/var/lib/xiaoxia-saas-production/.env /host/var/lib/xiaoxia-saas-production/repo/.env' - name: Build production images run: | - docker build \ - -t xiaoxia-saas-api:${{ github.sha }} \ - -t xiaoxia-saas-api:${{ github.ref_name }} \ - -t xiaoxia-saas-api:latest \ - -f /var/lib/xiaoxia-saas-production/repo/infra/docker/api.Dockerfile \ - /var/lib/xiaoxia-saas-production/repo - docker build \ - -t xiaoxia-saas-worker:${{ github.sha }} \ - -t xiaoxia-saas-worker:${{ github.ref_name }} \ - -t xiaoxia-saas-worker:latest \ - -f /var/lib/xiaoxia-saas-production/repo/infra/docker/worker.Dockerfile \ - /var/lib/xiaoxia-saas-production/repo + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /:/host \ + docker:27-cli sh -lc ' + docker build \ + -t xiaoxia-saas-api:${{ github.sha }} \ + -t xiaoxia-saas-api:${{ github.ref_name }} \ + -t xiaoxia-saas-api:latest \ + -f /host/var/lib/xiaoxia-saas-production/repo/infra/docker/api.Dockerfile \ + /host/var/lib/xiaoxia-saas-production/repo && \ + docker build \ + -t xiaoxia-saas-worker:${{ github.sha }} \ + -t xiaoxia-saas-worker:${{ github.ref_name }} \ + -t xiaoxia-saas-worker:latest \ + -f /host/var/lib/xiaoxia-saas-production/repo/infra/docker/worker.Dockerfile \ + /host/var/lib/xiaoxia-saas-production/repo + ' - name: Deploy production containers run: | - cd /var/lib/xiaoxia-saas-production/repo/infra/docker - API_IMAGE=xiaoxia-saas-api:latest \ - WORKER_IMAGE=xiaoxia-saas-worker:latest \ - docker compose up -d postgres redis api worker - docker compose ps + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /:/host \ + -w /host/var/lib/xiaoxia-saas-production/repo/infra/docker \ + -e API_IMAGE=xiaoxia-saas-api:latest \ + -e WORKER_IMAGE=xiaoxia-saas-worker:latest \ + docker:27-cli sh -lc ' + docker compose up -d postgres redis api worker && \ + docker compose ps + ' - name: Verify production health run: | - for i in $(seq 1 30); do - if curl -fsS http://127.0.0.1:8000/api/v1/health; then - exit 0 - fi - sleep 2 - done - exit 1 + docker run --rm --network host curlimages/curl:8.8.0 sh -lc ' + for i in $(seq 1 30); do + if curl -fsS http://127.0.0.1:8000/api/v1/health; then + exit 0 + fi + sleep 2 + done + exit 1 + ' diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index 178f75a98..c28b527e6 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -22,15 +22,17 @@ jobs: git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" git checkout --force FETCH_HEAD - - name: Show Python runtime + - name: Install Python tooling run: | + apt-get update + apt-get install -y python3-pip python3-venv python3 --version python3 -m pip --version - name: Install dependencies run: | - python3 -m pip install --upgrade pip - python3 -m pip install -r requirements.txt -r requirements-dev.txt + python3 -m pip install --break-system-packages --upgrade pip + python3 -m pip install --break-system-packages -r requirements.txt -r requirements-dev.txt - name: Run tests run: | @@ -51,15 +53,17 @@ jobs: git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" git checkout --force FETCH_HEAD - - name: Show Python runtime + - name: Install Python tooling run: | + apt-get update + apt-get install -y python3-pip python3-venv python3 --version python3 -m pip --version - name: Install linting tools run: | - python3 -m pip install --upgrade pip - python3 -m pip install -r requirements.txt -r requirements-dev.txt + python3 -m pip install --break-system-packages --upgrade pip + python3 -m pip install --break-system-packages -r requirements.txt -r requirements-dev.txt - name: Run Black (check only) run: | -- 2.54.0 From 33fb16a75a91f4d41c9bea4a4efbe75c8757539a Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 20:20:03 +0800 Subject: [PATCH 015/112] fix: stream deploy sources into host workspace --- .gitea/workflows/deploy.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 3853be0fd..ce5ee70f5 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -33,17 +33,15 @@ jobs: shell: bash run: | set -euo pipefail - docker run --rm \ + tar --exclude=.git -cf - . | docker run --rm -i \ -v /:/host \ - -v "$PWD":/workspace-src \ alpine:3.20 \ sh -lc ' set -eu mkdir -p /host/var/lib/xiaoxia-saas-staging rm -rf /host/var/lib/xiaoxia-saas-staging/repo mkdir -p /host/var/lib/xiaoxia-saas-staging/repo - cd /workspace-src - tar --exclude=.git -cf - . | tar -xf - -C /host/var/lib/xiaoxia-saas-staging/repo + tar -xf - -C /host/var/lib/xiaoxia-saas-staging/repo ' - name: Verify staging env file @@ -123,17 +121,15 @@ jobs: shell: bash run: | set -euo pipefail - docker run --rm \ + tar --exclude=.git -cf - . | docker run --rm -i \ -v /:/host \ - -v "$PWD":/workspace-src \ alpine:3.20 \ sh -lc ' set -eu mkdir -p /host/var/lib/xiaoxia-saas-production rm -rf /host/var/lib/xiaoxia-saas-production/repo mkdir -p /host/var/lib/xiaoxia-saas-production/repo - cd /workspace-src - tar --exclude=.git -cf - . | tar -xf - -C /host/var/lib/xiaoxia-saas-production/repo + tar -xf - -C /host/var/lib/xiaoxia-saas-production/repo ' - name: Verify production env file -- 2.54.0 From d3d90d3775b29ac480af61b68ce059d2ceb9794f Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 20:27:12 +0800 Subject: [PATCH 016/112] fix: use self-contained workflow images --- .gitea/workflows/deploy.yml | 190 +++++++++++++++++------------------- .gitea/workflows/tests.yml | 113 ++++++++++++++------- 2 files changed, 167 insertions(+), 136 deletions(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index ce5ee70f5..3f445f771 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -10,29 +10,24 @@ jobs: deploy-staging: name: Deploy Staging runs-on: ubuntu-latest + container: + image: docker:27-cli if: github.ref == 'refs/heads/main' steps: - name: Checkout code - shell: bash + shell: sh run: | - set -euo pipefail - repo_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - auth_header="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 -w0)" - git init . - git remote add origin "$repo_url" - git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" - git checkout --force FETCH_HEAD - - - name: Prepare deploy tooling - run: | - apt-get update - apt-get install -y docker.io curl + set -eu + archive_url="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/archive/${GITHUB_SHA}.tar.gz" + wget --header="Authorization: token ${GITHUB_TOKEN}" -O /tmp/repo.tar.gz "$archive_url" + tar -xzf /tmp/repo.tar.gz --strip-components=1 -C . + rm -f /tmp/repo.tar.gz - name: Sync code to staging workspace - shell: bash + shell: sh run: | - set -euo pipefail + set -eu tar --exclude=.git -cf - . | docker run --rm -i \ -v /:/host \ alpine:3.20 \ @@ -45,82 +40,77 @@ jobs: ' - name: Verify staging env file + shell: sh run: | + set -eu docker run --rm -v /:/host alpine:3.20 test -f /host/var/lib/xiaoxia-saas-staging/.env - name: Prepare staging env + shell: sh run: | + set -eu docker run --rm -v /:/host alpine:3.20 sh -lc 'cp /host/var/lib/xiaoxia-saas-staging/.env /host/var/lib/xiaoxia-saas-staging/repo/.env' - name: Build staging images + shell: sh run: | - docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /:/host \ - docker:27-cli sh -lc ' - docker build \ - -t xiaoxia-saas-api:${{ github.sha }} \ - -t xiaoxia-saas-api:staging \ - -f /host/var/lib/xiaoxia-saas-staging/repo/infra/docker/api.Dockerfile \ - /host/var/lib/xiaoxia-saas-staging/repo && \ - docker build \ - -t xiaoxia-saas-worker:${{ github.sha }} \ - -t xiaoxia-saas-worker:staging \ - -f /host/var/lib/xiaoxia-saas-staging/repo/infra/docker/worker.Dockerfile \ - /host/var/lib/xiaoxia-saas-staging/repo - ' + set -eu + docker build \ + -t xiaoxia-saas-api:${GITHUB_SHA} \ + -t xiaoxia-saas-api:staging \ + -f /var/lib/xiaoxia-saas-staging/repo/infra/docker/api.Dockerfile \ + /var/lib/xiaoxia-saas-staging/repo + docker build \ + -t xiaoxia-saas-worker:${GITHUB_SHA} \ + -t xiaoxia-saas-worker:staging \ + -f /var/lib/xiaoxia-saas-staging/repo/infra/docker/worker.Dockerfile \ + /var/lib/xiaoxia-saas-staging/repo - name: Deploy staging containers + shell: sh run: | - docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /:/host \ - -w /host/var/lib/xiaoxia-saas-staging/repo/infra/docker \ - -e API_IMAGE=xiaoxia-saas-api:staging \ - -e WORKER_IMAGE=xiaoxia-saas-worker:staging \ - docker:27-cli sh -lc ' - docker compose up -d postgres redis api worker && \ - docker compose ps - ' + set -eu + cd /var/lib/xiaoxia-saas-staging/repo/infra/docker + API_IMAGE=xiaoxia-saas-api:staging \ + WORKER_IMAGE=xiaoxia-saas-worker:staging \ + docker compose up -d postgres redis api worker + docker compose ps - name: Verify staging health + shell: sh run: | - docker run --rm --network host curlimages/curl:8.8.0 sh -lc ' - for i in $(seq 1 30); do - if curl -fsS http://127.0.0.1:8000/api/v1/health; then - exit 0 - fi - sleep 2 - done - exit 1 - ' + set -eu + i=0 + while [ "$i" -lt 30 ]; do + if wget -qO- http://127.0.0.1:8000/api/v1/health; then + exit 0 + fi + i=$((i + 1)) + sleep 2 + done + exit 1 deploy-production: name: Deploy Production runs-on: ubuntu-latest + container: + image: docker:27-cli if: startsWith(github.ref, 'refs/tags/v') steps: - name: Checkout code - shell: bash + shell: sh run: | - set -euo pipefail - repo_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - auth_header="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 -w0)" - git init . - git remote add origin "$repo_url" - git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" - git checkout --force FETCH_HEAD - - - name: Prepare deploy tooling - run: | - apt-get update - apt-get install -y docker.io curl + set -eu + archive_url="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/archive/${GITHUB_SHA}.tar.gz" + wget --header="Authorization: token ${GITHUB_TOKEN}" -O /tmp/repo.tar.gz "$archive_url" + tar -xzf /tmp/repo.tar.gz --strip-components=1 -C . + rm -f /tmp/repo.tar.gz - name: Sync code to production workspace - shell: bash + shell: sh run: | - set -euo pipefail + set -eu tar --exclude=.git -cf - . | docker run --rm -i \ -v /:/host \ alpine:3.20 \ @@ -133,54 +123,54 @@ jobs: ' - name: Verify production env file + shell: sh run: | + set -eu docker run --rm -v /:/host alpine:3.20 test -f /host/var/lib/xiaoxia-saas-production/.env - name: Prepare production env + shell: sh run: | + set -eu docker run --rm -v /:/host alpine:3.20 sh -lc 'cp /host/var/lib/xiaoxia-saas-production/.env /host/var/lib/xiaoxia-saas-production/repo/.env' - name: Build production images + shell: sh run: | - docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /:/host \ - docker:27-cli sh -lc ' - docker build \ - -t xiaoxia-saas-api:${{ github.sha }} \ - -t xiaoxia-saas-api:${{ github.ref_name }} \ - -t xiaoxia-saas-api:latest \ - -f /host/var/lib/xiaoxia-saas-production/repo/infra/docker/api.Dockerfile \ - /host/var/lib/xiaoxia-saas-production/repo && \ - docker build \ - -t xiaoxia-saas-worker:${{ github.sha }} \ - -t xiaoxia-saas-worker:${{ github.ref_name }} \ - -t xiaoxia-saas-worker:latest \ - -f /host/var/lib/xiaoxia-saas-production/repo/infra/docker/worker.Dockerfile \ - /host/var/lib/xiaoxia-saas-production/repo - ' + set -eu + docker build \ + -t xiaoxia-saas-api:${GITHUB_SHA} \ + -t xiaoxia-saas-api:${GITHUB_REF_NAME} \ + -t xiaoxia-saas-api:latest \ + -f /var/lib/xiaoxia-saas-production/repo/infra/docker/api.Dockerfile \ + /var/lib/xiaoxia-saas-production/repo + docker build \ + -t xiaoxia-saas-worker:${GITHUB_SHA} \ + -t xiaoxia-saas-worker:${GITHUB_REF_NAME} \ + -t xiaoxia-saas-worker:latest \ + -f /var/lib/xiaoxia-saas-production/repo/infra/docker/worker.Dockerfile \ + /var/lib/xiaoxia-saas-production/repo - name: Deploy production containers + shell: sh run: | - docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /:/host \ - -w /host/var/lib/xiaoxia-saas-production/repo/infra/docker \ - -e API_IMAGE=xiaoxia-saas-api:latest \ - -e WORKER_IMAGE=xiaoxia-saas-worker:latest \ - docker:27-cli sh -lc ' - docker compose up -d postgres redis api worker && \ - docker compose ps - ' + set -eu + cd /var/lib/xiaoxia-saas-production/repo/infra/docker + API_IMAGE=xiaoxia-saas-api:latest \ + WORKER_IMAGE=xiaoxia-saas-worker:latest \ + docker compose up -d postgres redis api worker + docker compose ps - name: Verify production health + shell: sh run: | - docker run --rm --network host curlimages/curl:8.8.0 sh -lc ' - for i in $(seq 1 30); do - if curl -fsS http://127.0.0.1:8000/api/v1/health; then - exit 0 - fi - sleep 2 - done - exit 1 - ' + set -eu + i=0 + while [ "$i" -lt 30 ]; do + if wget -qO- http://127.0.0.1:8000/api/v1/health; then + exit 0 + fi + i=$((i + 1)) + sleep 2 + done + exit 1 diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index c28b527e6..23c1b3e2e 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -9,70 +9,111 @@ on: jobs: test: runs-on: ubuntu-latest + container: + image: python:3.12-slim steps: - name: Checkout code - shell: bash + shell: sh run: | - set -euo pipefail - repo_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - auth_header="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 -w0)" - git init . - git remote add origin "$repo_url" - git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" - git checkout --force FETCH_HEAD + set -eu + python - <<'PY' + import io + import os + import tarfile + import urllib.request - - name: Install Python tooling + url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" + request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' + for member in tar.getmembers(): + name = member.name + if name == root_prefix[:-1]: + continue + if name.startswith(root_prefix): + member.name = name[len(root_prefix):] + if member.name: + tar.extract(member, '.') + PY + + - name: Show Python version + shell: sh run: | - apt-get update - apt-get install -y python3-pip python3-venv - python3 --version - python3 -m pip --version + set -eu + python --version + python -m pip --version - name: Install dependencies + shell: sh run: | - python3 -m pip install --break-system-packages --upgrade pip - python3 -m pip install --break-system-packages -r requirements.txt -r requirements-dev.txt + set -eu + python -m pip install --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com + python -m pip install -r requirements.txt -r requirements-dev.txt -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - name: Run tests + shell: sh run: | - python3 -m pytest tests/integration/ -v --cov=packages --cov=apps --cov-report=xml --cov-report=term + set -eu + python -m pytest tests/integration/ -v --cov=packages --cov=apps --cov-report=xml --cov-report=term lint: runs-on: ubuntu-latest + container: + image: python:3.12-slim steps: - name: Checkout code - shell: bash + shell: sh run: | - set -euo pipefail - repo_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - auth_header="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 -w0)" - git init . - git remote add origin "$repo_url" - git -c http.https://api.xiaoxiajianji.com/.extraheader="$auth_header" fetch --depth=1 origin "$GITHUB_SHA" - git checkout --force FETCH_HEAD + set -eu + python - <<'PY' + import io + import os + import tarfile + import urllib.request - - name: Install Python tooling - run: | - apt-get update - apt-get install -y python3-pip python3-venv - python3 --version - python3 -m pip --version + url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" + request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() - - name: Install linting tools + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' + for member in tar.getmembers(): + name = member.name + if name == root_prefix[:-1]: + continue + if name.startswith(root_prefix): + member.name = name[len(root_prefix):] + if member.name: + tar.extract(member, '.') + PY + + - name: Install dependencies + shell: sh run: | - python3 -m pip install --break-system-packages --upgrade pip - python3 -m pip install --break-system-packages -r requirements.txt -r requirements-dev.txt + set -eu + python -m pip install --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com + python -m pip install -r requirements.txt -r requirements-dev.txt -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - name: Run Black (check only) + shell: sh run: | - python3 -m black --check packages/ apps/ tests/ + set -eu + python -m black --check packages/ apps/ tests/ - name: Run Flake8 + shell: sh run: | - python3 -m flake8 packages/ apps/ tests/ --max-line-length=120 --extend-ignore=E203,W503 + set -eu + python -m flake8 packages/ apps/ tests/ --max-line-length=120 --extend-ignore=E203,W503 - name: Run MyPy + shell: sh run: | - python3 -m mypy packages/ apps/ --ignore-missing-imports + set -eu + python -m mypy packages/ apps/ --ignore-missing-imports -- 2.54.0 From 35a29d85d39c48ca9582f4b47cba249f937f59e3 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 20:29:35 +0800 Subject: [PATCH 017/112] fix: run deploy against host via docker cli --- .gitea/workflows/deploy.yml | 96 ++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 38 deletions(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 3f445f771..324e65ec2 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -30,7 +30,7 @@ jobs: set -eu tar --exclude=.git -cf - . | docker run --rm -i \ -v /:/host \ - alpine:3.20 \ + docker:27-cli \ sh -lc ' set -eu mkdir -p /host/var/lib/xiaoxia-saas-staging @@ -43,38 +43,48 @@ jobs: shell: sh run: | set -eu - docker run --rm -v /:/host alpine:3.20 test -f /host/var/lib/xiaoxia-saas-staging/.env + docker run --rm -v /:/host docker:27-cli sh -lc 'test -f /host/var/lib/xiaoxia-saas-staging/.env' - name: Prepare staging env shell: sh run: | set -eu - docker run --rm -v /:/host alpine:3.20 sh -lc 'cp /host/var/lib/xiaoxia-saas-staging/.env /host/var/lib/xiaoxia-saas-staging/repo/.env' + docker run --rm -v /:/host docker:27-cli sh -lc 'cp /host/var/lib/xiaoxia-saas-staging/.env /host/var/lib/xiaoxia-saas-staging/repo/.env' - name: Build staging images shell: sh run: | set -eu - docker build \ - -t xiaoxia-saas-api:${GITHUB_SHA} \ - -t xiaoxia-saas-api:staging \ - -f /var/lib/xiaoxia-saas-staging/repo/infra/docker/api.Dockerfile \ - /var/lib/xiaoxia-saas-staging/repo - docker build \ - -t xiaoxia-saas-worker:${GITHUB_SHA} \ - -t xiaoxia-saas-worker:staging \ - -f /var/lib/xiaoxia-saas-staging/repo/infra/docker/worker.Dockerfile \ - /var/lib/xiaoxia-saas-staging/repo + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /:/host \ + docker:27-cli sh -lc ' + docker build \ + -t xiaoxia-saas-api:${GITHUB_SHA} \ + -t xiaoxia-saas-api:staging \ + -f /host/var/lib/xiaoxia-saas-staging/repo/infra/docker/api.Dockerfile \ + /host/var/lib/xiaoxia-saas-staging/repo && \ + docker build \ + -t xiaoxia-saas-worker:${GITHUB_SHA} \ + -t xiaoxia-saas-worker:staging \ + -f /host/var/lib/xiaoxia-saas-staging/repo/infra/docker/worker.Dockerfile \ + /host/var/lib/xiaoxia-saas-staging/repo + ' - name: Deploy staging containers shell: sh run: | set -eu - cd /var/lib/xiaoxia-saas-staging/repo/infra/docker - API_IMAGE=xiaoxia-saas-api:staging \ - WORKER_IMAGE=xiaoxia-saas-worker:staging \ - docker compose up -d postgres redis api worker - docker compose ps + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /:/host \ + -w /host/var/lib/xiaoxia-saas-staging/repo/infra/docker \ + -e API_IMAGE=xiaoxia-saas-api:staging \ + -e WORKER_IMAGE=xiaoxia-saas-worker:staging \ + docker:27-cli sh -lc ' + docker compose up -d postgres redis api worker && \ + docker compose ps + ' - name: Verify staging health shell: sh @@ -113,7 +123,7 @@ jobs: set -eu tar --exclude=.git -cf - . | docker run --rm -i \ -v /:/host \ - alpine:3.20 \ + docker:27-cli \ sh -lc ' set -eu mkdir -p /host/var/lib/xiaoxia-saas-production @@ -126,40 +136,50 @@ jobs: shell: sh run: | set -eu - docker run --rm -v /:/host alpine:3.20 test -f /host/var/lib/xiaoxia-saas-production/.env + docker run --rm -v /:/host docker:27-cli sh -lc 'test -f /host/var/lib/xiaoxia-saas-production/.env' - name: Prepare production env shell: sh run: | set -eu - docker run --rm -v /:/host alpine:3.20 sh -lc 'cp /host/var/lib/xiaoxia-saas-production/.env /host/var/lib/xiaoxia-saas-production/repo/.env' + docker run --rm -v /:/host docker:27-cli sh -lc 'cp /host/var/lib/xiaoxia-saas-production/.env /host/var/lib/xiaoxia-saas-production/repo/.env' - name: Build production images shell: sh run: | set -eu - docker build \ - -t xiaoxia-saas-api:${GITHUB_SHA} \ - -t xiaoxia-saas-api:${GITHUB_REF_NAME} \ - -t xiaoxia-saas-api:latest \ - -f /var/lib/xiaoxia-saas-production/repo/infra/docker/api.Dockerfile \ - /var/lib/xiaoxia-saas-production/repo - docker build \ - -t xiaoxia-saas-worker:${GITHUB_SHA} \ - -t xiaoxia-saas-worker:${GITHUB_REF_NAME} \ - -t xiaoxia-saas-worker:latest \ - -f /var/lib/xiaoxia-saas-production/repo/infra/docker/worker.Dockerfile \ - /var/lib/xiaoxia-saas-production/repo + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /:/host \ + docker:27-cli sh -lc ' + docker build \ + -t xiaoxia-saas-api:${GITHUB_SHA} \ + -t xiaoxia-saas-api:${GITHUB_REF_NAME} \ + -t xiaoxia-saas-api:latest \ + -f /host/var/lib/xiaoxia-saas-production/repo/infra/docker/api.Dockerfile \ + /host/var/lib/xiaoxia-saas-production/repo && \ + docker build \ + -t xiaoxia-saas-worker:${GITHUB_SHA} \ + -t xiaoxia-saas-worker:${GITHUB_REF_NAME} \ + -t xiaoxia-saas-worker:latest \ + -f /host/var/lib/xiaoxia-saas-production/repo/infra/docker/worker.Dockerfile \ + /host/var/lib/xiaoxia-saas-production/repo + ' - name: Deploy production containers shell: sh run: | set -eu - cd /var/lib/xiaoxia-saas-production/repo/infra/docker - API_IMAGE=xiaoxia-saas-api:latest \ - WORKER_IMAGE=xiaoxia-saas-worker:latest \ - docker compose up -d postgres redis api worker - docker compose ps + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /:/host \ + -w /host/var/lib/xiaoxia-saas-production/repo/infra/docker \ + -e API_IMAGE=xiaoxia-saas-api:latest \ + -e WORKER_IMAGE=xiaoxia-saas-worker:latest \ + docker:27-cli sh -lc ' + docker compose up -d postgres redis api worker && \ + docker compose ps + ' - name: Verify production health shell: sh -- 2.54.0 From 2c78407f7ff8078df55ab95fd107cc145fb21f4e Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 15 Jun 2026 20:32:02 +0800 Subject: [PATCH 018/112] fix: pass commit metadata into deploy builds --- .gitea/workflows/deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 324e65ec2..6c32404cc 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -56,6 +56,7 @@ jobs: run: | set -eu docker run --rm \ + -e GITHUB_SHA="$GITHUB_SHA" \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /:/host \ docker:27-cli sh -lc ' @@ -149,6 +150,8 @@ jobs: run: | set -eu docker run --rm \ + -e GITHUB_SHA="$GITHUB_SHA" \ + -e GITHUB_REF_NAME="$GITHUB_REF_NAME" \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /:/host \ docker:27-cli sh -lc ' -- 2.54.0 From 355dd0ffbb5a9af18fb7815f3340a14e55540309 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Tue, 16 Jun 2026 11:02:06 +0800 Subject: [PATCH 019/112] feat: add project management module (tasks, milestones, issues) - Domain: Task, Milestone, TaskIssue entities with business logic - Ports: TaskRepository, MilestoneRepository, TaskIssueRepository interfaces - Adapters: In-Memory and SQLAlchemy implementations - Application: Use cases for task/milestone/issue operations - API: FastAPI routes for project management - Database: Alembic migration 002 for new tables - Tests: 7 integration tests all passing --- STATUS.md | 143 +++++++ .../versions/002_add_project_management.py | 91 ++++ apps/api/app/api/router.py | 7 + apps/api/app/api/routes/project_management.py | 405 ++++++++++++++++++ .../project_management_repositories.py | 91 ++++ packages/adapters/sqlalchemy_impl/models.py | 55 ++- .../project_management_repositories.py | 243 +++++++++++ .../project_management_use_cases.py | 151 +++++++ packages/domain/__init__.py | 6 + packages/domain/project_management.py | 241 +++++++++++ packages/ports/__init__.py | 4 + .../ports/project_management_repositories.py | 101 +++++ tests/integration/test_project_management.py | 196 +++++++++ 13 files changed, 1733 insertions(+), 1 deletion(-) create mode 100644 STATUS.md create mode 100644 alembic/versions/002_add_project_management.py create mode 100644 apps/api/app/api/routes/project_management.py create mode 100644 packages/adapters/in_memory/project_management_repositories.py create mode 100644 packages/adapters/sqlalchemy_impl/project_management_repositories.py create mode 100644 packages/application/project_management_use_cases.py create mode 100644 packages/domain/project_management.py create mode 100644 packages/ports/project_management_repositories.py create mode 100644 tests/integration/test_project_management.py diff --git a/STATUS.md b/STATUS.md new file mode 100644 index 000000000..f2cd3a60e --- /dev/null +++ b/STATUS.md @@ -0,0 +1,143 @@ +# Xiaoxia SaaS - 项目状态 + +> 最后更新:2026-06-16 +> 这是项目当前状态的单页真相,每次新会话优先读这个文件。 + +--- + +## 当前阶段 + +**Phase 1 已完成** - 核心平台层已搭建完成并稳定化中 + +--- + +## 已完成 + +### 核心架构 +- ✅ Clean Architecture 骨架(packages/domain + ports + application + adapters) +- ✅ 三层应用工程(apps/api + worker + web) +- ✅ 双持久化实现(In-Memory 测试 + PostgreSQL 生产) +- ✅ Alembic 数据库迁移 +- ✅ Docker Compose 开发环境 + +### 核心业务对象 +- ✅ User(用户) +- ✅ Workspace(工作空间) +- ✅ Project(项目) +- ✅ AssetLibrary(素材库:视频/音频) +- ✅ Asset(素材) +- ✅ IngestJob(入库任务:PENDING → PROCESSING → COMPLETED/FAILED) +- ✅ ClassificationJob(分类任务:含 9 种分类) + +### 核心业务流程 +- ✅ 上传 → 入库 → Asset 创建链路 +- ✅ 分类任务链路 +- ✅ 完整异步任务处理(Celery + Redis) + +### 基础设施 +- ✅ **MinIO 真实文件存储**(`apps/api/app/core/storage.py`) +- ✅ **Gitea CI/CD workflows**(`.gitea/workflows/tests.yml` + `deploy.yml`) +- ✅ 环境配置体系(`.env.staging` + `.env.production`) +- ✅ API 文档(Swagger) + +### 测试与质量 +- ✅ 8 个集成测试全绿 +- ✅ 测试策略:集成测试优先,覆盖核心业务流程 + +--- + +## 进行中 + +### 稳定化与部署打磨 +- 🔄 Gitea workflows 持续优化(最近 10+ commits) +- 🔄 部署流程与环境隔离稳定化 +- 🔄 Docker 构建优化(国内镜像源、多阶段构建) + +### 前端页面 +- 🔄 Next.js 前端占位已建立,实际页面待补 + +--- + +## 下一步优先级 + +1. **前端页面补齐** + - 项目/素材库管理页面 + - 素材上传/预览页面 + - 任务状态监控页面 + +2. **更完整的 SAAS 产品化能力** + - 多租户/组织/权限体系 + - 账号体系与认证 + - 订阅/计费体系 + +3. **业务能力扩展** + - 真实视频分类模型接入 + - 更多素材处理能力(剪辑/合成/配音) + - 批量任务调度 + +--- + +## 关键目录 + +**核心业务逻辑** +- `packages/domain/` - 核心实体与业务规则 +- `packages/application/` - 用例层 +- `packages/ports/` - 接口定义 +- `packages/adapters/` - 接口实现 + +**应用层** +- `apps/api/` - FastAPI REST API +- `apps/worker/` - Celery 异步任务 +- `apps/web/` - Next.js 前端 + +**基础设施** +- `infra/docker/` - Docker Compose 配置 +- `.gitea/workflows/` - CI/CD 流水线 + +**测试** +- `tests/integration/` - 集成测试 + +**数据库** +- `alembic/` - 数据库迁移 + +**文档** +- `docs/` - 项目文档 +- `README.md` - 项目总览与开发指南 + +--- + +## 关键决策记录 + +- 新 SaaS 与旧桌面版完全物理隔离 +- 旧桌面版仅作为业务参考,不再作为未来主线 +- 新系统从第一天起就遵循 Clean Architecture +- 持久化层提供双实现(in-memory + SQLAlchemy),方便测试 +- 测试策略:集成测试优先,覆盖核心业务流程 +- 数据库迁移从第一天起就版本化管理 +- CI/CD 基于 Gitea Actions + 自建 runner + +--- + +## 仓库信息 + +- **本地路径**: `F:\openclaw-saas` +- **远程仓库**: `xiaoxia-server:/var/lib/xiaoxia-ci/xiaoxia-saas.git` +- **分支**: `main` +- **最新提交**: `2c78407 fix: pass commit metadata into deploy builds` + +--- + +## 技术栈 + +**Backend**: Python 3.12 + FastAPI + Pydantic + SQLAlchemy + Alembic +**Worker**: Celery + Redis +**Database**: PostgreSQL + SQLite(测试) +**Storage**: MinIO(S3-compatible) +**Frontend**: Next.js 14 + TypeScript + React 18 +**CI/CD**: Gitea Actions +**Architecture**: Clean Architecture + Ports/Adapters +**Testing**: pytest + 集成测试优先 + +--- + +**以后每次新会话,先读这个文件 + `README.md`,快速对齐项目现状。** diff --git a/alembic/versions/002_add_project_management.py b/alembic/versions/002_add_project_management.py new file mode 100644 index 000000000..4f06e8472 --- /dev/null +++ b/alembic/versions/002_add_project_management.py @@ -0,0 +1,91 @@ +"""Add project management tables + +Revision ID: 002 +Revises: 001 +Create Date: 2026-06-16 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '002' +down_revision: Union[str, None] = '001' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Create tasks table + op.create_table( + 'tasks', + sa.Column('id', sa.String(32), nullable=False), + sa.Column('project_id', sa.String(32), nullable=False), + sa.Column('workspace_id', sa.String(32), nullable=False), + sa.Column('name', sa.String(200), nullable=False), + sa.Column('description', sa.Text(), nullable=False, server_default=''), + sa.Column('status', sa.String(20), nullable=False, server_default='pending'), + sa.Column('priority', sa.String(20), nullable=False, server_default='medium'), + sa.Column('parent_task_id', sa.String(32), nullable=False, server_default=''), + sa.Column('assignee_user_id', sa.String(32), nullable=False, server_default=''), + sa.Column('progress', sa.Float(), nullable=False, server_default='0'), + sa.Column('planned_start_date', sa.DateTime(), nullable=True), + sa.Column('planned_end_date', sa.DateTime(), nullable=True), + sa.Column('actual_start_date', sa.DateTime(), nullable=True), + sa.Column('actual_end_date', sa.DateTime(), nullable=True), + sa.Column('tags_json', sa.Text(), nullable=False, server_default='[]'), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_tasks_project_id'), 'tasks', ['project_id'], unique=False) + op.create_index(op.f('ix_tasks_workspace_id'), 'tasks', ['workspace_id'], unique=False) + op.create_index(op.f('ix_tasks_parent_task_id'), 'tasks', ['parent_task_id'], unique=False) + op.create_index(op.f('ix_tasks_status'), 'tasks', ['status'], unique=False) + + # Create milestones table + op.create_table( + 'milestones', + sa.Column('id', sa.String(32), nullable=False), + sa.Column('project_id', sa.String(32), nullable=False), + sa.Column('workspace_id', sa.String(32), nullable=False), + sa.Column('name', sa.String(200), nullable=False), + sa.Column('description', sa.Text(), nullable=False, server_default=''), + sa.Column('target_date', sa.DateTime(), nullable=True), + sa.Column('completed', sa.Boolean(), nullable=False, server_default='0'), + sa.Column('completed_at', sa.DateTime(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_milestones_project_id'), 'milestones', ['project_id'], unique=False) + op.create_index(op.f('ix_milestones_workspace_id'), 'milestones', ['workspace_id'], unique=False) + + # Create task_issues table + op.create_table( + 'task_issues', + sa.Column('id', sa.String(32), nullable=False), + sa.Column('task_id', sa.String(32), nullable=False), + sa.Column('project_id', sa.String(32), nullable=False), + sa.Column('workspace_id', sa.String(32), nullable=False), + sa.Column('title', sa.String(200), nullable=False), + sa.Column('description', sa.Text(), nullable=False, server_default=''), + sa.Column('resolved', sa.Boolean(), nullable=False, server_default='0'), + sa.Column('resolved_at', sa.DateTime(), nullable=True), + sa.Column('created_by_user_id', sa.String(32), nullable=False, server_default=''), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_task_issues_task_id'), 'task_issues', ['task_id'], unique=False) + op.create_index(op.f('ix_task_issues_project_id'), 'task_issues', ['project_id'], unique=False) + op.create_index(op.f('ix_task_issues_workspace_id'), 'task_issues', ['workspace_id'], unique=False) + + +def downgrade() -> None: + op.drop_table('task_issues') + op.drop_table('milestones') + op.drop_table('tasks') diff --git a/apps/api/app/api/router.py b/apps/api/app/api/router.py index 64221f71f..4f8d0ace4 100644 --- a/apps/api/app/api/router.py +++ b/apps/api/app/api/router.py @@ -4,6 +4,7 @@ from app.api.routes.asset_libraries import router as asset_libraries_router from app.api.routes.assets import router as assets_router from app.api.routes.health import router as health_router from app.api.routes.ingest_jobs import router as ingest_jobs_router +from app.api.routes.project_management import router as project_management_router from app.api.routes.projects import router as projects_router from app.api.routes.upload import router as upload_router @@ -50,3 +51,9 @@ api_router.include_router( prefix="/upload", tags=["文件上传"], ) + +# Project Management (Tasks, Milestones, Issues) +api_router.include_router( + project_management_router, + tags=["项目推进管理"], +) diff --git a/apps/api/app/api/routes/project_management.py b/apps/api/app/api/routes/project_management.py new file mode 100644 index 000000000..431c2bad1 --- /dev/null +++ b/apps/api/app/api/routes/project_management.py @@ -0,0 +1,405 @@ +"""项目管理 API 路由""" +from datetime import datetime +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + +from packages.adapters.in_memory.project_management_repositories import ( + InMemoryMilestoneRepository, + InMemoryTaskIssueRepository, + InMemoryTaskRepository, +) +from packages.application.project_management_use_cases import ( + CreateMilestoneUseCase, + CreateTaskIssueUseCase, + CreateTaskUseCase, + ListProjectMilestonesUseCase, + ListProjectTasksUseCase, + ListTaskIssuesUseCase, + ResolveTaskIssueUseCase, + UpdateTaskProgressUseCase, + UpdateTaskStatusUseCase, +) +from packages.domain import TaskPriority, TaskStatus + +router = APIRouter(prefix="/api/v1/project-management", tags=["project-management"]) + +# 简易依赖注入(生产环境应走数据库) +_task_repo = InMemoryTaskRepository() +_milestone_repo = InMemoryMilestoneRepository() +_issue_repo = InMemoryTaskIssueRepository() + + +def get_task_repo(): + return _task_repo + + +def get_milestone_repo(): + return _milestone_repo + + +def get_issue_repo(): + return _issue_repo + + +# ========== Request/Response Models ========== + + +class CreateTaskRequest(BaseModel): + project_id: str + workspace_id: str + name: str + description: str = "" + priority: TaskPriority = TaskPriority.MEDIUM + parent_task_id: str = "" + assignee_user_id: str = "" + + +class TaskResponse(BaseModel): + id: str + project_id: str + workspace_id: str + name: str + description: str + status: TaskStatus + priority: TaskPriority + parent_task_id: str + assignee_user_id: str + progress: float + planned_start_date: datetime | None + planned_end_date: datetime | None + actual_start_date: datetime | None + actual_end_date: datetime | None + tags: list[str] + created_at: datetime + updated_at: datetime + + +class UpdateTaskStatusRequest(BaseModel): + status: TaskStatus + + +class UpdateTaskProgressRequest(BaseModel): + progress: Annotated[float, Field(ge=0, le=100)] + + +class CreateMilestoneRequest(BaseModel): + project_id: str + workspace_id: str + name: str + description: str = "" + + +class MilestoneResponse(BaseModel): + id: str + project_id: str + workspace_id: str + name: str + description: str + target_date: datetime | None + completed: bool + completed_at: datetime | None + created_at: datetime + updated_at: datetime + + +class CreateTaskIssueRequest(BaseModel): + task_id: str + project_id: str + workspace_id: str + title: str + description: str = "" + created_by_user_id: str = "" + + +class TaskIssueResponse(BaseModel): + id: str + task_id: str + project_id: str + workspace_id: str + title: str + description: str + resolved: bool + resolved_at: datetime | None + created_by_user_id: str + created_at: datetime + updated_at: datetime + + +# ========== Task Endpoints ========== + + +@router.post("/tasks", response_model=TaskResponse) +def create_task( + req: CreateTaskRequest, + task_repo=Depends(get_task_repo), +): + """创建任务""" + use_case = CreateTaskUseCase(task_repo) + task = use_case.execute( + project_id=req.project_id, + workspace_id=req.workspace_id, + name=req.name, + description=req.description, + priority=req.priority, + parent_task_id=req.parent_task_id, + assignee_user_id=req.assignee_user_id, + ) + return TaskResponse( + id=task.id, + project_id=task.project_id, + workspace_id=task.workspace_id, + name=task.name, + description=task.description, + status=task.status, + priority=task.priority, + parent_task_id=task.parent_task_id, + assignee_user_id=task.assignee_user_id, + progress=task.progress, + planned_start_date=task.planned_start_date, + planned_end_date=task.planned_end_date, + actual_start_date=task.actual_start_date, + actual_end_date=task.actual_end_date, + tags=task.tags, + created_at=task.created_at, + updated_at=task.updated_at, + ) + + +@router.get("/tasks", response_model=list[TaskResponse]) +def list_tasks( + project_id: str, + task_repo=Depends(get_task_repo), +): + """获取项目任务列表""" + use_case = ListProjectTasksUseCase(task_repo) + tasks = use_case.execute(project_id) + return [ + TaskResponse( + id=t.id, + project_id=t.project_id, + workspace_id=t.workspace_id, + name=t.name, + description=t.description, + status=t.status, + priority=t.priority, + parent_task_id=t.parent_task_id, + assignee_user_id=t.assignee_user_id, + progress=t.progress, + planned_start_date=t.planned_start_date, + planned_end_date=t.planned_end_date, + actual_start_date=t.actual_start_date, + actual_end_date=t.actual_end_date, + tags=t.tags, + created_at=t.created_at, + updated_at=t.updated_at, + ) + for t in tasks + ] + + +@router.patch("/tasks/{task_id}/status", response_model=TaskResponse) +def update_task_status( + task_id: str, + req: UpdateTaskStatusRequest, + task_repo=Depends(get_task_repo), +): + """更新任务状态""" + use_case = UpdateTaskStatusUseCase(task_repo) + try: + task = use_case.execute(task_id, req.status) + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + return TaskResponse( + id=task.id, + project_id=task.project_id, + workspace_id=task.workspace_id, + name=task.name, + description=task.description, + status=task.status, + priority=task.priority, + parent_task_id=task.parent_task_id, + assignee_user_id=task.assignee_user_id, + progress=task.progress, + planned_start_date=task.planned_start_date, + planned_end_date=task.planned_end_date, + actual_start_date=task.actual_start_date, + actual_end_date=task.actual_end_date, + tags=task.tags, + created_at=task.created_at, + updated_at=task.updated_at, + ) + + +@router.patch("/tasks/{task_id}/progress", response_model=TaskResponse) +def update_task_progress( + task_id: str, + req: UpdateTaskProgressRequest, + task_repo=Depends(get_task_repo), +): + """更新任务进度""" + use_case = UpdateTaskProgressUseCase(task_repo) + try: + task = use_case.execute(task_id, req.progress) + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + return TaskResponse( + id=task.id, + project_id=task.project_id, + workspace_id=task.workspace_id, + name=task.name, + description=task.description, + status=task.status, + priority=task.priority, + parent_task_id=task.parent_task_id, + assignee_user_id=task.assignee_user_id, + progress=task.progress, + planned_start_date=task.planned_start_date, + planned_end_date=task.planned_end_date, + actual_start_date=task.actual_start_date, + actual_end_date=task.actual_end_date, + tags=task.tags, + created_at=task.created_at, + updated_at=task.updated_at, + ) + + +# ========== Milestone Endpoints ========== + + +@router.post("/milestones", response_model=MilestoneResponse) +def create_milestone( + req: CreateMilestoneRequest, + milestone_repo=Depends(get_milestone_repo), +): + """创建里程碑""" + use_case = CreateMilestoneUseCase(milestone_repo) + milestone = use_case.execute( + project_id=req.project_id, + workspace_id=req.workspace_id, + name=req.name, + description=req.description, + ) + return MilestoneResponse( + id=milestone.id, + project_id=milestone.project_id, + workspace_id=milestone.workspace_id, + name=milestone.name, + description=milestone.description, + target_date=milestone.target_date, + completed=milestone.completed, + completed_at=milestone.completed_at, + created_at=milestone.created_at, + updated_at=milestone.updated_at, + ) + + +@router.get("/milestones", response_model=list[MilestoneResponse]) +def list_milestones( + project_id: str, + milestone_repo=Depends(get_milestone_repo), +): + """获取项目里程碑列表""" + use_case = ListProjectMilestonesUseCase(milestone_repo) + milestones = use_case.execute(project_id) + return [ + MilestoneResponse( + id=m.id, + project_id=m.project_id, + workspace_id=m.workspace_id, + name=m.name, + description=m.description, + target_date=m.target_date, + completed=m.completed, + completed_at=m.completed_at, + created_at=m.created_at, + updated_at=m.updated_at, + ) + for m in milestones + ] + + +# ========== Task Issue Endpoints ========== + + +@router.post("/issues", response_model=TaskIssueResponse) +def create_issue( + req: CreateTaskIssueRequest, + issue_repo=Depends(get_issue_repo), +): + """创建任务问题""" + use_case = CreateTaskIssueUseCase(issue_repo) + issue = use_case.execute( + task_id=req.task_id, + project_id=req.project_id, + workspace_id=req.workspace_id, + title=req.title, + description=req.description, + created_by_user_id=req.created_by_user_id, + ) + return TaskIssueResponse( + id=issue.id, + task_id=issue.task_id, + project_id=issue.project_id, + workspace_id=issue.workspace_id, + title=issue.title, + description=issue.description, + resolved=issue.resolved, + resolved_at=issue.resolved_at, + created_by_user_id=issue.created_by_user_id, + created_at=issue.created_at, + updated_at=issue.updated_at, + ) + + +@router.get("/issues", response_model=list[TaskIssueResponse]) +def list_issues( + task_id: str, + issue_repo=Depends(get_issue_repo), +): + """获取任务问题列表""" + use_case = ListTaskIssuesUseCase(issue_repo) + issues = use_case.execute(task_id) + return [ + TaskIssueResponse( + id=i.id, + task_id=i.task_id, + project_id=i.project_id, + workspace_id=i.workspace_id, + title=i.title, + description=i.description, + resolved=i.resolved, + resolved_at=i.resolved_at, + created_by_user_id=i.created_by_user_id, + created_at=i.created_at, + updated_at=i.updated_at, + ) + for i in issues + ] + + +@router.patch("/issues/{issue_id}/resolve", response_model=TaskIssueResponse) +def resolve_issue( + issue_id: str, + issue_repo=Depends(get_issue_repo), +): + """解决任务问题""" + use_case = ResolveTaskIssueUseCase(issue_repo) + try: + issue = use_case.execute(issue_id) + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + return TaskIssueResponse( + id=issue.id, + task_id=issue.task_id, + project_id=issue.project_id, + workspace_id=issue.workspace_id, + title=issue.title, + description=issue.description, + resolved=issue.resolved, + resolved_at=issue.resolved_at, + created_by_user_id=issue.created_by_user_id, + created_at=issue.created_at, + updated_at=issue.updated_at, + ) diff --git a/packages/adapters/in_memory/project_management_repositories.py b/packages/adapters/in_memory/project_management_repositories.py new file mode 100644 index 000000000..44257d050 --- /dev/null +++ b/packages/adapters/in_memory/project_management_repositories.py @@ -0,0 +1,91 @@ +"""项目管理 In-Memory Repository 实现""" +from packages.domain import Milestone, Task, TaskIssue +from packages.ports.project_management_repositories import ( + MilestoneRepository, + TaskIssueRepository, + TaskRepository, +) + + +class InMemoryTaskRepository(TaskRepository): + """任务 In-Memory 仓储实现""" + + def __init__(self): + self._store: dict[str, Task] = {} + + def create(self, task: Task) -> Task: + self._store[task.id] = task + return task + + def get_by_id(self, task_id: str) -> Task | None: + return self._store.get(task_id) + + def list_by_project(self, project_id: str) -> list[Task]: + return [t for t in self._store.values() if t.project_id == project_id] + + def list_by_parent(self, parent_task_id: str) -> list[Task]: + return [t for t in self._store.values() if t.parent_task_id == parent_task_id] + + def update(self, task: Task) -> Task: + if task.id not in self._store: + raise ValueError(f"Task {task.id} not found") + self._store[task.id] = task + return task + + def delete(self, task_id: str) -> None: + self._store.pop(task_id, None) + + +class InMemoryMilestoneRepository(MilestoneRepository): + """里程碑 In-Memory 仓储实现""" + + def __init__(self): + self._store: dict[str, Milestone] = {} + + def create(self, milestone: Milestone) -> Milestone: + self._store[milestone.id] = milestone + return milestone + + def get_by_id(self, milestone_id: str) -> Milestone | None: + return self._store.get(milestone_id) + + def list_by_project(self, project_id: str) -> list[Milestone]: + return [m for m in self._store.values() if m.project_id == project_id] + + def update(self, milestone: Milestone) -> Milestone: + if milestone.id not in self._store: + raise ValueError(f"Milestone {milestone.id} not found") + self._store[milestone.id] = milestone + return milestone + + def delete(self, milestone_id: str) -> None: + self._store.pop(milestone_id, None) + + +class InMemoryTaskIssueRepository(TaskIssueRepository): + """任务问题 In-Memory 仓储实现""" + + def __init__(self): + self._store: dict[str, TaskIssue] = {} + + def create(self, issue: TaskIssue) -> TaskIssue: + self._store[issue.id] = issue + return issue + + def get_by_id(self, issue_id: str) -> TaskIssue | None: + return self._store.get(issue_id) + + def list_by_task(self, task_id: str) -> list[TaskIssue]: + return [i for i in self._store.values() if i.task_id == task_id] + + def list_by_project(self, project_id: str) -> list[TaskIssue]: + return [i for i in self._store.values() if i.project_id == project_id] + + def update(self, issue: TaskIssue) -> TaskIssue: + if issue.id not in self._store: + raise ValueError(f"TaskIssue {issue.id} not found") + self._store[issue.id] = issue + return issue + + def delete(self, issue_id: str) -> None: + self._store.pop(issue_id, None) diff --git a/packages/adapters/sqlalchemy_impl/models.py b/packages/adapters/sqlalchemy_impl/models.py index 93a172b0a..b6081916c 100644 --- a/packages/adapters/sqlalchemy_impl/models.py +++ b/packages/adapters/sqlalchemy_impl/models.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, DateTime, String, Text, create_engine +from sqlalchemy import Boolean, Column, DateTime, Float, String, Text, create_engine from sqlalchemy.orm import declarative_base from datetime import datetime, timezone @@ -53,3 +53,56 @@ class IngestJobModel(Base): result_asset_id = Column(String(32), nullable=False, default="") created_at = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc)) updated_at = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class TaskModel(Base): + __tablename__ = "tasks" + + id = Column(String(32), primary_key=True) + project_id = Column(String(32), nullable=False, index=True) + workspace_id = Column(String(32), nullable=False, index=True) + name = Column(String(200), nullable=False) + description = Column(Text, nullable=False, default="") + status = Column(String(20), nullable=False, default="pending", index=True) + priority = Column(String(20), nullable=False, default="medium") + parent_task_id = Column(String(32), nullable=False, default="", index=True) + assignee_user_id = Column(String(32), nullable=False, default="") + progress = Column(Float, nullable=False, default=0.0) + planned_start_date = Column(DateTime, nullable=True) + planned_end_date = Column(DateTime, nullable=True) + actual_start_date = Column(DateTime, nullable=True) + actual_end_date = Column(DateTime, nullable=True) + tags_json = Column(Text, nullable=False, default="[]") + created_at = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc)) + updated_at = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class MilestoneModel(Base): + __tablename__ = "milestones" + + id = Column(String(32), primary_key=True) + project_id = Column(String(32), nullable=False, index=True) + workspace_id = Column(String(32), nullable=False, index=True) + name = Column(String(200), nullable=False) + description = Column(Text, nullable=False, default="") + target_date = Column(DateTime, nullable=True) + completed = Column(Boolean, nullable=False, default=False) + completed_at = Column(DateTime, nullable=True) + created_at = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc)) + updated_at = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class TaskIssueModel(Base): + __tablename__ = "task_issues" + + id = Column(String(32), primary_key=True) + task_id = Column(String(32), nullable=False, index=True) + project_id = Column(String(32), nullable=False, index=True) + workspace_id = Column(String(32), nullable=False, index=True) + title = Column(String(200), nullable=False) + description = Column(Text, nullable=False, default="") + resolved = Column(Boolean, nullable=False, default=False) + resolved_at = Column(DateTime, nullable=True) + created_by_user_id = Column(String(32), nullable=False, default="") + created_at = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc)) + updated_at = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc)) diff --git a/packages/adapters/sqlalchemy_impl/project_management_repositories.py b/packages/adapters/sqlalchemy_impl/project_management_repositories.py new file mode 100644 index 000000000..3790234b4 --- /dev/null +++ b/packages/adapters/sqlalchemy_impl/project_management_repositories.py @@ -0,0 +1,243 @@ +"""项目管理 SQLAlchemy Repository 实现""" +import json +from sqlalchemy.orm import Session + +from packages.domain import Milestone, Task, TaskIssue +from packages.ports.project_management_repositories import ( + MilestoneRepository, + TaskIssueRepository, + TaskRepository, +) +from .models import MilestoneModel, TaskIssueModel, TaskModel + + +class SQLAlchemyTaskRepository(TaskRepository): + """任务 SQLAlchemy 仓储实现""" + + def __init__(self, session: Session): + self._session = session + + def create(self, task: Task) -> Task: + model = TaskModel( + id=task.id, + project_id=task.project_id, + workspace_id=task.workspace_id, + name=task.name, + description=task.description, + status=task.status.value, + priority=task.priority.value, + parent_task_id=task.parent_task_id, + assignee_user_id=task.assignee_user_id, + progress=task.progress, + planned_start_date=task.planned_start_date, + planned_end_date=task.planned_end_date, + actual_start_date=task.actual_start_date, + actual_end_date=task.actual_end_date, + tags_json=json.dumps(task.tags, ensure_ascii=False), + created_at=task.created_at, + updated_at=task.updated_at, + ) + self._session.add(model) + self._session.commit() + return task + + def get_by_id(self, task_id: str) -> Task | None: + model = self._session.query(TaskModel).filter(TaskModel.id == task_id).first() + if not model: + return None + return self._model_to_entity(model) + + def list_by_project(self, project_id: str) -> list[Task]: + models = self._session.query(TaskModel).filter(TaskModel.project_id == project_id).all() + return [self._model_to_entity(m) for m in models] + + def list_by_parent(self, parent_task_id: str) -> list[Task]: + models = self._session.query(TaskModel).filter(TaskModel.parent_task_id == parent_task_id).all() + return [self._model_to_entity(m) for m in models] + + def update(self, task: Task) -> Task: + model = self._session.query(TaskModel).filter(TaskModel.id == task.id).first() + if not model: + raise ValueError(f"Task {task.id} not found") + + model.name = task.name + model.description = task.description + model.status = task.status.value + model.priority = task.priority.value + model.parent_task_id = task.parent_task_id + model.assignee_user_id = task.assignee_user_id + model.progress = task.progress + model.planned_start_date = task.planned_start_date + model.planned_end_date = task.planned_end_date + model.actual_start_date = task.actual_start_date + model.actual_end_date = task.actual_end_date + model.tags_json = json.dumps(task.tags, ensure_ascii=False) + model.updated_at = task.updated_at + + self._session.commit() + return task + + def delete(self, task_id: str) -> None: + self._session.query(TaskModel).filter(TaskModel.id == task_id).delete() + self._session.commit() + + def _model_to_entity(self, model: TaskModel) -> Task: + from packages.domain.project_management import TaskPriority, TaskStatus + return Task( + id=model.id, + project_id=model.project_id, + workspace_id=model.workspace_id, + name=model.name, + description=model.description, + status=TaskStatus(model.status), + priority=TaskPriority(model.priority), + parent_task_id=model.parent_task_id, + assignee_user_id=model.assignee_user_id, + progress=model.progress, + planned_start_date=model.planned_start_date, + planned_end_date=model.planned_end_date, + actual_start_date=model.actual_start_date, + actual_end_date=model.actual_end_date, + tags=json.loads(model.tags_json), + created_at=model.created_at, + updated_at=model.updated_at, + ) + + +class SQLAlchemyMilestoneRepository(MilestoneRepository): + """里程碑 SQLAlchemy 仓储实现""" + + def __init__(self, session: Session): + self._session = session + + def create(self, milestone: Milestone) -> Milestone: + model = MilestoneModel( + id=milestone.id, + project_id=milestone.project_id, + workspace_id=milestone.workspace_id, + name=milestone.name, + description=milestone.description, + target_date=milestone.target_date, + completed=milestone.completed, + completed_at=milestone.completed_at, + created_at=milestone.created_at, + updated_at=milestone.updated_at, + ) + self._session.add(model) + self._session.commit() + return milestone + + def get_by_id(self, milestone_id: str) -> Milestone | None: + model = self._session.query(MilestoneModel).filter(MilestoneModel.id == milestone_id).first() + if not model: + return None + return self._model_to_entity(model) + + def list_by_project(self, project_id: str) -> list[Milestone]: + models = self._session.query(MilestoneModel).filter(MilestoneModel.project_id == project_id).all() + return [self._model_to_entity(m) for m in models] + + def update(self, milestone: Milestone) -> Milestone: + model = self._session.query(MilestoneModel).filter(MilestoneModel.id == milestone.id).first() + if not model: + raise ValueError(f"Milestone {milestone.id} not found") + + model.name = milestone.name + model.description = milestone.description + model.target_date = milestone.target_date + model.completed = milestone.completed + model.completed_at = milestone.completed_at + model.updated_at = milestone.updated_at + + self._session.commit() + return milestone + + def delete(self, milestone_id: str) -> None: + self._session.query(MilestoneModel).filter(MilestoneModel.id == milestone_id).delete() + self._session.commit() + + def _model_to_entity(self, model: MilestoneModel) -> Milestone: + return Milestone( + id=model.id, + project_id=model.project_id, + workspace_id=model.workspace_id, + name=model.name, + description=model.description, + target_date=model.target_date, + completed=model.completed, + completed_at=model.completed_at, + created_at=model.created_at, + updated_at=model.updated_at, + ) + + +class SQLAlchemyTaskIssueRepository(TaskIssueRepository): + """任务问题 SQLAlchemy 仓储实现""" + + def __init__(self, session: Session): + self._session = session + + def create(self, issue: TaskIssue) -> TaskIssue: + model = TaskIssueModel( + id=issue.id, + task_id=issue.task_id, + project_id=issue.project_id, + workspace_id=issue.workspace_id, + title=issue.title, + description=issue.description, + resolved=issue.resolved, + resolved_at=issue.resolved_at, + created_by_user_id=issue.created_by_user_id, + created_at=issue.created_at, + updated_at=issue.updated_at, + ) + self._session.add(model) + self._session.commit() + return issue + + def get_by_id(self, issue_id: str) -> TaskIssue | None: + model = self._session.query(TaskIssueModel).filter(TaskIssueModel.id == issue_id).first() + if not model: + return None + return self._model_to_entity(model) + + def list_by_task(self, task_id: str) -> list[TaskIssue]: + models = self._session.query(TaskIssueModel).filter(TaskIssueModel.task_id == task_id).all() + return [self._model_to_entity(m) for m in models] + + def list_by_project(self, project_id: str) -> list[TaskIssue]: + models = self._session.query(TaskIssueModel).filter(TaskIssueModel.project_id == project_id).all() + return [self._model_to_entity(m) for m in models] + + def update(self, issue: TaskIssue) -> TaskIssue: + model = self._session.query(TaskIssueModel).filter(TaskIssueModel.id == issue.id).first() + if not model: + raise ValueError(f"TaskIssue {issue.id} not found") + + model.title = issue.title + model.description = issue.description + model.resolved = issue.resolved + model.resolved_at = issue.resolved_at + model.updated_at = issue.updated_at + + self._session.commit() + return issue + + def delete(self, issue_id: str) -> None: + self._session.query(TaskIssueModel).filter(TaskIssueModel.id == issue_id).delete() + self._session.commit() + + def _model_to_entity(self, model: TaskIssueModel) -> TaskIssue: + return TaskIssue( + id=model.id, + task_id=model.task_id, + project_id=model.project_id, + workspace_id=model.workspace_id, + title=model.title, + description=model.description, + resolved=model.resolved, + resolved_at=model.resolved_at, + created_by_user_id=model.created_by_user_id, + created_at=model.created_at, + updated_at=model.updated_at, + ) diff --git a/packages/application/project_management_use_cases.py b/packages/application/project_management_use_cases.py new file mode 100644 index 000000000..9efdba7f6 --- /dev/null +++ b/packages/application/project_management_use_cases.py @@ -0,0 +1,151 @@ +"""项目管理 Use Cases""" +from packages.domain import Milestone, Task, TaskIssue, TaskPriority, TaskStatus +from packages.ports import MilestoneRepository, TaskIssueRepository, TaskRepository + + +class CreateTaskUseCase: + """创建任务用例""" + + def __init__(self, task_repo: TaskRepository): + self.task_repo = task_repo + + def execute( + self, + project_id: str, + workspace_id: str, + name: str, + description: str = "", + priority: TaskPriority = TaskPriority.MEDIUM, + parent_task_id: str = "", + assignee_user_id: str = "", + ) -> Task: + task = Task.create( + project_id=project_id, + workspace_id=workspace_id, + name=name, + description=description, + priority=priority, + parent_task_id=parent_task_id, + assignee_user_id=assignee_user_id, + ) + return self.task_repo.create(task) + + +class ListProjectTasksUseCase: + """获取项目任务列表用例""" + + def __init__(self, task_repo: TaskRepository): + self.task_repo = task_repo + + def execute(self, project_id: str) -> list[Task]: + return self.task_repo.list_by_project(project_id) + + +class UpdateTaskStatusUseCase: + """更新任务状态用例""" + + def __init__(self, task_repo: TaskRepository): + self.task_repo = task_repo + + def execute(self, task_id: str, new_status: TaskStatus) -> Task: + task = self.task_repo.get_by_id(task_id) + if not task: + raise ValueError(f"Task {task_id} not found") + task.update_status(new_status) + return self.task_repo.update(task) + + +class UpdateTaskProgressUseCase: + """更新任务进度用例""" + + def __init__(self, task_repo: TaskRepository): + self.task_repo = task_repo + + def execute(self, task_id: str, progress: float) -> Task: + task = self.task_repo.get_by_id(task_id) + if not task: + raise ValueError(f"Task {task_id} not found") + task.update_progress(progress) + return self.task_repo.update(task) + + +class CreateMilestoneUseCase: + """创建里程碑用例""" + + def __init__(self, milestone_repo: MilestoneRepository): + self.milestone_repo = milestone_repo + + def execute( + self, + project_id: str, + workspace_id: str, + name: str, + description: str = "", + ) -> Milestone: + milestone = Milestone.create( + project_id=project_id, + workspace_id=workspace_id, + name=name, + description=description, + ) + return self.milestone_repo.create(milestone) + + +class ListProjectMilestonesUseCase: + """获取项目里程碑列表用例""" + + def __init__(self, milestone_repo: MilestoneRepository): + self.milestone_repo = milestone_repo + + def execute(self, project_id: str) -> list[Milestone]: + return self.milestone_repo.list_by_project(project_id) + + +class CreateTaskIssueUseCase: + """创建任务问题用例""" + + def __init__(self, issue_repo: TaskIssueRepository): + self.issue_repo = issue_repo + + def execute( + self, + task_id: str, + project_id: str, + workspace_id: str, + title: str, + description: str = "", + created_by_user_id: str = "", + ) -> TaskIssue: + issue = TaskIssue.create( + task_id=task_id, + project_id=project_id, + workspace_id=workspace_id, + title=title, + description=description, + created_by_user_id=created_by_user_id, + ) + return self.issue_repo.create(issue) + + +class ListTaskIssuesUseCase: + """获取任务问题列表用例""" + + def __init__(self, issue_repo: TaskIssueRepository): + self.issue_repo = issue_repo + + def execute(self, task_id: str) -> list[TaskIssue]: + return self.issue_repo.list_by_task(task_id) + + +class ResolveTaskIssueUseCase: + """解决任务问题用例""" + + def __init__(self, issue_repo: TaskIssueRepository): + self.issue_repo = issue_repo + + def execute(self, issue_id: str) -> TaskIssue: + issue = self.issue_repo.get_by_id(issue_id) + if not issue: + raise ValueError(f"TaskIssue {issue_id} not found") + issue.mark_resolved() + return self.issue_repo.update(issue) diff --git a/packages/domain/__init__.py b/packages/domain/__init__.py index 087eb05b4..7b081b6b1 100644 --- a/packages/domain/__init__.py +++ b/packages/domain/__init__.py @@ -2,6 +2,7 @@ from .classification import AssetClassification, ClassificationJob, ClassificationJobStatus from .entities import Asset, AssetLibrary, AssetLibraryKind, IngestJob, IngestJobStatus, Project, User, Workspace +from .project_management import Milestone, Task, TaskIssue, TaskPriority, TaskStatus __all__ = [ "Asset", @@ -12,7 +13,12 @@ __all__ = [ "ClassificationJobStatus", "IngestJob", "IngestJobStatus", + "Milestone", "Project", + "Task", + "TaskIssue", + "TaskPriority", + "TaskStatus", "User", "Workspace", ] diff --git a/packages/domain/project_management.py b/packages/domain/project_management.py new file mode 100644 index 000000000..331bb3ccf --- /dev/null +++ b/packages/domain/project_management.py @@ -0,0 +1,241 @@ +"""项目管理领域对象:任务、里程碑、项目阶段""" +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import StrEnum +from uuid import uuid4 + + +class TaskStatus(StrEnum): + """任务状态""" + PENDING = "pending" # 待开始 + IN_PROGRESS = "in_progress" # 进行中 + BLOCKED = "blocked" # 阻塞 + COMPLETED = "completed" # 已完成 + CANCELLED = "cancelled" # 已取消 + + +class TaskPriority(StrEnum): + """任务优先级""" + LOW = "low" + MEDIUM = "medium" + HIGH = "high" + URGENT = "urgent" + + +@dataclass(slots=True) +class Task: + """任务实体""" + id: str + project_id: str + workspace_id: str + name: str + description: str = "" + status: TaskStatus = TaskStatus.PENDING + priority: TaskPriority = TaskPriority.MEDIUM + parent_task_id: str = "" # 父任务ID(支持子任务层级) + assignee_user_id: str = "" # 负责人 + progress: float = 0.0 # 进度 0-100 + planned_start_date: datetime | None = None + planned_end_date: datetime | None = None + actual_start_date: datetime | None = None + actual_end_date: datetime | None = None + tags: list[str] = field(default_factory=list) + created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + + @classmethod + def create( + cls, + project_id: str, + workspace_id: str, + name: str, + description: str = "", + priority: TaskPriority = TaskPriority.MEDIUM, + parent_task_id: str = "", + assignee_user_id: str = "", + planned_start_date: datetime | None = None, + planned_end_date: datetime | None = None, + ) -> "Task": + """创建任务""" + clean_name = name.strip() + if not clean_name: + raise ValueError("任务名称不能为空") + if not project_id.strip(): + raise ValueError("project_id 不能为空") + if not workspace_id.strip(): + raise ValueError("workspace_id 不能为空") + + return cls( + id=uuid4().hex, + project_id=project_id.strip(), + workspace_id=workspace_id.strip(), + name=clean_name, + description=description.strip(), + priority=priority, + parent_task_id=parent_task_id.strip(), + assignee_user_id=assignee_user_id.strip(), + planned_start_date=planned_start_date, + planned_end_date=planned_end_date, + ) + + def update_status(self, new_status: TaskStatus) -> None: + """更新任务状态""" + self.status = new_status + self.updated_at = datetime.now(timezone.utc) + + # 自动设置实际开始/结束时间 + if new_status == TaskStatus.IN_PROGRESS and self.actual_start_date is None: + self.actual_start_date = datetime.now(timezone.utc) + elif new_status == TaskStatus.COMPLETED and self.actual_end_date is None: + self.actual_end_date = datetime.now(timezone.utc) + self.progress = 100.0 + + def update_progress(self, progress: float) -> None: + """更新任务进度""" + if not 0 <= progress <= 100: + raise ValueError("进度必须在 0-100 之间") + self.progress = progress + self.updated_at = datetime.now(timezone.utc) + + # 自动更新状态 + if progress > 0 and self.status == TaskStatus.PENDING: + self.status = TaskStatus.IN_PROGRESS + if progress == 100 and self.status != TaskStatus.COMPLETED: + self.status = TaskStatus.COMPLETED + if self.actual_end_date is None: + self.actual_end_date = datetime.now(timezone.utc) + + def add_tag(self, tag: str) -> None: + """添加标签""" + clean_tag = tag.strip() + if not clean_tag: + raise ValueError("标签不能为空") + if clean_tag not in self.tags: + self.tags.append(clean_tag) + self.updated_at = datetime.now(timezone.utc) + + def remove_tag(self, tag: str) -> None: + """删除标签""" + clean_tag = tag.strip() + if clean_tag in self.tags: + self.tags.remove(clean_tag) + self.updated_at = datetime.now(timezone.utc) + + +@dataclass(slots=True) +class Milestone: + """里程碑实体""" + id: str + project_id: str + workspace_id: str + name: str + description: str = "" + target_date: datetime | None = None + completed: bool = False + completed_at: datetime | None = None + created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + + @classmethod + def create( + cls, + project_id: str, + workspace_id: str, + name: str, + description: str = "", + target_date: datetime | None = None, + ) -> "Milestone": + """创建里程碑""" + clean_name = name.strip() + if not clean_name: + raise ValueError("里程碑名称不能为空") + if not project_id.strip(): + raise ValueError("project_id 不能为空") + if not workspace_id.strip(): + raise ValueError("workspace_id 不能为空") + + return cls( + id=uuid4().hex, + project_id=project_id.strip(), + workspace_id=workspace_id.strip(), + name=clean_name, + description=description.strip(), + target_date=target_date, + ) + + def mark_completed(self) -> None: + """标记为已完成""" + if not self.completed: + self.completed = True + self.completed_at = datetime.now(timezone.utc) + self.updated_at = datetime.now(timezone.utc) + + def reopen(self) -> None: + """重新打开里程碑""" + if self.completed: + self.completed = False + self.completed_at = None + self.updated_at = datetime.now(timezone.utc) + + +@dataclass(slots=True) +class TaskIssue: + """任务问题/卡点实体""" + id: str + task_id: str + project_id: str + workspace_id: str + title: str + description: str = "" + resolved: bool = False + resolved_at: datetime | None = None + created_by_user_id: str = "" + created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + + @classmethod + def create( + cls, + task_id: str, + project_id: str, + workspace_id: str, + title: str, + description: str = "", + created_by_user_id: str = "", + ) -> "TaskIssue": + """创建任务问题""" + clean_title = title.strip() + if not clean_title: + raise ValueError("问题标题不能为空") + if not task_id.strip(): + raise ValueError("task_id 不能为空") + if not project_id.strip(): + raise ValueError("project_id 不能为空") + if not workspace_id.strip(): + raise ValueError("workspace_id 不能为空") + + return cls( + id=uuid4().hex, + task_id=task_id.strip(), + project_id=project_id.strip(), + workspace_id=workspace_id.strip(), + title=clean_title, + description=description.strip(), + created_by_user_id=created_by_user_id.strip(), + ) + + def mark_resolved(self) -> None: + """标记为已解决""" + if not self.resolved: + self.resolved = True + self.resolved_at = datetime.now(timezone.utc) + self.updated_at = datetime.now(timezone.utc) + + def reopen(self) -> None: + """重新打开问题""" + if self.resolved: + self.resolved = False + self.resolved_at = None + self.updated_at = datetime.now(timezone.utc) diff --git a/packages/ports/__init__.py b/packages/ports/__init__.py index 6a6259f64..f53fe2cb4 100644 --- a/packages/ports/__init__.py +++ b/packages/ports/__init__.py @@ -3,11 +3,15 @@ from .asset_library_repository import AssetLibraryRepository from .asset_repository import AssetRepository from .ingest_job_repository import IngestJobRepository +from .project_management_repositories import MilestoneRepository, TaskIssueRepository, TaskRepository from .project_repository import ProjectRepository __all__ = [ "AssetLibraryRepository", "AssetRepository", "IngestJobRepository", + "MilestoneRepository", "ProjectRepository", + "TaskIssueRepository", + "TaskRepository", ] diff --git a/packages/ports/project_management_repositories.py b/packages/ports/project_management_repositories.py new file mode 100644 index 000000000..83e087faa --- /dev/null +++ b/packages/ports/project_management_repositories.py @@ -0,0 +1,101 @@ +"""项目管理 Repository 接口定义""" +from abc import ABC, abstractmethod + +from packages.domain import Milestone, Task, TaskIssue + + +class TaskRepository(ABC): + """任务仓储接口""" + + @abstractmethod + def create(self, task: Task) -> Task: + """创建任务""" + pass + + @abstractmethod + def get_by_id(self, task_id: str) -> Task | None: + """根据ID获取任务""" + pass + + @abstractmethod + def list_by_project(self, project_id: str) -> list[Task]: + """获取项目下的所有任务""" + pass + + @abstractmethod + def list_by_parent(self, parent_task_id: str) -> list[Task]: + """获取子任务列表""" + pass + + @abstractmethod + def update(self, task: Task) -> Task: + """更新任务""" + pass + + @abstractmethod + def delete(self, task_id: str) -> None: + """删除任务""" + pass + + +class MilestoneRepository(ABC): + """里程碑仓储接口""" + + @abstractmethod + def create(self, milestone: Milestone) -> Milestone: + """创建里程碑""" + pass + + @abstractmethod + def get_by_id(self, milestone_id: str) -> Milestone | None: + """根据ID获取里程碑""" + pass + + @abstractmethod + def list_by_project(self, project_id: str) -> list[Milestone]: + """获取项目下的所有里程碑""" + pass + + @abstractmethod + def update(self, milestone: Milestone) -> Milestone: + """更新里程碑""" + pass + + @abstractmethod + def delete(self, milestone_id: str) -> None: + """删除里程碑""" + pass + + +class TaskIssueRepository(ABC): + """任务问题仓储接口""" + + @abstractmethod + def create(self, issue: TaskIssue) -> TaskIssue: + """创建任务问题""" + pass + + @abstractmethod + def get_by_id(self, issue_id: str) -> TaskIssue | None: + """根据ID获取任务问题""" + pass + + @abstractmethod + def list_by_task(self, task_id: str) -> list[TaskIssue]: + """获取任务下的所有问题""" + pass + + @abstractmethod + def list_by_project(self, project_id: str) -> list[TaskIssue]: + """获取项目下的所有问题""" + pass + + @abstractmethod + def update(self, issue: TaskIssue) -> TaskIssue: + """更新任务问题""" + pass + + @abstractmethod + def delete(self, issue_id: str) -> None: + """删除任务问题""" + pass diff --git a/tests/integration/test_project_management.py b/tests/integration/test_project_management.py new file mode 100644 index 000000000..8be018887 --- /dev/null +++ b/tests/integration/test_project_management.py @@ -0,0 +1,196 @@ +"""项目管理功能集成测试""" +import pytest + +from packages.adapters.in_memory.project_management_repositories import ( + InMemoryMilestoneRepository, + InMemoryTaskIssueRepository, + InMemoryTaskRepository, +) +from packages.application.project_management_use_cases import ( + CreateMilestoneUseCase, + CreateTaskIssueUseCase, + CreateTaskUseCase, + ListProjectTasksUseCase, + ListTaskIssuesUseCase, + ResolveTaskIssueUseCase, + UpdateTaskProgressUseCase, + UpdateTaskStatusUseCase, +) +from packages.domain import TaskPriority, TaskStatus + + +def test_create_task(): + """测试创建任务""" + repo = InMemoryTaskRepository() + use_case = CreateTaskUseCase(repo) + + task = use_case.execute( + project_id="proj_1", + workspace_id="ws_1", + name="开发登录功能", + description="实现用户登录功能", + priority=TaskPriority.HIGH, + ) + + assert task.id is not None + assert task.name == "开发登录功能" + assert task.status == TaskStatus.PENDING + assert task.priority == TaskPriority.HIGH + assert task.progress == 0.0 + + +def test_list_tasks(): + """测试获取任务列表""" + repo = InMemoryTaskRepository() + create_use_case = CreateTaskUseCase(repo) + + # 创建两个任务 + create_use_case.execute( + project_id="proj_1", + workspace_id="ws_1", + name="任务1", + ) + create_use_case.execute( + project_id="proj_1", + workspace_id="ws_1", + name="任务2", + ) + + # 查询任务列表 + list_use_case = ListProjectTasksUseCase(repo) + tasks = list_use_case.execute("proj_1") + + assert len(tasks) == 2 + assert tasks[0].name == "任务1" + assert tasks[1].name == "任务2" + + +def test_update_task_status(): + """测试更新任务状态""" + repo = InMemoryTaskRepository() + create_use_case = CreateTaskUseCase(repo) + update_use_case = UpdateTaskStatusUseCase(repo) + + # 创建任务 + task = create_use_case.execute( + project_id="proj_1", + workspace_id="ws_1", + name="测试任务", + ) + + # 更新状态为进行中 + updated_task = update_use_case.execute(task.id, TaskStatus.IN_PROGRESS) + + assert updated_task.status == TaskStatus.IN_PROGRESS + assert updated_task.actual_start_date is not None + + +def test_update_task_progress(): + """测试更新任务进度""" + repo = InMemoryTaskRepository() + create_use_case = CreateTaskUseCase(repo) + progress_use_case = UpdateTaskProgressUseCase(repo) + + # 创建任务 + task = create_use_case.execute( + project_id="proj_1", + workspace_id="ws_1", + name="测试任务", + ) + + # 更新进度到 50% + updated_task = progress_use_case.execute(task.id, 50.0) + + assert updated_task.progress == 50.0 + assert updated_task.status == TaskStatus.IN_PROGRESS + + # 更新进度到 100% + completed_task = progress_use_case.execute(task.id, 100.0) + + assert completed_task.progress == 100.0 + assert completed_task.status == TaskStatus.COMPLETED + assert completed_task.actual_end_date is not None + + +def test_create_milestone(): + """测试创建里程碑""" + repo = InMemoryMilestoneRepository() + use_case = CreateMilestoneUseCase(repo) + + milestone = use_case.execute( + project_id="proj_1", + workspace_id="ws_1", + name="V1.0 发布", + description="第一个正式版本", + ) + + assert milestone.id is not None + assert milestone.name == "V1.0 发布" + assert milestone.completed is False + + +def test_create_and_resolve_issue(): + """测试创建和解决任务问题""" + repo = InMemoryTaskIssueRepository() + create_use_case = CreateTaskIssueUseCase(repo) + resolve_use_case = ResolveTaskIssueUseCase(repo) + list_use_case = ListTaskIssuesUseCase(repo) + + # 创建问题 + issue = create_use_case.execute( + task_id="task_1", + project_id="proj_1", + workspace_id="ws_1", + title="接口报错", + description="调用登录接口返回 500", + ) + + assert issue.id is not None + assert issue.title == "接口报错" + assert issue.resolved is False + + # 解决问题 + resolved_issue = resolve_use_case.execute(issue.id) + + assert resolved_issue.resolved is True + assert resolved_issue.resolved_at is not None + + # 查询任务问题列表 + issues = list_use_case.execute("task_1") + assert len(issues) == 1 + assert issues[0].resolved is True + + +def test_task_hierarchy(): + """测试任务层级关系""" + repo = InMemoryTaskRepository() + create_use_case = CreateTaskUseCase(repo) + + # 创建父任务 + parent_task = create_use_case.execute( + project_id="proj_1", + workspace_id="ws_1", + name="开发用户模块", + ) + + # 创建子任务 + child_task_1 = create_use_case.execute( + project_id="proj_1", + workspace_id="ws_1", + name="登录功能", + parent_task_id=parent_task.id, + ) + + child_task_2 = create_use_case.execute( + project_id="proj_1", + workspace_id="ws_1", + name="注册功能", + parent_task_id=parent_task.id, + ) + + # 查询子任务 + children = repo.list_by_parent(parent_task.id) + + assert len(children) == 2 + assert children[0].parent_task_id == parent_task.id + assert children[1].parent_task_id == parent_task.id -- 2.54.0 From 9dd36f308f1c4a78c4dbddff960eb640e505582a Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Tue, 16 Jun 2026 12:25:37 +0800 Subject: [PATCH 020/112] feat: add project management frontend pages - Home page with navigation - Projects page with task list table - Task status/priority/progress display - Real-time API integration - Responsive layout and styling --- STATUS.md | 35 +++-- apps/web/app/globals.css | 26 ++++ apps/web/app/layout.tsx | 15 +- apps/web/app/page.tsx | 54 ++++++-- apps/web/app/projects/page.tsx | 243 +++++++++++++++++++++++++++++++++ 5 files changed, 348 insertions(+), 25 deletions(-) create mode 100644 apps/web/app/globals.css create mode 100644 apps/web/app/projects/page.tsx diff --git a/STATUS.md b/STATUS.md index f2cd3a60e..43c541c61 100644 --- a/STATUS.md +++ b/STATUS.md @@ -7,7 +7,8 @@ ## 当前阶段 -**Phase 1 已完成** - 核心平台层已搭建完成并稳定化中 +**Phase 1 已完成** - 核心平台层已搭建完成并稳定化中 +**Phase 2 进行中** - 项目管理模块已落地 --- @@ -28,11 +29,17 @@ - ✅ Asset(素材) - ✅ IngestJob(入库任务:PENDING → PROCESSING → COMPLETED/FAILED) - ✅ ClassificationJob(分类任务:含 9 种分类) +- ✅ **Task(任务:支持状态/进度/优先级/子任务/计划时间/实际时间)** +- ✅ **Milestone(里程碑)** +- ✅ **TaskIssue(任务问题/卡点)** ### 核心业务流程 - ✅ 上传 → 入库 → Asset 创建链路 - ✅ 分类任务链路 - ✅ 完整异步任务处理(Celery + Redis) +- ✅ **任务创建 → 状态更新 → 进度跟踪链路** +- ✅ **里程碑管理** +- ✅ **任务问题/卡点记录与解决** ### 基础设施 - ✅ **MinIO 真实文件存储**(`apps/api/app/core/storage.py`) @@ -41,29 +48,34 @@ - ✅ API 文档(Swagger) ### 测试与质量 -- ✅ 8 个集成测试全绿 +- ✅ **15 个集成测试全绿**(素材管理 8 个 + 项目管理 7 个) - ✅ 测试策略:集成测试优先,覆盖核心业务流程 --- ## 进行中 +### 项目推进器前端页面 +- 🔄 Next.js 页面设计与开发(下一步) +- 🔄 树形任务视图 +- 🔄 甘特图视图 +- 🔄 里程碑时间轴 +- 🔄 问题卡点面板 + ### 稳定化与部署打磨 -- 🔄 Gitea workflows 持续优化(最近 10+ commits) +- 🔄 Gitea workflows 持续优化 - 🔄 部署流程与环境隔离稳定化 - 🔄 Docker 构建优化(国内镜像源、多阶段构建) -### 前端页面 -- 🔄 Next.js 前端占位已建立,实际页面待补 - --- ## 下一步优先级 -1. **前端页面补齐** - - 项目/素材库管理页面 - - 素材上传/预览页面 - - 任务状态监控页面 +1. **项目推进器前端页面** + - 任务列表与详情页 + - 里程碑管理页 + - 问题卡点面板 + - 甘特图/时间线视图 2. **更完整的 SAAS 产品化能力** - 多租户/组织/权限体系 @@ -115,6 +127,7 @@ - 测试策略:集成测试优先,覆盖核心业务流程 - 数据库迁移从第一天起就版本化管理 - CI/CD 基于 Gitea Actions + 自建 runner +- **旧项目推进器(纯前端 HTML)已废弃,项目管理功能重新在 SAAS 里实现** --- @@ -123,7 +136,7 @@ - **本地路径**: `F:\openclaw-saas` - **远程仓库**: `xiaoxia-server:/var/lib/xiaoxia-ci/xiaoxia-saas.git` - **分支**: `main` -- **最新提交**: `2c78407 fix: pass commit metadata into deploy builds` +- **最新提交**: `355dd0f feat: add project management module (tasks, milestones, issues)` --- diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css new file mode 100644 index 000000000..6209f4970 --- /dev/null +++ b/apps/web/app/globals.css @@ -0,0 +1,26 @@ +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, "Microsoft YaHei", sans-serif; + background: #F5F7FA; + color: #1D2129; + height: 100vh; + overflow: hidden; +} + +:root { + --primary: #165DFF; + --primary-hover: #0e48d1; + --border: #E5E6EB; + --bg-white: #fff; + --bg-gray: #F5F7FA; + --text-primary: #1D2129; + --text-secondary: #6E7681; + --success: #00B42A; + --warning: #FF7D00; + --error: #F53F3F; +} diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx index 928393710..87f087c02 100644 --- a/apps/web/app/layout.tsx +++ b/apps/web/app/layout.tsx @@ -1,9 +1,16 @@ -export const metadata = { - title: 'xiaoxia-saas', - description: 'Xiaoxia SaaS scaffold', +import type { Metadata } from "next"; +import "./globals.css"; + +export const metadata: Metadata = { + title: "小虾 SaaS - 项目推进器", + description: "AI 视频自动化剪辑系统 - 项目管理", }; -export default function RootLayout({ children }: { children: React.ReactNode }) { +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { return ( {children} diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx index 5015b7760..6afcdd944 100644 --- a/apps/web/app/page.tsx +++ b/apps/web/app/page.tsx @@ -1,14 +1,48 @@ export default function HomePage() { return ( -
-

xiaoxia-saas

-

SaaS 新主线前端骨架已创建。

-
    -
  • 项目管理
  • -
  • 素材管理
  • -
  • 配音管理
  • -
  • 任务与成片
  • -
-
+
+

+ 📁 小虾 SaaS 项目推进器 +

+

+ 项目管理模块开发中... +

+ +
); } diff --git a/apps/web/app/projects/page.tsx b/apps/web/app/projects/page.tsx new file mode 100644 index 000000000..2a3dddfe2 --- /dev/null +++ b/apps/web/app/projects/page.tsx @@ -0,0 +1,243 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import Link from 'next/link'; + +interface Task { + id: string; + name: string; + status: string; + priority: string; + progress: number; + assignee_user_id: string; + created_at: string; + updated_at: string; +} + +const API_BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000'; + +export default function ProjectsPage() { + const [tasks, setTasks] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(''); + + // 模拟项目ID,生产环境应该从路由或上下文获取 + const projectId = 'demo_project_1'; + + useEffect(() => { + fetchTasks(); + }, []); + + const fetchTasks = async () => { + try { + const res = await fetch(`${API_BASE}/api/v1/project-management/tasks?project_id=${projectId}`); + if (!res.ok) throw new Error('获取任务列表失败'); + const data = await res.json(); + setTasks(data); + } catch (err: any) { + setError(err.message); + } finally { + setLoading(false); + } + }; + + const getStatusColor = (status: string) => { + const colors: Record = { + pending: '#86909C', + in_progress: '#165DFF', + completed: '#00B42A', + blocked: '#F53F3F', + cancelled: '#6E7681', + }; + return colors[status] || '#6E7681'; + }; + + const getStatusText = (status: string) => { + const texts: Record = { + pending: '待开始', + in_progress: '进行中', + completed: '已完成', + blocked: '阻塞', + cancelled: '已取消', + }; + return texts[status] || status; + }; + + const getPriorityText = (priority: string) => { + const texts: Record = { + low: '低', + medium: '中', + high: '高', + urgent: '紧急', + }; + return texts[priority] || priority; + }; + + if (loading) { + return ( +
+

加载中...

+
+ ); + } + + return ( +
+ {/* Header */} +
+
+ + 📁 项目推进器 + + Demo 项目 +
+
+ + +
+
+ + {/* Main Content */} +
+ {error && ( +
+ {error} +
+ )} + + {tasks.length === 0 ? ( +
+

暂无任务

+

点击右上角"+ 新增任务"创建第一个任务

+
+ ) : ( +
+ + + + + + + + + + + + {tasks.map((task) => ( + + + + + + + + ))} + +
任务名称状态优先级进度创建时间
{task.name} + + {getStatusText(task.status)} + + + {getPriorityText(task.priority)} + +
+
+
+
+ + {task.progress}% + +
+
+ {new Date(task.created_at).toLocaleDateString('zh-CN')} +
+
+ )} +
+ + {/* Footer */} +
+
当前项目:Demo 项目
+
总计任务:{tasks.length}
+
+
+ ); +} -- 2.54.0 From bda4db5e5bf945a298ec8b3440c2cffad56f4cae Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Tue, 16 Jun 2026 12:40:12 +0800 Subject: [PATCH 021/112] feat: add task creation form and detail page - CreateTaskForm component with validation - Task detail page with issue list - Integrated create form into projects page - Task name links to detail page - Added workspace_id parameter - Improved state management --- apps/web/app/components/CreateTaskForm.tsx | 200 +++++++++++++++++++ apps/web/app/projects/page.tsx | 28 ++- apps/web/app/tasks/[id]/page.tsx | 213 +++++++++++++++++++++ 3 files changed, 436 insertions(+), 5 deletions(-) create mode 100644 apps/web/app/components/CreateTaskForm.tsx create mode 100644 apps/web/app/tasks/[id]/page.tsx diff --git a/apps/web/app/components/CreateTaskForm.tsx b/apps/web/app/components/CreateTaskForm.tsx new file mode 100644 index 000000000..d130f9800 --- /dev/null +++ b/apps/web/app/components/CreateTaskForm.tsx @@ -0,0 +1,200 @@ +'use client'; + +import { useState } from 'react'; +import { useRouter } from 'next/navigation'; + +const API_BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000'; + +interface CreateTaskFormProps { + projectId: string; + workspaceId: string; + onSuccess?: () => void; + onCancel?: () => void; +} + +export default function CreateTaskForm({ projectId, workspaceId, onSuccess, onCancel }: CreateTaskFormProps) { + const router = useRouter(); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(''); + const [formData, setFormData] = useState({ + name: '', + description: '', + priority: 'medium', + assignee_user_id: '', + parent_task_id: '', + }); + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + setLoading(true); + setError(''); + + try { + const res = await fetch(`${API_BASE}/api/v1/project-management/tasks`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + project_id: projectId, + workspace_id: workspaceId, + ...formData, + }), + }); + + if (!res.ok) { + const data = await res.json(); + throw new Error(data.detail || '创建失败'); + } + + if (onSuccess) { + onSuccess(); + } else { + router.push('/projects'); + } + } catch (err: any) { + setError(err.message); + } finally { + setLoading(false); + } + }; + + return ( +
+

新增任务

+ + {error && ( +
+ {error} +
+ )} + +
+ + setFormData({ ...formData, name: e.target.value })} + style={{ + width: '100%', + padding: '8px 12px', + border: '1px solid var(--border)', + borderRadius: '4px', + fontSize: '14px', + }} + placeholder="输入任务名称" + /> +
+ +
+ +