Sync develop → main (v0.1.101) #121

Merged
xiaoxia merged 10 commits from develop into main 2026-06-30 11:10:59 +08:00
13 changed files with 240 additions and 37 deletions
+15
View File
@@ -169,6 +169,21 @@ jobs:
scp -i "$key_path" "dist/release-artifacts/xiaoxia-web-${GITHUB_REF_NAME}.tar" \
"$production_user@$production_host:/var/lib/xiaoxia-saas-production/web-${GITHUB_REF_NAME}.tar"
- name: Cleanup old Docker images
if: always()
shell: sh
run: |
set -eu
if [ -f scripts/cleanup_old_images.sh ]; then
chmod +x scripts/cleanup_old_images.sh
scripts/cleanup_old_images.sh
else
echo "Cleanup script not found, doing basic prune..."
docker image prune -f 2>/dev/null || true
fi
echo "Disk usage after cleanup:"
df -h / | tail -1
deploy-production:
name: Deploy Production
runs-on: runtime-builder:host
+1 -1
View File
@@ -202,7 +202,7 @@ def get_project_asset_diagnosis(
if project is None:
raise HTTPException(status_code=404, detail=f"Project {project_id} not found")
libraries = asset_library_repository.list_by_project(project_id)
libraries = asset_library_repository.find_by_project(project_id)
assets: list[Asset] = []
for library in libraries:
assets.extend(asset_repository.list_by_library(library.id))
+1 -1
View File
@@ -101,7 +101,7 @@ def _require_project_and_library(
if project is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found")
libraries = asset_library_repository.list_by_project(project_id)
libraries = asset_library_repository.find_by_project(project_id)
if not any(item.id == library_id for item in libraries):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Asset library not found")
+33
View File
@@ -24,6 +24,7 @@ async def readiness_check():
checks = {
"database": await _check_database(),
"redis": await _check_redis(),
"oss": _check_oss(),
}
all_healthy = all(check["status"] == "healthy" for check in checks.values())
response = {
@@ -97,6 +98,38 @@ async def _check_redis() -> dict:
}
def _check_oss() -> dict:
try:
from app.core.storage import get_storage_service
svc = get_storage_service()
if not svc.access_key_id or not svc.access_key_secret:
return {
"status": "unhealthy",
"type": "oss",
"message": "OSS credentials not configured (OSS_ACCESS_KEY_ID / OSS_ACCESS_KEY_SECRET missing)",
}
if svc.bucket is None:
return {
"status": "unhealthy",
"type": "oss",
"message": "OSS SDK (oss2) not installed or bucket client init failed",
}
# Try a lightweight OSS API call to verify connectivity & credentials
svc.bucket.get_bucket_info()
return {
"status": "healthy",
"type": "oss",
"message": f"OSS connected: endpoint={svc.endpoint} bucket={svc.bucket_name}",
}
except Exception as error:
return {
"status": "unhealthy",
"type": "oss",
"message": f"OSS check failed: {type(error).__name__}: {error}",
}
async def _check_migrations() -> dict:
if settings.USE_IN_MEMORY_DB:
return {
+35 -7
View File
@@ -1,3 +1,4 @@
import logging
from typing import Any
from uuid import uuid4
@@ -24,6 +25,8 @@ from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, s
from packages.application import GetProjectUseCase, SubmitIngestJobCommand, SubmitIngestJobUseCase
logger = logging.getLogger(__name__)
router = APIRouter()
# 允许上传的文件 MIME 类型
@@ -70,7 +73,7 @@ def _require_project_and_library(
if project is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found")
libraries = asset_library_repository.list_by_project(project_id)
libraries = asset_library_repository.find_by_project(project_id)
if not any(item.id == library_id for item in libraries):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Asset library not found")
@@ -131,7 +134,14 @@ async def prepare_direct_upload(
expires_seconds=settings.OSS_DIRECT_UPLOAD_EXPIRE_SECONDS,
)
except RuntimeError as error:
logger.error("OSS not configured for direct upload prepare: %s", error)
raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(error)) from error
except Exception as error:
logger.exception("Unexpected error in direct upload prepare: %s", error)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to prepare upload: {type(error).__name__}",
) from error
return DirectUploadPrepareResponse(
upload_url=str(payload["url"]),
@@ -162,7 +172,15 @@ async def complete_direct_upload(
normalized_key = storage_service._normalize_storage_key(request.storage_key)
if not normalized_key.startswith("uploads/"):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid upload key")
if not storage_service.file_exists(normalized_key):
try:
file_exists = storage_service.file_exists(normalized_key)
except Exception as error:
logger.exception("OSS error checking file existence for key=%s: %s", normalized_key, error)
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Storage service unavailable",
) from error
if not file_exists:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Uploaded file not found")
job = _submit_ingest_job(
@@ -201,11 +219,21 @@ async def upload_asset(
safe_filename = file.filename.replace("/", "_").replace("\\", "_") if file.filename else "unknown"
storage_key = f"uploads/{file_id}/{safe_filename}"
file_url = storage_service.upload_file(
file.file,
storage_key,
content_type=validated_content_type,
)
try:
file_url = storage_service.upload_file(
file.file,
storage_key,
content_type=validated_content_type,
)
except RuntimeError as error:
logger.error("OSS not configured for upload: %s", error)
raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(error)) from error
except Exception as error:
logger.exception("Unexpected error uploading file to OSS: %s", error)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to upload file: {type(error).__name__}",
) from error
job = _submit_ingest_job(
project_id=project_id,
+6 -3
View File
@@ -1,7 +1,7 @@
import os
from typing import Optional
from pydantic import field_validator
from pydantic import AliasChoices, Field, field_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -83,8 +83,11 @@ class Settings(BaseSettings):
OSS_ACCESS_KEY_ID: str = ""
OSS_ACCESS_KEY_SECRET: str = ""
OSS_BUCKET_NAME: str = "xiaoxia-autocut"
OSS_DIRECT_UPLOAD_MAX_MB: int = 800
OSS_DIRECT_UPLOAD_EXPRESS_SECRET: int = 900
OSS_DIRECT_UPLOAD_MAX_MB: int = Field(
default=2000,
validation_alias=AliasChoices("OSS_DIRECT_UPLOAD_MAX_MB", "MAX_UPLOAD_SIZE_MB"),
)
OSS_DIRECT_UPLOAD_EXPIRE_SECONDS: int = 900
LOG_LEVEL: str = "INFO"
CORS_ORIGINS_RAW: str = (
+31 -10
View File
@@ -28,17 +28,38 @@ class OSSStorageService:
self.local_url_prefix = os.getenv("GENERATED_FILES_URL_PREFIX", "/generated-files")
self.bucket = None
if settings.OSS_ACCESS_KEY_ID and settings.OSS_ACCESS_KEY_SECRET:
has_key_id = bool(settings.OSS_ACCESS_KEY_ID)
has_key_secret = bool(settings.OSS_ACCESS_KEY_SECRET)
if has_key_id and has_key_secret:
if oss2 is not None:
auth = oss2.Auth(
settings.OSS_ACCESS_KEY_ID,
settings.OSS_ACCESS_KEY_SECRET,
)
self.bucket = oss2.Bucket(
auth,
settings.OSS_ENDPOINT,
settings.OSS_BUCKET_NAME,
)
try:
auth = oss2.Auth(
settings.OSS_ACCESS_KEY_ID,
settings.OSS_ACCESS_KEY_SECRET,
)
self.bucket = oss2.Bucket(
auth,
settings.OSS_ENDPOINT,
settings.OSS_BUCKET_NAME,
)
logger.info(
"OSS initialized: endpoint=%s bucket=%s",
settings.OSS_ENDPOINT,
settings.OSS_BUCKET_NAME,
)
except Exception as error:
logger.error("Failed to initialize OSS bucket client: %s", error)
else:
logger.error("oss2 SDK is not installed — OSS operations will fail")
else:
missing = []
if not has_key_id:
missing.append("OSS_ACCESS_KEY_ID")
if not has_key_secret:
missing.append("OSS_ACCESS_KEY_SECRET")
logger.error("OSS credentials not configured — missing: %s", ", ".join(missing))
self.access_key_id = settings.OSS_ACCESS_KEY_ID
self.access_key_secret = settings.OSS_ACCESS_KEY_SECRET
self.endpoint = settings.OSS_ENDPOINT
+7
View File
@@ -154,6 +154,7 @@ export const uploadAsset = async (
/** 预签名直传准备 */
export const prepareDirectUpload = async (data: {
project_id: string;
library_id: string;
filename: string;
content_type: string;
@@ -172,6 +173,7 @@ export const prepareDirectUpload = async (data: {
/** 直传完成确认 */
export const completeDirectUpload = async (data: {
project_id: string;
library_id: string;
storage_key: string;
}): Promise<{ storage_key: string; ingest_job_id: string }> => {
@@ -184,7 +186,11 @@ export const uploadAssetDirect = async (data: {
file: File;
library_id: string;
}): Promise<{ storage_key: string; ingest_job_id: string }> => {
// 后端要求 project_id,前端自动获取默认项目
const project = await getOrCreateDefaultProject();
const prepared = await prepareDirectUpload({
project_id: project.id,
library_id: data.library_id,
filename: data.file.name,
content_type: data.file.type || 'application/octet-stream',
@@ -206,6 +212,7 @@ export const uploadAssetDirect = async (data: {
}
return completeDirectUpload({
project_id: project.id,
library_id: data.library_id,
storage_key: prepared.storage_key,
});
+43 -10
View File
@@ -37,12 +37,18 @@ import {
getAssets,
deleteAsset,
uploadAsset,
uploadAssetDirect,
} from '@/api/assets';
import { getOrCreateDefaultProject } from '@/api/projects';
const { Title, Text } = Typography;
const { Dragger } = Upload;
/** 最大文件大小:2048MB(与后端 OSS_DIRECT_UPLOAD_MAX_MB=2000 对齐,留少量余量) */
const MAX_FILE_SIZE = 2048 * 1024 * 1024; // 2147483648 bytes
/** 大文件阈值:100MB,超过此值走 OSS 直传通道 */
const LARGE_FILE_THRESHOLD = 100 * 1024 * 1024; // 104857600 bytes
/** 素材库类型图标 */
const KindIcon: React.FC<{ kind: string }> = ({ kind }) => {
switch (kind) {
@@ -99,7 +105,7 @@ const AssetLibrary: React.FC = () => {
onError: (err: any) => { if (!err?.__msgShown) message.error('创建失败') },
});
// 上传素材
// 上传素材(小文件表单上传)
const uploadMutation = useMutation({
mutationFn: uploadAsset,
onSuccess: () => {
@@ -110,6 +116,17 @@ const AssetLibrary: React.FC = () => {
onError: (err: any) => { if (!err?.__msgShown) message.error('上传失败') },
});
// 大文件直传(OSS 预签名)
const directUploadMutation = useMutation({
mutationFn: uploadAssetDirect,
onSuccess: () => {
message.success('上传成功');
queryClient.invalidateQueries({ queryKey: ['assets', activeLibrary] });
queryClient.invalidateQueries({ queryKey: ['asset-libraries'] });
},
onError: (err: any) => { if (!err?.__msgShown) message.error('上传失败') },
});
// 删除素材
const deleteMutation = useMutation({
mutationFn: deleteAsset,
@@ -127,16 +144,32 @@ const AssetLibrary: React.FC = () => {
message.warning('请先选择素材库');
return false;
}
// 文件大小校验:上限 2GB
if (file.size > MAX_FILE_SIZE) {
message.error('文件大小不能超过 2GB');
return false;
}
setUploading(true);
try {
const project = await getOrCreateDefaultProject();
const formData = new FormData();
formData.append('file', file);
formData.append('library_id', activeLibrary);
formData.append('project_id', project.id);
await uploadMutation.mutateAsync(formData);
if (file.size > LARGE_FILE_THRESHOLD) {
// 大文件(>100MB)走 OSS 预签名直传通道
await directUploadMutation.mutateAsync({
file,
library_id: activeLibrary,
});
} else {
// 小文件走表单上传
const project = await getOrCreateDefaultProject();
const formData = new FormData();
formData.append('file', file);
formData.append('library_id', activeLibrary);
formData.append('project_id', project.id);
await uploadMutation.mutateAsync(formData);
}
} catch {
// uploadMutation.onError 已处理错误提示
// mutation.onError 已处理错误提示
} finally {
setUploading(false);
}
@@ -216,7 +249,7 @@ const AssetLibrary: React.FC = () => {
beforeUpload={handleUpload}
showUploadList={false}
multiple
disabled={uploading || uploadMutation.isPending}
disabled={uploading || uploadMutation.isPending || directUploadMutation.isPending}
style={{ marginBottom: 24 }}
>
<p className="ant-upload-drag-icon">
@@ -226,7 +259,7 @@ const AssetLibrary: React.FC = () => {
{uploading ? '正在上传,请稍候...' : '点击或拖拽文件到此区域上传'}
</p>
<p className="ant-upload-hint">
{kindLabel[currentLib?.kind || 'video']}
{kindLabel[currentLib?.kind || 'video']} 2GB
</p>
</Dragger>
)}
+1 -1
View File
@@ -8,7 +8,7 @@ server {
gzip_vary on;
gzip_min_length 1024;
gzip_types text/plain text/css text/xml text/javascript application/javascript application/json application/xml+rss;
client_max_body_size 800m;
client_max_body_size 2g;
location /api/ {
proxy_pass http://xiaoxia-api-production:8000/api/;
+3 -3
View File
@@ -28,7 +28,7 @@ server {
include /etc/letsencrypt/options-ssl-nginx.conf;
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
client_max_body_size 800m;
client_max_body_size 2g;
location / {
proxy_pass http://127.0.0.1:3002/;
@@ -51,7 +51,7 @@ server {
include /etc/letsencrypt/options-ssl-nginx.conf;
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
client_max_body_size 800m;
client_max_body_size 2g;
location /api/ {
proxy_pass http://127.0.0.1:8001;
@@ -112,7 +112,7 @@ server {
include /etc/letsencrypt/options-ssl-nginx.conf;
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
client_max_body_size 800m;
client_max_body_size 2g;
location / {
proxy_pass http://127.0.0.1:3000/;
@@ -16,7 +16,7 @@ class InMemoryAssetLibraryRepository:
def get(self, library_id: str) -> AssetLibrary | None:
return self._libraries.get(library_id)
def list_by_project(self, project_id: str, kind: AssetLibraryKind | None = None) -> list[AssetLibrary]:
def find_by_project(self, project_id: str, kind: AssetLibraryKind | None = None) -> list[AssetLibrary]:
items = [library for library in self._libraries.values() if library.project_id == project_id]
if kind is not None:
items = [library for library in items if library.kind == kind]
+63
View File
@@ -0,0 +1,63 @@
#!/bin/sh
# cleanup_old_images.sh
# 清理构建服务器上的旧 Docker 镜像和 Registry 旧版本
# 保留最近 KEEP_VERSIONS 个版本(默认 2
# 在 CI 构建完成后调用,防止磁盘空间耗尽
set -eu
KEEP_VERSIONS="${KEEP_VERSIONS:-2}"
REGISTRY_HOST="${REGISTRY_HOST:-172.30.18.198:5000}"
SERVICES="xiaoxia-saas-api xiaoxia-saas-worker xiaoxia-saas-web"
echo "=== Docker Image Cleanup ==="
echo "Keeping last ${KEEP_VERSIONS} versions per service"
echo ""
for svc in $SERVICES; do
# 收集所有 v0.N.N 格式的版本号(去重,按版本号排序)
versions=$(docker images --format "{{.Repository}}:{{.Tag}}" | \
grep "${svc}" | \
grep -E "v[0-9]+\.[0-9]+\.[0-9]+" | \
sed -E "s/.*:${svc}:(v[0-9]+\.[0-9]+\.[0-9]+).*/\1/" | \
sed -E "s/.*:v([0-9]+\.[0-9]+\.[0-9]+).*/v\1/" | \
sort -t. -k1,1V -k2,2n -k3,3n | \
uniq)
total=$(echo "$versions" | grep -c "^v" || true)
if [ "$total" -gt "$KEEP_VERSIONS" ]; then
remove_count=$((total - KEEP_VERSIONS))
to_remove=$(echo "$versions" | head -n "$remove_count")
echo "[Registry] Cleaning old blobs for ${svc}..."
for ver in $to_remove; do
repo_name=$(echo "$svc" | sed "s/xiaoxia-saas-//")
manifest_url="http://admin:Xiaoxia2026@localhost:5000/v2/${repo_name}/manifests/${ver}"
digest=$(curl -sI -H "Accept: application/vnd.docker.distribution.manifest.v2+json" "$manifest_url" | grep -i "^docker-content-digest:" | tr -d "\r" | awk "{print \$2}")
if [ -n "$digest" ]; then
curl -s -X DELETE -H "Accept: application/vnd.docker.distribution.manifest.v2+json" "http://admin:Xiaoxia2026@localhost:5000/v2/${repo_name}/manifests/${digest}" > /dev/null 2>&1 || true
echo " Deleted registry tag: ${svc}:${ver}"
fi
done
echo "[Local] Removing old local images for ${svc}..."
for ver in $to_remove; do
docker rmi "${svc}:${ver}" 2>/dev/null && echo " Removed local: ${svc}:${ver}" || true
docker rmi "${REGISTRY_HOST}/${svc}:${ver}" 2>/dev/null && echo " Removed registry-ref: ${REGISTRY_HOST}/${svc}:${ver}" || true
done
else
echo "[${svc}] ${total} version(s) found, within keep limit (${KEEP_VERSIONS})"
fi
echo ""
done
# 清理悬空镜像(构建中间层)
echo "=== Pruning dangling images ==="
pruned=$(docker image prune -f 2>&1)
echo "$pruned" | tail -1
# 清理 dev 标签的旧层(dev 标签会被下次构建覆盖)
echo ""
echo "=== Cleanup complete ==="
echo "Current images:"
docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" | grep -E "(xiaoxia|REPOSITORY)" || true