diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 7ed922cd3..588a03f98 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -91,31 +91,7 @@ jobs: else key_path="/root/.ssh/xiaoxia_runtime_builder" fi - ssh -i "$key_path" "$staging_user@$staging_host" "GITHUB_SHA='${GITHUB_SHA}' sh -s" <<'REMOTE_DEPLOY' - set -eu - artifact="/var/lib/xiaoxia-saas-staging/artifacts/xiaoxia-staging-${GITHUB_SHA}.tar.gz" - image_tar="/var/lib/xiaoxia-saas-staging/artifacts/xiaoxia-web-staging-${GITHUB_SHA}.tar" - test -f "$artifact" - test -f "$image_tar" - test -f /var/lib/xiaoxia-saas-staging/.env - docker load -i "$image_tar" - rm -rf /var/lib/xiaoxia-saas-staging/repo - mkdir -p /var/lib/xiaoxia-saas-staging/repo - tar -xzf "$artifact" -C /var/lib/xiaoxia-saas-staging/repo - test -f /var/lib/xiaoxia-saas-staging/repo/apps/web/dist/index.html - cp /var/lib/xiaoxia-saas-staging/.env /var/lib/xiaoxia-saas-staging/repo/.env - chmod +x /var/lib/xiaoxia-saas-staging/repo/infra/docker/deploy-staging.sh - WEB_IMAGE="xiaoxia-saas-web:staging-${GITHUB_SHA}" HOST_PREFIX= WEB_PORT=3001 REBUILD_BACKEND=0 BUILD_WEB=0 RUN_MIGRATIONS=0 /var/lib/xiaoxia-saas-staging/repo/infra/docker/deploy-staging.sh - i=0 - while [ "$i" -lt 30 ]; do - if wget -qO- http://127.0.0.1:8000/health; then - exit 0 - fi - i=$((i + 1)) - sleep 2 - done - exit 1 -REMOTE_DEPLOY + echo 'c2V0IC1ldQphcnRpZmFjdD0iL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvYXJ0aWZhY3RzL3hpYW94aWEtc3RhZ2luZy0ke0dJVEhVQl9TSEF9LnRhci5neiIKaW1hZ2VfdGFyPSIvdmFyL2xpYi94aWFveGlhLXNhYXMtc3RhZ2luZy9hcnRpZmFjdHMveGlhb3hpYS13ZWItc3RhZ2luZy0ke0dJVEhVQl9TSEF9LnRhciIKdGVzdCAtZiAiJGFydGlmYWN0Igp0ZXN0IC1mICIkaW1hZ2VfdGFyIgp0ZXN0IC1mIC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nLy5lbnYKZG9ja2VyIGxvYWQgLWkgIiRpbWFnZV90YXIiCnJtIC1yZiAvdmFyL2xpYi94aWFveGlhLXNhYXMtc3RhZ2luZy9yZXBvCm1rZGlyIC1wIC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nL3JlcG8KdGFyIC14emYgIiRhcnRpZmFjdCIgLUMgL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvcmVwbwp0ZXN0IC1mIC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nL3JlcG8vYXBwcy93ZWIvZGlzdC9pbmRleC5odG1sCmNwIC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nLy5lbnYgL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvcmVwby8uZW52CmNobW9kICt4IC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nL3JlcG8vaW5mcmEvZG9ja2VyL2RlcGxveS1zdGFnaW5nLnNoCldFQl9JTUFHRT0ieGlhb3hpYS1zYWFzLXdlYjpzdGFnaW5nLSR7R0lUSFVCX1NIQX0iIEhPU1RfUFJFRklYPSBXRUJfUE9SVD0zMDAxIFJFQlVJTERfQkFDS0VORD0wIEJVSUxEX1dFQj0wIFJVTl9NSUdSQVRJT05TPTAgL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvcmVwby9pbmZyYS9kb2NrZXIvZGVwbG95LXN0YWdpbmcuc2gKaT0wCndoaWxlIFsgIiRpIiAtbHQgMzAgXTsgZG8KICBpZiB3Z2V0IC1xTy0gaHR0cDovLzEyNy4wLjAuMTo4MDAwL2hlYWx0aDsgdGhlbgogICAgZXhpdCAwCiAgZmkKICBpPSQoKGkgKyAxKSkKICBzbGVlcCAyCmRvbmUKZXhpdCAxCg==' | base64 -d | ssh -i "$key_path" "$staging_user@$staging_host" "GITHUB_SHA='${GITHUB_SHA}' sh" build-production-runtime-images: name: Build Production Runtime Images @@ -220,51 +196,7 @@ REMOTE_DEPLOY test -f "$key_path" fi ssh-keyscan -H "$production_host" >> ~/.ssh/known_hosts - ssh -i "$key_path" "$production_user@$production_host" \ - "RELEASE_VERSION='${GITHUB_REF_NAME}' sh -s" <<'REMOTE_DEPLOY' - set -eu - release_tar="/var/lib/xiaoxia-saas-production/release-${RELEASE_VERSION}.tar.gz" - test -f "$release_tar" - test -f "/var/lib/xiaoxia-saas-production/runtime-images-${RELEASE_VERSION}.tar" - test -f "/var/lib/xiaoxia-saas-production/web-${RELEASE_VERSION}.tar" - mkdir -p /var/lib/xiaoxia-saas-production - old_assets_dir="/tmp/xiaoxia-previous-web-assets-${RELEASE_VERSION}" - rm -rf "$old_assets_dir" - mkdir -p "$old_assets_dir" - if docker inspect xiaoxia-web-production >/dev/null 2>&1; then - docker cp xiaoxia-web-production:/usr/share/nginx/html/assets/. "$old_assets_dir"/ 2>/dev/null || true - fi - if [ -d /var/lib/xiaoxia-saas-production/repo/apps/web/dist/assets ]; then - cp -a /var/lib/xiaoxia-saas-production/repo/apps/web/dist/assets/. "$old_assets_dir"/ - fi - rm -rf /var/lib/xiaoxia-saas-production/repo - mkdir -p /var/lib/xiaoxia-saas-production/repo - tar -xzf "$release_tar" -C /var/lib/xiaoxia-saas-production/repo - test -f /var/lib/xiaoxia-saas-production/repo/apps/web/dist/index.html - if [ -d "$old_assets_dir" ]; then - mkdir -p /var/lib/xiaoxia-saas-production/repo/apps/web/dist/assets - for asset in "$old_assets_dir"/*; do - [ -e "$asset" ] || continue - name="$(basename "$asset")" - if [ ! -e "/var/lib/xiaoxia-saas-production/repo/apps/web/dist/assets/$name" ]; then - cp -a "$asset" "/var/lib/xiaoxia-saas-production/repo/apps/web/dist/assets/$name" - fi - done - rm -rf "$old_assets_dir" - fi - test -f /var/lib/xiaoxia-saas-production/.env - cp /var/lib/xiaoxia-saas-production/.env /var/lib/xiaoxia-saas-production/repo/.env - HOST_PREFIX= WEB_IMAGE="xiaoxia-saas-web:${RELEASE_VERSION}" WEB_IMAGE_TAR="/var/lib/xiaoxia-saas-production/web-${RELEASE_VERSION}.tar" sh /var/lib/xiaoxia-saas-production/repo/infra/docker/deploy-production.sh - i=0 - while [ "$i" -lt 30 ]; do - if wget -qO- http://127.0.0.1:8001/health; then - exit 0 - fi - i=$((i + 1)) - sleep 2 - done - exit 1 -REMOTE_DEPLOY + echo 'c2V0IC1ldQpyZWxlYXNlX3Rhcj0iL3Zhci9saWIveGlhb3hpYS1zYWFzLXByb2R1Y3Rpb24vcmVsZWFzZS0ke1JFTEVBU0VfVkVSU0lPTn0udGFyLmd6Igp0ZXN0IC1mICIkcmVsZWFzZV90YXIiCnRlc3QgLWYgIi92YXIvbGliL3hpYW94aWEtc2Fhcy1wcm9kdWN0aW9uL3J1bnRpbWUtaW1hZ2VzLSR7UkVMRUFTRV9WRVJTSU9OfS50YXIiCnRlc3QgLWYgIi92YXIvbGliL3hpYW94aWEtc2Fhcy1wcm9kdWN0aW9uL3dlYi0ke1JFTEVBU0VfVkVSU0lPTn0udGFyIgpta2RpciAtcCAvdmFyL2xpYi94aWFveGlhLXNhYXMtcHJvZHVjdGlvbgpvbGRfYXNzZXRzX2Rpcj0iL3RtcC94aWFveGlhLXByZXZpb3VzLXdlYi1hc3NldHMtJHtSRUxFQVNFX1ZFUlNJT059IgpybSAtcmYgIiRvbGRfYXNzZXRzX2RpciIKbWtkaXIgLXAgIiRvbGRfYXNzZXRzX2RpciIKaWYgZG9ja2VyIGluc3BlY3QgeGlhb3hpYS13ZWItcHJvZHVjdGlvbiA+L2Rldi9udWxsIDI+JjE7IHRoZW4KICBkb2NrZXIgY3AgeGlhb3hpYS13ZWItcHJvZHVjdGlvbjovdXNyL3NoYXJlL25naW54L2h0bWwvYXNzZXRzLy4gIiRvbGRfYXNzZXRzX2RpciIvIDI+L2Rldi9udWxsIHx8IHRydWUKZmkKaWYgWyAtZCAvdmFyL2xpYi94aWFveGlhLXNhYXMtcHJvZHVjdGlvbi9yZXBvL2FwcHMvd2ViL2Rpc3QvYXNzZXRzIF07IHRoZW4KICBjcCAtYSAvdmFyL2xpYi94aWFveGlhLXNhYXMtcHJvZHVjdGlvbi9yZXBvL2FwcHMvd2ViL2Rpc3QvYXNzZXRzLy4gIiRvbGRfYXNzZXRzX2RpciIvCmZpCnJtIC1yZiAvdmFyL2xpYi94aWFveGlhLXNhYXMtcHJvZHVjdGlvbi9yZXBvCm1rZGlyIC1wIC92YXIvbGliL3hpYW94aWEtc2Fhcy1wcm9kdWN0aW9uL3JlcG8KdGFyIC14emYgIiRyZWxlYXNlX3RhciIgLUMgL3Zhci9saWIveGlhb3hpYS1zYWFzLXByb2R1Y3Rpb24vcmVwbwp0ZXN0IC1mIC92YXIvbGliL3hpYW94aWEtc2Fhcy1wcm9kdWN0aW9uL3JlcG8vYXBwcy93ZWIvZGlzdC9pbmRleC5odG1sCmlmIFsgLWQgIiRvbGRfYXNzZXRzX2RpciIgXTsgdGhlbgogIG1rZGlyIC1wIC92YXIvbGliL3hpYW94aWEtc2Fhcy1wcm9kdWN0aW9uL3JlcG8vYXBwcy93ZWIvZGlzdC9hc3NldHMKICBmb3IgYXNzZXQgaW4gIiRvbGRfYXNzZXRzX2RpciIvKjsgZG8KICAgIFsgLWUgIiRhc3NldCIgXSB8fCBjb250aW51ZQogICAgbmFtZT0iJChiYXNlbmFtZSAiJGFzc2V0IikiCiAgICBpZiBbICEgLWUgIi92YXIvbGliL3hpYW94aWEtc2Fhcy1wcm9kdWN0aW9uL3JlcG8vYXBwcy93ZWIvZGlzdC9hc3NldHMvJG5hbWUiIF07IHRoZW4KICAgICAgY3AgLWEgIiRhc3NldCIgIi92YXIvbGliL3hpYW94aWEtc2Fhcy1wcm9kdWN0aW9uL3JlcG8vYXBwcy93ZWIvZGlzdC9hc3NldHMvJG5hbWUiCiAgICBmaQogIGRvbmUKICBybSAtcmYgIiRvbGRfYXNzZXRzX2RpciIKZmkKdGVzdCAtZiAvdmFyL2xpYi94aWFveGlhLXNhYXMtcHJvZHVjdGlvbi8uZW52CmNwIC92YXIvbGliL3hpYW94aWEtc2Fhcy1wcm9kdWN0aW9uLy5lbnYgL3Zhci9saWIveGlhb3hpYS1zYWFzLXByb2R1Y3Rpb24vcmVwby8uZW52CkhPU1RfUFJFRklYPSBXRUJfSU1BR0U9InhpYW94aWEtc2Fhcy13ZWI6JHtSRUxFQVNFX1ZFUlNJT059IiBXRUJfSU1BR0VfVEFSPSIvdmFyL2xpYi94aWFveGlhLXNhYXMtcHJvZHVjdGlvbi93ZWItJHtSRUxFQVNFX1ZFUlNJT059LnRhciIgc2ggL3Zhci9saWIveGlhb3hpYS1zYWFzLXByb2R1Y3Rpb24vcmVwby9pbmZyYS9kb2NrZXIvZGVwbG95LXByb2R1Y3Rpb24uc2gKaT0wCndoaWxlIFsgIiRpIiAtbHQgMzAgXTsgZG8KICBpZiB3Z2V0IC1xTy0gaHR0cDovLzEyNy4wLjAuMTo4MDAxL2hlYWx0aDsgdGhlbgogICAgZXhpdCAwCiAgZmkKICBpPSQoKGkgKyAxKSkKICBzbGVlcCAyCmRvbmUKZXhpdCAxCg==' | base64 -d | ssh -i "$key_path" "$production_user@$production_host" "RELEASE_VERSION='${GITHUB_REF_NAME}' sh" production-e2e: name: Production Browser E2E diff --git a/apps/web/src/pages/workspace/WorkspaceDetail.tsx b/apps/web/src/pages/workspace/WorkspaceDetail.tsx index 6ac291b06..c69efc974 100644 --- a/apps/web/src/pages/workspace/WorkspaceDetail.tsx +++ b/apps/web/src/pages/workspace/WorkspaceDetail.tsx @@ -23,7 +23,7 @@ const WorkspaceDetail: React.FC = () => { const projectsQuery = useQuery({ queryKey: ['projects', id], - queryFn: () => getProjects(id!), + queryFn: () => getProjects(), enabled: !!id, }); @@ -44,7 +44,6 @@ const WorkspaceDetail: React.FC = () => { const handleCreateProject = async () => { const values = await projectForm.validateFields(); createProjectMutation.mutate({ - workspace_id: id!, name: values.name, description: values.description || '', }); diff --git a/apps/web/src/router/index.tsx b/apps/web/src/router/index.tsx index 668d13bb0..f2566f405 100644 --- a/apps/web/src/router/index.tsx +++ b/apps/web/src/router/index.tsx @@ -52,6 +52,10 @@ export const router = createBrowserRouter([ index: true, element: , }, + { + path: 'projects', + lazy: () => import('@/pages/workspace/WorkspaceList'), + }, { path: 'projects/:id/assets', lazy: () => import('@/pages/workspace/ProjectAssets'), diff --git a/apps/worker/celery_app.py b/apps/worker/celery_app.py deleted file mode 100644 index 5db9732fa..000000000 --- a/apps/worker/celery_app.py +++ /dev/null @@ -1,12 +0,0 @@ -import os -from celery import Celery - - -def create_celery_app() -> Celery: - app = Celery("xiaoxia_saas_worker") - app.conf.broker_url = os.getenv("CELERY_BROKER_URL", "redis://redis:6379/0") - app.conf.result_backend = os.getenv("CELERY_RESULT_BACKEND", "redis://redis:6379/1") - return app - - -celery_app = create_celery_app() diff --git a/apps/worker/video_processing/dedup.py b/apps/worker/video_processing/dedup.py index 068d0f858..7468e0164 100644 --- a/apps/worker/video_processing/dedup.py +++ b/apps/worker/video_processing/dedup.py @@ -14,16 +14,12 @@ from celery import Task from sqlalchemy.orm import Session from packages.adapters.sqlalchemy_impl.generated_video_repository import SQLAlchemyGeneratedVideoRepository -from packages.adapters.sqlalchemy_impl.session import SessionLocal, build_session_factory -from packages.shared.config import get_shared_settings from packages.shared.storage import get_storage_service -from apps.worker.celery_app import celery_app +from worker_app.celery_app import celery_app +from worker_app.db import SessionLocal logger = logging.getLogger(__name__) -settings = get_shared_settings() -if SessionLocal is None: - build_session_factory(settings.database_url) def compute_phash(image: np.ndarray, hash_size: int = 8) -> str: diff --git a/apps/worker/worker_app/celery_app.py b/apps/worker/worker_app/celery_app.py index a14dd2e29..5dd182a0e 100644 --- a/apps/worker/worker_app/celery_app.py +++ b/apps/worker/worker_app/celery_app.py @@ -5,9 +5,12 @@ settings = get_settings() celery_app = Celery(settings.worker_name) celery_app.conf.broker_url = settings.broker_url celery_app.conf.result_backend = settings.result_backend +celery_app.conf.broker_connection_retry_on_startup = True celery_app.conf.imports = ( "worker_app.tasks.health", "worker_app.tasks.ingest", "worker_app.tasks.classification", "worker_app.tasks.generation", + "worker_app.tasks.voice_extraction", + "apps.worker.video_processing.dedup", ) diff --git a/apps/worker/worker_app/tasks/classification.py b/apps/worker/worker_app/tasks/classification.py index 88a409c5b..091d1d65c 100755 --- a/apps/worker/worker_app/tasks/classification.py +++ b/apps/worker/worker_app/tasks/classification.py @@ -13,12 +13,14 @@ from packages.domain import ( ClassificationStatus, ) +from worker_app.celery_app import celery_app from .asset_analyzer import classify_asset_real logger = get_task_logger(__name__) -def classify_asset(job_id: str) -> dict: +@celery_app.task(bind=True, name="worker.classify_asset", max_retries=2) +def classify_asset(self, job_id: str) -> dict: """ Classify asset task. diff --git a/apps/worker/worker_app/tasks/generation.py b/apps/worker/worker_app/tasks/generation.py index 57c7242af..f8cd1a5a4 100755 --- a/apps/worker/worker_app/tasks/generation.py +++ b/apps/worker/worker_app/tasks/generation.py @@ -14,6 +14,8 @@ from typing import Optional import oss2 +from worker_app.celery_app import celery_app + OUTPUT_WIDTH = 1280 OUTPUT_HEIGHT = 720 OUTPUT_FPS = 25.0 @@ -214,29 +216,37 @@ def _process_with_editing_mode( ) -def generate_video( - task_id: str, - workspace_id: str, - project_id: str, - asset_library_id: str, - voice_library_id: str = "", - mode: str = "one_take", -) -> dict: +@celery_app.task(bind=True, name="worker.generate_video", max_retries=2) +def generate_video(self, task_id: str) -> dict: """ 生成视频任务 Args: - task_id: 任务 ID - workspace_id: 工作空间 ID - project_id: 项目 ID - asset_library_id: 素材库 ID - voice_library_id: 配音库 ID（可选） - mode: 剪辑模式，默认 one_take + task_id: 任务 ID（从数据库加载完整任务信息） Returns: 生成结果字典 """ from packages.domain import GeneratedVideo, GenerationMode, GenerationTaskStatus + from packages.adapters.sqlalchemy_impl.generation_task_repository import ( + SQLAlchemyGenerationTaskRepository, + ) + from worker_app.db import SessionLocal + + # 从数据库加载任务信息 + session = SessionLocal() + try: + task_repo = SQLAlchemyGenerationTaskRepository(session) + gen_task = task_repo.get(task_id) + if gen_task is None: + return {"status": "failed", "error": f"generation task {task_id} not found"} + workspace_id = gen_task.workspace_id + project_id = gen_task.project_id + asset_library_id = gen_task.asset_library_id + voice_library_id = gen_task.voice_library_id or "" + mode = gen_task.strategy_id or "one_take" + finally: + session.close() try: editing_mode = GenerationMode(mode) diff --git a/apps/worker/worker_app/tasks/voice_extraction.py b/apps/worker/worker_app/tasks/voice_extraction.py index b3cae3ec9..38641e66b 100644 --- a/apps/worker/worker_app/tasks/voice_extraction.py +++ b/apps/worker/worker_app/tasks/voice_extraction.py @@ -10,16 +10,12 @@ from celery import Task from sqlalchemy.orm import Session from packages.adapters.sqlalchemy_impl.asset_repository import SQLAlchemyAssetRepository -from packages.adapters.sqlalchemy_impl.session import SessionLocal, build_session_factory -from packages.shared.config import get_shared_settings from packages.shared.storage import get_storage_service -from .celery_app import celery_app +from worker_app.celery_app import celery_app +from worker_app.db import SessionLocal logger = logging.getLogger(__name__) -settings = get_shared_settings() -if SessionLocal is None: - build_session_factory(settings.database_url) class VoiceExtractor: diff --git a/infra/docker/worker.Dockerfile b/infra/docker/worker.Dockerfile index 80b167dab..e76503555 100644 --- a/infra/docker/worker.Dockerfile +++ b/infra/docker/worker.Dockerfile @@ -36,6 +36,13 @@ COPY migrations/ /app/migrations/ ENV PYTHONPATH=/app ENV PYTHONUNBUFFERED=1 +# 创建非 root 用户运行 Worker +RUN groupadd -r celery && useradd -r -g celery -d /app -s /sbin/nologin celery \ + && chown -R celery:celery /app +RUN mkdir -p /app/generated && chown celery:celery /app/generated + +USER celery + # Worker 入口点 WORKDIR /app/apps/worker -CMD ["celery", "-A", "celery_app", "worker", "--loglevel=info", "--concurrency=2"] +CMD ["celery", "-A", "worker_app.celery_app", "worker", "--loglevel=info", "--concurrency=2"]