feat(deploy): #1978 GPU节点自动部署配置文件入库

新增 deploy/gpu_worker/ 下 3 个 systemd 单元 + 3 个运维脚本： - musetalk-worker.service: MuseTalk 推理 API systemd 服务 - gpu-poll.service + gpu-poll.timer: 每30秒轮询Gitea最新commit - scripts/update-gpu-worker.sh: 备份→拉代码→重启→健康检查→失败回滚 - scripts/poll_and_update.sh: SHA比对触发update - scripts/setup-gpu-node.sh: 新节点一键初始化(apt依赖+目录+systemd+sudo免密+首次启动) - README.md 追加第七章「自动部署」说明服务架构/部署步骤/更新机制/日志/注意事项不改动现有 gpu_worker.py/musetalk_server.py/requirements.txt/xiaoxia-gpu-worker.service/.env.example 不改动CI/Docker/镜像构建。脚本路径写死/home/ying，后续多节点再参数化。
2026-09-20 10:28:24 +08:00
parent 60cacdf280
commit db621b4fcb
7 changed files with 337 additions and 0 deletions
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+REPO_API="https://git.xiaoxiajianji.com/api/v1/repos/xiaoxia/xiaoxia-saas/commits?sha=develop&path=deploy/gpu_worker&limit=1"
+STATE_FILE="/home/ying/projects/gpu-webhook/.last_commit"
+UPDATE_SCRIPT="/home/ying/projects/update-gpu-worker.sh"
+LOG_FILE="/tmp/gpu-poll.log"
+
+log() {
+    echo "[$(date +"%Y-%m-%d %H:%M:%S")] $*" >> "$LOG_FILE"
+}
+
+LATEST_SHA=$(curl -sk --max-time 10 "$REPO_API" | python3 -c "
+import sys, json
+try:
+    data = json.load(sys.stdin)
+    if isinstance(data, list) and len(data) > 0:
+        print(data[0].get('sha', ''))
+    else:
+        print('')
+except:
+    print('')
+" 2>/dev/null)
+
+if [ -z "$LATEST_SHA" ]; then
+    log "get latest commit failed, skip"
+    exit 0
+fi
+
+LAST_SHA=""
+if [ -f "$STATE_FILE" ]; then
+    LAST_SHA=$(cat "$STATE_FILE")
+fi
+
+if [ "$LATEST_SHA" = "$LAST_SHA" ]; then
+    exit 0
+fi
+
+if [ -z "$LAST_SHA" ]; then
+    echo "$LATEST_SHA" > "$STATE_FILE"
+    log "first run, recording SHA: $LATEST_SHA"
+    exit 0
+fi
+
+log "new commit detected: $LAST_SHA -> $LATEST_SHA, triggering update"
+echo "$LATEST_SHA" > "$STATE_FILE"
+bash "$UPDATE_SCRIPT" >> "$LOG_FILE" 2>&1
+log "update completed"
@@ -0,0 +1,62 @@
+#!/bin/bash
+# GPU节点一键初始化脚本 - 在全新GPU机器上执行
+
+set -e
+
+echo "=== 1. 安装系统依赖 ==="
+sudo apt-get update -qq
+sudo apt-get install -y -qq python3 python3-pip python3-venv ffmpeg wget curl git
+
+echo "=== 2. 创建目录 ==="
+mkdir -p ~/projects/MuseTalk ~/projects/gpu-webhook /opt/xiaoxia-gpu-worker
+
+echo "=== 3. 安装nvidia-container-toolkit（如需要Docker）==="
+# 可选，当前不使用Docker，跳过
+# distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
+# curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
+# curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
+# sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
+# sudo nvidia-ctk runtime configure --runtime=docker
+# sudo systemctl restart docker
+
+echo "=== 4. 拉取服务配置和脚本 ==="
+REPO_URL="https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/raw/branch/develop/deploy/gpu_worker"
+wget -q -O /tmp/musetalk-worker.service "$REPO_URL/musetalk-worker.service"
+wget -q -O /tmp/gpu-poll.service "$REPO_URL/gpu-poll.service"
+wget -q -O /tmp/gpu-poll.timer "$REPO_URL/gpu-poll.timer"
+wget -q -O ~/projects/update-gpu-worker.sh "$REPO_URL/scripts/update-gpu-worker.sh"
+wget -q -O ~/projects/gpu-webhook/poll_and_update.sh "$REPO_URL/scripts/poll_and_update.sh"
+chmod +x ~/projects/update-gpu-worker.sh ~/projects/gpu-webhook/poll_and_update.sh
+
+echo "=== 5. 安装systemd服务 ==="
+sudo cp /tmp/musetalk-worker.service /etc/systemd/system/
+sudo cp /tmp/gpu-poll.service /etc/systemd/system/
+sudo cp /tmp/gpu-poll.timer /etc/systemd/system/
+
+echo "=== 6. 配置sudo免密 ==="
+sudo bash -c 'cat > /etc/sudoers.d/ying-gpu-update << EOF
+ying ALL=(ALL) NOPASSWD: /bin/systemctl restart musetalk-worker
+ying ALL=(ALL) NOPASSWD: /bin/systemctl restart xiaoxia-gpu-worker
+ying ALL=(ALL) NOPASSWD: /bin/systemctl status musetalk-worker
+ying ALL=(ALL) NOPASSWD: /bin/systemctl status xiaoxia-gpu-worker
+ying ALL=(ALL) NOPASSWD: /bin/systemctl daemon-reload
+ying ALL=(ALL) NOPASSWD: /usr/bin/journalctl
+ying ALL=(ALL) NOPASSWD: /bin/cp
+ying ALL=(ALL) NOPASSWD: /bin/chmod
+ying ALL=(ALL) NOPASSWD: /usr/bin/tee
+EOF'
+sudo chmod 440 /etc/sudoers.d/ying-gpu-update
+
+echo "=== 7. 首次拉取代码并启动服务 ==="
+bash ~/projects/update-gpu-worker.sh
+sudo systemctl daemon-reload
+sudo systemctl enable musetalk-worker xiaoxia-gpu-worker gpu-poll.timer
+sudo systemctl start musetalk-worker xiaoxia-gpu-worker gpu-poll.timer
+
+echo "=== 完成！ ==="
+echo "检查服务状态："
+echo "  sudo systemctl status musetalk-worker"
+echo "  sudo systemctl status xiaoxia-gpu-worker"
+echo "  sudo systemctl status gpu-poll.timer"
+echo "健康检查：curl http://127.0.0.1:7861/health"
+echo "更新日志：tail -f /tmp/gpu-worker-update.log"
@@ -0,0 +1,62 @@
+#!/bin/bash
+set -e
+
+REPO_URL="https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/raw/branch/develop/deploy/gpu_worker"
+MUSE_DIR="/home/ying/projects/MuseTalk"
+WORKER_DIR="/opt/xiaoxia-gpu-worker"
+LOG_FILE="/tmp/gpu-worker-update.log"
+
+log() {
+    local NOW
+    NOW=$(date +"%Y-%m-%d %H:%M:%S")
+    echo "[$NOW] $*" | tee -a "$LOG_FILE"
+}
+
+log "========== start update =========="
+
+BAK_SUFFIX=$(date +"%Y%m%d%H%M%S")
+cp "$MUSE_DIR/musetalk_server.py" "$MUSE_DIR/musetalk_server.py.bak.$BAK_SUFFIX"
+cp "$WORKER_DIR/gpu_worker.py" "$WORKER_DIR/gpu_worker.py.bak.$BAK_SUFFIX"
+log "backup done ($BAK_SUFFIX)"
+
+wget -q -O "$MUSE_DIR/musetalk_server.py" "$REPO_URL/musetalk_server.py"
+log "musetalk_server.py updated"
+
+wget -q -O "$WORKER_DIR/gpu_worker.py" "$REPO_URL/gpu_worker.py"
+log "gpu_worker.py updated"
+
+wget -q -O /tmp/gpu-requirements.txt "$REPO_URL/requirements.txt"
+if [ -f "$WORKER_DIR/requirements.txt" ] && ! diff -q "$WORKER_DIR/requirements.txt" /tmp/gpu-requirements.txt > /dev/null 2>&1; then
+    log "requirements changed, updating..."
+    cp /tmp/gpu-requirements.txt "$WORKER_DIR/requirements.txt"
+    "$WORKER_DIR/venv/bin/pip" install -r "$WORKER_DIR/requirements.txt" -q
+    log "pip install done"
+else
+    log "requirements no change, skip pip"
+fi
+
+sudo systemctl restart musetalk-worker
+log "musetalk restarted"
+sleep 5
+
+sudo systemctl restart xiaoxia-gpu-worker
+log "gpu-worker restarted"
+sleep 8
+
+HEALTH=$(curl -s http://127.0.0.1:7861/health 2>/dev/null)
+if echo "$HEALTH" | grep -q "healthy\|ok"; then
+    log "health check OK"
+    log "========== update done =========="
+    exit 0
+else
+    log "health check FAILED, rolling back..."
+    LATEST_MUSE_BAK=$(ls -t "$MUSE_DIR/musetalk_server.py.bak."* 2>/dev/null | head -1)
+    LATEST_WORKER_BAK=$(ls -t "$WORKER_DIR/gpu_worker.py.bak."* 2>/dev/null | head -1)
+    [ -n "$LATEST_MUSE_BAK" ] && cp "$LATEST_MUSE_BAK" "$MUSE_DIR/musetalk_server.py"
+    [ -n "$LATEST_WORKER_BAK" ] && cp "$LATEST_WORKER_BAK" "$WORKER_DIR/gpu_worker.py"
+    sudo systemctl restart musetalk-worker
+    sleep 5
+    sudo systemctl restart xiaoxia-gpu-worker
+    log "rolled back"
+    exit 1
+fi