From 223912ec0c7ad47ccb6ed7cfd968cf31a4bf8e6e Mon Sep 17 00:00:00 2001 From: CI Bot Date: Sun, 19 Jul 2026 20:09:49 +0800 Subject: [PATCH] =?UTF-8?q?fix(ci):=20deploy=E8=84=9A=E6=9C=AC=E7=9B=B4?= =?UTF-8?q?=E6=8E=A5=E4=BD=BF=E7=94=A8=E5=AE=8C=E6=95=B4registry=E9=95=9C?= =?UTF-8?q?=E5=83=8F=E5=90=8D=EF=BC=8C=E5=88=A0=E9=99=A4re-tag=E6=AD=A5?= =?UTF-8?q?=E9=AA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - staging deploy: 移除 docker tag 成本地名步骤，直接用完整ACR路径启动容器 - production deploy: 从base64内嵌改为外部脚本文件，同样使用完整registry路径 - 修复watchtower自动更新失效问题（短镜像名导致watchtower去docker.io查找） - 提高deploy脚本可维护性（production不再是base64黑盒） --- .gitea/workflows/ci-pipeline.yml | 14 +- scripts/ci_production_deploy.sh | 443 +++++++++++++++++++++++++++++++ scripts/ci_staging_deploy.sh | 18 +- 3 files changed, 456 insertions(+), 19 deletions(-) create mode 100644 scripts/ci_production_deploy.sh diff --git a/.gitea/workflows/ci-pipeline.yml b/.gitea/workflows/ci-pipeline.yml index dd619e769..b32bc8c3e 100755 --- a/.gitea/workflows/ci-pipeline.yml +++ b/.gitea/workflows/ci-pipeline.yml @@ -772,10 +772,12 @@ jobs: apt-get update -qq && apt-get install -y -qq openssh-client >/dev/null 2>&1 echo "openssh-client installed" - name: Deploy production over SSH (Registry pull) + if: success() shell: sh env: PRODUCTION_SSH_HOST: ${{ secrets.PRODUCTION_SSH_HOST }} PRODUCTION_SSH_USER: ${{ secrets.PRODUCTION_SSH_USER }} + PRODUCTION_SSH_PORT: ${{ secrets.PRODUCTION_SSH_PORT }} PRODUCTION_SSH_KEY: ${{ secrets.PRODUCTION_SSH_KEY }} ACR_USERNAME: ${{ secrets.ACR_USERNAME }} ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} @@ -783,10 +785,12 @@ jobs: set -eux production_host="${PRODUCTION_SSH_HOST:-47.98.113.167}" production_user="${PRODUCTION_SSH_USER:-root}" + production_port="${PRODUCTION_SSH_PORT:-22222}" echo "Host: $production_host" - echo "User: $production_user" + echo "Port: $production_port" mkdir -p ~/.ssh + key_path="" if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then key_path="/root/.ssh/xiaoxia_runtime_builder" @@ -806,15 +810,13 @@ jobs: exit 1 fi - ssh-keyscan -p 22222 -H "$production_host" >> ~/.ssh/known_hosts 2>/dev/null + ssh-keyscan -p "$production_port" -H "$production_host" >> ~/.ssh/known_hosts 2>/dev/null echo "SSH keyscan done" - ssh -p 22222 -i "$key_path" -o StrictHostKeyChecking=no "${production_user}@${production_host}" "echo SSH_CONNECTION_OK && hostname" + ssh -p "$production_port" -i "$key_path" -o StrictHostKeyChecking=no "${production_user}@${production_host}" "echo SSH_CONNECTION_OK && hostname" echo "SSH connection verified" - # Production deploy script (base64 encoded) - DEPLOY_B64="IyEvYmluL3NoCnNldCAtZXUKCklNQUdFX1RBRz0iJHtJTUFHRV9UQUc6LX0iClJFR0lTVFJZPSIke1JFR0lTVFJZOi1naXQueGlhb3hpYWppYW5qaS5jb20veGlhb3hpYS94aWFveGlhLXNhYXN9IgpSRUdJU1RSWV9VU0U9IiR7UkVHSVNUUllfVVNFUjp-eGlhb3hpYX0iClJFR0lTVFJZX1RPS0VOPSIke1JFR0lTVFJZX1RPS0VOOi19IgpFTlZfRklMRT0iJHtFTlZfRklMRTotL3Zhci9saWIveGlhb3hpYS1zYWFzLXByb2R1Y3Rpb24vLmVudn0iCkdFTkVSQVRFRF9ESVI9IiR7R0VORVJBVEVEX0RJUjotL3Zhci9saWIveGlhb3hpYS1zYWFzLXByb2R1Y3Rpb24vZ2VuZXJhdGVkfSIKTEVHQUNZX0FTU0VUU19ESVI9IiR7TEVHQUNZX0FTU0VUU19ESVI6LS92YXIvbGliL3hpYW94aWEtc2Fhcy1wcm9kdWN0aW9uL2xlZ2FjeS1hc3NldHN9IgoKaWYgWyAteiAiJElNQUdFX1RBRyIgXTsgdGhlbgogIGVjaG8gIkVSUk9SOiBJTUFHRV9UQUcgaXMgcmVxdWlyZWQiCiAgZXhpdCAxCmZpCgp0ZXN0IC1mICIkRU5WX0ZJTEUiCm1rZGlyIC1wICIkR0VORVJBVEVEX0RJUiIKbWtkaXIgLXAgIiRMRUdBQ1lfQVNTRVRTX0RJUiIKCiMgTG9naW4gdG8gcmVnaXN0cnkKaWYgWyAtbiAiJFJFR0lTVFJZX1RPS0VOIiBdOyB0aGVuCiAgUkVHSVNUUllfSE9TVD0kKGVjaG8gIiRSRUdJU1RSWSIgfCBjdXQgLWQvIC1mMSkKICBwcmludGYgJXMgIiRSRUdJU1RSWV9UT0tFTiIgfCBkb2NrZXIgbG9naW4gIiRSRUdJU1RSWV9IT1NUIiAtdSAiJFJFR0lTVFJZX1VTRVIiIC0tcGFzc3dvcmQtc3RkaW4gMj4vZGV2L251bGwgfHwgewogICAgZWNobyAiV0FSTjogZG9ja2VyIGxvZ2luIGZhaWxlZCwgd2lsbCB0cnkgdG8gcHVsbCBhbnl3YXkiCiAgfQpmaQoKIyBQdWxsIGltYWdlcwpSRUdJU1RSWV9BUEk9IiR7UkVHSVNUUll9L3hpYW94aWEtc2Fhcy1hcGk6JHtJTUFHRV9UQUd9IgpSRUdJU1RSWV9XT1JLRVI9IiR7UkVHSVNUUll9L3hpYW94aWEtc2Fhcy13b3JrZXI6JHtJTUFHRV9UQUd9IgpSRUdJU1RSWV9XRUI9IiR7UkVHSVNUUll9L3hpYW94aWEtc2Fhcy13ZWI6JHtJTUFHRV9UQUd9IgoKTE9DQUxfQVBJPSJ4aWFveGlhLXNhYXMtYXBpOiR7SU1BR0VfVEFHfSIKTE9DQUxfV09SS0VSPSJ4aWFveGlhLXNhYXMtd29ya2VyOiR7SU1BR0VfVEFHfSIKTE9DQUxfV0VCPSJ4aWFveGlhLXNhYXMtd2ViOiR7SU1BR0VfVEFHfSIKCmVjaG8gIlB1bGxpbmcgQVBJIGltYWdlLi4uIgpkb2NrZXIgcHVsbCAiJFJFR0lTVFJZX0FQSSIKZWNobyAiUHVsbGluZyBXb3JrZXIgaW1hZ2UuLi4iCmRvY2tlciBwdWxsICIkUkVHSVNUUllfV09SS0VSCmVjaG8gIlB1bGxpbmcgV2ViIGltYWdlLi4uIgpkb2NrZXIgcHVsbCAiJFJFR0lTVFJZX1dFQiIKCiMgUmUtdGFnCmRvY2tlciB0YWcgIiRSRUdJU1RSWV9BUEkiICIkTE9DQUxfQVBJIgpkb2NrZXIgdGFnICIkUkVHSVNUUllfV09SS0VSIiAiJExPQ0FMX1dPUktFUiIKZG9ja2VyIHRhZyAiJFJFR0lTVFJZX1dFQiAiJExPQ0FMX1dFQiIKZWNobyAiQWxsIGltYWdlcyBwdWxsZWQgYW5kIHRhZ2dlZC4iCgojIEJhY2sgdXAgbGVnYWN5IGFzc2V0cwplY2hvICJCYWNraW5nIHVwIGxlZ2FjeSBhc3NldHMgZnJvbSBjdXJyZW50IHdlYiBjb250YWluZXIuLi4iCmlmIGRvY2tlciBpbnNwZWN0IHhhb3hpYS13ZWItcHJvZHVjdGlvbiA+L2Rldi9udWxsIDI+JjE7IHRoZW4KICBfdG1wZGlyPSIvdG1wL2xlZ2FjeS1hc3NldHMtJCQiCiAgcm0gLXJmICIkX3RtcGRpciIKICBta2RpciAtcCAiJF90bXBkaXIiCiAgZG9ja2VyIGNwIHhhb3hpYS13ZWItcHJvZHVjdGlvbjovdXNyL3NoYXJlL25naW54L2h0bWwvYXNzZXRzLy4gIiRfdG1wZGlyLyIgMj4vZGV2L251bGwgfHwgdHJ1ZQogIGlmIFsgLWQgIiRfdG1wZGlyIl0gJiYgWyAiJChscyAtQSAiJF90bXBkaXIiIDI+L2Rldi9udWxsKSIgXTsgdGhlbgogICAgY3AgLWFuICIkX3RtcGRpciIvLiAiJExFR0FDWV9BU1NFVFNfRElSIi8gMj4vZGV2L251bGwgfHwgdHJ1ZQogICAgZWNobyAiTGVnYWN5IGFzc2V0cyBiYWNrZWQgdXA6ICQobHMgIiRfdG1wZGlyIiB8IHdjIC1sKSBmaWxlcyIKICBmaQogIHJtIC1yZiAiJF90bXBkaXIiCmVsc2UKICBlY2hvICJObyBleGlzdGluZyB3ZWIgY29udGFpbmVyLCBza2lwcGluZyBsZWdhY3kgYXNzZXRzIGJhY2t1cCIKZmkKCiMgQ2xlYW4gb2xkZXIgbGVnYWN5IGFzc2V0cyAoNyBkYXlzKQppZiBbIC1kICIkTEVHQUNZX0FTU0VUU19ESVIiBdOyB0aGVuCiAgZmluZCAiJExFR0FDWV9BU1NFVFNfRElSIiAtdHlwZSBmIC1tdGltZSArNyAtZGVsZXRlIDI+L2Rldi9udWxsIHx8IHRydWUKICBlY2hvICJMZWdhY3kgYXNzZXRzIGNsZWFudXAgZG9uZSAocmV0YWluIDcgZGF5cykiCmZpCgojIENoZWNrIGluZnJhc3RydWN0dXJlCmZvciBjIGluIHhhb3hpYS1wb3N0Z3Jlcy1wcm9kdWN0aW9uIHhhb3hpYS1yZWRpcy1wcm9kdWN0aW9uOyBkbwogIGlmICEgZG9ja2VyIGluc3BlY3QgIiRjIiA+L2Rldi9udWxsIDI+JjE7IHRoZW4KICAgIGVjaG8gIkVSUk9SOiBSZXF1aXJlZCBjb250YWluZXIgbm90IGZvdW5kOiAkYyIKICAgIGV4aXQgMQogIGZpCmRvbmUKCmRvY2tlciBuZXR3b3JrIGNyZWF0ZSB4aWFveGlhLW5ldC1wcm9kdWN0aW9uIDI+L2Rldi9udWxsIHx8IHRydWUKCiMgUnVuIG1pZ3JhdGlvbnMKZWNobyAiUnVubmluZyBkYXRhYmFzZSBtaWdyYXRpb25zLi4uIgpkb2NrZXIgcnVuIC0tcm0gXAogIC0tZW52LWZpbGUgIiRFTlZfRklMRSIgXAogIC0tbmV0d29yayB4aWFveGlhLW5ldC1wcm9kdWN0aW9uIFwKICAtZSBBUFBfRU5WPXByb2R1Y3Rpb24gXAogICIkTE9DQUxfQVBJIihzaCAtYyAiY2QgL2FwcCAmJiBhbGVtYmljIHVwZ3JhZGUgaGVhZCIKZWNobyAiTWlncmF0aW9ucyBjb21wbGV0ZWQuIgoKIyBTdG9wIG9sZCBjb250YWluZXJzCmRvY2tlciBybSAtZiB4aWFveGlhLWFwaS1wcm9kdWN0aW9uIDI+L2Rldi9udWxsIHx8IHRydWUKZG9ja2VyIHJtIC1mIHhhb3hpYS13b3JrZXItcHJvZHVjdGlvbiAyPi9kZXYvbnVsbCB8fCB0cnVlCmRvY2tlciBybSAtZiB4aWFveGlhLXdlYi1wcm9kdWN0aW9uIDI+L2Rldi9udWxsIHx8IHRydWUKCiMgTG9nIG9wdGlvbnMKTE9HX09QVFM9Ii0tbG9nLWRyaXZlciBqc29uLWZpbGUgLS1sb2ctb3B0IG1heC1zaXplPTUwbSAtLWxvZy1vcHQgbWF4LWZpbGU9MyIKCiMgU3RhcnQgQVBJCmVjaG8gIlN0YXJ0aW5nIEFQSSBjb250YWluZXIuLi4iCmRvY2tlciBydW4gLWQgXAogIC0tbmFtZSB4aWFveGlhLWFwaS1wcm9kdWN0aW9uIFwKICAtLWVudi1maWxlICIkRU5WX0ZJTEUiIFwKICAtLW5ldHdvcmsgeGlhb3hpYS1uZXQtcHJvZHVjdGlvbiBcCiAgLXAgMTI3LjAuMC4xOjgwMDE6ODAwMCBcCiAgZSBBUFBfRU5WPXByb2R1Y3Rpb24gXAogIC1lIEFQUF9WRVJTSU9OPSIkSU1BR0VfVEFHIiBcCiAgLWUgR0VORVJBVEVEX0ZJTEVTX0RJUj0vYXBwL2dlbmVyYXRlZCBcCiAgLWUgR0VORVJBVEVEX0ZJTEVTX1VSTF9QUkVGSVg9L2dlbmVyYXRlZC1maWxlcyBcCiAgLWBQVUJMSUNfQVBJX0JBU0VfVVJMPWh0dHBzOi8vYXBpLnhpYW94aWFqaWFuamkuY29tIFwKICAtdiAiJEdFTkVSQVRFRF9ESVI6L2FwcC9nZW5lcmF0ZWQiIFwKICAtLXJlc3RhcnQgdW5sZXNzLXN0b3BwZWQgXAogIC0tY3B1cyAyIFwKIC0tbWVtb3J5IDJnIFwKIC0taGVhbHRoLWNtZCAicHl0aG9uIC1jIFwiaW1wb3J0IHVybGxpYi5yZXF1ZXN0OyB1cmxsaWIucmVxdWVzdC51cmxvcGVuKCdodHRwOi8vbG9jYWxob3N0OjgwMDAvaGVhbHRoJywgdGltZW91dD01KVwiIiBcCiAgLS1oZWFsdGctaW50ZXJ2YWwgMzBzIFwKICAtLWhlYWx0aC10aW1lb3V0IDEwcyBcCiAgLS1oZWFsdGgtcmV0cmllcyAzIFwKIC0taGVhbHRoLXN0YXJ0LXBlcmlvZCA0MHMgXAogICRMTE9HX09QVFMgXAogICIkTE9DQUxfQVBJIgoKIyBTdGFydCBXb3JrZXIKZWNobyAiU3RhcnRpbmcgV29ya2VyIGNvbnRhaW5lci4uLiIKZG9ja2VyIHJ1biAtZCBcCiAgLS1uYW1lIHhhb3hpYS13b3JrZXItcHJvZHVjdGlvbiBcCiAgLS1lbnYtZmlsZSAiJEVOVl9GSUxFIiBcCiAgLS1uZXR3b3JrIHhhb3hpYS1uZXQtcHJvZHVjdGlvbiBcCiAgZSBBUFBfRU5WPXByb2R1Y3Rpb24gXAogIC1lIEFQUF9WRVJTSU9OPSIkSU1BR0VfVEFHIiBcCiAgLWUgV09SS0VSX0NPTkNVUlJFTkN5PTEgXAogIC1lIFdPUktFUl9NQVhfVEFTS1NfUEVSX0NISUxEPTEwMCBcCiAgLWVuZGVkX0ZJTEVTX0RJUj0vYXBwL2dlbmVyYXRlZCBcCiAgLWVfRklMRVNfVVJMX1BSRUZJWD0vZ2VuZXJhdGVkLWZpbGVzIFwKICAtZSBQVUJMSUNfQVBJX0JBU0VfVVJMPWh0dHBzOi8vYXBpLnhpYW94aWFqaWFuamkuY29tIFwKICAtdiAiJEdFTkVSQVRFRF9ESVI6L2FwcC9nZW5lcmF0ZWQiIFwKICAtLXJlc3RhcnQgdW5sZXNzLXN0b3BwZWQgXAogIC0tY3B1cyAyIFwKIC0tbWVtb3J5IDJnIFwKIC0taGVhbHRoLWNtZCAic2ggLWMgXCJncmVwIC1xIGNsZXJ5IC9wcm9jLzEvY21kbGluZSB8fCBleGl0IDFcIiIgXAogIC0taGVhbHRoLWludGVydmFsIDMwcyBcCiAgLS1oZWFsdGgtdGltZW91dCAxMHMgXAogIC0taGVhbHRoLXJldHJpZXMgMyBcCiAgLS1oZWFsdGgtc3RhcnQtcGVyaW9kIDMwcyBcCiAgJExPR19PUFRTIFwKICAiJExPQ0FMX1dPUktFUiIKCiMgU3RhcnQgV2ViCiMgTGVnYWN5IGFzc2V0cyBtb3VudDogdW5kZXIgbmdpbnggYXNzZXRzLWxlZ2FjeS9hc3NldHMKTEVHQUNZX1ZPTFVNRT0iIgppZiBbIC1kICIkTEVHQUNZX0FTU0VUU19ESVIiIF0gJiYgWyAiJChscyAtQSAiJExFR0FDWV9BU1NFVFNfRElSIiAyPi9kZXYvbnVsbCkiIF07IHRoZW4KICBMRUdBQ1lfVk9MVU1FPSItdiAke0xFR0FDWV9BU1NFVFNfRElSfTovdXNyL3NoYXJlL25naW54L2h0bWwvYXNzZXRzLWxlZ2FjeS9hc3NldHM6cm8iCiAgZWNobyAiV2ViIGNvbnRhaW5lcjogbGVnYWN5IGFzc2V0cyBtb3VudGVkIChmYWxsYmFjaykiCmVsc2UKICBlY2hvICJXZWIgY29udGFpbmVyOiBubyBsZWdhY3kgYXNzZXRzIHRvIG1vdW50IgpmaQoKZWNobyAiU3RhcnRpbmcgV2ViIGNvbnRhaW5lci4uLiIKZG9ja2VyIHJ1biAtZCBcCiAgLS1uYW1lIHhhb3hpYS13ZWItcHJvZHVjdGlvbiBcCiAgLS1uZXR3b3JrIHhhb3hpYS1uZXQtcHJvZHVjdGlvbiBcCiAgLXAgMTI3LjAuMC4xOjMwMDI6ODAgXAogIC0tcmVzdGFydCB1bmxlc3Mtc3RvcHBlZCBcCiAgLS1jcHVzIDAuNSBcCiAgLS1tZW1vcnkgNTEybSBcCiAgJExFR0FDWV9WT0xVTUUgXAogIC0taGVhbHRoLWNtZCAid2dldCAtLXNwaWRlciAtcSBodHRwOi8vMTI3LjAuMC4xOjgwIiBcCiAgLS1oZWFsdGhtaW50ZXJ2YWwgMzBzIFwKICAtLWhlYWx0aC10aW1lb3V0IDVzIFwKIC0taGVhbHRoLXJldHJpZXMgMyBcCiAgJExPR19PUFRTIFwKICAiJExPQ0FMX1dFQiIKCiMgV2FpdCBmb3IgaGVhbHRoCmVjaG8gIldhaXRpbmcgZm9yIEFQSSB0byBiZWNvbWUgaGVhbHRoeS4uLiIKaT0wCndoaWxlIFsgIiRpIiAtbHQgNDAgXTsgZG8KICBpZiBjdXJsIC1zZiAtLW1heC10aW1lIDUgaHR0cDovLzEyNy4wLjAuMTo4MDAxL2hlYWx0aCA+L2Rldi9udWxsIDI+JjE7IHRoZW4KICAgIGVjaG8gIkFQSSBpcyBoZWFsdGh5ISIKICAgIGJyZWFrCiAgZmkKICBpPSQoKGkgKyAxKSkKICBlY2hvICIgIFdhaXRpbmcuLi4gKCQpL0QwKSIKICBzbGVlcCAzCmRvbmUKCmlmIFsgIiRpIiAtZ2UgNDAgXTsgdGhlbgogIGVjaG8gIkVSUk9SOiBBUEkgZGlkIG5vdCBiZWNvbWUgaGVhbHRoeSB3aXRoaW4gMTIwc3MiCiAgZG9ja2VyIGxvZ3MgLS10YWlsIDUwIHhhb3hpYS1hcGktcHJvZHVjdGlvbgogIGV4aXQgMQpmaQoKZWNobyAiV2FpdGluZyBmb3IgV2ViIHRvIGJlY29tZSBoZWFsdGh5Li4uIgppPTAKd2hpbGUgWyAiJGkiIC1sdCAxNSBdOyBkbwogIGlmIGN1cmwgLXNmIC0tbWF4LXRpbWUgNSBodHRwOi8vMTI3LjAuMC4xOjMwMDIvID4vZGV2L251bGwgMj4mMTsgdGhlbgogICAgZWNobyAiV2ViIGlzIGhlYWx0aHkhIgogICAgYnJlYWsKICBmaQogIGk9JCgoaSArIDEpKQogIGVjaG8gIiAgV2FpdGluZy4uLiAoJGkvMTUpIgogIHNsZWVwIDIKZG9uZQoKaWYgWyAiJGkiIC1nZSAxNSBdOyB0aGVuCiAgZWNobyAiRVJST1I6IFdlYiBkaWQgbm90IGJlY29tZSBoZWFsdGh5IHdpdGhpbiAzMHMiCiAgZG9ja2VyIGxvZ3MgLS10YWlsIDMwIHhhb3hpYS13ZWItcHJvZHVjdGlvbgogIGV4aXQgMQpmaQoKIyBDbGVhbiB1cA==" - echo "$DEPLOY_B64" | base64 -d | ssh -p 22222 -i "$key_path" "${production_user}@${production_host}" "IMAGE_TAG='${GITHUB_REF_NAME}' REGISTRY_TOKEN='${REGISTRY_TOKEN}' sh" + cat scripts/ci_production_deploy.sh | ssh -p "$production_port" -i "$key_path" -o StrictHostKeyChecking=no "${production_user}@${production_host}" "IMAGE_TAG='${GITHUB_REF_NAME}' ACR_USERNAME='${ACR_USERNAME}' ACR_PASSWORD='${ACR_PASSWORD}' sh" - name: Production health check + auto rollback if: success() diff --git a/scripts/ci_production_deploy.sh b/scripts/ci_production_deploy.sh new file mode 100644 index 000000000..dec3e9032 --- /dev/null +++ b/scripts/ci_production_deploy.sh @@ -0,0 +1,443 @@ +#!/bin/sh +# =========================================== +# Production 部署脚本（SSH 模式，支持自动回滚） +# =========================================== +# 通过 SSH 在 production 服务器上执行 +# +# 环境变量: +# IMAGE_TAG - 镜像版本 tag（如 commit SHA 或分支名） +# REGISTRY_TOKEN - Registry 访问令牌 +# REGISTRY - Registry 地址（默认 xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji） +# REGISTRY_USER - Registry 用户名（默认 xiaoxia） +# ENV_FILE - 环境变量文件路径 +# GENERATED_DIR - 生成文件目录 +# SKIP_MIGRATION - 跳过数据库迁移（true/false，默认 false） +# SKIP_ROLLBACK - 失败时跳过自动回滚（true/false，默认 false） + +set -eu + +# ---- 重试工具函数 ---- +retry_cmd() { + local max_attempts=$1 + local backoff=$2 + shift 2 + local attempt=1 + while [ $attempt -le $max_attempts ]; do + if "$@"; then + return 0 + fi + echo " attempt $attempt/$max_attempts failed, retrying in ${backoff}s..." + sleep $backoff + backoff=$((backoff * 2)) + attempt=$((attempt + 1)) + done + echo " ERROR: failed after $max_attempts retries" + return 1 +} + +retry_docker_login() { + echo "Logging in to registry (up to 3 retries)" + if retry_cmd 3 5 sh -c "printf %s "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin"; then + return 0 + fi + echo "WARN: docker login failed after retries, will try pull anyway" + return 0 +} + +retry_docker_pull() { + local image=$1 + echo "Pulling $image (up to 3 retries)" + retry_cmd 3 10 docker pull "$image" +} + +IMAGE_TAG="${IMAGE_TAG:-}" +REGISTRY="${REGISTRY:-xiaoxia-registry.cn-hangzhou.cr.aliyuncs.com/xiaoxiakeji}" +REGISTRY_USER="${ACR_USERNAME:-${REGISTRY_USER:-nick0415343655}}" +REGISTRY_TOKEN="${ACR_PASSWORD:-${REGISTRY_TOKEN:-}}" + +ENV_FILE="${ENV_FILE:-/var/lib/xiaoxia-saas-production/.env}" +GENERATED_DIR="${GENERATED_DIR:-/var/lib/xiaoxia-saas-production/generated}" +LEGACY_ASSETS_DIR="${LEGACY_ASSETS_DIR:-/var/lib/xiaoxia-saas-production/legacy-assets}" + +SKIP_MIGRATION="${SKIP_MIGRATION:-false}" +SKIP_ROLLBACK="${SKIP_ROLLBACK:-false}" + +if [ -z "$IMAGE_TAG" ]; then + echo "ERROR: IMAGE_TAG is required" + exit 1 +fi + +test -f "$ENV_FILE" +mkdir -p "$GENERATED_DIR" +mkdir -p "$LEGACY_ASSETS_DIR" + +echo "===========================================" +echo " Production 部署 - $IMAGE_TAG" +echo "===========================================" + +# ---- 记录当前运行的镜像版本（用于回滚） ---- +echo "Recording current image versions for rollback..." +PREV_API_IMAGE="" +PREV_WORKER_IMAGE="" +PREV_WEB_IMAGE="" +for c in xiaoxia-api-production xiaoxia-worker-production xiaoxia-web-production; do + if docker inspect "$c" >/dev/null 2>&1; then + img=$(docker inspect -f '{{.Config.Image}}' "$c") + case "$c" in + xiaoxia-api-production) PREV_API_IMAGE="$img" ;; + xiaoxia-worker-production) PREV_WORKER_IMAGE="$img" ;; + xiaoxia-web-production) PREV_WEB_IMAGE="$img" ;; + esac + echo " $c -> $img" + else + echo " $c -> (not running)" + fi +done + +# ---- 回滚函数 ---- +rollback() { + echo "" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo " 部署失败，正在自动回滚到上一版本..." + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "" + + if [ "$SKIP_ROLLBACK" = "true" ]; then + echo "SKIP_ROLLBACK=true，跳过自动回滚" + exit 1 + fi + + # 停止当前（失败的）新容器 + echo "Stopping new containers..." + docker rm -f xiaoxia-api-production 2>/dev/null || true + docker rm -f xiaoxia-worker-production 2>/dev/null || true + docker rm -f xiaoxia-web-production 2>/dev/null || true + + LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" + + # 恢复 API + if [ -n "$PREV_API_IMAGE" ]; then + echo "Rolling back API to: $PREV_API_IMAGE" + docker run -d \ + --name xiaoxia-api-production \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-production \ + -p 127.0.0.1:8001:8000 \ + -e APP_ENV=production \ + -e APP_VERSION="$(echo $PREV_API_IMAGE | grep -oE '[^:]+$')" \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --cpus 2 \ + --memory 2g \ + --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 40s \ + $LOG_OPTS \ + "$PREV_API_IMAGE" + else + echo "No previous API image to roll back to" + fi + + # 恢复 Worker + if [ -n "$PREV_WORKER_IMAGE" ]; then + echo "Rolling back Worker to: $PREV_WORKER_IMAGE" + docker run -d \ + --name xiaoxia-worker-production \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-production \ + -e APP_ENV=production \ + -e APP_VERSION="$(echo $PREV_WORKER_IMAGE | grep -oE '[^:]+$')" \ + -e WORKER_CONCURRENCY=1 \ + -e WORKER_MAX_TASKS_PER_CHILD=100 \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --cpus 2 \ + --memory 2g \ + --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 30s \ + $LOG_OPTS \ + "$PREV_WORKER_IMAGE" + else + echo "No previous Worker image to roll back to" + fi + + # 恢复 Web + if [ -n "$PREV_WEB_IMAGE" ]; then + echo "Rolling back Web to: $PREV_WEB_IMAGE" + LEGACY_VOLUME="" + if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then + LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" + fi + docker run -d \ + --name xiaoxia-web-production \ + --network xiaoxia-net-production \ + -p 127.0.0.1:3002:80 \ + --restart unless-stopped \ + --cpus 0.5 \ + --memory 512m \ + $LEGACY_VOLUME \ + --health-cmd "wget --spider -q http://127.0.0.1:80" \ + --health-interval 30s \ + --health-timeout 5s \ + --health-retries 3 \ + $LOG_OPTS \ + "$PREV_WEB_IMAGE" + else + echo "No previous Web image to roll back to" + fi + + # 等待 API 回滚后恢复健康 + if [ -n "$PREV_API_IMAGE" ]; then + echo "Waiting for rolled-back API to become healthy..." + i=0 + while [ "$i" -lt 40 ]; do + if curl -sf --max-time 5 http://127.0.0.1:8001/health >/dev/null 2>&1; then + echo "Rolled-back API is healthy!" + break + fi + i=$((i + 1)) + echo " Waiting... ($i/40)" + sleep 3 + done + if [ "$i" -ge 40 ]; then + echo "WARN: Rolled-back API did not become healthy within 120s" + docker logs --tail 30 xiaoxia-api-production + fi + fi + + echo "" + echo "===========================================" + echo " 回滚完成" + echo "===========================================" + echo "Previous API: ${PREV_API_IMAGE:-none}" + echo "Previous Worker: ${PREV_WORKER_IMAGE:-none}" + echo "Previous Web: ${PREV_WEB_IMAGE:-none}" + echo "" + echo "部署失败，已自动回滚到上一版本" + docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep production + exit 1 +} + +# ---- 登录 Registry ---- +if [ -n "$REGISTRY_TOKEN" ]; then + echo "==========================================" + echo " Login to Registry (with retries)" + echo "==========================================" + REGISTRY_HOST=$(echo "$REGISTRY" | cut -d/ -f1) + retry_docker_login +fi + +# ---- Pull 新版本镜像 ---- +REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" +REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" +REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" + +echo "==========================================" +echo " Pull images (with retries)" +echo "==========================================" +retry_docker_pull "$REGISTRY_API" +retry_docker_pull "$REGISTRY_WORKER" +retry_docker_pull "$REGISTRY_WEB" + +echo "All images pulled."" + +# ---- 备份 legacy assets ---- +echo "Backing up legacy assets from current web container..." +if docker inspect xiaoxia-web-production >/dev/null 2>&1; then + _tmpdir="/tmp/legacy-assets-$$" + rm -rf "$_tmpdir" + mkdir -p "$_tmpdir" + docker cp xiaoxia-web-production:/usr/share/nginx/html/assets/. "$_tmpdir/" 2>/dev/null || true + # 只有目录非空才拷贝，避免覆盖有内容的 legacy assets + if [ -d "$_tmpdir" ] && [ "$(ls -A "$_tmpdir" 2>/dev/null)" ]; then + cp -an "$_tmpdir"/. "$LEGACY_ASSETS_DIR"/ 2>/dev/null || true + echo "Legacy assets backed up: $(ls "$_tmpdir" | wc -l) files" + fi + rm -rf "$_tmpdir" +else + echo "No existing web container, skipping legacy assets backup" +fi + +# 清理 7 天前的 legacy assets +if [ -d "$LEGACY_ASSETS_DIR" ]; then + find "$LEGACY_ASSETS_DIR" -type f -mtime +7 -delete 2>/dev/null || true + echo "Legacy assets cleanup done (retain 7 days)" +fi + +# ---- 检查基础设施容器 ---- +echo "Checking infrastructure containers..." +for c in xiaoxia-postgres-production xiaoxia-redis-production; do + if ! docker inspect "$c" >/dev/null 2>&1; then + echo "ERROR: Required container not found: $c" + exit 1 + fi + state=$(docker inspect -f '{{.State.Status}}' "$c") + if [ "$state" != "running" ]; then + echo "ERROR: Container not running: $c ($state)" + exit 1 + fi +done + +# ---- 创建网络（不存在则创建） ---- +docker network create xiaoxia-net-production 2>/dev/null || true + +# ---- 数据库迁移 ---- +if [ "$SKIP_MIGRATION" != "true" ]; then + echo "Running database migrations..." + docker run --rm \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-production \ + -e APP_ENV=production \ + "$REGISTRY_API" sh -c "cd /app && alembic upgrade head" || { + echo "ERROR: Database migration failed" + echo "Note: Migration failures are NOT automatically rolled back (data safety)" + echo "Please manually check and fix the migration, then redeploy" + exit 1 + } + echo "Migrations completed." +else + echo "Skipping migrations (SKIP_MIGRATION=true)" +fi + +# ---- 停止旧容器 ---- +echo "Stopping old containers..." +docker rm -f xiaoxia-api-production 2>/dev/null || true +docker rm -f xiaoxia-worker-production 2>/dev/null || true +docker rm -f xiaoxia-web-production 2>/dev/null || true + +LOG_OPTS="--log-driver json-file --log-opt max-size=50m --log-opt max-file=3" + +# ---- 启动 API ---- +echo "Starting API container..." +docker run -d \ + --name xiaoxia-api-production \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-production \ + -p 127.0.0.1:8001:8000 \ + -e APP_ENV=production \ + -e APP_VERSION="$IMAGE_TAG" \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --cpus 2 \ + --memory 2g \ + --health-cmd "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=5)\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 40s \ + $LOG_OPTS \ + "$REGISTRY_API" || rollback + +# ---- 启动 Worker ---- +echo "Starting Worker container..." +docker run -d \ + --name xiaoxia-worker-production \ + --env-file "$ENV_FILE" \ + --network xiaoxia-net-production \ + -e APP_ENV=production \ + -e APP_VERSION="$IMAGE_TAG" \ + -e WORKER_CONCURRENCY=1 \ + -e WORKER_MAX_TASKS_PER_CHILD=100 \ + -e GENERATED_FILES_DIR=/app/generated \ + -e GENERATED_FILES_URL_PREFIX=/generated-files \ + -e PUBLIC_API_BASE_URL=https://production-api.xiaoxiajianji.com \ + -v "$GENERATED_DIR:/app/generated" \ + --restart unless-stopped \ + --cpus 2 \ + --memory 2g \ + --health-cmd "sh -c \"grep -q celery /proc/1/cmdline || exit 1\"" \ + --health-interval 30s \ + --health-timeout 10s \ + --health-retries 3 \ + --health-start-period 30s \ + $LOG_OPTS \ + "$REGISTRY_WORKER" || rollback + +# ---- 启动 Web ---- +LEGACY_VOLUME="" +if [ -d "$LEGACY_ASSETS_DIR" ] && [ "$(ls -A "$LEGACY_ASSETS_DIR" 2>/dev/null)" ]; then + LEGACY_VOLUME="-v ${LEGACY_ASSETS_DIR}:/usr/share/nginx/html/assets-legacy/assets:ro" + echo "Web container: legacy assets mounted (fallback)" +else + echo "Web container: no legacy assets to mount" +fi + +echo "Starting Web container..." +docker run -d \ + --name xiaoxia-web-production \ + --network xiaoxia-net-production \ + -p 127.0.0.1:3002:80 \ + --restart unless-stopped \ + --cpus 0.5 \ + --memory 512m \ + $LEGACY_VOLUME \ + --health-cmd "wget --spider -q http://127.0.0.1:80" \ + --health-interval 30s \ + --health-timeout 5s \ + --health-retries 3 \ + $LOG_OPTS \ + "$REGISTRY_WEB" || rollback + +# ---- 等待 API 健康 ---- +echo "Waiting for API to become healthy..." +i=0 +while [ "$i" -lt 40 ]; do + if curl -sf --max-time 5 http://127.0.0.1:8001/health >/dev/null 2>&1; then + echo "API is healthy!" + break + fi + i=$((i + 1)) + echo " Waiting... ($i/40)" + sleep 3 +done + +if [ "$i" -ge 40 ]; then + echo "ERROR: API did not become healthy within 120s" + docker logs --tail 50 xiaoxia-api-production + rollback +fi + +# ---- 等待 Web 健康 ---- +echo "Waiting for Web to become healthy..." +i=0 +while [ "$i" -lt 15 ]; do + if curl -sf --max-time 5 http://127.0.0.1:3002/ >/dev/null 2>&1; then + echo "Web is healthy!" + break + fi + i=$((i + 1)) + echo " Waiting... ($i/15)" + sleep 2 +done + +if [ "$i" -ge 15 ]; then + echo "ERROR: Web did not become healthy within 30s" + docker logs --tail 30 xiaoxia-web-production + rollback +fi + +# ---- 清理旧镜像 ---- +echo "Cleaning up old images..." +docker image prune -af --filter "until=168h" 2>/dev/null || true +docker builder prune -af --filter "until=168h" 2>/dev/null || true + +echo "" +echo "=== Production deployment complete ===" +echo "API: http://127.0.0.1:8000" +echo "Web: http://127.0.0.1:3001" +echo "Version: $IMAGE_TAG" +docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" | grep production diff --git a/scripts/ci_staging_deploy.sh b/scripts/ci_staging_deploy.sh index 65fc22644..f91613257 100755 --- a/scripts/ci_staging_deploy.sh +++ b/scripts/ci_staging_deploy.sh @@ -237,10 +237,6 @@ REGISTRY_API="${REGISTRY}/xiaoxia-saas-api:${IMAGE_TAG}" REGISTRY_WORKER="${REGISTRY}/xiaoxia-saas-worker:${IMAGE_TAG}" REGISTRY_WEB="${REGISTRY}/xiaoxia-saas-web:${IMAGE_TAG}" -LOCAL_API="xiaoxia-saas-api:${IMAGE_TAG}" -LOCAL_WORKER="xiaoxia-saas-worker:${IMAGE_TAG}" -LOCAL_WEB="xiaoxia-saas-web:${IMAGE_TAG}" - echo "==========================================" echo " Pull images (with retries)" echo "==========================================" @@ -248,11 +244,7 @@ retry_docker_pull "$REGISTRY_API" retry_docker_pull "$REGISTRY_WORKER" retry_docker_pull "$REGISTRY_WEB" -# Re-tag 成本地名 -docker tag "$REGISTRY_API" "$LOCAL_API" -docker tag "$REGISTRY_WORKER" "$LOCAL_WORKER" -docker tag "$REGISTRY_WEB" "$LOCAL_WEB" -echo "All images pulled and tagged." +echo "All images pulled."" # ---- 备份 legacy assets ---- echo "Backing up legacy assets from current web container..." @@ -301,7 +293,7 @@ if [ "$SKIP_MIGRATION" != "true" ]; then --env-file "$ENV_FILE" \ --network xiaoxia-net-staging \ -e APP_ENV=staging \ - "$LOCAL_API" sh -c "cd /app && alembic upgrade head" || { + "$REGISTRY_API" sh -c "cd /app && alembic upgrade head" || { echo "ERROR: Database migration failed" echo "Note: Migration failures are NOT automatically rolled back (data safety)" echo "Please manually check and fix the migration, then redeploy" @@ -340,7 +332,7 @@ docker run -d \ --health-retries 3 \ --health-start-period 40s \ $LOG_OPTS \ - "$LOCAL_API" || rollback + "$REGISTRY_API" || rollback # ---- 启动 Worker ---- echo "Starting Worker container..." @@ -363,7 +355,7 @@ docker run -d \ --health-retries 3 \ --health-start-period 30s \ $LOG_OPTS \ - "$LOCAL_WORKER" || rollback + "$REGISTRY_WORKER" || rollback # ---- 启动 Web ---- LEGACY_VOLUME="" @@ -386,7 +378,7 @@ docker run -d \ --health-timeout 5s \ --health-retries 3 \ $LOG_OPTS \ - "$LOCAL_WEB" || rollback + "$REGISTRY_WEB" || rollback # ---- 等待 API 健康 ---- echo "Waiting for API to become healthy..." -- 2.54.0