ci: staging部署改为Watchtower自动更新,砍掉SSH部署步骤
CI/CD Pipeline / Staging E2E Tests (push) Failing after 38h33m15s
CI/CD Pipeline / Build & Push Staging (Watchtower auto-deploy) (push) Failing after 38h35m28s
CI/CD Pipeline / Frontend Lint (push) Failing after 38h36m58s
CI/CD Pipeline / Validate Code Quality And Tests (push) Failing after 38h37m47s
CI/CD Pipeline / Production Browser E2E (push) Failing after 1697h11m14s
CI/CD Pipeline / Build Production Runtime Images (push) Failing after 1697h11m25s
CI/CD Pipeline / Deploy Production (push) Failing after 1697h42m51s
CI/CD Pipeline / Staging E2E Tests (push) Failing after 38h33m15s
CI/CD Pipeline / Build & Push Staging (Watchtower auto-deploy) (push) Failing after 38h35m28s
CI/CD Pipeline / Frontend Lint (push) Failing after 38h36m58s
CI/CD Pipeline / Validate Code Quality And Tests (push) Failing after 38h37m47s
CI/CD Pipeline / Production Browser E2E (push) Failing after 1697h11m14s
CI/CD Pipeline / Build Production Runtime Images (push) Failing after 1697h11m25s
CI/CD Pipeline / Deploy Production (push) Failing after 1697h42m51s
This commit is contained in:
+53
-127
@@ -267,7 +267,7 @@ jobs:
|
||||
docker.m.daocloud.io/library/node:20 \
|
||||
sh -lc 'npx vitest run src/test'
|
||||
deploy-staging:
|
||||
name: Deploy Staging
|
||||
name: Build & Push Staging (Watchtower auto-deploy)
|
||||
runs-on: saas
|
||||
needs: [validate, frontend-lint]
|
||||
|
||||
@@ -280,11 +280,10 @@ jobs:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -eu
|
||||
python3 - <<'PY'
|
||||
python3 - <<'INNERPY'
|
||||
import io, os, tarfile, time, urllib.request, urllib.error
|
||||
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
|
||||
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
|
||||
# Retry up to 5 times with backoff for transient 5xx errors
|
||||
last_err = None
|
||||
for attempt in range(5):
|
||||
try:
|
||||
@@ -306,7 +305,6 @@ jobs:
|
||||
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
raise
|
||||
else:
|
||||
raise last_err
|
||||
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
|
||||
@@ -319,14 +317,8 @@ jobs:
|
||||
member.name = name[len(root_prefix):]
|
||||
if member.name:
|
||||
tar.extract(member, '.')
|
||||
PY
|
||||
INNERPY
|
||||
|
||||
- name: Install SSH client
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
apt-get update -qq && apt-get install -y -qq openssh-client >/dev/null 2>&1
|
||||
echo "openssh-client installed"
|
||||
- name: Build and push all images to Gitea Registry
|
||||
shell: sh
|
||||
env:
|
||||
@@ -337,139 +329,73 @@ jobs:
|
||||
ALLOW_SHARED_PRODUCTION_BUILD_HOST=true REGISTRY_TOKEN="${REGISTRY_TOKEN}" \
|
||||
scripts/build_release_images.sh "${GITHUB_SHA}"
|
||||
|
||||
- name: Tag and push :staging images (for Watchtower auto-update)
|
||||
- name: Tag and push :staging images (Watchtower auto-update)
|
||||
shell: sh
|
||||
env:
|
||||
REGISTRY_TOKEN: ${{{{ secrets.REGISTRY_TOKEN }}}}
|
||||
run: |
|
||||
set -eu
|
||||
REGISTRY="git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas"
|
||||
if [ -n "${{REGISTRY_TOKEN:-}}" ]; then
|
||||
printf '%s' "${{REGISTRY_TOKEN}}" | docker login git.xiaoxiajianji.com -u xiaoxia --password-stdin 2>/dev/null
|
||||
fi
|
||||
for svc in api worker web; do
|
||||
docker tag "${{REGISTRY}}/xiaoxia-saas-${{svc}}:${{GITHUB_SHA}}" "${{REGISTRY}}/xiaoxia-saas-${{svc}}:staging"
|
||||
docker push "${{REGISTRY}}/xiaoxia-saas-${{svc}}:staging"
|
||||
done
|
||||
echo "All :staging images pushed to registry"
|
||||
|
||||
- name: Deploy staging via Registry pull
|
||||
shell: sh
|
||||
env:
|
||||
STAGING_SSH_HOST: ${{ secrets.STAGING_SSH_HOST }}
|
||||
STAGING_SSH_USER: ${{ secrets.STAGING_SSH_USER }}
|
||||
STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
|
||||
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
staging_host="${STAGING_SSH_HOST:-47.98.113.167}"
|
||||
staging_user="${STAGING_SSH_USER:-root}"
|
||||
mkdir -p ~/.ssh
|
||||
if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
|
||||
key_path="/root/.ssh/xiaoxia_runtime_builder"
|
||||
elif [ -n "${STAGING_SSH_KEY:-}" ]; then
|
||||
key_path="$HOME/.ssh/id_ed25519"
|
||||
printf '%s\n' "$STAGING_SSH_KEY" > "$key_path"
|
||||
chmod 600 "$key_path"
|
||||
else
|
||||
echo "ERROR: No SSH key available"
|
||||
exit 1
|
||||
REGISTRY="git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas"
|
||||
if [ -n "${REGISTRY_TOKEN:-}" ]; then
|
||||
printf '%s' "${REGISTRY_TOKEN}" | docker login git.xiaoxiajianji.com -u xiaoxia --password-stdin 2>/dev/null
|
||||
fi
|
||||
ssh-keyscan -H "$staging_host" >> ~/.ssh/known_hosts 2>/dev/null || true
|
||||
for svc in api worker web; do
|
||||
docker tag "${REGISTRY}/xiaoxia-saas-${svc}:${GITHUB_SHA}" "${REGISTRY}/xiaoxia-saas-${svc}:staging"
|
||||
docker push "${REGISTRY}/xiaoxia-saas-${svc}:staging"
|
||||
done
|
||||
echo "All :staging images pushed. Watchtower will auto-deploy within 60s."
|
||||
|
||||
# 上传部署脚本(小文件,非源码包)
|
||||
scp -i "$key_path" infra/docker/deploy-staging-registry.sh \
|
||||
"$staging_user@$staging_host:/tmp/deploy-staging-registry.sh"
|
||||
|
||||
# SSH 执行部署
|
||||
ssh -i "$key_path" "$staging_user@$staging_host" \
|
||||
"IMAGE_TAG='${GITHUB_SHA}' REGISTRY_TOKEN='${REGISTRY_TOKEN}' sh /tmp/deploy-staging-registry.sh"
|
||||
|
||||
- name: Post-deploy smoke test
|
||||
- name: Wait for Watchtower update + smoke test
|
||||
shell: sh
|
||||
env:
|
||||
STAGING_SSH_HOST: ${{ secrets.STAGING_SSH_HOST }}
|
||||
STAGING_SSH_USER: ${{ secrets.STAGING_SSH_USER }}
|
||||
STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
|
||||
run: |
|
||||
set -eu
|
||||
staging_host="${STAGING_SSH_HOST:-47.98.113.167}"
|
||||
staging_user="${STAGING_SSH_USER:-root}"
|
||||
if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then
|
||||
key_path="/root/.ssh/xiaoxia_runtime_builder"
|
||||
elif [ -n "${STAGING_SSH_KEY:-}" ]; then
|
||||
key_path="$HOME/.ssh/id_ed25519"
|
||||
else
|
||||
echo "ERROR: No SSH key available"
|
||||
echo "Waiting 90s for Watchtower to detect new image and restart containers..."
|
||||
sleep 90
|
||||
|
||||
echo "--- Smoke test 1: Health check ---"
|
||||
for i in $(seq 1 12); do
|
||||
HEALTH=$(curl -sf --max-time 10 https://staging-api.xiaoxiajianji.com/health) && break
|
||||
echo " Attempt $i/12: not ready yet, waiting 5s..."
|
||||
sleep 5
|
||||
done
|
||||
if [ -z "$HEALTH" ]; then
|
||||
echo "FAIL: health endpoint unreachable after 60s"
|
||||
exit 1
|
||||
fi
|
||||
echo "Health OK: $HEALTH"
|
||||
|
||||
echo "Running post-deploy smoke tests on staging..."
|
||||
echo "--- Smoke test 2: Login API (expect 401) ---"
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 -X POST \
|
||||
https://staging-api.xiaoxiajianji.com/api/v1/auth/login \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"email":"smoke@test.com","password":"wrong"}')
|
||||
if [ "$HTTP_CODE" != "401" ] && [ "$HTTP_CODE" != "422" ]; then
|
||||
echo "FAIL: login returned HTTP $HTTP_CODE (expected 401 or 422)"
|
||||
exit 1
|
||||
fi
|
||||
echo "Login API OK: HTTP $HTTP_CODE"
|
||||
|
||||
# Wait for service to fully start
|
||||
sleep 5
|
||||
echo "--- Smoke test 3: API docs endpoint ---"
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 https://staging-api.xiaoxiajianji.com/docs)
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "FAIL: /docs returned HTTP $HTTP_CODE (expected 200)"
|
||||
exit 1
|
||||
fi
|
||||
echo "Docs endpoint OK: HTTP $HTTP_CODE"
|
||||
|
||||
# Run smoke tests via SSH on the business host
|
||||
ssh -i "$key_path" "$staging_user@$staging_host" '
|
||||
echo "--- Smoke test 1: Health check ---"
|
||||
HEALTH=$(curl -sf --max-time 10 http://127.0.0.1:8000/health) || {
|
||||
echo "FAIL: health endpoint unreachable"
|
||||
exit 1
|
||||
}
|
||||
echo "Health OK: $HEALTH"
|
||||
echo "--- Smoke test 4: Web frontend ---"
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 https://staging.xiaoxiajianji.com/)
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "FAIL: web frontend returned HTTP $HTTP_CODE (expected 200)"
|
||||
exit 1
|
||||
fi
|
||||
echo "Web frontend OK: HTTP $HTTP_CODE"
|
||||
|
||||
echo "--- Smoke test 2: Login API (expect 401) ---"
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 -X POST \
|
||||
http://127.0.0.1:8000/api/v1/auth/login \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"email\":\"smoke@test.com\",\"password\":\"wrong\"}")
|
||||
echo ""
|
||||
echo "=== All smoke tests passed! ==="
|
||||
echo "Branch: ${GITHUB_REF_NAME}"
|
||||
echo "Commit: ${GITHUB_SHA}"
|
||||
|
||||
if [ "$HTTP_CODE" != "401" ] && [ "$HTTP_CODE" != "422" ]; then
|
||||
echo "FAIL: login returned HTTP $HTTP_CODE (expected 401 or 422)"
|
||||
exit 1
|
||||
fi
|
||||
echo "Login API OK: HTTP $HTTP_CODE"
|
||||
|
||||
echo "--- Smoke test 3: API docs endpoint ---"
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 http://127.0.0.1:8000/docs)
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "FAIL: /docs returned HTTP $HTTP_CODE (expected 200)"
|
||||
exit 1
|
||||
fi
|
||||
echo "Docs endpoint OK: HTTP $HTTP_CODE"
|
||||
|
||||
echo "--- Smoke test 4: Network isolation verification ---"
|
||||
# Verify staging containers are on the staging network
|
||||
STAGING_NET=$(docker inspect xiaoxia-api-staging --format="{{json .NetworkSettings.Networks}}" 2>/dev/null)
|
||||
if [ -z "$STAGING_NET" ]; then
|
||||
echo "WARN: Could not inspect staging container networks (container may not exist yet)"
|
||||
else
|
||||
echo "Staging API container networks: $STAGING_NET"
|
||||
if echo "$STAGING_NET" | grep -q "xiaoxia-net-staging"; then
|
||||
echo "Network isolation OK: staging containers on xiaoxia-net-staging"
|
||||
else
|
||||
echo "WARN: staging containers not on expected xiaoxia-net-staging network"
|
||||
echo " Current networks: $STAGING_NET"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Verify cross-environment DNS isolation
|
||||
# staging API should resolve to staging container, not production
|
||||
STAGING_API_IP=$(docker exec xiaoxia-web-staging getent hosts xiaoxia-api-staging 2>/dev/null | awk "{print \$1}" || true)
|
||||
PRODUCTION_API_IP=$(docker exec xiaoxia-web-staging getent hosts xiaoxia-api-production 2>/dev/null | awk "{print \$1}" || true)
|
||||
if [ -n "$STAGING_API_IP" ]; then
|
||||
echo "Staging API resolves to: $STAGING_API_IP (from web container)"
|
||||
fi
|
||||
if [ -n "$PRODUCTION_API_IP" ]; then
|
||||
echo "FAIL: staging web container can resolve production API address ($PRODUCTION_API_IP) - network isolation broken!"
|
||||
exit 1
|
||||
else
|
||||
echo "Network isolation OK: staging web cannot resolve xiaoxia-api-production"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== All smoke tests passed! ==="
|
||||
'
|
||||
|
||||
staging-e2e:
|
||||
name: Staging E2E Tests
|
||||
|
||||
Reference in New Issue
Block a user