diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index c98ef2fd9..0dc76baa4 100755 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -166,7 +166,7 @@ jobs: echo "ERROR: No SSH key available" exit 1 fi - echo 'c2V0IC1ldQphcnRpZmFjdD0iL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvYXJ0aWZhY3RzL3hpYW94aWEtc3RhZ2luZy0ke0dJVEhVQl9TSEF9LnRhci5neiIKaW1hZ2VfdGFyPSIvdmFyL2xpYi94aWFveGlhLXNhYXMtc3RhZ2luZy9hcnRpZmFjdHMveGlhb3hpYS13ZWItc3RhZ2luZy0ke0dJVEhVQl9TSEF9LnRhciIKdGVzdCAtZiAiJGFydGlmYWN0Igp0ZXN0IC1mICIkaW1hZ2VfdGFyIgp0ZXN0IC1mIC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nLy5lbnYKZG9ja2VyIGxvYWQgLWkgIiRpbWFnZV90YXIiCnJtIC1yZiAvdmFyL2xpYi94aWFveGlhLXNhYXMtc3RhZ2luZy9yZXBvCm1rZGlyIC1wIC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nL3JlcG8KdGFyIC14emYgIiRhcnRpZmFjdCIgLUMgL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvcmVwbwp0ZXN0IC1mIC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nL3JlcG8vYXBwcy93ZWIvZGlzdC9pbmRleC5odG1sCmNwIC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nLy5lbnYgL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvcmVwby8uZW52CmNobW9kICt4IC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nL3JlcG8vaW5mcmEvZG9ja2VyL2RlcGxveS1zdGFnaW5nLnNoCiMgRW5zdXJlIGlzb2xhdGVkIHN0YWdpbmcgbmV0d29yayBleGlzdHMgYmVmb3JlIGRlcGxveQpkb2NrZXIgbmV0d29yayBjcmVhdGUgeGlhb3hpYS1uZXQtc3RhZ2luZyAyPi9kZXYvbnVsbCB8fCB0cnVlClJFR0lTVFJZPSIxNzIuMzAuMTguMTk4OjUwMDAiIEFQSV9JTUFHRT0iJHtSRUdJU1RSWX0veGlhb3hpYS1zYWFzLWFwaTpkZXYiIFdPUktFUl9JTUFHRT0iJHtSRUdJU1RSWX0veGlhb3hpYS1zYWFzLXdvcmtlcjpkZXYiIFdFQl9JTUFHRT0ieGlhb3hpYS1zYWFzLXdlYjpzdGFnaW5nLSR7R0lUSFVCX1NIQX0iIEhPU1RfUFJFRklYPSBFTlY9c3RhZ2luZyBXRUJfUE9SVD0zMDAxIFJFQlVJTERfQkFDS0VORD0wIEJVSUxEX1dFQj0wIFJVTl9NSUdSQVRJT05TPTAgL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvcmVwby9pbmZyYS9kb2NrZXIvZGVwbG95LXN0YWdpbmcuc2gKaT0wCndoaWxlIFsgIiRpIiAtbHQgMzAgXTsgZG8KICBpZiB3Z2V0IC1xTy0gaHR0cDovLzEyNy4wLjAuMTo4MDAwL2hlYWx0aDsgdGhlbgogICAgZXhpdCAwCiAgZmkKICBpPSQoKGkgKyAxKSkKICBzbGVlcCAyCmRvbmUKZXhpdCAxCg==' | base64 -d | ssh -i "$key_path" "$staging_user@$staging_host" "GITHUB_SHA='${GITHUB_SHA}' sh" + echo 'c2V0IC1ldQphcnRpZmFjdD0iL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvYXJ0aWZhY3RzL3hpYW94aWEtc3RhZ2luZy0ke0dJVEhVQl9TSEF9LnRhci5neiIKaW1hZ2VfdGFyPSIvdmFyL2xpYi94aWFveGlhLXNhYXMtc3RhZ2luZy9hcnRpZmFjdHMveGlhb3hpYS13ZWItc3RhZ2luZy0ke0dJVEhVQl9TSEF9LnRhciIKdGVzdCAtZiAiJGFydGlmYWN0Igp0ZXN0IC1mICIkaW1hZ2VfdGFyIgp0ZXN0IC1mIC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nLy5lbnYKZG9ja2VyIGxvYWQgLWkgIiRpbWFnZV90YXIiCnJtIC1yZiAvdmFyL2xpYi94aWFveGlhLXNhYXMtc3RhZ2luZy9yZXBvCm1rZGlyIC1wIC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nL3JlcG8KdGFyIC14emYgIiRhcnRpZmFjdCIgLUMgL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvcmVwbwp0ZXN0IC1mIC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nL3JlcG8vYXBwcy93ZWIvZGlzdC9pbmRleC5odG1sCmNwIC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nLy5lbnYgL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvcmVwby8uZW52CmNobW9kICt4IC92YXIvbGliL3hpYW94aWEtc2Fhcy1zdGFnaW5nL3JlcG8vaW5mcmEvZG9ja2VyL2RlcGxveS1zdGFnaW5nLnNoCiMgRW5zdXJlIGlzb2xhdGVkIHN0YWdpbmcgbmV0d29yayBleGlzdHMgYmVmb3JlIGRlcGxveQpkb2NrZXIgbmV0d29yayBjcmVhdGUgeGlhb3hpYS1uZXQtc3RhZ2luZyAyPi9kZXYvbnVsbCB8fCB0cnVlClJFR0lTVFJZPSIxNzIuMzAuMTguMTk4OjUwMDAiIEFQSV9JTUFHRT0iJHtSRUdJU1RSWX0veGlhb3hpYS1zYWFzLWFwaTpkZXYiIFdPUktFUl9JTUFHRT0iJHtSRUdJU1RSWX0veGlhb3hpYS1zYWFzLXdvcmtlcjpkZXYiIFdFQl9JTUFHRT0ieGlhb3hpYS1zYWFzLXdlYjpzdGFnaW5nLSR7R0lUSFVCX1NIQX0iIEhPU1RfUFJFRklYPSBFTlY9c3RhZ2luZyBXRUJfUE9SVD0zMDAxIFJFQlVJTERfQkFDS0VORD0wIEJVSUxEX1dFQj0wIFJVTl9NSUdSQVRJT05TPTAgUkVHSVNUUllfUEFTUz0iWGlhb3hpYVJlZ2lzdHJ5MjAyNiIgL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvcmVwby9pbmZyYS9kb2NrZXIvZGVwbG95LXN0YWdpbmcuc2gKaT0wCndoaWxlIFsgIiRpIiAtbHQgMzAgXTsgZG8KICBpZiB3Z2V0IC1xTy0gaHR0cDovLzEyNy4wLjAuMTo4MDAwL2hlYWx0aDsgdGhlbgogICAgZXhpdCAwCiAgZmkKICBpPSQoKGkgKyAxKSkKICBzbGVlcCAyCmRvbmUKZXhpdCAxCg==' | base64 -d | ssh -i "$key_path" "$staging_user@$staging_host" "GITHUB_SHA='${GITHUB_SHA}' sh" - name: Post-deploy smoke test shell: sh @@ -261,35 +261,48 @@ jobs: needs: deploy-staging steps: - - name: Checkout code + - name: Install SSH client + shell: sh + run: | + set -eu + apt-get update -qq && apt-get install -y -qq openssh-client >/dev/null 2>&1 + echo "openssh-client installed" + + - name: Run Playwright E2E on staging server shell: sh env: - GITHUB_TOKEN: ${{ github.token }} + STAGING_SSH_HOST: ${{ secrets.STAGING_SSH_HOST }} + STAGING_SSH_USER: ${{ secrets.STAGING_SSH_USER }} + STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }} run: | set -eu - python3 - <<'PY' - import io, os, tarfile, urllib.request - url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" - request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() - with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: - root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' - for member in tar.getmembers(): - name = member.name - if name == root_prefix[:-1]: - continue - if name.startswith(root_prefix): - member.name = name[len(root_prefix):] - if member.name: - tar.extract(member, '.') - PY + staging_host="${STAGING_SSH_HOST:-47.98.113.167}" + staging_user="${STAGING_SSH_USER:-root}" + if [ -f /root/.ssh/xiaoxia_runtime_builder ]; then + key_path="/root/.ssh/xiaoxia_runtime_builder" + elif [ -n "${STAGING_SSH_KEY:-}" ]; then + key_path="$HOME/.ssh/id_ed25519" + printf '%s\n' "$STAGING_SSH_KEY" > "$key_path" + chmod 600 "$key_path" + else + echo "ERROR: No SSH key available" + exit 1 + fi + ssh-keyscan -H "$staging_host" >> ~/.ssh/known_hosts - - name: Run Playwright E2E against staging - shell: sh - run: | - set -eu - docker run --rm -e E2E_BASE_URL=http://127.0.0.1:3001 -e E2E_API_BASE=http://127.0.0.1:8000/api/v1 -e E2E_BROWSER_CHANNEL=chromium -v "$PWD:/workspace" -w /workspace/apps/web --network host mcr.microsoft.com/playwright:v1.45.0-jammy sh -lc 'npm ci && npx playwright test --reporter=line --project=chromium' + # 在业务服务器上跑 Playwright E2E(用 host 网络访问 staging 3001/8000 端口) + ssh -i "$key_path" "$staging_user@$staging_host" ' + cd /var/lib/xiaoxia-saas-staging/repo + docker run --rm \ + -e E2E_BASE_URL=http://127.0.0.1:3001 \ + -e E2E_API_BASE=http://127.0.0.1:8000/api/v1 \ + -e E2E_BROWSER_CHANNEL=chromium \ + -v "$PWD:/workspace" \ + -w /workspace/apps/web \ + --network host \ + mcr.microsoft.com/playwright:v1.45.0-jammy \ + sh -lc "npm ci && npx playwright test --reporter=line --project=chromium" + ' build-production-runtime-images: name: Build Production Runtime Images @@ -323,10 +336,12 @@ jobs: - name: Build runtime image artifact shell: sh + env: + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} run: | set -eu chmod +x scripts/build_release_images.sh - scripts/build_release_images.sh "${GITHUB_REF_NAME}" + REGISTRY_TOKEN="${REGISTRY_TOKEN}" scripts/build_release_images.sh "${GITHUB_REF_NAME}" - name: Build production web artifact shell: sh diff --git a/infra/docker/deploy-staging.sh b/infra/docker/deploy-staging.sh index 9b9323d94..39039cc45 100755 --- a/infra/docker/deploy-staging.sh +++ b/infra/docker/deploy-staging.sh @@ -41,6 +41,13 @@ export COMPOSE_DOCKER_CLI_BUILD=0 # Ensure isolated staging network exists docker network create xiaoxia-net-staging 2>/dev/null || true +# 登录内网 Registry(拉取 API/Worker 镜像需要认证) +REGISTRY_USER="${REGISTRY_USER:-admin}" +REGISTRY_PASS="${REGISTRY_PASS:-}" +if [ -n "$REGISTRY_PASS" ]; then + printf '%s' "$REGISTRY_PASS" | docker login "$REGISTRY" -u "$REGISTRY_USER" --password-stdin 2>/dev/null || true +fi + # Set default image names with registry prefix if not provided export API_IMAGE="${API_IMAGE:-${REGISTRY}/xiaoxia-saas-api:dev}" export WORKER_IMAGE="${WORKER_IMAGE:-${REGISTRY}/xiaoxia-saas-worker:dev}" diff --git a/infra/docker/worker.Dockerfile b/infra/docker/worker.Dockerfile index 6eaa9e63b..4283c5d7c 100755 --- a/infra/docker/worker.Dockerfile +++ b/infra/docker/worker.Dockerfile @@ -24,13 +24,14 @@ WORKDIR /app # ---- 依赖分层:基础依赖(变化少,缓存命中率高)---- COPY requirements-base.txt /tmp/requirements-base.txt -RUN pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements-base.txt \ +RUN python -m venv /opt/venv \ + && /opt/venv/bin/pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements-base.txt \ && rm /tmp/requirements-base.txt # ---- 依赖分层:业务依赖(变化频繁)---- COPY requirements.txt /tmp/requirements.txt -RUN pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements.txt \ +RUN /opt/venv/bin/pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements.txt \ && rm /tmp/requirements.txt # 复制应用代码 @@ -42,6 +43,7 @@ COPY alembic.ini /app/alembic.ini COPY migrations/ /app/migrations/ # 设置 Python 路径 +ENV PATH="/opt/venv/bin:$PATH" ENV PYTHONPATH=/app ENV PYTHONUNBUFFERED=1 diff --git a/scripts/build_release_images.sh b/scripts/build_release_images.sh index 83b379d28..d65e5b48d 100755 --- a/scripts/build_release_images.sh +++ b/scripts/build_release_images.sh @@ -28,8 +28,43 @@ if docker ps --format '{{.Names}}' | grep -Eq '^(xiaoxia-(api|web|worker|postgre fi fi -docker build --pull=false -f infra/docker/api.Dockerfile -t "$API_IMAGE" -t "$API_LATEST" . -docker build --pull=false -f infra/docker/worker.Dockerfile -t "$WORKER_IMAGE" -t "$WORKER_LATEST" . +# ---- BuildKit 缓存优化 ---- +CACHE_REGISTRY="${CACHE_REGISTRY:-git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas}" +CACHE_TAG="${CACHE_TAG:-release}" +USE_CACHE=0 + +# 检查 buildx 和 Gitea Registry 认证是否可用 +if docker buildx version >/dev/null 2>&1; then + # 尝试登录缓存 Registry(有 token 才启用缓存) + if [ -n "${REGISTRY_TOKEN:-}" ]; then + printf '%s' "${REGISTRY_TOKEN}" | docker login git.xiaoxiajianji.com -u xiaoxia --password-stdin 2>/dev/null && USE_CACHE=1 + fi + # 确保使用 docker driver(共享 daemon 凭证) + docker buildx use default 2>/dev/null || true +fi + +if [ "$USE_CACHE" -eq 1 ]; then + echo "Using buildx with distributed cache (${CACHE_REGISTRY})" + docker buildx build \ + --cache-from "type=registry,ref=${CACHE_REGISTRY}/api-cache:${CACHE_TAG},ignore-error=true" \ + --cache-to "type=registry,ref=${CACHE_REGISTRY}/api-cache:${CACHE_TAG},mode=max" \ + -f infra/docker/api.Dockerfile \ + -t "$API_IMAGE" -t "$API_LATEST" \ + --load \ + . + docker buildx build \ + --cache-from "type=registry,ref=${CACHE_REGISTRY}/worker-cache:${CACHE_TAG},ignore-error=true" \ + --cache-to "type=registry,ref=${CACHE_REGISTRY}/worker-cache:${CACHE_TAG},mode=max" \ + -f infra/docker/worker.Dockerfile \ + -t "$WORKER_IMAGE" -t "$WORKER_LATEST" \ + --load \ + . +else + echo "Buildx cache not available, using plain docker build" + docker build --pull=false -f infra/docker/api.Dockerfile -t "$API_IMAGE" -t "$API_LATEST" . + docker build --pull=false -f infra/docker/worker.Dockerfile -t "$WORKER_IMAGE" -t "$WORKER_LATEST" . +fi + docker save "$API_IMAGE" "$API_LATEST" "$WORKER_IMAGE" "$WORKER_LATEST" -o "$TAR_PATH" printf '%s\n' "$TAR_PATH"