From 4eff7ea434bcd4c1b9aeb4be07573b1ff2316ae1 Mon Sep 17 00:00:00 2001 From: Xiaoxia AI Date: Mon, 22 Jun 2026 19:28:43 +0800 Subject: [PATCH] docs(release): record v0.1.9 automation retrospective --- docs/PRODUCTION-RELEASE-CHECKLIST.md | 36 +++++--- ...ASE-AUTOMATION-RETROSPECTIVE-2026-06-22.md | 92 +++++++++++++++++++ tests/unit/test_release_scripts.py | 30 ++++++ 3 files changed, 143 insertions(+), 15 deletions(-) create mode 100644 docs/RELEASE-AUTOMATION-RETROSPECTIVE-2026-06-22.md diff --git a/docs/PRODUCTION-RELEASE-CHECKLIST.md b/docs/PRODUCTION-RELEASE-CHECKLIST.md index 6920a1f91..6a492ce36 100644 --- a/docs/PRODUCTION-RELEASE-CHECKLIST.md +++ b/docs/PRODUCTION-RELEASE-CHECKLIST.md @@ -143,28 +143,34 @@ OSS 生命周期规则必须在云控制台或 IaC 中绑定到生产 bucket， - 只能通过 `v*` tag 触发 production deploy。 - 生产 API/Worker 必须使用预构建 runtime image tar，不允许生产机临时 build。 -- 在专用构建机执行： +- 生产 Web 必须使用预构建 `apps/web/dist`，不允许生产机运行 `npm`/Node build。 +- 当前标准链路： + 1. 推送 `v*` tag。 + 2. Gitea `build-production-runtime-images` job 在 `runtime-builder` 上运行。 + 3. `runtime-builder` 执行 `scripts/build_release_images.sh ` 生成 `runtime-images-.tar`。 + 4. `runtime-builder` 用 Node Docker 容器构建 `apps/web/dist`。 + 5. `runtime-builder` 打包 `release-.tar.gz`，必须保留 `apps/web/dist/index.html`，同时排除根目录 `./dist`，避免把 runtime tar 再包进 release tar。 + 6. `runtime-builder` 上传两个产物到生产： + - `/var/lib/xiaoxia-saas-production/runtime-images-.tar` + - `/var/lib/xiaoxia-saas-production/release-.tar.gz` + 7. Gitea `deploy-production` job 也在 `runtime-builder` 上运行，通过 SSH 驱动生产机部署。 + 8. SSH 部署必须传 `HOST_PREFIX=`，因为 `infra/docker/deploy-production.sh` 默认 `/host` 是给 Docker-in-Docker CI 场景用的。 + 9. 生产机只解包 artifact、复制 `.env`、`docker load` runtime images、运行 Alembic、重启容器和 health check。 -```bash -scripts/build_release_images.sh v0.1.x -``` - -- 将输出的 `dist/release-images/xiaoxia-runtime-images-v0.1.x.tar` 上传到： - -```bash -/var/lib/xiaoxia-saas-production/runtime-images-v0.1.x.tar -``` - -- Gitea production deploy 会把 `RELEASE_VERSION=${GITHUB_REF_NAME}` 传给 `infra/docker/deploy-production.sh`；如果缺少对应 runtime image tar，部署必须失败，禁止静默复用旧 API/Worker 镜像。 - 发布期间持续观察 Gitea Actions deploy log。 - 部署后确认： ```bash -curl -fsS http://127.0.0.1:8000/health -docker ps --format '{{.Names}} {{.Status}}' | grep xiaoxia -docker exec xiaoxia-postgres psql -U xiaoxia -d xiaoxia_saas -Atc 'select version_num from alembic_version;' +curl -fsS http://127.0.0.1:8001/health +docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep xiaoxia +docker exec xiaoxia-postgres-production psql -U xiaoxia -d xiaoxia_saas -Atc 'select version_num from alembic_version;' +ls -lh /var/lib/xiaoxia-saas-production/runtime-images-.tar /var/lib/xiaoxia-saas-production/release-.tar.gz ``` +- 缺少 runtime image tar、release tar 或 `apps/web/dist/index.html` 时，生产部署必须失败，禁止静默复用旧镜像或在生产机补 build。 + +- `v0.1.6`、`v0.1.7`、`v0.1.8` 是 release-chain 修复过程中的失败/中间态 tag，不作为回滚目标；当前可用回滚基准是已验证的 `v0.1.5` 和 `v0.1.9`。 + ## 8. Smoke 测试 生产发布后至少验证： diff --git a/docs/RELEASE-AUTOMATION-RETROSPECTIVE-2026-06-22.md b/docs/RELEASE-AUTOMATION-RETROSPECTIVE-2026-06-22.md new file mode 100644 index 000000000..fc130a727 --- /dev/null +++ b/docs/RELEASE-AUTOMATION-RETROSPECTIVE-2026-06-22.md @@ -0,0 +1,92 @@ +# v0.1.9 Release Automation Retrospective + +Date: 2026-06-22 +Branch: `develop` +Production release: `v0.1.9` + +## Result + +`v0.1.9` is the first verified end-to-end automatic production release for 小虾 SaaS: + +1. Gitea tag push starts production release workflow. +2. Dedicated `runtime-builder-01` builds API/Worker runtime images. +3. Dedicated builder builds Web `apps/web/dist` in a Node Docker container. +4. Dedicated builder uploads both artifacts to production. +5. Dedicated builder drives production deployment over SSH. +6. Production host only extracts artifacts, loads runtime images, runs migrations, restarts containers, and health checks. +7. Public Web/API smoke tests pass. + +## Verified Production State + +After `v0.1.9`: + +- `xiaoxia-api-production` runs `xiaoxia-saas-api:v0.1.9` and is healthy. +- `xiaoxia-worker-production` runs `xiaoxia-saas-worker:v0.1.9`. +- `xiaoxia-web-production` is healthy and serves prebuilt Web dist. +- `/var/lib/xiaoxia-saas-production/runtime-images-v0.1.9.tar` is present, about 325 MB. +- `/var/lib/xiaoxia-saas-production/release-v0.1.9.tar.gz` is present, about 883 KB. + +Public smoke results: + +```text +python scripts\smoke_public_auth_flow.py +public_auth_flow=ok + +python scripts\smoke_public_upload_flow.py +public_upload_flow=ok + +public project detail smoke +public_project_get=ok +``` + +## What Repeated + +The repeated tags were not feature releases; they were release-chain verification tags. + +- `v0.1.6`: runtime-builder checkout/token fix worked, but Gitea skipped downstream production deploy after cross-runner `needs`. +- `v0.1.7`: production deploy moved to `runtime-builder` over SSH; build/upload succeeded, but deploy failed because `deploy-production.sh` defaulted to `/host` paths while SSH uses host paths directly. +- `v0.1.8`: SSH deploy used `HOST_PREFIX=`, but release artifact packaging used broad `--exclude=dist`, which removed `apps/web/dist` and correctly failed the Web artifact gate. +- `v0.1.9`: artifact packaging changed to `--exclude=./dist`, preserving `apps/web/dist`; automatic release succeeded. + +Usable rollback tags: + +- `v0.1.5`: previously verified stable production release. +- `v0.1.9`: current verified automatic production release. + +Do not use `v0.1.6`, `v0.1.7`, or `v0.1.8` as rollback targets unless intentionally reproducing release-chain failures. + +## Root Causes Fixed + +1. `GITHUB_TOKEN` was not implicitly available in act_runner shell scripts. + - Fix: checkout steps explicitly bind `GITHUB_TOKEN: ${{ github.token }}`. + +2. Gitea `needs` with production deploy on a different runner path skipped the downstream deploy job. + - Fix: production deploy also runs on `runtime-builder`, then SSHes to production. + +3. Production deploy script defaults to `/host` for Docker-in-Docker CI execution. + - Fix: SSH deploy passes `HOST_PREFIX=` so the script uses normal host paths. + +4. Release source artifact accidentally included build output tarballs. + - Fix: exclude root `./dist` from release tar. + +5. A broad `--exclude=dist` also excluded Web dist. + - Fix: use `--exclude=./dist`, not `--exclude=dist`. + +## Current Release Contract + +For production `v*` tag releases: + +- Production host must not build API or Worker images. +- Production host must not build frontend assets. +- `runtime-builder-01` is the build and deployment driver. +- `runtime-images-.tar` must exist before production deploy. +- `release-.tar.gz` must include `apps/web/dist/index.html`. +- Production deploy must fail if either artifact is missing. +- Production smoke is required before saying release is done. + +## Follow-Ups + +- Add an explicit dry-run workflow or staging release rehearsal to reduce tag-based trial and error. +- Keep `v0.1.6` to `v0.1.8` documented as failed release-chain probes. +- Consider adding an API version endpoint that reports the Docker image tag; current `/health` version text can lag image tags. +- Review temporary credential posture for builder SSH key and old registration tokens. diff --git a/tests/unit/test_release_scripts.py b/tests/unit/test_release_scripts.py index b1acc42ec..aa3be7cf7 100644 --- a/tests/unit/test_release_scripts.py +++ b/tests/unit/test_release_scripts.py @@ -234,6 +234,36 @@ def test_runtime_image_release_scripts_keep_builds_off_production(): assert "docker compose --env-file \"$ENV_FILE\" build --pull=false worker" not in deploy_script +def test_production_release_checklist_matches_automatic_release_contract(): + checklist = Path("docs/PRODUCTION-RELEASE-CHECKLIST.md").read_text(encoding="utf-8") + + assert "runtime-builder" in checklist + assert "release-.tar.gz" in checklist + assert "apps/web/dist/index.html" in checklist + assert "HOST_PREFIX=" in checklist + assert "curl -fsS http://127.0.0.1:8001/health" in checklist + assert "xiaoxia-postgres-production" in checklist + assert "v0.1.6" in checklist + assert "v0.1.9" in checklist + assert "生产机补 build" in checklist + + +def test_release_automation_retrospective_records_failed_probe_tags(): + retrospective = Path("docs/RELEASE-AUTOMATION-RETROSPECTIVE-2026-06-22.md").read_text( + encoding="utf-8" + ) + + assert "v0.1.9" in retrospective + assert "first verified end-to-end automatic production release" in retrospective + assert "v0.1.6" in retrospective + assert "v0.1.7" in retrospective + assert "v0.1.8" in retrospective + assert "Do not use `v0.1.6`, `v0.1.7`, or `v0.1.8` as rollback targets" in retrospective + assert "HOST_PREFIX=" in retrospective + assert "--exclude=./dist" in retrospective + assert "public_upload_flow=ok" in retrospective + + def test_backup_postgres_writes_manifest_and_version(): script = Path("scripts/backup_postgres.sh").read_text(encoding="utf-8")