diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index eb05c57c7..da7b66197 100755 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -27,7 +27,7 @@ jobs: POSTGRES_PASSWORD: postgres POSTGRES_DB: xiaoxia_saas ports: - - 5432:5432 + - 5433:5432 options: >- --health-cmd "pg_isready -U postgres" --health-interval 10s @@ -35,7 +35,7 @@ jobs: --health-retries 5 env: - DATABASE_URL: postgresql+psycopg://postgres:postgres@127.0.0.1:5432/xiaoxia_saas + DATABASE_URL: postgresql+psycopg://postgres:postgres@127.0.0.1:5433/xiaoxia_saas USE_IN_MEMORY_DB: "false" steps: @@ -46,11 +46,34 @@ jobs: run: | set -eu python3 - <<'PY' - import io, os, tarfile, urllib.request + import io, os, tarfile, time, urllib.request, urllib.error url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() + # Retry up to 5 times with backoff for transient 5xx errors + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' for member in tar.getmembers(): @@ -149,7 +172,20 @@ jobs: run: | set -eu archive_url="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/archive/${GITHUB_SHA}.tar.gz" - wget --header="Authorization: token ${GITHUB_TOKEN}" -O /tmp/repo.tar.gz "$archive_url" + # Retry up to 5 times with backoff for transient 5xx errors + for i in 1 2 3 4 5; do + if wget --header="Authorization: token ${GITHUB_TOKEN}" -O /tmp/repo.tar.gz "$archive_url" 2>&1; then + break + fi + if [ "$i" -lt 5 ]; then + wait=$((2 ** i)) + echo "Checkout failed (attempt $i/5), retrying in ${wait}s..." + sleep "$wait" + else + echo "Checkout failed after 5 attempts" + exit 1 + fi + done tar -xzf /tmp/repo.tar.gz --strip-components=1 -C . rm -f /tmp/repo.tar.gz diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 0dc76baa4..593aea546 100755 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -23,11 +23,34 @@ jobs: run: | set -eu python3 - <<'PY' - import io, os, tarfile, urllib.request + import io, os, tarfile, time, urllib.request, urllib.error url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() + # Retry up to 5 times with backoff for transient 5xx errors + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' for member in tar.getmembers(): @@ -317,11 +340,34 @@ jobs: run: | set -eu python3 - <<'PY' - import io, os, tarfile, urllib.request + import io, os, tarfile, time, urllib.request, urllib.error url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() + # Retry up to 5 times with backoff for transient 5xx errors + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' for member in tar.getmembers(): @@ -471,11 +517,34 @@ jobs: run: | set -eu python3 - <<'PY' - import io, os, tarfile, urllib.request + import io, os, tarfile, time, urllib.request, urllib.error url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() + # Retry up to 5 times with backoff for transient 5xx errors + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' for member in tar.getmembers(): diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index 33bf05593..f8ffeb555 100755 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -17,12 +17,37 @@ jobs: import io import os import tarfile + import time + import urllib.error import urllib.request url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() + # Retry up to 5 times with backoff for transient 5xx errors + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/' @@ -74,12 +99,37 @@ jobs: import io import os import tarfile + import time + import urllib.error import urllib.request url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz" request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"}) - with urllib.request.urlopen(request, timeout=120) as response: - archive = response.read() + # Retry up to 5 times with backoff for transient 5xx errors + last_err = None + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + archive = response.read() + break + except urllib.error.HTTPError as e: + last_err = e + if e.code >= 500 and attempt < 4: + wait = 2 ** attempt + print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + except Exception as e: + last_err = e + if attempt < 4: + wait = 2 ** attempt + print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...") + time.sleep(wait) + continue + raise + else: + raise last_err with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'