xiaoxia-saas/scripts/production_resource_check.sh

#!/bin/sh
set -eu

REPORT_PATH="${REPORT_PATH:-/var/lib/xiaoxia-ci/duty_report.json}"
API_HEALTH_URL="${API_HEALTH_URL:-http://127.0.0.1:8001/health}"
WEB_HEALTH_URL="${WEB_HEALTH_URL:-http://127.0.0.1:3002/}"
LOAD_WARN_PER_CPU="${LOAD_WARN_PER_CPU:-1.5}"
MEM_AVAILABLE_WARN_MB="${MEM_AVAILABLE_WARN_MB:-256}"
DISK_WARN_PERCENT="${DISK_WARN_PERCENT:-85}"
SWAP_USED_WARN_PERCENT="${SWAP_USED_WARN_PERCENT:-60}"
EXPECTED_VERSION="${EXPECTED_VERSION:-}"

mkdir -p "$(dirname "$REPORT_PATH")"

python3 - <<'PY'
import json
import os
import shutil
import subprocess
import time
import urllib.request
from datetime import datetime, timezone

report_path = os.environ.get("REPORT_PATH", "/var/lib/xiaoxia-ci/duty_report.json")
api_health_url = os.environ.get("API_HEALTH_URL", "http://127.0.0.1:8001/health")
web_health_url = os.environ.get("WEB_HEALTH_URL", "http://127.0.0.1:3002/")
load_warn_per_cpu = float(os.environ.get("LOAD_WARN_PER_CPU", "1.5"))
mem_available_warn_mb = int(os.environ.get("MEM_AVAILABLE_WARN_MB", "256"))
disk_warn_percent = int(os.environ.get("DISK_WARN_PERCENT", "85"))
swap_used_warn_percent = int(os.environ.get("SWAP_USED_WARN_PERCENT", "60"))
expected_version = os.environ.get("EXPECTED_VERSION", "").strip()

alerts = []
info = []
actions = []
metrics = {}


def run(command):
    return subprocess.run(command, text=True, capture_output=True, check=False)


def add_alert(message):
    alerts.append(message)


def read_meminfo():
    data = {}
    with open("/proc/meminfo", "r", encoding="utf-8") as file:
        for line in file:
            key, value = line.split(":", 1)
            data[key] = int(value.strip().split()[0])
    return data


cpu_count = os.cpu_count() or 1
load1, load5, load15 = os.getloadavg()
metrics["load"] = {"1m": load1, "5m": load5, "15m": load15, "cpu_count": cpu_count}
if load5 > cpu_count * load_warn_per_cpu:
    add_alert(f"主机 5 分钟负载偏高：{load5:.2f} / CPU {cpu_count}")
else:
    info.append(f"负载正常：5m={load5:.2f}, cpu={cpu_count}")

meminfo = read_meminfo()
available_mb = meminfo.get("MemAvailable", 0) // 1024
swap_total_mb = meminfo.get("SwapTotal", 0) // 1024
swap_free_mb = meminfo.get("SwapFree", 0) // 1024
swap_used_percent = 0 if swap_total_mb == 0 else round((swap_total_mb - swap_free_mb) * 100 / swap_total_mb, 2)
metrics["memory"] = {
    "available_mb": available_mb,
    "swap_total_mb": swap_total_mb,
    "swap_free_mb": swap_free_mb,
    "swap_used_percent": swap_used_percent,
}
if available_mb < mem_available_warn_mb:
    add_alert(f"可用内存偏低：{available_mb}MiB")
else:
    info.append(f"可用内存：{available_mb}MiB")
if swap_total_mb == 0:
    add_alert("未启用 swap，内存抖动时可能导致 SSH/TLS 卡死")
elif swap_used_percent >= swap_used_warn_percent:
    add_alert(f"swap 使用率偏高：{swap_used_percent}%")
else:
    info.append(f"swap 正常：{swap_used_percent}% used")

usage = shutil.disk_usage("/")
disk_used_percent = round(usage.used * 100 / usage.total, 2)
metrics["disk"] = {"root_used_percent": disk_used_percent, "root_free_gb": round(usage.free / 1024 / 1024 / 1024, 2)}
if disk_used_percent >= disk_warn_percent:
    add_alert(f"根分区磁盘使用率偏高：{disk_used_percent}%")
else:
    info.append(f"根分区磁盘：{disk_used_percent}% used")


def check_url(name, url, expected_version=None):
    try:
        with urllib.request.urlopen(url, timeout=8) as response:
            body = response.read(4096).decode("utf-8", "replace")
            code = response.status
        metrics[f"{name}_status_code"] = code
        if code >= 400:
            add_alert(f"{name} 健康检查异常：HTTP {code}")
            return
        info.append(f"{name} 健康检查 OK：HTTP {code}")
        if expected_version:
            try:
                payload = json.loads(body)
                version = str(payload.get("version", ""))
                metrics[f"{name}_version"] = version
                if version != expected_version:
                    add_alert(f"{name} 版本不一致：当前 {version}，期望 {expected_version}")
                else:
                    info.append(f"{name} 版本 OK：{version}")
            except json.JSONDecodeError:
                add_alert(f"{name} 未返回 JSON，无法校验版本")
    except Exception as error:
        add_alert(f"{name} 健康检查失败：{error}")


check_url("API", api_health_url, expected_version or None)
check_url("Web", web_health_url)

containers = [
    "xiaoxia-api-production",
    "xiaoxia-worker-production",
    "xiaoxia-web-production",
    "xiaoxia-postgres-production",
    "xiaoxia-redis-production",
]
container_results = {}
for name in containers:
    result = run(["docker", "inspect", "-f", "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}no-health{{end}}", name])
    if result.returncode != 0:
        add_alert(f"容器不存在或不可检查：{name}")
        container_results[name] = "missing"
        continue
    state = result.stdout.strip()
    container_results[name] = state
    if not state.startswith("running"):
        add_alert(f"容器未运行：{name} ({state})")
    elif "unhealthy" in state:
        add_alert(f"容器健康检查失败：{name} ({state})")
    else:
        info.append(f"容器正常：{name} ({state})")
metrics["containers"] = container_results

status = "healthy" if not alerts else "warning"
report = {
    "check_time": datetime.now(timezone.utc).astimezone().isoformat(),
    "status": status,
    "alerts": alerts,
    "info": info,
    "actions": actions,
    "metrics": metrics,
}

tmp_path = f"{report_path}.tmp.{int(time.time())}"
with open(tmp_path, "w", encoding="utf-8") as file:
    json.dump(report, file, ensure_ascii=False, indent=2)
os.replace(tmp_path, report_path)
print(json.dumps(report, ensure_ascii=False, indent=2))
PY