#!/bin/sh set -eu REPORT_PATH="${REPORT_PATH:-/var/lib/xiaoxia-ci/duty_report.json}" API_HEALTH_URL="${API_HEALTH_URL:-http://127.0.0.1:8001/health}" WEB_HEALTH_URL="${WEB_HEALTH_URL:-http://127.0.0.1:3002/}" LOAD_WARN_PER_CPU="${LOAD_WARN_PER_CPU:-1.5}" MEM_AVAILABLE_WARN_MB="${MEM_AVAILABLE_WARN_MB:-256}" DISK_WARN_PERCENT="${DISK_WARN_PERCENT:-85}" SWAP_USED_WARN_PERCENT="${SWAP_USED_WARN_PERCENT:-60}" EXPECTED_VERSION="${EXPECTED_VERSION:-}" mkdir -p "$(dirname "$REPORT_PATH")" python3 - <<'PY' import json import os import shutil import subprocess import time import urllib.request from datetime import datetime, timezone report_path = os.environ.get("REPORT_PATH", "/var/lib/xiaoxia-ci/duty_report.json") api_health_url = os.environ.get("API_HEALTH_URL", "http://127.0.0.1:8001/health") web_health_url = os.environ.get("WEB_HEALTH_URL", "http://127.0.0.1:3002/") load_warn_per_cpu = float(os.environ.get("LOAD_WARN_PER_CPU", "1.5")) mem_available_warn_mb = int(os.environ.get("MEM_AVAILABLE_WARN_MB", "256")) disk_warn_percent = int(os.environ.get("DISK_WARN_PERCENT", "85")) swap_used_warn_percent = int(os.environ.get("SWAP_USED_WARN_PERCENT", "60")) expected_version = os.environ.get("EXPECTED_VERSION", "").strip() alerts = [] info = [] actions = [] metrics = {} def run(command): return subprocess.run(command, text=True, capture_output=True, check=False) def add_alert(message): alerts.append(message) def read_meminfo(): data = {} with open("/proc/meminfo", "r", encoding="utf-8") as file: for line in file: key, value = line.split(":", 1) data[key] = int(value.strip().split()[0]) return data cpu_count = os.cpu_count() or 1 load1, load5, load15 = os.getloadavg() metrics["load"] = {"1m": load1, "5m": load5, "15m": load15, "cpu_count": cpu_count} if load5 > cpu_count * load_warn_per_cpu: add_alert(f"主机 5 分钟负载偏高:{load5:.2f} / CPU {cpu_count}") else: info.append(f"负载正常:5m={load5:.2f}, cpu={cpu_count}") meminfo = read_meminfo() available_mb = meminfo.get("MemAvailable", 0) // 1024 swap_total_mb = meminfo.get("SwapTotal", 0) // 1024 swap_free_mb = meminfo.get("SwapFree", 0) // 1024 swap_used_percent = 0 if swap_total_mb == 0 else round((swap_total_mb - swap_free_mb) * 100 / swap_total_mb, 2) metrics["memory"] = { "available_mb": available_mb, "swap_total_mb": swap_total_mb, "swap_free_mb": swap_free_mb, "swap_used_percent": swap_used_percent, } if available_mb < mem_available_warn_mb: add_alert(f"可用内存偏低:{available_mb}MiB") else: info.append(f"可用内存:{available_mb}MiB") if swap_total_mb == 0: add_alert("未启用 swap,内存抖动时可能导致 SSH/TLS 卡死") elif swap_used_percent >= swap_used_warn_percent: add_alert(f"swap 使用率偏高:{swap_used_percent}%") else: info.append(f"swap 正常:{swap_used_percent}% used") usage = shutil.disk_usage("/") disk_used_percent = round(usage.used * 100 / usage.total, 2) metrics["disk"] = {"root_used_percent": disk_used_percent, "root_free_gb": round(usage.free / 1024 / 1024 / 1024, 2)} if disk_used_percent >= disk_warn_percent: add_alert(f"根分区磁盘使用率偏高:{disk_used_percent}%") else: info.append(f"根分区磁盘:{disk_used_percent}% used") def check_url(name, url, expected_version=None): try: with urllib.request.urlopen(url, timeout=8) as response: body = response.read(4096).decode("utf-8", "replace") code = response.status metrics[f"{name}_status_code"] = code if code >= 400: add_alert(f"{name} 健康检查异常:HTTP {code}") return info.append(f"{name} 健康检查 OK:HTTP {code}") if expected_version: try: payload = json.loads(body) version = str(payload.get("version", "")) metrics[f"{name}_version"] = version if version != expected_version: add_alert(f"{name} 版本不一致:当前 {version},期望 {expected_version}") else: info.append(f"{name} 版本 OK:{version}") except json.JSONDecodeError: add_alert(f"{name} 未返回 JSON,无法校验版本") except Exception as error: add_alert(f"{name} 健康检查失败:{error}") check_url("API", api_health_url, expected_version or None) check_url("Web", web_health_url) containers = [ "xiaoxia-api-production", "xiaoxia-worker-production", "xiaoxia-web-production", "xiaoxia-postgres-production", "xiaoxia-redis-production", ] container_results = {} for name in containers: result = run(["docker", "inspect", "-f", "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}no-health{{end}}", name]) if result.returncode != 0: add_alert(f"容器不存在或不可检查:{name}") container_results[name] = "missing" continue state = result.stdout.strip() container_results[name] = state if not state.startswith("running"): add_alert(f"容器未运行:{name} ({state})") elif "unhealthy" in state: add_alert(f"容器健康检查失败:{name} ({state})") else: info.append(f"容器正常:{name} ({state})") metrics["containers"] = container_results status = "healthy" if not alerts else "warning" report = { "check_time": datetime.now(timezone.utc).astimezone().isoformat(), "status": status, "alerts": alerts, "info": info, "actions": actions, "metrics": metrics, } tmp_path = f"{report_path}.tmp.{int(time.time())}" with open(tmp_path, "w", encoding="utf-8") as file: json.dump(report, file, ensure_ascii=False, indent=2) os.replace(tmp_path, report_path) print(json.dumps(report, ensure_ascii=False, indent=2)) PY