diff --git a/.gitea/workflows/ci-health-daily.yml b/.gitea/workflows/ci-health-daily.yml index d60326f2c..26a02ab8a 100644 --- a/.gitea/workflows/ci-health-daily.yml +++ b/.gitea/workflows/ci-health-daily.yml @@ -1,19 +1,15 @@ name: CI Health Daily Report - on: schedule: - cron: '0 1 * * *' # UTC 01:00 = 北京时间 09:00 workflow_dispatch: - permissions: contents: read - jobs: ci-health-report: name: CI健康度每日巡检 runs-on: saas - timeout-minutes: 10 - + timeout-minutes: 15 steps: - name: Checkout code shell: sh @@ -61,6 +57,34 @@ jobs: tar.extract(member, '.') PY + - name: Generate CI Dashboard HTML + shell: sh + env: + GITEA_TOKEN: ${{ github.token }} + run: | + set +e + echo "=== 生成 CI 健康度 HTML 看板 ===" + echo "时间: $(date '+%Y-%m-%d %H:%M:%S')" + echo "" + python3 scripts/ci/ci_dashboard.py --days 7 --html --html-output ci_dashboard.html + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ] && [ -f ci_dashboard.html ]; then + HTML_SIZE=$(wc -c < ci_dashboard.html) + echo "" + echo "✅ HTML 看板生成成功 (${HTML_SIZE} bytes)" + echo "路径: $(pwd)/ci_dashboard.html" + # 输出文件内容前几行，方便在 Actions 日志中确认 + echo "" + echo "--- 看板预览 (前 5 行) ---" + head -5 ci_dashboard.html + echo "...（完整内容见产物文件）" + else + echo "❌ HTML 看板生成失败 (exit code: $EXIT_CODE)" + fi + echo "" + # 永远成功，看板生成失败不影响主流程 + exit 0 + - name: Run CI health check and report shell: sh env: @@ -71,10 +95,8 @@ jobs: echo "=== CI健康度每日巡检 ===" echo "时间: $(date '+%Y-%m-%d %H:%M:%S')" echo "" - python3 scripts/ci/ci_health_report.py --limit 30 EXIT_CODE=$? - echo "" echo "巡检完成 (exit code: $EXIT_CODE)" # 永远成功，不影响CI状态（通知失败不应该标红） diff --git a/scripts/ci/ci_dashboard.py b/scripts/ci/ci_dashboard.py index a8bcaac5c..3c7c1f6f8 100644 --- a/scripts/ci/ci_dashboard.py +++ b/scripts/ci/ci_dashboard.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 """ -CI 可观测性看板 - 从 Gitea Actions API 拉取数据并生成 Markdown 日报 - +CI 可观测性看板 - 从 Gitea Actions API 拉取数据并生成 Markdown/HTML 日报 用法: python3 scripts/ci/ci_dashboard.py --days 7 python3 scripts/ci/ci_dashboard.py --days 30 --output ci_report.md python3 scripts/ci/ci_dashboard.py --workflow ci-cd.yml --days 7 - + python3 scripts/ci/ci_dashboard.py --days 7 --html --html-output dashboard.html 环境变量: GITEA_URL Gitea 地址 (默认 https://git.xiaoxiajianji.com) GITEA_REPO 仓库 (默认 xiaoxia/xiaoxia-saas) @@ -177,18 +176,14 @@ def fetch_runs_in_range(ga, start_date, end_date, workflow_filter=None): all_runs = [] page = 1 print(f"[INFO] 拉取 {start_date} ~ {end_date} 的 CI runs...", file=sys.stderr) - while True: runs, total = ga.list_runs(status="completed", page=page, limit=PAGE_LIMIT) if not runs: break - if workflow_filter: runs = [r for r in runs if workflow_filter in r.get("path", "")] - in_range = [] out_range_old = False - for run in runs: started = to_shanghai(parse_datetime(run.get("started_at"))) if not started: @@ -198,27 +193,22 @@ def fetch_runs_in_range(ga, start_date, end_date, workflow_filter=None): in_range.append(run) elif run_date < start_date: out_range_old = True - all_runs.extend(in_range) print( f"[INFO] 第 {page} 页: {len(runs)} 条, 范围内 {len(in_range)} 条, 累计 {len(all_runs)} 条", file=sys.stderr ) - if out_range_old or len(runs) < PAGE_LIMIT: break - page += 1 if page > 100: print("[WARN] 超过100页，停止拉取", file=sys.stderr) break - print(f"[INFO] 共获取 {len(all_runs)} 条 run 数据", file=sys.stderr) return all_runs def enrich_with_jobs(ga, runs, max_failures=50): """为 runs 补充 job 详情（失败原因分析 + runner 统计） - 失败 run 按时间倒序取最近 N 个（避免 API 调用过多）， 成功 run 采样用于 runner 分布统计。 """ @@ -226,13 +216,11 @@ def enrich_with_jobs(ga, runs, max_failures=50): failure_runs = [r for r in runs if r.get("conclusion") != "success"] failure_runs = failure_runs[:max_failures] # 已经是时间倒序 print(f"[INFO] 为最近 {len(failure_runs)} 个失败 run 拉取 job 详情...", file=sys.stderr) - for i, run in enumerate(failure_runs): jobs = ga.get_run_jobs(run["id"]) run["_jobs"] = jobs if (i + 1) % 10 == 0: print(f"[INFO] 已处理 {i+1}/{len(failure_runs)}", file=sys.stderr) - # 成功 run 采样用于 runner 分布 success_runs = [r for r in runs if r.get("conclusion") == "success"] sample_size = min(50, len(success_runs)) @@ -243,7 +231,6 @@ def enrich_with_jobs(ga, runs, max_failures=50): if "_jobs" not in run: jobs = ga.get_run_jobs(run["id"]) run["_jobs"] = jobs - return runs @@ -268,7 +255,6 @@ def analyze_runs(runs): if d and d > 0: durations.append(d) durations.sort() - avg_dur = statistics.mean(durations) if durations else None median_dur = percentile(durations, 50) p95_dur = percentile(durations, 95) @@ -316,8 +302,31 @@ def analyze_runs(runs): # 失败原因 + runner + job 耗时（需要 _jobs 数据） failure_categories = defaultdict(int) failed_jobs_by_name = defaultdict(int) + job_success_stats = defaultdict(lambda: {"total": 0, "success": 0, "failure": 0}) runner_stats = defaultdict(lambda: {"jobs": 0, "success": 0, "failure": 0, "durations": []}) job_time_stats = defaultdict(list) + infra_failures = 0 + business_failures = 0 + other_failures_count = 0 + + # 基础设施关键词（与 ci_health_check.py 保持一致的分类逻辑） + infra_job_keywords = ["checkout", "build", "deploy", "cleanup", "setup", "cache", "install", "docker"] + business_job_keywords = [ + "unit test", + "pytest", + "vitest", + "jest", + "lint", + "eslint", + "prettier", + "integration", + "e2e", + "validate", + "code quality", + "mypy", + "ruff", + "flake8", + ] for r in runs: jobs = r.get("_jobs", []) @@ -326,27 +335,45 @@ def analyze_runs(runs): for job in jobs: runner = job.get("runner_name", "unknown") conclusion = job.get("conclusion", "unknown") + job_name = job.get("name", "unknown") + job_name_lower = job_name.lower() + runner_stats[runner]["jobs"] += 1 + job_success_stats[job_name]["total"] += 1 if conclusion == "success": runner_stats[runner]["success"] += 1 + job_success_stats[job_name]["success"] += 1 elif conclusion == "failure": runner_stats[runner]["failure"] += 1 + job_success_stats[job_name]["failure"] += 1 jd = duration_seconds(job.get("started_at"), job.get("completed_at")) if jd and jd > 0: runner_stats[runner]["durations"].append(jd) - job_time_stats[job.get("name", "unknown")].append(jd) + job_time_stats[job_name].append(jd) if conclusion == "failure": - failed_jobs_by_name[job.get("name", "unknown")] += 1 + failed_jobs_by_name[job_name] += 1 failed_step = None for step in job.get("steps", []): if step.get("conclusion") == "failure": failed_step = step.get("name") break - category = classify_failure(job.get("name", ""), failed_step) + category = classify_failure(job_name, failed_step) failure_categories[category] += 1 + # 基础设施 vs 业务代码分类 + is_infra = any(k in job_name_lower for k in infra_job_keywords) and not any( + k in job_name_lower for k in business_job_keywords + ) + is_business = any(k in job_name_lower for k in business_job_keywords) + if is_infra: + infra_failures += 1 + elif is_business: + business_failures += 1 + else: + other_failures_count += 1 + return { "total": total, "success": success, @@ -365,14 +392,17 @@ def analyze_runs(runs): "failed_jobs_top": dict(sorted(failed_jobs_by_name.items(), key=lambda x: -x[1])[:15]), "runner_stats": dict(runner_stats), "job_time_stats": dict(job_time_stats), + "job_success_stats": dict(job_success_stats), + "infra_failures": infra_failures, + "business_failures": business_failures, + "other_failures_combined": other_failures_count, } -# ── 报表生成 ───────────────────────────────────────── +# ── Markdown 报表生成 ──────────────────────────────── def generate_markdown(stats, start_date, end_date, repo): """生成 Markdown 格式的日报""" lines = [] - lines.append("# CI 运行状态看板") lines.append("") lines.append(f"> 统计周期: **{start_date} ~ {end_date}**") @@ -526,6 +556,359 @@ def generate_markdown(stats, start_date, end_date, repo): return "\n".join(lines) +# ── HTML 看板生成 ──────────────────────────────────── +def generate_html(stats, start_date, end_date, repo): + """生成 HTML 格式的可视化看板（内嵌 ECharts）""" + # 准备图表数据 + + # 1. 每日成功率趋势 + daily_dates = list(stats["daily_stats"].keys()) + daily_success_rates = [] + daily_run_counts = [] + for day in daily_dates: + s = stats["daily_stats"][day] + rate = (s["success"] / s["total"] * 100) if s["total"] > 0 else 0 + daily_success_rates.append(round(rate, 1)) + daily_run_counts.append(s["total"]) + + # 2. 各 Workflow 耗时对比 + wf_sorted = sorted(stats["workflow_stats"].items(), key=lambda x: -x[1]["total"]) + wf_names = [] + wf_avg_durations = [] + for wf, s in wf_sorted: + wf_short = wf.split("/")[-1] if "/" in wf else wf + wf_names.append(wf_short) + avg = statistics.mean(s["durations"]) if s["durations"] else 0 + wf_avg_durations.append(round(avg / 60, 1)) # 转为分钟 + + # 3. 失败原因分布（饼图数据 - 基础设施 vs 业务 vs 其他） + total_infra_biz = stats["infra_failures"] + stats["business_failures"] + stats["other_failures_combined"] + infra_rate = (stats["infra_failures"] / total_infra_biz * 100) if total_infra_biz > 0 else 0 + failure_pie_data = [ + {"value": stats["infra_failures"], "name": "基础设施问题"}, + {"value": stats["business_failures"], "name": "业务代码问题"}, + {"value": stats["other_failures_combined"], "name": "其他"}, + ] + + # 4. 各 Job 成功率排行（横向柱状图，取成功率最低的 Top 15） + job_stats_list = [] + for name, s in stats["job_success_stats"].items(): + if s["total"] >= 3: # 至少有3次才统计 + rate = (s["success"] / s["total"] * 100) if s["total"] > 0 else 0 + job_stats_list.append( + { + "name": name, + "rate": round(rate, 1), + "total": s["total"], + "success": s["success"], + } + ) + job_stats_list.sort(key=lambda x: x["rate"]) + job_stats_list = job_stats_list[:15] # 取成功率最低的15个 + job_names = [j["name"] for j in job_stats_list] + job_rates = [j["rate"] for j in job_stats_list] + + # 核心指标 + total_runs = stats["total"] + success_rate = round(stats["success_rate"], 1) + avg_dur_min = round(stats["avg_duration"] / 60, 1) if stats["avg_duration"] else 0 + infra_fail_rate = round(infra_rate, 1) + + now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + # 序列化数据为 JSON（供 JS 使用） + data_json = json.dumps( + { + "daily_dates": daily_dates, + "daily_success_rates": daily_success_rates, + "daily_run_counts": daily_run_counts, + "wf_names": wf_names, + "wf_avg_durations": wf_avg_durations, + "failure_pie_data": failure_pie_data, + "job_names": job_names, + "job_rates": job_rates, + }, + ensure_ascii=False, + ) + + # HTML 模板（注意：不使用 f-string，避免与 CSS/JS 的大括号冲突） + html_parts = [] + html_parts.append("") + html_parts.append('') + html_parts.append("") + html_parts.append(' ') + html_parts.append(' ') + html_parts.append(f" CI 健康度看板 - {repo}") + html_parts.append(' ') + html_parts.append(" ") + html_parts.append("") + html_parts.append("") + html_parts.append('
') + html_parts.append('
') + html_parts.append("

📊 CI 健康度看板

") + html_parts.append(f'
仓库: {repo}
') + html_parts.append(f'
统计周期: {start_date} ~ {end_date} | 生成时间: {now_str}
') + html_parts.append("
") + html_parts.append('
') + html_parts.append('
') + html_parts.append('
总成功率
') + html_parts.append(f'
{success_rate}%
') + html_parts.append("
") + html_parts.append('
') + html_parts.append('
总 Run 数
') + html_parts.append(f'
{total_runs}次
') + html_parts.append("
") + html_parts.append('
') + html_parts.append('
平均耗时
') + html_parts.append(f'
{avg_dur_min}分钟
') + html_parts.append("
") + html_parts.append('
') + html_parts.append('
基础设施故障率
') + html_parts.append(f'
{infra_fail_rate}%
') + html_parts.append("
") + html_parts.append("
") + html_parts.append('
') + html_parts.append('
') + html_parts.append("

📈 CI 成功率趋势

") + html_parts.append('
') + html_parts.append("
") + html_parts.append("
") + html_parts.append('
') + html_parts.append('
') + html_parts.append("

⏱️ 各 Workflow 平均耗时

") + html_parts.append('
') + html_parts.append("
") + html_parts.append('
') + html_parts.append("

❌ 失败原因分布

") + html_parts.append('
') + html_parts.append("
") + html_parts.append("
") + html_parts.append('
') + html_parts.append('
') + html_parts.append("

📋 各 Job 成功率排行（最低 15 名）

") + html_parts.append('
') + html_parts.append("
") + html_parts.append("
") + html_parts.append('
') + html_parts.append('
') + html_parts.append("

📊 每日 Run 数量趋势

") + html_parts.append('
') + html_parts.append("
") + html_parts.append("
") + html_parts.append('
') + html_parts.append(" 由 ci_dashboard.py 自动生成 | ECharts 可视化") + html_parts.append("
") + html_parts.append("
") + html_parts.append(" ") + html_parts.append("") + html_parts.append("") + + return "\n".join(html_parts) + + # ── 主函数 ─────────────────────────────────────────── def main(): parser = argparse.ArgumentParser(description="CI 可观测性看板 - 生成 Gitea Actions 运行状态报表") @@ -539,6 +922,11 @@ def main(): parser.add_argument("--password", type=str, default=os.environ.get("GITEA_PASSWORD")) parser.add_argument("--no-job-detail", action="store_true", help="不拉取 job 详情") parser.add_argument("--max-failures", type=int, default=50, help="最多分析多少个失败 run 的 job 详情 (默认 50)") + + # HTML 输出相关参数 + parser.add_argument("--html", action="store_true", help="生成 HTML 可视化看板") + parser.add_argument("--html-output", type=str, help="HTML 输出文件路径 (默认 ci_dashboard.html)") + args = parser.parse_args() ga = GiteaActions( @@ -561,8 +949,18 @@ def main(): runs = enrich_with_jobs(ga, runs, max_failures=args.max_failures) stats = analyze_runs(runs) - md = generate_markdown(stats, start_date, end_date, args.repo) + # HTML 模式 + if args.html: + html = generate_html(stats, start_date, end_date, args.repo) + html_output = args.html_output or args.output or "ci_dashboard.html" + with open(html_output, "w", encoding="utf-8") as f: + f.write(html) + print(f"[INFO] HTML 看板已保存到 {html_output}", file=sys.stderr) + return + + # 默认 Markdown 模式（向后兼容） + md = generate_markdown(stats, start_date, end_date, args.repo) if args.output: with open(args.output, "w", encoding="utf-8") as f: f.write(md) diff --git a/scripts/ci/generate_ci_dashboard.sh b/scripts/ci/generate_ci_dashboard.sh new file mode 100644 index 000000000..fcd7abf8f --- /dev/null +++ b/scripts/ci/generate_ci_dashboard.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# +# CI 健康度看板一键生成脚本 +# - 从 Gitea Actions API 拉取数据 +# - 生成 HTML 可视化看板 +# - 输出文件路径 +# +# 用法: +# bash scripts/ci/generate_ci_dashboard.sh [--days 7] [--output ci_dashboard.html] +# +# 环境变量: +# GITEA_TOKEN API Token（必需） +# GITEA_URL Gitea 地址（可选，默认 https://git.xiaoxiajianji.com） +# GITEA_REPO 仓库（可选，默认 xiaoxia/xiaoxia-saas） +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +# 默认参数 +DAYS=7 +OUTPUT="ci_dashboard.html" + +# 解析参数 +while [[ $# -gt 0 ]]; do + case "$1" in + --days) + DAYS="$2" + shift 2 + ;; + --output|-o) + OUTPUT="$2" + shift 2 + ;; + --help|-h) + echo "用法: bash scripts/ci/generate_ci_dashboard.sh [--days 7] [--output ci_dashboard.html]" + echo "" + echo "选项:" + echo " --days N 统计最近 N 天 (默认 7)" + echo " --output PATH HTML 输出路径 (默认 ci_dashboard.html)" + echo " --help 显示帮助" + echo "" + echo "环境变量:" + echo " GITEA_TOKEN API Token（必需）" + echo " GITEA_URL Gitea 地址" + echo " GITEA_REPO 仓库" + exit 0 + ;; + *) + echo "未知参数: $1" + exit 1 + ;; + esac +done + +# 检查 Python +if ! command -v python3 &> /dev/null; then + echo "[ERROR] 未找到 python3，请先安装 Python 3" + exit 1 +fi + +# 检查 Token +if [[ -z "${GITEA_TOKEN:-}" ]]; then + echo "[ERROR] 请设置 GITEA_TOKEN 环境变量" + exit 1 +fi + +echo "========================================" +echo " CI 健康度看板生成器" +echo "========================================" +echo "" +echo "统计天数: ${DAYS} 天" +echo "输出文件: ${OUTPUT}" +echo "" + +# 生成 HTML 看板 +echo "[INFO] 正在拉取数据并生成看板..." +python3 "${SCRIPT_DIR}/ci_dashboard.py" \ + --days "${DAYS}" \ + --html \ + --html-output "${OUTPUT}" + +echo "" +echo "========================================" +echo " ✅ 看板生成完成！" +echo "========================================" +echo "" +echo "文件路径: $(realpath "${OUTPUT}")" +echo "" + +# 如果在 macOS 上，尝试打开 +if [[ "$(uname)" == "Darwin" ]]; then + echo "[INFO] 正在打开浏览器..." + open "${OUTPUT}" +fi