From 6f1d7f250848bda36000f1c276f71c1e09dd9a72 Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Fri, 17 Jul 2026 09:29:38 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat(ci):=20P1-5=20CI=E8=A7=A6=E5=8F=91?= =?UTF-8?q?=E5=8F=AF=E9=9D=A0=E6=80=A7=E7=9B=91=E6=8E=A7=20-=20=E5=AE=9A?= =?UTF-8?q?=E6=97=B6=E6=A3=80=E6=B5=8BPR=E6=9C=AA=E8=A7=A6=E5=8F=91CI?= =?UTF-8?q?=E5=B9=B6=E5=91=8A=E8=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/ci-trigger-monitor.yml | 36 ++++ scripts/ci_trigger_monitor.py | 247 ++++++++++++++++++++++++ 2 files changed, 283 insertions(+) create mode 100644 .gitea/workflows/ci-trigger-monitor.yml create mode 100644 scripts/ci_trigger_monitor.py diff --git a/.gitea/workflows/ci-trigger-monitor.yml b/.gitea/workflows/ci-trigger-monitor.yml new file mode 100644 index 000000000..69234b47a --- /dev/null +++ b/.gitea/workflows/ci-trigger-monitor.yml @@ -0,0 +1,36 @@ +name: CI Trigger Monitor + +on: + schedule: + - cron: '*/5 * * * *' # 每5分钟检查一次 + workflow_dispatch: + inputs: + stale_threshold: + description: 'CI未触发告警阈值（分钟）' + required: false + default: '5' + +permissions: + contents: read + +jobs: + monitor: + name: Monitor CI Trigger Reliability + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Check CI trigger status for all open PRs + env: + GITEA_API_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }} + GITEA_URL: https://git.xiaoxiajianji.com + GITEA_REPO: xiaoxia/xiaoxia-saas + CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }} + STALE_THRESHOLD_MIN: ${{ inputs.stale_threshold || 5 }} + run: | + set +e + python3 scripts/ci_trigger_monitor.py + # 监控脚本永远不fail，避免告警风暴 + exit 0 diff --git a/scripts/ci_trigger_monitor.py b/scripts/ci_trigger_monitor.py new file mode 100644 index 000000000..0d1061aa0 --- /dev/null +++ b/scripts/ci_trigger_monitor.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +""" +CI触发可靠性监控 - 定时检查PR的CI触发状态 +- 监控open PR的最新commit是否在5分钟内触发了CI +- 异常时通过飞书webhook告警 + +环境变量: + GITEA_API_TOKEN - Gitea API Token (必填) + GITEA_REPO - 仓库路径，如 xiaoxia/xiaoxia-saas + GITEA_URL - Gitea地址，如 https://git.xiaoxiajianji.com + CI_NOTIFY_WEBHOOK - 飞书告警webhook (必填) + CHECK_INTERVAL_MIN - 检查间隔（分钟），默认5 + STALE_THRESHOLD_MIN - CI未触发告警阈值（分钟），默认5 +""" + +import json +import os +import sys +import time +import urllib.request +import urllib.error + + +def get_env(name, default=""): + return os.environ.get(name, default) + + +def api_get(path): + """调用Gitea API""" + token = get_env("GITEA_API_TOKEN") + base_url = get_env("GITEA_URL", "https://git.xiaoxiajianji.com") + repo = get_env("GITEA_REPO", "xiaoxia/xiaoxia-saas") + + url = f"{base_url}/api/v1/repos/{repo}{path}" + req = urllib.request.Request(url) + req.add_header("Authorization", f"token {token}") + + for attempt in range(3): + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read()) + except urllib.error.HTTPError as e: + if e.code >= 500 and attempt < 2: + time.sleep(2 ** attempt) + continue + raise + except Exception as e: + if attempt < 2: + time.sleep(2 ** attempt) + continue + raise + + +def get_open_prs(): + """获取所有open PR""" + prs = [] + page = 1 + while True: + batch = api_get(f"/pulls?state=open&sort=updated&direction=desc&limit=50&page={page}") + if not batch: + break + prs.extend(batch) + if len(batch) < 50: + break + page += 1 + return prs + + +def get_commit_status(sha): + """获取commit的CI状态""" + try: + return api_get(f"/commits/{sha}/status") + except Exception as e: + print(f" ⚠️ 获取commit状态失败: {e}") + return {"state": "error", "statuses": []} + + +def has_ci_started(statuses): + """判断是否有CI job已经启动（pending/running/success/failure都算启动了）""" + pr_statuses = [s for s in statuses if 'pull_request' in s.get('context', '')] + if not pr_statuses: + return False + # 只要有非pending且非空的状态，就算启动了 + for s in pr_statuses: + if s.get('status') in ['success', 'failure', 'running']: + return True + if s.get('status') == 'pending' and 'Has started running' in s.get('description', ''): + return True + # 全是"Blocked by required conditions"的pending也算（说明CI系统收到了事件） + for s in pr_statuses: + if 'Blocked' in s.get('description', ''): + return True + return False + + +def send_alert(pr_num, pr_title, pr_url, head_sha, commit_age_min): + """发送飞书告警""" + webhook = get_env("CI_NOTIFY_WEBHOOK") + if not webhook: + print(" ⚠️ 未配置CI_NOTIFY_WEBHOOK，跳过告警") + return + + gitea_url = get_env("GITEA_URL", "https://git.xiaoxiajianji.com") + + content = { + "msg_type": "interactive", + "card": { + "header": { + "title": { + "tag": "plain_text", + "content": f"⚠️ CI告警 - PR#{pr_num} CI未触发" + }, + "template": "red" + }, + "elements": [ + { + "tag": "div", + "text": { + "tag": "lark_md", + "content": f"**PR**: [{pr_title}]({pr_url})\n**最新commit**: `{head_sha[:12]}`\n**已等待**: {commit_age_min:.0f} 分钟仍无CI启动\n**可能原因**: Gitea Actions事件丢失 / Webhook失败 / Runner资源不足" + } + }, + { + "tag": "action", + "actions": [ + { + "tag": "button", + "text": {"tag": "plain_text", "content": "查看PR"}, + "url": pr_url, + "type": "primary" + }, + { + "tag": "button", + "text": {"tag": "plain_text", "content": "查看Actions"}, + "url": f"{pr_url}/files", + "type": "default" + } + ] + }, + { + "tag": "note", + "elements": [ + {"tag": "plain_text", "content": f"CI触发监控 | 检测时间: {time.strftime('%Y-%m-%d %H:%M:%S')}"} + ] + } + ] + } + } + + try: + data = json.dumps(content).encode() + req = urllib.request.Request(webhook, data=data, method='POST') + req.add_header("Content-Type", "application/json") + with urllib.request.urlopen(req, timeout=10) as resp: + resp.read() + print(f" 📢 告警已发送: PR#{pr_num}") + except Exception as e: + print(f" ⚠️ 告警发送失败: {e}") + + +def main(): + stale_threshold = int(get_env("STALE_THRESHOLD_MIN", "5")) + + print("=" * 60) + print(f"CI触发监控 - 检测时间: {time.strftime('%Y-%m-%d %H:%M:%S')}") + print(f"告警阈值: {stale_threshold}分钟无CI启动") + print("=" * 60) + + # 获取open PR列表 + try: + prs = get_open_prs() + except Exception as e: + print(f"❌ 获取PR列表失败: {e}") + sys.exit(0) # 告警脚本不阻断CI + + print(f"\n共 {len(prs)} 个open PR\n") + + stale_prs = [] + now = time.time() + + for pr in prs: + pr_num = pr['number'] + pr_title = pr['title'] + pr_url = pr['html_url'] + head_sha = pr['head']['sha'] + updated_at = pr['updated_at'] + + # 解析updated_at（ISO格式） + try: + # 2026-07-17T09:22:43+08:00 + from datetime import datetime, timezone, timedelta + # 简化处理：直接用字符串解析 + ts_str = updated_at.replace('Z', '+00:00') + # 手动解析 + dt = datetime.fromisoformat(ts_str) + commit_time = dt.timestamp() + except Exception as e: + print(f" ⚠️ PR#{pr_num} 时间解析失败: {e}") + continue + + age_min = (now - commit_time) / 60 + + print(f"PR#{pr_num:3d} | {pr_title[:45]:45s} | 更新于 {age_min:.0f}min前") + + # 少于2分钟的跳过，给CI一点启动时间 + if age_min < 2: + print(f" ⏳ 刚更新，等待CI启动...") + continue + + # 获取commit状态 + status = get_commit_status(head_sha) + statuses = status.get('statuses', []) + + if has_ci_started(statuses): + print(f" ✅ CI已启动 (state={status.get('state')})") + continue + + # CI未启动，判断是否超过阈值 + if age_min >= stale_threshold: + print(f" 🚨 CI未触发！已等待 {age_min:.0f} 分钟") + stale_prs.append({ + 'num': pr_num, + 'title': pr_title, + 'url': pr_url, + 'sha': head_sha, + 'age_min': age_min + }) + else: + print(f" ⏳ CI尚未启动 ({age_min:.0f}min < {stale_threshold}min阈值)") + + # 发送告警 + print(f"\n{'=' * 60}") + print(f"检测结果: {len(stale_prs)} 个PR CI未触发超过阈值") + + if stale_prs: + print("\n告警列表:") + for pr in stale_prs: + print(f" - PR#{pr['num']}: {pr['title'][:40]} ({pr['age_min']:.0f}min)") + send_alert(pr['num'], pr['title'], pr['url'], pr['sha'], pr['age_min']) + else: + print("✅ 所有PR CI触发正常") + + print("=" * 60) + + +if __name__ == "__main__": + main() -- 2.54.0 From 29cb357fc87f40534d3daea75fb2f3274893ab42 Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Fri, 17 Jul 2026 09:44:38 +0800 Subject: [PATCH 2/3] =?UTF-8?q?style:=20black=E6=A0=BC=E5=BC=8F=E5=8C=96ci?= =?UTF-8?q?=5Ftrigger=5Fmonitor.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/ci_trigger_monitor.py | 106 ++++++++++++++++------------------ 1 file changed, 49 insertions(+), 57 deletions(-) diff --git a/scripts/ci_trigger_monitor.py b/scripts/ci_trigger_monitor.py index 0d1061aa0..83de9b3a2 100644 --- a/scripts/ci_trigger_monitor.py +++ b/scripts/ci_trigger_monitor.py @@ -30,23 +30,23 @@ def api_get(path): token = get_env("GITEA_API_TOKEN") base_url = get_env("GITEA_URL", "https://git.xiaoxiajianji.com") repo = get_env("GITEA_REPO", "xiaoxia/xiaoxia-saas") - + url = f"{base_url}/api/v1/repos/{repo}{path}" req = urllib.request.Request(url) req.add_header("Authorization", f"token {token}") - + for attempt in range(3): try: with urllib.request.urlopen(req, timeout=30) as resp: return json.loads(resp.read()) except urllib.error.HTTPError as e: if e.code >= 500 and attempt < 2: - time.sleep(2 ** attempt) + time.sleep(2**attempt) continue raise except Exception as e: if attempt < 2: - time.sleep(2 ** attempt) + time.sleep(2**attempt) continue raise @@ -77,18 +77,18 @@ def get_commit_status(sha): def has_ci_started(statuses): """判断是否有CI job已经启动（pending/running/success/failure都算启动了）""" - pr_statuses = [s for s in statuses if 'pull_request' in s.get('context', '')] + pr_statuses = [s for s in statuses if "pull_request" in s.get("context", "")] if not pr_statuses: return False # 只要有非pending且非空的状态，就算启动了 for s in pr_statuses: - if s.get('status') in ['success', 'failure', 'running']: + if s.get("status") in ["success", "failure", "running"]: return True - if s.get('status') == 'pending' and 'Has started running' in s.get('description', ''): + if s.get("status") == "pending" and "Has started running" in s.get("description", ""): return True # 全是"Blocked by required conditions"的pending也算（说明CI系统收到了事件） for s in pr_statuses: - if 'Blocked' in s.get('description', ''): + if "Blocked" in s.get("description", ""): return True return False @@ -99,26 +99,23 @@ def send_alert(pr_num, pr_title, pr_url, head_sha, commit_age_min): if not webhook: print(" ⚠️ 未配置CI_NOTIFY_WEBHOOK，跳过告警") return - + gitea_url = get_env("GITEA_URL", "https://git.xiaoxiajianji.com") - + content = { "msg_type": "interactive", "card": { "header": { - "title": { - "tag": "plain_text", - "content": f"⚠️ CI告警 - PR#{pr_num} CI未触发" - }, - "template": "red" + "title": {"tag": "plain_text", "content": f"⚠️ CI告警 - PR#{pr_num} CI未触发"}, + "template": "red", }, "elements": [ { "tag": "div", "text": { "tag": "lark_md", - "content": f"**PR**: [{pr_title}]({pr_url})\n**最新commit**: `{head_sha[:12]}`\n**已等待**: {commit_age_min:.0f} 分钟仍无CI启动\n**可能原因**: Gitea Actions事件丢失 / Webhook失败 / Runner资源不足" - } + "content": f"**PR**: [{pr_title}]({pr_url})\n**最新commit**: `{head_sha[:12]}`\n**已等待**: {commit_age_min:.0f} 分钟仍无CI启动\n**可能原因**: Gitea Actions事件丢失 / Webhook失败 / Runner资源不足", + }, }, { "tag": "action", @@ -127,29 +124,29 @@ def send_alert(pr_num, pr_title, pr_url, head_sha, commit_age_min): "tag": "button", "text": {"tag": "plain_text", "content": "查看PR"}, "url": pr_url, - "type": "primary" + "type": "primary", }, { "tag": "button", "text": {"tag": "plain_text", "content": "查看Actions"}, "url": f"{pr_url}/files", - "type": "default" - } - ] + "type": "default", + }, + ], }, { "tag": "note", "elements": [ {"tag": "plain_text", "content": f"CI触发监控 | 检测时间: {time.strftime('%Y-%m-%d %H:%M:%S')}"} - ] - } - ] - } + ], + }, + ], + }, } - + try: data = json.dumps(content).encode() - req = urllib.request.Request(webhook, data=data, method='POST') + req = urllib.request.Request(webhook, data=data, method="POST") req.add_header("Content-Type", "application/json") with urllib.request.urlopen(req, timeout=10) as resp: resp.read() @@ -160,86 +157,81 @@ def send_alert(pr_num, pr_title, pr_url, head_sha, commit_age_min): def main(): stale_threshold = int(get_env("STALE_THRESHOLD_MIN", "5")) - + print("=" * 60) print(f"CI触发监控 - 检测时间: {time.strftime('%Y-%m-%d %H:%M:%S')}") print(f"告警阈值: {stale_threshold}分钟无CI启动") print("=" * 60) - + # 获取open PR列表 try: prs = get_open_prs() except Exception as e: print(f"❌ 获取PR列表失败: {e}") sys.exit(0) # 告警脚本不阻断CI - + print(f"\n共 {len(prs)} 个open PR\n") - + stale_prs = [] now = time.time() - + for pr in prs: - pr_num = pr['number'] - pr_title = pr['title'] - pr_url = pr['html_url'] - head_sha = pr['head']['sha'] - updated_at = pr['updated_at'] - + pr_num = pr["number"] + pr_title = pr["title"] + pr_url = pr["html_url"] + head_sha = pr["head"]["sha"] + updated_at = pr["updated_at"] + # 解析updated_at（ISO格式） try: # 2026-07-17T09:22:43+08:00 from datetime import datetime, timezone, timedelta + # 简化处理：直接用字符串解析 - ts_str = updated_at.replace('Z', '+00:00') + ts_str = updated_at.replace("Z", "+00:00") # 手动解析 dt = datetime.fromisoformat(ts_str) commit_time = dt.timestamp() except Exception as e: print(f" ⚠️ PR#{pr_num} 时间解析失败: {e}") continue - + age_min = (now - commit_time) / 60 - + print(f"PR#{pr_num:3d} | {pr_title[:45]:45s} | 更新于 {age_min:.0f}min前") - + # 少于2分钟的跳过，给CI一点启动时间 if age_min < 2: print(f" ⏳ 刚更新，等待CI启动...") continue - + # 获取commit状态 status = get_commit_status(head_sha) - statuses = status.get('statuses', []) - + statuses = status.get("statuses", []) + if has_ci_started(statuses): print(f" ✅ CI已启动 (state={status.get('state')})") continue - + # CI未启动，判断是否超过阈值 if age_min >= stale_threshold: print(f" 🚨 CI未触发！已等待 {age_min:.0f} 分钟") - stale_prs.append({ - 'num': pr_num, - 'title': pr_title, - 'url': pr_url, - 'sha': head_sha, - 'age_min': age_min - }) + stale_prs.append({"num": pr_num, "title": pr_title, "url": pr_url, "sha": head_sha, "age_min": age_min}) else: print(f" ⏳ CI尚未启动 ({age_min:.0f}min < {stale_threshold}min阈值)") - + # 发送告警 print(f"\n{'=' * 60}") print(f"检测结果: {len(stale_prs)} 个PR CI未触发超过阈值") - + if stale_prs: print("\n告警列表:") for pr in stale_prs: print(f" - PR#{pr['num']}: {pr['title'][:40]} ({pr['age_min']:.0f}min)") - send_alert(pr['num'], pr['title'], pr['url'], pr['sha'], pr['age_min']) + send_alert(pr["num"], pr["title"], pr["url"], pr["sha"], pr["age_min"]) else: print("✅ 所有PR CI触发正常") - + print("=" * 60) -- 2.54.0 From d689b23997d1231b6258429ff9149252b14ff56e Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Fri, 17 Jul 2026 09:58:08 +0800 Subject: [PATCH 3/3] =?UTF-8?q?style:=20isort=E4=BF=AE=E5=A4=8D=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E6=8E=92=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/ci_trigger_monitor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci_trigger_monitor.py b/scripts/ci_trigger_monitor.py index 83de9b3a2..ad147bac7 100644 --- a/scripts/ci_trigger_monitor.py +++ b/scripts/ci_trigger_monitor.py @@ -17,8 +17,8 @@ import json import os import sys import time -import urllib.request import urllib.error +import urllib.request def get_env(name, default=""): @@ -185,7 +185,7 @@ def main(): # 解析updated_at（ISO格式） try: # 2026-07-17T09:22:43+08:00 - from datetime import datetime, timezone, timedelta + from datetime import datetime, timedelta, timezone # 简化处理：直接用字符串解析 ts_str = updated_at.replace("Z", "+00:00") -- 2.54.0