fix(ci): 定时任务防僵尸——urlopen补超时+扫描墙钟上限+降频 [P0] #1561

Merged
xiaoxia merged 1 commits from ci/fix-scheduled-jobs-zombie-timeout into develop 2026-08-31 11:57:46 +08:00
5 changed files with 49 additions and 20 deletions
+2 -2
View File
@@ -2,13 +2,13 @@ name: CI Trigger Monitor
on:
schedule:
- cron: '*/5 * * * *' # 每5分钟检查一次
- cron: '*/10 * * * *' # 每10分钟检查一次(与pr-auto-scan同步降频)
workflow_dispatch:
inputs:
stale_threshold:
description: 'CI未触发告警阈值(分钟)'
required: false
default: '5'
default: '10'
permissions:
contents: read
+1 -1
View File
@@ -3,7 +3,7 @@ name: PR Auto Scan
# 作为短作业模式的兜底,防止事件驱动遗漏
on:
schedule:
- cron: "*/5 * * * *" # 每5分钟扫描一次
- cron: "*/10 * * * *" # 每10分钟扫描一次(脚本自带240s墙钟上限,降频减负)
workflow_dispatch:
permissions:
+7 -7
View File
@@ -79,7 +79,7 @@ def get_token(repo, action="pull"):
token_url = AUTH_URL + "?service=" + SERVICE + "&scope=" + scope
req = urllib.request.Request(token_url)
req.add_header("Authorization", "Basic " + base64.b64encode((USERNAME + ":" + PASSWORD).encode()).decode())
with urllib.request.urlopen(req) as resp:
with urllib.request.urlopen(req, timeout=30) as resp:
data = json.loads(resp.read())
return data.get("token", "")
@@ -89,7 +89,7 @@ def get_tags(repo, token):
url = "https://" + REGISTRY + "/v2/" + NAMESPACE + "/" + repo + "/tags/list?n=1000"
req = urllib.request.Request(url)
req.add_header("Authorization", "Bearer " + token)
with urllib.request.urlopen(req) as resp:
with urllib.request.urlopen(req, timeout=30) as resp:
data = json.loads(resp.read())
return data.get("tags", []) or []
@@ -99,7 +99,7 @@ def http_get_json(url, token, accept_header):
req = urllib.request.Request(url)
req.add_header("Authorization", "Bearer " + token)
req.add_header("Accept", accept_header)
with urllib.request.urlopen(req) as resp:
with urllib.request.urlopen(req, timeout=30) as resp:
return json.loads(resp.read()), resp.headers
@@ -194,7 +194,7 @@ def delete_manifest(repo, digest, token):
req.add_header("Accept", ACCEPT_MANIFEST_OCI)
req.add_header("Accept", ACCEPT_MANIFEST_V2)
try:
with urllib.request.urlopen(req) as resp:
with urllib.request.urlopen(req, timeout=30) as resp:
return True, resp.status
except urllib.error.HTTPError as e:
return False, str(e.code) + " " + e.read().decode()[:200]
@@ -216,7 +216,7 @@ def gitea_get_open_prs():
req = urllib.request.Request(url)
req.add_header("Authorization", "token " + GITEA_TOKEN)
try:
with urllib.request.urlopen(req) as resp:
with urllib.request.urlopen(req, timeout=30) as resp:
data = json.loads(resp.read())
if not data:
break
@@ -241,7 +241,7 @@ def gitea_get_pr_commits(pr_number):
req = urllib.request.Request(url)
req.add_header("Authorization", "token " + GITEA_TOKEN)
try:
with urllib.request.urlopen(req) as resp:
with urllib.request.urlopen(req, timeout=30) as resp:
data = json.loads(resp.read())
return [c.get("sha", "") for c in data]
except Exception as e:
@@ -408,7 +408,7 @@ def cleanup_repo(repo, keep_count, dry_run, protected_tags, pr_sha=None, pr_open
req = urllib.request.Request(url)
req.add_header("Authorization", "token " + GITEA_TOKEN)
try:
with urllib.request.urlopen(req) as resp:
with urllib.request.urlopen(req, timeout=30) as resp:
data = json.loads(resp.read())
if not data:
break
+5 -5
View File
@@ -46,7 +46,7 @@ def ensure_git_repo(api_url, repo, token, pr_number):
# 获取PR的源分支
pr_api_url = f"{api_url}/repos/{repo}/pulls/{pr_number}"
req_obj = urllib.request.Request(pr_api_url, headers={"Authorization": f"token {token}"})
with urllib.request.urlopen(req_obj) as resp:
with urllib.request.urlopen(req_obj, timeout=15) as resp:
pr = json.loads(resp.read())
head_branch = pr["head"]["ref"]
@@ -120,7 +120,7 @@ def get_changed_files(pr_number, api_url, token):
"""获取PR中变更的文件列表"""
url = f"{api_url}/pulls/{pr_number}/files?limit=100"
req = urllib.request.Request(url, headers={"Authorization": f"token {token}"})
with urllib.request.urlopen(req) as resp:
with urllib.request.urlopen(req, timeout=15) as resp:
files = json.loads(resp.read())
return [f["filename"] for f in files if f["status"] != "removed"]
@@ -129,7 +129,7 @@ def get_pr_head_branch(pr_number, api_url, token):
"""获取PR的来源分支名"""
url = f"{api_url}/pulls/{pr_number}"
req = urllib.request.Request(url, headers={"Authorization": f"token {token}"})
with urllib.request.urlopen(req) as resp:
with urllib.request.urlopen(req, timeout=15) as resp:
pr = json.loads(resp.read())
return pr["head"]["ref"]
@@ -244,7 +244,7 @@ def main():
# 获取PR信息
pr_info_url = f"{api_url}/repos/{repo}/pulls/{pr_number}"
req_pr = urllib.request.Request(pr_info_url, headers={"Authorization": f"token {token}"})
with urllib.request.urlopen(req_pr) as resp:
with urllib.request.urlopen(req_pr, timeout=15) as resp:
pr_info = json.loads(resp.read())
pr_author = pr_info.get("user", {}).get("login", "")
print(f"PR作者: {pr_author}")
@@ -256,7 +256,7 @@ def main():
try:
commits_url = f"{api_url}/repos/{repo}/pulls/{pr_number}/commits?limit=3"
req_commits = urllib.request.Request(commits_url, headers={"Authorization": f"token {token}"})
with urllib.request.urlopen(req_commits) as resp_commits:
with urllib.request.urlopen(req_commits, timeout=15) as resp_commits:
commits = json.loads(resp_commits.read())
latest_msg = commits[0].get("commit", {}).get("message", "") if commits else ""
if skip_marker in latest_msg:
+34 -5
View File
@@ -9,10 +9,16 @@ PR自动扫描器:扫描所有open PR,对CI全绿的进行自动审批/合
import argparse
import json
import re
import socket
import time
import urllib.error
import urllib.request
# 单次HTTP请求超时(秒),防止网络异常时永久阻塞占住runner
HTTP_TIMEOUT = 15
# 整次扫描墙钟上限(秒),到点主动退出(Gitea 1.26的timeout-minutes不可靠,脚本自保)
DEFAULT_WALL_SECONDS = 240
def api_request(token, repo, endpoint, method="GET", data=None):
"""Gitea API请求"""
@@ -29,7 +35,7 @@ def api_request(token, repo, endpoint, method="GET", data=None):
ctx.verify_mode = ssl.CERT_NONE
try:
resp = urllib.request.urlopen(req, context=ctx)
resp = urllib.request.urlopen(req, context=ctx, timeout=HTTP_TIMEOUT)
return json.loads(resp.read().decode()), resp.status
except urllib.error.HTTPError as e:
body = e.read().decode()
@@ -39,6 +45,9 @@ def api_request(token, repo, endpoint, method="GET", data=None):
except json.JSONDecodeError:
return {"error": body}, e.code
return {"error": str(e)}, e.code
except (urllib.error.URLError, socket.timeout, TimeoutError, OSError) as e:
# 网络不可达/超时:返回599让调用方按失败处理,绝不永久挂起
return {"error": f"request-failed: {e}"}, 599
def get_open_prs(token, repo, base="develop"):
@@ -223,10 +232,12 @@ def add_pr_label(token, repo, pr_number, label):
return code in (200, 201)
def merge_pr(token, repo, pr_number):
def merge_pr(token, repo, pr_number, deadline=None):
"""合并PRsquash merge"""
# 等待几秒让状态同步
time.sleep(30)
# 等待几秒让状态同步(可被墙钟上限打断,最多等30秒)
wait_end = min(time.monotonic() + 30, deadline) if deadline else time.monotonic() + 30
while time.monotonic() < wait_end:
time.sleep(2)
# 检查PR状态
pr_data, code = api_request(token, repo, f"pulls/{pr_number}")
@@ -268,11 +279,23 @@ def main():
parser.add_argument("--dry-run", default="false", help="试运行模式")
parser.add_argument("--max-prs", type=int, default=20, help="最多处理的PR数")
parser.add_argument("--skip-ai-review", action="store_true", help="跳过AI审查检查(强制审批)")
parser.add_argument(
"--max-wall-seconds",
type=int,
default=DEFAULT_WALL_SECONDS,
help="整次扫描墙钟上限(秒),到点主动退出,默认240",
)
args = parser.parse_args()
dry_run = args.dry_run.lower() == "true"
# 墙钟自保:Gitea 1.26 的 timeout-minutes 对卡死 job 不生效,脚本自己兜底
wall_deadline = time.monotonic() + args.max_wall_seconds
def wall_expired():
return time.monotonic() >= wall_deadline
# required contexts(与分支保护一致)
REQUIRED_CONTEXTS_FULL = [
# 统一使用CI Gate作为合并门禁(与pr-automation和分支保护保持一致)
@@ -301,6 +324,9 @@ def main():
ai_blocked_count = 0
for pr in prs[: args.max_prs]:
if wall_expired():
print(f"\n⏰ 达到墙钟上限 {args.max_wall_seconds}s,停止处理剩余PR(下次调度继续)")
break
pr_num = pr["number"]
pr_title = pr["title"]
head_sha = pr["head"]["sha"]
@@ -379,11 +405,14 @@ def main():
approved = has_approval(args.token, args.repo, pr_num)
if merge_ok and approved and not merge_failed:
if wall_expired():
print("⏰ 达到墙钟上限,跳过本次合并(下次调度继续)")
break
if dry_run:
print(" 🎯 [DRY-RUN] 将自动合并")
else:
print(" 🎯 执行自动合并...")
ok, msg = merge_pr(args.token, args.repo, pr_num)
ok, msg = merge_pr(args.token, args.repo, pr_num, deadline=wall_deadline)
if ok:
print(f" ✅ 合并成功: {msg}")
merged_count += 1