|
@@ -47,6 +47,43 @@ send_mail() {
|
|
|
fi
|
|
fi
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+# ---- Push 推送(push.rrbrr.com WebSocket 通道)----
|
|
|
|
|
+PUSH_SCRIPT="${PUSH_SCRIPT:-/usr/local/bin/health-push.py}"
|
|
|
|
|
+send_push() {
|
|
|
|
|
+ local title="$1"
|
|
|
|
|
+ local body="$2"
|
|
|
|
|
+
|
|
|
|
|
+ if [ "${PUSH_ENABLED:-true}" != "true" ]; then
|
|
|
|
|
+ log "SKIP push: PUSH_ENABLED != true"
|
|
|
|
|
+ return 0
|
|
|
|
|
+ fi
|
|
|
|
|
+ if [ ! -x "$PUSH_SCRIPT" ]; then
|
|
|
|
|
+ log "PUSH FAIL: $PUSH_SCRIPT 不存在或不可执行"
|
|
|
|
|
+ return 1
|
|
|
|
|
+ fi
|
|
|
|
|
+
|
|
|
|
|
+ local out
|
|
|
|
|
+ out=$(PUSH_API_URL="$PUSH_API_URL" PUSH_API_KEY="$PUSH_API_KEY" \
|
|
|
|
|
+ PUSH_USER_ID="$PUSH_USER_ID" PUSH_CHANNEL="$PUSH_CHANNEL" \
|
|
|
|
|
+ python3 "$PUSH_SCRIPT" "$title" <<< "$body" 2>&1)
|
|
|
|
|
+ local rc=$?
|
|
|
|
|
+ if [ $rc -eq 0 ]; then
|
|
|
|
|
+ log "PUSH OK: $title | $out"
|
|
|
|
|
+ return 0
|
|
|
|
|
+ else
|
|
|
|
|
+ log "PUSH FAIL (rc=$rc): $title | $out"
|
|
|
|
|
+ return 1
|
|
|
|
|
+ fi
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+# ---- 统一告警入口:同时走 mail + push ----
|
|
|
|
|
+send_alert() {
|
|
|
|
|
+ local subject="$1"
|
|
|
|
|
+ local body="$2"
|
|
|
|
|
+ send_mail "$subject" "$body"
|
|
|
|
|
+ send_push "$subject" "$body"
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
# ---- 加载配置 ----
|
|
# ---- 加载配置 ----
|
|
|
CONFIG_FILE="/etc/health-monitor.env"
|
|
CONFIG_FILE="/etc/health-monitor.env"
|
|
|
if [ ! -f "$CONFIG_FILE" ]; then
|
|
if [ ! -f "$CONFIG_FILE" ]; then
|
|
@@ -103,7 +140,7 @@ if [ -z "$fail_detail" ]; then
|
|
|
# 恢复通知
|
|
# 恢复通知
|
|
|
if [ "$prev_status" = "down" ]; then
|
|
if [ "$prev_status" = "down" ]; then
|
|
|
log "RECOVERED: $HEALTH_URL + $API_URL 均恢复"
|
|
log "RECOVERED: $HEALTH_URL + $API_URL 均恢复"
|
|
|
- send_mail "✅ [book] 服务已恢复" \
|
|
|
|
|
|
|
+ send_alert "✅ [book] 服务已恢复" \
|
|
|
"时间: $(date '+%F %T')\n主机: $(hostname)\nURL: $HEALTH_URL\n之前: 连续 ${prev_consec} 次失败\n当前: 全部 200 OK"
|
|
"时间: $(date '+%F %T')\n主机: $(hostname)\nURL: $HEALTH_URL\n之前: 连续 ${prev_consec} 次失败\n当前: 全部 200 OK"
|
|
|
fi
|
|
fi
|
|
|
|
|
|
|
@@ -127,7 +164,7 @@ log "FAIL: $fail_detail (consec=$new_consec)"
|
|
|
# 达到阈值才发,且要过冷却期
|
|
# 达到阈值才发,且要过冷却期
|
|
|
if [ "$new_consec" -ge "$CONSEC_FAIL_THRESHOLD" ] && [ "$elapsed" -ge "$ALERT_COOLDOWN_SEC" ]; then
|
|
if [ "$new_consec" -ge "$CONSEC_FAIL_THRESHOLD" ] && [ "$elapsed" -ge "$ALERT_COOLDOWN_SEC" ]; then
|
|
|
log "ALERT: 触发告警 (consec=$new_consec, elapsed=${elapsed}s)"
|
|
log "ALERT: 触发告警 (consec=$new_consec, elapsed=${elapsed}s)"
|
|
|
- send_mail "🚨 [book] 服务异常 - $fail_detail" \
|
|
|
|
|
|
|
+ send_alert "🚨 [book] 服务异常 - $fail_detail" \
|
|
|
"时间: $(date '+%F %T')\n主机: $(hostname)\nURL: $HEALTH_URL\nAPI: $API_URL\n\n检测结果:\n health: $health\n api: $api\n\n连续失败次数: $new_consec\n上次告警: $((elapsed / 60)) 分钟前\n\n可能原因:\n 1) PM2 进程退出: pm2 list\n 2) 端口被占用: ss -lntp | grep 3100\n 3) 进程 OOM: dmesg | tail -20\n 4) 磁盘满: df -h /\n\nssh 排查: ssh -p 22622 root@8.159.134.106 'pm2 logs server --lines 50'"
|
|
"时间: $(date '+%F %T')\n主机: $(hostname)\nURL: $HEALTH_URL\nAPI: $API_URL\n\n检测结果:\n health: $health\n api: $api\n\n连续失败次数: $new_consec\n上次告警: $((elapsed / 60)) 分钟前\n\n可能原因:\n 1) PM2 进程退出: pm2 list\n 2) 端口被占用: ss -lntp | grep 3100\n 3) 进程 OOM: dmesg | tail -20\n 4) 磁盘满: df -h /\n\nssh 排查: ssh -p 22622 root@8.159.134.106 'pm2 logs server --lines 50'"
|
|
|
|
|
|
|
|
if [ $? -eq 0 ]; then
|
|
if [ $? -eq 0 ]; then
|