audio_codebuddy/deploy-package/health-monitor/health-monitor.sh

#!/bin/bash
#
# health-monitor.sh — 线上后端健康监控 + 邮件告警
#
# 部署路径：/usr/local/bin/health-monitor.sh
# 凭证路径：/etc/health-monitor.env   (QQ 邮箱授权码,chmod 600)
# 状态路径：/var/lib/health-monitor/state
# 日志路径：/var/log/health-monitor.log
# cron 路径：/etc/cron.d/health-monitor  (*/5 * * * *)
#
# 告警策略：
#   - 连续 2 次失败才发告警（防抖动）
#   - 同一故障 30 分钟内只发一次（防告警风暴）
#   - 服务恢复时单独发一条 ✅
#
# 邮件走 QQ 邮箱 SMTP 465 (SSL)，阿里云默认屏蔽 25 出站但 465/587 通畅

set -u

# ---- 邮件发送（调用 Python smtplib，避免 openssl s_client 交互卡死）----
MAIL_SCRIPT="${MAIL_SCRIPT:-/usr/local/bin/health-send-mail.py}"
send_mail() {
  local subject="$1"
  local body="$2"

  if [ -z "${QQ_USER:-}" ] || [ -z "${QQ_AUTH_CODE:-}" ]; then
    log "SKIP mail: QQ_USER 或 QQ_AUTH_CODE 未配置"
    return 1
  fi
  if [ ! -x "$MAIL_SCRIPT" ]; then
    log "MAIL FAIL: $MAIL_SCRIPT 不存在或不可执行"
    return 1
  fi

  local out
  out=$(QQ_USER="$QQ_USER" QQ_AUTH_CODE="$QQ_AUTH_CODE" \
        SMTP_HOST="$SMTP_HOST" SMTP_PORT="$SMTP_PORT" \
        MAIL_FROM="$MAIL_FROM" \
        python3 "$MAIL_SCRIPT" "$subject" <<< "$body" 2>&1)
  local rc=$?
  if [ $rc -eq 0 ]; then
    log "MAIL OK: $subject | $out"
    return 0
  else
    log "MAIL FAIL (rc=$rc): $subject | $out"
    return 1
  fi
}

# ---- Push 推送（push.rrbrr.com WebSocket 通道）----
PUSH_SCRIPT="${PUSH_SCRIPT:-/usr/local/bin/health-push.py}"
send_push() {
  local title="$1"
  local body="$2"

  if [ "${PUSH_ENABLED:-true}" != "true" ]; then
    log "SKIP push: PUSH_ENABLED != true"
    return 0
  fi
  if [ ! -x "$PUSH_SCRIPT" ]; then
    log "PUSH FAIL: $PUSH_SCRIPT 不存在或不可执行"
    return 1
  fi

  local out
  out=$(PUSH_API_URL="$PUSH_API_URL" PUSH_API_KEY="$PUSH_API_KEY" \
        PUSH_USER_ID="$PUSH_USER_ID" PUSH_CHANNEL="$PUSH_CHANNEL" \
        python3 "$PUSH_SCRIPT" "$title" <<< "$body" 2>&1)
  local rc=$?
  if [ $rc -eq 0 ]; then
    log "PUSH OK: $title | $out"
    return 0
  else
    log "PUSH FAIL (rc=$rc): $title | $out"
    return 1
  fi
}

# ---- 统一告警入口：同时走 mail + push ----
send_alert() {
  local subject="$1"
  local body="$2"
  send_mail "$subject" "$body"
  send_push  "$subject" "$body"
}

# ---- 加载配置 ----
CONFIG_FILE="/etc/health-monitor.env"
if [ ! -f "$CONFIG_FILE" ]; then
  echo "[$(date '+%F %T')] FATAL: $CONFIG_FILE 不存在，请创建并填 QQ 邮箱授权码" >> /var/log/health-monitor.log
  exit 2
fi
# shellcheck disable=SC1090
source "$CONFIG_FILE"

: "${HEALTH_URL:=http://127.0.0.1:3100/health}"
: "${API_URL:=http://127.0.0.1:3100/api/navigation/config}"
: "${SMTP_HOST:=smtp.qq.com}"
: "${SMTP_PORT:=465}"
: "${MAIL_FROM:=$QQ_USER}"
: "${CONSEC_FAIL_THRESHOLD:=2}"
: "${ALERT_COOLDOWN_SEC:=1800}"
: "${LOG_FILE:=/var/log/health-monitor.log}"
: "${STATE_DIR:=/var/lib/health-monitor}"

STATE_FILE="$STATE_DIR/state"

mkdir -p "$STATE_DIR"
touch "$LOG_FILE"

log() {
  echo "[$(date '+%F %T')] $*" >> "$LOG_FILE"
}

# ---- 检测 ----
check_endpoint() {
  local url="$1"
  local code
  code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 6 "$url" 2>/dev/null)
  if [ "$code" = "200" ]; then
    echo "ok"
  else
    echo "fail:$code"
  fi
}

health=$(check_endpoint "$HEALTH_URL")
api=$(check_endpoint "$API_URL")

# 提取详细失败原因
fail_detail=""
if [[ "$health" != ok ]]; then fail_detail+="health=$health "; fi
if [[ "$api" != ok ]]; then fail_detail+="api=$api "; fi

if [ -z "$fail_detail" ]; then
  # ---- 正常 ----
  prev_consec=$(cat "$STATE_FILE" 2>/dev/null | awk -F: '{print $1}' || echo 0)
  prev_status=$(cat "$STATE_FILE" 2>/dev/null | awk -F: '{print $2}' || echo ok)

  # 恢复通知
  if [ "$prev_status" = "down" ]; then
    log "RECOVERED: $HEALTH_URL + $API_URL 均恢复"
    send_alert "✅ [book] 服务已恢复" \
      "时间: $(date '+%F %T')\n主机: $(hostname)\nURL: $HEALTH_URL\n之前: 连续 ${prev_consec} 次失败\n当前: 全部 200 OK"
  fi

  echo "0:ok" > "$STATE_FILE"
  log "OK: health=$health api=$api"
  exit 0
fi

# ---- 故障 ----
prev_consec=$(cat "$STATE_FILE" 2>/dev/null | awk -F: '{print $1}' || echo 0)
prev_status=$(cat "$STATE_FILE" 2>/dev/null | awk -F: '{print $2}' || echo ok)

new_consec=$((prev_consec + 1))
last_alert_at=$(cat "$STATE_DIR/last_alert_at" 2>/dev/null || echo 0)
now=$(date +%s)
elapsed=$((now - last_alert_at))

echo "${new_consec}:down" > "$STATE_FILE"
log "FAIL: $fail_detail (consec=$new_consec)"

# 达到阈值才发,且要过冷却期
if [ "$new_consec" -ge "$CONSEC_FAIL_THRESHOLD" ] && [ "$elapsed" -ge "$ALERT_COOLDOWN_SEC" ]; then
  log "ALERT: 触发告警 (consec=$new_consec, elapsed=${elapsed}s)"
  send_alert "🚨 [book] 服务异常 - $fail_detail" \
    "时间: $(date '+%F %T')\n主机: $(hostname)\nURL: $HEALTH_URL\nAPI:  $API_URL\n\n检测结果:\n  health: $health\n  api:    $api\n\n连续失败次数: $new_consec\n上次告警: $((elapsed / 60)) 分钟前\n\n可能原因:\n  1) PM2 进程退出: pm2 list\n  2) 端口被占用: ss -lntp | grep 3100\n  3) 进程 OOM: dmesg | tail -20\n  4) 磁盘满: df -h /\n\nssh 排查: ssh -p 22622 root@8.159.134.106 'pm2 logs server --lines 50'"

  if [ $? -eq 0 ]; then
    echo "$now" > "$STATE_DIR/last_alert_at"
  fi
fi

# (send_mail 函数定义在脚本顶部,供 check 调用)