feat(monitor): 5 分钟健康监控 + QQ 邮箱 SMTP 告警

- health-monitor.sh: 检测 /health + /api/navigation/config
- 健康策略: 连续 2 次失败才告警(防抖动),30 分钟冷却(防告警风暴),恢复单独发 ✅
- health-send-mail.py: Python smtplib 走 465 SSL (阿里云屏蔽 25 出站)
- install.sh: 一键部署到 /usr/local/bin + /etc/cron.d/health-monitor
- README.md: 含授权码获取步骤 + 验证告警链路流程
- 凭证: 从 stock_trading/.env 提取 QQ_AUTH_CODE,记到 vault api-keys.md

deploy-package/health-monitor/README.md

@@ -0,0 +1,82 @@
+# 线上后端健康监控
+
+> 后端挂掉时邮件告警到 f1f3@qq.com。带去重防告警风暴，服务恢复时单独发 ✅。
+
+## 文件清单
+
+| 文件 | 部署到 | 说明 |
+|------|--------|------|
+| `health-monitor.sh` | `/usr/local/bin/` | 主脚本 |
+| `health-monitor.env.example` | `/etc/health-monitor.env` (改名) | 凭证模板 |
+| `install.sh` | (服务器上跑一次) | 一键部署 |
+
+## 部署步骤
+
+```bash
+# 1. 在本地打包
+cd deploy-package/health-monitor && tar czf /tmp/health-monitor.tar.gz .
+
+# 2. 上传到服务器
+scp -P 22622 /tmp/health-monitor.tar.gz root@8.159.134.106:/tmp/
+
+# 3. 服务器上安装
+ssh -p 22622 root@8.159.134.106
+cd /tmp && tar xzf health-monitor.tar.gz && cd health-monitor
+
+# 4. 填 QQ 邮箱授权码
+cp health-monitor.env.example /etc/health-monitor.env
+vim /etc/health-monitor.env       # 填入 QQ_AUTH_CODE
+chmod 600 /etc/health-monitor.env
+
+# 5. 部署
+bash install.sh
+```
+
+## 获取 QQ 邮箱授权码
+
+1. 网页登录 https://mail.qq.com
+2. 设置 → 账户 → POP3/IMAP/SMTP/Exchange/CardDAV/CalDAV服务
+3. 开启 **SMTP 服务**（手机短信验证）
+4. 复制"授权码"（16 位字符串，**不是你的 QQ 密码**）
+
+## 工作原理
+
+- cron 每 5 分钟跑一次（错开整点 :03）
+- 同时检测 `/health` 和 `/api/navigation/config`
+- **连续 2 次失败** 才触发告警（防抖动）
+- **30 分钟冷却期** — 同一故障不会反复骚扰
+- 服务**恢复**时单独发一条 ✅
+
+## 排查命令
+
+```bash
+tail -f /var/log/health-monitor.log          # 实时日志
+pm2 list                                      # PM2 进程状态
+ss -lntp | grep 3100                          # 端口监听
+curl -sS http://127.0.0.1:3100/health         # 直接验证
+bash /usr/local/bin/health-monitor.sh         # 手动跑一次
+```
+
+## 验证告警链路
+
+```bash
+# 1. 暂停后端
+pm2 stop server
+
+# 2. 触发检测
+bash /usr/local/bin/health-monitor.sh
+bash /usr/local/bin/health-monitor.sh   # 第二次会发邮件
+
+# 3. 启动后端
+pm2 start server
+
+# 4. 触发恢复检测
+bash /usr/local/bin/health-monitor.sh   # 会发"已恢复"邮件
+```
+
+## 设计取舍
+
+- **不依赖 Node.js 全局包** — 纯 bash + openssl，零依赖
+- **走 465 SSL** — 阿里云屏蔽 25 出站（防垃圾邮件），465 通畅
+- **状态文件 `/var/lib/health-monitor/state`** — 记录连续失败次数 + 上次告警时间，去重逻辑
+- **错开 :03 而非整点** — 避免大量实例同时打到云服务

deploy-package/health-monitor/health-monitor.env.example

@@ -0,0 +1,22 @@
+# health-monitor.env — QQ 邮箱 SMTP 凭证
+#
+# ⚠️ 必须 chmod 600,只能 root 读写
+# ⚠️ 不要 commit 到 git(.gitignore 已忽略 *.env)
+#
+# 获取 QQ 邮箱授权码：
+#   1. 网页登录 https://mail.qq.com
+#   2. 设置 → 账户 → 找到 POP3/IMAP/SMTP/Exchange/CardDAV/CalDAV服务
+#   3. 开启 SMTP 服务（手机短信验证）
+#   4. 复制"授权码"（16位字符串，**不是你的 QQ 密码**）
+
+QQ_USER=f1f3@qq.com
+QQ_AUTH_CODE=把你的QQ授权码填这里
+
+# 可选：覆盖默认（一般不用改）
+# HEALTH_URL=http://127.0.0.1:3100/health
+# API_URL=http://127.0.0.1:3100/api/navigation/config
+# SMTP_HOST=smtp.qq.com
+# SMTP_PORT=465
+# MAIL_FROM=$QQ_USER
+# CONSEC_FAIL_THRESHOLD=2
+# ALERT_COOLDOWN_SEC=1800

deploy-package/health-monitor/health-monitor.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+#
+# health-monitor.sh — 线上后端健康监控 + 邮件告警
+#
+# 部署路径：/usr/local/bin/health-monitor.sh
+# 凭证路径：/etc/health-monitor.env   (QQ 邮箱授权码,chmod 600)
+# 状态路径：/var/lib/health-monitor/state
+# 日志路径：/var/log/health-monitor.log
+# cron 路径：/etc/cron.d/health-monitor  (*/5 * * * *)
+#
+# 告警策略：
+#   - 连续 2 次失败才发告警（防抖动）
+#   - 同一故障 30 分钟内只发一次（防告警风暴）
+#   - 服务恢复时单独发一条 ✅
+#
+# 邮件走 QQ 邮箱 SMTP 465 (SSL)，阿里云默认屏蔽 25 出站但 465/587 通畅
+
+set -u
+
+# ---- 邮件发送（调用 Python smtplib，避免 openssl s_client 交互卡死）----
+MAIL_SCRIPT="${MAIL_SCRIPT:-/usr/local/bin/health-send-mail.py}"
+send_mail() {
+  local subject="$1"
+  local body="$2"
+
+  if [ -z "${QQ_USER:-}" ] || [ -z "${QQ_AUTH_CODE:-}" ]; then
+    log "SKIP mail: QQ_USER 或 QQ_AUTH_CODE 未配置"
+    return 1
+  fi
+  if [ ! -x "$MAIL_SCRIPT" ]; then
+    log "MAIL FAIL: $MAIL_SCRIPT 不存在或不可执行"
+    return 1
+  fi
+
+  local out
+  out=$(QQ_USER="$QQ_USER" QQ_AUTH_CODE="$QQ_AUTH_CODE" \
+        SMTP_HOST="$SMTP_HOST" SMTP_PORT="$SMTP_PORT" \
+        MAIL_FROM="$MAIL_FROM" \
+        python3 "$MAIL_SCRIPT" "$subject" <<< "$body" 2>&1)
+  local rc=$?
+  if [ $rc -eq 0 ]; then
+    log "MAIL OK: $subject | $out"
+    return 0
+  else
+    log "MAIL FAIL (rc=$rc): $subject | $out"
+    return 1
+  fi
+}
+
+# ---- 加载配置 ----
+CONFIG_FILE="/etc/health-monitor.env"
+if [ ! -f "$CONFIG_FILE" ]; then
+  echo "[$(date '+%F %T')] FATAL: $CONFIG_FILE 不存在，请创建并填 QQ 邮箱授权码" >> /var/log/health-monitor.log
+  exit 2
+fi
+# shellcheck disable=SC1090
+source "$CONFIG_FILE"
+
+: "${HEALTH_URL:=http://127.0.0.1:3100/health}"
+: "${API_URL:=http://127.0.0.1:3100/api/navigation/config}"
+: "${SMTP_HOST:=smtp.qq.com}"
+: "${SMTP_PORT:=465}"
+: "${MAIL_FROM:=$QQ_USER}"
+: "${CONSEC_FAIL_THRESHOLD:=2}"
+: "${ALERT_COOLDOWN_SEC:=1800}"
+: "${LOG_FILE:=/var/log/health-monitor.log}"
+: "${STATE_DIR:=/var/lib/health-monitor}"
+
+STATE_FILE="$STATE_DIR/state"
+
+mkdir -p "$STATE_DIR"
+touch "$LOG_FILE"
+
+log() {
+  echo "[$(date '+%F %T')] $*" >> "$LOG_FILE"
+}
+
+# ---- 检测 ----
+check_endpoint() {
+  local url="$1"
+  local code
+  code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 6 "$url" 2>/dev/null)
+  if [ "$code" = "200" ]; then
+    echo "ok"
+  else
+    echo "fail:$code"
+  fi
+}
+
+health=$(check_endpoint "$HEALTH_URL")
+api=$(check_endpoint "$API_URL")
+
+# 提取详细失败原因
+fail_detail=""
+if [[ "$health" != ok ]]; then fail_detail+="health=$health "; fi
+if [[ "$api" != ok ]]; then fail_detail+="api=$api "; fi
+
+if [ -z "$fail_detail" ]; then
+  # ---- 正常 ----
+  prev_consec=$(cat "$STATE_FILE" 2>/dev/null | awk -F: '{print $1}' || echo 0)
+  prev_status=$(cat "$STATE_FILE" 2>/dev/null | awk -F: '{print $2}' || echo ok)
+
+  # 恢复通知
+  if [ "$prev_status" = "down" ]; then
+    log "RECOVERED: $HEALTH_URL + $API_URL 均恢复"
+    send_mail "✅ [book] 服务已恢复" \
+      "时间: $(date '+%F %T')\n主机: $(hostname)\nURL: $HEALTH_URL\n之前: 连续 ${prev_consec} 次失败\n当前: 全部 200 OK"
+  fi
+
+  echo "0:ok" > "$STATE_FILE"
+  log "OK: health=$health api=$api"
+  exit 0
+fi
+
+# ---- 故障 ----
+prev_consec=$(cat "$STATE_FILE" 2>/dev/null | awk -F: '{print $1}' || echo 0)
+prev_status=$(cat "$STATE_FILE" 2>/dev/null | awk -F: '{print $2}' || echo ok)
+
+new_consec=$((prev_consec + 1))
+last_alert_at=$(cat "$STATE_DIR/last_alert_at" 2>/dev/null || echo 0)
+now=$(date +%s)
+elapsed=$((now - last_alert_at))
+
+echo "${new_consec}:down" > "$STATE_FILE"
+log "FAIL: $fail_detail (consec=$new_consec)"
+
+# 达到阈值才发,且要过冷却期
+if [ "$new_consec" -ge "$CONSEC_FAIL_THRESHOLD" ] && [ "$elapsed" -ge "$ALERT_COOLDOWN_SEC" ]; then
+  log "ALERT: 触发告警 (consec=$new_consec, elapsed=${elapsed}s)"
+  send_mail "🚨 [book] 服务异常 - $fail_detail" \
+    "时间: $(date '+%F %T')\n主机: $(hostname)\nURL: $HEALTH_URL\nAPI:  $API_URL\n\n检测结果:\n  health: $health\n  api:    $api\n\n连续失败次数: $new_consec\n上次告警: $((elapsed / 60)) 分钟前\n\n可能原因:\n  1) PM2 进程退出: pm2 list\n  2) 端口被占用: ss -lntp | grep 3100\n  3) 进程 OOM: dmesg | tail -20\n  4) 磁盘满: df -h /\n\nssh 排查: ssh -p 22622 root@8.159.134.106 'pm2 logs server --lines 50'"
+
+  if [ $? -eq 0 ]; then
+    echo "$now" > "$STATE_DIR/last_alert_at"
+  fi
+fi
+
+# (send_mail 函数定义在脚本顶部,供 check 调用)

deploy-package/health-monitor/health-send-mail.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""
+health-send-mail.py — 通过 QQ 邮箱 SMTP 465 SSL 发邮件
+
+Usage:
+  QQ_USER=f1f3@qq.com QQ_AUTH_CODE=xxx SMTP_HOST=smtp.qq.com SMTP_PORT=465 \
+    python3 health-send-mail.py <subject> <<< "body"
+
+Env required:
+  QQ_USER         发件邮箱 (= 收件邮箱)
+  QQ_AUTH_CODE    QQ 邮箱授权码 (不是登录密码)
+  SMTP_HOST       默认 smtp.qq.com
+  SMTP_PORT       默认 465
+"""
+
+import os
+import sys
+import smtplib
+import ssl
+import datetime
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from email.utils import formataddr
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: health-send-mail.py <subject>", file=sys.stderr)
+        sys.exit(2)
+
+    subject = sys.argv[1]
+    body = sys.stdin.read() if not sys.stdin.isatty() else "(empty)"
+
+    user = os.environ.get('QQ_USER', '').strip()
+    pwd = os.environ.get('QQ_AUTH_CODE', '').strip()
+    host = os.environ.get('SMTP_HOST', 'smtp.qq.com').strip()
+    port = int(os.environ.get('SMTP_PORT', '465'))
+    sender_name = os.environ.get('SENDER_NAME', 'book 监控')
+
+    if not user or not pwd:
+        print("MISSING: QQ_USER / QQ_AUTH_CODE env not set", file=sys.stderr)
+        sys.exit(3)
+
+    msg = MIMEMultipart()
+    msg['From'] = formataddr((sender_name, user))
+    msg['To'] = user
+    msg['Subject'] = subject
+    msg['Date'] = datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S %z')
+    msg.attach(MIMEText(body, 'plain', 'utf-8'))
+
+    ctx = ssl.create_default_context()
+    try:
+        with smtplib.SMTP_SSL(host, port, timeout=15, context=ctx) as s:
+            s.login(user, pwd)
+            s.sendmail(user, [user], msg.as_string())
+        print(f"OK: sent '{subject}' to {user}")
+        sys.exit(0)
+    except smtplib.SMTPAuthenticationError as e:
+        print(f"AUTH FAIL: {e} (check QQ_AUTH_CODE, may need reissue at mail.qq.com)", file=sys.stderr)
+        sys.exit(4)
+    except Exception as e:
+        print(f"SMTP FAIL [{type(e).__name__}]: {e}", file=sys.stderr)
+        sys.exit(5)
+
+
+if __name__ == '__main__':
+    main()

deploy-package/health-monitor/install.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# install.sh — 部署健康监控到服务器
+#
+# 用法（在服务器上 root 跑）：
+#   cd /tmp && tar xzf health-monitor.tar.gz && cd health-monitor && bash install.sh
+#
+# 前置条件：
+#   1. /etc/health-monitor.env 已存在并填好 QQ_AUTH_CODE（参考 health-monitor.env.example）
+
+set -e
+
+if [ "$(id -u)" -ne 0 ]; then
+  echo "必须 root 运行" >&2
+  exit 1
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+# 1. 凭证
+if [ ! -f /etc/health-monitor.env ]; then
+  echo "ERROR: /etc/health-monitor.env 不存在，请按以下步骤创建：" >&2
+  echo "  cp $SCRIPT_DIR/health-monitor.env.example /etc/health-monitor.env" >&2
+  echo "  vim /etc/health-monitor.env   # 填入 QQ_AUTH_CODE" >&2
+  echo "  chmod 600 /etc/health-monitor.env" >&2
+  exit 1
+fi
+chmod 600 /etc/health-monitor.env
+
+# 检查 QQ_AUTH_CODE 是否还是占位符
+if grep -q 'QQ_AUTH_CODE=把你的' /etc/health-monitor.env; then
+  echo "ERROR: /etc/health-monitor.env 还是占位符，请填入真实 QQ 邮箱授权码" >&2
+  exit 1
+fi
+
+# 2. 主脚本
+install -m 755 "$SCRIPT_DIR/health-monitor.sh" /usr/local/bin/health-monitor.sh
+install -m 755 "$SCRIPT_DIR/health-send-mail.py" /usr/local/bin/health-send-mail.py
+
+# 3. 状态目录
+mkdir -p /var/lib/health-monitor /var/log
+
+# 4. cron (每 5 分钟，错开 :03 避开整点)
+cat > /etc/cron.d/health-monitor <<'EOF'
+# 健康监控：每 5 分钟（错开 :03）检测 /health + /api/navigation/config
+# 失败连续 2 次才告警，同一故障 30 分钟内只发一次
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+MAILTO=""
+
+3,8,13,18,23,28,33,38,43,48,53,58 * * * * root /usr/local/bin/health-monitor.sh >> /var/log/health-monitor.log 2>&1
+EOF
+chmod 644 /etc/cron.d/health-monitor
+
+# 5. 立即跑一次 dry-run（用真实凭证）验证通路
+echo ""
+echo "=== 安装完成，立即跑一次 smoke test ==="
+bash /usr/local/bin/health-monitor.sh
+echo ""
+echo "退出码: $?"
+echo "日志:   tail -20 /var/log/health-monitor.log"
+echo ""
+echo "查看 cron:   cat /etc/cron.d/health-monitor"
+echo "手动触发:   bash /usr/local/bin/health-monitor.sh"
+echo "测试告警:   pm2 stop server && bash /usr/local/bin/health-monitor.sh"