# All files / modules/book-generator/utils content-cleaner.ts

0% Statements 0/106
100% Branches 1/1
100% Functions 1/1
0% Lines 0/106

Press n or j to go to the next uncovered block, b, p or k for the previous block.

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    132
    133
    134
    135
    136
    137
    138
    139
    140
    141
    142
    143
    144
    145
    146
    147
    148
    149
    150
    151
    152
    153
    154
    155
    156
    157
    158
    159
    160
    161
    162
    163
    164
    165
    166
    167
    168
    169

     /**
     * 内容清洗器
     *
     * 解决 issue #5（思考文本清理靠正则不可靠）和 issue #4（内容截断太粗暴）
     * 提供健壮的 LLM 输出清洗和智能截断功能。
     */

    /**
     * 清理 LLM 输出中的思考标签和无关文本。
     * 多模式联合匹配，覆盖不同模型的思考输出变体。
     */
    export function cleanThinkingText(content: string): string {
      if (!content) return '';

      let cleaned = content;

      // 1. 移除 HTML/XML 标签（但保留 markdown 格式）
      cleaned = cleaned.replace(/<(?!\/?(h[1-6]|p|ul|ol|li|code|pre|strong|em|blockquote|a|img|br|hr|table|thead|tbody|tr|td|th)\b)[^>]*>[\s\S]*?<\/[^>]*>/gi, '');
      // 自闭合标签
      cleaned = cleaned.replace(/<(?!\/?(h[1-6]|p|ul|ol|li|code|pre|strong|em|blockquote|a|img|br|hr|table|thead|tbody|tr|td|th)\b)[^>]*\/>/gi, '');

      // 2. 移除常见的思考块标记
      const thinkBlockPatterns = [
        /<think>[\s\S]*?<\/think>/gi,
        /<thinking>[\s\S]*?<\/thinking>/gi,
        /<thought>[\s\S]*?<\/thought>/gi,
        /\[think\][\s\S]*?\[\/think\]/gi,
        /\[thinking\][\s\S]*?\[\/thinking\]/gi,
        /\[inkleb\]:[\s\S]*?\[\/inkleb\]/gi,
        /<antthinking>[\s\S]*?<\/antthinking>/gi,
        /<reasoning>[\s\S]*?<\/reasoning>/gi,
      ];

      for (const pattern of thinkBlockPatterns) {
        cleaned = cleaned.replace(pattern, '');
      }

      // 3. 移除未闭合的思考块开始标记及其后所有内容
      cleaned = cleaned.replace(/<think>[\s\S]*/gi, '');
      cleaned = cleaned.replace(/<thinking>[\s\S]*/gi, '');
      cleaned = cleaned.replace(/<thought>[\s\S]*/gi, '');

      // 4. 移除孤立的闭合标签
      cleaned = cleaned.replace(/<\/think>/gi, '');
      cleaned = cleaned.replace(/<\/thinking>/gi, '');
      cleaned = cleaned.replace(/<\/thought>/gi, '');

      // 5. 如果内容以 LLM 常见的思考/确认性语言开头，尝试提取正文
      // 匹配模式：以"好的"、"我现在"、"让我"等开头，直到第一个 # 标题
      const thinkingPrefixPatterns = [
        /^(好的|好的[，,。])[^\n]*?(?=\n#)/s,
        /^(我现在|现在我来|我来|让我)[^\n]*?开始撰写[^\n]*?(?=\n#)/s,
        /^(让我|我来|我现在)[^\n]*?(?=\n#)/s,
        /^(收到|明白|了解|懂了)[^\n]*?(?=\n#)/s,
        /^(以下是|下面是)[^\n]*?(?=\n#)/s,
        /^(根据|按照)[^\n]*?(?=\n#)/s,
      ];

      for (const pattern of thinkingPrefixPatterns) {
        const before = cleaned;
        cleaned = cleaned.replace(pattern, '');
        if (cleaned !== before) break; // 只执行第一个匹配到的
      }

      // 6. 如果正文前面有超过 150 字符的非标题内容，可能是思考过程
      const titleMatch = cleaned.match(/\n(#{1,4}\s+.+)/);
      if (titleMatch && titleMatch.index !== undefined && titleMatch.index > 150) {
        cleaned = cleaned.substring(titleMatch.index).trim();
      }

      // 7. 压缩多余空行
      cleaned = cleaned.replace(/\n{3,}/g, '\n\n');

      // 8. 去除首尾空白
      cleaned = cleaned.trim();

      return cleaned;
    }

    /**
     * 按段落/句子边界智能截断，避免拦腰截断。
     *
     * @param content 原始内容
     * @param maxChars 最大字符数
     * @param strategy 截断策略：'paragraph' 按段落边界，'sentence' 按句子边界
     */
    export function truncateAtBoundary(
      content: string,
      maxChars: number,
      strategy: 'paragraph' | 'sentence' = 'paragraph'
    ): string {
      if (!content || content.length <= maxChars) return content;

      if (strategy === 'paragraph') {
        // 按段落截断：找到最后一个完整的段落
        const paragraphs = content.split(/\n\n+/);

        let result = '';
        for (const para of paragraphs) {
          if ((result + para).length > maxChars) {
            // 当前段落放不下
            if (!result) {
              // 第一个段落就超了，降级到按句子截断
              return truncateAtBoundary(content, maxChars, 'sentence');
            }
            break;
          }
          result += (result ? '\n\n' : '') + para;
        }

        if (result) return result;

        // fallback: 句子级别
        return truncateAtBoundary(content, maxChars, 'sentence');
      }

      // 按句子截断
      const sentences = content.split(/(?<=[。！？.!?\n])\s*/);
      let result = '';

      for (const sent of sentences) {
        if ((result + sent).length > maxChars) {
          if (!result) {
            // 第一个句子就超了，只能硬截但保留完整词
            const hardCut = content.substring(0, maxChars);
            // 找最后一个空格或标点
            const lastBreak = Math.max(
              hardCut.lastIndexOf('。'),
              hardCut.lastIndexOf('！'),
              hardCut.lastIndexOf('？'),
              hardCut.lastIndexOf('.'),
              hardCut.lastIndexOf('\n'),
              hardCut.lastIndexOf(' '),
            );
            return lastBreak > maxChars * 0.7
              ? hardCut.substring(0, lastBreak + 1)
              : hardCut;
          }
          break;
        }
        result += sent;
      }

      return result.trim();
    }

    /**
     * 综合清洗 + 截断的便捷方法。
     * 先清洗思考文本，再按预算截断。
     */
    export function cleanAndTruncate(
      content: string,
      maxChars: number,
      strategy: 'paragraph' | 'sentence' = 'paragraph'
    ): { cleaned: string; wasTruncated: boolean; originalLength: number } {
      const cleaned = cleanThinkingText(content);
      const originalLength = cleaned.length;

      if (cleaned.length <= maxChars) {
        return { cleaned, wasTruncated: false, originalLength };
      }

      return {
        cleaned: truncateAtBoundary(cleaned, maxChars, strategy),
        wasTruncated: true,
        originalLength,
      };
    }