Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | /** * 内容清洗器 * * 解决 issue #5（思考文本清理靠正则不可靠）和 issue #4（内容截断太粗暴） * 提供健壮的 LLM 输出清洗和智能截断功能。 */ /** * 清理 LLM 输出中的思考标签和无关文本。 * 多模式联合匹配，覆盖不同模型的思考输出变体。 */ export function cleanThinkingText(content: string): string { if (!content) return ''; let cleaned = content; // 1. 移除 HTML/XML 标签（但保留 markdown 格式） cleaned = cleaned.replace(/<(?!\/?(h[1-6]|p|ul|ol|li|code|pre|strong|em|blockquote|a|img|br|hr|table|thead|tbody|tr|td|th)\b)[^>]*>[\s\S]*?<\/[^>]*>/gi, ''); // 自闭合标签 cleaned = cleaned.replace(/<(?!\/?(h[1-6]|p|ul|ol|li|code|pre|strong|em|blockquote|a|img|br|hr|table|thead|tbody|tr|td|th)\b)[^>]*\/>/gi, ''); // 2. 移除常见的思考块标记 const thinkBlockPatterns = [ /<think>[\s\S]*?<\/think>/gi, /<thinking>[\s\S]*?<\/thinking>/gi, /<thought>[\s\S]*?<\/thought>/gi, /\[think\][\s\S]*?\[\/think\]/gi, /\[thinking\][\s\S]*?\[\/thinking\]/gi, /\[inkleb\]:[\s\S]*?\[\/inkleb\]/gi, /<antthinking>[\s\S]*?<\/antthinking>/gi, /<reasoning>[\s\S]*?<\/reasoning>/gi, ]; for (const pattern of thinkBlockPatterns) { cleaned = cleaned.replace(pattern, ''); } // 3. 移除未闭合的思考块开始标记及其后所有内容 cleaned = cleaned.replace(/<think>[\s\S]*/gi, ''); cleaned = cleaned.replace(/<thinking>[\s\S]*/gi, ''); cleaned = cleaned.replace(/<thought>[\s\S]*/gi, ''); // 4. 移除孤立的闭合标签 cleaned = cleaned.replace(/<\/think>/gi, ''); cleaned = cleaned.replace(/<\/thinking>/gi, ''); cleaned = cleaned.replace(/<\/thought>/gi, ''); // 5. 如果内容以 LLM 常见的思考/确认性语言开头，尝试提取正文 // 匹配模式：以"好的"、"我现在"、"让我"等开头，直到第一个 # 标题 const thinkingPrefixPatterns = [ /^(好的|好的[，,。])[^\n]*?(?=\n#)/s, /^(我现在|现在我来|我来|让我)[^\n]*?开始撰写[^\n]*?(?=\n#)/s, /^(让我|我来|我现在)[^\n]*?(?=\n#)/s, /^(收到|明白|了解|懂了)[^\n]*?(?=\n#)/s, /^(以下是|下面是)[^\n]*?(?=\n#)/s, /^(根据|按照)[^\n]*?(?=\n#)/s, ]; for (const pattern of thinkingPrefixPatterns) { const before = cleaned; cleaned = cleaned.replace(pattern, ''); if (cleaned !== before) break; // 只执行第一个匹配到的 } // 6. 如果正文前面有超过 150 字符的非标题内容，可能是思考过程 const titleMatch = cleaned.match(/\n(#{1,4}\s+.+)/); if (titleMatch && titleMatch.index !== undefined && titleMatch.index > 150) { cleaned = cleaned.substring(titleMatch.index).trim(); } // 7. 压缩多余空行 cleaned = cleaned.replace(/\n{3,}/g, '\n\n'); // 8. 去除首尾空白 cleaned = cleaned.trim(); return cleaned; } /** * 按段落/句子边界智能截断，避免拦腰截断。 * * @param content 原始内容 * @param maxChars 最大字符数 * @param strategy 截断策略：'paragraph' 按段落边界，'sentence' 按句子边界 */ export function truncateAtBoundary( content: string, maxChars: number, strategy: 'paragraph' | 'sentence' = 'paragraph' ): string { if (!content || content.length <= maxChars) return content; if (strategy === 'paragraph') { // 按段落截断：找到最后一个完整的段落 const paragraphs = content.split(/\n\n+/); let result = ''; for (const para of paragraphs) { if ((result + para).length > maxChars) { // 当前段落放不下 if (!result) { // 第一个段落就超了，降级到按句子截断 return truncateAtBoundary(content, maxChars, 'sentence'); } break; } result += (result ? '\n\n' : '') + para; } if (result) return result; // fallback: 句子级别 return truncateAtBoundary(content, maxChars, 'sentence'); } // 按句子截断 const sentences = content.split(/(?<=[。！？.!?\n])\s*/); let result = ''; for (const sent of sentences) { if ((result + sent).length > maxChars) { if (!result) { // 第一个句子就超了，只能硬截但保留完整词 const hardCut = content.substring(0, maxChars); // 找最后一个空格或标点 const lastBreak = Math.max( hardCut.lastIndexOf('。'), hardCut.lastIndexOf('！'), hardCut.lastIndexOf('？'), hardCut.lastIndexOf('.'), hardCut.lastIndexOf('\n'), hardCut.lastIndexOf(' '), ); return lastBreak > maxChars * 0.7 ? hardCut.substring(0, lastBreak + 1) : hardCut; } break; } result += sent; } return result.trim(); } /** * 综合清洗 + 截断的便捷方法。 * 先清洗思考文本，再按预算截断。 */ export function cleanAndTruncate( content: string, maxChars: number, strategy: 'paragraph' | 'sentence' = 'paragraph' ): { cleaned: string; wasTruncated: boolean; originalLength: number } { const cleaned = cleanThinkingText(content); const originalLength = cleaned.length; if (cleaned.length <= maxChars) { return { cleaned, wasTruncated: false, originalLength }; } return { cleaned: truncateAtBoundary(cleaned, maxChars, strategy), wasTruncated: true, originalLength, }; } |