audio_codebuddy/server/debug-parse4.js

// Verify: the truncation is due to max_tokens limit
// Check roughly how many tokens 9867 Chinese chars would be
const content = `每一个200字，都是对一个日常思维盲区的温柔颠覆——让你在下一个转角遇见不同的自己`;
// Rough estimate: Chinese chars ~= tokens when using BPE-like tokenizers
// For MiniMax M2.7 with 128k context, 9867 chars would be roughly 5000+ tokens

// Check: does the response have roughly the right size for 4096 tokens?
const fs = require('fs');
const fileContent = fs.readFileSync('temp/rich-outline-fail_57_1779014018760.txt', 'utf-8');
const marker = '========== AI 原始响应 ';
const markerIdx = fileContent.indexOf(marker);
const headerEndIdx = fileContent.indexOf(' ==========', markerIdx);
const aiStart = fileContent.indexOf('\n', headerEndIdx) + 1;
const aiEnd = fileContent.indexOf('\n\n', aiStart);
const aiResponse = fileContent.substring(aiStart, aiEnd);

console.log('AI response length (chars):', aiResponse.length);

// Chinese chars ratio
const chineseChars = (aiResponse.match(/[一-鿿]/g) || []).length;
console.log('Chinese chars:', chineseChars);
console.log('Non-Chinese chars:', aiResponse.length - chineseChars);

// Rough token estimate: Chinese chars ~= 1.5-2 tokens each for MiniMax
// Plus English/punctuation ~= 0.5 tokens each
const estimatedTokens = chineseChars * 1.5 + (aiResponse.length - chineseChars) * 0.5;
console.log('Estimated tokens (rough):', Math.round(estimatedTokens));

// The model has maxTokens=4096, so if estimatedTokens > 4096, truncation happened
console.log('\nmaxTokens setting: 4096');
console.log('Would be truncated:', estimatedTokens > 4096 ? 'YES' : 'NO');