audio_codebuddy/server/debug-parse12.js

const fs = require('fs');
const content = fs.readFileSync('temp/rich-outline-fail_57_1779015509994.txt', 'utf-8');
const marker = '========== AI 原始响应 ';
const markerIdx = content.indexOf(marker);
const headerEndIdx = content.indexOf(' ==========', markerIdx);
const aiStart = content.indexOf('\n', headerEndIdx) + 1;
const aiEnd = content.indexOf('\n\n', aiStart);
const aiResponse = content.substring(aiStart, aiEnd);

console.log('String length (UTF-16 code units):', aiResponse.length);

// The issue: JSON.parse uses UTF-16 code unit positions, but the JSON is actually UTF-8 encoded
// when transmitted over the network. The LLM returns UTF-8 bytes, but in Node.js when we get
// the response as a string, it's decoded to UTF-16.

// Wait - actually JSON.parse should work correctly with UTF-16 strings in JavaScript.
// Let me check if there's actually a malformed character somewhere.

// Try using a simple character-by-character JSON parser approach to find the issue
const pos = 1624;
console.log('\nAnalyzing around position', pos);

// Get the actual bytes via TextEncoder
const encoder = new TextEncoder();
const decoder = new TextDecoder('utf-8');

// Let's check the raw bytes of the string
const uint8Array = encoder.encode(aiResponse);
console.log('Encoded bytes length:', uint8Array.length);

// Check bytes around position 1624 (in UTF-16 string)
console.log('\nBytes around UTF-16 position', pos, ':');
for (let i = Math.max(0, pos - 5); i < Math.min(uint8Array.length, pos + 10); i++) {
  console.log('  byte', i, ':', uint8Array[i], '(char:', String.fromCharCode(uint8Array[i]), ')');
}

// Now let's try to find where the actual byte-level position differs from UTF-16 position
// by counting UTF-8 multi-byte characters before position 1624

let utf8Offset = 0;
let utf16Offset = 0;
let lastAsciiBefore1624 = -1;
let lastAsciiUtf8BytePos = -1;

for (let i = 0; i < aiResponse.length && utf16Offset < 1624; i++) {
  const code = aiResponse.charCodeAt(i);
  if (code <= 0x7F) {
    // ASCII - 1 byte in UTF-8, 1 code unit in UTF-16
    if (utf16Offset === 1623) {
      lastAsciiBefore1624 = code;
      lastAsciiUtf8BytePos = utf8Offset;
    }
    utf8Offset += 1;
    utf16Offset += 1;
  } else if (code <= 0x7FF) {
    // 2-byte UTF-8
    utf8Offset += 2;
    utf16Offset += 1;
  } else if (code <= 0xFFFF) {
    // 3-byte UTF-8 (most CJK characters)
    utf8Offset += 3;
    utf16Offset += 1;
  } else {
    // 4-byte UTF-8 (emoji, etc)
    utf8Offset += 4;
    utf16Offset += 2;
  }
}

console.log('\nAt UTF-16 position 1624:');
console.log('  Corresponding UTF-8 byte position would be around:', utf8Offset);
console.log('  Last ASCII char (at UTF-16 1623):', lastAsciiBefore1624, '(code:', lastAsciiBefore1624, ')');

// Now let's check what JSON.parse says the error position actually means
// In JSON parse errors, "position" is the character offset in the string
// Let's see if position 1624 in the string is actually where the issue is

console.log('\n=== Direct parse attempt ===');
try {
  JSON.parse(aiResponse);
} catch(e) {
  console.log('Error:', e.message);
  const match = e.message.match(/position (\d+)/);
  if (match) {
    const errorPos = parseInt(match[1]);
    console.log('Error position:', errorPos);
    console.log('Character at error position:', JSON.stringify(aiResponse[errorPos]));
    console.log('Context:', JSON.stringify(aiResponse.substring(Math.max(0, errorPos-50), errorPos+50)));

    // Check if char at errorPos is the start of a multi-byte sequence
    const code = aiResponse.charCodeAt(errorPos);
    console.log('Code point at errorPos:', code, 'hex:', code.toString(16));
  }
}