| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165 |
- PATH = '/data/ai/audio/server/dist/services/llm/index.js'
- with open(PATH, 'r', encoding='utf-8') as f:
- src = f.read()
- # 替换整个 invokeWithRetry 函数(旧的 + trySwitchModel 调用 + tried.has(resolveProviderKey(next)) 检查)
- # 找到从 "async function invokeWithRetry" 到下一个 "async function callLLM" 的整段
- OLD = """async function invokeWithRetry(fn, modelId, callType = 'llm_chat') {
- const tried = new Map();
- let cur = modelId, lastErr = null;
- for (let round = 0; round < exports.HA_RETRY_CONFIG.maxFallbackRounds; round++) {
- const provider = resolveProviderKey(cur);
- // tried check is now done in resolveNodeForModel below; keep this as a guard
- if (tried.has(provider))
- continue;
- let providerError = null;
- for (let attempt = 0; attempt < exports.HA_RETRY_CONFIG.maxRetriesPerProvider; attempt++) {
- // 用 AsyncLocalStorage 传递 tried set, 让 findVendorForModel 跳过已试 node
- const node = resolveNodeForModel(cur, tried);
- if (!node) break;
- const fp = node.provider.vendor + ':' + (node.provider._apiKey || '').slice(-12);
- tried.set(fp, { model: cur });
- try {
- const result = await _runWithTriedNodes(new Set(tried.keys()), () => fn(cur));
- return result;
- }
- catch (error) {
- const em = error?.message || String(error);
- lastErr = error;
- providerError = error;
- if (isBusinessError(em) && !isNetworkError(em)) {
- console.error(`[LLM-HA] ${provider}/${cur} 业务错误,立即抛出: ${em.substring(0, 100)}`);
- throw error;
- }
- if (isNetworkError(em)) {
- console.warn(`[LLM-HA] ${provider}/${cur} 网络错误,立即切换: ${em.substring(0, 100)}`);
- break;
- }
- if (attempt < exports.HA_RETRY_CONFIG.maxRetriesPerProvider - 1) {
- const delay = backoffWithJitter(attempt);
- console.log(`[LLM-HA] ${provider}/${cur} 第${attempt + 1}次失败: ${em.substring(0, 100)}, ${delay}ms后重试`);
- await new Promise(r => setTimeout(r, delay));
- }
- }
- }
- const em = providerError?.message || 'unknown';
- const et = isNetworkError(em) ? 'network' : (isBusinessError(em) ? 'business' : 'unknown');
- tried.set(provider, { model: cur, error: em, attempts: exports.HA_RETRY_CONFIG.maxRetriesPerProvider, errorType: et });
- console.log(`[LLM-HA] ${provider}/${cur} 失败 (${et}),尝试切换供应商`);
- const next = trySwitchModel(cur, providerError);
- if (!next)
- break;
- if (tried.has(resolveProviderKey(next))) {
- console.warn(`[LLM-HA] 下一供应商已尝试过,停止 fallback`);
- break;
- }
- cur = next;
- console.log(`[LLM-HA] 切换到 ${resolveProviderKey(cur)}/${cur}`);
- }
- throw new AllProvidersFailedError(Array.from(tried.entries()).map(([provider, info]) => ({ provider, ...info })));
- }"""
- NEW = """async function invokeWithRetry(fn, modelId, callType = 'llm_chat') {
- const tried = new Map();
- let cur = modelId, lastErr = null;
- for (let round = 0; round < exports.HA_RETRY_CONFIG.maxFallbackRounds; round++) {
- // 关键: 选下一个未 tried 的 node (而不是用 resolveProviderKey 取第一个匹配)
- const node = resolveNodeForModel(cur, tried);
- if (!node) {
- console.warn(`[LLM-HA] 已尝试所有 node,停止 fallback`);
- break;
- }
- const fp = node.provider.vendor + ':' + (node.provider._apiKey || '').slice(-12);
- tried.set(fp, { model: cur });
- let providerError = null;
- for (let attempt = 0; attempt < exports.HA_RETRY_CONFIG.maxRetriesPerProvider; attempt++) {
- try {
- const result = await _runWithTriedNodes(new Set(tried.keys()), () => fn(cur));
- return result;
- }
- catch (error) {
- const em = error?.message || String(error);
- lastErr = error;
- providerError = error;
- if (isBusinessError(em) && !isNetworkError(em)) {
- console.error(`[LLM-HA] ${fp}/${cur} 业务错误,立即抛出: ${em.substring(0, 100)}`);
- throw error;
- }
- if (isNetworkError(em)) {
- console.warn(`[LLM-HA] ${fp}/${cur} 网络错误,立即切换: ${em.substring(0, 100)}`);
- break;
- }
- if (attempt < exports.HA_RETRY_CONFIG.maxRetriesPerProvider - 1) {
- const delay = backoffWithJitter(attempt);
- console.log(`[LLM-HA] ${fp}/${cur} 第${attempt + 1}次失败: ${em.substring(0, 100)}, ${delay}ms后重试`);
- await new Promise(r => setTimeout(r, delay));
- }
- }
- }
- // 当前 node 失败, 标记其 info
- const em = providerError?.message || 'unknown';
- const et = isNetworkError(em) ? 'network' : (isBusinessError(em) ? 'business' : 'unknown');
- tried.set(fp, { model: cur, error: em, attempts: exports.HA_RETRY_CONFIG.maxRetriesPerProvider, errorType: et });
- console.log(`[LLM-HA] ${fp}/${cur} 失败 (${et}),尝试切换供应商`);
- // 关键: 自己挑下一个未 tried 的 node, 复用 switchToNextVendorModel 的逻辑但跳过 tried
- const next = pickNextModelAfterFailure(cur, tried, node);
- if (!next) {
- console.warn(`[LLM-HA] 找不到下一个未尝试的供应商`);
- break;
- }
- cur = next;
- console.log(`[LLM-HA] 切换到 ${next}`);
- }
- throw new AllProvidersFailedError(Array.from(tried.entries()).map(([provider, info]) => ({ provider, ...info })));
- }
- /** 从当前 node 找下一个未 tried 的 node, 返回 next modelId */
- function pickNextModelAfterFailure(currentModelId, tried, currentNode) {
- const registry = (0, provider_registry_1.getLlmRegistry)();
- const available = registry.listAvailable();
- if (available.length === 0) return null;
- // 关键: 完全按 node 顺序, 跳过 tried 节点
- const startIndex = currentNode ? (available.indexOf(currentNode) + 1) % available.length : 0;
- for (let i = 0; i < available.length; i++) {
- const idx = (startIndex + i) % available.length;
- const node = available[idx];
- if (node === currentNode) continue;
- const fp = node.provider.vendor + ':' + (node.provider._apiKey || '').slice(-12);
- if (tried.has(fp)) continue;
- // 1. 同名 model
- if (node.provider.hasModel(currentModelId)) {
- console.log(`[LLM] 同模型切换到供应商 ${node.provider.displayName},模型 ${currentModelId}`);
- return currentModelId;
- }
- // 2. 别名 model
- for (const modelId of node.provider.textModels) {
- const cfg = node.provider.getModelConfig(modelId);
- if (cfg?.canonicalModel === currentModelId) {
- console.log(`[LLM] 别名模型切换到供应商 ${node.provider.displayName},模型 ${modelId}`);
- return modelId;
- }
- }
- // 3. 第一个 text model
- if (node.provider.textModels.length > 0) {
- const nextId = node.provider.textModels[0];
- console.log(`[LLM] 切换到供应商 ${node.provider.displayName},模型 ${nextId}`);
- return nextId;
- }
- }
- return null;
- }"""
- if OLD not in src:
- print("ERROR: OLD invokeWithRetry not found exactly")
- # 看一下当前 invokeWithRetry 实际
- import re
- m = re.search(r'async function invokeWithRetry.*?throw new AllProvidersFailedError.*?;', src, re.DOTALL)
- if m:
- print("CURRENT (first 500 chars):")
- print(m.group(0)[:500])
- raise SystemExit(1)
- src = src.replace(OLD, NEW)
- with open(PATH, 'w', encoding='utf-8') as f:
- f.write(src)
- print("OK: invokeWithRetry v3 patched (skip tried nodes via pickNextModelAfterFailure)")
|