import path from 'path'; import fs from 'fs'; import { v4 as uuidv4 } from 'uuid'; import { config } from '../../config'; import { prisma } from '../../models'; import { VoiceParams, Voice } from '../../types'; import { getMiniMaxVoice } from './minimax.provider'; import { AudioMerger } from './audio-merger'; import { aiSummaryService } from './ai-summary.service'; import { storageService } from '../../services/storage.service'; import { getTtsRegistry, getAvailableTtsProvider, startTtsHealthCheck } from './provider.registry'; import { ITtsProvider } from './provider.interface'; import { CircuitBreakerOpenError } from '../../common/circuit-breaker'; import { ProviderNode } from '../../common/provider-registry'; import axios from 'axios'; // 日志文件路径 const LOG_FILE = path.join(process.cwd(), 'tts-debug.log'); function logToFile(msg: string) { const timestamp = new Date().toISOString(); fs.appendFileSync(LOG_FILE, `[${timestamp}] ${msg}\n`); } // 可用音色列表(CosyVoice v3-flash - 67个音色) // 数据来源:阿里云百炼官方文档 2026-05 export const VOICES: Voice[] = [ // 标杆音色 { id: 'longanyang', name: '龙安洋', gender: 'male', description: '阳光大男孩 20~30岁' }, { id: 'longanhuan', name: '龙安欢', gender: 'female', description: '欢脱元气女 20~30岁' }, { id: 'longhuhu_v3', name: '龙呼呼', gender: 'female', description: '天真烂漫女童 6~10岁' }, // 有声书 - 男声 { id: 'longsanshu_v3', name: '龙三叔', gender: 'male', description: '沉稳质感男 25~45岁' }, { id: 'longxiu_v3', name: '龙修', gender: 'male', description: '博才说书男 25~35岁' }, { id: 'longnan_v3', name: '龙楠', gender: 'male', description: '睿智青年男 25~30岁' }, { id: 'longyichen_v3', name: '龙逸尘', gender: 'male', description: '洒脱活力男 20~30岁' }, { id: 'longlaobo_v3', name: '龙老伯', gender: 'male', description: '沧桑岁月爷 60岁以上' }, // 有声书 - 女声 { id: 'longmiao_v3', name: '龙妙', gender: 'female', description: '抑扬顿挫女 25~30岁' }, { id: 'longyuan_v3', name: '龙媛', gender: 'female', description: '温暖治愈女 35~40岁' }, { id: 'longyue_v3', name: '龙悦', gender: 'female', description: '温暖磁性女 30~35岁' }, { id: 'longwanjun_v3', name: '龙婉君', gender: 'female', description: '细腻柔声女 20~30岁' }, { id: 'longlaoyi_v3', name: '龙老姨', gender: 'female', description: '烟火从容阿姨 60岁以上' }, // 客服/电话 { id: 'longyingxun_v3', name: '龙应询', gender: 'male', description: '年轻青涩男 20~25岁' }, { id: 'longyingjing_v3', name: '龙应静', gender: 'female', description: '低调冷静女 25~35岁' }, { id: 'longyingling_v3', name: '龙应聆', gender: 'female', description: '温和共情女 25~30岁' }, { id: 'longyingtao_v3', name: '龙应桃', gender: 'female', description: '温柔淡定女 25~30岁' }, { id: 'longyingmu_v3', name: '龙应沐', gender: 'female', description: '优雅知性女 25~30岁' }, { id: 'longyingxiao_v3', name: '龙应笑', gender: 'female', description: '清甜推销女 20~25岁' }, // 语音助手 { id: 'longxiaochun_v3', name: '龙小淳', gender: 'female', description: '知性积极女 25~30岁' }, { id: 'longxiaoxia_v3', name: '龙小夏', gender: 'female', description: '沉稳权威女 25~30岁' }, { id: 'longanyun_v3', name: '龙安昀', gender: 'male', description: '居家暖男 30~35岁' }, { id: 'longanwen_v3', name: '龙安温', gender: 'female', description: '优雅知性女 25~35岁' }, { id: 'longanli_v3', name: '龙安莉', gender: 'female', description: '利落从容女 25~35岁' }, { id: 'longanlang_v3', name: '龙安朗', gender: 'male', description: '清爽利落男 20~25岁' }, { id: 'longyumi_v3', name: 'YUMI', gender: 'female', description: '正经青年女 20~25岁' }, // 社交陪伴 { id: 'longanqin_v3', name: '龙安亲', gender: 'female', description: '亲和活泼女 20~25岁' }, { id: 'longanya_v3', name: '龙安雅', gender: 'female', description: '高雅气质女 25~35岁' }, { id: 'longanzhi_v3', name: '龙安智', gender: 'male', description: '睿智轻熟男 25~35岁' }, { id: 'longanling_v3', name: '龙安灵', gender: 'female', description: '思维灵动女 20~30岁' }, { id: 'longantai_v3', name: '龙安台', gender: 'female', description: '嗲甜台湾女 20~25岁' }, { id: 'longhua_v3', name: '龙华', gender: 'female', description: '元气甜美女 20~25岁' }, { id: 'longcheng_v3', name: '龙橙', gender: 'male', description: '智慧青年男 20~25岁' }, { id: 'longze_v3', name: '龙泽', gender: 'male', description: '温暖元气男 25~30岁' }, { id: 'longzhe_v3', name: '龙哲', gender: 'male', description: '呆板大暖男 25~30岁' }, { id: 'longyan_v3', name: '龙颜', gender: 'female', description: '温暖春风女 30~35岁' }, { id: 'longxing_v3', name: '龙星', gender: 'female', description: '温婉邻家女 20~25岁' }, { id: 'longtian_v3', name: '龙天', gender: 'male', description: '磁性理智男 30~35岁' }, { id: 'longwan_v3', name: '龙婉', gender: 'female', description: '细腻柔声女 20~30岁' }, { id: 'longqiang_v3', name: '龙嫱', gender: 'female', description: '浪漫风情女 30~35岁' }, { id: 'longfeifei_v3', name: '龙菲菲', gender: 'female', description: '甜美娇气女 20~25岁' }, { id: 'longhao_v3', name: '龙浩', gender: 'male', description: '多情忧郁男 30~35岁' }, { id: 'longanrou_v3', name: '龙安柔', gender: 'female', description: '温柔闺蜜女 20~35岁' }, { id: 'longhan_v3', name: '龙寒', gender: 'male', description: '温暖痴情男 30~35岁' }, // 童声 { id: 'longpaopao_v3', name: '龙泡泡', gender: 'female', description: '飞天泡泡音 6~15岁' }, { id: 'longjielidou_v3', name: '龙杰力豆', gender: 'male', description: '阳光顽皮男 10岁' }, { id: 'longxian_v3', name: '龙仙', gender: 'female', description: '豪放可爱女 12岁' }, { id: 'longling_v3', name: '龙铃', gender: 'female', description: '稚气呆板女 10岁' }, { id: 'longshanshan_v3', name: '龙闪闪', gender: 'female', description: '戏剧化童声 6~15岁' }, { id: 'longniuniu_v3', name: '龙牛牛', gender: 'male', description: '阳光男童声 6~15岁' }, // 方言 { id: 'longjiaxin_v3', name: '龙嘉欣', gender: 'female', description: '优雅粤语女 30~35岁' }, { id: 'longjiayi_v3', name: '龙嘉怡', gender: 'female', description: '知性粤语女 25~30岁' }, { id: 'longanyue_v3', name: '龙安粤', gender: 'male', description: '欢脱粤语男 25~35岁' }, { id: 'longlaotie_v3', name: '龙老铁', gender: 'male', description: '东北直率男 25~30岁' }, { id: 'longshange_v3', name: '龙陕哥', gender: 'male', description: '陕北原味男 25~35岁' }, // 新闻/直播 { id: 'longshuo_v3', name: '龙硕', gender: 'male', description: '博才干练男 25~30岁' }, { id: 'longshu_v3', name: '龙书', gender: 'male', description: '沉稳青年男 20~25岁' }, { id: 'loongbella_v3', name: 'Bella3.0',gender: 'female', description: '精准干练女 25~30岁' }, { id: 'longanran_v3', name: '龙安燃', gender: 'female', description: '活泼质感女 30~40岁' }, { id: 'longanxuan_v3', name: '龙安宣', gender: 'female', description: '经典直播女 30~40岁' }, // 特色 { id: 'longjiqi_v3', name: '龙机器', gender: 'male', description: '呆萌机器人 20~30岁' }, { id: 'longhouge_v3', name: '龙猴哥', gender: 'male', description: '经典猴哥 20~25岁' }, { id: 'longdaiyu_v3', name: '龙黛玉', gender: 'female', description: '娇率才女音 15~25岁' }, { id: 'longfei_v3', name: '龙飞', gender: 'male', description: '热血磁性男 30~35岁' }, // 外语 { id: 'loongkyong_v3', name: '韩语女', gender: 'female', description: '韩语女声' }, { id: 'loongriko_v3', name: 'Riko霓虹', gender: 'female', description: '二次元日语女' }, { id: 'loongabby_v3', name: '美语女', gender: 'female', description: '美式英文女' }, { id: 'loongandy_v3', name: '美语男', gender: 'male', description: '美式英文男' }, ]; // CosyVoice v3-flash 音色ID直接用作API参数名,无需映射 export function getAliyunVoice(voiceId: string): string { return voiceId; } // 获取或创建默认"我的音频"书籍 async function getOrCreateDefaultBook(userId: string): Promise { const userIdNum = parseInt(userId); const bookTitle = '我的音频'; // 先尝试查找现有默认书籍 let book = await prisma.book.findFirst({ where: { userId: userIdNum, title: bookTitle, }, }); if (book) { return book.id; } // 不存在则创建 book = await prisma.book.create({ data: { title: bookTitle, description: '我的语音合成音频收藏', userId: userIdNum, genStage: 'content_completed', bookScale: 'short', totalChapters: 0, estimatedWords: 0, }, }); console.log(`📚 创建默认书籍: bookId=${book.id}, userId=${userIdNum}`); return book.id; } // 判断是否使用长文本模式(>5000字符且启用realtime) // 注意:WebSocket realtime 模式需要特殊的API权限和配置,如果连接失败会导致生成失败。 // 暂时强制禁用,使用 HTTP 分段模式 export function shouldUseLongText(text: string): boolean { return false; // 强制返回 false,禁用 realtime 模式 } // 文本分段 - 统一 1000 字符上限,优先在标点处断开 // 策略:按自然句子累加,接近上限时在最后一个标点处截断,避免在字中间硬切 // 好处:TTS 在句末韵律自然,拼接后听感流畅 // // 1000 字的依据(主流 TTS 接口上限,大多以千为单位): // MiniMax 同步: 1万 | 阿里云 CosyVoice: 2万 | 讯飞 流式: ~4000字 // 百度 短文本: 5120字 | OpenAI TTS: 4096字 | Google Cloud: 5000字节 // ElevenLabs: 3000~5000字 | Azure: ~3000~5000字 // 不支持1000的接口(应走异步长文本API,而非短文本接口): // 火山引擎 非流式: 300字 | 腾讯云 基础: 150字 | 百度 短文本: 60字 const SEGMENT_MAX_LENGTH = 1000; // 句末标点(中文 + 英文),用于寻找断点 const SENTENCE_END_RE = /[。!?;\n.!?;]/; export function splitText(text: string, maxLength: number = SEGMENT_MAX_LENGTH): string[] { const segments: string[] = []; // 清理文本 const cleanText = text.replace(/\r/g, ''); // 第一步:按换行拆成段落,再按句末标点拆成句子 // 这样保留了段落边界,又不会在一个句子中间断开 const rawSentences: string[] = []; const paragraphs = cleanText.split(/\n+/); for (const para of paragraphs) { if (para.length === 0) continue; // 按句末标点拆分,保留标点 const sentences = para.match(/[^。!?;.!?;]+[。!?;.!?;]?/g) || [para]; for (const s of sentences) { if (s.trim().length > 0) { rawSentences.push(s); } } } // 第二步:将句子累加成段,接近上限时在最后一个标点处截断 let current = ''; for (const sentence of rawSentences) { // 当前段 + 这句还能放下 if ((current + sentence).length <= maxLength) { current += sentence; continue; } // 放不下了,先把当前段输出 if (current) { segments.push(current); current = ''; } // 如果这句话本身就超长,需要在句内找标点断点 if (sentence.length > maxLength) { let remaining = sentence; while (remaining.length > 0) { if (remaining.length <= maxLength) { current = remaining; break; } // 在上限附近向前找最后一个标点作为断点 let breakPos = -1; for (let i = maxLength; i > Math.max(0, maxLength - 100); i--) { if (SENTENCE_END_RE.test(remaining[i])) { breakPos = i + 1; // 标点后一位 break; } } // 找不到标点,尝试逗号/顿号等次级标点 if (breakPos === -1) { for (let i = maxLength; i > Math.max(0, maxLength - 100); i--) { if (/[,、,]/.test(remaining[i])) { breakPos = i + 1; break; } } } // 实在找不到标点,只能在上限处硬切(最后手段) if (breakPos === -1) { breakPos = maxLength; } segments.push(remaining.slice(0, breakPos)); remaining = remaining.slice(breakPos); } } else { current = sentence; } } if (current) segments.push(current); return segments; } // ============ TTS Provider 调度(使用注册表 + 熔断器)============ // // 优先级顺序由 Registry 中的 priority 决定: // 1. MiniMax (priority=1) - 异步长文本,业界最长 // 2. 阿里云 Qwen-TTS (priority=2) - 同步快速 // 3. Mock (priority=100) - 降级兜底 // // 熔断器: 连续 3 次失败自动熔断,MiniMax 冷却 60s,阿里云 30s // // 分段策略: 统一 1000 字符/段,兼容所有 TTS 模型 // 各家 TTS API 限制参考(2026年5月): // ┌──────────────┬────────────────────┬───────────┬──────────────────────┐ // │ 服务商 │ 接口类型 │ 最大长度 │ 备注 │ // ├──────────────┼────────────────────┼───────────┼──────────────────────┤ // │ MiniMax │ 异步长文本 │ 1,000,000 │ 业界最长 │ // │ 阿里云 │ 非流式/单向流式 │ 20,000 │ CosyVoice │ // │ 火山引擎/豆包 │ 异步长文本 │ 100,000 │ 音频保存7天 │ // │ 百度 │ 长文本异步 │ 100,000 │ 一次性合成 │ // │ 讯飞 │ 长文本TTS │ 100,000 │ 万字级别快速合成 │ // │ 腾讯云 │ 长文本语音合成 │ 10,000+ │ 每个speak标签≤150字 │ // │ OpenAI │ TTS-1/HD │ 4,096 │ 隐藏限制 │ // └──────────────┴────────────────────┴───────────┴──────────────────────┘ // 生成音频(异步模式,不再创建 Audio 记录) export async function generateAudio( userId: string, text: string, voiceId: string, voiceParams: VoiceParams, onComplete?: (audioUrl: string, duration: number) => void, options?: { bookId?: string; chapterId?: number; chapterTitle?: string; ttsProvider?: 'aliyun' | 'minimax'; } ): Promise<{ audioId: string; audioUrl: string; bookId: number; }> { // 使用 UUID 作为音频 ID const audioId = uuidv4(); const audioDir = path.join(config.upload.dir, audioId); if (!fs.existsSync(audioDir)) { fs.mkdirSync(audioDir, { recursive: true }); } console.log('📝 开始音频生成:', audioId, '文本长度:', text.length, 'bookId:', options?.bookId); // 解析或创建目标书籍 ID let targetBookId: number; if (options?.bookId) { targetBookId = parseInt(options.bookId); } else { targetBookId = await getOrCreateDefaultBook(userId); } // 创建 AudioRecord 记录(状态为 processing) const wordCount = (text.match(/[\u4e00-\u9fa5]/g) || []).length; try { await prisma.audioRecord.create({ data: { audioId, userId: userId ? parseInt(userId) : null, title: options.chapterTitle || '未命名音频', text: text.length > 10000 ? text.substring(0, 10000) : text, wordCount, voiceId, voiceParams: JSON.stringify(voiceParams), status: 'processing', bookId: targetBookId, }, }); console.log('💾 AudioRecord 已创建:', audioId); } catch (error) { console.error('❌ 创建 AudioRecord 失败:', error); } // 等待音频生成完成 try { const result = await processAudioGeneration(audioId, text, voiceId, voiceParams, audioDir, onComplete, { ...options, userId, bookId: String(targetBookId), ttsProvider: options?.ttsProvider, }); return result; } catch (error: any) { console.error('❌ 音频生成失败:', error.message); // 更新 AudioRecord 状态为失败 try { await prisma.audioRecord.update({ where: { audioId }, data: { status: 'failed', errorMsg: error.message }, }); } catch (dbError) { console.error('❌ 更新 AudioRecord 失败状态失败:', dbError); } // 创建失败标记文件 const failedMarker = path.join(audioDir, 'failed'); fs.writeFileSync(failedMarker, error.message); // 推送 WebSocket 事件(音频生成失败) if (options?.bookId && options?.chapterId) { const { pushAudioGenerationComplete } = await import('../../services/websocket.service.js'); pushAudioGenerationComplete(options.bookId, options.chapterId, 'failed'); } throw error; } } /** * 异步处理音频生成(使用 Provider 注册表 + 熔断器) */ async function processAudioGeneration( audioId: string, text: string, voiceId: string, voiceParams: VoiceParams, audioDir: string, onComplete?: (audioUrl: string, duration: number) => void, options?: { userId?: string; bookId?: string; chapterId?: number; chapterTitle?: string; ttsProvider?: 'aliyun' | 'minimax'; } ) { const logMsg = `🔄 开始处理音频 ID: ${audioId}, 文本长度: ${text.length}, voiceId: ${voiceId}`; console.log(logMsg); logToFile(logMsg); const registry = getTtsRegistry(); // 解析首选 Provider 名称(与注册表命名规则一致:${vendorKey}-tts) // 兼容前端传 'aliyun' → 实际 vendor key 是 'bailian' const vendorAlias: Record = { aliyun: 'bailian' }; const vendorKey = vendorAlias[options?.ttsProvider ?? ''] ?? options?.ttsProvider; const preferredName = vendorKey ? `${vendorKey}-tts` : undefined; // 带降级的执行:优先使用 preferredName,失败后通过 getNextAvailable 轮转 // 与 LLM 的 invokeWithRetry 保持一致:可恢复错误先重试,再切换供应商 const getInitialNode = (): ProviderNode => { if (preferredName) { const pref = registry.get(preferredName); if (pref && pref.enabled && !pref.breaker.isOpen()) return pref; const next = registry.getNextAvailable(preferredName); if (next) return next; } const primary = registry.getPrimary(); if (!primary) throw new Error('没有可用的 TTS Provider'); return primary; }; const visited = new Set(); let currentNode = getInitialNode(); const maxProviders = registry.listEnabled().length; while (currentNode && visited.size < maxProviders) { if (visited.has(currentNode.provider.name)) break; visited.add(currentNode.provider.name); const tts = currentNode.provider; const node = currentNode; try { console.log(`🔊 使用 TTS Provider: ${tts.name} (vendor=${tts.vendor}, mode=${tts.mode})`); logToFile(`Provider: ${tts.name}, vendor=${tts.vendor}, mode=${tts.mode}`); // 解析音色名称(MiniMax 和阿里云用不同的音色映射表) const voiceName = tts.vendor.startsWith('minimax') ? getMiniMaxVoice(voiceId) : getAliyunVoice(voiceId); // 按模型配置分段:有 maxTextLength 就用它的 80%,没配置就用默认 1000 const segmentMax = tts.maxTextLength ? Math.floor(tts.maxTextLength * 0.8) : 1000; const segments = splitText(text, segmentMax); console.log(`📝 文本已分段: ${segments.length} 段, 阈值=${segmentMax}字 (Provider: ${tts.name})`); // 并行生成各段音频(使用 Provider 自身的并发设置,每段带重试) const audioFiles: string[] = []; const cloudUrls: string[] = []; const concurrency = tts.concurrency; for (let i = 0; i < segments.length; i += concurrency) { const batch = segments.slice(i, i + concurrency); // 统一调用 synthesizeSegmentWithRetry(熔断保护 + 可恢复错误重试1次) const results = await Promise.all( batch.map(async (segment, idx) => { const segPath = path.join(audioDir, `segment_${i + idx}.mp3`); return synthesizeSegmentWithRetry(tts, node, segment, voiceName, voiceParams, segPath); }) ); results.forEach(r => { if (r.startsWith('cloud:')) { cloudUrls.push(r.substring(6)); } else { audioFiles.push(r); } }); } console.log(`📁 生成了 ${audioFiles.length} 个音频文件, ${cloudUrls.length} 个云端URL`); let audioUrl = ''; let duration = 0; let size = 0; if (cloudUrls.length > 0 && audioFiles.length === 0 && cloudUrls.length === 1) { console.log('☁️ 下载单个云端音频并上传...'); const cloudUrl = cloudUrls[0]; const response = await axios.get(cloudUrl, { responseType: 'arraybuffer', timeout: 60000 }); const buffer = Buffer.from(response.data); const tempPath = path.join(audioDir, 'output.mp3'); if (!fs.existsSync(audioDir)) fs.mkdirSync(audioDir, { recursive: true }); fs.writeFileSync(tempPath, buffer); audioUrl = await storageService.uploadAudio(tempPath, audioId); duration = await AudioMerger.getDuration(tempPath); size = fs.statSync(tempPath).size; } else { // 下载所有云端URL到本地 const allLocalFiles = [...audioFiles]; if (cloudUrls.length > 0) { for (let ci = 0; ci < cloudUrls.length; ci++) { try { const response = await axios.get(cloudUrls[ci], { responseType: 'arraybuffer', timeout: 60000 }); const localPath = path.join(audioDir, `cloud_segment_${ci}.mp3`); fs.writeFileSync(localPath, Buffer.from(response.data)); allLocalFiles.push(localPath); } catch (dlErr: any) { console.error(`❌ 云端音频 ${ci + 1} 下载失败:`, dlErr.message); } } } if (allLocalFiles.length === 0) { throw new Error('所有音频文件下载失败,无法生成音频'); } const outputPath = path.join(audioDir, 'output.mp3'); const mergedFile = await AudioMerger.merge(allLocalFiles, outputPath); size = fs.statSync(mergedFile).size; duration = await AudioMerger.getDuration(mergedFile); audioUrl = await storageService.uploadAudio(mergedFile, audioId); console.log(`📁 音频已合并并上传 (${duration}s):`, audioUrl); } // AI 生成标题、摘要和标签 const [title] = await Promise.all([ aiSummaryService.generateTitle(text), aiSummaryService.generateSummary(text, 200), aiSummaryService.extractTags(text), ]); const finalAudioUrl = audioUrl; // LRC 歌词时间轴 let lrcLyrics = ''; try { const localAudioPath = path.join(audioDir, 'output.mp3'); lrcLyrics = generateLrc(text, duration, localAudioPath); } catch (lrcErr: any) { console.error(`❌ LRC 生成异常:`, lrcErr.message); } // 保存到书籍章节 const targetBookId = options?.bookId ? parseInt(options.bookId) : null; const targetChapterId = options?.chapterId ?? null; if (targetBookId && targetChapterId) { try { await prisma.bookChapter.update({ where: { id: targetChapterId }, data: { audioUrl: finalAudioUrl, audioDuration: duration, lrcLyrics: lrcLyrics || null, generatedAt: new Date(), }, }); console.log(`✅ 已更新书籍章节音频 (chapterId: ${targetChapterId})`); } catch (error) { console.error('❌ 保存到书籍章节失败:', error); } } // 更新 AudioRecord 为完成 try { await prisma.audioRecord.update({ where: { audioId }, data: { status: 'completed', audioUrl: finalAudioUrl, audioDuration: duration, audioSize: size, title: options?.chapterTitle || title, }, }); } catch (error) { console.error('❌ 更新 AudioRecord 失败:', error); } // 完成回调 if (onComplete) { try { await onComplete(finalAudioUrl, duration); } catch (callbackErr) { console.error(`[TTS] onComplete回调失败:`, callbackErr); } } // WebSocket 推送 if (options?.bookId && options?.chapterId) { const { pushAudioGenerationComplete } = await import('../../services/websocket.service.js'); pushAudioGenerationComplete(options.bookId, options.chapterId, 'completed'); } console.log('✅ 音频生成完成:', audioId, 'URL:', finalAudioUrl); return { audioId, audioUrl: finalAudioUrl, bookId: targetBookId }; } catch (error: any) { // 自动降级:与 LLM 共用 shouldSwitchModel 判断可恢复错误 const errorMsg = error?.message || ''; const isRecoverable = error instanceof CircuitBreakerOpenError || config.models.shouldSwitchModel(error); // 额度耗尽立即标记(4小时自动恢复) if (['quota', 'balance', 'insufficient', 'usage limit'].some(k => errorMsg.toLowerCase().includes(k))) { registry.markExhausted(tts.name, errorMsg, 4 * 60 * 60 * 1000); } if (isRecoverable) { console.error(`❌ Provider ${tts.name} 失败(可恢复),通过 getNextAvailable 降级...`); currentNode = registry.getNextAvailable(tts.name); continue; } // 非可恢复错误,直接抛出 console.error(`❌ processAudioGeneration 失败(非可恢复错误):`, errorMsg); throw error; } } // 所有 Provider 都试过了,仍然失败 throw new Error('所有 TTS Provider 都已尝试,均无法生成音频'); } /** * 片段级合成(带熔断保护 + 可恢复错误重试1次) * 与 LLM invokeWithRetry 策略一致:瞬时波动先重试,持续性故障再切换 */ async function synthesizeSegmentWithRetry( tts: ITtsProvider, node: ProviderNode, segment: string, voiceName: string, voiceParams: VoiceParams, segPath: string, ): Promise { try { return await node.breaker.call(() => tts.synthesize(segment, voiceName, voiceParams, segPath), ); } catch (err: any) { // 熔断器开启 → 直接抛出,让外层切换 Provider if (err instanceof CircuitBreakerOpenError) throw err; const errorMsg = err?.message || ''; // 不可恢复错误 → 直接抛出(如认证失败、参数错误) if (!config.models.shouldSwitchModel(err)) throw err; // 可恢复错误:延迟后重试1次(与 LLM 相同的 2s 延迟) console.log(`[TTS] 片段合成失败(${errorMsg.substring(0, 80)}),2s后重试...`); logToFile(`[Retry] 片段合成失败,2s后重试: ${errorMsg.substring(0, 100)}`); await new Promise(resolve => setTimeout(resolve, 2000)); return await node.breaker.call(() => tts.synthesize(segment, voiceName, voiceParams, segPath), ); } } /** * 获取音频状态(基于文件系统) */ export async function getAudioStatus(audioId: string): Promise<{ status: string; audio?: any }> { const audioDir = path.join(config.upload.dir, audioId); const outputPath = path.join(audioDir, 'output.mp3'); const failedMarker = path.join(audioDir, 'failed'); // 检查是否失败 if (fs.existsSync(failedMarker)) { const errorMsg = fs.readFileSync(failedMarker, 'utf-8'); return { status: 'failed', audio: { error: errorMsg } }; } // 检查是否生成完成 if (fs.existsSync(outputPath)) { const stats = fs.statSync(outputPath); const duration = await AudioMerger.getDuration(outputPath); return { status: 'completed', audio: { audioUrl: `/uploads/${audioId}/output.mp3`, audioDuration: duration, audioSize: stats.size, }, }; } // 检查目录是否存在 if (fs.existsSync(audioDir)) { // 检查是否为僵尸任务:目录存在超过 5 分钟但没有任何文件 const dirStat = fs.statSync(audioDir); const dirAge = Date.now() - dirStat.mtimeMs; const files = fs.readdirSync(audioDir); if (files.length === 0 && dirAge > 120000) { // 空目录且超过 2 分钟,判定为僵尸任务 console.warn(`⚠️ 检测到僵尸任务: ${audioId},目录创建于 ${Math.round(dirAge / 1000)}s 前,无任何文件`); // 更新数据库状态为失败 try { await prisma.audioRecord.update({ where: { audioId }, data: { status: 'failed', errorMsg: '任务超时:生成进程异常终止' }, }); } catch (e) {} return { status: 'failed', audio: { error: '生成进程异常终止' } }; } return { status: 'processing' }; } // 不存在 return { status: 'not_found' }; } // 获取可用音色 export function getVoices(): Voice[] { return VOICES; } // ============ LRC 歌词生成 ============ /** 格式化秒数为 [MM:SS.XX] */ function formatLrcTimestamp(seconds: number): string { const m = Math.floor(seconds / 60); const s = seconds % 60; const cs = Math.round((s - Math.floor(s)) * 100); const sec = Math.floor(s); return `${m.toString().padStart(2, '0')}:${sec.toString().padStart(2, '0')}.${cs.toString().padStart(2, '0')}`; } /** 清洗文本中的换行符(LRC每行只能有一个时间戳,文本中不能含换行) */ function sanitizeLrcText(s: string): string { return s.replace(/\n+/g, ' ').trim(); } /** 统计可见字符数(去除空白) */ function countVisible(s: string): number { return s.replace(/\s/g, '').length; } /** 最大每行字符数(超过则进一步拆分) */ const MAX_CHARS_PER_LINE = 50; /** * 将文本拆分为适合 LRC 显示的小句子 * 策略: * 1. 先按 Markdown 标题(#/##)切分,标题独立一行 * 2. 按句末标点(。!?)切分 * 3. 按分号/冒号(;:)切分 * 4. 按逗号(,,)切分 * 5. 按换行切分 * 6. 兜底:按固定字符数切分 * 每行不超过 MAX_CHARS_PER_LINE 字符 */ function splitIntoLrcSentences(text: string): string[] { const result: string[] = []; const maxChars = MAX_CHARS_PER_LINE; // 第一步:按 Markdown 标题切分,标题独立成行 const titleParts = text.split(/(?=^#{1,3}\s)/m); for (const part of titleParts) { const trimmed = part.trim(); if (!trimmed) continue; // 如果是标题行(单独一行) const titleMatch = trimmed.match(/^(#{1,3}\s+.+?)(\n|$)/); if (titleMatch) { const titleLine = titleMatch[1].trim(); const rest = trimmed.substring(titleMatch[0].length).trim(); // 标题单独一行 result.push(titleLine); // 处理标题后面的正文 if (rest) { result.push(...splitBodyIntoSentences(rest, maxChars)); } } else { // 没有标题,直接处理正文 result.push(...splitBodyIntoSentences(trimmed, maxChars)); } } return result.filter(s => countVisible(s) > 0); } /** * 将正文拆分为小句子 * 优先级:句号 > 分号/冒号 > 逗号 > 换行 > 固定长度 */ function splitBodyIntoSentences(text: string, maxChars: number): string[] { const result: string[] = []; // 先按段落(空行/换行)粗分 const paragraphs = text.split(/\n+/).filter(p => p.trim()); for (const para of paragraphs) { const trimmed = para.trim(); if (!trimmed) continue; // 如果整段就小于 maxChars,直接加入 if (countVisible(trimmed) <= maxChars) { result.push(trimmed); continue; } // 按句末标点切分(中文:。!?,英文:.!?) let sentences = splitByPunctuation(trimmed, /[。!?.!?]+/, maxChars); // 如果切分后仍有超长句子,按顿号再切 sentences = furtherSplit(sentences, /[、]/, maxChars); // 兜底:强制按字符数切分 sentences = forceSplitByCharCount(sentences, maxChars); result.push(...sentences); } return result; } /** 按指定标点切分,标点附在前面那句末尾 */ function splitByPunctuation(text: string, punctRegex: RegExp, maxChars: number): string[] { const parts: string[] = []; let remaining = text; while (remaining.length > 0) { // 查找下一个标点位置 const match = remaining.match(punctRegex); if (!match || match.index === undefined) { // 没有更多标点,剩余部分整体加入 parts.push(remaining); break; } const cutPos = match.index + match[0].length; const sentence = remaining.substring(0, cutPos).trim(); if (sentence) { parts.push(sentence); } remaining = remaining.substring(cutPos).trim(); } return parts; } /** 对已切分的句子,如果某些句子仍然超长,用更细粒度的标点再切 */ function furtherSplit(sentences: string[], punctRegex: RegExp, maxChars: number): string[] { const result: string[] = []; for (const s of sentences) { if (countVisible(s) <= maxChars) { result.push(s); continue; } // 用更细粒度标点再切 const subParts = splitByPunctuation(s, punctRegex, maxChars); result.push(...subParts); } return result; } /** 兜底:强制按字符数切分超长句子 */ function forceSplitByCharCount(sentences: string[], maxChars: number): string[] { const result: string[] = []; for (const s of sentences) { if (countVisible(s) <= maxChars) { result.push(s); continue; } // 按可见字符数强制切分 let buf = ''; let visibleCount = 0; for (const ch of s) { buf += ch; if (!/\s/.test(ch)) visibleCount++; if (visibleCount >= maxChars) { result.push(buf.trim()); buf = ''; visibleCount = 0; } } if (buf.trim()) result.push(buf.trim()); } return result; } /** * 生成 LRC 歌词时间轴 * * 核心算法:**按句子拆分 + 均匀语速分配** * * 原理: * - TTS 生成的音频语速相对均匀 * - 用 总时长/总字数 得到真实平均语速(秒/字) * - 每句时长 = 该句字数 × 平均语速 * - 这样保证所有句子时间之和 = 总时长,且短句少分时间、长句多分时间 */ export function generateLrc(text: string, duration: number, audioPath?: string): string { if (!text || duration <= 0) return ''; try { const sentences = splitIntoLrcSentences(text); if (sentences.length === 0) return ''; logToFile(`🎵 LRC 拆分: ${sentences.length} 句, 总时长=${duration}s`); // 计算总可见字符数 const totalChars = sentences.reduce((sum, s) => sum + countVisible(s), 0); if (totalChars === 0) return ''; // 核心:均匀语速 = 总时长 / 总字数 const speechRate = duration / totalChars; // 秒/字 logToFile(`🎵 语速: ${speechRate.toFixed(3)}s/字, 总字数=${totalChars}`); const lines: string[] = []; let currentTime = 0; for (let i = 0; i < sentences.length; i++) { const charCount = countVisible(sentences[i]); const isTitle = /^#{1,3}\s/.test(sentences[i].trim()); // 标题行语速较慢(通常有停顿),给予 1.5 倍时长系数 const titleBoost = isTitle ? 1.5 : 1.0; const lineDuration = charCount * speechRate * titleBoost; lines.push(`[${formatLrcTimestamp(currentTime)}] ${sanitizeLrcText(sentences[i])}`); logToFile(` [${formatLrcTimestamp(currentTime)}] ${charCount}字 ${sentences[i].substring(0, 30)}...`); currentTime += lineDuration; } // 由于标题行增加了时长系数,实际计算总时长可能超过音频时长 // 用线性缩放将所有时间戳归一化到 [0, duration] 区间 if (currentTime > duration && lines.length > 0) { const scaleFactor = duration / currentTime; const rescaledLines: string[] = []; let rescaledTime = 0; for (let i = 0; i < sentences.length; i++) { const charCount = countVisible(sentences[i]); const isTitle = /^#{1,3}\s/.test(sentences[i].trim()); const titleBoost = isTitle ? 1.5 : 1.0; const rawLineDuration = charCount * speechRate * titleBoost; const scaledLineDuration = rawLineDuration * scaleFactor; rescaledLines.push(`[${formatLrcTimestamp(rescaledTime)}] ${sanitizeLrcText(sentences[i])}`); rescaledTime += scaledLineDuration; } logToFile(`🎵 LRC 标题补偿缩放: 原始总时长=${currentTime.toFixed(2)}s, 缩放因子=${scaleFactor.toFixed(3)}`); return rescaledLines.join('\n'); } // 确保最后一行不超过总时长 logToFile(`🎵 LRC 完成: ${lines.length} 行, 末尾时间=${currentTime.toFixed(2)}s, 音频时长=${duration}s`); return lines.join('\n'); } catch (err: any) { logToFile(`❌ generateLrc 异常: ${err.message}`); console.error(`❌ generateLrc 异常:`, err.message); return buildSimpleLrc(text, duration); } } /** 最简降级:按句号或换行分割,再不行按固定长度切分 */ function buildSimpleLrc(text: string, duration: number): string { // 中英文句末标点(句号、感叹号、问号) let sentences = text.split(/(?<=[。!?.!?])\s*/).filter(s => s.trim()); if (sentences.length <= 1) { sentences = text.split(/\n+/).filter(s => s.trim()); } if (sentences.length <= 1 && text.length > 20) { // 英文按句号切分后再按固定长度兜底 const englishSentences = text.split(/(?<=[.!?])\s+/).filter(s => s.trim().length > 0); if (englishSentences.length > 1) { sentences = englishSentences; } else { const chunkSize = 40; sentences = []; for (let i = 0; i < text.length; i += chunkSize) { sentences.push(text.substring(i, i + chunkSize)); } } } if (sentences.length === 0) return ''; const totalChars = sentences.reduce((sum, s) => sum + countVisible(s), 0); if (totalChars === 0) return ''; const speechRate = duration / totalChars; const lines: string[] = []; let t = 0; for (const s of sentences) { lines.push(`[${formatLrcTimestamp(t)}] ${sanitizeLrcText(s)}`); t += countVisible(s) * speechRate; } return lines.join('\n'); } // 获取可用 TTS 服务商(从注册表读取) export function getAvailableProviders() { const registry = getTtsRegistry(); return registry.listEnabled().map(n => ({ id: n.provider.vendor, name: n.provider.name, enabled: n.enabled, state: n.breaker.getState(), })); } // 生成预览音频(短文本) export async function generatePreview( voiceId: string, voiceParams?: { speed?: number; pitch?: number; volume?: number }, providerType?: 'aliyun' | 'minimax' ): Promise<{ audioId: string; audioUrl: string }> { const previewText = '你好,欢迎使用AI有声书'; const audioId = 'preview-' + uuidv4(); const audioDir = path.join(config.upload.dir, audioId); if (!fs.existsSync(audioDir)) { fs.mkdirSync(audioDir, { recursive: true }); } const params = { speed: voiceParams?.speed ?? 1.0, pitch: voiceParams?.pitch ?? 0, volume: voiceParams?.volume ?? 50, }; console.log(`[TTS Preview] params: speed=${params.speed}, pitch=${params.pitch}, volume=${params.volume}`); const registry = getTtsRegistry(); // 优先使用的 Provider const preferredName = providerType === 'minimax' ? 'minimax-tts' : providerType === 'aliyun' ? 'bailian-tts' : undefined; const startNode = preferredName ? registry.get(preferredName) : undefined; // 按优先级尝试所有可用 Provider const candidates = startNode ? [startNode, ...registry.listAvailable().filter(n => n !== startNode)] : registry.listAvailable(); let lastError: Error | null = null; for (const node of candidates) { if (node.exhausted || node.breaker.isOpen()) continue; const tts = node.provider; const voiceName = tts.vendor.startsWith('minimax') ? getMiniMaxVoice(voiceId) : getAliyunVoice(voiceId); if (tts.mode === 'mock') { const mockPath = path.join(audioDir, 'preview.mp3'); fs.writeFileSync(mockPath, Buffer.alloc(1024)); return { audioId, audioUrl: `/uploads/${audioId}/preview.mp3` }; } try { // 先创建空文件让 aliyun 下载能定位目录 const ext = tts.vendor.startsWith('minimax') ? 'mp3' : 'wav'; const outputPath = path.join(audioDir, `preview.${ext}`); if (!fs.existsSync(outputPath)) { fs.writeFileSync(outputPath, Buffer.alloc(0)); } const actualPath = await node.breaker.call(() => tts.synthesize(previewText, voiceName, params, outputPath)); const audioUrl = await storageService.uploadAudio(actualPath, audioId); console.log(`✅ [TTS Preview] ${tts.name} 生成成功`); return { audioId, audioUrl }; } catch (error: any) { console.error(`❌ [TTS Preview] ${tts.name} 失败:`, error.message); lastError = error; // 额度耗尽标记 if (['quota', 'balance', 'insufficient', 'usage limit'].some(k => error.message?.toLowerCase().includes(k))) { registry.markExhausted(tts.name, error.message, 4 * 60 * 60 * 1000); } continue; } } console.error('❌ 预览生成失败:', lastError?.message); throw lastError || new Error('所有 TTS Provider 都已尝试'); } // ============ 初始化 ============ // 首次调用时初始化注册表并启动健康检查 startTtsHealthCheck();