| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058 |
- import path from 'path';
- import fs from 'fs';
- import { v4 as uuidv4 } from 'uuid';
- import { config } from '../../config';
- import { prisma } from '../../models';
- import { VoiceParams, Voice } from '../../types';
- import { getMiniMaxVoice } from './minimax.provider';
- import { AudioMerger } from './audio-merger';
- import { aiSummaryService } from './ai-summary.service';
- import { storageService } from '../../services/storage.service';
- import { getTtsRegistry, getAvailableTtsProvider, startTtsHealthCheck } from './provider.registry';
- import { ITtsProvider } from './provider.interface';
- import { CircuitBreakerOpenError } from '../../common/circuit-breaker';
- import { ProviderNode } from '../../common/provider-registry';
- import axios from 'axios';
- // 日志文件路径
- const LOG_FILE = path.join(process.cwd(), 'tts-debug.log');
- function logToFile(msg: string) {
- const timestamp = new Date().toISOString();
- fs.appendFileSync(LOG_FILE, `[${timestamp}] ${msg}\n`);
- }
- // 可用音色列表(CosyVoice v3-flash - 67个音色)
- // 数据来源:阿里云百炼官方文档 2026-05
- export const VOICES: Voice[] = [
- // 标杆音色
- { id: 'longanyang', name: '龙安洋', gender: 'male', description: '阳光大男孩 20~30岁' },
- { id: 'longanhuan', name: '龙安欢', gender: 'female', description: '欢脱元气女 20~30岁' },
- { id: 'longhuhu_v3', name: '龙呼呼', gender: 'female', description: '天真烂漫女童 6~10岁' },
- // 有声书 - 男声
- { id: 'longsanshu_v3', name: '龙三叔', gender: 'male', description: '沉稳质感男 25~45岁' },
- { id: 'longxiu_v3', name: '龙修', gender: 'male', description: '博才说书男 25~35岁' },
- { id: 'longnan_v3', name: '龙楠', gender: 'male', description: '睿智青年男 25~30岁' },
- { id: 'longyichen_v3', name: '龙逸尘', gender: 'male', description: '洒脱活力男 20~30岁' },
- { id: 'longlaobo_v3', name: '龙老伯', gender: 'male', description: '沧桑岁月爷 60岁以上' },
- // 有声书 - 女声
- { id: 'longmiao_v3', name: '龙妙', gender: 'female', description: '抑扬顿挫女 25~30岁' },
- { id: 'longyuan_v3', name: '龙媛', gender: 'female', description: '温暖治愈女 35~40岁' },
- { id: 'longyue_v3', name: '龙悦', gender: 'female', description: '温暖磁性女 30~35岁' },
- { id: 'longwanjun_v3', name: '龙婉君', gender: 'female', description: '细腻柔声女 20~30岁' },
- { id: 'longlaoyi_v3', name: '龙老姨', gender: 'female', description: '烟火从容阿姨 60岁以上' },
- // 客服/电话
- { id: 'longyingxun_v3', name: '龙应询', gender: 'male', description: '年轻青涩男 20~25岁' },
- { id: 'longyingjing_v3', name: '龙应静', gender: 'female', description: '低调冷静女 25~35岁' },
- { id: 'longyingling_v3', name: '龙应聆', gender: 'female', description: '温和共情女 25~30岁' },
- { id: 'longyingtao_v3', name: '龙应桃', gender: 'female', description: '温柔淡定女 25~30岁' },
- { id: 'longyingmu_v3', name: '龙应沐', gender: 'female', description: '优雅知性女 25~30岁' },
- { id: 'longyingxiao_v3', name: '龙应笑', gender: 'female', description: '清甜推销女 20~25岁' },
- // 语音助手
- { id: 'longxiaochun_v3', name: '龙小淳', gender: 'female', description: '知性积极女 25~30岁' },
- { id: 'longxiaoxia_v3', name: '龙小夏', gender: 'female', description: '沉稳权威女 25~30岁' },
- { id: 'longanyun_v3', name: '龙安昀', gender: 'male', description: '居家暖男 30~35岁' },
- { id: 'longanwen_v3', name: '龙安温', gender: 'female', description: '优雅知性女 25~35岁' },
- { id: 'longanli_v3', name: '龙安莉', gender: 'female', description: '利落从容女 25~35岁' },
- { id: 'longanlang_v3', name: '龙安朗', gender: 'male', description: '清爽利落男 20~25岁' },
- { id: 'longyumi_v3', name: 'YUMI', gender: 'female', description: '正经青年女 20~25岁' },
- // 社交陪伴
- { id: 'longanqin_v3', name: '龙安亲', gender: 'female', description: '亲和活泼女 20~25岁' },
- { id: 'longanya_v3', name: '龙安雅', gender: 'female', description: '高雅气质女 25~35岁' },
- { id: 'longanzhi_v3', name: '龙安智', gender: 'male', description: '睿智轻熟男 25~35岁' },
- { id: 'longanling_v3', name: '龙安灵', gender: 'female', description: '思维灵动女 20~30岁' },
- { id: 'longantai_v3', name: '龙安台', gender: 'female', description: '嗲甜台湾女 20~25岁' },
- { id: 'longhua_v3', name: '龙华', gender: 'female', description: '元气甜美女 20~25岁' },
- { id: 'longcheng_v3', name: '龙橙', gender: 'male', description: '智慧青年男 20~25岁' },
- { id: 'longze_v3', name: '龙泽', gender: 'male', description: '温暖元气男 25~30岁' },
- { id: 'longzhe_v3', name: '龙哲', gender: 'male', description: '呆板大暖男 25~30岁' },
- { id: 'longyan_v3', name: '龙颜', gender: 'female', description: '温暖春风女 30~35岁' },
- { id: 'longxing_v3', name: '龙星', gender: 'female', description: '温婉邻家女 20~25岁' },
- { id: 'longtian_v3', name: '龙天', gender: 'male', description: '磁性理智男 30~35岁' },
- { id: 'longwan_v3', name: '龙婉', gender: 'female', description: '细腻柔声女 20~30岁' },
- { id: 'longqiang_v3', name: '龙嫱', gender: 'female', description: '浪漫风情女 30~35岁' },
- { id: 'longfeifei_v3', name: '龙菲菲', gender: 'female', description: '甜美娇气女 20~25岁' },
- { id: 'longhao_v3', name: '龙浩', gender: 'male', description: '多情忧郁男 30~35岁' },
- { id: 'longanrou_v3', name: '龙安柔', gender: 'female', description: '温柔闺蜜女 20~35岁' },
- { id: 'longhan_v3', name: '龙寒', gender: 'male', description: '温暖痴情男 30~35岁' },
- // 童声
- { id: 'longpaopao_v3', name: '龙泡泡', gender: 'female', description: '飞天泡泡音 6~15岁' },
- { id: 'longjielidou_v3', name: '龙杰力豆', gender: 'male', description: '阳光顽皮男 10岁' },
- { id: 'longxian_v3', name: '龙仙', gender: 'female', description: '豪放可爱女 12岁' },
- { id: 'longling_v3', name: '龙铃', gender: 'female', description: '稚气呆板女 10岁' },
- { id: 'longshanshan_v3', name: '龙闪闪', gender: 'female', description: '戏剧化童声 6~15岁' },
- { id: 'longniuniu_v3', name: '龙牛牛', gender: 'male', description: '阳光男童声 6~15岁' },
- // 方言
- { id: 'longjiaxin_v3', name: '龙嘉欣', gender: 'female', description: '优雅粤语女 30~35岁' },
- { id: 'longjiayi_v3', name: '龙嘉怡', gender: 'female', description: '知性粤语女 25~30岁' },
- { id: 'longanyue_v3', name: '龙安粤', gender: 'male', description: '欢脱粤语男 25~35岁' },
- { id: 'longlaotie_v3', name: '龙老铁', gender: 'male', description: '东北直率男 25~30岁' },
- { id: 'longshange_v3', name: '龙陕哥', gender: 'male', description: '陕北原味男 25~35岁' },
- // 新闻/直播
- { id: 'longshuo_v3', name: '龙硕', gender: 'male', description: '博才干练男 25~30岁' },
- { id: 'longshu_v3', name: '龙书', gender: 'male', description: '沉稳青年男 20~25岁' },
- { id: 'loongbella_v3', name: 'Bella3.0',gender: 'female', description: '精准干练女 25~30岁' },
- { id: 'longanran_v3', name: '龙安燃', gender: 'female', description: '活泼质感女 30~40岁' },
- { id: 'longanxuan_v3', name: '龙安宣', gender: 'female', description: '经典直播女 30~40岁' },
- // 特色
- { id: 'longjiqi_v3', name: '龙机器', gender: 'male', description: '呆萌机器人 20~30岁' },
- { id: 'longhouge_v3', name: '龙猴哥', gender: 'male', description: '经典猴哥 20~25岁' },
- { id: 'longdaiyu_v3', name: '龙黛玉', gender: 'female', description: '娇率才女音 15~25岁' },
- { id: 'longfei_v3', name: '龙飞', gender: 'male', description: '热血磁性男 30~35岁' },
- // 外语
- { id: 'loongkyong_v3', name: '韩语女', gender: 'female', description: '韩语女声' },
- { id: 'loongriko_v3', name: 'Riko霓虹', gender: 'female', description: '二次元日语女' },
- { id: 'loongabby_v3', name: '美语女', gender: 'female', description: '美式英文女' },
- { id: 'loongandy_v3', name: '美语男', gender: 'male', description: '美式英文男' },
- ];
- // CosyVoice v3-flash 音色ID直接用作API参数名,无需映射
- export function getAliyunVoice(voiceId: string): string {
- return voiceId;
- }
- // 获取或创建默认"我的音频"书籍
- async function getOrCreateDefaultBook(userId: string): Promise<number> {
- const userIdNum = parseInt(userId);
- const bookTitle = '我的音频';
- // 先尝试查找现有默认书籍
- let book = await prisma.book.findFirst({
- where: {
- userId: userIdNum,
- title: bookTitle,
- },
- });
- if (book) {
- return book.id;
- }
- // 不存在则创建
- book = await prisma.book.create({
- data: {
- title: bookTitle,
- description: '我的语音合成音频收藏',
- userId: userIdNum,
- genStage: 'content_completed',
- bookScale: 'short',
- totalChapters: 0,
- estimatedWords: 0,
- },
- });
- console.log(`📚 创建默认书籍: bookId=${book.id}, userId=${userIdNum}`);
- return book.id;
- }
- // 判断是否使用长文本模式(>5000字符且启用realtime)
- // 注意:WebSocket realtime 模式需要特殊的API权限和配置,如果连接失败会导致生成失败。
- // 暂时强制禁用,使用 HTTP 分段模式
- export function shouldUseLongText(text: string): boolean {
- return false; // 强制返回 false,禁用 realtime 模式
- }
- // 文本分段 - 统一 1000 字符上限,优先在标点处断开
- // 策略:按自然句子累加,接近上限时在最后一个标点处截断,避免在字中间硬切
- // 好处:TTS 在句末韵律自然,拼接后听感流畅
- //
- // 1000 字的依据(主流 TTS 接口上限,大多以千为单位):
- // MiniMax 同步: 1万 | 阿里云 CosyVoice: 2万 | 讯飞 流式: ~4000字
- // 百度 短文本: 5120字 | OpenAI TTS: 4096字 | Google Cloud: 5000字节
- // ElevenLabs: 3000~5000字 | Azure: ~3000~5000字
- // 不支持1000的接口(应走异步长文本API,而非短文本接口):
- // 火山引擎 非流式: 300字 | 腾讯云 基础: 150字 | 百度 短文本: 60字
- const SEGMENT_MAX_LENGTH = 1000;
- // 句末标点(中文 + 英文),用于寻找断点
- const SENTENCE_END_RE = /[。!?;\n.!?;]/;
- export function splitText(text: string, maxLength: number = SEGMENT_MAX_LENGTH): string[] {
- const segments: string[] = [];
- // 清理文本
- const cleanText = text.replace(/\r/g, '');
- // 第一步:按换行拆成段落,再按句末标点拆成句子
- // 这样保留了段落边界,又不会在一个句子中间断开
- const rawSentences: string[] = [];
- const paragraphs = cleanText.split(/\n+/);
- for (const para of paragraphs) {
- if (para.length === 0) continue;
- // 按句末标点拆分,保留标点
- const sentences = para.match(/[^。!?;.!?;]+[。!?;.!?;]?/g) || [para];
- for (const s of sentences) {
- if (s.trim().length > 0) {
- rawSentences.push(s);
- }
- }
- }
- // 第二步:将句子累加成段,接近上限时在最后一个标点处截断
- let current = '';
- for (const sentence of rawSentences) {
- // 当前段 + 这句还能放下
- if ((current + sentence).length <= maxLength) {
- current += sentence;
- continue;
- }
- // 放不下了,先把当前段输出
- if (current) {
- segments.push(current);
- current = '';
- }
- // 如果这句话本身就超长,需要在句内找标点断点
- if (sentence.length > maxLength) {
- let remaining = sentence;
- while (remaining.length > 0) {
- if (remaining.length <= maxLength) {
- current = remaining;
- break;
- }
- // 在上限附近向前找最后一个标点作为断点
- let breakPos = -1;
- for (let i = maxLength; i > Math.max(0, maxLength - 100); i--) {
- if (SENTENCE_END_RE.test(remaining[i])) {
- breakPos = i + 1; // 标点后一位
- break;
- }
- }
- // 找不到标点,尝试逗号/顿号等次级标点
- if (breakPos === -1) {
- for (let i = maxLength; i > Math.max(0, maxLength - 100); i--) {
- if (/[,、,]/.test(remaining[i])) {
- breakPos = i + 1;
- break;
- }
- }
- }
- // 实在找不到标点,只能在上限处硬切(最后手段)
- if (breakPos === -1) {
- breakPos = maxLength;
- }
- segments.push(remaining.slice(0, breakPos));
- remaining = remaining.slice(breakPos);
- }
- } else {
- current = sentence;
- }
- }
- if (current) segments.push(current);
- return segments;
- }
- // ============ TTS Provider 调度(使用注册表 + 熔断器)============
- //
- // 优先级顺序由 Registry 中的 priority 决定:
- // 1. MiniMax (priority=1) - 异步长文本,业界最长
- // 2. 阿里云 Qwen-TTS (priority=2) - 同步快速
- // 3. Mock (priority=100) - 降级兜底
- //
- // 熔断器: 连续 3 次失败自动熔断,MiniMax 冷却 60s,阿里云 30s
- //
- // 分段策略: 统一 1000 字符/段,兼容所有 TTS 模型
- // 各家 TTS API 限制参考(2026年5月):
- // ┌──────────────┬────────────────────┬───────────┬──────────────────────┐
- // │ 服务商 │ 接口类型 │ 最大长度 │ 备注 │
- // ├──────────────┼────────────────────┼───────────┼──────────────────────┤
- // │ MiniMax │ 异步长文本 │ 1,000,000 │ 业界最长 │
- // │ 阿里云 │ 非流式/单向流式 │ 20,000 │ CosyVoice │
- // │ 火山引擎/豆包 │ 异步长文本 │ 100,000 │ 音频保存7天 │
- // │ 百度 │ 长文本异步 │ 100,000 │ 一次性合成 │
- // │ 讯飞 │ 长文本TTS │ 100,000 │ 万字级别快速合成 │
- // │ 腾讯云 │ 长文本语音合成 │ 10,000+ │ 每个speak标签≤150字 │
- // │ OpenAI │ TTS-1/HD │ 4,096 │ 隐藏限制 │
- // └──────────────┴────────────────────┴───────────┴──────────────────────┘
- // 生成音频(异步模式,不再创建 Audio 记录)
- export async function generateAudio(
- userId: string,
- text: string,
- voiceId: string,
- voiceParams: VoiceParams,
- onComplete?: (audioUrl: string, duration: number) => void,
- options?: {
- bookId?: string;
- chapterId?: number;
- chapterTitle?: string;
- ttsProvider?: 'aliyun' | 'minimax';
- }
- ): Promise<{
- audioId: string;
- audioUrl: string;
- bookId: number;
- }> {
- // 使用 UUID 作为音频 ID
- const audioId = uuidv4();
- const audioDir = path.join(config.upload.dir, audioId);
- if (!fs.existsSync(audioDir)) {
- fs.mkdirSync(audioDir, { recursive: true });
- }
- console.log('📝 开始音频生成:', audioId, '文本长度:', text.length, 'bookId:', options?.bookId);
- // 解析或创建目标书籍 ID
- let targetBookId: number;
- if (options?.bookId) {
- targetBookId = parseInt(options.bookId);
- } else {
- targetBookId = await getOrCreateDefaultBook(userId);
- }
- // 创建 AudioRecord 记录(状态为 processing)
- const wordCount = (text.match(/[\u4e00-\u9fa5]/g) || []).length;
- try {
- await prisma.audioRecord.create({
- data: {
- audioId,
- userId: userId ? parseInt(userId) : null,
- title: options.chapterTitle || '未命名音频',
- text: text.length > 10000 ? text.substring(0, 10000) : text,
- wordCount,
- voiceId,
- voiceParams: JSON.stringify(voiceParams),
- status: 'processing',
- bookId: targetBookId,
- },
- });
- console.log('💾 AudioRecord 已创建:', audioId);
- } catch (error) {
- console.error('❌ 创建 AudioRecord 失败:', error);
- }
- // 等待音频生成完成
- try {
- const result = await processAudioGeneration(audioId, text, voiceId, voiceParams, audioDir, onComplete, {
- ...options,
- userId,
- bookId: String(targetBookId),
- ttsProvider: options?.ttsProvider,
- });
- return result;
- } catch (error: any) {
- console.error('❌ 音频生成失败:', error.message);
- // 更新 AudioRecord 状态为失败
- try {
- await prisma.audioRecord.update({
- where: { audioId },
- data: { status: 'failed', errorMsg: error.message },
- });
- } catch (dbError) {
- console.error('❌ 更新 AudioRecord 失败状态失败:', dbError);
- }
- // 创建失败标记文件
- const failedMarker = path.join(audioDir, 'failed');
- fs.writeFileSync(failedMarker, error.message);
- // 推送 WebSocket 事件(音频生成失败)
- if (options?.bookId && options?.chapterId) {
- const { pushAudioGenerationComplete } = await import('../../services/websocket.service.js');
- pushAudioGenerationComplete(options.bookId, options.chapterId, 'failed');
- }
- throw error;
- }
- }
- /**
- * 异步处理音频生成(使用 Provider 注册表 + 熔断器)
- */
- async function processAudioGeneration(
- audioId: string,
- text: string,
- voiceId: string,
- voiceParams: VoiceParams,
- audioDir: string,
- onComplete?: (audioUrl: string, duration: number) => void,
- options?: {
- userId?: string;
- bookId?: string;
- chapterId?: number;
- chapterTitle?: string;
- ttsProvider?: 'aliyun' | 'minimax';
- }
- ) {
- const logMsg = `🔄 开始处理音频 ID: ${audioId}, 文本长度: ${text.length}, voiceId: ${voiceId}`;
- console.log(logMsg);
- logToFile(logMsg);
- const registry = getTtsRegistry();
- // 解析首选 Provider 名称(与注册表命名规则一致:${vendorKey}-tts)
- // 兼容前端传 'aliyun' → 实际 vendor key 是 'bailian'
- const vendorAlias: Record<string, string> = { aliyun: 'bailian' };
- const vendorKey = vendorAlias[options?.ttsProvider ?? ''] ?? options?.ttsProvider;
- const preferredName = vendorKey ? `${vendorKey}-tts` : undefined;
- // 带降级的执行:优先使用 preferredName,失败后通过 getNextAvailable 轮转
- // 与 LLM 的 invokeWithRetry 保持一致:可恢复错误先重试,再切换供应商
- const getInitialNode = (): ProviderNode<ITtsProvider> => {
- if (preferredName) {
- const pref = registry.get(preferredName);
- if (pref && pref.enabled && !pref.breaker.isOpen()) return pref;
- const next = registry.getNextAvailable(preferredName);
- if (next) return next;
- }
- const primary = registry.getPrimary();
- if (!primary) throw new Error('没有可用的 TTS Provider');
- return primary;
- };
- const visited = new Set<string>();
- let currentNode = getInitialNode();
- const maxProviders = registry.listEnabled().length;
- while (currentNode && visited.size < maxProviders) {
- if (visited.has(currentNode.provider.name)) break;
- visited.add(currentNode.provider.name);
- const tts = currentNode.provider;
- const node = currentNode;
- try {
- console.log(`🔊 使用 TTS Provider: ${tts.name} (vendor=${tts.vendor}, mode=${tts.mode})`);
- logToFile(`Provider: ${tts.name}, vendor=${tts.vendor}, mode=${tts.mode}`);
- // 解析音色名称(MiniMax 和阿里云用不同的音色映射表)
- const voiceName = tts.vendor.startsWith('minimax') ? getMiniMaxVoice(voiceId) : getAliyunVoice(voiceId);
- // 按模型配置分段:有 maxTextLength 就用它的 80%,没配置就用默认 1000
- const segmentMax = tts.maxTextLength ? Math.floor(tts.maxTextLength * 0.8) : 1000;
- const segments = splitText(text, segmentMax);
- console.log(`📝 文本已分段: ${segments.length} 段, 阈值=${segmentMax}字 (Provider: ${tts.name})`);
- // 并行生成各段音频(使用 Provider 自身的并发设置,每段带重试)
- const audioFiles: string[] = [];
- const cloudUrls: string[] = [];
- const concurrency = tts.concurrency;
- for (let i = 0; i < segments.length; i += concurrency) {
- const batch = segments.slice(i, i + concurrency);
- // 统一调用 synthesizeSegmentWithRetry(熔断保护 + 可恢复错误重试1次)
- const results = await Promise.all(
- batch.map(async (segment, idx) => {
- const segPath = path.join(audioDir, `segment_${i + idx}.mp3`);
- return synthesizeSegmentWithRetry(tts, node, segment, voiceName, voiceParams, segPath);
- })
- );
- results.forEach(r => {
- if (r.startsWith('cloud:')) {
- cloudUrls.push(r.substring(6));
- } else {
- audioFiles.push(r);
- }
- });
- }
- console.log(`📁 生成了 ${audioFiles.length} 个音频文件, ${cloudUrls.length} 个云端URL`);
- let audioUrl = '';
- let duration = 0;
- let size = 0;
- if (cloudUrls.length > 0 && audioFiles.length === 0 && cloudUrls.length === 1) {
- console.log('☁️ 下载单个云端音频并上传...');
- const cloudUrl = cloudUrls[0];
- const response = await axios.get(cloudUrl, { responseType: 'arraybuffer', timeout: 60000 });
- const buffer = Buffer.from(response.data);
- const tempPath = path.join(audioDir, 'output.mp3');
- if (!fs.existsSync(audioDir)) fs.mkdirSync(audioDir, { recursive: true });
- fs.writeFileSync(tempPath, buffer);
- audioUrl = await storageService.uploadAudio(tempPath, audioId);
- duration = await AudioMerger.getDuration(tempPath);
- size = fs.statSync(tempPath).size;
- } else {
- // 下载所有云端URL到本地
- const allLocalFiles = [...audioFiles];
- if (cloudUrls.length > 0) {
- for (let ci = 0; ci < cloudUrls.length; ci++) {
- try {
- const response = await axios.get(cloudUrls[ci], { responseType: 'arraybuffer', timeout: 60000 });
- const localPath = path.join(audioDir, `cloud_segment_${ci}.mp3`);
- fs.writeFileSync(localPath, Buffer.from(response.data));
- allLocalFiles.push(localPath);
- } catch (dlErr: any) {
- console.error(`❌ 云端音频 ${ci + 1} 下载失败:`, dlErr.message);
- }
- }
- }
- if (allLocalFiles.length === 0) {
- throw new Error('所有音频文件下载失败,无法生成音频');
- }
- const outputPath = path.join(audioDir, 'output.mp3');
- const mergedFile = await AudioMerger.merge(allLocalFiles, outputPath);
- size = fs.statSync(mergedFile).size;
- duration = await AudioMerger.getDuration(mergedFile);
- audioUrl = await storageService.uploadAudio(mergedFile, audioId);
- console.log(`📁 音频已合并并上传 (${duration}s):`, audioUrl);
- }
- // AI 生成标题、摘要和标签
- const [title] = await Promise.all([
- aiSummaryService.generateTitle(text),
- aiSummaryService.generateSummary(text, 200),
- aiSummaryService.extractTags(text),
- ]);
- const finalAudioUrl = audioUrl;
- // LRC 歌词时间轴
- let lrcLyrics = '';
- try {
- const localAudioPath = path.join(audioDir, 'output.mp3');
- lrcLyrics = generateLrc(text, duration, localAudioPath);
- } catch (lrcErr: any) {
- console.error(`❌ LRC 生成异常:`, lrcErr.message);
- }
- // 保存到书籍章节
- const targetBookId = options?.bookId ? parseInt(options.bookId) : null;
- const targetChapterId = options?.chapterId ?? null;
- if (targetBookId && targetChapterId) {
- try {
- await prisma.bookChapter.update({
- where: { id: targetChapterId },
- data: {
- audioUrl: finalAudioUrl,
- audioDuration: duration,
- lrcLyrics: lrcLyrics || null,
- generatedAt: new Date(),
- },
- });
- console.log(`✅ 已更新书籍章节音频 (chapterId: ${targetChapterId})`);
- } catch (error) {
- console.error('❌ 保存到书籍章节失败:', error);
- }
- }
- // 更新 AudioRecord 为完成
- try {
- await prisma.audioRecord.update({
- where: { audioId },
- data: {
- status: 'completed',
- audioUrl: finalAudioUrl,
- audioDuration: duration,
- audioSize: size,
- title: options?.chapterTitle || title,
- },
- });
- } catch (error) {
- console.error('❌ 更新 AudioRecord 失败:', error);
- }
- // 完成回调
- if (onComplete) {
- try { await onComplete(finalAudioUrl, duration); } catch (callbackErr) {
- console.error(`[TTS] onComplete回调失败:`, callbackErr);
- }
- }
- // WebSocket 推送
- if (options?.bookId && options?.chapterId) {
- const { pushAudioGenerationComplete } = await import('../../services/websocket.service.js');
- pushAudioGenerationComplete(options.bookId, options.chapterId, 'completed');
- }
- console.log('✅ 音频生成完成:', audioId, 'URL:', finalAudioUrl);
- return { audioId, audioUrl: finalAudioUrl, bookId: targetBookId };
- } catch (error: any) {
- // 自动降级:与 LLM 共用 shouldSwitchModel 判断可恢复错误
- const errorMsg = error?.message || '';
- const isRecoverable = error instanceof CircuitBreakerOpenError
- || config.models.shouldSwitchModel(error);
- // 额度耗尽立即标记(4小时自动恢复)
- if (['quota', 'balance', 'insufficient', 'usage limit'].some(k => errorMsg.toLowerCase().includes(k))) {
- registry.markExhausted(tts.name, errorMsg, 4 * 60 * 60 * 1000);
- }
- if (isRecoverable) {
- console.error(`❌ Provider ${tts.name} 失败(可恢复),通过 getNextAvailable 降级...`);
- currentNode = registry.getNextAvailable(tts.name);
- continue;
- }
- // 非可恢复错误,直接抛出
- console.error(`❌ processAudioGeneration 失败(非可恢复错误):`, errorMsg);
- throw error;
- }
- }
- // 所有 Provider 都试过了,仍然失败
- throw new Error('所有 TTS Provider 都已尝试,均无法生成音频');
- }
- /**
- * 片段级合成(带熔断保护 + 可恢复错误重试1次)
- * 与 LLM invokeWithRetry 策略一致:瞬时波动先重试,持续性故障再切换
- */
- async function synthesizeSegmentWithRetry(
- tts: ITtsProvider,
- node: ProviderNode<ITtsProvider>,
- segment: string,
- voiceName: string,
- voiceParams: VoiceParams,
- segPath: string,
- ): Promise<string> {
- try {
- return await node.breaker.call(() =>
- tts.synthesize(segment, voiceName, voiceParams, segPath),
- );
- } catch (err: any) {
- // 熔断器开启 → 直接抛出,让外层切换 Provider
- if (err instanceof CircuitBreakerOpenError) throw err;
- const errorMsg = err?.message || '';
- // 不可恢复错误 → 直接抛出(如认证失败、参数错误)
- if (!config.models.shouldSwitchModel(err)) throw err;
- // 可恢复错误:延迟后重试1次(与 LLM 相同的 2s 延迟)
- console.log(`[TTS] 片段合成失败(${errorMsg.substring(0, 80)}),2s后重试...`);
- logToFile(`[Retry] 片段合成失败,2s后重试: ${errorMsg.substring(0, 100)}`);
- await new Promise(resolve => setTimeout(resolve, 2000));
- return await node.breaker.call(() =>
- tts.synthesize(segment, voiceName, voiceParams, segPath),
- );
- }
- }
- /**
- * 获取音频状态(基于文件系统)
- */
- export async function getAudioStatus(audioId: string): Promise<{ status: string; audio?: any }> {
- const audioDir = path.join(config.upload.dir, audioId);
- const outputPath = path.join(audioDir, 'output.mp3');
- const failedMarker = path.join(audioDir, 'failed');
- // 检查是否失败
- if (fs.existsSync(failedMarker)) {
- const errorMsg = fs.readFileSync(failedMarker, 'utf-8');
- return { status: 'failed', audio: { error: errorMsg } };
- }
- // 检查是否生成完成
- if (fs.existsSync(outputPath)) {
- const stats = fs.statSync(outputPath);
- const duration = await AudioMerger.getDuration(outputPath);
- return {
- status: 'completed',
- audio: {
- audioUrl: `/uploads/${audioId}/output.mp3`,
- audioDuration: duration,
- audioSize: stats.size,
- },
- };
- }
- // 检查目录是否存在
- if (fs.existsSync(audioDir)) {
- // 检查是否为僵尸任务:目录存在超过 5 分钟但没有任何文件
- const dirStat = fs.statSync(audioDir);
- const dirAge = Date.now() - dirStat.mtimeMs;
- const files = fs.readdirSync(audioDir);
- if (files.length === 0 && dirAge > 120000) {
- // 空目录且超过 2 分钟,判定为僵尸任务
- console.warn(`⚠️ 检测到僵尸任务: ${audioId},目录创建于 ${Math.round(dirAge / 1000)}s 前,无任何文件`);
- // 更新数据库状态为失败
- try {
- await prisma.audioRecord.update({
- where: { audioId },
- data: { status: 'failed', errorMsg: '任务超时:生成进程异常终止' },
- });
- } catch (e) {}
- return { status: 'failed', audio: { error: '生成进程异常终止' } };
- }
- return { status: 'processing' };
- }
- // 不存在
- return { status: 'not_found' };
- }
- // 获取可用音色
- export function getVoices(): Voice[] {
- return VOICES;
- }
- // ============ LRC 歌词生成 ============
- /** 格式化秒数为 [MM:SS.XX] */
- function formatLrcTimestamp(seconds: number): string {
- const m = Math.floor(seconds / 60);
- const s = seconds % 60;
- const cs = Math.round((s - Math.floor(s)) * 100);
- const sec = Math.floor(s);
- return `${m.toString().padStart(2, '0')}:${sec.toString().padStart(2, '0')}.${cs.toString().padStart(2, '0')}`;
- }
- /** 清洗文本中的换行符(LRC每行只能有一个时间戳,文本中不能含换行) */
- function sanitizeLrcText(s: string): string {
- return s.replace(/\n+/g, ' ').trim();
- }
- /** 统计可见字符数(去除空白) */
- function countVisible(s: string): number {
- return s.replace(/\s/g, '').length;
- }
- /** 最大每行字符数(超过则进一步拆分) */
- const MAX_CHARS_PER_LINE = 50;
- /**
- * 将文本拆分为适合 LRC 显示的小句子
- * 策略:
- * 1. 先按 Markdown 标题(#/##)切分,标题独立一行
- * 2. 按句末标点(。!?)切分
- * 3. 按分号/冒号(;:)切分
- * 4. 按逗号(,,)切分
- * 5. 按换行切分
- * 6. 兜底:按固定字符数切分
- * 每行不超过 MAX_CHARS_PER_LINE 字符
- */
- function splitIntoLrcSentences(text: string): string[] {
- const result: string[] = [];
- const maxChars = MAX_CHARS_PER_LINE;
- // 第一步:按 Markdown 标题切分,标题独立成行
- const titleParts = text.split(/(?=^#{1,3}\s)/m);
-
- for (const part of titleParts) {
- const trimmed = part.trim();
- if (!trimmed) continue;
- // 如果是标题行(单独一行)
- const titleMatch = trimmed.match(/^(#{1,3}\s+.+?)(\n|$)/);
- if (titleMatch) {
- const titleLine = titleMatch[1].trim();
- const rest = trimmed.substring(titleMatch[0].length).trim();
-
- // 标题单独一行
- result.push(titleLine);
-
- // 处理标题后面的正文
- if (rest) {
- result.push(...splitBodyIntoSentences(rest, maxChars));
- }
- } else {
- // 没有标题,直接处理正文
- result.push(...splitBodyIntoSentences(trimmed, maxChars));
- }
- }
- return result.filter(s => countVisible(s) > 0);
- }
- /**
- * 将正文拆分为小句子
- * 优先级:句号 > 分号/冒号 > 逗号 > 换行 > 固定长度
- */
- function splitBodyIntoSentences(text: string, maxChars: number): string[] {
- const result: string[] = [];
- // 先按段落(空行/换行)粗分
- const paragraphs = text.split(/\n+/).filter(p => p.trim());
-
- for (const para of paragraphs) {
- const trimmed = para.trim();
- if (!trimmed) continue;
- // 如果整段就小于 maxChars,直接加入
- if (countVisible(trimmed) <= maxChars) {
- result.push(trimmed);
- continue;
- }
- // 按句末标点切分(中文:。!?,英文:.!?)
- let sentences = splitByPunctuation(trimmed, /[。!?.!?]+/, maxChars);
- // 如果切分后仍有超长句子,按顿号再切
- sentences = furtherSplit(sentences, /[、]/, maxChars);
-
- // 兜底:强制按字符数切分
- sentences = forceSplitByCharCount(sentences, maxChars);
-
- result.push(...sentences);
- }
- return result;
- }
- /** 按指定标点切分,标点附在前面那句末尾 */
- function splitByPunctuation(text: string, punctRegex: RegExp, maxChars: number): string[] {
- const parts: string[] = [];
- let remaining = text;
- while (remaining.length > 0) {
- // 查找下一个标点位置
- const match = remaining.match(punctRegex);
- if (!match || match.index === undefined) {
- // 没有更多标点,剩余部分整体加入
- parts.push(remaining);
- break;
- }
- const cutPos = match.index + match[0].length;
- const sentence = remaining.substring(0, cutPos).trim();
- if (sentence) {
- parts.push(sentence);
- }
- remaining = remaining.substring(cutPos).trim();
- }
- return parts;
- }
- /** 对已切分的句子,如果某些句子仍然超长,用更细粒度的标点再切 */
- function furtherSplit(sentences: string[], punctRegex: RegExp, maxChars: number): string[] {
- const result: string[] = [];
- for (const s of sentences) {
- if (countVisible(s) <= maxChars) {
- result.push(s);
- continue;
- }
- // 用更细粒度标点再切
- const subParts = splitByPunctuation(s, punctRegex, maxChars);
- result.push(...subParts);
- }
- return result;
- }
- /** 兜底:强制按字符数切分超长句子 */
- function forceSplitByCharCount(sentences: string[], maxChars: number): string[] {
- const result: string[] = [];
- for (const s of sentences) {
- if (countVisible(s) <= maxChars) {
- result.push(s);
- continue;
- }
- // 按可见字符数强制切分
- let buf = '';
- let visibleCount = 0;
- for (const ch of s) {
- buf += ch;
- if (!/\s/.test(ch)) visibleCount++;
- if (visibleCount >= maxChars) {
- result.push(buf.trim());
- buf = '';
- visibleCount = 0;
- }
- }
- if (buf.trim()) result.push(buf.trim());
- }
- return result;
- }
- /**
- * 生成 LRC 歌词时间轴
- *
- * 核心算法:**按句子拆分 + 均匀语速分配**
- *
- * 原理:
- * - TTS 生成的音频语速相对均匀
- * - 用 总时长/总字数 得到真实平均语速(秒/字)
- * - 每句时长 = 该句字数 × 平均语速
- * - 这样保证所有句子时间之和 = 总时长,且短句少分时间、长句多分时间
- */
- export function generateLrc(text: string, duration: number, audioPath?: string): string {
- if (!text || duration <= 0) return '';
- try {
- const sentences = splitIntoLrcSentences(text);
- if (sentences.length === 0) return '';
- logToFile(`🎵 LRC 拆分: ${sentences.length} 句, 总时长=${duration}s`);
- // 计算总可见字符数
- const totalChars = sentences.reduce((sum, s) => sum + countVisible(s), 0);
- if (totalChars === 0) return '';
- // 核心:均匀语速 = 总时长 / 总字数
- const speechRate = duration / totalChars; // 秒/字
- logToFile(`🎵 语速: ${speechRate.toFixed(3)}s/字, 总字数=${totalChars}`);
- const lines: string[] = [];
- let currentTime = 0;
- for (let i = 0; i < sentences.length; i++) {
- const charCount = countVisible(sentences[i]);
- const isTitle = /^#{1,3}\s/.test(sentences[i].trim());
- // 标题行语速较慢(通常有停顿),给予 1.5 倍时长系数
- const titleBoost = isTitle ? 1.5 : 1.0;
- const lineDuration = charCount * speechRate * titleBoost;
- lines.push(`[${formatLrcTimestamp(currentTime)}] ${sanitizeLrcText(sentences[i])}`);
-
- logToFile(` [${formatLrcTimestamp(currentTime)}] ${charCount}字 ${sentences[i].substring(0, 30)}...`);
-
- currentTime += lineDuration;
- }
- // 由于标题行增加了时长系数,实际计算总时长可能超过音频时长
- // 用线性缩放将所有时间戳归一化到 [0, duration] 区间
- if (currentTime > duration && lines.length > 0) {
- const scaleFactor = duration / currentTime;
- const rescaledLines: string[] = [];
- let rescaledTime = 0;
- for (let i = 0; i < sentences.length; i++) {
- const charCount = countVisible(sentences[i]);
- const isTitle = /^#{1,3}\s/.test(sentences[i].trim());
- const titleBoost = isTitle ? 1.5 : 1.0;
- const rawLineDuration = charCount * speechRate * titleBoost;
- const scaledLineDuration = rawLineDuration * scaleFactor;
- rescaledLines.push(`[${formatLrcTimestamp(rescaledTime)}] ${sanitizeLrcText(sentences[i])}`);
- rescaledTime += scaledLineDuration;
- }
- logToFile(`🎵 LRC 标题补偿缩放: 原始总时长=${currentTime.toFixed(2)}s, 缩放因子=${scaleFactor.toFixed(3)}`);
- return rescaledLines.join('\n');
- }
- // 确保最后一行不超过总时长
- logToFile(`🎵 LRC 完成: ${lines.length} 行, 末尾时间=${currentTime.toFixed(2)}s, 音频时长=${duration}s`);
- return lines.join('\n');
- } catch (err: any) {
- logToFile(`❌ generateLrc 异常: ${err.message}`);
- console.error(`❌ generateLrc 异常:`, err.message);
- return buildSimpleLrc(text, duration);
- }
- }
- /** 最简降级:按句号或换行分割,再不行按固定长度切分 */
- function buildSimpleLrc(text: string, duration: number): string {
- // 中英文句末标点(句号、感叹号、问号)
- let sentences = text.split(/(?<=[。!?.!?])\s*/).filter(s => s.trim());
- if (sentences.length <= 1) {
- sentences = text.split(/\n+/).filter(s => s.trim());
- }
- if (sentences.length <= 1 && text.length > 20) {
- // 英文按句号切分后再按固定长度兜底
- const englishSentences = text.split(/(?<=[.!?])\s+/).filter(s => s.trim().length > 0);
- if (englishSentences.length > 1) {
- sentences = englishSentences;
- } else {
- const chunkSize = 40;
- sentences = [];
- for (let i = 0; i < text.length; i += chunkSize) {
- sentences.push(text.substring(i, i + chunkSize));
- }
- }
- }
- if (sentences.length === 0) return '';
- const totalChars = sentences.reduce((sum, s) => sum + countVisible(s), 0);
- if (totalChars === 0) return '';
- const speechRate = duration / totalChars;
- const lines: string[] = [];
- let t = 0;
- for (const s of sentences) {
- lines.push(`[${formatLrcTimestamp(t)}] ${sanitizeLrcText(s)}`);
- t += countVisible(s) * speechRate;
- }
- return lines.join('\n');
- }
- // 获取可用 TTS 服务商(从注册表读取)
- export function getAvailableProviders() {
- const registry = getTtsRegistry();
- return registry.listEnabled().map(n => ({
- id: n.provider.vendor,
- name: n.provider.name,
- enabled: n.enabled,
- state: n.breaker.getState(),
- }));
- }
- // 生成预览音频(短文本)
- export async function generatePreview(
- voiceId: string,
- voiceParams?: { speed?: number; pitch?: number; volume?: number },
- providerType?: 'aliyun' | 'minimax'
- ): Promise<{ audioId: string; audioUrl: string }> {
- const previewText = '你好,欢迎使用AI有声书';
- const audioId = 'preview-' + uuidv4();
- const audioDir = path.join(config.upload.dir, audioId);
- if (!fs.existsSync(audioDir)) {
- fs.mkdirSync(audioDir, { recursive: true });
- }
- const params = {
- speed: voiceParams?.speed ?? 1.0,
- pitch: voiceParams?.pitch ?? 0,
- volume: voiceParams?.volume ?? 50,
- };
- console.log(`[TTS Preview] params: speed=${params.speed}, pitch=${params.pitch}, volume=${params.volume}`);
- const registry = getTtsRegistry();
- // 优先使用的 Provider
- const preferredName = providerType === 'minimax' ? 'minimax-tts' : providerType === 'aliyun' ? 'bailian-tts' : undefined;
- const startNode = preferredName ? registry.get(preferredName) : undefined;
- // 按优先级尝试所有可用 Provider
- const candidates = startNode
- ? [startNode, ...registry.listAvailable().filter(n => n !== startNode)]
- : registry.listAvailable();
- let lastError: Error | null = null;
- for (const node of candidates) {
- if (node.exhausted || node.breaker.isOpen()) continue;
- const tts = node.provider;
- const voiceName = tts.vendor.startsWith('minimax') ? getMiniMaxVoice(voiceId) : getAliyunVoice(voiceId);
- if (tts.mode === 'mock') {
- const mockPath = path.join(audioDir, 'preview.mp3');
- fs.writeFileSync(mockPath, Buffer.alloc(1024));
- return { audioId, audioUrl: `/uploads/${audioId}/preview.mp3` };
- }
- try {
- // 先创建空文件让 aliyun 下载能定位目录
- const ext = tts.vendor.startsWith('minimax') ? 'mp3' : 'wav';
- const outputPath = path.join(audioDir, `preview.${ext}`);
- if (!fs.existsSync(outputPath)) {
- fs.writeFileSync(outputPath, Buffer.alloc(0));
- }
- const actualPath = await node.breaker.call(() => tts.synthesize(previewText, voiceName, params, outputPath));
- const audioUrl = await storageService.uploadAudio(actualPath, audioId);
- console.log(`✅ [TTS Preview] ${tts.name} 生成成功`);
- return { audioId, audioUrl };
- } catch (error: any) {
- console.error(`❌ [TTS Preview] ${tts.name} 失败:`, error.message);
- lastError = error;
- // 额度耗尽标记
- if (['quota', 'balance', 'insufficient', 'usage limit'].some(k => error.message?.toLowerCase().includes(k))) {
- registry.markExhausted(tts.name, error.message, 4 * 60 * 60 * 1000);
- }
- continue;
- }
- }
- console.error('❌ 预览生成失败:', lastError?.message);
- throw lastError || new Error('所有 TTS Provider 都已尝试');
- }
- // ============ 初始化 ============
- // 首次调用时初始化注册表并启动健康检查
- startTtsHealthCheck();
|