|
|
@@ -12,6 +12,7 @@ import { ITtsProvider } from './provider.interface';
|
|
|
import { CircuitBreakerOpenError } from '../../common/circuit-breaker';
|
|
|
import { ProviderNode } from '../../common/provider-registry';
|
|
|
import { ttsLogger } from './tts-logger';
|
|
|
+import { callLLMWithMessages, ChatMessage } from '../../services/llm';
|
|
|
import axios from 'axios';
|
|
|
|
|
|
// ============ CosyVoice 官方 Instruct 指令生成 ============
|
|
|
@@ -487,6 +488,108 @@ export function getVoiceInstruct(text: string, voiceId?: string): string {
|
|
|
return emotionOnlyInstruct(emotion);
|
|
|
}
|
|
|
|
|
|
+// ============ LLM 全文一次分析(折中方案:LLM优先 → 关键词降级)============
|
|
|
+
|
|
|
+/** 单段内容分析结果 */
|
|
|
+interface SegmentAnalysis {
|
|
|
+ index: number;
|
|
|
+ emotion?: string; // happy/sad/angry/fearful/surprised/disgusted/neutral
|
|
|
+ scene?: string; // 新闻播报/闲聊互动/...
|
|
|
+ role?: string; // 一个旁白
|
|
|
+ identity?: string; // 故事机
|
|
|
+}
|
|
|
+
|
|
|
+/** LLM 分析超时时间(ms) */
|
|
|
+const LLM_ANALYSIS_TIMEOUT = 4000;
|
|
|
+
|
|
|
+/**
|
|
|
+ * 用 LLM 一次性分析全文所有分段的情感/场景/角色/身份
|
|
|
+ * 输出 JSON 数组,每段一条记录
|
|
|
+ * 超时或失败返回 null,调用方降级到关键词匹配
|
|
|
+ */
|
|
|
+async function analyzeWithLLM(fullText: string, segments: string[]): Promise<SegmentAnalysis[] | null> {
|
|
|
+ if (segments.length === 0) return null;
|
|
|
+
|
|
|
+ const segmentList = segments.map((seg, i) => `[段${i}] ${seg.substring(0, 150)}${seg.length > 150 ? '…' : ''}`).join('\n---\n');
|
|
|
+
|
|
|
+ const systemPrompt = `你是文本情感和场景分析专家。分析每段文本,输出严格JSON数组(不要markdown包裹)。
|
|
|
+
|
|
|
+场景必须从以下7个中选择最匹配的:闲聊互动、新闻播报、广告促销、比赛解说、一些儿童内容解说、语音导航、脱口秀表演
|
|
|
+情感必须从以下7个中选择:neutral、happy、sad、angry、fearful、surprised、disgusted
|
|
|
+角色:旁白类内容填"一个旁白",否则null
|
|
|
+身份:儿童/童话类内容填"故事机",否则null
|
|
|
+优先级:有身份时role必须为null
|
|
|
+
|
|
|
+每段输出格式:{"index":段号,"emotion":"情感","scene":"场景","role":角色|null,"identity":身份|null}`;
|
|
|
+
|
|
|
+ const userPrompt = `分析以下${segments.length}段文本:\n\n${segmentList}`;
|
|
|
+
|
|
|
+ try {
|
|
|
+ const result = await Promise.race([
|
|
|
+ callLLMWithMessages([
|
|
|
+ { role: 'system', content: systemPrompt },
|
|
|
+ { role: 'user', content: userPrompt },
|
|
|
+ ], undefined, 512),
|
|
|
+ new Promise<null>((_, reject) =>
|
|
|
+ setTimeout(() => reject(new Error('LLM_ANALYSIS_TIMEOUT')), LLM_ANALYSIS_TIMEOUT)
|
|
|
+ ),
|
|
|
+ ]);
|
|
|
+
|
|
|
+ if (result === null) return null;
|
|
|
+
|
|
|
+ // 清理 JSON(去除可能的 markdown 包裹)
|
|
|
+ const jsonStr = result.replace(/```json\s*/gi, '').replace(/```\s*/g, '').trim();
|
|
|
+ const parsed = JSON.parse(jsonStr);
|
|
|
+
|
|
|
+ if (!Array.isArray(parsed)) return null;
|
|
|
+
|
|
|
+ // 校验并规范化
|
|
|
+ const validEmotions = ['neutral', 'fearful', 'angry', 'sad', 'surprised', 'happy', 'disgusted'];
|
|
|
+ const validScenes = ['闲聊互动', '新闻播报', '广告促销', '比赛解说', '一些儿童内容解说', '语音导航', '脱口秀表演'];
|
|
|
+
|
|
|
+ return parsed.map((item: any) => ({
|
|
|
+ index: typeof item.index === 'number' ? item.index : parseInt(item.index),
|
|
|
+ emotion: validEmotions.includes(item.emotion) ? item.emotion : 'neutral',
|
|
|
+ scene: validScenes.includes(item.scene) ? item.scene : '闲聊互动',
|
|
|
+ role: item.role === '一个旁白' ? '一个旁白' : undefined,
|
|
|
+ identity: item.identity === '故事机' ? '故事机' : undefined,
|
|
|
+ }));
|
|
|
+ } catch (err: any) {
|
|
|
+ const reason = err.message === 'LLM_ANALYSIS_TIMEOUT' ? '超时' : (err.message || '未知错误');
|
|
|
+ console.log(`🤖 [LLM分析] 降级到关键词 → ${reason}`);
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * 为所有分段生成 Instruct 指令(LLM优先 + 关键词降级)
|
|
|
+ * 返回与 segments 等长的数组,每项为对应段的 instructText 或 undefined
|
|
|
+ */
|
|
|
+async function analyzeAllSegments(
|
|
|
+ segments: string[],
|
|
|
+ voice: Voice,
|
|
|
+): Promise<(string | undefined)[]> {
|
|
|
+ // 第一步:尝试 LLM 一次分析
|
|
|
+ const fullText = segments.join('');
|
|
|
+ const llmResults = await analyzeWithLLM(fullText, segments);
|
|
|
+
|
|
|
+ // 第二步:逐段组装 Instruct
|
|
|
+ return segments.map((segment, i) => {
|
|
|
+ if (llmResults && llmResults[i]) {
|
|
|
+ const r = llmResults[i];
|
|
|
+ const instruct = generateInstruct({
|
|
|
+ identity: r.identity || voice.identity,
|
|
|
+ role: r.role || voice.role,
|
|
|
+ scene: r.scene || voice.scene,
|
|
|
+ emotion: r.emotion && r.emotion !== 'neutral' ? r.emotion : (voice.cosyEmotion || 'neutral'),
|
|
|
+ });
|
|
|
+ if (instruct) return instruct;
|
|
|
+ }
|
|
|
+ // 降级:关键词匹配
|
|
|
+ return generateSegmentInstruct(segment, voice);
|
|
|
+ });
|
|
|
+}
|
|
|
+
|
|
|
// ============ 官方 Instruct 支持的 7 个场景 ============
|
|
|
const OFFICIAL_SCENES = ['闲聊互动', '新闻播报', '广告促销', '比赛解说', '一些儿童内容解说', '语音导航', '脱口秀表演'] as const;
|
|
|
type OfficialScene = typeof OFFICIAL_SCENES[number];
|
|
|
@@ -1136,12 +1239,25 @@ async function processAudioGeneration(
|
|
|
const segments = splitText(cleanText, segmentMax);
|
|
|
console.log(`📝 文本已分段: ${segments.length} 段, 阈值=${segmentMax}字 (Provider: ${tts.name})`);
|
|
|
|
|
|
- // 🎭 逐段 Instruct:为每段文本独立分析内容,动态匹配场景/角色/身份+情感
|
|
|
+ // 🎭 逐段 Instruct:LLM 全文一次分析(优先)+ 关键词匹配(降级)
|
|
|
const baseVoiceForInstruct = UNIFIED_VOICES.find(v => v.id === effectiveVoiceId)
|
|
|
|| UNIFIED_VOICES.find(v => v.id === voiceId);
|
|
|
// 全局 instruct 作为兜底(用户手动指定 或 音色预设 优先)
|
|
|
const globalInstruct = (voiceParams as any).instructText;
|
|
|
|
|
|
+ // ⭐ 折中方案:LLM 一次分析所有分段(所有 provider 都运行,通过日志验证)
|
|
|
+ const canUseInstruct = INSTRUCT_ENABLED && baseVoiceForInstruct && !(voiceParams as any).emotion;
|
|
|
+ const segmentInstructs = canUseInstruct
|
|
|
+ ? await analyzeAllSegments(segments, baseVoiceForInstruct)
|
|
|
+ : [];
|
|
|
+ // 记录分析来源(LLM 或 关键词降级)
|
|
|
+ if (canUseInstruct && segments.length > 0) {
|
|
|
+ console.log(`🤖 [LLM分析] 完成 ${segments.length} 段分析,provider=${tts.vendor},instruct=${tts.vendor === 'bailian' ? '启用' : '仅日志'}`);
|
|
|
+ segmentInstructs.forEach((si, idx) => {
|
|
|
+ if (si) console.log(`🤖 [LLM分析] 段${idx} → ${si.substring(0, 60)}...`);
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
// 并行生成各段音频(使用 Provider 自身的并发设置,每段带重试)
|
|
|
const audioFiles: string[] = [];
|
|
|
const cloudUrls: string[] = [];
|
|
|
@@ -1154,11 +1270,10 @@ async function processAudioGeneration(
|
|
|
const results = await Promise.all(
|
|
|
batch.map(async (segment, idx) => {
|
|
|
const segPath = path.join(audioDir, `segment_${i + idx}.mp3`);
|
|
|
- // 🎭 逐段 Instruct:每段文本独立分析内容 → 动态生成场景+情感指令
|
|
|
+ // 🎭 使用预分析的 Instruct 结果
|
|
|
let segVoiceParams = voiceParams;
|
|
|
- if (INSTRUCT_ENABLED && tts.vendor === 'bailian' && baseVoiceForInstruct && !(voiceParams as any).emotion) {
|
|
|
- // 用户手动指定情感时 跳过逐段检测,使用用户选择
|
|
|
- const segInstruct = generateSegmentInstruct(segment, baseVoiceForInstruct);
|
|
|
+ if (canUseInstruct && tts.vendor === 'bailian') {
|
|
|
+ const segInstruct = segmentInstructs[i + idx];
|
|
|
if (segInstruct && segInstruct !== globalInstruct) {
|
|
|
segVoiceParams = { ...voiceParams, instructText: segInstruct };
|
|
|
console.log(`🎭 [SegInstruct] 段${i+idx} 「${segment.substring(0, 25)}...」→ ${segInstruct.substring(0, 50)}...`);
|