/** * TTS (Text-to-Speech) Provider Implementation * * Factory pattern for routing TTS requests to appropriate provider implementations. * Follows the same architecture as lib/ai/providers.ts for consistency. * * Currently Supported Providers: * - OpenAI TTS: https://platform.openai.com/docs/guides/text-to-speech * - Azure TTS: https://learn.microsoft.com/en-us/azure/ai-services/speech-service/text-to-speech * - GLM TTS: https://docs.bigmodel.cn/cn/guide/models/sound-and-video/glm-tts * - Qwen TTS: https://bailian.console.aliyun.com/ * - MiniMax TTS: https://platform.minimaxi.com/docs/api-reference/speech-t2a-http * - Doubao TTS: https://www.volcengine.com/docs/6561/1257543 * - ElevenLabs TTS: https://elevenlabs.io/docs/api-reference/text-to-speech/convert * - Browser Native: Web Speech API (client-side only) * * HOW TO ADD A NEW PROVIDER: * * 1. Add provider ID to TTSProviderId in lib/audio/types.ts * Example: | 'elevenlabs-tts' * * 2. Add provider configuration to lib/audio/constants.ts * Example: * 'elevenlabs-tts': { * id: 'elevenlabs-tts', * name: 'ElevenLabs', * requiresApiKey: true, * defaultBaseUrl: 'https://api.elevenlabs.io/v1', * icon: '/logos/elevenlabs.svg', * voices: [...], * supportedFormats: ['mp3', 'pcm'], * speedRange: { min: 0.5, max: 2.0, default: 1.0 } * } * * 3. Implement provider function in this file * Pattern: async function generateXxxTTS(config, text): Promise * - Validate config and build API request * - Handle API authentication (apiKey, headers) * - Convert provider-specific parameters (voice, speed, format) * - Return { audio: Uint8Array, format: string } * * Example: * async function generateElevenLabsTTS( * config: TTSModelConfig, * text: string * ): Promise { * const baseUrl = config.baseUrl || TTS_PROVIDERS['elevenlabs-tts'].defaultBaseUrl; * * const response = await fetch(`${baseUrl}/text-to-speech/${config.voice}`, { * method: 'POST', * headers: { * 'xi-api-key': config.apiKey!, * 'Content-Type': 'application/json', * }, * body: JSON.stringify({ * text, * model_id: 'eleven_multilingual_v2', * voice_settings: { * stability: 0.5, * similarity_boost: 0.75, * } * }), * }); * * if (!response.ok) { * throw new Error(`ElevenLabs TTS API error: ${response.statusText}`); * } * * const arrayBuffer = await response.arrayBuffer(); * return { * audio: new Uint8Array(arrayBuffer), * format: 'mp3', * }; * } * * 4. Add case to generateTTS() switch statement * case 'elevenlabs-tts': * return await generateElevenLabsTTS(config, text); * * 5. Add i18n translations in lib/i18n.ts * providerElevenLabsTTS: { zh: 'ElevenLabs TTS', en: 'ElevenLabs TTS' } * * Error Handling Patterns: * - Always validate API key if requiresApiKey is true * - Throw descriptive errors for API failures * - Include response.statusText or error messages from API * - For client-only providers (browser-native), throw error directing to client-side usage * * API Call Patterns: * - Direct API: Use fetch with appropriate headers and body format (recommended for better encoding support) * - SSML: For Azure-like providers requiring SSML markup * - URL-based: For providers returning audio URL (download in second step) */ import type { TTSModelConfig } from './types'; import { TTS_PROVIDERS } from './constants'; /** * Result of TTS generation */ export interface TTSGenerationResult { audio: Uint8Array; format: string; } /** * Thrown when a TTS provider returns a rate-limit / concurrency-quota error. * Allows downstream consumers to distinguish rate-limit errors from other TTS failures. * * TODO: The API route currently catches all errors uniformly as GENERATION_FAILED. * This class enables future retry/backoff logic without changing the throw sites. */ export class TTSRateLimitError extends Error { constructor( public readonly provider: string, message: string, ) { super(message); this.name = 'TTSRateLimitError'; } } /** * Generate speech using specified TTS provider */ export async function generateTTS( config: TTSModelConfig, text: string, ): Promise { const provider = TTS_PROVIDERS[config.providerId]; if (!provider) { throw new Error(`Unknown TTS provider: ${config.providerId}`); } // Validate API key if required if (provider.requiresApiKey && !config.apiKey) { throw new Error(`API key required for TTS provider: ${config.providerId}`); } switch (config.providerId) { case 'openai-tts': return await generateOpenAITTS(config, text); case 'azure-tts': return await generateAzureTTS(config, text); case 'glm-tts': return await generateGLMTTS(config, text); case 'qwen-tts': return await generateQwenTTS(config, text); case 'minimax-tts': return await generateMiniMaxTTS(config, text); case 'doubao-tts': return await generateDoubaoTTS(config, text); case 'elevenlabs-tts': return await generateElevenLabsTTS(config, text); case 'browser-native-tts': throw new Error( 'Browser Native TTS must be handled client-side using Web Speech API. This provider cannot be used on the server.', ); default: throw new Error(`Unsupported TTS provider: ${config.providerId}`); } } /** * OpenAI TTS implementation (direct API call with explicit UTF-8 encoding) */ async function generateOpenAITTS( config: TTSModelConfig, text: string, ): Promise { const baseUrl = config.baseUrl || TTS_PROVIDERS['openai-tts'].defaultBaseUrl; // Use gpt-4o-mini-tts for best quality and intelligent realtime applications const response = await fetch(`${baseUrl}/audio/speech`, { method: 'POST', headers: { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json; charset=utf-8', }, body: JSON.stringify({ model: config.modelId || 'gpt-4o-mini-tts', input: text, voice: config.voice, speed: config.speed || 1.0, }), }); if (!response.ok) { const error = await response.json().catch(() => ({ error: response.statusText })); throw new Error(`OpenAI TTS API error: ${error.error?.message || response.statusText}`); } const arrayBuffer = await response.arrayBuffer(); return { audio: new Uint8Array(arrayBuffer), format: 'mp3', }; } /** * Azure TTS implementation (direct API call with SSML) */ async function generateAzureTTS( config: TTSModelConfig, text: string, ): Promise { const baseUrl = config.baseUrl || TTS_PROVIDERS['azure-tts'].defaultBaseUrl; // Build SSML const rate = config.speed ? `${((config.speed - 1) * 100).toFixed(0)}%` : '0%'; const ssml = ` ${escapeXml(text)} `.trim(); const response = await fetch(`${baseUrl}/cognitiveservices/v1`, { method: 'POST', headers: { 'Ocp-Apim-Subscription-Key': config.apiKey!, 'Content-Type': 'application/ssml+xml; charset=utf-8', 'X-Microsoft-OutputFormat': 'audio-16khz-128kbitrate-mono-mp3', }, body: ssml, }); if (!response.ok) { throw new Error(`Azure TTS API error: ${response.statusText}`); } const arrayBuffer = await response.arrayBuffer(); return { audio: new Uint8Array(arrayBuffer), format: 'mp3', }; } /** * GLM TTS implementation (GLM API) */ async function generateGLMTTS(config: TTSModelConfig, text: string): Promise { const baseUrl = config.baseUrl || TTS_PROVIDERS['glm-tts'].defaultBaseUrl; const response = await fetch(`${baseUrl}/audio/speech`, { method: 'POST', headers: { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json; charset=utf-8', }, body: JSON.stringify({ model: config.modelId || 'glm-tts', input: text, voice: config.voice, speed: config.speed || 1.0, volume: 1.0, response_format: 'wav', }), }); if (!response.ok) { const errorText = await response.text().catch(() => response.statusText); let errorMessage = `GLM TTS API error: ${errorText}`; try { const errorJson = JSON.parse(errorText); if (errorJson.error?.message) { errorMessage = `GLM TTS API error: ${errorJson.error.message} (code: ${errorJson.error.code})`; } } catch { // If not JSON, use the text as is } throw new Error(errorMessage); } const arrayBuffer = await response.arrayBuffer(); return { audio: new Uint8Array(arrayBuffer), format: 'wav', }; } /** * Qwen TTS implementation (DashScope API - Qwen3 TTS Flash) */ async function generateQwenTTS(config: TTSModelConfig, text: string): Promise { const baseUrl = config.baseUrl || TTS_PROVIDERS['qwen-tts'].defaultBaseUrl; // Calculate speed: Qwen3 uses rate parameter from -500 to 500 // speed 1.0 = rate 0, speed 2.0 = rate 500, speed 0.5 = rate -250 const rate = Math.round(((config.speed || 1.0) - 1.0) * 500); const response = await fetch(`${baseUrl}/services/aigc/multimodal-generation/generation`, { method: 'POST', headers: { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json; charset=utf-8', }, body: JSON.stringify({ model: config.modelId || 'qwen3-tts-flash', input: { text, voice: config.voice, language_type: 'Chinese', // Default to Chinese, can be made configurable }, parameters: { rate, // Speech rate from -500 to 500 }, }), }); if (!response.ok) { const errorText = await response.text().catch(() => response.statusText); throw new Error(`Qwen TTS API error: ${errorText}`); } const data = await response.json(); // Check for audio URL in response if (!data.output?.audio?.url) { throw new Error(`Qwen TTS error: No audio URL in response. Response: ${JSON.stringify(data)}`); } // Download audio from URL const audioUrl = data.output.audio.url; const audioResponse = await fetch(audioUrl); if (!audioResponse.ok) { throw new Error(`Failed to download audio from URL: ${audioResponse.statusText}`); } const arrayBuffer = await audioResponse.arrayBuffer(); return { audio: new Uint8Array(arrayBuffer), format: 'wav', // Qwen3 TTS returns WAV format }; } /** * MiniMax TTS implementation (synchronous HTTP API) */ async function generateMiniMaxTTS( config: TTSModelConfig, text: string, ): Promise { const baseUrl = (config.baseUrl || TTS_PROVIDERS['minimax-tts'].defaultBaseUrl || '').replace( /\/$/, '', ); const response = await fetch(`${baseUrl}/v1/t2a_v2`, { method: 'POST', headers: { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json; charset=utf-8', }, body: JSON.stringify({ model: config.modelId || 'speech-2.8-hd', text, stream: false, output_format: 'hex', voice_setting: { voice_id: config.voice, speed: config.speed || 1.0, vol: 1, pitch: 0, }, audio_setting: { sample_rate: 32000, bitrate: 128000, format: config.format || 'mp3', channel: 1, }, language_boost: 'auto', }), }); if (!response.ok) { const errorText = await response.text().catch(() => response.statusText); throw new Error(`MiniMax TTS API error: ${errorText}`); } const data = await response.json(); const hexAudio = data?.data?.audio; if (!hexAudio || typeof hexAudio !== 'string') { throw new Error(`MiniMax TTS error: No audio returned. Response: ${JSON.stringify(data)}`); } const cleanedHex = hexAudio.trim(); if (cleanedHex.length % 2 !== 0) { throw new Error('MiniMax TTS error: invalid hex audio payload length'); } const audio = new Uint8Array( cleanedHex.match(/.{1,2}/g)?.map((byte: string) => parseInt(byte, 16)) || [], ); return { audio, format: data?.extra_info?.audio_format || config.format || 'mp3', }; } /** * ElevenLabs TTS implementation (direct API call with voice-specific endpoint) */ async function generateElevenLabsTTS( config: TTSModelConfig, text: string, ): Promise { const baseUrl = config.baseUrl || TTS_PROVIDERS['elevenlabs-tts'].defaultBaseUrl; const requestedFormat = config.format || 'mp3'; const clampedSpeed = Math.min(1.2, Math.max(0.7, config.speed || 1.0)); const outputFormatMap: Record = { mp3: 'mp3_44100_128', opus: 'opus_48000_96', pcm: 'pcm_44100', wav: 'wav_44100', ulaw: 'ulaw_8000', alaw: 'alaw_8000', }; const outputFormat = outputFormatMap[requestedFormat] || outputFormatMap.mp3; const response = await fetch( `${baseUrl}/text-to-speech/${encodeURIComponent(config.voice)}?output_format=${outputFormat}`, { method: 'POST', headers: { 'xi-api-key': config.apiKey!, 'Content-Type': 'application/json; charset=utf-8', }, body: JSON.stringify({ text, model_id: config.modelId || 'eleven_multilingual_v2', voice_settings: { stability: 0.5, similarity_boost: 0.75, speed: clampedSpeed, }, }), }, ); if (!response.ok) { const errorText = await response.text().catch(() => response.statusText); throw new Error(`ElevenLabs TTS API error: ${errorText || response.statusText}`); } const arrayBuffer = await response.arrayBuffer(); return { audio: new Uint8Array(arrayBuffer), format: requestedFormat, }; } /** * Get current TTS configuration from settings store * Note: This function should only be called in browser context */ export async function getCurrentTTSConfig(): Promise { if (typeof window === 'undefined') { throw new Error('getCurrentTTSConfig() can only be called in browser context'); } // Lazy import to avoid circular dependency const { useSettingsStore } = await import('@/lib/store/settings'); const { ttsProviderId, ttsVoice, ttsSpeed, ttsProvidersConfig } = useSettingsStore.getState(); const providerConfig = ttsProvidersConfig?.[ttsProviderId]; return { providerId: ttsProviderId, modelId: providerConfig?.modelId || TTS_PROVIDERS[ttsProviderId]?.defaultModelId || '', apiKey: providerConfig?.apiKey, baseUrl: providerConfig?.baseUrl, voice: ttsVoice, speed: ttsSpeed, }; } // Re-export from constants for convenience export { getAllTTSProviders, getTTSProvider, getTTSVoices } from './constants'; /** * Doubao TTS 2.0 implementation (Volcengine Seed-TTS 2.0) */ async function generateDoubaoTTS( config: TTSModelConfig, text: string, ): Promise { const colonIdx = (config.apiKey || '').indexOf(':'); if (colonIdx <= 0) { throw new Error( 'Doubao TTS requires API key in format "appId:accessKey". Get both from the Volcengine console.', ); } const appId = config.apiKey!.slice(0, colonIdx); const accessKey = config.apiKey!.slice(colonIdx + 1); const baseUrl = config.baseUrl || TTS_PROVIDERS['doubao-tts'].defaultBaseUrl; const speechRate = Math.round(((config.speed || 1.0) - 1.0) * 100); const response = await fetch(`${baseUrl}/unidirectional`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Api-App-Id': appId, 'X-Api-Access-Key': accessKey, 'X-Api-Resource-Id': 'seed-tts-2.0', }, body: JSON.stringify({ user: { uid: 'openmaic' }, req_params: { text, speaker: config.voice, audio_params: { format: 'mp3', sample_rate: 24000, speech_rate: speechRate }, }, }), }); if (!response.ok) { const errorText = await response.text().catch(() => response.statusText); throw new Error(`Doubao TTS API error (${response.status}): ${errorText}`); } const responseText = await response.text(); const audioChunks: Uint8Array[] = []; let depth = 0; let start = -1; for (let i = 0; i < responseText.length; i++) { if (responseText[i] === '{') { if (depth === 0) start = i; depth++; } else if (responseText[i] === '}') { depth--; if (depth === 0 && start >= 0) { let chunk: { code: number; message?: string; data?: string }; try { chunk = JSON.parse(responseText.slice(start, i + 1)); } catch { start = -1; continue; } start = -1; if (chunk.code === 0 && chunk.data) { audioChunks.push(new Uint8Array(Buffer.from(chunk.data, 'base64'))); } else if (chunk.code === 20000000) { break; } else if (chunk.code && chunk.code !== 0) { if (chunk.code === 45000000 || chunk.code === 45000292) { throw new TTSRateLimitError( 'doubao-tts', chunk.message || 'concurrency quota exceeded', ); } throw new Error(`Doubao TTS error: ${chunk.message || 'unknown'} (code: ${chunk.code})`); } } } } if (audioChunks.length === 0) { throw new Error('Doubao TTS: no audio data received'); } const totalLength = audioChunks.reduce((sum, c) => sum + c.length, 0); const combined = new Uint8Array(totalLength); let offset = 0; for (const chunk of audioChunks) { combined.set(chunk, offset); offset += chunk.length; } return { audio: combined, format: 'mp3' }; } /** * Escape XML special characters for SSML */ function escapeXml(text: string): string { return text .replace(/&/g, '&') .replace(//g, '>') .replace(/"/g, '"') .replace(/'/g, '''); }