Codex
471 lines
15 KiB
Python
471 lines
15 KiB
Python
"""
|
||
语音识别服务
|
||
|
||
支持：
|
||
- 阿里云百炼 DashScope ASR（推荐）
|
||
- OpenAI Whisper API
|
||
"""
|
||
import logging
|
||
import os
|
||
import tempfile
|
||
import threading
|
||
import time
|
||
from pathlib import Path
|
||
from typing import Optional
|
||
|
||
import dashscope
|
||
from dashscope.audio.asr import Recognition, RecognitionCallback, RecognitionResult
|
||
from sqlalchemy import select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.config import settings
|
||
from ai_platform.models import LLMProvider
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class SpeechService:
|
||
"""
|
||
语音识别服务
|
||
|
||
支持多种语音识别后端：
|
||
- 阿里云百炼 DashScope（默认，推荐）
|
||
- OpenAI Whisper API
|
||
"""
|
||
|
||
def __init__(self, db: Optional[AsyncSession] = None):
|
||
self._db = db
|
||
# 延迟解析：优先从数据库获取，初始化时先从 settings 读取作为兜底（settings 已自动从环境变量读取）
|
||
self.dashscope_api_key = getattr(settings, 'DASHSCOPE_API_KEY', None)
|
||
self._resolved = False
|
||
|
||
async def _resolve_dashscope_api_key(self):
|
||
"""
|
||
异步解析 DashScope API Key
|
||
|
||
优先级：
|
||
1. 数据库中 qwen 类型提供商的 API Key
|
||
2. settings / 环境变量中的 DASHSCOPE_API_KEY（兜底）
|
||
"""
|
||
if self._resolved:
|
||
return
|
||
self._resolved = True
|
||
|
||
if not self._db:
|
||
return
|
||
|
||
try:
|
||
result = await self._db.execute(
|
||
select(LLMProvider).where(
|
||
LLMProvider.provider_type == 'qwen',
|
||
LLMProvider.is_active == True,
|
||
LLMProvider.is_deleted == False
|
||
)
|
||
)
|
||
provider = result.scalar_one_or_none()
|
||
if provider and provider.api_key:
|
||
self.dashscope_api_key = provider.api_key
|
||
logger.info(f"Speech: Using API key from database provider: {provider.name}")
|
||
except Exception as e:
|
||
logger.debug(f"Speech: Failed to get provider from database: {e}")
|
||
|
||
def transcribe(
|
||
self,
|
||
audio_file,
|
||
language: str = 'zh',
|
||
provider: str = 'dashscope',
|
||
) -> dict:
|
||
"""
|
||
语音转文字
|
||
|
||
Args:
|
||
audio_file: 音频文件（Django UploadedFile 或文件路径或 bytes）
|
||
language: 语言代码，如 'zh', 'en'
|
||
provider: 服务提供商，'dashscope' 或 'openai'
|
||
|
||
Returns:
|
||
{
|
||
'success': bool,
|
||
'text': str,
|
||
'duration': float,
|
||
'error': str,
|
||
}
|
||
"""
|
||
if provider == 'dashscope':
|
||
return self._transcribe_dashscope(audio_file, language)
|
||
elif provider == 'openai':
|
||
return self._transcribe_openai(audio_file, language)
|
||
else:
|
||
return {
|
||
'success': False,
|
||
'text': '',
|
||
'error': f'不支持的提供商: {provider}',
|
||
}
|
||
|
||
def _transcribe_dashscope(self, audio_file, language: str) -> dict:
|
||
"""
|
||
使用阿里云百炼 DashScope 进行语音识别
|
||
|
||
使用 fun-asr-realtime 模型，支持多种音频格式
|
||
"""
|
||
try:
|
||
if not self.dashscope_api_key:
|
||
return {
|
||
'success': False,
|
||
'text': '',
|
||
'error': '未配置 DASHSCOPE_API_KEY',
|
||
}
|
||
|
||
# 设置 API Key
|
||
dashscope.api_key = self.dashscope_api_key
|
||
dashscope.base_websocket_api_url = 'wss://dashscope.aliyuncs.com/api-ws/v1/inference'
|
||
|
||
# 获取音频数据和格式
|
||
audio_data, audio_format, sample_rate = self._prepare_audio(audio_file)
|
||
|
||
if not audio_data:
|
||
return {
|
||
'success': False,
|
||
'text': '',
|
||
'error': '无法读取音频文件',
|
||
}
|
||
|
||
# 使用同步方式收集识别结果
|
||
result_text = []
|
||
error_message = []
|
||
completed = threading.Event()
|
||
|
||
class SyncCallback(RecognitionCallback):
|
||
def on_complete(self) -> None:
|
||
completed.set()
|
||
|
||
def on_error(self, result: RecognitionResult) -> None:
|
||
error_message.append(result.message if hasattr(result, 'message') else str(result))
|
||
completed.set()
|
||
|
||
def on_event(self, result: RecognitionResult) -> None:
|
||
sentence = result.get_sentence()
|
||
if 'text' in sentence and RecognitionResult.is_sentence_end(sentence):
|
||
result_text.append(sentence['text'])
|
||
|
||
callback = SyncCallback()
|
||
|
||
# 创建识别实例
|
||
recognition = Recognition(
|
||
model='paraformer-realtime-v2', # 使用 paraformer 模型，效果更好
|
||
format=audio_format,
|
||
sample_rate=sample_rate,
|
||
callback=callback,
|
||
)
|
||
|
||
# 开始识别
|
||
recognition.start()
|
||
|
||
# 分块发送音频数据
|
||
chunk_size = 3200 # 每次发送 3200 字节
|
||
offset = 0
|
||
while offset < len(audio_data):
|
||
chunk = audio_data[offset:offset + chunk_size]
|
||
recognition.send_audio_frame(chunk)
|
||
offset += chunk_size
|
||
time.sleep(0.05) # 稍微延迟，模拟实时流
|
||
|
||
# 停止识别
|
||
recognition.stop()
|
||
|
||
# 等待完成（最多 30 秒）
|
||
completed.wait(timeout=30)
|
||
|
||
if error_message:
|
||
return {
|
||
'success': False,
|
||
'text': '',
|
||
'error': error_message[0],
|
||
}
|
||
|
||
final_text = ''.join(result_text)
|
||
|
||
return {
|
||
'success': True,
|
||
'text': final_text,
|
||
'duration': len(audio_data) / (sample_rate * 2), # 估算时长
|
||
'error': '',
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.exception(f'阿里云语音识别失败: {e}')
|
||
return {
|
||
'success': False,
|
||
'text': '',
|
||
'error': str(e),
|
||
}
|
||
|
||
def _prepare_audio(self, audio_file) -> tuple:
|
||
"""
|
||
准备音频数据
|
||
|
||
前端已直接录制 WAV 格式（16kHz, 16bit, 单声道），无需转换
|
||
阿里云 DashScope ASR 支持的格式：pcm, wav, mp3, opus, speex, aac, amr
|
||
|
||
Returns:
|
||
(audio_data: bytes, format: str, sample_rate: int)
|
||
"""
|
||
audio_data = None
|
||
audio_format = 'wav'
|
||
sample_rate = 16000
|
||
|
||
if isinstance(audio_file, bytes):
|
||
audio_data = audio_file
|
||
elif hasattr(audio_file, 'read'):
|
||
# Django UploadedFile
|
||
audio_data = audio_file.read()
|
||
file_name = getattr(audio_file, 'name', 'audio.wav').lower()
|
||
|
||
# 根据文件名判断格式
|
||
if file_name.endswith('.mp3'):
|
||
audio_format = 'mp3'
|
||
elif file_name.endswith('.pcm'):
|
||
audio_format = 'pcm'
|
||
elif file_name.endswith('.opus'):
|
||
audio_format = 'opus'
|
||
else:
|
||
audio_format = 'wav'
|
||
elif isinstance(audio_file, (str, Path)):
|
||
# 文件路径
|
||
file_path = Path(audio_file)
|
||
with open(file_path, 'rb') as f:
|
||
audio_data = f.read()
|
||
|
||
suffix = file_path.suffix.lower()
|
||
if suffix == '.mp3':
|
||
audio_format = 'mp3'
|
||
elif suffix == '.pcm':
|
||
audio_format = 'pcm'
|
||
elif suffix == '.opus':
|
||
audio_format = 'opus'
|
||
else:
|
||
audio_format = 'wav'
|
||
|
||
return audio_data, audio_format, sample_rate
|
||
|
||
def _transcribe_openai(self, audio_file, language: str, provider_config: dict = None) -> dict:
|
||
"""使用 OpenAI Whisper API 进行语音识别"""
|
||
try:
|
||
import openai
|
||
|
||
if not provider_config:
|
||
return {
|
||
'success': False,
|
||
'text': '',
|
||
'error': '未配置 OpenAI 提供商',
|
||
}
|
||
|
||
# 创建客户端
|
||
client = openai.OpenAI(
|
||
api_key=provider_config.get('api_key', ''),
|
||
base_url=provider_config.get('api_base') or None,
|
||
)
|
||
|
||
# 处理文件
|
||
if isinstance(audio_file, bytes):
|
||
# bytes 数据，写入临时文件
|
||
with tempfile.NamedTemporaryFile(
|
||
suffix='.wav',
|
||
delete=False,
|
||
) as tmp:
|
||
tmp.write(audio_file)
|
||
tmp_path = tmp.name
|
||
|
||
try:
|
||
with open(tmp_path, 'rb') as f:
|
||
response = client.audio.transcriptions.create(
|
||
model='whisper-1',
|
||
file=f,
|
||
language=language,
|
||
response_format='verbose_json',
|
||
)
|
||
finally:
|
||
Path(tmp_path).unlink(missing_ok=True)
|
||
elif hasattr(audio_file, 'read'):
|
||
# UploadedFile
|
||
file_content = audio_file.read()
|
||
file_name = getattr(audio_file, 'name', 'audio.webm')
|
||
|
||
# 写入临时文件
|
||
with tempfile.NamedTemporaryFile(
|
||
suffix=Path(file_name).suffix or '.webm',
|
||
delete=False,
|
||
) as tmp:
|
||
tmp.write(file_content)
|
||
tmp_path = tmp.name
|
||
|
||
try:
|
||
with open(tmp_path, 'rb') as f:
|
||
response = client.audio.transcriptions.create(
|
||
model='whisper-1',
|
||
file=f,
|
||
language=language,
|
||
response_format='verbose_json',
|
||
)
|
||
finally:
|
||
Path(tmp_path).unlink(missing_ok=True)
|
||
else:
|
||
with open(audio_file, 'rb') as f:
|
||
response = client.audio.transcriptions.create(
|
||
model='whisper-1',
|
||
file=f,
|
||
language=language,
|
||
response_format='verbose_json',
|
||
)
|
||
|
||
return {
|
||
'success': True,
|
||
'text': response.text,
|
||
'duration': getattr(response, 'duration', 0),
|
||
'error': '',
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.exception(f'OpenAI 语音识别失败: {e}')
|
||
return {
|
||
'success': False,
|
||
'text': '',
|
||
'error': str(e),
|
||
}
|
||
|
||
def text_to_speech(
|
||
self,
|
||
text: str,
|
||
voice: str = 'sambert-zhichu-v1',
|
||
provider: str = 'dashscope',
|
||
) -> dict:
|
||
"""
|
||
文字转语音（TTS）
|
||
|
||
Args:
|
||
text: 要转换的文字
|
||
voice: 声音类型
|
||
provider: 服务提供商
|
||
|
||
Returns:
|
||
{
|
||
'success': bool,
|
||
'audio_data': bytes,
|
||
'content_type': str,
|
||
'error': str,
|
||
}
|
||
"""
|
||
if provider == 'dashscope':
|
||
return self._tts_dashscope(text, voice)
|
||
elif provider == 'openai':
|
||
return self._tts_openai(text, voice)
|
||
else:
|
||
return {
|
||
'success': False,
|
||
'audio_data': b'',
|
||
'error': f'不支持的提供商: {provider}',
|
||
}
|
||
|
||
def _tts_dashscope(self, text: str, voice: str) -> dict:
|
||
"""使用阿里云百炼 TTS"""
|
||
try:
|
||
from dashscope.audio.tts import SpeechSynthesizer
|
||
|
||
if not self.dashscope_api_key:
|
||
return {
|
||
'success': False,
|
||
'audio_data': b'',
|
||
'error': '未配置 DASHSCOPE_API_KEY',
|
||
}
|
||
|
||
dashscope.api_key = self.dashscope_api_key
|
||
|
||
# 调用 TTS
|
||
result = SpeechSynthesizer.call(
|
||
model=voice, # sambert-zhichu-v1, sambert-zhimiao-emo-v1 等
|
||
text=text,
|
||
sample_rate=16000,
|
||
format='wav',
|
||
)
|
||
|
||
if result.get_audio_data():
|
||
return {
|
||
'success': True,
|
||
'audio_data': result.get_audio_data(),
|
||
'content_type': 'audio/wav',
|
||
'error': '',
|
||
}
|
||
else:
|
||
return {
|
||
'success': False,
|
||
'audio_data': b'',
|
||
'error': '语音合成失败',
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.exception(f'阿里云 TTS 失败: {e}')
|
||
return {
|
||
'success': False,
|
||
'audio_data': b'',
|
||
'error': str(e),
|
||
}
|
||
|
||
def _tts_openai(self, text: str, voice: str, provider_config: dict = None) -> dict:
|
||
"""使用 OpenAI TTS API"""
|
||
try:
|
||
import openai
|
||
|
||
if not provider_config:
|
||
return {
|
||
'success': False,
|
||
'audio_data': b'',
|
||
'error': '未配置 OpenAI 提供商',
|
||
}
|
||
|
||
client = openai.OpenAI(
|
||
api_key=provider_config.get('api_key', ''),
|
||
base_url=provider_config.get('api_base') or None,
|
||
)
|
||
|
||
response = client.audio.speech.create(
|
||
model='tts-1',
|
||
voice=voice, # alloy, echo, fable, onyx, nova, shimmer
|
||
input=text,
|
||
)
|
||
|
||
return {
|
||
'success': True,
|
||
'audio_data': response.content,
|
||
'content_type': 'audio/mpeg',
|
||
'error': '',
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.exception(f'OpenAI TTS 失败: {e}')
|
||
return {
|
||
'success': False,
|
||
'audio_data': b'',
|
||
'error': str(e),
|
||
}
|
||
|
||
async def get_openai_provider_config(self) -> dict:
|
||
"""异步获取OpenAI提供商配置"""
|
||
if not self._db:
|
||
return {}
|
||
|
||
result = await self._db.execute(
|
||
select(LLMProvider).where(
|
||
LLMProvider.provider_type == 'openai',
|
||
LLMProvider.is_active == True,
|
||
LLMProvider.is_deleted == False
|
||
)
|
||
)
|
||
provider = result.scalar_one_or_none()
|
||
if not provider:
|
||
return {}
|
||
|
||
return {
|
||
'api_key': provider.api_key,
|
||
'api_base': provider.api_base,
|
||
}
|