Build lightweight AI agent admin
This commit is contained in:
Codex
@@ -0,0 +1,470 @@
|
||||
"""
|
||||
语音识别服务
|
||||
|
||||
支持：
|
||||
- 阿里云百炼 DashScope ASR（推荐）
|
||||
- OpenAI Whisper API
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import dashscope
|
||||
from dashscope.audio.asr import Recognition, RecognitionCallback, RecognitionResult
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from ai_platform.models import LLMProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SpeechService:
|
||||
"""
|
||||
语音识别服务
|
||||
|
||||
支持多种语音识别后端：
|
||||
- 阿里云百炼 DashScope（默认，推荐）
|
||||
- OpenAI Whisper API
|
||||
"""
|
||||
|
||||
def __init__(self, db: Optional[AsyncSession] = None):
|
||||
self._db = db
|
||||
# 延迟解析：优先从数据库获取，初始化时先从 settings 读取作为兜底（settings 已自动从环境变量读取）
|
||||
self.dashscope_api_key = getattr(settings, 'DASHSCOPE_API_KEY', None)
|
||||
self._resolved = False
|
||||
|
||||
async def _resolve_dashscope_api_key(self):
|
||||
"""
|
||||
异步解析 DashScope API Key
|
||||
|
||||
优先级：
|
||||
1. 数据库中 qwen 类型提供商的 API Key
|
||||
2. settings / 环境变量中的 DASHSCOPE_API_KEY（兜底）
|
||||
"""
|
||||
if self._resolved:
|
||||
return
|
||||
self._resolved = True
|
||||
|
||||
if not self._db:
|
||||
return
|
||||
|
||||
try:
|
||||
result = await self._db.execute(
|
||||
select(LLMProvider).where(
|
||||
LLMProvider.provider_type == 'qwen',
|
||||
LLMProvider.is_active == True,
|
||||
LLMProvider.is_deleted == False
|
||||
)
|
||||
)
|
||||
provider = result.scalar_one_or_none()
|
||||
if provider and provider.api_key:
|
||||
self.dashscope_api_key = provider.api_key
|
||||
logger.info(f"Speech: Using API key from database provider: {provider.name}")
|
||||
except Exception as e:
|
||||
logger.debug(f"Speech: Failed to get provider from database: {e}")
|
||||
|
||||
def transcribe(
|
||||
self,
|
||||
audio_file,
|
||||
language: str = 'zh',
|
||||
provider: str = 'dashscope',
|
||||
) -> dict:
|
||||
"""
|
||||
语音转文字
|
||||
|
||||
Args:
|
||||
audio_file: 音频文件（Django UploadedFile 或文件路径或 bytes）
|
||||
language: 语言代码，如 'zh', 'en'
|
||||
provider: 服务提供商，'dashscope' 或 'openai'
|
||||
|
||||
Returns:
|
||||
{
|
||||
'success': bool,
|
||||
'text': str,
|
||||
'duration': float,
|
||||
'error': str,
|
||||
}
|
||||
"""
|
||||
if provider == 'dashscope':
|
||||
return self._transcribe_dashscope(audio_file, language)
|
||||
elif provider == 'openai':
|
||||
return self._transcribe_openai(audio_file, language)
|
||||
else:
|
||||
return {
|
||||
'success': False,
|
||||
'text': '',
|
||||
'error': f'不支持的提供商: {provider}',
|
||||
}
|
||||
|
||||
def _transcribe_dashscope(self, audio_file, language: str) -> dict:
|
||||
"""
|
||||
使用阿里云百炼 DashScope 进行语音识别
|
||||
|
||||
使用 fun-asr-realtime 模型，支持多种音频格式
|
||||
"""
|
||||
try:
|
||||
if not self.dashscope_api_key:
|
||||
return {
|
||||
'success': False,
|
||||
'text': '',
|
||||
'error': '未配置 DASHSCOPE_API_KEY',
|
||||
}
|
||||
|
||||
# 设置 API Key
|
||||
dashscope.api_key = self.dashscope_api_key
|
||||
dashscope.base_websocket_api_url = 'wss://dashscope.aliyuncs.com/api-ws/v1/inference'
|
||||
|
||||
# 获取音频数据和格式
|
||||
audio_data, audio_format, sample_rate = self._prepare_audio(audio_file)
|
||||
|
||||
if not audio_data:
|
||||
return {
|
||||
'success': False,
|
||||
'text': '',
|
||||
'error': '无法读取音频文件',
|
||||
}
|
||||
|
||||
# 使用同步方式收集识别结果
|
||||
result_text = []
|
||||
error_message = []
|
||||
completed = threading.Event()
|
||||
|
||||
class SyncCallback(RecognitionCallback):
|
||||
def on_complete(self) -> None:
|
||||
completed.set()
|
||||
|
||||
def on_error(self, result: RecognitionResult) -> None:
|
||||
error_message.append(result.message if hasattr(result, 'message') else str(result))
|
||||
completed.set()
|
||||
|
||||
def on_event(self, result: RecognitionResult) -> None:
|
||||
sentence = result.get_sentence()
|
||||
if 'text' in sentence and RecognitionResult.is_sentence_end(sentence):
|
||||
result_text.append(sentence['text'])
|
||||
|
||||
callback = SyncCallback()
|
||||
|
||||
# 创建识别实例
|
||||
recognition = Recognition(
|
||||
model='paraformer-realtime-v2', # 使用 paraformer 模型，效果更好
|
||||
format=audio_format,
|
||||
sample_rate=sample_rate,
|
||||
callback=callback,
|
||||
)
|
||||
|
||||
# 开始识别
|
||||
recognition.start()
|
||||
|
||||
# 分块发送音频数据
|
||||
chunk_size = 3200 # 每次发送 3200 字节
|
||||
offset = 0
|
||||
while offset < len(audio_data):
|
||||
chunk = audio_data[offset:offset + chunk_size]
|
||||
recognition.send_audio_frame(chunk)
|
||||
offset += chunk_size
|
||||
time.sleep(0.05) # 稍微延迟，模拟实时流
|
||||
|
||||
# 停止识别
|
||||
recognition.stop()
|
||||
|
||||
# 等待完成（最多 30 秒）
|
||||
completed.wait(timeout=30)
|
||||
|
||||
if error_message:
|
||||
return {
|
||||
'success': False,
|
||||
'text': '',
|
||||
'error': error_message[0],
|
||||
}
|
||||
|
||||
final_text = ''.join(result_text)
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'text': final_text,
|
||||
'duration': len(audio_data) / (sample_rate * 2), # 估算时长
|
||||
'error': '',
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f'阿里云语音识别失败: {e}')
|
||||
return {
|
||||
'success': False,
|
||||
'text': '',
|
||||
'error': str(e),
|
||||
}
|
||||
|
||||
def _prepare_audio(self, audio_file) -> tuple:
|
||||
"""
|
||||
准备音频数据
|
||||
|
||||
前端已直接录制 WAV 格式（16kHz, 16bit, 单声道），无需转换
|
||||
阿里云 DashScope ASR 支持的格式：pcm, wav, mp3, opus, speex, aac, amr
|
||||
|
||||
Returns:
|
||||
(audio_data: bytes, format: str, sample_rate: int)
|
||||
"""
|
||||
audio_data = None
|
||||
audio_format = 'wav'
|
||||
sample_rate = 16000
|
||||
|
||||
if isinstance(audio_file, bytes):
|
||||
audio_data = audio_file
|
||||
elif hasattr(audio_file, 'read'):
|
||||
# Django UploadedFile
|
||||
audio_data = audio_file.read()
|
||||
file_name = getattr(audio_file, 'name', 'audio.wav').lower()
|
||||
|
||||
# 根据文件名判断格式
|
||||
if file_name.endswith('.mp3'):
|
||||
audio_format = 'mp3'
|
||||
elif file_name.endswith('.pcm'):
|
||||
audio_format = 'pcm'
|
||||
elif file_name.endswith('.opus'):
|
||||
audio_format = 'opus'
|
||||
else:
|
||||
audio_format = 'wav'
|
||||
elif isinstance(audio_file, (str, Path)):
|
||||
# 文件路径
|
||||
file_path = Path(audio_file)
|
||||
with open(file_path, 'rb') as f:
|
||||
audio_data = f.read()
|
||||
|
||||
suffix = file_path.suffix.lower()
|
||||
if suffix == '.mp3':
|
||||
audio_format = 'mp3'
|
||||
elif suffix == '.pcm':
|
||||
audio_format = 'pcm'
|
||||
elif suffix == '.opus':
|
||||
audio_format = 'opus'
|
||||
else:
|
||||
audio_format = 'wav'
|
||||
|
||||
return audio_data, audio_format, sample_rate
|
||||
|
||||
def _transcribe_openai(self, audio_file, language: str, provider_config: dict = None) -> dict:
|
||||
"""使用 OpenAI Whisper API 进行语音识别"""
|
||||
try:
|
||||
import openai
|
||||
|
||||
if not provider_config:
|
||||
return {
|
||||
'success': False,
|
||||
'text': '',
|
||||
'error': '未配置 OpenAI 提供商',
|
||||
}
|
||||
|
||||
# 创建客户端
|
||||
client = openai.OpenAI(
|
||||
api_key=provider_config.get('api_key', ''),
|
||||
base_url=provider_config.get('api_base') or None,
|
||||
)
|
||||
|
||||
# 处理文件
|
||||
if isinstance(audio_file, bytes):
|
||||
# bytes 数据，写入临时文件
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix='.wav',
|
||||
delete=False,
|
||||
) as tmp:
|
||||
tmp.write(audio_file)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
with open(tmp_path, 'rb') as f:
|
||||
response = client.audio.transcriptions.create(
|
||||
model='whisper-1',
|
||||
file=f,
|
||||
language=language,
|
||||
response_format='verbose_json',
|
||||
)
|
||||
finally:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
elif hasattr(audio_file, 'read'):
|
||||
# UploadedFile
|
||||
file_content = audio_file.read()
|
||||
file_name = getattr(audio_file, 'name', 'audio.webm')
|
||||
|
||||
# 写入临时文件
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=Path(file_name).suffix or '.webm',
|
||||
delete=False,
|
||||
) as tmp:
|
||||
tmp.write(file_content)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
with open(tmp_path, 'rb') as f:
|
||||
response = client.audio.transcriptions.create(
|
||||
model='whisper-1',
|
||||
file=f,
|
||||
language=language,
|
||||
response_format='verbose_json',
|
||||
)
|
||||
finally:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
else:
|
||||
with open(audio_file, 'rb') as f:
|
||||
response = client.audio.transcriptions.create(
|
||||
model='whisper-1',
|
||||
file=f,
|
||||
language=language,
|
||||
response_format='verbose_json',
|
||||
)
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'text': response.text,
|
||||
'duration': getattr(response, 'duration', 0),
|
||||
'error': '',
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f'OpenAI 语音识别失败: {e}')
|
||||
return {
|
||||
'success': False,
|
||||
'text': '',
|
||||
'error': str(e),
|
||||
}
|
||||
|
||||
def text_to_speech(
|
||||
self,
|
||||
text: str,
|
||||
voice: str = 'sambert-zhichu-v1',
|
||||
provider: str = 'dashscope',
|
||||
) -> dict:
|
||||
"""
|
||||
文字转语音（TTS）
|
||||
|
||||
Args:
|
||||
text: 要转换的文字
|
||||
voice: 声音类型
|
||||
provider: 服务提供商
|
||||
|
||||
Returns:
|
||||
{
|
||||
'success': bool,
|
||||
'audio_data': bytes,
|
||||
'content_type': str,
|
||||
'error': str,
|
||||
}
|
||||
"""
|
||||
if provider == 'dashscope':
|
||||
return self._tts_dashscope(text, voice)
|
||||
elif provider == 'openai':
|
||||
return self._tts_openai(text, voice)
|
||||
else:
|
||||
return {
|
||||
'success': False,
|
||||
'audio_data': b'',
|
||||
'error': f'不支持的提供商: {provider}',
|
||||
}
|
||||
|
||||
def _tts_dashscope(self, text: str, voice: str) -> dict:
|
||||
"""使用阿里云百炼 TTS"""
|
||||
try:
|
||||
from dashscope.audio.tts import SpeechSynthesizer
|
||||
|
||||
if not self.dashscope_api_key:
|
||||
return {
|
||||
'success': False,
|
||||
'audio_data': b'',
|
||||
'error': '未配置 DASHSCOPE_API_KEY',
|
||||
}
|
||||
|
||||
dashscope.api_key = self.dashscope_api_key
|
||||
|
||||
# 调用 TTS
|
||||
result = SpeechSynthesizer.call(
|
||||
model=voice, # sambert-zhichu-v1, sambert-zhimiao-emo-v1 等
|
||||
text=text,
|
||||
sample_rate=16000,
|
||||
format='wav',
|
||||
)
|
||||
|
||||
if result.get_audio_data():
|
||||
return {
|
||||
'success': True,
|
||||
'audio_data': result.get_audio_data(),
|
||||
'content_type': 'audio/wav',
|
||||
'error': '',
|
||||
}
|
||||
else:
|
||||
return {
|
||||
'success': False,
|
||||
'audio_data': b'',
|
||||
'error': '语音合成失败',
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f'阿里云 TTS 失败: {e}')
|
||||
return {
|
||||
'success': False,
|
||||
'audio_data': b'',
|
||||
'error': str(e),
|
||||
}
|
||||
|
||||
def _tts_openai(self, text: str, voice: str, provider_config: dict = None) -> dict:
|
||||
"""使用 OpenAI TTS API"""
|
||||
try:
|
||||
import openai
|
||||
|
||||
if not provider_config:
|
||||
return {
|
||||
'success': False,
|
||||
'audio_data': b'',
|
||||
'error': '未配置 OpenAI 提供商',
|
||||
}
|
||||
|
||||
client = openai.OpenAI(
|
||||
api_key=provider_config.get('api_key', ''),
|
||||
base_url=provider_config.get('api_base') or None,
|
||||
)
|
||||
|
||||
response = client.audio.speech.create(
|
||||
model='tts-1',
|
||||
voice=voice, # alloy, echo, fable, onyx, nova, shimmer
|
||||
input=text,
|
||||
)
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'audio_data': response.content,
|
||||
'content_type': 'audio/mpeg',
|
||||
'error': '',
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f'OpenAI TTS 失败: {e}')
|
||||
return {
|
||||
'success': False,
|
||||
'audio_data': b'',
|
||||
'error': str(e),
|
||||
}
|
||||
|
||||
async def get_openai_provider_config(self) -> dict:
|
||||
"""异步获取OpenAI提供商配置"""
|
||||
if not self._db:
|
||||
return {}
|
||||
|
||||
result = await self._db.execute(
|
||||
select(LLMProvider).where(
|
||||
LLMProvider.provider_type == 'openai',
|
||||
LLMProvider.is_active == True,
|
||||
LLMProvider.is_deleted == False
|
||||
)
|
||||
)
|
||||
provider = result.scalar_one_or_none()
|
||||
if not provider:
|
||||
return {}
|
||||
|
||||
return {
|
||||
'api_key': provider.api_key,
|
||||
'api_base': provider.api_base,
|
||||
}
|
||||
Reference in New Issue
Block a user