bakaEC/olivebot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复麦克风&文字回复问题

Browse Source 
1、修复助理版文字输入不读取人设回复问题；
2、修复助理版文字输入不读取qa回复问题；
3、增强麦克风接入稳定性。
This commit is contained in:
xszyou 2023-07-12 18:56:17 +08:00
parent 5c5e9d08bd
commit b53bc6e124
5 changed files with 150 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -142,6 +142,12 @@ Fay（服务端）与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127
## **三、升级日志**
**2023.07.12：**
+ 修复助理版文字输入不读取人设回复问题；
+ 修复助理版文字输入不读取qa回复问题；
+ 增强麦克风接入稳定性。
**2023.07.05：**
+ 修复无法运行唇型算法而导致的不播放声音问题。

83
core/fay_core.py
View File

@ -1,4 +1,5 @@
import difflib
import imp
import math
import os
import random
@ -21,12 +22,13 @@ from core.interact import Interact
from core.tts_voice import EnumVoice
from scheduler.thread_manager import MyThread
from utils import util, storer, config_util
from core import qa_service
import pygame
from utils import config_util as cfg
from core.content_db import Content_Db
from datetime import datetime
from ai_module import nlp_rasa
from ai_module import nlp_chatgpt
from ai_module import nlp_gpt
@ -34,6 +36,7 @@ from ai_module import nlp_yuan
from ai_module import yolov8
from ai_module import nlp_VisualGLM
import platform
if platform.system() == "Windows":
import sys
@ -89,8 +92,21 @@ def determine_nlp_strategy(sendto,msg):
#文本消息处理
def send_for_answer(msg,sendto):
contentdb = Content_Db()
contentdb.add_content('member','send',msg)
text,textlist = determine_nlp_strategy(sendto,msg)
contentdb.add_content('member','send',msg)
textlist = []
text = None
# 人设问答
keyword = qa_service.question('Persona',msg)
if keyword is not None:
text = config_util.config["attribute"][keyword]
# 全局问答
if text is None:
answer = qa_service.question('qa',msg)
if answer is not None:
text = answer
else:
text,textlist = determine_nlp_strategy(sendto,msg)
contentdb.add_content('fay','send',text)
wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":text}})
@ -121,27 +137,6 @@ class FeiFei:
# self.W = np.array([0.01577594,1.16119452,0.75828,0.207746,1.25017864,0.1044121,0.4294899,0.2770932]).reshape(-1,1) #适应模型变量矩阵
self.W = np.array([0.0, 0.6, 0.1, 0.7, 0.3, 0.0, 0.0, 0.0]).reshape(-1, 1) # 适应模型变量矩阵
self.command_keyword = [
[['播放歌曲', '播放音乐', '唱首歌', '放首歌', '听音乐', '你会唱歌吗', '我想首听歌'], 'playSong'],
[['关闭', '再见', '你走吧'], 'stop'],
[['静音', '闭嘴', '我想静静'], 'mute'],
[['取消静音', '你在哪呢', '你可以说话了'], 'unmute'],
[['换个性别', '换个声音'], 'changeVoice']
]
# 人设提问关键字
self.attribute_keyword = [
[['你叫什么名字', '你的名字是什么'], 'name'],
[['你是男的还是女的', '你是男生还是女生', '你的性别是什么', '你是男生吗', '你是女生吗', '你是男的吗', '你是女的吗', '你是男孩子吗', '你是女孩子吗', ], 'gender', ],
[['你今年多大了', '你多大了', '你今年多少岁', '你几岁了', '你今年几岁了', '你今年几岁了', '你什么时候出生', '你的生日是什么', '你的年龄'], 'age', ],
[['你的家乡在哪', '你的家乡是什么', '你家在哪', '你住在哪', '你出生在哪', '你的出生地在哪', '你的出生地是什么', ], 'birth', ],
[['你的生肖是什么', '你属什么', ], 'zodiac', ],
[['你是什么座', '你是什么星座', '你的星座是什么', ], 'constellation', ],
[['你是做什么的', '你的职业是什么', '你是干什么的', '你的职位是什么', '你的工作是什么', '你是做什么工作的'], 'job', ],
[['你的爱好是什么', '你有爱好吗', '你喜欢什么', '你喜欢做什么'], 'hobby'],
[['联系方式', '联系你们', '怎么联系客服', '有没有客服'], 'contact']
]
self.wsParam = None
self.wss = None
self.sp = Speech()
@ -156,36 +151,6 @@ class FeiFei:
self.playing = False
self.muting = False
def __string_similar(self, s1, s2):
return difflib.SequenceMatcher(None, s1, s2).quick_ratio()
def __read_qna(self, filename) -> list:
qna = []
try:
wb = load_workbook(filename)
sheets = wb.worksheets # 获取当前所有的sheet
sheet = sheets[0]
for row in sheet.rows:
if len(row) >= 2:
qna.append([row[0].value.split(";"), row[1].value])
except BaseException as e:
print("无法读取Q&A文件 {} -> ".format(filename) + str(e))
return qna
def __get_keyword(self, keyword_dict, text):
last_similar = 0
last_answer = ''
for qa in keyword_dict:
for quest in qa[0]:
similar = self.__string_similar(text, quest)
if quest in text:
similar += 0.3
if similar > last_similar:
last_similar = similar
last_answer = qa[1]
if last_similar >= 0.6:
return last_answer
return None
def __play_song(self):
self.playing = True
@ -197,7 +162,7 @@ class FeiFei:
def __get_answer(self, interleaver, text):
if interleaver == "mic":
#指令
keyword = self.__get_keyword(self.command_keyword, text)
keyword = qa_service.question('command',text)
if keyword is not None:
if keyword == "playSong":
MyThread(target=self.__play_song).start()
@ -225,14 +190,14 @@ class FeiFei:
config_util.save_config(config_util.config)
wsa_server.get_web_instance().add_cmd({"panelMsg": ""})
return "NO_ANSWER"
# 人设问答
keyword = self.__get_keyword(self.attribute_keyword, text)
keyword = qa_service.question('Persona',text)
if keyword is not None:
return config_util.config["attribute"][keyword]
answer = None
# 全局问答
answer = self.__get_keyword(self.__read_qna(config_util.config['interact']['QnA']), text)
answer = qa_service.question('qa',text)
if answer is not None:
return answer

83
core/qa_service.py Normal file
View File

@ -0,0 +1,83 @@
from winreg import QueryInfoKey
from openpyxl import load_workbook
import difflib
import shlex
import subprocess
from utils import config_util as cfg
def question(query_type,text):
qa = QAService()
answer = qa.question(query_type,text)
return answer
class QAService:
def __init__(self):
# 人设提问关键字
self.attribute_keyword = [
[['你叫什么名字', '你的名字是什么'], 'name'],
[['你是男的还是女的', '你是男生还是女生', '你的性别是什么', '你是男生吗', '你是女生吗', '你是男的吗', '你是女的吗', '你是男孩子吗', '你是女孩子吗', ], 'gender', ],
[['你今年多大了', '你多大了', '你今年多少岁', '你几岁了', '你今年几岁了', '你今年几岁了', '你什么时候出生', '你的生日是什么', '你的年龄'], 'age', ],
[['你的家乡在哪', '你的家乡是什么', '你家在哪', '你住在哪', '你出生在哪', '你的出生地在哪', '你的出生地是什么', ], 'birth', ],
[['你的生肖是什么', '你属什么', ], 'zodiac', ],
[['你是什么座', '你是什么星座', '你的星座是什么', ], 'constellation', ],
[['你是做什么的', '你的职业是什么', '你是干什么的', '你的职位是什么', '你的工作是什么', '你是做什么工作的'], 'job', ],
[['你的爱好是什么', '你有爱好吗', '你喜欢什么', '你喜欢做什么'], 'hobby'],
[['联系方式', '联系你们', '怎么联系客服', '有没有客服'], 'contact']
]
self.command_keyword = [
[['播放歌曲', '播放音乐', '唱首歌', '放首歌', '听音乐', '你会唱歌吗', '我想首听歌'], 'playSong'],
[['关闭', '再见', '你走吧'], 'stop'],
[['静音', '闭嘴', '我想静静'], 'mute'],
[['取消静音', '你在哪呢', '你可以说话了'], 'unmute'],
[['换个性别', '换个声音'], 'changeVoice']
]
def question(self, query_type, text):
if query_type == 'qa':
answer_dict = self.__read_qna(cfg.config['interact']['QnA'])
answer = self.__get_keyword(answer_dict, text)
elif query_type == 'Persona':
answer_dict = self.attribute_keyword
answer = self.__get_keyword(answer_dict, text)
elif query_type == 'command':
answer = self.__get_keyword(self.command_keyword, text)
print(answer)
return answer
def __read_qna(self, filename) -> list:
qna = []
try:
wb = load_workbook(filename)
sheets = wb.worksheets
sheet = sheets[0]
for row in sheet.rows:
if len(row) >= 2:
qna.append([row[0].value.split(";"), row[1].value])
except BaseException as e:
print("无法读取Q&A文件 {} -> ".format(filename) + str(e))
return qna
def __get_keyword(self, keyword_dict, text):
last_similar = 0
last_answer = ''
for qa in keyword_dict:
for quest in qa[0]:
similar = self.__string_similar(text, quest)
if quest in text:
similar += 0.3
if similar > last_similar:
last_similar = similar
last_answer = qa[1]
if last_similar >= 0.6:
return last_answer
return None
def __string_similar(self, s1, s2):
return difflib.SequenceMatcher(None, s1, s2).quick_ratio()

33
core/recorder.py
View File

@ -73,7 +73,7 @@ class Recorder:
text += "-"
print(text + " [" + str(int(per * 100)) + "%]")
def __waitingResult(self, iat):
def __waitingResult(self, iat:asrclient):
if self.__fay.playing:
return
self.processing = True
@ -95,20 +95,37 @@ class Recorder:
def __record(self):
stream = self.get_stream() #把get stream的方式封装出来方便实现麦克风录制及网络流等不同的流录制子类
try:
stream = self.get_stream() #把get stream的方式封装出来方便实现麦克风录制及网络流等不同的流录制子类
except Exception as e:
print(e)
util.log(1, "请检查设备是否有误，再重新启动!")
return
isSpeaking = False
last_mute_time = time.time()
last_speaking_time = time.time()
data = None
while self.__running:
data = stream.read(1024, exception_on_overflow=False)
if not data:
try:
data = stream.read(1024, exception_on_overflow=False)
except Exception as e:
data = None
print(e)
util.log(1, "请检查设备是否有误，再重新启动!")
return
if data is None:
continue
if cfg.config['source']['record'].get("channels"):
if cfg.config['source']['record']['enabled']:
if len(cfg.config['source']['record'])<3:
channels = 1
else:
channels = int(cfg.config['source']['record']['channels'])
#只获取第一声道
data = np.frombuffer(data, dtype=np.int16)
data = np.reshape(data, (-1, cfg.config['source']['record']['channels'])) # reshaping the array to split the channels
data = np.reshape(data, (-1, channels)) # reshaping the array to split the channels
mono = data[:, 0] # taking the first channel
data = mono.tobytes()

16
fay_booter.py
View File

@ -34,8 +34,10 @@ class RecorderListener(Recorder):
device_id,devInfo = self.__findInternalRecordingDevice(self.paudio)
if device_id < 0:
return
rate = int(devInfo['defaultSampleRate'])
channels = int(devInfo['maxInputChannels'])
if channels == 0:
util.log(1, '请检查设备是否有误，再重新启动!')
return
self.stream = self.paudio.open(input_device_index=device_id, rate=self.__RATE, format=self.__FORMAT, channels=channels, input=True)
return self.stream
@ -51,9 +53,15 @@ class RecorderListener(Recorder):
def stop(self):
super().stop()
self.stream.stop_stream()
self.stream.close()
self.paudio.terminate()
try:
self.stream.stop_stream()
self.stream.close()
self.paudio.terminate()
except Exception as e:
print(e)
util.log(1, "请检查设备是否有误，再重新启动!")
#Edit by xszyou on 20230113:录制远程设备音频输入并传给aliyun