助理版多处优化
1、解决多声道麦克风兼容问题； 2、重构fay_core.py及fay_booter.py代码； 3、ui适应布局调整； 4、恢复男女声音选择； 5、”思考中...“显示逻辑修复。
This commit is contained in:
xszyou
parent
ae1d2ae292
commit
7c67bb5858
16
README.md
16
README.md
@ -28,6 +28,8 @@ UE5工程：https://github.com/xszyou/fay-ue5
|
||||
|
||||
|
||||
|
||||
助理版Fay控制器使用：语音沟通，语音和文字回复；文字沟通，文字回复。
|
||||
|
||||
|
||||
|
||||
### **PC远程助理** [`PC demo`](https://github.com/TheRamU/Fay/tree/main/python_connector_demo)
|
||||
@ -45,9 +47,12 @@ UE5工程：https://github.com/xszyou/fay-ue5
|
||||
|
||||
下载工程: [https://pan.baidu.com/s/1RBo2Pie6A5yTrCf1cn_Tuw?pwd=ck99](https://pan.baidu.com/s/1RBo2Pie6A5yTrCf1cn_Tuw?pwd=ck99)
|
||||
|
||||
|
||||
下载windows运行包: [https://pan.baidu.com/s/1CsJ647uV5rS2NjQH3QT0Iw?pwd=s9s8](https://pan.baidu.com/s/1CsJ647uV5rS2NjQH3QT0Iw?pwd=s9s8)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
工程及运行包：https://github.com/xszyou/fay-ue5
|
||||
@ -105,7 +110,6 @@ UE5工程：https://github.com/xszyou/fay-ue5
|
||||
│ ├── fay_core.py # 数字人核心模块
|
||||
│ ├── recorder.py # 录音器
|
||||
│ ├── tts_voice.py # 语音生源枚举
|
||||
│ ├── viewer.py # 抖音直播间接入模块
|
||||
│ └── wsa_server.py # WebSocket 服务端
|
||||
├── gui # 图形界面
|
||||
│ ├── flask_server.py # Flask 服务端
|
||||
@ -124,6 +128,14 @@ UE5工程：https://github.com/xszyou/fay-ue5
|
||||
|
||||
## **三、升级日志**
|
||||
|
||||
**2023.06.14：**
|
||||
|
||||
+ 解决多声道麦克风兼容问题；
|
||||
+ 重构fay_core.py及fay_booter.py代码；
|
||||
+ ui适应布局调整；
|
||||
+ 恢复声音选择；
|
||||
+ ”思考中...“显示逻辑修复。
|
||||
|
||||
**2023.05.27：**
|
||||
|
||||
+ 修复多个bug：消息框换行及空格问题、语音识别优化；
|
||||
|
||||
19
config.json
19
config.json
@ -1,13 +1,13 @@
|
||||
{
|
||||
"attribute": {
|
||||
"age": "\u6210\u5e74",
|
||||
"birth": "\u4e2d\u56fd",
|
||||
"birth": "Github",
|
||||
"constellation": "\u6c34\u74f6\u5ea7",
|
||||
"contact": "qq467665317",
|
||||
"gender": "\u7537",
|
||||
"gender": "\u5973",
|
||||
"hobby": "\u53d1\u5446",
|
||||
"job": "\u4ea7\u54c1\u5e03\u9053\u8005",
|
||||
"name": "\u9648\u5347",
|
||||
"job": "\u52a9\u7406",
|
||||
"name": "\u83f2\u83f2",
|
||||
"voice": "XIAO_XIAO",
|
||||
"zodiac": "\u86c7"
|
||||
},
|
||||
@ -15,18 +15,19 @@
|
||||
"QnA": "qa_demo.xlsx",
|
||||
"maxInteractTime": 15,
|
||||
"perception": {
|
||||
"chat": 7,
|
||||
"chat": 10,
|
||||
"follow": 10,
|
||||
"gift": 50,
|
||||
"gift": 10,
|
||||
"indifferent": 10,
|
||||
"join": 10
|
||||
},
|
||||
"playSound": true
|
||||
"playSound": true,
|
||||
"visualization": false
|
||||
},
|
||||
"items": [
|
||||
{
|
||||
"QnA": "qa_demo.xlsx",
|
||||
"demoVideo": "C:/Demo.mp4",
|
||||
"QnA": "",
|
||||
"demoVideo": "",
|
||||
"enabled": false,
|
||||
"explain": {
|
||||
"character": "",
|
||||
|
||||
259
core/fay_core.py
259
core/fay_core.py
@ -35,11 +35,10 @@ from ai_module import nlp_VisualGLM as VisualGLM
|
||||
#文本消息处理
|
||||
def send_for_answer(msg,sendto):
|
||||
contentdb = Content_Db()
|
||||
contentdb.add_content('member','send', msg)
|
||||
contentdb.add_content('member','send',msg)
|
||||
text = ''
|
||||
textlist = []
|
||||
try:
|
||||
#wsa_server.get_web_instance().add_cmd({"panelMsg": "思考中..."})
|
||||
util.log(1, '自然语言处理...')
|
||||
tm = time.time()
|
||||
cfg.load_config()
|
||||
@ -54,23 +53,21 @@ def send_for_answer(msg,sendto):
|
||||
text = chatgpt.question(msg)
|
||||
elif cfg.key_chat_module == 'rasa':
|
||||
textlist = nlp_rasa.question(msg)
|
||||
text = textlist[0]['text']
|
||||
text = textlist[0]['text']
|
||||
elif cfg.key_chat_module == "VisualGLM":
|
||||
text = VisualGLM.question(msg)
|
||||
text = VisualGLM.question(msg)
|
||||
|
||||
|
||||
else:
|
||||
raise RuntimeError('讯飞key、yuan key、chatgpt key都没有配置！')
|
||||
util.log(1, '自然语言处理完成. 耗时: {} ms'.format(math.floor((time.time() - tm) * 1000)))
|
||||
if text == '哎呀，你这么说我也不懂，详细点呗' or text == '':
|
||||
util.log(1, '[!] 自然语言无语了！')
|
||||
text = '哎呀，你这么说我也不懂，详细点呗'
|
||||
# wsa_server.get_web_instance().add_cmd({"panelMsg": ""})
|
||||
|
||||
text = '哎呀，你这么说我也不懂，详细点呗'
|
||||
except BaseException as e:
|
||||
print(e)
|
||||
util.log(1, '自然语言处理错误！')
|
||||
text = '哎呀，你这么说我也不懂，详细点呗'
|
||||
# wsa_server.get_web_instance().add_cmd({"panelMsg": ""})
|
||||
|
||||
now = datetime.now()
|
||||
timetext = str(now.strftime("%Y-%m-%d %H:%M:%S"))
|
||||
@ -183,10 +180,10 @@ class FeiFei:
|
||||
self.playing = False
|
||||
wsa_server.get_web_instance().add_cmd({"panelMsg": ""})
|
||||
|
||||
#检查是否命中指令或q&a
|
||||
def __get_answer(self, interleaver, text):
|
||||
|
||||
if interleaver == "mic":
|
||||
# 命令
|
||||
#指令
|
||||
keyword = self.__get_keyword(self.command_keyword, text)
|
||||
if keyword is not None:
|
||||
if keyword == "playSong":
|
||||
@ -225,34 +222,7 @@ class FeiFei:
|
||||
answer = self.__get_keyword(self.__read_qna(config_util.config['interact']['QnA']), text)
|
||||
if answer is not None:
|
||||
return answer
|
||||
|
||||
items = self.__get_item_list()
|
||||
|
||||
if len(items) > 0:
|
||||
item = items[self.item_index]
|
||||
|
||||
# 跨商品物品问答匹配
|
||||
for ite in items:
|
||||
name = ite["name"]
|
||||
if name != item["name"]:
|
||||
if name in text or self.__string_similar(text, name) > 0.6:
|
||||
item = ite
|
||||
break
|
||||
|
||||
# 商品介绍问答
|
||||
keyword = self.__get_keyword(self.explain_keyword, text)
|
||||
if keyword is not None:
|
||||
try:
|
||||
return item["explain"][keyword]
|
||||
except BaseException as e:
|
||||
print(e)
|
||||
|
||||
# 商品问答
|
||||
answer = self.__get_keyword(self.__read_qna(item["QnA"]), text)
|
||||
if answer is not None:
|
||||
return answer
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def __get_list_answer(self, answers, text):
|
||||
last_similar = 0
|
||||
@ -272,26 +242,19 @@ class FeiFei:
|
||||
return None
|
||||
|
||||
def __auto_speak(self):
|
||||
i = 0
|
||||
script_index = 0
|
||||
while self.__running:
|
||||
time.sleep(0.8)
|
||||
if self.speaking or self.sleep:
|
||||
continue
|
||||
|
||||
try:
|
||||
# 简化逻辑：默认执行带货脚本，带货脚本执行其间有人互动，则执行完当前脚本就回应最后三条互动，回应完继续执行带货脚本
|
||||
if i <= 3 and len(self.interactive) > i:
|
||||
i += 1
|
||||
interact: Interact = self.interactive[0 - i]
|
||||
if interact.interact_type == 1:
|
||||
self.q_msg = interact.data["msg"]
|
||||
if len(self.interactive) > 0:
|
||||
interact: Interact = self.interactive.pop()
|
||||
index = interact.interact_type
|
||||
# print("index:{0}".format(index))
|
||||
user_name = interact.data["user"]
|
||||
# self.__isExecute = True #!!!!
|
||||
|
||||
if index == 1:
|
||||
self.q_msg = interact.data["msg"]
|
||||
|
||||
#fay eyes
|
||||
fay_eyes = yolov8.new_instance()
|
||||
if fay_eyes.get_status():#YOLO正在运行
|
||||
person_count, stand_count, sit_count = fay_eyes.get_counts()
|
||||
@ -299,15 +262,14 @@ class FeiFei:
|
||||
wsa_server.get_web_instance().add_cmd({"panelMsg": "不是有且只有一个人，不互动"})
|
||||
continue
|
||||
|
||||
answer = self.__get_answer(interact.interleaver, self.q_msg)
|
||||
answer = self.__get_answer(interact.interleaver, self.q_msg)#确定是否命中指令或q&a
|
||||
if(self.muting): #静音指令正在执行
|
||||
wsa_server.get_web_instance().add_cmd({"panelMsg": "静音指令正在执行，不互动"})
|
||||
continue
|
||||
|
||||
|
||||
contentdb = Content_Db()
|
||||
contentdb.add_content('member','speak',self.q_msg)
|
||||
wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":self.q_msg}})
|
||||
|
||||
text = ''
|
||||
textlist = []
|
||||
if answer is None:
|
||||
@ -327,7 +289,6 @@ class FeiFei:
|
||||
text = textlist[0]['text']
|
||||
elif cfg.key_chat_module == "VisualGLM":
|
||||
text = VisualGLM.question(self.q_msg)
|
||||
|
||||
else:
|
||||
raise RuntimeError('讯飞key、yuan key、chatgpt key都没有配置！')
|
||||
util.log(1, '自然语言处理完成. 耗时: {} ms'.format(math.floor((time.time() - tm) * 1000)))
|
||||
@ -340,14 +301,9 @@ class FeiFei:
|
||||
util.log(1, '自然语言处理错误！')
|
||||
wsa_server.get_web_instance().add_cmd({"panelMsg": ""})
|
||||
continue
|
||||
elif answer != 'NO_ANSWER':
|
||||
elif answer != 'NO_ANSWER': #语音内容没有命中指令,回复q&a内容
|
||||
text = answer
|
||||
|
||||
if len(user_name) == 0:
|
||||
self.a_msg = text
|
||||
else:
|
||||
self.a_msg = user_name + '，' + text
|
||||
|
||||
self.a_msg = text
|
||||
contentdb.add_content('fay','speak',self.a_msg)
|
||||
wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":self.a_msg}})
|
||||
if len(textlist) > 1:
|
||||
@ -356,130 +312,19 @@ class FeiFei:
|
||||
contentdb.add_content('fay','speak',textlist[i]['text'])
|
||||
wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":textlist[i]['text']}})
|
||||
i+= 1
|
||||
|
||||
elif index == 2:
|
||||
self.a_msg = ['我们的直播间越来越多人咯', '感谢{}的到来'.format(user_name), '欢印{}来到我们的直播间'.format(user_name)][
|
||||
random.randint(0, 2)]
|
||||
|
||||
elif index == 3:
|
||||
gift = interact.data["gift"]
|
||||
self.a_msg = '感谢感谢，感谢 {}送给我的{}个{}'.format(interact.data["user"], interact.data["amount"], gift[1])
|
||||
|
||||
elif index == 4:
|
||||
self.a_msg = '感谢关注'
|
||||
|
||||
elif index == 5:
|
||||
msg = ""
|
||||
for i in range(0, len(interact.data["gifts"])):
|
||||
user = interact.data["gifts"][i]["user"]
|
||||
gift = interact.data["gifts"][i]["gift"]
|
||||
amount = interact.data["gifts"][i]["amount"]
|
||||
msg += "{}送给我的{}个{}".format(user, amount, gift[1])
|
||||
self.a_msg = '感谢感谢，感谢' + msg
|
||||
wsa_server.get_web_instance().add_cmd({"panelMsg": self.a_msg})
|
||||
self.last_speak_data = self.a_msg
|
||||
self.speaking = True
|
||||
MyThread(target=self.__say, args=['interact']).start()
|
||||
else:
|
||||
i = 0
|
||||
self.interactive.clear()
|
||||
config_items = config_util.config["items"]
|
||||
items = []
|
||||
for item in config_items:
|
||||
if item["enabled"]:
|
||||
items.append(item)
|
||||
if len(items) > 0:
|
||||
if self.item_index >= len(items):
|
||||
self.item_index = 0
|
||||
script_index = 0
|
||||
item = items[self.item_index]
|
||||
script_index = script_index + 1
|
||||
explain_key = self.__get_explain_from_index(script_index)
|
||||
if explain_key is None:
|
||||
self.item_index = self.item_index + 1
|
||||
script_index = 0
|
||||
if self.item_index >= len(items):
|
||||
self.item_index = 0
|
||||
explain_key = self.__get_explain_from_index(script_index)
|
||||
explain = item["explain"][explain_key]
|
||||
if len(explain) > 0:
|
||||
self.a_msg = explain
|
||||
self.last_speak_data = self.a_msg
|
||||
self.speaking = True
|
||||
MyThread(target=self.__say, args=['script']).start()
|
||||
|
||||
except BaseException as e:
|
||||
print(e)
|
||||
|
||||
def __get_item_list(self) -> list:
|
||||
items = []
|
||||
for item in config_util.config["items"]:
|
||||
if item["enabled"]:
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
def __get_explain_from_index(self, index: int):
|
||||
if index == 0:
|
||||
return "character"
|
||||
if index == 1:
|
||||
return "discount"
|
||||
if index == 2:
|
||||
return "intro"
|
||||
if index == 3:
|
||||
return "price"
|
||||
if index == 4:
|
||||
return "promise"
|
||||
if index == 5:
|
||||
return "usage"
|
||||
return None
|
||||
|
||||
def on_interact(self, interact: Interact):
|
||||
|
||||
# 合并同类交互
|
||||
# 进入
|
||||
if interact.interact_type == 2:
|
||||
itr = self.__get_interactive(2)
|
||||
if itr is None:
|
||||
self.interactive.append(interact)
|
||||
else:
|
||||
newItr = (2, itr.data["user"] + ', ' + interact.data["user"], itr.data["msg"])
|
||||
self.interactive.remove(itr)
|
||||
self.interactive.append(newItr)
|
||||
|
||||
# 送礼
|
||||
elif interact.interact_type == 3:
|
||||
gifts = []
|
||||
rm_list = []
|
||||
for itr in self.interactive:
|
||||
if itr.interact_type == 3:
|
||||
gifts.append({
|
||||
"user": itr.data["user"],
|
||||
"gift": itr.data["gift"],
|
||||
"amount": itr.data["amount"]
|
||||
})
|
||||
rm_list.append(itr)
|
||||
elif itr.interact_type == 5:
|
||||
for gift in itr.data["gifts"]:
|
||||
gifts.append(gift)
|
||||
rm_list.append(itr)
|
||||
if len(rm_list) > 0:
|
||||
for itr in rm_list:
|
||||
self.interactive.remove(itr)
|
||||
self.interactive.append(Interact("live", 5, {"gifts": gifts}))
|
||||
|
||||
# 关注
|
||||
elif interact.interact_type == 4:
|
||||
if self.__get_interactive(2) is None:
|
||||
self.interactive.append(interact)
|
||||
|
||||
else:
|
||||
self.interactive.append(interact)
|
||||
self.interactive.append(interact)
|
||||
MyThread(target=self.__update_mood, args=[interact.interact_type]).start()
|
||||
MyThread(target=storer.storage_live_interact, args=[interact]).start()
|
||||
|
||||
def __get_interactive(self, interactType) -> Interact:
|
||||
for interact in self.interactive:
|
||||
if interact is Interact and interact.interact_type == interactType:
|
||||
return interact
|
||||
return None
|
||||
|
||||
# 适应模型计算
|
||||
def __fay(self, index):
|
||||
@ -529,7 +374,7 @@ class FeiFei:
|
||||
if self.mood <= -1:
|
||||
self.mood = -1
|
||||
|
||||
def __get_mood(self):
|
||||
def __get_mood_voice(self):
|
||||
voice = tts_voice.get_voice_of(config_util.config["attribute"]["voice"])
|
||||
if voice is None:
|
||||
voice = EnumVoice.XIAO_XIAO
|
||||
@ -547,14 +392,13 @@ class FeiFei:
|
||||
sayType = styleList["cheerful"]
|
||||
return sayType
|
||||
|
||||
# 合成声音，加上type代表是脚本还是互动
|
||||
# 合成声音
|
||||
def __say(self, styleType):
|
||||
try:
|
||||
if len(self.a_msg) < 1:
|
||||
self.speaking = False
|
||||
else:
|
||||
# print(self.__get_mood().name + self.a_msg)
|
||||
util.printInfo(1, '菲菲', '({}) {}'.format(self.__get_mood(), self.a_msg))
|
||||
util.printInfo(1, '菲菲', '({}) {}'.format(self.__get_mood_voice(), self.a_msg))
|
||||
MyThread(target=storer.storage_live_interact, args=[Interact('Fay', 0, {'user': 'Fay', 'msg': self.a_msg})]).start()
|
||||
util.log(1, '合成音频...')
|
||||
tm = time.time()
|
||||
@ -562,14 +406,13 @@ class FeiFei:
|
||||
if not config_util.config["interact"]["playSound"]: # 非展板播放
|
||||
content = {'Topic': 'Unreal', 'Data': {'Key': 'text', 'Value': self.a_msg}}
|
||||
wsa_server.get_instance().add_cmd(content)
|
||||
result = self.sp.to_sample(self.a_msg, self.__get_mood())
|
||||
result = self.sp.to_sample(self.a_msg, self.__get_mood_voice())
|
||||
util.log(1, '合成音频完成. 耗时: {} ms 文件:{}'.format(math.floor((time.time() - tm) * 1000), result))
|
||||
if result is not None:
|
||||
MyThread(target=self.__send_audio, args=[result, styleType]).start()
|
||||
MyThread(target=self.__send_or_play_audio, args=[result, styleType]).start()
|
||||
return result
|
||||
except BaseException as e:
|
||||
print(e)
|
||||
# print("tts失败！！！！！！！！！！！！！")
|
||||
self.speaking = False
|
||||
return None
|
||||
|
||||
@ -579,7 +422,8 @@ class FeiFei:
|
||||
pygame.mixer.music.load(file_url)
|
||||
pygame.mixer.music.play()
|
||||
|
||||
def __send_audio(self, file_url, say_type):
|
||||
|
||||
def __send_or_play_audio(self, file_url, say_type):
|
||||
try:
|
||||
audio_length = eyed3.load(file_url).info.time_secs #mp3音频长度
|
||||
# with wave.open(file_url, 'rb') as wav_file: #wav音频长度
|
||||
@ -605,14 +449,11 @@ class FeiFei:
|
||||
util.log(1, "远程音频发送完成：{}".format(total))
|
||||
except socket.error as serr:
|
||||
util.log(1,"远程音频输入输出设备已经断开：{}".format(serr))
|
||||
|
||||
|
||||
|
||||
wsa_server.get_web_instance().add_cmd({"panelMsg": self.a_msg})
|
||||
time.sleep(audio_length + 0.5)
|
||||
wsa_server.get_web_instance().add_cmd({"panelMsg": ""})
|
||||
if config_util.config["interact"]["playSound"]:
|
||||
util.log(1, '结束播放！')
|
||||
time.sleep(audio_length + 0.5)
|
||||
wsa_server.get_web_instance().add_cmd({"panelMsg": ""})
|
||||
if config_util.config["interact"]["playSound"]:
|
||||
util.log(1, '结束播放！')
|
||||
self.speaking = False
|
||||
except Exception as e:
|
||||
print(e)
|
||||
@ -642,50 +483,12 @@ class FeiFei:
|
||||
except Exception as err:
|
||||
pass
|
||||
|
||||
def __waiting_speaking(self, file_url):
|
||||
try:
|
||||
time.sleep(5)
|
||||
print('[' + str(int(time.time())) + '][菲菲] [S] [开始发言]')
|
||||
with wave.open(file_url, 'rb') as wav_file:
|
||||
wav_length = wav_file.getnframes() / float(wav_file.getframerate())
|
||||
time.sleep(wav_length)
|
||||
self.last_interact_time = time.time()
|
||||
self.speaking = False
|
||||
print('[' + str(int(time.time())) + '][菲菲] [E] [结束发言]')
|
||||
time.sleep(30)
|
||||
os.remove(file_url)
|
||||
except:
|
||||
self.last_interact_time = time.time()
|
||||
self.speaking = False
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# 冷场情绪更新
|
||||
def __update_mood_runnable(self):
|
||||
while self.__running:
|
||||
time.sleep(10)
|
||||
update = config_util.config["interact"]["perception"]["indifferent"] / 100
|
||||
if len(self.interactive) < 1:
|
||||
if self.mood > 0:
|
||||
if self.mood > update:
|
||||
self.mood = self.mood - update
|
||||
else:
|
||||
self.mood = 0
|
||||
elif self.mood < 0:
|
||||
if self.mood < -update:
|
||||
self.mood = self.mood + update
|
||||
else:
|
||||
self.mood = 0
|
||||
|
||||
def set_sleep(self, sleep):
|
||||
self.sleep = sleep
|
||||
|
||||
def start(self):
|
||||
MyThread(target=self.__send_mood).start()
|
||||
MyThread(target=self.__auto_speak).start()
|
||||
MyThread(target=self.__update_mood_runnable).start()
|
||||
|
||||
def stop(self):
|
||||
self.__running = False
|
||||
|
||||
@ -9,8 +9,7 @@ from core import wsa_server
|
||||
from scheduler.thread_manager import MyThread
|
||||
from utils import util
|
||||
from utils import config_util as cfg
|
||||
|
||||
|
||||
import numpy as np
|
||||
# 启动时间 (秒)
|
||||
_ATTACK = 0.2
|
||||
|
||||
@ -105,6 +104,11 @@ class Recorder:
|
||||
data = stream.read(1024, exception_on_overflow=False)
|
||||
if not data:
|
||||
continue
|
||||
#只获取第一声道
|
||||
data = np.frombuffer(data, dtype=np.int16)
|
||||
data = np.reshape(data, (-1, cfg.config['source']['record']['channels'])) # reshaping the array to split the channels
|
||||
mono = data[:, 0] # taking the first channel
|
||||
data = mono.tobytes()
|
||||
|
||||
level = audioop.rms(data, 2)
|
||||
if len(self.__history_data) >= 5:
|
||||
@ -147,10 +151,7 @@ class Recorder:
|
||||
self.__waitingResult(self.__aLiNls)
|
||||
if not soon and isSpeaking:
|
||||
self.__aLiNls.send(data)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def set_processing(self, processing):
|
||||
self.__processing = processing
|
||||
|
||||
126
core/viewer.py
126
core/viewer.py
@ -1,126 +0,0 @@
|
||||
from abc import abstractmethod
|
||||
import json
|
||||
import random
|
||||
import time
|
||||
import requests
|
||||
import websocket
|
||||
import ssl
|
||||
|
||||
from core.interact import Interact
|
||||
from scheduler.thread_manager import MyThread
|
||||
from utils import config_util, util
|
||||
|
||||
USER_URL = 'https://www.douyin.com/user/'
|
||||
|
||||
interact_datas = []
|
||||
import json
|
||||
import time
|
||||
import ssl
|
||||
import websocket
|
||||
|
||||
|
||||
running = False
|
||||
class WS_Client:
|
||||
def __init__(self, host):
|
||||
self.__ws = None
|
||||
self.__host = host
|
||||
self.__connect(host)
|
||||
|
||||
def on_message(self, ws, message):
|
||||
global interact_datas
|
||||
try:
|
||||
|
||||
data = json.loads(message)
|
||||
if data["Type"] == 1:#留言
|
||||
if len(interact_datas) >= 5:
|
||||
interact_datas.pop()
|
||||
interact = Interact("live", 1, {"user": json.loads(data["Data"])["User"]["Nickname"], "msg": json.loads(data["Data"])["Content"]})
|
||||
interact_datas.append(interact)
|
||||
if data["Type"] == 3:#进入
|
||||
if len(interact_datas) >= 5:
|
||||
interact_datas.pop()
|
||||
interact_datas.append(Interact("live", 2, {"user": json.loads(data["Data"])["User"]["Nickname"], "msg": "来了"}))
|
||||
#...
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
def on_close(self, ws, code, msg):
|
||||
pass
|
||||
|
||||
def on_error(self, ws, error):
|
||||
util.log(1, "弹幕监听WebSocket error. Reconnecting...")
|
||||
time.sleep(5)
|
||||
self.__connect(self.__host)
|
||||
|
||||
def on_open(self, ws):
|
||||
pass
|
||||
|
||||
def __connect(self, host):
|
||||
global running
|
||||
while running:
|
||||
try:
|
||||
self.__ws = websocket.WebSocketApp(host,
|
||||
on_message=self.on_message,
|
||||
on_error=self.on_error,
|
||||
on_close=self.on_close)
|
||||
self.__ws.on_open = self.on_open
|
||||
self.__ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE})
|
||||
util.log(1, "弹幕监听WebSocket success.")
|
||||
break
|
||||
except Exception as e:
|
||||
break
|
||||
|
||||
def close(self):
|
||||
self.__ws.close()
|
||||
|
||||
|
||||
class Viewer:
|
||||
|
||||
def __init__(self):
|
||||
global running
|
||||
running = True
|
||||
self.live_started = False
|
||||
self.dy_msg_ws = None
|
||||
|
||||
def __start(self):
|
||||
MyThread(target=self.__run_dy_msg_ws).start() #获取抖音监听内容
|
||||
self.live_started = True
|
||||
MyThread(target=self.__get_package_listen_interact_runnable).start()
|
||||
|
||||
def __run_dy_msg_ws(self):
|
||||
self.dy_msg_ws = WS_Client('ws://127.0.0.1:8888')
|
||||
|
||||
def start(self):
|
||||
MyThread(target=self.__start).start()
|
||||
|
||||
def is_live_started(self):
|
||||
return self.live_started
|
||||
|
||||
|
||||
#Add by xszyou on 20230412.通过抓包监测互动数据
|
||||
def __get_package_listen_interact_runnable(self):
|
||||
global interact_datas
|
||||
global running
|
||||
while running:
|
||||
if not self.live_started:
|
||||
continue
|
||||
|
||||
for interact in interact_datas:
|
||||
MyThread(target=self.on_interact, args=[interact, time.time()]).start()
|
||||
interact_datas.clear()
|
||||
|
||||
def stop(self):
|
||||
global running
|
||||
running = False
|
||||
if self.dy_msg_ws:
|
||||
self.dy_msg_ws.close()
|
||||
self.dy_msg_ws = None
|
||||
|
||||
|
||||
@abstractmethod
|
||||
def on_interact(self, interact, event_time):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def on_change_state(self, is_live_started):
|
||||
pass
|
||||
BIN
favicon.ico
Normal file
BIN
favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
@ -1,52 +1,17 @@
|
||||
import time
|
||||
from io import BytesIO
|
||||
import socket
|
||||
import pyaudio
|
||||
import numpy as np
|
||||
import scipy.io.wavfile as wav
|
||||
import wave
|
||||
|
||||
from core.interact import Interact
|
||||
from core.recorder import Recorder
|
||||
from core.fay_core import FeiFei
|
||||
from core.viewer import Viewer
|
||||
from scheduler.thread_manager import MyThread
|
||||
from utils import util, config_util, stream_util, ngrok_util
|
||||
from core.wsa_server import MyServer
|
||||
|
||||
|
||||
|
||||
|
||||
feiFei: FeiFei = None
|
||||
viewerListener: Viewer = None
|
||||
recorderListener: Recorder = None
|
||||
|
||||
__running = True
|
||||
|
||||
|
||||
class ViewerListener(Viewer):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def on_interact(self, interact: Interact, event_time):
|
||||
type_names = {
|
||||
1: '发言',
|
||||
2: '进入',
|
||||
3: '送礼',
|
||||
4: '关注'
|
||||
}
|
||||
util.printInfo(1, type_names[interact.interact_type], '{}: {}'.format(interact.data["user"], interact.data["msg"]), event_time)
|
||||
if interact.interact_type == 1:
|
||||
feiFei.last_quest_time = time.time()
|
||||
thr = MyThread(target=feiFei.on_interact, args=[interact])
|
||||
thr.start()
|
||||
thr.join()
|
||||
|
||||
def on_change_state(self, is_live_started):
|
||||
feiFei.set_sleep(not is_live_started)
|
||||
pass
|
||||
|
||||
#录制麦克风音频输入并传给aliyun
|
||||
class RecorderListener(Recorder):
|
||||
|
||||
@ -54,7 +19,6 @@ class RecorderListener(Recorder):
|
||||
self.__device = device
|
||||
self.__RATE = 16000
|
||||
self.__FORMAT = pyaudio.paInt16
|
||||
self.__CHANNELS = 1
|
||||
|
||||
super().__init__(fei)
|
||||
|
||||
@ -67,19 +31,23 @@ class RecorderListener(Recorder):
|
||||
|
||||
def get_stream(self):
|
||||
self.paudio = pyaudio.PyAudio()
|
||||
device_id = self.__findInternalRecordingDevice(self.paudio)
|
||||
device_id,devInfo = self.__findInternalRecordingDevice(self.paudio)
|
||||
if device_id < 0:
|
||||
return
|
||||
self.stream = self.paudio.open(input_device_index=device_id, rate=self.__RATE, format=self.__FORMAT, channels=self.__CHANNELS, input=True)
|
||||
rate = int(devInfo['defaultSampleRate'])
|
||||
channels = int(devInfo['maxInputChannels'])
|
||||
self.stream = self.paudio.open(input_device_index=device_id, rate=self.__RATE, format=self.__FORMAT, channels=channels, input=True)
|
||||
return self.stream
|
||||
|
||||
def __findInternalRecordingDevice(self, p):
|
||||
for i in range(p.get_device_count()):
|
||||
devInfo = p.get_device_info_by_index(i)
|
||||
if devInfo['name'].find(self.__device) >= 0 and devInfo['hostApi'] == 0:
|
||||
return i
|
||||
config_util.config['source']['record']['channels'] = devInfo['maxInputChannels']
|
||||
config_util.save_config(config_util.config)
|
||||
return i, devInfo
|
||||
util.log(1, '[!] 无法找到内录设备!')
|
||||
return -1
|
||||
return -1, None
|
||||
|
||||
def stop(self):
|
||||
super().stop()
|
||||
@ -119,6 +87,8 @@ class DeviceInputListener(Recorder):
|
||||
time.sleep(1)
|
||||
|
||||
def on_speaking(self, text):
|
||||
global feiFei
|
||||
|
||||
if len(text) > 1:
|
||||
interact = Interact("mic", 1, {'user': '', 'msg': text})
|
||||
util.printInfo(3, "语音", '{}'.format(interact.data["msg"]), time.time())
|
||||
@ -146,12 +116,7 @@ class DeviceInputListener(Recorder):
|
||||
|
||||
|
||||
def console_listener():
|
||||
type_names = {
|
||||
1: '发言',
|
||||
2: '进入',
|
||||
3: '送礼',
|
||||
4: '关注'
|
||||
}
|
||||
global feiFei
|
||||
while __running:
|
||||
text = input()
|
||||
args = text.split(' ')
|
||||
@ -177,19 +142,9 @@ def console_listener():
|
||||
if len(args) == 1:
|
||||
util.log(1, '错误的参数！')
|
||||
msg = text[3:len(text)]
|
||||
i = 1
|
||||
try:
|
||||
i = int(msg)
|
||||
except:
|
||||
pass
|
||||
if i < 1:
|
||||
i = 1
|
||||
if i > 4:
|
||||
i = 4
|
||||
util.printInfo(1, type_names[i], '{}: {}'.format('控制台', msg))
|
||||
if i == 1:
|
||||
feiFei.last_quest_time = time.time()
|
||||
interact = Interact("console", i, {'user': '', 'msg': msg})
|
||||
util.printInfo(3, "控制台", '{}: {}'.format('控制台', msg))
|
||||
feiFei.last_quest_time = time.time()
|
||||
interact = Interact("console", 1, {'user': '', 'msg': msg})
|
||||
thr = MyThread(target=feiFei.on_interact, args=[interact])
|
||||
thr.start()
|
||||
thr.join()
|
||||
@ -200,16 +155,12 @@ def console_listener():
|
||||
#停止服务
|
||||
def stop():
|
||||
global feiFei
|
||||
global viewerListener
|
||||
global recorderListener
|
||||
global __running
|
||||
global deviceInputListener
|
||||
|
||||
util.log(1, '正在关闭服务...')
|
||||
__running = False
|
||||
if viewerListener is not None:
|
||||
util.log(1, '正在关闭直播服务...')
|
||||
viewerListener.stop()
|
||||
if recorderListener is not None:
|
||||
util.log(1, '正在关闭录音服务...')
|
||||
recorderListener.stop()
|
||||
@ -222,26 +173,21 @@ def stop():
|
||||
|
||||
|
||||
def start():
|
||||
# global ws_server
|
||||
global feiFei
|
||||
global viewerListener
|
||||
global recorderListener
|
||||
global __running
|
||||
global deviceInputListener
|
||||
|
||||
util.log(1, '开启服务...')
|
||||
__running = True
|
||||
|
||||
util.log(1, '读取配置...')
|
||||
config_util.load_config()
|
||||
|
||||
|
||||
|
||||
|
||||
util.log(1, '开启核心服务...')
|
||||
feiFei = FeiFei()
|
||||
feiFei.start()
|
||||
|
||||
liveRoom = config_util.config['source']['liveRoom']
|
||||
record = config_util.config['source']['record']
|
||||
|
||||
if record['enabled']:
|
||||
@ -265,6 +211,5 @@ def start():
|
||||
if __name__ == '__main__':
|
||||
ws_server: MyServer = None
|
||||
feiFei: FeiFei = None
|
||||
viewerListener: Viewer = None
|
||||
recorderListener: Recorder = None
|
||||
start()
|
||||
|
||||
@ -42,6 +42,14 @@ def api_submit():
|
||||
data = request.values.get('data')
|
||||
# print(data)
|
||||
config_data = json.loads(data)
|
||||
if(config_data['config']['source']['record']['enabled']):
|
||||
config_data['config']['source']['record']['channels'] = 0
|
||||
audio = pyaudio.PyAudio()
|
||||
for i in range(audio.get_device_count()):
|
||||
devInfo = audio.get_device_info_by_index(i)
|
||||
if devInfo['name'].find(config_data['config']['source']['record']['device']) >= 0 and devInfo['hostApi'] == 0:
|
||||
config_data['config']['source']['record']['channels'] = devInfo['maxInputChannels']
|
||||
|
||||
config_util.save_config(config_data['config'])
|
||||
|
||||
|
||||
|
||||
@ -540,6 +540,11 @@ new Vue({
|
||||
//滚动条置底
|
||||
let height = document.querySelector('.content').scrollHeight;
|
||||
document.querySelector(".content").scrollTop = height;
|
||||
this.timer = setTimeout(()=>{ //设置延迟执行
|
||||
//滚动条置底
|
||||
let height = document.querySelector('.content').scrollHeight;
|
||||
document.querySelector(".content").scrollTop = height;
|
||||
},1000)
|
||||
}
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
|
||||
@ -20,8 +20,7 @@
|
||||
<title>Fay</title>
|
||||
<style>
|
||||
.container {
|
||||
max-height: calc(100vh - 55px);
|
||||
min-height: 953px;
|
||||
height: 953px;
|
||||
width: 913px;
|
||||
border-radius: 4px;
|
||||
border: 0.5px solid #e0e0e0;
|
||||
@ -240,7 +239,15 @@
|
||||
</div>
|
||||
<div class="character_right">
|
||||
<ul>
|
||||
|
||||
<li>
|
||||
<p>声音选择：{{attribute_voice}}</p>
|
||||
<el-select v-model="attribute_voice" placeholder="请选择">
|
||||
<el-option v-for="item in voice_list" :key="item.value"
|
||||
:label="item.label" :value="item.value">
|
||||
</el-option>
|
||||
</el-select>
|
||||
</li>
|
||||
<br>
|
||||
<li>
|
||||
<p>敏感度：</p>
|
||||
<el-slider v-model="interact_perception_follow"></el-slider>
|
||||
@ -263,10 +270,7 @@
|
||||
<div class="character_box">
|
||||
<p>Q&A文件：</p>
|
||||
<el-input v-model="interact_QnA" placeholder="请输入内容"></el-input>
|
||||
<!-- <el-upload class="upload-demo" action="http://127.0.0.1:5000/"-->
|
||||
<!-- :on-success="handlePreview">-->
|
||||
<!-- <el-input v-model="interact_QnA" placeholder="请输入内容"></el-input>-->
|
||||
<!-- </el-upload>-->
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -350,31 +354,10 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
// function send() {
|
||||
// let text = document.querySelector('#textarea').value;
|
||||
// if (!text) {
|
||||
// alert('请输入内容');
|
||||
// return;
|
||||
// }
|
||||
// // text = text.replace(/\s/g, "<br/>");
|
||||
// text = text.replace(/\n/g, "<br/>");
|
||||
// text = text.replace(/\r\n/g, "<br/>");
|
||||
// let item = document.createElement('div');
|
||||
// item.className = 'item item-right';
|
||||
// item.innerHTML = `<div class="bubble bubble-right">${text}</div><div class="avatar"><img src="{{ url_for('static',filename='from.jpg') }}" /></div>`;
|
||||
// document.querySelector('.content').appendChild(item);
|
||||
// document.querySelector('#textarea').value = '';
|
||||
// document.querySelector('#textarea').focus();
|
||||
// //滚动条置底
|
||||
// let height = document.querySelector('.content').scrollHeight;
|
||||
// document.querySelector(".content").scrollTop = height;
|
||||
// }
|
||||
</script>
|
||||
|
||||
</body>
|
||||
<!-- 开发环境vue.js -->
|
||||
<script src="{{ url_for('static',filename='js/vue.js') }}"></script>
|
||||
<!--<script src="https://cdn.jsdelivr.net/npm/vue@2/dist/vue.js"></script>-->
|
||||
<!-- 发行环境vue.js -->
|
||||
<!-- <script src="https://cdn.jsdelivr.net/npm/vue@2"></script> -->
|
||||
<!-- 引入element-ui组件库 -->
|
||||
|
||||
4
main.py
4
main.py
@ -12,6 +12,8 @@ from gui.window import MainWindow
|
||||
from utils import config_util
|
||||
from scheduler.thread_manager import MyThread
|
||||
from core.content_db import Content_Db
|
||||
import sys
|
||||
sys.setrecursionlimit(sys.getrecursionlimit() * 5)
|
||||
|
||||
def __clear_samples():
|
||||
if not os.path.exists("./samples"):
|
||||
@ -29,8 +31,6 @@ def __clear_songs():
|
||||
os.remove('./songs/' + file_name)
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
__clear_samples()
|
||||
__clear_songs()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user