diff --git a/README.md b/README.md index 403185b..fa9c87e 100644 --- a/README.md +++ b/README.md @@ -109,8 +109,9 @@ Fay（服务端）与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127 │   ├── ms_tts_sdk.py # 微软 文本转语音 │   ├── nlp_lingju.py # 灵聚 人机交互-自然语言处理 │   ├── xf_aiui.py # 讯飞 人机交互-自然语言处理 -│   ├── chatgpt.py # gpt3.5对接 -│   ├── yuan_1_0.py # 浪潮.源大模型对接 +│   ├── nlp_gpt.py # gpt api对接 +│   ├── nlp_chatgpt.py # chat.openai.com逆向对接 +│   ├── nlp_yuan.py # 浪潮.源大模型对接 │   ├── nlp_rasa.py # ChatGLM-6B的基础上前置Rasa会话管理(强烈推荐) │   ├── nlp_VisualGLM.py # 对接多模态大语言模型VisualGLM-6B │   ├── yolov8.py # yolov8资态识别 @@ -141,6 +142,12 @@ Fay（服务端）与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127 ## **三、升级日志** +**2023.06.28：** + ++ 重构NLP模块管理逻辑，便于自由扩展； ++ gpt：拆分为ChatGPT及GPT、更换新的GPT接口、可单独配置代理服务器； ++ 指定yolov8包版本，解决yolo不兼容问题； ++ 修复：自言自语bug、接收多个待处理消息bug。 **2023.06.21：** diff --git a/ai_module/nlp_chatgpt.py b/ai_module/nlp_chatgpt.py new file mode 100644 index 0000000..6a94595 --- /dev/null +++ b/ai_module/nlp_chatgpt.py @@ -0,0 +1,30 @@ +from revChatGPT.V1 import Chatbot +from utils import config_util as cfg +import time + +count = 0 +def question(cont): + global count + try: + chatbot = Chatbot(config={ + "access_token": cfg.key_gpt_access_token, + "paid": False, + "collect_analytics": True, + "proxy": cfg.proxy_config, + "model": "gpt-4", + "conversation_id":cfg.key_gpt_conversation_id + },conversation_id=cfg.key_gpt_conversation_id, + parent_id=None) + + prompt = cont + response = "" + for data in chatbot.ask(prompt): + response = data["message"] + count = 0 + return response + except Exception as e: + count += 1 + if count < 3: + time.sleep(15) + return question(cont) + return 'gpt当前繁忙，请稍后重试' + e diff --git a/ai_module/nlp_gpt.py b/ai_module/nlp_gpt.py index 4fdc36d..0a51e73 100644 --- a/ai_module/nlp_gpt.py +++ b/ai_module/nlp_gpt.py @@ -1,5 +1,4 @@ -from revChatGPT.V1 import Chatbot -from core.content_db import Content_Db +from revChatGPT.V3 import Chatbot from utils import config_util as cfg import time @@ -7,19 +6,8 @@ count = 0 def question(cont): global count try: - chatbot = Chatbot(config={ - "access_token": cfg.key_gpt_access_token, - "paid": False, - "collect_analytics": True, - "model": "gpt-4", - "conversation_id":cfg.key_gpt_conversation_id - },conversation_id=cfg.key_gpt_conversation_id, - parent_id=None) - - prompt = cont - response = "" - for data in chatbot.ask(prompt): - response = data["message"] + chatbot = Chatbot(proxy = cfg.proxy_config, api_key = cfg.key_chatgpt_api_key) + response = chatbot.ask(cont) count = 0 return response except Exception as e: diff --git a/ai_module/yuan_1_0.py b/ai_module/nlp_yuan.py similarity index 100% rename from ai_module/yuan_1_0.py rename to ai_module/nlp_yuan.py diff --git a/ai_module/yolov8.py b/ai_module/yolov8.py index f5fb503..1213314 100644 --- a/ai_module/yolov8.py +++ b/ai_module/yolov8.py @@ -100,10 +100,6 @@ class FeiEyes: cv2.putText(operated_frame, f"{res.names[int(cls.item())]}", (int(x1.item()), int(y1.item()) - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) if res.keypoints is not None and res.keypoints.size(0) > 0: # check if keypoints exist keypoints = res.keypoints[0] - - #TODO人脸相似性的比较，待优化 - keypoints_np = keypoints[0:5].cpu().numpy() - mtx1, mtx2, disparity = procrustes(keypoints_np[:, :2], self.my_face) #总人数 person_count += 1 #坐着的人数 @@ -142,5 +138,6 @@ def new_instance(): return __fei_eyes + diff --git a/core/fay_core.py b/core/fay_core.py index ebcced5..d665a29 100644 --- a/core/fay_core.py +++ b/core/fay_core.py @@ -21,64 +21,77 @@ from core.interact import Interact from core.tts_voice import EnumVoice from scheduler.thread_manager import MyThread from utils import util, storer, config_util -from ai_module import yuan_1_0 -from ai_module import chatgpt + + import pygame from utils import config_util as cfg from core.content_db import Content_Db from datetime import datetime from ai_module import nlp_rasa +from ai_module import nlp_chatgpt from ai_module import nlp_gpt +from ai_module import nlp_yuan from ai_module import yolov8 -from ai_module import nlp_VisualGLM as VisualGLM +from ai_module import nlp_VisualGLM import platform if platform.system() == "Windows": import sys sys.path.append("test/ovr_lipsync") from test_olipsync import LipSyncGenerator + from ai_module import nlp_lingju +modules = { + "nlp_yuan": nlp_yuan, + "nlp_gpt": nlp_gpt, + "nlp_chatgpt": nlp_chatgpt, + "nlp_rasa": nlp_rasa, + "nlp_VisualGLM": nlp_VisualGLM, + "nlp_lingju": nlp_lingju +} + + +def determine_nlp_strategy(sendto,msg): + text = '' + textlist = [] + try: + util.log(1, '自然语言处理...') + tm = time.time() + cfg.load_config() + if sendto == 2: + text = nlp_chatgpt.question(msg) + else: + module_name = "nlp_" + cfg.key_chat_module + selected_module = modules.get(module_name) + if selected_module is None: + raise RuntimeError('灵聚key、yuan key、gpt key都没有配置！') + if cfg.key_chat_module == 'rasa': + textlist = selected_module.question(msg) + text = textlist[0]['text'] + else: + text = selected_module.question(msg) + util.log(1, '自然语言处理完成. 耗时: {} ms'.format(math.floor((time.time() - tm) * 1000))) + if text == '哎呀，你这么说我也不懂，详细点呗' or text == '': + util.log(1, '[!] 自然语言无语了！') + text = '哎呀，你这么说我也不懂，详细点呗' + except BaseException as e: + print(e) + util.log(1, '自然语言处理错误！') + text = '哎呀，你这么说我也不懂，详细点呗' + + return text,textlist + + + + + #文本消息处理 def send_for_answer(msg,sendto): contentdb = Content_Db() contentdb.add_content('member','send',msg) - text = '' - textlist = [] - try: - util.log(1, '自然语言处理...') - tm = time.time() - cfg.load_config() - if sendto == 2: - text = nlp_gpt.question(msg) - else: - - if cfg.key_chat_module == 'yuan': - text = yuan_1_0.question(msg) - elif cfg.key_chat_module == 'chatgpt': - text = chatgpt.question(msg) - elif cfg.key_chat_module == 'rasa': - textlist = nlp_rasa.question(msg) - text = textlist[0]['text'] - elif cfg.key_chat_module == "VisualGLM": - text = VisualGLM.question(msg) - elif cfg.key_chat_module == "lingju": - text = nlp_lingju.question(msg) + text,textlist = determine_nlp_strategy(sendto,msg) - - else: - raise RuntimeError('灵聚key、yuan key、gpt key都没有配置！') - util.log(1, '自然语言处理完成. 耗时: {} ms'.format(math.floor((time.time() - tm) * 1000))) - if text == '哎呀，你这么说我也不懂，详细点呗' or text == '': - util.log(1, '[!] 自然语言无语了！') - text = '哎呀，你这么说我也不懂，详细点呗' - except BaseException as e: - print(e) - util.log(1, '自然语言处理错误！') - text = '哎呀，你这么说我也不懂，详细点呗' - - now = datetime.now() - timetext = str(now.strftime("%Y-%m-%d %H:%M:%S")) contentdb.add_content('fay','send',text) wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":text}}) if len(textlist) > 1: @@ -88,6 +101,8 @@ def send_for_answer(msg,sendto): wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":textlist[i]['text']}}) i+= 1 return text + + class FeiFei: def __init__(self): pygame.mixer.init() @@ -238,8 +253,8 @@ class FeiFei: fay_eyes = yolov8.new_instance() if fay_eyes.get_status():#YOLO正在运行 person_count, stand_count, sit_count = fay_eyes.get_counts() - if person_count != 1: #不是有且只有一个人，不互动 - wsa_server.get_web_instance().add_cmd({"panelMsg": "不是有且只有一个人，不互动"}) + if person_count < 1: #看不到人，不互动 + wsa_server.get_web_instance().add_cmd({"panelMsg": "看不到人，不互动"}) continue answer = self.__get_answer(interact.interleaver, self.q_msg)#确定是否命中指令或q&a @@ -252,35 +267,10 @@ class FeiFei: wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":self.q_msg}}) text = '' textlist = [] + self.speaking = True if answer is None: - try: - wsa_server.get_web_instance().add_cmd({"panelMsg": "思考中..."}) - util.log(1, '自然语言处理...') - tm = time.time() - cfg.load_config() - if cfg.key_chat_module == 'yuan': - text = yuan_1_0.question(self.q_msg) - elif cfg.key_chat_module == 'chatgpt': - text = chatgpt.question(self.q_msg) - elif cfg.key_chat_module == 'rasa': - textlist = nlp_rasa.question(self.q_msg) - text = textlist[0]['text'] - elif cfg.key_chat_module == "VisualGLM": - text = VisualGLM.question(self.q_msg) - elif cfg.key_chat_module == "lingju": - text = nlp_lingju.question(self.q_msg) - else: - raise RuntimeError('灵聚key、yuan key、gpt key都没有配置！') - util.log(1, '自然语言处理完成. 耗时: {} ms'.format(math.floor((time.time() - tm) * 1000))) - if text == '哎呀，你这么说我也不懂，详细点呗' or text == '': - util.log(1, '[!] 自然语言无语了！') - wsa_server.get_web_instance().add_cmd({"panelMsg": ""}) - continue - except BaseException as e: - print(e) - util.log(1, '自然语言处理错误！') - wsa_server.get_web_instance().add_cmd({"panelMsg": ""}) - continue + wsa_server.get_web_instance().add_cmd({"panelMsg": "思考中..."}) + text,textlist = determine_nlp_strategy(1,self.q_msg) elif answer != 'NO_ANSWER': #语音内容没有命中指令,回复q&a内容 text = answer self.a_msg = text @@ -293,8 +283,7 @@ class FeiFei: wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":textlist[i]['text']}}) i+= 1 wsa_server.get_web_instance().add_cmd({"panelMsg": self.a_msg}) - self.last_speak_data = self.a_msg - self.speaking = True + self.last_speak_data = self.a_msg MyThread(target=self.__say, args=['interact']).start() except BaseException as e: diff --git a/core/recorder.py b/core/recorder.py index e68401e..46d8130 100644 --- a/core/recorder.py +++ b/core/recorder.py @@ -28,7 +28,7 @@ class Recorder: self.__processing = False self.__history_level = [] self.__history_data = [] - self.__dynamic_threshold = 0.35 # 声音识别的音量阈值 + self.__dynamic_threshold = 0.7 # 声音识别的音量阈值 self.__MAX_LEVEL = 25000 self.__MAX_BLOCK = 100 diff --git a/requirements.txt b/requirements.txt index f24e6c3..33b79ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests~=2.26.0 -numpy~=1.22.0 +numpy pyaudio~=0.2.11 websockets~=10.2 ws4py~=0.5.1 diff --git a/system.conf b/system.conf index b8751d2..eed369e 100644 --- a/system.conf +++ b/system.conf @@ -20,20 +20,20 @@ ms_tts_region= xf_ltp_app_id= xf_ltp_api_key= -#NLP五选一:lingju、yuan、chatgpt、rasa(需启动chatglm及rasa，https://m.bilibili.com/video/BV1D14y1f7pr)、VisualGLM +#NLP五选一:yuan、gpt、chatgpt、rasa(需启动chatglm及rasa，https://m.bilibili.com/video/BV1D14y1f7pr)、lingju chat_module=lingju -#灵聚 服务密钥(NLP多选1) https://open.lingju.ai -lingju_api_key= -lingju_api_authcode= -#浪.潮源大模型 服务密钥(NLP5多1) https://air.inspur.com/ +#浪.潮源大模型 服务密钥(NLP5选1) https://air.inspur.com/ yuan_1_0_account= yuan_1_0_phone= -#gpt 服务密钥(NLP多选1) https://openai.com/ +#gpt 服务密钥(NLP5选1) https://openai.com/ chatgpt_api_key= +#灵聚 服务密钥 https://open.lingju.ai +lingju_api_key= +lingju_api_authcode= #ngrok内网穿透id，远程设备可以通过互联网连接Fay（非必须）http://ngrok.cc ngrok_cc_id= @@ -42,3 +42,6 @@ ngrok_cc_id= gpt_access_token= gpt_conversation_id= +#gpt代理 +proxy_config=http://192.168.1.13:7890 + diff --git a/utils/config_util.py b/utils/config_util.py index 3304199..62028af 100644 --- a/utils/config_util.py +++ b/utils/config_util.py @@ -21,6 +21,7 @@ key_chatgpt_api_key = None key_chat_module = None key_gpt_access_token = None key_gpt_conversation_id = None +proxy_config = None ASR_mode = None local_asr_ip = None @@ -45,6 +46,7 @@ def load_config(): global key_gpt_conversation_id global key_lingju_api_key global key_lingju_api_authcode + global proxy_config global ASR_mode global local_asr_ip @@ -73,6 +75,8 @@ def load_config(): local_asr_ip = system_config.get('key', 'local_asr_ip') local_asr_port = system_config.get('key', 'local_asr_port') + proxy_config = system_config.get('key', 'proxy_config') + config = json.load(codecs.open('config.json', encoding='utf-8'))