diff --git a/config.json b/config.json index 1214e07..a7dfbd4 100644 --- a/config.json +++ b/config.json @@ -8,11 +8,11 @@ "hobby": "\u53d1\u5446", "job": "\u52a9\u7406", "name": "\u83f2\u83f2", - "voice": "\u6653\u6653(azure)", + "voice": "abin", "zodiac": "\u86c7" }, "interact": { - "QnA": "", + "QnA": "qa.csv", "maxInteractTime": 15, "perception": { "chat": 10, @@ -21,8 +21,7 @@ "indifferent": 10, "join": 10 }, - "playSound": true, - "sound_synthesis_enabled": false, + "playSound": false, "visualization": false }, "items": [], @@ -34,12 +33,11 @@ "url": "" }, "record": { - "channels": 0, "device": "", - "enabled": true + "enabled": false }, "wake_word": "\u4f60\u597d", - "wake_word_enabled": true, + "wake_word_enabled": false, "wake_word_type": "front" } } \ No newline at end of file diff --git a/core/fay_core.py b/core/fay_core.py index c99b6b2..ec4874b 100644 --- a/core/fay_core.py +++ b/core/fay_core.py @@ -29,6 +29,7 @@ from llm import nlp_xingchen from llm import nlp_langchain from llm import nlp_ollama_api from llm import nlp_coze +from llm.agent import fay_agent from core import member_db import threading import functools @@ -60,7 +61,8 @@ modules = { "nlp_xingchen": nlp_xingchen, "nlp_langchain": nlp_langchain, "nlp_ollama_api": nlp_ollama_api, - "nlp_coze": nlp_coze + "nlp_coze": nlp_coze, + "nlp_agent": fay_agent } @@ -145,9 +147,9 @@ class FeiFei: uid = member_db.new_instance().find_user(username) #记录用户问题 - content_db.new_instance().add_content('member','speak',interact.data["msg"], username, uid) + content_id = content_db.new_instance().add_content('member','speak',interact.data["msg"], username, uid) if wsa_server.get_web_instance().is_connected(username): - wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":interact.data["msg"], "username":username, "uid":uid}, "Username" : username}) + wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":interact.data["msg"], "username":username, "uid":uid, "id":content_id}, "Username" : username}) #确定是否命中q&a answer = self.__get_answer(interact.interleaver, interact.data["msg"]) @@ -163,18 +165,17 @@ class FeiFei: wsa_server.get_instance().add_cmd(content) text,textlist = handle_chat_message(interact.data["msg"], username, interact.data.get("observation", "")) - # qa_service.QAService().record_qapair(interact.data["msg"], text)#沟通记录缓存到qa文件 else: text = answer #记录回复 self.write_to_file("./logs", "answer_result.txt", text) - content_db.new_instance().add_content('fay','speak',text, username, uid) + content_id = content_db.new_instance().add_content('fay','speak',text, username, uid) #文字输出：面板、聊天窗、log、数字人 if wsa_server.get_web_instance().is_connected(username): wsa_server.get_web_instance().add_cmd({"panelMsg": text, "Username" : username, 'robot': f'http://{cfg.fay_url}:5000/robot/Speaking.jpg'}) - wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":text, "username":username, "uid":uid}, "Username" : username}) + wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":text, "username":username, "uid":uid, "id":content_id}, "Username" : username}) if len(textlist) > 1: i = 1 while i < len(textlist): @@ -198,8 +199,6 @@ class FeiFei: if member_db.new_instance().is_username_exist(username) == "notexists": member_db.new_instance().add_user(username) uid = member_db.new_instance().find_user(username) - - #TODO 这里可以通过qa来触发指定的脚本操作，如ppt翻页等 if interact.data.get("text"): #记录回复 @@ -217,7 +216,8 @@ class FeiFei: wsa_server.get_instance().add_cmd(content) #声音输出 - MyThread(target=self.say, args=[interact, text]).start() + MyThread(target=self.say, args=[interact, text]).start() + except BaseException as e: print(e) @@ -319,9 +319,6 @@ class FeiFei: if audio_url is not None: file_name = 'sample-' + str(int(time.time() * 1000)) + '.wav' result = self.download_wav(audio_url, './samples/', file_name) - - elif not wsa_server.get_instance().get_client_output(interact.data.get('user')): - result = None elif config_util.config["interact"]["playSound"] or wsa_server.get_instance().is_connected(interact.data.get("user")) or self.__is_send_remote_device_audio(interact):#tts util.printInfo(1, interact.data.get('user'), '合成音频...') tm = time.time() diff --git a/core/member_db.py b/core/member_db.py index 294e6ad..53a3c84 100644 --- a/core/member_db.py +++ b/core/member_db.py @@ -83,6 +83,7 @@ class Member_Db: else: return "notexists" + #根据username查询uid def find_user(self, username): conn = sqlite3.connect('user_profiles.db') c = conn.cursor() @@ -93,6 +94,20 @@ class Member_Db: return 0 else: return result[0] + + #根据uid查询username + def find_username_by_uid(self, uid): + conn = sqlite3.connect('user_profiles.db') + c = conn.cursor() + c.execute('SELECT username FROM T_Member WHERE id = ?', (uid,)) + result = c.fetchone() + conn.close() + if result is None: + return 0 + else: + return result[0] + + @synchronized def query(self, sql): diff --git a/core/qa_service.py b/core/qa_service.py index 8ca0711..f60dbef 100644 --- a/core/qa_service.py +++ b/core/qa_service.py @@ -33,7 +33,7 @@ class QAService: def question(self, query_type, text): if query_type == 'qa': - answer_dict = self.__read_qna(cfg.config['interact']['QnA']) + answer_dict = self.__read_qna(cfg.config['interact'].get('QnA')) answer, action = self.__get_keyword(answer_dict, text, query_type) if action: MyThread(target=self.__run, args=[action]).start() @@ -61,7 +61,7 @@ class QAService: if len(row) >= 2: qna.append([row[0].split(";"), row[1], row[2] if len(row) >= 3 else None]) except Exception as e: - util.log(1, 'qa文件没有指定，不匹配qa') + pass return qna def record_qapair(self, question, answer): diff --git a/core/recorder.py b/core/recorder.py index c2cdee8..a6a56a2 100644 --- a/core/recorder.py +++ b/core/recorder.py @@ -46,6 +46,8 @@ class Recorder: self.username = 'User' #默认用户，子类实现时会重写 self.channels = 1 self.sample_rate = 16000 + self.is_reading = False + self.stream = None def asrclient(self): if self.ASRMode == "ali": @@ -204,7 +206,7 @@ class Recorder: cfg.load_config() record = cfg.config['source']['record'] if not record['enabled'] and not self.is_remote: - time.sleep(0.1) + time.sleep(1) continue self.is_reading = True data = stream.read(1024, exception_on_overflow=False) diff --git a/fay_booter.py b/fay_booter.py index 6e1c6c8..e20e0a1 100644 --- a/fay_booter.py +++ b/fay_booter.py @@ -14,6 +14,7 @@ from utils import util, config_util, stream_util from core.wsa_server import MyServer from core import wsa_server from core import socket_bridge_service +from llm.agent import agent_service feiFei: fay_core.FeiFei = None recorderListener: Recorder = None @@ -96,9 +97,10 @@ class RecorderListener(Recorder): try: while self.is_reading: time.sleep(0.1) - self.stream.stop_stream() - self.stream.close() - self.paudio.terminate() + if self.stream is not None: + self.stream.stop_stream() + self.stream.close() + self.paudio.terminate() except Exception as e: print(e) util.log(1, "请检查设备是否有误，再重新启动!") @@ -186,7 +188,7 @@ def device_socket_keep_alive(): if wsa_server.get_web_instance().is_connected(value.username): wsa_server.get_web_instance().add_cmd({"remote_audio_connect": True, "Username" : value.username}) except Exception as serr: - util.printInfo(3, value.username, "远程音频输入输出设备已经断开：{}".format(key)) + util.printInfo(1, value.username, "远程音频输入输出设备已经断开：{}".format(key)) value.stop() delkey = key break @@ -222,6 +224,8 @@ def accept_audio_device_output_connect(): #数字人端请求获取最新的自动播放消息，若自动播放服务关闭会自动退出自动播放 def start_auto_play_service(): #TODO 评估一下有无优化的空间 + if config_util.config['source'].get('automatic_player_url') is None or config_util.config['source'].get('automatic_player_status') is None: + return url = f"{config_util.config['source']['automatic_player_url']}/get_auto_play_item" user = "User" #TODO 临时固死了 is_auto_server_error = False @@ -290,6 +294,11 @@ def stop(): socket_service_instance = None except: pass + + if config_util.key_chat_module == "agent": + util.log(1, '正在关闭agent服务...') + agent_service.agent_stop() + util.log(1, '正在关闭核心服务...') feiFei.stop() util.log(1, '服务已关闭！') @@ -325,18 +334,22 @@ def start(): record = config_util.config['source']['record'] if record['enabled']: util.log(1, '开启录音服务...') - recorderListener = RecorderListener(record['device'], feiFei) # 监听麦克风 + recorderListener = RecorderListener('device', feiFei) # 监听麦克风 recorderListener.start() #启动声音沟通接口服务 util.log(1,'启动声音沟通接口服务...') deviceSocketThread = MyThread(target=accept_audio_device_output_connect) deviceSocketThread.start() - socket_service_instance = socket_bridge_service.new_instance() socket_bridge_service_Thread = MyThread(target=socket_service_instance.start_service) socket_bridge_service_Thread.start() + #启动agent服务 + if config_util.key_chat_module == "agent": + util.log(1,'启动agent服务...') + agent_service.agent_start() + #启动自动播放服务 util.log(1,'启动自动播放服务...') MyThread(target=start_auto_play_service).start() diff --git a/gui/flask_server.py b/gui/flask_server.py index 1266c6a..0b06ca3 100644 --- a/gui/flask_server.py +++ b/gui/flask_server.py @@ -46,6 +46,7 @@ def verify_password(username, password): if username in users and users[username] == password: return username + def __get_template(): try: return render_template('index.html') @@ -68,6 +69,7 @@ def __get_device_list(): print(f"Error getting device list: {e}") return [] + @__app.route('/api/submit', methods=['post']) def api_submit(): data = request.values.get('data') @@ -252,7 +254,7 @@ def api_send(): if not username or not msg: return jsonify({'result': 'error', 'message': '用户名和消息内容不能为空'}) interact = Interact("text", 1, {'user': username, 'msg': msg}) - util.printInfo(3, "文字发送按钮", '{}'.format(interact.data["msg"]), time.time()) + util.printInfo(1, username, '[文字发送按钮]{}'.format(interact.data["msg"]), time.time()) fay_booter.feiFei.on_interact(interact) return '{"result":"successful"}' except json.JSONDecodeError: @@ -263,11 +265,12 @@ def api_send(): # 获取指定用户的消息记录 @__app.route('/api/get-msg', methods=['post']) def api_get_Msg(): - data = request.form.get('data') - if not data: - return jsonify({'list': [], 'message': '未提供数据'}) try: - data = json.loads(data) + data = request.form.get('data') + if data is None: + data = request.get_json() + else: + data = json.loads(data) uid = member_db.new_instance().find_user(data["username"]) contentdb = content_db.new_instance() if uid == 0: @@ -310,7 +313,7 @@ def api_send_v1_chat_completions(): model = data.get('model', 'fay') observation = data.get('observation', '') interact = Interact("text", 1, {'user': username, 'msg': last_content, 'observation': observation}) - util.printInfo(3, "文字沟通接口", '{}'.format(interact.data["msg"]), time.time()) + util.printInfo(1, username, '[文字沟通接口]{}'.format(interact.data["msg"]), time.time()) text = fay_booter.feiFei.on_interact(interact) if model == 'fay-streaming': @@ -393,7 +396,7 @@ def stream_response(text): yield f"data: {json.dumps(message)}\n\n" time.sleep(0.1) yield 'data: [DONE]\n\n' - + return Response(generate(), mimetype='text/event-stream') def non_streaming_response(last_content, text): diff --git a/gui/static/js/index.js b/gui/static/js/index.js index 3fc1f17..6e97143 100644 --- a/gui/static/js/index.js +++ b/gui/static/js/index.js @@ -190,6 +190,7 @@ class FayInterface { } if (vueInstance.selectedUser && data.panelReply.username === vueInstance.selectedUser[1]) { vueInstance.messages.push({ + id: data.panelReply.id, username: data.panelReply.username, content: data.panelReply.content, type: data.panelReply.type, diff --git a/gui/templates/index.html b/gui/templates/index.html index 14506ce..e980e1f 100644 --- a/gui/templates/index.html +++ b/gui/templates/index.html @@ -14,7 +14,7 @@
-