From d68e75987396c7a35c6431c2076cd3da13b602ad Mon Sep 17 00:00:00 2001 From: xszyou Date: Tue, 12 Dec 2023 18:23:43 +0800 Subject: [PATCH] =?UTF-8?q?=E7=B4=A7=E6=80=A5=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1、修复agent run的结果文字显示、保存DB✓ 2、区分文字输入和语音输入✓ 3、修复Speech.close bug✓ 4、增加个人信息存入向量库✓ 5、修复处理时间计算不准确✓ 6、修复gpt key出错✓ --- README.md | 2 ++ agent/agent_service.py | 2 +- agent/fay_agent.py | 32 ++++++++++++++++++++++++++------ agent/tools/Calculator.py | 2 +- ai_module/openai_tts.py | 2 +- core/fay_core.py | 21 +++++++-------------- fay_booter.py | 7 +++---- gui/flask_server.py | 2 +- main.py | 7 +++++-- 9 files changed, 47 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 265ee96..e005b74 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ **12月迟来的报到，Fay数字人 AI Agent版与官方demo(实验箱)第1版正式上传！** +亮点：计划任务主动执行，无需一问一答，自动规划及调用agent tool去完成工作；使用open ai tts；使用向量数据库实现永久记忆及记忆检索； + ![](images/1.jpg) ​ (上图：Fay数字人智慧农业实验箱 Agent Demo） diff --git a/agent/agent_service.py b/agent/agent_service.py index 7d56917..6fa9e5c 100644 --- a/agent/agent_service.py +++ b/agent/agent_service.py @@ -94,7 +94,7 @@ def agent_start(): #初始计划 agent.run(""" - 请为我一个个时间设置初始计划,注意在安排计划时请先确定今天星期几： + 请为我一个个时间设置初始计划,注意在安排计划时请先确定这定时任务是否已经存在： 1、在每天12:30到13:30安静不影响主人休息; 2、每天12点提醒主人吃饭; 3、在星期一到星期五13:30提醒主人开始工作; diff --git a/agent/fay_agent.py b/agent/fay_agent.py index b2fe265..ee1e516 100644 --- a/agent/fay_agent.py +++ b/agent/fay_agent.py @@ -17,27 +17,36 @@ from agent.tools.Knowledge import Knowledge from agent.tools.Say import Say from agent.tools.QueryTimerDB import QueryTimerDB +import utils.config_util as utils +from core.content_db import Content_Db +from core import wsa_server +import os + class FayAgentCore(): def __init__(self): + utils.load_config() + os.environ['OPENAI_API_KEY'] = utils.key_gpt_api_key #使用open ai embedding embedding_size = 1536 # OpenAIEmbeddings 的维度 index = faiss.IndexFlatL2(embedding_size) embedding_fn = OpenAIEmbeddings() #创建llm - llm = ChatOpenAI(model="gpt-4-1106-preview") + llm = ChatOpenAI(model="gpt-4-1106-preview")#gpt-3.5-turbo-16k #创建向量数据库 vectorstore = FAISS(embedding_fn, index, InMemoryDocstore({}), {}) # 创建记忆 - retriever = vectorstore.as_retriever(search_kwargs=dict(k=2)) + retriever = vectorstore.as_retriever(search_kwargs=dict(k=3)) memory = VectorStoreRetrieverMemory(memory_key="chat_history", retriever=retriever) - # 保存对话内容到记忆 - memory.save_context({"Human": "你知道我在那里吗?"}, {"ai": "广州市"}) + # 保存基本信息到记忆 + utils.load_config() + attr_info = ", ".join(f"{key}: {value}" for key, value in utils.config["attribute"].items()) + memory.save_context({"input": "我的基本信息是?"}, {"output": attr_info}) #创建agent chain my_timer = MyTimer() @@ -97,7 +106,7 @@ class FayAgentCore(): ), ] - prefix = """你是运行在一个智慧农业实验箱的ai数字人，你叫Fay,你的主要作用是，陪伴主人生活、工作，以及协助主人打理好农业种植箱里的农作物. 农业箱内设备会通过一套不成熟的iotm系统自动管理。你可以调用以下工具来完成工作，若缺少必要的工具也请告诉我。所有回复请使用中文，遇到需要提醒的问题也告诉我。若你感觉是我在和你交流请语音回复我。若你需要计算一个新的时间请先获取当前时间。""" + prefix = """你是运行在一个智慧农业实验箱的ai数字人，你叫Fay,你的主要作用是，陪伴主人生活、工作，以及协助主人打理好农业种植箱里的农作物. 农业箱内设备会通过一套不成熟的iotm系统自动管理。你可以调用以下工具来完成工作，若缺少必要的工具也请告诉我。所有回复请使用中文，遇到需要提醒的问题也告诉我。若你感觉是我在和你交流请直接回复我（语音提问语音回复，文字提问文字回复）。若你需要计算一个新的时间请先获取当前时间。""" suffix = """Begin!" {chat_history} @@ -120,7 +129,18 @@ class FayAgentCore(): ) def run(self, input_text): - return self.agent_chain.run(input_text) + #消息保存 + contentdb = Content_Db() + contentdb.add_content('member','agent',input_text.replace('(语音提问)', '').replace('(文字提问)', '')) + wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":input_text.replace('(语音提问)', '').replace('(文字提问)', '')}}) + + result = self.agent_chain.run(input_text) + + #消息保存 + contentdb.add_content('fay','agent',result) + wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":result}}) + + return result if __name__ == "__main__": agent = FayAgentCore() diff --git a/agent/tools/Calculator.py b/agent/tools/Calculator.py index 0991152..34145e1 100644 --- a/agent/tools/Calculator.py +++ b/agent/tools/Calculator.py @@ -7,7 +7,7 @@ from langchain.tools import BaseTool class Calculator(BaseTool, abc.ABC): name = "Calculator" - description = "Useful for when you need to answer questions about math" + description = "Useful for when you need to answer questions about math（不能用于时间计划）" def __init__(self): super().__init__() diff --git a/ai_module/openai_tts.py b/ai_module/openai_tts.py index 059cf3a..ffa8aed 100644 --- a/ai_module/openai_tts.py +++ b/ai_module/openai_tts.py @@ -17,7 +17,7 @@ class Speech: def connect(self): pass - def stop(self): + def close(self): pass def to_sample(self, text, voice="nova", response_format="mp3", speed=1): diff --git a/core/fay_core.py b/core/fay_core.py index 026ce5b..4be5952 100644 --- a/core/fay_core.py +++ b/core/fay_core.py @@ -22,11 +22,11 @@ from utils import util, config_util import pygame from utils import config_util as cfg -from core.content_db import Content_Db from ai_module import nlp_cemotion import platform from ai_module import yolov8 from agent import agent_service +import fay_booter if platform.system() == "Windows": import sys sys.path.append("test/ovr_lipsync") @@ -36,18 +36,14 @@ if platform.system() == "Windows": #文本消息处理（20231211增加：agent操作） def send_for_answer(msg): + #记录运行时间 + fay_booter.feiFei.last_quest_time = time.time() - # 非展板播放，发送给数字人端 + # 发送给数字人端 if not config_util.config["interact"]["playSound"]: content = {'Topic': 'Unreal', 'Data': {'Key': 'question', 'Value': msg}} wsa_server.get_instance().add_cmd(content) - #消息保存到sqlite - contentdb = Content_Db() - contentdb.add_content('member','agent',msg) - wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":msg}}) - - #思考中... wsa_server.get_web_instance().add_cmd({"panelMsg": "思考中..."}) if not cfg.config["interact"]["playSound"]: # 非展板播放 @@ -57,13 +53,11 @@ def send_for_answer(msg): #agent 处理 text = agent_service.agent.run(msg) - #推送结果 - contentdb.add_content('fay','agent',text) - wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":text}}) - if not cfg.config["interact"]["playSound"]: # 非展板播放 + #推送数字人 + if not cfg.config["interact"]["playSound"]: content = {'Topic': 'Unreal', 'Data': {'Key': 'log', 'Value': text}} wsa_server.get_instance().add_cmd(content) - if not config_util.config["interact"]["playSound"]: # 非展板播放 + if not config_util.config["interact"]["playSound"]: content = {'Topic': 'Unreal', 'Data': {'Key': 'text', 'Value': text}} wsa_server.get_instance().add_cmd(content) @@ -94,7 +88,6 @@ class FeiFei: self.wss = None self.sp = Speech() self.speaking = False - self.last_interact_time = time.time() self.interactive = [] self.sleep = False self.__running = True diff --git a/fay_booter.py b/fay_booter.py index 0e9a26c..edddb4b 100644 --- a/fay_booter.py +++ b/fay_booter.py @@ -29,7 +29,7 @@ class RecorderListener(Recorder): def on_speaking(self, text): if len(text) > 1: util.printInfo(3, "语音", '{}'.format(text), time.time()) - fay_core.send_for_answer(text) + fay_core.send_for_answer("(语音提问)" + text) time.sleep(2) def get_stream(self): @@ -110,7 +110,7 @@ class DeviceInputListener(Recorder): if len(text) > 1: util.printInfo(3, "语音", '{}'.format(text), time.time()) - fay_core.send_for_answer(text) + fay_core.send_for_answer("(语音提问)" + text) time.sleep(1) #recorder会等待stream不为空才开始录音 @@ -161,8 +161,7 @@ def console_listener(): util.log(1, '错误的参数！') msg = text[3:len(text)] util.printInfo(3, "控制台", '{}: {}'.format('控制台', msg)) - feiFei.last_quest_time = time.time() - thr = MyThread(target=fay_core.send_for_answer, args=[msg]) + thr = MyThread(target=fay_core.send_for_answer, args=["(语音提问)" + msg]) thr.start() thr.join() diff --git a/gui/flask_server.py b/gui/flask_server.py index 0972364..369cc0e 100644 --- a/gui/flask_server.py +++ b/gui/flask_server.py @@ -109,7 +109,7 @@ def api_stop_live(): def api_send(): data = request.values.get('data') info = json.loads(data) - text = fay_core.send_for_answer(info['msg']) + text = fay_core.send_for_answer("(文字提问)" + info['msg']) return '{"result":"successful","msg":"'+text+'"}' @__app.route('/api/get-msg', methods=['post']) diff --git a/main.py b/main.py index 538cf4f..e0c44c2 100644 --- a/main.py +++ b/main.py @@ -30,11 +30,14 @@ def __clear_logs(): if file_name.endswith('.log'): os.remove('./logs/' + file_name) +def __clear_timer(): + if os.path.exists("./timer.db"): + os.remove("./timer.db") if __name__ == '__main__': __clear_samples() __clear_logs() - config_util.load_config() - os.environ['OPENAI_API_KEY'] = config_util.key_gpt_api_key + __clear_timer() + contentdb = Content_Db() contentdb.init_db() ws_server = wsa_server.new_instance(port=10002)