diff --git a/README.md b/README.md index e005b74..2b612ac 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,54 @@
Fay

Fay数字人 AI Agent版

+ “agent”即代理，它能够代替你完成决策规划并执行，这一切都依赖目前最强的大语言模型的ReAct能力。 -[`带货完整版`](https://github.com/TheRamU/Fay/tree/fay-sales-edition) [`助理完整版`](https://github.com/TheRamU/Fay/tree/fay-assistant-edition) +**12月迟来的报到，Fay数字人 AI Agent版（含智慧农业应用demo）第3版正式上传！** + +如果你需要是一个线上线下的销售员，请移步[`带货完整版`](https://github.com/TheRamU/Fay/tree/fay-sales-edition) + +如果你需要的是一个人机交互的数字人助理（当然，你也可以命令它开关设备），请移步 [`助理完整版`](https://github.com/TheRamU/Fay/tree/fay-assistant-edition) + + ***“优秀的产品都值得用数字人从新做一遍”*** -**12月迟来的报到，Fay数字人 AI Agent版与官方demo(实验箱)第1版正式上传！** - 亮点：计划任务主动执行，无需一问一答，自动规划及调用agent tool去完成工作；使用open ai tts；使用向量数据库实现永久记忆及记忆检索； -![](images/1.jpg) +![](images/agent_demo.gif) -​ (上图：Fay数字人智慧农业实验箱 Agent Demo） +​ (上图：实测ReAct能力） + +## **安装说明** + + +### **环境要求** + +- Python 3.9、3.10 +- Windows、macos、linux + +### **安装依赖** + +```shell +pip install -r requirements.txt +``` + +### **配置应用密钥** + ++ 将GPT-4 key填入 `./system.conf` 中 + +### **启动控制器** + +启动Fay控制器 + +```shell +python main.py +``` + +### **启动数字人（非必须）** + ++ 启动数字人[xszyou/fay-ue5: 可对接fay数字人的ue5工程 (github.com)](https://github.com/xszyou/fay-ue5) ### **联系** diff --git a/agent/agent_service.py b/agent/agent_service.py index 6fa9e5c..92d876e 100644 --- a/agent/agent_service.py +++ b/agent/agent_service.py @@ -14,6 +14,7 @@ def init_db(): cursor = conn.cursor() cursor.execute(''' CREATE TABLE IF NOT EXISTS timer ( + id INTEGER PRIMARY KEY AUTOINCREMENT, time TEXT NOT NULL, repeat_rule TEXT NOT NULL, content TEXT NOT NULL @@ -32,7 +33,7 @@ def insert_test_data(): conn.commit() conn.close() -# 解析重复规则 +# 解析重复规则返回待执行时间，None代表不在今天的待执行计划 def parse_repeat_rule(rule, task_time): today = datetime.datetime.now() if rule == '0000000': # 不重复 @@ -49,8 +50,10 @@ def parse_repeat_rule(rule, task_time): return None # 执行任务 -def execute_task(task_time, content): - print(f"Executing task scheduled for {task_time}: {content} --> {agent.run(content)}") +def execute_task(task_time, id, content): + content = content + agent.run(content) + del scheduled_tasks[id] # 如果不重复，执行后删除记录 conn = sqlite3.connect('timer.db') cursor = conn.cursor() @@ -59,26 +62,23 @@ def execute_task(task_time, content): conn.close() -# 检查并执行任务 +# 30秒扫描一次数据库，当扫描到今天的不存在于定时任务列表的记录，则添加到定时任务列表。执行完的记录从定时任务列表中清除。 def check_and_execute(): while agent_running: conn = sqlite3.connect('timer.db') cursor = conn.cursor() - cursor.execute("SELECT time, repeat_rule, content FROM timer") + cursor.execute("SELECT * FROM timer") rows = cursor.fetchall() for row in rows: - task_time_str, repeat_rule, content = row + id, task_time_str, repeat_rule, content = row task_time = datetime.datetime.strptime(task_time_str, '%H:%M').time() next_execution = parse_repeat_rule(repeat_rule, task_time) - if next_execution and (task_time_str not in scheduled_tasks or scheduled_tasks[task_time_str] < next_execution): - # 更新线程 - if task_time_str in scheduled_tasks: - scheduled_tasks[task_time_str].cancel() - timer_thread = threading.Timer((next_execution - datetime.datetime.now()).total_seconds(), execute_task, [next_execution, content]) + if next_execution and id not in scheduled_tasks: + timer_thread = threading.Timer((next_execution - datetime.datetime.now()).total_seconds(), execute_task, [next_execution, id, content]) timer_thread.start() - scheduled_tasks[task_time_str] = next_execution + scheduled_tasks[id] = timer_thread conn.close() time.sleep(30) # 30秒扫描一次 @@ -86,6 +86,8 @@ def check_and_execute(): # agent启动 def agent_start(): global agent_running + global agent + agent_running = True init_db() # insert_test_data() @@ -93,8 +95,8 @@ def agent_start(): check_and_execute_thread.start() #初始计划 - agent.run(""" - 请为我一个个时间设置初始计划,注意在安排计划时请先确定这定时任务是否已经存在： + agent.run("""执行任务--> + 请为我一个个时间设置初始计划： 1、在每天12:30到13:30安静不影响主人休息; 2、每天12点提醒主人吃饭; 3、在星期一到星期五13:30提醒主人开始工作; diff --git a/agent/fay_agent.py b/agent/fay_agent.py index ee1e516..3fd8fa9 100644 --- a/agent/fay_agent.py +++ b/agent/fay_agent.py @@ -4,7 +4,7 @@ from langchain.memory import VectorStoreRetrieverMemory import faiss from langchain.docstore import InMemoryDocstore from langchain.vectorstores import FAISS -from langchain.agents import AgentExecutor, Tool, ZeroShotAgent, initialize_agent +from langchain.agents import AgentExecutor, Tool, ZeroShotAgent, initialize_agent, agent_types from langchain.chains import LLMChain from agent.tools.MyTimer import MyTimer @@ -16,6 +16,7 @@ from agent.tools.Switch import Switch from agent.tools.Knowledge import Knowledge from agent.tools.Say import Say from agent.tools.QueryTimerDB import QueryTimerDB +from agent.tools.DeleteTimer import DeleteTimer import utils.config_util as utils from core.content_db import Content_Db @@ -34,13 +35,13 @@ class FayAgentCore(): embedding_fn = OpenAIEmbeddings() #创建llm - llm = ChatOpenAI(model="gpt-4-1106-preview")#gpt-3.5-turbo-16k + llm = ChatOpenAI(verbose=True)#model="gpt-4-1106-preview" #创建向量数据库 vectorstore = FAISS(embedding_fn, index, InMemoryDocstore({}), {}) # 创建记忆 - retriever = vectorstore.as_retriever(search_kwargs=dict(k=3)) + retriever = vectorstore.as_retriever(search_kwargs=dict(k=2)) memory = VectorStoreRetrieverMemory(memory_key="chat_history", retriever=retriever) # 保存基本信息到记忆 @@ -58,6 +59,7 @@ class FayAgentCore(): knowledge_tool = Knowledge() say_tool = Say() query_timer_db_tool = QueryTimerDB() + delete_timer_tool = DeleteTimer() tools = [ Tool( name=my_timer.name, @@ -104,40 +106,34 @@ class FayAgentCore(): func=query_timer_db_tool.run, description=query_timer_db_tool.description ), - + Tool( + name=delete_timer_tool.name, + func=delete_timer_tool.run, + description=delete_timer_tool.description + ), ] - prefix = """你是运行在一个智慧农业实验箱的ai数字人，你叫Fay,你的主要作用是，陪伴主人生活、工作，以及协助主人打理好农业种植箱里的农作物. 农业箱内设备会通过一套不成熟的iotm系统自动管理。你可以调用以下工具来完成工作，若缺少必要的工具也请告诉我。所有回复请使用中文，遇到需要提醒的问题也告诉我。若你感觉是我在和你交流请直接回复我（语音提问语音回复，文字提问文字回复）。若你需要计算一个新的时间请先获取当前时间。""" - suffix = """Begin!" - {chat_history} - Question: {input} - {agent_scratchpad}""" - prompt = ZeroShotAgent.create_prompt( - tools, - prefix=prefix, - suffix=suffix, - input_variables=["input", "chat_history", "agent_scratchpad"], - ) - llm_chain = LLMChain(llm=llm, prompt=prompt, verbose=True) - agent = ZeroShotAgent(llm_chain=llm_chain, tools=tools, verbose=True) - # agent = initialize_agent(agent="chat-conversational-react-description", - # tools=tools, llm=llm, verbose=True, - # max_iterations=3, early_stopping_method="generate", memory=memory, handle_parsing_errors=True) - self.agent_chain = AgentExecutor.from_agent_and_tools( - agent=agent, tools=tools, verbose=True, memory=memory, handle_parsing_errors=True - ) + self.agent = initialize_agent(agent_types=agent_types.AgentType.CHAT_CONVERSATIONAL_REACT_DESCRIPTION, + tools=tools, llm=llm, verbose=True, + max_history=5, + memory=memory, handle_parsing_errors=True) def run(self, input_text): #消息保存 contentdb = Content_Db() - contentdb.add_content('member','agent',input_text.replace('(语音提问)', '').replace('(文字提问)', '')) - wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":input_text.replace('(语音提问)', '').replace('(文字提问)', '')}}) - - result = self.agent_chain.run(input_text) + contentdb.add_content('member', 'agent', input_text.replace('主人语音说了：', '').replace('主人文字说了：', '')) + wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":input_text.replace('主人语音说了：', '').replace('主人文字说了：', '')}}) + result = None + try: + result = self.agent.run(input_text.replace('执行任务-->', '')) + except Exception as e: + print(e) + result = "执行完毕" if result is None or result == "N/A" else result + #消息保存 - contentdb.add_content('fay','agent',result) + contentdb.add_content('fay','agent', result) wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":result}}) return result diff --git a/agent/tools/Calculator.py b/agent/tools/Calculator.py index 34145e1..6599694 100644 --- a/agent/tools/Calculator.py +++ b/agent/tools/Calculator.py @@ -7,7 +7,7 @@ from langchain.tools import BaseTool class Calculator(BaseTool, abc.ABC): name = "Calculator" - description = "Useful for when you need to answer questions about math（不能用于时间计划）" + description = "Useful for when you need to answer questions about math（不能用于非数字计算）" def __init__(self): super().__init__() diff --git a/agent/tools/CheckSensor.py b/agent/tools/CheckSensor.py index 4740445..1f04b03 100644 --- a/agent/tools/CheckSensor.py +++ b/agent/tools/CheckSensor.py @@ -7,7 +7,7 @@ from langchain.tools import BaseTool class CheckSensor(BaseTool): name = "CheckSensor" - description = "此工具用于查询传感器数据及设备开关状态" + description = "此工具用于查询农业箱传感器数据及设备开关状态" def __init__(self): super().__init__() diff --git a/agent/tools/DeleteTimer.py b/agent/tools/DeleteTimer.py new file mode 100644 index 0000000..0ef19e1 --- /dev/null +++ b/agent/tools/DeleteTimer.py @@ -0,0 +1,39 @@ +import abc +import sqlite3 +from typing import Any +import ast + +from langchain.tools import BaseTool + +from agent import agent_service + + +class DeleteTimer(BaseTool, abc.ABC): + name = "DeleteTimer" + description = "用于删除某一个定时任务，接受任务id作为参数，如：('2')" + + def __init__(self): + super().__init__() + + async def _arun(self, *args: Any, **kwargs: Any) -> Any: + # 用例中没有用到 arun 不予具体实现 + pass + + + def _run(self, para) -> str: + para = ast.literal_eval(para) + + del agent_service.scheduled_tasks[int(para[0])] + conn = sqlite3.connect('timer.db') + cursor = conn.cursor() + cursor.execute(f"DELETE FROM timer WHERE id = {id}") + conn.commit() + conn.close() + + return f"{id}任务取消成功" + + +if __name__ == "__main__": + tool = DeleteTimer() + result = tool.run("1") + print(result) diff --git a/agent/tools/QueryTimerDB.py b/agent/tools/QueryTimerDB.py index 4885c43..4600194 100644 --- a/agent/tools/QueryTimerDB.py +++ b/agent/tools/QueryTimerDB.py @@ -9,7 +9,7 @@ from langchain.tools import BaseTool class QueryTimerDB(BaseTool, abc.ABC): name = "QueryTimerDB" - description = "用于查询所有定时任务，结果包含3个参数，第1个参数是时间,第2个参数是循环规则（如:'1000100'代表星期一和星期五循环，'0000000'代表不循环），第3个参数代表要执行的事项,如：('15:15', '0000001', '提醒主人叫咖啡')" + description = "用于查询所有定时任务，返回的数据里包含3个参数:时间、循环规则（如:'1000100'代表星期一和星期五循环，'0000000'代表不循环）、执行的事项" def __init__(self): super().__init__() @@ -29,13 +29,13 @@ class QueryTimerDB(BaseTool, abc.ABC): # 拼接结果 result = "" for row in rows: - result = result + "\n" + str(row) + result = result + str(row) + "\n" conn.commit() conn.close() return result if __name__ == "__main__": - calculator_tool = MyTimer() - result = calculator_tool.run("sqrt(2) + 3") + tool = QueryTimerDB() + result = tool.run("") print(result) diff --git a/agent/tools/Say.py b/agent/tools/Say.py index ce939b0..b624946 100644 --- a/agent/tools/Say.py +++ b/agent/tools/Say.py @@ -9,7 +9,7 @@ from core.interact import Interact class Say(BaseTool): name = "Say" - description = """此工具用于语音输出内容，用于与主人沟通，使用时请传入说话内容作为参数，例如：“该下班了，请注意休息”""" + description = """此工具用于语音输出内容，用于与主人沟通及提醒主人，使用时请传入说话内容作为参数，例如：“该下班了，请注意休息”""" def __init__(self): super().__init__() diff --git a/agent/tools/Switch.py b/agent/tools/Switch.py index 4b2222c..4c91125 100644 --- a/agent/tools/Switch.py +++ b/agent/tools/Switch.py @@ -7,7 +7,7 @@ from langchain.tools import BaseTool class Switch(BaseTool): name = "Switch" - description = "此工具用于控制箱内制冷设备（A）、制热设备(B)、内外通风设备(C)、浇水设备(D)、补光设备(E)、二氧化碳设备(F)的开关状态" + description = "此工具用于控制箱内制冷设备（A）、制热设备(B)、内外通风设备(C)、浇水设备(D)、补光设备(E)、二氧化碳设备(F)的开关状态，参数格式:('A':'on')" def __init__(self): super().__init__() diff --git a/fay_booter.py b/fay_booter.py index edddb4b..e236e0d 100644 --- a/fay_booter.py +++ b/fay_booter.py @@ -29,7 +29,7 @@ class RecorderListener(Recorder): def on_speaking(self, text): if len(text) > 1: util.printInfo(3, "语音", '{}'.format(text), time.time()) - fay_core.send_for_answer("(语音提问)" + text) + fay_core.send_for_answer("主人语音说了：" + text) time.sleep(2) def get_stream(self): @@ -110,7 +110,7 @@ class DeviceInputListener(Recorder): if len(text) > 1: util.printInfo(3, "语音", '{}'.format(text), time.time()) - fay_core.send_for_answer("(语音提问)" + text) + fay_core.send_for_answer("主人语音说了：" + text) time.sleep(1) #recorder会等待stream不为空才开始录音 @@ -161,7 +161,7 @@ def console_listener(): util.log(1, '错误的参数！') msg = text[3:len(text)] util.printInfo(3, "控制台", '{}: {}'.format('控制台', msg)) - thr = MyThread(target=fay_core.send_for_answer, args=["(语音提问)" + msg]) + thr = MyThread(target=fay_core.send_for_answer, args=["请语音回复：" + msg]) thr.start() thr.join() diff --git a/gui/flask_server.py b/gui/flask_server.py index 369cc0e..0adf89e 100644 --- a/gui/flask_server.py +++ b/gui/flask_server.py @@ -109,7 +109,7 @@ def api_stop_live(): def api_send(): data = request.values.get('data') info = json.loads(data) - text = fay_core.send_for_answer("(文字提问)" + info['msg']) + text = fay_core.send_for_answer("主人文字说了：" + info['msg']) return '{"result":"successful","msg":"'+text+'"}' @__app.route('/api/get-msg', methods=['post']) diff --git a/images/agent_demo.gif b/images/agent_demo.gif new file mode 100644 index 0000000..bb71a7f Binary files /dev/null and b/images/agent_demo.gif differ diff --git a/main.py b/main.py index e0c44c2..e48a871 100644 --- a/main.py +++ b/main.py @@ -44,7 +44,6 @@ if __name__ == '__main__': ws_server.start_server() web_ws_server = wsa_server.new_web_instance(port=10003) web_ws_server.start_server() - #Edit by xszyou in 20230516:增加本地asr后，aliyun调成可选配置 if config_util.ASR_mode == "ali": ali_nls.start() flask_server.start()