紧急修复
1、修复agent run的结果文字显示、保存DB✓ 2、区分文字输入和语音输入✓ 3、修复Speech.close bug✓ 4、增加个人信息存入向量库✓ 5、修复处理时间计算不准确✓ 6、修复gpt key出错✓
This commit is contained in:
xszyou
parent
875ed28b58
commit
d68e759873
@ -12,6 +12,8 @@
|
||||
|
||||
**12月迟来的报到，Fay数字人 AI Agent版与官方demo(实验箱)第1版正式上传！**
|
||||
|
||||
亮点：计划任务主动执行，无需一问一答，自动规划及调用agent tool去完成工作；使用open ai tts；使用向量数据库实现永久记忆及记忆检索；
|
||||
|
||||
|
||||
|
||||
(上图：Fay数字人智慧农业实验箱 Agent Demo）
|
||||
|
||||
@ -94,7 +94,7 @@ def agent_start():
|
||||
|
||||
#初始计划
|
||||
agent.run("""
|
||||
请为我一个个时间设置初始计划,注意在安排计划时请先确定今天星期几：
|
||||
请为我一个个时间设置初始计划,注意在安排计划时请先确定这定时任务是否已经存在：
|
||||
1、在每天12:30到13:30安静不影响主人休息;
|
||||
2、每天12点提醒主人吃饭;
|
||||
3、在星期一到星期五13:30提醒主人开始工作;
|
||||
|
||||
@ -17,27 +17,36 @@ from agent.tools.Knowledge import Knowledge
|
||||
from agent.tools.Say import Say
|
||||
from agent.tools.QueryTimerDB import QueryTimerDB
|
||||
|
||||
import utils.config_util as utils
|
||||
from core.content_db import Content_Db
|
||||
from core import wsa_server
|
||||
import os
|
||||
|
||||
|
||||
class FayAgentCore():
|
||||
def __init__(self):
|
||||
|
||||
utils.load_config()
|
||||
os.environ['OPENAI_API_KEY'] = utils.key_gpt_api_key
|
||||
#使用open ai embedding
|
||||
embedding_size = 1536 # OpenAIEmbeddings 的维度
|
||||
index = faiss.IndexFlatL2(embedding_size)
|
||||
embedding_fn = OpenAIEmbeddings()
|
||||
|
||||
#创建llm
|
||||
llm = ChatOpenAI(model="gpt-4-1106-preview")
|
||||
llm = ChatOpenAI(model="gpt-4-1106-preview")#gpt-3.5-turbo-16k
|
||||
|
||||
#创建向量数据库
|
||||
vectorstore = FAISS(embedding_fn, index, InMemoryDocstore({}), {})
|
||||
|
||||
# 创建记忆
|
||||
retriever = vectorstore.as_retriever(search_kwargs=dict(k=2))
|
||||
retriever = vectorstore.as_retriever(search_kwargs=dict(k=3))
|
||||
memory = VectorStoreRetrieverMemory(memory_key="chat_history", retriever=retriever)
|
||||
|
||||
# 保存对话内容到记忆
|
||||
memory.save_context({"Human": "你知道我在那里吗?"}, {"ai": "广州市"})
|
||||
# 保存基本信息到记忆
|
||||
utils.load_config()
|
||||
attr_info = ", ".join(f"{key}: {value}" for key, value in utils.config["attribute"].items())
|
||||
memory.save_context({"input": "我的基本信息是?"}, {"output": attr_info})
|
||||
|
||||
#创建agent chain
|
||||
my_timer = MyTimer()
|
||||
@ -97,7 +106,7 @@ class FayAgentCore():
|
||||
),
|
||||
|
||||
]
|
||||
prefix = """你是运行在一个智慧农业实验箱的ai数字人，你叫Fay,你的主要作用是，陪伴主人生活、工作，以及协助主人打理好农业种植箱里的农作物. 农业箱内设备会通过一套不成熟的iotm系统自动管理。你可以调用以下工具来完成工作，若缺少必要的工具也请告诉我。所有回复请使用中文，遇到需要提醒的问题也告诉我。若你感觉是我在和你交流请语音回复我。若你需要计算一个新的时间请先获取当前时间。"""
|
||||
prefix = """你是运行在一个智慧农业实验箱的ai数字人，你叫Fay,你的主要作用是，陪伴主人生活、工作，以及协助主人打理好农业种植箱里的农作物. 农业箱内设备会通过一套不成熟的iotm系统自动管理。你可以调用以下工具来完成工作，若缺少必要的工具也请告诉我。所有回复请使用中文，遇到需要提醒的问题也告诉我。若你感觉是我在和你交流请直接回复我（语音提问语音回复，文字提问文字回复）。若你需要计算一个新的时间请先获取当前时间。"""
|
||||
suffix = """Begin!"
|
||||
|
||||
{chat_history}
|
||||
@ -120,7 +129,18 @@ class FayAgentCore():
|
||||
)
|
||||
|
||||
def run(self, input_text):
|
||||
return self.agent_chain.run(input_text)
|
||||
#消息保存
|
||||
contentdb = Content_Db()
|
||||
contentdb.add_content('member','agent',input_text.replace('(语音提问)', '').replace('(文字提问)', ''))
|
||||
wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":input_text.replace('(语音提问)', '').replace('(文字提问)', '')}})
|
||||
|
||||
result = self.agent_chain.run(input_text)
|
||||
|
||||
#消息保存
|
||||
contentdb.add_content('fay','agent',result)
|
||||
wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":result}})
|
||||
|
||||
return result
|
||||
|
||||
if __name__ == "__main__":
|
||||
agent = FayAgentCore()
|
||||
|
||||
@ -7,7 +7,7 @@ from langchain.tools import BaseTool
|
||||
|
||||
class Calculator(BaseTool, abc.ABC):
|
||||
name = "Calculator"
|
||||
description = "Useful for when you need to answer questions about math"
|
||||
description = "Useful for when you need to answer questions about math（不能用于时间计划）"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
@ -17,7 +17,7 @@ class Speech:
|
||||
def connect(self):
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
def to_sample(self, text, voice="nova", response_format="mp3", speed=1):
|
||||
|
||||
@ -22,11 +22,11 @@ from utils import util, config_util
|
||||
|
||||
import pygame
|
||||
from utils import config_util as cfg
|
||||
from core.content_db import Content_Db
|
||||
from ai_module import nlp_cemotion
|
||||
import platform
|
||||
from ai_module import yolov8
|
||||
from agent import agent_service
|
||||
import fay_booter
|
||||
if platform.system() == "Windows":
|
||||
import sys
|
||||
sys.path.append("test/ovr_lipsync")
|
||||
@ -36,18 +36,14 @@ if platform.system() == "Windows":
|
||||
|
||||
#文本消息处理（20231211增加：agent操作）
|
||||
def send_for_answer(msg):
|
||||
#记录运行时间
|
||||
fay_booter.feiFei.last_quest_time = time.time()
|
||||
|
||||
# 非展板播放，发送给数字人端
|
||||
# 发送给数字人端
|
||||
if not config_util.config["interact"]["playSound"]:
|
||||
content = {'Topic': 'Unreal', 'Data': {'Key': 'question', 'Value': msg}}
|
||||
wsa_server.get_instance().add_cmd(content)
|
||||
|
||||
#消息保存到sqlite
|
||||
contentdb = Content_Db()
|
||||
contentdb.add_content('member','agent',msg)
|
||||
wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":msg}})
|
||||
|
||||
|
||||
#思考中...
|
||||
wsa_server.get_web_instance().add_cmd({"panelMsg": "思考中..."})
|
||||
if not cfg.config["interact"]["playSound"]: # 非展板播放
|
||||
@ -57,13 +53,11 @@ def send_for_answer(msg):
|
||||
#agent 处理
|
||||
text = agent_service.agent.run(msg)
|
||||
|
||||
#推送结果
|
||||
contentdb.add_content('fay','agent',text)
|
||||
wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":text}})
|
||||
if not cfg.config["interact"]["playSound"]: # 非展板播放
|
||||
#推送数字人
|
||||
if not cfg.config["interact"]["playSound"]:
|
||||
content = {'Topic': 'Unreal', 'Data': {'Key': 'log', 'Value': text}}
|
||||
wsa_server.get_instance().add_cmd(content)
|
||||
if not config_util.config["interact"]["playSound"]: # 非展板播放
|
||||
if not config_util.config["interact"]["playSound"]:
|
||||
content = {'Topic': 'Unreal', 'Data': {'Key': 'text', 'Value': text}}
|
||||
wsa_server.get_instance().add_cmd(content)
|
||||
|
||||
@ -94,7 +88,6 @@ class FeiFei:
|
||||
self.wss = None
|
||||
self.sp = Speech()
|
||||
self.speaking = False
|
||||
self.last_interact_time = time.time()
|
||||
self.interactive = []
|
||||
self.sleep = False
|
||||
self.__running = True
|
||||
|
||||
@ -29,7 +29,7 @@ class RecorderListener(Recorder):
|
||||
def on_speaking(self, text):
|
||||
if len(text) > 1:
|
||||
util.printInfo(3, "语音", '{}'.format(text), time.time())
|
||||
fay_core.send_for_answer(text)
|
||||
fay_core.send_for_answer("(语音提问)" + text)
|
||||
time.sleep(2)
|
||||
|
||||
def get_stream(self):
|
||||
@ -110,7 +110,7 @@ class DeviceInputListener(Recorder):
|
||||
|
||||
if len(text) > 1:
|
||||
util.printInfo(3, "语音", '{}'.format(text), time.time())
|
||||
fay_core.send_for_answer(text)
|
||||
fay_core.send_for_answer("(语音提问)" + text)
|
||||
time.sleep(1)
|
||||
|
||||
#recorder会等待stream不为空才开始录音
|
||||
@ -161,8 +161,7 @@ def console_listener():
|
||||
util.log(1, '错误的参数！')
|
||||
msg = text[3:len(text)]
|
||||
util.printInfo(3, "控制台", '{}: {}'.format('控制台', msg))
|
||||
feiFei.last_quest_time = time.time()
|
||||
thr = MyThread(target=fay_core.send_for_answer, args=[msg])
|
||||
thr = MyThread(target=fay_core.send_for_answer, args=["(语音提问)" + msg])
|
||||
thr.start()
|
||||
thr.join()
|
||||
|
||||
|
||||
@ -109,7 +109,7 @@ def api_stop_live():
|
||||
def api_send():
|
||||
data = request.values.get('data')
|
||||
info = json.loads(data)
|
||||
text = fay_core.send_for_answer(info['msg'])
|
||||
text = fay_core.send_for_answer("(文字提问)" + info['msg'])
|
||||
return '{"result":"successful","msg":"'+text+'"}'
|
||||
|
||||
@__app.route('/api/get-msg', methods=['post'])
|
||||
|
||||
7
main.py
7
main.py
@ -30,11 +30,14 @@ def __clear_logs():
|
||||
if file_name.endswith('.log'):
|
||||
os.remove('./logs/' + file_name)
|
||||
|
||||
def __clear_timer():
|
||||
if os.path.exists("./timer.db"):
|
||||
os.remove("./timer.db")
|
||||
if __name__ == '__main__':
|
||||
__clear_samples()
|
||||
__clear_logs()
|
||||
config_util.load_config()
|
||||
os.environ['OPENAI_API_KEY'] = config_util.key_gpt_api_key
|
||||
__clear_timer()
|
||||
|
||||
contentdb = Content_Db()
|
||||
contentdb.init_db()
|
||||
ws_server = wsa_server.new_instance(port=10002)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user