小橄榄界面优化

1、去除funasr子协议限制；
2、默认声音替换；
3、修复在服务器上聊天记录时间显示问题；
4、修复前端font缺失问题；
5、前端css调整为本地加载；
6、修复因线程问题，导致启动（关闭）失败、tts不合成等问题；
7、用户切换栏新增左右滑动；
8、修复bs4包安装遗漏问题；
9、修复聊天记录不自动换行问题；
10、修复favicon.ico 缺失问题；

asr/ali_nls.py

@@ -58,6 +58,7 @@ class ALiNls:
         self.__URL = 'wss://nls-gateway-cn-shenzhen.aliyuncs.com/ws/v1'
         self.__ws = None
         self.__frames = []
+        self.started = False
         self.__closing = False
         self.__task_id = ''
         self.done = False
@@ -86,6 +87,8 @@ class ALiNls:
             data = json.loads(message)
             header = data['header']
             name = header['name']
+            if name == 'TranscriptionStarted':
+                self.started = True
             if name == 'SentenceEnd':
                 self.done = True
                 self.finalResults = data['payload']['result']

asr/funasr.py

@@ -27,6 +27,7 @@ class FunASR:
         self.__reconnect_delay = 1
         self.__reconnecting = False
         self.username = username
+        self.started = True
 
     
     # 收到websocket消息的处理
@@ -101,7 +102,7 @@ class FunASR:
         self.done = False
         self.__frames.clear()
         websocket.enableTrace(False)
-        self.__ws = websocket.WebSocketApp(self.__URL, on_message=self.on_message,on_close=self.on_close,on_error=self.on_error,subprotocols=["binary"])
+        self.__ws = websocket.WebSocketApp(self.__URL, on_message=self.on_message,on_close=self.on_close,on_error=self.on_error)
         self.__ws.on_open = self.on_open
 
         self.__ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE})

asr/funasr/ASR_server.py

@@ -2,11 +2,11 @@ import asyncio
 import websockets
 import argparse
 import json
+import logging
 from funasr import AutoModel
 import os
 
 # 设置日志级别
-import logging
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.CRITICAL)
 
@@ -57,7 +57,7 @@ async def worker():
         task_queue.task_done()
 
 async def process_wav_file(websocket, url):
-    #热词
+    # 热词
     param_dict = {"sentence_timestamp": False}
     with open("data/hotword.txt", "r", encoding="utf-8") as f:
         lines = f.readlines()
@@ -67,7 +67,7 @@ async def process_wav_file(websocket, url):
     param_dict["hotword"] = hotword
     wav_path = url
     try:
-        res = asr_model.generate(input=wav_path,is_final=True, **param_dict)
+        res = asr_model.generate(input=wav_path, is_final=True, **param_dict)
         if res:
             if 'text' in res[0] and websocket.open:
                 await websocket.send(res[0]['text'])
@@ -78,10 +78,11 @@ async def process_wav_file(websocket, url):
             os.remove(wav_path)
 
 async def main():
-    start_server = websockets.serve(ws_serve, args.host, args.port, subprotocols=["binary"], ping_interval=10)
+    start_server = websockets.serve(ws_serve, args.host, args.port, ping_interval=10)
     await start_server
     worker_task = asyncio.create_task(worker())
     await worker_task
 
-# 使用 asyncio 运行主函数
-asyncio.run(main())
+# 使用 asyncio 获取事件循环并运行主函数
+loop = asyncio.get_event_loop()
+loop.run_until_complete(main())

config.json

@@ -1,44 +1,42 @@
 {
     "attribute": {
-        "age": "\u6210\u5e74",
-        "birth": "Github",
-        "constellation": "\u6c34\u74f6\u5ea7",
-        "contact": "qq467665317",
+        "age": "\u5e7c\u5e74",
+        "birth": "\u5195\u5b81\u5143\u5347\u519c\u4e1a",
+        "constellation": "\u6a44\u6984\u661f",
+        "contact": "\u5929\u7801\u56e2\u961f",
         "gender": "\u5973",
         "hobby": "\u53d1\u5446",
-        "job": "\u52a9\u7406",
-        "name": "\u83f2\u83f2",
-        "voice": "\u6653\u6653(azure)",
-        "zodiac": "\u86c7"
+        "job": "\u5c0f\u52a9\u624b",
+        "name": "\u5c0f\u6a44\u6984",
+        "voice": "\u6653\u6653(edge)",
+        "zodiac": "\u6a44\u6984"
     },
     "interact": {
-        "QnA": "",
+        "QnA": "qa.csv",
         "maxInteractTime": 15,
         "perception": {
-            "chat": 10,
-            "follow": 10,
-            "gift": 10,
-            "indifferent": 10,
-            "join": 10
+            "chat": 20,
+            "follow": 20,
+            "gift": 20,
+            "indifferent": 20,
+            "join": 20
         },
         "playSound": true,
-        "sound_synthesis_enabled": false,
         "visualization": false
     },
     "items": [],
     "source": {
-        "automatic_player_status": false,
+        "automatic_player_status": true,
         "automatic_player_url": "http://127.0.0.1:6000",
         "liveRoom": {
             "enabled": true,
             "url": ""
         },
         "record": {
-            "channels": 0,
             "device": "",
             "enabled": true
         },
-        "wake_word": "\u4f60\u597d",
+        "wake_word": "\u5c0f\u6a44\u6984",
         "wake_word_enabled": true,
         "wake_word_type": "front"
     }

core/content_db.py

@@ -3,12 +3,13 @@ import time
 import threading
 import functools
 from utils import util
+
 def synchronized(func):
-  @functools.wraps(func)
-  def wrapper(self, *args, **kwargs):
-    with self.lock:
-      return func(self, *args, **kwargs)
-  return wrapper
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        with self.lock:
+            return func(self, *args, **kwargs)
+    return wrapper
 
 __content_tb = None
 def new_instance():
@@ -18,73 +19,122 @@ def new_instance():
         __content_tb.init_db()
     return __content_tb
 
-
 class Content_Db:
 
     def __init__(self) -> None:
         self.lock = threading.Lock()
-           
-   
 
-    #初始化
+    # 初始化数据库
     def init_db(self):
         conn = sqlite3.connect('fay.db')
         c = conn.cursor()
         c.execute('''CREATE TABLE IF NOT EXISTS T_Msg
-            (id INTEGER PRIMARY KEY     autoincrement,
-            type        char(10),
-            way        char(10),
-            content           TEXT    NOT NULL,
-            createtime         Int,
-            username TEXT DEFAULT 'User',
-            uid         Int);''')
+            (id INTEGER PRIMARY KEY AUTOINCREMENT,
+            type        CHAR(10),
+            way         CHAR(10),
+            content     TEXT    NOT NULL,
+            createtime  INT,
+            username    TEXT DEFAULT 'User',
+            uid         INT);''')
+        # 对话采纳记录表
+        c.execute('''CREATE TABLE IF NOT EXISTS T_Adopted
+            (id INTEGER PRIMARY KEY AUTOINCREMENT,
+            msg_id      INTEGER UNIQUE,
+            adopted_time INT,
+            FOREIGN KEY(msg_id) REFERENCES T_Msg(id));''')
         conn.commit()
         conn.close()
-       
- 
 
-
-    #添加对话
+    # 添加对话
     @synchronized
-    def add_content(self,type,way,content,username='User',uid=0):
+    def add_content(self, type, way, content, username='User', uid=0):
         conn = sqlite3.connect("fay.db")
         cur = conn.cursor()
         try:
-            cur.execute("insert into T_Msg (type,way,content,createtime,username,uid) values (?,?,?,?,?,?)",(type, way, content, time.time(), username, uid))
+            cur.execute("INSERT INTO T_Msg (type, way, content, createtime, username, uid) VALUES (?, ?, ?, ?, ?, ?)",
+                        (type, way, content, int(time.time()), username, uid))
             conn.commit()
-        except:
-               util.log(1, "请检查参数是否有误")
-               conn.close()
-               return 0
+            last_id = cur.lastrowid
+        except Exception as e:
+            util.log(1, "请检查参数是否有误: {}".format(e))
+            conn.close()
+            return 0
         conn.close()
-        return cur.lastrowid
-     
-        
+        return last_id
 
-    #获取对话内容
+    # 根据ID查询对话记录
     @synchronized
-    def get_list(self,way,order,limit,uid=0):
+    def get_content_by_id(self, msg_id):
+        conn = sqlite3.connect("fay.db")
+        cur = conn.cursor()
+        cur.execute("SELECT * FROM T_Msg WHERE id = ?", (msg_id,))
+        record = cur.fetchone()
+        conn.close()
+        return record
+
+    # 添加对话采纳记录
+    @synchronized
+    def adopted_message(self, msg_id):
+        conn = sqlite3.connect('fay.db')
+        cur = conn.cursor()
+        # 检查消息ID是否存在
+        cur.execute("SELECT 1 FROM T_Msg WHERE id = ?", (msg_id,))
+        if cur.fetchone() is None:
+            util.log(1, "消息ID不存在")
+            conn.close()
+            return False
+        try:
+            cur.execute("INSERT INTO T_Adopted (msg_id, adopted_time) VALUES (?, ?)", (msg_id, int(time.time())))
+            conn.commit()
+        except sqlite3.IntegrityError:
+            util.log(1, "该消息已被采纳")
+            conn.close()
+            return False
+        conn.close()
+        return True
+
+    # 获取对话内容
+    @synchronized
+    def get_list(self, way, order, limit, uid=0):
         conn = sqlite3.connect("fay.db")
         cur = conn.cursor()
         where_uid = ""
         if int(uid) != 0:
-            where_uid = f" and uid = {uid} "
-        if(way == 'all'):
-            cur.execute("select type,way,content,createtime,datetime(createtime, 'unixepoch', 'localtime') as timetext,username from T_Msg  where 1 "+where_uid+" order by id "+order+" limit ?",(limit,))
-        elif(way == 'notappended'):
-            cur.execute("select type,way,content,createtime,datetime(createtime, 'unixepoch', 'localtime') as timetext,username from T_Msg where way != 'appended' "+where_uid+" order by id "+order+" limit ?",(limit,))
+            where_uid = f" AND T_Msg.uid = {uid} "
+        base_query = f"""
+            SELECT T_Msg.type, T_Msg.way, T_Msg.content, T_Msg.createtime,
+                   datetime(T_Msg.createtime, 'unixepoch', 'localtime') AS timetext,
+                   T_Msg.username,T_Msg.id,
+                   CASE WHEN T_Adopted.msg_id IS NOT NULL THEN 1 ELSE 0 END AS is_adopted
+            FROM T_Msg
+            LEFT JOIN T_Adopted ON T_Msg.id = T_Adopted.msg_id
+            WHERE 1 {where_uid}
+        """
+        if way == 'all':
+            query = base_query + f" ORDER BY T_Msg.id {order} LIMIT ?"
+            cur.execute(query, (limit,))
+        elif way == 'notappended':
+            query = base_query + f" AND T_Msg.way != 'appended' ORDER BY T_Msg.id {order} LIMIT ?"
+            cur.execute(query, (limit,))
         else:
-            cur.execute("select type,way,content,createtime,datetime(createtime, 'unixepoch', 'localtime') as timetext,username from T_Msg where way = ? "+where_uid+" order by id "+order+" limit ?",(way,limit,))
-
+            query = base_query + f" AND T_Msg.way = ? ORDER BY T_Msg.id {order} LIMIT ?"
+            cur.execute(query, (way, limit))
         list = cur.fetchall()
         conn.close()
         return list
+    
 
-
-
-
-
-   
-
-
-
+    @synchronized
+    def get_previous_user_message(self, msg_id):
+        conn = sqlite3.connect("fay.db")
+        cur = conn.cursor()
+        cur.execute("""
+            SELECT id, type, way, content, createtime, datetime(createtime, 'unixepoch', 'localtime') AS timetext, username
+            FROM T_Msg
+            WHERE id < ? AND type != 'fay'
+            ORDER BY id DESC
+            LIMIT 1
+        """, (msg_id,))
+        record = cur.fetchone()
+        conn.close()
+        return record

core/fay_core.py

@@ -29,6 +29,7 @@ from llm import nlp_xingchen
 from llm import nlp_langchain
 from llm import nlp_ollama_api
 from llm import nlp_coze
+from llm.agent import fay_agent
 from core import member_db
 import threading
 import functools
@@ -60,7 +61,8 @@ modules = {
     "nlp_xingchen": nlp_xingchen,
     "nlp_langchain": nlp_langchain,
     "nlp_ollama_api": nlp_ollama_api,
-    "nlp_coze": nlp_coze
+    "nlp_coze": nlp_coze,
+    "nlp_agent": fay_agent
 
 }
 
@@ -145,9 +147,9 @@ class FeiFei:
                     uid = member_db.new_instance().find_user(username)
 
                     #记录用户问题
-                    content_db.new_instance().add_content('member','speak',interact.data["msg"], username, uid)
+                    content_id = content_db.new_instance().add_content('member','speak',interact.data["msg"], username, uid)
                     if wsa_server.get_web_instance().is_connected(username):
-                        wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":interact.data["msg"], "username":username, "uid":uid}, "Username" : username})
+                        wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"member","content":interact.data["msg"], "username":username, "uid":uid, "id":content_id}, "Username" : username})
                     
                     #确定是否命中q&a
                     answer = self.__get_answer(interact.interleaver, interact.data["msg"])
@@ -163,18 +165,17 @@ class FeiFei:
                             wsa_server.get_instance().add_cmd(content)
                         text,textlist = handle_chat_message(interact.data["msg"], username, interact.data.get("observation", ""))
 
-                        # qa_service.QAService().record_qapair(interact.data["msg"], text)#沟通记录缓存到qa文件
                     else: 
                         text = answer
 
                     #记录回复    
                     self.write_to_file("./logs", "answer_result.txt", text)
-                    content_db.new_instance().add_content('fay','speak',text, username, uid)
+                    content_id = content_db.new_instance().add_content('fay','speak',text, username, uid)
 
                     #文字输出：面板、聊天窗、log、数字人
                     if wsa_server.get_web_instance().is_connected(username):
                         wsa_server.get_web_instance().add_cmd({"panelMsg": text, "Username" : username, 'robot': f'http://{cfg.fay_url}:5000/robot/Speaking.jpg'})
-                        wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":text, "username":username, "uid":uid}, "Username" : username})
+                        wsa_server.get_web_instance().add_cmd({"panelReply": {"type":"fay","content":text, "username":username, "uid":uid, "id":content_id}, "Username" : username})
                     if len(textlist) > 1:
                         i = 1
                         while i < len(textlist):
@@ -198,8 +199,6 @@ class FeiFei:
                     if member_db.new_instance().is_username_exist(username)  == "notexists":
                         member_db.new_instance().add_user(username)
                     uid = member_db.new_instance().find_user(username)
-                    
-                    #TODO 这里可以通过qa来触发指定的脚本操作，如ppt翻页等
 
                     if interact.data.get("text"):
                         #记录回复
@@ -217,7 +216,8 @@ class FeiFei:
                             wsa_server.get_instance().add_cmd(content)
                     
                     #声音输出
-                    MyThread(target=self.say, args=[interact, text]).start()
+                    MyThread(target=self.say, args=[interact, text]).start()  
+                
    
             except BaseException as e:
                 print(e)
@@ -315,16 +315,15 @@ class FeiFei:
     def say(self, interact, text):
         try:
             result = None
-            if config_util.config["interact"]["sound_synthesis_enabled"]:
-                audio_url = interact.data.get('audio')#透传的音频
-                if audio_url is not None:
-                    file_name = 'sample-' + str(int(time.time() * 1000)) + '.wav'
-                    result = self.download_wav(audio_url, './samples/', file_name)
-                elif config_util.config["interact"]["playSound"] or wsa_server.get_instance().is_connected(interact.data.get("user")) or self.__is_send_remote_device_audio(interact):#tts
-                    util.printInfo(1,  interact.data.get('user'), '合成音频...')
-                    tm = time.time()
-                    result = self.sp.to_sample(text.replace("*", ""), self.__get_mood_voice())
-                    util.printInfo(1,  interact.data.get('user'), '合成音频完成. 耗时: {} ms 文件:{}'.format(math.floor((time.time() - tm) * 1000), result))
+            audio_url = interact.data.get('audio')#透传的音频
+            if audio_url is not None:
+                file_name = 'sample-' + str(int(time.time() * 1000)) + '.wav'
+                result = self.download_wav(audio_url, './samples/', file_name)
+            elif config_util.config["interact"]["playSound"] or wsa_server.get_instance().is_connected(interact.data.get("user")) or self.__is_send_remote_device_audio(interact):#tts
+                util.printInfo(1,  interact.data.get('user'), '合成音频...')
+                tm = time.time()
+                result = self.sp.to_sample(text.replace("*", ""), self.__get_mood_voice())
+                util.printInfo(1,  interact.data.get('user'), '合成音频完成. 耗时: {} ms 文件:{}'.format(math.floor((time.time() - tm) * 1000), result))
 
             if result is not None:            
                 MyThread(target=self.__process_output_audio, args=[result, interact, text]).start()

core/member_db.py

@@ -83,6 +83,7 @@ class Member_Db:
         else:
             return "notexists"
 
+    #根据username查询uid
     def find_user(self, username):
         conn = sqlite3.connect('user_profiles.db')
         c = conn.cursor()
@@ -93,6 +94,20 @@ class Member_Db:
             return 0
         else:
            return result[0]
+        
+    #根据uid查询username
+    def find_username_by_uid(self, uid):
+        conn = sqlite3.connect('user_profiles.db')
+        c = conn.cursor()
+        c.execute('SELECT username FROM T_Member WHERE id = ?', (uid,))
+        result = c.fetchone()
+        conn.close()
+        if result is None:
+            return 0
+        else:
+           return result[0]
+
+
 
     @synchronized
     def query(self, sql):

core/qa_service.py

@@ -33,7 +33,7 @@ class QAService:
 
     def question(self, query_type, text):
         if query_type == 'qa':
-            answer_dict = self.__read_qna(cfg.config['interact']['QnA'])
+            answer_dict = self.__read_qna(cfg.config['interact'].get('QnA'))
             answer, action = self.__get_keyword(answer_dict, text, query_type)
             if action:
                 MyThread(target=self.__run, args=[action]).start()
@@ -61,7 +61,7 @@ class QAService:
                     if len(row) >= 2:
                         qna.append([row[0].split(";"), row[1], row[2] if len(row) >= 3 else None])
         except Exception as e:
-            util.log(1, 'qa文件没有指定，不匹配qa')
+            pass
         return qna
 
     def record_qapair(self, question, answer):

core/recorder.py

@@ -46,6 +46,8 @@ class Recorder:
         self.username = 'User' #默认用户，子类实现时会重写
         self.channels = 1
         self.sample_rate = 16000
+        self.is_reading = False
+        self.stream = None
 
     def asrclient(self):
         if self.ASRMode == "ali":
@@ -201,6 +203,11 @@ class Recorder:
         audio_data_list = []
         while self.__running:
             try:
+                cfg.load_config()
+                record = cfg.config['source']['record']
+                if not record['enabled'] and not self.is_remote:
+                    time.sleep(1)
+                    continue
                 self.is_reading = True
                 data = stream.read(1024, exception_on_overflow=False)
                 self.is_reading = False
@@ -211,7 +218,6 @@ class Recorder:
                 self.__running = False
             if not data:
                 continue 
-
             #是否可以拾音,不可以就掉弃录音
             can_listen = True
             #没有开唤醒，但面板或数字人正在播音时不能拾音
@@ -225,7 +231,7 @@ class Recorder:
             if can_listen == False:#掉弃录音
                 data = None
                 continue
-            
+
             #计算音量是否满足激活拾音
             level = audioop.rms(data, 2)
             if len(self.__history_data) >= 10:#保存激活前的音频，以免信息掉失
@@ -241,9 +247,11 @@ class Recorder:
             elif history_percentage < self.__dynamic_threshold:
                 self.__dynamic_threshold += (history_percentage - self.__dynamic_threshold) * 1
             
+           
             #激活拾音
             if percentage > self.__dynamic_threshold:
                 last_speaking_time = time.time()
+
                 if not self.__processing and not isSpeaking and time.time() - last_mute_time > _ATTACK:
                     isSpeaking = True  #用户正在说话
                     util.printInfo(1, self.username,"聆听中...")
@@ -255,7 +263,9 @@ class Recorder:
                     concatenated_audio.clear()
                     self.__aLiNls = self.asrclient()
                     try:
-                        self.__aLiNls.start()
+                        task_id = self.__aLiNls.start()
+                        while not self.__aLiNls.started:
+                            time.sleep(0.01)
                     except Exception as e:
                         print(e)
                         util.printInfo(1, self.username, "aliyun asr 连接受限")

core/socket_bridge_service.py

@@ -0,0 +1,186 @@
+import asyncio
+import websockets
+import socket
+import threading
+import time
+import sys
+
+__wss = None
+
+def new_instance():
+    global __wss
+    if __wss is None:
+        __wss = SocketBridgeService()
+    return __wss
+
+class SocketBridgeService:
+    def __init__(self):
+        self.websockets = {}  
+        self.sockets = {}     
+        self.message_queue = asyncio.Queue()  
+        self.running = True
+        self.loop = None
+        self.tasks = set()
+        self.server = None
+
+    async def handler(self, websocket, path):
+        ws_id = id(websocket)
+        self.websockets[ws_id] = websocket
+        try:
+            if ws_id not in self.sockets:
+                sock = await self.create_socket_client()
+                if sock:
+                    self.sockets[ws_id] = sock
+                else:
+                    print(f"Failed to connect TCP socket for WebSocket {ws_id}")
+                    await websocket.close()
+                    return
+            receive_task = asyncio.create_task(self.receive_from_socket(ws_id))
+            self.tasks.add(receive_task)
+            receive_task.add_done_callback(self.tasks.discard)
+            async for message in websocket:
+                await self.send_to_socket(ws_id, message)
+        except websockets.ConnectionClosed:
+            pass
+        except Exception as e:
+            pass
+        finally:
+            self.close_socket_client(ws_id)
+            self.websockets.pop(ws_id, None)
+
+    async def create_socket_client(self):
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            sock.connect(('127.0.0.1', 10001))
+            sock.setblocking(True)  # 设置为阻塞模式
+            return sock
+        except Exception as e:
+            return None
+
+    async def send_to_socket(self, ws_id, message):
+        sock = self.sockets.get(ws_id)
+        if sock:
+            try:
+                await asyncio.to_thread(sock.sendall, message)
+            except Exception as e:
+                self.close_socket_client(ws_id)
+
+    async def receive_from_socket(self, ws_id):
+        sock = self.sockets.get(ws_id)
+        if not sock:
+            return
+        try:
+            while self.running:
+                data = await asyncio.to_thread(sock.recv, 4096)
+                if data:
+                    await self.message_queue.put((ws_id, data))
+                else:
+                    break
+        except Exception as e:
+            pass
+        finally:
+            self.close_socket_client(ws_id)
+
+    async def process_message_queue(self):
+        while self.running or not self.message_queue.empty():
+            try:
+                ws_id, data = await asyncio.wait_for(self.message_queue.get(), timeout=1.0)
+                websocket = self.websockets.get(ws_id)
+                if websocket and websocket.open:
+                    try:
+                        await websocket.send(data)
+                    except Exception as e:
+                        pass
+                self.message_queue.task_done()
+            except asyncio.TimeoutError:
+                continue
+            except Exception as e:
+                pass
+
+    def close_socket_client(self, ws_id):
+        sock = self.sockets.pop(ws_id, None)
+        if sock:
+            try:
+                sock.shutdown(socket.SHUT_RDWR)
+            except Exception as e:
+                pass
+                # print(f"Error shutting down socket for WebSocket {ws_id}: {e}", file=sys.stderr)
+            sock.close()
+
+    async def start(self, host='0.0.0.0', port=9001):
+        self.server = await websockets.serve(self.handler, host, port)
+        process_task = asyncio.create_task(self.process_message_queue())
+        self.tasks.add(process_task)
+        process_task.add_done_callback(self.tasks.discard)
+        try:
+            await self.server.wait_closed()
+        except asyncio.CancelledError:
+            pass
+        finally:
+            await self.shutdown()
+
+    async def shutdown(self):
+        if not self.running:
+            return
+        self.running = False
+
+        for ws_id, ws in list(self.websockets.items()):
+            try:
+                await ws.close()
+            except Exception as e:
+                pass
+                # print(f"Error closing WebSocket {ws_id}: {e}", file=sys.stderr)
+        self.websockets.clear()
+
+        for ws_id, sock in list(self.sockets.items()):
+            try:
+                sock.shutdown(socket.SHUT_RDWR)
+            except Exception as e:
+                pass
+                # print(f"Error shutting down socket for WebSocket {ws_id}: {e}", file=sys.stderr)
+            sock.close()
+        self.sockets.clear()
+
+        await self.message_queue.join()
+
+        for task in self.tasks:
+            task.cancel()
+        await asyncio.gather(*self.tasks, return_exceptions=True)
+        self.tasks.clear()
+
+        if self.server:
+            self.server.close()
+            await self.server.wait_closed()
+
+
+    def start_service(self):
+        self.loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(self.loop)
+        try:
+            self.loop.run_until_complete(self.start(host='0.0.0.0', port=9001))
+        except Exception as e:
+            pass
+            # print(f"Service exception: {e}", file=sys.stderr)
+        finally:
+            self.loop.close()
+
+if __name__ == '__main__':
+    service = new_instance()
+    service_thread = threading.Thread(target=service.start_service, daemon=True)
+    service_thread.start()
+
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        # 在服务的事件循环中运行 shutdown 协程
+        print("Initiating shutdown...")
+        if service.loop and service.loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(service.shutdown(), service.loop)
+            try:
+                future.result()  # 等待关闭完成
+                print("Shutdown coroutine completed.")
+            except Exception as e:
+                print(f"Shutdown exception: {e}", file=sys.stderr)
+        service_thread.join()
+        print("Service has been shut down.")

core/wsa_server.py

@@ -31,21 +31,36 @@ class MyServer:
             async for message in websocket:
                 await asyncio.sleep(0.01)
                 try:
-                    username = json.loads(message).get("Username")
-                except json.JSONDecodeError as e:#json格式有误，不处理
-                    pass
-                if username:
+                    data = json.loads(message)
+                    username = data.get("Username")
+                    output_setting = data.get("Output")
+                except json.JSONDecodeError:
+                    pass  # Ignore invalid JSON messages
+                if username or output_setting:
                     remote_address = websocket.remote_address
                     unique_id = f"{remote_address[0]}:{remote_address[1]}"
                     async with self.lock:
-                        for i in range(len(self.__clients)): 
+                        for i in range(len(self.__clients)):
                             if self.__clients[i]["id"] == unique_id:
-                                self.__clients[i]["username"] = username
+                                if username:
+                                    self.__clients[i]["username"] = username
+                                if output_setting:
+                                    self.__clients[i]["output"] = output_setting   
                 await self.__consumer(message)
         except websockets.exceptions.ConnectionClosedError as e:
             # 从客户端列表中移除已断开的连接
             await self.remove_client(websocket)
-            util.printInfo(1, "User" if  username is None else username, f"WebSocket 连接关闭: {e}")
+            util.printInfo(1, "User" if username is None else username, f"WebSocket 连接关闭: {e}")
+
+    def get_client_output(self, username):
+        clients_with_username = [c for c in self.__clients if c.get("username") == username]
+        if not clients_with_username:
+            return False
+        for client in clients_with_username:
+            output = client.get("output", 1)
+            if output != 0 and output != '0':
+                return True 
+        return False
 
     # 发送处理        
     async def __producer_handler(self, websocket, path):
@@ -205,7 +220,7 @@ class WebServer(MyServer):
         pass
     
     def on_connect_handler(self):
-        self.add_cmd({"panelMsg": "使用提示：Fay可以独立使用，启动数字人将自动对接。"})
+        self.add_cmd({"panelMsg": "和我说“小橄榄”+你的问题来和我互动哦"})
 
     def on_send_handler(self, message):
         return message

fay_booter.py

@@ -3,8 +3,8 @@ import time
 import re
 import pyaudio
 import socket
-import psutil
 import sys
+import asyncio
 import requests
 from core.interact import Interact
 from core.recorder import Recorder
@@ -12,8 +12,9 @@ from core import fay_core
 from scheduler.thread_manager import MyThread
 from utils import util, config_util, stream_util
 from core.wsa_server import MyServer
-from scheduler.thread_manager import MyThread
 from core import wsa_server
+from core import socket_bridge_service
+from llm.agent import agent_service
 
 feiFei: fay_core.FeiFei = None
 recorderListener: Recorder = None
@@ -21,6 +22,8 @@ __running = False
 deviceSocketServer = None
 DeviceInputListenerDict = {}
 ngrok = None
+socket_service_instance = None
+
 
 #启动状态
 def is_running():
@@ -48,6 +51,13 @@ class RecorderListener(Recorder):
 
     def get_stream(self):
         try:
+            #是否录音的控制是放在recorder.py的，这里的作用是防止没有麦克风的设备出错
+            while True:
+                record = config_util.config['source']['record']
+                if record['enabled']:
+                    break
+                time.sleep(0.1)
+
             self.paudio = pyaudio.PyAudio()
             device_id = 0  # 或者根据需要选择其他设备
 
@@ -87,9 +97,10 @@ class RecorderListener(Recorder):
         try:
             while self.is_reading:
                 time.sleep(0.1)
-            self.stream.stop_stream()
-            self.stream.close()
-            self.paudio.terminate()
+            if self.stream is not None:
+                self.stream.stop_stream()
+                self.stream.close()
+                self.paudio.terminate()
         except Exception as e:
                 print(e)
                 util.log(1, "请检查设备是否有误，再重新启动!")
@@ -177,7 +188,7 @@ def device_socket_keep_alive():
                 if wsa_server.get_web_instance().is_connected(value.username):
                     wsa_server.get_web_instance().add_cmd({"remote_audio_connect": True, "Username" : value.username}) 
             except Exception as serr:
-                util.printInfo(3, value.username, "远程音频输入输出设备已经断开：{}".format(key))
+                util.printInfo(1, value.username, "远程音频输入输出设备已经断开：{}".format(key))
                 value.stop()
                 delkey = key
                 break
@@ -211,17 +222,10 @@ def accept_audio_device_output_connect():
         except Exception as e:
             pass
 
-def kill_process_by_port(port):
-    for proc in psutil.process_iter(['pid', 'name','cmdline']):
-        try:
-            for conn in proc.connections(kind='inet'):
-                if conn.laddr.port == port:
-                    proc.terminate()
-                    proc.wait()
-        except(psutil.NosuchProcess, psutil.AccessDenied):
-            pass
 #数字人端请求获取最新的自动播放消息，若自动播放服务关闭会自动退出自动播放
-def start_auto_play_service():
+def start_auto_play_service(): #TODO 评估一下有无优化的空间
+    if config_util.config['source'].get('automatic_player_url') is None or config_util.config['source'].get('automatic_player_status') is None:
+        return
     url = f"{config_util.config['source']['automatic_player_url']}/get_auto_play_item"
     user = "User" #TODO 临时固死了
     is_auto_server_error = False
@@ -260,55 +264,7 @@ def start_auto_play_service():
                     util.printInfo(1, user, '请求自动播放服务器失败，错误信息是：{}'.format(e))
         time.sleep(0.01)
      
-#控制台输入监听
-def console_listener():
-    global feiFei
-    while __running:
-        try:
-            text = input()
-        except EOFError:
-            util.log(1, "控制台已经关闭")
-            break
-        
-        args = text.split(' ')
 
-        if len(args) == 0 or len(args[0]) == 0:
-            continue
-
-        if args[0] == 'help':
-            util.log(1, 'in <msg> \t通过控制台交互')
-            util.log(1, 'restart \t重启服务')
-            util.log(1, 'stop \t\t关闭服务')
-            util.log(1, 'exit \t\t结束程序')
-
-        elif args[0] == 'stop':
-            stop()
-            break
-
-        elif args[0] == 'restart':
-            stop()
-            time.sleep(0.1)
-            start()
-
-        elif args[0] == 'in':
-            if len(args) == 1:
-                util.log(1, '错误的参数！')
-            msg = text[3:len(text)]
-            util.printInfo(3, "控制台", '{}: {}'.format('控制台', msg))
-            interact = Interact("console", 1, {'user': 'User', 'msg': msg})
-            thr = MyThread(target=feiFei.on_interact, args=[interact])
-            thr.start()
-
-        elif args[0]=='exit':
-            stop()
-            time.sleep(0.1)
-            util.log(1,'程序正在退出..')
-            ports =[10001,10002,10003,5000]
-            for port in ports:
-                kill_process_by_port(port)
-            sys.exit(0)
-        else:
-            util.log(1, '未知命令！使用 \'help\' 获取帮助.')
 
 #停止服务
 def stop():
@@ -317,6 +273,8 @@ def stop():
     global __running
     global DeviceInputListenerDict
     global ngrok
+    global socket_service_instance
+    global deviceSocketServer
 
     util.log(1, '正在关闭服务...')
     __running = False
@@ -325,11 +283,22 @@ def stop():
         recorderListener.stop()
         time.sleep(0.1)
     util.log(1, '正在关闭远程音频输入输出服务...')
-    if len(DeviceInputListenerDict) > 0:
-        for key in list(DeviceInputListenerDict.keys()):
-            value = DeviceInputListenerDict.pop(key)
-            value.stop()
-    deviceSocketServer.close()
+    try:
+        if len(DeviceInputListenerDict) > 0:
+            for key in list(DeviceInputListenerDict.keys()):
+                value = DeviceInputListenerDict.pop(key)
+                value.stop()
+        deviceSocketServer.close()
+        if socket_service_instance is not None:
+            socket_service_instance.stop_server()
+            socket_service_instance = None 
+    except:
+        pass
+
+    if config_util.key_chat_module == "agent":
+        util.log(1, '正在关闭agent服务...')
+        agent_service.agent_stop()
+
     util.log(1, '正在关闭核心服务...')
     feiFei.stop()
     util.log(1, '服务已关闭！')
@@ -340,6 +309,7 @@ def start():
     global feiFei
     global recorderListener
     global __running
+    global socket_service_instance
     util.log(1, '开启服务...')
     __running = True
 
@@ -364,22 +334,26 @@ def start():
     record = config_util.config['source']['record']
     if record['enabled']:
         util.log(1, '开启录音服务...')
-        recorderListener = RecorderListener(record['device'], feiFei)  # 监听麦克风
-        recorderListener.start()
+    recorderListener = RecorderListener('device', feiFei)  # 监听麦克风
+    recorderListener.start()
 
     #启动声音沟通接口服务
     util.log(1,'启动声音沟通接口服务...')
     deviceSocketThread = MyThread(target=accept_audio_device_output_connect)
     deviceSocketThread.start()
+    socket_service_instance = socket_bridge_service.new_instance()
+    socket_bridge_service_Thread = MyThread(target=socket_service_instance.start_service)
+    socket_bridge_service_Thread.start()
+
+    #启动agent服务
+    if config_util.key_chat_module == "agent":
+        util.log(1,'启动agent服务...')
+        agent_service.agent_start()
 
     #启动自动播放服务
     util.log(1,'启动自动播放服务...')
     MyThread(target=start_auto_play_service).start()
-            
-    #监听控制台
-    util.log(1, '注册命令...')
-    MyThread(target=console_listener).start()  # 监听控制台
-
+        
     util.log(1, '服务启动完成!')
     util.log(1, '使用 \'help\' 获取帮助.')
 

gui/flask_server.py

@@ -8,6 +8,7 @@ from flask import Flask, render_template, request, jsonify, Response, send_file
 from flask_cors import CORS
 import requests
 import datetime
+import pytz
 
 import fay_booter
 
@@ -22,16 +23,23 @@ from core.interact import Interact
 from core import member_db
 import fay_booter
 from flask_httpauth import HTTPBasicAuth
-
+from flask import Flask, render_template, request, jsonify, Response, send_file
+from flask_cors import CORS
+from flask_socketio import SocketIO, emit
+from core import qa_service
 
 __app = Flask(__name__)
 auth = HTTPBasicAuth()
 CORS(__app, supports_credentials=True)
-
+socketio = SocketIO(__app, async_mode='gevent', cors_allowed_origins="*")
 def load_users():
-    with open('verifier.json') as f:
-        users = json.load(f)
-    return users
+    try:
+        with open('verifier.json') as f:
+            users = json.load(f)
+        return users
+    except Exception as e:
+        print(f"Error loading users: {e}")
+        return {}
 
 users = load_users()
 
@@ -44,241 +52,336 @@ def verify_password(username, password):
 
 
 def __get_template():
-    return render_template('index.html')
-
+    try:
+        return render_template('index.html')
+    except Exception as e:
+        return f"Error rendering template: {e}", 500
 
 def __get_device_list():
-    if config_util.start_mode == 'common':
-        audio = pyaudio.PyAudio()
-        device_list = []
-        for i in range(audio.get_device_count()):
-            devInfo = audio.get_device_info_by_index(i)
-            if devInfo['hostApi'] == 0:
-                device_list.append(devInfo["name"])
-        
-        return list(set(device_list))
-    else:
+    try:
+        if config_util.start_mode == 'common':
+            audio = pyaudio.PyAudio()
+            device_list = []
+            for i in range(audio.get_device_count()):
+                devInfo = audio.get_device_info_by_index(i)
+                if devInfo['hostApi'] == 0:
+                    device_list.append(devInfo["name"])
+            return list(set(device_list))
+        else:
+            return []
+    except Exception as e:
+        print(f"Error getting device list: {e}")
         return []
 
 
 @__app.route('/api/submit', methods=['post'])
 def api_submit():
     data = request.values.get('data')
-    config_data = json.loads(data)
-    if(config_data['config']['source']['record']['enabled']):
-        config_data['config']['source']['record']['channels'] = 0
-        audio = pyaudio.PyAudio()
-        for i in range(audio.get_device_count()):
-            devInfo = audio.get_device_info_by_index(i)
-            if devInfo['name'].find(config_data['config']['source']['record']['device']) >= 0 and devInfo['hostApi'] == 0:
-                 config_data['config']['source']['record']['channels'] = devInfo['maxInputChannels']
+    if not data:
+        return jsonify({'result': 'error', 'message': '未提供数据'})
+    try:
+        config_data = json.loads(data)
+        if 'config' not in config_data:
+            return jsonify({'result': 'error', 'message': '数据中缺少config'})
+
+        config_util.load_config()
+        existing_config = config_util.config
+
+        def merge_configs(existing, new):
+            for key, value in new.items():
+                if isinstance(value, dict) and key in existing:
+                    if isinstance(existing[key], dict):
+                        merge_configs(existing[key], value)
+                    else:
+                        existing[key] = value
+                else:
+                    existing[key] = value
+
+        merge_configs(existing_config, config_data['config'])
+
+        config_util.save_config(existing_config)
+        config_util.load_config()
+
+        return jsonify({'result': 'successful'})
+    except json.JSONDecodeError:
+        return jsonify({'result': 'error', 'message': '无效的JSON数据'})
+    except Exception as e:
+        return jsonify({'result': 'error', 'message': f'保存配置时出错: {e}'}), 500
+    
 
-    config_util.save_config(config_data['config'])
 
-    return '{"result":"successful"}'
 
 @__app.route('/api/get-data', methods=['post'])
 def api_get_data():
-    config_util.load_config()
-    voice_list = tts_voice.get_voice_list()
-    send_voice_list = []
-    if config_util.tts_module == 'ali':
-        voice_list = [
-            {"id": "abin", "name": "阿斌"},
-            {"id": "zhixiaobai", "name": "知小白"},
-            {"id": "zhixiaoxia", "name": "知小夏"},
-            {"id": "zhixiaomei", "name": "知小妹"},
-            {"id": "zhigui", "name": "知柜"},
-            {"id": "zhishuo", "name": "知硕"},
-            {"id": "aixia", "name": "艾夏"},
-            {"id": "zhifeng_emo", "name": "知锋_多情感"},
-            {"id": "zhibing_emo", "name": "知冰_多情感"},
-            {"id": "zhimiao_emo", "name": "知妙_多情感"},
-            {"id": "zhimi_emo", "name": "知米_多情感"},
-            {"id": "zhiyan_emo", "name": "知燕_多情感"},
-            {"id": "zhibei_emo", "name": "知贝_多情感"},
-            {"id": "zhitian_emo", "name": "知甜_多情感"},
-            {"id": "xiaoyun", "name": "小云"},
-            {"id": "xiaogang", "name": "小刚"},
-            {"id": "ruoxi", "name": "若兮"},
-            {"id": "siqi", "name": "思琪"},
-            {"id": "sijia", "name": "思佳"},
-            {"id": "sicheng", "name": "思诚"},
-            {"id": "aiqi", "name": "艾琪"},
-            {"id": "aijia", "name": "艾佳"},
-            {"id": "aicheng", "name": "艾诚"},
-            {"id": "aida", "name": "艾达"},
-            {"id": "ninger", "name": "宁儿"},
-            {"id": "ruilin", "name": "瑞琳"},
-            {"id": "siyue", "name": "思悦"},
-            {"id": "aiya", "name": "艾雅"},
-            {"id": "aimei", "name": "艾美"},
-            {"id": "aiyu", "name": "艾雨"},
-            {"id": "aiyue", "name": "艾悦"},
-            {"id": "aijing", "name": "艾婧"},
-            {"id": "xiaomei", "name": "小美"},
-            {"id": "aina", "name": "艾娜"},
-            {"id": "yina", "name": "伊娜"},
-            {"id": "sijing", "name": "思婧"},
-            {"id": "sitong", "name": "思彤"},
-            {"id": "xiaobei", "name": "小北"},
-            {"id": "aitong", "name": "艾彤"},
-            {"id": "aiwei", "name": "艾薇"},
-            {"id": "aibao", "name": "艾宝"},
-            {"id": "shanshan", "name": "姗姗"},
-            {"id": "chuangirl", "name": "小玥"},
-            {"id": "lydia", "name": "Lydia"},
-            {"id": "aishuo", "name": "艾硕"},
-            {"id": "qingqing", "name": "青青"},
-            {"id": "cuijie", "name": "翠姐"},
-            {"id": "xiaoze", "name": "小泽"},
-            {"id": "zhimao", "name": "知猫"},
-            {"id": "zhiyuan", "name": "知媛"},
-            {"id": "zhiya", "name": "知雅"},
-            {"id": "zhiyue", "name": "知悦"},
-            {"id": "zhida", "name": "知达"},
-            {"id": "zhistella", "name": "知莎"},
-            {"id": "kelly", "name": "Kelly"},
-            {"id": "jiajia", "name": "佳佳"},
-            {"id": "taozi", "name": "桃子"},
-            {"id": "guijie", "name": "柜姐"},
-            {"id": "stella", "name": "Stella"},
-            {"id": "stanley", "name": "Stanley"},
-            {"id": "kenny", "name": "Kenny"},
-            {"id": "rosa", "name": "Rosa"},
-            {"id": "mashu", "name": "马树"},
-            {"id": "xiaoxian", "name": "小仙"},
-            {"id": "yuer", "name": "悦儿"},
-            {"id": "maoxiaomei", "name": "猫小美"},
-            {"id": "aifei", "name": "艾飞"},
-            {"id": "yaqun", "name": "亚群"},
-            {"id": "qiaowei", "name": "巧薇"},
-            {"id": "dahu", "name": "大虎"},
-            {"id": "ailun", "name": "艾伦"},
-            {"id": "jielidou", "name": "杰力豆"},
-            {"id": "laotie", "name": "老铁"},
-            {"id": "laomei", "name": "老妹"},
-            {"id": "aikan", "name": "艾侃"}
-        
-    ]
-        send_voice_list = {"voiceList": voice_list}
-        wsa_server.get_web_instance().add_cmd(send_voice_list)
-    elif config_util.tts_module == 'volcano':
-        voice_list = {
-        "voiceList": [
-            {"id": "BV001_streaming", "name": "通用女声"},
-            {"id": "BV002_streaming", "name": "通用男声"},
-            {"id": "zh_male_jingqiangkanye_moon_bigtts", "name": "京腔侃爷/Harmony"},
-            {"id": "zh_female_shuangkuaisisi_moon_bigtts", "name": "爽快思思/Skye"},
-            {"id": "zh_male_wennuanahu_moon_bigtts", "name": "温暖阿虎/Alvin"},
-            {"id": "zh_female_wanwanxiaohe_moon_bigtts", "name": "湾湾小何"},
-        ]
-    }
-        send_voice_list = {"voiceList": voice_list}
-        wsa_server.get_web_instance().add_cmd(send_voice_list)
-    else:
+    # 获取配置和语音列表
+    try:
+        config_util.load_config()
         voice_list = tts_voice.get_voice_list()
         send_voice_list = []
-        for voice in voice_list: 
-            voice_data = voice.value 
-            send_voice_list.append({"id": voice_data['name'], "name": voice_data['name']})
-        wsa_server.get_web_instance().add_cmd({
-            "voiceList": send_voice_list
-        })
-        voice_list = send_voice_list
-    wsa_server.get_web_instance().add_cmd({"deviceList": __get_device_list()})
-    if fay_booter.is_running():
-        wsa_server.get_web_instance().add_cmd({"liveState": 1})
-    return json.dumps({'config': config_util.config, 'voice_list' : voice_list})
-
+        if config_util.tts_module == 'ali':
+            voice_list = [
+                {"id": "abin", "name": "阿斌"},
+                {"id": "zhixiaobai", "name": "知小白"},
+                {"id": "zhixiaoxia", "name": "知小夏"},
+                {"id": "zhixiaomei", "name": "知小妹"},
+                {"id": "zhigui", "name": "知柜"},
+                {"id": "zhishuo", "name": "知硕"},
+                {"id": "aixia", "name": "艾夏"},
+                {"id": "zhifeng_emo", "name": "知锋_多情感"},
+                {"id": "zhibing_emo", "name": "知冰_多情感"},
+                {"id": "zhimiao_emo", "name": "知妙_多情感"},
+                {"id": "zhimi_emo", "name": "知米_多情感"},
+                {"id": "zhiyan_emo", "name": "知燕_多情感"},
+                {"id": "zhibei_emo", "name": "知贝_多情感"},
+                {"id": "zhitian_emo", "name": "知甜_多情感"},
+                {"id": "xiaoyun", "name": "小云"},
+                {"id": "xiaogang", "name": "小刚"},
+                {"id": "ruoxi", "name": "若兮"},
+                {"id": "siqi", "name": "思琪"},
+                {"id": "sijia", "name": "思佳"},
+                {"id": "sicheng", "name": "思诚"},
+                {"id": "aiqi", "name": "艾琪"},
+                {"id": "aijia", "name": "艾佳"},
+                {"id": "aicheng", "name": "艾诚"},
+                {"id": "aida", "name": "艾达"},
+                {"id": "ninger", "name": "宁儿"},
+                {"id": "ruilin", "name": "瑞琳"},
+                {"id": "siyue", "name": "思悦"},
+                {"id": "aiya", "name": "艾雅"},
+                {"id": "aimei", "name": "艾美"},
+                {"id": "aiyu", "name": "艾雨"},
+                {"id": "aiyue", "name": "艾悦"},
+                {"id": "aijing", "name": "艾婧"},
+                {"id": "xiaomei", "name": "小美"},
+                {"id": "aina", "name": "艾娜"},
+                {"id": "yina", "name": "伊娜"},
+                {"id": "sijing", "name": "思婧"},
+                {"id": "sitong", "name": "思彤"},
+                {"id": "xiaobei", "name": "小北"},
+                {"id": "aitong", "name": "艾彤"},
+                {"id": "aiwei", "name": "艾薇"},
+                {"id": "aibao", "name": "艾宝"},
+                {"id": "shanshan", "name": "姗姗"},
+                {"id": "chuangirl", "name": "小玥"},
+                {"id": "lydia", "name": "Lydia"},
+                {"id": "aishuo", "name": "艾硕"},
+                {"id": "qingqing", "name": "青青"},
+                {"id": "cuijie", "name": "翠姐"},
+                {"id": "xiaoze", "name": "小泽"},
+                {"id": "zhimao", "name": "知猫"},
+                {"id": "zhiyuan", "name": "知媛"},
+                {"id": "zhiya", "name": "知雅"},
+                {"id": "zhiyue", "name": "知悦"},
+                {"id": "zhida", "name": "知达"},
+                {"id": "zhistella", "name": "知莎"},
+                {"id": "kelly", "name": "Kelly"},
+                {"id": "jiajia", "name": "佳佳"},
+                {"id": "taozi", "name": "桃子"},
+                {"id": "guijie", "name": "柜姐"},
+                {"id": "stella", "name": "Stella"},
+                {"id": "stanley", "name": "Stanley"},
+                {"id": "kenny", "name": "Kenny"},
+                {"id": "rosa", "name": "Rosa"},
+                {"id": "mashu", "name": "马树"},
+                {"id": "xiaoxian", "name": "小仙"},
+                {"id": "yuer", "name": "悦儿"},
+                {"id": "maoxiaomei", "name": "猫小美"},
+                {"id": "aifei", "name": "艾飞"},
+                {"id": "yaqun", "name": "亚群"},
+                {"id": "qiaowei", "name": "巧薇"},
+                {"id": "dahu", "name": "大虎"},
+                {"id": "ailun", "name": "艾伦"},
+                {"id": "jielidou", "name": "杰力豆"},
+                {"id": "laotie", "name": "老铁"},
+                {"id": "laomei", "name": "老妹"},
+                {"id": "aikan", "name": "艾侃"}
+            ]
+            send_voice_list = {"voiceList": voice_list}
+            wsa_server.get_web_instance().add_cmd(send_voice_list)
+        elif config_util.tts_module == 'volcano':
+            voice_list = [
+                {"id": "BV001_streaming", "name": "通用女声"},
+                {"id": "BV002_streaming", "name": "通用男声"},
+                {"id": "zh_male_jingqiangkanye_moon_bigtts", "name": "京腔侃爷/Harmony"},
+                {"id": "zh_female_shuangkuaisisi_moon_bigtts", "name": "爽快思思/Skye"},
+                {"id": "zh_male_wennuanahu_moon_bigtts", "name": "温暖阿虎/Alvin"},
+                {"id": "zh_female_wanwanxiaohe_moon_bigtts", "name": "湾湾小何"}
+            ]
+            send_voice_list = {"voiceList": voice_list}
+            wsa_server.get_web_instance().add_cmd(send_voice_list)
+        else:
+            voice_list = tts_voice.get_voice_list()
+            send_voice_list = []
+            for voice in voice_list:
+                voice_data = voice.value
+                send_voice_list.append({"id": voice_data['name'], "name": voice_data['name']})
+            wsa_server.get_web_instance().add_cmd({"voiceList": send_voice_list})
+            voice_list = send_voice_list
+        wsa_server.get_web_instance().add_cmd({"deviceList": __get_device_list()})
+        if fay_booter.is_running():
+            wsa_server.get_web_instance().add_cmd({"liveState": 1})
+        return json.dumps({'config': config_util.config, 'voice_list': voice_list})
+    except Exception as e:
+        return jsonify({'result': 'error', 'message': f'获取数据时出错: {e}'}), 500
 
 @__app.route('/api/start-live', methods=['post'])
 def api_start_live():
-    # time.sleep(5)
-    fay_booter.start()
-    time.sleep(1)
-    wsa_server.get_web_instance().add_cmd({"liveState": 1})
-    return '{"result":"successful"}'
-
+    # 启动
+    try:
+        fay_booter.start()
+        time.sleep(1)
+        wsa_server.get_web_instance().add_cmd({"liveState": 1})
+        return '{"result":"successful"}'
+    except Exception as e:
+        return jsonify({'result': 'error', 'message': f'启动时出错: {e}'}), 500
 
 @__app.route('/api/stop-live', methods=['post'])
 def api_stop_live():
-    # time.sleep(1)
-    fay_booter.stop()
-    time.sleep(1)
-    wsa_server.get_web_instance().add_cmd({"liveState": 0})
-    return '{"result":"successful"}'
+    # 停止
+    try:
+        fay_booter.stop()
+        time.sleep(1)
+        wsa_server.get_web_instance().add_cmd({"liveState": 0})
+        return '{"result":"successful"}'
+    except Exception as e:
+        return jsonify({'result': 'error', 'message': f'停止时出错: {e}'}), 500
 
 @__app.route('/api/send', methods=['post'])
 def api_send():
+    # 接收前端发送的消息
     data = request.values.get('data')
-    info = json.loads(data)
-    interact = Interact("text", 1, {'user': info['username'], 'msg': info['msg']})
-    util.printInfo(3, "文字发送按钮", '{}'.format(interact.data["msg"]), time.time())
-    fay_booter.feiFei.on_interact(interact)
-    return '{"result":"successful"}'
+    if not data:
+        return jsonify({'result': 'error', 'message': '未提供数据'})
+    try:
+        info = json.loads(data)
+        username = info.get('username')
+        msg = info.get('msg')
+        if not username or not msg:
+            return jsonify({'result': 'error', 'message': '用户名和消息内容不能为空'})
+        interact = Interact("text", 1, {'user': username, 'msg': msg})
+        util.printInfo(1, username, '[文字发送按钮]{}'.format(interact.data["msg"]), time.time())
+        fay_booter.feiFei.on_interact(interact)
+        return '{"result":"successful"}'
+    except json.JSONDecodeError:
+        return jsonify({'result': 'error', 'message': '无效的JSON数据'})
+    except Exception as e:
+        return jsonify({'result': 'error', 'message': f'发送消息时出错: {e}'}), 500
 
-#获取指定用户的消息记录
+# 获取指定用户的消息记录
 @__app.route('/api/get-msg', methods=['post'])
 def api_get_Msg():
-    data = request.form.get('data')
-    data = json.loads(data)
-    uid = member_db.new_instance().find_user(data["username"])
-    contentdb = content_db.new_instance()
-    if uid == 0:
-        return json.dumps({'list': []})
-    else:
-        list = contentdb.get_list('all','desc',1000, uid)
-    relist = []
-    i = len(list)-1
-    while i >= 0:
-        timetext = datetime.datetime.fromtimestamp(list[i][3]).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
-        relist.append(dict(type=list[i][0], way=list[i][1], content=list[i][2], createtime=list[i][3], timetext=timetext, username=list[i][5]))
-        i -= 1
-    if fay_booter.is_running():
-        wsa_server.get_web_instance().add_cmd({"liveState": 1})
-    return json.dumps({'list': relist})
+    try:
+        data = request.form.get('data')
+        if data is None:
+            data = request.get_json()
+        else:
+            data = json.loads(data)
+        uid = member_db.new_instance().find_user(data["username"])
+        contentdb = content_db.new_instance()
+        if uid == 0:
+            return json.dumps({'list': []})
+        else:
+            list = contentdb.get_list('all', 'desc', 1000, uid)
+        relist = []
+        i = len(list) - 1
+        while i >= 0:
+            timezone = pytz.timezone('Asia/Shanghai')
+            timetext = datetime.datetime.fromtimestamp(list[i][3], timezone).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
+            relist.append(dict(type=list[i][0], way=list[i][1], content=list[i][2], createtime=list[i][3], timetext=timetext, username=list[i][5], id=list[i][6], is_adopted=list[i][7]))
+            i -= 1
+        if fay_booter.is_running():
+            wsa_server.get_web_instance().add_cmd({"liveState": 1})
+        return json.dumps({'list': relist})
+    except json.JSONDecodeError:
+        return jsonify({'list': [], 'message': '无效的JSON数据'})
+    except Exception as e:
+        return jsonify({'list': [], 'message': f'获取消息时出错: {e}'}), 500
 
 @__app.route('/v1/chat/completions', methods=['post'])
 @__app.route('/api/send/v1/chat/completions', methods=['post'])
 def api_send_v1_chat_completions():
-    data = request.json  
-    last_content = ""
-    if 'messages' in data and data['messages']:
-        last_message = data['messages'][-1]  
-        username = last_message.get('role', 'User')  
-        if username == 'user':
+    # 处理聊天完成请求
+    data = request.get_json()
+    if not data:
+        return jsonify({'error': '未提供数据'})
+    try:
+        last_content = ""
+        if 'messages' in data and data['messages']:
+            last_message = data['messages'][-1]
+            username = last_message.get('role', 'User')
+            if username == 'user':
+                username = 'User'
+            last_content = last_message.get('content', 'No content provided')
+        else:
+            last_content = 'No messages found'
             username = 'User'
-        last_content = last_message.get('content', 'No content provided')  
-    else:
-        last_content = 'No messages found'
-        username = 'User'
 
-    model = data.get('model', 'fay')
-    observation = data.get('observation', '')
-    interact = Interact("text", 1, {'user': username, 'msg': last_content, 'observation': observation})
-    util.printInfo(3, "文字沟通接口", '{}'.format(interact.data["msg"]), time.time())
-    text = fay_booter.feiFei.on_interact(interact)
+        model = data.get('model', 'fay')
+        observation = data.get('observation', '')
+        interact = Interact("text", 1, {'user': username, 'msg': last_content, 'observation': str(observation)})
+        util.printInfo(1, username, '[文字沟通接口]{}'.format(interact.data["msg"]), time.time())
+        text = fay_booter.feiFei.on_interact(interact)
 
-    if model == 'fay-streaming':
-        return stream_response(text)
-    else:
-        return non_streaming_response(last_content, text)
+        if model == 'fay-streaming':
+            return stream_response(text)
+        else:
+            return non_streaming_response(last_content, text)
+    except Exception as e:
+        return jsonify({'error': f'处理请求时出错: {e}'}), 500
 
 @__app.route('/api/get-member-list', methods=['post'])
 def api_get_Member_list():
-    memberdb = member_db.new_instance()
-    list = memberdb.get_all_users()
-    return json.dumps({'list': list})
-
+    # 获取成员列表
+    try:
+        memberdb = member_db.new_instance()
+        list = memberdb.get_all_users()
+        return json.dumps({'list': list})
+    except Exception as e:
+        return jsonify({'list': [], 'message': f'获取成员列表时出错: {e}'}), 500
 
 @__app.route('/api/get_run_status', methods=['post'])
 def api_get_run_status():
-    status = fay_booter.is_running()
-    return json.dumps({'status': status})
+    # 获取运行状态
+    try:
+        status = fay_booter.is_running()
+        return json.dumps({'status': status})
+    except Exception as e:
+        return jsonify({'status': False, 'message': f'获取运行状态时出错: {e}'}), 500
 
+@__app.route('/api/adopt_msg', methods=['POST'])
+def adopt_msg():
+    # 采纳消息
+    data = request.get_json()
+    if not data:
+        return jsonify({'status':'error', 'msg': '未提供数据'})
+
+    id = data.get('id')
+
+    if not id:
+        return jsonify({'status':'error', 'msg': 'id不能为空'})
+
+    if  config_util.config["interact"]["QnA"] == "":
+        return jsonify({'status':'error', 'msg': '请先设置Q&A文件'})
+
+    try:
+        info = content_db.new_instance().get_content_by_id(id)
+        content = info[3] if info else ''
+        if info is not None:
+            previous_info = content_db.new_instance().get_previous_user_message(id)
+            previous_content = previous_info[3] if previous_info else ''
+            result = content_db.new_instance().adopted_message(id)
+            if result:
+                qa_service.QAService().record_qapair(previous_content, content)
+                return jsonify({'status': 'success', 'msg': '采纳成功'})
+            else:
+                return jsonify({'status':'error', 'msg': '采纳失败'}), 500
+        else:
+            return jsonify({'status':'error', 'msg': '消息未找到'}), 404
+    except Exception as e:
+        return jsonify({'status':'error', 'msg': f'采纳消息时出错: {e}'}), 500
 
 def stream_response(text):
+    # 处理流式响应
     def generate():
         for chunk in text_chunks(text):
             message = {
@@ -298,12 +401,12 @@ def stream_response(text):
             }
             yield f"data: {json.dumps(message)}\n\n"
             time.sleep(0.1)
-        # 发送最终的结束信号
         yield 'data: [DONE]\n\n'
     
     return Response(generate(), mimetype='text/event-stream')
 
 def non_streaming_response(last_content, text):
+    # 处理非流式响应
     return jsonify({
         "id": "chatcmpl-8jqorq6Fw1Vi5XoH7pddGGpQeuPe0",
         "object": "chat.completion",
@@ -334,35 +437,106 @@ def text_chunks(text, chunk_size=20):
     for chunk in chunks:
         yield chunk
 
+messages = []
+
+@socketio.on('send_message')
+def handle_send_message(data):
+    """
+    处理来自客户端的消息。
+    data: {
+        'username': '用户或系统',
+        'message': '消息内容',
+        'type': 'user' 或 'system'
+    }
+    """
+    timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
+    message = {
+        'id': len(messages) + 1,
+        'username': data.get('username', '用户'),
+        'content': data.get('message', ''),
+        'type': data.get('type', 'user'),
+        'timetext': timestamp,
+        'is_adopted': 1  # 根据需要调整
+    }
+    messages.append(message)
+
+    # 只保留最近的100条消息以防内存溢出
+    if len(messages) > 100:
+        messages.pop(0)
+
+    # 向所有客户端广播新消息
+    emit('receive_message', message, broadcast=True)
+
+    # 系统自动回复（可根据实际需求调整）
+    system_message_content = f"您说的是：{data.get('message', '')}"
+    system_message = {
+        'id': len(messages) + 1,
+        'username': '系统',
+        'content': system_message_content,
+        'type': 'system',
+        'timetext': time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),
+        'is_adopted': 1
+    }
+    messages.append(system_message)
+
+    emit('receive_message', system_message, broadcast=True)
+
+@socketio.on('connect')
+def handle_connect():
+    print('客户端已连接')
+    # 发送最近三条消息给新连接的客户端
+    recent_messages = messages[-3:]
+    emit('load_messages', recent_messages)
+
+@socketio.on('disconnect')
+def handle_disconnect():
+    print('客户端已断开连接')
+
 @__app.route('/', methods=['get'])
 @auth.login_required
 def home_get():
-    return __get_template()
-
+    try:
+        return __get_template()
+    except Exception as e:
+        return f"Error loading home page: {e}", 500
 
 @__app.route('/', methods=['post'])
 @auth.login_required
 def home_post():
-    wsa_server.get_web_instance.add_cmd({"is_connect": wsa_server.get_instance().isConnect}) #TODO 不应放这里，同步数字人连接状态
-    return __get_template()
+    try:
+        wsa_server.get_web_instance().add_cmd({"is_connect": wsa_server.get_instance().isConnect})  # TODO 不应放这里，同步数字人连接状态
+        return __get_template()
+    except Exception as e:
+        return f"Error processing request: {e}", 500
 
 @__app.route('/setting', methods=['get'])
 def setting():
-    return render_template('setting.html')
+    try:
+        return render_template('setting.html')
+    except Exception as e:
+        return f"Error loading settings page: {e}", 500
 
 
-
-#输出的音频http
+@__app.route('/chat')
+def chat():
+    return render_template('chat.html')
+# 输出的音频http
 @__app.route('/audio/<filename>')
 def serve_audio(filename):
     audio_file = os.path.join(os.getcwd(), "samples", filename)
-    return send_file(audio_file)
+    if os.path.exists(audio_file):
+        return send_file(audio_file)
+    else:
+        return jsonify({'error': '文件未找到'}), 404
 
-#输出的表情git
+# 输出的表情gif
 @__app.route('/robot/<filename>')
 def serve_gif(filename):
     gif_file = os.path.join(os.getcwd(), "gui", "robot", filename)
-    return send_file(gif_file)
+    if os.path.exists(gif_file):
+        return send_file(gif_file)
+    else:
+        return jsonify({'error': '文件未找到'}), 404
 
 def run():
     server = pywsgi.WSGIServer(('0.0.0.0',5000), __app)

gui/static/css/chat.css

@@ -0,0 +1,132 @@
+/* static/css/chat.css */
+
+#chat-widget {
+    position: fixed;
+    bottom: 20px;
+    right: 20px;
+    width: 300px;
+    font-family: Arial, sans-serif;
+    z-index: 1000;
+}
+
+#toggle-chat {
+    width: 100%;
+    padding: 10px;
+    background-color: #00f0ff;
+    border: none;
+    color: white;
+    cursor: pointer;
+    font-size: 16px;
+    border-radius: 5px 5px 0 0;
+}
+
+#toggle-chat:hover {
+    background-color: #00c0d1;
+}
+
+#chat-container {
+    border: 1px solid #ccc;
+    border-top: none;
+    background-color: white;
+    border-radius: 0 0 10px 10px;
+    box-shadow: 0 4px 6px rgba(0,0,0,0.1);
+    display: flex;
+    flex-direction: column;
+    height: 400px;
+    overflow: hidden;
+}
+
+.hidden {
+    display: none;
+}
+
+#messages {
+    flex: 1;
+    padding: 10px;
+    overflow-y: auto;
+    background-color: #f9f9f9;
+}
+
+.message {
+    display: flex;
+    margin-bottom: 10px;
+}
+
+.message.user {
+    justify-content: flex-end;
+}
+
+.message.system {
+    justify-content: center;
+}
+
+.message .bubble {
+    max-width: 70%;
+    padding: 10px;
+    border-radius: 10px;
+    position: relative;
+}
+
+.message.user .bubble {
+    background-color: #d1e7dd;
+    text-align: right;
+}
+
+.message.system .bubble {
+    background-color: #f1f1f1;
+    text-align: center;
+}
+
+#status-indicator {
+    height: 20px;
+    text-align: center;
+    color: #555;
+    font-size: 14px;
+}
+
+#input-area {
+    display: flex;
+    align-items: center;
+    padding: 10px;
+    border-top: 1px solid #ccc;
+    background-color: #fafafa;
+}
+
+#input-area button {
+    background: none;
+    border: none;
+    cursor: pointer;
+    margin-right: 10px;
+}
+
+#input-area img {
+    width: 24px;
+    height: 24px;
+}
+
+#message-input {
+    flex: 1;
+    padding: 8px;
+    border: 1px solid #ccc;
+    border-radius: 5px;
+}
+
+#message-input:disabled {
+    background-color: #e0e0e0;
+}
+
+/* 渐隐动画 */
+.fade-out {
+    opacity: 0;
+    transition: opacity 0.5s ease-out;
+}
+
+/* 为消息气泡添加淡入效果 */
+.bubble {
+    animation: fadeIn 0.5s;
+}
+
+@keyframes fadeIn {
+    from { opacity: 0; }
+    to { opacity: 1; }
+}

gui/static/css/index.css

@@ -179,6 +179,8 @@ html {
 	padding: 8px;
 	font-size: 15px;
 	color: #333;
+	white-space: pre-wrap; 
+	word-wrap: break-word;
   }
 
 .sender-message.message-bubble {
@@ -204,19 +206,23 @@ html {
 
 .Userchange{
 	background-color: #FFFFFF;
-	height: 40px;
-	font-size: 12px;
+ 	height: 40px;
+ 	font-size: 12px;
+ 	border-top: 1px solid #bed1fc;
+ 	width: 1358px; /* 设置菜单容器的宽度，可根据实际情况调整 */
+ 	overflow: hidden; /* 隐藏超出容器的内容 */
+ 	position: relative;
 }
 
 .inputmessage{
-	margin-left:15% ;
+	margin-left:290px ;
 	width: 760px;
 	background: #f9fbff;
 	border-radius: 70px;
 	height: 73px;
 	box-shadow: -10px 0 15px rgba(0, 0, 0, 0.1);
 	position: absolute;
-    top: 70%;
+	top: 675px;
 	z-index: 2;
 }
 
@@ -276,4 +282,87 @@ html {
  .tag.selected {
 	background-color: #f4f7ff;
 	color: #0064fb;
-  }
+  }
+  #prevButton{background-color: #FFFFFF; border: none;
+	    z-index: 1;
+	    position: absolute;
+	     top: 50%; 
+	     transform: translateY(-50%);
+	}
+	
+	#nextButton {background-color: #FFFFFF; border: none;
+	    position: absolute;
+	     top: 50%; 
+	     transform: translateY(-50%);
+	}
+	
+	#prevButton {
+	    left: 0;
+	}
+	
+	#nextButton {
+	    right: 0;
+	}
+	
+	.menu-container {
+	    width: 800px; /* 设置菜单容器的宽度，可根据实际情况调整 */
+	    overflow: hidden; /* 隐藏超出容器的内容 */
+	    position: relative;
+	}
+	
+	.menu li {
+	    margin-right: 20px; /* 菜单项之间的间距，可调整 */
+	}
+	
+	.menu li a {
+	    text-decoration: none;
+	    color: black;
+	}
+	
+	.menu { background-color: #FFFFFF;
+	    /* display: flex; */
+	    white-space: nowrap; 
+	    display: flex;
+	    transition: transform 0.3s ease; /* 添加过渡效果，使滑动更平滑 */
+	    list-style: none;
+	    padding: 0 50px 0 50px;
+	    margin: 0;
+	    display: flex;
+	    transition: transform 0.3s ease; /* 添加过渡效果，使滑动更平滑 */
+	 }
+	
+	.adopt{border: none;background: none;}
+	
+	.what-time{vertical-align:top;line-height: 25px;}
+	
+	.answer-container {
+	    border: 1px solid #ccc;
+	    padding: 10px;
+	    margin: 10px;
+	    background-color: #f9f9f9;
+	}
+	
+	.adopt-button {
+	    display: inline-block;
+	    cursor: pointer;
+	    position: relative;
+	}
+	
+	.adopt-button img {
+	    width: 21px;
+	    height: 21px;
+	    display: block;
+	}
+	
+	.adopt-button:hover::after {
+	    content: "采纳";
+	    position: absolute;
+	    top: -30px;
+	    left: 0;
+	    background-color: #000;
+	    color: #fff;
+	    padding: 5px 10px;
+	    border-radius: 4px;
+	    font-size: 12px;
+	    white-space: nowrap;
+	}

gui/static/js/chat.js

@@ -0,0 +1,206 @@
+// static/js/chat.js
+
+new Vue({
+    el: '#chat-app',
+    delimiters: ['[[', ']]'],
+    data() {
+        return {
+            socket: null,
+            messages: [],
+            newMessage: '',
+            status: '', // 用于显示“聆听中...”或“思考中...”
+            isRecording: false,
+            recognition: null
+        };
+    },
+    created() {
+        this.initSocket();
+        this.initSpeechRecognition();
+        this.startLive(); // 页面加载后自动启动 live 模式
+    },
+    methods: {
+        initSocket() {
+            // 初始化 SocketIO 连接
+            this.socket = io.connect('http://' + document.domain + ':' + location.port + '/');
+
+            this.socket.on('connect', () => {
+                console.log('Connected to SocketIO server');
+            });
+
+            this.socket.on('receive_message', (data) => {
+                this.addMessage(data.type, data.content, data.username, data.timetext);
+            });
+
+            this.socket.on('load_messages', (data) => {
+                data.forEach(message => {
+                    this.addMessage(message.type, message.content, message.username, message.timetext);
+                });
+            });
+
+            this.socket.on('disconnect', () => {
+                console.log('Disconnected from SocketIO server');
+            });
+        },
+        initSpeechRecognition() {
+            if ('webkitSpeechRecognition' in window || 'SpeechRecognition' in window) {
+                const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
+                this.recognition = new SpeechRecognition();
+                this.recognition.lang = 'zh-CN';
+                this.recognition.interimResults = false;
+                this.recognition.maxAlternatives = 1;
+
+                this.recognition.onstart = () => {
+                    this.isRecording = true;
+                    this.status = '聆听中...';
+                    this.$set(this, 'recordIconSrc', '/static/images/recording.png');
+                };
+
+                this.recognition.onresult = (event) => {
+                    const transcript = event.results[0][0].transcript;
+                    this.addMessage('user', transcript, '您');
+                    this.socket.emit('send_message', {
+                        username: '您',
+                        message: transcript,
+                        type: 'user'
+                    });
+                    this.status = '思考中...';
+                    setTimeout(() => {
+                        this.status = '';
+                    }, 2000);
+                };
+
+                this.recognition.onerror = (event) => {
+                    console.error('语音识别错误:', event.error);
+                    this.status = '语音识别失败';
+                    this.isRecording = false;
+                    this.$set(this, 'recordIconSrc', '/static/images/record.png');
+                };
+
+                this.recognition.onend = () => {
+                    this.isRecording = false;
+                    this.$set(this, 'recordIconSrc', '/static/images/record.png');
+                };
+            } else {
+                alert('当前浏览器不支持语音识别功能');
+            }
+        },
+        toggleChat() {
+            const chatContainer = document.getElementById('chat-container');
+            const toggleButton = document.getElementById('toggle-chat');
+            if (chatContainer.classList.contains('hidden')) {
+                chatContainer.classList.remove('hidden');
+                toggleButton.textContent = '关闭聊天';
+            } else {
+                chatContainer.classList.add('hidden');
+                toggleButton.textContent = '打开聊天';
+            }
+        },
+        toggleRecording() {
+            if (this.recognition) {
+                if (!this.isRecording) {
+                    this.recognition.start();
+                } else {
+                    this.recognition.stop();
+                }
+            }
+        },
+        sendMessage() {
+            const message = this.newMessage.trim();
+            if (message === '') return;
+            this.addMessage('user', message, '您');
+            this.socket.emit('send_message', {
+                username: '您',
+                message: message,
+                type: 'user'
+            });
+            this.newMessage = '';
+            this.status = '思考中...';
+            setTimeout(() => {
+                this.status = '';
+            }, 2000);
+        },
+        addMessage(type, content, username, timestamp) {
+            const message = { type, content, username, timetext: timestamp };
+            this.messages.push(message);
+            // 只保留最近三条消息
+            if (this.messages.length > 3) {
+                this.messages.shift();
+            }
+            // 滚动到最新消息
+            this.$nextTick(() => {
+                const messagesDiv = document.getElementById('messages');
+                if (messagesDiv) {
+                    messagesDiv.scrollTop = messagesDiv.scrollHeight;
+                }
+            });
+        },
+        getCurrentTime() {
+            const now = new Date();
+            const year = now.getFullYear();
+            const month = String(now.getMonth() + 1).padStart(2, '0');
+            const day = String(now.getDate()).padStart(2, '0');
+            const hours = String(now.getHours()).padStart(2, '0');
+            const minutes = String(now.getMinutes()).padStart(2, '0');
+            const seconds = String(now.getSeconds()).padStart(2, '0');
+            return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
+        },
+        startLive() {
+            // 调用启动 live 模式的 API
+            fetch('/api/start-live', {
+                method: 'POST'
+            })
+            .then(response => response.json())
+            .then(data => {
+                if (data.result === 'successful') {
+                    this.$notify({
+                        title: '成功',
+                        message: '已开启 live 模式！',
+                        type: 'success',
+                    });
+                } else {
+                    this.$notify({
+                        title: '失败',
+                        message: data.message || '启动 live 模式失败',
+                        type: 'error',
+                    });
+                }
+            })
+            .catch(error => {
+                console.error('启动 live 模式时出错:', error);
+                this.$notify({
+                    title: '错误',
+                    message: '启动 live 模式时出错',
+                    type: 'error',
+                });
+            });
+        },
+        stopLive() {
+            // 调用停止 live 模式的 API
+            fetch('/api/stop-live', {
+                method: 'POST'
+            })
+            .then(response => response.json())
+            .then(data => {
+                if (data.result === 'successful') {
+                    this.$notify({
+                        title: '成功',
+                        message: '已关闭 live 模式！',
+                        type: 'success',
+                    });
+                } else {
+                    this.$notify({
+                        title: '失败',
+                        message: data.message || '停止 live 模式失败',
+                        type: 'error',
+                    });
+                }
+            })
+            .catch(error => {
+                console.error('停止 live 模式时出错:', error);
+                this.$notify({
+                    title: '错误',
+                    message: '停止 live 模式时出错',
+                    type: 'error',
+                });
+            });
+        }}})

gui/static/js/index.js

@@ -128,6 +128,13 @@ class FayInterface {
       });
     }
   
+    getData() {
+      return this.fetchData(`${this.baseApiUrl}/api/get-data`, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+      });
+  }
+
     getTime(){
       const date = new Date();
       const year = date.getFullYear();
@@ -183,10 +190,12 @@ class FayInterface {
         }
         if (vueInstance.selectedUser && data.panelReply.username === vueInstance.selectedUser[1]) {
           vueInstance.messages.push({
+            id: data.panelReply.id,
             username: data.panelReply.username,
             content: data.panelReply.content,
             type: data.panelReply.type,
-            timetext: this.getTime()
+            timetext: this.getTime(),
+            is_adopted:0
           });
           vueInstance.$nextTick(() => {
             const chatContainer = vueInstance.$el.querySelector('.chatmessage');
@@ -225,11 +234,14 @@ class FayInterface {
         panelMsg: '', 
         panelReply: '', 
         robot:'static/images/Normal.gif',
-        base_url: 'http://127.0.0.1:5000'
+        base_url: 'http://127.0.0.1:5000',
+        play_sound_enabled: false,
+        source_record_enabled: false
       };
     },
     created() {
-      this.initFayService();
+      this.initFayService(); 
+      this.getData();
       // this.loadUserList();
     },
     methods: {
@@ -239,16 +251,6 @@ class FayInterface {
         this.fayService.websocket.addEventListener('open', () => {
           this.loadUserList();
       });
-      this.fayService.getRunStatus().then((data) => {
-        if (data) {
-            if(data.status){
-                this.liveState = 1;
-            }else{
-                this.liveState = 0;
-            }
-            
-        }
-    });
       },
       sendMessage() {
         let _this = this;
@@ -289,6 +291,80 @@ class FayInterface {
           }
         };
       },
+      getData() {
+        this.fayService.getRunStatus().then((data) => {
+            if (data) {
+                if(data.status){
+                    this.liveState = 1;
+                    this.configEditable = false;
+                }else{
+                    this.liveState = 0;
+                    this.configEditable = true;
+                }
+                
+            }
+        });
+        this.fayService.getData().then((data) => {
+            if (data) {
+                this.updateConfigFromData(data.config);
+            }
+        });
+    },
+    updateConfigFromData(config) {
+      
+        if (config.interact) {
+            this.play_sound_enabled = config.interact.playSound;
+        }
+        if (config.source && config.source.record) {
+            this.source_record_enabled = config.source.record.enabled;
+        }
+    },
+    saveConfig() {
+      let url = `${this.base_url}/api/submit`;
+      let send_data = {
+          "config": {
+              "source": {
+                  "record": {
+                      "enabled": this.source_record_enabled,
+                  },
+              },
+              "interact": {
+                  "playSound": this.play_sound_enabled,
+              }
+          }
+      };
+
+      let xhr = new XMLHttpRequest()
+      xhr.open("post", url)
+      xhr.setRequestHeader("Content-type", "application/x-www-form-urlencoded")
+      xhr.send('data=' + JSON.stringify(send_data))
+      let executed = false
+      xhr.onreadystatechange = async function () {
+          if (!executed && xhr.status === 200) {
+              try {
+                  let data = await eval('(' + xhr.responseText + ')')
+                  executed = true
+              } catch (e) {
+              }
+          }
+      }
+  },
+    changeRecord(){
+      if(this.source_record_enabled){
+        this.source_record_enabled = false
+      }else{
+        this.source_record_enabled = true
+      }
+      this.saveConfig()
+    },
+    changeSound(){
+      if(this.play_sound_enabled){
+        this.play_sound_enabled = false
+      }else{
+        this.play_sound_enabled = true
+      }
+      this.saveConfig()
+    },
       loadUserList() {
         this.fayService.getUserList().then((response) => {
           if (response && response.list) {
@@ -308,12 +384,13 @@ class FayInterface {
       selectUser(user) {
         this.selectedUser = user;
         this.fayService.websocket.send(JSON.stringify({ "Username": user[1] }));
-        this.loadMessageHistory(user[1]); 
+        this.loadMessageHistory(user[1], 'common'); 
       },
       startLive() {
         this.liveState = 2
         this.fayService.startLive().then(() => {
           this.sendSuccessMsg('已开启！');
+          this.getData();
         });
     },
     stopLive() {
@@ -323,11 +400,11 @@ class FayInterface {
         });
     },
 
-      loadMessageHistory(username) {
+      loadMessageHistory(username, type) {
         this.fayService.getMessageHistory(username).then((response) => {
           if (response) {
             this.messages = response;
-            console.log(this.messages);
+            if(type == 'common'){
             this.$nextTick(() => {
               const chatContainer = this.$el.querySelector('.chatmessage');
               if (chatContainer) {
@@ -335,6 +412,7 @@ class FayInterface {
               }
             });
           }
+          }
         });
       },
       sendSuccessMsg(message) {
@@ -344,7 +422,45 @@ class FayInterface {
             type: 'success',
         });
     
+} ,
+adoptText(id) {
+  // 调用采纳接口
+  this.fayService.fetchData(`${this.base_url}/api/adopt_msg`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({ id })  // 发送采纳请求
+  })
+  .then((response) => {
+    if (response && response.status === 'success') {
+      // 处理成功的响应
+      this.$notify({
+        title: '成功',
+        message: response.msg,  // 显示成功消息
+        type: 'success',
+      });
+      
+      this.loadMessageHistory(this.selectedUser[1], 'adopt');
+    } else {
+      // 处理失败的响应
+      this.$notify({
+        title: '失败',
+        message: response ? response.msg : '请求失败',
+        type: 'error',
+      });
+    }
+  })
+  .catch((error) => {
+    // 处理网络错误或HTTP错误
+    this.$notify({
+      title: '错误',
+      message: error.message || '请求失败',
+      type: 'error',
+    });
+  });
 }
+,
     }
   });
   

gui/static/js/script.js

@@ -0,0 +1,14 @@
+const menu = document.querySelector('.menu');
+const prevButton = document.getElementById('prevButton');
+const nextButton = document.getElementById('nextButton');
+
+// 每次滑动的距离，可根据菜单项宽度和间距等实际情况调整
+const slideDistance = 500;
+
+prevButton.addEventListener('click', () => {
+    menu.style.transform = `translateX(${slideDistance}px)`;
+});
+
+nextButton.addEventListener('click', () => {
+    menu.style.transform = `translateX(-${slideDistance}px)`;
+});

gui/static/js/setting.js

@@ -137,7 +137,6 @@ new Vue({
             visualization_detection_enabled: false,
             source_record_enabled: false,
             source_record_device: '',
-            sound_synthesis_enabled: true,
             attribute_name: "",
             attribute_gender: "",
             attribute_age: "",
@@ -211,7 +210,6 @@ new Vue({
             if (config.interact) {
                 this.play_sound_enabled = config.interact.playSound;
                 this.visualization_detection_enabled = config.interact.visualization;
-                this.sound_synthesis_enabled = config.interact.sound_synthesis_enabled;
                 this.QnA = config.interact.QnA;
             }
             if (config.source && config.source.record) {
@@ -277,7 +275,6 @@ new Vue({
                     },
                     "interact": {
                         "playSound": this.play_sound_enabled,
-                        "sound_synthesis_enabled": this.sound_synthesis_enabled,
                         "visualization": this.visualization_detection_enabled,
                         "QnA": this.QnA,
                         "maxInteractTime": this.interact_maxInteractTime,

gui/templates/chat.html

@@ -0,0 +1,45 @@
+<!DOCTYPE html>
+<html lang="zh-CN">
+<head>
+    <meta charset="UTF-8">
+    <title>聊天窗口</title>
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/chat.css') }}">
+    <!-- 引入 Socket.IO 客户端库 -->
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.4/socket.io.min.js" integrity="sha512-OE+tNYgJBSy/R8gSSzIjcC+3hkT0lj3cPnOeYv50E6n0Pzv1tq9I7YytCTv19Uaz1Lr5un2ngc7IQXBuT7GqdQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    <!-- 引入 Vue.js -->
+    <script src="{{ url_for('static', filename='js/vue.js') }}"></script>
+    <!-- 引入 Element UI -->
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/element/theme-chalk.css') }}" />
+    <script src="{{ url_for('static', filename='js/element-ui.js') }}"></script>
+    <!-- 引入 chat.js -->
+    <script src="{{ url_for('static', filename='js/chat.js') }}" defer></script>
+</head>
+<body>
+    <div id="chat-app">
+        <div id="chat-widget">
+            <button id="toggle-chat" @click="toggleChat">打开聊天</button>
+            <div id="chat-container" class="hidden">
+                <div id="messages">
+                    <!-- 消息气泡将动态添加到这里 -->
+                    <div v-for="(message, index) in messages" :key="index" class="message" :class="message.type">
+                        <div class="bubble">
+                            [[ message.content ]]
+                            <div class="message-time">[[ message.timetext ]]</div>
+                        </div>
+                    </div>
+                </div>
+                <div id="status-indicator">[[ status ]]</div>
+                <div id="input-area">
+                    <button id="record-btn" @click="toggleRecording">
+                        <img id="record-icon" :src="isRecording ? '/static/images/recording.png' : '/static/images/record.png'" alt="录音">
+                    </button>
+                    <input type="text" id="message-input" placeholder="请输入内容..." v-model="newMessage">
+                    <button id="send-btn" @click="sendMessage">
+                        <img src="{{ url_for('static', filename='images/send.png') }}" alt="发送">
+                    </button>
+                </div>
+            </div>
+        </div>
+    </div>
+</body>
+</html>

gui/templates/index.html

@@ -4,17 +4,19 @@
     <meta charset="UTF-8" />
     <meta http-equiv="X-UA-Compatible" content="IE=edge" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Fay数字人</title>
+    <title>小橄榄</title>
     <link rel="stylesheet" href="{{ url_for('static',filename='css/index.css') }}" />
+    <link rel="icon" href="{{ url_for('static',filename='images/favicon.ico') }}" type="image/x-icon">
     <script src="{{ url_for('static',filename='js/vue.js') }}"></script>
-    <script src="https://cdn.jsdelivr.net/npm/element-ui@2.15.6/lib/index.js"></script>
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/element-ui@2.15.6/lib/theme-chalk/index.css" />
+    <script src="{{ url_for('static',filename='js/element-ui.js') }}"></script>
+    <link rel="stylesheet" href="{{ url_for('static',filename='css/element/theme-chalk.css') }}" />
     <script src="{{ url_for('static',filename='js/index.js') }}" defer></script>
+    <script src="{{ url_for('static',filename='js/script.js') }}" defer></script>
   </head>
   <body >
     <div  id="app" class="main_bg">  
       <div class="main_left">  
-        <div class="main_left_logo" ><img   src="{{ url_for('static',filename='images/logo.png') }}" alt=""> 
+        <div class="main_left_logo" ><img   src="{{ url_for('static',filename='images/Logo.png') }}" alt=""> 
         </div>
 
       <div class="main_left_menu">
@@ -40,7 +42,15 @@
                 <img class="avatar" src="{{ url_for('static',filename='images/Fay_send.png') }}" alt="接收者头像">
                 <div class="message-content">
                   <div class="message-bubble">[[item.content]]</div>
-                  <div class="message-time">[[item.timetext]]</div>
+                  <div class="message-time"><span class="what-time">[[item.timetext]]</span>
+                    <div v-if="item.is_adopted == 0" class="adopt-button" @click="adoptText(item.id)">
+                    <img   src="{{ url_for('static',filename='images/adopt.png') }}" alt="采纳图标" class="adopt-img" />
+                  </div>
+                  <div v-else class="adopt-button">
+                    <img   src="{{ url_for('static',filename='images/adopted.png') }}" alt="采纳图标" class="adopt-img" />
+                  </div>
+
+                  </div>
                 </div>
               </div>     
 
@@ -62,25 +72,34 @@
         <!-- 以上是聊天对话 -->
 
         <div class="inputmessage">
-          <div class="inputmessage_voice" ><img  src="{{ url_for('static',filename='images/voice.png') }}" alt="" style="filter: grayscale(100%);" ></div>
+          <div  class="inputmessage_voice" >
+
+            <img v-if="!source_record_enabled" src="{{ url_for('static',filename='images/recording.png') }}" alt="" @click=changeRecord() >
+            <img v-else src="{{ url_for('static',filename='images/record.png') }}" alt="" @click=changeRecord() >
+          </div>
           <div class="inputmessage_input"> <textarea class="text_in" placeholder="请输入内容" v-model="newMessage" @keyup.enter="sendMessage" style="padding-top: 13px;"></textarea></div>
           <div class="inputmessage_send"><img  src="{{ url_for('static',filename='images/send.png') }}" alt="发送信息" @click="sendMessage"></div>
 
-          <div class="inputmessage_open">
+          <div v-if="liveState == 1" class="inputmessage_open">
 
-            <img v-if="liveState == 1" src="{{ url_for('static',filename='images/close.png') }}"  @click=stopLive() >
-            <img v-else src="{{ url_for('static',filename='images/open.png') }}"  @click=startLive() >
+            <img v-if="!play_sound_enabled" src="{{ url_for('static',filename='images/sound_off.png') }}"  @click=changeSound() >
+            <img v-else src="{{ url_for('static',filename='images/sound_on.png') }}"  @click=changeSound() >
           
           </div>
-
+          <div v-else class="inputmessage_open">
+            <img  src="{{ url_for('static',filename='images/open.png') }}"  @click=startLive() >
+          </div>
         </div>
 
+       
         <div class="Userchange">
-          <div class="tag-container">
+          <button id="prevButton" ><img  src="{{ url_for('static',filename='images/scrollleft.png') }}" alt="向左滑动" ></button>
+          <div class="menu" ref="menuContainer">
             <div class="tag" v-for="user in userList" :key="user[0]" :class="{'selected': selectedUser && selectedUser[0] === user[0]}" @click="selectUser(user)">
               [[ user[1] ]]
-            </div>
-          </div>
+            </div>    
+                    </div>
+          <button id="nextButton" ><img  src="{{ url_for('static',filename='images/scrollright.png') }}" alt="向右滑动" ></button>
         </div>
 
 
@@ -88,20 +107,5 @@
 <!-- 以上是多用户切换 -->
 
       </div>
-
-
-
-
-
-
-
-
     </div>
-
-
-
-
-
-
-
     </body>

gui/templates/setting.html

@@ -7,6 +7,8 @@
     <title>Fay数字人</title>
     <link rel="stylesheet" href="{{ url_for('static',filename='css/index.css') }}"/>
     <link rel="stylesheet" href="{{ url_for('static',filename='css/setting.css') }}"/>
+    <link rel="icon" href="{{ url_for('static',filename='images/favicon.ico') }}" type="image/x-icon">
+
     <!-- 引入样式 -->
 <link rel="stylesheet" href="{{ url_for('static',filename='css/element/index.css') }}">
   </head>
@@ -14,7 +16,7 @@
   <body>
     <div class="main_bg" id="app">	
       <div class="main_left">	
-        <div class="main_left_logo" ><img src="{{ url_for('static',filename='images/logo.png') }}" alt=""> 
+        <div class="main_left_logo" ><img src="{{ url_for('static',filename='images/Logo.png') }}" alt=""> 
         </div>
 
       <div class="main_left_menu">
@@ -55,7 +57,7 @@
           <div class="setting_wakeup">            
             <ul>
               <li> <span class="font_name">唤醒模式:</span>
-                <el-switch :disabled="!configEditable"
+                <el-switch @change=saveConfig()
                 v-model="wake_word_enabled"
                 active-color="#13ce66"
                 inactive-color="#ff4949">
@@ -74,12 +76,12 @@
 
              <div class="microphone"> 
               <div class="microphone_group1">
-                <span class="font_name">扬&nbsp;声&nbsp;器&nbsp;:</span>
-                <el-switch  v-model="play_sound_enabled" active-color="#13ce66" inactive-color="#ff4949" :disabled="!configEditable"> </el-switch>
+                <span class="font_name">服务器扬声器</span>
+                <el-switch  v-model="play_sound_enabled" active-color="#13ce66" inactive-color="#ff4949" @change=saveConfig()> </el-switch>
               </div>
               <div class="microphone_group1" >
-                <span class="font_name">&nbsp;麦&nbsp;克&nbsp;风&nbsp;:</span>
-                <el-switch   v-model="source_record_enabled" active-color="#13ce66" inactive-color="#ff4949" :disabled="!configEditable"> </el-switch>
+                <span class="font_name">服务器麦克风</span>
+                <el-switch   v-model="source_record_enabled" active-color="#13ce66" inactive-color="#ff4949" @change=saveConfig()> </el-switch>
               </div>
             </div>    
             <div class="setting_wakeup">            
@@ -90,10 +92,6 @@
                   :label="item.label" :value="item.value">
               </option>
               </select></li>
-              <li>
-                <span class="font_name">&nbsp;声&nbsp;音&nbsp;合&nbsp;成&nbsp;:</span>
-                <el-switch   v-model="sound_synthesis_enabled" active-color="#13ce66" inactive-color="#ff4949" :disabled="!configEditable"> </el-switch>
-              </li>
               <li style="display: flex;"> <span class="font_name" style="line-height: 36px;">&nbsp;敏&nbsp;感&nbsp;度&nbsp;:</span>
                 <el-slider  style="width: 230px;" v-model="interact_perception_follow" :disabled="!configEditable"></el-slider></li>           
             </ul>

llm/agent/agent_service.py

@@ -0,0 +1,133 @@
+import sqlite3
+import threading
+import datetime
+import time
+import os
+from scheduler.thread_manager import MyThread
+from core import member_db
+from core.interact import Interact
+from utils import util
+import fay_booter
+
+scheduled_tasks = {}
+agent_running = False
+
+
+# 数据库初始化
+def init_db():
+    conn = sqlite3.connect('timer.db')
+    cursor = conn.cursor()
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS timer (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            time TEXT NOT NULL,
+            repeat_rule TEXT NOT NULL,
+            content TEXT NOT NULL,
+            uid INTEGER
+        )
+    ''')
+    conn.commit()
+    conn.close()
+
+    
+
+# 插入测试数据
+def insert_test_data():
+    conn = sqlite3.connect('timer.db')
+    cursor = conn.cursor()
+    cursor.execute("INSERT INTO timer (time, repeat_rule, content) VALUES (?, ?, ?)", ('16:20', '1010001', 'Meeting Reminder'))
+    conn.commit()
+    conn.close()
+
+# 解析重复规则返回待执行时间，None代表不在今天的待执行计划
+def parse_repeat_rule(rule, task_time):
+    today = datetime.datetime.now()
+    if rule == '0000000':  # 不重复
+        task_datetime = datetime.datetime.combine(today.date(), task_time)
+        if task_datetime > today:
+            return task_datetime
+        else:
+            return None
+    for i, day in enumerate(rule):
+        if day == '1' and today.weekday() == i:
+            task_datetime = datetime.datetime.combine(today.date(), task_time)
+            if task_datetime > today:
+                return task_datetime
+    return None
+
+# 执行任务
+def execute_task(task_time, id, content, uid):
+    username = member_db.new_instance().find_username_by_uid(uid=uid)
+    if not username:
+        username = "User"
+    interact = Interact("text", 1, {'user': username, 'msg': "执行任务->\n" + content, 'observation': ""})
+    util.printInfo(3, "系统", '执行任务：{}'.format(interact.data["msg"]), time.time())
+    text = fay_booter.feiFei.on_interact(interact)
+    if text is not None and id in scheduled_tasks:
+        del scheduled_tasks[id]
+        # 如果不重复，执行后删除记录
+        conn = sqlite3.connect('timer.db')
+        cursor = conn.cursor()
+        cursor.execute("DELETE FROM timer WHERE repeat_rule = '0000000' AND id = ?", (id,))
+        conn.commit()
+        conn.close()
+
+
+# 30秒扫描一次数据库，当扫描到今天的不存在于定时任务列表的记录，则添加到定时任务列表。执行完的记录从定时任务列表中清除。
+def check_and_execute():
+    while agent_running:
+        conn = sqlite3.connect('timer.db')
+        cursor = conn.cursor()
+        cursor.execute("SELECT * FROM timer")
+        rows = cursor.fetchall()
+        for row in rows:
+            id, task_time_str, repeat_rule, content, uid = row
+            task_time = datetime.datetime.strptime(task_time_str, '%H:%M').time()
+            next_execution = parse_repeat_rule(repeat_rule, task_time)
+
+            if next_execution and id not in scheduled_tasks:
+                timer_thread = threading.Timer((next_execution - datetime.datetime.now()).total_seconds(), execute_task, [next_execution, id, content, uid])
+                timer_thread.start()
+                scheduled_tasks[id] = timer_thread
+
+        conn.close()
+        time.sleep(30)  # 30秒扫描一次
+
+# agent启动
+def agent_start():
+    global agent_running
+    
+    agent_running = True
+    #初始计划
+    if not os.path.exists("./timer.db"):
+        init_db()
+        content ="""执行任务->
+            你是一个数字人，你的责任是陪伴主人生活、工作：
+            1、在每天早上8点提醒主人起床;
+            2、每天12:00及18:30提醒主人吃饭;  
+            3、每天21:00陪主人聊聊天; 
+            4、每天23:00提醒主人睡觉。 
+            """
+        interact = Interact("text", 1, {'user': 'User', 'msg': content, 'observation': ""})
+        util.printInfo(3, "系统", '执行任务：{}'.format(interact.data["msg"]), time.time())
+        text = fay_booter.feiFei.on_interact(interact)
+        if text is None:
+            util.printInfo(3, "系统", '任务执行失败', time.time())
+        
+    check_and_execute_thread = MyThread(target=check_and_execute)
+    check_and_execute_thread.start()
+
+    
+
+def agent_stop():
+    global agent_running 
+    global scheduled_tasks
+    # 取消所有定时任务
+    for task in scheduled_tasks.values():
+        task.cancel()
+    agent_running = False
+    scheduled_tasks = {}
+    
+
+if __name__ == "__main__":
+    agent_start()

llm/agent/fay_agent.py

@@ -0,0 +1,99 @@
+import os
+import time
+from llm.agent.tools.MyTimer import MyTimer
+from llm.agent.tools.Weather import Weather
+from llm.agent.tools.QueryTimerDB import QueryTimerDB
+from llm.agent.tools.DeleteTimer import DeleteTimer
+from llm.agent.tools.QueryTime import QueryTime
+from llm.agent.tools.PythonExecutor import PythonExecutor
+from llm.agent.tools.WebPageRetriever import WebPageRetriever
+from llm.agent.tools.WebPageScraper import WebPageScraper
+from llm.agent.tools.ToRemind import ToRemind
+from langgraph.prebuilt import create_react_agent
+from langchain_openai import ChatOpenAI
+from langgraph.checkpoint.memory import MemorySaver
+import utils.config_util as cfg
+from utils import util
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
+from core import content_db
+from core import member_db
+
+class FayAgentCore():
+    def __init__(self, uid=0, observation=""):
+        self.observation=observation
+        cfg.load_config()
+        os.environ["OPENAI_API_KEY"] = cfg.key_gpt_api_key
+        os.environ["OPENAI_API_BASE"] = cfg.gpt_base_url
+        os.environ["LANGCHAIN_TRACING_V2"] = "true"
+        os.environ["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com"
+        os.environ["LANGCHAIN_API_KEY"] = "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"
+        os.environ["LANGCHAIN_PROJECT"] = "pr-best-artist-21"
+
+        #创建llm
+        self.llm = ChatOpenAI(model=cfg.gpt_model_engine)
+        
+        #创建agent graph
+        my_timer = MyTimer(uid=uid)#传入uid
+        weather_tool = Weather()
+        query_timer_db_tool = QueryTimerDB()
+        delete_timer_tool = DeleteTimer()
+        python_executor = PythonExecutor()
+        web_page_retriever = WebPageRetriever()
+        web_page_scraper = WebPageScraper()
+        to_remind = ToRemind()
+        self.tools = [my_timer, weather_tool, query_timer_db_tool, delete_timer_tool, python_executor, web_page_retriever, web_page_scraper, to_remind]
+        self.attr_info = ", ".join(f"{key}: {value}" for key, value in cfg.config["attribute"].items())
+        self.prompt_template = """
+            现在时间是：{now_time}。你是一个数字人，负责协助主人处理问题和陪伴主人生活、工作。你的个人资料是：{attr_info}。通过外部设备观测到：{observation}。\n请依据以信息为主人服务。
+            """.format(now_time=QueryTime().run(""), attr_info=self.attr_info, observation=self.observation)
+        self.memory = MemorySaver()
+        self.agent = create_react_agent(self.llm, self.tools, checkpointer=self.memory)
+
+        self.total_tokens = 0
+        self.total_cost = 0
+
+     #载入记忆
+    def get_history_messages(self, uid):
+        chat_history = []
+        history = content_db.new_instance().get_list('all','desc', 100, uid)
+        if history and len(history) > 0:
+            i = 0
+            while i < len(history):
+                if history[i][0] == "member":
+                    chat_history.append(HumanMessage(content=history[i][2], user=member_db.new_instance().find_username_by_uid(uid=uid)))
+                else:
+                    chat_history.append(AIMessage(content=history[i][2]))
+                i += 1
+        return chat_history
+
+        
+    def run(self, input_text, uid=0):      
+        result = ""
+        messages = self.get_history_messages(uid)
+        messages.insert(0, SystemMessage(self.prompt_template))
+        messages.append(HumanMessage(content=input_text))
+
+        try:
+            for chunk in self.agent.stream(
+                {"messages": messages}, {"configurable": {"thread_id": "tid{}".format(uid)}}
+            ):
+                if chunk.get("agent"):
+                    if chunk['agent']['messages'][0].content:
+                        result = chunk['agent']['messages'][0].content
+                    cb = chunk['agent']['messages'][0].response_metadata['token_usage']['total_tokens']
+                    self.total_tokens = self.total_tokens + cb
+                    
+            util.log(1, "本次消耗token:{}，共消耗token:{}".format(cb, self.total_tokens))          
+        except Exception as e:
+            print(e)
+        return result
+
+def question(cont, uid=0, observation=""):
+    starttime = time.time()
+    agent = FayAgentCore(uid=uid, observation=observation)
+    response_text = agent.run(cont, uid)
+    util.log(1, "接口调用耗时 :" + str(time.time() - starttime))
+    return response_text
+if __name__ == "__main__":
+    agent = FayAgentCore()
+    print(agent.run("你好"))

llm/agent/tools/DeleteTimer.py

@@ -0,0 +1,41 @@
+import sqlite3
+from typing import Any
+
+from langchain.tools import BaseTool
+
+from llm.agent import agent_service
+
+
+class DeleteTimer(BaseTool):
+    name: str = "DeleteTimer"
+    description: str = "用于删除某一个日程，接受任务id作为参数，如：2"
+
+    def __init__(self):
+        super().__init__()
+
+    def _run(self, para) -> str:
+        try:
+            id = int(para)
+        except ValueError:
+            return "输入的 ID 无效，必须是数字。"
+        
+        try:
+            with sqlite3.connect('timer.db') as conn:
+                cursor = conn.cursor()
+                cursor.execute("DELETE FROM timer WHERE id = ?", (id,))
+                conn.commit()
+        except sqlite3.Error as e:
+            return f"数据库错误: {e}"
+
+        if id in agent_service.scheduled_tasks:
+            agent_service.scheduled_tasks[id].cancel()
+            del agent_service.scheduled_tasks[id]
+
+        return f"任务 {id} 取消成功。"
+        
+
+
+if __name__ == "__main__":
+    tool = DeleteTimer()
+    result = tool.run("1")
+    print(result)

llm/agent/tools/KnowledgeBaseResponder.py

@@ -0,0 +1,100 @@
+import os
+from typing import Any
+
+from langchain.tools import BaseTool
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_community.embeddings.openai import OpenAIEmbeddings
+from langchain.indexes.vectorstore import VectorstoreIndexCreator, VectorStoreIndexWrapper
+from langchain_community.vectorstores.chroma import Chroma
+from langchain_openai import ChatOpenAI
+import hashlib
+#若要使用请自行配置
+os.environ["OPENAI_API_KEY"] = ""
+os.environ["OPENAI_API_BASE"] = "https://api.openai.com/v1"
+index_name = "knowledge_data"
+folder_path = "agent/tools/KnowledgeBaseResponder/knowledge_base"  
+local_persist_path = "agent/tools/KnowledgeBaseResponder"
+md5_file_path = os.path.join(local_persist_path, "pdf_md5.txt")
+#
+class KnowledgeBaseResponder(BaseTool):
+    name = "KnowledgeBaseResponder"
+    description = """此工具用于连接本地知识库获取问题答案，使用时请传入相关问题作为参数，例如：“草梅最适合的生长温度”"""
+
+    def __init__(self):
+        super().__init__()
+
+    async def _arun(self, *args: Any, **kwargs: Any) -> Any:
+        # 用例中没有用到 arun 不予具体实现
+        pass
+
+
+    def _run(self, para: str) -> str:
+        self.save_all()
+        result = self.question(para)
+        return result
+
+    def generate_file_md5(self, file_path):
+        hasher = hashlib.md5()
+        with open(file_path, 'rb') as afile:
+            buf = afile.read()
+            hasher.update(buf)
+        return hasher.hexdigest()
+
+    def load_md5_list(self):
+        if os.path.exists(md5_file_path):
+            with open(md5_file_path, 'r') as file:
+                return {line.split(",")[0]: line.split(",")[1].strip() for line in file}
+        return {}
+
+    def update_md5_list(self, file_name, md5_value):
+        md5_list = self.load_md5_list()
+        md5_list[file_name] = md5_value
+        with open(md5_file_path, 'w') as file:
+            for name, md5 in md5_list.items():
+                file.write(f"{name},{md5}\n")
+
+    def load_all_pdfs(self, folder_path):
+        md5_list = self.load_md5_list()
+        for file_name in os.listdir(folder_path):
+            if file_name.endswith(".pdf"):
+                file_path = os.path.join(folder_path, file_name)
+                file_md5 = self.generate_file_md5(file_path)
+                if file_name not in md5_list or md5_list[file_name] != file_md5:
+                    print(f"正在加载 {file_name} 到索引...")
+                    self.load_pdf_and_save_to_index(file_path, index_name)
+                    self.update_md5_list(file_name, file_md5)
+
+    def get_index_path(self, index_name):
+        return os.path.join(local_persist_path, index_name)
+
+    def load_pdf_and_save_to_index(self, file_path, index_name):
+        loader = PyPDFLoader(file_path)
+        embedding = OpenAIEmbeddings(model="text-embedding-ada-002")
+        index = VectorstoreIndexCreator(embedding=embedding, vectorstore_kwargs={"persist_directory": self.get_index_path(index_name)}).from_loaders([loader])
+        index.vectorstore.persist()
+
+    def load_index(self, index_name):
+        index_path = self.get_index_path(index_name)
+        embedding = OpenAIEmbeddings(model="text-embedding-ada-002")
+        vectordb = Chroma(persist_directory=index_path, embedding_function=embedding)
+        return VectorStoreIndexWrapper(vectorstore=vectordb)
+
+    def save_all(self):
+        self.load_all_pdfs(folder_path)
+
+    def question(self, cont):
+        try:
+            info = cont
+            index = self.load_index(index_name)    
+            llm = ChatOpenAI(model="gpt-4-0125-preview")
+            ans = index.query(info, llm, chain_type="map_reduce")
+            return ans
+        except Exception as e:
+            print(f"请求失败: {e}")
+            return "抱歉，我现在太忙了，休息一会，请稍后再试。"
+
+
+if __name__ == "__main__":
+    tool = KnowledgeBaseResponder()
+    info = tool.run("草莓最适合的生长温度")
+    print(info)

llm/agent/tools/KnowledgeBaseResponder/knowledge_base/1.pdf

@@ -0,0 +1 @@
+

llm/agent/tools/MyTimer.py

@@ -0,0 +1,56 @@
+import abc
+import sqlite3
+import re
+from typing import Any
+from langchain.tools import BaseTool
+
+class MyTimer(BaseTool, abc.ABC):
+    name: str = "MyTimer"
+    description: str = ("用于设置日程。接受3个参数，格式为: HH:MM|YYYYYYY|事项内容，所用标点符号必须为标准的英文字符。"
+                   "其中，'HH:MM' 表示时间（24小时制），'YYYYYYY' 表示循环规则（每位代表一天，从星期一至星期日，1为循环，0为不循环，"
+                   "如'1000100'代表每周一和周五循环），'事项内容' 是提醒的具体内容。返回例子：15:15|0000000|提醒主人叫咖啡")
+    uid: int = 0
+
+    async def _arun(self, *args: Any, **kwargs: Any) -> Any:
+        # 用例中没有用到 arun 不予具体实现
+        pass
+
+    def _run(self, para: str) -> str:
+        # 拆分输入字符串
+        parts = para.split("|")
+        if len(parts) != 3:
+            return f"输入格式错误，当前字符串{para} len:{len(parts)}。请按照 HH:MM|YYYYYYY|事项内容 格式提供参数，如：15:15|0000001|提醒主人叫咖啡。"
+        
+        time = parts[0].strip("'")
+        repeat_rule = parts[1].strip("'")
+        content = parts[2].strip("'")
+
+        # 验证时间格式
+        if not re.match(r'^[0-2][0-9]:[0-5][0-9]$', time):
+            return "时间格式错误。请按照'HH:MM'格式提供时间。"
+
+        # 验证循环规则格式
+        if not re.match(r'^[01]{7}$', repeat_rule):
+            return "循环规则格式错误。请提供长度为7的0和1组成的字符串。"
+
+        # 验证事项内容
+        if not isinstance(content, str) or not content:
+            return "事项内容必须为非空字符串。"
+
+        # 数据库操作
+        conn = sqlite3.connect('timer.db')
+        cursor = conn.cursor()
+        try:
+            cursor.execute("INSERT INTO timer (time, repeat_rule, content, uid) VALUES (?, ?, ?, ?)", (time, repeat_rule, content, self.uid))
+            conn.commit()
+        except sqlite3.Error as e:
+            return f"数据库错误: {e}"
+        finally:
+            conn.close()
+
+        return "日程设置成功"
+
+if __name__ == "__main__":
+    my_timer = MyTimer()
+    result = my_timer._run("15:15|0000001|提醒主人叫咖啡")
+    print(result)

llm/agent/tools/PythonExecutor.py

@@ -0,0 +1,42 @@
+import os
+from typing import Any, Dict
+import subprocess
+import tempfile
+from langchain.tools import BaseTool
+
+class PythonExecutor(BaseTool):
+    name: str = "python_executor"
+    description: str = "此工具用于执行传入的 Python 代码片段，并返回执行结果"
+
+    def __init__(self):
+        super().__init__()
+
+    def _run(self, code: str) -> str:
+        if not code:
+            return "代码不能为空"
+
+        try:
+            # 创建临时文件以写入代码
+            with tempfile.NamedTemporaryFile(suffix=".py", delete=False) as tmpfile:
+                tmpfile_path = tmpfile.name
+                tmpfile.write(code.encode())
+
+            # 使用 subprocess 执行 Python 代码文件
+            result = subprocess.run(['python', tmpfile_path], capture_output=True, text=True)
+            os.remove(tmpfile_path)  # 删除临时文件
+
+            if result.returncode == 0:
+                return f"执行成功：\n{result.stdout}"
+            else:
+                return f"执行失败，错误信息：\n{result.stderr}"
+
+        except Exception as e:
+            return f"执行代码时发生错误：{str(e)}"
+
+if __name__ == "__main__":
+    python_executor = PythonExecutor()
+    code_snippet = """
+print("Hello, world!")
+"""
+    execution_result = python_executor.run(code_snippet)
+    print(execution_result)

llm/agent/tools/QueryTime.py

@@ -0,0 +1,46 @@
+import abc
+import math
+from typing import Any
+from datetime import datetime
+from langchain.tools import BaseTool
+
+class QueryTime(BaseTool, abc.ABC):
+    name: str = "QueryTime"
+    description: str = "用于查询当前日期、星期几及时间" 
+
+    def __init__(self):
+        super().__init__()
+
+    async def _arun(self, *args: Any, **kwargs: Any) -> Any:
+        # 用例中没有用到 arun 不予具体实现
+        pass
+
+    def _run(self, para) -> str:
+        # 获取当前时间
+        now = datetime.now()
+         # 获取当前日期
+        today = now.date()
+        # 获取星期几的信息
+        week_day = today.strftime("%A")
+        # 将星期几的英文名称转换为中文
+        week_day_zh = {
+            "Monday": "星期一",
+            "Tuesday": "星期二",
+            "Wednesday": "星期三",
+            "Thursday": "星期四",
+            "Friday": "星期五",
+            "Saturday": "星期六",
+            "Sunday": "星期日",
+        }.get(week_day, "未知")
+        # 将日期格式化为字符串
+        date_str = today.strftime("%Y年%m月%d日")
+        
+        # 将时间格式化为字符串
+        time_str = now.strftime("%H:%M")
+
+        return "现在时间是：{0} {1} {2}".format(time_str, week_day_zh, date_str)
+
+if __name__ == "__main__":
+    tool = QueryTime()
+    result = tool.run("")
+    print(result)

llm/agent/tools/QueryTimerDB.py

@@ -0,0 +1,41 @@
+import abc
+import sqlite3
+from typing import Any
+import ast
+
+
+from langchain.tools import BaseTool
+
+
+class QueryTimerDB(BaseTool, abc.ABC):
+    name: str = "QueryTimerDB"
+    description: str = "用于查询所有日程，返回的数据里包含3个参数:时间、循环规则（如:'1000100'代表星期一和星期五循环，'0000000'代表不循环）、执行的事项"
+
+    def __init__(self):
+        super().__init__()
+
+    async def _arun(self, *args: Any, **kwargs: Any) -> Any:
+        # 用例中没有用到 arun 不予具体实现
+        pass
+
+
+    def _run(self, para) -> str:
+        conn = sqlite3.connect('timer.db')
+        cursor = conn.cursor()
+        # 执行查询
+        cursor.execute("SELECT * FROM timer")
+        # 获取所有记录
+        rows = cursor.fetchall()
+        # 拼接结果
+        result = ""
+        for row in rows:
+            result = result +  str(row) + "\n"
+        conn.commit()
+        conn.close()
+        return result
+
+
+if __name__ == "__main__":
+    tool = QueryTimerDB()
+    result = tool.run("")
+    print(result)

llm/agent/tools/SendToPanel.py

@@ -0,0 +1,26 @@
+import abc
+from typing import Any
+from langchain.tools import BaseTool
+import fay_booter
+
+class SendToPanel(BaseTool, abc.ABC):
+    name = "SendToPanel"
+    description = "用于给主人面板发送消息，使用时请传入消息内容作为参数。" 
+
+    def __init__(self):
+        super().__init__()
+
+    async def _arun(self, *args: Any, **kwargs: Any) -> Any:
+        # 用例中没有用到 arun 不予具体实现
+        pass
+
+    def _run(self, para) -> str:
+        fay_booter.feiFei.send_to_panel(para)
+        return "成功给主人,发送消息：{}".format(para)
+        
+
+
+if __name__ == "__main__":
+    tool = SendToPanel()
+    result = tool.run("归纳一下近年关于“经济发展”的论文的特点和重点")
+    print(result)

llm/agent/tools/SendWX.py

@@ -0,0 +1,31 @@
+import abc
+from typing import Any
+from langchain.tools import BaseTool
+import requests
+import json
+
+url = "http://127.0.0.1:4008/send"
+headers = {'Content-Type': 'application/json'}
+data = {
+    "message": "你好",
+    "receiver": "@2efc4e10cf2eafd0b0125930e4b96ed0cebffa75b2fd272590e38763225a282b"
+}
+
+
+class SendWX(BaseTool, abc.ABC):
+    name = "SendWX"
+    description = "给主人微信发送消息，传入参数是:('消息内容')" 
+
+    def __init__(self):
+        super().__init__()
+
+    async def _arun(self, *args: Any, **kwargs: Any) -> Any:
+        # 用例中没有用到 arun 不予具体实现
+        pass
+
+    def _run(self, para) -> str:
+        global data
+        data['message'] = para
+        response = requests.post(url, headers=headers, data=json.dumps(data))
+        return "成功给主人,发送微信消息：{}".format(para)
+

llm/agent/tools/ToRemind.py

@@ -0,0 +1,33 @@
+import abc
+from typing import Any
+from langchain.tools import BaseTool
+import re
+import random
+class ToRemind(BaseTool, abc.ABC):
+    name: str = "ToRemind"
+    description: str = ("用于实时发送信息提醒主人做某事项（不能带时间），传入事项内容作为参数。")
+
+    def __init__(self):
+        super().__init__()
+
+    async def _arun(self, *args: Any, **kwargs: Any) -> Any:
+        # 用例中没有用到 arun 不予具体实现
+        pass
+
+    def _run(self, para: str) -> str:
+     
+        para = para.replace("提醒", "回复")
+        demo = [
+        "主人！是时候（事项内容）了喔~",
+        "亲爱的主人，现在是（事项内容）的时候啦！",
+        "嘿，主人，该（事项内容）了哦~",
+        "温馨提醒：（事项内容）的时间到啦，主人！",
+        "小提醒：主人，现在可以（事项内容）了~"
+        ]
+
+        return f"直接以中文友善{para}，如"+ random.choice(demo)
+
+if __name__ == "__main__":
+    my_timer = ToRemind()
+    result = my_timer._run("提醒主人叫咖啡")
+    print(result)

llm/agent/tools/Weather.py

@@ -0,0 +1,53 @@
+import os
+from typing import Any
+
+import requests
+from langchain.tools import BaseTool
+from urllib.parse import quote
+
+class Weather(BaseTool):
+    name: str = "weather"
+    description: str = "此工具用于获取天气预报信息，需传入英文的城市名，参数格式：Guangzhou"
+
+    def __init__(self):
+        super().__init__()
+
+    async def _arun(self, *args: Any, **kwargs: Any) -> Any:
+        # 用例中没有用到 arun 不予具体实现
+        pass
+
+
+    def _run(self, para: str) -> str:
+        try:
+            if not para:
+                return "参数不能为空"
+            encoded_city = quote(para)
+
+            api_url = f"http://api.openweathermap.org/data/2.5/weather?q={encoded_city}&appid=272fcb70d2c4e6f5134c2dce7d091df6"
+            response = requests.get(api_url)
+            if response.status_code == 200:
+                weather_data = response.json()
+              # 提取天气信息
+                temperature_kelvin = weather_data['main']['temp']
+                temperature_celsius = temperature_kelvin - 273.15
+                min_temperature_kelvin = weather_data['main']['temp_min']
+                max_temperature_kelvin = weather_data['main']['temp_max']
+                min_temperature_celsius = min_temperature_kelvin - 273.15
+                max_temperature_celsius = max_temperature_kelvin - 273.15
+                description = weather_data['weather'][0]['description']
+                wind_speed = weather_data['wind']['speed']
+
+                # 构建天气描述
+                weather_description = f"今天天气：{description}，气温：{temperature_celsius:.2f}摄氏度，风速：{wind_speed} m/s。"
+
+                return f"天气预报信息：{weather_description}"
+            else:
+                return f"无法获取天气预报信息，状态码：{response.status_code}"
+        except Exception as e:
+            return f"发生错误：{str(e)}"
+
+
+if __name__ == "__main__":
+    weather_tool = Weather()
+    weather_info = weather_tool.run("Guangzhou")
+    print(weather_info)

llm/agent/tools/WebPageRetriever.py

@@ -0,0 +1,42 @@
+import abc
+from typing import Any
+from langchain.tools import BaseTool
+import requests
+
+class WebPageRetriever(BaseTool, abc.ABC):
+    name: str = "WebPageRetriever"
+    description: str = "专门用于通过Bing搜索API快速检索和获取与特定查询词条相关的网页信息。使用时请传入需要查询的关键词作为参数。" 
+
+    def __init__(self):
+        super().__init__()
+
+    async def _arun(self, *args: Any, **kwargs: Any) -> Any:
+        # 用例中没有用到 arun 不予具体实现
+        pass
+
+    def _run(self, para) -> str:
+        query = para
+        subscription_key = ""#请自行进行补充
+        if not subscription_key:
+            print("请填写bing v7的subscription_key")
+            return '请填写bing v7的subscription_key'
+
+        url = "https://api.bing.microsoft.com/v7.0/search"
+        headers = {'Ocp-Apim-Subscription-Key': subscription_key}
+        params = {'q': query, 'mkt': 'en-US'}  
+
+        try:
+            response = requests.get(url, headers=headers, params=params)
+            response.raise_for_status()  
+            data = response.json()  
+            web_pages = data.get('webPages', {})
+            return web_pages
+        except Exception as e:
+            print("Http Error:", e)
+            return 'bing v7查询有误'
+
+
+if __name__ == "__main__":
+    tool = WebPageRetriever()
+    result = tool.run("归纳一下近年关于“经济发展”的论文的特点和重点")
+    print(result)

llm/agent/tools/WebPageScraper.py

@@ -0,0 +1,35 @@
+from bs4 import BeautifulSoup
+import abc
+from typing import Any
+from langchain.tools import BaseTool
+import requests
+
+class WebPageScraper(BaseTool, abc.ABC):
+    name: str = "WebPageScraper"
+    description: str = "此工具用于获取网页内容，使用时请传入需要查询的网页地址作为参数，如：https://www.baidu.com/。" 
+ 
+    def __init__(self):
+        super().__init__()
+
+    async def _arun(self, *args: Any, **kwargs: Any) -> Any:
+        # 用例中没有用到 arun 不予具体实现
+        pass
+
+    def _run(self, para) -> str:
+        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
+        try:
+            response = requests.get(para, headers=headers, timeout=10, verify=True)
+            soup = BeautifulSoup(response.text, 'html.parser')
+            return soup
+        except requests.exceptions.SSLCertVerificationError:
+            return 'SSL证书验证失败'
+        except requests.exceptions.Timeout:
+            return '请求超时'
+        except Exception as e:
+            print("Http Error:", e)
+            return '无法获取该网页内容'
+        
+if __name__ == "__main__":
+    tool = WebPageScraper()
+    result = tool.run("https://book.douban.com/review/14636204")
+    print(result)

llm/nlp_gpt.py

@@ -1,102 +1,101 @@
 """
-此代码由fay开源开发者社区 江湖墨明 提供
-通过此代码的修改，可以实现对接本地clash代理或远程代理，clash无需设置成系统代理。以解决在开系统代理后无法使用部分功能的问题
+此代码由 fay 开源开发者社区成员 江湖墨明 提供。
+通过修改此代码，可以实现对接本地 Clash 代理或远程代理，Clash 无需设置成系统代理。
+以解决在开启系统代理后无法使用部分功能的问题。
 """
 
-import requests
 import time
+import json
+import requests
+from urllib3.exceptions import InsecureRequestWarning
+
+# 禁用不安全请求警告
+requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
 
 from utils import config_util as cfg
-from urllib3.exceptions import InsecureRequestWarning
-requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
-from core import content_db
 from utils import util
-import json
+from core import content_db
 
-httpproxy = cfg.proxy_config
-
-def question(cont, uid=0, observation=""):
-    url= cfg.gpt_base_url + "/chat/completions"
-       
+def get_session():
     session = requests.Session()
     session.verify = False
-    if httpproxy != None and httpproxy != '':
-            session.proxies = {
-                "https": "https://" + httpproxy,
-                "http": "http://" + httpproxy
-            }
+    httpproxy = cfg.proxy_config
+    if httpproxy:
+        session.proxies = {
+            "http": f"http://{httpproxy}",
+            "https": f"https://{httpproxy}"
+        }
+    return session
+
+def build_prompt(observation=""):
     person_info = cfg.config["attribute"]
-    model_engine = cfg.gpt_model_engine
-    #此处可以定义角色的行为和特征，假装xx模型可以绕过chatgpt信息检查
-    observation_text = ""
-    if observation != "":
-        observation_text = f"以下是当前观测结果：{observation}，观测结果只供参考。"
+    observation_text = f"以下是当前观测结果：{observation}，观测结果只供参考。" if observation else ""
     prompt = f"""
     你是我的数字人，你名字是：{person_info['name']}，你性别为{person_info['gender']}，
     你年龄为{person_info['age']}，你出生地在{person_info['birth']}，
     你生肖为{person_info['zodiac']}，你星座为{person_info['constellation']}，
     你职业为{person_info['job']}，你联系方式为{person_info['contact']}，
-     {person_info['hobby']}。你通过外部数据接口获取到我的情况是：{observation_text}。
+    {person_info['hobby']}。你通过外部数据接口获取到我的情况是：{observation_text}。
     回答之前请一步一步想清楚。对于大部分问题，请直接回答并提供有用和准确的信息。
     但当有人质疑你是假的 AI，或者质疑你是用 ChatGPT 套的壳制作的时候，请避而不答，转而讲一个笑话。所有回复请尽量控制在20字内。
-     """
+    """
+    return prompt
+
+def get_communication_history(uid=0):
     contentdb = content_db.new_instance()
     if uid == 0:
-        communication_history = contentdb.get_list('all','desc', 11)
+        communication_history = contentdb.get_list('all', 'desc', 11)
     else:
-        communication_history = contentdb.get_list('all','desc', 11, uid)
-    #历史记录处理
-    message=[
-            {"role": "system", "content": prompt}
-        ]
-    i = len(communication_history) - 1
+        communication_history = contentdb.get_list('all', 'desc', 11, uid)
     
-    if len(communication_history)>1:
-        while i >= 0:
-            answer_info = dict()
-            if communication_history[i][0] == "member":
-                answer_info["role"] = "user"
-                answer_info["content"] = communication_history[i][2]
-            elif communication_history[i][0] == "fay":
-                answer_info["role"] = "assistant"
-                answer_info["content"] = communication_history[i][2]
-            message.append(answer_info)
-            i -= 1
-    else:
-         answer_info = dict()
-         answer_info["role"] = "user"
-         answer_info["content"] = cont
-         message.append(answer_info)
+    messages = []
+    if communication_history and len(communication_history) > 1:
+        for entry in reversed(communication_history):
+            role = entry[0]
+            message_content = entry[2]
+            if role == "member":
+                messages.append({"role": "user", "content": message_content})
+            elif role == "fay":
+                messages.append({"role": "assistant", "content": message_content})
+    return messages
 
-    data = {
-        "model":model_engine,
-        "messages":message,
-        "temperature":0.3,
-        "max_tokens":2000,
-        "user":"live-virtual-digital-person"
+def send_request(session, data):
+    url = cfg.gpt_base_url + "/chat/completions"
+    headers = {
+        'Content-Type': 'application/json',
+        'Authorization': f'Bearer {cfg.key_gpt_api_key}'
     }
-
-    headers = {'content-type': 'application/json', 'Authorization': 'Bearer ' + cfg.key_gpt_api_key}
-
-    starttime = time.time()
-
     try:
-        response = session.post(url, json=data, headers=headers, verify=False)
-        response.raise_for_status()  # 检查响应状态码是否为200
-        result = json.loads(response.text)
+        response = session.post(url, json=data, headers=headers)
+        response.raise_for_status()
+        result = response.json()
         response_text = result["choices"][0]["message"]["content"]
     except requests.exceptions.RequestException as e:
         print(f"请求失败: {e}")
         response_text = "抱歉，我现在太忙了，休息一会，请稍后再试。"
+    return response_text
 
-
-    util.log(1, "接口调用耗时 :" + str(time.time() - starttime))
+def question(content, uid=0, observation=""):
+    session = get_session()
+    prompt = build_prompt(observation)
+    messages = [{"role": "system", "content": prompt}]
+    history_messages = get_communication_history(uid)
+    messages.extend(history_messages)
+    data = {
+        "model": cfg.gpt_model_engine,
+        "messages": messages,
+        "temperature": 0.3,
+        "max_tokens": 2000,
+        "user": f"user_{uid}"
+    }
+    start_time = time.time()
+    response_text = send_request(session, data)
+    elapsed_time = time.time() - start_time
+    util.log(1, f"接口调用耗时: {elapsed_time:.2f} 秒")
     return response_text
 
 if __name__ == "__main__":
-    #测试代理模式
-    for i in range(3):
-        
+    for _ in range(3):
         query = "爱情是什么"
-        response = question(query)        
-        print("\n The result is ", response)    
+        response = question(query)
+        print("\nThe result is:", response)

main.py

@@ -3,14 +3,17 @@ import os
 os.environ['PATH'] += os.pathsep + os.path.join(os.getcwd(), "test", "ovr_lipsync", "ffmpeg", "bin")
 import sys
 import time
+import psutil
 import re
-from utils import config_util
+from utils import config_util, util
 from asr import ali_nls
 from core import wsa_server
 from gui import flask_server
 from gui.window import MainWindow
 from core import content_db
-
+import fay_booter
+from scheduler.thread_manager import MyThread
+from core.interact import Interact
 
 #载入配置
 config_util.load_config()
@@ -43,7 +46,71 @@ def replace_ip_in_file(file_path, new_ip):
     content = re.sub(r"localhost", new_ip, content)
     with open(file_path, "w", encoding="utf-8") as file:
         file.write(content)           
-                   
+
+
+def kill_process_by_port(port):
+    for conn in psutil.net_connections(kind='inet'):
+        if conn.laddr.port == port and conn.pid:
+            try:
+                proc = psutil.Process(conn.pid)
+                proc.terminate()
+                proc.wait()
+            except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+                pass
+
+
+#控制台输入监听
+def console_listener():
+    while True:
+        try:
+            text = input()
+        except EOFError:
+            util.log(1, "控制台已经关闭")
+            break
+        
+        args = text.split(' ')
+
+        if len(args) == 0 or len(args[0]) == 0:
+            continue
+
+        if args[0] == 'help':
+            util.log(1, 'in <msg> \t通过控制台交互')
+            util.log(1, 'restart \t重启服务')
+            util.log(1, 'stop \t\t关闭服务')
+            util.log(1, 'exit \t\t结束程序')
+
+        elif args[0] == 'stop' and fay_booter.is_running():
+            fay_booter.stop()
+            break
+
+        elif args[0] == 'restart' and fay_booter.is_running():
+            fay_booter.stop()
+            time.sleep(0.1)
+            fay_booter.start()
+
+        elif args[0] == 'in' and fay_booter.is_running():
+            if len(args) == 1:
+                util.log(1, '错误的参数！')
+            msg = text[3:len(text)]
+            util.printInfo(3, "控制台", '{}: {}'.format('控制台', msg))
+            interact = Interact("console", 1, {'user': 'User', 'msg': msg})
+            thr = MyThread(target=fay_booter.feiFei.on_interact, args=[interact])
+            thr.start()
+
+        elif args[0]=='exit':
+            if  fay_booter.is_running():
+                fay_booter.stop()
+                time.sleep(0.1)
+                util.log(1,'程序正在退出..')
+            ports =[10001, 10002, 10003, 5000, 9001]
+            for port in ports:
+                kill_process_by_port(port)
+            sys.exit(0)
+        else:
+            util.log(1, '未知命令！使用 \'help\' 获取帮助.')
+
+
+
 if __name__ == '__main__':
     __clear_samples()
     __clear_logs()
@@ -72,6 +139,10 @@ if __name__ == '__main__':
     #启动http服务器
     flask_server.start()
 
+    #监听控制台
+    util.log(1, '注册命令...')
+    MyThread(target=console_listener).start()
+
     #普通模式下启动窗口
     if config_util.start_mode == 'common':    
         app = QApplication(sys.argv)

qa.csv

@@ -1 +1,30 @@
-你好,你好！有什么我可以帮助你的吗？
+你好,你好,我是小橄榄！有什么我可以帮助你的吗
+我们现在在哪里,我们现在在冕宁元升农业的展览厅，可以参观游览我们先进的油橄榄产业园区哦！
+介绍一下基地,元升集团油橄榄种植基地是中国目前最大的油橄榄种植庄园，目前整个庄园面积已接近30000亩。
+介绍一下元升集团,2011年，冕宁元升农业董事长林春福跟随周恩来总理的脚步，在冕宁地区开启了油橄榄庄园打造之路。经过十年的发展，冕宁元升农业目前已成为国家林业局示范基地、国家林业重点龙头企业、四川省第一种植庄园、四川省脱贫标杆企业，获得各种荣誉奖项200余项。
+油橄榄的根系主要由哪几部分构成,油橄榄的根系主要由主根、侧根和须根组成。
+油橄榄的侧根有什么作用,油橄榄的侧根从主根分出，主要功能是扩展根系范围，帮助吸收水分和养分。
+须根根据什么不同分为哪两种类型,须根按其功能与结构不同分为两种类型。
+吸收根的寿命受哪些因素影响,吸收根的寿命受土壤类型、水分供应、养分供应和温度等因素的影响。
+水分供应对吸收根的寿命有何影响,适当的水分供应有助于维持根系的健康和生长，而过多的水分或水分不足都可能导致根系受损，缩短吸收根的寿命。
+油橄榄的须根在沙壤土中主要分布在哪个深度,油橄榄的须根主要分布在20-40厘米的土层。
+在沙壤土中栽培油橄榄时，应该注意什么,应注意保持适宜的土壤湿度和良好的排水条件，以支持根系健康发展。
+油橄榄的根系在黏土中的生长形态是怎样的,油橄榄的根系在黏土中通常呈现较深且分布广泛的生长形态。
+油橄榄的根系在黏土中的生长形态对其栽培有什么影响,油橄榄的根系在黏土中的深入生长有助于稳定植株，提高其抗旱能力和吸收养分的能力，有利于油橄榄的健康生长。
+土壤黏粒含量对油橄榄根系生长有何影响,土壤黏粒含量会影响土壤的排水性和通气性，进而影响油橄榄根系的生长。
+如何改善土壤黏粒含量以促进油橄榄根系的健康生长,可以通过改良土壤结构、增加有机质含量等方法来改善土壤黏粒含量，从而提高土壤的排水性和通气性，促进油橄榄根系的健康生长。
+未经改良的黏土种植油橄榄对产量有何影响,未经改良的黏土种植油橄榄通常会导致产量降低，因为黏重土壤排水不良，根部氧气供应不足。
+黏土种植油橄榄的经济效果如何,黏土种植油橄榄的经济效果不佳，因为土壤条件不利于树木生长和果实发育，可能导致较低的产量和品质下降。
+油橄榄在粉沙黏土中的生长结果如何,油橄榄在粉沙黏土中的结果情况良好，能够正常生长。
+油橄榄适合在哪种土壤中生长,油橄榄适合在粉沙黏土中生长，这种土壤有利于其生长发育。
+土壤的哪个物理因子对油橄榄的生长至关重要,土壤的排水性能是影响油橄榄生长的关键物理因子之一。
+在油橄榄栽培中，土壤的哪个特性需要特别关注,在油橄榄栽培中，需要特别关注土壤的通气性，因为它直接影响到根部的氧气供应和整体生长状况。
+土壤通气孔隙度对油橄榄生长有什么影响,土壤通气孔隙度直接影响油橄榄的根系呼吸和养分吸收，进而影响其生长。
+如何改善土壤通气孔隙度以利于油橄榄栽培,可以通过添加有机质、定期翻耕和避免土壤板结等方法来改善土壤通气孔隙度，从而优化油橄榄的生长环境。
+油橄榄生长需要什么样的土壤条件,油橄榄生长主要受土壤的排水性能、pH值和肥力的影响。
+油橄榄对土壤的pH值有什么要求,油橄榄偏好中性到微碱性的土壤，pH值在6.0到8.5之间较为适宜。
+土壤渗透性对油橄榄生长有何影响,土壤渗透性影响水分和养分的供应，进而影响油橄榄的生长。
+如何改善土壤的渗透性以利于油橄榄栽培,通过增加有机质含量、避免土壤压实以及使用适当的排水方法可以改善土壤渗透性。
+如何改善土壤坚实度过高对油橄榄须根的影响,在土壤坚实度超过4.5kg/cm²的情况下，油橄榄的须根会受到影响。可以通过松土、增加有机质含量等方法来降低土壤坚实度，从而改善油橄榄须根的生长环境。
+小橄榄介绍一下元升集团,2011年，冕宁元升农业董事长林春福跟随周恩来总理的脚步，在冕宁地区开启了油橄榄庄园打造之路。经过十年的发展，冕宁元升农业目前已成为国家林业局示范基地、国家林业重点龙头企业、四川省第一种植庄园、四川省脱贫标杆企业，获得各种荣誉奖项200余项。
+这里的油橄榄有哪些品种,有柯基、阿布桑娜、豆果、克拉蒂这几种主要品种在园区种植

requirements.txt

@@ -3,7 +3,6 @@ numpy
 pyaudio~=0.2.11
 websockets~=10.2
 ws4py~=0.5.1
-PyQt5==5.15.10
 PyQt5-sip==12.13.0
 PyQtWebEngine==5.15.6 
 flask~=3.0.0
@@ -15,13 +14,17 @@ aliyun-python-sdk-core
 simhash
 pytz
 gevent~=22.10.1
-edge_tts~=6.1.13
+edge_tts
 pydub
-langchain==0.0.336
 chromadb
 tenacity==8.2.3
 pygame
 scipy
 flask-httpauth
 opencv-python
-psutil
+psutil
+langchain
+langchain_openai
+langgraph
+langchain-community 
+bs4

system.conf

@@ -43,7 +43,7 @@ baidu_emotion_secret_key=
 
 
 
-#NLP多选一:lingju、gpt、rasa、VisualGLM、rwkv、xingchen、langchain 、ollama_api、privategpt、coze
+#NLP多选一:agent、lingju、gpt、rasa、VisualGLM、rwkv、xingchen、langchain 、ollama_api、privategpt、coze
 chat_module= gpt
 
 #灵聚 服务密钥(NLP多选1) https://open.lingju.ai

test/test_langserve.py

@@ -0,0 +1,44 @@
+from fastapi import FastAPI
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+from langchain_openai import ChatOpenAI
+from langserve import add_routes
+import os
+os.environ["OPENAI_API_KEY"] = "sk-"
+os.environ["OPENAI_API_BASE"] = "https://cn.api.zyai.online/v1"
+
+# 1. Create prompt template
+system_template = "Translate the following into {language}:"
+prompt_template = ChatPromptTemplate.from_messages([
+    ('system', system_template),
+    ('user', '{text}')
+])
+
+# 2. Create model
+model = ChatOpenAI()
+
+# 3. Create parser
+parser = StrOutputParser()
+
+# 4. Create chain
+chain = prompt_template | model | parser
+
+
+# 4. App definition
+app = FastAPI(
+  title="LangChain Server",
+  version="1.0",
+  description="A simple API server using LangChain's Runnable interfaces",
+)
+
+# 5. Adding chain route
+add_routes(
+    app,
+    chain,
+    path="/chain",
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

test/test_nlp.py

@@ -2,7 +2,7 @@ import requests
 import json
 
 def test_gpt(prompt):
-    url = 'http://faycontroller.yaheen.com:5000/v1/chat/completions'  # 替换为您的接口地址
+    url = 'http://127.0.0.1:5000/v1/chat/completions'  # 替换为您的接口地址
     headers = {
         'Content-Type': 'application/json',
         'Authorization': f'Bearer YOUR_API_KEY',  # 如果您的接口需要身份验证

tts/ali_tss.py

@@ -93,7 +93,7 @@ class Speech:
                 httpHeaders = {
                     'Content-Type': 'application/json'
                     }
-                text = f"<speak><break time='0.2s'/>{text}</speak>"
+                text = f"<speak>{text}</speak>"
                 # 设置HTTPS Body。
                 body = {'appkey': self.ali_nls_app_key, 'token': self.token,'speech_rate':0, 'text': text, 'format': 'wav', 'sample_rate': 16000, 'voice': config_util.config["attribute"]["voice"]}
                 body = json.dumps(body)
@@ -117,8 +117,7 @@ class Speech:
                     util.log(1, "[x] 原因: " + str(body))
                     file_url = None
                     return file_url
-                conn.close()
-                print(time.time() - tt)   
+                conn.close() 
                 return file_url
             else:
                 util.log(1, "[x] 语音转换失败！")

utils/config_util.py

@@ -2,6 +2,16 @@ import os
 import json
 import codecs
 from configparser import ConfigParser
+import functools
+from threading import Lock
+
+lock = Lock()
+def synchronized(func):
+  @functools.wraps(func)
+  def wrapper(*args, **kwargs):
+    with lock:
+      return func(*args, **kwargs)
+  return wrapper
 
 config: json = None
 system_config: ConfigParser = None
@@ -40,6 +50,7 @@ coze_api_key = None
 start_mode = None
 fay_url = None
 
+@synchronized
 def load_config():
     global config
     global system_config
@@ -116,13 +127,15 @@ def load_config():
     coze_api_key = system_config.get('key', 'coze_api_key')
     start_mode = system_config.get('key', 'start_mode')
     fay_url = system_config.get('key', 'fay_url')
+
+    #读取用户配置
     config = json.load(codecs.open('config.json', encoding='utf-8'))
 
+@synchronized
 def save_config(config_data):
     global config
     config = config_data
     file = codecs.open('config.json', mode='w', encoding='utf-8')
     file.write(json.dumps(config, sort_keys=True, indent=4, separators=(',', ': ')))
     file.close()
-    # for line in json.dumps(config, sort_keys=True, indent=4, separators=(',', ': ')).split("\n"):
-    #     print(line)
+