diff --git a/README.md b/README.md index 1421a2b..8829256 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ Fay数字人助理版是fay开源项目的重要分支，专注于构建智能 ## **推荐玩法** + + +灵聚NLP api(支持GPT3.5及多应用)：b站"Fay数字人集成灵聚NLP api(支持GPT3.5及多应用)" + 集成本地唇型算法：https://www.bilibili.com/video/BV1Zh4y1g7o7/?buvid=XXDD0B5DD6C43C070DF9E7E67930FC48B24DF&is_story_h5=false&mid=Pvwl%2Ft1ahPM726k1L4%2FnRA%3D%3D&plat_id=202&share_from=ugc&share_medium=android&share_plat=android&share_source=WEIXIN&share_tag=s_i×tamp=1686926382&unique_k=Jdqazy3&up_id=2111554564 给数字人加上眼睛（集成yolo+VisualGLM)：B站视频 @@ -85,7 +89,7 @@ Fay（服务端）与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127 ## **二、Fay控制器核心逻辑** -![](images/luoji.png) +![](images/luoji.jpg) **注：** @@ -103,9 +107,9 @@ Fay（服务端）与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127 ├── ai_module │   ├── ali_nls.py # 阿里云 实时语音 │   ├── ms_tts_sdk.py # 微软 文本转语音 +│   ├── nlp_lingju.py # 灵聚 人机交互-自然语言处理 │   ├── xf_aiui.py # 讯飞 人机交互-自然语言处理 │   ├── chatgpt.py # gpt3.5对接 -│   ├── nlp_gpt.py # 对接chat.openai.com(免key) │   ├── yuan_1_0.py # 浪潮.源大模型对接 │   ├── nlp_rasa.py # ChatGLM-6B的基础上前置Rasa会话管理(强烈推荐) │   ├── nlp_VisualGLM.py # 对接多模态大语言模型VisualGLM-6B @@ -116,6 +120,10 @@ Fay（服务端）与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127 │   ├── fay_core.py # 数字人核心模块 │   ├── recorder.py # 录音器 │   ├── tts_voice.py # 语音生源枚举 +│   ├── authorize_tb.py # fay.db认证表管理 +│   ├── content_db.py # fay.db内容表管理 +│   ├── interact.py # 互动（消息）对象 +│   ├── song_player.py # 音乐播放（暂不可用） │   └── wsa_server.py # WebSocket 服务端 ├── gui # 图形界面 │   ├── flask_server.py # Flask 服务端 @@ -134,6 +142,11 @@ Fay（服务端）与数字人的通讯接口: [`ws://127.0.0.1:10002`](ws://127 ## **三、升级日志** +**2023.06.21：** + ++ 集成灵聚NLP api(支持GPT3.5及多应用)； ++ ui修正。 + **2023.06.17：** + 集成本地唇型算法。 @@ -195,9 +208,9 @@ python main.py | ./ai_module/ms_tts_sdk.py | 微软 文本转情绪语音（非必须，不配置时使用免费的edge-tts） | https://azure.microsoft.com/zh-cn/services/cognitive-services/text-to-speech/ | | ./ai_module/xf_ltp.py | 讯飞 情感分析 | https://www.xfyun.cn/service/emotion-analysis | | ./utils/ngrok_util.py | ngrok.cc 外网穿透（可选） | http://ngrok.cc | +| ./ai_module/nlp_lingju.py | 灵聚NLP api(支持GPT3.5及多应用)（NLP多选1） | https://open.lingju.ai 需联系客服务开通gpt3.5权限| | ./ai_module/yuan_1_0.py | 浪潮源大模型（NLP 多选1） | https://air.inspur.com/ | | ./ai_module/chatgpt.py | ChatGPT（NLP多选1） | ******* | -| ./ai_module/xf_aiui.py | 讯飞自然语言处理（NLP多选1） | https://aiui.xfyun.cn/solution/webapi | | ./ai_module/nlp_rasa.py | ChatGLM-6B的基础上前置Rasa会话管理（NLP 多选1） | https://m.bilibili.com/video/BV1D14y1f7pr | | ./ai_module/nlp_VisualGLM.py | 对接VisualGLM-6B多模态单机离线大语言模型（NLP 多选1） | B站视频 | diff --git a/ai_module/nlp_lingju.py b/ai_module/nlp_lingju.py new file mode 100644 index 0000000..2968f27 --- /dev/null +++ b/ai_module/nlp_lingju.py @@ -0,0 +1,100 @@ +import json +import requests +import uuid +from datetime import datetime, timedelta +import time +from utils import util +from utils import config_util as cfg +from core.authorize_tb import Authorize_Tb + +def question(cont): + lingju = Lingju() + answer = lingju.question(cont) + return answer + +class Lingju: + + def __init__(self): + self.userid = str(uuid.getnode()) + self.authorize_tb = Authorize_Tb() + + def question(self, cont): + token = self.__check_token() + if token is None or token == 'expired': + token_info = self.__get_token() + if token_info is not None: + #转换过期时间 + updated_in_seconds = time.time() + updated_datetime = datetime.fromtimestamp(updated_in_seconds) + expires_timedelta = timedelta(days=token_info['data']['expires']) + expiry_datetime = updated_datetime + expires_timedelta + expiry_timestamp_in_seconds = expiry_datetime.timestamp() + expiry_timestamp_in_milliseconds = int(expiry_timestamp_in_seconds) * 1000 + token = token_info['data']['accessToken'] + if token == 'expired': + self.authorize_tb.update_by_userid(self.userid, token_info['data']['accessToken'], expiry_timestamp_in_milliseconds) + else: + self.authorize_tb.add(self.userid, token_info['data']['accessToken'], expiry_timestamp_in_milliseconds) + else: + token = None + + if token is not None: + try: + lat,lng,city = self.__get_location() + url="https://dev.lingju.ai/httpapi/ljchat.do" + req = json.dumps({"accessToken": token, "lat": lat, "lng": lng, "input": cont}) + headers = {'Content-Type':'application/json;charset=UTF-8'} + r = requests.post(url, headers=headers, data=req) + if r.status_code != 200: + util.log(1, f"灵聚api对接有误: {r.text}") + return "哎呀，出错了！请重新发一下" + info = json.loads(r.text) + if info['status'] != 0: + return info['description'] + else: + answer = json.loads(info['answer']) + return answer['rtext'] + except Exception as e: + util.log(1, f"灵聚api对接有误： {str(e)}") + return "哎呀，出错了！请重新发一下" + + def __check_token(self): + self.authorize_tb.init_tb() + info = self.authorize_tb.find_by_userid(self.userid) + if info is not None: + if info[1] >= int(time.time())*1000: + return info[0] + else: + return 'expired' + else: + return None + + def __get_token(self): + try: + cfg.load_config() + url=f"https://dev.lingju.ai/httpapi/authorize.do?appkey={cfg.key_lingju_api_key}&userid={self.userid}&authcode={cfg.key_lingju_api_authcode}" + headers = {'Content-Type':'application/json;charset=UTF-8'} + r = requests.post(url, headers=headers) + if r.status_code != 200: + util.log(1, f"灵聚api对接有误: {r.text}") + return None + info = json.loads(r.text) + if info['status'] != 0: + util.log(1, f"灵聚api对接有误：{info['description']}") + return None + else: + return info + except Exception as e: + util.log(1, f"灵聚api对接有误： {str(e)}") + return None + + def __get_location(self): + try: + response = requests.get('http://ip-api.com/json/') + data = response.json() + return data['lat'], data['lon'], data['city'] + except requests.exceptions.RequestException as e: + util.log(1, f"获取位置失败: {str(e)}") + return 0, 0, "北京" + + diff --git a/core/authorize_tb.py b/core/authorize_tb.py new file mode 100644 index 0000000..5af0afc --- /dev/null +++ b/core/authorize_tb.py @@ -0,0 +1,66 @@ +import sqlite3 +import time +import threading +import functools +def synchronized(func): + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + with self.lock: + return func(self, *args, **kwargs) + return wrapper +class Authorize_Tb: + + def __init__(self) -> None: + self.lock = threading.Lock() + + + + #初始化 + def init_tb(self): + conn = sqlite3.connect('fay.db') + c = conn.cursor() + try: + c.execute('SELECT * FROM T_Authorize') + except sqlite3.OperationalError: + c.execute(''' + CREATE TABLE T_Authorize + (id INTEGER PRIMARY KEY autoincrement, + userid char(100), + accesstoken TEXT, + expirestime BigInt, + createtime Int); + ''') + conn.commit() + finally: + conn.close() + + #添加 + @synchronized + def add(self,userid,accesstoken,expirestime): + conn = sqlite3.connect("fay.db") + cur = conn.cursor() + cur.execute("insert into T_Authorize (userid,accesstoken,expirestime,createtime) values (?,?,?,?)",(userid,accesstoken,expirestime,int(time.time()))) + + conn.commit() + conn.close() + return cur.lastrowid + + #查询 + @synchronized + def find_by_userid(self,userid): + conn = sqlite3.connect("fay.db") + cur = conn.cursor() + cur.execute("select accesstoken,expirestime from T_Authorize where userid = ? order by id desc limit 1",(userid,)) + info = cur.fetchone() + conn.close() + return info + + # 更新token + @synchronized + def update_by_userid(self, userid, new_accesstoken, new_expirestime): + conn = sqlite3.connect("fay.db") + cur = conn.cursor() + cur.execute("UPDATE T_Authorize SET accesstoken = ?, expirestime = ? WHERE userid = ?", + (new_accesstoken, new_expirestime, userid)) + conn.commit() + conn.close() \ No newline at end of file diff --git a/core/fay_core.py b/core/fay_core.py index f8e804f..ebcced5 100644 --- a/core/fay_core.py +++ b/core/fay_core.py @@ -14,7 +14,6 @@ from openpyxl import load_workbook import numpy as np # import tensorflow as tf import fay_booter -from ai_module import xf_aiui from ai_module import xf_ltp from ai_module.ms_tts_sdk import Speech from core import wsa_server, tts_voice, song_player @@ -38,6 +37,7 @@ if platform.system() == "Windows": import sys sys.path.append("test/ovr_lipsync") from test_olipsync import LipSyncGenerator +from ai_module import nlp_lingju #文本消息处理 def send_for_answer(msg,sendto): @@ -52,9 +52,8 @@ def send_for_answer(msg,sendto): if sendto == 2: text = nlp_gpt.question(msg) else: - if cfg.key_chat_module == 'xfaiui': - text = xf_aiui.question(msg) - elif cfg.key_chat_module == 'yuan': + + if cfg.key_chat_module == 'yuan': text = yuan_1_0.question(msg) elif cfg.key_chat_module == 'chatgpt': text = chatgpt.question(msg) @@ -63,10 +62,12 @@ def send_for_answer(msg,sendto): text = textlist[0]['text'] elif cfg.key_chat_module == "VisualGLM": text = VisualGLM.question(msg) + elif cfg.key_chat_module == "lingju": + text = nlp_lingju.question(msg) else: - raise RuntimeError('讯飞key、yuan key、chatgpt key都没有配置！') + raise RuntimeError('灵聚key、yuan key、gpt key都没有配置！') util.log(1, '自然语言处理完成. 耗时: {} ms'.format(math.floor((time.time() - tm) * 1000))) if text == '哎呀，你这么说我也不懂，详细点呗' or text == '': util.log(1, '[!] 自然语言无语了！') @@ -126,16 +127,6 @@ class FeiFei: [['联系方式', '联系你们', '怎么联系客服', '有没有客服'], 'contact'] ] - # 商品提问关键字 - self.explain_keyword = [ - [['是什么'], 'intro'], - [['怎么用', '使用场景', '有什么作用'], 'usage'], - [['怎么卖', '多少钱', '售价'], 'price'], - [['便宜点', '优惠', '折扣', '促销'], 'discount'], - [['质量', '保证', '担保'], 'promise'], - [['特点', '优点'], 'character'], - ] - self.wsParam = None self.wss = None self.sp = Speech() @@ -229,24 +220,6 @@ class FeiFei: answer = self.__get_keyword(self.__read_qna(config_util.config['interact']['QnA']), text) if answer is not None: return answer - - - def __get_list_answer(self, answers, text): - last_similar = 0 - last_answer = '' - for mlist in answers: - for quest in mlist[0]: - similar = self.__string_similar(text, quest) - if quest in text: - similar += 0.3 - if similar > last_similar: - last_similar = similar - answer_list = mlist[1] - last_answer = answer_list[random.randint(0, len(answer_list) - 1)] - # print("相似度: {}, 回答: {}".format(last_similar, last_answer)) - if last_similar >= 0.6: - return last_answer - return None def __auto_speak(self): while self.__running: @@ -285,9 +258,7 @@ class FeiFei: util.log(1, '自然语言处理...') tm = time.time() cfg.load_config() - if cfg.key_chat_module == 'xfaiui': - text = xf_aiui.question(self.q_msg) - elif cfg.key_chat_module == 'yuan': + if cfg.key_chat_module == 'yuan': text = yuan_1_0.question(self.q_msg) elif cfg.key_chat_module == 'chatgpt': text = chatgpt.question(self.q_msg) @@ -296,8 +267,10 @@ class FeiFei: text = textlist[0]['text'] elif cfg.key_chat_module == "VisualGLM": text = VisualGLM.question(self.q_msg) + elif cfg.key_chat_module == "lingju": + text = nlp_lingju.question(self.q_msg) else: - raise RuntimeError('讯飞key、yuan key、chatgpt key都没有配置！') + raise RuntimeError('灵聚key、yuan key、gpt key都没有配置！') util.log(1, '自然语言处理完成. 耗时: {} ms'.format(math.floor((time.time() - tm) * 1000))) if text == '哎呀，你这么说我也不懂，详细点呗' or text == '': util.log(1, '[!] 自然语言无语了！') @@ -435,6 +408,7 @@ class FeiFei: audio_length = eyed3.load(file_url).info.time_secs #mp3音频长度 # with wave.open(file_url, 'rb') as wav_file: #wav音频长度 # audio_length = wav_file.getnframes() / float(wav_file.getframerate()) + # print(audio_length) # if audio_length <= config_util.config["interact"]["maxInteractTime"] or say_type == "script": if config_util.config["interact"]["playSound"]: # 展板播放 self.__play_sound(file_url) diff --git a/gui/static/css/index.css b/gui/static/css/index.css index 4e414be..1cc0f39 100644 --- a/gui/static/css/index.css +++ b/gui/static/css/index.css @@ -1,6 +1,5 @@ #app { width: 1920px; - height: 1080px; margin: 0; padding: 0; } @@ -11,7 +10,6 @@ ul { .main { width: 1920px; - height: 1080px; display: flex; flex-direction: column; /* flex-wrap: wrap; */ @@ -259,4 +257,164 @@ ul { border-color: #E4E7ED; color: #000206 !important; cursor: not-allowed; +} +.container { + height: 902px; + width: 913px; + border-radius: 4px; + border: 0.5px solid #e0e0e0; + background-color: #f5f5f5; + display: flex; + flex-flow: column; + overflow: hidden; +} + +.content { + width: calc(100% - 20px); + padding: 20px; + overflow-y: scroll; + flex: 1; + white-space: pre-wrap; +} + +.content:hover::-webkit-scrollbar-thumb { + background: rgba(0, 0, 0, 0.1); +} + +.bubble { + /* max-width: 400px; */ + max-width: 60%; + padding: 10px; + border-radius: 5px; + position: relative; + color: #000; + word-wrap: break-word; + word-break: normal; +} + +.item-left .bubble { + margin-left: 15px; + background-color: #fff; +} + +.item-left .bubble:before { + content: ""; + position: absolute; + width: 0; + height: 0; + border-left: 10px solid transparent; + border-top: 10px solid transparent; + border-right: 10px solid #fff; + border-bottom: 10px solid transparent; + left: -20px; +} + +.item-right .bubble { + margin-right: 15px; + background-color: #63f5a1; +} + +.item-right .bubble:before { + content: ""; + position: absolute; + width: 0; + height: 0; + border-left: 10px solid #63f5a1; + border-top: 10px solid transparent; + border-right: 10px solid transparent; + border-bottom: 10px solid transparent; + right: -20px; +} + +.item { + margin-top: 15px; + display: flex; + width: 100%; +} + +.item.item-right { + justify-content: flex-end; +} + +.item.item-center { + justify-content: center; +} + +.item.item-center span { + font-size: 12px; + padding: 2px 4px; + color: #fff; + background-color: #dadada; + border-radius: 3px; + -moz-user-select: none; + /*火狐*/ + -webkit-user-select: none; + /*webkit浏览器*/ + -ms-user-select: none; + /*IE10*/ + -khtml-user-select: none; + /*早期浏览器*/ + user-select: none; +} + +.avatar img { + width: 42px; + height: 42px; + border-radius: 50%; +} + +.input-area { + border-top: 0.5px solid #e0e0e0; + height: 150px; + display: flex; + flex-flow: column; + background-color: #fff; +} + +textarea { + flex: 1; + padding: 5px; + font-size: 14px; + border: none; + cursor: pointer; + overflow-y: auto; + overflow-x: hidden; + outline: none; + resize: none; +} + +.button-area { + display: flex; + height: 40px; + margin-right: 10px; + line-height: 40px; + padding: 5px; + justify-content: flex-end; +} + +.button-area button { + width: 80px; + border: none; + outline: none; + border-radius: 4px; + float: right; + cursor: pointer; +} + +/* 设置滚动条的样式 */ +::-webkit-scrollbar { + width: 10px; +} + +/* 滚动槽 */ +::-webkit-scrollbar-track { + -webkit-box-shadow: inset006pxrgba(0, 0, 0, 0.3); + border-radius: 8px; +} + +/* 滚动条滑块 */ +::-webkit-scrollbar-thumb { + border-radius: 10px; + background: rgba(0, 0, 0, 0); + -webkit-box-shadow: inset006pxrgba(0, 0, 0, 0.5); } \ No newline at end of file diff --git a/gui/templates/index.html b/gui/templates/index.html index ae98e2f..19dc9cb 100644 --- a/gui/templates/index.html +++ b/gui/templates/index.html @@ -15,171 +15,10 @@ + Fay - @@ -226,19 +65,30 @@

职业：

-
  • -

    喜好：

    - -
    • -

  • 联系方式：

    • +
  • +

    喜好：

    + +
    • -
    -

    Q&A文件：

    - - -
    +
    @@ -344,7 +184,7 @@
    - +
    diff --git a/images/luoji.jpg b/images/luoji.jpg new file mode 100644 index 0000000..b45fa26 Binary files /dev/null and b/images/luoji.jpg differ diff --git a/images/luoji.png b/images/luoji.png deleted file mode 100644 index 3fd1596..0000000 Binary files a/images/luoji.png and /dev/null differ diff --git a/system.conf b/system.conf index 744e946..b8751d2 100644 --- a/system.conf +++ b/system.conf @@ -3,7 +3,7 @@ #funasr / ali ASR_mode = ali #ASR二选一（需要运行fay/test/funasr服务）集成达摩院asr项目、感谢中科大脑算法工程师张聪聪提供集成代码 -local_asr_ip=127.0.0.1 +local_asr_ip=192.168.1.13 local_asr_port=10197 # ASR二选一（第1次运行建议用这个，免费3个月）， 阿里云 实时语音识别 服务密钥（必须）https://ai.aliyun.com/nls/trans @@ -12,7 +12,6 @@ ali_nls_key_secret= ali_nls_app_key= - # 微软 文字转语音 服务密钥（非必须，使用可产生不同情绪的音频）https://azure.microsoft.com/zh-cn/services/cognitive-services/text-to-speech/ ms_tts_key= ms_tts_region= @@ -21,23 +20,25 @@ ms_tts_region= xf_ltp_app_id= xf_ltp_api_key= -#NLP多选一:xfaiui、yuan、chatgpt、rasa(需启动chatglm及rasa，https://m.bilibili.com/video/BV1D14y1f7pr)、VisualGLM -chat_module=xfaiui +#NLP五选一:lingju、yuan、chatgpt、rasa(需启动chatglm及rasa，https://m.bilibili.com/video/BV1D14y1f7pr)、VisualGLM +chat_module=lingju -# 讯飞 自然语言处理 服务密钥(NLP3选1) https://aiui.xfyun.cn/solution/webapi/ -xf_aiui_app_id= -xf_aiui_api_key= +#灵聚 服务密钥(NLP多选1) https://open.lingju.ai +lingju_api_key= +lingju_api_authcode= -#浪.潮源大模型 服务密钥(NLP3选1) https://air.inspur.com/ +#浪.潮源大模型 服务密钥(NLP5多1) https://air.inspur.com/ yuan_1_0_account= yuan_1_0_phone= -#gpt 服务密钥(NLP3选1) https://openai.com/ +#gpt 服务密钥(NLP多选1) https://openai.com/ chatgpt_api_key= + #ngrok内网穿透id，远程设备可以通过互联网连接Fay（非必须）http://ngrok.cc ngrok_cc_id= #revChatGPT对接（非必须，https://chat.openai.com登录后访问https://chat.openai.com/api/auth/session获取） gpt_access_token= -gpt_conversation_id= \ No newline at end of file +gpt_conversation_id= + diff --git a/utils/config_util.py b/utils/config_util.py index 797d1f8..8dd5edd 100644 --- a/utils/config_util.py +++ b/utils/config_util.py @@ -47,6 +47,8 @@ def load_config(): global key_chat_module global key_gpt_access_token global key_gpt_conversation_id + global key_lingju_api_key + global key_lingju_api_authcode global ASR_mode global local_asr_ip @@ -70,6 +72,8 @@ def load_config(): key_chat_module = system_config.get('key', 'chat_module') key_gpt_access_token = system_config.get('key', 'gpt_access_token') key_gpt_conversation_id = system_config.get('key', 'gpt_conversation_id') + key_lingju_api_key = system_config.get('key', 'lingju_api_key') + key_lingju_api_authcode = system_config.get('key', 'lingju_api_authcode') ASR_mode = system_config.get('key', 'ASR_mode') local_asr_ip = system_config.get('key', 'local_asr_ip')