From b1030a527a3abf1865e46e37e43a48677ea4802f Mon Sep 17 00:00:00 2001 From: divid Date: Mon, 20 Nov 2023 21:51:59 +0800 Subject: [PATCH 01/11] blacklist --- channel/chat_channel.py | 5 +++++ config-template.json | 4 ++++ config.py | 1 + 3 files changed, 10 insertions(+) diff --git a/channel/chat_channel.py b/channel/chat_channel.py index 8ed5f4f..4cedffd 100644 --- a/channel/chat_channel.py +++ b/channel/chat_channel.py @@ -99,6 +99,7 @@ class ChatChannel(Channel): # 校验关键字 match_prefix = check_prefix(content, conf().get("group_chat_prefix")) match_contain = check_contain(content, conf().get("group_chat_keyword")) + group_name_black_list = config.get("group_name_black_list", []) flag = False if context["msg"].to_user_id != context["msg"].actual_user_id: if match_prefix is not None or match_contain is not None: @@ -107,6 +108,10 @@ class ChatChannel(Channel): content = content.replace(match_prefix, "", 1).strip() if context["msg"].is_at: logger.info("[WX]receive group at") + if context["msg"].actual_user_nickname in group_name_black_list: + print('黑名单2',context["msg"].actual_user_nickname) + logger.info("[WX]Is In BlackList") + return None if not conf().get("group_at_off", False): flag = True pattern = f"@{re.escape(self.name)}(\u2005|\u0020)" diff --git a/config-template.json b/config-template.json index dd07d25..f211762 100644 --- a/config-template.json +++ b/config-template.json @@ -19,6 +19,10 @@ "ChatGPT测试群", "ChatGPT测试群2" ], + "group_name_black_list": [ + "测试昵称", + "测试昵称2" + ], "group_chat_in_one_session": [ "ChatGPT测试群" ], diff --git a/config.py b/config.py index 8db08cd..448c5c9 100644 --- a/config.py +++ b/config.py @@ -32,6 +32,7 @@ available_setting = { "group_name_white_list": ["ChatGPT测试群", "ChatGPT测试群2"], # 开启自动回复的群名称列表 "group_name_keyword_white_list": [], # 开启自动回复的群名称关键词列表 "group_chat_in_one_session": ["ChatGPT测试群"], # 支持会话上下文共享的群名称 + "group_name_black_list": [], # 黑名单 "group_welcome_msg": "", # 配置新人进群固定欢迎语，不配置则使用随机风格欢迎 "trigger_by_self": False, # 是否允许机器人触发 "text_to_image": "dall-e-2", # 图片生成模型，可选 dall-e-2, dall-e-3 From abf9a9048dab4c20a99890f722b3b4d6141ce65d Mon Sep 17 00:00:00 2001 From: divid Date: Mon, 20 Nov 2023 21:59:00 +0800 Subject: [PATCH 02/11] feat:blasklist --- channel/chat_channel.py | 1 - 1 file changed, 1 deletion(-) diff --git a/channel/chat_channel.py b/channel/chat_channel.py index 4cedffd..b8f7e90 100644 --- a/channel/chat_channel.py +++ b/channel/chat_channel.py @@ -109,7 +109,6 @@ class ChatChannel(Channel): if context["msg"].is_at: logger.info("[WX]receive group at") if context["msg"].actual_user_nickname in group_name_black_list: - print('黑名单2',context["msg"].actual_user_nickname) logger.info("[WX]Is In BlackList") return None if not conf().get("group_at_off", False): From 86a58c3d804c55463ab6bce76923cf2b2d005c75 Mon Sep 17 00:00:00 2001 From: Saboteur7 Date: Tue, 21 Nov 2023 22:41:54 +0800 Subject: [PATCH 03/11] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=A3=9E=E4=B9=A6?= =?UTF-8?q?=E5=BA=94=E7=94=A8=E9=80=9A=E9=81=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 支持自建机器人的私聊和群聊 - 支持图片生成 - 支持文件总结 --- app.py | 6 +- channel/channel_factory.py | 8 +- channel/chat_channel.py | 3 +- channel/feishu/feishu_channel.py | 250 +++++++++++++++++++++++++++++++ channel/feishu/feishu_message.py | 92 ++++++++++++ common/const.py | 3 + config.py | 7 + utils/file_util.py | 8 + 8 files changed, 371 insertions(+), 6 deletions(-) create mode 100644 channel/feishu/feishu_channel.py create mode 100644 channel/feishu/feishu_message.py create mode 100644 utils/file_util.py diff --git a/app.py b/app.py index 1bd6dad..19acdcd 100644 --- a/app.py +++ b/app.py @@ -5,8 +5,8 @@ import signal import sys from channel import channel_factory -from common.log import logger -from config import conf, load_config +from common import const +from config import load_config from plugins import * @@ -43,7 +43,7 @@ def run(): # os.environ['WECHATY_PUPPET_SERVICE_ENDPOINT'] = '127.0.0.1:9001' channel = channel_factory.create_channel(channel_name) - if channel_name in ["wx", "wxy", "terminal", "wechatmp", "wechatmp_service", "wechatcom_app", "wework"]: + if channel_name in ["wx", "wxy", "terminal", "wechatmp", "wechatmp_service", "wechatcom_app", "wework", const.FEISHU]: PluginManager().load_plugins() # startup channel diff --git a/channel/channel_factory.py b/channel/channel_factory.py index 8c45045..7044b9a 100644 --- a/channel/channel_factory.py +++ b/channel/channel_factory.py @@ -1,7 +1,7 @@ """ channel factory """ - +from common import const def create_channel(channel_type): """ @@ -35,6 +35,10 @@ def create_channel(channel_type): return WechatComAppChannel() elif channel_type == "wework": from channel.wework.wework_channel import WeworkChannel - return WeworkChannel() + + elif channel_type == const.FEISHU: + from channel.feishu.feishu_channel import FeiShuChanel + return FeiShuChanel() + raise RuntimeError diff --git a/channel/chat_channel.py b/channel/chat_channel.py index 8ed5f4f..c511b7e 100644 --- a/channel/chat_channel.py +++ b/channel/chat_channel.py @@ -238,7 +238,8 @@ class ChatChannel(Channel): reply = super().build_text_to_voice(reply.content) return self._decorate_reply(context, reply) if context.get("isgroup", False): - reply_text = "@" + context["msg"].actual_user_nickname + "\n" + reply_text.strip() + if not context.get("no_need_at", False): + reply_text = "@" + context["msg"].actual_user_nickname + "\n" + reply_text.strip() reply_text = conf().get("group_chat_reply_prefix", "") + reply_text + conf().get("group_chat_reply_suffix", "") else: reply_text = conf().get("single_chat_reply_prefix", "") + reply_text + conf().get("single_chat_reply_suffix", "") diff --git a/channel/feishu/feishu_channel.py b/channel/feishu/feishu_channel.py new file mode 100644 index 0000000..aed9436 --- /dev/null +++ b/channel/feishu/feishu_channel.py @@ -0,0 +1,250 @@ +""" +飞书通道接入 + +@author Saboteur7 +@Date 2023/11/19 +""" + +# -*- coding=utf-8 -*- +import io +import os +import time +import uuid + +import requests +import web +from channel.feishu.feishu_message import FeishuMessage +from bridge.context import Context +from bridge.reply import Reply, ReplyType +from common.log import logger +from common.singleton import singleton +from config import conf +from common.expired_dict import ExpiredDict +from bridge.context import ContextType +from channel.chat_channel import ChatChannel, check_prefix +from utils import file_util +import json +import os + +URL_VERIFICATION = "url_verification" + + +@singleton +class FeiShuChanel(ChatChannel): + feishu_app_id = conf().get('feishu_app_id') + feishu_app_secret = conf().get('feishu_app_secret') + feishu_token = conf().get('feishu_token') + + def __init__(self): + super().__init__() + # 历史消息id暂存，用于幂等控制 + self.receivedMsgs = ExpiredDict(60 * 60 * 7.1) + logger.info("[FeiShu] app_id={}, app_secret={} verification_token={}".format( + self.feishu_app_id, self.feishu_app_secret, self.feishu_token)) + # 无需群校验和前缀 + conf()["group_name_white_list"] = ["ALL_GROUP"] + conf()["single_chat_prefix"] = [] + + def startup(self): + urls = ( + '/', 'channel.feishu.feishu_channel.FeishuController' + ) + app = web.application(urls, globals(), autoreload=False) + port = conf().get("feishu_port", 9891) + web.httpserver.runsimple(app.wsgifunc(), ("0.0.0.0", port)) + + def send(self, reply: Reply, context: Context): + msg = context["msg"] + is_group = context["isgroup"] + headers = { + "Authorization": "Bearer " + msg.access_token, + "Content-Type": "application/json", + } + msg_type = "text" + logger.info(f"[FeiShu] start send reply message, type={context.type}, content={reply.content}") + reply_content = reply.content + content_key = "text" + if reply.type == ReplyType.IMAGE_URL: + # 图片上传 + reply_content = self._upload_image_url(reply.content, msg.access_token) + if not reply_content: + logger.warning("[FeiShu] upload file failed") + return + msg_type = "image" + content_key = "image_key" + if is_group: + # 群聊中直接回复 + url = f"https://open.feishu.cn/open-apis/im/v1/messages/{msg.msg_id}/reply" + data = { + "msg_type": msg_type, + "content": json.dumps({content_key: reply_content}) + } + res = requests.post(url=url, headers=headers, json=data, timeout=(5, 10)) + else: + url = "https://open.feishu.cn/open-apis/im/v1/messages" + params = {"receive_id_type": context.get("receive_id_type")} + data = { + "receive_id": context.get("receiver"), + "msg_type": msg_type, + "content": json.dumps({content_key: reply_content}) + } + res = requests.post(url=url, headers=headers, params=params, json=data, timeout=(5, 10)) + res = res.json() + if res.get("code") == 0: + logger.info(f"[FeiShu] send message success") + else: + logger.error(f"[FeiShu] send message failed, code={res.get('code')}, msg={res.get('msg')}") + + + def fetch_access_token(self) -> str: + url = "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal/" + headers = { + "Content-Type": "application/json" + } + req_body = { + "app_id": self.feishu_app_id, + "app_secret": self.feishu_app_secret + } + data = bytes(json.dumps(req_body), encoding='utf8') + response = requests.post(url=url, data=data, headers=headers) + if response.status_code == 200: + res = response.json() + if res.get("code") != 0: + logger.error(f"[FeiShu] get tenant_access_token error, code={res.get('code')}, msg={res.get('msg')}") + return "" + else: + return res.get("tenant_access_token") + else: + logger.error(f"[FeiShu] fetch token error, res={response}") + + + def _upload_image_url(self, img_url, access_token): + logger.debug(f"[WX] start download image, img_url={img_url}") + response = requests.get(img_url) + suffix = file_util.get_path_suffix(img_url) + temp_name = str(uuid.uuid4()) + "." + suffix + if response.status_code == 200: + # 将图片内容保存为临时文件 + with open(temp_name, "wb") as file: + file.write(response.content) + + # upload + upload_url = "https://open.feishu.cn/open-apis/im/v1/images" + data = { + 'image_type': 'message' + } + headers = { + 'Authorization': f'Bearer {access_token}', + } + with open(temp_name, "rb") as file: + upload_response = requests.post(upload_url, files={"image": file}, data=data, headers=headers) + logger.info(f"[FeiShu] upload file, res={upload_response.content}") + os.remove(temp_name) + return upload_response.json().get("data").get("image_key") + + + +class FeishuController: + # 类常量 + FAILED_MSG = '{"success": false}' + SUCCESS_MSG = '{"success": true}' + MESSAGE_RECEIVE_TYPE = "im.message.receive_v1" + + def GET(self): + return "Feishu service start success!" + + def POST(self): + try: + channel = FeiShuChanel() + + request = json.loads(web.data().decode("utf-8")) + logger.debug(f"[FeiShu] receive request: {request}") + + # 1.事件订阅回调验证 + if request.get("type") == URL_VERIFICATION: + varify_res = {"challenge": request.get("challenge")} + return json.dumps(varify_res) + + # 2.消息接收处理 + # token 校验 + header = request.get("header") + if not header or header.get("token") != channel.feishu_token: + return self.FAILED_MSG + + # 处理消息事件 + event = request.get("event") + if header.get("event_type") == self.MESSAGE_RECEIVE_TYPE and event: + if not event.get("message") or not event.get("sender"): + logger.warning(f"[FeiShu] invalid message, msg={request}") + return self.FAILED_MSG + msg = event.get("message") + + # 幂等判断 + if channel.receivedMsgs.get(msg.get("message_id")): + logger.warning(f"[FeiShu] repeat msg filtered, event_id={header.get('event_id')}") + return self.SUCCESS_MSG + channel.receivedMsgs[msg.get("message_id")] = True + + is_group = False + chat_type = msg.get("chat_type") + if chat_type == "group": + if not msg.get("mentions"): + # 群聊中未@不响应 + return self.SUCCESS_MSG + # 群聊 + is_group = True + receive_id_type = "chat_id" + elif chat_type == "p2p": + receive_id_type = "open_id" + else: + logger.warning("[FeiShu] message ignore") + return self.SUCCESS_MSG + # 构造飞书消息对象 + feishu_msg = FeishuMessage(event, is_group=is_group, access_token=channel.fetch_access_token()) + if not feishu_msg: + return self.SUCCESS_MSG + + context = self._compose_context( + feishu_msg.ctype, + feishu_msg.content, + isgroup=is_group, + msg=feishu_msg, + receive_id_type=receive_id_type, + no_need_at=True + ) + if context: + channel.produce(context) + logger.info(f"[FeiShu] query={feishu_msg.content}, type={feishu_msg.ctype}") + return self.SUCCESS_MSG + + except Exception as e: + logger.error(e) + return self.FAILED_MSG + + def _compose_context(self, ctype: ContextType, content, **kwargs): + context = Context(ctype, content) + context.kwargs = kwargs + if "origin_ctype" not in context: + context["origin_ctype"] = ctype + + cmsg = context["msg"] + context["session_id"] = cmsg.from_user_id + context["receiver"] = cmsg.other_user_id + + if ctype == ContextType.TEXT: + # 1.文本请求 + # 图片生成处理 + img_match_prefix = check_prefix(content, conf().get("image_create_prefix")) + if img_match_prefix: + content = content.replace(img_match_prefix, "", 1) + context.type = ContextType.IMAGE_CREATE + else: + context.type = ContextType.TEXT + context.content = content.strip() + + elif context.type == ContextType.VOICE: + # 2.语音请求 + if "desire_rtype" not in context and conf().get("voice_reply_voice"): + context["desire_rtype"] = ReplyType.VOICE + + return context diff --git a/channel/feishu/feishu_message.py b/channel/feishu/feishu_message.py new file mode 100644 index 0000000..fa6057e --- /dev/null +++ b/channel/feishu/feishu_message.py @@ -0,0 +1,92 @@ +from bridge.context import ContextType +from channel.chat_message import ChatMessage +import json +import requests +from common.log import logger +from common.tmp_dir import TmpDir +from utils import file_util + + +class FeishuMessage(ChatMessage): + def __init__(self, event: dict, is_group=False, access_token=None): + super().__init__(event) + msg = event.get("message") + sender = event.get("sender") + self.access_token = access_token + self.msg_id = msg.get("message_id") + self.create_time = msg.get("create_time") + self.is_group = is_group + msg_type = msg.get("message_type") + + if msg_type == "text": + self.ctype = ContextType.TEXT + content = json.loads(msg.get('content')) + self.content = content.get("text").strip() + elif msg_type == "file": + self.ctype = ContextType.FILE + content = json.loads(msg.get("content")) + file_key = content.get("file_key") + file_name = content.get("file_name") + + self.content = TmpDir().path() + file_key + "." + file_util.get_path_suffix(file_name) + + def _download_file(): + # 如果响应状态码是200，则将响应内容写入本地文件 + url = f"https://open.feishu.cn/open-apis/im/v1/messages/{self.msg_id}/resources/{file_key}" + headers = { + "Authorization": "Bearer " + access_token, + } + params = { + "type": "file" + } + response = requests.get(url=url, headers=headers, params=params) + if response.status_code == 200: + with open(self.content, "wb") as f: + f.write(response.content) + else: + logger.info(f"[FeiShu] Failed to download file, key={file_key}, res={response.text}") + self._prepare_fn = _download_file + + # elif msg.type == "voice": + # self.ctype = ContextType.VOICE + # self.content = TmpDir().path() + msg.media_id + "." + msg.format # content直接存临时目录路径 + # + # def download_voice(): + # # 如果响应状态码是200，则将响应内容写入本地文件 + # response = client.media.download(msg.media_id) + # if response.status_code == 200: + # with open(self.content, "wb") as f: + # f.write(response.content) + # else: + # logger.info(f"[wechatcom] Failed to download voice file, {response.content}") + # + # self._prepare_fn = download_voice + # elif msg.type == "image": + # self.ctype = ContextType.IMAGE + # self.content = TmpDir().path() + msg.media_id + ".png" # content直接存临时目录路径 + # + # def download_image(): + # # 如果响应状态码是200，则将响应内容写入本地文件 + # response = client.media.download(msg.media_id) + # if response.status_code == 200: + # with open(self.content, "wb") as f: + # f.write(response.content) + # else: + # logger.info(f"[wechatcom] Failed to download image file, {response.content}") + # + # self._prepare_fn = download_image + else: + raise NotImplementedError("Unsupported message type: Type:{} ".format(msg_type)) + + self.from_user_id = sender.get("sender_id").get("open_id") + self.to_user_id = event.get("app_id") + if is_group: + # 群聊 + self.other_user_id = msg.get("chat_id") + self.actual_user_id = self.from_user_id + self.content = self.content.replace("@_user_1", "").strip() + self.actual_user_nickname = "" + else: + # 私聊 + self.other_user_id = self.from_user_id + self.actual_user_id = self.from_user_id diff --git a/common/const.py b/common/const.py index a46765e..b8d701a 100644 --- a/common/const.py +++ b/common/const.py @@ -17,3 +17,6 @@ TTS_1 = "tts-1" TTS_1_HD = "tts-1-hd" MODEL_LIST = ["gpt-3.5-turbo", "gpt-3.5-turbo-16k", "gpt-4", "wenxin", "wenxin-4", "xunfei", "claude", "gpt-4-turbo", GPT4_TURBO_PREVIEW] + +# channel +FEISHU = "feishu" diff --git a/config.py b/config.py index 8db08cd..fd6bb93 100644 --- a/config.py +++ b/config.py @@ -115,6 +115,13 @@ available_setting = { "wechatcomapp_secret": "", # 企业微信app的secret "wechatcomapp_agent_id": "", # 企业微信app的agent_id "wechatcomapp_aes_key": "", # 企业微信app的aes_key + + # 飞书配置 + "feishu_port": 80, # 飞书bot监听端口 + "feishu_app_id": "", + "feishu_app_secret": "", + "feishu_token": "", + # chatgpt指令自定义触发词 "clear_memory_commands": ["#清除记忆"], # 重置会话指令，必须以#开头 # channel配置 diff --git a/utils/file_util.py b/utils/file_util.py new file mode 100644 index 0000000..6db659c --- /dev/null +++ b/utils/file_util.py @@ -0,0 +1,8 @@ +from urllib.parse import urlparse +import os + + +# 获取url后缀 +def get_path_suffix(path): + path = urlparse(path).path + return os.path.splitext(path)[-1].lstrip('.') From ddcfcf21fe82d120c2082120d5b9d1c63a9ffee5 Mon Sep 17 00:00:00 2001 From: Saboteur7 Date: Thu, 23 Nov 2023 22:05:10 +0800 Subject: [PATCH 04/11] =?UTF-8?q?=E7=BE=A4=E8=81=8A=E5=8F=AA=E6=9C=89?= =?UTF-8?q?=E8=89=BE=E7=89=B9=E6=9C=BA=E5=99=A8=E4=BA=BA=E6=89=8D=E5=9B=9E?= =?UTF-8?q?=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- channel/feishu/feishu_channel.py | 8 ++++---- config.py | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/channel/feishu/feishu_channel.py b/channel/feishu/feishu_channel.py index aed9436..5e7e4fc 100644 --- a/channel/feishu/feishu_channel.py +++ b/channel/feishu/feishu_channel.py @@ -6,9 +6,6 @@ """ # -*- coding=utf-8 -*- -import io -import os -import time import uuid import requests @@ -188,9 +185,12 @@ class FeishuController: is_group = False chat_type = msg.get("chat_type") if chat_type == "group": - if not msg.get("mentions"): + if not msg.get("mentions") and msg.get("message_type") == "text": # 群聊中未@不响应 return self.SUCCESS_MSG + if msg.get("mentions")[0].get("name") != conf().get("feishu_bot_name") and msg.get("message_type") == "text": + # 不是@机器人，不响应 + return self.SUCCESS_MSG # 群聊 is_group = True receive_id_type = "chat_id" diff --git a/config.py b/config.py index fd6bb93..3b6d130 100644 --- a/config.py +++ b/config.py @@ -118,9 +118,10 @@ available_setting = { # 飞书配置 "feishu_port": 80, # 飞书bot监听端口 - "feishu_app_id": "", - "feishu_app_secret": "", - "feishu_token": "", + "feishu_app_id": "", # 飞书机器人应用APP Id + "feishu_app_secret": "", # 飞书机器人APP secret + "feishu_token": "", # 飞书 verification token + "feishu_bot_name": "", # 飞书机器人的名字 # chatgpt指令自定义触发词 "clear_memory_commands": ["#清除记忆"], # 重置会话指令，必须以#开头 From 4e675b84fbd24d49df5c848bde42fc7ea43a2a45 Mon Sep 17 00:00:00 2001 From: zhayujie Date: Mon, 27 Nov 2023 12:47:00 +0800 Subject: [PATCH 05/11] feat: image input and session optimize --- bot/linkai/link_ai_bot.py | 124 +++++++++++++++++++++++++++++++++++--- bot/session_manager.py | 4 +- channel/chat_channel.py | 11 ++-- common/memory.py | 3 + common/utils.py | 7 ++- plugins/linkai/summary.py | 1 - 6 files changed, 134 insertions(+), 16 deletions(-) create mode 100644 common/memory.py diff --git a/bot/linkai/link_ai_bot.py b/bot/linkai/link_ai_bot.py index 1dc5df2..22f5172 100644 --- a/bot/linkai/link_ai_bot.py +++ b/bot/linkai/link_ai_bot.py @@ -13,6 +13,9 @@ from bridge.reply import Reply, ReplyType from common.log import logger from config import conf, pconf import threading +from common import memory, utils +import base64 + class LinkAIBot(Bot): # authentication failed @@ -21,7 +24,7 @@ class LinkAIBot(Bot): def __init__(self): super().__init__() - self.sessions = SessionManager(ChatGPTSession, model=conf().get("model") or "gpt-3.5-turbo") + self.sessions = LinkAISessionManager(LinkAISession, model=conf().get("model") or "gpt-3.5-turbo") self.args = {} def reply(self, query, context: Context = None) -> Reply: @@ -61,17 +64,25 @@ class LinkAIBot(Bot): linkai_api_key = conf().get("linkai_api_key") session_id = context["session_id"] + session_message = self.sessions.session_msg_query(query, session_id) + logger.debug(f"[LinkAI] session={session_message}, session_id={session_id}") + + # image process + img_cache = memory.USER_IMAGE_CACHE.get(session_id) + if img_cache: + messages = self._process_image_msg(app_code=app_code, session_id=session_id, query=query, img_cache=img_cache) + if messages: + session_message = messages - session = self.sessions.session_query(query, session_id) model = conf().get("model") # remove system message - if session.messages[0].get("role") == "system": + if session_message[0].get("role") == "system": if app_code or model == "wenxin": - session.messages.pop(0) + session_message.pop(0) body = { "app_code": app_code, - "messages": session.messages, + "messages": session_message, "model": model, # 对话模型的名称, 支持 gpt-3.5-turbo, gpt-3.5-turbo-16k, gpt-4, wenxin, xunfei "temperature": conf().get("temperature"), "top_p": conf().get("top_p", 1), @@ -94,7 +105,7 @@ class LinkAIBot(Bot): reply_content = response["choices"][0]["message"]["content"] total_tokens = response["usage"]["total_tokens"] logger.info(f"[LINKAI] reply={reply_content}, total_tokens={total_tokens}") - self.sessions.session_reply(reply_content, session_id, total_tokens) + self.sessions.session_reply(reply_content, session_id, total_tokens, query=query) agent_suffix = self._fetch_agent_suffix(response) if agent_suffix: @@ -130,6 +141,54 @@ class LinkAIBot(Bot): logger.warn(f"[LINKAI] do retry, times={retry_count}") return self._chat(query, context, retry_count + 1) + def _process_image_msg(self, app_code: str, session_id: str, query:str, img_cache: dict): + try: + enable_image_input = False + app_info = self._fetch_app_info(app_code) + if not app_info: + logger.debug(f"[LinkAI] not found app, can't process images, app_code={app_code}") + return None + plugins = app_info.get("data").get("plugins") + for plugin in plugins: + if plugin.get("input_type") and "IMAGE" in plugin.get("input_type"): + enable_image_input = True + if not enable_image_input: + return + msg = img_cache.get("msg") + path = img_cache.get("path") + msg.prepare() + logger.info(f"[LinkAI] query with images, path={path}") + messages = self._build_vision_msg(query, path) + memory.USER_IMAGE_CACHE[session_id] = None + return messages + except Exception as e: + logger.exception(e) + + + def _build_vision_msg(self, query: str, path: str): + try: + suffix = utils.get_path_suffix(path) + with open(path, "rb") as file: + base64_str = base64.b64encode(file.read()).decode('utf-8') + messages = [{ + "role": "user", + "content": [ + { + "type": "text", + "text": query + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/{suffix};base64,{base64_str}" + } + } + ] + }] + return messages + except Exception as e: + logger.exception(e) + def reply_text(self, session: ChatGPTSession, app_code="", retry_count=0) -> dict: if retry_count >= 2: # exit from retry 2 times @@ -195,6 +254,16 @@ class LinkAIBot(Bot): logger.warn(f"[LINKAI] do retry, times={retry_count}") return self.reply_text(session, app_code, retry_count + 1) + def _fetch_app_info(self, app_code: str): + headers = {"Authorization": "Bearer " + conf().get("linkai_api_key")} + # do http request + base_url = conf().get("linkai_api_base", "https://api.link-ai.chat") + params = {"app_code": app_code} + res = requests.get(url=base_url + "/v1/app/info", params=params, headers=headers, timeout=(5, 10)) + if res.status_code == 200: + return res.json() + else: + logger.warning(f"[LinkAI] find app info exception, res={res}") def create_img(self, query, retry_count=0, api_key=None): try: @@ -239,6 +308,7 @@ class LinkAIBot(Bot): except Exception as e: logger.exception(e) + def _fetch_agent_suffix(self, response): try: plugin_list = [] @@ -275,4 +345,44 @@ class LinkAIBot(Bot): reply = Reply(ReplyType.IMAGE_URL, url) channel.send(reply, context) except Exception as e: - logger.error(e) \ No newline at end of file + logger.error(e) + + +class LinkAISessionManager(SessionManager): + def session_msg_query(self, query, session_id): + session = self.build_session(session_id) + messages = session.messages + [{"role": "user", "content": query}] + return messages + + def session_reply(self, reply, session_id, total_tokens=None, query=None): + session = self.build_session(session_id) + if query: + session.add_query(query) + session.add_reply(reply) + try: + max_tokens = conf().get("conversation_max_tokens", 2500) + tokens_cnt = session.discard_exceeding(max_tokens, total_tokens) + logger.info(f"[LinkAI] chat history discard, before tokens={total_tokens}, now tokens={tokens_cnt}") + except Exception as e: + logger.warning("Exception when counting tokens precisely for session: {}".format(str(e))) + return session + + +class LinkAISession(ChatGPTSession): + def calc_tokens(self): + try: + cur_tokens = super().calc_tokens() + except Exception as e: + logger.debug("Exception when counting tokens precisely for query: {}".format(e)) + cur_tokens = len(str(self.messages)) + return cur_tokens + + def discard_exceeding(self, max_tokens, cur_tokens=None): + cur_tokens = self.calc_tokens() + if cur_tokens > max_tokens: + for i in range(0, len(self.messages)): + if i > 0 and self.messages[i].get("role") == "assistant" and self.messages[i - 1].get("role") == "user": + self.messages.pop(i) + self.messages.pop(i - 1) + return self.calc_tokens() + return cur_tokens diff --git a/bot/session_manager.py b/bot/session_manager.py index 8d70886..a6e89f9 100644 --- a/bot/session_manager.py +++ b/bot/session_manager.py @@ -69,7 +69,7 @@ class SessionManager(object): total_tokens = session.discard_exceeding(max_tokens, None) logger.debug("prompt tokens used={}".format(total_tokens)) except Exception as e: - logger.debug("Exception when counting tokens precisely for prompt: {}".format(str(e))) + logger.warning("Exception when counting tokens precisely for prompt: {}".format(str(e))) return session def session_reply(self, reply, session_id, total_tokens=None): @@ -80,7 +80,7 @@ class SessionManager(object): tokens_cnt = session.discard_exceeding(max_tokens, total_tokens) logger.debug("raw total_tokens={}, savesession tokens={}".format(total_tokens, tokens_cnt)) except Exception as e: - logger.debug("Exception when counting tokens precisely for session: {}".format(str(e))) + logger.warning("Exception when counting tokens precisely for session: {}".format(str(e))) return session def clear_session(self, session_id): diff --git a/channel/chat_channel.py b/channel/chat_channel.py index 8ed5f4f..ab574c6 100644 --- a/channel/chat_channel.py +++ b/channel/chat_channel.py @@ -9,8 +9,7 @@ from bridge.context import * from bridge.reply import * from channel.channel import Channel from common.dequeue import Dequeue -from common.log import logger -from config import conf +from common import memory from plugins import * try: @@ -205,14 +204,16 @@ class ChatChannel(Channel): else: return elif context.type == ContextType.IMAGE: # 图片消息，当前仅做下载保存到本地的逻辑 - cmsg = context["msg"] - cmsg.prepare() + memory.USER_IMAGE_CACHE[context["session_id"]] = { + "path": context.content, + "msg": context.get("msg") + } elif context.type == ContextType.SHARING: # 分享信息，当前无默认逻辑 pass elif context.type == ContextType.FUNCTION or context.type == ContextType.FILE: # 文件消息及函数调用等，当前无默认逻辑 pass else: - logger.error("[WX] unknown context type: {}".format(context.type)) + logger.warning("[WX] unknown context type: {}".format(context.type)) return return reply diff --git a/common/memory.py b/common/memory.py new file mode 100644 index 0000000..026bed2 --- /dev/null +++ b/common/memory.py @@ -0,0 +1,3 @@ +from common.expired_dict import ExpiredDict + +USER_IMAGE_CACHE = ExpiredDict(60 * 3) \ No newline at end of file diff --git a/common/utils.py b/common/utils.py index 966a7cf..dd69c9d 100644 --- a/common/utils.py +++ b/common/utils.py @@ -1,6 +1,6 @@ import io import os - +from urllib.parse import urlparse from PIL import Image @@ -49,3 +49,8 @@ def split_string_by_utf8_length(string, max_length, max_split=0): result.append(encoded[start:end].decode("utf-8")) start = end return result + + +def get_path_suffix(path): + path = urlparse(path).path + return os.path.splitext(path)[-1].lstrip('.') diff --git a/plugins/linkai/summary.py b/plugins/linkai/summary.py index c945896..5711fd9 100644 --- a/plugins/linkai/summary.py +++ b/plugins/linkai/summary.py @@ -91,5 +91,4 @@ class LinkSummary: for support_url in support_list: if url.strip().startswith(support_url): return True - logger.debug(f"[LinkSum] unsupported url, no need to process, url={url}") return False From a12507abbd9f110fac787d2cb82cb6e8c4e1de4c Mon Sep 17 00:00:00 2001 From: zhayujie Date: Mon, 27 Nov 2023 14:07:14 +0800 Subject: [PATCH 06/11] feat: default close image summary --- plugins/linkai/README.md | 2 +- plugins/linkai/config.json.template | 2 +- plugins/linkai/linkai.py | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/plugins/linkai/README.md b/plugins/linkai/README.md index 043d436..2ac80b1 100644 --- a/plugins/linkai/README.md +++ b/plugins/linkai/README.md @@ -26,7 +26,7 @@ "enabled": true, # 文档总结和对话功能开关 "group_enabled": true, # 是否支持群聊开启 "max_file_size": 5000, # 文件的大小限制，单位KB，默认为5M，超过该大小直接忽略 - "type": ["FILE", "SHARING", "IMAGE"] # 支持总结的类型，分别表示 文件、分享链接、图片 + "type": ["FILE", "SHARING", "IMAGE"] # 支持总结的类型，分别表示 文件、分享链接、图片，其中文件和链接默认打开，图片默认关闭 } } ``` diff --git a/plugins/linkai/config.json.template b/plugins/linkai/config.json.template index b6c7a04..547b8ef 100644 --- a/plugins/linkai/config.json.template +++ b/plugins/linkai/config.json.template @@ -15,6 +15,6 @@ "enabled": true, "group_enabled": true, "max_file_size": 5000, - "type": ["FILE", "SHARING", "IMAGE"] + "type": ["FILE", "SHARING"] } } diff --git a/plugins/linkai/linkai.py b/plugins/linkai/linkai.py index 9e35bcd..7978743 100644 --- a/plugins/linkai/linkai.py +++ b/plugins/linkai/linkai.py @@ -192,9 +192,7 @@ class LinkAI(Plugin): return False if context.kwargs.get("isgroup") and not self.sum_config.get("group_enabled"): return False - support_type = self.sum_config.get("type") - if not support_type: - return True + support_type = self.sum_config.get("type") or ["FILE", "SHARING"] if context.type.name not in support_type: return False return True From 21ad51ffbf6dc5a372c021ca9821e98808fbb4d5 Mon Sep 17 00:00:00 2001 From: zhayujie Date: Mon, 27 Nov 2023 14:24:26 +0800 Subject: [PATCH 07/11] fix: remove repeat util --- channel/feishu/feishu_channel.py | 4 ++-- channel/feishu/feishu_message.py | 4 ++-- utils/file_util.py | 8 -------- 3 files changed, 4 insertions(+), 12 deletions(-) delete mode 100644 utils/file_util.py diff --git a/channel/feishu/feishu_channel.py b/channel/feishu/feishu_channel.py index 5e7e4fc..85e40d7 100644 --- a/channel/feishu/feishu_channel.py +++ b/channel/feishu/feishu_channel.py @@ -19,7 +19,7 @@ from config import conf from common.expired_dict import ExpiredDict from bridge.context import ContextType from channel.chat_channel import ChatChannel, check_prefix -from utils import file_util +from common import utils import json import os @@ -118,7 +118,7 @@ class FeiShuChanel(ChatChannel): def _upload_image_url(self, img_url, access_token): logger.debug(f"[WX] start download image, img_url={img_url}") response = requests.get(img_url) - suffix = file_util.get_path_suffix(img_url) + suffix = utils.get_path_suffix(img_url) temp_name = str(uuid.uuid4()) + "." + suffix if response.status_code == 200: # 将图片内容保存为临时文件 diff --git a/channel/feishu/feishu_message.py b/channel/feishu/feishu_message.py index fa6057e..73285f2 100644 --- a/channel/feishu/feishu_message.py +++ b/channel/feishu/feishu_message.py @@ -4,7 +4,7 @@ import json import requests from common.log import logger from common.tmp_dir import TmpDir -from utils import file_util +from common import utils class FeishuMessage(ChatMessage): @@ -28,7 +28,7 @@ class FeishuMessage(ChatMessage): file_key = content.get("file_key") file_name = content.get("file_name") - self.content = TmpDir().path() + file_key + "." + file_util.get_path_suffix(file_name) + self.content = TmpDir().path() + file_key + "." + utils.get_path_suffix(file_name) def _download_file(): # 如果响应状态码是200，则将响应内容写入本地文件 diff --git a/utils/file_util.py b/utils/file_util.py deleted file mode 100644 index 6db659c..0000000 --- a/utils/file_util.py +++ /dev/null @@ -1,8 +0,0 @@ -from urllib.parse import urlparse -import os - - -# 获取url后缀 -def get_path_suffix(path): - path = urlparse(path).path - return os.path.splitext(path)[-1].lstrip('.') From d4da4d25759f9e7360b84f3ed04554e44df58496 Mon Sep 17 00:00:00 2001 From: zhayujie Date: Mon, 27 Nov 2023 14:38:45 +0800 Subject: [PATCH 08/11] fix: nick name config name --- channel/chat_channel.py | 7 ++++--- config-template.json | 4 ---- config.py | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/channel/chat_channel.py b/channel/chat_channel.py index 63abb12..3402c0c 100644 --- a/channel/chat_channel.py +++ b/channel/chat_channel.py @@ -98,7 +98,7 @@ class ChatChannel(Channel): # 校验关键字 match_prefix = check_prefix(content, conf().get("group_chat_prefix")) match_contain = check_contain(content, conf().get("group_chat_keyword")) - group_name_black_list = config.get("group_name_black_list", []) + nick_name_black_list = conf().get("nick_name_black_list", []) flag = False if context["msg"].to_user_id != context["msg"].actual_user_id: if match_prefix is not None or match_contain is not None: @@ -107,8 +107,9 @@ class ChatChannel(Channel): content = content.replace(match_prefix, "", 1).strip() if context["msg"].is_at: logger.info("[WX]receive group at") - if context["msg"].actual_user_nickname in group_name_black_list: - logger.info("[WX]Is In BlackList") + nick_name = context["msg"].actual_user_nickname + if nick_name and nick_name in nick_name_black_list: + logger.info(f"[WX] Nickname {nick_name} in In BlackList, ignore") return None if not conf().get("group_at_off", False): flag = True diff --git a/config-template.json b/config-template.json index f211762..dd07d25 100644 --- a/config-template.json +++ b/config-template.json @@ -19,10 +19,6 @@ "ChatGPT测试群", "ChatGPT测试群2" ], - "group_name_black_list": [ - "测试昵称", - "测试昵称2" - ], "group_chat_in_one_session": [ "ChatGPT测试群" ], diff --git a/config.py b/config.py index 7e9a945..6cc5708 100644 --- a/config.py +++ b/config.py @@ -32,7 +32,7 @@ available_setting = { "group_name_white_list": ["ChatGPT测试群", "ChatGPT测试群2"], # 开启自动回复的群名称列表 "group_name_keyword_white_list": [], # 开启自动回复的群名称关键词列表 "group_chat_in_one_session": ["ChatGPT测试群"], # 支持会话上下文共享的群名称 - "group_name_black_list": [], # 黑名单 + "nick_name_black_list": [], # 用户昵称黑名单 "group_welcome_msg": "", # 配置新人进群固定欢迎语，不配置则使用随机风格欢迎 "trigger_by_self": False, # 是否允许机器人触发 "text_to_image": "dall-e-2", # 图片生成模型，可选 dall-e-2, dall-e-3 From da87fd9e206f3e34bdb5f825ad68001572d005eb Mon Sep 17 00:00:00 2001 From: zhayujie Date: Mon, 27 Nov 2023 14:45:25 +0800 Subject: [PATCH 09/11] feat: add single chat blacklist --- channel/chat_channel.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/channel/chat_channel.py b/channel/chat_channel.py index 3402c0c..c664435 100644 --- a/channel/chat_channel.py +++ b/channel/chat_channel.py @@ -94,11 +94,11 @@ class ChatChannel(Channel): logger.debug("[WX]reference query skipped") return None + nick_name_black_list = conf().get("nick_name_black_list", []) if context.get("isgroup", False): # 群聊 # 校验关键字 match_prefix = check_prefix(content, conf().get("group_chat_prefix")) match_contain = check_contain(content, conf().get("group_chat_keyword")) - nick_name_black_list = conf().get("nick_name_black_list", []) flag = False if context["msg"].to_user_id != context["msg"].actual_user_id: if match_prefix is not None or match_contain is not None: @@ -106,11 +106,13 @@ class ChatChannel(Channel): if match_prefix: content = content.replace(match_prefix, "", 1).strip() if context["msg"].is_at: - logger.info("[WX]receive group at") nick_name = context["msg"].actual_user_nickname if nick_name and nick_name in nick_name_black_list: - logger.info(f"[WX] Nickname {nick_name} in In BlackList, ignore") + # 黑名单过滤 + logger.warning(f"[WX] Nickname {nick_name} in In BlackList, ignore") return None + + logger.info("[WX]receive group at") if not conf().get("group_at_off", False): flag = True pattern = f"@{re.escape(self.name)}(\u2005|\u0020)" @@ -129,6 +131,12 @@ class ChatChannel(Channel): logger.info("[WX]receive group voice, but checkprefix didn't match") return None else: # 单聊 + nick_name = context["msg"].from_user_nickname + if nick_name and nick_name in nick_name_black_list: + # 黑名单过滤 + logger.warning(f"[WX] Nickname '{nick_name}' in In BlackList, ignore") + return None + match_prefix = check_prefix(content, conf().get("single_chat_prefix", [""])) if match_prefix is not None: # 判断如果匹配到自定义前缀，则返回过滤掉前缀+空格后的内容 content = content.replace(match_prefix, "", 1).strip() From 0b2ce485860c9bbf53ef422466f3a4b4dfbc7552 Mon Sep 17 00:00:00 2001 From: zhayujie Date: Mon, 27 Nov 2023 18:20:52 +0800 Subject: [PATCH 10/11] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f46c357..ff47ae3 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ Demo made by [Visionn](https://www.wangpc.cc/) # 更新日志 +>**2023.11.10：** [1.5.2版本](https://github.com/zhayujie/chatgpt-on-wechat/releases/tag/1.5.2)，新增飞书通道、图像识别对话、黑名单配置 + >**2023.11.10：** [1.5.0版本](https://github.com/zhayujie/chatgpt-on-wechat/releases/tag/1.5.0)，新增 `gpt-4-turbo`, `dall-e-3`, `tts` 模型接入，完善图像理解&生成、语音识别&生成的多模态能力 >**2023.10.16：** 支持通过意图识别使用LinkAI联网搜索、数学计算、网页访问等插件，参考[插件文档](https://docs.link-ai.tech/platform/plugins) From 291f9360979ff8415e3a2154b4a5d4962ebc8c70 Mon Sep 17 00:00:00 2001 From: zhayujie Date: Mon, 27 Nov 2023 20:24:42 +0800 Subject: [PATCH 11/11] Update README.md --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ff47ae3..30d2f52 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,10 @@ 最新版本支持的功能如下： -- [x] **多端部署：** 有多种部署方式可选择且功能完备，目前已支持个人微信，微信公众号和企业微信应用等部署方式 +- [x] **多端部署：** 有多种部署方式可选择且功能完备，目前已支持个人微信、微信公众号和、业微信、飞书等部署方式 - [x] **基础对话：** 私聊及群聊的消息智能回复，支持多轮会话上下文记忆，支持 GPT-3.5, GPT-4, claude, 文心一言, 讯飞星火 -- [x] **语音识别：** 可识别语音消息，通过文字或语音回复，支持 azure, baidu, google, openai(whisper/tts) 等多种语音模型 -- [x] **图片生成：** 支持图片生成 和 图生图（如照片修复），可选择 Dall-E, stable diffusion, replicate, midjourney模型 +- [x] **语音能力：** 可识别语音消息，通过文字或语音回复，支持 azure, baidu, google, openai(whisper/tts) 等多种语音模型 +- [x] **图像能力：** 支持图片生成、图片识别、图生图（如照片修复），可选择 Dall-E-3, stable diffusion, replicate, midjourney, vision模型 - [x] **丰富插件：** 支持个性化插件扩展，已实现多角色切换、文字冒险、敏感词过滤、聊天记录总结、文档总结和对话等插件 - [X] **Tool工具：** 与操作系统和互联网交互，支持最新信息搜索、数学计算、天气和资讯查询、网页总结，基于 [chatgpt-tool-hub](https://github.com/goldfishh/chatgpt-tool-hub) 实现 - [x] **知识库：** 通过上传知识库文件自定义专属机器人，可作为数字分身、领域知识库、智能客服使用，基于 [LinkAI](https://link-ai.tech/console) 实现 @@ -54,6 +54,8 @@ Demo made by [Visionn](https://www.wangpc.cc/) # 快速开始 +快速开始文档：[项目搭建文档](https://docs.link-ai.tech/cow/quick-start) + ## 准备 ### 1. 账号注册