BJ_ZTEITS-Developers / parking-cloud-yudao

Commit a5a0383f10af82df8154ced0eb6d3e15e05e1fee

Authored by YunaiV
1 parent 3578e0bb

feat：【ai 大模型】对话增加附件的支持

Inline Side-by-side
Showing 6 changed files with 133 additions and 19 deletions
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/AiChatMessageController.http
  View file @a5a0383
... ... @@ -20,6 +20,30 @@ tenant-id: {{adminTenantId}}
20 20 "content": "1+1=?"
21 21 }
22 22  
  23 +### 发送消息（流式）【带文件】
  24 +POST {{baseUrl}}/ai/chat/message/send-stream
  25 +Content-Type: application/json
  26 +Authorization: {{token}}
  27 +tenant-id: {{adminTenantId}}
  28 +
  29 +{
  30 + "conversationId": "1781604279872581797",
  31 + "content": "图片里有什么？",
  32 + "attachmentUrls": ["http://test.yudao.iocoder.cn/1755531278.jpeg"]
  33 +}
  34 +
  35 +### 发送消息（流式）【追问带文件】
  36 +POST {{baseUrl}}/ai/chat/message/send-stream
  37 +Content-Type: application/json
  38 +Authorization: {{token}}
  39 +tenant-id: {{adminTenantId}}
  40 +
  41 +{
  42 + "conversationId": "1781604279872581797",
  43 + "content": "说下图片里，有哪些字？",
  44 + "useContext": true
  45 +}
  46 +
23 47 ### 获得指定对话的消息列表
24 48 GET {{baseUrl}}/ai/chat/message/list-by-conversation-id?conversationId=1781604279872581649
25 49 Authorization: {{token}}
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/vo/message/AiChatMessageRespVO.java
  View file @a5a0383
... ... @@ -49,6 +49,9 @@ public class AiChatMessageRespVO {
49 49 @Schema(description = "知识库段落数组")
50 50 private List<KnowledgeSegment> segments;
51 51  
  52 + @Schema(description = "附件 URL 数组", example = "https://www.iocoder.cn/1.png")
  53 + private List<String> attachmentUrls;
  54 +
52 55 @Schema(description = "创建时间", requiredMode = Schema.RequiredMode.REQUIRED, example = "2024-05-12 12:51")
53 56 private LocalDateTime createTime;
54 57  
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/vo/message/AiChatMessageSendReqVO.java
  View file @a5a0383
... ... @@ -3,9 +3,9 @@ package cn.iocoder.yudao.module.ai.controller.admin.chat.vo.message;
3 3 import io.swagger.v3.oas.annotations.media.Schema;
4 4 import jakarta.validation.constraints.NotEmpty;
5 5 import jakarta.validation.constraints.NotNull;
6   -import jakarta.validation.constraints.Size;
7 6 import lombok.Data;
8   -import lombok.experimental.Accessors;
  7 +
  8 +import java.util.List;
9 9  
10 10 @Schema(description = "管理后台 - AI 聊天消息发送 Request VO")
11 11 @Data
... ... @@ -22,4 +22,7 @@ public class AiChatMessageSendReqVO {
22 22 @Schema(description = "是否携带上下文", example = "true")
23 23 private Boolean useContext;
24 24  
  25 + @Schema(description = "附件 URL 数组", example = "https://www.iocoder.cn/1.png")
  26 + private List<String> attachmentUrls;
  27 +
25 28 }
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/chat/AiChatMessageDO.java
  View file @a5a0383
... ... @@ -2,6 +2,7 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.chat;
2 2  
3 3 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
4 4 import cn.iocoder.yudao.framework.mybatis.core.type.LongListTypeHandler;
  5 +import cn.iocoder.yudao.framework.mybatis.core.type.StringListTypeHandler;
5 6 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
6 7 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatRoleDO;
7 8 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiModelDO;
... ... @@ -105,4 +106,10 @@ public class AiChatMessageDO extends BaseDO {
105 106 @TableField(typeHandler = LongListTypeHandler.class)
106 107 private List<Long> segmentIds;
107 108  
  109 + /**
  110 + * 附件 URL 数组
  111 + */
  112 + @TableField(typeHandler = StringListTypeHandler.class)
  113 + private List<String> attachmentUrls;
  114 +
108 115 }
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImpl.java
  View file @a5a0383
1 1 package cn.iocoder.yudao.module.ai.service.chat;
2 2  
  3 +import cn.hutool.core.codec.Base64;
3 4 import cn.hutool.core.collection.CollUtil;
  5 +import cn.hutool.core.io.file.FileNameUtil;
4 6 import cn.hutool.core.util.ObjUtil;
5 7 import cn.hutool.core.util.StrUtil;
  8 +import cn.hutool.http.HttpUtil;
6 9 import cn.iocoder.yudao.framework.common.pojo.CommonResult;
7 10 import cn.iocoder.yudao.framework.common.pojo.PageResult;
8 11 import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
... ... @@ -28,6 +31,8 @@ import cn.iocoder.yudao.module.ai.service.model.AiChatRoleService;
28 31 import cn.iocoder.yudao.module.ai.service.model.AiModelService;
29 32 import cn.iocoder.yudao.module.ai.service.model.AiToolService;
30 33 import cn.iocoder.yudao.module.ai.util.AiUtils;
  34 +import cn.iocoder.yudao.module.infra.framework.file.core.utils.FileTypeUtils;
  35 +import com.google.common.collect.Maps;
31 36 import jakarta.annotation.Resource;
32 37 import lombok.extern.slf4j.Slf4j;
33 38 import org.springframework.ai.chat.messages.Message;
... ... @@ -64,6 +69,8 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.CHAT_MESSAGE_N
64 69 @Slf4j
65 70 public class AiChatMessageServiceImpl implements AiChatMessageService {
66 71  
  72 + // TODO @芋艿：后续优化下对话的 Prompt 整体结构
  73 +
67 74 /**
68 75 * 知识库转 {@link UserMessage} 的内容模版
69 76 */
... ... @@ -71,6 +78,14 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
71 78 "%s\n\n" + // 多个 <Reference></Reference> 的拼接
72 79 "回答要求：\n- 避免提及你是从 <Reference></Reference> 获取的知识。";
73 80  
  81 + /**
  82 + * 附件转 ${@link UserMessage} 的内容模版
  83 + */
  84 + @SuppressWarnings("TextBlockMigration")
  85 + private static final String Attachment_USER_MESSAGE_TEMPLATE = "使用 <Attachment></Attachment> 标记用户对话上传的附件内容:\n\n" +
  86 + "%s\n\n" + // 多个 <Attachment></Attachment> 的拼接
  87 + "回答要求：\n- 避免提及 <Attachment></Attachment> 附件的编码格式。";
  88 +
74 89 @Resource
75 90 private AiChatMessageMapper chatMessageMapper;
76 91  
... ... @@ -101,17 +116,18 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
101 116 ChatModel chatModel = modalService.getChatModel(model.getId());
102 117  
103 118 // 2. 知识库找回
104   - List<AiKnowledgeSegmentSearchRespBO> knowledgeSegments = recallKnowledgeSegment(sendReqVO.getContent(), conversation);
  119 + List<AiKnowledgeSegmentSearchRespBO> knowledgeSegments = recallKnowledgeSegment(
  120 + sendReqVO.getContent(), conversation);
105 121  
106 122 // 3. 插入 user 发送消息
107 123 AiChatMessageDO userMessage = createChatMessage(conversation.getId(), null, model,
108 124 userId, conversation.getRoleId(), MessageType.USER, sendReqVO.getContent(), sendReqVO.getUseContext(),
109   - null);
  125 + null, sendReqVO.getAttachmentUrls());
110 126  
111 127 // 3.1 插入 assistant 接收消息
112 128 AiChatMessageDO assistantMessage = createChatMessage(conversation.getId(), userMessage.getId(), model,
113 129 userId, conversation.getRoleId(), MessageType.ASSISTANT, "", sendReqVO.getUseContext(),
114   - knowledgeSegments);
  130 + knowledgeSegments, null);
115 131  
116 132 // 3.2 创建 chat 需要的 Prompt
117 133 Prompt prompt = buildPrompt(conversation, historyMessages, knowledgeSegments, model, sendReqVO);
... ... @@ -151,18 +167,18 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
151 167 StreamingChatModel chatModel = modalService.getChatModel(model.getId());
152 168  
153 169 // 2. 知识库找回
154   - List<AiKnowledgeSegmentSearchRespBO> knowledgeSegments = recallKnowledgeSegment(sendReqVO.getContent(),
155   - conversation);
  170 + List<AiKnowledgeSegmentSearchRespBO> knowledgeSegments = recallKnowledgeSegment(
  171 + sendReqVO.getContent(), conversation);
156 172  
157 173 // 3. 插入 user 发送消息
158 174 AiChatMessageDO userMessage = createChatMessage(conversation.getId(), null, model,
159 175 userId, conversation.getRoleId(), MessageType.USER, sendReqVO.getContent(), sendReqVO.getUseContext(),
160   - null);
  176 + null, sendReqVO.getAttachmentUrls());
161 177  
162 178 // 4.1 插入 assistant 接收消息
163 179 AiChatMessageDO assistantMessage = createChatMessage(conversation.getId(), userMessage.getId(), model,
164 180 userId, conversation.getRoleId(), MessageType.ASSISTANT, "", sendReqVO.getUseContext(),
165   - knowledgeSegments);
  181 + knowledgeSegments, null);
166 182  
167 183 // 4.2 构建 Prompt，并进行调用
168 184 Prompt prompt = buildPrompt(conversation, historyMessages, knowledgeSegments, model, sendReqVO);
... ... @@ -243,8 +259,13 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
243 259  
244 260 // 1.2 历史 history message 历史消息
245 261 List<AiChatMessageDO> contextMessages = filterContextMessages(messages, conversation, sendReqVO);
246   - contextMessages
247   - .forEach(message -> chatMessages.add(AiUtils.buildMessage(message.getType(), message.getContent())));
  262 + contextMessages.forEach(message -> {
  263 + chatMessages.add(AiUtils.buildMessage(message.getType(), message.getContent()));
  264 + UserMessage attachmentUserMessage = buildAttachmentUserMessage(message.getAttachmentUrls());
  265 + if (attachmentUserMessage != null) {
  266 + chatMessages.add(attachmentUserMessage);
  267 + }
  268 + });
248 269  
249 270 // 1.3 当前 user message 新发送消息
250 271 chatMessages.add(new UserMessage(sendReqVO.getContent()));
... ... @@ -257,6 +278,14 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
257 278 chatMessages.add(new UserMessage(String.format(KNOWLEDGE_USER_MESSAGE_TEMPLATE, reference)));
258 279 }
259 280  
  281 + // 1.5 附件，通过 UserMessage 实现
  282 + if (CollUtil.isNotEmpty(sendReqVO.getAttachmentUrls())) {
  283 + UserMessage attachmentUserMessage = buildAttachmentUserMessage(sendReqVO.getAttachmentUrls());
  284 + if (attachmentUserMessage != null) {
  285 + chatMessages.add(attachmentUserMessage);
  286 + }
  287 + }
  288 +
260 289 // 2.1 查询 tool 工具
261 290 Set<String> toolNames = null;
262 291 Map<String,Object> toolContext = Map.of();
... ... @@ -314,14 +343,52 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
314 343 return contextMessages;
315 344 }
316 345  
  346 + private UserMessage buildAttachmentUserMessage(List<String> attachmentUrls) {
  347 + if (CollUtil.isEmpty(attachmentUrls)) {
  348 + return null;
  349 + }
  350 + // 读取文件内容
  351 + Map<String, String> attachmentContents = Maps.newLinkedHashMapWithExpectedSize(attachmentUrls.size());
  352 + for (String attachmentUrl : attachmentUrls) {
  353 + try {
  354 + String name = FileNameUtil.getName(attachmentUrl);
  355 + String mineType = FileTypeUtils.getMineType(name);
  356 + String content;
  357 + if (FileTypeUtils.isImage(mineType)) {
  358 + // 特殊：图片则转为 Base64
  359 + byte[] bytes = HttpUtil.downloadBytes(attachmentUrl);
  360 + content = Base64.encode(bytes);
  361 + } else {
  362 + content = knowledgeDocumentService.readUrl(attachmentUrl);
  363 + }
  364 + if (StrUtil.isNotEmpty(content)) {
  365 + attachmentContents.put(name, content);
  366 + }
  367 + } catch (Exception e) {
  368 + log.error("[buildAttachmentUserMessage][读取附件({}) 发生异常]", attachmentUrl, e);
  369 + }
  370 + }
  371 + if (CollUtil.isEmpty(attachmentContents)) {
  372 + return null;
  373 + }
  374 +
  375 + // 拼接 UserMessage 消息
  376 + String attachment = attachmentContents.entrySet().stream()
  377 + .map(entry -> "<Attachment name=\"" + entry.getKey() + "\">" + entry.getValue() + "</Attachment>")
  378 + .collect(Collectors.joining("\n\n"));
  379 + return new UserMessage(String.format(Attachment_USER_MESSAGE_TEMPLATE, attachment));
  380 + }
  381 +
317 382 private AiChatMessageDO createChatMessage(Long conversationId, Long replyId,
318   - AiModelDO model, Long userId, Long roleId,
319   - MessageType messageType, String content, Boolean useContext,
320   - List<AiKnowledgeSegmentSearchRespBO> knowledgeSegments) {
  383 + AiModelDO model, Long userId, Long roleId,
  384 + MessageType messageType, String content, Boolean useContext,
  385 + List<AiKnowledgeSegmentSearchRespBO> knowledgeSegments,
  386 + List<String> attachmentUrls) {
321 387 AiChatMessageDO message = new AiChatMessageDO().setConversationId(conversationId).setReplyId(replyId)
322 388 .setModel(model.getModel()).setModelId(model.getId()).setUserId(userId).setRoleId(roleId)
323 389 .setType(messageType.getValue()).setContent(content).setUseContext(useContext)
324   - .setSegmentIds(convertList(knowledgeSegments, AiKnowledgeSegmentSearchRespBO::getId));
  390 + .setSegmentIds(convertList(knowledgeSegments, AiKnowledgeSegmentSearchRespBO::getId))
  391 + .setAttachmentUrls(attachmentUrls);
325 392 message.setCreateTime(LocalDateTime.now());
326 393 chatMessageMapper.insert(message);
327 394 return message;
... ...
yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/core/utils/FileTypeUtils.java
  View file @a5a0383
... ... @@ -80,17 +80,17 @@ public class FileTypeUtils {
80 80 */
81 81 public static void writeAttachment(HttpServletResponse response, String filename, byte[] content) throws IOException {
82 82 // 设置 header 和 contentType
83   - String contentType = getMineType(content, filename);
84   - response.setContentType(contentType);
  83 + String mineType = getMineType(content, filename);
  84 + response.setContentType(mineType);
85 85 // 设置内容显示、下载文件名：https://www.cnblogs.com/wq-9/articles/12165056.html
86   - if (StrUtil.containsIgnoreCase(contentType, "image/")) {
  86 + if (isImage(mineType)) {
87 87 // 参见 https://github.com/YunaiV/ruoyi-vue-pro/issues/692 讨论
88 88 response.setHeader("Content-Disposition", "inline;filename=" + HttpUtils.encodeUtf8(filename));
89 89 } else {
90 90 response.setHeader("Content-Disposition", "attachment;filename=" + HttpUtils.encodeUtf8(filename));
91 91 }
92 92 // 针对 video 的特殊处理，解决视频地址在移动端播放的兼容性问题
93   - if (StrUtil.containsIgnoreCase(contentType, "video")) {
  93 + if (StrUtil.containsIgnoreCase(mineType, "video")) {
94 94 response.setHeader("Content-Length", String.valueOf(content.length));
95 95 response.setHeader("Content-Range", "bytes 0-" + (content.length - 1) + "/" + content.length);
96 96 response.setHeader("Accept-Ranges", "bytes");
... ... @@ -99,4 +99,14 @@ public class FileTypeUtils {
99 99 IoUtil.write(response.getOutputStream(), false, content);
100 100 }
101 101  
  102 + /**
  103 + * 判断是否是图片
  104 + *
  105 + * @param mineType 类型
  106 + * @return 是否是图片
  107 + */
  108 + public static boolean isImage(String mineType) {
  109 + return StrUtil.startWith(mineType, "image/");
  110 + }
  111 +
102 112 }
... ...