BJ_ZTEITS-Developers / parking-cloud-yudao

Commit 0b1889a8e88c82890762cceb2ae93abe6b18598e

Authored by YunaiV
1 parent 842bbb55

feat(ai): 支持模型输入模态配置与图片多模态传参

Inline Side-by-side
Showing 16 changed files with 741 additions and 27 deletions
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/AiModelController.java
  View file @0b1889a
... ... @@ -83,7 +83,8 @@ public class AiModelController {
83 83 List<AiModelDO> list = modelService.getModelListByStatusAndType(
84 84 CommonStatusEnum.ENABLE.getStatus(), type, platform);
85 85 return success(convertList(list, model -> new AiModelRespVO().setId(model.getId())
86   - .setName(model.getName()).setModel(model.getModel()).setPlatform(model.getPlatform())));
  86 + .setName(model.getName()).setModel(model.getModel()).setPlatform(model.getPlatform())
  87 + .setInputModalities(model.getInputModalities())));
87 88 }
88 89  
89 90 }
90 91 \ No newline at end of file
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/vo/model/AiModelRespVO.java
  View file @0b1889a
... ... @@ -4,6 +4,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
4 4 import lombok.Data;
5 5  
6 6 import java.time.LocalDateTime;
  7 +import java.util.List;
7 8  
8 9 @Schema(description = "管理后台 - AI 模型 Response VO")
9 10 @Data
... ... @@ -42,6 +43,9 @@ public class AiModelRespVO {
42 43 @Schema(description = "上下文的最大 Message 数量", example = "8192")
43 44 private Integer maxContexts;
44 45  
  46 + @Schema(description = "输入模态数组", example = "[\"text\", \"image\"]")
  47 + private List<String> inputModalities;
  48 +
45 49 @Schema(description = "创建时间", requiredMode = Schema.RequiredMode.REQUIRED)
46 50 private LocalDateTime createTime;
47 51  
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/vo/model/AiModelSaveReqVO.java
  View file @0b1889a
1 1 package cn.iocoder.yudao.module.ai.controller.admin.model.vo.model;
2 2  
  3 +import cn.iocoder.yudao.module.ai.enums.model.AiModelInputModalityEnum;
3 4 import cn.iocoder.yudao.module.ai.enums.model.AiModelTypeEnum;
4 5 import cn.iocoder.yudao.module.ai.enums.model.AiPlatformEnum;
5 6 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
... ... @@ -9,6 +10,8 @@ import jakarta.validation.constraints.NotEmpty;
9 10 import jakarta.validation.constraints.NotNull;
10 11 import lombok.Data;
11 12  
  13 +import java.util.List;
  14 +
12 15 @Schema(description = "管理后台 - API 模型新增/修改 Request VO")
13 16 @Data
14 17 public class AiModelSaveReqVO {
... ... @@ -56,4 +59,8 @@ public class AiModelSaveReqVO {
56 59 @Schema(description = "上下文的最大 Message 数量", example = "8192")
57 60 private Integer maxContexts;
58 61  
  62 + @Schema(description = "输入模态数组", example = "[\"text\", \"image\"]")
  63 + @InEnum(AiModelInputModalityEnum.class)
  64 + private List<String> inputModalities;
  65 +
59 66 }
60 67 \ No newline at end of file
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/model/AiModelDO.java
  View file @0b1889a
1 1 package cn.iocoder.yudao.module.ai.dal.dataobject.model;
2 2  
  3 +import cn.iocoder.yudao.module.ai.enums.model.AiModelInputModalityEnum;
3 4 import cn.iocoder.yudao.module.ai.enums.model.AiModelTypeEnum;
4 5 import cn.iocoder.yudao.module.ai.enums.model.AiPlatformEnum;
5 6 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
6 7 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
  8 +import cn.iocoder.yudao.framework.mybatis.core.type.StringListTypeHandler;
7 9 import com.baomidou.mybatisplus.annotation.KeySequence;
  10 +import com.baomidou.mybatisplus.annotation.TableField;
8 11 import com.baomidou.mybatisplus.annotation.TableId;
9 12 import com.baomidou.mybatisplus.annotation.TableName;
10 13 import lombok.*;
11 14  
  15 +import java.util.List;
  16 +
12 17 /**
13 18 * AI 模型 DO
14 19 *
... ... @@ -17,7 +22,7 @@ import lombok.*;
17 22 * @author fansili
18 23 * @since 2024/4/24 19:39
19 24 */
20   -@TableName("ai_model")
  25 +@TableName(value = "ai_model", autoResultMap = true)
21 26 @KeySequence("ai_model_seq") // 用于 Oracle、PostgreSQL、Kingbase、DB2、H2 数据库的主键自增。如果是 MySQL 等数据库，可不写。
22 27 @Data
23 28 @Builder
... ... @@ -84,5 +89,12 @@ public class AiModelDO extends BaseDO {
84 89 * 上下文的最大 Message 数量
85 90 */
86 91 private Integer maxContexts;
  92 + /**
  93 + * 输入模态数组
  94 + *
  95 + * 枚举 {@link AiModelInputModalityEnum}，仅 {@link AiModelTypeEnum#CHAT} 使用；为空时视为仅支持 text
  96 + */
  97 + @TableField(typeHandler = StringListTypeHandler.class)
  98 + private List<String> inputModalities;
87 99  
88 100 }
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java
  View file @0b1889a
... ... @@ -31,6 +31,7 @@ public interface ErrorCodeConstants {
31 31 // ========== API 聊天消息 1-040-004-000 ==========
32 32 ErrorCode CHAT_MESSAGE_NOT_EXIST = new ErrorCode(1_040_004_000, "消息不存在!");
33 33 ErrorCode CHAT_STREAM_ERROR = new ErrorCode(1_040_004_001, "对话生成异常!");
  34 + ErrorCode CHAT_MESSAGE_IMAGE_READ_FAIL = new ErrorCode(1_040_004_002, "读取图片附件({})失败，请重新上传后重试");
34 35  
35 36 // ========== API 绘画 1-040-005-000 ==========
36 37 ErrorCode IMAGE_NOT_EXISTS = new ErrorCode(1_040_005_000, "图片不存在!");
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/enums/model/AiModelInputModalityEnum.java 0 → 100644
  View file @0b1889a
  1 +package cn.iocoder.yudao.module.ai.enums.model;
  2 +
  3 +import cn.hutool.core.collection.CollUtil;
  4 +import cn.iocoder.yudao.framework.common.core.ArrayValuable;
  5 +import lombok.Getter;
  6 +import lombok.RequiredArgsConstructor;
  7 +
  8 +import java.util.Arrays;
  9 +import java.util.List;
  10 +
  11 +/**
  12 + * AI 模型输入模态的枚举
  13 + *
  14 + * 用于 {@link AiModelTypeEnum#CHAT} 对话模型，描述模型可以接收的输入类型，由用户按需配置
  15 + *
  16 + * @author 芋道源码
  17 + */
  18 +@Getter
  19 +@RequiredArgsConstructor
  20 +public enum AiModelInputModalityEnum implements ArrayValuable<String> {
  21 +
  22 + TEXT("text", "文本"),
  23 + IMAGE("image", "图片"),
  24 + VIDEO("video", "视频");
  25 +
  26 + /**
  27 + * 模态
  28 + */
  29 + private final String modality;
  30 + /**
  31 + * 模态名
  32 + */
  33 + private final String name;
  34 +
  35 + public static final String[] ARRAYS = Arrays.stream(values()).map(AiModelInputModalityEnum::getModality).toArray(String[]::new);
  36 +
  37 + @Override
  38 + public String[] array() {
  39 + return ARRAYS;
  40 + }
  41 +
  42 + /**
  43 + * 判断是否支持该输入模态；为空时（历史数据），视为仅支持 text
  44 + *
  45 + * @param modalities 模型配置的输入模态数组
  46 + * @param modality 输入模态
  47 + * @return 是否支持
  48 + */
  49 + public static boolean isSupported(List<String> modalities, AiModelInputModalityEnum modality) {
  50 + if (CollUtil.isEmpty(modalities)) {
  51 + return modality == TEXT;
  52 + }
  53 + return modalities.contains(modality.getModality());
  54 + }
  55 +
  56 +}
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/minimax/MiniMaxChatModel.java
  View file @0b1889a
... ... @@ -20,7 +20,7 @@ public class MiniMaxChatModel implements ChatModel {
20 20  
21 21 public static final String BASE_URL = "https://api.minimaxi.com/v1";
22 22  
23   - public static final String MODEL_DEFAULT = "MiniMax-M3";
  23 + public static final String MODEL_DEFAULT = "MiniMax-M3.1-Flash";
24 24  
25 25 /**
26 26 * 兼容 OpenAI 接口，复用 DeepSeek 客户端
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/moonshot/MoonshotChatModel.java
  View file @0b1889a
... ... @@ -22,7 +22,7 @@ public class MoonshotChatModel implements ChatModel {
22 22  
23 23 public static final String COMPLETE_PATH = "/v1/chat/completions";
24 24  
25   - public static final String MODEL_DEFAULT = "kimi-k2.6";
  25 + public static final String MODEL_DEFAULT = "kimi-k3.1-pro";
26 26  
27 27 /**
28 28 * 兼容 OpenAI 接口，复用 DeepSeek 客户端
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/zhipu/ZhiPuChatModel.java
  View file @0b1889a
... ... @@ -20,7 +20,7 @@ public class ZhiPuChatModel implements ChatModel {
20 20  
21 21 public static final String BASE_URL = "https://open.bigmodel.cn/api/paas/v4";
22 22  
23   - public static final String MODEL_DEFAULT = "glm-5.2";
  23 + public static final String MODEL_DEFAULT = "glm-5.3-flash";
24 24  
25 25 /**
26 26 * 兼容 OpenAI 接口，复用 DeepSeek 客户端
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImpl.java
  View file @0b1889a
... ... @@ -6,8 +6,10 @@ import cn.hutool.core.io.file.FileNameUtil;
6 6 import cn.hutool.core.util.ObjUtil;
7 7 import cn.hutool.core.util.StrUtil;
8 8 import cn.hutool.http.HttpUtil;
  9 +import cn.iocoder.yudao.framework.common.exception.ServiceException;
9 10 import cn.iocoder.yudao.framework.common.pojo.CommonResult;
10 11 import cn.iocoder.yudao.framework.common.pojo.PageResult;
  12 +import cn.iocoder.yudao.framework.common.util.http.HttpUtils;
11 13 import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
12 14 import cn.iocoder.yudao.framework.tenant.core.util.TenantUtils;
13 15 import cn.iocoder.yudao.module.ai.controller.admin.chat.vo.message.AiChatMessagePageReqVO;
... ... @@ -22,6 +24,7 @@ import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiModelDO;
22 24 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiToolDO;
23 25 import cn.iocoder.yudao.module.ai.dal.mysql.chat.AiChatMessageMapper;
24 26 import cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants;
  27 +import cn.iocoder.yudao.module.ai.enums.model.AiModelInputModalityEnum;
25 28 import cn.iocoder.yudao.module.ai.enums.model.AiPlatformEnum;
26 29 import cn.iocoder.yudao.module.ai.framework.ai.core.webserch.AiWebSearchClient;
27 30 import cn.iocoder.yudao.module.ai.framework.ai.core.webserch.AiWebSearchRequest;
... ... @@ -48,13 +51,16 @@ import org.springframework.ai.chat.model.ChatResponse;
48 51 import org.springframework.ai.chat.model.StreamingChatModel;
49 52 import org.springframework.ai.chat.prompt.ChatOptions;
50 53 import org.springframework.ai.chat.prompt.Prompt;
  54 +import org.springframework.ai.content.Media;
51 55 import org.springframework.ai.mcp.SyncMcpToolCallbackProvider;
52 56 import org.springframework.ai.mcp.client.common.autoconfigure.properties.McpClientCommonProperties;
53 57 import org.springframework.ai.tool.ToolCallback;
54 58 import org.springframework.ai.tool.resolution.ToolCallbackResolver;
55 59 import org.springframework.beans.factory.annotation.Autowired;
  60 +import org.springframework.core.io.ByteArrayResource;
56 61 import org.springframework.stereotype.Service;
57 62 import org.springframework.transaction.annotation.Transactional;
  63 +import org.springframework.util.MimeTypeUtils;
58 64 import reactor.core.publisher.Flux;
59 65  
60 66 import java.time.LocalDateTime;
... ... @@ -70,6 +76,7 @@ import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.
70 76 import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertSet;
71 77 import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.CHAT_CONVERSATION_NOT_EXISTS;
72 78 import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.CHAT_MESSAGE_NOT_EXIST;
  79 +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.CHAT_MESSAGE_IMAGE_READ_FAIL;
73 80  
74 81 /**
75 82 * AI 聊天消息 Service 实现类
... ... @@ -149,6 +156,8 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
149 156 // 1.2 校验模型
150 157 AiModelDO model = modalService.validateModel(conversation.getModelId());
151 158 ChatModel chatModel = modalService.getChatModel(model.getId());
  159 + // 1.3 选出上下文消息：附件按各自所属消息组装
  160 + List<AiChatMessageDO> contextMessages = filterContextMessages(historyMessages, conversation, sendReqVO);
152 161  
153 162 // 2.1 知识库召回
154 163 List<AiKnowledgeSegmentSearchRespBO> knowledgeSegments = recallKnowledgeSegment(
... ... @@ -159,6 +168,9 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
159 168 webSearchClient.search(new AiWebSearchRequest().setQuery(sendReqVO.getContent())
160 169 .setSummary(true).setCount(WEB_SEARCH_COUNT)) : null;
161 170  
  171 + // 2.3 创建 chat 需要的 Prompt：在插入消息前构建，图片读取失败时不留下消息
  172 + Prompt prompt = buildPrompt(conversation, contextMessages, knowledgeSegments, webSearchResponse, model, sendReqVO);
  173 +
162 174 // 3. 插入 user 发送消息
163 175 AiChatMessageDO userMessage = createChatMessage(conversation.getId(), null, model,
164 176 userId, conversation.getRoleId(), MessageType.USER, sendReqVO.getContent(), sendReqVO.getUseContext(),
... ... @@ -169,8 +181,7 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
169 181 userId, conversation.getRoleId(), MessageType.ASSISTANT, "", sendReqVO.getUseContext(),
170 182 knowledgeSegments, null, webSearchResponse);
171 183  
172   - // 4.2 创建 chat 需要的 Prompt
173   - Prompt prompt = buildPrompt(conversation, historyMessages, knowledgeSegments, webSearchResponse, model, sendReqVO);
  184 + // 4.2 调用模型
174 185 ChatResponse chatResponse = chatModel.call(prompt);
175 186  
176 187 // 4.3 更新响应内容
... ... @@ -206,6 +217,8 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
206 217 // 1.2 校验模型
207 218 AiModelDO model = modalService.validateModel(conversation.getModelId());
208 219 StreamingChatModel chatModel = modalService.getChatModel(model.getId());
  220 + // 1.3 选出上下文消息：附件按各自所属消息组装
  221 + List<AiChatMessageDO> contextMessages = filterContextMessages(historyMessages, conversation, sendReqVO);
209 222  
210 223 // 2.1 知识库找回
211 224 List<AiKnowledgeSegmentSearchRespBO> knowledgeSegments = recallKnowledgeSegment(
... ... @@ -216,6 +229,14 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
216 229 webSearchClient.search(new AiWebSearchRequest().setQuery(sendReqVO.getContent())
217 230 .setSummary(true).setCount(WEB_SEARCH_COUNT)) : null;
218 231  
  232 + // 2.3 构建 Prompt：在插入消息前构建，图片读取失败时返回错误帧，不留下消息
  233 + Prompt prompt;
  234 + try {
  235 + prompt = buildPrompt(conversation, contextMessages, knowledgeSegments, webSearchResponse, model, sendReqVO);
  236 + } catch (ServiceException ex) {
  237 + return Flux.just(error(ex.getCode(), ex.getMessage()));
  238 + }
  239 +
219 240 // 3. 插入 user 发送消息
220 241 AiChatMessageDO userMessage = createChatMessage(conversation.getId(), null, model,
221 242 userId, conversation.getRoleId(), MessageType.USER, sendReqVO.getContent(), sendReqVO.getUseContext(),
... ... @@ -226,8 +247,7 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
226 247 userId, conversation.getRoleId(), MessageType.ASSISTANT, "", sendReqVO.getUseContext(),
227 248 knowledgeSegments, null, webSearchResponse);
228 249  
229   - // 4.2 构建 Prompt，并进行调用
230   - Prompt prompt = buildPrompt(conversation, historyMessages, knowledgeSegments, webSearchResponse, model, sendReqVO);
  250 + // 4.2 调用模型
231 251 Flux<ChatResponse> streamResponse = chatModel.stream(prompt);
232 252  
233 253 // 4.3 流式返回
... ... @@ -322,7 +342,7 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
322 342 return knowledgeSegments;
323 343 }
324 344  
325   - private Prompt buildPrompt(AiChatConversationDO conversation, List<AiChatMessageDO> messages,
  345 + private Prompt buildPrompt(AiChatConversationDO conversation, List<AiChatMessageDO> contextMessages,
326 346 List<AiKnowledgeSegmentSearchRespBO> knowledgeSegments,
327 347 AiWebSearchResponse webSearchResponse,
328 348 AiModelDO model, AiChatMessageSendReqVO sendReqVO) {
... ... @@ -332,19 +352,20 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
332 352 chatMessages.add(new SystemMessage(conversation.getSystemMessage()));
333 353 }
334 354  
335   - // 1.2 历史 history message 历史消息
336   - List<AiChatMessageDO> contextMessages = filterContextMessages(messages, conversation, sendReqVO);
  355 + // 1.2 历史 history message 历史消息（已由 filterContextMessages 选出）；user 消息带上该轮的图片
337 356 contextMessages.forEach(message -> {
338   - chatMessages.add(AiUtils.buildMessage(message.getType(), message.getContent()));
339   - UserMessage attachmentUserMessage = buildAttachmentUserMessage(message.getAttachmentUrls());
  357 + chatMessages.add(MessageType.USER.getValue().equals(message.getType())
  358 + ? buildUserMessage(message.getContent(), message.getAttachmentUrls(), model)
  359 + : AiUtils.buildMessage(message.getType(), message.getContent()));
  360 + UserMessage attachmentUserMessage = buildAttachmentUserMessage(message.getAttachmentUrls(), model);
340 361 if (attachmentUserMessage != null) {
341 362 chatMessages.add(attachmentUserMessage);
342 363 }
343 364 // TODO @芋艿：历史的知识库；历史的搜索，要不要拼接？
344 365 });
345 366  
346   - // 1.3 当前 user message 新发送消息
347   - chatMessages.add(new UserMessage(sendReqVO.getContent()));
  367 + // 1.3 当前 user message 新发送消息：文本和本轮图片放在同一条 UserMessage
  368 + chatMessages.add(buildUserMessage(sendReqVO.getContent(), sendReqVO.getAttachmentUrls(), model));
348 369  
349 370 // 1.4 知识库，通过 UserMessage 实现
350 371 if (CollUtil.isNotEmpty(knowledgeSegments)) {
... ... @@ -369,7 +390,7 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
369 390  
370 391 // 1.6 附件，通过 UserMessage 实现
371 392 if (CollUtil.isNotEmpty(sendReqVO.getAttachmentUrls())) {
372   - UserMessage attachmentUserMessage = buildAttachmentUserMessage(sendReqVO.getAttachmentUrls());
  393 + UserMessage attachmentUserMessage = buildAttachmentUserMessage(sendReqVO.getAttachmentUrls(), model);
373 394 if (attachmentUserMessage != null) {
374 395 chatMessages.add(attachmentUserMessage);
375 396 }
... ... @@ -381,12 +402,24 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
381 402 : Map.of();
382 403 // 2.2 构建 ChatOptions 对象
383 404 AiPlatformEnum platform = AiPlatformEnum.validatePlatform(model.getPlatform());
  405 + boolean hasMedia = CollUtil.anyMatch(chatMessages, message -> message instanceof UserMessage
  406 + && CollUtil.isNotEmpty(((UserMessage) message).getMedia()));
384 407 ChatOptions chatOptions = AiUtils.buildChatOptions(platform, model.getModel(),
385 408 conversation.getTemperature(), conversation.getMaxTokens(),
386   - toolCallbacks, toolContext);
  409 + toolCallbacks, toolContext, hasMedia);
387 410 return new Prompt(chatMessages, chatOptions);
388 411 }
389 412  
  413 +
  414 + /**
  415 + * 获得角色关联的工具回调列表
  416 + *
  417 + * 1. 通过角色的 toolIds，解析本地注册的工具
  418 + * 2. 通过角色的 mcpClientNames，匹配已连接的 MCP Client，获取它们提供的工具
  419 + *
  420 + * @param roleId 角色编号
  421 + * @return 工具回调列表；角色编号为空、或角色不存在时，返回 null
  422 + */
390 423 private List<ToolCallback> getToolCallbackListByRoleId(Long roleId) {
391 424 if (roleId == null) {
392 425 return null;
... ... @@ -464,15 +497,19 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
464 497 return contextMessages;
465 498 }
466 499  
467   - private UserMessage buildAttachmentUserMessage(List<String> attachmentUrls) {
  500 + private UserMessage buildAttachmentUserMessage(List<String> attachmentUrls, AiModelDO model) {
468 501 if (CollUtil.isEmpty(attachmentUrls)) {
469 502 return null;
470 503 }
471 504 // 读取文件内容
472 505 Map<String, String> attachmentContents = Maps.newLinkedHashMapWithExpectedSize(attachmentUrls.size());
473 506 for (String attachmentUrl : attachmentUrls) {
  507 + // 模型配置了 image 输入的图片，已通过 UserMessage 的 Media 传输
  508 + if (isMediaImage(attachmentUrl, model)) {
  509 + continue;
  510 + }
474 511 try {
475   - String name = FileNameUtil.getName(attachmentUrl);
  512 + String name = getAttachmentFileName(attachmentUrl);
476 513 String mineType = FileTypeUtils.getMineType(name);
477 514 String content;
478 515 if (FileTypeUtils.isImage(mineType)) {
... ... @@ -500,6 +537,64 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
500 537 return new UserMessage(String.format(Attachment_USER_MESSAGE_TEMPLATE, attachment));
501 538 }
502 539  
  540 + /**
  541 + * 构建 user 消息：文本和该轮的图片放在同一条 {@link UserMessage}，图片以 Resource 形式作为 {@link Media}
  542 + *
  543 + * @param content 文本
  544 + * @param attachmentUrls 该轮附件 URL 数组
  545 + * @param model 模型
  546 + * @return user 消息；没有图片时为纯文本消息。图片读取失败时抛出异常，不降级为纯文本
  547 + */
  548 + private UserMessage buildUserMessage(String content, List<String> attachmentUrls, AiModelDO model) {
  549 + // 1. 读取该轮图片：仅模型配置了 image 输入的图片，以 Resource 作为 Media；读取失败直接报错，不降级为纯文本
  550 + List<Media> medias = new ArrayList<>();
  551 + if (CollUtil.isNotEmpty(attachmentUrls)) {
  552 + for (String url : attachmentUrls) {
  553 + if (!isMediaImage(url, model)) {
  554 + continue;
  555 + }
  556 + try {
  557 + String mineType = FileTypeUtils.getMineType(getAttachmentFileName(url));
  558 + medias.add(new Media(MimeTypeUtils.parseMimeType(mineType), new ByteArrayResource(HttpUtil.downloadBytes(url))));
  559 + } catch (Exception e) {
  560 + log.error("[buildUserMessage][读取图片({}) 发生异常]", url, e);
  561 + throw exception(CHAT_MESSAGE_IMAGE_READ_FAIL, getAttachmentFileName(url));
  562 + }
  563 + }
  564 + }
  565 + // 2.1 没有图片：纯文本的 user 消息
  566 + if (CollUtil.isEmpty(medias)) {
  567 + return new UserMessage(content);
  568 + }
  569 + // 2.2 有图片：文本和图片放在同一条 user 消息
  570 + return UserMessage.builder().text(content).media(medias).build();
  571 + }
  572 +
  573 + /**
  574 + * 判断附件是否作为 {@link Media} 传输：是图片，并且模型配置了 image 输入
  575 + *
  576 + * 其它附件（未配置 image 的图片、视频、文档）走原来的 Attachment 文本逻辑
  577 + *
  578 + * @param url 附件 URL
  579 + * @param model 模型
  580 + * @return 是否作为 Media 传输
  581 + */
  582 + private static boolean isMediaImage(String url, AiModelDO model) {
  583 + return FileTypeUtils.isImage(FileTypeUtils.getMineType(getAttachmentFileName(url)))
  584 + && AiModelInputModalityEnum.isSupported(model.getInputModalities(), AiModelInputModalityEnum.IMAGE);
  585 + }
  586 +
  587 + /**
  588 + * 获得附件的文件名：复用 {@link HttpUtils#removeUrlPathQueryAndFragment(String)} 去掉 ?query 和 #fragment
  589 + * （例如签名 URL 的 X-Amz-Signature），再取文件名
  590 + *
  591 + * @param url 附件 URL
  592 + * @return 文件名
  593 + */
  594 + private static String getAttachmentFileName(String url) {
  595 + return FileNameUtil.getName(HttpUtils.removeUrlPathQueryAndFragment(url));
  596 + }
  597 +
503 598 private AiChatMessageDO createChatMessage(Long conversationId, Long replyId,
504 599 AiModelDO model, Long userId, Long roleId,
505 600 MessageType messageType, String content, Boolean useContext,
... ...
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/util/AiUtils.java
  View file @0b1889a
... ... @@ -93,6 +93,17 @@ public class AiUtils {
93 93  
94 94 public static ChatOptions buildChatOptions(AiPlatformEnum platform, String model, Double temperature, Integer maxTokens,
95 95 List<ToolCallback> toolCallbacks, Map<String, Object> toolContext) {
  96 + return buildChatOptions(platform, model, temperature, maxTokens, toolCallbacks, toolContext, false);
  97 + }
  98 +
  99 + /**
  100 + * 构建 ChatOptions
  101 + *
  102 + * @param hasMedia 本次 Prompt 是否含 Media（包括选入上下文的历史消息）。通义含 Media 时必须走多模态接口
  103 + */
  104 + public static ChatOptions buildChatOptions(AiPlatformEnum platform, String model, Double temperature, Integer maxTokens,
  105 + List<ToolCallback> toolCallbacks, Map<String, Object> toolContext,
  106 + boolean hasMedia) {
96 107 toolCallbacks = ObjUtil.defaultIfNull(toolCallbacks, Collections.emptyList());
97 108 toolContext = ObjUtil.defaultIfNull(toolContext, Collections.emptyMap());
98 109 // noinspection EnhancedSwitchMigration
... ... @@ -100,7 +111,7 @@ public class AiUtils {
100 111 case TONG_YI:
101 112 return DashScopeChatOptions.builder().model(model).temperature(temperature).maxToken(maxTokens)
102 113 .enableThinking(true) // TODO 芋艿：默认都开启 thinking 模式，后续可以让用户配置
103   - .multiModel(TONG_YI_MULTI_MODELS.contains(model)) // 是否多模态模型
  114 + .multiModel(hasMedia || TONG_YI_MULTI_MODELS.contains(model)) // 含 Media 必须走多模态接口；纯文本按名单保持原通道
104 115 .toolCallbacks(toolCallbacks).toolContext(toolContext).build();
105 116 case DEEP_SEEK:
106 117 case DOU_BAO: // 复用 DeepSeek 客户端
... ...
yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/chat/DeepSeekChatModelTests.java
  View file @0b1889a
... ... @@ -29,7 +29,7 @@ public class DeepSeekChatModelTests {
29 29 private static final String API_KEY = SystemUtil.get("DEEPSEEK_API_KEY",
30 30 "sk-xxxx");
31 31 private static final String MODEL = SystemUtil.get("DEEPSEEK_MODEL",
32   - "deepseek-v4-flash");
  32 + "deepseek-flash");
33 33  
34 34 private final DeepSeekChatModel chatModel = DeepSeekChatModel.builder()
35 35 .deepSeekApi(DeepSeekApi.builder()
... ...
yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/chat/TongYiChatModelTests.java
  View file @0b1889a
1 1 package cn.iocoder.yudao.module.ai.framework.ai.core.model.chat;
2 2  
  3 +import cn.hutool.system.SystemUtil;
3 4 import cn.iocoder.yudao.framework.common.util.json.JsonUtils;
4 5 import com.alibaba.cloud.ai.dashscope.api.DashScopeApi;
5 6 import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatModel;
... ... @@ -32,13 +33,16 @@ import static java.util.Arrays.asList;
32 33 */
33 34 public class TongYiChatModelTests {
34 35  
  36 + private static final String API_KEY = SystemUtil.get("DASHSCOPE_API_KEY",
  37 + "sk-xxxx"); // 按需改成你的 DashScope API Key
  38 +
35 39 private final DashScopeChatModel chatModel = DashScopeChatModel.builder()
36 40 .dashScopeApi(DashScopeApi.builder()
37   - .apiKey("sk-cd9f39e99ea54840bd1888282325f55a") // https://bailian.console.aliyun.com/cn-beijing/?tab=model#/api-key 获取密钥
  41 + .apiKey(API_KEY) // https://bailian.console.aliyun.com/cn-beijing/?tab=model#/api-key 获取密钥
38 42 .build())
39 43 .defaultOptions(DashScopeChatOptions.builder()
40   - .multiModel(true) // 注意：当使用 qwen3.6-plus 等多模态模型，需要设置为 true，可见 https://help.aliyun.com/zh/model-studio/error-code#error-url 链接
41   - .model("qwen3.6-plus") // 模型
  44 + .multiModel(true) // 注意：当使用 qwen3.8-max 等多模态模型，需要设置为 true，可见 https://help.aliyun.com/zh/model-studio/error-code#error-url 链接
  45 + .model("qwen3.8-max") // 模型
42 46 // .model("deepseek-r1") // 模型（deepseek-r1）
43 47 // .model("deepseek-v3") // 模型（deepseek-v3）
44 48 // .model("deepseek-r1-distill-qwen-1.5b") // 模型（deepseek-r1-distill-qwen-1.5b）
... ... @@ -85,7 +89,7 @@ public class TongYiChatModelTests {
85 89 List<Message> messages = new ArrayList<>();
86 90 messages.add(new UserMessage("详细分析下，如何设计一个电商系统？"));
87 91 DashScopeChatOptions options = DashScopeChatOptions.builder()
88   - .model("qwen3.6-plus").multiModel(true)
  92 + .model("qwen3.8-max").multiModel(true)
89 93 // .withModel("qwen-max-2025-01-25")
90 94 .enableThinking(true) // 必须设置，否则会报错
91 95 .build();
... ... @@ -105,7 +109,7 @@ public class TongYiChatModelTests {
105 109 // 准备环境
106 110 RerankModel rerankModel = new DashScopeRerankModel(
107 111 DashScopeApi.builder()
108   - .apiKey("sk-47aa124781be4bfb95244cc62f63f7d0")
  112 + .apiKey(API_KEY)
109 113 .build());
110 114 // 准备参数
111 115 String query = "spring";
... ...
yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImplTest.java
  View file @0b1889a
1 1 package cn.iocoder.yudao.module.ai.service.chat;
2 2  
  3 +import cn.hutool.core.codec.Base64;
  4 +import cn.hutool.core.util.StrUtil;
  5 +import cn.hutool.http.HttpUtil;
  6 +import cn.iocoder.yudao.module.ai.enums.model.AiModelInputModalityEnum;
  7 +import org.mockito.MockedStatic;
  8 +import org.springframework.ai.chat.messages.AssistantMessage;
  9 +import org.springframework.ai.chat.messages.Message;
  10 +import org.springframework.ai.chat.messages.MessageType;
  11 +import org.springframework.ai.chat.messages.UserMessage;
  12 +import org.springframework.ai.chat.model.ChatModel;
  13 +import org.springframework.ai.chat.model.ChatResponse;
  14 +import org.springframework.ai.chat.model.Generation;
  15 +import org.springframework.ai.chat.prompt.Prompt;
  16 +import org.springframework.ai.content.Media;
  17 +import java.util.ArrayList;
  18 +import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatOptions;
3 19 import cn.hutool.core.collection.CollUtil;
4 20 import cn.iocoder.yudao.framework.common.pojo.CommonResult;
5 21 import cn.iocoder.yudao.framework.security.core.LoginUser;
... ... @@ -50,6 +66,8 @@ import java.util.concurrent.CopyOnWriteArrayList;
50 66 import java.util.concurrent.atomic.AtomicLong;
51 67 import java.util.stream.Collectors;
52 68  
  69 +import static cn.iocoder.yudao.framework.test.core.util.AssertUtils.assertServiceException;
  70 +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.CHAT_MESSAGE_IMAGE_READ_FAIL;
53 71 import static cn.iocoder.yudao.framework.test.core.util.RandomUtils.randomLongId;
54 72 import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.CHAT_STREAM_ERROR;
55 73 import static cn.iocoder.yudao.module.ai.util.AiUtils.TOOL_CONTEXT_LOGIN_USER;
... ... @@ -377,6 +395,348 @@ public class AiChatMessageServiceImplTest extends BaseMockitoUnitTest {
377 395 : ChatCompletionFinishReason.STOP;
378 396 }
379 397  
  398 + // ========== 附件多模态：模型配置 image 的图片走 Media，其它附件走原 Attachment 文本 ==========
  399 +
  400 + /**
  401 + * 自定义型号：不在任何已知名单里，是否传图只看模型配置的 inputModalities
  402 + */
  403 + private static final String CUSTOM_MODEL_NAME = "my-vlm-x";
  404 + private static final String IMAGE_URL = "https://www.iocoder.cn/a.png?X-Amz-Signature=xxx#v1";
  405 + private static final String OLD_IMAGE_URL = "https://www.iocoder.cn/old.png";
  406 + private static final String PDF_URL = "https://www.iocoder.cn/b.pdf";
  407 + private static final String VIDEO_URL = "https://www.iocoder.cn/c.mp4";
  408 + private static final byte[] IMAGE_BYTES = new byte[]{1, 2, 3};
  409 + private static final List<String> TEXT_IMAGE = asList(AiModelInputModalityEnum.TEXT.getModality(),
  410 + AiModelInputModalityEnum.IMAGE.getModality());
  411 +
  412 + @Mock
  413 + private ChatModel chatModel;
  414 +
  415 + @Test
  416 + public void testSendMessage_imageWithImageModality() {
  417 + // 准备参数：签名 URL 的图片 + pdf 文档
  418 + Long userId = randomLongId();
  419 + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId())
  420 + .setContent("这是我的车？").setAttachmentUrls(asList(IMAGE_URL, PDF_URL));
  421 + // mock 对话、模型：自定义型号配置了 image
  422 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE);
  423 + // mock 方法
  424 + when(knowledgeDocumentService.readUrl(eq(PDF_URL))).thenReturn("文档内容");
  425 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse());
  426 +
  427 + try (MockedStatic<HttpUtil> httpUtilMock = mockStatic(HttpUtil.class)) {
  428 + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL))).thenReturn(IMAGE_BYTES);
  429 + // 调用
  430 + chatMessageService.sendMessage(reqVO, userId);
  431 + }
  432 + // 断言：图片和问题在同一条 UserMessage 中，以 Media 传输
  433 + Prompt prompt = captureCallPrompt();
  434 + UserMessage currentMessage = findUserMessage(prompt, reqVO.getContent());
  435 + assertEquals(1, currentMessage.getMedia().size());
  436 + Media media = CollUtil.getFirst(currentMessage.getMedia());
  437 + assertEquals("image/png", media.getMimeType().toString());
  438 + assertArrayEquals(IMAGE_BYTES, (byte[]) media.getData());
  439 + // 断言：文档走 Attachment 文本，图片不再拼进文本
  440 + UserMessage attachmentMessage = findUserMessage(prompt, "<Attachment");
  441 + assertTrue(attachmentMessage.getText().contains("文档内容"));
  442 + assertFalse(attachmentMessage.getText().contains("a.png"));
  443 + verify(knowledgeDocumentService, never()).readUrl(eq(IMAGE_URL));
  444 + }
  445 +
  446 + @Test
  447 + public void testSendMessage_imageWithoutImageModality() {
  448 + // 准备参数
  449 + Long userId = randomLongId();
  450 + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId())
  451 + .setContent("这是我的车？").setAttachmentUrls(singletonList(IMAGE_URL));
  452 + // mock 对话、模型：未配置 image
  453 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK,
  454 + singletonList(AiModelInputModalityEnum.TEXT.getModality()));
  455 + // mock 方法
  456 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse());
  457 +
  458 + try (MockedStatic<HttpUtil> httpUtilMock = mockStatic(HttpUtil.class)) {
  459 + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL))).thenReturn(IMAGE_BYTES);
  460 + // 调用：不拒绝
  461 + chatMessageService.sendMessage(reqVO, userId);
  462 + }
  463 + // 断言：没有 Media，图片沿用原逻辑，以 Base64 拼进 Attachment 文本
  464 + Prompt prompt = captureCallPrompt();
  465 + assertNoMedia(prompt);
  466 + UserMessage attachmentMessage = findUserMessage(prompt, "<Attachment");
  467 + assertTrue(attachmentMessage.getText().contains("name=\"a.png\""));
  468 + assertTrue(attachmentMessage.getText().contains(Base64.encode(IMAGE_BYTES)));
  469 + }
  470 +
  471 + @Test
  472 + public void testSendMessage_historyImage() {
  473 + // 准备参数
  474 + Long userId = randomLongId();
  475 + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId())
  476 + .setContent("这是我的车？").setUseContext(true);
  477 + // mock 对话、模型：maxContexts = 1，只选入最近一组
  478 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE);
  479 + when(chatMessageMapper.selectListByConversationId(eq(reqVO.getConversationId())))
  480 + .thenReturn(buildHistoryMessages());
  481 + // mock 方法
  482 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse());
  483 +
  484 + try (MockedStatic<HttpUtil> httpUtilMock = mockStatic(HttpUtil.class)) {
  485 + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL))).thenReturn(IMAGE_BYTES);
  486 + // 调用
  487 + chatMessageService.sendMessage(reqVO, userId);
  488 + // 断言：未选入上下文的历史图片不下载
  489 + httpUtilMock.verify(() -> HttpUtil.downloadBytes(eq(OLD_IMAGE_URL)), never());
  490 + }
  491 + // 断言：历史图片挂在它所属的 user 消息上；当前消息没有图片
  492 + Prompt prompt = captureCallPrompt();
  493 + assertEquals(1, findUserMessage(prompt, "上一问").getMedia().size());
  494 + assertTrue(CollUtil.isEmpty(findUserMessage(prompt, reqVO.getContent()).getMedia()));
  495 + assertNull(findUserMessageOrNull(prompt, "旧问题"));
  496 + }
  497 +
  498 + @Test
  499 + public void testSendMessage_historyImageWithoutContext() {
  500 + // 准备参数：不使用上下文
  501 + Long userId = randomLongId();
  502 + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId())
  503 + .setContent("这是我的车？").setUseContext(false);
  504 + // mock 对话、模型
  505 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE);
  506 + when(chatMessageMapper.selectListByConversationId(eq(reqVO.getConversationId())))
  507 + .thenReturn(buildHistoryMessages());
  508 + // mock 方法
  509 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse());
  510 +
  511 + try (MockedStatic<HttpUtil> httpUtilMock = mockStatic(HttpUtil.class)) {
  512 + // 调用
  513 + chatMessageService.sendMessage(reqVO, userId);
  514 + // 断言：历史图片不下载
  515 + httpUtilMock.verify(() -> HttpUtil.downloadBytes(any(String.class)), never());
  516 + }
  517 + // 断言：没有 Media，也没有历史消息
  518 + Prompt prompt = captureCallPrompt();
  519 + assertNoMedia(prompt);
  520 + assertNull(findUserMessageOrNull(prompt, "上一问"));
  521 + }
  522 +
  523 + @Test
  524 + public void testSendMessage_imageReadFail() {
  525 + // 准备参数
  526 + Long userId = randomLongId();
  527 + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId())
  528 + .setContent("这是我的车？").setAttachmentUrls(singletonList(IMAGE_URL));
  529 + // mock 对话、模型：配置了 image
  530 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE);
  531 +
  532 + try (MockedStatic<HttpUtil> httpUtilMock = mockStatic(HttpUtil.class)) {
  533 + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL)))
  534 + .thenThrow(new IllegalStateException("下载失败"));
  535 + // 调用，并断言异常
  536 + assertServiceException(() -> chatMessageService.sendMessage(reqVO, userId),
  537 + CHAT_MESSAGE_IMAGE_READ_FAIL, "a.png");
  538 + }
  539 + // 断言：不调用模型、不写消息
  540 + verify(chatModel, never()).call(any(Prompt.class));
  541 + verify(chatMessageMapper, never()).insert(any(AiChatMessageDO.class));
  542 + }
  543 +
  544 + @Test
  545 + public void testSendChatMessageStream_imageReadFail() {
  546 + // 准备参数
  547 + Long userId = randomLongId();
  548 + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId())
  549 + .setContent("这是我的车？").setAttachmentUrls(singletonList(IMAGE_URL));
  550 + // mock 对话、模型：配置了 image
  551 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE);
  552 +
  553 + List<CommonResult<AiChatMessageSendRespVO>> results;
  554 + try (MockedStatic<HttpUtil> httpUtilMock = mockStatic(HttpUtil.class)) {
  555 + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL)))
  556 + .thenThrow(new IllegalStateException("下载失败"));
  557 + // 调用
  558 + results = chatMessageService.sendChatMessageStream(reqVO, userId).collectList().block();
  559 + }
  560 + // 断言：返回错误帧，不调用模型、不写消息
  561 + assertNotNull(results);
  562 + assertEquals(1, results.size());
  563 + assertEquals(CHAT_MESSAGE_IMAGE_READ_FAIL.getCode(), results.get(0).getCode());
  564 + verify(chatModel, never()).stream(any(Prompt.class));
  565 + verify(chatMessageMapper, never()).insert(any(AiChatMessageDO.class));
  566 + }
  567 +
  568 + @Test
  569 + public void testSendChatMessageStream_imageWithImageModality() {
  570 + // 准备参数
  571 + Long userId = randomLongId();
  572 + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId())
  573 + .setContent("这是我的车？").setAttachmentUrls(singletonList(IMAGE_URL));
  574 + // mock 对话、模型：配置了 image
  575 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE);
  576 + // mock 方法
  577 + when(chatModel.stream(any(Prompt.class))).thenReturn(Flux.just(buildChatResponse()));
  578 +
  579 + List<CommonResult<AiChatMessageSendRespVO>> results;
  580 + try (MockedStatic<HttpUtil> httpUtilMock = mockStatic(HttpUtil.class)) {
  581 + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL))).thenReturn(IMAGE_BYTES);
  582 + // 调用
  583 + results = chatMessageService.sendChatMessageStream(reqVO, userId).collectList().block();
  584 + }
  585 + // 断言：正常返回，图片以 Media 传输
  586 + assertNotNull(results);
  587 + assertTrue(CollUtil.getLast(results).isSuccess());
  588 + ArgumentCaptor<Prompt> promptCaptor = ArgumentCaptor.forClass(Prompt.class);
  589 + verify(chatModel).stream(promptCaptor.capture());
  590 + assertEquals(1, findUserMessage(promptCaptor.getValue(), reqVO.getContent()).getMedia().size());
  591 + }
  592 +
  593 + @Test
  594 + public void testSendMessage_video() {
  595 + // 准备参数
  596 + Long userId = randomLongId();
  597 + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId())
  598 + .setContent("视频里有什么？").setAttachmentUrls(singletonList(VIDEO_URL));
  599 + // mock 对话、模型：即使配置了 video，本期也不实现视频传输
  600 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK,
  601 + asList(AiModelInputModalityEnum.TEXT.getModality(), AiModelInputModalityEnum.IMAGE.getModality(),
  602 + AiModelInputModalityEnum.VIDEO.getModality()));
  603 + // mock 方法
  604 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse());
  605 +
  606 + try (MockedStatic<HttpUtil> httpUtilMock = mockStatic(HttpUtil.class)) {
  607 + // 调用：不拒绝
  608 + chatMessageService.sendMessage(reqVO, userId);
  609 + // 断言：视频不当作图片下载
  610 + httpUtilMock.verify(() -> HttpUtil.downloadBytes(any(String.class)), never());
  611 + }
  612 + // 断言：没有 Media，视频沿用原 Attachment 逻辑
  613 + assertNoMedia(captureCallPrompt());
  614 + verify(knowledgeDocumentService).readUrl(eq(VIDEO_URL));
  615 + }
  616 +
  617 + @Test
  618 + public void testSendMessage_textOnly() {
  619 + // 准备参数
  620 + Long userId = randomLongId();
  621 + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId())
  622 + .setContent("你好");
  623 + // mock 对话、模型
  624 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE);
  625 + // mock 方法
  626 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse());
  627 +
  628 + // 调用
  629 + chatMessageService.sendMessage(reqVO, userId);
  630 + // 断言：纯文本，没有 Media
  631 + Prompt prompt = captureCallPrompt();
  632 + assertNoMedia(prompt);
  633 + assertEquals(reqVO.getContent(), findUserMessage(prompt, reqVO.getContent()).getText());
  634 + }
  635 +
  636 + @Test
  637 + public void testSendMessage_tongYiMultiModelByMedia() {
  638 + // 准备参数：当前消息纯文本，历史消息带图
  639 + Long userId = randomLongId();
  640 + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId())
  641 + .setContent("这是我的车？").setUseContext(true);
  642 + // mock 对话、模型：通义的自定义型号，不在 TONG_YI_MULTI_MODELS 名单
  643 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.TONG_YI, TEXT_IMAGE);
  644 + when(chatMessageMapper.selectListByConversationId(eq(reqVO.getConversationId())))
  645 + .thenReturn(buildHistoryMessages());
  646 + // mock 方法
  647 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse());
  648 +
  649 + try (MockedStatic<HttpUtil> httpUtilMock = mockStatic(HttpUtil.class)) {
  650 + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL))).thenReturn(IMAGE_BYTES);
  651 + // 调用
  652 + chatMessageService.sendMessage(reqVO, userId);
  653 + }
  654 + // 断言：Prompt 含历史 Media，走多模态接口
  655 + DashScopeChatOptions options = (DashScopeChatOptions) captureCallPrompt().getOptions();
  656 + assertEquals(Boolean.TRUE, options.getMultiModel());
  657 + }
  658 +
  659 + @Test
  660 + public void testSendMessage_tongYiTextOnly() {
  661 + // 准备参数
  662 + Long userId = randomLongId();
  663 + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId())
  664 + .setContent("你好");
  665 + // mock 对话、模型：通义的自定义型号
  666 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.TONG_YI, TEXT_IMAGE);
  667 + // mock 方法
  668 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse());
  669 +
  670 + // 调用
  671 + chatMessageService.sendMessage(reqVO, userId);
  672 + // 断言：纯文本保持原通道
  673 + DashScopeChatOptions options = (DashScopeChatOptions) captureCallPrompt().getOptions();
  674 + assertEquals(Boolean.FALSE, options.getMultiModel());
  675 + }
  676 +
  677 +
  678 + private void mockMediaConversation(Long conversationId, Long userId, AiPlatformEnum platform,
  679 + List<String> inputModalities) {
  680 + // mock 对话
  681 + AiChatConversationDO conversation = new AiChatConversationDO().setId(conversationId).setUserId(userId)
  682 + .setModelId(randomLongId()).setTemperature(TEMPERATURE).setMaxTokens(MAX_TOKENS).setMaxContexts(1);
  683 + when(chatConversationService.validateChatConversationExists(eq(conversationId))).thenReturn(conversation);
  684 + // mock 模型：mock 的 ChatModel，用于捕获 Prompt
  685 + AiModelDO model = new AiModelDO().setId(conversation.getModelId()).setName("自定义视觉模型")
  686 + .setModel(CUSTOM_MODEL_NAME).setPlatform(platform.getPlatform()).setInputModalities(inputModalities);
  687 + when(modalService.validateModel(eq(model.getId()))).thenReturn(model);
  688 + when(modalService.getChatModel(eq(model.getId()))).thenReturn(chatModel);
  689 + }
  690 +
  691 + /**
  692 + * 构建历史消息：两组 user + assistant，各带一张图片
  693 + */
  694 + private static List<AiChatMessageDO> buildHistoryMessages() {
  695 + List<AiChatMessageDO> messages = new ArrayList<>();
  696 + messages.add(new AiChatMessageDO().setId(11L).setType(MessageType.USER.getValue())
  697 + .setContent("旧问题").setAttachmentUrls(singletonList(OLD_IMAGE_URL)));
  698 + messages.add(new AiChatMessageDO().setId(12L).setReplyId(11L).setType(MessageType.ASSISTANT.getValue())
  699 + .setContent("旧回答"));
  700 + messages.add(new AiChatMessageDO().setId(13L).setType(MessageType.USER.getValue())
  701 + .setContent("上一问").setAttachmentUrls(singletonList(IMAGE_URL)));
  702 + messages.add(new AiChatMessageDO().setId(14L).setReplyId(13L).setType(MessageType.ASSISTANT.getValue())
  703 + .setContent("上一答"));
  704 + return messages;
  705 + }
  706 +
  707 + private Prompt captureCallPrompt() {
  708 + ArgumentCaptor<Prompt> promptCaptor = ArgumentCaptor.forClass(Prompt.class);
  709 + verify(chatModel).call(promptCaptor.capture());
  710 + return promptCaptor.getValue();
  711 + }
  712 +
  713 + private static UserMessage findUserMessage(Prompt prompt, String text) {
  714 + UserMessage message = findUserMessageOrNull(prompt, text);
  715 + assertNotNull(message, "未找到包含(" + text + ")的 UserMessage");
  716 + return message;
  717 + }
  718 +
  719 + private static UserMessage findUserMessageOrNull(Prompt prompt, String text) {
  720 + for (Message message : prompt.getInstructions()) {
  721 + if (message instanceof UserMessage && StrUtil.contains(message.getText(), text)) {
  722 + return (UserMessage) message;
  723 + }
  724 + }
  725 + return null;
  726 + }
  727 +
  728 + private static void assertNoMedia(Prompt prompt) {
  729 + for (Message message : prompt.getInstructions()) {
  730 + if (message instanceof UserMessage) {
  731 + assertTrue(CollUtil.isEmpty(((UserMessage) message).getMedia()));
  732 + }
  733 + }
  734 + }
  735 +
  736 + private static ChatResponse buildChatResponse() {
  737 + return new ChatResponse(singletonList(new Generation(new AssistantMessage("好的"))));
  738 + }
  739 +
380 740 @Data
381 741 public static class WeatherRequest {
382 742  
... ...
yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/service/model/AiModelServiceImplTest.java 0 → 100644
  View file @0b1889a
  1 +package cn.iocoder.yudao.module.ai.service.model;
  2 +
  3 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
  4 +import cn.iocoder.yudao.framework.common.util.validation.ValidationUtils;
  5 +import cn.iocoder.yudao.framework.test.core.ut.BaseMockitoUnitTest;
  6 +import cn.iocoder.yudao.module.ai.controller.admin.model.vo.model.AiModelSaveReqVO;
  7 +import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiModelDO;
  8 +import cn.iocoder.yudao.module.ai.dal.mysql.model.AiChatMapper;
  9 +import cn.iocoder.yudao.module.ai.enums.model.AiModelTypeEnum;
  10 +import cn.iocoder.yudao.module.ai.enums.model.AiPlatformEnum;
  11 +import cn.iocoder.yudao.module.ai.framework.ai.core.model.AiModelFactory;
  12 +import jakarta.validation.ConstraintViolationException;
  13 +import org.junit.jupiter.api.Test;
  14 +import org.mockito.ArgumentCaptor;
  15 +import org.mockito.InjectMocks;
  16 +import org.mockito.Mock;
  17 +
  18 +import java.util.List;
  19 +
  20 +import static cn.iocoder.yudao.framework.test.core.util.RandomUtils.randomLongId;
  21 +import static java.util.Arrays.asList;
  22 +import static org.junit.jupiter.api.Assertions.*;
  23 +import static org.mockito.ArgumentMatchers.eq;
  24 +import static org.mockito.Mockito.verify;
  25 +import static org.mockito.Mockito.when;
  26 +
  27 +/**
  28 + * {@link AiModelServiceImpl} 的单元测试类
  29 + *
  30 + * 重点：输入模态 inputModalities 按前端传入的数组原样保存，后端不做规范化
  31 + *
  32 + * @author 芋道源码
  33 + */
  34 +public class AiModelServiceImplTest extends BaseMockitoUnitTest {
  35 +
  36 + @InjectMocks
  37 + private AiModelServiceImpl modelService;
  38 +
  39 + @Mock
  40 + private AiApiKeyService apiKeyService;
  41 + @Mock
  42 + private AiChatMapper modelMapper;
  43 + @Mock
  44 + private AiModelFactory modelFactory;
  45 +
  46 + @Test
  47 + public void testCreateModel_inputModalities() {
  48 + // 准备参数：顺序、重复按前端原样
  49 + List<String> inputModalities = asList("image", "text");
  50 + AiModelSaveReqVO reqVO = buildSaveReqVO(null).setInputModalities(inputModalities);
  51 +
  52 + // 调用
  53 + modelService.createModel(reqVO);
  54 + // 断言
  55 + ArgumentCaptor<AiModelDO> modelCaptor = ArgumentCaptor.forClass(AiModelDO.class);
  56 + verify(modelMapper).insert(modelCaptor.capture());
  57 + assertEquals(inputModalities, modelCaptor.getValue().getInputModalities());
  58 + }
  59 +
  60 + @Test
  61 + public void testUpdateModel_inputModalities() {
  62 + // mock 数据：库里原有 image、video
  63 + Long id = randomLongId();
  64 + when(modelMapper.selectById(eq(id))).thenReturn(new AiModelDO().setId(id)
  65 + .setInputModalities(asList("text", "image", "video")));
  66 + // 准备参数：前端只传 text
  67 + List<String> inputModalities = asList("text");
  68 + AiModelSaveReqVO reqVO = buildSaveReqVO(id).setInputModalities(inputModalities);
  69 +
  70 + // 调用
  71 + modelService.updateModel(reqVO);
  72 + // 断言：直接采用前端数组，不合并库里旧值
  73 + ArgumentCaptor<AiModelDO> modelCaptor = ArgumentCaptor.forClass(AiModelDO.class);
  74 + verify(modelMapper).updateById(modelCaptor.capture());
  75 + assertEquals(inputModalities, modelCaptor.getValue().getInputModalities());
  76 + }
  77 +
  78 + @Test
  79 + public void testSaveReqVO_inputModalitiesValid() {
  80 + // 准备参数
  81 + AiModelSaveReqVO reqVO = buildSaveReqVO(null).setInputModalities(asList("text", "image", "video"));
  82 +
  83 + // 调用，并断言：合法枚举值通过
  84 + assertDoesNotThrow(() -> ValidationUtils.validate(reqVO));
  85 + }
  86 +
  87 + @Test
  88 + public void testSaveReqVO_inputModalitiesInvalid() {
  89 + // 准备参数：audio 不在枚举中
  90 + AiModelSaveReqVO reqVO = buildSaveReqVO(null).setInputModalities(asList("text", "audio"));
  91 +
  92 + // 调用，并断言：@InEnum 拒绝非法值
  93 + assertThrows(ConstraintViolationException.class, () -> ValidationUtils.validate(reqVO));
  94 + }
  95 +
  96 + private static AiModelSaveReqVO buildSaveReqVO(Long id) {
  97 + AiModelSaveReqVO reqVO = new AiModelSaveReqVO();
  98 + reqVO.setId(id);
  99 + reqVO.setKeyId(randomLongId());
  100 + reqVO.setName("自定义视觉模型");
  101 + reqVO.setModel("my-vlm-x");
  102 + reqVO.setPlatform(AiPlatformEnum.DEEP_SEEK.getPlatform());
  103 + reqVO.setType(AiModelTypeEnum.CHAT.getType());
  104 + reqVO.setSort(1);
  105 + reqVO.setStatus(CommonStatusEnum.ENABLE.getStatus());
  106 + return reqVO;
  107 + }
  108 +
  109 +}
... ...
yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/util/AiUtilsTest.java 0 → 100644
  View file @0b1889a
  1 +package cn.iocoder.yudao.module.ai.util;
  2 +
  3 +import cn.iocoder.yudao.module.ai.enums.model.AiPlatformEnum;
  4 +import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatOptions;
  5 +import org.junit.jupiter.api.Test;
  6 +
  7 +import static org.junit.jupiter.api.Assertions.assertEquals;
  8 +
  9 +/**
  10 + * {@link AiUtils} 的单元测试类
  11 + *
  12 + * 重点：通义的 multiModel 按本次 Prompt 是否含 Media 选择请求通道，不受型号名单限制
  13 + *
  14 + * @author 芋道源码
  15 + */
  16 +public class AiUtilsTest {
  17 +
  18 + /**
  19 + * 自定义型号：不在 {@link AiUtils#TONG_YI_MULTI_MODELS} 名单中
  20 + */
  21 + private static final String CUSTOM_MODEL = "my-qwen-vl";
  22 + /**
  23 + * 名单内的型号
  24 + */
  25 + private static final String LISTED_MODEL = "qwen3-vl-plus";
  26 +
  27 + @Test
  28 + public void testBuildChatOptions_tongYiCustomModelWithMedia() {
  29 + // 调用
  30 + DashScopeChatOptions options = (DashScopeChatOptions) AiUtils.buildChatOptions(AiPlatformEnum.TONG_YI,
  31 + CUSTOM_MODEL, 0.7, 1024, null, null, true);
  32 + // 断言：含 Media 走多模态接口
  33 + assertEquals(Boolean.TRUE, options.getMultiModel());
  34 + }
  35 +
  36 + @Test
  37 + public void testBuildChatOptions_tongYiCustomModelTextOnly() {
  38 + // 调用
  39 + DashScopeChatOptions options = (DashScopeChatOptions) AiUtils.buildChatOptions(AiPlatformEnum.TONG_YI,
  40 + CUSTOM_MODEL, 0.7, 1024, null, null, false);
  41 + // 断言：纯文本保持原通道
  42 + assertEquals(Boolean.FALSE, options.getMultiModel());
  43 + }
  44 +
  45 + @Test
  46 + public void testBuildChatOptions_tongYiListedModelTextOnly() {
  47 + // 调用：原有重载，默认不含 Media
  48 + DashScopeChatOptions options = (DashScopeChatOptions) AiUtils.buildChatOptions(AiPlatformEnum.TONG_YI,
  49 + LISTED_MODEL, 0.7, 1024);
  50 + // 断言：名单内型号纯文本仍按名单走多模态接口，与改动前一致
  51 + assertEquals(Boolean.TRUE, options.getMultiModel());
  52 + }
  53 +
  54 +}
... ...