diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/AiModelController.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/AiModelController.java index 86dd4d0..5660cdd 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/AiModelController.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/AiModelController.java @@ -83,7 +83,8 @@ public class AiModelController { List list = modelService.getModelListByStatusAndType( CommonStatusEnum.ENABLE.getStatus(), type, platform); return success(convertList(list, model -> new AiModelRespVO().setId(model.getId()) - .setName(model.getName()).setModel(model.getModel()).setPlatform(model.getPlatform()))); + .setName(model.getName()).setModel(model.getModel()).setPlatform(model.getPlatform()) + .setInputModalities(model.getInputModalities()))); } } \ No newline at end of file diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/vo/model/AiModelRespVO.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/vo/model/AiModelRespVO.java index b50b70a..fcf6994 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/vo/model/AiModelRespVO.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/vo/model/AiModelRespVO.java @@ -4,6 +4,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import java.time.LocalDateTime; +import java.util.List; @Schema(description = "管理后台 - AI 模型 Response VO") @Data @@ -42,6 +43,9 @@ public class AiModelRespVO { @Schema(description = "上下文的最大 Message 数量", example = "8192") private Integer maxContexts; + @Schema(description = "输入模态数组", example = "[\"text\", \"image\"]") + private List inputModalities; + @Schema(description = "创建时间", requiredMode = Schema.RequiredMode.REQUIRED) private LocalDateTime createTime; diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/vo/model/AiModelSaveReqVO.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/vo/model/AiModelSaveReqVO.java index 95ed6bf..6b6d069 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/vo/model/AiModelSaveReqVO.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/model/vo/model/AiModelSaveReqVO.java @@ -1,5 +1,6 @@ package cn.iocoder.yudao.module.ai.controller.admin.model.vo.model; +import cn.iocoder.yudao.module.ai.enums.model.AiModelInputModalityEnum; import cn.iocoder.yudao.module.ai.enums.model.AiModelTypeEnum; import cn.iocoder.yudao.module.ai.enums.model.AiPlatformEnum; import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; @@ -9,6 +10,8 @@ import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import lombok.Data; +import java.util.List; + @Schema(description = "管理后台 - API 模型新增/修改 Request VO") @Data public class AiModelSaveReqVO { @@ -56,4 +59,8 @@ public class AiModelSaveReqVO { @Schema(description = "上下文的最大 Message 数量", example = "8192") private Integer maxContexts; + @Schema(description = "输入模态数组", example = "[\"text\", \"image\"]") + @InEnum(AiModelInputModalityEnum.class) + private List inputModalities; + } \ No newline at end of file diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/model/AiModelDO.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/model/AiModelDO.java index 9e54f94..7e9e020 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/model/AiModelDO.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/model/AiModelDO.java @@ -1,14 +1,19 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.model; +import cn.iocoder.yudao.module.ai.enums.model.AiModelInputModalityEnum; import cn.iocoder.yudao.module.ai.enums.model.AiModelTypeEnum; import cn.iocoder.yudao.module.ai.enums.model.AiPlatformEnum; import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; +import cn.iocoder.yudao.framework.mybatis.core.type.StringListTypeHandler; import com.baomidou.mybatisplus.annotation.KeySequence; +import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import lombok.*; +import java.util.List; + /** * AI 模型 DO * @@ -17,7 +22,7 @@ import lombok.*; * @author fansili * @since 2024/4/24 19:39 */ -@TableName("ai_model") +@TableName(value = "ai_model", autoResultMap = true) @KeySequence("ai_model_seq") // 用于 Oracle、PostgreSQL、Kingbase、DB2、H2 数据库的主键自增。如果是 MySQL 等数据库，可不写。 @Data @Builder @@ -84,5 +89,12 @@ public class AiModelDO extends BaseDO { * 上下文的最大 Message 数量 */ private Integer maxContexts; + /** + * 输入模态数组 + * + * 枚举 {@link AiModelInputModalityEnum}，仅 {@link AiModelTypeEnum#CHAT} 使用；为空时视为仅支持 text + */ + @TableField(typeHandler = StringListTypeHandler.class) + private List inputModalities; } diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java index 4658a08..70b99d3 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java @@ -31,6 +31,7 @@ public interface ErrorCodeConstants { // ========== API 聊天消息 1-040-004-000 ========== ErrorCode CHAT_MESSAGE_NOT_EXIST = new ErrorCode(1_040_004_000, "消息不存在!"); ErrorCode CHAT_STREAM_ERROR = new ErrorCode(1_040_004_001, "对话生成异常!"); + ErrorCode CHAT_MESSAGE_IMAGE_READ_FAIL = new ErrorCode(1_040_004_002, "读取图片附件({})失败，请重新上传后重试"); // ========== API 绘画 1-040-005-000 ========== ErrorCode IMAGE_NOT_EXISTS = new ErrorCode(1_040_005_000, "图片不存在!"); diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/enums/model/AiModelInputModalityEnum.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/enums/model/AiModelInputModalityEnum.java new file mode 100644 index 0000000..d810b1e --- /dev/null +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/enums/model/AiModelInputModalityEnum.java @@ -0,0 +1,56 @@ +package cn.iocoder.yudao.module.ai.enums.model; + +import cn.hutool.core.collection.CollUtil; +import cn.iocoder.yudao.framework.common.core.ArrayValuable; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.Arrays; +import java.util.List; + +/** + * AI 模型输入模态的枚举 + * + * 用于 {@link AiModelTypeEnum#CHAT} 对话模型，描述模型可以接收的输入类型，由用户按需配置 + * + * @author 芋道源码 + */ +@Getter +@RequiredArgsConstructor +public enum AiModelInputModalityEnum implements ArrayValuable { + + TEXT("text", "文本"), + IMAGE("image", "图片"), + VIDEO("video", "视频"); + + /** + * 模态 + */ + private final String modality; + /** + * 模态名 + */ + private final String name; + + public static final String[] ARRAYS = Arrays.stream(values()).map(AiModelInputModalityEnum::getModality).toArray(String[]::new); + + @Override + public String[] array() { + return ARRAYS; + } + + /** + * 判断是否支持该输入模态；为空时（历史数据），视为仅支持 text + * + * @param modalities 模型配置的输入模态数组 + * @param modality 输入模态 + * @return 是否支持 + */ + public static boolean isSupported(List modalities, AiModelInputModalityEnum modality) { + if (CollUtil.isEmpty(modalities)) { + return modality == TEXT; + } + return modalities.contains(modality.getModality()); + } + +} diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/minimax/MiniMaxChatModel.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/minimax/MiniMaxChatModel.java index d421aab..36497c1 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/minimax/MiniMaxChatModel.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/minimax/MiniMaxChatModel.java @@ -20,7 +20,7 @@ public class MiniMaxChatModel implements ChatModel { public static final String BASE_URL = "https://api.minimaxi.com/v1"; - public static final String MODEL_DEFAULT = "MiniMax-M3"; + public static final String MODEL_DEFAULT = "MiniMax-M3.1-Flash"; /** * 兼容 OpenAI 接口，复用 DeepSeek 客户端 diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/moonshot/MoonshotChatModel.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/moonshot/MoonshotChatModel.java index 27287e3..d4aa126 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/moonshot/MoonshotChatModel.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/moonshot/MoonshotChatModel.java @@ -22,7 +22,7 @@ public class MoonshotChatModel implements ChatModel { public static final String COMPLETE_PATH = "/v1/chat/completions"; - public static final String MODEL_DEFAULT = "kimi-k2.6"; + public static final String MODEL_DEFAULT = "kimi-k3.1-pro"; /** * 兼容 OpenAI 接口，复用 DeepSeek 客户端 diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/zhipu/ZhiPuChatModel.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/zhipu/ZhiPuChatModel.java index 858f22c..87e37fb 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/zhipu/ZhiPuChatModel.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/zhipu/ZhiPuChatModel.java @@ -20,7 +20,7 @@ public class ZhiPuChatModel implements ChatModel { public static final String BASE_URL = "https://open.bigmodel.cn/api/paas/v4"; - public static final String MODEL_DEFAULT = "glm-5.2"; + public static final String MODEL_DEFAULT = "glm-5.3-flash"; /** * 兼容 OpenAI 接口，复用 DeepSeek 客户端 diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImpl.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImpl.java index 6a7f54a..9310cfc 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImpl.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImpl.java @@ -6,8 +6,10 @@ import cn.hutool.core.io.file.FileNameUtil; import cn.hutool.core.util.ObjUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.http.HttpUtil; +import cn.iocoder.yudao.framework.common.exception.ServiceException; import cn.iocoder.yudao.framework.common.pojo.CommonResult; import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.common.util.http.HttpUtils; import cn.iocoder.yudao.framework.common.util.object.BeanUtils; import cn.iocoder.yudao.framework.tenant.core.util.TenantUtils; import cn.iocoder.yudao.module.ai.controller.admin.chat.vo.message.AiChatMessagePageReqVO; @@ -22,6 +24,7 @@ import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiModelDO; import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiToolDO; import cn.iocoder.yudao.module.ai.dal.mysql.chat.AiChatMessageMapper; import cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants; +import cn.iocoder.yudao.module.ai.enums.model.AiModelInputModalityEnum; import cn.iocoder.yudao.module.ai.enums.model.AiPlatformEnum; import cn.iocoder.yudao.module.ai.framework.ai.core.webserch.AiWebSearchClient; import cn.iocoder.yudao.module.ai.framework.ai.core.webserch.AiWebSearchRequest; @@ -48,13 +51,16 @@ import org.springframework.ai.chat.model.ChatResponse; import org.springframework.ai.chat.model.StreamingChatModel; import org.springframework.ai.chat.prompt.ChatOptions; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.content.Media; import org.springframework.ai.mcp.SyncMcpToolCallbackProvider; import org.springframework.ai.mcp.client.common.autoconfigure.properties.McpClientCommonProperties; import org.springframework.ai.tool.ToolCallback; import org.springframework.ai.tool.resolution.ToolCallbackResolver; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ByteArrayResource; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.MimeTypeUtils; import reactor.core.publisher.Flux; import java.time.LocalDateTime; @@ -70,6 +76,7 @@ import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils. import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertSet; import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.CHAT_CONVERSATION_NOT_EXISTS; import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.CHAT_MESSAGE_NOT_EXIST; +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.CHAT_MESSAGE_IMAGE_READ_FAIL; /** * AI 聊天消息 Service 实现类 @@ -149,6 +156,8 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { // 1.2 校验模型 AiModelDO model = modalService.validateModel(conversation.getModelId()); ChatModel chatModel = modalService.getChatModel(model.getId()); + // 1.3 选出上下文消息：附件按各自所属消息组装 + List contextMessages = filterContextMessages(historyMessages, conversation, sendReqVO); // 2.1 知识库召回 List knowledgeSegments = recallKnowledgeSegment( @@ -159,6 +168,9 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { webSearchClient.search(new AiWebSearchRequest().setQuery(sendReqVO.getContent()) .setSummary(true).setCount(WEB_SEARCH_COUNT)) : null; + // 2.3 创建 chat 需要的 Prompt：在插入消息前构建，图片读取失败时不留下消息 + Prompt prompt = buildPrompt(conversation, contextMessages, knowledgeSegments, webSearchResponse, model, sendReqVO); + // 3. 插入 user 发送消息 AiChatMessageDO userMessage = createChatMessage(conversation.getId(), null, model, userId, conversation.getRoleId(), MessageType.USER, sendReqVO.getContent(), sendReqVO.getUseContext(), @@ -169,8 +181,7 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { userId, conversation.getRoleId(), MessageType.ASSISTANT, "", sendReqVO.getUseContext(), knowledgeSegments, null, webSearchResponse); - // 4.2 创建 chat 需要的 Prompt - Prompt prompt = buildPrompt(conversation, historyMessages, knowledgeSegments, webSearchResponse, model, sendReqVO); + // 4.2 调用模型 ChatResponse chatResponse = chatModel.call(prompt); // 4.3 更新响应内容 @@ -206,6 +217,8 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { // 1.2 校验模型 AiModelDO model = modalService.validateModel(conversation.getModelId()); StreamingChatModel chatModel = modalService.getChatModel(model.getId()); + // 1.3 选出上下文消息：附件按各自所属消息组装 + List contextMessages = filterContextMessages(historyMessages, conversation, sendReqVO); // 2.1 知识库找回 List knowledgeSegments = recallKnowledgeSegment( @@ -216,6 +229,14 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { webSearchClient.search(new AiWebSearchRequest().setQuery(sendReqVO.getContent()) .setSummary(true).setCount(WEB_SEARCH_COUNT)) : null; + // 2.3 构建 Prompt：在插入消息前构建，图片读取失败时返回错误帧，不留下消息 + Prompt prompt; + try { + prompt = buildPrompt(conversation, contextMessages, knowledgeSegments, webSearchResponse, model, sendReqVO); + } catch (ServiceException ex) { + return Flux.just(error(ex.getCode(), ex.getMessage())); + } + // 3. 插入 user 发送消息 AiChatMessageDO userMessage = createChatMessage(conversation.getId(), null, model, userId, conversation.getRoleId(), MessageType.USER, sendReqVO.getContent(), sendReqVO.getUseContext(), @@ -226,8 +247,7 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { userId, conversation.getRoleId(), MessageType.ASSISTANT, "", sendReqVO.getUseContext(), knowledgeSegments, null, webSearchResponse); - // 4.2 构建 Prompt，并进行调用 - Prompt prompt = buildPrompt(conversation, historyMessages, knowledgeSegments, webSearchResponse, model, sendReqVO); + // 4.2 调用模型 Flux streamResponse = chatModel.stream(prompt); // 4.3 流式返回 @@ -322,7 +342,7 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { return knowledgeSegments; } - private Prompt buildPrompt(AiChatConversationDO conversation, List messages, + private Prompt buildPrompt(AiChatConversationDO conversation, List contextMessages, List knowledgeSegments, AiWebSearchResponse webSearchResponse, AiModelDO model, AiChatMessageSendReqVO sendReqVO) { @@ -332,19 +352,20 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { chatMessages.add(new SystemMessage(conversation.getSystemMessage())); } - // 1.2 历史 history message 历史消息 - List contextMessages = filterContextMessages(messages, conversation, sendReqVO); + // 1.2 历史 history message 历史消息（已由 filterContextMessages 选出）；user 消息带上该轮的图片 contextMessages.forEach(message -> { - chatMessages.add(AiUtils.buildMessage(message.getType(), message.getContent())); - UserMessage attachmentUserMessage = buildAttachmentUserMessage(message.getAttachmentUrls()); + chatMessages.add(MessageType.USER.getValue().equals(message.getType()) + ? buildUserMessage(message.getContent(), message.getAttachmentUrls(), model) + : AiUtils.buildMessage(message.getType(), message.getContent())); + UserMessage attachmentUserMessage = buildAttachmentUserMessage(message.getAttachmentUrls(), model); if (attachmentUserMessage != null) { chatMessages.add(attachmentUserMessage); } // TODO @芋艿：历史的知识库；历史的搜索，要不要拼接？ }); - // 1.3 当前 user message 新发送消息 - chatMessages.add(new UserMessage(sendReqVO.getContent())); + // 1.3 当前 user message 新发送消息：文本和本轮图片放在同一条 UserMessage + chatMessages.add(buildUserMessage(sendReqVO.getContent(), sendReqVO.getAttachmentUrls(), model)); // 1.4 知识库，通过 UserMessage 实现 if (CollUtil.isNotEmpty(knowledgeSegments)) { @@ -369,7 +390,7 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { // 1.6 附件，通过 UserMessage 实现 if (CollUtil.isNotEmpty(sendReqVO.getAttachmentUrls())) { - UserMessage attachmentUserMessage = buildAttachmentUserMessage(sendReqVO.getAttachmentUrls()); + UserMessage attachmentUserMessage = buildAttachmentUserMessage(sendReqVO.getAttachmentUrls(), model); if (attachmentUserMessage != null) { chatMessages.add(attachmentUserMessage); } @@ -381,12 +402,24 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { : Map.of(); // 2.2 构建 ChatOptions 对象 AiPlatformEnum platform = AiPlatformEnum.validatePlatform(model.getPlatform()); + boolean hasMedia = CollUtil.anyMatch(chatMessages, message -> message instanceof UserMessage + && CollUtil.isNotEmpty(((UserMessage) message).getMedia())); ChatOptions chatOptions = AiUtils.buildChatOptions(platform, model.getModel(), conversation.getTemperature(), conversation.getMaxTokens(), - toolCallbacks, toolContext); + toolCallbacks, toolContext, hasMedia); return new Prompt(chatMessages, chatOptions); } + + /** + * 获得角色关联的工具回调列表 + * + * 1. 通过角色的 toolIds，解析本地注册的工具 + * 2. 通过角色的 mcpClientNames，匹配已连接的 MCP Client，获取它们提供的工具 + * + * @param roleId 角色编号 + * @return 工具回调列表；角色编号为空、或角色不存在时，返回 null + */ private List getToolCallbackListByRoleId(Long roleId) { if (roleId == null) { return null; @@ -464,15 +497,19 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { return contextMessages; } - private UserMessage buildAttachmentUserMessage(List attachmentUrls) { + private UserMessage buildAttachmentUserMessage(List attachmentUrls, AiModelDO model) { if (CollUtil.isEmpty(attachmentUrls)) { return null; } // 读取文件内容 Map attachmentContents = Maps.newLinkedHashMapWithExpectedSize(attachmentUrls.size()); for (String attachmentUrl : attachmentUrls) { + // 模型配置了 image 输入的图片，已通过 UserMessage 的 Media 传输 + if (isMediaImage(attachmentUrl, model)) { + continue; + } try { - String name = FileNameUtil.getName(attachmentUrl); + String name = getAttachmentFileName(attachmentUrl); String mineType = FileTypeUtils.getMineType(name); String content; if (FileTypeUtils.isImage(mineType)) { @@ -500,6 +537,64 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { return new UserMessage(String.format(Attachment_USER_MESSAGE_TEMPLATE, attachment)); } + /** + * 构建 user 消息：文本和该轮的图片放在同一条 {@link UserMessage}，图片以 Resource 形式作为 {@link Media} + * + * @param content 文本 + * @param attachmentUrls 该轮附件 URL 数组 + * @param model 模型 + * @return user 消息；没有图片时为纯文本消息。图片读取失败时抛出异常，不降级为纯文本 + */ + private UserMessage buildUserMessage(String content, List attachmentUrls, AiModelDO model) { + // 1. 读取该轮图片：仅模型配置了 image 输入的图片，以 Resource 作为 Media；读取失败直接报错，不降级为纯文本 + List medias = new ArrayList<>(); + if (CollUtil.isNotEmpty(attachmentUrls)) { + for (String url : attachmentUrls) { + if (!isMediaImage(url, model)) { + continue; + } + try { + String mineType = FileTypeUtils.getMineType(getAttachmentFileName(url)); + medias.add(new Media(MimeTypeUtils.parseMimeType(mineType), new ByteArrayResource(HttpUtil.downloadBytes(url)))); + } catch (Exception e) { + log.error("[buildUserMessage][读取图片({}) 发生异常]", url, e); + throw exception(CHAT_MESSAGE_IMAGE_READ_FAIL, getAttachmentFileName(url)); + } + } + } + // 2.1 没有图片：纯文本的 user 消息 + if (CollUtil.isEmpty(medias)) { + return new UserMessage(content); + } + // 2.2 有图片：文本和图片放在同一条 user 消息 + return UserMessage.builder().text(content).media(medias).build(); + } + + /** + * 判断附件是否作为 {@link Media} 传输：是图片，并且模型配置了 image 输入 + * + * 其它附件（未配置 image 的图片、视频、文档）走原来的 Attachment 文本逻辑 + * + * @param url 附件 URL + * @param model 模型 + * @return 是否作为 Media 传输 + */ + private static boolean isMediaImage(String url, AiModelDO model) { + return FileTypeUtils.isImage(FileTypeUtils.getMineType(getAttachmentFileName(url))) + && AiModelInputModalityEnum.isSupported(model.getInputModalities(), AiModelInputModalityEnum.IMAGE); + } + + /** + * 获得附件的文件名：复用 {@link HttpUtils#removeUrlPathQueryAndFragment(String)} 去掉 ?query 和 #fragment + * （例如签名 URL 的 X-Amz-Signature），再取文件名 + * + * @param url 附件 URL + * @return 文件名 + */ + private static String getAttachmentFileName(String url) { + return FileNameUtil.getName(HttpUtils.removeUrlPathQueryAndFragment(url)); + } + private AiChatMessageDO createChatMessage(Long conversationId, Long replyId, AiModelDO model, Long userId, Long roleId, MessageType messageType, String content, Boolean useContext, diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/util/AiUtils.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/util/AiUtils.java index 95fe446..c8fed63 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/util/AiUtils.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/util/AiUtils.java @@ -93,6 +93,17 @@ public class AiUtils { public static ChatOptions buildChatOptions(AiPlatformEnum platform, String model, Double temperature, Integer maxTokens, List toolCallbacks, Map toolContext) { + return buildChatOptions(platform, model, temperature, maxTokens, toolCallbacks, toolContext, false); + } + + /** + * 构建 ChatOptions + * + * @param hasMedia 本次 Prompt 是否含 Media（包括选入上下文的历史消息）。通义含 Media 时必须走多模态接口 + */ + public static ChatOptions buildChatOptions(AiPlatformEnum platform, String model, Double temperature, Integer maxTokens, + List toolCallbacks, Map toolContext, + boolean hasMedia) { toolCallbacks = ObjUtil.defaultIfNull(toolCallbacks, Collections.emptyList()); toolContext = ObjUtil.defaultIfNull(toolContext, Collections.emptyMap()); // noinspection EnhancedSwitchMigration @@ -100,7 +111,7 @@ public class AiUtils { case TONG_YI: return DashScopeChatOptions.builder().model(model).temperature(temperature).maxToken(maxTokens) .enableThinking(true) // TODO 芋艿：默认都开启 thinking 模式，后续可以让用户配置 - .multiModel(TONG_YI_MULTI_MODELS.contains(model)) // 是否多模态模型 + .multiModel(hasMedia || TONG_YI_MULTI_MODELS.contains(model)) // 含 Media 必须走多模态接口；纯文本按名单保持原通道 .toolCallbacks(toolCallbacks).toolContext(toolContext).build(); case DEEP_SEEK: case DOU_BAO: // 复用 DeepSeek 客户端 diff --git a/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/chat/DeepSeekChatModelTests.java b/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/chat/DeepSeekChatModelTests.java index 64a28f8..f29057d 100644 --- a/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/chat/DeepSeekChatModelTests.java +++ b/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/chat/DeepSeekChatModelTests.java @@ -29,7 +29,7 @@ public class DeepSeekChatModelTests { private static final String API_KEY = SystemUtil.get("DEEPSEEK_API_KEY", "sk-xxxx"); private static final String MODEL = SystemUtil.get("DEEPSEEK_MODEL", - "deepseek-v4-flash"); + "deepseek-flash"); private final DeepSeekChatModel chatModel = DeepSeekChatModel.builder() .deepSeekApi(DeepSeekApi.builder() diff --git a/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/chat/TongYiChatModelTests.java b/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/chat/TongYiChatModelTests.java index 056709d..3b56ec3 100644 --- a/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/chat/TongYiChatModelTests.java +++ b/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/framework/ai/core/model/chat/TongYiChatModelTests.java @@ -1,5 +1,6 @@ package cn.iocoder.yudao.module.ai.framework.ai.core.model.chat; +import cn.hutool.system.SystemUtil; import cn.iocoder.yudao.framework.common.util.json.JsonUtils; import com.alibaba.cloud.ai.dashscope.api.DashScopeApi; import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatModel; @@ -32,13 +33,16 @@ import static java.util.Arrays.asList; */ public class TongYiChatModelTests { + private static final String API_KEY = SystemUtil.get("DASHSCOPE_API_KEY", + "sk-xxxx"); // 按需改成你的 DashScope API Key + private final DashScopeChatModel chatModel = DashScopeChatModel.builder() .dashScopeApi(DashScopeApi.builder() - .apiKey("sk-cd9f39e99ea54840bd1888282325f55a") // https://bailian.console.aliyun.com/cn-beijing/?tab=model#/api-key 获取密钥 + .apiKey(API_KEY) // https://bailian.console.aliyun.com/cn-beijing/?tab=model#/api-key 获取密钥 .build()) .defaultOptions(DashScopeChatOptions.builder() - .multiModel(true) // 注意：当使用 qwen3.6-plus 等多模态模型，需要设置为 true，可见 https://help.aliyun.com/zh/model-studio/error-code#error-url 链接 - .model("qwen3.6-plus") // 模型 + .multiModel(true) // 注意：当使用 qwen3.8-max 等多模态模型，需要设置为 true，可见 https://help.aliyun.com/zh/model-studio/error-code#error-url 链接 + .model("qwen3.8-max") // 模型 // .model("deepseek-r1") // 模型（deepseek-r1） // .model("deepseek-v3") // 模型（deepseek-v3） // .model("deepseek-r1-distill-qwen-1.5b") // 模型（deepseek-r1-distill-qwen-1.5b） @@ -85,7 +89,7 @@ public class TongYiChatModelTests { List messages = new ArrayList<>(); messages.add(new UserMessage("详细分析下，如何设计一个电商系统？")); DashScopeChatOptions options = DashScopeChatOptions.builder() - .model("qwen3.6-plus").multiModel(true) + .model("qwen3.8-max").multiModel(true) // .withModel("qwen-max-2025-01-25") .enableThinking(true) // 必须设置，否则会报错 .build(); @@ -105,7 +109,7 @@ public class TongYiChatModelTests { // 准备环境 RerankModel rerankModel = new DashScopeRerankModel( DashScopeApi.builder() - .apiKey("sk-47aa124781be4bfb95244cc62f63f7d0") + .apiKey(API_KEY) .build()); // 准备参数 String query = "spring"; diff --git a/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImplTest.java b/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImplTest.java index b1d0f02..43eeba4 100644 --- a/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImplTest.java +++ b/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImplTest.java @@ -1,5 +1,21 @@ package cn.iocoder.yudao.module.ai.service.chat; +import cn.hutool.core.codec.Base64; +import cn.hutool.core.util.StrUtil; +import cn.hutool.http.HttpUtil; +import cn.iocoder.yudao.module.ai.enums.model.AiModelInputModalityEnum; +import org.mockito.MockedStatic; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.MessageType; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.content.Media; +import java.util.ArrayList; +import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatOptions; import cn.hutool.core.collection.CollUtil; import cn.iocoder.yudao.framework.common.pojo.CommonResult; import cn.iocoder.yudao.framework.security.core.LoginUser; @@ -50,6 +66,8 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; +import static cn.iocoder.yudao.framework.test.core.util.AssertUtils.assertServiceException; +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.CHAT_MESSAGE_IMAGE_READ_FAIL; import static cn.iocoder.yudao.framework.test.core.util.RandomUtils.randomLongId; import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.CHAT_STREAM_ERROR; import static cn.iocoder.yudao.module.ai.util.AiUtils.TOOL_CONTEXT_LOGIN_USER; @@ -377,6 +395,348 @@ public class AiChatMessageServiceImplTest extends BaseMockitoUnitTest { : ChatCompletionFinishReason.STOP; } + // ========== 附件多模态：模型配置 image 的图片走 Media，其它附件走原 Attachment 文本 ========== + + /** + * 自定义型号：不在任何已知名单里，是否传图只看模型配置的 inputModalities + */ + private static final String CUSTOM_MODEL_NAME = "my-vlm-x"; + private static final String IMAGE_URL = "https://www.iocoder.cn/a.png?X-Amz-Signature=xxx#v1"; + private static final String OLD_IMAGE_URL = "https://www.iocoder.cn/old.png"; + private static final String PDF_URL = "https://www.iocoder.cn/b.pdf"; + private static final String VIDEO_URL = "https://www.iocoder.cn/c.mp4"; + private static final byte[] IMAGE_BYTES = new byte[]{1, 2, 3}; + private static final List TEXT_IMAGE = asList(AiModelInputModalityEnum.TEXT.getModality(), + AiModelInputModalityEnum.IMAGE.getModality()); + + @Mock + private ChatModel chatModel; + + @Test + public void testSendMessage_imageWithImageModality() { + // 准备参数：签名 URL 的图片 + pdf 文档 + Long userId = randomLongId(); + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId()) + .setContent("这是我的车？").setAttachmentUrls(asList(IMAGE_URL, PDF_URL)); + // mock 对话、模型：自定义型号配置了 image + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE); + // mock 方法 + when(knowledgeDocumentService.readUrl(eq(PDF_URL))).thenReturn("文档内容"); + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse()); + + try (MockedStatic httpUtilMock = mockStatic(HttpUtil.class)) { + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL))).thenReturn(IMAGE_BYTES); + // 调用 + chatMessageService.sendMessage(reqVO, userId); + } + // 断言：图片和问题在同一条 UserMessage 中，以 Media 传输 + Prompt prompt = captureCallPrompt(); + UserMessage currentMessage = findUserMessage(prompt, reqVO.getContent()); + assertEquals(1, currentMessage.getMedia().size()); + Media media = CollUtil.getFirst(currentMessage.getMedia()); + assertEquals("image/png", media.getMimeType().toString()); + assertArrayEquals(IMAGE_BYTES, (byte[]) media.getData()); + // 断言：文档走 Attachment 文本，图片不再拼进文本 + UserMessage attachmentMessage = findUserMessage(prompt, " httpUtilMock = mockStatic(HttpUtil.class)) { + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL))).thenReturn(IMAGE_BYTES); + // 调用：不拒绝 + chatMessageService.sendMessage(reqVO, userId); + } + // 断言：没有 Media，图片沿用原逻辑，以 Base64 拼进 Attachment 文本 + Prompt prompt = captureCallPrompt(); + assertNoMedia(prompt); + UserMessage attachmentMessage = findUserMessage(prompt, " httpUtilMock = mockStatic(HttpUtil.class)) { + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL))).thenReturn(IMAGE_BYTES); + // 调用 + chatMessageService.sendMessage(reqVO, userId); + // 断言：未选入上下文的历史图片不下载 + httpUtilMock.verify(() -> HttpUtil.downloadBytes(eq(OLD_IMAGE_URL)), never()); + } + // 断言：历史图片挂在它所属的 user 消息上；当前消息没有图片 + Prompt prompt = captureCallPrompt(); + assertEquals(1, findUserMessage(prompt, "上一问").getMedia().size()); + assertTrue(CollUtil.isEmpty(findUserMessage(prompt, reqVO.getContent()).getMedia())); + assertNull(findUserMessageOrNull(prompt, "旧问题")); + } + + @Test + public void testSendMessage_historyImageWithoutContext() { + // 准备参数：不使用上下文 + Long userId = randomLongId(); + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId()) + .setContent("这是我的车？").setUseContext(false); + // mock 对话、模型 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE); + when(chatMessageMapper.selectListByConversationId(eq(reqVO.getConversationId()))) + .thenReturn(buildHistoryMessages()); + // mock 方法 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse()); + + try (MockedStatic httpUtilMock = mockStatic(HttpUtil.class)) { + // 调用 + chatMessageService.sendMessage(reqVO, userId); + // 断言：历史图片不下载 + httpUtilMock.verify(() -> HttpUtil.downloadBytes(any(String.class)), never()); + } + // 断言：没有 Media，也没有历史消息 + Prompt prompt = captureCallPrompt(); + assertNoMedia(prompt); + assertNull(findUserMessageOrNull(prompt, "上一问")); + } + + @Test + public void testSendMessage_imageReadFail() { + // 准备参数 + Long userId = randomLongId(); + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId()) + .setContent("这是我的车？").setAttachmentUrls(singletonList(IMAGE_URL)); + // mock 对话、模型：配置了 image + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE); + + try (MockedStatic httpUtilMock = mockStatic(HttpUtil.class)) { + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL))) + .thenThrow(new IllegalStateException("下载失败")); + // 调用，并断言异常 + assertServiceException(() -> chatMessageService.sendMessage(reqVO, userId), + CHAT_MESSAGE_IMAGE_READ_FAIL, "a.png"); + } + // 断言：不调用模型、不写消息 + verify(chatModel, never()).call(any(Prompt.class)); + verify(chatMessageMapper, never()).insert(any(AiChatMessageDO.class)); + } + + @Test + public void testSendChatMessageStream_imageReadFail() { + // 准备参数 + Long userId = randomLongId(); + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId()) + .setContent("这是我的车？").setAttachmentUrls(singletonList(IMAGE_URL)); + // mock 对话、模型：配置了 image + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE); + + List> results; + try (MockedStatic httpUtilMock = mockStatic(HttpUtil.class)) { + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL))) + .thenThrow(new IllegalStateException("下载失败")); + // 调用 + results = chatMessageService.sendChatMessageStream(reqVO, userId).collectList().block(); + } + // 断言：返回错误帧，不调用模型、不写消息 + assertNotNull(results); + assertEquals(1, results.size()); + assertEquals(CHAT_MESSAGE_IMAGE_READ_FAIL.getCode(), results.get(0).getCode()); + verify(chatModel, never()).stream(any(Prompt.class)); + verify(chatMessageMapper, never()).insert(any(AiChatMessageDO.class)); + } + + @Test + public void testSendChatMessageStream_imageWithImageModality() { + // 准备参数 + Long userId = randomLongId(); + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId()) + .setContent("这是我的车？").setAttachmentUrls(singletonList(IMAGE_URL)); + // mock 对话、模型：配置了 image + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE); + // mock 方法 + when(chatModel.stream(any(Prompt.class))).thenReturn(Flux.just(buildChatResponse())); + + List> results; + try (MockedStatic httpUtilMock = mockStatic(HttpUtil.class)) { + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL))).thenReturn(IMAGE_BYTES); + // 调用 + results = chatMessageService.sendChatMessageStream(reqVO, userId).collectList().block(); + } + // 断言：正常返回，图片以 Media 传输 + assertNotNull(results); + assertTrue(CollUtil.getLast(results).isSuccess()); + ArgumentCaptor promptCaptor = ArgumentCaptor.forClass(Prompt.class); + verify(chatModel).stream(promptCaptor.capture()); + assertEquals(1, findUserMessage(promptCaptor.getValue(), reqVO.getContent()).getMedia().size()); + } + + @Test + public void testSendMessage_video() { + // 准备参数 + Long userId = randomLongId(); + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId()) + .setContent("视频里有什么？").setAttachmentUrls(singletonList(VIDEO_URL)); + // mock 对话、模型：即使配置了 video，本期也不实现视频传输 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, + asList(AiModelInputModalityEnum.TEXT.getModality(), AiModelInputModalityEnum.IMAGE.getModality(), + AiModelInputModalityEnum.VIDEO.getModality())); + // mock 方法 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse()); + + try (MockedStatic httpUtilMock = mockStatic(HttpUtil.class)) { + // 调用：不拒绝 + chatMessageService.sendMessage(reqVO, userId); + // 断言：视频不当作图片下载 + httpUtilMock.verify(() -> HttpUtil.downloadBytes(any(String.class)), never()); + } + // 断言：没有 Media，视频沿用原 Attachment 逻辑 + assertNoMedia(captureCallPrompt()); + verify(knowledgeDocumentService).readUrl(eq(VIDEO_URL)); + } + + @Test + public void testSendMessage_textOnly() { + // 准备参数 + Long userId = randomLongId(); + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId()) + .setContent("你好"); + // mock 对话、模型 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.DEEP_SEEK, TEXT_IMAGE); + // mock 方法 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse()); + + // 调用 + chatMessageService.sendMessage(reqVO, userId); + // 断言：纯文本，没有 Media + Prompt prompt = captureCallPrompt(); + assertNoMedia(prompt); + assertEquals(reqVO.getContent(), findUserMessage(prompt, reqVO.getContent()).getText()); + } + + @Test + public void testSendMessage_tongYiMultiModelByMedia() { + // 准备参数：当前消息纯文本，历史消息带图 + Long userId = randomLongId(); + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId()) + .setContent("这是我的车？").setUseContext(true); + // mock 对话、模型：通义的自定义型号，不在 TONG_YI_MULTI_MODELS 名单 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.TONG_YI, TEXT_IMAGE); + when(chatMessageMapper.selectListByConversationId(eq(reqVO.getConversationId()))) + .thenReturn(buildHistoryMessages()); + // mock 方法 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse()); + + try (MockedStatic httpUtilMock = mockStatic(HttpUtil.class)) { + httpUtilMock.when(() -> HttpUtil.downloadBytes(eq(IMAGE_URL))).thenReturn(IMAGE_BYTES); + // 调用 + chatMessageService.sendMessage(reqVO, userId); + } + // 断言：Prompt 含历史 Media，走多模态接口 + DashScopeChatOptions options = (DashScopeChatOptions) captureCallPrompt().getOptions(); + assertEquals(Boolean.TRUE, options.getMultiModel()); + } + + @Test + public void testSendMessage_tongYiTextOnly() { + // 准备参数 + Long userId = randomLongId(); + AiChatMessageSendReqVO reqVO = new AiChatMessageSendReqVO().setConversationId(randomLongId()) + .setContent("你好"); + // mock 对话、模型：通义的自定义型号 + mockMediaConversation(reqVO.getConversationId(), userId, AiPlatformEnum.TONG_YI, TEXT_IMAGE); + // mock 方法 + when(chatModel.call(any(Prompt.class))).thenReturn(buildChatResponse()); + + // 调用 + chatMessageService.sendMessage(reqVO, userId); + // 断言：纯文本保持原通道 + DashScopeChatOptions options = (DashScopeChatOptions) captureCallPrompt().getOptions(); + assertEquals(Boolean.FALSE, options.getMultiModel()); + } + + + private void mockMediaConversation(Long conversationId, Long userId, AiPlatformEnum platform, + List inputModalities) { + // mock 对话 + AiChatConversationDO conversation = new AiChatConversationDO().setId(conversationId).setUserId(userId) + .setModelId(randomLongId()).setTemperature(TEMPERATURE).setMaxTokens(MAX_TOKENS).setMaxContexts(1); + when(chatConversationService.validateChatConversationExists(eq(conversationId))).thenReturn(conversation); + // mock 模型：mock 的 ChatModel，用于捕获 Prompt + AiModelDO model = new AiModelDO().setId(conversation.getModelId()).setName("自定义视觉模型") + .setModel(CUSTOM_MODEL_NAME).setPlatform(platform.getPlatform()).setInputModalities(inputModalities); + when(modalService.validateModel(eq(model.getId()))).thenReturn(model); + when(modalService.getChatModel(eq(model.getId()))).thenReturn(chatModel); + } + + /** + * 构建历史消息：两组 user + assistant，各带一张图片 + */ + private static List buildHistoryMessages() { + List messages = new ArrayList<>(); + messages.add(new AiChatMessageDO().setId(11L).setType(MessageType.USER.getValue()) + .setContent("旧问题").setAttachmentUrls(singletonList(OLD_IMAGE_URL))); + messages.add(new AiChatMessageDO().setId(12L).setReplyId(11L).setType(MessageType.ASSISTANT.getValue()) + .setContent("旧回答")); + messages.add(new AiChatMessageDO().setId(13L).setType(MessageType.USER.getValue()) + .setContent("上一问").setAttachmentUrls(singletonList(IMAGE_URL))); + messages.add(new AiChatMessageDO().setId(14L).setReplyId(13L).setType(MessageType.ASSISTANT.getValue()) + .setContent("上一答")); + return messages; + } + + private Prompt captureCallPrompt() { + ArgumentCaptor promptCaptor = ArgumentCaptor.forClass(Prompt.class); + verify(chatModel).call(promptCaptor.capture()); + return promptCaptor.getValue(); + } + + private static UserMessage findUserMessage(Prompt prompt, String text) { + UserMessage message = findUserMessageOrNull(prompt, text); + assertNotNull(message, "未找到包含(" + text + ")的 UserMessage"); + return message; + } + + private static UserMessage findUserMessageOrNull(Prompt prompt, String text) { + for (Message message : prompt.getInstructions()) { + if (message instanceof UserMessage && StrUtil.contains(message.getText(), text)) { + return (UserMessage) message; + } + } + return null; + } + + private static void assertNoMedia(Prompt prompt) { + for (Message message : prompt.getInstructions()) { + if (message instanceof UserMessage) { + assertTrue(CollUtil.isEmpty(((UserMessage) message).getMedia())); + } + } + } + + private static ChatResponse buildChatResponse() { + return new ChatResponse(singletonList(new Generation(new AssistantMessage("好的")))); + } + @Data public static class WeatherRequest { diff --git a/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/service/model/AiModelServiceImplTest.java b/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/service/model/AiModelServiceImplTest.java new file mode 100644 index 0000000..38c42d6 --- /dev/null +++ b/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/service/model/AiModelServiceImplTest.java @@ -0,0 +1,109 @@ +package cn.iocoder.yudao.module.ai.service.model; + +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.common.util.validation.ValidationUtils; +import cn.iocoder.yudao.framework.test.core.ut.BaseMockitoUnitTest; +import cn.iocoder.yudao.module.ai.controller.admin.model.vo.model.AiModelSaveReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiModelDO; +import cn.iocoder.yudao.module.ai.dal.mysql.model.AiChatMapper; +import cn.iocoder.yudao.module.ai.enums.model.AiModelTypeEnum; +import cn.iocoder.yudao.module.ai.enums.model.AiPlatformEnum; +import cn.iocoder.yudao.module.ai.framework.ai.core.model.AiModelFactory; +import jakarta.validation.ConstraintViolationException; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; + +import java.util.List; + +import static cn.iocoder.yudao.framework.test.core.util.RandomUtils.randomLongId; +import static java.util.Arrays.asList; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * {@link AiModelServiceImpl} 的单元测试类 + * + * 重点：输入模态 inputModalities 按前端传入的数组原样保存，后端不做规范化 + * + * @author 芋道源码 + */ +public class AiModelServiceImplTest extends BaseMockitoUnitTest { + + @InjectMocks + private AiModelServiceImpl modelService; + + @Mock + private AiApiKeyService apiKeyService; + @Mock + private AiChatMapper modelMapper; + @Mock + private AiModelFactory modelFactory; + + @Test + public void testCreateModel_inputModalities() { + // 准备参数：顺序、重复按前端原样 + List inputModalities = asList("image", "text"); + AiModelSaveReqVO reqVO = buildSaveReqVO(null).setInputModalities(inputModalities); + + // 调用 + modelService.createModel(reqVO); + // 断言 + ArgumentCaptor modelCaptor = ArgumentCaptor.forClass(AiModelDO.class); + verify(modelMapper).insert(modelCaptor.capture()); + assertEquals(inputModalities, modelCaptor.getValue().getInputModalities()); + } + + @Test + public void testUpdateModel_inputModalities() { + // mock 数据：库里原有 image、video + Long id = randomLongId(); + when(modelMapper.selectById(eq(id))).thenReturn(new AiModelDO().setId(id) + .setInputModalities(asList("text", "image", "video"))); + // 准备参数：前端只传 text + List inputModalities = asList("text"); + AiModelSaveReqVO reqVO = buildSaveReqVO(id).setInputModalities(inputModalities); + + // 调用 + modelService.updateModel(reqVO); + // 断言：直接采用前端数组，不合并库里旧值 + ArgumentCaptor modelCaptor = ArgumentCaptor.forClass(AiModelDO.class); + verify(modelMapper).updateById(modelCaptor.capture()); + assertEquals(inputModalities, modelCaptor.getValue().getInputModalities()); + } + + @Test + public void testSaveReqVO_inputModalitiesValid() { + // 准备参数 + AiModelSaveReqVO reqVO = buildSaveReqVO(null).setInputModalities(asList("text", "image", "video")); + + // 调用，并断言：合法枚举值通过 + assertDoesNotThrow(() -> ValidationUtils.validate(reqVO)); + } + + @Test + public void testSaveReqVO_inputModalitiesInvalid() { + // 准备参数：audio 不在枚举中 + AiModelSaveReqVO reqVO = buildSaveReqVO(null).setInputModalities(asList("text", "audio")); + + // 调用，并断言：@InEnum 拒绝非法值 + assertThrows(ConstraintViolationException.class, () -> ValidationUtils.validate(reqVO)); + } + + private static AiModelSaveReqVO buildSaveReqVO(Long id) { + AiModelSaveReqVO reqVO = new AiModelSaveReqVO(); + reqVO.setId(id); + reqVO.setKeyId(randomLongId()); + reqVO.setName("自定义视觉模型"); + reqVO.setModel("my-vlm-x"); + reqVO.setPlatform(AiPlatformEnum.DEEP_SEEK.getPlatform()); + reqVO.setType(AiModelTypeEnum.CHAT.getType()); + reqVO.setSort(1); + reqVO.setStatus(CommonStatusEnum.ENABLE.getStatus()); + return reqVO; + } + +} diff --git a/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/util/AiUtilsTest.java b/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/util/AiUtilsTest.java new file mode 100644 index 0000000..35a8e62 --- /dev/null +++ b/yudao-module-ai/src/test/java/cn/iocoder/yudao/module/ai/util/AiUtilsTest.java @@ -0,0 +1,54 @@ +package cn.iocoder.yudao.module.ai.util; + +import cn.iocoder.yudao.module.ai.enums.model.AiPlatformEnum; +import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatOptions; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * {@link AiUtils} 的单元测试类 + * + * 重点：通义的 multiModel 按本次 Prompt 是否含 Media 选择请求通道，不受型号名单限制 + * + * @author 芋道源码 + */ +public class AiUtilsTest { + + /** + * 自定义型号：不在 {@link AiUtils#TONG_YI_MULTI_MODELS} 名单中 + */ + private static final String CUSTOM_MODEL = "my-qwen-vl"; + /** + * 名单内的型号 + */ + private static final String LISTED_MODEL = "qwen3-vl-plus"; + + @Test + public void testBuildChatOptions_tongYiCustomModelWithMedia() { + // 调用 + DashScopeChatOptions options = (DashScopeChatOptions) AiUtils.buildChatOptions(AiPlatformEnum.TONG_YI, + CUSTOM_MODEL, 0.7, 1024, null, null, true); + // 断言：含 Media 走多模态接口 + assertEquals(Boolean.TRUE, options.getMultiModel()); + } + + @Test + public void testBuildChatOptions_tongYiCustomModelTextOnly() { + // 调用 + DashScopeChatOptions options = (DashScopeChatOptions) AiUtils.buildChatOptions(AiPlatformEnum.TONG_YI, + CUSTOM_MODEL, 0.7, 1024, null, null, false); + // 断言：纯文本保持原通道 + assertEquals(Boolean.FALSE, options.getMultiModel()); + } + + @Test + public void testBuildChatOptions_tongYiListedModelTextOnly() { + // 调用：原有重载，默认不含 Media + DashScopeChatOptions options = (DashScopeChatOptions) AiUtils.buildChatOptions(AiPlatformEnum.TONG_YI, + LISTED_MODEL, 0.7, 1024); + // 断言：名单内型号纯文本仍按名单走多模态接口，与改动前一致 + assertEquals(Boolean.TRUE, options.getMultiModel()); + } + +}