Commit acd36258be5691d577d5f045ea3dc9c87fc1272b

Authored by 芋道源码
Committed by Gitee
2 parents f7bed6d8 1596ca33

!1096 合并最新知识库的进展

Merge pull request !1096 from 芋道源码/master-jdk21-ai
Showing 32 changed files with 456 additions and 202 deletions
yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/AiChatRoleEnum.java
@@ -34,7 +34,12 @@ public enum AiChatRoleEnum { @@ -34,7 +34,12 @@ public enum AiChatRoleEnum {
34 ### 支付宝 34 ### 支付宝
35 ### 微信 35 ### 微信
36 除此之外不要任何解释性语句。 36 除此之外不要任何解释性语句。
37 - """); 37 + """),
  38 +
  39 + AI_KNOWLEDGE_ROLE("知识库助手", """
  40 + 给你提供一些数据参考:{info},请回答我的问题。
  41 + 请你跟进数据参考与工具返回结果回复用户的请求。
  42 + """);
38 43
39 /** 44 /**
40 * 角色名 45 * 角色名
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/vo/conversation/AiChatConversationCreateMyReqVO.java
@@ -10,4 +10,7 @@ public class AiChatConversationCreateMyReqVO { @@ -10,4 +10,7 @@ public class AiChatConversationCreateMyReqVO {
10 @Schema(description = "聊天角色编号", example = "666") 10 @Schema(description = "聊天角色编号", example = "666")
11 private Long roleId; 11 private Long roleId;
12 12
  13 + @Schema(description = "知识库编号", example = "1204")
  14 + private Long knowledgeId;
  15 +
13 } 16 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/vo/conversation/AiChatConversationUpdateMyReqVO.java
@@ -21,6 +21,9 @@ public class AiChatConversationUpdateMyReqVO { @@ -21,6 +21,9 @@ public class AiChatConversationUpdateMyReqVO {
21 @Schema(description = "模型编号", example = "1") 21 @Schema(description = "模型编号", example = "1")
22 private Long modelId; 22 private Long modelId;
23 23
  24 + @Schema(description = "知识库编号", example = "1")
  25 + private Long knowledgeId;
  26 +
24 @Schema(description = "角色设定", example = "一个快乐的程序员") 27 @Schema(description = "角色设定", example = "一个快乐的程序员")
25 private String systemMessage; 28 private String systemMessage;
26 29
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java
1 package cn.iocoder.yudao.module.ai.controller.admin.knowledge; 1 package cn.iocoder.yudao.module.ai.controller.admin.knowledge;
2 2
3 import cn.iocoder.yudao.framework.common.pojo.CommonResult; 3 import cn.iocoder.yudao.framework.common.pojo.CommonResult;
4 -import cn.iocoder.yudao.framework.common.pojo.PageParam;  
5 import cn.iocoder.yudao.framework.common.pojo.PageResult; 4 import cn.iocoder.yudao.framework.common.pojo.PageResult;
6 import cn.iocoder.yudao.framework.common.util.object.BeanUtils; 5 import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
7 -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateMyReqVO; 6 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateReqVO;
  7 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO;
8 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeRespVO; 8 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeRespVO;
9 -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateMyReqVO; 9 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateReqVO;
10 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; 10 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
11 import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeService; 11 import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeService;
12 import io.swagger.v3.oas.annotations.Operation; 12 import io.swagger.v3.oas.annotations.Operation;
@@ -28,24 +28,23 @@ public class AiKnowledgeController { @@ -28,24 +28,23 @@ public class AiKnowledgeController {
28 @Resource 28 @Resource
29 private AiKnowledgeService knowledgeService; 29 private AiKnowledgeService knowledgeService;
30 30
31 - @GetMapping("/my-page")  
32 - @Operation(summary = "获取【我的】知识库分页")  
33 - public CommonResult<PageResult<AiKnowledgeRespVO>> getKnowledgePageMy(@Validated PageParam pageReqVO) {  
34 - PageResult<AiKnowledgeDO> pageResult = knowledgeService.getKnowledgePageMy(getLoginUserId(), pageReqVO); 31 + @GetMapping("/page")
  32 + @Operation(summary = "获取知识库分页")
  33 + public CommonResult<PageResult<AiKnowledgeRespVO>> getKnowledgePage(@Valid AiKnowledgePageReqVO pageReqVO) {
  34 + PageResult<AiKnowledgeDO> pageResult = knowledgeService.getKnowledgePage(getLoginUserId(), pageReqVO);
35 return success(BeanUtils.toBean(pageResult, AiKnowledgeRespVO.class)); 35 return success(BeanUtils.toBean(pageResult, AiKnowledgeRespVO.class));
36 } 36 }
37 37
38 - @PostMapping("/create-my")  
39 - @Operation(summary = "创建【我的】知识库")  
40 - public CommonResult<Long> createKnowledgeMy(@RequestBody @Valid AiKnowledgeCreateMyReqVO createReqVO) {  
41 - return success(knowledgeService.createKnowledgeMy(createReqVO, getLoginUserId())); 38 + @PostMapping("/create")
  39 + @Operation(summary = "创建知识库")
  40 + public CommonResult<Long> createKnowledge(@RequestBody @Valid AiKnowledgeCreateReqVO createReqVO) {
  41 + return success(knowledgeService.createKnowledge(createReqVO, getLoginUserId()));
42 } 42 }
43 43
44 - @PutMapping("/update-my")  
45 - @Operation(summary = "更新【我的】知识库")  
46 - public CommonResult<Boolean> updateKnowledgeMy(@RequestBody @Valid AiKnowledgeUpdateMyReqVO updateReqVO) {  
47 - knowledgeService.updateKnowledgeMy(updateReqVO, getLoginUserId()); 44 + @PutMapping("/update")
  45 + @Operation(summary = "更新知识库")
  46 + public CommonResult<Boolean> updateKnowledge(@RequestBody @Valid AiKnowledgeUpdateReqVO updateReqVO) {
  47 + knowledgeService.updateKnowledge(updateReqVO, getLoginUserId());
48 return success(true); 48 return success(true);
49 } 49 }
50 -  
51 } 50 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeDocumentController.java
@@ -36,7 +36,7 @@ public class AiKnowledgeDocumentController { @@ -36,7 +36,7 @@ public class AiKnowledgeDocumentController {
36 36
37 @GetMapping("/page") 37 @GetMapping("/page")
38 @Operation(summary = "获取文档分页") 38 @Operation(summary = "获取文档分页")
39 - public CommonResult<PageResult<AiKnowledgeDocumentRespVO>> getKnowledgeDocumentPageMy(@Valid AiKnowledgeDocumentPageReqVO pageReqVO) { 39 + public CommonResult<PageResult<AiKnowledgeDocumentRespVO>> getKnowledgeDocumentPage(@Valid AiKnowledgeDocumentPageReqVO pageReqVO) {
40 PageResult<AiKnowledgeDocumentDO> pageResult = documentService.getKnowledgeDocumentPage(pageReqVO); 40 PageResult<AiKnowledgeDocumentDO> pageResult = documentService.getKnowledgeDocumentPage(pageReqVO);
41 return success(BeanUtils.toBean(pageResult, AiKnowledgeDocumentRespVO.class)); 41 return success(BeanUtils.toBean(pageResult, AiKnowledgeDocumentRespVO.class));
42 } 42 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.java
@@ -29,7 +29,7 @@ public class AiKnowledgeSegmentController { @@ -29,7 +29,7 @@ public class AiKnowledgeSegmentController {
29 29
30 @GetMapping("/page") 30 @GetMapping("/page")
31 @Operation(summary = "获取段落分页") 31 @Operation(summary = "获取段落分页")
32 - public CommonResult<PageResult<AiKnowledgeSegmentRespVO>> getKnowledgeSegmentPageMy(@Valid AiKnowledgeSegmentPageReqVO pageReqVO) { 32 + public CommonResult<PageResult<AiKnowledgeSegmentRespVO>> getKnowledgeSegmentPage(@Valid AiKnowledgeSegmentPageReqVO pageReqVO) {
33 PageResult<AiKnowledgeSegmentDO> pageResult = segmentService.getKnowledgeSegmentPage(pageReqVO); 33 PageResult<AiKnowledgeSegmentDO> pageResult = segmentService.getKnowledgeSegmentPage(pageReqVO);
34 return success(BeanUtils.toBean(pageResult, AiKnowledgeSegmentRespVO.class)); 34 return success(BeanUtils.toBean(pageResult, AiKnowledgeSegmentRespVO.class));
35 } 35 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeCreateMyReqVO.java renamed to yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeCreateReqVO.java
@@ -7,9 +7,9 @@ import lombok.Data; @@ -7,9 +7,9 @@ import lombok.Data;
7 7
8 import java.util.List; 8 import java.util.List;
9 9
10 -@Schema(description = "管理后台 - AI 知识库创建【我的】 Request VO") 10 +@Schema(description = "管理后台 - AI 知识库创建 Request VO")
11 @Data 11 @Data
12 -public class AiKnowledgeCreateMyReqVO { 12 +public class AiKnowledgeCreateReqVO {
13 13
14 @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南") 14 @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南")
15 @NotBlank(message = "知识库名称不能为空") 15 @NotBlank(message = "知识库名称不能为空")
@@ -18,11 +18,19 @@ public class AiKnowledgeCreateMyReqVO { @@ -18,11 +18,19 @@ public class AiKnowledgeCreateMyReqVO {
18 @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "存储 ruoyi-vue-pro 操作文档") 18 @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "存储 ruoyi-vue-pro 操作文档")
19 private String description; 19 private String description;
20 20
21 - @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]") 21 + @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1,2,3]")
22 private List<Long> visibilityPermissions; 22 private List<Long> visibilityPermissions;
23 23
24 @Schema(description = "嵌入模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") 24 @Schema(description = "嵌入模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
25 @NotNull(message = "嵌入模型不能为空") 25 @NotNull(message = "嵌入模型不能为空")
26 private Long modelId; 26 private Long modelId;
27 27
  28 + @Schema(description = "相似性阈值", requiredMode = Schema.RequiredMode.REQUIRED, example = "0.5")
  29 + @NotNull(message = "相似性阈值不能为空")
  30 + private Double similarityThreshold;
  31 +
  32 + @Schema(description = "topK", requiredMode = Schema.RequiredMode.REQUIRED, example = "3")
  33 + @NotNull(message = "topK 不能为空")
  34 + private Integer topK;
  35 +
28 } 36 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeDocumentCreateReqVO.java
@@ -23,4 +23,24 @@ public class AiKnowledgeDocumentCreateReqVO { @@ -23,4 +23,24 @@ public class AiKnowledgeDocumentCreateReqVO {
23 @URL(message = "文档 URL 格式不正确") 23 @URL(message = "文档 URL 格式不正确")
24 private String url; 24 private String url;
25 25
  26 + @Schema(description = "每个段落的目标 token 数", requiredMode = Schema.RequiredMode.REQUIRED, example = "800")
  27 + @NotNull(message = "每个段落的目标 token 数不能为空")
  28 + private Integer defaultSegmentTokens;
  29 +
  30 + @Schema(description = "每个段落的最小字符数", requiredMode = Schema.RequiredMode.REQUIRED, example = "350")
  31 + @NotNull(message = "每个段落的最小字符数不能为空")
  32 + private Integer minSegmentWordCount;
  33 +
  34 + @Schema(description = "丢弃阈值:低于此阈值的段落会被丢弃", requiredMode = Schema.RequiredMode.REQUIRED, example = "5")
  35 + @NotNull(message = "丢弃阈值不能为空")
  36 + private Integer minChunkLengthToEmbed;
  37 +
  38 + @Schema(description = "最大段落数", requiredMode = Schema.RequiredMode.REQUIRED, example = "10000")
  39 + @NotNull(message = "最大段落数不能为空")
  40 + private Integer maxNumSegments;
  41 +
  42 + @Schema(description = "分块是否保留分隔符", requiredMode = Schema.RequiredMode.REQUIRED, example = "true")
  43 + @NotNull(message = "分块是否保留分隔符不能为空")
  44 + private Boolean keepSeparator;
  45 +
26 } 46 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgePageReqVO.java 0 → 100644
  1 +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge;
  2 +
  3 +import cn.iocoder.yudao.framework.common.pojo.PageParam;
  4 +import io.swagger.v3.oas.annotations.media.Schema;
  5 +import lombok.Data;
  6 +
  7 +@Schema(description = "管理后台 - AI 知识库的分页 Request VO")
  8 +@Data
  9 +public class AiKnowledgePageReqVO extends PageParam {
  10 +
  11 + @Schema(description = "知识库名称", example = "Java 开发手册")
  12 + private String name;
  13 +
  14 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeUpdateMyReqVO.java renamed to yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeUpdateReqVO.java
@@ -9,7 +9,7 @@ import java.util.List; @@ -9,7 +9,7 @@ import java.util.List;
9 9
10 @Schema(description = "管理后台 - AI 知识库更新【我的】 Request VO") 10 @Schema(description = "管理后台 - AI 知识库更新【我的】 Request VO")
11 @Data 11 @Data
12 -public class AiKnowledgeUpdateMyReqVO { 12 +public class AiKnowledgeUpdateReqVO {
13 13
14 @Schema(description = "对话编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204") 14 @Schema(description = "对话编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204")
15 @NotNull(message = "知识库编号不能为空") 15 @NotNull(message = "知识库编号不能为空")
@@ -22,7 +22,7 @@ public class AiKnowledgeUpdateMyReqVO { @@ -22,7 +22,7 @@ public class AiKnowledgeUpdateMyReqVO {
22 @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "") 22 @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "")
23 private String description; 23 private String description;
24 24
25 - @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]") 25 + @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "1,2,3")
26 private List<Long> visibilityPermissions; 26 private List<Long> visibilityPermissions;
27 27
28 @Schema(description = "嵌入模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") 28 @Schema(description = "嵌入模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentSearchReqVO.java 0 → 100644
  1 +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment;
  2 +
  3 +import io.swagger.v3.oas.annotations.media.Schema;
  4 +import lombok.Data;
  5 +
  6 +
  7 +@Schema(description = "管理后台 - AI 知识库段落召回 Request VO")
  8 +@Data
  9 +public class AiKnowledgeSegmentSearchReqVO {
  10 +
  11 + @Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790")
  12 + private Long knowledgeId;
  13 +
  14 + @Schema(description = "内容", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 学习路线")
  15 + private String content;
  16 +
  17 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/chat/AiChatConversationDO.java
1 package cn.iocoder.yudao.module.ai.dal.dataobject.chat; 1 package cn.iocoder.yudao.module.ai.dal.dataobject.chat;
2 2
3 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; 3 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
  4 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
4 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; 5 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
5 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatRoleDO; 6 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatRoleDO;
6 import com.baomidou.mybatisplus.annotation.KeySequence; 7 import com.baomidou.mybatisplus.annotation.KeySequence;
@@ -65,6 +66,13 @@ public class AiChatConversationDO extends BaseDO { @@ -65,6 +66,13 @@ public class AiChatConversationDO extends BaseDO {
65 private Long roleId; 66 private Long roleId;
66 67
67 /** 68 /**
  69 + * 知识库编号
  70 + * <p>
  71 + * 关联 {@link AiKnowledgeDO#getId()}
  72 + */
  73 + private Long knowledgeId;
  74 +
  75 + /**
68 * 模型编号 76 * 模型编号
69 * 77 *
70 * 关联 {@link AiChatModelDO#getId()} 字段 78 * 关联 {@link AiChatModelDO#getId()} 字段
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/chat/AiChatMessageDO.java
1 package cn.iocoder.yudao.module.ai.dal.dataobject.chat; 1 package cn.iocoder.yudao.module.ai.dal.dataobject.chat;
2 2
3 -import com.baomidou.mybatisplus.annotation.TableId;  
4 -import org.springframework.ai.chat.messages.MessageType;  
5 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; 3 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
  4 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
6 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; 5 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
7 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatRoleDO; 6 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatRoleDO;
8 import com.baomidou.mybatisplus.annotation.KeySequence; 7 import com.baomidou.mybatisplus.annotation.KeySequence;
  8 +import com.baomidou.mybatisplus.annotation.TableField;
  9 +import com.baomidou.mybatisplus.annotation.TableId;
9 import com.baomidou.mybatisplus.annotation.TableName; 10 import com.baomidou.mybatisplus.annotation.TableName;
  11 +import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
10 import lombok.*; 12 import lombok.*;
  13 +import org.springframework.ai.chat.messages.MessageType;
  14 +
  15 +import java.util.List;
11 16
12 /** 17 /**
13 * AI Chat 消息 DO 18 * AI Chat 消息 DO
@@ -66,6 +71,15 @@ public class AiChatMessageDO extends BaseDO { @@ -66,6 +71,15 @@ public class AiChatMessageDO extends BaseDO {
66 */ 71 */
67 private Long roleId; 72 private Long roleId;
68 73
  74 +
  75 + /**
  76 + * 段落编号数组
  77 + *
  78 + * 关联 {@link AiKnowledgeSegmentDO#getId()} 字段
  79 + */
  80 + @TableField(typeHandler = JacksonTypeHandler.class)
  81 + private List<Long> segmentIds;
  82 +
69 /** 83 /**
70 * 模型标志 84 * 模型标志
71 */ 85 */
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java
@@ -2,10 +2,10 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; @@ -2,10 +2,10 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
2 2
3 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; 3 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
4 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; 4 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
  5 +import cn.iocoder.yudao.framework.mybatis.core.type.LongListTypeHandler;
5 import com.baomidou.mybatisplus.annotation.TableField; 6 import com.baomidou.mybatisplus.annotation.TableField;
6 import com.baomidou.mybatisplus.annotation.TableId; 7 import com.baomidou.mybatisplus.annotation.TableId;
7 import com.baomidou.mybatisplus.annotation.TableName; 8 import com.baomidou.mybatisplus.annotation.TableName;
8 -import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;  
9 import lombok.Data; 9 import lombok.Data;
10 10
11 import java.util.List; 11 import java.util.List;
@@ -38,11 +38,13 @@ public class AiKnowledgeDO extends BaseDO { @@ -38,11 +38,13 @@ public class AiKnowledgeDO extends BaseDO {
38 * 知识库描述 38 * 知识库描述
39 */ 39 */
40 private String description; 40 private String description;
41 - // TODO @新:如果全部可见,需要怎么设置? 41 +
42 /** 42 /**
43 - * 可见权限,只能选择哪些人可见 43 + * 可见权限,选择哪些人可见
  44 + * <p>
  45 + * -1 所有人可见,其他为各自用户编号
44 */ 46 */
45 - @TableField(typeHandler = JacksonTypeHandler.class) 47 + @TableField(typeHandler = LongListTypeHandler.class)
46 private List<Long> visibilityPermissions; 48 private List<Long> visibilityPermissions;
47 /** 49 /**
48 * 嵌入模型编号 50 * 嵌入模型编号
@@ -52,10 +54,21 @@ public class AiKnowledgeDO extends BaseDO { @@ -52,10 +54,21 @@ public class AiKnowledgeDO extends BaseDO {
52 * 模型标识 54 * 模型标识
53 */ 55 */
54 private String model; 56 private String model;
  57 +
  58 + /**
  59 + * topK
  60 + */
  61 + private Integer topK;
  62 + /**
  63 + * 相似度阈值
  64 + */
  65 + private Double similarityThreshold;
  66 +
55 /** 67 /**
56 * 状态 68 * 状态
57 * <p> 69 * <p>
58 * 枚举 {@link CommonStatusEnum} 70 * 枚举 {@link CommonStatusEnum}
59 */ 71 */
60 private Integer status; 72 private Integer status;
  73 +
61 } 74 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java
@@ -23,7 +23,7 @@ public class AiKnowledgeDocumentDO extends BaseDO { @@ -23,7 +23,7 @@ public class AiKnowledgeDocumentDO extends BaseDO {
23 private Long id; 23 private Long id;
24 /** 24 /**
25 * 知识库编号 25 * 知识库编号
26 - * 26 + * <p>
27 * 关联 {@link AiKnowledgeDO#getId()} 27 * 关联 {@link AiKnowledgeDO#getId()}
28 */ 28 */
29 private Long knowledgeId; 29 private Long knowledgeId;
@@ -40,13 +40,39 @@ public class AiKnowledgeDocumentDO extends BaseDO { @@ -40,13 +40,39 @@ public class AiKnowledgeDocumentDO extends BaseDO {
40 */ 40 */
41 private String url; 41 private String url;
42 /** 42 /**
43 - * token 数量 43 + * 文档 token 数量
44 */ 44 */
45 private Integer tokens; 45 private Integer tokens;
46 /** 46 /**
47 - * 字符数 47 + * 文档字符数
48 */ 48 */
49 private Integer wordCount; 49 private Integer wordCount;
  50 +
  51 +
  52 + // ========== 自定义分段所用参数 ==========
  53 + // TODO @新:3)defaultChunkSize、defaultChunkSize、minChunkSizeChars、maxNumChunks 这几个字段的命名,可能要微信一起讨论下。尽量命名保持风格统一哈。
  54 + /**
  55 + * 每个文本块的目标 token 数
  56 + */
  57 + private Integer defaultSegmentTokens;
  58 + /**
  59 + * 每个文本块的最小字符数
  60 + */
  61 + private Integer minSegmentWordCount;
  62 + /**
  63 + * 低于此值的块会被丢弃
  64 + */
  65 + private Integer minChunkLengthToEmbed;
  66 + /**
  67 + * 最大块数
  68 + */
  69 + private Integer maxNumSegments;
  70 + /**
  71 + * 分块是否保留分隔符
  72 + */
  73 + private Boolean keepSeparator;
  74 + // ===================================
  75 +
50 /** 76 /**
51 * 切片状态 77 * 切片状态
52 * <p> 78 * <p>
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java
@@ -28,13 +28,13 @@ public class AiKnowledgeSegmentDO extends BaseDO { @@ -28,13 +28,13 @@ public class AiKnowledgeSegmentDO extends BaseDO {
28 private String vectorId; 28 private String vectorId;
29 /** 29 /**
30 * 知识库编号 30 * 知识库编号
31 - * 31 + * <p>
32 * 关联 {@link AiKnowledgeDO#getId()} 32 * 关联 {@link AiKnowledgeDO#getId()}
33 */ 33 */
34 private Long knowledgeId; 34 private Long knowledgeId;
35 /** 35 /**
36 * 文档编号 36 * 文档编号
37 - * 37 + * <p>
38 * 关联 {@link AiKnowledgeDocumentDO#getId()} 38 * 关联 {@link AiKnowledgeDocumentDO#getId()}
39 */ 39 */
40 private Long documentId; 40 private Long documentId;
@@ -52,7 +52,7 @@ public class AiKnowledgeSegmentDO extends BaseDO { @@ -52,7 +52,7 @@ public class AiKnowledgeSegmentDO extends BaseDO {
52 private Integer tokens; 52 private Integer tokens;
53 /** 53 /**
54 * 状态 54 * 状态
55 - * 55 + * <p>
56 * 枚举 {@link CommonStatusEnum} 56 * 枚举 {@link CommonStatusEnum}
57 */ 57 */
58 private Integer status; 58 private Integer status;
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeMapper.java
1 package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; 1 package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
2 2
3 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; 3 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
4 -import cn.iocoder.yudao.framework.common.pojo.PageParam;  
5 import cn.iocoder.yudao.framework.common.pojo.PageResult; 4 import cn.iocoder.yudao.framework.common.pojo.PageResult;
6 import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; 5 import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
7 import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; 6 import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
  7 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO;
8 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; 8 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
9 import org.apache.ibatis.annotations.Mapper; 9 import org.apache.ibatis.annotations.Mapper;
10 10
@@ -16,10 +16,11 @@ import org.apache.ibatis.annotations.Mapper; @@ -16,10 +16,11 @@ import org.apache.ibatis.annotations.Mapper;
16 @Mapper 16 @Mapper
17 public interface AiKnowledgeMapper extends BaseMapperX<AiKnowledgeDO> { 17 public interface AiKnowledgeMapper extends BaseMapperX<AiKnowledgeDO> {
18 18
19 - default PageResult<AiKnowledgeDO> selectPageByMy(Long userId, PageParam pageReqVO) { 19 + default PageResult<AiKnowledgeDO> selectPage(Long userId, AiKnowledgePageReqVO pageReqVO) {
20 return selectPage(pageReqVO, new LambdaQueryWrapperX<AiKnowledgeDO>() 20 return selectPage(pageReqVO, new LambdaQueryWrapperX<AiKnowledgeDO>()
21 - .eq(AiKnowledgeDO::getUserId, userId)  
22 .eq(AiKnowledgeDO::getStatus, CommonStatusEnum.ENABLE.getStatus()) 21 .eq(AiKnowledgeDO::getStatus, CommonStatusEnum.ENABLE.getStatus())
  22 + .likeIfPresent(AiKnowledgeDO::getName, pageReqVO.getName())
  23 + .and(e -> e.apply("FIND_IN_SET(" + userId + ",visibility_permissions)").or(m -> m.apply("FIND_IN_SET(-1,visibility_permissions)")))
23 .orderByDesc(AiKnowledgeDO::getId)); 24 .orderByDesc(AiKnowledgeDO::getId));
24 } 25 }
25 } 26 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java
@@ -7,6 +7,8 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowle @@ -7,6 +7,8 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowle
7 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; 7 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
8 import org.apache.ibatis.annotations.Mapper; 8 import org.apache.ibatis.annotations.Mapper;
9 9
  10 +import java.util.List;
  11 +
10 /** 12 /**
11 * AI 知识库-分片 Mapper 13 * AI 知识库-分片 Mapper
12 * 14 *
@@ -22,4 +24,11 @@ public interface AiKnowledgeSegmentMapper extends BaseMapperX&lt;AiKnowledgeSegment @@ -22,4 +24,11 @@ public interface AiKnowledgeSegmentMapper extends BaseMapperX&lt;AiKnowledgeSegment
22 .likeIfPresent(AiKnowledgeSegmentDO::getContent, reqVO.getKeyword()) 24 .likeIfPresent(AiKnowledgeSegmentDO::getContent, reqVO.getKeyword())
23 .orderByDesc(AiKnowledgeSegmentDO::getId)); 25 .orderByDesc(AiKnowledgeSegmentDO::getId));
24 } 26 }
  27 +
  28 + default List<AiKnowledgeSegmentDO> selectListByVectorIds(List<String> vectorIdList) {
  29 + return selectList(new LambdaQueryWrapperX<AiKnowledgeSegmentDO>()
  30 + .in(AiKnowledgeSegmentDO::getVectorId, vectorIdList)
  31 + .orderByDesc(AiKnowledgeSegmentDO::getId));
  32 + }
  33 +
25 } 34 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/chat/AiChatConversationServiceImpl.java
@@ -13,6 +13,7 @@ import cn.iocoder.yudao.module.ai.dal.dataobject.chat.AiChatConversationDO; @@ -13,6 +13,7 @@ import cn.iocoder.yudao.module.ai.dal.dataobject.chat.AiChatConversationDO;
13 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; 13 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
14 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatRoleDO; 14 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatRoleDO;
15 import cn.iocoder.yudao.module.ai.dal.mysql.chat.AiChatConversationMapper; 15 import cn.iocoder.yudao.module.ai.dal.mysql.chat.AiChatConversationMapper;
  16 +import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeService;
16 import cn.iocoder.yudao.module.ai.service.model.AiChatModelService; 17 import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;
17 import cn.iocoder.yudao.module.ai.service.model.AiChatRoleService; 18 import cn.iocoder.yudao.module.ai.service.model.AiChatRoleService;
18 import jakarta.annotation.Resource; 19 import jakarta.annotation.Resource;
@@ -22,6 +23,7 @@ import org.springframework.validation.annotation.Validated; @@ -22,6 +23,7 @@ import org.springframework.validation.annotation.Validated;
22 23
23 import java.time.LocalDateTime; 24 import java.time.LocalDateTime;
24 import java.util.List; 25 import java.util.List;
  26 +import java.util.Objects;
25 27
26 import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; 28 import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
27 import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertList; 29 import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertList;
@@ -45,6 +47,8 @@ public class AiChatConversationServiceImpl implements AiChatConversationService @@ -45,6 +47,8 @@ public class AiChatConversationServiceImpl implements AiChatConversationService
45 private AiChatModelService chatModalService; 47 private AiChatModelService chatModalService;
46 @Resource 48 @Resource
47 private AiChatRoleService chatRoleService; 49 private AiChatRoleService chatRoleService;
  50 + @Resource
  51 + private AiKnowledgeService knowledgeService;
48 52
49 @Override 53 @Override
50 public Long createChatConversationMy(AiChatConversationCreateMyReqVO createReqVO, Long userId) { 54 public Long createChatConversationMy(AiChatConversationCreateMyReqVO createReqVO, Long userId) {
@@ -56,9 +60,14 @@ public class AiChatConversationServiceImpl implements AiChatConversationService @@ -56,9 +60,14 @@ public class AiChatConversationServiceImpl implements AiChatConversationService
56 Assert.notNull(model, "必须找到默认模型"); 60 Assert.notNull(model, "必须找到默认模型");
57 validateChatModel(model); 61 validateChatModel(model);
58 62
  63 + // 1.3 校验知识库
  64 + if (Objects.nonNull(createReqVO.getKnowledgeId())) {
  65 + knowledgeService.validateKnowledgeExists(createReqVO.getKnowledgeId());
  66 + }
  67 +
59 // 2. 创建 AiChatConversationDO 聊天对话 68 // 2. 创建 AiChatConversationDO 聊天对话
60 AiChatConversationDO conversation = new AiChatConversationDO().setUserId(userId).setPinned(false) 69 AiChatConversationDO conversation = new AiChatConversationDO().setUserId(userId).setPinned(false)
61 - .setModelId(model.getId()).setModel(model.getModel()) 70 + .setModelId(model.getId()).setModel(model.getModel()).setKnowledgeId(createReqVO.getKnowledgeId())
62 .setTemperature(model.getTemperature()).setMaxTokens(model.getMaxTokens()).setMaxContexts(model.getMaxContexts()); 71 .setTemperature(model.getTemperature()).setMaxTokens(model.getMaxTokens()).setMaxContexts(model.getMaxContexts());
63 if (role != null) { 72 if (role != null) {
64 conversation.setTitle(role.getName()).setRoleId(role.getId()).setSystemMessage(role.getSystemMessage()); 73 conversation.setTitle(role.getName()).setRoleId(role.getId()).setSystemMessage(role.getSystemMessage());
@@ -82,6 +91,11 @@ public class AiChatConversationServiceImpl implements AiChatConversationService @@ -82,6 +91,11 @@ public class AiChatConversationServiceImpl implements AiChatConversationService
82 model = chatModalService.validateChatModel(updateReqVO.getModelId()); 91 model = chatModalService.validateChatModel(updateReqVO.getModelId());
83 } 92 }
84 93
  94 + // 1.3 校验知识库是否存在
  95 + if (updateReqVO.getKnowledgeId() != null) {
  96 + knowledgeService.validateKnowledgeExists(updateReqVO.getKnowledgeId());
  97 + }
  98 +
85 // 2. 更新对话信息 99 // 2. 更新对话信息
86 AiChatConversationDO updateObj = BeanUtils.toBean(updateReqVO, AiChatConversationDO.class); 100 AiChatConversationDO updateObj = BeanUtils.toBean(updateReqVO, AiChatConversationDO.class);
87 if (Boolean.TRUE.equals(updateReqVO.getPinned())) { 101 if (Boolean.TRUE.equals(updateReqVO.getPinned())) {
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImpl.java
@@ -12,21 +12,29 @@ import cn.iocoder.yudao.framework.tenant.core.util.TenantUtils; @@ -12,21 +12,29 @@ import cn.iocoder.yudao.framework.tenant.core.util.TenantUtils;
12 import cn.iocoder.yudao.module.ai.controller.admin.chat.vo.message.AiChatMessagePageReqVO; 12 import cn.iocoder.yudao.module.ai.controller.admin.chat.vo.message.AiChatMessagePageReqVO;
13 import cn.iocoder.yudao.module.ai.controller.admin.chat.vo.message.AiChatMessageSendReqVO; 13 import cn.iocoder.yudao.module.ai.controller.admin.chat.vo.message.AiChatMessageSendReqVO;
14 import cn.iocoder.yudao.module.ai.controller.admin.chat.vo.message.AiChatMessageSendRespVO; 14 import cn.iocoder.yudao.module.ai.controller.admin.chat.vo.message.AiChatMessageSendRespVO;
  15 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentSearchReqVO;
15 import cn.iocoder.yudao.module.ai.dal.dataobject.chat.AiChatConversationDO; 16 import cn.iocoder.yudao.module.ai.dal.dataobject.chat.AiChatConversationDO;
16 import cn.iocoder.yudao.module.ai.dal.dataobject.chat.AiChatMessageDO; 17 import cn.iocoder.yudao.module.ai.dal.dataobject.chat.AiChatMessageDO;
  18 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
17 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; 19 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
18 import cn.iocoder.yudao.module.ai.dal.mysql.chat.AiChatMessageMapper; 20 import cn.iocoder.yudao.module.ai.dal.mysql.chat.AiChatMessageMapper;
  21 +import cn.iocoder.yudao.module.ai.enums.AiChatRoleEnum;
19 import cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants; 22 import cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants;
  23 +import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeSegmentService;
20 import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService; 24 import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService;
21 import cn.iocoder.yudao.module.ai.service.model.AiChatModelService; 25 import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;
22 import jakarta.annotation.Resource; 26 import jakarta.annotation.Resource;
23 import lombok.extern.slf4j.Slf4j; 27 import lombok.extern.slf4j.Slf4j;
24 -import org.springframework.ai.chat.messages.*; 28 +import org.springframework.ai.chat.messages.Message;
  29 +import org.springframework.ai.chat.messages.MessageType;
  30 +import org.springframework.ai.chat.messages.SystemMessage;
  31 +import org.springframework.ai.chat.messages.UserMessage;
25 import org.springframework.ai.chat.model.ChatModel; 32 import org.springframework.ai.chat.model.ChatModel;
26 import org.springframework.ai.chat.model.ChatResponse; 33 import org.springframework.ai.chat.model.ChatResponse;
27 import org.springframework.ai.chat.model.StreamingChatModel; 34 import org.springframework.ai.chat.model.StreamingChatModel;
28 import org.springframework.ai.chat.prompt.ChatOptions; 35 import org.springframework.ai.chat.prompt.ChatOptions;
29 import org.springframework.ai.chat.prompt.Prompt; 36 import org.springframework.ai.chat.prompt.Prompt;
  37 +import org.springframework.ai.chat.prompt.PromptTemplate;
30 import org.springframework.stereotype.Service; 38 import org.springframework.stereotype.Service;
31 import org.springframework.transaction.annotation.Transactional; 39 import org.springframework.transaction.annotation.Transactional;
32 import reactor.core.publisher.Flux; 40 import reactor.core.publisher.Flux;
@@ -59,6 +67,8 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { @@ -59,6 +67,8 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
59 private AiChatModelService chatModalService; 67 private AiChatModelService chatModalService;
60 @Resource 68 @Resource
61 private AiApiKeyService apiKeyService; 69 private AiApiKeyService apiKeyService;
  70 + @Resource
  71 + private AiKnowledgeSegmentService knowledgeSegmentService;
62 72
63 @Transactional(rollbackFor = Exception.class) 73 @Transactional(rollbackFor = Exception.class)
64 public AiChatMessageSendRespVO sendMessage(AiChatMessageSendReqVO sendReqVO, Long userId) { 74 public AiChatMessageSendRespVO sendMessage(AiChatMessageSendReqVO sendReqVO, Long userId) {
@@ -80,13 +90,16 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { @@ -80,13 +90,16 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
80 AiChatMessageDO assistantMessage = createChatMessage(conversation.getId(), userMessage.getId(), model, 90 AiChatMessageDO assistantMessage = createChatMessage(conversation.getId(), userMessage.getId(), model,
81 userId, conversation.getRoleId(), MessageType.ASSISTANT, "", sendReqVO.getUseContext()); 91 userId, conversation.getRoleId(), MessageType.ASSISTANT, "", sendReqVO.getUseContext());
82 92
83 - // 3.2 创建 chat 需要的 Prompt  
84 - Prompt prompt = buildPrompt(conversation, historyMessages, model, sendReqVO); 93 + // 3.2 召回段落
  94 + List<AiKnowledgeSegmentDO> segmentList = recallSegment(sendReqVO.getContent(), conversation.getKnowledgeId());
  95 +
  96 + // 3.3 创建 chat 需要的 Prompt
  97 + Prompt prompt = buildPrompt(conversation, historyMessages, segmentList, model, sendReqVO);
85 ChatResponse chatResponse = chatModel.call(prompt); 98 ChatResponse chatResponse = chatModel.call(prompt);
86 99
87 - // 3.3 段式返回 100 + // 3.4 段式返回
88 String newContent = chatResponse.getResult().getOutput().getContent(); 101 String newContent = chatResponse.getResult().getOutput().getContent();
89 - chatMessageMapper.updateById(new AiChatMessageDO().setId(assistantMessage.getId()).setContent(newContent)); 102 + chatMessageMapper.updateById(new AiChatMessageDO().setId(assistantMessage.getId()).setSegmentIds(convertList(segmentList, AiKnowledgeSegmentDO::getId)).setContent(newContent));
90 return new AiChatMessageSendRespVO().setSend(BeanUtils.toBean(userMessage, AiChatMessageSendRespVO.Message.class)) 103 return new AiChatMessageSendRespVO().setSend(BeanUtils.toBean(userMessage, AiChatMessageSendRespVO.Message.class))
91 .setReceive(BeanUtils.toBean(assistantMessage, AiChatMessageSendRespVO.Message.class).setContent(newContent)); 104 .setReceive(BeanUtils.toBean(assistantMessage, AiChatMessageSendRespVO.Message.class).setContent(newContent));
92 } 105 }
@@ -111,11 +124,15 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { @@ -111,11 +124,15 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
111 AiChatMessageDO assistantMessage = createChatMessage(conversation.getId(), userMessage.getId(), model, 124 AiChatMessageDO assistantMessage = createChatMessage(conversation.getId(), userMessage.getId(), model,
112 userId, conversation.getRoleId(), MessageType.ASSISTANT, "", sendReqVO.getUseContext()); 125 userId, conversation.getRoleId(), MessageType.ASSISTANT, "", sendReqVO.getUseContext());
113 126
114 - // 3.2 构建 Prompt,并进行调用  
115 - Prompt prompt = buildPrompt(conversation, historyMessages, model, sendReqVO); 127 +
  128 + // 3.2 召回段落
  129 + List<AiKnowledgeSegmentDO> segmentList = recallSegment(sendReqVO.getContent(), conversation.getKnowledgeId());
  130 +
  131 + // 3.3 构建 Prompt,并进行调用
  132 + Prompt prompt = buildPrompt(conversation, historyMessages, segmentList, model, sendReqVO);
116 Flux<ChatResponse> streamResponse = chatModel.stream(prompt); 133 Flux<ChatResponse> streamResponse = chatModel.stream(prompt);
117 134
118 - // 3.3 流式返回 135 + // 3.4 流式返回
119 // TODO 注意:Schedulers.immediate() 目的是,避免默认 Schedulers.parallel() 并发消费 chunk 导致 SSE 响应前端会乱序问题 136 // TODO 注意:Schedulers.immediate() 目的是,避免默认 Schedulers.parallel() 并发消费 chunk 导致 SSE 响应前端会乱序问题
120 StringBuffer contentBuffer = new StringBuffer(); 137 StringBuffer contentBuffer = new StringBuffer();
121 return streamResponse.map(chunk -> { 138 return streamResponse.map(chunk -> {
@@ -128,7 +145,8 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { @@ -128,7 +145,8 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
128 }).doOnComplete(() -> { 145 }).doOnComplete(() -> {
129 // 忽略租户,因为 Flux 异步无法透传租户 146 // 忽略租户,因为 Flux 异步无法透传租户
130 TenantUtils.executeIgnore(() -> 147 TenantUtils.executeIgnore(() ->
131 - chatMessageMapper.updateById(new AiChatMessageDO().setId(assistantMessage.getId()).setContent(contentBuffer.toString()))); 148 + chatMessageMapper.updateById(new AiChatMessageDO().setId(assistantMessage.getId()).setSegmentIds(convertList(segmentList, AiKnowledgeSegmentDO::getId))
  149 + .setContent(contentBuffer.toString())));
132 }).doOnError(throwable -> { 150 }).doOnError(throwable -> {
133 log.error("[sendChatMessageStream][userId({}) sendReqVO({}) 发生异常]", userId, sendReqVO, throwable); 151 log.error("[sendChatMessageStream][userId({}) sendReqVO({}) 发生异常]", userId, sendReqVO, throwable);
134 // 忽略租户,因为 Flux 异步无法透传租户 152 // 忽略租户,因为 Flux 异步无法透传租户
@@ -137,18 +155,35 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { @@ -137,18 +155,35 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
137 }).onErrorResume(error -> Flux.just(error(ErrorCodeConstants.CHAT_STREAM_ERROR))); 155 }).onErrorResume(error -> Flux.just(error(ErrorCodeConstants.CHAT_STREAM_ERROR)));
138 } 156 }
139 157
140 - private Prompt buildPrompt(AiChatConversationDO conversation, List<AiChatMessageDO> messages, 158 + private List<AiKnowledgeSegmentDO> recallSegment(String content, Long knowledgeId) {
  159 + if (Objects.isNull(knowledgeId)) {
  160 + return Collections.emptyList();
  161 + }
  162 + return knowledgeSegmentService.similaritySearch(new AiKnowledgeSegmentSearchReqVO().setKnowledgeId(knowledgeId).setContent(content));
  163 + }
  164 +
  165 + private Prompt buildPrompt(AiChatConversationDO conversation, List<AiChatMessageDO> messages,List<AiKnowledgeSegmentDO> segmentList,
141 AiChatModelDO model, AiChatMessageSendReqVO sendReqVO) { 166 AiChatModelDO model, AiChatMessageSendReqVO sendReqVO) {
142 // 1. 构建 Prompt Message 列表 167 // 1. 构建 Prompt Message 列表
143 List<Message> chatMessages = new ArrayList<>(); 168 List<Message> chatMessages = new ArrayList<>();
144 - // 1.1 system context 角色设定 169 +
  170 + // 1.1 召回内容消息构建
  171 + if (CollUtil.isNotEmpty(segmentList)) {
  172 + PromptTemplate promptTemplate = new PromptTemplate(AiChatRoleEnum.AI_KNOWLEDGE_ROLE.getSystemMessage());
  173 + StringBuilder infoBuilder = StrUtil.builder();
  174 + segmentList.forEach(segment -> infoBuilder.append(System.lineSeparator()).append(segment.getContent()));
  175 + Message message = promptTemplate.createMessage(Map.of("info", infoBuilder.toString()));
  176 + chatMessages.add(message);
  177 + }
  178 +
  179 + // 1.2 system context 角色设定
145 if (StrUtil.isNotBlank(conversation.getSystemMessage())) { 180 if (StrUtil.isNotBlank(conversation.getSystemMessage())) {
146 chatMessages.add(new SystemMessage(conversation.getSystemMessage())); 181 chatMessages.add(new SystemMessage(conversation.getSystemMessage()));
147 } 182 }
148 - // 1.2 history message 历史消息 183 + // 1.3 history message 历史消息
149 List<AiChatMessageDO> contextMessages = filterContextMessages(messages, conversation, sendReqVO); 184 List<AiChatMessageDO> contextMessages = filterContextMessages(messages, conversation, sendReqVO);
150 contextMessages.forEach(message -> chatMessages.add(AiUtils.buildMessage(message.getType(), message.getContent()))); 185 contextMessages.forEach(message -> chatMessages.add(AiUtils.buildMessage(message.getType(), message.getContent())));
151 - // 1.3 user message 新发送消息 186 + // 1.4 user message 新发送消息
152 chatMessages.add(new UserMessage(sendReqVO.getContent())); 187 chatMessages.add(new UserMessage(sendReqVO.getContent()));
153 188
154 // 2. 构建 ChatOptions 对象 189 // 2. 构建 ChatOptions 对象
@@ -160,12 +195,12 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { @@ -160,12 +195,12 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
160 195
161 /** 196 /**
162 * 从历史消息中,获得倒序的 n 组消息作为消息上下文 197 * 从历史消息中,获得倒序的 n 组消息作为消息上下文
163 - * 198 + * <p>
164 * n 组:指的是 user + assistant 形成一组 199 * n 组:指的是 user + assistant 形成一组
165 * 200 *
166 - * @param messages 消息列表 201 + * @param messages 消息列表
167 * @param conversation 对话 202 * @param conversation 对话
168 - * @param sendReqVO 发送请求 203 + * @param sendReqVO 发送请求
169 * @return 消息上下文 204 * @return 消息上下文
170 */ 205 */
171 private List<AiChatMessageDO> filterContextMessages(List<AiChatMessageDO> messages, 206 private List<AiChatMessageDO> filterContextMessages(List<AiChatMessageDO> messages,
@@ -182,7 +217,7 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { @@ -182,7 +217,7 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
182 } 217 }
183 AiChatMessageDO userMessage = CollUtil.get(messages, i - 1); 218 AiChatMessageDO userMessage = CollUtil.get(messages, i - 1);
184 if (userMessage == null || ObjUtil.notEqual(assistantMessage.getReplyId(), userMessage.getId()) 219 if (userMessage == null || ObjUtil.notEqual(assistantMessage.getReplyId(), userMessage.getId())
185 - || StrUtil.isEmpty(assistantMessage.getContent())) { 220 + || StrUtil.isEmpty(assistantMessage.getContent())) {
186 continue; 221 continue;
187 } 222 }
188 // 由于后续要 reverse 反转,所以先添加 assistantMessage 223 // 由于后续要 reverse 反转,所以先添加 assistantMessage
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java
@@ -9,15 +9,11 @@ import cn.iocoder.yudao.framework.common.util.object.BeanUtils; @@ -9,15 +9,11 @@ import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
9 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO; 9 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO;
10 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO; 10 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO;
11 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO; 11 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO;
12 -import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;  
13 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; 12 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
14 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; 13 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
15 -import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;  
16 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper; 14 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper;
17 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper; 15 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper;
18 import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum; 16 import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum;
19 -import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService;  
20 -import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;  
21 import jakarta.annotation.Resource; 17 import jakarta.annotation.Resource;
22 import lombok.extern.slf4j.Slf4j; 18 import lombok.extern.slf4j.Slf4j;
23 import org.springframework.ai.document.Document; 19 import org.springframework.ai.document.Document;
@@ -49,23 +45,15 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic @@ -49,23 +45,15 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
49 private AiKnowledgeSegmentMapper segmentMapper; 45 private AiKnowledgeSegmentMapper segmentMapper;
50 46
51 @Resource 47 @Resource
52 - private TokenTextSplitter tokenTextSplitter;  
53 - @Resource  
54 private TokenCountEstimator tokenCountEstimator; 48 private TokenCountEstimator tokenCountEstimator;
55 -  
56 - @Resource  
57 - private AiApiKeyService apiKeyService;  
58 @Resource 49 @Resource
59 private AiKnowledgeService knowledgeService; 50 private AiKnowledgeService knowledgeService;
60 - @Resource  
61 - private AiChatModelService chatModelService;  
62 51
63 @Override 52 @Override
64 @Transactional(rollbackFor = Exception.class) 53 @Transactional(rollbackFor = Exception.class)
65 public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) { 54 public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) {
66 - // 0. 校验  
67 - AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(createReqVO.getKnowledgeId());  
68 - AiChatModelDO model = chatModelService.validateChatModel(knowledge.getModelId()); 55 + // 0. 校验并获取向量存储实例
  56 + VectorStore vectorStore = knowledgeService.getVectorStoreById(createReqVO.getKnowledgeId());
69 57
70 // 1.1 下载文档 58 // 1.1 下载文档
71 TikaDocumentReader loader = new TikaDocumentReader(downloadFile(createReqVO.getUrl())); 59 TikaDocumentReader loader = new TikaDocumentReader(downloadFile(createReqVO.getUrl()));
@@ -82,6 +70,9 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic @@ -82,6 +70,9 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
82 return documentId; 70 return documentId;
83 } 71 }
84 72
  73 + // 2 构造文本分段器
  74 + TokenTextSplitter tokenTextSplitter = new TokenTextSplitter(createReqVO.getDefaultSegmentTokens(), createReqVO.getMinSegmentWordCount(), createReqVO.getMinChunkLengthToEmbed(),
  75 + createReqVO.getMaxNumSegments(), createReqVO.getKeepSeparator());
85 // 2.1 文档分段 76 // 2.1 文档分段
86 List<Document> segments = tokenTextSplitter.apply(documents); 77 List<Document> segments = tokenTextSplitter.apply(documents);
87 // 2.2 分段内容入库 78 // 2.2 分段内容入库
@@ -92,9 +83,7 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic @@ -92,9 +83,7 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
92 .setStatus(CommonStatusEnum.ENABLE.getStatus())); 83 .setStatus(CommonStatusEnum.ENABLE.getStatus()));
93 segmentMapper.insertBatch(segmentDOList); 84 segmentMapper.insertBatch(segmentDOList);
94 85
95 - // 3.1 获取向量存储实例  
96 - VectorStore vectorStore = apiKeyService.getOrCreateVectorStore(model.getKeyId());  
97 - // 3.2 向量化并存储 86 + // 3. 向量化并存储
98 segments.forEach(segment -> segment.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, createReqVO.getKnowledgeId())); 87 segments.forEach(segment -> segment.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, createReqVO.getKnowledgeId()));
99 vectorStore.add(segments); 88 vectorStore.add(segments);
100 return documentId; 89 return documentId;
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java
@@ -2,10 +2,13 @@ package cn.iocoder.yudao.module.ai.service.knowledge; @@ -2,10 +2,13 @@ package cn.iocoder.yudao.module.ai.service.knowledge;
2 2
3 import cn.iocoder.yudao.framework.common.pojo.PageResult; 3 import cn.iocoder.yudao.framework.common.pojo.PageResult;
4 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO; 4 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO;
  5 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentSearchReqVO;
5 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO; 6 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO;
6 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO; 7 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO;
7 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; 8 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
8 9
  10 +import java.util.List;
  11 +
9 /** 12 /**
10 * AI 知识库段落 Service 接口 13 * AI 知识库段落 Service 接口
11 * 14 *
@@ -35,4 +38,12 @@ public interface AiKnowledgeSegmentService { @@ -35,4 +38,12 @@ public interface AiKnowledgeSegmentService {
35 */ 38 */
36 void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO); 39 void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO);
37 40
  41 + /**
  42 + * 召回段落
  43 + *
  44 + * @param reqVO 召回请求信息
  45 + * @return 召回的段落
  46 + */
  47 + List<AiKnowledgeSegmentDO> similaritySearch(AiKnowledgeSegmentSearchReqVO reqVO);
  48 +
38 } 49 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
1 package cn.iocoder.yudao.module.ai.service.knowledge; 1 package cn.iocoder.yudao.module.ai.service.knowledge;
2 2
  3 +import cn.hutool.core.collection.CollUtil;
  4 +import cn.hutool.core.collection.ListUtil;
  5 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
3 import cn.iocoder.yudao.framework.common.pojo.PageResult; 6 import cn.iocoder.yudao.framework.common.pojo.PageResult;
4 import cn.iocoder.yudao.framework.common.util.object.BeanUtils; 7 import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
5 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO; 8 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO;
  9 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentSearchReqVO;
6 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO; 10 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO;
7 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO; 11 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO;
  12 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
8 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; 13 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
  14 +import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
9 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper; 15 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper;
  16 +import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService;
  17 +import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;
10 import jakarta.annotation.Resource; 18 import jakarta.annotation.Resource;
11 import lombok.extern.slf4j.Slf4j; 19 import lombok.extern.slf4j.Slf4j;
  20 +import org.springframework.ai.document.Document;
  21 +import org.springframework.ai.vectorstore.SearchRequest;
  22 +import org.springframework.ai.vectorstore.VectorStore;
  23 +import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder;
12 import org.springframework.stereotype.Service; 24 import org.springframework.stereotype.Service;
13 25
  26 +import java.util.List;
  27 +import java.util.Objects;
  28 +
  29 +import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
  30 +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_NOT_EXISTS;
  31 +
14 /** 32 /**
15 * AI 知识库分片 Service 实现类 33 * AI 知识库分片 Service 实现类
16 * 34 *
@@ -23,6 +41,13 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService @@ -23,6 +41,13 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
23 @Resource 41 @Resource
24 private AiKnowledgeSegmentMapper segmentMapper; 42 private AiKnowledgeSegmentMapper segmentMapper;
25 43
  44 + @Resource
  45 + private AiKnowledgeService knowledgeService;
  46 + @Resource
  47 + private AiChatModelService chatModelService;
  48 + @Resource
  49 + private AiApiKeyService apiKeyService;
  50 +
26 @Override 51 @Override
27 public PageResult<AiKnowledgeSegmentDO> getKnowledgeSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO) { 52 public PageResult<AiKnowledgeSegmentDO> getKnowledgeSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO) {
28 return segmentMapper.selectPage(pageReqVO); 53 return segmentMapper.selectPage(pageReqVO);
@@ -30,13 +55,80 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService @@ -30,13 +55,80 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
30 55
31 @Override 56 @Override
32 public void updateKnowledgeSegment(AiKnowledgeSegmentUpdateReqVO reqVO) { 57 public void updateKnowledgeSegment(AiKnowledgeSegmentUpdateReqVO reqVO) {
33 - segmentMapper.updateById(BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class));  
34 - // TODO @xin 重新向量化 58 + // 1. 校验
  59 + AiKnowledgeSegmentDO oldKnowledgeSegment = validateKnowledgeSegmentExists(reqVO.getId());
  60 +
  61 + // 2.1 获取知识库向量实例
  62 + VectorStore vectorStore = knowledgeService.getVectorStoreById(oldKnowledgeSegment.getKnowledgeId());
  63 + // 2.2 删除原向量
  64 + vectorStore.delete(List.of(oldKnowledgeSegment.getVectorId()));
  65 + // 2.3 重新向量化
  66 + Document document = new Document(reqVO.getContent());
  67 + document.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, oldKnowledgeSegment.getKnowledgeId());
  68 + vectorStore.add(List.of(document));
  69 +
  70 + // 3. 更新段落内容
  71 + AiKnowledgeSegmentDO knowledgeSegment = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class);
  72 + knowledgeSegment.setVectorId(document.getId());
  73 + segmentMapper.updateById(knowledgeSegment);
35 } 74 }
36 75
37 @Override 76 @Override
38 public void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO) { 77 public void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO) {
39 - segmentMapper.updateById(BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class));  
40 - // TODO @xin 1.禁用删除向量 2.启用重新向量化 78 + // 0 校验
  79 + AiKnowledgeSegmentDO oldKnowledgeSegment = validateKnowledgeSegmentExists(reqVO.getId());
  80 + // 1 获取知识库向量实例
  81 + VectorStore vectorStore = knowledgeService.getVectorStoreById(oldKnowledgeSegment.getKnowledgeId());
  82 + AiKnowledgeSegmentDO knowledgeSegment = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class);
  83 +
  84 + if (Objects.equals(reqVO.getStatus(), CommonStatusEnum.ENABLE.getStatus())) {
  85 + // 2.1 启用重新向量化
  86 + Document document = new Document(oldKnowledgeSegment.getContent());
  87 + document.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, oldKnowledgeSegment.getKnowledgeId());
  88 + vectorStore.add(List.of(document));
  89 + knowledgeSegment.setVectorId(document.getId());
  90 + } else {
  91 + // 2.2 禁用删除向量
  92 + vectorStore.delete(List.of(oldKnowledgeSegment.getVectorId()));
  93 + knowledgeSegment.setVectorId("");
  94 + }
  95 + // 3 更新段落状态
  96 + segmentMapper.updateById(knowledgeSegment);
41 } 97 }
  98 +
  99 + @Override
  100 + public List<AiKnowledgeSegmentDO> similaritySearch(AiKnowledgeSegmentSearchReqVO reqVO) {
  101 + // 1. 校验
  102 + AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(reqVO.getKnowledgeId());
  103 + AiChatModelDO model = chatModelService.validateChatModel(knowledge.getModelId());
  104 +
  105 + // 2. 获取向量存储实例
  106 + VectorStore vectorStore = apiKeyService.getOrCreateVectorStore(model.getKeyId());
  107 +
  108 + // 3.1 向量检索
  109 + List<Document> documentList = vectorStore.similaritySearch(SearchRequest.query(reqVO.getContent())
  110 + .withTopK(knowledge.getTopK())
  111 + .withSimilarityThreshold(knowledge.getSimilarityThreshold())
  112 + .withFilterExpression(new FilterExpressionBuilder().eq(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, reqVO.getKnowledgeId()).build()));
  113 + if (CollUtil.isEmpty(documentList)) {
  114 + return ListUtil.empty();
  115 + }
  116 + // 3.2 段落召回
  117 + return segmentMapper.selectListByVectorIds(CollUtil.getFieldValues(documentList, "id", String.class));
  118 + }
  119 +
  120 + /**
  121 + * 校验段落是否存在
  122 + *
  123 + * @param id 文档编号
  124 + * @return 段落信息
  125 + */
  126 + private AiKnowledgeSegmentDO validateKnowledgeSegmentExists(Long id) {
  127 + AiKnowledgeSegmentDO knowledgeSegment = segmentMapper.selectById(id);
  128 + if (knowledgeSegment == null) {
  129 + throw exception(KNOWLEDGE_SEGMENT_NOT_EXISTS);
  130 + }
  131 + return knowledgeSegment;
  132 + }
  133 +
42 } 134 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java
1 package cn.iocoder.yudao.module.ai.service.knowledge; 1 package cn.iocoder.yudao.module.ai.service.knowledge;
2 2
3 -import cn.iocoder.yudao.framework.common.pojo.PageParam;  
4 import cn.iocoder.yudao.framework.common.pojo.PageResult; 3 import cn.iocoder.yudao.framework.common.pojo.PageResult;
5 -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateMyReqVO;  
6 -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateMyReqVO; 4 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateReqVO;
  5 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO;
  6 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateReqVO;
7 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; 7 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
  8 +import org.springframework.ai.vectorstore.VectorStore;
8 9
9 /** 10 /**
10 * AI 知识库-基础信息 Service 接口 11 * AI 知识库-基础信息 Service 接口
@@ -14,23 +15,21 @@ import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; @@ -14,23 +15,21 @@ import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
14 public interface AiKnowledgeService { 15 public interface AiKnowledgeService {
15 16
16 /** 17 /**
17 - * 创建【我的】知识库 18 + * 创建知识库
18 * 19 *
19 * @param createReqVO 创建信息 20 * @param createReqVO 创建信息
20 * @param userId 用户编号 21 * @param userId 用户编号
21 * @return 编号 22 * @return 编号
22 */ 23 */
23 - Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId);  
24 - 24 + Long createKnowledge(AiKnowledgeCreateReqVO createReqVO, Long userId);
25 25
26 /** 26 /**
27 - * 创建【我的】知识库 27 + * 更新知识库
28 * 28 *
29 * @param updateReqVO 更新信息 29 * @param updateReqVO 更新信息
30 * @param userId 用户编号 30 * @param userId 用户编号
31 */ 31 */
32 - void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId);  
33 - 32 + void updateKnowledge(AiKnowledgeUpdateReqVO updateReqVO, Long userId);
34 33
35 /** 34 /**
36 * 校验知识库是否存在 35 * 校验知识库是否存在
@@ -40,11 +39,20 @@ public interface AiKnowledgeService { @@ -40,11 +39,20 @@ public interface AiKnowledgeService {
40 AiKnowledgeDO validateKnowledgeExists(Long id); 39 AiKnowledgeDO validateKnowledgeExists(Long id);
41 40
42 /** 41 /**
43 - * 获得【我的】知识库分页 42 + * 获得知识库分页
44 * 43 *
45 - * @param userId 用户编号  
46 - * @param pageReqVO 分页查询 44 + * @param userId 用户编号
  45 + * @param pageReqVO 分页查询
47 * @return 知识库分页 46 * @return 知识库分页
48 */ 47 */
49 - PageResult<AiKnowledgeDO> getKnowledgePageMy(Long userId, PageParam pageReqVO); 48 + PageResult<AiKnowledgeDO> getKnowledgePage(Long userId, AiKnowledgePageReqVO pageReqVO);
  49 +
  50 + /**
  51 + * 根据知识库编号获取向量存储实例
  52 + *
  53 + * @param id 知识库编号
  54 + * @return 向量存储实例
  55 + */
  56 + VectorStore getVectorStoreById(Long id);
  57 +
50 } 58 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java
@@ -2,17 +2,19 @@ package cn.iocoder.yudao.module.ai.service.knowledge; @@ -2,17 +2,19 @@ package cn.iocoder.yudao.module.ai.service.knowledge;
2 2
3 import cn.hutool.core.util.ObjUtil; 3 import cn.hutool.core.util.ObjUtil;
4 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; 4 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
5 -import cn.iocoder.yudao.framework.common.pojo.PageParam;  
6 import cn.iocoder.yudao.framework.common.pojo.PageResult; 5 import cn.iocoder.yudao.framework.common.pojo.PageResult;
7 import cn.iocoder.yudao.framework.common.util.object.BeanUtils; 6 import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
8 -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateMyReqVO;  
9 -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateMyReqVO; 7 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateReqVO;
  8 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO;
  9 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateReqVO;
10 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; 10 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
11 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; 11 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
12 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeMapper; 12 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeMapper;
  13 +import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService;
13 import cn.iocoder.yudao.module.ai.service.model.AiChatModelService; 14 import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;
14 import jakarta.annotation.Resource; 15 import jakarta.annotation.Resource;
15 import lombok.extern.slf4j.Slf4j; 16 import lombok.extern.slf4j.Slf4j;
  17 +import org.springframework.ai.vectorstore.VectorStore;
16 import org.springframework.stereotype.Service; 18 import org.springframework.stereotype.Service;
17 19
18 import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; 20 import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
@@ -28,15 +30,17 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_ @@ -28,15 +30,17 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_
28 public class AiKnowledgeServiceImpl implements AiKnowledgeService { 30 public class AiKnowledgeServiceImpl implements AiKnowledgeService {
29 31
30 @Resource 32 @Resource
31 - private AiChatModelService chatModalService; 33 + private AiKnowledgeMapper knowledgeMapper;
32 34
33 @Resource 35 @Resource
34 - private AiKnowledgeMapper knowledgeMapper; 36 + private AiChatModelService chatModelService;
  37 + @Resource
  38 + private AiApiKeyService apiKeyService;
35 39
36 @Override 40 @Override
37 - public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) { 41 + public Long createKnowledge(AiKnowledgeCreateReqVO createReqVO, Long userId) {
38 // 1. 校验模型配置 42 // 1. 校验模型配置
39 - AiChatModelDO model = chatModalService.validateChatModel(createReqVO.getModelId()); 43 + AiChatModelDO model = chatModelService.validateChatModel(createReqVO.getModelId());
40 44
41 // 2. 插入知识库 45 // 2. 插入知识库
42 AiKnowledgeDO knowledgeBase = BeanUtils.toBean(createReqVO, AiKnowledgeDO.class) 46 AiKnowledgeDO knowledgeBase = BeanUtils.toBean(createReqVO, AiKnowledgeDO.class)
@@ -46,14 +50,14 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService { @@ -46,14 +50,14 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService {
46 } 50 }
47 51
48 @Override 52 @Override
49 - public void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId) { 53 + public void updateKnowledge(AiKnowledgeUpdateReqVO updateReqVO, Long userId) {
50 // 1.1 校验知识库存在 54 // 1.1 校验知识库存在
51 AiKnowledgeDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId()); 55 AiKnowledgeDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId());
52 if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) { 56 if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) {
53 throw exception(KNOWLEDGE_NOT_EXISTS); 57 throw exception(KNOWLEDGE_NOT_EXISTS);
54 } 58 }
55 // 1.2 校验模型配置 59 // 1.2 校验模型配置
56 - AiChatModelDO model = chatModalService.validateChatModel(updateReqVO.getModelId()); 60 + AiChatModelDO model = chatModelService.validateChatModel(updateReqVO.getModelId());
57 61
58 // 2. 更新知识库 62 // 2. 更新知识库
59 AiKnowledgeDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeDO.class); 63 AiKnowledgeDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeDO.class);
@@ -71,8 +75,16 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService { @@ -71,8 +75,16 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService {
71 } 75 }
72 76
73 @Override 77 @Override
74 - public PageResult<AiKnowledgeDO> getKnowledgePageMy(Long userId, PageParam pageReqVO) {  
75 - return knowledgeMapper.selectPageByMy(userId, pageReqVO); 78 + public PageResult<AiKnowledgeDO> getKnowledgePage(Long userId, AiKnowledgePageReqVO pageReqVO) {
  79 + return knowledgeMapper.selectPage(userId, pageReqVO);
  80 + }
  81 +
  82 + @Override
  83 + public VectorStore getVectorStoreById(Long id) {
  84 + AiKnowledgeDO knowledge = validateKnowledgeExists(id);
  85 + AiChatModelDO model = chatModelService.validateChatModel(knowledge.getModelId());
  86 + // 创建或获取 VectorStore 对象
  87 + return apiKeyService.getOrCreateVectorStore(model.getKeyId());
76 } 88 }
77 89
78 } 90 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiApiKeyServiceImpl.java
@@ -2,7 +2,6 @@ package cn.iocoder.yudao.module.ai.service.model; @@ -2,7 +2,6 @@ package cn.iocoder.yudao.module.ai.service.model;
2 2
3 import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum; 3 import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum;
4 import cn.iocoder.yudao.framework.ai.core.factory.AiModelFactory; 4 import cn.iocoder.yudao.framework.ai.core.factory.AiModelFactory;
5 -import cn.iocoder.yudao.framework.ai.core.factory.AiVectorStoreFactory;  
6 import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi; 5 import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi;
7 import cn.iocoder.yudao.framework.ai.core.model.suno.api.SunoApi; 6 import cn.iocoder.yudao.framework.ai.core.model.suno.api.SunoApi;
8 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; 7 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
@@ -39,8 +38,6 @@ public class AiApiKeyServiceImpl implements AiApiKeyService { @@ -39,8 +38,6 @@ public class AiApiKeyServiceImpl implements AiApiKeyService {
39 38
40 @Resource 39 @Resource
41 private AiModelFactory modelFactory; 40 private AiModelFactory modelFactory;
42 - @Resource  
43 - private AiVectorStoreFactory vectorFactory;  
44 41
45 @Override 42 @Override
46 public Long createApiKey(AiApiKeySaveReqVO createReqVO) { 43 public Long createApiKey(AiApiKeySaveReqVO createReqVO) {
@@ -149,7 +146,8 @@ public class AiApiKeyServiceImpl implements AiApiKeyService { @@ -149,7 +146,8 @@ public class AiApiKeyServiceImpl implements AiApiKeyService {
149 public VectorStore getOrCreateVectorStore(Long id) { 146 public VectorStore getOrCreateVectorStore(Long id) {
150 AiApiKeyDO apiKey = validateApiKey(id); 147 AiApiKeyDO apiKey = validateApiKey(id);
151 AiPlatformEnum platform = AiPlatformEnum.validatePlatform(apiKey.getPlatform()); 148 AiPlatformEnum platform = AiPlatformEnum.validatePlatform(apiKey.getPlatform());
152 - return vectorFactory.getOrCreateVectorStore(getEmbeddingModel(id), platform, apiKey.getApiKey(), apiKey.getUrl()); 149 + // 创建或获取 VectorStore 对象
  150 + return modelFactory.getOrCreateVectorStore(getEmbeddingModel(id), platform, apiKey.getApiKey(), apiKey.getUrl());
153 } 151 }
154 152
155 } 153 }
156 \ No newline at end of file 154 \ No newline at end of file
yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml
@@ -46,11 +46,13 @@ @@ -46,11 +46,13 @@
46 </dependency> 46 </dependency>
47 47
48 <!-- 向量化,基于 Redis 存储,Tika 解析内容 --> 48 <!-- 向量化,基于 Redis 存储,Tika 解析内容 -->
49 - <dependency>  
50 - <groupId>${spring-ai.groupId}</groupId>  
51 - <artifactId>spring-ai-transformers-spring-boot-starter</artifactId>  
52 - <version>${spring-ai.version}</version>  
53 - </dependency> 49 +
  50 + <!-- 暂不做经济型,先注释 TODO 经济型是啥呀? -->
  51 + <!-- <dependency>-->
  52 + <!-- <groupId>${spring-ai.groupId}</groupId>-->
  53 + <!-- <artifactId>spring-ai-transformers-spring-boot-starter</artifactId>-->
  54 + <!-- <version>${spring-ai.version}</version>-->
  55 + <!-- </dependency>-->
54 <dependency> 56 <dependency>
55 <groupId>${spring-ai.groupId}</groupId> 57 <groupId>${spring-ai.groupId}</groupId>
56 <artifactId>spring-ai-tika-document-reader</artifactId> 58 <artifactId>spring-ai-tika-document-reader</artifactId>
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java
@@ -2,8 +2,6 @@ package cn.iocoder.yudao.framework.ai.config; @@ -2,8 +2,6 @@ package cn.iocoder.yudao.framework.ai.config;
2 2
3 import cn.iocoder.yudao.framework.ai.core.factory.AiModelFactory; 3 import cn.iocoder.yudao.framework.ai.core.factory.AiModelFactory;
4 import cn.iocoder.yudao.framework.ai.core.factory.AiModelFactoryImpl; 4 import cn.iocoder.yudao.framework.ai.core.factory.AiModelFactoryImpl;
5 -import cn.iocoder.yudao.framework.ai.core.factory.AiVectorStoreFactory;  
6 -import cn.iocoder.yudao.framework.ai.core.factory.AiVectorStoreFactoryImpl;  
7 import cn.iocoder.yudao.framework.ai.core.model.deepseek.DeepSeekChatModel; 5 import cn.iocoder.yudao.framework.ai.core.model.deepseek.DeepSeekChatModel;
8 import cn.iocoder.yudao.framework.ai.core.model.deepseek.DeepSeekChatOptions; 6 import cn.iocoder.yudao.framework.ai.core.model.deepseek.DeepSeekChatOptions;
9 import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi; 7 import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi;
@@ -38,11 +36,6 @@ public class YudaoAiAutoConfiguration { @@ -38,11 +36,6 @@ public class YudaoAiAutoConfiguration {
38 return new AiModelFactoryImpl(); 36 return new AiModelFactoryImpl();
39 } 37 }
40 38
41 - @Bean  
42 - public AiVectorStoreFactory aiVectorFactory() {  
43 - return new AiVectorStoreFactoryImpl();  
44 - }  
45 -  
46 39
47 // ========== 各种 AI Client 创建 ========== 40 // ========== 各种 AI Client 创建 ==========
48 41
@@ -89,7 +82,7 @@ public class YudaoAiAutoConfiguration { @@ -89,7 +82,7 @@ public class YudaoAiAutoConfiguration {
89 // TODO @xin 免费版本 82 // TODO @xin 免费版本
90 // @Bean 83 // @Bean
91 // @Lazy // TODO 芋艿:临时注释,避免无法启动」 84 // @Lazy // TODO 芋艿:临时注释,避免无法启动」
92 -// public EmbeddingModel transformersEmbeddingClient() { 85 +// public TransformersEmbeddingModel transformersEmbeddingClient() {
93 // return new TransformersEmbeddingModel(MetadataMode.EMBED); 86 // return new TransformersEmbeddingModel(MetadataMode.EMBED);
94 // } 87 // }
95 88
@@ -98,23 +91,24 @@ public class YudaoAiAutoConfiguration { @@ -98,23 +91,24 @@ public class YudaoAiAutoConfiguration {
98 */ 91 */
99 // @Bean 92 // @Bean
100 // @Lazy // TODO 芋艿:临时注释,避免无法启动 93 // @Lazy // TODO 芋艿:临时注释,避免无法启动
101 -// public RedisVectorStore vectorStore(TongYiTextEmbeddingModel tongYiTextEmbeddingModel, RedisVectorStoreProperties properties, 94 +// public RedisVectorStore vectorStore(TransformersEmbeddingModel embeddingModel, RedisVectorStoreProperties properties,
102 // RedisProperties redisProperties) { 95 // RedisProperties redisProperties) {
103 // var config = RedisVectorStore.RedisVectorStoreConfig.builder() 96 // var config = RedisVectorStore.RedisVectorStoreConfig.builder()
104 // .withIndexName(properties.getIndex()) 97 // .withIndexName(properties.getIndex())
105 // .withPrefix(properties.getPrefix()) 98 // .withPrefix(properties.getPrefix())
  99 +// .withMetadataFields(new RedisVectorStore.MetadataField("knowledgeId", Schema.FieldType.NUMERIC))
106 // .build(); 100 // .build();
107 // 101 //
108 -// RedisVectorStore redisVectorStore = new RedisVectorStore(config, tongYiTextEmbeddingModel, 102 +// RedisVectorStore redisVectorStore = new RedisVectorStore(config, embeddingModel,
109 // new JedisPooled(redisProperties.getHost(), redisProperties.getPort()), 103 // new JedisPooled(redisProperties.getHost(), redisProperties.getPort()),
110 // properties.isInitializeSchema()); 104 // properties.isInitializeSchema());
111 // redisVectorStore.afterPropertiesSet(); 105 // redisVectorStore.afterPropertiesSet();
112 // return redisVectorStore; 106 // return redisVectorStore;
113 // } 107 // }
114 -  
115 @Bean 108 @Bean
116 @Lazy // TODO 芋艿:临时注释,避免无法启动 109 @Lazy // TODO 芋艿:临时注释,避免无法启动
117 public TokenTextSplitter tokenTextSplitter() { 110 public TokenTextSplitter tokenTextSplitter() {
  111 + //TODO @xin 配置提取
118 return new TokenTextSplitter(500, 100, 5, 10000, true); 112 return new TokenTextSplitter(500, 100, 5, 10000, true);
119 } 113 }
120 114
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactory.java
@@ -6,6 +6,7 @@ import cn.iocoder.yudao.framework.ai.core.model.suno.api.SunoApi; @@ -6,6 +6,7 @@ import cn.iocoder.yudao.framework.ai.core.model.suno.api.SunoApi;
6 import org.springframework.ai.chat.model.ChatModel; 6 import org.springframework.ai.chat.model.ChatModel;
7 import org.springframework.ai.embedding.EmbeddingModel; 7 import org.springframework.ai.embedding.EmbeddingModel;
8 import org.springframework.ai.image.ImageModel; 8 import org.springframework.ai.image.ImageModel;
  9 +import org.springframework.ai.vectorstore.VectorStore;
9 10
10 /** 11 /**
11 * AI Model 模型工厂的接口类 12 * AI Model 模型工厂的接口类
@@ -92,4 +93,17 @@ public interface AiModelFactory { @@ -92,4 +93,17 @@ public interface AiModelFactory {
92 */ 93 */
93 EmbeddingModel getOrCreateEmbeddingModel(AiPlatformEnum platform, String apiKey, String url); 94 EmbeddingModel getOrCreateEmbeddingModel(AiPlatformEnum platform, String apiKey, String url);
94 95
  96 + /**
  97 + * 基于指定配置,获得 VectorStore 对象
  98 + * <p>
  99 + * 如果不存在,则进行创建
  100 + *
  101 + * @param embeddingModel 嵌入模型
  102 + * @param platform 平台
  103 + * @param apiKey API KEY
  104 + * @param url API URL
  105 + * @return VectorStore 对象
  106 + */
  107 + VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url);
  108 +
95 } 109 }
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactoryImpl.java
@@ -13,6 +13,7 @@ import cn.iocoder.yudao.framework.ai.core.model.deepseek.DeepSeekChatModel; @@ -13,6 +13,7 @@ import cn.iocoder.yudao.framework.ai.core.model.deepseek.DeepSeekChatModel;
13 import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi; 13 import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi;
14 import cn.iocoder.yudao.framework.ai.core.model.suno.api.SunoApi; 14 import cn.iocoder.yudao.framework.ai.core.model.suno.api.SunoApi;
15 import cn.iocoder.yudao.framework.ai.core.model.xinghuo.XingHuoChatModel; 15 import cn.iocoder.yudao.framework.ai.core.model.xinghuo.XingHuoChatModel;
  16 +import cn.iocoder.yudao.framework.common.util.spring.SpringUtils;
16 import com.alibaba.cloud.ai.tongyi.TongYiAutoConfiguration; 17 import com.alibaba.cloud.ai.tongyi.TongYiAutoConfiguration;
17 import com.alibaba.cloud.ai.tongyi.TongYiConnectionProperties; 18 import com.alibaba.cloud.ai.tongyi.TongYiConnectionProperties;
18 import com.alibaba.cloud.ai.tongyi.chat.TongYiChatModel; 19 import com.alibaba.cloud.ai.tongyi.chat.TongYiChatModel;
@@ -54,13 +55,18 @@ import org.springframework.ai.qianfan.api.QianFanApi; @@ -54,13 +55,18 @@ import org.springframework.ai.qianfan.api.QianFanApi;
54 import org.springframework.ai.qianfan.api.QianFanImageApi; 55 import org.springframework.ai.qianfan.api.QianFanImageApi;
55 import org.springframework.ai.stabilityai.StabilityAiImageModel; 56 import org.springframework.ai.stabilityai.StabilityAiImageModel;
56 import org.springframework.ai.stabilityai.api.StabilityAiApi; 57 import org.springframework.ai.stabilityai.api.StabilityAiApi;
  58 +import org.springframework.ai.vectorstore.RedisVectorStore;
  59 +import org.springframework.ai.vectorstore.VectorStore;
57 import org.springframework.ai.zhipuai.ZhiPuAiChatModel; 60 import org.springframework.ai.zhipuai.ZhiPuAiChatModel;
58 import org.springframework.ai.zhipuai.ZhiPuAiImageModel; 61 import org.springframework.ai.zhipuai.ZhiPuAiImageModel;
59 import org.springframework.ai.zhipuai.api.ZhiPuAiApi; 62 import org.springframework.ai.zhipuai.api.ZhiPuAiApi;
60 import org.springframework.ai.zhipuai.api.ZhiPuAiImageApi; 63 import org.springframework.ai.zhipuai.api.ZhiPuAiImageApi;
  64 +import org.springframework.boot.autoconfigure.data.redis.RedisProperties;
61 import org.springframework.retry.support.RetryTemplate; 65 import org.springframework.retry.support.RetryTemplate;
62 import org.springframework.web.client.ResponseErrorHandler; 66 import org.springframework.web.client.ResponseErrorHandler;
63 import org.springframework.web.client.RestClient; 67 import org.springframework.web.client.RestClient;
  68 +import redis.clients.jedis.JedisPooled;
  69 +import redis.clients.jedis.search.Schema;
64 70
65 import java.util.List; 71 import java.util.List;
66 72
@@ -191,6 +197,25 @@ public class AiModelFactoryImpl implements AiModelFactory { @@ -191,6 +197,25 @@ public class AiModelFactoryImpl implements AiModelFactory {
191 }); 197 });
192 } 198 }
193 199
  200 + @Override
  201 + public VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url) {
  202 + String cacheKey = buildClientCacheKey(VectorStore.class, platform, apiKey, url);
  203 + return Singleton.get(cacheKey, (Func0<VectorStore>) () -> {
  204 + String prefix = StrUtil.format("{}#{}:", platform.getPlatform(), apiKey);
  205 + var config = RedisVectorStore.RedisVectorStoreConfig.builder()
  206 + .withIndexName(cacheKey)
  207 + .withPrefix(prefix)
  208 + .withMetadataFields(new RedisVectorStore.MetadataField("knowledgeId", Schema.FieldType.NUMERIC))
  209 + .build();
  210 + RedisProperties redisProperties = SpringUtils.getBean(RedisProperties.class);
  211 + RedisVectorStore redisVectorStore = new RedisVectorStore(config, embeddingModel,
  212 + new JedisPooled(redisProperties.getHost(), redisProperties.getPort()),
  213 + true);
  214 + redisVectorStore.afterPropertiesSet();
  215 + return redisVectorStore;
  216 + });
  217 + }
  218 +
194 private static String buildClientCacheKey(Class<?> clazz, Object... params) { 219 private static String buildClientCacheKey(Class<?> clazz, Object... params) {
195 if (ArrayUtil.isEmpty(params)) { 220 if (ArrayUtil.isEmpty(params)) {
196 return clazz.getName(); 221 return clazz.getName();
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiVectorStoreFactory.java deleted
1 -package cn.iocoder.yudao.framework.ai.core.factory;  
2 -  
3 -import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum;  
4 -import org.springframework.ai.embedding.EmbeddingModel;  
5 -import org.springframework.ai.vectorstore.VectorStore;  
6 -  
7 -// TODO @xin:也放到 AiModelFactory 里面好了,后续改成 AiFactory  
8 -/**  
9 - * AI Vector 模型工厂的接口类  
10 - *  
11 - * @author xiaoxin  
12 - */  
13 -public interface AiVectorStoreFactory {  
14 -  
15 - /**  
16 - * 基于指定配置,获得 VectorStore 对象  
17 - * <p>  
18 - * 如果不存在,则进行创建  
19 - *  
20 - * @param embeddingModel 嵌入模型  
21 - * @param platform 平台  
22 - * @param apiKey API KEY  
23 - * @param url API URL  
24 - * @return VectorStore 对象  
25 - */  
26 - VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url);  
27 -  
28 -}  
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiVectorStoreFactoryImpl.java deleted
1 -package cn.iocoder.yudao.framework.ai.core.factory;  
2 -  
3 -import cn.hutool.core.lang.Singleton;  
4 -import cn.hutool.core.lang.func.Func0;  
5 -import cn.hutool.core.util.ArrayUtil;  
6 -import cn.hutool.core.util.StrUtil;  
7 -import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum;  
8 -import cn.iocoder.yudao.framework.common.util.spring.SpringUtils;  
9 -import org.springframework.ai.embedding.EmbeddingModel;  
10 -import org.springframework.ai.vectorstore.RedisVectorStore;  
11 -import org.springframework.ai.vectorstore.VectorStore;  
12 -import org.springframework.boot.autoconfigure.data.redis.RedisProperties;  
13 -import redis.clients.jedis.JedisPooled;  
14 -  
15 -/**  
16 - * AI Vector 模型工厂的实现类  
17 - * 使用 redisVectorStore 实现 VectorStore  
18 - *  
19 - * @author xiaoxin  
20 - */  
21 -public class AiVectorStoreFactoryImpl implements AiVectorStoreFactory {  
22 -  
23 - @Override  
24 - public VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url) {  
25 - String cacheKey = buildClientCacheKey(VectorStore.class, platform, apiKey, url);  
26 - return Singleton.get(cacheKey, (Func0<VectorStore>) () -> {  
27 - // TODO 芋艿 @xin 这两个配置取哪好呢  
28 - // TODO 不同模型的向量维度可能会不一样,目前看貌似是以 index 来做区分的,维度不一样存不到一个 index 上  
29 - // TODO 回复:好的哈  
30 - String index = "default-index";  
31 - String prefix = "default:";  
32 - var config = RedisVectorStore.RedisVectorStoreConfig.builder()  
33 - .withIndexName(index)  
34 - .withPrefix(prefix)  
35 - .build();  
36 - RedisProperties redisProperties = SpringUtils.getBean(RedisProperties.class);  
37 - RedisVectorStore redisVectorStore = new RedisVectorStore(config, embeddingModel,  
38 - new JedisPooled(redisProperties.getHost(), redisProperties.getPort()),  
39 - true);  
40 - redisVectorStore.afterPropertiesSet();  
41 - return redisVectorStore;  
42 - });  
43 - }  
44 -  
45 - private static String buildClientCacheKey(Class<?> clazz, Object... params) {  
46 - if (ArrayUtil.isEmpty(params)) {  
47 - return clazz.getName();  
48 - }  
49 - return StrUtil.format("{}#{}", clazz.getName(), ArrayUtil.join(params, "_"));  
50 - }  
51 -  
52 -}