BJ_ZTEITS-Developers / parking-cloud-yudao

Commit 0b1d9ce251932f9940a7f797f6c5cb43e09adc89

Authored by 芋道源码
Committed by Gitee
2 parents 059f64ac 9b8136ef

!20 【新增】AI 知识库: 段落召回、段落启禁用、配置自定义 

Merge pull request !20 from 小新/master-jdk21-ai
Inline Side-by-side
Showing 19 changed files with 277 additions and 124 deletions
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeCreateMyReqVO.java
  View file @0b1d9ce
... ... @@ -25,4 +25,11 @@ public class AiKnowledgeCreateMyReqVO {
25 25 @NotNull(message = "嵌入模型不能为空")
26 26 private Long modelId;
27 27  
  28 + @Schema(description = "相似性阈值", requiredMode = Schema.RequiredMode.REQUIRED, example = "0.5")
  29 + @NotNull(message = "相似性阈值不能为空")
  30 + private Double similarityThreshold;
  31 +
  32 + @Schema(description = "topK", requiredMode = Schema.RequiredMode.REQUIRED, example = "3")
  33 + @NotNull(message = "topK 不能为空")
  34 + private Integer topK;
28 35 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeDocumentCreateReqVO.java
  View file @0b1d9ce
... ... @@ -23,4 +23,23 @@ public class AiKnowledgeDocumentCreateReqVO {
23 23 @URL(message = "文档 URL 格式不正确")
24 24 private String url;
25 25  
  26 + @Schema(description = "每个文本块的目标 token 数", requiredMode = Schema.RequiredMode.REQUIRED, example = "800")
  27 + @NotNull(message = "每个文本块的目标 token 数不能为空")
  28 + private Integer defaultChunkSize;
  29 +
  30 + @Schema(description = "每个文本块的最小字符数", requiredMode = Schema.RequiredMode.REQUIRED, example = "350")
  31 + @NotNull(message = "每个文本块的最小字符数不能为空")
  32 + private Integer minChunkSizeChars;
  33 +
  34 + @Schema(description = "丢弃阈值", requiredMode = Schema.RequiredMode.REQUIRED, example = "5")
  35 + @NotNull(message = "丢弃阈值不能为空")
  36 + private Integer minChunkLengthToEmbed;
  37 +
  38 + @Schema(description = "最大块数", requiredMode = Schema.RequiredMode.REQUIRED, example = "10000")
  39 + @NotNull(message = "最大块数不能为空")
  40 + private Integer maxNumChunks;
  41 +
  42 + @Schema(description = "分块是否保留分隔符", requiredMode = Schema.RequiredMode.REQUIRED, example = "true")
  43 + @NotNull(message = "分块是否保留分隔符不能为空")
  44 + private Boolean keepSeparator;
26 45 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentSearchReqVO.java 0 → 100644
  View file @0b1d9ce
  1 +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment;
  2 +
  3 +import io.swagger.v3.oas.annotations.media.Schema;
  4 +import lombok.Data;
  5 +
  6 +
  7 +@Schema(description = "管理后台 - AI 知识库段落召回 Request VO")
  8 +@Data
  9 +public class AiKnowledgeSegmentSearchReqVO {
  10 +
  11 + @Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790")
  12 + private Long knowledgeId;
  13 +
  14 + @Schema(description = "内容", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 学习路线")
  15 + private String content;
  16 +
  17 +}
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java
  View file @0b1d9ce
... ... @@ -52,6 +52,18 @@ public class AiKnowledgeDO extends BaseDO {
52 52 * 模型标识
53 53 */
54 54 private String model;
  55 +
  56 + /**
  57 + * topK
  58 + */
  59 + private Integer topK;
  60 +
  61 + /**
  62 + * 相似度阈值
  63 + */
  64 + private Double similarityThreshold;
  65 +
  66 +
55 67 /**
56 68 * 状态
57 69 * <p>
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java
  View file @0b1d9ce
... ... @@ -23,7 +23,7 @@ public class AiKnowledgeDocumentDO extends BaseDO {
23 23 private Long id;
24 24 /**
25 25 * 知识库编号
26   - *
  26 + * <p>
27 27 * 关联 {@link AiKnowledgeDO#getId()}
28 28 */
29 29 private Long knowledgeId;
... ... @@ -48,6 +48,26 @@ public class AiKnowledgeDocumentDO extends BaseDO {
48 48 */
49 49 private Integer wordCount;
50 50 /**
  51 + * 每个文本块的目标 token 数
  52 + */
  53 + private Integer defaultChunkSize;
  54 + /**
  55 + * 每个文本块的最小字符数
  56 + */
  57 + private Integer minChunkSizeChars;
  58 + /**
  59 + * 低于此值的块会被丢弃
  60 + */
  61 + private Integer minChunkLengthToEmbed;
  62 + /**
  63 + * 最大块数
  64 + */
  65 + private Integer maxNumChunks;
  66 + /**
  67 + * 分块是否保留分隔符
  68 + */
  69 + private Boolean keepSeparator;
  70 + /**
51 71 * 切片状态
52 72 * <p>
53 73 * 枚举 {@link AiKnowledgeDocumentStatusEnum}
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java
  View file @0b1d9ce
... ... @@ -2,6 +2,8 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
2 2  
3 3 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
4 4 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
  5 +import com.baomidou.mybatisplus.annotation.FieldStrategy;
  6 +import com.baomidou.mybatisplus.annotation.TableField;
5 7 import com.baomidou.mybatisplus.annotation.TableId;
6 8 import com.baomidou.mybatisplus.annotation.TableName;
7 9 import lombok.Data;
... ... @@ -25,16 +27,17 @@ public class AiKnowledgeSegmentDO extends BaseDO {
25 27 /**
26 28 * 向量库的编号
27 29 */
  30 + @TableField(updateStrategy = FieldStrategy.ALWAYS)
28 31 private String vectorId;
29 32 /**
30 33 * 知识库编号
31   - *
  34 + * <p>
32 35 * 关联 {@link AiKnowledgeDO#getId()}
33 36 */
34 37 private Long knowledgeId;
35 38 /**
36 39 * 文档编号
37   - *
  40 + * <p>
38 41 * 关联 {@link AiKnowledgeDocumentDO#getId()}
39 42 */
40 43 private Long documentId;
... ... @@ -52,7 +55,7 @@ public class AiKnowledgeSegmentDO extends BaseDO {
52 55 private Integer tokens;
53 56 /**
54 57 * 状态
55   - *
  58 + * <p>
56 59 * 枚举 {@link CommonStatusEnum}
57 60 */
58 61 private Integer status;
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java
  View file @0b1d9ce
... ... @@ -7,6 +7,8 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowle
7 7 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
8 8 import org.apache.ibatis.annotations.Mapper;
9 9  
  10 +import java.util.List;
  11 +
10 12 /**
11 13 * AI 知识库-分片 Mapper
12 14 *
... ... @@ -22,4 +24,10 @@ public interface AiKnowledgeSegmentMapper extends BaseMapperX&lt;AiKnowledgeSegment
22 24 .likeIfPresent(AiKnowledgeSegmentDO::getContent, reqVO.getKeyword())
23 25 .orderByDesc(AiKnowledgeSegmentDO::getId));
24 26 }
  27 +
  28 + default List<AiKnowledgeSegmentDO> selectList(List<String> vectorIdList) {
  29 + return selectList(new LambdaQueryWrapperX<AiKnowledgeSegmentDO>()
  30 + .in(AiKnowledgeSegmentDO::getVectorId, vectorIdList)
  31 + .orderByDesc(AiKnowledgeSegmentDO::getId));
  32 + }
25 33 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java
  View file @0b1d9ce
... ... @@ -9,15 +9,11 @@ import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
9 9 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO;
10 10 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO;
11 11 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO;
12   -import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
13 12 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
14 13 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
15   -import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
16 14 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper;
17 15 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper;
18 16 import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum;
19   -import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService;
20   -import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;
21 17 import jakarta.annotation.Resource;
22 18 import lombok.extern.slf4j.Slf4j;
23 19 import org.springframework.ai.document.Document;
... ... @@ -49,23 +45,15 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
49 45 private AiKnowledgeSegmentMapper segmentMapper;
50 46  
51 47 @Resource
52   - private TokenTextSplitter tokenTextSplitter;
53   - @Resource
54 48 private TokenCountEstimator tokenCountEstimator;
55   -
56   - @Resource
57   - private AiApiKeyService apiKeyService;
58 49 @Resource
59 50 private AiKnowledgeService knowledgeService;
60   - @Resource
61   - private AiChatModelService chatModelService;
62 51  
63 52 @Override
64 53 @Transactional(rollbackFor = Exception.class)
65 54 public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) {
66   - // 0. 校验
67   - AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(createReqVO.getKnowledgeId());
68   - AiChatModelDO model = chatModelService.validateChatModel(knowledge.getModelId());
  55 + // 0. 校验并获取向量存储实例
  56 + VectorStore vectorStore = knowledgeService.getVectorStoreById(createReqVO.getKnowledgeId());
69 57  
70 58 // 1.1 下载文档
71 59 TikaDocumentReader loader = new TikaDocumentReader(downloadFile(createReqVO.getUrl()));
... ... @@ -82,6 +70,9 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
82 70 return documentId;
83 71 }
84 72  
  73 + // 2 构造文本分段器
  74 + TokenTextSplitter tokenTextSplitter = new TokenTextSplitter(createReqVO.getDefaultChunkSize(), createReqVO.getMinChunkSizeChars(), createReqVO.getMinChunkLengthToEmbed(),
  75 + createReqVO.getMaxNumChunks(), createReqVO.getKeepSeparator());
85 76 // 2.1 文档分段
86 77 List<Document> segments = tokenTextSplitter.apply(documents);
87 78 // 2.2 分段内容入库
... ... @@ -92,8 +83,6 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
92 83 .setStatus(CommonStatusEnum.ENABLE.getStatus()));
93 84 segmentMapper.insertBatch(segmentDOList);
94 85  
95   - // 3.1 获取向量存储实例
96   - VectorStore vectorStore = apiKeyService.getOrCreateVectorStore(model.getKeyId());
97 86 // 3.2 向量化并存储
98 87 segments.forEach(segment -> segment.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, createReqVO.getKnowledgeId()));
99 88 vectorStore.add(segments);
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java
  View file @0b1d9ce
... ... @@ -2,10 +2,13 @@ package cn.iocoder.yudao.module.ai.service.knowledge;
2 2  
3 3 import cn.iocoder.yudao.framework.common.pojo.PageResult;
4 4 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO;
  5 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentSearchReqVO;
5 6 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO;
6 7 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO;
7 8 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
8 9  
  10 +import java.util.List;
  11 +
9 12 /**
10 13 * AI 知识库段落 Service 接口
11 14 *
... ... @@ -35,4 +38,13 @@ public interface AiKnowledgeSegmentService {
35 38 */
36 39 void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO);
37 40  
  41 +
  42 + /**
  43 + * 段落召回
  44 + *
  45 + * @param reqVO 召回请求信息
  46 + * @return 召回的段落
  47 + */
  48 + List<AiKnowledgeSegmentDO> similaritySearch(AiKnowledgeSegmentSearchReqVO reqVO);
  49 +
38 50 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
  View file @0b1d9ce
1 1 package cn.iocoder.yudao.module.ai.service.knowledge;
2 2  
  3 +import cn.hutool.core.collection.CollUtil;
  4 +import cn.hutool.core.collection.ListUtil;
  5 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
3 6 import cn.iocoder.yudao.framework.common.pojo.PageResult;
4 7 import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
5 8 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO;
  9 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentSearchReqVO;
6 10 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO;
7 11 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO;
  12 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
8 13 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
  14 +import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
9 15 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper;
  16 +import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService;
  17 +import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;
10 18 import jakarta.annotation.Resource;
11 19 import lombok.extern.slf4j.Slf4j;
  20 +import org.springframework.ai.document.Document;
  21 +import org.springframework.ai.vectorstore.SearchRequest;
  22 +import org.springframework.ai.vectorstore.VectorStore;
  23 +import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder;
12 24 import org.springframework.stereotype.Service;
13 25  
  26 +import java.util.List;
  27 +import java.util.Objects;
  28 +
  29 +import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
  30 +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_NOT_EXISTS;
  31 +
14 32 /**
15 33 * AI 知识库分片 Service 实现类
16 34 *
... ... @@ -23,6 +41,13 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
23 41 @Resource
24 42 private AiKnowledgeSegmentMapper segmentMapper;
25 43  
  44 + @Resource
  45 + private AiKnowledgeService knowledgeService;
  46 + @Resource
  47 + private AiChatModelService chatModelService;
  48 + @Resource
  49 + private AiApiKeyService apiKeyService;
  50 +
26 51 @Override
27 52 public PageResult<AiKnowledgeSegmentDO> getKnowledgeSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO) {
28 53 return segmentMapper.selectPage(pageReqVO);
... ... @@ -30,13 +55,80 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
30 55  
31 56 @Override
32 57 public void updateKnowledgeSegment(AiKnowledgeSegmentUpdateReqVO reqVO) {
33   - segmentMapper.updateById(BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class));
34   - // TODO @xin 重新向量化
  58 + // 0 校验
  59 + AiKnowledgeSegmentDO oldKnowledgeSegment = validateKnowledgeSegmentExists(reqVO.getId());
  60 + // 2.1 获取知识库向量实例
  61 + VectorStore vectorStore = knowledgeService.getVectorStoreById(oldKnowledgeSegment.getKnowledgeId());
  62 + // 2.2 删除原向量
  63 + vectorStore.delete(List.of(oldKnowledgeSegment.getVectorId()));
  64 +
  65 + // 2.3 重新向量化
  66 + Document document = new Document(reqVO.getContent());
  67 + document.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, oldKnowledgeSegment.getKnowledgeId());
  68 + vectorStore.add(List.of(document));
  69 +
  70 + // 2.1 更新段落内容
  71 + AiKnowledgeSegmentDO knowledgeSegment = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class);
  72 + knowledgeSegment.setVectorId(document.getId());
  73 + segmentMapper.updateById(knowledgeSegment);
35 74 }
36 75  
37 76 @Override
38 77 public void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO) {
39   - segmentMapper.updateById(BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class));
40   - // TODO @xin 1.禁用删除向量 2.启用重新向量化
  78 + // 0 校验
  79 + AiKnowledgeSegmentDO oldKnowledgeSegment = validateKnowledgeSegmentExists(reqVO.getId());
  80 + // 1 获取知识库向量实例
  81 + VectorStore vectorStore = knowledgeService.getVectorStoreById(oldKnowledgeSegment.getKnowledgeId());
  82 + AiKnowledgeSegmentDO knowledgeSegment = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class);
  83 +
  84 + if (Objects.equals(reqVO.getStatus(), CommonStatusEnum.ENABLE.getStatus())) {
  85 + // 2.1 启用重新向量化
  86 + Document document = new Document(oldKnowledgeSegment.getContent());
  87 + document.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, oldKnowledgeSegment.getKnowledgeId());
  88 + vectorStore.add(List.of(document));
  89 + knowledgeSegment.setVectorId(document.getId());
  90 + } else {
  91 + // 2.2 禁用删除向量
  92 + vectorStore.delete(List.of(oldKnowledgeSegment.getVectorId()));
  93 + knowledgeSegment.setVectorId(null);
  94 + }
  95 + // 3 更新段落状态
  96 + segmentMapper.updateById(knowledgeSegment);
  97 + }
  98 +
  99 + @Override
  100 + public List<AiKnowledgeSegmentDO> similaritySearch(AiKnowledgeSegmentSearchReqVO reqVO) {
  101 + // 0. 校验
  102 + AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(reqVO.getKnowledgeId());
  103 + AiChatModelDO model = chatModelService.validateChatModel(knowledge.getModelId());
  104 +
  105 + // 1.1 获取向量存储实例
  106 + VectorStore vectorStore = apiKeyService.getOrCreateVectorStore(model.getKeyId());
  107 +
  108 + // 1.2 向量检索
  109 + List<Document> documentList = vectorStore.similaritySearch(SearchRequest.query(reqVO.getContent())
  110 + .withTopK(knowledge.getTopK())
  111 + .withSimilarityThreshold(knowledge.getSimilarityThreshold())
  112 + .withFilterExpression(new FilterExpressionBuilder().eq(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, reqVO.getKnowledgeId()).build()));
  113 + if (CollUtil.isEmpty(documentList)) {
  114 + return ListUtil.empty();
  115 + }
  116 + // 2.1 段落召回
  117 + return segmentMapper.selectList(CollUtil.getFieldValues(documentList, "id", String.class));
  118 + }
  119 +
  120 +
  121 + /**
  122 + * 校验段落是否存在
  123 + *
  124 + * @param id 文档编号
  125 + * @return 段落信息
  126 + */
  127 + private AiKnowledgeSegmentDO validateKnowledgeSegmentExists(Long id) {
  128 + AiKnowledgeSegmentDO knowledgeSegment = segmentMapper.selectById(id);
  129 + if (knowledgeSegment == null) {
  130 + throw exception(KNOWLEDGE_SEGMENT_NOT_EXISTS);
  131 + }
  132 + return knowledgeSegment;
41 133 }
42 134 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java
  View file @0b1d9ce
... ... @@ -5,6 +5,7 @@ import cn.iocoder.yudao.framework.common.pojo.PageResult;
5 5 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateMyReqVO;
6 6 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateMyReqVO;
7 7 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
  8 +import org.springframework.ai.vectorstore.VectorStore;
8 9  
9 10 /**
10 11 * AI 知识库-基础信息 Service 接口
... ... @@ -47,4 +48,12 @@ public interface AiKnowledgeService {
47 48 * @return 知识库分页
48 49 */
49 50 PageResult<AiKnowledgeDO> getKnowledgePageMy(Long userId, PageParam pageReqVO);
  51 +
  52 + /**
  53 + * 根据知识库编号获取向量存储实例
  54 + *
  55 + * @param knowledgeId 知识库编号
  56 + * @return 向量存储实例
  57 + */
  58 + VectorStore getVectorStoreById(Long knowledgeId);
50 59 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java
  View file @0b1d9ce
... ... @@ -10,9 +10,11 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnow
10 10 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
11 11 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
12 12 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeMapper;
  13 +import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService;
13 14 import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;
14 15 import jakarta.annotation.Resource;
15 16 import lombok.extern.slf4j.Slf4j;
  17 +import org.springframework.ai.vectorstore.VectorStore;
16 18 import org.springframework.stereotype.Service;
17 19  
18 20 import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
... ... @@ -32,6 +34,10 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService {
32 34  
33 35 @Resource
34 36 private AiKnowledgeMapper knowledgeMapper;
  37 + @Resource
  38 + private AiChatModelService chatModelService;
  39 + @Resource
  40 + private AiApiKeyService apiKeyService;
35 41  
36 42 @Override
37 43 public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) {
... ... @@ -75,4 +81,11 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService {
75 81 return knowledgeMapper.selectPageByMy(userId, pageReqVO);
76 82 }
77 83  
  84 + @Override
  85 + public VectorStore getVectorStoreById(Long knowledgeId) {
  86 + AiKnowledgeDO knowledge = validateKnowledgeExists(knowledgeId);
  87 + AiChatModelDO model = chatModelService.validateChatModel(knowledge.getModelId());
  88 + return apiKeyService.getOrCreateVectorStore(model.getKeyId());
  89 + }
  90 +
78 91 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiApiKeyServiceImpl.java
  View file @0b1d9ce
... ... @@ -2,7 +2,6 @@ package cn.iocoder.yudao.module.ai.service.model;
2 2  
3 3 import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum;
4 4 import cn.iocoder.yudao.framework.ai.core.factory.AiModelFactory;
5   -import cn.iocoder.yudao.framework.ai.core.factory.AiVectorStoreFactory;
6 5 import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi;
7 6 import cn.iocoder.yudao.framework.ai.core.model.suno.api.SunoApi;
8 7 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
... ... @@ -39,8 +38,6 @@ public class AiApiKeyServiceImpl implements AiApiKeyService {
39 38  
40 39 @Resource
41 40 private AiModelFactory modelFactory;
42   - @Resource
43   - private AiVectorStoreFactory vectorFactory;
44 41  
45 42 @Override
46 43 public Long createApiKey(AiApiKeySaveReqVO createReqVO) {
... ... @@ -149,7 +146,7 @@ public class AiApiKeyServiceImpl implements AiApiKeyService {
149 146 public VectorStore getOrCreateVectorStore(Long id) {
150 147 AiApiKeyDO apiKey = validateApiKey(id);
151 148 AiPlatformEnum platform = AiPlatformEnum.validatePlatform(apiKey.getPlatform());
152   - return vectorFactory.getOrCreateVectorStore(getEmbeddingModel(id), platform, apiKey.getApiKey(), apiKey.getUrl());
  149 + return modelFactory.getOrCreateVectorStore(getEmbeddingModel(id), platform, apiKey.getApiKey(), apiKey.getUrl());
153 150 }
154 151  
155 152 }
156 153 \ No newline at end of file
... ...
yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml
  View file @0b1d9ce
... ... @@ -46,11 +46,13 @@
46 46 </dependency>
47 47  
48 48 <!-- 向量化，基于 Redis 存储，Tika 解析内容 -->
49   - <dependency>
50   - <groupId>${spring-ai.groupId}</groupId>
51   - <artifactId>spring-ai-transformers-spring-boot-starter</artifactId>
52   - <version>${spring-ai.version}</version>
53   - </dependency>
  49 +
  50 + <!-- 暂不做经济型，先注释 -->
  51 + <!-- <dependency>-->
  52 + <!-- <groupId>${spring-ai.groupId}</groupId>-->
  53 + <!-- <artifactId>spring-ai-transformers-spring-boot-starter</artifactId>-->
  54 + <!-- <version>${spring-ai.version}</version>-->
  55 + <!-- </dependency>-->
54 56 <dependency>
55 57 <groupId>${spring-ai.groupId}</groupId>
56 58 <artifactId>spring-ai-tika-document-reader</artifactId>
... ...
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java
  View file @0b1d9ce
... ... @@ -2,8 +2,6 @@ package cn.iocoder.yudao.framework.ai.config;
2 2  
3 3 import cn.iocoder.yudao.framework.ai.core.factory.AiModelFactory;
4 4 import cn.iocoder.yudao.framework.ai.core.factory.AiModelFactoryImpl;
5   -import cn.iocoder.yudao.framework.ai.core.factory.AiVectorStoreFactory;
6   -import cn.iocoder.yudao.framework.ai.core.factory.AiVectorStoreFactoryImpl;
7 5 import cn.iocoder.yudao.framework.ai.core.model.deepseek.DeepSeekChatModel;
8 6 import cn.iocoder.yudao.framework.ai.core.model.deepseek.DeepSeekChatOptions;
9 7 import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi;
... ... @@ -38,11 +36,6 @@ public class YudaoAiAutoConfiguration {
38 36 return new AiModelFactoryImpl();
39 37 }
40 38  
41   - @Bean
42   - public AiVectorStoreFactory aiVectorFactory() {
43   - return new AiVectorStoreFactoryImpl();
44   - }
45   -
46 39  
47 40 // ========== 各种 AI Client 创建 ==========
48 41  
... ... @@ -89,7 +82,7 @@ public class YudaoAiAutoConfiguration {
89 82 // TODO @xin 免费版本
90 83 // @Bean
91 84 // @Lazy // TODO 芋艿：临时注释，避免无法启动」
92   -// public EmbeddingModel transformersEmbeddingClient() {
  85 +// public TransformersEmbeddingModel transformersEmbeddingClient() {
93 86 // return new TransformersEmbeddingModel(MetadataMode.EMBED);
94 87 // }
95 88  
... ... @@ -98,23 +91,24 @@ public class YudaoAiAutoConfiguration {
98 91 */
99 92 // @Bean
100 93 // @Lazy // TODO 芋艿：临时注释，避免无法启动
101   -// public RedisVectorStore vectorStore(TongYiTextEmbeddingModel tongYiTextEmbeddingModel, RedisVectorStoreProperties properties,
  94 +// public RedisVectorStore vectorStore(TransformersEmbeddingModel embeddingModel, RedisVectorStoreProperties properties,
102 95 // RedisProperties redisProperties) {
103 96 // var config = RedisVectorStore.RedisVectorStoreConfig.builder()
104 97 // .withIndexName(properties.getIndex())
105 98 // .withPrefix(properties.getPrefix())
  99 +// .withMetadataFields(new RedisVectorStore.MetadataField("knowledgeId", Schema.FieldType.NUMERIC))
106 100 // .build();
107 101 //
108   -// RedisVectorStore redisVectorStore = new RedisVectorStore(config, tongYiTextEmbeddingModel,
  102 +// RedisVectorStore redisVectorStore = new RedisVectorStore(config, embeddingModel,
109 103 // new JedisPooled(redisProperties.getHost(), redisProperties.getPort()),
110 104 // properties.isInitializeSchema());
111 105 // redisVectorStore.afterPropertiesSet();
112 106 // return redisVectorStore;
113 107 // }
114   -
115 108 @Bean
116 109 @Lazy // TODO 芋艿：临时注释，避免无法启动
117 110 public TokenTextSplitter tokenTextSplitter() {
  111 + //TODO @xin 配置提取
118 112 return new TokenTextSplitter(500, 100, 5, 10000, true);
119 113 }
120 114  
... ...
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactory.java
  View file @0b1d9ce
... ... @@ -6,6 +6,7 @@ import cn.iocoder.yudao.framework.ai.core.model.suno.api.SunoApi;
6 6 import org.springframework.ai.chat.model.ChatModel;
7 7 import org.springframework.ai.embedding.EmbeddingModel;
8 8 import org.springframework.ai.image.ImageModel;
  9 +import org.springframework.ai.vectorstore.VectorStore;
9 10  
10 11 /**
11 12 * AI Model 模型工厂的接口类
... ... @@ -92,4 +93,17 @@ public interface AiModelFactory {
92 93 */
93 94 EmbeddingModel getOrCreateEmbeddingModel(AiPlatformEnum platform, String apiKey, String url);
94 95  
  96 + /**
  97 + * 基于指定配置，获得 VectorStore 对象
  98 + * <p>
  99 + * 如果不存在，则进行创建
  100 + *
  101 + * @param embeddingModel 嵌入模型
  102 + * @param platform 平台
  103 + * @param apiKey API KEY
  104 + * @param url API URL
  105 + * @return VectorStore 对象
  106 + */
  107 + VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url);
  108 +
95 109 }
... ...
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactoryImpl.java
  View file @0b1d9ce
... ... @@ -13,6 +13,7 @@ import cn.iocoder.yudao.framework.ai.core.model.deepseek.DeepSeekChatModel;
13 13 import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi;
14 14 import cn.iocoder.yudao.framework.ai.core.model.suno.api.SunoApi;
15 15 import cn.iocoder.yudao.framework.ai.core.model.xinghuo.XingHuoChatModel;
  16 +import cn.iocoder.yudao.framework.common.util.spring.SpringUtils;
16 17 import com.alibaba.cloud.ai.tongyi.TongYiAutoConfiguration;
17 18 import com.alibaba.cloud.ai.tongyi.TongYiConnectionProperties;
18 19 import com.alibaba.cloud.ai.tongyi.chat.TongYiChatModel;
... ... @@ -54,13 +55,18 @@ import org.springframework.ai.qianfan.api.QianFanApi;
54 55 import org.springframework.ai.qianfan.api.QianFanImageApi;
55 56 import org.springframework.ai.stabilityai.StabilityAiImageModel;
56 57 import org.springframework.ai.stabilityai.api.StabilityAiApi;
  58 +import org.springframework.ai.vectorstore.RedisVectorStore;
  59 +import org.springframework.ai.vectorstore.VectorStore;
57 60 import org.springframework.ai.zhipuai.ZhiPuAiChatModel;
58 61 import org.springframework.ai.zhipuai.ZhiPuAiImageModel;
59 62 import org.springframework.ai.zhipuai.api.ZhiPuAiApi;
60 63 import org.springframework.ai.zhipuai.api.ZhiPuAiImageApi;
  64 +import org.springframework.boot.autoconfigure.data.redis.RedisProperties;
61 65 import org.springframework.retry.support.RetryTemplate;
62 66 import org.springframework.web.client.ResponseErrorHandler;
63 67 import org.springframework.web.client.RestClient;
  68 +import redis.clients.jedis.JedisPooled;
  69 +import redis.clients.jedis.search.Schema;
64 70  
65 71 import java.util.List;
66 72  
... ... @@ -191,6 +197,25 @@ public class AiModelFactoryImpl implements AiModelFactory {
191 197 });
192 198 }
193 199  
  200 + @Override
  201 + public VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url) {
  202 + String cacheKey = buildClientCacheKey(VectorStore.class, platform, apiKey, url);
  203 + return Singleton.get(cacheKey, (Func0<VectorStore>) () -> {
  204 + String prefix = StrUtil.format("{}#{}:", platform.getPlatform(), apiKey);
  205 + var config = RedisVectorStore.RedisVectorStoreConfig.builder()
  206 + .withIndexName(cacheKey)
  207 + .withPrefix(prefix)
  208 + .withMetadataFields(new RedisVectorStore.MetadataField("knowledgeId", Schema.FieldType.NUMERIC))
  209 + .build();
  210 + RedisProperties redisProperties = SpringUtils.getBean(RedisProperties.class);
  211 + RedisVectorStore redisVectorStore = new RedisVectorStore(config, embeddingModel,
  212 + new JedisPooled(redisProperties.getHost(), redisProperties.getPort()),
  213 + true);
  214 + redisVectorStore.afterPropertiesSet();
  215 + return redisVectorStore;
  216 + });
  217 + }
  218 +
194 219 private static String buildClientCacheKey(Class<?> clazz, Object... params) {
195 220 if (ArrayUtil.isEmpty(params)) {
196 221 return clazz.getName();
... ...
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiVectorStoreFactory.java deleted
View file @059f64a
1   -package cn.iocoder.yudao.framework.ai.core.factory;
2   -
3   -import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum;
4   -import org.springframework.ai.embedding.EmbeddingModel;
5   -import org.springframework.ai.vectorstore.VectorStore;
6   -
7   -// TODO @xin：也放到 AiModelFactory 里面好了，后续改成 AiFactory
8   -/**
9   - * AI Vector 模型工厂的接口类
10   - *
11   - * @author xiaoxin
12   - */
13   -public interface AiVectorStoreFactory {
14   -
15   - /**
16   - * 基于指定配置，获得 VectorStore 对象
17   - * <p>
18   - * 如果不存在，则进行创建
19   - *
20   - * @param embeddingModel 嵌入模型
21   - * @param platform 平台
22   - * @param apiKey API KEY
23   - * @param url API URL
24   - * @return VectorStore 对象
25   - */
26   - VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url);
27   -
28   -}
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiVectorStoreFactoryImpl.java deleted
View file @059f64a
1   -package cn.iocoder.yudao.framework.ai.core.factory;
2   -
3   -import cn.hutool.core.lang.Singleton;
4   -import cn.hutool.core.lang.func.Func0;
5   -import cn.hutool.core.util.ArrayUtil;
6   -import cn.hutool.core.util.StrUtil;
7   -import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum;
8   -import cn.iocoder.yudao.framework.common.util.spring.SpringUtils;
9   -import org.springframework.ai.embedding.EmbeddingModel;
10   -import org.springframework.ai.vectorstore.RedisVectorStore;
11   -import org.springframework.ai.vectorstore.VectorStore;
12   -import org.springframework.boot.autoconfigure.data.redis.RedisProperties;
13   -import redis.clients.jedis.JedisPooled;
14   -
15   -/**
16   - * AI Vector 模型工厂的实现类
17   - * 使用 redisVectorStore 实现 VectorStore
18   - *
19   - * @author xiaoxin
20   - */
21   -public class AiVectorStoreFactoryImpl implements AiVectorStoreFactory {
22   -
23   - @Override
24   - public VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url) {
25   - String cacheKey = buildClientCacheKey(VectorStore.class, platform, apiKey, url);
26   - return Singleton.get(cacheKey, (Func0<VectorStore>) () -> {
27   - // TODO 芋艿 @xin 这两个配置取哪好呢
28   - // TODO 不同模型的向量维度可能会不一样，目前看貌似是以 index 来做区分的，维度不一样存不到一个 index 上
29   - // TODO 回复：好的哈
30   - String index = "default-index";
31   - String prefix = "default:";
32   - var config = RedisVectorStore.RedisVectorStoreConfig.builder()
33   - .withIndexName(index)
34   - .withPrefix(prefix)
35   - .build();
36   - RedisProperties redisProperties = SpringUtils.getBean(RedisProperties.class);
37   - RedisVectorStore redisVectorStore = new RedisVectorStore(config, embeddingModel,
38   - new JedisPooled(redisProperties.getHost(), redisProperties.getPort()),
39   - true);
40   - redisVectorStore.afterPropertiesSet();
41   - return redisVectorStore;
42   - });
43   - }
44   -
45   - private static String buildClientCacheKey(Class<?> clazz, Object... params) {
46   - if (ArrayUtil.isEmpty(params)) {
47   - return clazz.getName();
48   - }
49   - return StrUtil.format("{}#{}", clazz.getName(), ArrayUtil.join(params, "_"));
50   - }
51   -
52   -}