BJ_ZTEITS-Developers / parking-cloud-yudao

Commit ed2296e4c772724004d92b0b0dc3b209cd70a78b

Authored by xiaoxin
1 parent 6cceab5b

【解决todo】AI 知识库

Inline Side-by-side
Showing 10 changed files with 39 additions and 105 deletions
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java
  View file @ed2296e
... ... @@ -3,7 +3,7 @@ package cn.iocoder.yudao.module.ai.controller.admin.knowledge;
3 3 import cn.iocoder.yudao.framework.common.pojo.CommonResult;
4 4 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO;
5 5 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO;
6   -import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeBaseService;
  6 +import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeService;
7 7 import io.swagger.v3.oas.annotations.Operation;
8 8 import io.swagger.v3.oas.annotations.tags.Tag;
9 9 import jakarta.annotation.Resource;
... ... @@ -19,7 +19,7 @@ import static cn.iocoder.yudao.framework.security.core.util.SecurityFrameworkUti
19 19 public class AiKnowledgeController {
20 20  
21 21 @Resource
22   - private AiKnowledgeBaseService knowledgeBaseService;
  22 + private AiKnowledgeService knowledgeBaseService;
23 23  
24 24 @PostMapping("/create-my")
25 25 @Operation(summary = "创建【我的】知识库")
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java renamed to yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java
  View file @ed2296e
... ... @@ -10,15 +10,14 @@ import lombok.Data;
10 10  
11 11 import java.util.List;
12 12  
13   -// TODO @xin：要不把 AiKnowledgeBaseDO 改成 AiKnowledgeDO。感觉 base 后缀，感觉有点奇怪（让人以为是基类）。然后，我们很多地方的外键编号，都是 knowledgeId
14 13 /**
15 14 * AI 知识库 DO
16 15 *
17 16 * @author xiaoxin
18 17 */
19   -@TableName(value = "ai_knowledge_base", autoResultMap = true)
  18 +@TableName(value = "ai_knowledge", autoResultMap = true)
20 19 @Data
21   -public class AiKnowledgeBaseDO extends BaseDO {
  20 +public class AiKnowledgeDO extends BaseDO {
22 21  
23 22 /**
24 23 * 编号
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java
  View file @ed2296e
... ... @@ -24,7 +24,7 @@ public class AiKnowledgeDocumentDO extends BaseDO {
24 24 /**
25 25 * 知识库编号
26 26 *
27   - * 关联 {@link AiKnowledgeBaseDO#getId()}
  27 + * 关联 {@link AiKnowledgeDO#getId()}
28 28 */
29 29 private Long knowledgeId;
30 30 /**
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java
  View file @ed2296e
1 1 package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
2 2  
3 3 import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
4   -import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO;
  4 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
5 5 import org.apache.ibatis.annotations.Mapper;
6 6  
7 7 /**
... ... @@ -10,5 +10,5 @@ import org.apache.ibatis.annotations.Mapper;
10 10 * @author xiaoxin
11 11 */
12 12 @Mapper
13   -public interface AiKnowledgeBaseMapper extends BaseMapperX<AiKnowledgeBaseDO> {
  13 +public interface AiKnowledgeBaseMapper extends BaseMapperX<AiKnowledgeDO> {
14 14 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java deleted
View file @6cceab5
1   -package cn.iocoder.yudao.module.ai.service.knowledge;
2   -
3   -import org.springframework.ai.document.Document;
4   -import org.springframework.ai.vectorstore.SearchRequest;
5   -
6   -import java.util.List;
7   -
8   -/**
9   - * AI 嵌入 Service 接口
10   - *
11   - * @author xiaoxin
12   - */
13   -public interface AiEmbeddingService {
14   -
15   - /**
16   - * 向量化文档并存储
17   - */
18   - void add(List<Document> documents);
19   -
20   - /**
21   - * 相似查询
22   - *
23   - * @param request 查询实体
24   - */
25   - List<Document> similaritySearch(SearchRequest request);
26   -
27   -}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java deleted
View file @6cceab5
1   -package cn.iocoder.yudao.module.ai.service.knowledge;
2   -
3   -import jakarta.annotation.Resource;
4   -import org.springframework.ai.document.Document;
5   -import org.springframework.ai.vectorstore.RedisVectorStore;
6   -import org.springframework.ai.vectorstore.SearchRequest;
7   -import org.springframework.stereotype.Service;
8   -
9   -import java.util.List;
10   -
11   -// TODO @xin：是不是不用 AiEmbeddingServiceImpl，直接 vectorStore 注入到需要的地方就好啦。通过 KnowledgeDocumentService 返回就好。
12   -/**
13   - * AI 嵌入 Service 实现类
14   - *
15   - * @author xiaoxin
16   - */
17   -@Service
18   -public class AiEmbeddingServiceImpl implements AiEmbeddingService {
19   -
20   - @Resource
21   - private RedisVectorStore vectorStore;
22   -
23   - @Override
24   -// @Async
25   - // TODO xiaoxin 报错先注释
26   - public void add(List<Document> documents) {
27   - vectorStore.add(documents);
28   - }
29   -
30   - @Override
31   - public List<Document> similaritySearch(SearchRequest request) {
32   - return vectorStore.similaritySearch(request);
33   - }
34   -
35   -}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java
  View file @ed2296e
... ... @@ -14,8 +14,9 @@ import jakarta.annotation.Resource;
14 14 import lombok.extern.slf4j.Slf4j;
15 15 import org.springframework.ai.document.Document;
16 16 import org.springframework.ai.reader.tika.TikaDocumentReader;
17   -import org.springframework.ai.tokenizer.JTokkitTokenCountEstimator;
  17 +import org.springframework.ai.tokenizer.TokenCountEstimator;
18 18 import org.springframework.ai.transformer.splitter.TokenTextSplitter;
  19 +import org.springframework.ai.vectorstore.RedisVectorStore;
19 20 import org.springframework.beans.factory.annotation.Value;
20 21 import org.springframework.stereotype.Service;
21 22 import org.springframework.transaction.annotation.Transactional;
... ... @@ -39,52 +40,49 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
39 40  
40 41 @Resource
41 42 private TokenTextSplitter tokenTextSplitter;
42   -
43 43 @Resource
44   - private AiEmbeddingService embeddingService;
  44 + private TokenCountEstimator TOKEN_COUNT_ESTIMATOR;
  45 + @Resource
  46 + private RedisVectorStore vectorStore;
45 47  
46   - // TODO @xin：@Resource 注入
47   - private static final JTokkitTokenCountEstimator TOKEN_COUNT_ESTIMATOR = new JTokkitTokenCountEstimator();
48 48  
49 49 // TODO xiaoxin 临时测试用，后续删
50 50 @Value("classpath:/webapp/test/Fel.pdf")
51 51 private org.springframework.core.io.Resource data;
52 52  
53 53 // TODO 芋艿：需要 review 下，代码格式；
54   - // TODO @xin：最好有 1、/2、/3 这种，让代码更有层次感
55 54 @Override
56 55 @Transactional(rollbackFor = Exception.class)
57 56 public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) {
58 57 // TODO xiaoxin 后续从 url 加载
59 58 TikaDocumentReader loader = new TikaDocumentReader(data);
60   - // 加载文档
  59 + // 1.1 加载文档
61 60 List<Document> documents = loader.get();
62 61 Document document = CollUtil.getFirst(documents);
63 62 // TODO @xin：是不是不存在，就抛出异常呀；厚泽 return 呀；
64   - // TODO 芋艿 文档层面有没有可能会比较大，这两个字段是否可以从分段表计算得出？回复：先直接算；
65 63 Integer tokens = Objects.nonNull(document) ? TOKEN_COUNT_ESTIMATOR.estimate(document.getContent()) : 0;
66 64 Integer wordCount = Objects.nonNull(document) ? document.getContent().length() : 0;
67 65  
68 66 AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class)
69 67 .setTokens(tokens).setWordCount(wordCount)
70 68 .setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus());
71   - // 文档记录入库
  69 + // 1.2 文档记录入库
72 70 documentMapper.insert(documentDO);
73 71 Long documentId = documentDO.getId();
74 72 if (CollUtil.isEmpty(documents)) {
75 73 return documentId;
76 74 }
77 75  
78   - // 文档分段
  76 + // 2.1 文档分段
79 77 List<Document> segments = tokenTextSplitter.apply(documents);
80   - // 分段内容入库
  78 + // 2.2 分段内容入库
81 79 List<AiKnowledgeSegmentDO> segmentDOList = CollectionUtils.convertList(segments,
82 80 segment -> new AiKnowledgeSegmentDO().setContent(segment.getContent()).setDocumentId(documentId)
83 81 .setTokens(TOKEN_COUNT_ESTIMATOR.estimate(segment.getContent())).setWordCount(segment.getContent().length())
84 82 .setStatus(CommonStatusEnum.ENABLE.getStatus()));
85 83 segmentMapper.insertBatch(segmentDOList);
86   - // 向量化并存储
87   - embeddingService.add(segments);
  84 + // 3 向量化并存储
  85 + vectorStore.add(segments);
88 86 return documentId;
89 87 }
90 88  
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java renamed to yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java
  View file @ed2296e
... ... @@ -7,7 +7,7 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdat
7 7 *
8 8 * @author xiaoxin
9 9 */
10   -public interface AiKnowledgeBaseService {
  10 +public interface AiKnowledgeService {
11 11  
12 12 /**
13 13 * 创建【我的】知识库
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java renamed to yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java
  View file @ed2296e
1 1 package cn.iocoder.yudao.module.ai.service.knowledge;
2 2  
3   -import cn.hutool.core.lang.Assert;
4 3 import cn.hutool.core.util.ObjUtil;
5 4 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
6 5 import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
7 6 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO;
8 7 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO;
9   -import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO;
  8 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
10 9 import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
11 10 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeBaseMapper;
12 11 import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;
... ... @@ -24,7 +23,7 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_
24 23 */
25 24 @Service
26 25 @Slf4j
27   -public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService {
  26 +public class AiKnowledgeServiceImpl implements AiKnowledgeService {
28 27  
29 28 @Resource
30 29 private AiChatModelService chatModalService;
... ... @@ -34,42 +33,34 @@ public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService {
34 33  
35 34 @Override
36 35 public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) {
37   - // TODO @xin：貌似直接调用 chatModalService.validateChatModel(id) 完事，不用搞个方法
38 36 // 1. 校验模型配置
39   - AiChatModelDO model = validateChatModel(createReqVO.getModelId());
  37 + AiChatModelDO model = chatModalService.validateChatModel(createReqVO.getModelId());
40 38  
41 39 // 2. 插入知识库
42   - // TODO @xin：不用 DO 结尾
43   - AiKnowledgeBaseDO knowledgeBaseDO = BeanUtils.toBean(createReqVO, AiKnowledgeBaseDO.class)
  40 + AiKnowledgeDO knowledgeBase = BeanUtils.toBean(createReqVO, AiKnowledgeDO.class)
44 41 .setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus());
45   - knowledgeBaseMapper.insert(knowledgeBaseDO);
46   - return knowledgeBaseDO.getId();
  42 + knowledgeBaseMapper.insert(knowledgeBase);
  43 + return knowledgeBase.getId();
47 44 }
48 45  
49 46 @Override
50 47 public void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId) {
51 48 // 1.1 校验知识库存在
52   - AiKnowledgeBaseDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId());
  49 + AiKnowledgeDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId());
53 50 if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) {
54 51 throw exception(KNOWLEDGE_NOT_EXISTS);
55 52 }
56 53 // 1.2 校验模型配置
57   - AiChatModelDO model = validateChatModel(updateReqVO.getModelId());
  54 + AiChatModelDO model = chatModalService.validateChatModel(updateReqVO.getModelId());
58 55  
59 56 // 2. 更新知识库
60   - AiKnowledgeBaseDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeBaseDO.class);
  57 + AiKnowledgeDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeDO.class);
61 58 updateDO.setModel(model.getModel());
62 59 knowledgeBaseMapper.updateById(updateDO);
63 60 }
64 61  
65   - private AiChatModelDO validateChatModel(Long id) {
66   - AiChatModelDO model = chatModalService.validateChatModel(id);
67   - Assert.notNull(model, "未找到对应嵌入模型");
68   - return model;
69   - }
70   -
71   - public AiKnowledgeBaseDO validateKnowledgeExists(Long id) {
72   - AiKnowledgeBaseDO knowledgeBase = knowledgeBaseMapper.selectById(id);
  62 + public AiKnowledgeDO validateKnowledgeExists(Long id) {
  63 + AiKnowledgeDO knowledgeBase = knowledgeBaseMapper.selectById(id);
73 64 if (knowledgeBase == null) {
74 65 throw exception(KNOWLEDGE_NOT_EXISTS);
75 66 }
... ...
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java
  View file @ed2296e
... ... @@ -13,6 +13,8 @@ import lombok.extern.slf4j.Slf4j;
13 13 import org.springframework.ai.autoconfigure.vectorstore.redis.RedisVectorStoreProperties;
14 14 import org.springframework.ai.document.MetadataMode;
15 15 import org.springframework.ai.embedding.EmbeddingModel;
  16 +import org.springframework.ai.tokenizer.JTokkitTokenCountEstimator;
  17 +import org.springframework.ai.tokenizer.TokenCountEstimator;
16 18 import org.springframework.ai.transformer.splitter.TokenTextSplitter;
17 19 import org.springframework.ai.transformers.TransformersEmbeddingModel;
18 20 import org.springframework.ai.vectorstore.RedisVectorStore;
... ... @@ -90,7 +92,7 @@ public class YudaoAiAutoConfiguration {
90 92 }
91 93  
92 94 /**
93   - * 我们启动有加载很多 Embedding 模型，不晓得取哪个好，先 new 个 TransformersEmbeddingModel 跑
  95 + * TODO @xin 抽离出去，根据具体模型走
94 96 */
95 97 @Bean
96 98 @Lazy // TODO 芋艿：临时注释，避免无法启动
... ... @@ -114,4 +116,10 @@ public class YudaoAiAutoConfiguration {
114 116 return new TokenTextSplitter(500, 100, 5, 10000, true);
115 117 }
116 118  
  119 + @Bean
  120 + @Lazy // TODO 芋艿：临时注释，避免无法启动
  121 + public TokenCountEstimator tokenCountEstimator() {
  122 + return new JTokkitTokenCountEstimator();
  123 + }
  124 +
117 125 }
118 126 \ No newline at end of file
... ...