BJ_ZTEITS-Developers / parking-cloud-yudao

Commit 6cceab5ba4ed23c5d29ee9992449e8539156fc38

Authored by YunaiV
1 parent 0db165f6

【代码评审】AI 大模型：知识库的逻辑

Inline Side-by-side
Showing 18 changed files with 53 additions and 52 deletions
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java
  View file @6cceab5
... ... @@ -7,9 +7,6 @@ import lombok.Data;
7 7  
8 8 import java.util.List;
9 9  
10   -/**
11   - * @author xiaoxin
12   - */
13 10 @Schema(description = "管理后台 - AI 知识库创建【我的】 Request VO")
14 11 @Data
15 12 public class AiKnowledgeCreateMyReqVO {
... ... @@ -21,10 +18,10 @@ public class AiKnowledgeCreateMyReqVO {
21 18 @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "存储 ruoyi-vue-pro 操作文档")
22 19 private String description;
23 20  
24   - @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]")
  21 + @Schema(description = "可见权限，只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]")
25 22 private List<Long> visibilityPermissions;
26 23  
27   - @Schema(description = "嵌入模型 id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
  24 + @Schema(description = "嵌入模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
28 25 @NotNull(message = "嵌入模型不能为空")
29 26 private Long modelId;
30 27  
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeDocumentCreateReqVO.java
  View file @6cceab5
... ... @@ -4,6 +4,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
4 4 import jakarta.validation.constraints.NotBlank;
5 5 import jakarta.validation.constraints.NotNull;
6 6 import lombok.Data;
  7 +import org.hibernate.validator.constraints.URL;
7 8  
8 9 /**
9 10 * @author xiaoxin
... ... @@ -12,7 +13,6 @@ import lombok.Data;
12 13 @Data
13 14 public class AiKnowledgeDocumentCreateReqVO {
14 15  
15   -
16 16 @Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204")
17 17 @NotNull(message = "知识库编号不能为空")
18 18 private Long knowledgeId;
... ... @@ -21,7 +21,8 @@ public class AiKnowledgeDocumentCreateReqVO {
21 21 @NotBlank(message = "文档名称不能为空")
22 22 private String name;
23 23  
24   - @Schema(description = "文档 url", requiredMode = Schema.RequiredMode.REQUIRED, example = "https://doc.iocoder.cn")
  24 + @Schema(description = "文档 URL", requiredMode = Schema.RequiredMode.REQUIRED, example = "https://doc.iocoder.cn")
  25 + @URL(message = "文档 URL 格式不正确")
25 26 private String url;
26 27  
27 28 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeUpdateMyReqVO.java
  View file @6cceab5
... ... @@ -7,14 +7,10 @@ import lombok.Data;
7 7  
8 8 import java.util.List;
9 9  
10   -/**
11   - * @author xiaoxin
12   - */
13 10 @Schema(description = "管理后台 - AI 知识库创建【我的】 Request VO")
14 11 @Data
15 12 public class AiKnowledgeUpdateMyReqVO {
16 13  
17   -
18 14 @Schema(description = "对话编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204")
19 15 @NotNull(message = "知识库编号不能为空")
20 16 private Long id;
... ... @@ -26,10 +22,10 @@ public class AiKnowledgeUpdateMyReqVO {
26 22 @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "")
27 23 private String description;
28 24  
29   - @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]")
  25 + @Schema(description = "可见权限，只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]")
30 26 private List<Long> visibilityPermissions;
31 27  
32   - @Schema(description = "嵌入模型 id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
  28 + @Schema(description = "嵌入模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
33 29 @NotNull(message = "嵌入模型不能为空")
34 30 private Long modelId;
35 31  
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/image/AiImageDO.java
  View file @6cceab5
... ... @@ -32,7 +32,7 @@ public class AiImageDO extends BaseDO {
32 32 /**
33 33 * 编号
34 34 */
35   - @TableId(type = IdType.AUTO)
  35 + @TableId
36 36 private Long id;
37 37  
38 38 /**
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java
  View file @6cceab5
1 1 package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
2 2  
3   -
4 3 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
5 4 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
6   -import com.baomidou.mybatisplus.annotation.IdType;
7 5 import com.baomidou.mybatisplus.annotation.TableField;
8 6 import com.baomidou.mybatisplus.annotation.TableId;
9 7 import com.baomidou.mybatisplus.annotation.TableName;
... ... @@ -12,19 +10,20 @@ import lombok.Data;
12 10  
13 11 import java.util.List;
14 12  
  13 +// TODO @xin：要不把 AiKnowledgeBaseDO 改成 AiKnowledgeDO。感觉 base 后缀，感觉有点奇怪（让人以为是基类）。然后，我们很多地方的外键编号，都是 knowledgeId
15 14 /**
16 15 * AI 知识库 DO
17 16 *
18 17 * @author xiaoxin
19 18 */
20   -@TableName(value = "ai_knowledge_base")
  19 +@TableName(value = "ai_knowledge_base", autoResultMap = true)
21 20 @Data
22 21 public class AiKnowledgeBaseDO extends BaseDO {
23 22  
24 23 /**
25 24 * 编号
26 25 */
27   - @TableId(type = IdType.AUTO)
  26 + @TableId
28 27 private Long id;
29 28 /**
30 29 * 用户编号
... ... @@ -40,6 +39,7 @@ public class AiKnowledgeBaseDO extends BaseDO {
40 39 * 知识库描述
41 40 */
42 41 private String description;
  42 + // TODO @新：如果全部可见，需要怎么设置？
43 43 /**
44 44 * 可见权限,只能选择哪些人可见
45 45 */
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java
  View file @6cceab5
... ... @@ -3,7 +3,6 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
3 3 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
4 4 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
5 5 import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum;
6   -import com.baomidou.mybatisplus.annotation.IdType;
7 6 import com.baomidou.mybatisplus.annotation.TableId;
8 7 import com.baomidou.mybatisplus.annotation.TableName;
9 8 import lombok.Data;
... ... @@ -20,10 +19,12 @@ public class AiKnowledgeDocumentDO extends BaseDO {
20 19 /**
21 20 * 编号
22 21 */
23   - @TableId(type = IdType.AUTO)
  22 + @TableId
24 23 private Long id;
25 24 /**
26 25 * 知识库编号
  26 + *
  27 + * 关联 {@link AiKnowledgeBaseDO#getId()}
27 28 */
28 29 private Long knowledgeId;
29 30 /**
... ... @@ -39,7 +40,7 @@ public class AiKnowledgeDocumentDO extends BaseDO {
39 40 */
40 41 private String url;
41 42 /**
42   - * token数量
  43 + * token 数量
43 44 */
44 45 private Integer tokens;
45 46 /**
... ... @@ -59,4 +60,5 @@ public class AiKnowledgeDocumentDO extends BaseDO {
59 60 * 枚举 {@link CommonStatusEnum}
60 61 */
61 62 private Integer status;
  63 +
62 64 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java
  View file @6cceab5
... ... @@ -2,7 +2,6 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
2 2  
3 3 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
4 4 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
5   -import com.baomidou.mybatisplus.annotation.IdType;
6 5 import com.baomidou.mybatisplus.annotation.TableId;
7 6 import com.baomidou.mybatisplus.annotation.TableName;
8 7 import lombok.Data;
... ... @@ -19,14 +18,17 @@ public class AiKnowledgeSegmentDO extends BaseDO {
19 18 /**
20 19 * 编号
21 20 */
22   - @TableId(type = IdType.AUTO)
  21 + @TableId
23 22 private Long id;
24 23 /**
25   - * 向量库的id
  24 + * 向量库的编号
26 25 */
27 26 private String vectorId;
  27 + // TODO @新：knowledgeId 加个，会方便点
28 28 /**
29 29 * 文档编号
  30 + *
  31 + * 关联 {@link AiKnowledgeDocumentDO#getId()}
30 32 */
31 33 private Long documentId;
32 34 /**
... ... @@ -38,7 +40,7 @@ public class AiKnowledgeSegmentDO extends BaseDO {
38 40 */
39 41 private Integer wordCount;
40 42 /**
41   - * token数量
  43 + * token 数量
42 44 */
43 45 private Integer tokens;
44 46 /**
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/mindmap/AiMindMapDO.java
  View file @6cceab5
... ... @@ -19,7 +19,7 @@ public class AiMindMapDO extends BaseDO {
19 19 /**
20 20 * 编号
21 21 */
22   - @TableId(type = IdType.AUTO)
  22 + @TableId
23 23 private Long id;
24 24  
25 25 /**
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/music/AiMusicDO.java
  View file @6cceab5
... ... @@ -25,7 +25,7 @@ public class AiMusicDO extends BaseDO {
25 25 /**
26 26 * 编号
27 27 */
28   - @TableId(type = IdType.AUTO)
  28 + @TableId
29 29 private Long id;
30 30  
31 31 /**
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/write/AiWriteDO.java
  View file @6cceab5
... ... @@ -20,7 +20,7 @@ public class AiWriteDO extends BaseDO {
20 20 /**
21 21 * 编号
22 22 */
23   - @TableId(type = IdType.AUTO)
  23 + @TableId
24 24 private Long id;
25 25  
26 26 /**
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java
  View file @6cceab5
... ... @@ -17,11 +17,11 @@ public interface AiEmbeddingService {
17 17 */
18 18 void add(List<Document> documents);
19 19  
20   -
21 20 /**
22 21 * 相似查询
23 22 *
24 23 * @param request 查询实体
25 24 */
26 25 List<Document> similaritySearch(SearchRequest request);
  26 +
27 27 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java
  View file @6cceab5
... ... @@ -8,6 +8,7 @@ import org.springframework.stereotype.Service;
8 8  
9 9 import java.util.List;
10 10  
  11 +// TODO @xin：是不是不用 AiEmbeddingServiceImpl，直接 vectorStore 注入到需要的地方就好啦。通过 KnowledgeDocumentService 返回就好。
11 12 /**
12 13 * AI 嵌入 Service 实现类
13 14 *
... ... @@ -30,4 +31,5 @@ public class AiEmbeddingServiceImpl implements AiEmbeddingService {
30 31 public List<Document> similaritySearch(SearchRequest request) {
31 32 return vectorStore.similaritySearch(request);
32 33 }
  34 +
33 35 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java
  View file @6cceab5
... ... @@ -9,7 +9,6 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdat
9 9 */
10 10 public interface AiKnowledgeBaseService {
11 11  
12   -
13 12 /**
14 13 * 创建【我的】知识库
15 14 *
... ... @@ -27,4 +26,5 @@ public interface AiKnowledgeBaseService {
27 26 * @param userId 用户编号
28 27 */
29 28 void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId);
  29 +
30 30 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java
  View file @6cceab5
... ... @@ -32,33 +32,36 @@ public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService {
32 32 @Resource
33 33 private AiKnowledgeBaseMapper knowledgeBaseMapper;
34 34  
35   -
36 35 @Override
37 36 public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) {
  37 + // TODO @xin：貌似直接调用 chatModalService.validateChatModel(id) 完事，不用搞个方法
  38 + // 1. 校验模型配置
38 39 AiChatModelDO model = validateChatModel(createReqVO.getModelId());
39 40  
40   - AiKnowledgeBaseDO knowledgeBaseDO = BeanUtils.toBean(createReqVO, AiKnowledgeBaseDO.class);
41   - knowledgeBaseDO.setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus());
42   -
  41 + // 2. 插入知识库
  42 + // TODO @xin：不用 DO 结尾
  43 + AiKnowledgeBaseDO knowledgeBaseDO = BeanUtils.toBean(createReqVO, AiKnowledgeBaseDO.class)
  44 + .setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus());
43 45 knowledgeBaseMapper.insert(knowledgeBaseDO);
44 46 return knowledgeBaseDO.getId();
45 47 }
46 48  
47 49 @Override
48 50 public void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId) {
49   -
  51 + // 1.1 校验知识库存在
50 52 AiKnowledgeBaseDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId());
51 53 if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) {
52 54 throw exception(KNOWLEDGE_NOT_EXISTS);
53 55 }
  56 + // 1.2 校验模型配置
54 57 AiChatModelDO model = validateChatModel(updateReqVO.getModelId());
  58 +
  59 + // 2. 更新知识库
55 60 AiKnowledgeBaseDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeBaseDO.class);
56 61 updateDO.setModel(model.getModel());
57   -
58 62 knowledgeBaseMapper.updateById(updateDO);
59 63 }
60 64  
61   -
62 65 private AiChatModelDO validateChatModel(Long id) {
63 66 AiChatModelDO model = chatModalService.validateChatModel(id);
64 67 Assert.notNull(model, "未找到对应嵌入模型");
... ... @@ -72,4 +75,5 @@ public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService {
72 75 }
73 76 return knowledgeBase;
74 77 }
  78 +
75 79 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java
  View file @6cceab5
... ... @@ -9,7 +9,6 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeDocum
9 9 */
10 10 public interface AiKnowledgeDocumentService {
11 11  
12   -
13 12 /**
14 13 * 创建文档
15 14 *
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java
  View file @6cceab5
... ... @@ -43,28 +43,30 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
43 43 @Resource
44 44 private AiEmbeddingService embeddingService;
45 45  
  46 + // TODO @xin：@Resource 注入
46 47 private static final JTokkitTokenCountEstimator TOKEN_COUNT_ESTIMATOR = new JTokkitTokenCountEstimator();
47 48  
48 49 // TODO xiaoxin 临时测试用，后续删
49 50 @Value("classpath:/webapp/test/Fel.pdf")
50 51 private org.springframework.core.io.Resource data;
51 52  
52   -
  53 + // TODO 芋艿：需要 review 下，代码格式；
  54 + // TODO @xin：最好有 1、/2、/3 这种，让代码更有层次感
53 55 @Override
54 56 @Transactional(rollbackFor = Exception.class)
55 57 public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) {
56   -
57 58 // TODO xiaoxin 后续从 url 加载
58 59 TikaDocumentReader loader = new TikaDocumentReader(data);
59 60 // 加载文档
60 61 List<Document> documents = loader.get();
61 62 Document document = CollUtil.getFirst(documents);
62   - // TODO 芋艿 文档层面有没有可能会比较大，这两个字段是否可以从分段表计算得出？
  63 + // TODO @xin：是不是不存在，就抛出异常呀；厚泽 return 呀；
  64 + // TODO 芋艿 文档层面有没有可能会比较大，这两个字段是否可以从分段表计算得出？回复：先直接算；
63 65 Integer tokens = Objects.nonNull(document) ? TOKEN_COUNT_ESTIMATOR.estimate(document.getContent()) : 0;
64 66 Integer wordCount = Objects.nonNull(document) ? document.getContent().length() : 0;
65 67  
66   - AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class);
67   - documentDO.setTokens(tokens).setWordCount(wordCount)
  68 + AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class)
  69 + .setTokens(tokens).setWordCount(wordCount)
68 70 .setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus());
69 71 // 文档记录入库
70 72 documentMapper.insert(documentDO);
... ... @@ -75,17 +77,15 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
75 77  
76 78 // 文档分段
77 79 List<Document> segments = tokenTextSplitter.apply(documents);
78   -
  80 + // 分段内容入库
79 81 List<AiKnowledgeSegmentDO> segmentDOList = CollectionUtils.convertList(segments,
80 82 segment -> new AiKnowledgeSegmentDO().setContent(segment.getContent()).setDocumentId(documentId)
81 83 .setTokens(TOKEN_COUNT_ESTIMATOR.estimate(segment.getContent())).setWordCount(segment.getContent().length())
82 84 .setStatus(CommonStatusEnum.ENABLE.getStatus()));
83   - // 分段内容入库
84 85 segmentMapper.insertBatch(segmentDOList);
85   -
86   - //向量化并存储
  86 + // 向量化并存储
87 87 embeddingService.add(segments);
88   -
89 88 return documentId;
90 89 }
  90 +
91 91 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java
  View file @6cceab5
1 1 package cn.iocoder.yudao.module.ai.service.knowledge;
2 2  
3 3 /**
4   - * AI 知识库-分片 Service 接口
  4 + * AI 知识库分片 Service 接口
5 5 *
6 6 * @author xiaoxin
7 7 */
8 8 public interface AiKnowledgeSegmentService {
9 9  
10   -
11 10 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
  View file @6cceab5
... ... @@ -4,7 +4,7 @@ import lombok.extern.slf4j.Slf4j;
4 4 import org.springframework.stereotype.Service;
5 5  
6 6 /**
7   - * AI 知识库-基础信息 Service 实现类
  7 + * AI 知识库分片 Service 实现类
8 8 *
9 9 * @author xiaoxin
10 10 */
... ... @@ -12,5 +12,4 @@ import org.springframework.stereotype.Service;
12 12 @Slf4j
13 13 public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService {
14 14  
15   -
16 15 }
... ...