BJ_ZTEITS-Developers / parking-cloud-yudao

Commit 5cd870748df348ef43b99b19de358d10aaac3b3c

Authored by xiaoxin
1 parent 8e56b81a

【解决todo】AI 知识库: 字段命名统一 补充注释

Inline Side-by-side
Showing 11 changed files with 38 additions and 41 deletions
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.java
  View file @5cd8707
... ... @@ -29,7 +29,7 @@ public class AiKnowledgeSegmentController {
29 29  
30 30 @GetMapping("/page")
31 31 @Operation(summary = "获取段落分页")
32   - public CommonResult<PageResult<AiKnowledgeSegmentRespVO>> getKnowledgeSegmentPageMy(@Valid AiKnowledgeSegmentPageReqVO pageReqVO) {
  32 + public CommonResult<PageResult<AiKnowledgeSegmentRespVO>> getKnowledgeSegmentPage(@Valid AiKnowledgeSegmentPageReqVO pageReqVO) {
33 33 PageResult<AiKnowledgeSegmentDO> pageResult = segmentService.getKnowledgeSegmentPage(pageReqVO);
34 34 return success(BeanUtils.toBean(pageResult, AiKnowledgeSegmentRespVO.class));
35 35 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeDocumentCreateReqVO.java
  View file @5cd8707
... ... @@ -23,21 +23,21 @@ public class AiKnowledgeDocumentCreateReqVO {
23 23 @URL(message = "文档 URL 格式不正确")
24 24 private String url;
25 25  
26   - @Schema(description = "每个文本块的目标 token 数", requiredMode = Schema.RequiredMode.REQUIRED, example = "800")
27   - @NotNull(message = "每个文本块的目标 token 数不能为空")
28   - private Integer defaultChunkSize;
  26 + @Schema(description = "每个段落的目标 token 数", requiredMode = Schema.RequiredMode.REQUIRED, example = "800")
  27 + @NotNull(message = "每个段落的目标 token 数不能为空")
  28 + private Integer defaultSegmentTokens;
29 29  
30   - @Schema(description = "每个文本块的最小字符数", requiredMode = Schema.RequiredMode.REQUIRED, example = "350")
31   - @NotNull(message = "每个文本块的最小字符数不能为空")
32   - private Integer minChunkSizeChars;
  30 + @Schema(description = "每个段落的最小字符数", requiredMode = Schema.RequiredMode.REQUIRED, example = "350")
  31 + @NotNull(message = "每个段落的最小字符数不能为空")
  32 + private Integer minSegmentWordCount;
33 33  
34   - @Schema(description = "丢弃阈值", requiredMode = Schema.RequiredMode.REQUIRED, example = "5")
  34 + @Schema(description = "丢弃阈值：低于此阈值的段落会被丢弃", requiredMode = Schema.RequiredMode.REQUIRED, example = "5")
35 35 @NotNull(message = "丢弃阈值不能为空")
36 36 private Integer minChunkLengthToEmbed;
37 37  
38   - @Schema(description = "最大块数", requiredMode = Schema.RequiredMode.REQUIRED, example = "10000")
39   - @NotNull(message = "最大块数不能为空")
40   - private Integer maxNumChunks;
  38 + @Schema(description = "最大段落数", requiredMode = Schema.RequiredMode.REQUIRED, example = "10000")
  39 + @NotNull(message = "最大段落数不能为空")
  40 + private Integer maxNumSegments;
41 41  
42 42 @Schema(description = "分块是否保留分隔符", requiredMode = Schema.RequiredMode.REQUIRED, example = "true")
43 43 @NotNull(message = "分块是否保留分隔符不能为空")
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java
  View file @5cd8707
... ... @@ -38,9 +38,11 @@ public class AiKnowledgeDO extends BaseDO {
38 38 * 知识库描述
39 39 */
40 40 private String description;
41   - // TODO @新：如果全部可见，需要怎么设置？
  41 +
42 42 /**
43   - * 可见权限,只能选择哪些人可见
  43 + * 可见权限,选择哪些人可见
  44 + * <p>
  45 + * -1 所有人可见，其他为各自用户编号
44 46 */
45 47 @TableField(typeHandler = JacksonTypeHandler.class)
46 48 private List<Long> visibilityPermissions;
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java
  View file @5cd8707
... ... @@ -40,23 +40,25 @@ public class AiKnowledgeDocumentDO extends BaseDO {
40 40 */
41 41 private String url;
42 42 /**
43   - * token 数量
  43 + * 文档 token 数量
44 44 */
45 45 private Integer tokens;
46 46 /**
47   - * 字符数
  47 + * 文档字符数
48 48 */
49 49 private Integer wordCount;
50   - // TODO @新：chunk 1）是不是 segment，这样命名保持一致会好点哈？2）Size 是不是改成 Tokens 会统一点；3）defaultChunkSize、defaultChunkSize、minChunkSizeChars、maxNumChunks 这几个字段的命名，可能要微信一起讨论下。尽量命名保持风格统一哈。
  50 +
  51 +
  52 + // ========== 自定义分段所用参数 ==========
  53 + // TODO @新：3）defaultChunkSize、defaultChunkSize、minChunkSizeChars、maxNumChunks 这几个字段的命名，可能要微信一起讨论下。尽量命名保持风格统一哈。
51 54 /**
52 55 * 每个文本块的目标 token 数
53 56 */
54   - private Integer defaultChunkSize;
55   - // TODO @xin：SizeChars 和 wordCount 好像是一个意思，是不是也要统一哈。
  57 + private Integer defaultSegmentTokens;
56 58 /**
57 59 * 每个文本块的最小字符数
58 60 */
59   - private Integer minChunkSizeChars;
  61 + private Integer minSegmentWordCount;
60 62 /**
61 63 * 低于此值的块会被丢弃
62 64 */
... ... @@ -64,11 +66,13 @@ public class AiKnowledgeDocumentDO extends BaseDO {
64 66 /**
65 67 * 最大块数
66 68 */
67   - private Integer maxNumChunks;
  69 + private Integer maxNumSegments;
68 70 /**
69 71 * 分块是否保留分隔符
70 72 */
71 73 private Boolean keepSeparator;
  74 + // ===================================
  75 +
72 76 /**
73 77 * 切片状态
74 78 * <p>
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java
  View file @5cd8707
... ... @@ -2,8 +2,6 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
2 2  
3 3 import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
4 4 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
5   -import com.baomidou.mybatisplus.annotation.FieldStrategy;
6   -import com.baomidou.mybatisplus.annotation.TableField;
7 5 import com.baomidou.mybatisplus.annotation.TableId;
8 6 import com.baomidou.mybatisplus.annotation.TableName;
9 7 import lombok.Data;
... ... @@ -27,7 +25,6 @@ public class AiKnowledgeSegmentDO extends BaseDO {
27 25 /**
28 26 * 向量库的编号
29 27 */
30   - @TableField(updateStrategy = FieldStrategy.ALWAYS) // TODO @新：尽量规避要这个注解。万一后面加个 status 单独更新，可能会踩坑。
31 28 private String vectorId;
32 29 /**
33 30 * 知识库编号
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java
  View file @5cd8707
... ... @@ -25,8 +25,7 @@ public interface AiKnowledgeSegmentMapper extends BaseMapperX&lt;AiKnowledgeSegment
25 25 .orderByDesc(AiKnowledgeSegmentDO::getId));
26 26 }
27 27  
28   - // TODO @新：selectListByXXX 哈
29   - default List<AiKnowledgeSegmentDO> selectList(List<String> vectorIdList) {
  28 + default List<AiKnowledgeSegmentDO> selectListByVectorIds(List<String> vectorIdList) {
30 29 return selectList(new LambdaQueryWrapperX<AiKnowledgeSegmentDO>()
31 30 .in(AiKnowledgeSegmentDO::getVectorId, vectorIdList)
32 31 .orderByDesc(AiKnowledgeSegmentDO::getId));
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java
  View file @5cd8707
... ... @@ -71,8 +71,8 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
71 71 }
72 72  
73 73 // 2 构造文本分段器
74   - TokenTextSplitter tokenTextSplitter = new TokenTextSplitter(createReqVO.getDefaultChunkSize(), createReqVO.getMinChunkSizeChars(), createReqVO.getMinChunkLengthToEmbed(),
75   - createReqVO.getMaxNumChunks(), createReqVO.getKeepSeparator());
  74 + TokenTextSplitter tokenTextSplitter = new TokenTextSplitter(createReqVO.getDefaultSegmentTokens(), createReqVO.getMinSegmentWordCount(), createReqVO.getMinChunkLengthToEmbed(),
  75 + createReqVO.getMaxNumSegments(), createReqVO.getKeepSeparator());
76 76 // 2.1 文档分段
77 77 List<Document> segments = tokenTextSplitter.apply(documents);
78 78 // 2.2 分段内容入库
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
  View file @5cd8707
... ... @@ -90,7 +90,7 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
90 90 } else {
91 91 // 2.2 禁用删除向量
92 92 vectorStore.delete(List.of(oldKnowledgeSegment.getVectorId()));
93   - knowledgeSegment.setVectorId(null);
  93 + knowledgeSegment.setVectorId("");
94 94 }
95 95 // 3 更新段落状态
96 96 segmentMapper.updateById(knowledgeSegment);
... ... @@ -114,7 +114,7 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
114 114 return ListUtil.empty();
115 115 }
116 116 // 3.2 段落召回
117   - return segmentMapper.selectList(CollUtil.getFieldValues(documentList, "id", String.class));
  117 + return segmentMapper.selectListByVectorIds(CollUtil.getFieldValues(documentList, "id", String.class));
118 118 }
119 119  
120 120 /**
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java
  View file @5cd8707
... ... @@ -47,13 +47,12 @@ public interface AiKnowledgeService {
47 47 */
48 48 PageResult<AiKnowledgeDO> getKnowledgePageMy(Long userId, PageParam pageReqVO);
49 49  
50   - // TODO @新：knowledgeId 和 validateKnowledgeExists 的 id 是同一个么？如果是的话，建议变量也用 id 哈，然后两边的 id 注释，保持一致
51 50 /**
52 51 * 根据知识库编号获取向量存储实例
53 52 *
54   - * @param knowledgeId 知识库编号
  53 + * @param id 知识库编号
55 54 * @return 向量存储实例
56 55 */
57   - VectorStore getVectorStoreById(Long knowledgeId);
  56 + VectorStore getVectorStoreById(Long id);
58 57  
59 58 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java
  View file @5cd8707
... ... @@ -30,20 +30,17 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_
30 30 public class AiKnowledgeServiceImpl implements AiKnowledgeService {
31 31  
32 32 @Resource
33   - private AiChatModelService chatModalService;
34   -
35   - @Resource
36 33 private AiKnowledgeMapper knowledgeMapper;
  34 +
37 35 @Resource
38 36 private AiChatModelService chatModelService;
39 37 @Resource
40 38 private AiApiKeyService apiKeyService;
41   - // TODO @新：chatModelService 和 apiKeyService 可以放到 33 行的 chatModalService 后面。尽量保持，想通类型的变量在一块。例如说，Service 一块，Mapper 一块。
42 39  
43 40 @Override
44 41 public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) {
45 42 // 1. 校验模型配置
46   - AiChatModelDO model = chatModalService.validateChatModel(createReqVO.getModelId());
  43 + AiChatModelDO model = chatModelService.validateChatModel(createReqVO.getModelId());
47 44  
48 45 // 2. 插入知识库
49 46 AiKnowledgeDO knowledgeBase = BeanUtils.toBean(createReqVO, AiKnowledgeDO.class)
... ... @@ -60,7 +57,7 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService {
60 57 throw exception(KNOWLEDGE_NOT_EXISTS);
61 58 }
62 59 // 1.2 校验模型配置
63   - AiChatModelDO model = chatModalService.validateChatModel(updateReqVO.getModelId());
  60 + AiChatModelDO model = chatModelService.validateChatModel(updateReqVO.getModelId());
64 61  
65 62 // 2. 更新知识库
66 63 AiKnowledgeDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeDO.class);
... ... @@ -83,8 +80,8 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService {
83 80 }
84 81  
85 82 @Override
86   - public VectorStore getVectorStoreById(Long knowledgeId) {
87   - AiKnowledgeDO knowledge = validateKnowledgeExists(knowledgeId);
  83 + public VectorStore getVectorStoreById(Long id) {
  84 + AiKnowledgeDO knowledge = validateKnowledgeExists(id);
88 85 AiChatModelDO model = chatModelService.validateChatModel(knowledge.getModelId());
89 86 // 创建或获取 VectorStore 对象
90 87 return apiKeyService.getOrCreateVectorStore(model.getKeyId());
... ...
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactoryImpl.java
  View file @5cd8707
... ... @@ -197,7 +197,6 @@ public class AiModelFactoryImpl implements AiModelFactory {
197 197 });
198 198 }
199 199  
200   - // TODO @新：貌似可以创建一个大的 VectorStore。然后搜的时候，通过 Filter.Expression 过滤对应的数据。
201 200 @Override
202 201 public VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url) {
203 202 String cacheKey = buildClientCacheKey(VectorStore.class, platform, apiKey, url);
... ...