BJ_ZTEITS-Developers / parking-cloud-yudao

Commit 8e54eef8af762e77084c4d7742307afa2cdf212a

Authored by xiaoxin
1 parent b4af042c

【新增】AI 知识库：文档切片向量化入库

Inline Side-by-side
Showing 14 changed files with 190 additions and 34 deletions
yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java 0 → 100644
  View file @8e54eef
  1 +package cn.iocoder.yudao.module.ai.enums.knowledge;
  2 +
  3 +import cn.iocoder.yudao.framework.common.core.IntArrayValuable;
  4 +import lombok.AllArgsConstructor;
  5 +import lombok.Getter;
  6 +
  7 +import java.util.Arrays;
  8 +
  9 +/**
  10 + * AI 知识库-文档状态的枚举
  11 + *
  12 + * @author xiaoxin
  13 + */
  14 +@AllArgsConstructor
  15 +@Getter
  16 +public enum AiKnowledgeDocumentStatusEnum implements IntArrayValuable {
  17 +
  18 + IN_PROGRESS(10, "索引中"),
  19 + SUCCESS(20, "可用"),
  20 + FAIL(30, "失败");
  21 +
  22 + /**
  23 + * 状态
  24 + */
  25 + private final Integer status;
  26 +
  27 + /**
  28 + * 状态名
  29 + */
  30 + private final String name;
  31 +
  32 + public static final int[] ARRAYS = Arrays.stream(values()).mapToInt(AiKnowledgeDocumentStatusEnum::getStatus).toArray();
  33 +
  34 + @Override
  35 + public int[] array() {
  36 + return ARRAYS;
  37 + }
  38 +
  39 +}
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java
  View file @8e54eef
... ... @@ -14,11 +14,11 @@ import java.util.List;
14 14 @Data
15 15 public class AiKnowledgeCreateMyReqVO {
16 16  
17   - @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "")
  17 + @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南")
18 18 @NotBlank(message = "知识库名称不能为空")
19 19 private String name;
20 20  
21   - @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "")
  21 + @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "存储 ruoyi-vue-pro 操作文档")
22 22 private String description;
23 23  
24 24 @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]")
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeDocumentCreateReqVO.java 0 → 100644
  View file @8e54eef
  1 +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo;
  2 +
  3 +import io.swagger.v3.oas.annotations.media.Schema;
  4 +import jakarta.validation.constraints.NotBlank;
  5 +import jakarta.validation.constraints.NotNull;
  6 +import lombok.Data;
  7 +
  8 +/**
  9 + * @author xiaoxin
  10 + */
  11 +@Schema(description = "管理后台 - AI 知识库【创建文档】 Request VO")
  12 +@Data
  13 +public class AiKnowledgeDocumentCreateReqVO {
  14 +
  15 +
  16 + @Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204")
  17 + @NotNull(message = "知识库编号不能为空")
  18 + private Long knowledgeId;
  19 +
  20 + @Schema(description = "文档名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "三方登陆")
  21 + @NotBlank(message = "文档名称不能为空")
  22 + private String name;
  23 +
  24 + @Schema(description = "文档 url", requiredMode = Schema.RequiredMode.REQUIRED, example = "https://doc.iocoder.cn")
  25 + private String url;
  26 +
  27 +}
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java
  View file @8e54eef
1 1 package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
2 2  
3 3  
  4 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
4 5 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
5 6 import com.baomidou.mybatisplus.annotation.IdType;
  7 +import com.baomidou.mybatisplus.annotation.TableField;
6 8 import com.baomidou.mybatisplus.annotation.TableId;
7 9 import com.baomidou.mybatisplus.annotation.TableName;
  10 +import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
8 11 import lombok.Data;
9 12  
10 13 import java.util.List;
... ... @@ -40,7 +43,8 @@ public class AiKnowledgeBaseDO extends BaseDO {
40 43 /**
41 44 * 可见权限,只能选择哪些人可见
42 45 */
43   - private List<String> visibilityPermissions;
  46 + @TableField(typeHandler = JacksonTypeHandler.class)
  47 + private List<Long> visibilityPermissions;
44 48 /**
45 49 * 嵌入模型编号，高质量模式时维护
46 50 */
... ... @@ -50,7 +54,9 @@ public class AiKnowledgeBaseDO extends BaseDO {
50 54 */
51 55 private String model;
52 56 /**
53   - * 是否启用
  57 + * 状态
  58 + * <p>
  59 + * 枚举 {@link CommonStatusEnum}
54 60 */
55   - private Boolean status;
  61 + private Integer status;
56 62 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java
  View file @8e54eef
1 1 package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
2 2  
  3 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
3 4 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
  5 +import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum;
4 6 import com.baomidou.mybatisplus.annotation.IdType;
5 7 import com.baomidou.mybatisplus.annotation.TableId;
6 8 import com.baomidou.mybatisplus.annotation.TableName;
... ... @@ -46,10 +48,15 @@ public class AiKnowledgeDocumentDO extends BaseDO {
46 48 private Integer wordCount;
47 49 /**
48 50 * 切片状态
  51 + * <p>
  52 + * 枚举 {@link AiKnowledgeDocumentStatusEnum}
49 53 */
50 54 private Integer sliceStatus;
  55 +
51 56 /**
52   - * 是否启用
  57 + * 状态
  58 + * <p>
  59 + * 枚举 {@link CommonStatusEnum}
53 60 */
54   - private Boolean status;
  61 + private Integer status;
55 62 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java
  View file @8e54eef
1 1 package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
2 2  
  3 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
3 4 import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
4 5 import com.baomidou.mybatisplus.annotation.IdType;
5 6 import com.baomidou.mybatisplus.annotation.TableId;
... ... @@ -41,8 +42,10 @@ public class AiKnowledgeSegmentDO extends BaseDO {
41 42 */
42 43 private Integer tokens;
43 44 /**
44   - * 是否启用
  45 + * 状态
  46 + * <p>
  47 + * 枚举 {@link CommonStatusEnum}
45 48 */
46   - private Boolean status;
  49 + private Integer status;
47 50  
48 51 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java
  View file @8e54eef
... ... @@ -2,11 +2,13 @@ package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
2 2  
3 3 import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
4 4 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO;
  5 +import org.apache.ibatis.annotations.Mapper;
5 6  
6 7 /**
7 8 * AI 知识库基础信息 Mapper
8 9 *
9 10 * @author xiaoxin
10 11 */
  12 +@Mapper
11 13 public interface AiKnowledgeBaseMapper extends BaseMapperX<AiKnowledgeBaseDO> {
12 14 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java
  View file @8e54eef
... ... @@ -2,11 +2,13 @@ package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
2 2  
3 3 import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
4 4 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
  5 +import org.apache.ibatis.annotations.Mapper;
5 6  
6 7 /**
7 8 * AI 知识库-文档 Mapper
8 9 *
9 10 * @author xiaoxin
10 11 */
  12 +@Mapper
11 13 public interface AiKnowledgeDocumentMapper extends BaseMapperX<AiKnowledgeDocumentDO> {
12 14 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java
  View file @8e54eef
... ... @@ -2,11 +2,13 @@ package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
2 2  
3 3 import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
4 4 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
  5 +import org.apache.ibatis.annotations.Mapper;
5 6  
6 7 /**
7 8 * AI 知识库-分片 Mapper
8 9 *
9 10 * @author xiaoxin
10 11 */
  12 +@Mapper
11 13 public interface AiKnowledgeSegmentMapper extends BaseMapperX<AiKnowledgeSegmentDO> {
12 14 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java
  View file @8e54eef
1 1 package cn.iocoder.yudao.module.ai.service.knowledge;
2 2  
3 3 import org.springframework.ai.document.Document;
  4 +import org.springframework.ai.vectorstore.SearchRequest;
4 5  
5 6 import java.util.List;
6 7  
... ... @@ -12,9 +13,9 @@ import java.util.List;
12 13 public interface AiEmbeddingService {
13 14  
14 15 /**
15   - * 向量化文档
  16 + * 向量化文档并存储
16 17 */
17   - void embeddingDoc();
  18 + void add(List<Document> documents);
18 19  
19 20  
20 21 /**
... ... @@ -22,5 +23,5 @@ public interface AiEmbeddingService {
22 23 *
23 24 * @param content 查询内容
24 25 */
25   - List<Document> similaritySearch(String content);
  26 + List<Document> similaritySearch(SearchRequest request);
26 27 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java
  View file @8e54eef
... ... @@ -2,11 +2,9 @@ package cn.iocoder.yudao.module.ai.service.knowledge;
2 2  
3 3 import jakarta.annotation.Resource;
4 4 import org.springframework.ai.document.Document;
5   -import org.springframework.ai.reader.tika.TikaDocumentReader;
6   -import org.springframework.ai.transformer.splitter.TokenTextSplitter;
7 5 import org.springframework.ai.vectorstore.RedisVectorStore;
8 6 import org.springframework.ai.vectorstore.SearchRequest;
9   -import org.springframework.beans.factory.annotation.Value;
  7 +import org.springframework.scheduling.annotation.Async;
10 8 import org.springframework.stereotype.Service;
11 9  
12 10 import java.util.List;
... ... @@ -21,27 +19,14 @@ public class AiEmbeddingServiceImpl implements AiEmbeddingService {
21 19  
22 20 @Resource
23 21 private RedisVectorStore vectorStore;
24   - @Resource
25   - private TokenTextSplitter tokenTextSplitter;
26   -
27   - // TODO @xin 临时测试用，后续删
28   - @Value("classpath:/webapp/test/Fel.pdf")
29   - private org.springframework.core.io.Resource data;
30 22  
31 23 @Override
32   - public void embeddingDoc() {
33   - // 读取文件
34   - TikaDocumentReader loader = new TikaDocumentReader(data);
35   - List<Document> documents = loader.get();
36   - // 文档分段
37   - List<Document> segments = tokenTextSplitter.apply(documents);
38   - // 向量化并存储
39   - vectorStore.add(segments);
  24 + public void add(List<Document> documents) {
  25 + vectorStore.add(documents);
40 26 }
41 27  
42 28 @Override
43   - public List<Document> similaritySearch(String content) {
44   - SearchRequest request = SearchRequest.query(content);
  29 + public List<Document> similaritySearch(SearchRequest request) {
45 30 return vectorStore.similaritySearch(request);
46 31 }
47 32 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java
  View file @8e54eef
... ... @@ -2,6 +2,7 @@ package cn.iocoder.yudao.module.ai.service.knowledge;
2 2  
3 3 import cn.hutool.core.lang.Assert;
4 4 import cn.hutool.core.util.ObjUtil;
  5 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
5 6 import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
6 7 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO;
7 8 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO;
... ... @@ -26,16 +27,18 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_
26 27 public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService {
27 28  
28 29 @Resource
29   - private AiKnowledgeBaseMapper knowledgeBaseMapper;
30   - @Resource
31 30 private AiChatModelService chatModalService;
32 31  
  32 + @Resource
  33 + private AiKnowledgeBaseMapper knowledgeBaseMapper;
  34 +
  35 +
33 36 @Override
34 37 public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) {
35 38 AiChatModelDO model = validateChatModel(createReqVO.getModelId());
36 39  
37 40 AiKnowledgeBaseDO knowledgeBaseDO = BeanUtils.toBean(createReqVO, AiKnowledgeBaseDO.class);
38   - knowledgeBaseDO.setModel(model.getModel()).setUserId(userId);
  41 + knowledgeBaseDO.setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus());
39 42  
40 43 knowledgeBaseMapper.insert(knowledgeBaseDO);
41 44 return knowledgeBaseDO.getId();
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java
  View file @8e54eef
1 1 package cn.iocoder.yudao.module.ai.service.knowledge;
2 2  
  3 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeDocumentCreateReqVO;
  4 +
3 5 /**
4 6 * AI 知识库-文档 Service 接口
5 7 *
... ... @@ -7,4 +9,13 @@ package cn.iocoder.yudao.module.ai.service.knowledge;
7 9 */
8 10 public interface AiKnowledgeDocumentService {
9 11  
  12 +
  13 + /**
  14 + * 创建文档
  15 + *
  16 + * @param createReqVO 文档创建 Request VO
  17 + * @return 文档编号
  18 + */
  19 + Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO);
  20 +
10 21 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java
  View file @8e54eef
1 1 package cn.iocoder.yudao.module.ai.service.knowledge;
2 2  
  3 +import cn.hutool.core.collection.CollUtil;
  4 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
  5 +import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
  6 +import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
  7 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeDocumentCreateReqVO;
  8 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
  9 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
  10 +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper;
  11 +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper;
  12 +import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum;
  13 +import jakarta.annotation.Resource;
3 14 import lombok.extern.slf4j.Slf4j;
  15 +import org.springframework.ai.document.Document;
  16 +import org.springframework.ai.reader.tika.TikaDocumentReader;
  17 +import org.springframework.ai.transformer.splitter.TokenTextSplitter;
  18 +import org.springframework.beans.factory.annotation.Value;
4 19 import org.springframework.stereotype.Service;
  20 +import org.springframework.transaction.annotation.Transactional;
  21 +
  22 +import java.util.List;
5 23  
6 24 /**
7 25 * AI 知识库-文档 Service 实现类
... ... @@ -12,5 +30,55 @@ import org.springframework.stereotype.Service;
12 30 @Slf4j
13 31 public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentService {
14 32  
  33 + @Resource
  34 + private AiKnowledgeDocumentMapper documentMapper;
  35 + @Resource
  36 + private AiKnowledgeSegmentMapper segmentMapper;
  37 +
  38 + @Resource
  39 + private TokenTextSplitter tokenTextSplitter;
  40 +
  41 + @Resource
  42 + private AiEmbeddingService embeddingService;
  43 +
  44 + // TODO @xin 临时测试用，后续删
  45 + @Value("classpath:/webapp/test/Fel.pdf")
  46 + private org.springframework.core.io.Resource data;
  47 +
  48 +
  49 + @Override
  50 + @Transactional(rollbackFor = Exception.class)
  51 + public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) {
  52 + AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class);
  53 + documentDO
  54 + //todo
  55 + .setTokens(0).setWordCount(0)
  56 + .setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus());
  57 + documentMapper.insert(documentDO);
  58 +
  59 + TikaDocumentReader loader = new TikaDocumentReader(data);
  60 + List<Document> documents = loader.get();
  61 + Long documentId = documentDO.getId();
  62 + if (CollUtil.isEmpty(documents)) {
  63 + log.info("文档内容为空");
  64 + return documentId;
  65 + }
  66 +
  67 + // 文档分段
  68 + List<Document> segments = tokenTextSplitter.apply(documents);
  69 +
  70 + List<AiKnowledgeSegmentDO> segmentDOList = CollectionUtils.convertList(segments,
  71 + segment -> new AiKnowledgeSegmentDO().setContent(segment.getContent()).setDocumentId(documentId)
  72 + //todo
  73 + .setTokens(0).setWordCount(0)
  74 + .setStatus(CommonStatusEnum.ENABLE.getStatus()));
  75 +
  76 + // 分段内容入库
  77 + segmentMapper.insertBatch(segmentDOList);
  78 +
  79 + //向量化并存储
  80 + embeddingService.add(segments);
15 81  
  82 + return documentId;
  83 + }
16 84 }
... ...