BJ_ZTEITS-Developers / parking-cloud-yudao

Commit 7918ba7d29698a39dc0141e20284d4946d13fa9a

Authored by YunaiV
1 parent 0a8c7562

【功能新增】AI：新增知识库文档的批量添加

Inline Side-by-side
Showing 8 changed files with 154 additions and 18 deletions
yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java
  View file @7918ba7
... ... @@ -56,6 +56,7 @@ public interface ErrorCodeConstants {
56 56  
57 57 ErrorCode KNOWLEDGE_DOCUMENT_NOT_EXISTS = new ErrorCode(1_022_008_101, "文档不存在!");
58 58 ErrorCode KNOWLEDGE_DOCUMENT_FILE_EMPTY = new ErrorCode(1_022_008_102, "文档内容为空!");
  59 + ErrorCode KNOWLEDGE_DOCUMENT_FILE_DOWNLOAD_FAIL = new ErrorCode(1_022_008_102, "文件下载失败!");
59 60 ErrorCode KNOWLEDGE_DOCUMENT_FILE_READ_FAIL = new ErrorCode(1_022_008_102, "文档加载失败!");
60 61  
61 62 ErrorCode KNOWLEDGE_SEGMENT_NOT_EXISTS = new ErrorCode(1_022_008_202, "段落不存在!");
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeDocumentController.http
  View file @7918ba7
... ... @@ -9,4 +9,27 @@ tenant-id: {{adminTenantId}}
9 9 "name": "测试文档",
10 10 "url": "https://static.iocoder.cn/README.md",
11 11 "segmentMaxTokens": 800
12   -}
13 12 \ No newline at end of file
  13 +}
  14 +
  15 +### 批量创建知识文档
  16 +POST {{baseUrl}}/ai/knowledge/document/create-list
  17 +Content-Type: application/json
  18 +Authorization: Bearer {{token}}
  19 +tenant-id: {{adminTenantId}}
  20 +
  21 +{
  22 + "knowledgeId": 1,
  23 + "list": [
  24 + {
  25 + "name": "测试文档1",
  26 + "url": "https://static.iocoder.cn/README.md",
  27 + "segmentMaxTokens": 800
  28 + },
  29 + {
  30 + "name": "测试文档2",
  31 + "url": "https://static.iocoder.cn/README_yudao.md",
  32 + "segmentMaxTokens": 400
  33 + }
  34 + ]
  35 +}
  36 +
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeDocumentController.java
  View file @7918ba7
... ... @@ -6,6 +6,7 @@ import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
6 6 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO;
7 7 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentRespVO;
8 8 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO;
  9 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateListReqVO;
9 10 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO;
10 11 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
11 12 import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeDocumentService;
... ... @@ -16,6 +17,8 @@ import jakarta.validation.Valid;
16 17 import org.springframework.validation.annotation.Validated;
17 18 import org.springframework.web.bind.annotation.*;
18 19  
  20 +import java.util.List;
  21 +
19 22 import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success;
20 23  
21 24 @Tag(name = "管理后台 - AI 知识库文档")
... ... @@ -38,8 +41,16 @@ public class AiKnowledgeDocumentController {
38 41 @PostMapping("/create")
39 42 @Operation(summary = "新建文档")
40 43 public CommonResult<Long> createKnowledgeDocument(@RequestBody @Valid AiKnowledgeDocumentCreateReqVO reqVO) {
41   - Long knowledgeDocumentId = documentService.createKnowledgeDocument(reqVO);
42   - return success(knowledgeDocumentId);
  44 + Long id = documentService.createKnowledgeDocument(reqVO);
  45 + return success(id);
  46 + }
  47 +
  48 + @PostMapping("/create-list")
  49 + @Operation(summary = "批量新建文档")
  50 + public CommonResult<List<Long>> createKnowledgeDocumentList(
  51 + @RequestBody @Valid AiKnowledgeDocumentCreateListReqVO reqVO) {
  52 + List<Long> ids = documentService.createKnowledgeDocumentList(reqVO);
  53 + return success(ids);
43 54 }
44 55  
45 56 @PutMapping("/update")
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeDocumentCreateListReqVO.java 0 → 100644
  View file @7918ba7
  1 +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge;
  2 +
  3 +import io.swagger.v3.oas.annotations.media.Schema;
  4 +import jakarta.validation.constraints.NotBlank;
  5 +import jakarta.validation.constraints.NotEmpty;
  6 +import jakarta.validation.constraints.NotNull;
  7 +import lombok.Data;
  8 +import org.hibernate.validator.constraints.URL;
  9 +
  10 +import java.util.List;
  11 +
  12 +@Schema(description = "管理后台 - AI 知识库文档批量创建 Request VO")
  13 +@Data
  14 +public class AiKnowledgeDocumentCreateListReqVO {
  15 +
  16 + @Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204")
  17 + @NotNull(message = "知识库编号不能为空")
  18 + private Long knowledgeId;
  19 +
  20 + @Schema(description = "文档列表", requiredMode = Schema.RequiredMode.REQUIRED)
  21 + @NotEmpty(message = "文档列表不能为空")
  22 + private List<Document> list;
  23 +
  24 + @Schema(description = "文档")
  25 + @Data
  26 + public static class Document {
  27 +
  28 + @Schema(description = "文档名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "三方登陆")
  29 + @NotBlank(message = "文档名称不能为空")
  30 + private String name;
  31 +
  32 + @Schema(description = "文档 URL", requiredMode = Schema.RequiredMode.REQUIRED, example = "https://doc.iocoder.cn")
  33 + @URL(message = "文档 URL 格式不正确")
  34 + private String url;
  35 +
  36 + @Schema(description = "分段的最大 Token 数", requiredMode = Schema.RequiredMode.REQUIRED, example = "800")
  37 + @NotNull(message = "分段的最大 Token 数不能为空")
  38 + private Integer segmentMaxTokens;
  39 +
  40 + }
  41 +
  42 +}
0 43 \ No newline at end of file
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java
  View file @7918ba7
... ... @@ -4,8 +4,11 @@ import cn.iocoder.yudao.framework.common.pojo.PageResult;
4 4 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO;
5 5 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO;
6 6 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO;
  7 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateListReqVO;
7 8 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
8 9  
  10 +import java.util.List;
  11 +
9 12 /**
10 13 * AI 知识库-文档 Service 接口
11 14 *
... ... @@ -22,6 +25,14 @@ public interface AiKnowledgeDocumentService {
22 25 Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO);
23 26  
24 27 /**
  28 + * 批量创建文档
  29 + *
  30 + * @param createListReqVO 批量创建 Request VO
  31 + * @return 文档编号列表
  32 + */
  33 + List<Long> createKnowledgeDocumentList(AiKnowledgeDocumentCreateListReqVO createListReqVO);
  34 +
  35 + /**
25 36 * 获取文档分页
26 37 *
27 38 * @param pageReqVO 分页参数
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java
  View file @7918ba7
... ... @@ -8,6 +8,7 @@ import cn.iocoder.yudao.framework.common.pojo.PageResult;
8 8 import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
9 9 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO;
10 10 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO;
  11 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateListReqVO;
11 12 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO;
12 13 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
13 14 import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper;
... ... @@ -21,9 +22,11 @@ import org.springframework.core.io.ByteArrayResource;
21 22 import org.springframework.stereotype.Service;
22 23 import org.springframework.transaction.annotation.Transactional;
23 24  
  25 +import java.util.ArrayList;
24 26 import java.util.List;
25 27  
26 28 import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
  29 +import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertList;
27 30 import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.*;
28 31  
29 32 /**
... ... @@ -54,26 +57,46 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
54 57 knowledgeService.validateKnowledgeExists(createReqVO.getKnowledgeId());
55 58  
56 59 // 2. 下载文档
57   - TikaDocumentReader loader = new TikaDocumentReader(downloadFile(createReqVO.getUrl()));
58   - List<Document> documents = loader.get();
59   - Document document = CollUtil.getFirst(documents);
60   - if (document == null || StrUtil.isEmpty(document.getText())) {
61   - throw exception(KNOWLEDGE_DOCUMENT_FILE_READ_FAIL);
62   - }
  60 + String content = readUrl(createReqVO.getUrl());
63 61  
64 62 // 3. 文档记录入库
65   - String content = document.getText();
66 63 AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class)
67 64 .setContent(content).setContentLength(content.length()).setTokens(tokenCountEstimator.estimate(content))
68 65 .setStatus(CommonStatusEnum.ENABLE.getStatus());
69 66 knowledgeDocumentMapper.insert(documentDO);
70 67  
71   - // 4. 文档切片入库
72   - knowledgeSegmentService.createKnowledgeSegmentBySplitContent(documentDO.getId(), document.getText());
  68 + // 4. 文档切片入库（同步）
  69 + knowledgeSegmentService.createKnowledgeSegmentBySplitContent(documentDO.getId(), content);
73 70 return documentDO.getId();
74 71 }
75 72  
76 73 @Override
  74 + @Transactional(rollbackFor = Exception.class)
  75 + public List<Long> createKnowledgeDocumentList(AiKnowledgeDocumentCreateListReqVO createListReqVO) {
  76 + // 1. 校验参数
  77 + knowledgeService.validateKnowledgeExists(createListReqVO.getKnowledgeId());
  78 +
  79 + // 2. 下载文档
  80 + List<String> contents = convertList(createListReqVO.getList(), document -> readUrl(document.getUrl()));
  81 +
  82 + // 3. 文档记录入库
  83 + List<AiKnowledgeDocumentDO> documentDOs = new ArrayList<>(createListReqVO.getList().size());
  84 + for (int i = 0; i < createListReqVO.getList().size(); i++) {
  85 + AiKnowledgeDocumentCreateListReqVO.Document documentVO = createListReqVO.getList().get(i);
  86 + String content = contents.get(i);
  87 + documentDOs.add(BeanUtils.toBean(documentVO, AiKnowledgeDocumentDO.class).setKnowledgeId(createListReqVO.getKnowledgeId())
  88 + .setContent(content).setContentLength(content.length()).setTokens(tokenCountEstimator.estimate(content))
  89 + .setStatus(CommonStatusEnum.ENABLE.getStatus()));
  90 + }
  91 + knowledgeDocumentMapper.insertBatch(documentDOs);
  92 +
  93 + // 4. 批量创建文档切片（异步）
  94 + documentDOs.forEach(documentDO ->
  95 + knowledgeSegmentService.createKnowledgeSegmentBySplitContentAsync(documentDO.getId(), documentDO.getContent()));
  96 + return convertList(documentDOs, AiKnowledgeDocumentDO::getId);
  97 + }
  98 +
  99 + @Override
77 100 public PageResult<AiKnowledgeDocumentDO> getKnowledgeDocumentPage(AiKnowledgeDocumentPageReqVO pageReqVO) {
78 101 return knowledgeDocumentMapper.selectPage(pageReqVO);
79 102 }
... ... @@ -97,17 +120,28 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
97 120 return knowledgeDocument;
98 121 }
99 122  
100   - private org.springframework.core.io.Resource downloadFile(String url) {
  123 + private static String readUrl(String url) {
  124 + // 下载文件
  125 + ByteArrayResource resource = null;
101 126 try {
102 127 byte[] bytes = HttpUtil.downloadBytes(url);
103 128 if (bytes.length == 0) {
104 129 throw exception(KNOWLEDGE_DOCUMENT_FILE_EMPTY);
105 130 }
106   - return new ByteArrayResource(bytes);
  131 + resource = new ByteArrayResource(bytes);
107 132 } catch (Exception e) {
108   - log.error("[downloadFile][url({}) 下载失败]", url, e);
109   - throw new RuntimeException(e);
  133 + log.error("[readUrl][url({}) 读取失败]", url, e);
  134 + throw exception(KNOWLEDGE_DOCUMENT_FILE_DOWNLOAD_FAIL);
  135 + }
  136 +
  137 + // 读取文件
  138 + TikaDocumentReader loader = new TikaDocumentReader(resource);
  139 + List<Document> documents = loader.get();
  140 + Document document = CollUtil.getFirst(documents);
  141 + if (document == null || StrUtil.isEmpty(document.getText())) {
  142 + throw exception(KNOWLEDGE_DOCUMENT_FILE_READ_FAIL);
110 143 }
  144 + return document.getText();
111 145 }
112 146  
113 147 }
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java
  View file @7918ba7
... ... @@ -6,6 +6,7 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowle
6 6 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO;
7 7 import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO;
8 8 import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
  9 +import org.springframework.scheduling.annotation.Async;
9 10  
10 11 import java.util.List;
11 12  
... ... @@ -33,6 +34,17 @@ public interface AiKnowledgeSegmentService {
33 34 void createKnowledgeSegmentBySplitContent(Long documentId, String content);
34 35  
35 36 /**
  37 + * 【异步】基于 content 内容，切片创建多个段落
  38 + *
  39 + * @param documentId 知识库文档编号
  40 + * @param content 文档内容
  41 + */
  42 + @Async
  43 + default void createKnowledgeSegmentBySplitContentAsync(Long documentId, String content) {
  44 + createKnowledgeSegmentBySplitContent(documentId, content);
  45 + }
  46 +
  47 + /**
36 48 * 更新段落的内容
37 49 *
38 50 * @param reqVO 更新内容
... ...
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
  View file @7918ba7
... ... @@ -110,8 +110,10 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
110 110 // 3.1 更新切片
111 111 AiKnowledgeSegmentDO segmentDO = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class);
112 112 segmentMapper.updateById(segmentDO);
113   - // 3.2 重新向量化
114   - writeVectorStore(vectorStore, segmentDO, new Document(segmentDO.getContent()));
  113 + // 3.2 重新向量化，必须开启状态
  114 + if (CommonStatusEnum.isEnable(segmentDO.getStatus())) {
  115 + writeVectorStore(vectorStore, segmentDO, new Document(segmentDO.getContent()));
  116 + }
115 117 }
116 118  
117 119 @Override
... ...