From 024109dac9237d06766daf8f1aa0e5347b5bf361 Mon Sep 17 00:00:00 2001 From: xiaoxin <718949661@qq.com> Date: Thu, 29 Aug 2024 10:12:47 +0800 Subject: [PATCH] 【优化】AI 知识库: 从url下载文档资源 --- yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java | 14 -------------- yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeMapper.java | 14 ++++++++++++++ yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java | 24 ++++++++++++++++-------- yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java | 4 ++-- yudao-module-ai/yudao-spring-boot-starter-ai/src/main/resources/webapp/test/Fel.pdf | Bin 352908 -> 0 bytes 5 files changed, 32 insertions(+), 24 deletions(-) delete mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeMapper.java delete mode 100755 yudao-module-ai/yudao-spring-boot-starter-ai/src/main/resources/webapp/test/Fel.pdf diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java deleted file mode 100644 index 710b654..0000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java +++ /dev/null @@ -1,14 +0,0 @@ -package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; - -import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; -import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; -import org.apache.ibatis.annotations.Mapper; - -/** - * AI 知识库基础信息 Mapper - * - * @author xiaoxin - */ -@Mapper -public interface AiKnowledgeBaseMapper extends BaseMapperX { -} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeMapper.java new file mode 100644 index 0000000..41e71cc --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeMapper.java @@ -0,0 +1,14 @@ +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; + +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; +import org.apache.ibatis.annotations.Mapper; + +/** + * AI 知识库基础信息 Mapper + * + * @author xiaoxin + */ +@Mapper +public interface AiKnowledgeMapper extends BaseMapperX { +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java index 69a73d6..2af8b9d 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java @@ -1,6 +1,7 @@ package cn.iocoder.yudao.module.ai.service.knowledge; import cn.hutool.core.collection.CollUtil; +import cn.hutool.http.HttpUtil; import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils; import cn.iocoder.yudao.framework.common.util.object.BeanUtils; @@ -17,7 +18,7 @@ import org.springframework.ai.reader.tika.TikaDocumentReader; import org.springframework.ai.tokenizer.TokenCountEstimator; import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.RedisVectorStore; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.ByteArrayResource; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -46,17 +47,14 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic private RedisVectorStore vectorStore; - // TODO xiaoxin 临时测试用，后续删 - @Value("classpath:/webapp/test/Fel.pdf") - private org.springframework.core.io.Resource data; - // TODO 芋艿：需要 review 下，代码格式； @Override @Transactional(rollbackFor = Exception.class) public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) { - // TODO xiaoxin 后续从 url 加载 - TikaDocumentReader loader = new TikaDocumentReader(data); - // 1.1 加载文档 + // 1.1 下载文档 + String url = createReqVO.getUrl(); + TikaDocumentReader loader = new TikaDocumentReader(downloadFile(url)); + // 1.2 加载文档 List documents = loader.get(); Document document = CollUtil.getFirst(documents); // TODO @xin：是不是不存在，就抛出异常呀；厚泽 return 呀； @@ -86,4 +84,14 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic return documentId; } + private org.springframework.core.io.Resource downloadFile(String url) { + try { + byte[] bytes = HttpUtil.downloadBytes(url); + return new ByteArrayResource(bytes); + } catch (Exception e) { + log.error("[downloadFile][url({}) 下载失败]", url, e); + throw new RuntimeException(e); + } + } + } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java index a981b87..5889bce 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java @@ -7,7 +7,7 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreat import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; -import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeBaseMapper; +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeMapper; import cn.iocoder.yudao.module.ai.service.model.AiChatModelService; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; @@ -29,7 +29,7 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService { private AiChatModelService chatModalService; @Resource - private AiKnowledgeBaseMapper knowledgeBaseMapper; + private AiKnowledgeMapper knowledgeBaseMapper; @Override public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) { diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/resources/webapp/test/Fel.pdf b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/resources/webapp/test/Fel.pdf deleted file mode 100755 index 405b67f..0000000 Binary files a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/resources/webapp/test/Fel.pdf and /dev/null differ -- libgit2 0.21.4