From ed2296e4c772724004d92b0b0dc3b209cd70a78b Mon Sep 17 00:00:00 2001 From: xiaoxin <718949661@qq.com> Date: Wed, 28 Aug 2024 16:01:10 +0800 Subject: [PATCH] 【解决todo】AI 知识库 --- yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java | 4 ++-- yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java | 62 -------------------------------------------------------------- yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java | 2 +- yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java | 4 ++-- yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java | 27 --------------------------- yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java | 35 ----------------------------------- yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java | 30 ------------------------------ yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java | 79 ------------------------------------------------------------------------------- yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java | 24 +++++++++++------------- yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java | 30 ++++++++++++++++++++++++++++++ yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java | 10 +++++++++- 13 files changed, 186 insertions(+), 252 deletions(-) delete mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java delete mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java delete mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java delete mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java delete mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java index 9d9c99a..9eae6b7 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java @@ -3,7 +3,7 @@ package cn.iocoder.yudao.module.ai.controller.admin.knowledge; import cn.iocoder.yudao.framework.common.pojo.CommonResult; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; -import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeBaseService; +import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.annotation.Resource; @@ -19,7 +19,7 @@ import static cn.iocoder.yudao.framework.security.core.util.SecurityFrameworkUti public class AiKnowledgeController { @Resource - private AiKnowledgeBaseService knowledgeBaseService; + private AiKnowledgeService knowledgeBaseService; @PostMapping("/create-my") @Operation(summary = "创建【我的】知识库") diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java deleted file mode 100644 index d33114f..0000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java +++ /dev/null @@ -1,62 +0,0 @@ -package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; - -import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; -import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler; -import lombok.Data; - -import java.util.List; - -// TODO @xin：要不把 AiKnowledgeBaseDO 改成 AiKnowledgeDO。感觉 base 后缀，感觉有点奇怪（让人以为是基类）。然后，我们很多地方的外键编号，都是 knowledgeId -/** - * AI 知识库 DO - * - * @author xiaoxin - */ -@TableName(value = "ai_knowledge_base", autoResultMap = true) -@Data -public class AiKnowledgeBaseDO extends BaseDO { - - /** - * 编号 - */ - @TableId - private Long id; - /** - * 用户编号 - *

- * 关联 AdminUserDO 的 userId 字段 - */ - private Long userId; - /** - * 知识库名称 - */ - private String name; - /** - * 知识库描述 - */ - private String description; - // TODO @新：如果全部可见，需要怎么设置？ - /** - * 可见权限,只能选择哪些人可见 - */ - @TableField(typeHandler = JacksonTypeHandler.class) - private List visibilityPermissions; - /** - * 嵌入模型编号，高质量模式时维护 - */ - private Long modelId; - /** - * 模型标识 - */ - private String model; - /** - * 状态 - *

- * 枚举 {@link CommonStatusEnum} - */ - private Integer status; -} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java new file mode 100644 index 0000000..89e7486 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java @@ -0,0 +1,61 @@ +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; + +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler; +import lombok.Data; + +import java.util.List; + +/** + * AI 知识库 DO + * + * @author xiaoxin + */ +@TableName(value = "ai_knowledge", autoResultMap = true) +@Data +public class AiKnowledgeDO extends BaseDO { + + /** + * 编号 + */ + @TableId + private Long id; + /** + * 用户编号 + *

+ * 关联 AdminUserDO 的 userId 字段 + */ + private Long userId; + /** + * 知识库名称 + */ + private String name; + /** + * 知识库描述 + */ + private String description; + // TODO @新：如果全部可见，需要怎么设置？ + /** + * 可见权限,只能选择哪些人可见 + */ + @TableField(typeHandler = JacksonTypeHandler.class) + private List visibilityPermissions; + /** + * 嵌入模型编号，高质量模式时维护 + */ + private Long modelId; + /** + * 模型标识 + */ + private String model; + /** + * 状态 + *

+ * 枚举 {@link CommonStatusEnum} + */ + private Integer status; +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java index 4866025..c5e526c 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java @@ -24,7 +24,7 @@ public class AiKnowledgeDocumentDO extends BaseDO { /** * 知识库编号 * - * 关联 {@link AiKnowledgeBaseDO#getId()} + * 关联 {@link AiKnowledgeDO#getId()} */ private Long knowledgeId; /** diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java index cad90fc..710b654 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java @@ -1,7 +1,7 @@ package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; -import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; import org.apache.ibatis.annotations.Mapper; /** @@ -10,5 +10,5 @@ import org.apache.ibatis.annotations.Mapper; * @author xiaoxin */ @Mapper -public interface AiKnowledgeBaseMapper extends BaseMapperX { +public interface AiKnowledgeBaseMapper extends BaseMapperX { } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java deleted file mode 100644 index ee4b3d0..0000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java +++ /dev/null @@ -1,27 +0,0 @@ -package cn.iocoder.yudao.module.ai.service.knowledge; - -import org.springframework.ai.document.Document; -import org.springframework.ai.vectorstore.SearchRequest; - -import java.util.List; - -/** - * AI 嵌入 Service 接口 - * - * @author xiaoxin - */ -public interface AiEmbeddingService { - - /** - * 向量化文档并存储 - */ - void add(List documents); - - /** - * 相似查询 - * - * @param request 查询实体 - */ - List similaritySearch(SearchRequest request); - -} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java deleted file mode 100644 index 689ccea..0000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java +++ /dev/null @@ -1,35 +0,0 @@ -package cn.iocoder.yudao.module.ai.service.knowledge; - -import jakarta.annotation.Resource; -import org.springframework.ai.document.Document; -import org.springframework.ai.vectorstore.RedisVectorStore; -import org.springframework.ai.vectorstore.SearchRequest; -import org.springframework.stereotype.Service; - -import java.util.List; - -// TODO @xin：是不是不用 AiEmbeddingServiceImpl，直接 vectorStore 注入到需要的地方就好啦。通过 KnowledgeDocumentService 返回就好。 -/** - * AI 嵌入 Service 实现类 - * - * @author xiaoxin - */ -@Service -public class AiEmbeddingServiceImpl implements AiEmbeddingService { - - @Resource - private RedisVectorStore vectorStore; - - @Override -// @Async - // TODO xiaoxin 报错先注释 - public void add(List documents) { - vectorStore.add(documents); - } - - @Override - public List similaritySearch(SearchRequest request) { - return vectorStore.similaritySearch(request); - } - -} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java deleted file mode 100644 index be96b09..0000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java +++ /dev/null @@ -1,30 +0,0 @@ -package cn.iocoder.yudao.module.ai.service.knowledge; -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; - -/** - * AI 知识库-基础信息 Service 接口 - * - * @author xiaoxin - */ -public interface AiKnowledgeBaseService { - - /** - * 创建【我的】知识库 - * - * @param createReqVO 创建信息 - * @param userId 用户编号 - * @return 编号 - */ - Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId); - - - /** - * 创建【我的】知识库 - * - * @param updateReqVO 更新信息 - * @param userId 用户编号 - */ - void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId); - -} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java deleted file mode 100644 index c208c92..0000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java +++ /dev/null @@ -1,79 +0,0 @@ -package cn.iocoder.yudao.module.ai.service.knowledge; - -import cn.hutool.core.lang.Assert; -import cn.hutool.core.util.ObjUtil; -import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; -import cn.iocoder.yudao.framework.common.util.object.BeanUtils; -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; -import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO; -import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; -import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeBaseMapper; -import cn.iocoder.yudao.module.ai.service.model.AiChatModelService; -import jakarta.annotation.Resource; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; -import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_EXISTS; - -/** - * AI 知识库-基础信息 Service 实现类 - * - * @author xiaoxin - */ -@Service -@Slf4j -public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService { - - @Resource - private AiChatModelService chatModalService; - - @Resource - private AiKnowledgeBaseMapper knowledgeBaseMapper; - - @Override - public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) { - // TODO @xin：貌似直接调用 chatModalService.validateChatModel(id) 完事，不用搞个方法 - // 1. 校验模型配置 - AiChatModelDO model = validateChatModel(createReqVO.getModelId()); - - // 2. 插入知识库 - // TODO @xin：不用 DO 结尾 - AiKnowledgeBaseDO knowledgeBaseDO = BeanUtils.toBean(createReqVO, AiKnowledgeBaseDO.class) - .setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus()); - knowledgeBaseMapper.insert(knowledgeBaseDO); - return knowledgeBaseDO.getId(); - } - - @Override - public void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId) { - // 1.1 校验知识库存在 - AiKnowledgeBaseDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId()); - if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) { - throw exception(KNOWLEDGE_NOT_EXISTS); - } - // 1.2 校验模型配置 - AiChatModelDO model = validateChatModel(updateReqVO.getModelId()); - - // 2. 更新知识库 - AiKnowledgeBaseDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeBaseDO.class); - updateDO.setModel(model.getModel()); - knowledgeBaseMapper.updateById(updateDO); - } - - private AiChatModelDO validateChatModel(Long id) { - AiChatModelDO model = chatModalService.validateChatModel(id); - Assert.notNull(model, "未找到对应嵌入模型"); - return model; - } - - public AiKnowledgeBaseDO validateKnowledgeExists(Long id) { - AiKnowledgeBaseDO knowledgeBase = knowledgeBaseMapper.selectById(id); - if (knowledgeBase == null) { - throw exception(KNOWLEDGE_NOT_EXISTS); - } - return knowledgeBase; - } - -} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java index 5370330..69a73d6 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java @@ -14,8 +14,9 @@ import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.document.Document; import org.springframework.ai.reader.tika.TikaDocumentReader; -import org.springframework.ai.tokenizer.JTokkitTokenCountEstimator; +import org.springframework.ai.tokenizer.TokenCountEstimator; import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.ai.vectorstore.RedisVectorStore; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -39,52 +40,49 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic @Resource private TokenTextSplitter tokenTextSplitter; - @Resource - private AiEmbeddingService embeddingService; + private TokenCountEstimator TOKEN_COUNT_ESTIMATOR; + @Resource + private RedisVectorStore vectorStore; - // TODO @xin：@Resource 注入 - private static final JTokkitTokenCountEstimator TOKEN_COUNT_ESTIMATOR = new JTokkitTokenCountEstimator(); // TODO xiaoxin 临时测试用，后续删 @Value("classpath:/webapp/test/Fel.pdf") private org.springframework.core.io.Resource data; // TODO 芋艿：需要 review 下，代码格式； - // TODO @xin：最好有 1、/2、/3 这种，让代码更有层次感 @Override @Transactional(rollbackFor = Exception.class) public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) { // TODO xiaoxin 后续从 url 加载 TikaDocumentReader loader = new TikaDocumentReader(data); - // 加载文档 + // 1.1 加载文档 List documents = loader.get(); Document document = CollUtil.getFirst(documents); // TODO @xin：是不是不存在，就抛出异常呀；厚泽 return 呀； - // TODO 芋艿 文档层面有没有可能会比较大，这两个字段是否可以从分段表计算得出？回复：先直接算； Integer tokens = Objects.nonNull(document) ? TOKEN_COUNT_ESTIMATOR.estimate(document.getContent()) : 0; Integer wordCount = Objects.nonNull(document) ? document.getContent().length() : 0; AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class) .setTokens(tokens).setWordCount(wordCount) .setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus()); - // 文档记录入库 + // 1.2 文档记录入库 documentMapper.insert(documentDO); Long documentId = documentDO.getId(); if (CollUtil.isEmpty(documents)) { return documentId; } - // 文档分段 + // 2.1 文档分段 List segments = tokenTextSplitter.apply(documents); - // 分段内容入库 + // 2.2 分段内容入库 List segmentDOList = CollectionUtils.convertList(segments, segment -> new AiKnowledgeSegmentDO().setContent(segment.getContent()).setDocumentId(documentId) .setTokens(TOKEN_COUNT_ESTIMATOR.estimate(segment.getContent())).setWordCount(segment.getContent().length()) .setStatus(CommonStatusEnum.ENABLE.getStatus())); segmentMapper.insertBatch(segmentDOList); - // 向量化并存储 - embeddingService.add(segments); + // 3 向量化并存储 + vectorStore.add(segments); return documentId; } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java new file mode 100644 index 0000000..91b0c9b --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java @@ -0,0 +1,30 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; + +/** + * AI 知识库-基础信息 Service 接口 + * + * @author xiaoxin + */ +public interface AiKnowledgeService { + + /** + * 创建【我的】知识库 + * + * @param createReqVO 创建信息 + * @param userId 用户编号 + * @return 编号 + */ + Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId); + + + /** + * 创建【我的】知识库 + * + * @param updateReqVO 更新信息 + * @param userId 用户编号 + */ + void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId); + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java new file mode 100644 index 0000000..a981b87 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java @@ -0,0 +1,70 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import cn.hutool.core.util.ObjUtil; +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; +import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeBaseMapper; +import cn.iocoder.yudao.module.ai.service.model.AiChatModelService; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_EXISTS; + +/** + * AI 知识库-基础信息 Service 实现类 + * + * @author xiaoxin + */ +@Service +@Slf4j +public class AiKnowledgeServiceImpl implements AiKnowledgeService { + + @Resource + private AiChatModelService chatModalService; + + @Resource + private AiKnowledgeBaseMapper knowledgeBaseMapper; + + @Override + public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) { + // 1. 校验模型配置 + AiChatModelDO model = chatModalService.validateChatModel(createReqVO.getModelId()); + + // 2. 插入知识库 + AiKnowledgeDO knowledgeBase = BeanUtils.toBean(createReqVO, AiKnowledgeDO.class) + .setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus()); + knowledgeBaseMapper.insert(knowledgeBase); + return knowledgeBase.getId(); + } + + @Override + public void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId) { + // 1.1 校验知识库存在 + AiKnowledgeDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId()); + if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) { + throw exception(KNOWLEDGE_NOT_EXISTS); + } + // 1.2 校验模型配置 + AiChatModelDO model = chatModalService.validateChatModel(updateReqVO.getModelId()); + + // 2. 更新知识库 + AiKnowledgeDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeDO.class); + updateDO.setModel(model.getModel()); + knowledgeBaseMapper.updateById(updateDO); + } + + public AiKnowledgeDO validateKnowledgeExists(Long id) { + AiKnowledgeDO knowledgeBase = knowledgeBaseMapper.selectById(id); + if (knowledgeBase == null) { + throw exception(KNOWLEDGE_NOT_EXISTS); + } + return knowledgeBase; + } + +} diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java index 543444f..8566a09 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java @@ -13,6 +13,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.ai.autoconfigure.vectorstore.redis.RedisVectorStoreProperties; import org.springframework.ai.document.MetadataMode; import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.tokenizer.JTokkitTokenCountEstimator; +import org.springframework.ai.tokenizer.TokenCountEstimator; import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.transformers.TransformersEmbeddingModel; import org.springframework.ai.vectorstore.RedisVectorStore; @@ -90,7 +92,7 @@ public class YudaoAiAutoConfiguration { } /** - * 我们启动有加载很多 Embedding 模型，不晓得取哪个好，先 new 个 TransformersEmbeddingModel 跑 + * TODO @xin 抽离出去，根据具体模型走 */ @Bean @Lazy // TODO 芋艿：临时注释，避免无法启动 @@ -114,4 +116,10 @@ public class YudaoAiAutoConfiguration { return new TokenTextSplitter(500, 100, 5, 10000, true); } + @Bean + @Lazy // TODO 芋艿：临时注释，避免无法启动 + public TokenCountEstimator tokenCountEstimator() { + return new JTokkitTokenCountEstimator(); + } + } \ No newline at end of file -- libgit2 0.21.4