BJ_ZTEITS-Developers / parking-cloud-yudao

Commit 0db165f6d97940a4b23af3fd453646b513092ec7

Authored by 芋道源码
Committed by Gitee
2 parents 83d86bcf 238f603f

!18 【新增】AI 知识库：文档切片向量化入库 

Merge pull request !18 from 小新/master-jdk21-ai
Inline Side-by-side
Showing 23 changed files with 691 additions and 52 deletions
yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java
  View file @0db165f
@@ -52,4 +52,8 @@ public interface ErrorCodeConstants { @@ -52,4 +52,8 @@ public interface ErrorCodeConstants {
52 // ========== API 思维导图 1-040-008-000 ========== 52 // ========== API 思维导图 1-040-008-000 ==========
53 ErrorCode MIND_MAP_NOT_EXISTS = new ErrorCode(1_040_008_000, "思维导图不存在!"); 53 ErrorCode MIND_MAP_NOT_EXISTS = new ErrorCode(1_040_008_000, "思维导图不存在!");
54 54
  55 +
  56 + // ========== API 知识库 1-022-008-000 ==========
  57 + ErrorCode KNOWLEDGE_NOT_EXISTS = new ErrorCode(1_022_008_000, "知识库不存在!");
  58 +
55 } 59 }
yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.enums.knowledge;
  2 +
  3 +import cn.iocoder.yudao.framework.common.core.IntArrayValuable;
  4 +import lombok.AllArgsConstructor;
  5 +import lombok.Getter;
  6 +
  7 +import java.util.Arrays;
  8 +
  9 +/**
  10 + * AI 知识库-文档状态的枚举
  11 + *
  12 + * @author xiaoxin
  13 + */
  14 +@AllArgsConstructor
  15 +@Getter
  16 +public enum AiKnowledgeDocumentStatusEnum implements IntArrayValuable {
  17 +
  18 + IN_PROGRESS(10, "索引中"),
  19 + SUCCESS(20, "可用"),
  20 + FAIL(30, "失败");
  21 +
  22 + /**
  23 + * 状态
  24 + */
  25 + private final Integer status;
  26 +
  27 + /**
  28 + * 状态名
  29 + */
  30 + private final String name;
  31 +
  32 + public static final int[] ARRAYS = Arrays.stream(values()).mapToInt(AiKnowledgeDocumentStatusEnum::getStatus).toArray();
  33 +
  34 + @Override
  35 + public int[] array() {
  36 + return ARRAYS;
  37 + }
  38 +
  39 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.controller.admin.knowledge;
  2 +
  3 +import cn.iocoder.yudao.framework.common.pojo.CommonResult;
  4 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO;
  5 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO;
  6 +import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeBaseService;
  7 +import io.swagger.v3.oas.annotations.Operation;
  8 +import io.swagger.v3.oas.annotations.tags.Tag;
  9 +import jakarta.annotation.Resource;
  10 +import jakarta.validation.Valid;
  11 +import org.springframework.web.bind.annotation.*;
  12 +
  13 +import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success;
  14 +import static cn.iocoder.yudao.framework.security.core.util.SecurityFrameworkUtils.getLoginUserId;
  15 +
  16 +@Tag(name = "管理后台 - AI 知识库")
  17 +@RestController
  18 +@RequestMapping("/ai/knowledge")
  19 +public class AiKnowledgeController {
  20 +
  21 + @Resource
  22 + private AiKnowledgeBaseService knowledgeBaseService;
  23 +
  24 + @PostMapping("/create-my")
  25 + @Operation(summary = "创建【我的】知识库")
  26 + public CommonResult<Long> createKnowledgeMy(@RequestBody @Valid AiKnowledgeCreateMyReqVO createReqVO) {
  27 + return success(knowledgeBaseService.createKnowledgeMy(createReqVO, getLoginUserId()));
  28 + }
  29 +
  30 +
  31 + @PutMapping("/update-my")
  32 + @Operation(summary = "更新【我的】知识库")
  33 + public CommonResult<Boolean> updateKnowledgeMy(@RequestBody @Valid AiKnowledgeUpdateMyReqVO updateReqVO) {
  34 + knowledgeBaseService.updateKnowledgeMy(updateReqVO, getLoginUserId());
  35 + return success(true);
  36 + }
  37 +
  38 +
  39 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo;
  2 +
  3 +import io.swagger.v3.oas.annotations.media.Schema;
  4 +import jakarta.validation.constraints.NotBlank;
  5 +import jakarta.validation.constraints.NotNull;
  6 +import lombok.Data;
  7 +
  8 +import java.util.List;
  9 +
  10 +/**
  11 + * @author xiaoxin
  12 + */
  13 +@Schema(description = "管理后台 - AI 知识库创建【我的】 Request VO")
  14 +@Data
  15 +public class AiKnowledgeCreateMyReqVO {
  16 +
  17 + @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南")
  18 + @NotBlank(message = "知识库名称不能为空")
  19 + private String name;
  20 +
  21 + @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "存储 ruoyi-vue-pro 操作文档")
  22 + private String description;
  23 +
  24 + @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]")
  25 + private List<Long> visibilityPermissions;
  26 +
  27 + @Schema(description = "嵌入模型 id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
  28 + @NotNull(message = "嵌入模型不能为空")
  29 + private Long modelId;
  30 +
  31 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeDocumentCreateReqVO.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo;
  2 +
  3 +import io.swagger.v3.oas.annotations.media.Schema;
  4 +import jakarta.validation.constraints.NotBlank;
  5 +import jakarta.validation.constraints.NotNull;
  6 +import lombok.Data;
  7 +
  8 +/**
  9 + * @author xiaoxin
  10 + */
  11 +@Schema(description = "管理后台 - AI 知识库【创建文档】 Request VO")
  12 +@Data
  13 +public class AiKnowledgeDocumentCreateReqVO {
  14 +
  15 +
  16 + @Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204")
  17 + @NotNull(message = "知识库编号不能为空")
  18 + private Long knowledgeId;
  19 +
  20 + @Schema(description = "文档名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "三方登陆")
  21 + @NotBlank(message = "文档名称不能为空")
  22 + private String name;
  23 +
  24 + @Schema(description = "文档 url", requiredMode = Schema.RequiredMode.REQUIRED, example = "https://doc.iocoder.cn")
  25 + private String url;
  26 +
  27 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeUpdateMyReqVO.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo;
  2 +
  3 +import io.swagger.v3.oas.annotations.media.Schema;
  4 +import jakarta.validation.constraints.NotBlank;
  5 +import jakarta.validation.constraints.NotNull;
  6 +import lombok.Data;
  7 +
  8 +import java.util.List;
  9 +
  10 +/**
  11 + * @author xiaoxin
  12 + */
  13 +@Schema(description = "管理后台 - AI 知识库创建【我的】 Request VO")
  14 +@Data
  15 +public class AiKnowledgeUpdateMyReqVO {
  16 +
  17 +
  18 + @Schema(description = "对话编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204")
  19 + @NotNull(message = "知识库编号不能为空")
  20 + private Long id;
  21 +
  22 + @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "")
  23 + @NotBlank(message = "知识库名称不能为空")
  24 + private String name;
  25 +
  26 + @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "")
  27 + private String description;
  28 +
  29 + @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]")
  30 + private List<Long> visibilityPermissions;
  31 +
  32 + @Schema(description = "嵌入模型 id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
  33 + @NotNull(message = "嵌入模型不能为空")
  34 + private Long modelId;
  35 +
  36 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
  2 +
  3 +
  4 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
  5 +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
  6 +import com.baomidou.mybatisplus.annotation.IdType;
  7 +import com.baomidou.mybatisplus.annotation.TableField;
  8 +import com.baomidou.mybatisplus.annotation.TableId;
  9 +import com.baomidou.mybatisplus.annotation.TableName;
  10 +import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
  11 +import lombok.Data;
  12 +
  13 +import java.util.List;
  14 +
  15 +/**
  16 + * AI 知识库 DO
  17 + *
  18 + * @author xiaoxin
  19 + */
  20 +@TableName(value = "ai_knowledge_base")
  21 +@Data
  22 +public class AiKnowledgeBaseDO extends BaseDO {
  23 +
  24 + /**
  25 + * 编号
  26 + */
  27 + @TableId(type = IdType.AUTO)
  28 + private Long id;
  29 + /**
  30 + * 用户编号
  31 + * <p>
  32 + * 关联 AdminUserDO 的 userId 字段
  33 + */
  34 + private Long userId;
  35 + /**
  36 + * 知识库名称
  37 + */
  38 + private String name;
  39 + /**
  40 + * 知识库描述
  41 + */
  42 + private String description;
  43 + /**
  44 + * 可见权限,只能选择哪些人可见
  45 + */
  46 + @TableField(typeHandler = JacksonTypeHandler.class)
  47 + private List<Long> visibilityPermissions;
  48 + /**
  49 + * 嵌入模型编号，高质量模式时维护
  50 + */
  51 + private Long modelId;
  52 + /**
  53 + * 模型标识
  54 + */
  55 + private String model;
  56 + /**
  57 + * 状态
  58 + * <p>
  59 + * 枚举 {@link CommonStatusEnum}
  60 + */
  61 + private Integer status;
  62 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
  2 +
  3 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
  4 +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
  5 +import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum;
  6 +import com.baomidou.mybatisplus.annotation.IdType;
  7 +import com.baomidou.mybatisplus.annotation.TableId;
  8 +import com.baomidou.mybatisplus.annotation.TableName;
  9 +import lombok.Data;
  10 +
  11 +/**
  12 + * AI 知识库-文档 DO
  13 + *
  14 + * @author xiaoxin
  15 + */
  16 +@TableName(value = "ai_knowledge_document")
  17 +@Data
  18 +public class AiKnowledgeDocumentDO extends BaseDO {
  19 +
  20 + /**
  21 + * 编号
  22 + */
  23 + @TableId(type = IdType.AUTO)
  24 + private Long id;
  25 + /**
  26 + * 知识库编号
  27 + */
  28 + private Long knowledgeId;
  29 + /**
  30 + * 文件名称
  31 + */
  32 + private String name;
  33 + /**
  34 + * 内容
  35 + */
  36 + private String content;
  37 + /**
  38 + * 文件 URL
  39 + */
  40 + private String url;
  41 + /**
  42 + * token数量
  43 + */
  44 + private Integer tokens;
  45 + /**
  46 + * 字符数
  47 + */
  48 + private Integer wordCount;
  49 + /**
  50 + * 切片状态
  51 + * <p>
  52 + * 枚举 {@link AiKnowledgeDocumentStatusEnum}
  53 + */
  54 + private Integer sliceStatus;
  55 +
  56 + /**
  57 + * 状态
  58 + * <p>
  59 + * 枚举 {@link CommonStatusEnum}
  60 + */
  61 + private Integer status;
  62 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
  2 +
  3 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
  4 +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
  5 +import com.baomidou.mybatisplus.annotation.IdType;
  6 +import com.baomidou.mybatisplus.annotation.TableId;
  7 +import com.baomidou.mybatisplus.annotation.TableName;
  8 +import lombok.Data;
  9 +
  10 +/**
  11 + * AI 知识库-文档分段 DO
  12 + *
  13 + * @author xiaoxin
  14 + */
  15 +@TableName(value = "ai_knowledge_segment")
  16 +@Data
  17 +public class AiKnowledgeSegmentDO extends BaseDO {
  18 +
  19 + /**
  20 + * 编号
  21 + */
  22 + @TableId(type = IdType.AUTO)
  23 + private Long id;
  24 + /**
  25 + * 向量库的id
  26 + */
  27 + private String vectorId;
  28 + /**
  29 + * 文档编号
  30 + */
  31 + private Long documentId;
  32 + /**
  33 + * 切片内容
  34 + */
  35 + private String content;
  36 + /**
  37 + * 字符数
  38 + */
  39 + private Integer wordCount;
  40 + /**
  41 + * token数量
  42 + */
  43 + private Integer tokens;
  44 + /**
  45 + * 状态
  46 + * <p>
  47 + * 枚举 {@link CommonStatusEnum}
  48 + */
  49 + private Integer status;
  50 +
  51 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
  2 +
  3 +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
  4 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO;
  5 +import org.apache.ibatis.annotations.Mapper;
  6 +
  7 +/**
  8 + * AI 知识库基础信息 Mapper
  9 + *
  10 + * @author xiaoxin
  11 + */
  12 +@Mapper
  13 +public interface AiKnowledgeBaseMapper extends BaseMapperX<AiKnowledgeBaseDO> {
  14 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
  2 +
  3 +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
  4 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
  5 +import org.apache.ibatis.annotations.Mapper;
  6 +
  7 +/**
  8 + * AI 知识库-文档 Mapper
  9 + *
  10 + * @author xiaoxin
  11 + */
  12 +@Mapper
  13 +public interface AiKnowledgeDocumentMapper extends BaseMapperX<AiKnowledgeDocumentDO> {
  14 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
  2 +
  3 +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
  4 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
  5 +import org.apache.ibatis.annotations.Mapper;
  6 +
  7 +/**
  8 + * AI 知识库-分片 Mapper
  9 + *
  10 + * @author xiaoxin
  11 + */
  12 +@Mapper
  13 +public interface AiKnowledgeSegmentMapper extends BaseMapperX<AiKnowledgeSegmentDO> {
  14 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.service.knowledge;
  2 +
  3 +import org.springframework.ai.document.Document;
  4 +import org.springframework.ai.vectorstore.SearchRequest;
  5 +
  6 +import java.util.List;
  7 +
  8 +/**
  9 + * AI 嵌入 Service 接口
  10 + *
  11 + * @author xiaoxin
  12 + */
  13 +public interface AiEmbeddingService {
  14 +
  15 + /**
  16 + * 向量化文档并存储
  17 + */
  18 + void add(List<Document> documents);
  19 +
  20 +
  21 + /**
  22 + * 相似查询
  23 + *
  24 + * @param request 查询实体
  25 + */
  26 + List<Document> similaritySearch(SearchRequest request);
  27 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.service.knowledge;
  2 +
  3 +import jakarta.annotation.Resource;
  4 +import org.springframework.ai.document.Document;
  5 +import org.springframework.ai.vectorstore.RedisVectorStore;
  6 +import org.springframework.ai.vectorstore.SearchRequest;
  7 +import org.springframework.stereotype.Service;
  8 +
  9 +import java.util.List;
  10 +
  11 +/**
  12 + * AI 嵌入 Service 实现类
  13 + *
  14 + * @author xiaoxin
  15 + */
  16 +@Service
  17 +public class AiEmbeddingServiceImpl implements AiEmbeddingService {
  18 +
  19 + @Resource
  20 + private RedisVectorStore vectorStore;
  21 +
  22 + @Override
  23 +// @Async
  24 + // TODO xiaoxin 报错先注释
  25 + public void add(List<Document> documents) {
  26 + vectorStore.add(documents);
  27 + }
  28 +
  29 + @Override
  30 + public List<Document> similaritySearch(SearchRequest request) {
  31 + return vectorStore.similaritySearch(request);
  32 + }
  33 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.service.knowledge;
  2 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO;
  3 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO;
  4 +
  5 +/**
  6 + * AI 知识库-基础信息 Service 接口
  7 + *
  8 + * @author xiaoxin
  9 + */
  10 +public interface AiKnowledgeBaseService {
  11 +
  12 +
  13 + /**
  14 + * 创建【我的】知识库
  15 + *
  16 + * @param createReqVO 创建信息
  17 + * @param userId 用户编号
  18 + * @return 编号
  19 + */
  20 + Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId);
  21 +
  22 +
  23 + /**
  24 + * 创建【我的】知识库
  25 + *
  26 + * @param updateReqVO 更新信息
  27 + * @param userId 用户编号
  28 + */
  29 + void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId);
  30 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.service.knowledge;
  2 +
  3 +import cn.hutool.core.lang.Assert;
  4 +import cn.hutool.core.util.ObjUtil;
  5 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
  6 +import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
  7 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO;
  8 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO;
  9 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO;
  10 +import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
  11 +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeBaseMapper;
  12 +import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;
  13 +import jakarta.annotation.Resource;
  14 +import lombok.extern.slf4j.Slf4j;
  15 +import org.springframework.stereotype.Service;
  16 +
  17 +import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
  18 +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_EXISTS;
  19 +
  20 +/**
  21 + * AI 知识库-基础信息 Service 实现类
  22 + *
  23 + * @author xiaoxin
  24 + */
  25 +@Service
  26 +@Slf4j
  27 +public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService {
  28 +
  29 + @Resource
  30 + private AiChatModelService chatModalService;
  31 +
  32 + @Resource
  33 + private AiKnowledgeBaseMapper knowledgeBaseMapper;
  34 +
  35 +
  36 + @Override
  37 + public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) {
  38 + AiChatModelDO model = validateChatModel(createReqVO.getModelId());
  39 +
  40 + AiKnowledgeBaseDO knowledgeBaseDO = BeanUtils.toBean(createReqVO, AiKnowledgeBaseDO.class);
  41 + knowledgeBaseDO.setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus());
  42 +
  43 + knowledgeBaseMapper.insert(knowledgeBaseDO);
  44 + return knowledgeBaseDO.getId();
  45 + }
  46 +
  47 + @Override
  48 + public void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId) {
  49 +
  50 + AiKnowledgeBaseDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId());
  51 + if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) {
  52 + throw exception(KNOWLEDGE_NOT_EXISTS);
  53 + }
  54 + AiChatModelDO model = validateChatModel(updateReqVO.getModelId());
  55 + AiKnowledgeBaseDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeBaseDO.class);
  56 + updateDO.setModel(model.getModel());
  57 +
  58 + knowledgeBaseMapper.updateById(updateDO);
  59 + }
  60 +
  61 +
  62 + private AiChatModelDO validateChatModel(Long id) {
  63 + AiChatModelDO model = chatModalService.validateChatModel(id);
  64 + Assert.notNull(model, "未找到对应嵌入模型");
  65 + return model;
  66 + }
  67 +
  68 + public AiKnowledgeBaseDO validateKnowledgeExists(Long id) {
  69 + AiKnowledgeBaseDO knowledgeBase = knowledgeBaseMapper.selectById(id);
  70 + if (knowledgeBase == null) {
  71 + throw exception(KNOWLEDGE_NOT_EXISTS);
  72 + }
  73 + return knowledgeBase;
  74 + }
  75 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.service.knowledge;
  2 +
  3 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeDocumentCreateReqVO;
  4 +
  5 +/**
  6 + * AI 知识库-文档 Service 接口
  7 + *
  8 + * @author xiaoxin
  9 + */
  10 +public interface AiKnowledgeDocumentService {
  11 +
  12 +
  13 + /**
  14 + * 创建文档
  15 + *
  16 + * @param createReqVO 文档创建 Request VO
  17 + * @return 文档编号
  18 + */
  19 + Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO);
  20 +
  21 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.service.knowledge;
  2 +
  3 +import cn.hutool.core.collection.CollUtil;
  4 +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
  5 +import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
  6 +import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
  7 +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeDocumentCreateReqVO;
  8 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
  9 +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
  10 +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper;
  11 +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper;
  12 +import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum;
  13 +import jakarta.annotation.Resource;
  14 +import lombok.extern.slf4j.Slf4j;
  15 +import org.springframework.ai.document.Document;
  16 +import org.springframework.ai.reader.tika.TikaDocumentReader;
  17 +import org.springframework.ai.tokenizer.JTokkitTokenCountEstimator;
  18 +import org.springframework.ai.transformer.splitter.TokenTextSplitter;
  19 +import org.springframework.beans.factory.annotation.Value;
  20 +import org.springframework.stereotype.Service;
  21 +import org.springframework.transaction.annotation.Transactional;
  22 +
  23 +import java.util.List;
  24 +import java.util.Objects;
  25 +
  26 +/**
  27 + * AI 知识库-文档 Service 实现类
  28 + *
  29 + * @author xiaoxin
  30 + */
  31 +@Service
  32 +@Slf4j
  33 +public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentService {
  34 +
  35 + @Resource
  36 + private AiKnowledgeDocumentMapper documentMapper;
  37 + @Resource
  38 + private AiKnowledgeSegmentMapper segmentMapper;
  39 +
  40 + @Resource
  41 + private TokenTextSplitter tokenTextSplitter;
  42 +
  43 + @Resource
  44 + private AiEmbeddingService embeddingService;
  45 +
  46 + private static final JTokkitTokenCountEstimator TOKEN_COUNT_ESTIMATOR = new JTokkitTokenCountEstimator();
  47 +
  48 + // TODO xiaoxin 临时测试用，后续删
  49 + @Value("classpath:/webapp/test/Fel.pdf")
  50 + private org.springframework.core.io.Resource data;
  51 +
  52 +
  53 + @Override
  54 + @Transactional(rollbackFor = Exception.class)
  55 + public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) {
  56 +
  57 + // TODO xiaoxin 后续从 url 加载
  58 + TikaDocumentReader loader = new TikaDocumentReader(data);
  59 + // 加载文档
  60 + List<Document> documents = loader.get();
  61 + Document document = CollUtil.getFirst(documents);
  62 + // TODO 芋艿 文档层面有没有可能会比较大，这两个字段是否可以从分段表计算得出？
  63 + Integer tokens = Objects.nonNull(document) ? TOKEN_COUNT_ESTIMATOR.estimate(document.getContent()) : 0;
  64 + Integer wordCount = Objects.nonNull(document) ? document.getContent().length() : 0;
  65 +
  66 + AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class);
  67 + documentDO.setTokens(tokens).setWordCount(wordCount)
  68 + .setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus());
  69 + // 文档记录入库
  70 + documentMapper.insert(documentDO);
  71 + Long documentId = documentDO.getId();
  72 + if (CollUtil.isEmpty(documents)) {
  73 + return documentId;
  74 + }
  75 +
  76 + // 文档分段
  77 + List<Document> segments = tokenTextSplitter.apply(documents);
  78 +
  79 + List<AiKnowledgeSegmentDO> segmentDOList = CollectionUtils.convertList(segments,
  80 + segment -> new AiKnowledgeSegmentDO().setContent(segment.getContent()).setDocumentId(documentId)
  81 + .setTokens(TOKEN_COUNT_ESTIMATOR.estimate(segment.getContent())).setWordCount(segment.getContent().length())
  82 + .setStatus(CommonStatusEnum.ENABLE.getStatus()));
  83 + // 分段内容入库
  84 + segmentMapper.insertBatch(segmentDOList);
  85 +
  86 + //向量化并存储
  87 + embeddingService.add(segments);
  88 +
  89 + return documentId;
  90 + }
  91 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java renamed to yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java
  View file @0db165f
1 package cn.iocoder.yudao.module.ai.service.knowledge; 1 package cn.iocoder.yudao.module.ai.service.knowledge;
2 2
3 /** 3 /**
4 - * AI 知识库 Service 接口 4 + * AI 知识库-分片 Service 接口
5 * 5 *
6 * @author xiaoxin 6 * @author xiaoxin
7 */ 7 */
8 -public interface DocService { 8 +public interface AiKnowledgeSegmentService {
9 9
10 - /**  
11 - * 向量化文档  
12 - */  
13 - void embeddingDoc();  
14 10
15 } 11 }
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java 0 → 100644
  View file @0db165f
  1 +package cn.iocoder.yudao.module.ai.service.knowledge;
  2 +
  3 +import lombok.extern.slf4j.Slf4j;
  4 +import org.springframework.stereotype.Service;
  5 +
  6 +/**
  7 + * AI 知识库-基础信息 Service 实现类
  8 + *
  9 + * @author xiaoxin
  10 + */
  11 +@Service
  12 +@Slf4j
  13 +public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService {
  14 +
  15 +
  16 +}
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java deleted
View file @83d86bc
1 -package cn.iocoder.yudao.module.ai.service.knowledge;  
2 -  
3 -import jakarta.annotation.Resource;  
4 -import lombok.extern.slf4j.Slf4j;  
5 -import org.springframework.ai.document.Document;  
6 -import org.springframework.ai.reader.tika.TikaDocumentReader;  
7 -import org.springframework.ai.transformer.splitter.TokenTextSplitter;  
8 -import org.springframework.ai.vectorstore.RedisVectorStore;  
9 -import org.springframework.beans.factory.annotation.Value;  
10 -  
11 -import java.util.List;  
12 -  
13 -/**  
14 - * AI 知识库 Service 实现类  
15 - *  
16 - * @author xiaoxin  
17 - */  
18 -//@Service // TODO 芋艿：临时注释，避免无法启动  
19 -@Slf4j  
20 -public class DocServiceImpl implements DocService {  
21 -  
22 - @Resource  
23 - private RedisVectorStore vectorStore;  
24 - @Resource  
25 - private TokenTextSplitter tokenTextSplitter;  
26 -  
27 - // TODO @xin 临时测试用，后续删  
28 - @Value("classpath:/webapp/test/Fel.pdf")  
29 - private org.springframework.core.io.Resource data;  
30 -  
31 - @Override  
32 - public void embeddingDoc() {  
33 - // 读取文件  
34 - TikaDocumentReader loader = new TikaDocumentReader(data);  
35 - List<Document> documents = loader.get();  
36 - // 文档分段  
37 - List<Document> segments = tokenTextSplitter.apply(documents);  
38 - // 向量化并存储  
39 - vectorStore.add(segments);  
40 - }  
41 -  
42 -}  
yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml
  View file @0db165f
@@ -61,11 +61,9 @@ @@ -61,11 +61,9 @@
61 <version>${spring-ai.version}</version> 61 <version>${spring-ai.version}</version>
62 </dependency> 62 </dependency>
63 63
64 - <!-- TODO @xin：引入我们项目的 starter -->  
65 <dependency> 64 <dependency>
66 - <groupId>org.springframework.data</groupId>  
67 - <artifactId>spring-data-redis</artifactId>  
68 - <optional>true</optional> 65 + <groupId>cn.iocoder.boot</groupId>
  66 + <artifactId>yudao-spring-boot-starter-redis</artifactId>
69 </dependency> 67 </dependency>
70 68
71 <dependency> 69 <dependency>
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java
  View file @0db165f
@@ -31,6 +31,7 @@ import redis.clients.jedis.JedisPooled; @@ -31,6 +31,7 @@ import redis.clients.jedis.JedisPooled;
31 * TODO @xin 先拿 spring-ai 最新代码覆盖，1.0.0-M1 跟 redis 自动配置会冲突 31 * TODO @xin 先拿 spring-ai 最新代码覆盖，1.0.0-M1 跟 redis 自动配置会冲突
32 * 32 *
33 * TODO 这个官方，有说啥时候 fix 哇？ 33 * TODO 这个官方，有说啥时候 fix 哇？
  34 + * TODO 看着是列在1.0.0-M2版本
34 * 35 *
35 * @author Christian Tzolov 36 * @author Christian Tzolov
36 * @author Eddú Meléndez 37 * @author Eddú Meléndez