Commit 0a8c75625a2501a21efcbea14a79784a5b03a10f
1 parent
deca69ad
【代码重构】AI:知识库相关的表结构
Showing
31 changed files
with
356 additions
and
365 deletions
yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java
| @@ -21,7 +21,7 @@ public interface ErrorCodeConstants { | @@ -21,7 +21,7 @@ public interface ErrorCodeConstants { | ||
| 21 | ErrorCode CHAT_MODEL_DISABLE = new ErrorCode(1_040_001_001, "模型({})已禁用!"); | 21 | ErrorCode CHAT_MODEL_DISABLE = new ErrorCode(1_040_001_001, "模型({})已禁用!"); |
| 22 | ErrorCode CHAT_MODEL_DEFAULT_NOT_EXISTS = new ErrorCode(1_040_001_002, "操作失败,找不到默认聊天模型"); | 22 | ErrorCode CHAT_MODEL_DEFAULT_NOT_EXISTS = new ErrorCode(1_040_001_002, "操作失败,找不到默认聊天模型"); |
| 23 | 23 | ||
| 24 | - // ========== API 聊天模型 1-040-002-000 ========== | 24 | + // ========== API 聊天角色 1-040-002-000 ========== |
| 25 | ErrorCode CHAT_ROLE_NOT_EXISTS = new ErrorCode(1_040_002_000, "聊天角色不存在"); | 25 | ErrorCode CHAT_ROLE_NOT_EXISTS = new ErrorCode(1_040_002_000, "聊天角色不存在"); |
| 26 | ErrorCode CHAT_ROLE_DISABLE = new ErrorCode(1_040_001_001, "聊天角色({})已禁用!"); | 26 | ErrorCode CHAT_ROLE_DISABLE = new ErrorCode(1_040_001_001, "聊天角色({})已禁用!"); |
| 27 | 27 | ||
| @@ -40,7 +40,6 @@ public interface ErrorCodeConstants { | @@ -40,7 +40,6 @@ public interface ErrorCodeConstants { | ||
| 40 | ErrorCode IMAGE_NOT_EXISTS = new ErrorCode(1_022_005_000, "图片不存在!"); | 40 | ErrorCode IMAGE_NOT_EXISTS = new ErrorCode(1_022_005_000, "图片不存在!"); |
| 41 | ErrorCode IMAGE_MIDJOURNEY_SUBMIT_FAIL = new ErrorCode(1_022_005_001, "Midjourney 提交失败!原因:{}"); | 41 | ErrorCode IMAGE_MIDJOURNEY_SUBMIT_FAIL = new ErrorCode(1_022_005_001, "Midjourney 提交失败!原因:{}"); |
| 42 | ErrorCode IMAGE_CUSTOM_ID_NOT_EXISTS = new ErrorCode(1_022_005_002, "Midjourney 按钮 customId 不存在! {}"); | 42 | ErrorCode IMAGE_CUSTOM_ID_NOT_EXISTS = new ErrorCode(1_022_005_002, "Midjourney 按钮 customId 不存在! {}"); |
| 43 | - ErrorCode IMAGE_FAIL = new ErrorCode(1_022_005_002, "图片绘画失败! {}"); | ||
| 44 | 43 | ||
| 45 | // ========== API 音乐 1-040-006-000 ========== | 44 | // ========== API 音乐 1-040-006-000 ========== |
| 46 | ErrorCode MUSIC_NOT_EXISTS = new ErrorCode(1_022_006_000, "音乐不存在!"); | 45 | ErrorCode MUSIC_NOT_EXISTS = new ErrorCode(1_022_006_000, "音乐不存在!"); |
| @@ -54,7 +53,11 @@ public interface ErrorCodeConstants { | @@ -54,7 +53,11 @@ public interface ErrorCodeConstants { | ||
| 54 | 53 | ||
| 55 | // ========== API 知识库 1-022-008-000 ========== | 54 | // ========== API 知识库 1-022-008-000 ========== |
| 56 | ErrorCode KNOWLEDGE_NOT_EXISTS = new ErrorCode(1_022_008_000, "知识库不存在!"); | 55 | ErrorCode KNOWLEDGE_NOT_EXISTS = new ErrorCode(1_022_008_000, "知识库不存在!"); |
| 57 | - ErrorCode KNOWLEDGE_DOCUMENT_NOT_EXISTS = new ErrorCode(1_022_008_001, "文档不存在!"); | ||
| 58 | - ErrorCode KNOWLEDGE_SEGMENT_NOT_EXISTS = new ErrorCode(1_022_008_002, "段落不存在!"); | 56 | + |
| 57 | + ErrorCode KNOWLEDGE_DOCUMENT_NOT_EXISTS = new ErrorCode(1_022_008_101, "文档不存在!"); | ||
| 58 | + ErrorCode KNOWLEDGE_DOCUMENT_FILE_EMPTY = new ErrorCode(1_022_008_102, "文档内容为空!"); | ||
| 59 | + ErrorCode KNOWLEDGE_DOCUMENT_FILE_READ_FAIL = new ErrorCode(1_022_008_102, "文档加载失败!"); | ||
| 60 | + | ||
| 61 | + ErrorCode KNOWLEDGE_SEGMENT_NOT_EXISTS = new ErrorCode(1_022_008_202, "段落不存在!"); | ||
| 59 | 62 | ||
| 60 | } | 63 | } |
yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java deleted
| 1 | -package cn.iocoder.yudao.module.ai.enums.knowledge; | ||
| 2 | - | ||
| 3 | -import cn.iocoder.yudao.framework.common.core.ArrayValuable; | ||
| 4 | -import lombok.AllArgsConstructor; | ||
| 5 | -import lombok.Getter; | ||
| 6 | - | ||
| 7 | -import java.util.Arrays; | ||
| 8 | - | ||
| 9 | -/** | ||
| 10 | - * AI 知识库-文档状态的枚举 | ||
| 11 | - * | ||
| 12 | - * @author xiaoxin | ||
| 13 | - */ | ||
| 14 | -@AllArgsConstructor | ||
| 15 | -@Getter | ||
| 16 | -public enum AiKnowledgeDocumentStatusEnum implements ArrayValuable<Integer> { | ||
| 17 | - | ||
| 18 | - IN_PROGRESS(10, "索引中"), | ||
| 19 | - SUCCESS(20, "可用"), | ||
| 20 | - FAIL(30, "失败"); | ||
| 21 | - | ||
| 22 | - /** | ||
| 23 | - * 状态 | ||
| 24 | - */ | ||
| 25 | - private final Integer status; | ||
| 26 | - | ||
| 27 | - /** | ||
| 28 | - * 状态名 | ||
| 29 | - */ | ||
| 30 | - private final String name; | ||
| 31 | - | ||
| 32 | - public static final Integer[] ARRAYS = Arrays.stream(values()).map(AiKnowledgeDocumentStatusEnum::getStatus).toArray(Integer[]::new); | ||
| 33 | - | ||
| 34 | - @Override | ||
| 35 | - public Integer[] array() { | ||
| 36 | - return ARRAYS; | ||
| 37 | - } | ||
| 38 | - | ||
| 39 | -} |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.http
0 → 100644
| 1 | +### 创建知识库 | ||
| 2 | +POST {{baseUrl}}/ai/knowledge/create | ||
| 3 | +Content-Type: application/json | ||
| 4 | +Authorization: {{token}} | ||
| 5 | +tenant-id: {{adminTenantId}} | ||
| 6 | + | ||
| 7 | +{ | ||
| 8 | + "name": "测试标题", | ||
| 9 | + "description": "测试描述", | ||
| 10 | + "embeddingModelId": 30, | ||
| 11 | + "topK": 3, | ||
| 12 | + "similarityThreshold": 0.5 | ||
| 13 | +} | ||
| 14 | + | ||
| 15 | +### 更新知识库 | ||
| 16 | +PUT {{baseUrl}}/ai/knowledge/update | ||
| 17 | +Content-Type: application/json | ||
| 18 | +Authorization: {{token}} | ||
| 19 | +tenant-id: {{adminTenantId}} | ||
| 20 | + | ||
| 21 | +{ | ||
| 22 | + "id": 1, | ||
| 23 | + "name": "测试标题(更新)", | ||
| 24 | + "description": "测试描述", | ||
| 25 | + "embeddingModelId": 30, | ||
| 26 | + "topK": 5, | ||
| 27 | + "similarityThreshold": 0.6 | ||
| 28 | +} | ||
| 29 | + | ||
| 30 | +### 获取知识库分页 | ||
| 31 | +GET {{baseUrl}}/ai/knowledge/page?pageNo=1&pageSize=10 | ||
| 32 | +Authorization: {{token}} | ||
| 33 | +tenant-id: {{adminTenantId}} | ||
| 0 | \ No newline at end of file | 34 | \ No newline at end of file |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java
| @@ -3,10 +3,9 @@ package cn.iocoder.yudao.module.ai.controller.admin.knowledge; | @@ -3,10 +3,9 @@ package cn.iocoder.yudao.module.ai.controller.admin.knowledge; | ||
| 3 | import cn.iocoder.yudao.framework.common.pojo.CommonResult; | 3 | import cn.iocoder.yudao.framework.common.pojo.CommonResult; |
| 4 | import cn.iocoder.yudao.framework.common.pojo.PageResult; | 4 | import cn.iocoder.yudao.framework.common.pojo.PageResult; |
| 5 | import cn.iocoder.yudao.framework.common.util.object.BeanUtils; | 5 | import cn.iocoder.yudao.framework.common.util.object.BeanUtils; |
| 6 | -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateReqVO; | ||
| 7 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO; | 6 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO; |
| 8 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeRespVO; | 7 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeRespVO; |
| 9 | -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateReqVO; | 8 | +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeSaveReqVO; |
| 10 | import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; | 9 | import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; |
| 11 | import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeService; | 10 | import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeService; |
| 12 | import io.swagger.v3.oas.annotations.Operation; | 11 | import io.swagger.v3.oas.annotations.Operation; |
| @@ -17,7 +16,6 @@ import org.springframework.validation.annotation.Validated; | @@ -17,7 +16,6 @@ import org.springframework.validation.annotation.Validated; | ||
| 17 | import org.springframework.web.bind.annotation.*; | 16 | import org.springframework.web.bind.annotation.*; |
| 18 | 17 | ||
| 19 | import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success; | 18 | import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success; |
| 20 | -import static cn.iocoder.yudao.framework.security.core.util.SecurityFrameworkUtils.getLoginUserId; | ||
| 21 | 19 | ||
| 22 | @Tag(name = "管理后台 - AI 知识库") | 20 | @Tag(name = "管理后台 - AI 知识库") |
| 23 | @RestController | 21 | @RestController |
| @@ -31,20 +29,22 @@ public class AiKnowledgeController { | @@ -31,20 +29,22 @@ public class AiKnowledgeController { | ||
| 31 | @GetMapping("/page") | 29 | @GetMapping("/page") |
| 32 | @Operation(summary = "获取知识库分页") | 30 | @Operation(summary = "获取知识库分页") |
| 33 | public CommonResult<PageResult<AiKnowledgeRespVO>> getKnowledgePage(@Valid AiKnowledgePageReqVO pageReqVO) { | 31 | public CommonResult<PageResult<AiKnowledgeRespVO>> getKnowledgePage(@Valid AiKnowledgePageReqVO pageReqVO) { |
| 34 | - PageResult<AiKnowledgeDO> pageResult = knowledgeService.getKnowledgePage(getLoginUserId(), pageReqVO); | 32 | + |
| 33 | + PageResult<AiKnowledgeDO> pageResult = knowledgeService.getKnowledgePage(pageReqVO); | ||
| 35 | return success(BeanUtils.toBean(pageResult, AiKnowledgeRespVO.class)); | 34 | return success(BeanUtils.toBean(pageResult, AiKnowledgeRespVO.class)); |
| 36 | } | 35 | } |
| 37 | 36 | ||
| 38 | @PostMapping("/create") | 37 | @PostMapping("/create") |
| 39 | @Operation(summary = "创建知识库") | 38 | @Operation(summary = "创建知识库") |
| 40 | - public CommonResult<Long> createKnowledge(@RequestBody @Valid AiKnowledgeCreateReqVO createReqVO) { | ||
| 41 | - return success(knowledgeService.createKnowledge(createReqVO, getLoginUserId())); | 39 | + public CommonResult<Long> createKnowledge(@RequestBody @Valid AiKnowledgeSaveReqVO createReqVO) { |
| 40 | + return success(knowledgeService.createKnowledge(createReqVO)); | ||
| 42 | } | 41 | } |
| 43 | 42 | ||
| 44 | @PutMapping("/update") | 43 | @PutMapping("/update") |
| 45 | @Operation(summary = "更新知识库") | 44 | @Operation(summary = "更新知识库") |
| 46 | - public CommonResult<Boolean> updateKnowledge(@RequestBody @Valid AiKnowledgeUpdateReqVO updateReqVO) { | ||
| 47 | - knowledgeService.updateKnowledge(updateReqVO, getLoginUserId()); | 45 | + public CommonResult<Boolean> updateKnowledge(@RequestBody @Valid AiKnowledgeSaveReqVO updateReqVO) { |
| 46 | + knowledgeService.updateKnowledge(updateReqVO); | ||
| 48 | return success(true); | 47 | return success(true); |
| 49 | } | 48 | } |
| 49 | + | ||
| 50 | } | 50 | } |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeDocumentController.http
0 → 100644
| 1 | +### 创建知识文档 | ||
| 2 | +POST {{baseUrl}}/ai/knowledge/document/create | ||
| 3 | +Content-Type: application/json | ||
| 4 | +Authorization: Bearer {{token}} | ||
| 5 | +tenant-id: {{adminTenantId}} | ||
| 6 | + | ||
| 7 | +{ | ||
| 8 | + "knowledgeId": 1, | ||
| 9 | + "name": "测试文档", | ||
| 10 | + "url": "https://static.iocoder.cn/README.md", | ||
| 11 | + "segmentMaxTokens": 800 | ||
| 12 | +} | ||
| 0 | \ No newline at end of file | 13 | \ No newline at end of file |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeDocumentController.java
| @@ -27,20 +27,21 @@ public class AiKnowledgeDocumentController { | @@ -27,20 +27,21 @@ public class AiKnowledgeDocumentController { | ||
| 27 | @Resource | 27 | @Resource |
| 28 | private AiKnowledgeDocumentService documentService; | 28 | private AiKnowledgeDocumentService documentService; |
| 29 | 29 | ||
| 30 | - @PostMapping("/create") | ||
| 31 | - @Operation(summary = "新建文档") | ||
| 32 | - public CommonResult<Long> createKnowledgeDocument(@Valid AiKnowledgeDocumentCreateReqVO reqVO) { | ||
| 33 | - Long knowledgeDocumentId = documentService.createKnowledgeDocument(reqVO); | ||
| 34 | - return success(knowledgeDocumentId); | ||
| 35 | - } | ||
| 36 | - | ||
| 37 | @GetMapping("/page") | 30 | @GetMapping("/page") |
| 38 | @Operation(summary = "获取文档分页") | 31 | @Operation(summary = "获取文档分页") |
| 39 | - public CommonResult<PageResult<AiKnowledgeDocumentRespVO>> getKnowledgeDocumentPage(@Valid AiKnowledgeDocumentPageReqVO pageReqVO) { | 32 | + public CommonResult<PageResult<AiKnowledgeDocumentRespVO>> getKnowledgeDocumentPage( |
| 33 | + @Valid AiKnowledgeDocumentPageReqVO pageReqVO) { | ||
| 40 | PageResult<AiKnowledgeDocumentDO> pageResult = documentService.getKnowledgeDocumentPage(pageReqVO); | 34 | PageResult<AiKnowledgeDocumentDO> pageResult = documentService.getKnowledgeDocumentPage(pageReqVO); |
| 41 | return success(BeanUtils.toBean(pageResult, AiKnowledgeDocumentRespVO.class)); | 35 | return success(BeanUtils.toBean(pageResult, AiKnowledgeDocumentRespVO.class)); |
| 42 | } | 36 | } |
| 43 | 37 | ||
| 38 | + @PostMapping("/create") | ||
| 39 | + @Operation(summary = "新建文档") | ||
| 40 | + public CommonResult<Long> createKnowledgeDocument(@RequestBody @Valid AiKnowledgeDocumentCreateReqVO reqVO) { | ||
| 41 | + Long knowledgeDocumentId = documentService.createKnowledgeDocument(reqVO); | ||
| 42 | + return success(knowledgeDocumentId); | ||
| 43 | + } | ||
| 44 | + | ||
| 44 | @PutMapping("/update") | 45 | @PutMapping("/update") |
| 45 | @Operation(summary = "更新文档") | 46 | @Operation(summary = "更新文档") |
| 46 | public CommonResult<Boolean> updateKnowledgeDocument(@Valid @RequestBody AiKnowledgeDocumentUpdateReqVO reqVO) { | 47 | public CommonResult<Boolean> updateKnowledgeDocument(@Valid @RequestBody AiKnowledgeDocumentUpdateReqVO reqVO) { |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentUpdateReqVO.java
| @@ -11,16 +11,15 @@ import lombok.Data; | @@ -11,16 +11,15 @@ import lombok.Data; | ||
| 11 | @Data | 11 | @Data |
| 12 | public class AiKnowledgeDocumentUpdateReqVO { | 12 | public class AiKnowledgeDocumentUpdateReqVO { |
| 13 | 13 | ||
| 14 | - | ||
| 15 | @Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "15583") | 14 | @Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "15583") |
| 16 | @NotNull(message = "编号不能为空") | 15 | @NotNull(message = "编号不能为空") |
| 17 | private Long id; | 16 | private Long id; |
| 18 | 17 | ||
| 18 | + @Schema(description = "名称", example = "Java 开发手册") | ||
| 19 | + private String name; | ||
| 20 | + | ||
| 19 | @Schema(description = "是否启用", example = "1") | 21 | @Schema(description = "是否启用", example = "1") |
| 20 | @InEnum(CommonStatusEnum.class) | 22 | @InEnum(CommonStatusEnum.class) |
| 21 | private Integer status; | 23 | private Integer status; |
| 22 | 24 | ||
| 23 | - @Schema(description = "名称", example = "Java 开发手册") | ||
| 24 | - private String name; | ||
| 25 | - | ||
| 26 | } | 25 | } |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeDocumentCreateReqVO.java
| @@ -23,24 +23,8 @@ public class AiKnowledgeDocumentCreateReqVO { | @@ -23,24 +23,8 @@ public class AiKnowledgeDocumentCreateReqVO { | ||
| 23 | @URL(message = "文档 URL 格式不正确") | 23 | @URL(message = "文档 URL 格式不正确") |
| 24 | private String url; | 24 | private String url; |
| 25 | 25 | ||
| 26 | - @Schema(description = "每个段落的目标 token 数", requiredMode = Schema.RequiredMode.REQUIRED, example = "800") | ||
| 27 | - @NotNull(message = "每个段落的目标 token 数不能为空") | ||
| 28 | - private Integer defaultSegmentTokens; | ||
| 29 | - | ||
| 30 | - @Schema(description = "每个段落的最小字符数", requiredMode = Schema.RequiredMode.REQUIRED, example = "350") | ||
| 31 | - @NotNull(message = "每个段落的最小字符数不能为空") | ||
| 32 | - private Integer minSegmentWordCount; | ||
| 33 | - | ||
| 34 | - @Schema(description = "丢弃阈值:低于此阈值的段落会被丢弃", requiredMode = Schema.RequiredMode.REQUIRED, example = "5") | ||
| 35 | - @NotNull(message = "丢弃阈值不能为空") | ||
| 36 | - private Integer minChunkLengthToEmbed; | ||
| 37 | - | ||
| 38 | - @Schema(description = "最大段落数", requiredMode = Schema.RequiredMode.REQUIRED, example = "10000") | ||
| 39 | - @NotNull(message = "最大段落数不能为空") | ||
| 40 | - private Integer maxNumSegments; | ||
| 41 | - | ||
| 42 | - @Schema(description = "分块是否保留分隔符", requiredMode = Schema.RequiredMode.REQUIRED, example = "true") | ||
| 43 | - @NotNull(message = "分块是否保留分隔符不能为空") | ||
| 44 | - private Boolean keepSeparator; | 26 | + @Schema(description = "分段的最大 Token 数", requiredMode = Schema.RequiredMode.REQUIRED, example = "800") |
| 27 | + @NotNull(message = "分段的最大 Token 数不能为空") | ||
| 28 | + private Integer segmentMaxTokens; | ||
| 45 | 29 | ||
| 46 | } | 30 | } |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeCreateReqVO.java renamed to yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeSaveReqVO.java
| @@ -5,11 +5,12 @@ import jakarta.validation.constraints.NotBlank; | @@ -5,11 +5,12 @@ import jakarta.validation.constraints.NotBlank; | ||
| 5 | import jakarta.validation.constraints.NotNull; | 5 | import jakarta.validation.constraints.NotNull; |
| 6 | import lombok.Data; | 6 | import lombok.Data; |
| 7 | 7 | ||
| 8 | -import java.util.List; | ||
| 9 | - | ||
| 10 | -@Schema(description = "管理后台 - AI 知识库创建 Request VO") | 8 | +@Schema(description = "管理后台 - AI 知识库新增/修改 Request VO") |
| 11 | @Data | 9 | @Data |
| 12 | -public class AiKnowledgeCreateReqVO { | 10 | +public class AiKnowledgeSaveReqVO { |
| 11 | + | ||
| 12 | + @Schema(description = "对话编号", example = "1204") | ||
| 13 | + private Long id; | ||
| 13 | 14 | ||
| 14 | @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南") | 15 | @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南") |
| 15 | @NotBlank(message = "知识库名称不能为空") | 16 | @NotBlank(message = "知识库名称不能为空") |
| @@ -18,19 +19,18 @@ public class AiKnowledgeCreateReqVO { | @@ -18,19 +19,18 @@ public class AiKnowledgeCreateReqVO { | ||
| 18 | @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "存储 ruoyi-vue-pro 操作文档") | 19 | @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "存储 ruoyi-vue-pro 操作文档") |
| 19 | private String description; | 20 | private String description; |
| 20 | 21 | ||
| 21 | - @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1,2,3]") | ||
| 22 | - private List<Long> visibilityPermissions; | 22 | + @Schema(description = "向量模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") |
| 23 | + @NotNull(message = "向量模型不能为空") | ||
| 24 | + private Long embeddingModelId; | ||
| 23 | 25 | ||
| 24 | - @Schema(description = "嵌入模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") | ||
| 25 | - @NotNull(message = "嵌入模型不能为空") | ||
| 26 | - private Long modelId; | 26 | + @Schema(description = "topK", requiredMode = Schema.RequiredMode.REQUIRED, example = "3") |
| 27 | + @NotNull(message = "topK 不能为空") | ||
| 28 | + private Integer topK; | ||
| 27 | 29 | ||
| 28 | @Schema(description = "相似性阈值", requiredMode = Schema.RequiredMode.REQUIRED, example = "0.5") | 30 | @Schema(description = "相似性阈值", requiredMode = Schema.RequiredMode.REQUIRED, example = "0.5") |
| 29 | @NotNull(message = "相似性阈值不能为空") | 31 | @NotNull(message = "相似性阈值不能为空") |
| 30 | private Double similarityThreshold; | 32 | private Double similarityThreshold; |
| 31 | 33 | ||
| 32 | - @Schema(description = "topK", requiredMode = Schema.RequiredMode.REQUIRED, example = "3") | ||
| 33 | - @NotNull(message = "topK 不能为空") | ||
| 34 | - private Integer topK; | 34 | + // TODO @芋艿:status |
| 35 | 35 | ||
| 36 | } | 36 | } |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeUpdateReqVO.java deleted
| 1 | -package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge; | ||
| 2 | - | ||
| 3 | -import io.swagger.v3.oas.annotations.media.Schema; | ||
| 4 | -import jakarta.validation.constraints.NotBlank; | ||
| 5 | -import jakarta.validation.constraints.NotNull; | ||
| 6 | -import lombok.Data; | ||
| 7 | - | ||
| 8 | -import java.util.List; | ||
| 9 | - | ||
| 10 | -@Schema(description = "管理后台 - AI 知识库更新【我的】 Request VO") | ||
| 11 | -@Data | ||
| 12 | -public class AiKnowledgeUpdateReqVO { | ||
| 13 | - | ||
| 14 | - @Schema(description = "对话编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204") | ||
| 15 | - @NotNull(message = "知识库编号不能为空") | ||
| 16 | - private Long id; | ||
| 17 | - | ||
| 18 | - @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "") | ||
| 19 | - @NotBlank(message = "知识库名称不能为空") | ||
| 20 | - private String name; | ||
| 21 | - | ||
| 22 | - @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "") | ||
| 23 | - private String description; | ||
| 24 | - | ||
| 25 | - @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "1,2,3") | ||
| 26 | - private List<Long> visibilityPermissions; | ||
| 27 | - | ||
| 28 | - @Schema(description = "嵌入模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") | ||
| 29 | - @NotNull(message = "嵌入模型不能为空") | ||
| 30 | - private Long modelId; | ||
| 31 | - | ||
| 32 | -} |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/chat/AiChatConversationDO.java
| @@ -80,6 +80,8 @@ public class AiChatConversationDO extends BaseDO { | @@ -80,6 +80,8 @@ public class AiChatConversationDO extends BaseDO { | ||
| 80 | private Long modelId; | 80 | private Long modelId; |
| 81 | /** | 81 | /** |
| 82 | * 模型标志 | 82 | * 模型标志 |
| 83 | + * | ||
| 84 | + * 冗余 {@link AiChatModelDO#getModel()} 字段 | ||
| 83 | */ | 85 | */ |
| 84 | private String model; | 86 | private String model; |
| 85 | 87 |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/chat/AiChatMessageDO.java
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/image/AiImageDO.java
| @@ -52,6 +52,7 @@ public class AiImageDO extends BaseDO { | @@ -52,6 +52,7 @@ public class AiImageDO extends BaseDO { | ||
| 52 | * 枚举 {@link cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum} | 52 | * 枚举 {@link cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum} |
| 53 | */ | 53 | */ |
| 54 | private String platform; | 54 | private String platform; |
| 55 | + // TODO @芋艿:modelId? | ||
| 55 | /** | 56 | /** |
| 56 | * 模型 | 57 | * 模型 |
| 57 | * | 58 | * |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java
| @@ -2,15 +2,12 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; | @@ -2,15 +2,12 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; | ||
| 2 | 2 | ||
| 3 | import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; | 3 | import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; |
| 4 | import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; | 4 | import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; |
| 5 | -import cn.iocoder.yudao.framework.mybatis.core.type.LongListTypeHandler; | 5 | +import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; |
| 6 | import com.baomidou.mybatisplus.annotation.KeySequence; | 6 | import com.baomidou.mybatisplus.annotation.KeySequence; |
| 7 | -import com.baomidou.mybatisplus.annotation.TableField; | ||
| 8 | import com.baomidou.mybatisplus.annotation.TableId; | 7 | import com.baomidou.mybatisplus.annotation.TableId; |
| 9 | import com.baomidou.mybatisplus.annotation.TableName; | 8 | import com.baomidou.mybatisplus.annotation.TableName; |
| 10 | import lombok.Data; | 9 | import lombok.Data; |
| 11 | 10 | ||
| 12 | -import java.util.List; | ||
| 13 | - | ||
| 14 | /** | 11 | /** |
| 15 | * AI 知识库 DO | 12 | * AI 知识库 DO |
| 16 | * | 13 | * |
| @@ -27,12 +24,6 @@ public class AiKnowledgeDO extends BaseDO { | @@ -27,12 +24,6 @@ public class AiKnowledgeDO extends BaseDO { | ||
| 27 | @TableId | 24 | @TableId |
| 28 | private Long id; | 25 | private Long id; |
| 29 | /** | 26 | /** |
| 30 | - * 用户编号 | ||
| 31 | - * <p> | ||
| 32 | - * 关联 AdminUserDO 的 userId 字段 | ||
| 33 | - */ | ||
| 34 | - private Long userId; | ||
| 35 | - /** | ||
| 36 | * 知识库名称 | 27 | * 知识库名称 |
| 37 | */ | 28 | */ |
| 38 | private String name; | 29 | private String name; |
| @@ -42,20 +33,17 @@ public class AiKnowledgeDO extends BaseDO { | @@ -42,20 +33,17 @@ public class AiKnowledgeDO extends BaseDO { | ||
| 42 | private String description; | 33 | private String description; |
| 43 | 34 | ||
| 44 | /** | 35 | /** |
| 45 | - * 可见权限,选择哪些人可见 | ||
| 46 | - * <p> | ||
| 47 | - * -1 所有人可见,其他为各自用户编号 | ||
| 48 | - */ | ||
| 49 | - @TableField(typeHandler = LongListTypeHandler.class) | ||
| 50 | - private List<Long> visibilityPermissions; | ||
| 51 | - /** | ||
| 52 | - * 嵌入模型编号 | 36 | + * 向量模型编号 |
| 37 | + * | ||
| 38 | + * 关联 {@link AiChatModelDO#getId()} | ||
| 53 | */ | 39 | */ |
| 54 | - private Long modelId; | 40 | + private Long embeddingModelId; |
| 55 | /** | 41 | /** |
| 56 | * 模型标识 | 42 | * 模型标识 |
| 43 | + * | ||
| 44 | + * 冗余 {@link AiChatModelDO#getModel()} | ||
| 57 | */ | 45 | */ |
| 58 | - private String model; | 46 | + private String embeddingModel; |
| 59 | 47 | ||
| 60 | /** | 48 | /** |
| 61 | * topK | 49 | * topK |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java
| @@ -2,7 +2,6 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; | @@ -2,7 +2,6 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; | ||
| 2 | 2 | ||
| 3 | import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; | 3 | import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; |
| 4 | import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; | 4 | import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; |
| 5 | -import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum; | ||
| 6 | import com.baomidou.mybatisplus.annotation.KeySequence; | 5 | import com.baomidou.mybatisplus.annotation.KeySequence; |
| 7 | import com.baomidou.mybatisplus.annotation.TableId; | 6 | import com.baomidou.mybatisplus.annotation.TableId; |
| 8 | import com.baomidou.mybatisplus.annotation.TableName; | 7 | import com.baomidou.mybatisplus.annotation.TableName; |
| @@ -34,53 +33,26 @@ public class AiKnowledgeDocumentDO extends BaseDO { | @@ -34,53 +33,26 @@ public class AiKnowledgeDocumentDO extends BaseDO { | ||
| 34 | */ | 33 | */ |
| 35 | private String name; | 34 | private String name; |
| 36 | /** | 35 | /** |
| 37 | - * 内容 | ||
| 38 | - */ | ||
| 39 | - private String content; | ||
| 40 | - /** | ||
| 41 | * 文件 URL | 36 | * 文件 URL |
| 42 | */ | 37 | */ |
| 43 | private String url; | 38 | private String url; |
| 44 | /** | 39 | /** |
| 45 | - * 文档 token 数量 | 40 | + * 内容 |
| 46 | */ | 41 | */ |
| 47 | - private Integer tokens; | 42 | + private String content; |
| 48 | /** | 43 | /** |
| 49 | - * 文档字符数 | 44 | + * 文档长度 |
| 50 | */ | 45 | */ |
| 51 | - private Integer wordCount; | ||
| 52 | - | 46 | + private Integer contentLength; |
| 53 | 47 | ||
| 54 | - // ========== 自定义分段所用参数 ========== | ||
| 55 | - // TODO @新:3)defaultChunkSize、defaultChunkSize、minChunkSizeChars、maxNumChunks 这几个字段的命名,可能要微信一起讨论下。尽量命名保持风格统一哈。 | ||
| 56 | - /** | ||
| 57 | - * 每个文本块的目标 token 数 | ||
| 58 | - */ | ||
| 59 | - private Integer defaultSegmentTokens; | ||
| 60 | /** | 48 | /** |
| 61 | - * 每个文本块的最小字符数 | ||
| 62 | - */ | ||
| 63 | - private Integer minSegmentWordCount; | ||
| 64 | - /** | ||
| 65 | - * 低于此值的块会被丢弃 | ||
| 66 | - */ | ||
| 67 | - private Integer minChunkLengthToEmbed; | ||
| 68 | - /** | ||
| 69 | - * 最大块数 | ||
| 70 | - */ | ||
| 71 | - private Integer maxNumSegments; | ||
| 72 | - /** | ||
| 73 | - * 分块是否保留分隔符 | 49 | + * 文档 token 数量 |
| 74 | */ | 50 | */ |
| 75 | - private Boolean keepSeparator; | ||
| 76 | - // =================================== | ||
| 77 | - | 51 | + private Integer tokens; |
| 78 | /** | 52 | /** |
| 79 | - * 切片状态 | ||
| 80 | - * <p> | ||
| 81 | - * 枚举 {@link AiKnowledgeDocumentStatusEnum} | 53 | + * 每个文本块的目标 token 数 |
| 82 | */ | 54 | */ |
| 83 | - private Integer sliceStatus; | 55 | + private Integer segmentMaxTokens; |
| 84 | 56 | ||
| 85 | /** | 57 | /** |
| 86 | * 状态 | 58 | * 状态 |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java
| @@ -17,7 +17,10 @@ import lombok.Data; | @@ -17,7 +17,10 @@ import lombok.Data; | ||
| 17 | @Data | 17 | @Data |
| 18 | public class AiKnowledgeSegmentDO extends BaseDO { | 18 | public class AiKnowledgeSegmentDO extends BaseDO { |
| 19 | 19 | ||
| 20 | - public static final String FIELD_KNOWLEDGE_ID = "knowledgeId"; | 20 | + /** |
| 21 | + * 向量库的编号 - 空值 | ||
| 22 | + */ | ||
| 23 | + public static final String VECTOR_ID_EMPTY = ""; | ||
| 21 | 24 | ||
| 22 | /** | 25 | /** |
| 23 | * 编号 | 26 | * 编号 |
| @@ -25,10 +28,6 @@ public class AiKnowledgeSegmentDO extends BaseDO { | @@ -25,10 +28,6 @@ public class AiKnowledgeSegmentDO extends BaseDO { | ||
| 25 | @TableId | 28 | @TableId |
| 26 | private Long id; | 29 | private Long id; |
| 27 | /** | 30 | /** |
| 28 | - * 向量库的编号 | ||
| 29 | - */ | ||
| 30 | - private String vectorId; | ||
| 31 | - /** | ||
| 32 | * 知识库编号 | 31 | * 知识库编号 |
| 33 | * <p> | 32 | * <p> |
| 34 | * 关联 {@link AiKnowledgeDO#getId()} | 33 | * 关联 {@link AiKnowledgeDO#getId()} |
| @@ -45,13 +44,19 @@ public class AiKnowledgeSegmentDO extends BaseDO { | @@ -45,13 +44,19 @@ public class AiKnowledgeSegmentDO extends BaseDO { | ||
| 45 | */ | 44 | */ |
| 46 | private String content; | 45 | private String content; |
| 47 | /** | 46 | /** |
| 48 | - * 字符数 | 47 | + * 切片内容长度 |
| 49 | */ | 48 | */ |
| 50 | - private Integer wordCount; | 49 | + private Integer contentLength; |
| 50 | + | ||
| 51 | + /** | ||
| 52 | + * 向量库的编号 | ||
| 53 | + */ | ||
| 54 | + private String vectorId; | ||
| 51 | /** | 55 | /** |
| 52 | * token 数量 | 56 | * token 数量 |
| 53 | */ | 57 | */ |
| 54 | private Integer tokens; | 58 | private Integer tokens; |
| 59 | + | ||
| 55 | /** | 60 | /** |
| 56 | * 状态 | 61 | * 状态 |
| 57 | * <p> | 62 | * <p> |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/mindmap/AiMindMapDO.java
| @@ -36,6 +36,7 @@ public class AiMindMapDO extends BaseDO { | @@ -36,6 +36,7 @@ public class AiMindMapDO extends BaseDO { | ||
| 36 | * 枚举 {@link AiPlatformEnum} | 36 | * 枚举 {@link AiPlatformEnum} |
| 37 | */ | 37 | */ |
| 38 | private String platform; | 38 | private String platform; |
| 39 | + // TODO @芋艿:modelId? | ||
| 39 | /** | 40 | /** |
| 40 | * 模型 | 41 | * 模型 |
| 41 | */ | 42 | */ |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/model/AiChatModelDO.java
| @@ -8,6 +8,7 @@ import com.baomidou.mybatisplus.annotation.TableId; | @@ -8,6 +8,7 @@ import com.baomidou.mybatisplus.annotation.TableId; | ||
| 8 | import com.baomidou.mybatisplus.annotation.TableName; | 8 | import com.baomidou.mybatisplus.annotation.TableName; |
| 9 | import lombok.*; | 9 | import lombok.*; |
| 10 | 10 | ||
| 11 | +// TODO @芋艿,需要改造,增加 type | ||
| 11 | /** | 12 | /** |
| 12 | * AI 聊天模型 DO | 13 | * AI 聊天模型 DO |
| 13 | * | 14 | * |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/music/AiMusicDO.java
| @@ -84,6 +84,7 @@ public class AiMusicDO extends BaseDO { | @@ -84,6 +84,7 @@ public class AiMusicDO extends BaseDO { | ||
| 84 | * 枚举 {@link AiPlatformEnum} | 84 | * 枚举 {@link AiPlatformEnum} |
| 85 | */ | 85 | */ |
| 86 | private String platform; | 86 | private String platform; |
| 87 | + // TODO @芋艿:modelId? | ||
| 87 | /** | 88 | /** |
| 88 | * 模型 | 89 | * 模型 |
| 89 | */ | 90 | */ |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/write/AiWriteDO.java
| @@ -44,6 +44,7 @@ public class AiWriteDO extends BaseDO { | @@ -44,6 +44,7 @@ public class AiWriteDO extends BaseDO { | ||
| 44 | * 枚举 {@link AiPlatformEnum} | 44 | * 枚举 {@link AiPlatformEnum} |
| 45 | */ | 45 | */ |
| 46 | private String platform; | 46 | private String platform; |
| 47 | + // TODO @芋艿:modelId? | ||
| 47 | /** | 48 | /** |
| 48 | * 模型 | 49 | * 模型 |
| 49 | */ | 50 | */ |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeMapper.java
| @@ -16,11 +16,9 @@ import org.apache.ibatis.annotations.Mapper; | @@ -16,11 +16,9 @@ import org.apache.ibatis.annotations.Mapper; | ||
| 16 | @Mapper | 16 | @Mapper |
| 17 | public interface AiKnowledgeMapper extends BaseMapperX<AiKnowledgeDO> { | 17 | public interface AiKnowledgeMapper extends BaseMapperX<AiKnowledgeDO> { |
| 18 | 18 | ||
| 19 | - default PageResult<AiKnowledgeDO> selectPage(Long userId, AiKnowledgePageReqVO pageReqVO) { | 19 | + default PageResult<AiKnowledgeDO> selectPage(AiKnowledgePageReqVO pageReqVO) { |
| 20 | return selectPage(pageReqVO, new LambdaQueryWrapperX<AiKnowledgeDO>() | 20 | return selectPage(pageReqVO, new LambdaQueryWrapperX<AiKnowledgeDO>() |
| 21 | .eq(AiKnowledgeDO::getStatus, CommonStatusEnum.ENABLE.getStatus()) | 21 | .eq(AiKnowledgeDO::getStatus, CommonStatusEnum.ENABLE.getStatus()) |
| 22 | - .likeIfPresent(AiKnowledgeDO::getName, pageReqVO.getName()) | ||
| 23 | - .and(e -> e.apply("FIND_IN_SET(" + userId + ",visibility_permissions)").or(m -> m.apply("FIND_IN_SET(-1,visibility_permissions)"))) | ||
| 24 | - .orderByDesc(AiKnowledgeDO::getId)); | 22 | + .likeIfPresent(AiKnowledgeDO::getName, pageReqVO.getName())); |
| 25 | } | 23 | } |
| 26 | } | 24 | } |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java
| @@ -21,7 +21,6 @@ public interface AiKnowledgeDocumentService { | @@ -21,7 +21,6 @@ public interface AiKnowledgeDocumentService { | ||
| 21 | */ | 21 | */ |
| 22 | Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO); | 22 | Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO); |
| 23 | 23 | ||
| 24 | - | ||
| 25 | /** | 24 | /** |
| 26 | * 获取文档分页 | 25 | * 获取文档分页 |
| 27 | * | 26 | * |
| @@ -36,4 +35,13 @@ public interface AiKnowledgeDocumentService { | @@ -36,4 +35,13 @@ public interface AiKnowledgeDocumentService { | ||
| 36 | * @param reqVO 更新信息 | 35 | * @param reqVO 更新信息 |
| 37 | */ | 36 | */ |
| 38 | void updateKnowledgeDocument(AiKnowledgeDocumentUpdateReqVO reqVO); | 37 | void updateKnowledgeDocument(AiKnowledgeDocumentUpdateReqVO reqVO); |
| 38 | + | ||
| 39 | + /** | ||
| 40 | + * 校验文档是否存在 | ||
| 41 | + * | ||
| 42 | + * @param id 文档编号 | ||
| 43 | + * @return 文档信息 | ||
| 44 | + */ | ||
| 45 | + AiKnowledgeDocumentDO validateKnowledgeDocumentExists(Long id); | ||
| 46 | + | ||
| 39 | } | 47 | } |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java
| 1 | package cn.iocoder.yudao.module.ai.service.knowledge; | 1 | package cn.iocoder.yudao.module.ai.service.knowledge; |
| 2 | 2 | ||
| 3 | import cn.hutool.core.collection.CollUtil; | 3 | import cn.hutool.core.collection.CollUtil; |
| 4 | +import cn.hutool.core.util.StrUtil; | ||
| 4 | import cn.hutool.http.HttpUtil; | 5 | import cn.hutool.http.HttpUtil; |
| 5 | import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; | 6 | import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; |
| 6 | import cn.iocoder.yudao.framework.common.pojo.PageResult; | 7 | import cn.iocoder.yudao.framework.common.pojo.PageResult; |
| 7 | -import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils; | ||
| 8 | import cn.iocoder.yudao.framework.common.util.object.BeanUtils; | 8 | import cn.iocoder.yudao.framework.common.util.object.BeanUtils; |
| 9 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO; | 9 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO; |
| 10 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO; | 10 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO; |
| 11 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO; | 11 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO; |
| 12 | import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; | 12 | import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; |
| 13 | -import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; | ||
| 14 | import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper; | 13 | import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper; |
| 15 | -import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper; | ||
| 16 | -import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum; | ||
| 17 | import jakarta.annotation.Resource; | 14 | import jakarta.annotation.Resource; |
| 18 | import lombok.extern.slf4j.Slf4j; | 15 | import lombok.extern.slf4j.Slf4j; |
| 19 | import org.springframework.ai.document.Document; | 16 | import org.springframework.ai.document.Document; |
| 20 | import org.springframework.ai.reader.tika.TikaDocumentReader; | 17 | import org.springframework.ai.reader.tika.TikaDocumentReader; |
| 21 | import org.springframework.ai.tokenizer.TokenCountEstimator; | 18 | import org.springframework.ai.tokenizer.TokenCountEstimator; |
| 22 | -import org.springframework.ai.transformer.splitter.TokenTextSplitter; | ||
| 23 | -import org.springframework.ai.vectorstore.VectorStore; | 19 | +import org.springframework.context.annotation.Lazy; |
| 24 | import org.springframework.core.io.ByteArrayResource; | 20 | import org.springframework.core.io.ByteArrayResource; |
| 25 | import org.springframework.stereotype.Service; | 21 | import org.springframework.stereotype.Service; |
| 26 | import org.springframework.transaction.annotation.Transactional; | 22 | import org.springframework.transaction.annotation.Transactional; |
| @@ -28,7 +24,7 @@ import org.springframework.transaction.annotation.Transactional; | @@ -28,7 +24,7 @@ import org.springframework.transaction.annotation.Transactional; | ||
| 28 | import java.util.List; | 24 | import java.util.List; |
| 29 | 25 | ||
| 30 | import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; | 26 | import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; |
| 31 | -import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_DOCUMENT_NOT_EXISTS; | 27 | +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.*; |
| 32 | 28 | ||
| 33 | /** | 29 | /** |
| 34 | * AI 知识库文档 Service 实现类 | 30 | * AI 知识库文档 Service 实现类 |
| @@ -40,58 +36,46 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_DOCU | @@ -40,58 +36,46 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_DOCU | ||
| 40 | public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentService { | 36 | public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentService { |
| 41 | 37 | ||
| 42 | @Resource | 38 | @Resource |
| 43 | - private AiKnowledgeDocumentMapper documentMapper; | ||
| 44 | - @Resource | ||
| 45 | - private AiKnowledgeSegmentMapper segmentMapper; | 39 | + private AiKnowledgeDocumentMapper knowledgeDocumentMapper; |
| 46 | 40 | ||
| 47 | @Resource | 41 | @Resource |
| 48 | private TokenCountEstimator tokenCountEstimator; | 42 | private TokenCountEstimator tokenCountEstimator; |
| 43 | + | ||
| 49 | @Resource | 44 | @Resource |
| 45 | + private AiKnowledgeSegmentService knowledgeSegmentService; | ||
| 46 | + @Resource | ||
| 47 | + @Lazy // 延迟加载,避免循环依赖 | ||
| 50 | private AiKnowledgeService knowledgeService; | 48 | private AiKnowledgeService knowledgeService; |
| 51 | 49 | ||
| 52 | @Override | 50 | @Override |
| 53 | @Transactional(rollbackFor = Exception.class) | 51 | @Transactional(rollbackFor = Exception.class) |
| 54 | public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) { | 52 | public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) { |
| 55 | - // 0. 校验并获取向量存储实例 | ||
| 56 | - VectorStore vectorStore = knowledgeService.getVectorStoreById(createReqVO.getKnowledgeId()); | 53 | + // 1. 校验参数 |
| 54 | + knowledgeService.validateKnowledgeExists(createReqVO.getKnowledgeId()); | ||
| 57 | 55 | ||
| 58 | - // 1.1 下载文档 | 56 | + // 2. 下载文档 |
| 59 | TikaDocumentReader loader = new TikaDocumentReader(downloadFile(createReqVO.getUrl())); | 57 | TikaDocumentReader loader = new TikaDocumentReader(downloadFile(createReqVO.getUrl())); |
| 60 | List<Document> documents = loader.get(); | 58 | List<Document> documents = loader.get(); |
| 61 | Document document = CollUtil.getFirst(documents); | 59 | Document document = CollUtil.getFirst(documents); |
| 62 | - // 1.2 文档记录入库 | 60 | + if (document == null || StrUtil.isEmpty(document.getText())) { |
| 61 | + throw exception(KNOWLEDGE_DOCUMENT_FILE_READ_FAIL); | ||
| 62 | + } | ||
| 63 | + | ||
| 64 | + // 3. 文档记录入库 | ||
| 63 | String content = document.getText(); | 65 | String content = document.getText(); |
| 64 | AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class) | 66 | AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class) |
| 65 | - .setTokens(tokenCountEstimator.estimate(content)).setWordCount(content.length()) | ||
| 66 | - .setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus()); | ||
| 67 | - documentMapper.insert(documentDO); | ||
| 68 | - Long documentId = documentDO.getId(); | ||
| 69 | - if (CollUtil.isEmpty(documents)) { | ||
| 70 | - return documentId; | ||
| 71 | - } | 67 | + .setContent(content).setContentLength(content.length()).setTokens(tokenCountEstimator.estimate(content)) |
| 68 | + .setStatus(CommonStatusEnum.ENABLE.getStatus()); | ||
| 69 | + knowledgeDocumentMapper.insert(documentDO); | ||
| 72 | 70 | ||
| 73 | - // 2 构造文本分段器 | ||
| 74 | - TokenTextSplitter tokenTextSplitter = new TokenTextSplitter(createReqVO.getDefaultSegmentTokens(), createReqVO.getMinSegmentWordCount(), createReqVO.getMinChunkLengthToEmbed(), | ||
| 75 | - createReqVO.getMaxNumSegments(), createReqVO.getKeepSeparator()); | ||
| 76 | - // 2.1 文档分段 | ||
| 77 | - List<Document> segments = tokenTextSplitter.apply(documents); | ||
| 78 | - // 2.2 分段内容入库 | ||
| 79 | - List<AiKnowledgeSegmentDO> segmentDOList = CollectionUtils.convertList(segments, | ||
| 80 | - segment -> new AiKnowledgeSegmentDO().setContent(segment.getText()).setDocumentId(documentId) | ||
| 81 | - .setKnowledgeId(createReqVO.getKnowledgeId()).setVectorId(segment.getId()) | ||
| 82 | - .setTokens(tokenCountEstimator.estimate(segment.getText())).setWordCount(segment.getText().length()) | ||
| 83 | - .setStatus(CommonStatusEnum.ENABLE.getStatus())); | ||
| 84 | - segmentMapper.insertBatch(segmentDOList); | ||
| 85 | - | ||
| 86 | - // 3. 向量化并存储 | ||
| 87 | - segments.forEach(segment -> segment.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, createReqVO.getKnowledgeId())); | ||
| 88 | - vectorStore.add(segments); | ||
| 89 | - return documentId; | 71 | + // 4. 文档切片入库 |
| 72 | + knowledgeSegmentService.createKnowledgeSegmentBySplitContent(documentDO.getId(), document.getText()); | ||
| 73 | + return documentDO.getId(); | ||
| 90 | } | 74 | } |
| 91 | 75 | ||
| 92 | @Override | 76 | @Override |
| 93 | public PageResult<AiKnowledgeDocumentDO> getKnowledgeDocumentPage(AiKnowledgeDocumentPageReqVO pageReqVO) { | 77 | public PageResult<AiKnowledgeDocumentDO> getKnowledgeDocumentPage(AiKnowledgeDocumentPageReqVO pageReqVO) { |
| 94 | - return documentMapper.selectPage(pageReqVO); | 78 | + return knowledgeDocumentMapper.selectPage(pageReqVO); |
| 95 | } | 79 | } |
| 96 | 80 | ||
| 97 | @Override | 81 | @Override |
| @@ -100,17 +84,13 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic | @@ -100,17 +84,13 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic | ||
| 100 | validateKnowledgeDocumentExists(reqVO.getId()); | 84 | validateKnowledgeDocumentExists(reqVO.getId()); |
| 101 | // 2. 更新文档 | 85 | // 2. 更新文档 |
| 102 | AiKnowledgeDocumentDO document = BeanUtils.toBean(reqVO, AiKnowledgeDocumentDO.class); | 86 | AiKnowledgeDocumentDO document = BeanUtils.toBean(reqVO, AiKnowledgeDocumentDO.class); |
| 103 | - documentMapper.updateById(document); | 87 | + knowledgeDocumentMapper.updateById(document); |
| 88 | + // TODO @芋艿:这里要处理状态的变更 | ||
| 104 | } | 89 | } |
| 105 | 90 | ||
| 106 | - /** | ||
| 107 | - * 校验文档是否存在 | ||
| 108 | - * | ||
| 109 | - * @param id 文档编号 | ||
| 110 | - * @return 文档信息 | ||
| 111 | - */ | ||
| 112 | - private AiKnowledgeDocumentDO validateKnowledgeDocumentExists(Long id) { | ||
| 113 | - AiKnowledgeDocumentDO knowledgeDocument = documentMapper.selectById(id); | 91 | + @Override |
| 92 | + public AiKnowledgeDocumentDO validateKnowledgeDocumentExists(Long id) { | ||
| 93 | + AiKnowledgeDocumentDO knowledgeDocument = knowledgeDocumentMapper.selectById(id); | ||
| 114 | if (knowledgeDocument == null) { | 94 | if (knowledgeDocument == null) { |
| 115 | throw exception(KNOWLEDGE_DOCUMENT_NOT_EXISTS); | 95 | throw exception(KNOWLEDGE_DOCUMENT_NOT_EXISTS); |
| 116 | } | 96 | } |
| @@ -120,6 +100,9 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic | @@ -120,6 +100,9 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic | ||
| 120 | private org.springframework.core.io.Resource downloadFile(String url) { | 100 | private org.springframework.core.io.Resource downloadFile(String url) { |
| 121 | try { | 101 | try { |
| 122 | byte[] bytes = HttpUtil.downloadBytes(url); | 102 | byte[] bytes = HttpUtil.downloadBytes(url); |
| 103 | + if (bytes.length == 0) { | ||
| 104 | + throw exception(KNOWLEDGE_DOCUMENT_FILE_EMPTY); | ||
| 105 | + } | ||
| 123 | return new ByteArrayResource(bytes); | 106 | return new ByteArrayResource(bytes); |
| 124 | } catch (Exception e) { | 107 | } catch (Exception e) { |
| 125 | log.error("[downloadFile][url({}) 下载失败]", url, e); | 108 | log.error("[downloadFile][url({}) 下载失败]", url, e); |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java
| @@ -25,6 +25,14 @@ public interface AiKnowledgeSegmentService { | @@ -25,6 +25,14 @@ public interface AiKnowledgeSegmentService { | ||
| 25 | PageResult<AiKnowledgeSegmentDO> getKnowledgeSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO); | 25 | PageResult<AiKnowledgeSegmentDO> getKnowledgeSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO); |
| 26 | 26 | ||
| 27 | /** | 27 | /** |
| 28 | + * 基于 content 内容,切片创建多个段落 | ||
| 29 | + * | ||
| 30 | + * @param documentId 知识库文档编号 | ||
| 31 | + * @param content 文档内容 | ||
| 32 | + */ | ||
| 33 | + void createKnowledgeSegmentBySplitContent(Long documentId, String content); | ||
| 34 | + | ||
| 35 | + /** | ||
| 28 | * 更新段落的内容 | 36 | * 更新段落的内容 |
| 29 | * | 37 | * |
| 30 | * @param reqVO 更新内容 | 38 | * @param reqVO 更新内容 |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
| @@ -2,6 +2,7 @@ package cn.iocoder.yudao.module.ai.service.knowledge; | @@ -2,6 +2,7 @@ package cn.iocoder.yudao.module.ai.service.knowledge; | ||
| 2 | 2 | ||
| 3 | import cn.hutool.core.collection.CollUtil; | 3 | import cn.hutool.core.collection.CollUtil; |
| 4 | import cn.hutool.core.collection.ListUtil; | 4 | import cn.hutool.core.collection.ListUtil; |
| 5 | +import cn.hutool.core.util.StrUtil; | ||
| 5 | import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; | 6 | import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; |
| 6 | import cn.iocoder.yudao.framework.common.pojo.PageResult; | 7 | import cn.iocoder.yudao.framework.common.pojo.PageResult; |
| 7 | import cn.iocoder.yudao.framework.common.util.object.BeanUtils; | 8 | import cn.iocoder.yudao.framework.common.util.object.BeanUtils; |
| @@ -10,23 +11,28 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowle | @@ -10,23 +11,28 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowle | ||
| 10 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO; | 11 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO; |
| 11 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO; | 12 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO; |
| 12 | import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; | 13 | import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; |
| 14 | +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; | ||
| 13 | import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; | 15 | import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; |
| 14 | -import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; | ||
| 15 | import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper; | 16 | import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper; |
| 16 | import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService; | 17 | import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService; |
| 17 | -import cn.iocoder.yudao.module.ai.service.model.AiChatModelService; | ||
| 18 | import jakarta.annotation.Resource; | 18 | import jakarta.annotation.Resource; |
| 19 | import lombok.extern.slf4j.Slf4j; | 19 | import lombok.extern.slf4j.Slf4j; |
| 20 | import org.springframework.ai.document.Document; | 20 | import org.springframework.ai.document.Document; |
| 21 | +import org.springframework.ai.tokenizer.TokenCountEstimator; | ||
| 22 | +import org.springframework.ai.transformer.splitter.TextSplitter; | ||
| 23 | +import org.springframework.ai.transformer.splitter.TokenTextSplitter; | ||
| 21 | import org.springframework.ai.vectorstore.SearchRequest; | 24 | import org.springframework.ai.vectorstore.SearchRequest; |
| 22 | import org.springframework.ai.vectorstore.VectorStore; | 25 | import org.springframework.ai.vectorstore.VectorStore; |
| 23 | import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder; | 26 | import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder; |
| 27 | +import org.springframework.context.annotation.Lazy; | ||
| 24 | import org.springframework.stereotype.Service; | 28 | import org.springframework.stereotype.Service; |
| 25 | 29 | ||
| 30 | +import java.util.Collections; | ||
| 26 | import java.util.List; | 31 | import java.util.List; |
| 27 | import java.util.Objects; | 32 | import java.util.Objects; |
| 28 | 33 | ||
| 29 | import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; | 34 | import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; |
| 35 | +import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertList; | ||
| 30 | import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_NOT_EXISTS; | 36 | import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_NOT_EXISTS; |
| 31 | 37 | ||
| 32 | /** | 38 | /** |
| @@ -38,85 +44,138 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGM | @@ -38,85 +44,138 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGM | ||
| 38 | @Slf4j | 44 | @Slf4j |
| 39 | public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService { | 45 | public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService { |
| 40 | 46 | ||
| 47 | + public static final String VECTOR_STORE_METADATA_KNOWLEDGE_ID = "knowledgeId"; | ||
| 48 | + public static final String VECTOR_STORE_METADATA_DOCUMENT_ID = "documentId"; | ||
| 49 | + public static final String VECTOR_STORE_METADATA_SEGMENT_ID = "segmentId"; | ||
| 50 | + | ||
| 41 | @Resource | 51 | @Resource |
| 42 | private AiKnowledgeSegmentMapper segmentMapper; | 52 | private AiKnowledgeSegmentMapper segmentMapper; |
| 43 | 53 | ||
| 44 | @Resource | 54 | @Resource |
| 45 | private AiKnowledgeService knowledgeService; | 55 | private AiKnowledgeService knowledgeService; |
| 46 | @Resource | 56 | @Resource |
| 47 | - private AiChatModelService chatModelService; | 57 | + @Lazy // 延迟加载,避免循环依赖 |
| 58 | + private AiKnowledgeDocumentService knowledgeDocumentService; | ||
| 48 | @Resource | 59 | @Resource |
| 49 | private AiApiKeyService apiKeyService; | 60 | private AiApiKeyService apiKeyService; |
| 50 | 61 | ||
| 62 | + @Resource | ||
| 63 | + private TokenCountEstimator tokenCountEstimator; | ||
| 64 | + | ||
| 51 | @Override | 65 | @Override |
| 52 | public PageResult<AiKnowledgeSegmentDO> getKnowledgeSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO) { | 66 | public PageResult<AiKnowledgeSegmentDO> getKnowledgeSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO) { |
| 53 | return segmentMapper.selectPage(pageReqVO); | 67 | return segmentMapper.selectPage(pageReqVO); |
| 54 | } | 68 | } |
| 55 | 69 | ||
| 56 | @Override | 70 | @Override |
| 71 | + public void createKnowledgeSegmentBySplitContent(Long documentId, String content) { | ||
| 72 | + // 1. 校验 | ||
| 73 | + AiKnowledgeDocumentDO documentDO = knowledgeDocumentService.validateKnowledgeDocumentExists(documentId); | ||
| 74 | + AiKnowledgeDO knowledgeDO = knowledgeService.validateKnowledgeExists(documentDO.getKnowledgeId()); | ||
| 75 | + VectorStore vectorStore = getVectorStoreById(knowledgeDO); | ||
| 76 | + | ||
| 77 | + // 2. 文档切片 | ||
| 78 | + Document document = new Document(content); | ||
| 79 | + TextSplitter textSplitter = buildTokenTextSplitter(documentDO.getSegmentMaxTokens()); | ||
| 80 | + List<Document> documentSegments = textSplitter.apply(Collections.singletonList(document)); | ||
| 81 | + | ||
| 82 | + // 3.1 存储切片 | ||
| 83 | + List<AiKnowledgeSegmentDO> segmentDOs = convertList(documentSegments, segment -> { | ||
| 84 | + if (StrUtil.isEmpty(segment.getText())) { | ||
| 85 | + return null; | ||
| 86 | + } | ||
| 87 | + return new AiKnowledgeSegmentDO().setKnowledgeId(documentDO.getKnowledgeId()).setDocumentId(documentId) | ||
| 88 | + .setContent(segment.getText()).setContentLength(segment.getText().length()) | ||
| 89 | + .setVectorId(AiKnowledgeSegmentDO.VECTOR_ID_EMPTY).setTokens(tokenCountEstimator.estimate(segment.getText())) | ||
| 90 | + .setStatus(CommonStatusEnum.ENABLE.getStatus()); | ||
| 91 | + }); | ||
| 92 | + segmentMapper.insertBatch(segmentDOs); | ||
| 93 | + // 3.2 切片向量化 | ||
| 94 | + for (int i = 0; i < documentSegments.size(); i++) { | ||
| 95 | + Document segment = documentSegments.get(i); | ||
| 96 | + AiKnowledgeSegmentDO segmentDO = segmentDOs.get(i); | ||
| 97 | + writeVectorStore(vectorStore, segmentDO, segment); | ||
| 98 | + } | ||
| 99 | + } | ||
| 100 | + | ||
| 101 | + @Override | ||
| 57 | public void updateKnowledgeSegment(AiKnowledgeSegmentUpdateReqVO reqVO) { | 102 | public void updateKnowledgeSegment(AiKnowledgeSegmentUpdateReqVO reqVO) { |
| 58 | // 1. 校验 | 103 | // 1. 校验 |
| 59 | - AiKnowledgeSegmentDO oldKnowledgeSegment = validateKnowledgeSegmentExists(reqVO.getId()); | ||
| 60 | - | ||
| 61 | - // 2.1 获取知识库向量实例 | ||
| 62 | - VectorStore vectorStore = knowledgeService.getVectorStoreById(oldKnowledgeSegment.getKnowledgeId()); | ||
| 63 | - // 2.2 删除原向量 | ||
| 64 | - vectorStore.delete(List.of(oldKnowledgeSegment.getVectorId())); | ||
| 65 | - // 2.3 重新向量化 | ||
| 66 | - Document document = new Document(reqVO.getContent()); | ||
| 67 | - document.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, oldKnowledgeSegment.getKnowledgeId()); | ||
| 68 | - vectorStore.add(List.of(document)); | ||
| 69 | - | ||
| 70 | - // 3. 更新段落内容 | ||
| 71 | - AiKnowledgeSegmentDO knowledgeSegment = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class); | ||
| 72 | - knowledgeSegment.setVectorId(document.getId()); | ||
| 73 | - segmentMapper.updateById(knowledgeSegment); | 104 | + AiKnowledgeSegmentDO segment = validateKnowledgeSegmentExists(reqVO.getId()); |
| 105 | + | ||
| 106 | + // 2. 删除向量 | ||
| 107 | + VectorStore vectorStore = getVectorStoreById(segment.getKnowledgeId()); | ||
| 108 | + deleteVectorStore(vectorStore, segment); | ||
| 109 | + | ||
| 110 | + // 3.1 更新切片 | ||
| 111 | + AiKnowledgeSegmentDO segmentDO = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class); | ||
| 112 | + segmentMapper.updateById(segmentDO); | ||
| 113 | + // 3.2 重新向量化 | ||
| 114 | + writeVectorStore(vectorStore, segmentDO, new Document(segmentDO.getContent())); | ||
| 74 | } | 115 | } |
| 75 | 116 | ||
| 76 | @Override | 117 | @Override |
| 77 | public void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO) { | 118 | public void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO) { |
| 78 | - // 0 校验 | ||
| 79 | - AiKnowledgeSegmentDO oldKnowledgeSegment = validateKnowledgeSegmentExists(reqVO.getId()); | ||
| 80 | - // 1 获取知识库向量实例 | ||
| 81 | - VectorStore vectorStore = knowledgeService.getVectorStoreById(oldKnowledgeSegment.getKnowledgeId()); | ||
| 82 | - AiKnowledgeSegmentDO knowledgeSegment = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class); | 119 | + // 1. 校验 |
| 120 | + AiKnowledgeSegmentDO segment = validateKnowledgeSegmentExists(reqVO.getId()); | ||
| 121 | + | ||
| 122 | + // 2. 获取知识库向量实例 | ||
| 123 | + VectorStore vectorStore = getVectorStoreById(segment.getKnowledgeId()); | ||
| 124 | + | ||
| 125 | + // 3. 更新状态 | ||
| 126 | + segmentMapper.updateById(new AiKnowledgeSegmentDO().setId(reqVO.getId()).setStatus(reqVO.getStatus())); | ||
| 83 | 127 | ||
| 128 | + // 4. 更新向量 | ||
| 84 | if (Objects.equals(reqVO.getStatus(), CommonStatusEnum.ENABLE.getStatus())) { | 129 | if (Objects.equals(reqVO.getStatus(), CommonStatusEnum.ENABLE.getStatus())) { |
| 85 | - // 2.1 启用重新向量化 | ||
| 86 | - Document document = new Document(oldKnowledgeSegment.getContent()); | ||
| 87 | - document.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, oldKnowledgeSegment.getKnowledgeId()); | ||
| 88 | - vectorStore.add(List.of(document)); | ||
| 89 | - knowledgeSegment.setVectorId(document.getId()); | 130 | + writeVectorStore(vectorStore, segment, new Document(segment.getContent())); |
| 90 | } else { | 131 | } else { |
| 91 | - // 2.2 禁用删除向量 | ||
| 92 | - vectorStore.delete(List.of(oldKnowledgeSegment.getVectorId())); | ||
| 93 | - knowledgeSegment.setVectorId(""); | 132 | + deleteVectorStore(vectorStore, segment); |
| 94 | } | 133 | } |
| 95 | - // 3 更新段落状态 | ||
| 96 | - segmentMapper.updateById(knowledgeSegment); | 134 | + } |
| 135 | + | ||
| 136 | + private void writeVectorStore(VectorStore vectorStore, AiKnowledgeSegmentDO segmentDO, Document segment) { | ||
| 137 | + // 1. 向量存储 | ||
| 138 | + segment.getMetadata().put(VECTOR_STORE_METADATA_KNOWLEDGE_ID, segmentDO.getKnowledgeId()); | ||
| 139 | + segment.getMetadata().put(VECTOR_STORE_METADATA_DOCUMENT_ID, segmentDO.getDocumentId()); | ||
| 140 | + segment.getMetadata().put(VECTOR_STORE_METADATA_SEGMENT_ID, segmentDO.getId()); | ||
| 141 | + vectorStore.add(List.of(segment)); | ||
| 142 | + | ||
| 143 | + // 2. 更新向量 ID | ||
| 144 | + segmentMapper.updateById(new AiKnowledgeSegmentDO().setId(segmentDO.getId()).setVectorId(segment.getId())); | ||
| 145 | + } | ||
| 146 | + | ||
| 147 | + private void deleteVectorStore(VectorStore vectorStore, AiKnowledgeSegmentDO segmentDO) { | ||
| 148 | + // 1. 更新向量 ID | ||
| 149 | + if (StrUtil.isEmpty(segmentDO.getVectorId())) { | ||
| 150 | + return; | ||
| 151 | + } | ||
| 152 | + segmentMapper.updateById(new AiKnowledgeSegmentDO().setId(segmentDO.getId()) | ||
| 153 | + .setVectorId(AiKnowledgeSegmentDO.VECTOR_ID_EMPTY)); | ||
| 154 | + | ||
| 155 | + // 2. 删除向量 | ||
| 156 | + vectorStore.delete(List.of(segmentDO.getVectorId())); | ||
| 97 | } | 157 | } |
| 98 | 158 | ||
| 99 | @Override | 159 | @Override |
| 100 | public List<AiKnowledgeSegmentDO> similaritySearch(AiKnowledgeSegmentSearchReqVO reqVO) { | 160 | public List<AiKnowledgeSegmentDO> similaritySearch(AiKnowledgeSegmentSearchReqVO reqVO) { |
| 101 | // 1. 校验 | 161 | // 1. 校验 |
| 102 | AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(reqVO.getKnowledgeId()); | 162 | AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(reqVO.getKnowledgeId()); |
| 103 | - AiChatModelDO model = chatModelService.validateChatModel(knowledge.getModelId()); | ||
| 104 | 163 | ||
| 105 | // 2. 获取向量存储实例 | 164 | // 2. 获取向量存储实例 |
| 106 | - VectorStore vectorStore = apiKeyService.getOrCreateVectorStore(model.getKeyId()); | 165 | + VectorStore vectorStore = apiKeyService.getOrCreateVectorStoreByModelId(knowledge.getEmbeddingModelId()); |
| 107 | 166 | ||
| 108 | // 3.1 向量检索 | 167 | // 3.1 向量检索 |
| 109 | - List<Document> documentList = vectorStore.similaritySearch(SearchRequest.builder() | 168 | + List<Document> documents = vectorStore.similaritySearch(SearchRequest.builder() |
| 110 | .query(reqVO.getContent()) | 169 | .query(reqVO.getContent()) |
| 111 | - .topK(knowledge.getTopK()) | ||
| 112 | - .similarityThreshold(knowledge.getSimilarityThreshold()) | ||
| 113 | - .filterExpression(new FilterExpressionBuilder().eq(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, reqVO.getKnowledgeId()).build()) | 170 | + .topK(knowledge.getTopK()).similarityThreshold(knowledge.getSimilarityThreshold()) |
| 171 | + .filterExpression(new FilterExpressionBuilder() | ||
| 172 | + .eq(VECTOR_STORE_METADATA_KNOWLEDGE_ID, reqVO.getKnowledgeId()).build()) | ||
| 114 | .build()); | 173 | .build()); |
| 115 | - if (CollUtil.isEmpty(documentList)) { | 174 | + if (CollUtil.isEmpty(documents)) { |
| 116 | return ListUtil.empty(); | 175 | return ListUtil.empty(); |
| 117 | } | 176 | } |
| 118 | // 3.2 段落召回 | 177 | // 3.2 段落召回 |
| 119 | - return segmentMapper.selectListByVectorIds(CollUtil.getFieldValues(documentList, "id", String.class)); | 178 | + return segmentMapper.selectListByVectorIds(convertList(documents, Document::getId)); |
| 120 | } | 179 | } |
| 121 | 180 | ||
| 122 | /** | 181 | /** |
| @@ -133,4 +192,22 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService | @@ -133,4 +192,22 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService | ||
| 133 | return knowledgeSegment; | 192 | return knowledgeSegment; |
| 134 | } | 193 | } |
| 135 | 194 | ||
| 195 | + private VectorStore getVectorStoreById(AiKnowledgeDO knowledge) { | ||
| 196 | + return apiKeyService.getOrCreateVectorStoreByModelId(knowledge.getEmbeddingModelId()); | ||
| 197 | + } | ||
| 198 | + | ||
| 199 | + private VectorStore getVectorStoreById(Long knowledgeId) { | ||
| 200 | + return getVectorStoreById(knowledgeService.validateKnowledgeExists(knowledgeId)); | ||
| 201 | + } | ||
| 202 | + | ||
| 203 | + private static TextSplitter buildTokenTextSplitter(Integer segmentMaxTokens) { | ||
| 204 | + return TokenTextSplitter.builder() | ||
| 205 | + .withChunkSize(segmentMaxTokens) | ||
| 206 | + .withMinChunkSizeChars(Integer.MAX_VALUE) // 忽略字符的截断 | ||
| 207 | + .withMinChunkLengthToEmbed(1) // 允许的最小有效分段长度 | ||
| 208 | + .withMaxNumChunks(Integer.MAX_VALUE) | ||
| 209 | + .withKeepSeparator(true) // 保留分隔符 | ||
| 210 | + .build(); | ||
| 211 | + } | ||
| 212 | + | ||
| 136 | } | 213 | } |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java
| 1 | package cn.iocoder.yudao.module.ai.service.knowledge; | 1 | package cn.iocoder.yudao.module.ai.service.knowledge; |
| 2 | 2 | ||
| 3 | import cn.iocoder.yudao.framework.common.pojo.PageResult; | 3 | import cn.iocoder.yudao.framework.common.pojo.PageResult; |
| 4 | -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateReqVO; | ||
| 5 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO; | 4 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO; |
| 6 | -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateReqVO; | 5 | +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeSaveReqVO; |
| 7 | import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; | 6 | import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; |
| 8 | -import org.springframework.ai.vectorstore.VectorStore; | ||
| 9 | 7 | ||
| 10 | /** | 8 | /** |
| 11 | * AI 知识库-基础信息 Service 接口 | 9 | * AI 知识库-基础信息 Service 接口 |
| @@ -18,18 +16,16 @@ public interface AiKnowledgeService { | @@ -18,18 +16,16 @@ public interface AiKnowledgeService { | ||
| 18 | * 创建知识库 | 16 | * 创建知识库 |
| 19 | * | 17 | * |
| 20 | * @param createReqVO 创建信息 | 18 | * @param createReqVO 创建信息 |
| 21 | - * @param userId 用户编号 | ||
| 22 | * @return 编号 | 19 | * @return 编号 |
| 23 | */ | 20 | */ |
| 24 | - Long createKnowledge(AiKnowledgeCreateReqVO createReqVO, Long userId); | 21 | + Long createKnowledge(AiKnowledgeSaveReqVO createReqVO); |
| 25 | 22 | ||
| 26 | /** | 23 | /** |
| 27 | * 更新知识库 | 24 | * 更新知识库 |
| 28 | * | 25 | * |
| 29 | * @param updateReqVO 更新信息 | 26 | * @param updateReqVO 更新信息 |
| 30 | - * @param userId 用户编号 | ||
| 31 | */ | 27 | */ |
| 32 | - void updateKnowledge(AiKnowledgeUpdateReqVO updateReqVO, Long userId); | 28 | + void updateKnowledge(AiKnowledgeSaveReqVO updateReqVO); |
| 33 | 29 | ||
| 34 | /** | 30 | /** |
| 35 | * 校验知识库是否存在 | 31 | * 校验知识库是否存在 |
| @@ -41,18 +37,9 @@ public interface AiKnowledgeService { | @@ -41,18 +37,9 @@ public interface AiKnowledgeService { | ||
| 41 | /** | 37 | /** |
| 42 | * 获得知识库分页 | 38 | * 获得知识库分页 |
| 43 | * | 39 | * |
| 44 | - * @param userId 用户编号 | ||
| 45 | * @param pageReqVO 分页查询 | 40 | * @param pageReqVO 分页查询 |
| 46 | * @return 知识库分页 | 41 | * @return 知识库分页 |
| 47 | */ | 42 | */ |
| 48 | - PageResult<AiKnowledgeDO> getKnowledgePage(Long userId, AiKnowledgePageReqVO pageReqVO); | ||
| 49 | - | ||
| 50 | - /** | ||
| 51 | - * 根据知识库编号获取向量存储实例 | ||
| 52 | - * | ||
| 53 | - * @param id 知识库编号 | ||
| 54 | - * @return 向量存储实例 | ||
| 55 | - */ | ||
| 56 | - VectorStore getVectorStoreById(Long id); | 43 | + PageResult<AiKnowledgeDO> getKnowledgePage(AiKnowledgePageReqVO pageReqVO); |
| 57 | 44 | ||
| 58 | } | 45 | } |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java
| 1 | package cn.iocoder.yudao.module.ai.service.knowledge; | 1 | package cn.iocoder.yudao.module.ai.service.knowledge; |
| 2 | 2 | ||
| 3 | -import cn.hutool.core.util.ObjUtil; | ||
| 4 | import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; | 3 | import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; |
| 5 | import cn.iocoder.yudao.framework.common.pojo.PageResult; | 4 | import cn.iocoder.yudao.framework.common.pojo.PageResult; |
| 6 | import cn.iocoder.yudao.framework.common.util.object.BeanUtils; | 5 | import cn.iocoder.yudao.framework.common.util.object.BeanUtils; |
| 7 | -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateReqVO; | ||
| 8 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO; | 6 | import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO; |
| 9 | -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateReqVO; | 7 | +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeSaveReqVO; |
| 10 | import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; | 8 | import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; |
| 11 | import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; | 9 | import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; |
| 12 | import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeMapper; | 10 | import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeMapper; |
| @@ -34,35 +32,30 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService { | @@ -34,35 +32,30 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService { | ||
| 34 | 32 | ||
| 35 | @Resource | 33 | @Resource |
| 36 | private AiChatModelService chatModelService; | 34 | private AiChatModelService chatModelService; |
| 37 | - @Resource | ||
| 38 | - private AiApiKeyService apiKeyService; | ||
| 39 | 35 | ||
| 40 | @Override | 36 | @Override |
| 41 | - public Long createKnowledge(AiKnowledgeCreateReqVO createReqVO, Long userId) { | 37 | + public Long createKnowledge(AiKnowledgeSaveReqVO createReqVO) { |
| 42 | // 1. 校验模型配置 | 38 | // 1. 校验模型配置 |
| 43 | - AiChatModelDO model = chatModelService.validateChatModel(createReqVO.getModelId()); | 39 | + AiChatModelDO model = chatModelService.validateChatModel(createReqVO.getEmbeddingModelId()); |
| 44 | 40 | ||
| 45 | // 2. 插入知识库 | 41 | // 2. 插入知识库 |
| 46 | - AiKnowledgeDO knowledgeBase = BeanUtils.toBean(createReqVO, AiKnowledgeDO.class) | ||
| 47 | - .setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus()); | ||
| 48 | - knowledgeMapper.insert(knowledgeBase); | ||
| 49 | - return knowledgeBase.getId(); | 42 | + AiKnowledgeDO knowledge = BeanUtils.toBean(createReqVO, AiKnowledgeDO.class) |
| 43 | + .setEmbeddingModel(model.getModel()).setStatus(CommonStatusEnum.ENABLE.getStatus()); | ||
| 44 | + knowledgeMapper.insert(knowledge); | ||
| 45 | + return knowledge.getId(); | ||
| 50 | } | 46 | } |
| 51 | 47 | ||
| 52 | @Override | 48 | @Override |
| 53 | - public void updateKnowledge(AiKnowledgeUpdateReqVO updateReqVO, Long userId) { | 49 | + public void updateKnowledge(AiKnowledgeSaveReqVO updateReqVO) { |
| 54 | // 1.1 校验知识库存在 | 50 | // 1.1 校验知识库存在 |
| 55 | - AiKnowledgeDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId()); | ||
| 56 | - if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) { | ||
| 57 | - throw exception(KNOWLEDGE_NOT_EXISTS); | ||
| 58 | - } | 51 | + validateKnowledgeExists(updateReqVO.getId()); |
| 59 | // 1.2 校验模型配置 | 52 | // 1.2 校验模型配置 |
| 60 | - AiChatModelDO model = chatModelService.validateChatModel(updateReqVO.getModelId()); | 53 | + AiChatModelDO model = chatModelService.validateChatModel(updateReqVO.getEmbeddingModelId()); |
| 61 | 54 | ||
| 62 | // 2. 更新知识库 | 55 | // 2. 更新知识库 |
| 63 | - AiKnowledgeDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeDO.class); | ||
| 64 | - updateDO.setModel(model.getModel()); | ||
| 65 | - knowledgeMapper.updateById(updateDO); | 56 | + AiKnowledgeDO updateObj = BeanUtils.toBean(updateReqVO, AiKnowledgeDO.class) |
| 57 | + .setEmbeddingModel(model.getModel()); | ||
| 58 | + knowledgeMapper.updateById(updateObj); | ||
| 66 | } | 59 | } |
| 67 | 60 | ||
| 68 | @Override | 61 | @Override |
| @@ -75,16 +68,8 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService { | @@ -75,16 +68,8 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService { | ||
| 75 | } | 68 | } |
| 76 | 69 | ||
| 77 | @Override | 70 | @Override |
| 78 | - public PageResult<AiKnowledgeDO> getKnowledgePage(Long userId, AiKnowledgePageReqVO pageReqVO) { | ||
| 79 | - return knowledgeMapper.selectPage(userId, pageReqVO); | ||
| 80 | - } | ||
| 81 | - | ||
| 82 | - @Override | ||
| 83 | - public VectorStore getVectorStoreById(Long id) { | ||
| 84 | - AiKnowledgeDO knowledge = validateKnowledgeExists(id); | ||
| 85 | - AiChatModelDO model = chatModelService.validateChatModel(knowledge.getModelId()); | ||
| 86 | - // 创建或获取 VectorStore 对象 | ||
| 87 | - return apiKeyService.getOrCreateVectorStore(model.getKeyId()); | 71 | + public PageResult<AiKnowledgeDO> getKnowledgePage(AiKnowledgePageReqVO pageReqVO) { |
| 72 | + return knowledgeMapper.selectPage(pageReqVO); | ||
| 88 | } | 73 | } |
| 89 | 74 | ||
| 90 | } | 75 | } |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiApiKeyService.java
| @@ -9,7 +9,6 @@ import cn.iocoder.yudao.module.ai.controller.admin.model.vo.apikey.AiApiKeySaveR | @@ -9,7 +9,6 @@ import cn.iocoder.yudao.module.ai.controller.admin.model.vo.apikey.AiApiKeySaveR | ||
| 9 | import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiApiKeyDO; | 9 | import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiApiKeyDO; |
| 10 | import jakarta.validation.Valid; | 10 | import jakarta.validation.Valid; |
| 11 | import org.springframework.ai.chat.model.ChatModel; | 11 | import org.springframework.ai.chat.model.ChatModel; |
| 12 | -import org.springframework.ai.embedding.EmbeddingModel; | ||
| 13 | import org.springframework.ai.image.ImageModel; | 12 | import org.springframework.ai.image.ImageModel; |
| 14 | import org.springframework.ai.vectorstore.VectorStore; | 13 | import org.springframework.ai.vectorstore.VectorStore; |
| 15 | 14 | ||
| @@ -114,19 +113,11 @@ public interface AiApiKeyService { | @@ -114,19 +113,11 @@ public interface AiApiKeyService { | ||
| 114 | SunoApi getSunoApi(); | 113 | SunoApi getSunoApi(); |
| 115 | 114 | ||
| 116 | /** | 115 | /** |
| 117 | - * 获得 EmbeddingModel 对象 | ||
| 118 | - * | ||
| 119 | - * @param id 编号 | ||
| 120 | - * @return EmbeddingModel 对象 | ||
| 121 | - */ | ||
| 122 | - EmbeddingModel getEmbeddingModel(Long id); | ||
| 123 | - | ||
| 124 | - /** | ||
| 125 | * 获得 VectorStore 对象 | 116 | * 获得 VectorStore 对象 |
| 126 | * | 117 | * |
| 127 | - * @param id 编号 | 118 | + * @param modelId 编号 |
| 128 | * @return VectorStore 对象 | 119 | * @return VectorStore 对象 |
| 129 | */ | 120 | */ |
| 130 | - VectorStore getOrCreateVectorStore(Long id); | 121 | + VectorStore getOrCreateVectorStoreByModelId(Long modelId); |
| 131 | 122 | ||
| 132 | } | 123 | } |
| 133 | \ No newline at end of file | 124 | \ No newline at end of file |
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiApiKeyServiceImpl.java
| @@ -10,12 +10,14 @@ import cn.iocoder.yudao.framework.common.util.object.BeanUtils; | @@ -10,12 +10,14 @@ import cn.iocoder.yudao.framework.common.util.object.BeanUtils; | ||
| 10 | import cn.iocoder.yudao.module.ai.controller.admin.model.vo.apikey.AiApiKeyPageReqVO; | 10 | import cn.iocoder.yudao.module.ai.controller.admin.model.vo.apikey.AiApiKeyPageReqVO; |
| 11 | import cn.iocoder.yudao.module.ai.controller.admin.model.vo.apikey.AiApiKeySaveReqVO; | 11 | import cn.iocoder.yudao.module.ai.controller.admin.model.vo.apikey.AiApiKeySaveReqVO; |
| 12 | import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiApiKeyDO; | 12 | import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiApiKeyDO; |
| 13 | +import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; | ||
| 13 | import cn.iocoder.yudao.module.ai.dal.mysql.model.AiApiKeyMapper; | 14 | import cn.iocoder.yudao.module.ai.dal.mysql.model.AiApiKeyMapper; |
| 14 | import jakarta.annotation.Resource; | 15 | import jakarta.annotation.Resource; |
| 15 | import org.springframework.ai.chat.model.ChatModel; | 16 | import org.springframework.ai.chat.model.ChatModel; |
| 16 | import org.springframework.ai.embedding.EmbeddingModel; | 17 | import org.springframework.ai.embedding.EmbeddingModel; |
| 17 | import org.springframework.ai.image.ImageModel; | 18 | import org.springframework.ai.image.ImageModel; |
| 18 | import org.springframework.ai.vectorstore.VectorStore; | 19 | import org.springframework.ai.vectorstore.VectorStore; |
| 20 | +import org.springframework.context.annotation.Lazy; | ||
| 19 | import org.springframework.stereotype.Service; | 21 | import org.springframework.stereotype.Service; |
| 20 | import org.springframework.validation.annotation.Validated; | 22 | import org.springframework.validation.annotation.Validated; |
| 21 | 23 | ||
| @@ -36,6 +38,11 @@ public class AiApiKeyServiceImpl implements AiApiKeyService { | @@ -36,6 +38,11 @@ public class AiApiKeyServiceImpl implements AiApiKeyService { | ||
| 36 | @Resource | 38 | @Resource |
| 37 | private AiApiKeyMapper apiKeyMapper; | 39 | private AiApiKeyMapper apiKeyMapper; |
| 38 | 40 | ||
| 41 | + // TODO @芋艿:后续要不要改? | ||
| 42 | + @Resource | ||
| 43 | + @Lazy // 延迟加载,解决渲染依赖 | ||
| 44 | + private AiChatModelService chatModelService; | ||
| 45 | + | ||
| 39 | @Resource | 46 | @Resource |
| 40 | private AiModelFactory modelFactory; | 47 | private AiModelFactory modelFactory; |
| 41 | 48 | ||
| @@ -136,18 +143,18 @@ public class AiApiKeyServiceImpl implements AiApiKeyService { | @@ -136,18 +143,18 @@ public class AiApiKeyServiceImpl implements AiApiKeyService { | ||
| 136 | } | 143 | } |
| 137 | 144 | ||
| 138 | @Override | 145 | @Override |
| 139 | - public EmbeddingModel getEmbeddingModel(Long id) { | ||
| 140 | - AiApiKeyDO apiKey = validateApiKey(id); | 146 | + public VectorStore getOrCreateVectorStoreByModelId(Long modelId) { |
| 147 | + // 获取模型 + 密钥 | ||
| 148 | + AiChatModelDO chatModel = chatModelService.validateChatModel(modelId); | ||
| 149 | + AiApiKeyDO apiKey = validateApiKey(chatModel.getKeyId()); | ||
| 141 | AiPlatformEnum platform = AiPlatformEnum.validatePlatform(apiKey.getPlatform()); | 150 | AiPlatformEnum platform = AiPlatformEnum.validatePlatform(apiKey.getPlatform()); |
| 142 | - return modelFactory.getOrCreateEmbeddingModel(platform, apiKey.getApiKey(), apiKey.getUrl()); | ||
| 143 | - } | ||
| 144 | 151 | ||
| 145 | - @Override | ||
| 146 | - public VectorStore getOrCreateVectorStore(Long id) { | ||
| 147 | - AiApiKeyDO apiKey = validateApiKey(id); | ||
| 148 | - AiPlatformEnum platform = AiPlatformEnum.validatePlatform(apiKey.getPlatform()); | 152 | + // 创建或获取 EmbeddingModel 对象 |
| 153 | + EmbeddingModel embeddingModel = modelFactory.getOrCreateEmbeddingModel(platform, apiKey.getApiKey(), | ||
| 154 | + apiKey.getUrl(), chatModel.getModel()); | ||
| 155 | + | ||
| 149 | // 创建或获取 VectorStore 对象 | 156 | // 创建或获取 VectorStore 对象 |
| 150 | - return modelFactory.getOrCreateVectorStore(getEmbeddingModel(id), platform, apiKey.getApiKey(), apiKey.getUrl()); | 157 | + return modelFactory.getOrCreateVectorStore(embeddingModel); |
| 151 | } | 158 | } |
| 152 | 159 | ||
| 153 | } | 160 | } |
| 154 | \ No newline at end of file | 161 | \ No newline at end of file |
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactory.java
| @@ -89,21 +89,19 @@ public interface AiModelFactory { | @@ -89,21 +89,19 @@ public interface AiModelFactory { | ||
| 89 | * @param platform 平台 | 89 | * @param platform 平台 |
| 90 | * @param apiKey API KEY | 90 | * @param apiKey API KEY |
| 91 | * @param url API URL | 91 | * @param url API URL |
| 92 | + * @param model 模型 | ||
| 92 | * @return ChatModel 对象 | 93 | * @return ChatModel 对象 |
| 93 | */ | 94 | */ |
| 94 | - EmbeddingModel getOrCreateEmbeddingModel(AiPlatformEnum platform, String apiKey, String url); | 95 | + EmbeddingModel getOrCreateEmbeddingModel(AiPlatformEnum platform, String apiKey, String url, String model); |
| 95 | 96 | ||
| 96 | /** | 97 | /** |
| 97 | * 基于指定配置,获得 VectorStore 对象 | 98 | * 基于指定配置,获得 VectorStore 对象 |
| 98 | * <p> | 99 | * <p> |
| 99 | * 如果不存在,则进行创建 | 100 | * 如果不存在,则进行创建 |
| 100 | * | 101 | * |
| 101 | - * @param embeddingModel 嵌入模型 | ||
| 102 | - * @param platform 平台 | ||
| 103 | - * @param apiKey API KEY | ||
| 104 | - * @param url API URL | 102 | + * @param embeddingModel 向量模型 |
| 105 | * @return VectorStore 对象 | 103 | * @return VectorStore 对象 |
| 106 | */ | 104 | */ |
| 107 | - VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url); | 105 | + VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel); |
| 108 | 106 | ||
| 109 | } | 107 | } |
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactoryImpl.java
| @@ -21,6 +21,7 @@ import com.alibaba.cloud.ai.dashscope.api.DashScopeApi; | @@ -21,6 +21,7 @@ import com.alibaba.cloud.ai.dashscope.api.DashScopeApi; | ||
| 21 | import com.alibaba.cloud.ai.dashscope.api.DashScopeImageApi; | 21 | import com.alibaba.cloud.ai.dashscope.api.DashScopeImageApi; |
| 22 | import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatModel; | 22 | import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatModel; |
| 23 | import com.alibaba.cloud.ai.dashscope.embedding.DashScopeEmbeddingModel; | 23 | import com.alibaba.cloud.ai.dashscope.embedding.DashScopeEmbeddingModel; |
| 24 | +import com.alibaba.cloud.ai.dashscope.embedding.DashScopeEmbeddingOptions; | ||
| 24 | import com.alibaba.cloud.ai.dashscope.image.DashScopeImageModel; | 25 | import com.alibaba.cloud.ai.dashscope.image.DashScopeImageModel; |
| 25 | import com.azure.ai.openai.OpenAIClientBuilder; | 26 | import com.azure.ai.openai.OpenAIClientBuilder; |
| 26 | import org.springframework.ai.autoconfigure.azure.openai.AzureOpenAiAutoConfiguration; | 27 | import org.springframework.ai.autoconfigure.azure.openai.AzureOpenAiAutoConfiguration; |
| @@ -33,10 +34,13 @@ import org.springframework.ai.autoconfigure.zhipuai.ZhiPuAiAutoConfiguration; | @@ -33,10 +34,13 @@ import org.springframework.ai.autoconfigure.zhipuai.ZhiPuAiAutoConfiguration; | ||
| 33 | import org.springframework.ai.autoconfigure.zhipuai.ZhiPuAiConnectionProperties; | 34 | import org.springframework.ai.autoconfigure.zhipuai.ZhiPuAiConnectionProperties; |
| 34 | import org.springframework.ai.azure.openai.AzureOpenAiChatModel; | 35 | import org.springframework.ai.azure.openai.AzureOpenAiChatModel; |
| 35 | import org.springframework.ai.chat.model.ChatModel; | 36 | import org.springframework.ai.chat.model.ChatModel; |
| 37 | +import org.springframework.ai.document.MetadataMode; | ||
| 36 | import org.springframework.ai.embedding.EmbeddingModel; | 38 | import org.springframework.ai.embedding.EmbeddingModel; |
| 37 | import org.springframework.ai.image.ImageModel; | 39 | import org.springframework.ai.image.ImageModel; |
| 38 | import org.springframework.ai.ollama.OllamaChatModel; | 40 | import org.springframework.ai.ollama.OllamaChatModel; |
| 41 | +import org.springframework.ai.ollama.OllamaEmbeddingModel; | ||
| 39 | import org.springframework.ai.ollama.api.OllamaApi; | 42 | import org.springframework.ai.ollama.api.OllamaApi; |
| 43 | +import org.springframework.ai.ollama.api.OllamaOptions; | ||
| 40 | import org.springframework.ai.openai.OpenAiChatModel; | 44 | import org.springframework.ai.openai.OpenAiChatModel; |
| 41 | import org.springframework.ai.openai.OpenAiImageModel; | 45 | import org.springframework.ai.openai.OpenAiImageModel; |
| 42 | import org.springframework.ai.openai.api.OpenAiApi; | 46 | import org.springframework.ai.openai.api.OpenAiApi; |
| @@ -184,13 +188,15 @@ public class AiModelFactoryImpl implements AiModelFactory { | @@ -184,13 +188,15 @@ public class AiModelFactoryImpl implements AiModelFactory { | ||
| 184 | } | 188 | } |
| 185 | 189 | ||
| 186 | @Override | 190 | @Override |
| 187 | - public EmbeddingModel getOrCreateEmbeddingModel(AiPlatformEnum platform, String apiKey, String url) { | ||
| 188 | - String cacheKey = buildClientCacheKey(EmbeddingModel.class, platform, apiKey, url); | 191 | + public EmbeddingModel getOrCreateEmbeddingModel(AiPlatformEnum platform, String apiKey, String url, String model) { |
| 192 | + String cacheKey = buildClientCacheKey(EmbeddingModel.class, platform, apiKey, url, model); | ||
| 189 | return Singleton.get(cacheKey, (Func0<EmbeddingModel>) () -> { | 193 | return Singleton.get(cacheKey, (Func0<EmbeddingModel>) () -> { |
| 190 | - // TODO @xin 先测试一个 | ||
| 191 | switch (platform) { | 194 | switch (platform) { |
| 192 | case TONG_YI: | 195 | case TONG_YI: |
| 193 | - return buildTongYiEmbeddingModel(apiKey); | 196 | + return buildTongYiEmbeddingModel(apiKey, model); |
| 197 | + case OLLAMA: | ||
| 198 | + return buildOllamaEmbeddingModel(url, model); | ||
| 199 | + // TODO @芋艿:各个平台的向量化能力; | ||
| 194 | default: | 200 | default: |
| 195 | throw new IllegalArgumentException(StrUtil.format("未知平台({})", platform)); | 201 | throw new IllegalArgumentException(StrUtil.format("未知平台({})", platform)); |
| 196 | } | 202 | } |
| @@ -198,13 +204,14 @@ public class AiModelFactoryImpl implements AiModelFactory { | @@ -198,13 +204,14 @@ public class AiModelFactoryImpl implements AiModelFactory { | ||
| 198 | } | 204 | } |
| 199 | 205 | ||
| 200 | @Override | 206 | @Override |
| 201 | - public VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url) { | ||
| 202 | - String cacheKey = buildClientCacheKey(VectorStore.class, platform, apiKey, url); | 207 | + public VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel) { |
| 208 | +// String cacheKey = buildClientCacheKey(VectorStore.class, platform, apiKey, url); | ||
| 209 | + String cacheKey = buildClientCacheKey(VectorStore.class, embeddingModel); | ||
| 203 | return Singleton.get(cacheKey, (Func0<VectorStore>) () -> { | 210 | return Singleton.get(cacheKey, (Func0<VectorStore>) () -> { |
| 204 | - String prefix = StrUtil.format("{}#{}:", platform.getPlatform(), apiKey); | ||
| 205 | // TODO @芋艿:先临时使用 store | 211 | // TODO @芋艿:先临时使用 store |
| 206 | return SimpleVectorStore.builder(embeddingModel).build(); | 212 | return SimpleVectorStore.builder(embeddingModel).build(); |
| 207 | // TODO @芋艿:@xin:后续看看,是不是切到阿里云之类的 | 213 | // TODO @芋艿:@xin:后续看看,是不是切到阿里云之类的 |
| 214 | +// String prefix = StrUtil.format("{}#{}:", platform.getPlatform(), apiKey); | ||
| 208 | // var config = RedisVectorStore.RedisVectorStoreConfig.builder() | 215 | // var config = RedisVectorStore.RedisVectorStoreConfig.builder() |
| 209 | // .withIndexName(cacheKey) | 216 | // .withIndexName(cacheKey) |
| 210 | // .withPrefix(prefix) | 217 | // .withPrefix(prefix) |
| @@ -388,9 +395,16 @@ public class AiModelFactoryImpl implements AiModelFactory { | @@ -388,9 +395,16 @@ public class AiModelFactoryImpl implements AiModelFactory { | ||
| 388 | /** | 395 | /** |
| 389 | * 可参考 {@link DashScopeAutoConfiguration} 的 dashscopeEmbeddingModel 方法 | 396 | * 可参考 {@link DashScopeAutoConfiguration} 的 dashscopeEmbeddingModel 方法 |
| 390 | */ | 397 | */ |
| 391 | - private EmbeddingModel buildTongYiEmbeddingModel(String apiKey) { | 398 | + private DashScopeEmbeddingModel buildTongYiEmbeddingModel(String apiKey, String model) { |
| 392 | DashScopeApi dashScopeApi = new DashScopeApi(apiKey); | 399 | DashScopeApi dashScopeApi = new DashScopeApi(apiKey); |
| 393 | - return new DashScopeEmbeddingModel(dashScopeApi); | 400 | + DashScopeEmbeddingOptions dashScopeEmbeddingOptions = DashScopeEmbeddingOptions.builder().withModel(model).build(); |
| 401 | + return new DashScopeEmbeddingModel(dashScopeApi, MetadataMode.EMBED, dashScopeEmbeddingOptions); | ||
| 402 | + } | ||
| 403 | + | ||
| 404 | + private OllamaEmbeddingModel buildOllamaEmbeddingModel(String url, String model) { | ||
| 405 | + OllamaApi ollamaApi = new OllamaApi(url); | ||
| 406 | + OllamaOptions ollamaOptions = OllamaOptions.builder().model(model).build(); | ||
| 407 | + return OllamaEmbeddingModel.builder().ollamaApi(ollamaApi).defaultOptions(ollamaOptions).build(); | ||
| 394 | } | 408 | } |
| 395 | 409 | ||
| 396 | } | 410 | } |