BJ_ZTEITS-Developers / parking-cloud-yudao

Commit d698ef70b5e8a0a5ca038b4fb7b881dedfe48902

Authored by xiaoxin
1 parent 2a984504

【新增】AI 知识库：支持读取 PDF、DOC/DOCX、PPT/PPTX 和 HTML等格式

Inline Side-by-side
Showing 3 changed files with 44 additions and 2 deletions
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java
  View file @d698ef7
... ... @@ -3,7 +3,7 @@ package cn.iocoder.yudao.module.ai.service.knowledge;
3 3 import jakarta.annotation.Resource;
4 4 import lombok.extern.slf4j.Slf4j;
5 5 import org.springframework.ai.document.Document;
6   -import org.springframework.ai.reader.TextReader;
  6 +import org.springframework.ai.reader.tika.TikaDocumentReader;
7 7 import org.springframework.ai.transformer.splitter.TokenTextSplitter;
8 8 import org.springframework.ai.vectorstore.RedisVectorStore;
9 9 import org.springframework.beans.factory.annotation.Value;
... ... @@ -34,7 +34,7 @@ public class DocServiceImpl implements DocService {
34 34 public void embeddingDoc() {
35 35 // 读取文件
36 36 org.springframework.core.io.Resource file = data;
37   - TextReader loader = new TextReader(file);
  37 + TikaDocumentReader loader = new TikaDocumentReader(file);
38 38 List<Document> documents = loader.get();
39 39 // 文档分段
40 40 List<Document> segments = tokenTextSplitter.apply(documents);
... ...
yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml
  View file @d698ef7
... ... @@ -39,6 +39,7 @@
39 39 <artifactId>spring-ai-stability-ai-spring-boot-starter</artifactId>
40 40 <version>${spring-ai.version}</version>
41 41 </dependency>
  42 +
42 43 <dependency>
43 44 <groupId>org.springframework.ai</groupId>
44 45 <artifactId>spring-ai-transformers-spring-boot-starter</artifactId>
... ... @@ -46,6 +47,11 @@
46 47 </dependency>
47 48 <dependency>
48 49 <groupId>org.springframework.ai</groupId>
  50 + <artifactId>spring-ai-tika-document-reader</artifactId>
  51 + <version>${spring-ai.version}</version>
  52 + </dependency>
  53 + <dependency>
  54 + <groupId>org.springframework.ai</groupId>
49 55 <artifactId>spring-ai-redis-store</artifactId>
50 56 <version>${spring-ai.version}</version>
51 57 </dependency>
... ...
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java
  View file @d698ef7
... ... @@ -10,11 +10,18 @@ import cn.iocoder.yudao.framework.ai.core.model.xinghuo.XingHuoChatModel;
10 10 import cn.iocoder.yudao.framework.ai.core.model.xinghuo.XingHuoChatOptions;
11 11 import com.alibaba.cloud.ai.tongyi.TongYiAutoConfiguration;
12 12 import lombok.extern.slf4j.Slf4j;
  13 +import org.springframework.ai.autoconfigure.vectorstore.redis.RedisVectorStoreProperties;
  14 +import org.springframework.ai.document.MetadataMode;
  15 +import org.springframework.ai.transformer.splitter.TokenTextSplitter;
  16 +import org.springframework.ai.transformers.TransformersEmbeddingModel;
  17 +import org.springframework.ai.vectorstore.RedisVectorStore;
13 18 import org.springframework.boot.autoconfigure.AutoConfiguration;
14 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
  20 +import org.springframework.boot.autoconfigure.data.redis.RedisProperties;
15 21 import org.springframework.boot.context.properties.EnableConfigurationProperties;
16 22 import org.springframework.context.annotation.Bean;
17 23 import org.springframework.context.annotation.Import;
  24 +import redis.clients.jedis.JedisPooled;
18 25  
19 26 /**
20 27 * 芋道 AI 自动配置
... ... @@ -73,4 +80,33 @@ public class YudaoAiAutoConfiguration {
73 80 return new SunoApi(yudaoAiProperties.getSuno().getBaseUrl());
74 81 }
75 82  
  83 + // ========== rag 相关 ==========
  84 + @Bean
  85 + public TransformersEmbeddingModel transformersEmbeddingClient() {
  86 + return new TransformersEmbeddingModel(MetadataMode.EMBED);
  87 + }
  88 +
  89 + /**
  90 + * 我们启动有加载很多 Embedding 模型，不晓得取哪个好，先 new 个 TransformersEmbeddingModel 跑
  91 + */
  92 + @Bean
  93 + public RedisVectorStore vectorStore(TransformersEmbeddingModel transformersEmbeddingModel, RedisVectorStoreProperties properties,
  94 + RedisProperties redisProperties) {
  95 + var config = RedisVectorStore.RedisVectorStoreConfig.builder()
  96 + .withIndexName(properties.getIndex())
  97 + .withPrefix(properties.getPrefix())
  98 + .build();
  99 +
  100 + RedisVectorStore redisVectorStore = new RedisVectorStore(config, transformersEmbeddingModel,
  101 + new JedisPooled(redisProperties.getHost(), redisProperties.getPort()),
  102 + properties.isInitializeSchema());
  103 + redisVectorStore.afterPropertiesSet();
  104 + return redisVectorStore;
  105 + }
  106 +
  107 + @Bean
  108 + public TokenTextSplitter tokenTextSplitter() {
  109 + return new TokenTextSplitter(500, 100, 5, 10000, true);
  110 + }
  111 +
76 112 }
77 113 \ No newline at end of file
... ...