敏感词的本地缓存，使用 Job 轮询，替代 MQ 广播

YunaiV
1 parent 5e7e3d81
Showing 8 changed files with 48 additions and 103 deletions
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/dal/mysql/sensitiveword/SensitiveWordMapper.java
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/mq/consumer/sensitiveword/SensitiveWordRefreshConsumer.java
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/mq/message/sensitiveword/SensitiveWordRefreshMessage.java
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/mq/producer/package-info.java
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/mq/producer/sensitiveword/SensitiveWordProducer.java
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/service/sensitiveword/SensitiveWordService.java
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/service/sensitiveword/SensitiveWordServiceImpl.java
yudao-module-system/yudao-module-system-biz/src/test/java/cn/iocoder/yudao/module/system/service/sensitiveword/SensitiveWordServiceImplTest.java
@@ -7,7 +7,9 @@ import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.Sensitiv
 import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordPageReqVO;
 import cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword.SensitiveWordDO;
 import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Select;
+import java.time.LocalDateTime;
 import java.util.List;
 /**
@@ -40,4 +42,7 @@ public interface SensitiveWordMapper extends BaseMapperX&lt;SensitiveWordDO&gt; {
         return selectOne(SensitiveWordDO::getName, name);
     }
+    @Select("SELECT COUNT(*) FROM system_sensitive_word WHERE update_time > #{maxUpdateTime}")
+    Long selectCountByUpdateTimeGt(LocalDateTime maxTime);
+
 }
-package cn.iocoder.yudao.module.system.mq.consumer.sensitiveword;
-
-import cn.iocoder.yudao.framework.mq.core.pubsub.AbstractChannelMessageListener;
-import cn.iocoder.yudao.module.system.mq.message.sensitiveword.SensitiveWordRefreshMessage;
-import cn.iocoder.yudao.module.system.service.sensitiveword.SensitiveWordService;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.Resource;
-
-/**
- * 针对 {@link SensitiveWordRefreshMessage} 的消费者
- *
- * @author 芋道源码
- */
-@Component
-@Slf4j
-public class SensitiveWordRefreshConsumer extends AbstractChannelMessageListener<SensitiveWordRefreshMessage> {
-
-    @Resource
-    private SensitiveWordService sensitiveWordService;
-
-    @Override
-    public void onMessage(SensitiveWordRefreshMessage message) {
-        log.info("[onMessage][收到 SensitiveWord 刷新消息]");
-        sensitiveWordService.initLocalCache();
-    }
-
-}
-package cn.iocoder.yudao.module.system.mq.message.sensitiveword;
-
-import cn.iocoder.yudao.framework.mq.core.pubsub.AbstractChannelMessage;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-
-/**
- * 敏感词的刷新 Message
- */
-@Data
-@EqualsAndHashCode(callSuper = true)
-public class SensitiveWordRefreshMessage extends AbstractChannelMessage {
-
-    @Override
-    public String getChannel() {
-        return "system.sensitive-word.refresh";
-    }
-
-}
-/**
- * 占位
- */
-package cn.iocoder.yudao.module.system.mq.producer;
-package cn.iocoder.yudao.module.system.mq.producer.sensitiveword;
-
-import cn.iocoder.yudao.framework.mq.core.RedisMQTemplate;
-import cn.iocoder.yudao.module.system.mq.message.sensitiveword.SensitiveWordRefreshMessage;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.Resource;
-
-/**
- * 敏感词相关的 Producer
- */
-@Component
-public class SensitiveWordProducer {
-
-    @Resource
-    private RedisMQTemplate redisMQTemplate;
-
-    /**
-     * 发送 {@link SensitiveWordRefreshMessage} 消息
-     */
-    public void sendSensitiveWordRefreshMessage() {
-        SensitiveWordRefreshMessage message = new SensitiveWordRefreshMessage();
-        redisMQTemplate.send(message);
-    }
-
-}
@@ -19,11 +19,6 @@ import java.util.Set;
 public interface SensitiveWordService {
     /**
-     * 初始化本地缓存
-     */
-    void initLocalCache();
-
-    /**
      * 创建敏感词
      *
      * @param createReqVO 创建信息
@@ -11,21 +11,24 @@ import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.Sensitiv
 import cn.iocoder.yudao.module.system.convert.sensitiveword.SensitiveWordConvert;
 import cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword.SensitiveWordDO;
 import cn.iocoder.yudao.module.system.dal.mysql.sensitiveword.SensitiveWordMapper;
-import cn.iocoder.yudao.module.system.mq.producer.sensitiveword.SensitiveWordProducer;
 import cn.iocoder.yudao.module.system.util.collection.SimpleTrie;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 import org.springframework.validation.annotation.Validated;
 import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
+import java.time.LocalDateTime;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
 import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.filterList;
+import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.getMaxValue;
 import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_WORD_EXISTS;
 import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_WORD_NOT_EXISTS;
@@ -40,6 +43,11 @@ import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_
 public class SensitiveWordServiceImpl implements SensitiveWordService {
     /**
+     * 敏感词列表缓存
+     */
+    @Getter
+    private volatile List<SensitiveWordDO> sensitiveWordCache = Collections.emptyList();
+    /**
      * 敏感词标签缓存
      * key：敏感词编号 {@link SensitiveWordDO#getId()}
      * <p>
@@ -51,9 +59,6 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
     @Resource
     private SensitiveWordMapper sensitiveWordMapper;
-    @Resource
-    private SensitiveWordProducer sensitiveWordProducer;
-
     /**
      * 默认的敏感词的字典树，包含所有敏感词
      */
@@ -68,7 +73,6 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
     /**
      * 初始化缓存
      */
-    @Override
     @PostConstruct
     public void initLocalCache() {
         // 第一步：查询数据
@@ -80,6 +84,7 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
         Set<String> tags = new HashSet<>();
         sensitiveWords.forEach(word -> tags.addAll(word.getTags()));
         sensitiveWordTagsCache = tags;
+        sensitiveWordCache = sensitiveWords;
         // 写入 defaultSensitiveWordTrie、tagSensitiveWordTries 缓存
         initSensitiveWordTrie(sensitiveWords);
     }
@@ -105,6 +110,26 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
         this.tagSensitiveWordTries = tagSensitiveWordTries;
     }
+    /**
+     * 通过定时任务轮询，刷新缓存
+     *
+     * 目的：多节点部署时，通过轮询”通知“所有节点，进行刷新
+     */
+    @Scheduled(initialDelay = 60, fixedRate = 60, timeUnit = TimeUnit.SECONDS)
+    public void refreshLocalCache() {
+        // 情况一：如果缓存里没有数据，则直接刷新缓存
+        if (CollUtil.isEmpty(sensitiveWordCache)) {
+            initLocalCache();
+            return;
+        }
+
+        // 情况二，如果缓存里数据，则通过 updateTime 判断是否有数据变更，有变更则刷新缓存
+        LocalDateTime maxTime = getMaxValue(sensitiveWordCache, SensitiveWordDO::getUpdateTime);
+        if (sensitiveWordMapper.selectCountByUpdateTimeGt(maxTime) > 0) {
+            initLocalCache();
+        }
+    }
+
     @Override
     public Long createSensitiveWord(SensitiveWordCreateReqVO createReqVO) {
         // 校验唯一性
@@ -113,8 +138,9 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
         // 插入
         SensitiveWordDO sensitiveWord = SensitiveWordConvert.INSTANCE.convert(createReqVO);
         sensitiveWordMapper.insert(sensitiveWord);
-        // 发送消息，刷新缓存
-        sensitiveWordProducer.sendSensitiveWordRefreshMessage();
+
+        // 刷新缓存
+        initLocalCache();
         return sensitiveWord.getId();
     }
@@ -127,8 +153,9 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
         // 更新
         SensitiveWordDO updateObj = SensitiveWordConvert.INSTANCE.convert(updateReqVO);
         sensitiveWordMapper.updateById(updateObj);
-        // 发送消息，刷新缓存
-        sensitiveWordProducer.sendSensitiveWordRefreshMessage();
+
+        // 刷新缓存
+        initLocalCache();
     }
     @Override
@@ -137,8 +164,9 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
         validateSensitiveWordExists(id);
         // 删除
         sensitiveWordMapper.deleteById(id);
-        // 发送消息，刷新缓存
-        sensitiveWordProducer.sendSensitiveWordRefreshMessage();
+
+        // 刷新缓存
+        initLocalCache();
     }
     private void validateSensitiveWordNameUnique(Long id, String name) {
@@ -10,9 +10,7 @@ import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.Sensitiv
 import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordUpdateReqVO;
 import cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword.SensitiveWordDO;
 import cn.iocoder.yudao.module.system.dal.mysql.sensitiveword.SensitiveWordMapper;
-import cn.iocoder.yudao.module.system.mq.producer.sensitiveword.SensitiveWordProducer;
 import org.junit.jupiter.api.Test;
-import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Import;
 import javax.annotation.Resource;
@@ -29,7 +27,6 @@ import static cn.iocoder.yudao.framework.test.core.util.RandomUtils.randomPojo;
 import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_WORD_NOT_EXISTS;
 import static java.util.Collections.singletonList;
 import static org.junit.jupiter.api.Assertions.*;
-import static org.mockito.Mockito.verify;
 /**
  * {@link SensitiveWordServiceImpl} 的单元测试类
@@ -45,9 +42,6 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
     @Resource
     private SensitiveWordMapper sensitiveWordMapper;
-    @MockBean
-    private SensitiveWordProducer sensitiveWordProducer;
-
     @Test
     public void testInitLocalCache() {
         SensitiveWordDO wordDO1 = randomPojo(SensitiveWordDO.class, o -> o.setName("傻瓜")
@@ -61,6 +55,10 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
         sensitiveWordService.initLocalCache();
         // 断言 sensitiveWordTagsCache 缓存
         assertEquals(SetUtils.asSet("论坛", "蔬菜"), sensitiveWordService.getSensitiveWordTagSet());
+        // 断言 sensitiveWordCache
+        assertEquals(2, sensitiveWordService.getSensitiveWordCache().size());
+        assertPojoEquals(wordDO1, sensitiveWordService.getSensitiveWordCache().get(0));
+        assertPojoEquals(wordDO2, sensitiveWordService.getSensitiveWordCache().get(1));
         // 断言 tagSensitiveWordTries 缓存
         assertNotNull(sensitiveWordService.getDefaultSensitiveWordTrie());
         assertEquals(2, sensitiveWordService.getTagSensitiveWordTries().size());
@@ -80,7 +78,6 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
         // 校验记录的属性是否正确
         SensitiveWordDO sensitiveWord = sensitiveWordMapper.selectById(sensitiveWordId);
         assertPojoEquals(reqVO, sensitiveWord);
-        verify(sensitiveWordProducer).sendSensitiveWordRefreshMessage();
     }
     @Test
@@ -98,7 +95,6 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
         // 校验是否更新正确
         SensitiveWordDO sensitiveWord = sensitiveWordMapper.selectById(reqVO.getId()); // 获取最新的
         assertPojoEquals(reqVO, sensitiveWord);
-        verify(sensitiveWordProducer).sendSensitiveWordRefreshMessage();
     }
     @Test
@@ -122,7 +118,6 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
         sensitiveWordService.deleteSensitiveWord(id);
         // 校验数据不存在了
         assertNull(sensitiveWordMapper.selectById(id));
-        verify(sensitiveWordProducer).sendSensitiveWordRefreshMessage();
     }
     @Test