BJ_ZTEITS-Developers / parking-cloud-yudao

Commit 361e50e5de3abd4551e05ba974ffc3c27d26c2c4

Authored by YunaiV
1 parent d7656535

feat：Redis Stream 增加清理 Job，避免占用内存过多

Inline Side-by-side
Showing 3 changed files with 87 additions and 3 deletions
yudao-framework/yudao-spring-boot-starter-mq/src/main/java/cn/iocoder/yudao/framework/mq/redis/config/YudaoRedisMQConsumerAutoConfiguration.java
  View file @361e50e
... ... @@ -6,6 +6,7 @@ import cn.hutool.system.SystemUtil;
6 6 import cn.iocoder.yudao.framework.common.enums.DocumentEnum;
7 7 import cn.iocoder.yudao.framework.mq.redis.core.RedisMQTemplate;
8 8 import cn.iocoder.yudao.framework.mq.redis.core.job.RedisPendingMessageResendJob;
  9 +import cn.iocoder.yudao.framework.mq.redis.core.job.RedisStreamMessageCleanupJob;
9 10 import cn.iocoder.yudao.framework.mq.redis.core.pubsub.AbstractRedisChannelMessageListener;
10 11 import cn.iocoder.yudao.framework.mq.redis.core.stream.AbstractRedisStreamMessageListener;
11 12 import cn.iocoder.yudao.framework.redis.config.YudaoRedisAutoConfiguration;
... ... @@ -74,6 +75,17 @@ public class YudaoRedisMQConsumerAutoConfiguration {
74 75 }
75 76  
76 77 /**
  78 + * 创建 Redis Stream 消息清理任务
  79 + */
  80 + @Bean
  81 + @ConditionalOnBean(AbstractRedisStreamMessageListener.class)
  82 + public RedisStreamMessageCleanupJob redisStreamMessageCleanupJob(List<AbstractRedisStreamMessageListener<?>> listeners,
  83 + RedisMQTemplate redisTemplate,
  84 + RedissonClient redissonClient) {
  85 + return new RedisStreamMessageCleanupJob(listeners, redisTemplate, redissonClient);
  86 + }
  87 +
  88 + /**
77 89 * 创建 Redis Stream 集群消费的容器
78 90 *
79 91 * 基础知识：<a href="https://www.geek-book.com/src/docs/redis/redis/redis.io/commands/xreadgroup.html">Redis Stream 的 xreadgroup 命令</a>
... ...
yudao-framework/yudao-spring-boot-starter-mq/src/main/java/cn/iocoder/yudao/framework/mq/redis/core/job/RedisPendingMessageResendJob.java
  View file @361e50e
... ... @@ -23,13 +23,13 @@ import java.util.Objects;
23 23 @AllArgsConstructor
24 24 public class RedisPendingMessageResendJob {
25 25  
26   - private static final String LOCK_KEY = "redis:pending:msg:lock";
  26 + private static final String LOCK_KEY = "redis:stream:pending-message-resend:lock";
27 27  
28 28 /**
29 29 * 消息超时时间，默认 5 分钟
30 30 *
31 31 * 1. 超时的消息才会被重新投递
32   - * 2. 由于定时任务 1 分钟一次，消息超时后不会被立即重投，极端情况下消息5分钟过期后，再等 1 分钟才会被扫瞄到
  32 + * 2. 由于定时任务 1 分钟一次，消息超时后不会被立即重投，极端情况下消息 5 分钟过期后，再等 1 分钟才会被扫瞄到
33 33 */
34 34 private static final int EXPIRE_TIME = 5 * 60;
35 35  
... ... @@ -39,7 +39,7 @@ public class RedisPendingMessageResendJob {
39 39 private final RedissonClient redissonClient;
40 40  
41 41 /**
42   - * 一分钟执行一次,这里选择每分钟的35秒执行，是为了避免整点任务过多的问题
  42 + * 一分钟执行一次,这里选择每分钟的 35 秒执行，是为了避免整点任务过多的问题
43 43 */
44 44 @Scheduled(cron = "35 * * * * ?")
45 45 public void messageResend() {
... ...
yudao-framework/yudao-spring-boot-starter-mq/src/main/java/cn/iocoder/yudao/framework/mq/redis/core/job/RedisStreamMessageCleanupJob.java 0 → 100644
  View file @361e50e
  1 +package cn.iocoder.yudao.framework.mq.redis.core.job;
  2 +
  3 +import cn.iocoder.yudao.framework.mq.redis.core.RedisMQTemplate;
  4 +import cn.iocoder.yudao.framework.mq.redis.core.stream.AbstractRedisStreamMessageListener;
  5 +import lombok.AllArgsConstructor;
  6 +import lombok.extern.slf4j.Slf4j;
  7 +import org.redisson.api.RLock;
  8 +import org.redisson.api.RedissonClient;
  9 +import org.springframework.data.redis.core.StreamOperations;
  10 +import org.springframework.scheduling.annotation.Scheduled;
  11 +
  12 +import java.util.List;
  13 +
  14 +/**
  15 + * Redis Stream 消息清理任务
  16 + * 用于定期清理已消费的消息，防止内存占用过大
  17 + *
  18 + * @see <a href="https://www.cnblogs.com/nanxiang/p/16179519.html">记一次 redis stream 数据类型内存不释放问题</a>
  19 + *
  20 + * @author 芋道源码
  21 + */
  22 +@Slf4j
  23 +@AllArgsConstructor
  24 +public class RedisStreamMessageCleanupJob {
  25 +
  26 + private static final String LOCK_KEY = "redis:stream:message-cleanup:lock";
  27 +
  28 + /**
  29 + * 保留的消息数量，默认保留最近 10000 条消息
  30 + */
  31 + private static final long MAX_COUNT = 10000;
  32 +
  33 + private final List<AbstractRedisStreamMessageListener<?>> listeners;
  34 + private final RedisMQTemplate redisTemplate;
  35 + private final RedissonClient redissonClient;
  36 +
  37 + /**
  38 + * 每小时执行一次清理任务
  39 + */
  40 + @Scheduled(cron = "0 0 * * * ?")
  41 + public void cleanup() {
  42 + RLock lock = redissonClient.getLock(LOCK_KEY);
  43 + // 尝试加锁
  44 + if (lock.tryLock()) {
  45 + try {
  46 + execute();
  47 + } catch (Exception ex) {
  48 + log.error("[cleanup][执行异常]", ex);
  49 + } finally {
  50 + lock.unlock();
  51 + }
  52 + }
  53 + }
  54 +
  55 + /**
  56 + * 执行清理逻辑
  57 + */
  58 + private void execute() {
  59 + StreamOperations<String, Object, Object> ops = redisTemplate.getRedisTemplate().opsForStream();
  60 + listeners.forEach(listener -> {
  61 + try {
  62 + // 使用 XTRIM 命令清理消息，只保留最近的 MAX_LEN 条消息
  63 + Long trimCount = ops.trim(listener.getStreamKey(), MAX_COUNT, true);
  64 + if (trimCount != null && trimCount > 0) {
  65 + log.info("[execute][Stream({}) 清理消息数量({})]", listener.getStreamKey(), trimCount);
  66 + }
  67 + } catch (Exception ex) {
  68 + log.error("[execute][Stream({}) 清理异常]", listener.getStreamKey(), ex);
  69 + }
  70 + });
  71 + }
  72 +}
0 73 \ No newline at end of file
... ...