BJ_ZTEITS-Developers / parking-cloud-yudao

Commit 46f3790492492bfaee83cf33039a445de9787b6d

Authored by gaibu
1 parent a6c92816

fix: 解决 redis mq 消息丢失问题

Inline Side-by-side
Showing 2 changed files with 52 additions and 43 deletions
yudao-framework/yudao-spring-boot-starter-mq/src/main/java/cn/iocoder/yudao/framework/mq/config/YudaoMQAutoConfiguration.java
  View file @46f3790
... ... @@ -8,9 +8,11 @@ import cn.iocoder.yudao.framework.mq.core.RedisMQTemplate;
8 8 import cn.iocoder.yudao.framework.mq.core.interceptor.RedisMessageInterceptor;
9 9 import cn.iocoder.yudao.framework.mq.core.pubsub.AbstractChannelMessageListener;
10 10 import cn.iocoder.yudao.framework.mq.core.stream.AbstractStreamMessageListener;
11   -import cn.iocoder.yudao.framework.mq.scheduler.PendingMessageScheduler;
  11 +import cn.iocoder.yudao.framework.mq.scheduler.RedisPendingMessageResendJob;
12 12 import cn.iocoder.yudao.framework.redis.config.YudaoRedisAutoConfiguration;
13 13 import lombok.extern.slf4j.Slf4j;
  14 +import org.redisson.api.RedissonClient;
  15 +import org.springframework.beans.factory.annotation.Value;
14 16 import org.springframework.boot.autoconfigure.AutoConfiguration;
15 17 import org.springframework.context.annotation.Bean;
16 18 import org.springframework.data.redis.connection.RedisServerCommands;
... ... @@ -25,6 +27,7 @@ import org.springframework.data.redis.listener.ChannelTopic;
25 27 import org.springframework.data.redis.listener.RedisMessageListenerContainer;
26 28 import org.springframework.data.redis.stream.DefaultStreamMessageListenerContainerX;
27 29 import org.springframework.data.redis.stream.StreamMessageListenerContainer;
  30 +import org.springframework.scheduling.annotation.EnableScheduling;
28 31  
29 32 import java.util.List;
30 33 import java.util.Properties;
... ... @@ -35,6 +38,7 @@ import java.util.Properties;
35 38 * @author 芋道源码
36 39 */
37 40 @Slf4j
  41 +@EnableScheduling // 启用定时任务，用于 RedisPendingMessageResendJob 重发消息
38 42 @AutoConfiguration(after = YudaoRedisAutoConfiguration.class)
39 43 public class YudaoMQAutoConfiguration {
40 44  
... ... @@ -70,17 +74,19 @@ public class YudaoMQAutoConfiguration {
70 74 }
71 75  
72 76 /**
73   - *
74   - * @return
  77 + * 创建 Redis Stream 重新消费的任务
75 78 */
76 79 @Bean
77   - public PendingMessageScheduler pendingMessageScheduler(){
78   - return new PendingMessageScheduler();
  80 + public RedisPendingMessageResendJob redisPendingMessageResendJob(List<AbstractStreamMessageListener<?>> listeners,
  81 + RedisMQTemplate redisTemplate,
  82 + @Value("${spring.application.name}") String groupName,
  83 + RedissonClient redissonClient) {
  84 + return new RedisPendingMessageResendJob(listeners, redisTemplate, groupName, redissonClient);
79 85 }
80 86  
81 87 /**
82 88 * 创建 Redis Stream 集群消费的容器
83   - *
  89 + * <p>
84 90 * Redis Stream 的 xreadgroup 命令：https://www.geek-book.com/src/docs/redis/redis/redis.io/commands/xreadgroup.html
85 91 */
86 92 @Bean(initMethod = "start", destroyMethod = "stop")
... ... @@ -108,7 +114,8 @@ public class YudaoMQAutoConfiguration {
108 114 // 创建 listener 对应的消费者分组
109 115 try {
110 116 redisTemplate.opsForStream().createGroup(listener.getStreamKey(), listener.getGroup());
111   - } catch (Exception ignore) {}
  117 + } catch (Exception ignore) {
  118 + }
112 119 // 设置 listener 对应的 redisTemplate
113 120 listener.setRedisMQTemplate(redisMQTemplate);
114 121 // 创建 Consumer 对象
... ...
yudao-framework/yudao-spring-boot-starter-mq/src/main/java/cn/iocoder/yudao/framework/mq/scheduler/PendingMessageScheduler.java renamed to yudao-framework/yudao-spring-boot-starter-mq/src/main/java/cn/iocoder/yudao/framework/mq/scheduler/RedisPendingMessageResendJob.java
  View file @46f3790
1 1 package cn.iocoder.yudao.framework.mq.scheduler;
2 2  
  3 +import cn.hutool.core.collection.CollUtil;
3 4 import cn.iocoder.yudao.framework.mq.core.RedisMQTemplate;
4 5 import cn.iocoder.yudao.framework.mq.core.stream.AbstractStreamMessageListener;
5 6 import lombok.extern.slf4j.Slf4j;
6 7 import org.redisson.api.RLock;
7 8 import org.redisson.api.RedissonClient;
8   -import org.springframework.beans.factory.annotation.Autowired;
9   -import org.springframework.beans.factory.annotation.Value;
10 9 import org.springframework.data.redis.connection.stream.Consumer;
11 10 import org.springframework.data.redis.connection.stream.MapRecord;
12 11 import org.springframework.data.redis.connection.stream.PendingMessagesSummary;
... ... @@ -14,51 +13,53 @@ import org.springframework.data.redis.connection.stream.ReadOffset;
14 13 import org.springframework.data.redis.connection.stream.StreamOffset;
15 14 import org.springframework.data.redis.connection.stream.StreamRecords;
16 15 import org.springframework.data.redis.core.StreamOperations;
17   -import org.springframework.scheduling.annotation.EnableScheduling;
18 16 import org.springframework.scheduling.annotation.Scheduled;
19 17  
20 18 import java.util.List;
21 19 import java.util.Map;
22   -import java.util.concurrent.TimeUnit;
23 20  
24 21 /**
25   - * 这个定时器用于处理，crash 之后的消费者未消费完的消息
  22 + * 这个任务用于处理，crash 之后的消费者未消费完的消息
26 23 */
27 24 @Slf4j
28   -@EnableScheduling
29   -public class PendingMessageScheduler {
  25 +public class RedisPendingMessageResendJob {
30 26 private static final String LOCK_KEY = "redis:pending:msg:lock";
31   - @Autowired
32   - private List<AbstractStreamMessageListener<?>> listeners;
33   - @Autowired
34   - private RedisMQTemplate redisTemplate;
35   - @Value("${spring.application.name}")
36   - private String groupName;
37   - @Autowired
38   - private RedissonClient redissonClient;
  27 +
  28 + private final List<AbstractStreamMessageListener<?>> listeners;
  29 + private final RedisMQTemplate redisTemplate;
  30 + private final String groupName;
  31 + private final RedissonClient redissonClient;
  32 +
  33 + public RedisPendingMessageResendJob(List<AbstractStreamMessageListener<?>> listeners, RedisMQTemplate redisTemplate, String groupName, RedissonClient redissonClient) {
  34 + this.listeners = listeners;
  35 + this.redisTemplate = redisTemplate;
  36 + this.groupName = groupName;
  37 + this.redissonClient = redissonClient;
  38 + }
39 39  
40 40 /**
41   - * 一分钟执行一次
  41 + * 一分钟执行一次,这里选择每分钟的35秒执行，是为了避免整点任务过多的问题
42 42 */
43   - @Scheduled(fixedRate = 60 * 1000)
44   - public void processPendingMessage() {
45   - final RLock lock = redissonClient.getLock(LOCK_KEY);
46   - try {
47   - // 尝试加锁，最多等待 30 秒，上锁以后 60 秒自动解锁
48   - boolean lockFlag = lock.tryLock(30, 60, TimeUnit.SECONDS);
49   - if (lockFlag) {
  43 + @Scheduled(cron = "35 * * * * ?")
  44 + public void messageResend() {
  45 + RLock lock = redissonClient.getLock(LOCK_KEY);
  46 + log.info("[messageResend][尝试获取锁]");
  47 + // 尝试加锁
  48 + if (lock.tryLock()) {
  49 + try {
50 50 execute();
  51 + } catch (Exception ex) {
  52 + log.error("[messageResend][执行异常]", ex);
  53 + } finally {
  54 + lock.unlock();
51 55 }
52   - } catch (InterruptedException e) {
53   - log.error("获取锁失败", e);
54 56 }
55   -
56 57 }
57 58  
58 59 private void execute() {
59 60 StreamOperations<String, Object, Object> ops = redisTemplate.getRedisTemplate().opsForStream();
60 61  
61   - for (AbstractStreamMessageListener<?> listener : listeners) {
  62 + listeners.forEach(listener -> {
62 63 PendingMessagesSummary pendingMessagesSummary = ops.pending(listener.getStreamKey(), groupName);
63 64 // 每个消费者的pending消息数量
64 65 Map<String, Long> pendingMessagesPerConsumer = pendingMessagesSummary.getPendingMessagesPerConsumer();
... ... @@ -69,17 +70,18 @@ public class PendingMessageScheduler {
69 70  
70 71 // 从消费者的pending队列中读取消息
71 72 List<MapRecord<String, Object, Object>> retVal = ops.read(Consumer.from(groupName, consumerName), StreamOffset.create(listener.getStreamKey(), ReadOffset.from("0")));
  73 + if (CollUtil.isNotEmpty(retVal)) {
  74 + for (MapRecord<String, Object, Object> record : retVal) {
  75 + // 重新投递消息
  76 + redisTemplate.getRedisTemplate().opsForStream().add(StreamRecords.newRecord()
  77 + .ofObject(record.getValue()) // 设置内容
  78 + .withStreamKey(listener.getStreamKey()));
72 79  
73   - for (MapRecord<String, Object, Object> record : retVal) {
74   - // 重新投递消息
75   - redisTemplate.getRedisTemplate().opsForStream().add(StreamRecords.newRecord()
76   - .ofObject(record.getValue()) // 设置内容
77   - .withStreamKey(listener.getStreamKey()));
78   -
79   - // ack 消息消费完成
80   - redisTemplate.getRedisTemplate().opsForStream().acknowledge(groupName, record);
  80 + // ack 消息消费完成
  81 + redisTemplate.getRedisTemplate().opsForStream().acknowledge(groupName, record);
  82 + }
81 83 }
82 84 });
83   - }
  85 + });
84 86 }
85 87 }
... ...