Repository: apache/rocketmq (Java)
Date: 2023-10-09 03:28:36+00:00
Task type: Bug Fix

Issue:
[Bug] The channel map was not cleared properly in proxy
### Before Creating the Bug Report

- [X] I found a bug, not just asking a question, which should be created in [GitHub Discussions](https://github.com/apache/rocketmq/discussions).

- [X] I have searched the [GitHub Issues](https://github.com/apache/rocketmq/issues) and [GitHub Discussions](https://github.com/apache/rocketmq/discussions)  of this repository and believe that this is not a duplicate.

- [X] I have confirmed that this bug belongs to the current repository, not other repositories of RocketMQ.


### Runtime platform environment

CentOS

### RocketMQ version

develop

### JDK Version

_No response_

### Describe the Bug

The remoteChannelMap of HeartbeatSyncer was not cleared properly.

### Steps to Reproduce

Start two proxies in cluster mode, start a consumer and then shut down the consumer.

### What Did You Expect to See?

The remoteChannelMap of HeartbeatSyncer was cleared properly.

### What Did You See Instead?

The remoteChannelMap of HeartbeatSyncer was not cleared properly.

### Additional Context

_No response_


Issue discussion:

Solution patch:
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
@@ -18,6 +18,7 @@
 package org.apache.rocketmq.proxy.service.sysmessage;
 
 import com.alibaba.fastjson.JSON;
+import io.netty.channel.Channel;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
@@ -73,16 +74,8 @@ protected void init() {
         );
         this.consumerManager.appendConsumerIdsChangeListener(new ConsumerIdsChangeListener() {
             @Override
-            public void handle(ConsumerGroupEvent event, String s, Object... args) {
-                if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) {
-                    if (args == null || args.length < 1) {
-                        return;
-                    }
-                    if (args[0] instanceof ClientChannelInfo) {
-                        ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
-                        remoteChannelMap.remove(clientChannelInfo.getChannel().id().asLongText());
-                    }
-                }
+            public void handle(ConsumerGroupEvent event, String group, Object... args) {
+                processConsumerGroupEvent(event, group, args);
             }
 
             @Override
@@ -98,6 +91,18 @@ public void shutdown() throws Exception {
         super.shutdown();
     }
 
+    protected void processConsumerGroupEvent(ConsumerGroupEvent event, String group, Object... args) {
+        if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) {
+            if (args == null || args.length < 1) {
+                return;
+            }
+            if (args[0] instanceof ClientChannelInfo) {
+                ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
+                remoteChannelMap.remove(buildKey(group, clientChannelInfo.getChannel()));
+            }
+        }
+    }
+
     public void onConsumerRegister(String consumerGroup, ClientChannelInfo clientChannelInfo,
         ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
         Set<SubscriptionData> subList) {
@@ -189,7 +194,7 @@ public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeCo
                 }
 
                 RemoteChannel decodedChannel = RemoteChannel.decode(data.getChannelData());
-                RemoteChannel channel = remoteChannelMap.computeIfAbsent(data.getGroup() + "@" + decodedChannel.id().asLongText(), key -> decodedChannel);
+                RemoteChannel channel = remoteChannelMap.computeIfAbsent(buildKey(data.getGroup(), decodedChannel), key -> decodedChannel);
                 channel.setExtendAttribute(decodedChannel.getChannelExtendAttribute());
                 ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
                     channel,
@@ -228,4 +233,8 @@ private String buildLocalProxyId() {
         // use local address, remoting port and grpc port to build unique local proxy Id
         return proxyConfig.getLocalServeAddr() + "%" + proxyConfig.getRemotingListenPort() + "%" + proxyConfig.getGrpcServerPort();
     }
+
+    private static String buildKey(String group, Channel channel) {
+        return group + "@" + channel.id().asLongText();
+    }
 }


Test patch:
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
@@ -27,6 +27,7 @@
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelId;
 import java.time.Duration;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -35,6 +36,7 @@
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.client.ConsumerGroupEvent;
 import org.apache.rocketmq.broker.client.ConsumerManager;
 import org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt;
 import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
@@ -320,6 +322,72 @@ public void testSyncRemotingChannel() throws Exception {
         }
     }
 
+    @Test
+    public void testProcessConsumerGroupEventForRemoting() {
+        String consumerGroup = "consumerGroup";
+        Channel channel = createMockChannel();
+        RemotingProxyOutClient remotingProxyOutClient = mock(RemotingProxyOutClient.class);
+        RemotingChannel remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, Collections.emptySet());
+        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+            remotingChannel,
+            clientId,
+            LanguageCode.JAVA,
+            4
+        );
+
+        testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo);
+    }
+
+    @Test
+    public void testProcessConsumerGroupEventForGrpcV2() {
+        String consumerGroup = "consumerGroup";
+        GrpcClientSettingsManager grpcClientSettingsManager = mock(GrpcClientSettingsManager.class);
+        GrpcChannelManager grpcChannelManager = mock(GrpcChannelManager.class);
+        GrpcClientChannel grpcClientChannel = new GrpcClientChannel(
+            proxyRelayService, grpcClientSettingsManager, grpcChannelManager,
+            ProxyContext.create().setRemoteAddress(remoteAddress).setLocalAddress(localAddress),
+            clientId);
+        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+            grpcClientChannel,
+            clientId,
+            LanguageCode.JAVA,
+            5
+        );
+
+        testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo);
+    }
+
+    private void testProcessConsumerGroupEvent(String consumerGroup, ClientChannelInfo clientChannelInfo) {
+        HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory, null);
+        SendResult okSendResult = new SendResult();
+        okSendResult.setSendStatus(SendStatus.SEND_OK);
+
+        ArgumentCaptor<Message> messageArgumentCaptor = ArgumentCaptor.forClass(Message.class);
+        doReturn(CompletableFuture.completedFuture(okSendResult)).when(this.mqClientAPIExt)
+            .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong());
+
+        heartbeatSyncer.onConsumerRegister(
+            consumerGroup,
+            clientChannelInfo,
+            ConsumeType.CONSUME_PASSIVELY,
+            MessageModel.CLUSTERING,
+            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,
+            Collections.emptySet()
+        );
+        await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 1);
+
+        // change local serve addr, to simulate other proxy receive messages
+        heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10);
+        ArgumentCaptor<ClientChannelInfo> channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);
+        doReturn(true).when(consumerManager).registerConsumer(anyString(), channelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean());
+
+        heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null);
+        assertEquals(1, heartbeatSyncer.remoteChannelMap.size());
+
+        heartbeatSyncer.processConsumerGroupEvent(ConsumerGroupEvent.CLIENT_UNREGISTER, consumerGroup, channelInfoArgumentCaptor.getValue());
+        assertTrue(heartbeatSyncer.remoteChannelMap.isEmpty());
+    }
+
     private MessageExt convertFromMessage(Message message) {
         MessageExt messageExt = new MessageExt();
         messageExt.setTopic(message.getTopic());