Repository: Alluxio/alluxio (language: java)

Issue:
Stale raft journal metrics after leadership change
**Alluxio Version:**

2.9.0-SNAPSHOT (873bcca82e8cc1abb421bbb316bd9ef99b306c16)

**Describe the bug**

After a leadership change, journal entries (which is read from metrics) will get stuck:
<img width="1296" alt="before" src="https://user-images.githubusercontent.com/5821159/210472225-b5855af5-ca70-49a3-8e09-a801c95cb16f.png">

(The WebUI is with the change in #16636)

In this screenshot, `localhost:19988` lost primacy, and its `Journal Entries` got stuck.

**To Reproduce**

1. Start a HA cluster
2. Use Ratis shell to transfer leadership
3. Perform some operations, to increase journal entries
4. Observe journal entries metrics in each master

**Expected behavior**

The master lost primacy should have its metrics up to date.

**Urgency**

Medium.

**Are you planning to fix it**

Yes.



Discussion / hints:

Fix patch:
diff --git a/core/server/common/src/main/java/alluxio/master/journal/raft/JournalStateMachine.java b/core/server/common/src/main/java/alluxio/master/journal/raft/JournalStateMachine.java
index a96f69717490..5088146a554a 100644
--- a/core/server/common/src/main/java/alluxio/master/journal/raft/JournalStateMachine.java
+++ b/core/server/common/src/main/java/alluxio/master/journal/raft/JournalStateMachine.java
@@ -367,6 +367,11 @@ public CompletableFuture<Message> query(Message request) {
   @Override
   public void close() {
     mClosed = true;
+    MetricsSystem.removeMetrics(MetricKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_LAST_INDEX.getName());
+    MetricsSystem.removeMetrics(MetricKey.MASTER_JOURNAL_ENTRIES_SINCE_CHECKPOINT.getName());
+    MetricsSystem.removeMetrics(MetricKey.MASTER_JOURNAL_LAST_CHECKPOINT_TIME.getName());
+    MetricsSystem.removeMetrics(MetricKey.MASTER_JOURNAL_LAST_APPLIED_COMMIT_INDEX.getName());
+    MetricsSystem.removeMetrics(MetricKey.MASTER_JOURNAL_CHECKPOINT_WARN.getName());
     synchronized (mSnapshotManager) {
       mSnapshotManager.notifyAll();
     }
diff --git a/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalSystem.java b/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalSystem.java
index eb744855c98c..7bdffcc363af 100644
--- a/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalSystem.java
+++ b/core/server/common/src/main/java/alluxio/master/journal/raft/RaftJournalSystem.java
@@ -1192,6 +1192,11 @@ synchronized RaftServer getRaftServer() {
     return mServer;
   }
 
+  @VisibleForTesting
+  ConcurrentHashMap<String, RaftJournal> getJournals() {
+    return mJournals;
+  }
+
   /**
    * Updates raft group with the current values from raft server.
    */


Test patch:
diff --git a/core/server/master/src/test/java/alluxio/master/journal/raft/RaftJournalSystemMetricsTest.java b/core/server/master/src/test/java/alluxio/master/journal/raft/RaftJournalSystemMetricsTest.java
index 858dc1a4ba20..4d89cf0c54b5 100644
--- a/core/server/master/src/test/java/alluxio/master/journal/raft/RaftJournalSystemMetricsTest.java
+++ b/core/server/master/src/test/java/alluxio/master/journal/raft/RaftJournalSystemMetricsTest.java
@@ -12,6 +12,8 @@
 package alluxio.master.journal.raft;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 
 import alluxio.conf.Configuration;
 import alluxio.conf.PropertyKey;
@@ -43,6 +45,39 @@ public void after() {
     Configuration.reloadProperties();
   }
 
+  @Test
+  public void journalStateMachineMetrics() throws Exception {
+    Configuration.set(PropertyKey.MASTER_EMBEDDED_JOURNAL_ADDRESSES,
+        "localhost:29200,localhost:29201,localhost:29202");
+    Configuration.set(PropertyKey.MASTER_HOSTNAME, "localhost");
+    Configuration.set(PropertyKey.MASTER_EMBEDDED_JOURNAL_PORT, 29200);
+    RaftJournalSystem system =
+        new RaftJournalSystem(mFolder.newFolder().toURI(), ServiceType.MASTER_RAFT);
+    String[] metricsNames = new String[] {
+        MetricKey.MASTER_EMBEDDED_JOURNAL_SNAPSHOT_LAST_INDEX.getName(),
+        MetricKey.MASTER_JOURNAL_ENTRIES_SINCE_CHECKPOINT.getName(),
+        MetricKey.MASTER_JOURNAL_LAST_CHECKPOINT_TIME.getName(),
+        MetricKey.MASTER_JOURNAL_LAST_APPLIED_COMMIT_INDEX.getName(),
+        MetricKey.MASTER_JOURNAL_CHECKPOINT_WARN.getName(),
+    };
+    JournalStateMachine stateMachine = new JournalStateMachine(system.getJournals(), system);
+    for (String name : metricsNames) {
+      assertNotNull(MetricsSystem.METRIC_REGISTRY.getGauges().get(name));
+    }
+    stateMachine.close();
+    for (String name : metricsNames) {
+      assertNull(MetricsSystem.METRIC_REGISTRY.getGauges().get(name));
+    }
+    JournalStateMachine newStateMachine = new JournalStateMachine(system.getJournals(), system);
+    for (String name : metricsNames) {
+      assertNotNull(MetricsSystem.METRIC_REGISTRY.getGauges().get(name));
+    }
+    newStateMachine.close();
+    for (String name : metricsNames) {
+      assertNull(MetricsSystem.METRIC_REGISTRY.getGauges().get(name));
+    }
+  }
+
   @Test
   public void metrics() throws Exception {
     Configuration.set(PropertyKey.MASTER_EMBEDDED_JOURNAL_ADDRESSES,