Repository: apache/gravitino (Java)

Issue:
[Improvement] Suggestion for Handling Stale Metrics in Gravitino
### What would you like to be improved?

Issue Description
There is a current limitation in the gravitino_server_http_request_duration_seconds{operation="xxx", quantile="0.xx"} metrics within Gravitino. If an operation is not called for a prolonged period after its last execution, the metric continues to display the duration of the last call. This behavior can lead to misinterpretations of the system's performance, as it does not accurately reflect the inactivity.

Suggested Improvement
It would be beneficial to modify the metric behavior to either reset to zero or drop the metric data when no calls are made to an operation within a predefined timeout period. This change would provide a more accurate representation of the system's current state and activity levels.

<img width="1227" alt="image" src="https://github.com/datastrato/gravitino/assets/58762426/368b4c47-7c25-4fee-a55f-878ba54c3d2d">

@FANNG1 +cc

### How should we improve?

_No response_


Discussion: thanks for reporting this issue.
@TEOTEO520 ,  #3341 propose a way to compuate Pxx according to the time sliding window, you could check it when you are free. I'm not sure whether to make it configurable for now.



Fix (patch):
diff --git a/common/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java b/common/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java
index 2f548cf4fe5..d88e9682ece 100644
--- a/common/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java
+++ b/common/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java
@@ -32,4 +32,6 @@ private ConfigConstants() {}
   public static final String VERSION_0_4_0 = "0.4.0";
   /** The version number for the 0.5.0 release. */
   public static final String VERSION_0_5_0 = "0.5.0";
+  /** The version number for the 0.5.1 release. */
+  public static final String VERSION_0_5_1 = "0.5.1";
 }
diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java
index e78d8860338..26619f03464 100644
--- a/core/src/main/java/com/datastrato/gravitino/Configs.java
+++ b/core/src/main/java/com/datastrato/gravitino/Configs.java
@@ -276,4 +276,12 @@ private Configs() {}
           .version(ConfigConstants.VERSION_0_5_0)
           .longConf()
           .createWithDefault(60 * 60 * 1000L);
+
+  public static final int DEFAULT_METRICS_TIME_SLIDING_WINDOW_SECONDS = 60;
+  public static final ConfigEntry<Integer> METRICS_TIME_SLIDING_WINDOW_SECONDS =
+      new ConfigBuilder("gravitino.metrics.timeSlidingWindowSecs")
+          .doc("The seconds of Gravitino metrics time sliding window")
+          .version(ConfigConstants.VERSION_0_5_1)
+          .intConf()
+          .createWithDefault(DEFAULT_METRICS_TIME_SLIDING_WINDOW_SECONDS);
 }
diff --git a/core/src/main/java/com/datastrato/gravitino/metrics/source/MetricsSource.java b/core/src/main/java/com/datastrato/gravitino/metrics/source/MetricsSource.java
index 047f1f298da..4ff58eda0d4 100644
--- a/core/src/main/java/com/datastrato/gravitino/metrics/source/MetricsSource.java
+++ b/core/src/main/java/com/datastrato/gravitino/metrics/source/MetricsSource.java
@@ -9,7 +9,12 @@
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SlidingTimeWindowArrayReservoir;
 import com.codahale.metrics.Timer;
+import com.datastrato.gravitino.Config;
+import com.datastrato.gravitino.Configs;
+import com.datastrato.gravitino.GravitinoEnv;
+import java.util.concurrent.TimeUnit;
 
 /**
  * MetricsSource provides utilities to collect specified kind metrics, all metrics must create with
@@ -24,10 +29,19 @@ public abstract class MetricsSource {
   public static final String JVM_METRIC_NAME = "jvm";
   private final MetricRegistry metricRegistry;
   private final String metricsSourceName;
+  private final int timeSlidingWindowSeconds;
 
   protected MetricsSource(String name) {
     this.metricsSourceName = name;
     metricRegistry = new MetricRegistry();
+    Config config = GravitinoEnv.getInstance().config();
+    if (config != null) {
+      this.timeSlidingWindowSeconds =
+          config.get(Configs.METRICS_TIME_SLIDING_WINDOW_SECONDS).intValue();
+    } else {
+      // Couldn't get config when testing
+      this.timeSlidingWindowSeconds = Configs.DEFAULT_METRICS_TIME_SLIDING_WINDOW_SECONDS;
+    }
   }
 
   /**
@@ -75,7 +89,12 @@ public Counter getCounter(String name) {
    * @return a new or pre-existing Histogram
    */
   public Histogram getHistogram(String name) {
-    return this.metricRegistry.histogram(name);
+    return this.metricRegistry.histogram(
+        name,
+        () ->
+            new Histogram(
+                new SlidingTimeWindowArrayReservoir(
+                    getTimeSlidingWindowSeconds(), TimeUnit.SECONDS)));
   }
 
   /**
@@ -85,6 +104,15 @@ public Histogram getHistogram(String name) {
    * @return a new or pre-existing Timer
    */
   public Timer getTimer(String name) {
-    return this.metricRegistry.timer(name);
+    return this.metricRegistry.timer(
+        name,
+        () ->
+            new Timer(
+                new SlidingTimeWindowArrayReservoir(
+                    getTimeSlidingWindowSeconds(), TimeUnit.SECONDS)));
+  }
+
+  protected int getTimeSlidingWindowSeconds() {
+    return timeSlidingWindowSeconds;
   }
 }
diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md
index 91a000d62de..6078d1f5378 100644
--- a/docs/gravitino-server-config.md
+++ b/docs/gravitino-server-config.md
@@ -131,6 +131,12 @@ For more details, please refer to the definition of the plugin.
 
 Refer to [security](security.md) for HTTPS and authentication configurations.
 
+### Metrics configuration
+
+| Property name                             | Description                                          | Default value | Required | Since Version |
+|-------------------------------------------|------------------------------------------------------|---------------|----------|---------------|
+| `gravitino.metrics.timeSlidingWindowSecs` | The seconds of Gravitino metrics time sliding window | 60            | No       | 0.5.1         |
+
 ## Gravitino catalog properties configuration
 
 There are three types of catalog properties:
diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/HttpServerMetricsSource.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/HttpServerMetricsSource.java
index 068d17085fa..58964a7a9ee 100644
--- a/server-common/src/main/java/com/datastrato/gravitino/server/web/HttpServerMetricsSource.java
+++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/HttpServerMetricsSource.java
@@ -5,15 +5,25 @@
 
 package com.datastrato.gravitino.server.web;
 
+import com.codahale.metrics.Clock;
+import com.codahale.metrics.SlidingTimeWindowArrayReservoir;
 import com.codahale.metrics.jersey2.InstrumentedResourceMethodApplicationListener;
 import com.datastrato.gravitino.metrics.MetricNames;
 import com.datastrato.gravitino.metrics.source.MetricsSource;
+import java.util.concurrent.TimeUnit;
 import org.glassfish.jersey.server.ResourceConfig;
 
 public class HttpServerMetricsSource extends MetricsSource {
   public HttpServerMetricsSource(String name, ResourceConfig resourceConfig, JettyServer server) {
     super(name);
-    resourceConfig.register(new InstrumentedResourceMethodApplicationListener(getMetricRegistry()));
+    resourceConfig.register(
+        new InstrumentedResourceMethodApplicationListener(
+            getMetricRegistry(),
+            Clock.defaultClock(),
+            false,
+            () ->
+                new SlidingTimeWindowArrayReservoir(
+                    getTimeSlidingWindowSeconds(), TimeUnit.SECONDS)));
     registerGauge(
         MetricNames.SERVER_IDLE_THREAD_NUM, () -> server.getThreadPool().getIdleThreads());
   }