Repository: quarkusio/quarkus
Issue: Trailing comma is lost from prometheus metrics

### Describe the bug

In all released versions (including 2.15.3.Final), prometheus metrics (accessible at 'q/metrics') has format of json object with a trailing comma. In the current main branch (tested on dc85cd0d16c44a264507529c9ee26e26c7ffe175) the format is changed, and the comma was lost. Is this an expected change? Prometheus format allows[1] for both options, but that change may broke parsing of logs and therefore it would be nice to have it documented


[1] https://prometheus.io/docs/prometheus/latest/querying/operators/#aggregation-operators

### Expected behavior

Example:
`http_server_requests_seconds_count{method="GET",outcome="SUCCESS",status="200",uri="/hello",} 1.0`


### Actual behavior

`http_server_requests_seconds_count{method="GET",outcome="SUCCESS",status="200",uri="/hello"} 6.0`

### How to Reproduce?

1. `mvn io.quarkus.platform:quarkus-maven-plugin:2.15.3.Final:create     -DprojectGroupId=org.acme     -DprojectArtifactId=micrometer-quickstart     -Dextensions='resteasy-reactive,micrometer-registry-prometheus'`
2. `cd  micrometer-quickstart`
3. `mvn quarkus:dev -Dquarkus.platform.version=999-SNAPSHOT`
4. `curl http://localhost:8080/hello/`
5. `curl http://localhost:8080/q/metrics | grep http_server_requests_seconds_count # example, many (all?) other metrics  also changed`

for comparision, replace step 3 with `mvn quarkus:dev -Dquarkus.platform.version=2.15.3.Final`

### Output of `uname -a` or `ver`

6.0.18-300.fc37.x86_64

### Output of `java -version`

17.0.4, vendor: GraalVM Community

### GraalVM version (if different from Java)

_No response_

### Quarkus version or git rev

dc85cd0d16c44a264507529c9ee26e26c7ffe175

### Build tool (ie. output of `mvnw --version` or `gradlew --version`)

Apache Maven 3.8.6 (84538c9988a25aec085021c365c560670ad80f63)

### Additional information

_No response_

Changed files:
extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/PrometheusRecorder.java
extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/handlers/PrometheusHandler.java
integration-tests/micrometer-prometheus/src/test/java/io/quarkus/it/micrometer/prometheus/PrometheusMetricsRegistryTest.java
extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/JsonRecorder.java

diff --git a/extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/JsonRecorder.java b/extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/JsonRecorder.java
index 9ab0a33fb7d..af127457e7d 100644
--- a/extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/JsonRecorder.java
+++ b/extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/JsonRecorder.java
@@ -22,7 +22,7 @@ public Consumer<Route> route() {
         return new Consumer<Route>() {
             @Override
             public void accept(Route route) {
-                route.order(2).produces("application/json");
+                route.order(3).produces("application/json");
             }
         };
     }
diff --git a/extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/PrometheusRecorder.java b/extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/PrometheusRecorder.java
index 43add789032..f6efecf5d9b 100644
--- a/extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/PrometheusRecorder.java
+++ b/extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/PrometheusRecorder.java
@@ -2,6 +2,7 @@
 
 import java.util.function.Consumer;
 
+import io.prometheus.client.exporter.common.TextFormat;
 import io.quarkus.micrometer.runtime.export.handlers.PrometheusHandler;
 import io.quarkus.runtime.annotations.Recorder;
 import io.vertx.core.Handler;
@@ -26,6 +27,7 @@ public Consumer<Route> route() {
             @Override
             public void accept(Route route) {
                 route.order(1).produces("text/plain");
+                route.order(2).produces(TextFormat.CONTENT_TYPE_OPENMETRICS_100);
             }
         };
     }
diff --git a/extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/handlers/PrometheusHandler.java b/extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/handlers/PrometheusHandler.java
index 391452fb929..71409f51d82 100644
--- a/extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/handlers/PrometheusHandler.java
+++ b/extensions/micrometer/runtime/src/main/java/io/quarkus/micrometer/runtime/export/handlers/PrometheusHandler.java
@@ -34,12 +34,13 @@ public void handle(RoutingContext routingContext) {
                     .setStatusMessage("Unable to resolve Prometheus registry instance");
         } else {
             ManagedContext requestContext = Arc.container().requestContext();
+            var acceptHeader = chooseContentType(routingContext.request().getHeader("Accept"));
             if (requestContext.isActive()) {
-                doHandle(response);
+                doHandle(response, acceptHeader);
             } else {
                 requestContext.activate();
                 try {
-                    doHandle(response);
+                    doHandle(response, acceptHeader);
                 } finally {
                     requestContext.terminate();
                 }
@@ -47,9 +48,19 @@ public void handle(RoutingContext routingContext) {
         }
     }
 
-    private void doHandle(HttpServerResponse response) {
-        response.putHeader("Content-Type", TextFormat.CONTENT_TYPE_OPENMETRICS_100)
-                .end(Buffer.buffer(registry.scrape(TextFormat.CONTENT_TYPE_OPENMETRICS_100)));
+    private String chooseContentType(String acceptHeader) {
+        if (acceptHeader == null) {
+            return TextFormat.CONTENT_TYPE_OPENMETRICS_100;
+        }
+        if (acceptHeader.startsWith("text/plain")) {
+            return TextFormat.CONTENT_TYPE_004;
+        }
+        return TextFormat.CONTENT_TYPE_OPENMETRICS_100;
+    }
+
+    private void doHandle(HttpServerResponse response, String acceptHeader) {
+        response.putHeader("Content-Type", acceptHeader)
+                .end(Buffer.buffer(registry.scrape(acceptHeader)));
     }
 
     private void setup() {
diff --git a/integration-tests/micrometer-prometheus/src/test/java/io/quarkus/it/micrometer/prometheus/PrometheusMetricsRegistryTest.java b/integration-tests/micrometer-prometheus/src/test/java/io/quarkus/it/micrometer/prometheus/PrometheusMetricsRegistryTest.java
index 944c7ab8dce..ce9ceef711f 100644
--- a/integration-tests/micrometer-prometheus/src/test/java/io/quarkus/it/micrometer/prometheus/PrometheusMetricsRegistryTest.java
+++ b/integration-tests/micrometer-prometheus/src/test/java/io/quarkus/it/micrometer/prometheus/PrometheusMetricsRegistryTest.java
@@ -9,7 +9,9 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestMethodOrder;
 
+import io.prometheus.client.exporter.common.TextFormat;
 import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.RestAssured;
 
 /**
  * Test functioning prometheus endpoint.
@@ -86,8 +88,102 @@ void testTemplatedPathOnClass() {
 
     @Test
     @Order(10)
-    void testPrometheusScrapeEndpoint() {
-        when().get("/q/metrics").then().statusCode(200)
+    void testPrometheusScrapeEndpointTextPlain() {
+        RestAssured.given().header("Accept", TextFormat.CONTENT_TYPE_004)
+                .when().get("/q/metrics")
+                .then().statusCode(200)
+
+                // Prometheus body has ALL THE THINGS in no particular order
+
+                .body(containsString("registry=\\"prometheus\\""))
+                .body(containsString("env=\\"test\\""))
+                .body(containsString("http_server_requests"))
+
+                .body(containsString("status=\\"404\\""))
+                .body(containsString("uri=\\"NOT_FOUND\\""))
+                .body(containsString("outcome=\\"CLIENT_ERROR\\""))
+
+                .body(containsString("status=\\"500\\""))
+                .body(containsString("uri=\\"/message/fail\\""))
+                .body(containsString("outcome=\\"SERVER_ERROR\\""))
+
+                .body(containsString("status=\\"200\\""))
+                .body(containsString("uri=\\"/message\\""))
+                .body(containsString("uri=\\"/message/item/{id}\\""))
+                .body(containsString("outcome=\\"SUCCESS\\""))
+                .body(containsString("uri=\\"/message/match/{id}/{sub}\\""))
+                .body(containsString("uri=\\"/message/match/{other}\\""))
+
+                .body(containsString(
+                        "http_server_requests_seconds_count{env=\\"test\\",method=\\"GET\\",outcome=\\"SUCCESS\\",registry=\\"prometheus\\",status=\\"200\\",uri=\\"/template/path/{value}\\""))
+
+                // Verify Hibernate Metrics
+                .body(containsString(
+                        "hibernate_sessions_open_total{entityManagerFactory=\\"<default>\\",env=\\"test\\",registry=\\"prometheus\\",} 2.0"))
+                .body(containsString(
+                        "hibernate_sessions_closed_total{entityManagerFactory=\\"<default>\\",env=\\"test\\",registry=\\"prometheus\\",} 2.0"))
+                .body(containsString(
+                        "hibernate_connections_obtained_total{entityManagerFactory=\\"<default>\\",env=\\"test\\",registry=\\"prometheus\\",}"))
+                .body(containsString(
+                        "hibernate_entities_inserts_total{entityManagerFactory=\\"<default>\\",env=\\"test\\",registry=\\"prometheus\\",} 3.0"))
+                .body(containsString(
+                        "hibernate_flushes_total{entityManagerFactory=\\"<default>\\",env=\\"test\\",registry=\\"prometheus\\",} 1.0"))
+
+                // Annotated counters
+                .body(not(containsString("metric_none")))
+                .body(containsString(
+                        "metric_all_total{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",exception=\\"none\\",extra=\\"tag\\",method=\\"countAllInvocations\\",registry=\\"prometheus\\",result=\\"success\\",} 1.0"))
+                .body(containsString(
+                        "metric_all_total{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",exception=\\"NullPointerException\\",extra=\\"tag\\",method=\\"countAllInvocations\\",registry=\\"prometheus\\",result=\\"failure\\",} 1.0"))
+                .body(containsString(
+                        "method_counted_total{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",exception=\\"NullPointerException\\",method=\\"emptyMetricName\\",registry=\\"prometheus\\",result=\\"failure\\",} 1.0"))
+                .body(containsString(
+                        "method_counted_total{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",exception=\\"none\\",method=\\"emptyMetricName\\",registry=\\"prometheus\\",result=\\"success\\",} 1.0"))
+                .body(not(containsString("async_none")))
+                .body(containsString(
+                        "async_all_total{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",exception=\\"NullPointerException\\",extra=\\"tag\\",method=\\"countAllAsyncInvocations\\",registry=\\"prometheus\\",result=\\"failure\\",} 1.0"))
+                .body(containsString(
+                        "async_all_total{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",exception=\\"none\\",extra=\\"tag\\",method=\\"countAllAsyncInvocations\\",registry=\\"prometheus\\",result=\\"success\\",} 1.0"))
+                .body(containsString(
+                        "method_counted_total{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",exception=\\"NullPointerException\\",method=\\"emptyAsyncMetricName\\",registry=\\"prometheus\\",result=\\"failure\\",} 1.0"))
+                .body(containsString(
+                        "method_counted_total{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",exception=\\"none\\",method=\\"emptyAsyncMetricName\\",registry=\\"prometheus\\",result=\\"success\\",} 1.0"))
+
+                // Annotated Timers
+                .body(containsString(
+                        "call_seconds_count{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",exception=\\"NullPointerException\\",extra=\\"tag\\",method=\\"call\\",registry=\\"prometheus\\",} 1.0"))
+                .body(containsString(
+                        "call_seconds_count{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",exception=\\"none\\",extra=\\"tag\\",method=\\"call\\",registry=\\"prometheus\\",}"))
+                .body(containsString(
+                        "async_call_seconds_count{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",exception=\\"NullPointerException\\",extra=\\"tag\\",method=\\"asyncCall\\",registry=\\"prometheus\\",} 1.0"))
+                .body(containsString(
+                        "async_call_seconds_count{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",exception=\\"none\\",extra=\\"tag\\",method=\\"asyncCall\\",registry=\\"prometheus\\",} 1.0"))
+                .body(containsString(
+                        "longCall_seconds_active_count{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",extra=\\"tag\\",method=\\"longCall\\",registry=\\"prometheus\\",}"))
+                .body(containsString(
+                        "async_longCall_seconds_duration_sum{class=\\"io.quarkus.it.micrometer.prometheus.AnnotatedResource\\",env=\\"test\\",extra=\\"tag\\",method=\\"longAsyncCall\\",registry=\\"prometheus\\",} 0.0"))
+
+                // Configured median, 95th percentile and histogram buckets
+                .body(containsString(
+                        "prime_number_test_seconds{env=\\"test\\",registry=\\"prometheus\\",quantile=\\"0.5\\",}"))
+                .body(containsString(
+                        "prime_number_test_seconds{env=\\"test\\",registry=\\"prometheus\\",quantile=\\"0.95\\",}"))
+                .body(containsString(
+                        "prime_number_test_seconds_bucket{env=\\"test\\",registry=\\"prometheus\\",le=\\"0.001\\",}"))
+
+                // this was defined by a tag to a non-matching registry, and should not be found
+                .body(not(containsString("class-should-not-match")))
+
+                // should not find this ignored uri
+                .body(not(containsString("uri=\\"/fruit/create\\"")));
+    }
+
+    @Test
+    @Order(11)
+    void testPrometheusScrapeEndpointOpenMetrics() {
+        RestAssured.given().header("Accept", TextFormat.CONTENT_TYPE_OPENMETRICS_100)
+                .when().get("/q/metrics")
+                .then().statusCode(200)
 
                 // Prometheus body has ALL THE THINGS in no particular order