Repository: cloudflare/daphne
Pull request #314 (2023-05-03 10:17:52)

Replaces aggregation_job_gauge with aggregation_job_counter
Allows for easier aggregation of the metrics pushed by the worker.


Patch:
diff --git a/daphne/src/metrics.rs b/daphne/src/metrics.rs
index 59daf58..04892be 100644
--- a/daphne/src/metrics.rs
+++ b/daphne/src/metrics.rs
@@ -4,10 +4,7 @@
 //! Daphne metrics.
 
 use crate::DapError;
-use prometheus::{
-    register_int_counter_vec_with_registry, register_int_gauge_vec_with_registry, IntCounterVec,
-    IntGaugeVec, Registry,
-};
+use prometheus::{register_int_counter_vec_with_registry, IntCounterVec, Registry};
 
 pub struct DaphneMetrics {
     /// Inbound request metrics: Successful requests served, broken down by type.
@@ -17,8 +14,8 @@ pub struct DaphneMetrics {
     /// a report is rejected, the failure type is recorded.
     report_counter: IntCounterVec,
 
-    /// Helper: Number of running aggregation jobs.
-    aggregation_job_gauge: IntGaugeVec,
+    /// Helper: Total number of aggregation jobs started and completed.
+    aggregation_job_counter: IntCounterVec,
 }
 
 impl DaphneMetrics {
@@ -45,17 +42,17 @@ impl DaphneMetrics {
             registry
         )?;
 
-        let aggregation_job_gauge = register_int_gauge_vec_with_registry!(
-            format!("{front}aggregation_job_gauge"),
-            "Number of running aggregation jobs.",
-            &["host"],
+        let aggregation_job_counter = register_int_counter_vec_with_registry!(
+            format!("{front}aggregation_job_counter"),
+            "Total number of aggregation jobs started and completed.",
+            &["host", "status"],
             registry
         )?;
 
         Ok(Self {
             inbound_request_counter,
             report_counter,
-            aggregation_job_gauge,
+            aggregation_job_counter,
         })
     }
 
@@ -94,18 +91,18 @@ impl ContextualizedDaphneMetrics<'_> {
             .inc_by(val);
     }
 
-    pub fn agg_job_inc(&self) {
+    pub fn agg_job_started_inc(&self) {
         self.metrics
-            .aggregation_job_gauge
-            .with_label_values(&[self.host])
+            .aggregation_job_counter
+            .with_label_values(&[self.host, "started"])
             .inc();
     }
 
-    pub fn agg_job_dec(&self) {
+    pub fn agg_job_completed_inc(&self) {
         self.metrics
-            .aggregation_job_gauge
-            .with_label_values(&[self.host])
-            .dec();
+            .aggregation_job_counter
+            .with_label_values(&[self.host, "completed"])
+            .inc();
     }
 }
 
diff --git a/daphne/src/roles.rs b/daphne/src/roles.rs
index 8ef1a8a..cc3d292 100644
--- a/daphne/src/roles.rs
+++ b/daphne/src/roles.rs
@@ -963,7 +963,7 @@ where
                     }
                 };
 
-                metrics.agg_job_inc();
+                metrics.agg_job_started_inc();
                 metrics.inbound_req_inc(DaphneRequestType::Aggregate);
                 Ok(DapResponse {
                     version: req.version,
@@ -1033,7 +1033,7 @@ where
                 };
 
                 metrics.report_inc_by("aggregated", out_shares_count);
-                metrics.agg_job_dec();
+                metrics.agg_job_completed_inc();
                 metrics.inbound_req_inc(DaphneRequestType::Aggregate);
                 Ok(DapResponse {
                     version: req.version,


Test patch:
diff --git a/daphne/src/roles_test.rs b/daphne/src/roles_test.rs
index 34503cf..16fd22b 100644
--- a/daphne/src/roles_test.rs
+++ b/daphne/src/roles_test.rs
@@ -983,7 +983,7 @@ async fn http_post_aggregate_failure_report_replayed(version: DapVersion) {
     assert_metrics_include!(t.prometheus_registry, {
         r#"test_helper_report_counter{host="helper.org",status="rejected_report_replayed"}"#: 1,
         r#"test_helper_inbound_request_counter{host="helper.org",type="aggregate"}"#: 1,
-        r#"test_helper_aggregation_job_gauge{host="helper.org"}"#: 1,
+        r#"test_helper_aggregation_job_counter{host="helper.org",status="started"}"#: 1,
     });
 }
 
@@ -1041,7 +1041,7 @@ async fn http_post_aggregate_failure_batch_collected(version: DapVersion) {
     assert_metrics_include!(t.prometheus_registry, {
         r#"test_helper_report_counter{host="helper.org",status="rejected_batch_collected"}"#: 1,
         r#"test_helper_inbound_request_counter{host="helper.org",type="aggregate"}"#: 1,
-        r#"test_helper_aggregation_job_gauge{host="helper.org"}"#: 1,
+        r#"test_helper_aggregation_job_counter{host="helper.org",status="started"}"#: 1,
     });
 }
 
@@ -1579,7 +1579,8 @@ async fn e2e_time_interval(version: DapVersion) {
         r#"test_helper_report_counter{host="helper.org",status="aggregated"}"#: 1,
         r#"test_leader_report_counter{host="leader.com",status="collected"}"#: 1,
         r#"test_helper_report_counter{host="helper.org",status="collected"}"#: 1,
-        r#"test_helper_aggregation_job_gauge{host="helper.org"}"#: 0,
+        r#"test_helper_aggregation_job_counter{host="helper.org",status="started"}"#: 1,
+        r#"test_helper_aggregation_job_counter{host="helper.org",status="completed"}"#: 1,
     });
 }
 
@@ -1612,7 +1613,8 @@ async fn e2e_fixed_size(version: DapVersion) {
         r#"test_helper_report_counter{host="helper.org",status="aggregated"}"#: 1,
         r#"test_leader_report_counter{host="leader.com",status="collected"}"#: 1,
         r#"test_helper_report_counter{host="helper.org",status="collected"}"#: 1,
-        r#"test_helper_aggregation_job_gauge{host="helper.org"}"#: 0,
+        r#"test_helper_aggregation_job_counter{host="helper.org",status="started"}"#: 1,
+        r#"test_helper_aggregation_job_counter{host="helper.org",status="completed"}"#: 1,
     });
 }
 
@@ -1713,7 +1715,8 @@ async fn e2e_taskprov(version: DapVersion) {
         r#"test_helper_report_counter{host="helper.org",status="aggregated"}"#: 1,
         r#"test_leader_report_counter{host="leader.com",status="collected"}"#: 1,
         r#"test_helper_report_counter{host="helper.org",status="collected"}"#: 1,
-        r#"test_helper_aggregation_job_gauge{host="helper.org"}"#: 0,
+        r#"test_helper_aggregation_job_counter{host="helper.org",status="started"}"#: 1,
+        r#"test_helper_aggregation_job_counter{host="helper.org",status="completed"}"#: 1,
     });
 }