Repository: cloudprober/cloudprober

Issue:
Use targets in SERVER mode external probe?
Hello! we're attempting to scale up the amount of probes our cloudprober box can handle and I'm unable to get the server mode external probes working. 

My desired flow would be to specify that we have a server mode probe for some targets - then, cloudprober will iterate through the targets and stdin which target it wants probed into the server. 

Currently, it seems for server mode probes, targets is just unused.

This way, we can keep using the targets api and also get to leverage server mode probes. 

Is there a way to do this already? I'm happy to open the PR myself but I'd like to make sure I'm not missing anything before I get hacking. Thanks!!

Discussion:
@antholeole: Figured it out! Target templating can be done in options. I'll submit a doc update soon - thank you!

Solution patch:
diff --git a/docs/content/docs/how-to/external-probe/index.md b/docs/content/docs/how-to/external-probe/index.md
index 23f03750..bef843f9 100644
--- a/docs/content/docs/how-to/external-probe/index.md
+++ b/docs/content/docs/how-to/external-probe/index.md
@@ -146,7 +146,7 @@ The probe that we created above forks out a new `redis_probe` process for every
 probe cycle. This can get expensive if probe frequency is high and the process is big (e.g. a Java binary). Also, what if you want to keep some state across probes, for example, lets say you want to monitor performance over HTTP/2 where you keep using the same TCP connection for multiple HTTP requests. A new process
 every time makes keeping state impossible.
 
-External probe's server mode provides a way to run the external probe process in daemon mode. Cloudprober communicates with this process over stdout/stdin (connected with OS pipes), using serialized protobuf messages. Cloudprober comes with a serverutils package that makes it easy to build external probe servers in Go.
+External probe's server mode provides a way to run the external probe process in daemon mode. Cloudprober communicates with this process over stdout/stdin (connected with OS pipes), using serialized protobuf messages. Cloudprober comes with a serverutils package that makes it easy to build external probe servers in Go. To pass target info into the probe, use the @label@ notation in `external_probe.options`.
 
 ![External Probe Server](external_probe_server.svg)
 
diff --git a/docs/layouts/index.html b/docs/layouts/index.html
index c72634d3..8192db96 100644
--- a/docs/layouts/index.html
+++ b/docs/layouts/index.html
@@ -4,7 +4,7 @@
     <div class="col-lg-8 text-center mt-1">
       <h3 class="h2 fw-bold mb-4">{{ .Params.lead | safeHTML }}</h3>
       <div class="mb-5" style="font-size: 1.05rem">
-        Cloudprober is an open-source, active monitoring tool designed to detect failures in software systems and services in real-time.
+        Cloudprober is an open-source active monitoring tool designed to detect failures in software systems and services in real-time.
       </div>
       <a
         class="btn btn-lg btn-primary mb-2"
diff --git a/examples/external/cloudprober_server.cfg b/examples/external/cloudprober_server.cfg
index 687f98ec..c1c85083 100644
--- a/examples/external/cloudprober_server.cfg
+++ b/examples/external/cloudprober_server.cfg
@@ -18,5 +18,12 @@ probe {
         }
       }
     }
+
+    # @target@ and friends are templated into options, which can be sent to the
+    # server probe.
+    options {
+      name: "target"
+      value: "@target@"
+    }
   }
 }
diff --git a/surfacers/internal/prometheus/prometheus.go b/surfacers/internal/prometheus/prometheus.go
index 2e939293..20192af6 100644
--- a/surfacers/internal/prometheus/prometheus.go
+++ b/surfacers/internal/prometheus/prometheus.go
@@ -185,7 +185,9 @@ func New(ctx context.Context, config *configpb.SurfacerConf, opts *options.Optio
 				ps.writeData(hw.w)
 				close(hw.doneChan)
 			case <-staleMetricDeleteTimer.C:
-				ps.deleteExpiredMetrics()
+				if !ps.disableMetricsExpiration() {
+					ps.deleteExpiredMetrics()
+				}
 			}
 		}
 	}()
@@ -216,6 +218,18 @@ func (ps *PromSurfacer) Write(_ context.Context, em *metrics.EventMetrics) {
 	}
 }
 
+func (ps *PromSurfacer) disableMetricsExpiration() bool {
+	if ps.c.DisableMetricsExpiration != nil {
+		return ps.c.GetDisableMetricsExpiration()
+	}
+
+	if !ps.c.GetIncludeTimestamp() {
+		return true
+	}
+
+	return false
+}
+
 func promType(em *metrics.EventMetrics) string {
 	switch em.Kind {
 	case metrics.CUMULATIVE:
diff --git a/surfacers/internal/prometheus/proto/config.pb.go b/surfacers/internal/prometheus/proto/config.pb.go
index 56f810c6..5eeb3f8c 100644
--- a/surfacers/internal/prometheus/proto/config.pb.go
+++ b/surfacers/internal/prometheus/proto/config.pb.go
@@ -35,6 +35,13 @@ type SurfacerConf struct {
 	// i.e. timestamps are not exported, prometheus associates scraped values with
 	// scrape timestamp.
 	IncludeTimestamp *bool `protobuf:"varint,2,opt,name=include_timestamp,json=includeTimestamp,def=1" json:"include_timestamp,omitempty"`
+	// We automatically delete metrics that haven't been updated for more than
+	// 10 min. We do that because prometheus generates warnings while scraping
+	// metrics that are more than 10m old and prometheus knows that metrics are
+	// more than 10m old because we include timestamp in metrics (by default).
+	// This option controls that behavior.
+	// It's automatically set to true if include_timestamp is false.
+	DisableMetricsExpiration *bool `protobuf:"varint,5,opt,name=disable_metrics_expiration,json=disableMetricsExpiration" json:"disable_metrics_expiration,omitempty"`
 	// URL that prometheus scrapes metrics from.
 	MetricsUrl *string `protobuf:"bytes,3,opt,name=metrics_url,json=metricsUrl,def=/metrics" json:"metrics_url,omitempty"`
 	// Prefix to add to all metric names. For example setting this field to
@@ -99,6 +106,13 @@ func (x *SurfacerConf) GetIncludeTimestamp() bool {
 	return Default_SurfacerConf_IncludeTimestamp
 }
 
+func (x *SurfacerConf) GetDisableMetricsExpiration() bool {
+	if x != nil && x.DisableMetricsExpiration != nil {
+		return *x.DisableMetricsExpiration
+	}
+	return false
+}
+
 func (x *SurfacerConf) GetMetricsUrl() string {
 	if x != nil && x.MetricsUrl != nil {
 		return *x.MetricsUrl
@@ -117,10 +131,11 @@ var File_github_com_cloudprober_cloudprober_surfacers_internal_prometheus_proto_
 
 const file_github_com_cloudprober_cloudprober_surfacers_internal_prometheus_proto_config_proto_rawDesc = "" +
 	"\n" +
-	"Sgithub.com/cloudprober/cloudprober/surfacers/internal/prometheus/proto/config.proto\x12\x1fcloudprober.surfacer.prometheus\"\xca\x01\n" +
+	"Sgithub.com/cloudprober/cloudprober/surfacers/internal/prometheus/proto/config.proto\x12\x1fcloudprober.surfacer.prometheus\"\x88\x02\n" +
 	"\fSurfacerConf\x125\n" +
 	"\x13metrics_buffer_size\x18\x01 \x01(\x03:\x0510000R\x11metricsBufferSize\x121\n" +
-	"\x11include_timestamp\x18\x02 \x01(\b:\x04trueR\x10includeTimestamp\x12)\n" +
+	"\x11include_timestamp\x18\x02 \x01(\b:\x04trueR\x10includeTimestamp\x12<\n" +
+	"\x1adisable_metrics_expiration\x18\x05 \x01(\bR\x18disableMetricsExpiration\x12)\n" +
 	"\vmetrics_url\x18\x03 \x01(\t:\b/metricsR\n" +
 	"metricsUrl\x12%\n" +
 	"\x0emetrics_prefix\x18\x04 \x01(\tR\rmetricsPrefixBHZFgithub.com/cloudprober/cloudprober/surfacers/internal/prometheus/proto"
diff --git a/surfacers/internal/prometheus/proto/config.proto b/surfacers/internal/prometheus/proto/config.proto
index 746c2152..5324d7fa 100644
--- a/surfacers/internal/prometheus/proto/config.proto
+++ b/surfacers/internal/prometheus/proto/config.proto
@@ -19,6 +19,14 @@ message SurfacerConf {
   // scrape timestamp.
   optional bool include_timestamp = 2 [default = true];
 
+  // We automatically delete metrics that haven't been updated for more than
+  // 10 min. We do that because prometheus generates warnings while scraping
+  // metrics that are more than 10m old and prometheus knows that metrics are
+  // more than 10m old because we include timestamp in metrics (by default).
+  // This option controls that behavior.
+  // It's automatically set to true if include_timestamp is false.
+  optional bool disable_metrics_expiration = 5;
+
   // URL that prometheus scrapes metrics from.
   optional string metrics_url = 3 [default = "/metrics"];
 



Test patch:
diff --git a/surfacers/internal/prometheus/prometheus_test.go b/surfacers/internal/prometheus/prometheus_test.go
index 5e2056a0..d7d047d0 100644
--- a/surfacers/internal/prometheus/prometheus_test.go
+++ b/surfacers/internal/prometheus/prometheus_test.go
@@ -379,3 +379,67 @@ func TestMain(m *testing.M) {
 
 	m.Run()
 }
+func TestDisableMetricsExpiration(t *testing.T) {
+	tests := []struct {
+		name                      string
+		includeTimestamp          *bool
+		disableMetricsExpiration  *bool
+		want                      bool
+	}{
+		{
+			name:                     "DisableMetricsExpiration unset, IncludeTimestamp true",
+			includeTimestamp:         proto.Bool(true),
+			disableMetricsExpiration: nil,
+			want:                     false,
+		},
+		{
+			name:                     "DisableMetricsExpiration unset, IncludeTimestamp false",
+			includeTimestamp:         proto.Bool(false),
+			disableMetricsExpiration: nil,
+			want:                     true,
+		},
+		{
+			name:                     "DisableMetricsExpiration true, IncludeTimestamp true",
+			includeTimestamp:         proto.Bool(true),
+			disableMetricsExpiration: proto.Bool(true),
+			want:                     true,
+		},
+		{
+			name:                     "DisableMetricsExpiration false, IncludeTimestamp true",
+			includeTimestamp:         proto.Bool(true),
+			disableMetricsExpiration: proto.Bool(false),
+			want:                     false,
+		},
+		{
+			name:                     "DisableMetricsExpiration true, IncludeTimestamp false",
+			includeTimestamp:         proto.Bool(false),
+			disableMetricsExpiration: proto.Bool(true),
+			want:                     true,
+		},
+		{
+			name:                     "DisableMetricsExpiration false, IncludeTimestamp false",
+			includeTimestamp:         proto.Bool(false),
+			disableMetricsExpiration: proto.Bool(false),
+			want:                     false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			conf := &configpb.SurfacerConf{}
+			if tt.includeTimestamp != nil {
+				conf.IncludeTimestamp = tt.includeTimestamp
+			}
+			if tt.disableMetricsExpiration != nil {
+				conf.DisableMetricsExpiration = tt.disableMetricsExpiration
+			}
+			ps := &PromSurfacer{
+				c: conf,
+			}
+			got := ps.disableMetricsExpiration()
+			if got != tt.want {
+				t.Errorf("disableMetricsExpiration() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
\ No newline at end of file