Repository: nats-io/nats-streaming-server
Pull request #1024 (2020-02-27 16:20:00)

[ADDED] Monitoring endpoint '/isFTActive' to evaluate whether the server is in FTActive mode
My use case for this feature is related to a kubernetes based deployment. We run nats in FT mode, which means that the monitor endpoints return useful information, or no information, depending on which node you make requests against. This feature will enable me to place some kind of reverse proxy in front of the nats nodes to proxy http requests on the monitor port to only the active node. 

With this PR in its current state, the only proxy i've found that i can configure to use appropriately is envoy. it seems most of the common reverse proxy solutions in use on k8s have varying levels of opinionatedness about how to configure health checks, and what constitutes "healthy". For instance, traefik considers any health check that returns a 2XX or 3XX to be healthy, and there's no way to alter that configuration. NGINX Plus supports active health checks that allow you to specify a path to a health check endpoint. The free NGINX offering only supports passive health checks, which evaluate traffic as a whole and removes upstream nodes from load balancing if a node begins throwing 5XX status codes. Envoy seems to be only proxy that allows you to fully configure a health check that fits inline with the expected status codes added by this feature.


Patch:
diff --git a/server/monitor.go b/server/monitor.go
index 714f721..18bc824 100644
--- a/server/monitor.go
+++ b/server/monitor.go
@@ -31,11 +31,12 @@ import (
 
 // Routes for the monitoring pages
 const (
-	RootPath     = "/streaming"
-	ServerPath   = RootPath + "/serverz"
-	StorePath    = RootPath + "/storez"
-	ClientsPath  = RootPath + "/clientsz"
-	ChannelsPath = RootPath + "/channelsz"
+	RootPath       = "/streaming"
+	ServerPath     = RootPath + "/serverz"
+	StorePath      = RootPath + "/storez"
+	ClientsPath    = RootPath + "/clientsz"
+	ChannelsPath   = RootPath + "/channelsz"
+	IsFTActivePath = RootPath + "/isFTActive"
 
 	defaultMonitorListLimit = 1024
 )
@@ -160,6 +161,7 @@ func (s *StanServer) startMonitoring(nOpts *natsd.Options) error {
 	mux.HandleFunc(StorePath, s.handleStorez)
 	mux.HandleFunc(ClientsPath, s.handleClientsz)
 	mux.HandleFunc(ChannelsPath, s.handleChannelsz)
+	mux.HandleFunc(IsFTActivePath, s.handleIsFTActivez)
 
 	return nil
 }
@@ -245,6 +247,17 @@ func (s *StanServer) handleServerz(w http.ResponseWriter, r *http.Request) {
 	s.sendResponse(w, r, serverz)
 }
 
+func (s *StanServer) handleIsFTActivez(w http.ResponseWriter, r *http.Request) {
+	s.mu.RLock()
+	state := s.state
+	s.mu.RUnlock()
+	if state == FTActive {
+		w.WriteHeader(http.StatusOK)
+	} else {
+		w.WriteHeader(http.StatusNoContent)
+	}
+}
+
 func myUptime(d time.Duration) string {
 	// Just use total seconds for uptime, and display days / years
 	tsecs := d / time.Second


Test patch:
diff --git a/server/monitor_test.go b/server/monitor_test.go
index 91fd82f..837d398 100644
--- a/server/monitor_test.go
+++ b/server/monitor_test.go
@@ -377,6 +377,48 @@ func TestMonitorServerz(t *testing.T) {
 	monitorExpectStatus(t, ServerPath, http.StatusInternalServerError)
 }
 
+func TestMonitorIsFTActiveFTServer(t *testing.T) {
+	for _, test := range []struct {
+		name           string
+		checkActive    bool
+		expectedStatus int
+	}{
+		{"active", true, http.StatusOK},
+		{"standby", false, http.StatusNoContent},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			resetPreviousHTTPConnections()
+
+			cleanupFTDatastore(t)
+			defer cleanupFTDatastore(t)
+
+			nopts := defaultMonitorOptions
+			var aNOpts *natsd.Options
+			var sNOpts *natsd.Options
+
+			if test.checkActive {
+				aNOpts = &nopts
+			} else {
+				sNOpts = &nopts
+			}
+
+			opts := getTestFTDefaultOptions()
+			active := runServerWithOpts(t, opts, aNOpts)
+			defer active.Shutdown()
+
+			getFTActiveServer(t, active)
+
+			opts = getTestFTDefaultOptions()
+			opts.NATSServerURL = "nats://127.0.0.1:4222"
+			standby := runServerWithOpts(t, opts, sNOpts)
+			defer standby.Shutdown()
+
+			resp, _ := getBodyEx(t, http.DefaultClient, "http", IsFTActivePath, test.expectedStatus, "")
+			defer resp.Body.Close()
+		})
+	}
+}
+
 func TestMonitorUptime(t *testing.T) {
 	expected := []string{"1y2d3h4m5s", "1d2h3m4s", "1h2m3s", "1m2s", "1s"}
 	durations := []time.Duration{