Repository: jaegertracing/jaeger

Issue:
Jaeger containers healthcheck support
## Requirement - what kind of business use case are you trying to solve?

I would like to make my container depend on the successful start up of jager containers (agent for instance, but applies to all jaeger containers). In AWS ECS deployment is unable to perform a container-level healthcheck on the jaeger containers. So declaring my app to depend on the "HEALTHY" startup of a jaeger container is not possible at this time. Nor is ECS able to determine if the deployment completed successfully since it cannot healthcheck the app at startup.

## Problem - what in Jaeger blocks you from solving the requirement?

Jaeger containers are stripped down and do not have a way to execute wget or curl to check the admin port. ECS needs a [command it can execute](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#container_definition_healthcheck) in the container that will exit 0.

## Proposal - what do you suggest to solve the problem or improve the existing situation?

Rather than including curl or wget which would bloat the containers needlessly. Add a built-in command to jager binaries to check their respective health (taking a minimal/identical set of environment/arguments to be compatible). The idea being we can very easily just have jaeger check it's own health and exit 0 if successful.

```
jaeger-agent health [--admin.http.host-port] [--config-file]
Checks the local agent admin endpoint to verify health.
```

## Any open questions to address

<!-- Questions that should be answered before proceeding with implementation. -->


Hints from issue comments:
@dekimsey: To be clear, while this is clearly possible via the admin endpoint with other tools (such as Consul), ECS itself doesn't support http requests as healthchecks and appears to require we shell-in to run a command in the container.@yurishkuro: Sounds like a reasonable solution. Are you interested in creating a pull request?@dekimsey: Yes, I think that's possible. Let me give it a go and we'll see what happens.@jpkrohling: > ECS itself doesn't support http requests as healthchecks and appears to require we shell-in to run a command in the container.

😯 Do they have a good reason for that? 

> Sounds like a reasonable solution. Are you interested in creating a pull request?

Sounds good to me as well.@Dheerajcode2016: What happened to this , can we do healthcheck now ?@cmnemoi: For anyone who got here from the Internet, accordingly to the documentation, with Jaeger v2 you can use the `/status` endpoint exposed on the port 13113 : https://www.jaegertracing.io/docs/2.11/deployment/#management-ports@yurishkuro: This issue is from 2020, I am curious if ECS still has this limitation that it needs a command to run instead of using an http health check. While this issue was resolved for Jaeger v1, there is no similar `status` command in Jaeger v2 (which is really OTEL Collector underneath).

Solution patch:
diff --git a/cmd/agent/main.go b/cmd/agent/main.go
index c01d04d3..dcd296e2 100644
--- a/cmd/agent/main.go
+++ b/cmd/agent/main.go
@@ -32,6 +32,7 @@ import (
 	"github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc"
 	"github.com/jaegertracing/jaeger/cmd/docs"
 	"github.com/jaegertracing/jaeger/cmd/flags"
+	"github.com/jaegertracing/jaeger/cmd/status"
 	"github.com/jaegertracing/jaeger/pkg/config"
 	"github.com/jaegertracing/jaeger/pkg/version"
 	"github.com/jaegertracing/jaeger/ports"
@@ -95,6 +96,7 @@ func main() {
 
 	command.AddCommand(version.Command())
 	command.AddCommand(docs.Command(v))
+	command.AddCommand(status.Command(v, ports.AgentAdminHTTP))
 
 	config.AddFlags(
 		v,
diff --git a/cmd/all-in-one/main.go b/cmd/all-in-one/main.go
index bf2611cc..f2565cb0 100644
--- a/cmd/all-in-one/main.go
+++ b/cmd/all-in-one/main.go
@@ -41,6 +41,7 @@ import (
 	"github.com/jaegertracing/jaeger/cmd/flags"
 	queryApp "github.com/jaegertracing/jaeger/cmd/query/app"
 	"github.com/jaegertracing/jaeger/cmd/query/app/querysvc"
+	"github.com/jaegertracing/jaeger/cmd/status"
 	"github.com/jaegertracing/jaeger/pkg/config"
 	"github.com/jaegertracing/jaeger/pkg/version"
 	ss "github.com/jaegertracing/jaeger/plugin/sampling/strategystore"
@@ -182,6 +183,7 @@ by default uses only in-memory database.`,
 	command.AddCommand(version.Command())
 	command.AddCommand(env.Command())
 	command.AddCommand(docs.Command(v))
+	command.AddCommand(status.Command(v, ports.CollectorAdminHTTP))
 
 	config.AddFlags(
 		v,
diff --git a/cmd/collector/main.go b/cmd/collector/main.go
index 66e19cd4..a476ff73 100644
--- a/cmd/collector/main.go
+++ b/cmd/collector/main.go
@@ -33,6 +33,7 @@ import (
 	"github.com/jaegertracing/jaeger/cmd/docs"
 	"github.com/jaegertracing/jaeger/cmd/env"
 	"github.com/jaegertracing/jaeger/cmd/flags"
+	"github.com/jaegertracing/jaeger/cmd/status"
 	"github.com/jaegertracing/jaeger/pkg/config"
 	"github.com/jaegertracing/jaeger/pkg/version"
 	ss "github.com/jaegertracing/jaeger/plugin/sampling/strategystore"
@@ -122,6 +123,7 @@ func main() {
 	command.AddCommand(version.Command())
 	command.AddCommand(env.Command())
 	command.AddCommand(docs.Command(v))
+	command.AddCommand(status.Command(v, ports.CollectorAdminHTTP))
 
 	config.AddFlags(
 		v,
diff --git a/cmd/ingester/main.go b/cmd/ingester/main.go
index 683236c7..e93f5d56 100644
--- a/cmd/ingester/main.go
+++ b/cmd/ingester/main.go
@@ -31,6 +31,7 @@ import (
 	"github.com/jaegertracing/jaeger/cmd/flags"
 	"github.com/jaegertracing/jaeger/cmd/ingester/app"
 	"github.com/jaegertracing/jaeger/cmd/ingester/app/builder"
+	"github.com/jaegertracing/jaeger/cmd/status"
 	"github.com/jaegertracing/jaeger/pkg/config"
 	"github.com/jaegertracing/jaeger/pkg/version"
 	"github.com/jaegertracing/jaeger/plugin/storage"
@@ -99,6 +100,7 @@ func main() {
 	command.AddCommand(version.Command())
 	command.AddCommand(env.Command())
 	command.AddCommand(docs.Command(v))
+	command.AddCommand(status.Command(v, ports.IngesterAdminHTTP))
 
 	config.AddFlags(
 		v,
diff --git a/cmd/query/main.go b/cmd/query/main.go
index a1a23f1b..2f2f00bf 100644
--- a/cmd/query/main.go
+++ b/cmd/query/main.go
@@ -34,6 +34,7 @@ import (
 	"github.com/jaegertracing/jaeger/cmd/flags"
 	"github.com/jaegertracing/jaeger/cmd/query/app"
 	"github.com/jaegertracing/jaeger/cmd/query/app/querysvc"
+	"github.com/jaegertracing/jaeger/cmd/status"
 	"github.com/jaegertracing/jaeger/pkg/config"
 	"github.com/jaegertracing/jaeger/pkg/version"
 	"github.com/jaegertracing/jaeger/plugin/storage"
@@ -134,6 +135,7 @@ func main() {
 	command.AddCommand(version.Command())
 	command.AddCommand(env.Command())
 	command.AddCommand(docs.Command(v))
+	command.AddCommand(status.Command(v, ports.QueryAdminHTTP))
 
 	config.AddFlags(
 		v,
diff --git a/cmd/status/command.go b/cmd/status/command.go
new file mode 100644
index 00000000..0be39d77
--- /dev/null
+++ b/cmd/status/command.go
@@ -0,0 +1,70 @@
+// Copyright (c) 2020 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package status
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/jaegertracing/jaeger/ports"
+)
+
+const statusHTTPHostPort = "status.http.host-port"
+
+// Command for check component status.
+func Command(v *viper.Viper, adminPort int) *cobra.Command {
+	c := &cobra.Command{
+		Use:   "status",
+		Short: "Print the status.",
+		Long:  `Print Jaeger component status information, exit non-zero on any error.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			url := convert(v.GetString(statusHTTPHostPort))
+			resp, err := http.Get(url)
+			if err != nil {
+				return err
+			}
+			defer resp.Body.Close()
+			body, _ := ioutil.ReadAll(resp.Body)
+			fmt.Println(string(body))
+			if resp.StatusCode != http.StatusOK {
+				return fmt.Errorf("abnormal value of http status code: %v", resp.StatusCode)
+			}
+			return nil
+		},
+	}
+	c.Flags().AddGoFlagSet(flags(&flag.FlagSet{}, adminPort))
+	v.BindPFlags(c.Flags())
+	return c
+}
+
+func flags(flagSet *flag.FlagSet, adminPort int) *flag.FlagSet {
+	adminPortStr := ports.PortToHostPort(adminPort)
+	flagSet.String(statusHTTPHostPort, adminPortStr, fmt.Sprintf(
+		"The host:port (e.g. 127.0.0.1%s or %s) for the health check", adminPortStr, adminPortStr))
+	return flagSet
+}
+
+func convert(httpHostPort string) string {
+	if strings.HasPrefix(httpHostPort, ":") {
+		return fmt.Sprintf("http://127.0.0.1%s", httpHostPort)
+	}
+	return fmt.Sprintf("http://%s", httpHostPort)
+}



Test patch:
diff --git a/cmd/status/command_test.go b/cmd/status/command_test.go
new file mode 100644
index 00000000..47af9ec2
--- /dev/null
+++ b/cmd/status/command_test.go
@@ -0,0 +1,72 @@
+// Copyright (c) 2020 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package status
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+)
+
+func readyHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte("{\"status\":\"Server available\"}"))
+}
+
+func unavailableHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusServiceUnavailable)
+	w.Write([]byte("{\"status\":\"Server not available\"}"))
+}
+
+func TestReady(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(readyHandler))
+	defer ts.Close()
+	v := viper.New()
+	cmd := Command(v, 80)
+	cmd.ParseFlags([]string{"--status.http.host-port=" + strings.TrimPrefix(ts.URL, "http://")})
+	err := cmd.Execute()
+	assert.NoError(t, err)
+}
+
+func TestOnlyPortConfig(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(readyHandler))
+	defer ts.Close()
+	v := viper.New()
+	cmd := Command(v, 80)
+	cmd.ParseFlags([]string{"--status.http.host-port=:" + strings.Split(ts.URL, ":")[len(strings.Split(ts.URL, ":"))-1]})
+	err := cmd.Execute()
+	assert.NoError(t, err)
+}
+
+func TestUnready(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(unavailableHandler))
+	defer ts.Close()
+	v := viper.New()
+	cmd := Command(v, 80)
+	cmd.ParseFlags([]string{"--status.http.host-port=" + strings.TrimPrefix(ts.URL, "http://")})
+	err := cmd.Execute()
+	assert.Error(t, err)
+}
+
+func TestNoService(t *testing.T) {
+	v := viper.New()
+	cmd := Command(v, 12345)
+	err := cmd.Execute()
+	assert.Error(t, err)
+}