Repository: docker/compose-cli
Pull request #713 (2020-10-02 09:27:16)

E2e test on metrics
Signed-off-by: Guillaume Tardif <guillaume.tardif@docker.com>

**What I did**
* Mock metrics with a custom socket for tests
* build flag to read test metrics socket from env
* add local e2e tests checking metric on moby context and example context

**Related issue**
https://github.com/docker/compose-cli/issues/686

<!-- optional tests
You can add a / mention to run tests executed by default only on main branch : 
* `test-aci` to run ACI E2E tests
* `test-ecs` to run ECS E2E tests
* `test-windows` to run tests & E2E tests on windows
-->
/test-windows

**(not mandatory) A picture of a cute animal, if possible in relation with what you did**
![](https://tutomoviles.com/wp-content/uploads/2020/02/1581813307_882_Hewan-hewan-baru-yang-gila-membayangkan-dengan-kekuatan-Photoshop.jpg)

gtardif: I will need to adapt the metrics test socket on windows with pipes
gtardif: > I will need to adapt the metrics test socket on windows with pipes

Actually we can also use unix sockets on windows to send metrics. Using this only for the server mock, not changing the Desktop current setup on windows. 

Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f7a88379..593f9c46 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -55,12 +55,12 @@ jobs:
 
       - name: Test
         env:
-          BUILD_TAGS: example,local,ecs
+          BUILD_TAGS: example,local
         run: make -f builder.Makefile test
 
-      - name: Build
+      - name: Build for local E2E
         env:
-          BUILD_TAGS: example,local,ecs
+          BUILD_TAGS: example,local,e2e
         run: make -f builder.Makefile cli
 
       - name: E2E Test
diff --git a/.github/workflows/optional-ci.yml b/.github/workflows/optional-ci.yml
index 0af57c43..00838fda 100644
--- a/.github/workflows/optional-ci.yml
+++ b/.github/workflows/optional-ci.yml
@@ -130,7 +130,7 @@ jobs:
 
       - name: Build
         env:
-          BUILD_TAGS: example,local
+          BUILD_TAGS: example,local,e2e
         run: make -f builder.Makefile cli
 
       - name: E2E Test
diff --git a/Makefile b/Makefile
index ef5205fc..61e61971 100644
--- a/Makefile
+++ b/Makefile
@@ -39,7 +39,7 @@ protos: ## Generate go code from .proto files
 cli: ## Compile the cli
 	@docker build . --target cli \
 	--platform local \
-	--build-arg BUILD_TAGS=example,local,ecs \
+	--build-arg BUILD_TAGS=example,local,e2e \
 	--build-arg GIT_TAG=$(GIT_TAG) \
 	--output ./bin
 
@@ -63,7 +63,7 @@ cross: ## Compile the CLI for linux, darwin and windows
 
 test: ## Run unit tests
 	@docker build . \
-	--build-arg BUILD_TAGS=example,local,ecs \
+	--build-arg BUILD_TAGS=example,local \
 	--build-arg GIT_TAG=$(GIT_TAG) \
 	--target test
 
diff --git a/go.mod b/go.mod
index ff6d7d76..316987ca 100644
--- a/go.mod
+++ b/go.mod
@@ -42,6 +42,8 @@ require (
 	github.com/gorilla/mux v1.7.4 // indirect
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/joho/godotenv v1.3.0
+	github.com/labstack/echo v3.3.10+incompatible
+	github.com/labstack/gommon v0.3.0 // indirect
 	github.com/moby/term v0.0.0-20200611042045-63b9a826fb74
 	github.com/morikuni/aec v1.0.0
 	github.com/onsi/gomega v1.10.1 // indirect
@@ -55,6 +57,7 @@ require (
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.6.1
+	github.com/valyala/fasttemplate v1.2.1 // indirect
 	golang.org/x/mod v0.3.0
 	golang.org/x/net v0.0.0-20200822124328-c89045814202
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
diff --git a/go.sum b/go.sum
index c393b60e..05b9e2f5 100644
--- a/go.sum
+++ b/go.sum
@@ -325,6 +325,10 @@ github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg=
+github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
+github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
+github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
@@ -334,6 +338,8 @@ github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljm
 github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw=
 github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
@@ -458,6 +464,11 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
+github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
@@ -600,6 +611,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/metrics/conn_other.go b/metrics/conn_other.go
index a648eebc..d51945f6 100644
--- a/metrics/conn_other.go
+++ b/metrics/conn_other.go
@@ -20,7 +20,7 @@ package metrics
 
 import "net"
 
-const (
+var (
 	socket = "/var/run/docker-cli.sock"
 )
 
diff --git a/metrics/conn_windows.go b/metrics/conn_windows.go
index be967a32..f10a5472 100644
--- a/metrics/conn_windows.go
+++ b/metrics/conn_windows.go
@@ -20,16 +20,20 @@ package metrics
 
 import (
 	"net"
+	"strings"
 	"time"
 
 	"github.com/Microsoft/go-winio"
 )
 
-const (
+var (
 	socket = `\\.\pipe\docker_cli`
 )
 
 func conn() (net.Conn, error) {
-	timeout := 200 * time.Millisecond
-	return winio.DialPipe(socket, &timeout)
+	if strings.HasPrefix(socket, `\\.\pipe\`) {
+		timeout := 200 * time.Millisecond
+		return winio.DialPipe(socket, &timeout)
+	}
+	return net.Dial("unix", socket)
 }


Test patch:
diff --git a/metrics/conn_e2e.go b/metrics/conn_e2e.go
new file mode 100644
index 00000000..86d83715
--- /dev/null
+++ b/metrics/conn_e2e.go
@@ -0,0 +1,31 @@
+// +build e2e
+
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package metrics
+
+import (
+	"os"
+)
+
+
+func init() {
+	testSocket, defined := os.LookupEnv("TEST_METRICS_SOCKET")
+	if defined {
+		socket = testSocket
+	}
+}
diff --git a/tests/aci-e2e/e2e-aci_test.go b/tests/aci-e2e/e2e-aci_test.go
index bf208ae5..8bd361ec 100644
--- a/tests/aci-e2e/e2e-aci_test.go
+++ b/tests/aci-e2e/e2e-aci_test.go
@@ -498,7 +498,8 @@ func TestUpUpdate(t *testing.T) {
 	composeAccountName = strings.ToLower(composeAccountName)
 
 	dstDir := filepath.Join(os.TempDir(), "e2e-aci-volume-"+composeAccountName)
-	err := fileutil.CopyDirs("../composefiles/aci-demo/", dstDir)
+	srcDir := filepath.Join("..", "composefiles", "aci-demo")
+	err := fileutil.CopyDirs(srcDir, dstDir)
 	assert.NilError(t, err)
 	t.Cleanup(func() {
 		assert.NilError(t, os.RemoveAll(dstDir))
diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go
index 5e7ed9d7..18dc00e3 100644
--- a/tests/e2e/e2e_test.go
+++ b/tests/e2e/e2e_test.go
@@ -158,6 +158,49 @@ func TestContextHelpACI(t *testing.T) {
 	})
 }
 
+func TestContextMetrics(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+	s := NewMetricsServer(c.MetricsSocket())
+	s.Start()
+	defer s.Stop()
+
+	t.Run("metrics on default context", func(t *testing.T) {
+		s.ResetUsage()
+
+		c.RunDockerCmd("ps")
+		c.RunDockerCmd("version")
+		c.RunDockerOrExitError("version", "--xxx")
+
+		usage := s.GetUsage()
+		assert.Equal(t, 3, len(usage))
+		assert.Equal(t, `{"command":"ps","context":"moby","source":"cli","status":"success"}`, usage[0])
+		assert.Equal(t, `{"command":"version","context":"moby","source":"cli","status":"success"}`, usage[1])
+		assert.Equal(t, `{"command":"version","context":"moby","source":"cli","status":"failure"}`, usage[2])
+	})
+
+	t.Run("metrics on other context type", func(t *testing.T) {
+		s.ResetUsage()
+
+		c.RunDockerCmd("context", "create", "example", "test-example")
+		c.RunDockerCmd("ps")
+		c.RunDockerCmd("context", "use", "test-example")
+		c.RunDockerCmd("ps")
+		c.RunDockerOrExitError("error")
+		c.RunDockerCmd("context", "use", "default")
+		c.RunDockerCmd("--context", "test-example", "ps")
+
+		usage := s.GetUsage()
+		assert.Equal(t, 7, len(usage))
+		assert.Equal(t, `{"command":"context create","context":"moby","source":"cli","status":"success"}`, usage[0])
+		assert.Equal(t, `{"command":"ps","context":"moby","source":"cli","status":"success"}`, usage[1])
+		assert.Equal(t, `{"command":"context use","context":"moby","source":"cli","status":"success"}`, usage[2])
+		assert.Equal(t, `{"command":"ps","context":"example","source":"cli","status":"success"}`, usage[3])
+		assert.Equal(t, `{"command":"error","context":"example","source":"cli","status":"failure"}`, usage[4])
+		assert.Equal(t, `{"command":"context use","context":"example","source":"cli","status":"success"}`, usage[5])
+		assert.Equal(t, `{"command":"ps","context":"example","source":"cli","status":"success"}`, usage[6])
+	})
+}
+
 func TestContextDuplicateACI(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 
diff --git a/tests/framework/e2e.go b/tests/framework/e2e.go
index d15313cf..4216e93b 100644
--- a/tests/framework/e2e.go
+++ b/tests/framework/e2e.go
@@ -147,6 +147,7 @@ func (c *E2eCLI) NewCmd(command string, args ...string) icmd.Cmd {
 	env := append(os.Environ(),
 		"DOCKER_CONFIG="+c.ConfigDir,
 		"KUBECONFIG=invalid",
+		"TEST_METRICS_SOCKET="+c.MetricsSocket(),
 		"PATH="+c.PathEnvVar(),
 	)
 	return icmd.Cmd{
@@ -155,6 +156,11 @@ func (c *E2eCLI) NewCmd(command string, args ...string) icmd.Cmd {
 	}
 }
 
+// MetricsSocket get the path where test metrics will be sent
+func (c *E2eCLI) MetricsSocket() string {
+	return filepath.Join(c.ConfigDir, "./docker-cli.sock")
+}
+
 // NewDockerCmd creates a docker cmd without running it
 func (c *E2eCLI) NewDockerCmd(args ...string) icmd.Cmd {
 	return c.NewCmd(filepath.Join(c.BinDir, DockerExecutableName), args...)
diff --git a/tests/framework/mockmetrics.go b/tests/framework/mockmetrics.go
new file mode 100644
index 00000000..71ed0192
--- /dev/null
+++ b/tests/framework/mockmetrics.go
@@ -0,0 +1,81 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package framework
+
+import (
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"strings"
+
+	"github.com/labstack/echo"
+)
+
+// MockMetricsServer a mock registring all metrics POST invocations
+type MockMetricsServer struct {
+	socket string
+	usage  []string
+	e      *echo.Echo
+}
+
+// NewMetricsServer instaniate a new MockMetricsServer
+func NewMetricsServer(socket string) *MockMetricsServer {
+	return &MockMetricsServer{
+		socket: socket,
+		e:      echo.New(),
+	}
+}
+
+// Handler
+func (s *MockMetricsServer) hello(c echo.Context) error {
+	body, error := ioutil.ReadAll(c.Request().Body)
+	if error != nil {
+		return error
+	}
+	cliUsage := string(body)
+	s.usage = append(s.usage, cliUsage)
+	return c.String(http.StatusOK, "")
+}
+
+// GetUsage get usage
+func (s *MockMetricsServer) GetUsage() []string {
+	return s.usage
+}
+
+// ResetUsage reset usage
+func (s *MockMetricsServer) ResetUsage() {
+	s.usage = []string{}
+}
+
+// Stop stop the mock server
+func (s *MockMetricsServer) Stop() {
+	_ = s.e.Close()
+}
+
+// Start start the mock server
+func (s *MockMetricsServer) Start() {
+	go func() {
+		listener, err := net.Listen("unix", strings.TrimPrefix(s.socket, "unix://"))
+		if err != nil {
+			log.Fatal(err)
+		}
+		s.e.Listener = listener
+		s.e.POST("/usage", s.hello)
+		_ = s.e.Start(":1323")
+	}()
+}