Repository: deis/deis

Issue:
Decouple Deis containers
Currently, controller and logger need to run on the same host because they share volumes: https://github.com/deis/deis/blob/master/controller/systemd/deis-controller.service#L11

This dependency should be removed, with each service using proper service discovery and connecting over the network.



Discussion:
Right now we have tight coupling between 2 sets of containers:
1. `deis/controller` and `deis/builder`
2. `deis/controller` and `deis/logger`

Today `deis/controller` and `deis/builder` must reside on the same machine because of the way the `deis` client constructs git remote URIs.  It's assumed the controller endpoint is the same as the builder endpoint.  Once we have both sets of traffic flowing through the platform router (see https://github.com/deis/deis/issues/535#issuecomment-43368074), we can remove the host constraint and decouple these containers.  Should be straightforward.

On the other hand, `deis/controller` and `deis/logger` must reside on the same machine because of the way `deis logs` works.  The controller responds to a `deis logs` API call by retrieving logs from the filesystem.  This was the fastest and easiest way to achieve the functionality we needed.  To facilitate this, we use Docker volumes and `--volumes-from` to share a data volume between the logger and controller.  Removing this is a trickier problem.

I think the cleanest solution is making the controller access log streams via an API call to `deis/logger`.  This would allow the component to reside anywhere in the cluster.  However, we'd have to modify `deis/logger` (currently a simple syslogd server) to add APIs endpoint for querying log data.  We also need to account for proper authentication and authorization.  Down the road we could make the log store an in-memory ring buffer to ensure we don't run out of disk space.

Thoughts?

I just read this http://www.tech-d.net/2014/07/17/libswarm-demotime-logging
Seems a way to remove the log service

development on this is blocked until the `store` component is implemented in #1384

deis-store won't have a way to mount a filesystem for the initial release. deferring.


Resolving patch:
diff --git a/controller/templates/confd_settings.py b/controller/templates/confd_settings.py
--- a/controller/templates/confd_settings.py
+++ b/controller/templates/confd_settings.py
@@ -38,7 +38,7 @@
 CACHE_URL = 'redis://{{ .deis_cache_host }}:{{ .deis_cache_port }}/0'
 
 # move log directory out of /app/deis
-DEIS_LOG_DIR = '/var/log/deis'
+DEIS_LOG_DIR = '/data/logs'
 
 {{ if .deis_controller_registrationEnabled }}
 REGISTRATION_ENABLED = bool({{ .deis_controller_registrationEnabled }})


Test patch:
diff --git a/controller/tests/controller_test.go b/controller/tests/controller_test.go
--- a/controller/tests/controller_test.go
+++ b/controller/tests/controller_test.go
@@ -29,10 +29,13 @@ func TestController(t *testing.T) {
 		"/deis/domains",
 	}
 	tag, etcdPort := utils.BuildTag(), utils.RandomPort()
+
+	//start etcd container
 	etcdName := "deis-etcd-" + tag
 	cli, stdout, stdoutPipe := dockercli.NewClient()
 	dockercli.RunTestEtcd(t, etcdName, etcdPort)
 	defer cli.CmdRm("-f", etcdName)
+
 	handler := etcdutils.InitEtcd(setdir, setkeys, etcdPort)
 	etcdutils.PublishEtcd(t, handler)
 	mock.RunMockDatabase(t, tag, etcdPort, utils.RandomPort())
diff --git a/logger/tests/logger_test.go b/logger/tests/logger_test.go
--- a/logger/tests/logger_test.go
+++ b/logger/tests/logger_test.go
@@ -13,13 +13,13 @@ import (
 func TestLogger(t *testing.T) {
 	var err error
 	tag, etcdPort := utils.BuildTag(), utils.RandomPort()
+
+	//start etcd container
 	etcdName := "deis-etcd-" + tag
 	cli, stdout, stdoutPipe := dockercli.NewClient()
 	dockercli.RunTestEtcd(t, etcdName, etcdPort)
 	defer cli.CmdRm("-f", etcdName)
-	dataName := "deis-logger-data-" + tag
-	dockercli.RunDeisDataTest(t, "--name", dataName,
-		"-v", "/var/log/deis", "ubuntu-debootstrap:14.04", "/bin/true")
+
 	host, port := utils.HostAddress(), utils.RandomPort()
 	fmt.Printf("--- Run deis/logger:%s at %s:%s\n", tag, host, port)
 	name := "deis-logger-" + tag
@@ -33,7 +33,6 @@ func TestLogger(t *testing.T) {
 			"-e", "EXTERNAL_PORT="+port,
 			"-e", "HOST="+host,
 			"-e", "ETCD_PORT="+etcdPort,
-			"--volumes-from", dataName,
 			"deis/logger:"+tag)
 	}()
 	dockercli.PrintToStdout(t, stdout, stdoutPipe, "deis-logger running")
diff --git a/store/tests/store_test.go b/store/tests/store_test.go
--- a/store/tests/store_test.go
+++ b/store/tests/store_test.go
@@ -26,7 +26,7 @@ func TestStore(t *testing.T) {
 	etcdutils.SetSingle(t, "/deis/store/hosts/"+host, hostname, etcdPort)
 
 	// test deis-store-monitor
-	fmt.Printf("--- Run deis/store-monitor:%s at %s:%s\n", tag, host, "6789")
+	fmt.Printf("--- Run deis/store-monitor:%s at %s\n", tag, host)
 	name := "deis-store-monitor-" + tag
 	defer cli.CmdRm("-f", name)
 	go func() {
@@ -34,7 +34,6 @@ func TestStore(t *testing.T) {
 		err = dockercli.RunContainer(cli,
 			"--name", name,
 			"--rm",
-			"-p", "6789",
 			"-e", "HOST="+host,
 			"-e", "ETCD_PORT="+etcdPort,
 			"-e", "NUM_STORES=1",
@@ -53,7 +52,7 @@ func TestStore(t *testing.T) {
 	etcdutils.VerifyEtcdValue(t, "/deis/store/monSetupComplete", "youBetcha", etcdPort)
 
 	// test deis-store-daemon
-	fmt.Printf("--- Run deis/store-daemon:%s at %s:%s\n", tag, host, "6800")
+	fmt.Printf("--- Run deis/store-daemon:%s at %s\n", tag, host)
 	name = "deis-store-daemon-" + tag
 	cli2, stdout2, stdoutPipe2 := dockercli.NewClient()
 	defer cli2.CmdRm("-f", "-v", name)
@@ -62,7 +61,6 @@ func TestStore(t *testing.T) {
 		err = dockercli.RunContainer(cli2,
 			"--name", name,
 			"--rm",
-			"-p", "6800",
 			"-e", "HOST="+host,
 			"-e", "ETCD_PORT="+etcdPort,
 			"--net=host",
@@ -77,15 +75,35 @@ func TestStore(t *testing.T) {
 	dockercli.DeisServiceTest(t, name, "6800", "tcp")
 	etcdutils.VerifyEtcdValue(t, "/deis/store/osds/"+host, "0", etcdPort)
 
-	// test deis-gateway
+	// test deis-store-metadata
+	fmt.Printf("--- Run deis/store-metadata:%s at %s\n", tag, host)
+	name = "deis-store-metadata-" + tag
+	cli3, stdout3, stdoutPipe3 := dockercli.NewClient()
+	defer cli3.CmdRm("-f", "-v", name)
+	go func() {
+		_ = cli3.CmdRm("-f", "-v", name)
+		err = dockercli.RunContainer(cli3,
+			"--name", name,
+			"--rm",
+			"-e", "HOST="+host,
+			"-e", "ETCD_PORT="+etcdPort,
+			"--net=host",
+			"deis/store-metadata:"+tag)
+	}()
+	dockercli.PrintToStdout(t, stdout3, stdoutPipe3, "mds.0.1 active_start")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// test deis-store-gateway
 	port := utils.RandomPort()
 	fmt.Printf("--- Run deis/store-gateway:%s at %s:%s\n", tag, host, port)
 	name = "deis-store-gateway-" + tag
-	cli3, stdout3, stdoutPipe3 := dockercli.NewClient()
-	defer cli3.CmdRm("-f", name)
+	cli4, stdout4, stdoutPipe4 := dockercli.NewClient()
+	defer cli4.CmdRm("-f", name)
 	go func() {
-		_ = cli3.CmdRm("-f", name)
-		err = dockercli.RunContainer(cli3,
+		_ = cli4.CmdRm("-f", name)
+		err = dockercli.RunContainer(cli4,
 			"--name", name,
 			"--rm",
 			"-h", "deis-store-gateway",
@@ -95,7 +113,7 @@ func TestStore(t *testing.T) {
 			"-e", "ETCD_PORT="+etcdPort,
 			"deis/store-gateway:"+tag)
 	}()
-	dockercli.PrintToStdout(t, stdout3, stdoutPipe3, "deis-store-gateway running...")
+	dockercli.PrintToStdout(t, stdout4, stdoutPipe4, "deis-store-gateway running...")
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/tests/mock/mock.go b/tests/mock/mock.go
--- a/tests/mock/mock.go
+++ b/tests/mock/mock.go
@@ -59,23 +59,22 @@ func RunMockCeph(t *testing.T, name string, cli *client.DockerCli, etcdPort stri
 	etcdutils.SetSingle(t, "/deis/store/hosts/"+utils.HostAddress(), utils.HostAddress(), etcdPort)
 
 	monitorName := name + "-monitor"
-	RunMockCephMonitor(t, monitorName, "6379", etcdPort)
+	RunMockCephMonitor(t, monitorName, etcdPort)
 
 	daemonName := name + "-daemon"
-	RunMockCephDaemon(t, daemonName, "6800", etcdPort)
+	RunMockCephDaemon(t, daemonName, etcdPort)
 
 	gatewayName := name + "-gateway"
 	RunMockCephGateway(t, gatewayName, utils.RandomPort(), etcdPort)
 }
 
 // RunMockCephMonitor runs a Ceph Monitor agent
-func RunMockCephMonitor(t *testing.T, name string, port string, etcdPort string) {
+func RunMockCephMonitor(t *testing.T, name string, etcdPort string) {
 	var err error
 	cli, stdout, stdoutPipe := dockercli.NewClient()
 	cephImage := "deis/store-monitor:" + utils.BuildTag()
 	ipaddr := utils.HostAddress()
-	cephAddr := ipaddr + ":" + port
-	fmt.Printf("--- Running deis/mock-ceph-monitor at %s\n", cephAddr)
+	fmt.Printf("--- Running deis/mock-ceph-monitor at %s\n", ipaddr)
 	done2 := make(chan bool, 1)
 	go func() {
 		done2 <- true
@@ -83,7 +82,6 @@ func RunMockCephMonitor(t *testing.T, name string, port string, etcdPort string)
 		err = dockercli.RunContainer(cli,
 			"--name", name,
 			"--rm",
-			"-p", port,
 			"-e", "HOST="+ipaddr,
 			"-e", "ETCD_PORT="+etcdPort,
 			"-e", "NUM_STORES=1",
@@ -97,13 +95,12 @@ func RunMockCephMonitor(t *testing.T, name string, port string, etcdPort string)
 }
 
 // RunMockCephDaemon sets up a single Ceph OSD
-func RunMockCephDaemon(t *testing.T, name string, port string, etcdPort string) {
+func RunMockCephDaemon(t *testing.T, name string, etcdPort string) {
 	var err error
 	cli, stdout, stdoutPipe := dockercli.NewClient()
 	cephImage := "deis/store-daemon:" + utils.BuildTag()
 	ipaddr := utils.HostAddress()
-	cephAddr := ipaddr + ":" + port
-	fmt.Printf("--- Running deis/mock-ceph-daemon at %s\n", cephAddr)
+	fmt.Printf("--- Running deis/mock-ceph-daemon at %s\n", ipaddr)
 	done := make(chan bool, 1)
 	go func() {
 		done <- true
@@ -111,7 +108,6 @@ func RunMockCephDaemon(t *testing.T, name string, port string, etcdPort string)
 		err = dockercli.RunContainer(cli,
 			"--name", name,
 			"--rm",
-			"-p", port,
 			"-e", "HOST="+ipaddr,
 			"-e", "ETCD_PORT="+etcdPort,
 			"--net=host",