Repository: deis/deis

Issue:
refactoring logs and announcers
I was wondering if the logs and announcer containers can be refactored in some way?

currently if I create an app and scale it to say 10 containers, there is about 30 containers to go through and maintain... then on top, I might have another 10 worker containers and 10 clock containers for the app, then I am dealing with 90 containers for just one app... 

in reality I would have thought just 10 containers for the web is sufficient and instead a separate monitor container can keep track of the containers and take care of announcing things to etcd and redirecting logs (eg https://github.com/discordianfish/docker-spotter). the monitor is attached to the docker daemon, so it would know about all the different docker containers that are in that instance and can also do extra things such as healthchecks, alerts, special actions to take when specific containers goes down etc.
The monitor can also take care of things such as total resource used by a particular Deis app by adding the values across the cluster, the quota for the containers, etc.

wdyt?



Discussion:

Resolving patch:
diff --git a/controller/scheduler/coreos.py b/controller/scheduler/coreos.py
--- a/controller/scheduler/coreos.py
+++ b/controller/scheduler/coreos.py
@@ -1,7 +1,6 @@
 import cStringIO
 import base64
 import copy
-import functools
 import json
 import httplib
 import paramiko
@@ -105,7 +104,6 @@ def create(self, name, image, command='', template=None, **kwargs):
         """Create a container"""
         self._create_container(name, image, command,
                                template or copy.deepcopy(CONTAINER_TEMPLATE), **kwargs)
-        self._create_log(name, image, command, copy.deepcopy(LOG_TEMPLATE))
 
     def _create_container(self, name, image, command, unit, **kwargs):
         l = locals().copy()
@@ -138,15 +136,6 @@ def _create_container(self, name, image, command, unit, **kwargs):
         # post unit to fleet
         self._put_unit(name, {"desiredState": "launched", "options": unit})
 
-    def _create_log(self, name, image, command, unit):
-        l = locals().copy()
-        l.update(re.match(MATCH, name).groupdict())
-        # construct unit from template
-        for f in unit:
-            f['value'] = f['value'].format(**l)
-        # post unit to fleet
-        self._put_unit(name+'-log', {"desiredState": "launched", "options": unit})
-
     def start(self, name):
         """Start a container"""
         self._wait_for_container(name)
@@ -181,23 +170,16 @@ def stop(self, name):
 
     def destroy(self, name):
         """Destroy a container"""
-        funcs = []
-        funcs.append(functools.partial(self._destroy_container, name))
-        funcs.append(functools.partial(self._destroy_log, name))
         # call all destroy functions, ignoring any errors
-        for f in funcs:
-            try:
-                f()
-            except:
-                pass
+        try:
+            self._destroy_container(name)
+        except:
+            pass
         self._wait_for_destroy(name)
 
     def _destroy_container(self, name):
         return self._delete_unit(name)
 
-    def _destroy_log(self, name):
-        return self._delete_unit(name+'-log')
-
     def run(self, name, image, entrypoint, command):  # noqa
         """Run a one-off command"""
         self._create_container(name, image, command, copy.deepcopy(RUN_TEMPLATE),
@@ -307,16 +289,6 @@ def attach(self, name):
 ]
 
 
-LOG_TEMPLATE = [
-    {"section": "Unit", "name": "Description", "value": "{name} log"},
-    {"section": "Unit", "name": "BindsTo", "value": "{name}.service"},
-    {"section": "Service", "name": "ExecStartPre", "value": '''/bin/sh -c "until docker inspect {name} >/dev/null 2>&1; do sleep 1; done"'''},  # noqa
-    {"section": "Service", "name": "ExecStart", "value": '''/bin/sh -c "docker logs -f {name} 2>&1 | logger -p local0.info -t {app}[{c_type}.{c_num}] --udp --server $(etcdctl get /deis/logs/host) --port $(etcdctl get /deis/logs/port)"'''},  # noqa
-    {"section": "Service", "name": "TimeoutStartSec", "value": "20m"},
-    {"section": "X-Fleet", "name": "MachineOf", "value": "{name}.service"},
-]
-
-
 RUN_TEMPLATE = [
     {"section": "Unit", "name": "Description", "value": "{name} admin command"},
     {"section": "Service", "name": "ExecStartPre", "value": '''/bin/sh -c "IMAGE=$(etcdctl get /deis/registry/host 2>&1):$(etcdctl get /deis/registry/port 2>&1)/{image}; docker pull $IMAGE"'''},  # noqa


Test patch:
diff --git a/logger/syslog/filehandler_test.go b/logger/syslog/filehandler_test.go
--- a/logger/syslog/filehandler_test.go
+++ b/logger/syslog/filehandler_test.go
@@ -2,7 +2,6 @@ package syslog
 
 import (
 	"testing"
-	"time"
 )
 
 type TestLogger struct{}
@@ -12,7 +11,7 @@ func (tl TestLogger) Printf(format string, v ...interface{}) {}
 func (tl TestLogger) Println(...interface{})                 {}
 
 func TestNewFileHandler(t *testing.T) {
-	fh := NewFileHandler("", 1, func(m *Message) bool { return true }, true)
+	fh := NewFileHandler("", 1, func(m SyslogMessage) bool { return true }, true)
 	if fh == nil {
 		t.Errorf("expected filehandler, got nil")
 	}
@@ -20,7 +19,7 @@ func TestNewFileHandler(t *testing.T) {
 
 func TestSetLogger(t *testing.T) {
 	tl := TestLogger{}
-	fh := NewFileHandler("", 1, func(m *Message) bool { return true }, true)
+	fh := NewFileHandler("", 1, func(m SyslogMessage) bool { return true }, true)
 	fh.SetLogger(tl)
 	if fh.l != tl {
 		t.Errorf("expected the logger to be set")
@@ -28,8 +27,8 @@ func TestSetLogger(t *testing.T) {
 }
 
 func TestHandle(t *testing.T) {
-	fh := NewFileHandler("/tmp/test", 1, func(m *Message) bool { return true }, true)
-	handle := fh.Handle(&Message{time.Now(), nil, 0, 0, time.Now(), "localhost", "test", "message", "", ""})
+	fh := NewFileHandler("/tmp/test", 1, func(m SyslogMessage) bool { return true }, true)
+	handle := fh.Handle(&Message{"localhost test message"})
 	if handle == nil {
 		t.Errorf("expected a handle, got nil")
 	}
diff --git a/logger/syslog/message_test.go b/logger/syslog/message_test.go
deleted file mode 100644
--- a/logger/syslog/message_test.go
+++ /dev/null
@@ -1,80 +0,0 @@
-package syslog
-
-import (
-	"net"
-	"testing"
-	"time"
-)
-
-func TestMessageNetSrc(t *testing.T) {
-	tcpAddress, err := net.ResolveTCPAddr("tcp", "localhost:1234")
-
-	if err != nil {
-		t.Error("could not resolve TCP address")
-	}
-
-	m := &Message{time.Now(), tcpAddress, 0, 0, time.Now(), "", "", "", "", ""}
-	if m.NetSrc() != "127.0.0.1" {
-		t.Errorf("expected 127.0.0.1, got %s", m.NetSrc())
-	}
-
-	udpAddress, err := net.ResolveUDPAddr("udp", "localhost:1234")
-
-	if err != nil {
-		t.Error("could not resolve UDP address")
-	}
-
-	m.Source = udpAddress
-	if m.NetSrc() != "127.0.0.1" {
-		t.Errorf("expected 127.0.0.1, got %s", m.NetSrc())
-	}
-
-	unixAddress, err := net.ResolveUnixAddr("unix", "/tmp/str")
-
-	if err != nil {
-		t.Error("could not resolve unix address")
-	}
-
-	m.Source = unixAddress
-	if m.NetSrc() != "/tmp/str" {
-		t.Errorf("expected /tmp/str, got %s", m.NetSrc())
-	}
-
-	unknownAddress, err := net.ResolveIPAddr("ip4", "localhost")
-
-	if err != nil {
-		t.Error("could not resolve unknown address")
-	}
-
-	m.Source = unknownAddress
-	if m.NetSrc() != "127.0.0.1" {
-		t.Errorf("expected 127.0.0.1, got %s", m.NetSrc())
-	}
-}
-
-func TestMessageFormat(t *testing.T) {
-	tcpAddress, err := net.ResolveTCPAddr("tcp", "localhost:1234")
-
-	if err != nil {
-		t.Errorf("could not resolve TCP address")
-	}
-
-	m := &Message{
-		time.Now(),
-		tcpAddress,
-		0,
-		0,
-		time.Now(),
-		"localhost",
-		"TEST",
-		"hello world",
-		"",
-		"",
-	}
-
-	timeLayout := "2006-01-02 15:04:05"
-	expectedOutput := m.Time.Format(timeLayout) + " localhost TESThello world"
-	if m.String() != expectedOutput {
-		t.Errorf("expected '" + expectedOutput + "', got '" + m.String() + "'.")
-	}
-}
diff --git a/tests/apps_test.go b/tests/apps_test.go
--- a/tests/apps_test.go
+++ b/tests/apps_test.go
@@ -80,7 +80,8 @@ func appsLogsTest(t *testing.T, params *utils.DeisTestConfig) {
 	}
 	utils.Execute(t, gitPushCmd, params, false, "")
 	utils.Curl(t, params)
-	utils.Execute(t, cmd, params, false, "")
+	utils.Execute(t, cmd, params, false, "created initial release")
+	utils.Execute(t, cmd, params, false, "listening on 5000...")
 	if err := utils.Chdir(".."); err != nil {
 		t.Fatal(err)
 	}