Repository: deis/deis

Issue:
Intermittent failures when scaling up/down processes
This isn't blocking for me, but I thought I'd document this. Following the EC2 tutorial and deploying the sinatra example app (on three m3.large instances with two routers), I tried `deis scale web=25`. Usually most workers come up except for three or four. `deis ps` might look like this

```
--- web:
web.1 initialized (v2)
web.2 up (v2)
web.3 up (v2)
web.4 up (v2)
web.5 up (v2)
web.6 up (v2)
web.7 up (v2)
web.8 up (v2)
web.9 up (v2)
web.10 up (v2)
web.11 up (v2)
web.12 up (v2)
web.13 down (v2)
web.14 up (v2)
web.15 up (v2)
web.16 up (v2)
web.17 down (v2)
web.18 down (v2)
web.19 up (v2)
web.20 up (v2)
web.21 up (v2)
web.22 up (v2)
web.23 up (v2)
web.24 up (v2)
web.25 up (v2)
```

and `fleetctl journal deis-controller` will contain this:

```
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: 2014-08-09 17:14:15 [98] [DEBUG] POST /api/apps/kingly-jailbird/scale
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,524: INFO/MainProcess] Received task: api.tasks.start_containers[5e7cfb11-a771-4f39-81c1-3f9ab8e5be4d]
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,575: WARNING/Worker-7] Creating kingly-jailbird_v2.web.5
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,577: WARNING/Worker-7] Creating kingly-jailbird_v2.web.3
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,600: WARNING/Worker-7] Creating kingly-jailbird_v2.web.6
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,602: WARNING/Worker-7] Creating kingly-jailbird_v2.web.4
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,639: WARNING/Worker-7] Creating kingly-jailbird_v2.web.7
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,665: WARNING/Worker-7] Creating kingly-jailbird_v2.web.9
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,703: WARNING/Worker-7] Creating kingly-jailbird_v2.web.10
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,664: WARNING/Worker-7] Creating kingly-jailbird_v2.web.8
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,734: WARNING/Worker-7] Creating kingly-jailbird_v2.web.11
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,803: WARNING/Worker-7] Creating kingly-jailbird_v2.web.12
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,829: WARNING/Worker-7] Creating kingly-jailbird_v2.web.13
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,859: WARNING/Worker-7] Creating kingly-jailbird_v2.web.14
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,908: WARNING/Worker-7] Creating kingly-jailbird_v2.web.15
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,947: WARNING/Worker-7] Creating kingly-jailbird_v2.web.16
Aug 09 23:14:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:15,991: WARNING/Worker-7] Creating kingly-jailbird_v2.web.17
Aug 09 23:14:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:16,052: WARNING/Worker-7] Creating kingly-jailbird_v2.web.18
Aug 09 23:14:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:16,088: WARNING/Worker-7] Creating kingly-jailbird_v2.web.19
Aug 09 23:14:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:16,153: WARNING/Worker-7] Creating kingly-jailbird_v2.web.20
Aug 09 23:14:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:16,259: WARNING/Worker-7] Creating kingly-jailbird_v2.web.21
Aug 09 23:14:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:16,316: WARNING/Worker-7] Creating kingly-jailbird_v2.web.22
Aug 09 23:14:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:16,395: WARNING/Worker-7] Creating kingly-jailbird_v2.web.23
Aug 09 23:14:17 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:16,829: WARNING/Worker-7] Creating kingly-jailbird_v2.web.24
Aug 09 23:14:18 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:18,165: WARNING/Worker-7] Creating kingly-jailbird_v2.web.25
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,082: WARNING/Worker-7] Starting kingly-jailbird_v2.web.3
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,098: WARNING/Worker-7] Starting kingly-jailbird_v2.web.4
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,116: WARNING/Worker-7] Starting kingly-jailbird_v2.web.5
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,137: WARNING/Worker-7] Starting kingly-jailbird_v2.web.6
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,162: WARNING/Worker-7] Starting kingly-jailbird_v2.web.7
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,208: WARNING/Worker-7] Starting kingly-jailbird_v2.web.8
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,232: WARNING/Worker-7] Starting kingly-jailbird_v2.web.9
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,252: WARNING/Worker-7] Starting kingly-jailbird_v2.web.10
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.3.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.4.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,286: WARNING/Worker-7] Starting kingly-jailbird_v2.web.11
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.6.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.7.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.5.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,507: WARNING/Worker-7] Starting kingly-jailbird_v2.web.12
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.9.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.8.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.3-log.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.10.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.11.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.6-log.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.4-log.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.7-log.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,598: WARNING/Worker-7] Starting kingly-jailbird_v2.web.13
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,871: WARNING/Worker-7] Starting kingly-jailbird_v2.web.14
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,878: WARNING/Worker-7] Starting kingly-jailbird_v2.web.15
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.5-log.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.3-announce.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.9-log.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.12.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.8-log.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:21,961: WARNING/Worker-7] Starting kingly-jailbird_v2.web.16
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.4-announce.service start
Aug 09 23:14:21 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.6-announce.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.7-announce.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.10-log.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.13.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.11-log.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.14.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.15.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.9-announce.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.16.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.10-announce.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:22,247: WARNING/Worker-7] Starting kingly-jailbird_v2.web.17
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.12-log.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.5-announce.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.13-log.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.14-log.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.15-log.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.17.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.8-announce.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.16-log.service start
Aug 09 23:14:22 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.11-announce.service start
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:22,730: WARNING/Worker-7] Starting kingly-jailbird_v2.web.18
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied (publickey,password,keyboard-interactive).
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.15-announce.service start
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied (publickey,password,keyboard-interactive).
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied (publickey,password,keyboard-interactive).
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.12-announce.service start
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.14-announce.service start
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:23,359: WARNING/Worker-7] Starting kingly-jailbird_v2.web.19
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.16-announce.service start
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:23,507: WARNING/Worker-7] Starting kingly-jailbird_v2.web.20
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:23,639: WARNING/Worker-7] Starting kingly-jailbird_v2.web.22
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied (publickey,password,keyboard-interactive).
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied (publickey,password,keyboard-interactive).
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:23,637: WARNING/Worker-7] Starting kingly-jailbird_v2.web.21
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:23,706: WARNING/Worker-7] Starting kingly-jailbird_v2.web.24
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:23,810: WARNING/Worker-7] Starting kingly-jailbird_v2.web.25
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:23,808: WARNING/Worker-7] Starting kingly-jailbird_v2.web.23
Aug 09 23:14:23 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.19.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.20.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:24,172: WARNING/Worker-7] Exception in thread Thread-32:
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Traceback (most recent call last):
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: self.run()
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 504, in run
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: self.__target(*self.__args, **self.__kwargs)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 47, in _inner
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: func(*args, **kwargs)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 267, in _change_state
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: return fsm_meta.field.change_state(instance, func, *args, **kwargs)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 161, in change_state
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: result = method(instance, *args, **kwargs)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 319, in start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: self._scheduler.start(self._job_id, self._command_announceable())
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 115, in start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: self._start_container(name, env)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 132, in _start_container
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: shell=True, env=env)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/subprocess.py", line 511, in check_call
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: raise CalledProcessError(retcode, cmd)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: CalledProcessError: Command 'fleetctl.sh start -no-block kingly-jailbird_v2.web.18.service' returned non-zero exit status 255
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:24,171: WARNING/Worker-7] Exception in thread Thread-22:
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Traceback (most recent call last):
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: self.run()
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 504, in run
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: self.__target(*self.__args, **self.__kwargs)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 47, in _inner
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: func(*args, **kwargs)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 267, in _change_state
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: return fsm_meta.field.change_state(instance, func, *args, **kwargs)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 161, in change_state
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: result = method(instance, *args, **kwargs)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 319, in start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: self._scheduler.start(self._job_id, self._command_announceable())
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 119, in start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: self._start_announcer(name, env)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 137, in _start_announcer
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: shell=True, env=env)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/subprocess.py", line 511, in check_call
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: raise CalledProcessError(retcode, cmd)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: CalledProcessError: Command 'fleetctl.sh start -no-block kingly-jailbird_v2.web.13-announce.service' returned non-zero exit status 255
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:14:24,171: WARNING/Worker-7] Exception in thread Thread-30:
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Traceback (most recent call last):
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: self.run()
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 504, in run
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: self.__target(*self.__args, **self.__kwargs)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 47, in _inner
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: func(*args, **kwargs)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 267, in _change_state
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: return fsm_meta.field.change_state(instance, func, *args, **kwargs)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 161, in change_state
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: result = method(instance, *args, **kwargs)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 319, in start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: self._scheduler.start(self._job_id, self._command_announceable())
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 116, in start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: self._start_log(name, env)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 127, in _start_log
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: shell=True, env=env)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/subprocess.py", line 511, in check_call
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: raise CalledProcessError(retcode, cmd)
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: CalledProcessError: Command 'fleetctl.sh start -no-block kingly-jailbird_v2.web.17-log.service' returned non-zero exit status 255
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.24.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.25.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.21.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.22.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.23.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.20-log.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.19-log.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.21-log.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.20-announce.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.25-log.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.24-log.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.22-log.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.23-log.service start
Aug 09 23:14:24 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.19-announce.service start
Aug 09 23:14:25 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.21-announce.service start
Aug 09 23:14:25 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.25-announce.service start
Aug 09 23:14:25 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.24-announce.service start
Aug 09 23:14:25 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.22-announce.service start
Aug 09 23:14:25 ip-10-0-182-162.ec2.internal sh[7880]: Triggered job kingly-jailbird_v2.web.23-announce.service start
Aug 09 23:15:18 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:15:18,108: INFO/MainProcess] Task api.tasks.start_containers[5e7cfb11-a771-4f39-81c1-3f9ab8e5be4d] succeeded in 62.580956961s: None
Aug 09 23:15:18 ip-10-0-182-162.ec2.internal sh[7880]: INFO kingly-jailbird: Containers scaled web=25
```

When scaling back down, a large number of exceptions appear in the journal

```
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,147: INFO/MainProcess] Received task: api.tasks.stop_containers[160c792e-0621-4651-8283-0ad34cf12e2e]
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,249: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.24
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,275: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.23
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,279: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.16
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,300: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.25
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,306: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.15
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,324: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.21
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,330: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.19
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,332: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.14
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,333: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.20
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,333: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.17
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,334: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.22
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,394: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.18
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.16-announce.service
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.24-announce.service
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,395: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.13
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,443: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.12
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.23-announce.service
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.15-announce.service
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.25-announce.service
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.21-announce.service
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,488: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.11
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.19-announce.service
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.22-announce.service
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.14-announce.service
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.18-announce.service
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.17-announce.service
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.20-announce.service
Aug 09 23:26:14 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.13-announce.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.16.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.24.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,657: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.9
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.23.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.12-announce.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.25.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.19.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.22.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.15.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.11-announce.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,696: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.8
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.14.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.17.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.18.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.20.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,694: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.10
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.13.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.21.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.12.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.24-log.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.23-log.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.16-log.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied (publickey,password,keyboard-interactive).
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied (publickey,password,keyboard-interactive).
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied (publickey,password,keyboard-interactive).
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied (publickey,password,keyboard-interactive).
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.22-log.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.9-announce.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:14,771: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.7
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.14-log.service
Aug 09 23:26:15 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.18-log.service
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied (publickey,password,keyboard-interactive).
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.8-announce.service
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied, please try again.
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Permission denied (publickey,password,keyboard-interactive).
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.10-announce.service
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.12-log.service
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:15,469: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.6
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.21-log.service
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:15,807: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.5
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:16,064: WARNING/Worker-8] Destroying kingly-jailbird_v2.web.4
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.9.service
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:16,383: WARNING/Worker-8] Exception in thread Thread-1:
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Traceback (most recent call last):
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self.run()
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 504, in run
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self.__target(*self.__args, **self.__kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 47, in _inner
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: func(*args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 267, in _change_state
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: return fsm_meta.field.change_state(instance, func, *args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 161, in change_state
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: result = method(instance, *args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 359, in destroy
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self._scheduler.destroy(self._job_id, self._command_announceable())
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 195, in destroy
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self._destroy_log(name, env)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 210, in _destroy_log
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: shell=True, env=env)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/subprocess.py", line 511, in check_call
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: raise CalledProcessError(retcode, cmd)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: CalledProcessError: Command 'fleetctl.sh destroy kingly-jailbird_v2.web.25-log.service' returned non-zero exit status 255
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.7-announce.service
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.20-log.service
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:16,504: WARNING/Worker-8] Exception in thread Thread-17:
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Traceback (most recent call last):
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self.run()
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 504, in run
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self.__target(*self.__args, **self.__kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 47, in _inner
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: func(*args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 267, in _change_state
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: return fsm_meta.field.change_state(instance, func, *args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 161, in change_state
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: result = method(instance, *args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 359, in destroy
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self._scheduler.destroy(self._job_id, self._command_announceable())
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 195, in destroy
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self._destroy_log(name, env)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 210, in _destroy_log
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: shell=True, env=env)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/subprocess.py", line 511, in check_call
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: raise CalledProcessError(retcode, cmd)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: CalledProcessError: Command 'fleetctl.sh destroy kingly-jailbird_v2.web.17-log.service' returned non-zero exit status 255
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.10.service
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.8.service
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:16,603: WARNING/Worker-8] Exception in thread Thread-25:
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Traceback (most recent call last):
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self.run()
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 504, in run
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self.__target(*self.__args, **self.__kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 47, in _inner
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: func(*args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 267, in _change_state
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: return fsm_meta.field.change_state(instance, func, *args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 161, in change_state
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: result = method(instance, *args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 359, in destroy
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self._scheduler.destroy(self._job_id, self._command_announceable())
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 195, in destroy
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self._destroy_log(name, env)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 210, in _destroy_log
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: shell=True, env=env)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/subprocess.py", line 511, in check_call
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: raise CalledProcessError(retcode, cmd)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: CalledProcessError: Command 'fleetctl.sh destroy kingly-jailbird_v2.web.13-log.service' returned non-zero exit status 255
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:16,606: WARNING/Worker-8] Exception in thread Thread-13:
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Traceback (most recent call last):
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self.run()
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 504, in run
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self.__target(*self.__args, **self.__kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 47, in _inner
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: func(*args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 267, in _change_state
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: return fsm_meta.field.change_state(instance, func, *args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 161, in change_state
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: result = method(instance, *args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 359, in destroy
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self._scheduler.destroy(self._job_id, self._command_announceable())
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 195, in destroy
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self._destroy_log(name, env)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 210, in _destroy_log
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: shell=True, env=env)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/subprocess.py", line 511, in check_call
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: raise CalledProcessError(retcode, cmd)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: CalledProcessError: Command 'fleetctl.sh destroy kingly-jailbird_v2.web.19-log.service' returned non-zero exit status 255
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:16,633: WARNING/Worker-8] Exception in thread Thread-21:
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Traceback (most recent call last):
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self.run()
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 504, in run
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self.__target(*self.__args, **self.__kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 47, in _inner
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: func(*args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 267, in _change_state
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: return fsm_meta.field.change_state(instance, func, *args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 161, in change_state
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: result = method(instance, *args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 359, in destroy
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self._scheduler.destroy(self._job_id, self._command_announceable())
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 195, in destroy
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self._destroy_log(name, env)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 210, in _destroy_log
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: shell=True, env=env)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/subprocess.py", line 511, in check_call
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: raise CalledProcessError(retcode, cmd)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: CalledProcessError: Command 'fleetctl.sh destroy kingly-jailbird_v2.web.15-log.service' returned non-zero exit status 255
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.4-announce.service
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:16,670: WARNING/Worker-8] Exception in thread Thread-29:
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Traceback (most recent call last):
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self.run()
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/threading.py", line 504, in run
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self.__target(*self.__args, **self.__kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 47, in _inner
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: func(*args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 267, in _change_state
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: return fsm_meta.field.change_state(instance, func, *args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/local/lib/python2.7/dist-packages/django_fsm/__init__.py", line 161, in change_state
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: result = method(instance, *args, **kwargs)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/api/models.py", line 359, in destroy
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self._scheduler.destroy(self._job_id, self._command_announceable())
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 194, in destroy
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: self._destroy_container(name, env)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/app/scheduler/coreos.py", line 200, in _destroy_container
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: shell=True, env=env)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: File "/usr/lib/python2.7/subprocess.py", line 511, in check_call
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: raise CalledProcessError(retcode, cmd)
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: CalledProcessError: Command 'fleetctl.sh destroy kingly-jailbird_v2.web.11.service' returned non-zero exit status 255
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.5-announce.service
Aug 09 23:26:16 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.6-announce.service
Aug 09 23:26:17 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.9-log.service
Aug 09 23:26:17 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.7.service
Aug 09 23:26:17 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.5.service
Aug 09 23:26:17 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.10-log.service
Aug 09 23:26:17 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.4.service
Aug 09 23:26:17 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.8-log.service
Aug 09 23:26:17 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.6.service
Aug 09 23:26:17 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.4-log.service
Aug 09 23:26:17 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.5-log.service
Aug 09 23:26:17 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.7-log.service
Aug 09 23:26:17 ip-10-0-182-162.ec2.internal sh[7880]: Destroyed kingly-jailbird_v2.web.6-log.service
Aug 09 23:26:17 ip-10-0-182-162.ec2.internal sh[7880]: [2014-08-09 17:26:17,792: INFO/MainProcess] Task api.tasks.stop_containers[160c792e-0621-4651-8283-0ad34cf12e2e] succeeded in 3.591019307s: None
Aug 09 23:26:18 ip-10-0-182-162.ec2.internal sh[7880]: INFO kingly-jailbird: Containers scaled web=3
```

`deis ps` will then only list three services. This isn't terrible for me, but it'd be a problem if I wanted to run 25 containers in productions.

PS: `fleetctl journal deis-controller` is also littered with lines saying things like `sh[7880]: Permission denied (publickey,password,keyboard-interactive).`. I guess this is benign?



Discussion:
Here's the output of `fleetctl list-units` after scaling up:

```
➜  fleetctl list-units
UNIT                        DSTATE      TMACHINE            STATE       MACHINE             ACTIVE
deis-builder-data.service           loaded      6481092d.../10.0.182.162    loaded      6481092d.../10.0.182.162    active
deis-builder.service                launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
deis-cache.service              launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
deis-controller.service             launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
deis-database-data.service          loaded      6481092d.../10.0.182.162    loaded      6481092d.../10.0.182.162    active
deis-database.service               launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
deis-logger-data.service            loaded      6481092d.../10.0.182.162    loaded      6481092d.../10.0.182.162    active
deis-logger.service             launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
deis-registry-data.service          loaded      e6de6880.../10.0.112.80     loaded      e6de6880.../10.0.112.80     active
deis-registry.service               launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
deis-router@1.service               launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
deis-router@2.service               loaded      af4287d4.../10.0.17.33      loaded      af4287d4.../10.0.17.33      inactive
kingly-jailbird_v2.web.10-announce.service  launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.10-log.service       launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.10.service       launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.11-announce.service  launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.11-log.service       launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.11.service       launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.12-announce.service  launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.12-log.service       launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.12.service       launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.13-announce.service  inactive    -               inactive    -               -
kingly-jailbird_v2.web.13-log.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.13.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.14-announce.service  launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.14-log.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.14.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.15-announce.service  launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.15-log.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.15.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.16-announce.service  launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.16-log.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.16.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.17-announce.service  inactive    -               inactive    -               -
kingly-jailbird_v2.web.17-log.service       inactive    -               inactive    -               -
kingly-jailbird_v2.web.17.service       launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
kingly-jailbird_v2.web.18-announce.service  inactive    -               inactive    -               -
kingly-jailbird_v2.web.18-log.service       inactive    -               inactive    -               -
kingly-jailbird_v2.web.18.service       inactive    -               inactive    -               -
kingly-jailbird_v2.web.19-announce.service  launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
kingly-jailbird_v2.web.19-log.service       launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
kingly-jailbird_v2.web.19.service       launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
kingly-jailbird_v2.web.2-announce.service   launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.2-log.service        launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.2.service        launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.20-announce.service  launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
kingly-jailbird_v2.web.20-log.service       launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
kingly-jailbird_v2.web.20.service       launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
kingly-jailbird_v2.web.21-announce.service  launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
kingly-jailbird_v2.web.21-log.service       launched    6481092d.../10.0.182.162    launched    6481092d.../10.0.182.162    active
kingly-jailbird_v2.web.22-announce.service  launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.22-log.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.22.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.23-announce.service  launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.23-log.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.23.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.24-announce.service  launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.24-log.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.24.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.25-announce.service  launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.25-log.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.25.service       launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.3-announce.service   launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.3-log.service        launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.3.service        launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.4-announce.service   launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.4-log.service        launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.4.service        launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.5-announce.service   launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.5-log.service        launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.5.service        launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.6-announce.service   launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.6-log.service        launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.6.service        launched    af4287d4.../10.0.17.33      launched    af4287d4.../10.0.17.33      active
kingly-jailbird_v2.web.7-announce.service   launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.7-log.service        launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.7.service        launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.8-announce.service   launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.8-log.service        launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.8.service        launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.9-announce.service   launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.9-log.service        launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
kingly-jailbird_v2.web.9.service        launched    e6de6880.../10.0.112.80     launched    e6de6880.../10.0.112.80     active
```

@jonashuckestein Could you show us your cluster? `deis clusters:info <clustername>`

I'm guessing you'll find that one of the IPs you've supplied to the `clusters:create` command was launched separately, with a different SSH key. This is why you're only occasionally seeing a `Permission denied (publickey,password,keyboard-interactive).` - each time the controller schedules a job to the scheduler, it chooses a host at random to connect to in order to submit the job.

I'll look it up when I get a chance. The list-units output does show my app containers coming up on all three instances though, doesn't it? 

Sent from my phone

> On Aug 10, 2014, at 10:55 AM, Chris Armstrong notifications@github.com wrote:
> 
> I'm guessing you'll find that one of the IPs you've supplied to the clusters:create command was launched separately, with a different SSH key. This is why you're only occasionally seeing a Permission denied (publickey,password,keyboard-interactive)..
> 
> —
> Reply to this email directly or view it on GitHub.


Resolving patch:
diff --git a/controller/api/models.py b/controller/api/models.py
--- a/controller/api/models.py
+++ b/controller/api/models.py
@@ -402,7 +402,7 @@ def _command_announceable(self):
 
     @transition(field=state, source=INITIALIZED, target=CREATED)
     def create(self):
-        image = self.release.image
+        image = self.release.image + ':v' + str(self.release.version)
         kwargs = {'memory': self.release.config.memory,
                   'cpu': self.release.config.cpu,
                   'tags': self.release.config.tags}
@@ -427,7 +427,7 @@ def deploy(self, new_release):
         self.release = new_release
         # deploy new container
         new_job_id = self._job_id
-        image = self.release.image
+        image = self.release.image + ':v' + str(self.release.version)
         c_type = self.type
         kwargs = {'memory': self.release.config.memory,
                   'cpu': self.release.config.cpu,
@@ -453,7 +453,8 @@ def destroy(self):
 
     def run(self, command):
         """Run a one-off command"""
-        rc, output = self._scheduler.run(self._job_id, self.release.image, command)
+        image = self.release.image + ':v' + str(self.release.version)
+        rc, output = self._scheduler.run(self._job_id, image, command)
         return rc, output
 
 
diff --git a/controller/scheduler/coreos.py b/controller/scheduler/coreos.py
--- a/controller/scheduler/coreos.py
+++ b/controller/scheduler/coreos.py
@@ -1,60 +1,102 @@
-from cStringIO import StringIO
+import cStringIO
 import base64
-import os
-import random
+import copy
+import json
+import httplib
+import paramiko
+import socket
 import re
-import subprocess
 import time
 
 
-ROOT_DIR = os.path.join(os.getcwd(), 'coreos')
-if not os.path.exists(ROOT_DIR):
-    os.mkdir(ROOT_DIR)
-
 MATCH = re.compile(
     '(?P<app>[a-z0-9-]+)_?(?P<version>v[0-9]+)?\.?(?P<c_type>[a-z]+)?.(?P<c_num>[0-9]+)')
 
 
-class FleetClient(object):
+class UHTTPConnection(httplib.HTTPConnection):
+    """Subclass of Python library HTTPConnection that uses a Unix domain socket.
+    """
+
+    def __init__(self, path):
+        httplib.HTTPConnection.__init__(self, 'localhost')
+        self.path = path
+
+    def connect(self):
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        sock.connect(self.path)
+        self.sock = sock
+
+
+class FleetHTTPClient(object):
 
     def __init__(self, cluster_name, hosts, auth, domain, options):
         self.name = cluster_name
         self.hosts = hosts
+        self.auth = auth
         self.domain = domain
         self.options = options
-        self.auth = auth
-        self.auth_path = os.path.join(ROOT_DIR, 'ssh-{cluster_name}'.format(**locals()))
-        with open(self.auth_path, 'w') as f:
-            f.write(base64.b64decode(auth))
-            os.chmod(self.auth_path, 0600)
-
-        self.env = {
-            'PATH': '/usr/local/bin:/usr/bin:/bin:{}'.format(
-                os.path.abspath(os.path.join(__file__, '..'))),
-            'FLEETW_KEY': self.auth_path,
-            'FLEETW_HOST': random.choice(self.hosts.split(','))}
+        # single global connection
+        self.conn = UHTTPConnection('/var/run/fleet.sock')
 
     # scheduler setup / teardown
 
     def setUp(self):
-        """
-        Setup a CoreOS cluster including router and log aggregator
-        """
-        return
+        pass
 
     def tearDown(self):
-        """
-        Tear down a CoreOS cluster including router and log aggregator
-        """
-        return
-
-    # announcer helpers
-
-    def _log_skipped_announcer(self, action, name):
-        """
-        Logs a message stating that this operation doesn't require an announcer
-        """
-        print "-- skipping announcer {} for {}".format(action, name)
+        pass
+
+    # connection helpers
+
+    def _put_unit(self, name, body):
+        headers = {'Content-Type': 'application/json'}
+        self.conn.request('PUT', '/v1-alpha/units/{name}.service'.format(**locals()),
+                          headers=headers, body=json.dumps(body))
+        resp = self.conn.getresponse()
+        data = resp.read()
+        if resp.status != 204:
+            errmsg = "Failed to create unit: {} {} - {}".format(
+                resp.status, resp.reason, data)
+            raise RuntimeError(errmsg)
+        return data
+
+    def _delete_unit(self, name):
+        headers = {'Content-Type': 'application/json'}
+        self.conn.request('DELETE', '/v1-alpha/units/{name}.service'.format(**locals()),
+                          headers=headers)
+        resp = self.conn.getresponse()
+        data = resp.read()
+        if resp.status not in (404, 204):
+            errmsg = "Failed to delete unit: {} {} - {}".format(
+                resp.status, resp.reason, data)
+            raise RuntimeError(errmsg)
+        return data
+
+    def _get_state(self, name=None):
+        headers = {'Content-Type': 'application/json'}
+        url = '/v1-alpha/state'
+        if name:
+            url += '?unitName={name}.service'.format(**locals())
+        self.conn.request('GET', url, headers=headers)
+        resp = self.conn.getresponse()
+        data = resp.read()
+        if resp.status not in (200,):
+            errmsg = "Failed to retrieve state: {} {} - {}".format(
+                resp.status, resp.reason, data)
+            raise RuntimeError(errmsg)
+        return json.loads(data)
+
+    def _get_machines(self):
+        headers = {'Content-Type': 'application/json'}
+        url = '/v1-alpha/machines'
+        self.conn.request('GET', url, headers=headers)
+        resp = self.conn.getresponse()
+        data = resp.read()
+        if resp.status not in (200,):
+            errmsg = "Failed to retrieve machines: {} {} - {}".format(
+                resp.status, resp.reason, data)
+            raise RuntimeError(errmsg)
+        return json.loads(data)
 
     # job api
 
@@ -62,17 +104,16 @@ def create(self, name, image, command='', template=None, use_announcer=True, **k
         """
         Create a new job
         """
-        print 'Creating {name}'.format(**locals())
-        env = self.env.copy()
-        self._create_container(name, image, command, template or CONTAINER_TEMPLATE, env, **kwargs)
-        self._create_log(name, image, command, LOG_TEMPLATE, env)
+        self._create_container(name, image, command,
+                               template or copy.deepcopy(CONTAINER_TEMPLATE), **kwargs)
+        self._create_log(name, image, command, copy.deepcopy(LOG_TEMPLATE))
+        self._wait_for_container(name)
 
         if use_announcer:
-            self._create_announcer(name, image, command, ANNOUNCE_TEMPLATE, env)
-        else:
-            self._log_skipped_announcer('create', name)
+            self._create_announcer(name, image, command, copy.deepcopy(ANNOUNCE_TEMPLATE))
+            self._wait_for_announcer(name)
 
-    def _create_container(self, name, image, command, template, env, **kwargs):
+    def _create_container(self, name, image, command, unit, **kwargs):
         l = locals().copy()
         l.update(re.match(MATCH, name).groupdict())
         # prepare memory limit for the container type
@@ -87,217 +128,223 @@ def _create_container(self, name, image, command, template, env, **kwargs):
             l.update({'cpu': '-c {}'.format(cpu)})
         else:
             l.update({'cpu': ''})
-        env.update({'FLEETW_UNIT': name + '.service'})
         # construct unit from template
-        unit = template.format(**l)
+        for f in unit:
+            f['value'] = f['value'].format(**l)
         # prepare tags only if one was provided
         tags = kwargs.get('tags', {})
         if tags:
             tagset = ' '.join(['"{}={}"'.format(k, v) for k, v in tags.items()])
-            unit = unit + '\n[X-Fleet]\nX-ConditionMachineMetadata={}\n'.format(tagset)
-        env.update({'FLEETW_UNIT_DATA': base64.b64encode(unit)})
-        return subprocess.check_call('fleetctl.sh submit {name}.service'.format(**l),
-                                     shell=True, env=env)
+            unit.append({"section": "X-Fleet", "name": "X-ConditionMachineMetadata",
+                         "value": tagset})
+        # post unit to fleet
+        self._put_unit(name, {"desiredState": "launched", "options": unit})
 
-    def _create_announcer(self, name, image, command, template, env):
+    def _create_log(self, name, image, command, unit):
         l = locals().copy()
         l.update(re.match(MATCH, name).groupdict())
-        env.update({'FLEETW_UNIT': name + '-announce' + '.service'})
-        env.update({'FLEETW_UNIT_DATA': base64.b64encode(template.format(**l))})
-        return subprocess.check_call('fleetctl.sh submit {name}-announce.service'.format(**l),  # noqa
-                                     shell=True, env=env)
+        # construct unit from template
+        for f in unit:
+            f['value'] = f['value'].format(**l)
+        # post unit to fleet
+        self._put_unit(name+'-log', {"desiredState": "launched", "options": unit})
 
-    def _create_log(self, name, image, command, template, env):
+    def _create_announcer(self, name, image, command, unit):
         l = locals().copy()
         l.update(re.match(MATCH, name).groupdict())
-        env.update({'FLEETW_UNIT': name + '-log' + '.service'})
-        env.update({'FLEETW_UNIT_DATA': base64.b64encode(template.format(**l))})
-        return subprocess.check_call('fleetctl.sh submit {name}-log.service'.format(**locals()),  # noqa
-                                     shell=True, env=env)
+        # construct unit from template
+        for f in unit:
+            f['value'] = f['value'].format(**l)
+        # post unit to fleet
+        self._put_unit(name+'-announce', {"desiredState": "launched", "options": unit})
 
     def start(self, name, use_announcer=True):
         """
         Start an idle job
         """
-        print 'Starting {name}'.format(**locals())
-        env = self.env.copy()
-        self._start_container(name, env)
-        self._start_log(name, env)
-
-        if use_announcer:
-            self._start_announcer(name, env)
-            self._wait_for_container(name, env)
-            self._wait_for_announcer(name, env)
-        else:
-            self._log_skipped_announcer('start', name)
-
-    def _start_log(self, name, env):
-        subprocess.check_call(
-            'fleetctl.sh start -no-block {name}-log.service'.format(**locals()),
-            shell=True, env=env)
+        pass
 
-    def _start_container(self, name, env):
-        return subprocess.check_call(
-            'fleetctl.sh start -no-block {name}.service'.format(**locals()),
-            shell=True, env=env)
-
-    def _start_announcer(self, name, env):
-        return subprocess.check_call(
-            'fleetctl.sh start -no-block {name}-announce.service'.format(**locals()),
-            shell=True, env=env)
-
-    def _wait_for_container(self, name, env):
-        status = None
+    def _wait_for_container(self, name):
         # we bump to 20 minutes here to match the timeout on the router and in the app unit files
         for _ in range(1200):
-            # check if the main container's running
-            status = subprocess.check_output(
-                "fleetctl.sh list-units --no-legend --fields unit,sub | grep {name}.service | awk '{{print $2}}'".format(**locals()),  # noqa
-                shell=True, env=env).strip('\n')
-            if status == 'running':
-                break
-            elif status == 'failed':
-                raise RuntimeError('Container failed to start')
+            states = self._get_state(name)
+            if states and len(states.get('states', [])) == 1:
+                state = states.get('states')[0]
+                subState = state.get('systemdSubState')
+                if subState == 'running' or subState == 'exited':
+                    break
+                elif subState == 'failed':
+                    raise RuntimeError('Container failed to start')
             time.sleep(1)
         else:
-            raise RuntimeError('Container timeout on start')
+            raise RuntimeError('Container failed to start')
 
-    def _wait_for_announcer(self, name, env):
-        status = None
+    def _wait_for_announcer(self, name):
         # wait a bit for the announcer to come up, otherwise we may have hit
         # https://github.com/docker/docker/issues/8022
         for _ in range(30):
-            # check if the main container's running
-            status = subprocess.check_output(
-                "fleetctl.sh list-units --no-legend --fields unit,sub | grep {name}-announce.service | awk '{{print $2}}'".format(**locals()),  # noqa
-                shell=True, env=env).strip('\n')
-            if status == 'running':
-                break
-            elif status == 'failed':
-                raise RuntimeError('Announcer failed to start')
+            states = self._get_state(name)
+            if states and len(states.get('states', [])) == 1:
+                state = states.get('states')[0]
+                subState = state.get('systemdSubState')
+                if subState == 'running':
+                    # wait for the router to be reconfigured
+                    time.sleep(10)
+                    break
+                elif subState == 'failed':
+                    raise RuntimeError('Announcer failed to start')
             time.sleep(1)
         else:
             raise RuntimeError('Announcer timeout on start')
 
+    def _wait_for_destroy(self, name):
+        for _ in range(1200):
+            states = self._get_state(name)
+            if not states:
+                break
+            time.sleep(1)
+        else:
+            raise RuntimeError('Timeout on container destroy')
+
     def stop(self, name, use_announcer=True):
         """
         Stop a running job
         """
-        print 'Stopping {name}'.format(**locals())
-        env = self.env.copy()
-
-        if use_announcer:
-            self._stop_announcer(name, env)
-        else:
-            self._log_skipped_announcer('stop', name)
-
-        self._stop_container(name, env)
-        self._stop_log(name, env)
-
-    def _stop_container(self, name, env):
-        return subprocess.check_call(
-            'fleetctl.sh stop -block-attempts=600 {name}.service'.format(**locals()),
-            shell=True, env=env)
-
-    def _stop_announcer(self, name, env):
-        return subprocess.check_call(
-            'fleetctl.sh stop -block-attempts=600 {name}-announce.service'.format(**locals()),
-            shell=True, env=env)
-
-    def _stop_log(self, name, env):
-        return subprocess.check_call(
-            'fleetctl.sh stop -block-attempts=600 {name}-log.service'.format(**locals()),
-            shell=True, env=env)
+        pass
 
     def destroy(self, name, use_announcer=True):
         """
         Destroy an existing job
         """
-        print 'Destroying {name}'.format(**locals())
-        env = self.env.copy()
-
         if use_announcer:
-            self._destroy_announcer(name, env)
-        else:
-            self._log_skipped_announcer('destroy', name)
-
-        self._destroy_container(name, env)
-        self._destroy_log(name, env)
+            self._destroy_announcer(name)
+        self._destroy_container(name)
+        self._destroy_log(name)
+        self._wait_for_destroy(name)
 
-    def _destroy_container(self, name, env):
-        return subprocess.check_call(
-            'fleetctl.sh destroy {name}.service'.format(**locals()),
-            shell=True, env=env)
+    def _destroy_container(self, name):
+        return self._delete_unit(name)
 
-    def _destroy_announcer(self, name, env):
-        return subprocess.check_call(
-            'fleetctl.sh destroy {name}-announce.service'.format(**locals()),
-            shell=True, env=env)
+    def _destroy_announcer(self, name):
+        return self._delete_unit(name+'-announce')
 
-    def _destroy_log(self, name, env):
-        return subprocess.check_call(
-            'fleetctl.sh destroy {name}-log.service'.format(**locals()),
-            shell=True, env=env)
+    def _destroy_log(self, name):
+        return self._delete_unit(name+'-log')
 
     def run(self, name, image, command):
         """
         Run a one-off command
         """
-        print 'Running {name}'.format(**locals())
-        output = subprocess.PIPE
-        p = subprocess.Popen('fleetrun.sh {command}'.format(**locals()), shell=True, env=self.env,
-                             stdout=output, stderr=subprocess.STDOUT)
-        rc = p.wait()
-        return rc, p.stdout.read()
+        self._create_container(name, image, command, copy.deepcopy(RUN_TEMPLATE))
+
+        # wait for the container to return something
+        for _ in range(1200):
+            states = self._get_state(name)
+            if states and len(states.get('states', [])) == 1:
+                state = states.get('states')[0]
+                subState = state.get('systemdSubState')
+                if subState == 'exited' or subState == 'failed' or subState == 'dead':
+                    break
+            time.sleep(1)
+        machineID = state.get('machineID')
+
+        # find the machine
+        machines = self._get_machines()
+        if not machines:
+            raise RuntimeError('No available hosts to run command')
+
+        # find the machine's primaryIP
+        primaryIP = None
+        for m in machines.get('machines', []):
+            if m['id'] == machineID:
+                primaryIP = m['primaryIP']
+        if not primaryIP:
+            raise RuntimeError('Could not find host')
+
+        # prepare ssh key
+        file_obj = cStringIO.StringIO(base64.b64decode(self.auth))
+        pkey = paramiko.RSAKey(file_obj=file_obj)
+
+        # grab output via docker logs over SSH
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(primaryIP, username="core", pkey=pkey)
+
+        # get a pty so stdout/stderr look right
+        tran = ssh.get_transport()
+        chan = tran.open_session()
+        chan.get_pty()
+        out = chan.makefile()
+
+        # exec the command to gather container output
+        chan.exec_command('docker logs {name}'.format(**locals()))
+        rc, output = chan.recv_exit_status(), out.read()
+        if rc != 0:
+            raise RuntimeError('Could not attach to container')
+
+        # use another channel to inspect the container
+        chan = tran.open_session()
+        chan.get_pty()
+        out = chan.makefile()
+        chan.exec_command('docker inspect {name}'.format(**locals()))
+        rc, inspect_output = chan.recv_exit_status(), out.read()
+        if rc != 0:
+            raise RuntimeError('Could not determine exit code')
+        container = json.loads(inspect_output)
+        rc = container[0]["State"]["ExitCode"]
+
+        # cleanup
+        self._destroy_container(name)
+        self._wait_for_destroy(name)
+
+        # return rc and output
+        return rc, output
 
     def attach(self, name):
         """
         Attach to a job's stdin, stdout and stderr
         """
-        return StringIO(), StringIO(), StringIO()
-
-SchedulerClient = FleetClient
-
-
-CONTAINER_TEMPLATE = """
-[Unit]
-Description={name}
-
-[Service]
-ExecStartPre=/bin/sh -c "IMAGE=$(etcdctl get /deis/registry/host 2>&1):$(etcdctl get /deis/registry/port 2>&1)/{image}; docker pull $IMAGE"
-ExecStartPre=/bin/sh -c "docker inspect {name} >/dev/null 2>&1 && docker rm -f {name} || true"
-ExecStart=/bin/sh -c "IMAGE=$(etcdctl get /deis/registry/host 2>&1):$(etcdctl get /deis/registry/port 2>&1)/{image}; port=$(docker inspect -f '{{{{range $k, $v := .ContainerConfig.ExposedPorts }}}}{{{{$k}}}}{{{{end}}}}' $IMAGE | cut -d/ -f1) ; docker run --name {name} {memory} {cpu} -P -e PORT=$port $IMAGE {command}"
-ExecStop=/usr/bin/docker rm -f {name}
-TimeoutStartSec=20m
-"""  # noqa
-
-# TODO revisit the "not getting a port" issue after we upgrade to Docker 1.1.0
-ANNOUNCE_TEMPLATE = """
-[Unit]
-Description={name} announce
-BindsTo={name}.service
-
-[Service]
-EnvironmentFile=/etc/environment
-ExecStartPre=/bin/sh -c "until docker inspect -f '{{{{range $i, $e := .NetworkSettings.Ports }}}}{{{{$p := index $e 0}}}}{{{{$p.HostPort}}}}{{{{end}}}}' {name} >/dev/null 2>&1; do sleep 2; done; port=$(docker inspect -f '{{{{range $i, $e := .NetworkSettings.Ports }}}}{{{{$p := index $e 0}}}}{{{{$p.HostPort}}}}{{{{end}}}}' {name}); if [[ -z $port ]]; then echo We have no port...; exit 1; fi; echo Waiting for $port/tcp...; until netstat -lnt | grep :$port >/dev/null; do sleep 1; done"
-ExecStart=/bin/sh -c "port=$(docker inspect -f '{{{{range $i, $e := .NetworkSettings.Ports }}}}{{{{$p := index $e 0}}}}{{{{$p.HostPort}}}}{{{{end}}}}' {name}); echo Connected to $COREOS_PRIVATE_IPV4:$port/tcp, publishing to etcd...; while netstat -lnt | grep :$port >/dev/null; do etcdctl set /deis/services/{app}/{name} $COREOS_PRIVATE_IPV4:$port --ttl 60 >/dev/null; sleep 45; done"
-ExecStop=/usr/bin/etcdctl rm --recursive /deis/services/{app}/{name}
-TimeoutStartSec=20m
-
-[X-Fleet]
-X-ConditionMachineOf={name}.service
-"""  # noqa
-
-LOG_TEMPLATE = """
-[Unit]
-Description={name} log
-BindsTo={name}.service
-
-[Service]
-ExecStartPre=/bin/sh -c "until docker inspect {name} >/dev/null 2>&1; do sleep 1; done"
-ExecStart=/bin/sh -c "docker logs -f {name} 2>&1 | logger -p local0.info -t {app}[{c_type}.{c_num}] --udp --server $(etcdctl get /deis/logs/host) --port $(etcdctl get /deis/logs/port)"
-TimeoutStartSec=20m
-
-[X-Fleet]
-X-ConditionMachineOf={name}.service
-"""  # noqa
+        raise NotImplementedError
+
+SchedulerClient = FleetHTTPClient
+
+
+CONTAINER_TEMPLATE = [
+    {"section": "Unit", "name": "Description", "value": "{name}"},
+    {"section": "Service", "name": "ExecStartPre", "value": '''/bin/sh -c "IMAGE=$(etcdctl get /deis/registry/host 2>&1):$(etcdctl get /deis/registry/port 2>&1)/{image}; docker pull $IMAGE"'''},  # noqa
+    {"section": "Service", "name": "ExecStartPre", "value": '''/bin/sh -c "docker inspect {name} >/dev/null 2>&1 && docker rm -f {name} || true"'''},  # noqa
+    {"section": "Service", "name": "ExecStart", "value": '''/bin/sh -c "IMAGE=$(etcdctl get /deis/registry/host 2>&1):$(etcdctl get /deis/registry/port 2>&1)/{image}; port=$(docker inspect -f '{{{{range $k, $v := .ContainerConfig.ExposedPorts }}}}{{{{$k}}}}{{{{end}}}}' $IMAGE | cut -d/ -f1) ; docker run --name {name} {memory} {cpu} -P -e PORT=$port $IMAGE {command}"'''},  # noqa
+    {"section": "Service", "name": "ExecStop", "value": '''/usr/bin/docker rm -f {name}'''},
+    {"section": "Service", "name": "TimeoutStartSec", "value": "20m"},
+]
+
+
+LOG_TEMPLATE = [
+    {"section": "Unit", "name": "Description", "value": "{name} log"},
+    {"section": "Unit", "name": "BindsTo", "value": "{name}.service"},
+    {"section": "Service", "name": "ExecStartPre", "value": '''/bin/sh -c "until docker inspect {name} >/dev/null 2>&1; do sleep 1; done"'''},  # noqa
+    {"section": "Service", "name": "ExecStart", "value": '''/bin/sh -c "docker logs -f {name} 2>&1 | logger -p local0.info -t {app}[{c_type}.{c_num}] --udp --server $(etcdctl get /deis/logs/host) --port $(etcdctl get /deis/logs/port)"'''},  # noqa
+    {"section": "Service", "name": "TimeoutStartSec", "value": "20m"},
+    {"section": "X-Fleet", "name": "X-ConditionMachineOf", "value": "{name}.service"},
+]
+
+
+ANNOUNCE_TEMPLATE = [
+    {"section": "Unit", "name": "Description", "value": "{name} announce"},
+    {"section": "Unit", "name": "BindsTo", "value": "{name}.service"},
+    {"section": "Service", "name": "EnvironmentFile", "value": "/etc/environment"},
+    {"section": "Service", "name": "ExecStartPre", "value": '''/bin/sh -c "until docker inspect -f '{{{{range $i, $e := .NetworkSettings.Ports }}}}{{{{$p := index $e 0}}}}{{{{$p.HostPort}}}}{{{{end}}}}' {name} >/dev/null 2>&1; do sleep 2; done; port=$(docker inspect -f '{{{{range $i, $e := .NetworkSettings.Ports }}}}{{{{$p := index $e 0}}}}{{{{$p.HostPort}}}}{{{{end}}}}' {name}); if [[ -z $port ]]; then echo We have no port...; exit 1; fi; echo Waiting for $port/tcp...; until netstat -lnt | grep :$port >/dev/null; do sleep 1; done"'''},  # noqa
+    {"section": "Service", "name": "ExecStart", "value": '''/bin/sh -c "port=$(docker inspect -f '{{{{range $i, $e := .NetworkSettings.Ports }}}}{{{{$p := index $e 0}}}}{{{{$p.HostPort}}}}{{{{end}}}}' {name}); echo Connected to $COREOS_PRIVATE_IPV4:$port/tcp, publishing to etcd...; while netstat -lnt | grep :$port >/dev/null; do etcdctl set /deis/services/{app}/{name} $COREOS_PRIVATE_IPV4:$port --ttl 60 >/dev/null; sleep 45; done"'''},  # noqa
+    {"section": "Service", "name": "ExecStop", "value": "/usr/bin/etcdctl rm --recursive /deis/services/{app}/{name}"},  # noqa
+    {"section": "Service", "name": "TimeoutStartSec", "value": "20m"},
+    {"section": "X-Fleet", "name": "X-ConditionMachineOf", "value": "{name}.service"},
+]
+
+
+RUN_TEMPLATE = [
+    {"section": "Unit", "name": "Description", "value": "{name} admin command"},
+    {"section": "Service", "name": "ExecStartPre", "value": '''/bin/sh -c "IMAGE=$(etcdctl get /deis/registry/host 2>&1):$(etcdctl get /deis/registry/port 2>&1)/{image}; docker pull $IMAGE"'''},  # noqa
+    {"section": "Service", "name": "ExecStartPre", "value": '''/bin/sh -c "docker inspect {name} >/dev/null 2>&1 && docker rm -f {name} || true"'''},  # noqa
+    {"section": "Service", "name": "ExecStart", "value": '''/bin/sh -c "IMAGE=$(etcdctl get /deis/registry/host 2>&1):$(etcdctl get /deis/registry/port 2>&1)/{image}; docker run --name {name} --entrypoint=/bin/bash -a stdout -a stderr $IMAGE -c '{command}'"'''},  # noqa
+    {"section": "Service", "name": "TimeoutStartSec", "value": "20m"},
+]


Test patch:
diff --git a/tests/apps_test.go b/tests/apps_test.go
--- a/tests/apps_test.go
+++ b/tests/apps_test.go
@@ -5,7 +5,6 @@ package tests
 import (
 	"os"
 	"testing"
-	"time"
 
 	"github.com/deis/deis/tests/utils"
 )
@@ -80,8 +79,6 @@ func appsLogsTest(t *testing.T, params *utils.DeisTestConfig) {
 		t.Fatal(err)
 	}
 	utils.Execute(t, gitPushCmd, params, false, "")
-	// TODO: nginx needs a few seconds to wake up here--fixme!
-	time.Sleep(5000 * time.Millisecond)
 	utils.Curl(t, params)
 	utils.Execute(t, cmd, params, false, "")
 	if err := utils.Chdir(".."); err != nil {
diff --git a/tests/config_test.go b/tests/config_test.go
--- a/tests/config_test.go
+++ b/tests/config_test.go
@@ -25,7 +25,7 @@ func TestConfig(t *testing.T) {
 	appsOpenTest(t, params)
 	limitsUnsetTest(t, params, 6)
 	appsOpenTest(t, params)
-	tagsTest(t, params, 8)
+	//tagsTest(t, params, 8)
 	appsOpenTest(t, params)
 	utils.AppsDestroyTest(t, params)
 }
diff --git a/tests/limits_test.go b/tests/limits_test.go
--- a/tests/limits_test.go
+++ b/tests/limits_test.go
@@ -3,9 +3,6 @@
 package tests
 
 import (
-	"fmt"
-	"os/exec"
-	"regexp"
 	"strings"
 	"testing"
 
@@ -34,16 +31,8 @@ func limitsSetTest(t *testing.T, cfg *utils.DeisTestConfig, ver int) {
 		memCmd = strings.Replace(memCmd, "web", "cmd", 1)
 	}
 	utils.Execute(t, cpuCmd, cfg, false, "512")
-	out := dockerInspect(t, cfg, ver)
-	if _, err := regexp.MatchString(output1, out); err != nil {
-		t.Fatal(err)
-	}
 	utils.Execute(t, limitsListCmd, cfg, false, "512")
 	utils.Execute(t, memCmd, cfg, false, "256M")
-	out = dockerInspect(t, cfg, ver+1)
-	if _, err := regexp.MatchString(output2, out); err != nil {
-		t.Fatal(err)
-	}
 	utils.Execute(t, limitsListCmd, cfg, false, "256M")
 }
 
@@ -54,32 +43,7 @@ func limitsUnsetTest(t *testing.T, cfg *utils.DeisTestConfig, ver int) {
 		memCmd = strings.Replace(memCmd, "web", "cmd", 1)
 	}
 	utils.Execute(t, cpuCmd, cfg, false, "Unlimited")
-	out := dockerInspect(t, cfg, ver)
-	if _, err := regexp.MatchString(output3, out); err != nil {
-		t.Fatal(err)
-	}
 	utils.Execute(t, limitsListCmd, cfg, false, "Unlimited")
 	utils.Execute(t, memCmd, cfg, false, "Unlimited")
-	out = dockerInspect(t, cfg, ver+1)
-	if _, err := regexp.MatchString(output4, out); err != nil {
-		t.Fatal(err)
-	}
 	utils.Execute(t, limitsListCmd, cfg, false, "Unlimited")
 }
-
-// dockerInspect creates an SSH session to the Deis controller
-// and runs "docker inspect" on the first app container.
-func dockerInspect(
-	t *testing.T, cfg *utils.DeisTestConfig, ver int) string {
-	cmd := fmt.Sprintf("docker inspect %s_v%d.web.1", cfg.AppName, ver)
-	sshCmd := exec.Command("ssh",
-		"-o", "StrictHostKeyChecking=no",
-		"-o", "UserKnownHostsFile=/dev/null",
-		"-o", "PasswordAuthentication=no",
-		"core@deis."+cfg.Domain, cmd)
-	out, err := sshCmd.Output()
-	if err != nil {
-		t.Fatal(out, err)
-	}
-	return string(out)
-}
diff --git a/tests/utils/itutils.go b/tests/utils/itutils.go
--- a/tests/utils/itutils.go
+++ b/tests/utils/itutils.go
@@ -85,6 +85,13 @@ func GetGlobalConfig() *DeisTestConfig {
 // Curl connects to a Deis endpoint to see if the example app is running.
 func Curl(t *testing.T, params *DeisTestConfig) {
 	url := "http://" + params.AppName + "." + params.Domain
+	// FIXME: make an initial request to nginx to remove stale worker
+	_, err := http.Get(url)
+	if err != nil {
+		t.Fatalf("not reachable:\n%v", err)
+	}
+	// FIXME: sleep a bit before curling
+	time.Sleep(5000 * time.Millisecond)
 	response, err := http.Get(url)
 	if err != nil {
 		t.Fatalf("not reachable:\n%v", err)