Repository: deis/deis

Issue:
Retry Fleet scheduling operations
The current controller implementation use the HTTP API to schedule services.
Scaling services up/down I see random errors "fleet server currently unavailable". This could be fine but I also see errors building images (after the image is constructed and is launching) so I only have one options: create a new commit (or an empty one) and do a new git push. The problem is the time of this task and the built image is useless

It's possible to "retry" the scheduling at least once?

This is one example:

``` console
Oct 02 09:16:55 nodo-2 sh[27393]: INFO app-demo: build app-demo-57a7550 created
Oct 02 09:16:55 nodo-2 sh[27393]: INFO app-demo: release app-demo-v24 created
Oct 02 09:16:56 nodo-2 sh[27393]: ERROR app-demo: app-demo_v24.web.1 (create): Failed to create unit: 503 Service Unavailable - {"error":{"code":503,"message":"fleet server currently unavailable"}}
Oct 02 09:16:56 nodo-2 sh[27393]: ERROR app-demo: aborting, failed to create some containers
Oct 02 09:16:56 nodo-2 sh[27393]: ERROR app-demo: app-demo_v24.web.1 (destroy): Failed to retrieve state: 503 Service Unavailable - {"error":{"code":503,"message":"fleet server currently unavailable"}}
Oct 02 09:16:56 nodo-2 sh[27393]: ERROR app-demo: aborting, failed to destroy some containers
```



Discussion:

Resolving patch:
diff --git a/controller/scheduler/coreos.py b/controller/scheduler/coreos.py
--- a/controller/scheduler/coreos.py
+++ b/controller/scheduler/coreos.py
@@ -11,6 +11,7 @@
 
 MATCH = re.compile(
     '(?P<app>[a-z0-9-]+)_?(?P<version>v[0-9]+)?\.?(?P<c_type>[a-z-_]+)?.(?P<c_num>[0-9]+)')
+RETRIES = 3
 
 
 class UHTTPConnection(httplib.HTTPConnection):
@@ -133,14 +134,21 @@ def _create_container(self, name, image, command, unit, **kwargs):
             tagset = ' '.join(['"{}={}"'.format(k, v) for k, v in tags.items()])
             unit.append({"section": "X-Fleet", "name": "MachineMetadata",
                          "value": tagset})
-        # post unit to fleet
-        self._put_unit(name, {"desiredState": "launched", "options": unit})
+        # post unit to fleet and retry
+        for attempt in range(RETRIES):
+            try:
+                self._put_unit(name, {"desiredState": "launched", "options": unit})
+                break
+            except:
+                if attempt == (RETRIES - 1):  # account for 0 indexing
+                    raise
 
     def start(self, name):
         """Start a container"""
         self._wait_for_container(name)
 
     def _wait_for_container(self, name):
+        failures = 0
         # we bump to 20 minutes here to match the timeout on the router and in the app unit files
         for _ in range(1200):
             states = self._get_state(name)
@@ -150,10 +158,13 @@ def _wait_for_container(self, name):
                 if subState == 'running' or subState == 'exited':
                     break
                 elif subState == 'failed':
-                    raise RuntimeError('container failed to start')
+                    # FIXME: fleet unit state reports failed when containers are fine
+                    failures += 1
+                    if failures == 10:
+                        raise RuntimeError('container failed to start')
             time.sleep(1)
         else:
-            raise RuntimeError('container failed to start')
+            raise RuntimeError('container timeout on start')
 
     def _wait_for_destroy(self, name):
         for _ in range(30):
@@ -178,7 +189,13 @@ def destroy(self, name):
         self._wait_for_destroy(name)
 
     def _destroy_container(self, name):
-        return self._delete_unit(name)
+        for attempt in range(RETRIES):
+            try:
+                self._delete_unit(name)
+                break
+            except:
+                if attempt == (RETRIES - 1):  # account for 0 indexing
+                    raise
 
     def run(self, name, image, entrypoint, command):  # noqa
         """Run a one-off command"""


Test patch: