Repository: buildbot/buildbot

Issue:
LatentDockerWorker: Fail if image is not found and cannot be pulled
For LatentDockerWorker, it would make sense if the build would fail if an image isn't found or can't be downloaded instead on continually retrying.

This could be used in tandem of #3071, so that is the image is not there, it tries a pull, and then if that fails, the build should fail or retry a couples times.

I have looked at the code for latent workers, and I don't really see a way to make it stop retrying.


Discussion:
I did not realize #3578 did not fix this.

I think we can do this by adding a special case here:

https://github.com/buildbot/buildbot/blob/master/master/buildbot/process/build.py#L322

```
        if ready_or_failure is not True:
            yield self.buildPreparationFailure(ready_or_failure, "worker_prepare")
            if self.stopped:
                self.buildFinished(["worker", "cancelled"], self.results)
+           elif ready_or_failure.check([interfaces.LatentWorkerCannotToSubstantiate]):
+                self.buildFinished(["worker", "cannot", "substanciate"], RETRY)
            else:
                self.buildFinished(["worker", "not", "available"], RETRY)
            return
```
We would add a ```LatentWorkerCannotToSubstantiate``` Exception, which would tell the core that it is not worth to retry the substantiation.

I could do that. Not sure on how I would do the unit tests.

Brainstorming here:
Would the worker tell the build process that it should give up, or should that be inherent to the
build process itself?

Resolving patch:
diff --git a/master/buildbot/interfaces.py b/master/buildbot/interfaces.py
--- a/master/buildbot/interfaces.py
+++ b/master/buildbot/interfaces.py
@@ -60,6 +60,10 @@ class LatentWorkerFailedToSubstantiate(Exception):
     pass
 
 
+class LatentWorkerCannotSubstantiate(Exception):
+    pass
+
+
 deprecatedWorkerModuleAttribute(
     locals(), LatentWorkerFailedToSubstantiate,
     compat_name="LatentBuildSlaveFailedToSubstantiate")
diff --git a/master/buildbot/process/build.py b/master/buildbot/process/build.py
--- a/master/buildbot/process/build.py
+++ b/master/buildbot/process/build.py
@@ -327,6 +327,8 @@ def startBuild(self, build_status, workerforbuilder):
             yield self.buildPreparationFailure(ready_or_failure, "worker_prepare")
             if self.stopped:
                 self.buildFinished(["worker", "cancelled"], self.results)
+            elif isinstance(ready_or_failure, Failure) and ready_or_failure.check(interfaces.LatentWorkerCannotSubstantiate):
+                self.buildFinished(["worker", "cannot", "substantiate"], EXCEPTION)
             else:
                 self.buildFinished(["worker", "not", "available"], RETRY)
             return
diff --git a/master/buildbot/worker/docker.py b/master/buildbot/worker/docker.py
--- a/master/buildbot/worker/docker.py
+++ b/master/buildbot/worker/docker.py
@@ -27,6 +27,7 @@
 from twisted.python import log
 
 from buildbot import config
+from buildbot.interfaces import LatentWorkerCannotSubstantiate
 from buildbot.interfaces import LatentWorkerFailedToSubstantiate
 from buildbot.util import unicode2bytes
 from buildbot.worker import AbstractLatentWorker
@@ -241,7 +242,7 @@ def _thd_start_instance(self, image, dockerfile, volumes):
 
         if (not self._image_exists(docker_client, image)):
             log.msg("Image '%s' not found" % image)
-            raise LatentWorkerFailedToSubstantiate(
+            raise LatentWorkerCannotSubstantiate(
                 'Image "%s" not found on docker host.' % image
             )
 


Test patch:
diff --git a/master/buildbot/test/integration/test_latent.py b/master/buildbot/test/integration/test_latent.py
--- a/master/buildbot/test/integration/test_latent.py
+++ b/master/buildbot/test/integration/test_latent.py
@@ -26,11 +26,13 @@
 from twisted.trial.unittest import SynchronousTestCase
 
 from buildbot.config import BuilderConfig
+from buildbot.interfaces import LatentWorkerCannotSubstantiate
 from buildbot.interfaces import LatentWorkerFailedToSubstantiate
 from buildbot.interfaces import LatentWorkerSubstantiatiationCancelled
 from buildbot.process.buildstep import BuildStep
 from buildbot.process.factory import BuildFactory
 from buildbot.process.results import CANCELLED
+from buildbot.process.results import EXCEPTION
 from buildbot.process.results import RETRY
 from buildbot.process.results import SUCCESS
 from buildbot.test.fake.latent import LatentController
@@ -209,6 +211,43 @@ def test_failed_substantiations_get_requeued(self):
         )
         controller.auto_stop(True)
 
+    @defer.inlineCallbacks
+    def test_failed_substantiations_get_exception(self):
+        """
+        If a latent worker fails to substantiate, the result is an exception.
+        """
+        controller = LatentController('local')
+        config_dict = {
+            'builders': [
+                BuilderConfig(name="testy",
+                              workernames=["local"],
+                              factory=BuildFactory(),
+                              ),
+            ],
+            'workers': [controller.worker],
+            'protocols': {'null': {}},
+            # Disable checks about missing scheduler.
+            'multiMaster': True,
+        }
+        master = self.getMaster(config_dict)
+        builder_id = self.successResultOf(
+            master.data.updates.findBuilderId('testy'))
+
+        # Trigger a buildrequest
+        self.createBuildrequest(master, [builder_id])
+
+        # The worker fails to substantiate.
+        controller.start_instance(
+            Failure(LatentWorkerCannotSubstantiate("substantiation failed")))
+        # Flush the errors logged by the failure.
+        self.flushLoggedErrors(LatentWorkerCannotSubstantiate)
+
+        dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1)
+
+        # When the substantiation fails, the result is an exception.
+        self.assertEqual(EXCEPTION, dbdict['results'])
+        controller.auto_stop(True)
+
     def test_worker_accepts_builds_after_failure(self):
         """
         If a latent worker fails to substantiate, the worker is still able to accept jobs.