Repository: buildbot/buildbot

Issue:
Allow slaves to substantiate in parallel
BuildRequestDistributor's _maybeStartBuildsOnBuilder() method is
decorated with @defer.inlineCallbacks.  This means that when it
uses "yield bldr.maybeStartBuild", the entire slave substantiation
process is sequentialized at that point.

slave substantiation can take considerable time, especially with
something like EC2 latent slaves.  Even without anything substantial
happening in the cloud init (user-data) script, it can easily take
a couple of minutes.  If any substantial configuration of the
OS takes place ins the user-data script, maybeStartBuild() can
easily take many minutes.

To enable parallel slave substantiation, we need to stop waiting on
bldr.maybeStartBuild() inline.  Instead we move the error
handling code into a callback function that can be added to the
Deferred that is returned by bldr.maybeStartBuild().

At least for for the EC2LatentBuildSlave class, everything else down
the chain is properly event based and/or threaded.  This small
code change allows slaves to substantiate in parallel.



Discussion:

Resolving patch:
diff --git a/master/buildbot/steps/source/git.py b/master/buildbot/steps/source/git.py
--- a/master/buildbot/steps/source/git.py
+++ b/master/buildbot/steps/source/git.py
@@ -115,6 +115,8 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental',
         assert self.repourl is not None
         if self.mode == 'full':
             assert self.method in ['clean', 'fresh', 'clobber', 'copy', None]
+        if self.shallow:
+            assert self.mode == 'full' and self.method == 'clobber'
         assert isinstance(self.getDescription, (bool, dict))
 
     def startVC(self, branch, revision, patch):
@@ -154,7 +156,7 @@ def full(self):
         updatable = yield self._sourcedirIsUpdatable()
         if not updatable:
             log.msg("No git repo present, making full clone")
-            yield self._doFull()
+            yield self._fullCloneOrFallback()
         elif self.method == 'clean':
             yield self.clean()
         elif self.method == 'fresh':
@@ -168,7 +170,7 @@ def incremental(self):
 
         # if not updateable, do a full checkout
         if not updatable:
-            yield self._doFull()
+            yield self._fullCloneOrFallback()
             return
 
         # test for existence of the revision; rc=1 indicates it does not exist
@@ -187,35 +189,27 @@ def incremental(self):
                 yield self._dovccmd(['branch', '-M', self.branch],
                         abandonOnFailure=False)
         else:
-            yield self._doFetch(None)
+            yield self._fetchOrFallback(None)
 
         yield self._updateSubmodule(None)
 
     def clean(self):
         command = ['clean', '-f', '-d']
         d = self._dovccmd(command)
-        d.addCallback(self._doFetch)
+        d.addCallback(self._fetchOrFallback)
         d.addCallback(self._updateSubmodule)
         d.addCallback(self._cleanSubmodule)
         return d
 
     def clobber(self):
-        cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir,
-                                                'logEnviron': self.logEnviron,})
-        cmd.useLog(self.stdio_log, False)
-        d = self.runCommand(cmd)
-        def checkRemoval(res):
-            if res != 0:
-                raise RuntimeError("Failed to delete directory")
-            return res
-        d.addCallback(lambda _: checkRemoval(cmd.rc))
-        d.addCallback(lambda _: self._doFull())
+        d = self._doClobber()
+        d.addCallback(lambda _: self._fullClone(shallowClone=self.shallow))
         return d
 
     def fresh(self):
         command = ['clean', '-f', '-d', '-x']
         d = self._dovccmd(command)
-        d.addCallback(self._doFetch)
+        d.addCallback(self._fetchOrFallback)
         d.addCallback(self._updateSubmodule)
         d.addCallback(self._cleanSubmodule)
         return d
@@ -342,13 +336,8 @@ def renameBranch(res):
             d.addCallback(renameBranch)
         return d
 
-    def patch(self, _, patch):
-        d = self._dovccmd(['apply', '--index', '-p', str(patch[0])],
-                initialStdin=patch[1])
-        return d
-
     @defer.inlineCallbacks
-    def _doFetch(self, _):
+    def _fetchOrFallback(self, _):
         """
         Handles fallbacks for failure of fetch,
         wrapper for self._fetch
@@ -360,12 +349,17 @@ def _doFetch(self, _):
         elif self.retryFetch:
             yield self._fetch(None)
         elif self.clobberOnFailure:
-            yield self.clobber()
+            yield self._doClobber()
+            yield self._fullClone()
         else:
             raise buildstep.BuildStepFailed()
 
-    def _full(self):
-        if self.shallow:
+    def _fullClone(self, shallowClone=False):
+        """Perform full clone and checkout to the revision if specified
+           In the case of shallow clones if any of the step fail abort whole build step.
+        """
+
+        if shallowClone:
             command = ['clone', '--depth', '1', '--branch', self.branch, self.repourl, '.']
         else:
             command = ['clone', '--branch', self.branch, self.repourl, '.']
@@ -373,26 +367,33 @@ def _full(self):
         if self.prog:
             command.append('--progress')
 
-        d = self._dovccmd(command, not self.clobberOnFailure)
+        # If it's a shallow clone abort build step
+        d = self._dovccmd(command, shallowClone)
         # If revision specified checkout that revision
         if self.revision:
             d.addCallback(lambda _: self._dovccmd(['reset', '--hard',
                                                    self.revision],
-                                                  not self.clobberOnFailure))
+                                                  shallowClone))
         # init and update submodules, recurisively. If there's not recursion
         # it will not do it.
         if self.submodules:
             d.addCallback(lambda _: self._dovccmd(['submodule', 'update',
                                                    '--init', '--recursive'],
-                                                  not self.clobberOnFailure))
+                                                  shallowClone))
         return d
 
-    def _doFull(self):
-        d = self._full()
+    def _fullCloneOrFallback(self):
+        """Wrapper for _fullClone(). In the case of failure, if clobberOnFailure 
+           is set to True remove the build directory and try a full clone again.
+        """
+
+        d = self._fullClone()
         def clobber(res):
             if res != 0:
                 if self.clobberOnFailure:
-                    return self.clobber()
+                    d = self._doClobber()
+                    d.addCallback(lambda _: self._fullClone())
+                    return d
                 else:
                     raise buildstep.BuildStepFailed()
             else:
@@ -400,6 +401,19 @@ def clobber(res):
         d.addCallback(clobber)
         return d
 
+    def _doClobber(self):
+        """Remove the work directory"""
+        cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir,
+                                                'logEnviron': self.logEnviron,})
+        cmd.useLog(self.stdio_log, False)
+        d = self.runCommand(cmd)
+        def checkRemoval(res):
+            if res != 0:
+                raise RuntimeError("Failed to delete directory")
+            return res
+        d.addCallback(lambda _: checkRemoval(cmd.rc))
+        return d
+
     def computeSourceRevision(self, changes):
         if not changes:
             return None
@@ -448,3 +462,8 @@ def check(res):
             return False
         d.addCallback(check)
         return d
+
+    def patch(self, _, patch):
+        d = self._dovccmd(['apply', '--index', '-p', str(patch[0])],
+                initialStdin=patch[1])
+        return d


Test patch:
diff --git a/master/buildbot/test/unit/test_steps_source_git.py b/master/buildbot/test/unit/test_steps_source_git.py
--- a/master/buildbot/test/unit/test_steps_source_git.py
+++ b/master/buildbot/test/unit/test_steps_source_git.py
@@ -530,6 +530,34 @@ def test_mode_full_clobber_shallow(self):
         self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source')
         return self.runStep()
 
+    def test_mode_full_clobber_no_shallow(self):
+        self.setupStep(
+                git.Git(repourl='http://github.com/buildbot/buildbot.git',
+                        mode='full', method='clobber'))
+
+        self.expectCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['git', '--version'])
+            + 0,
+            Expect('rmdir', dict(dir='wkdir',
+                                 logEnviron=True))
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['git', 'clone',
+                                 '--branch', 'HEAD',
+                                 'http://github.com/buildbot/buildbot.git',
+                                 '.'])
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['git', 'rev-parse', 'HEAD'])
+            + ExpectShell.log('stdio',
+                stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d')
+            + 0,
+        )
+        self.expectOutcome(result=SUCCESS, status_text=["update"])
+        self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source')
+        return self.runStep()
+
     def test_mode_incremental_retryFetch(self):
         self.setupStep(
                 git.Git(repourl='http://github.com/buildbot/buildbot.git',
@@ -690,7 +718,7 @@ def test_mode_incremental_clobberOnFailure_branch(self):
     def test_mode_full_copy(self):
         self.setupStep(
                 git.Git(repourl='http://github.com/buildbot/buildbot.git',
-                        mode='full', method='copy', shallow=True))
+                        mode='full', method='copy'))
 
         self.expectCommands(
             ExpectShell(workdir='wkdir',
@@ -722,6 +750,10 @@ def test_mode_full_copy(self):
         self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source')
         return self.runStep()
 
+    def test_mode_full_copy_shallow(self):
+        self.assertRaises(AssertionError, lambda :
+                git.Git(repourl='http://github.com/buildbot/buildbot.git',
+                        mode='full', method='copy', shallow=True))
 
     def test_mode_incremental_no_existing_repo(self):
         self.setupStep(