Repository: buildbot/buildbot

Issue:
new Buildbot presentation style
This is basically a simple import of reveal.js, with some Buildbot-specific skinning.  I'd like to use it for a few talks I'm planning to give, and of course drop them in the media/ directory, too.  The big part here is `nut.png`, which will be copied about unchanged and thus not take any additional space in the repository.



Discussion:

Resolving patch:
diff --git a/master/buildbot/clients/tryclient.py b/master/buildbot/clients/tryclient.py
--- a/master/buildbot/clients/tryclient.py
+++ b/master/buildbot/clients/tryclient.py
@@ -179,7 +179,10 @@ class MercurialExtractor(SourceStampExtractor):
     vcexe = "hg"
 
     def getBaseRevision(self):
-        d = self.dovc(["parents", "--template", "{node}\\n"])
+        upstream = ""
+        if self.repository:
+            upstream = "r'%s'" % self.repository
+        d = self.dovc(["log", "--template", "{node}\\n", "-r", "limit(parents(outgoing(%s) and branch(parents())) or parents(), 1)" % upstream])
         d.addCallback(self.parseStatus)
         return d
 
@@ -188,7 +191,7 @@ def parseStatus(self, output):
         self.baserev = m.group(0)
 
     def getPatch(self, res):
-        d = self.dovc(["diff"])
+        d = self.dovc(["diff", "-r", self.baserev])
         d.addCallback(self.readPatch, self.patchlevel)
         return d
 
diff --git a/master/buildbot/steps/source/mercurial.py b/master/buildbot/steps/source/mercurial.py
--- a/master/buildbot/steps/source/mercurial.py
+++ b/master/buildbot/steps/source/mercurial.py
@@ -111,6 +111,10 @@ def checkInstall(hgInstalled):
             d.addCallback(lambda _: self.full())
         elif self.mode == 'incremental':
             d.addCallback(lambda _: self.incremental())
+
+        if patch:
+            d.addCallback(self.patch, patch)
+
         d.addCallback(self.parseGotRevision)
         d.addCallback(self.finish)
         d.addErrback(self.failed)
@@ -204,11 +208,11 @@ def _checkBranchChange(self, _):
             else:
                 msg += ' Updating.'
                 log.msg(msg)
-                yield self._update(None)
+                yield self._removeAddedFilesAndUpdate(None)
         else:
             msg += ' Updating.'
             log.msg(msg)
-            yield self._update(None)
+            yield self._removeAddedFilesAndUpdate(None)
 
     def _pullUpdate(self, res):
         command = ['pull' , self.repourl]
@@ -218,14 +222,16 @@ def _pullUpdate(self, res):
         d.addCallback(self._checkBranchChange)
         return d
 
-    def _dovccmd(self, command, collectStdout=False):
+    def _dovccmd(self, command, collectStdout=False, initialStdin=None, successfulRC=(0,)):
         if not command:
             raise ValueError("No command specified")
         cmd = buildstep.RemoteShellCommand(self.workdir, ['hg', '--verbose'] + command,
                                            env=self.env,
                                            logEnviron=self.logEnviron,
                                            timeout=self.timeout,
-                                           collectStdout=collectStdout)
+                                           collectStdout=collectStdout,
+                                           initialStdin=initialStdin,
+                                           successfulRC=successfulRC)
         cmd.useLog(self.stdio_log, False)
         log.msg("Starting mercurial command : hg %s" % (" ".join(command), ))
         d = self.runCommand(cmd)
@@ -253,6 +259,11 @@ def computeSourceRevision(self, changes):
                     "the most recent" % len(changes))
         return changes[-1].revision
 
+    def patch(self, _, patch):
+        d = self._dovccmd(['import', '--no-commit', '-p', str(patch[0]), '-'],
+                initialStdin=patch[1])
+        return d
+
     def _getCurrentBranch(self):
         if self.branchType == 'dirname':
             return defer.succeed(self.branch)
@@ -283,6 +294,43 @@ def _fail(tmp):
         d.addCallback(_fail)
         return d
 
+    def _removeAddedFilesAndUpdate(self, _):
+        command = ['locate', 'set:added()']
+        d = self._dovccmd(command, collectStdout=True, successfulRC=(0,1,))
+        def parseAndRemove(stdout):
+            files = []
+            for filename in stdout.splitlines() :
+                filename = self.workdir+'/'+filename
+                files.append(filename)
+            if len(files) == 0:
+                d = defer.succeed(0)
+            else:
+                if self.slaveVersionIsOlderThan('rmdir', '2.14'):
+                    d = self.removeFiles(files)
+                else:
+                    cmd = buildstep.RemoteCommand('rmdir', {'dir': files,
+                                                            'logEnviron':
+                                                            self.logEnviron,})
+                    cmd.useLog(self.stdio_log, False)
+                    d = self.runCommand(cmd)
+                    d.addCallback(lambda _: cmd.rc)
+            return d
+        d.addCallback(parseAndRemove)
+        d.addCallback(self._update)
+        return d
+
+    @defer.inlineCallbacks
+    def removeFiles(self, files):
+        for filename in files:
+            cmd = buildstep.RemoteCommand('rmdir', {'dir': filename,
+                                                    'logEnviron': self.logEnviron,})
+            cmd.useLog(self.stdio_log, False)
+            yield self.runCommand(cmd)
+            if cmd.rc != 0:
+                defer.returnValue(cmd.rc)
+                return
+        defer.returnValue(0)
+
     def _update(self, _):
         command = ['update', '--clean']
         if self.revision:


Test patch:
diff --git a/master/buildbot/test/unit/test_steps_source_mercurial.py b/master/buildbot/test/unit/test_steps_source_mercurial.py
--- a/master/buildbot/test/unit/test_steps_source_mercurial.py
+++ b/master/buildbot/test/unit/test_steps_source_mercurial.py
@@ -28,6 +28,9 @@ def setUp(self):
     def tearDown(self):
         return self.tearDownSourceStep()
 
+    def patch_slaveVersionIsOlderThan(self, result):
+        self.patch(mercurial.Mercurial, 'slaveVersionIsOlderThan', lambda x, y, z: result)
+
     def test_no_repourl(self):
         self.assertRaises(config.ConfigErrors, lambda :
                 mercurial.Mercurial(mode="full"))
@@ -71,6 +74,9 @@ def test_mode_full_clean(self):
             + ExpectShell.log('stdio',
                 stdout='default')
             + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'])
+            + 1,
             ExpectShell(workdir='wkdir',
                         command=['hg', '--verbose', 'update',
                                  '--clean'])
@@ -115,6 +121,10 @@ def test_mode_full_clean_timeout(self):
             + ExpectShell.log('stdio',
                 stdout='default')
             + 0,
+            ExpectShell(workdir='wkdir',
+                        timeout=1,
+                        command=['hg', '--verbose', 'locate', 'set:added()'])
+            + 1,
             ExpectShell(workdir='wkdir',
                         timeout=1,
                         command=['hg', '--verbose', 'update',
@@ -132,6 +142,95 @@ def test_mode_full_clean_timeout(self):
         self.expectOutcome(result=SUCCESS, status_text=["update"])
         return self.runStep()
 
+    def test_mode_full_clean_patch(self):
+        self.setupStep(
+                mercurial.Mercurial(repourl='http://hg.mozilla.org',
+                                    mode='full', method='clean', branchType='inrepo'),
+                patch=(1, 'patch'))
+        self.expectCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', '--version'])
+            + 0,
+            Expect('stat', dict(file='wkdir/.hg',
+                                      logEnviron=True))
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', '--config',
+                                 'extensions.purge=', 'purge'])
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'pull',
+                                 'http://hg.mozilla.org'])
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'identify', '--branch'])
+            + ExpectShell.log('stdio',
+                stdout='default')
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'])
+            + 1,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'update',
+                                 '--clean'])
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'import',
+                                 '--no-commit', '-p', '1', '-'],
+                        initialStdin='patch')
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'parents',
+                                    '--template', '{node}\\n'])
+            + ExpectShell.log('stdio', stdout='\n')
+            + ExpectShell.log('stdio',
+                stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d')
+            + 0,
+        )
+        self.expectOutcome(result=SUCCESS, status_text=["update"])
+        return self.runStep()
+    
+    def test_mode_full_clean_patch_fail(self):
+        self.setupStep(
+                mercurial.Mercurial(repourl='http://hg.mozilla.org',
+                                    mode='full', method='clean', branchType='inrepo'),
+                patch=(1, 'patch'))
+        self.expectCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', '--version'])
+            + 0,
+            Expect('stat', dict(file='wkdir/.hg',
+                                      logEnviron=True))
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', '--config',
+                                 'extensions.purge=', 'purge'])
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'pull',
+                                 'http://hg.mozilla.org'])
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'identify', '--branch'])
+            + ExpectShell.log('stdio',
+                stdout='default')
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'])
+            + 1,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'update',
+                                 '--clean'])
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'import',
+                                 '--no-commit', '-p', '1', '-'],
+                        initialStdin='patch')
+            + 1,
+        )
+        self.expectOutcome(result=FAILURE, status_text=["updating"])
+        return self.runStep()
+
     def test_mode_full_clean_no_existing_repo(self):
         self.setupStep(
                 mercurial.Mercurial(repourl='http://hg.mozilla.org',
@@ -211,6 +310,9 @@ def test_mode_full_fresh(self):
             + ExpectShell.log('stdio',
                 stdout='default')
             + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'])
+            + 1,
             ExpectShell(workdir='wkdir',
                         command=['hg', '--verbose', 'update',
                                  '--clean'])
@@ -268,6 +370,9 @@ def test_mode_incremental_no_existing_repo_dirname(self):
                         command=['hg', '--verbose', 'clone',
                                  'http://hg.mozilla.org', '.', '--noupdate'])
             + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'])
+            + 1,
             ExpectShell(workdir='wkdir',
                         command=['hg', '--verbose', 'update', '--clean'])
             + 0,
@@ -341,6 +446,9 @@ def test_mode_incremental_no_existing_repo_inrepo(self):
                         command=['hg', '--verbose', 'identify', '--branch'])
             + ExpectShell.log('stdio', stdout='default')
             + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'])
+            + 1,
             ExpectShell(workdir='wkdir',
                         command=['hg', '--verbose', 'update', '--clean'])
             + 0,
@@ -374,6 +482,93 @@ def test_mode_incremental_existing_repo(self):
                         command=['hg', '--verbose', 'identify', '--branch'])
             + ExpectShell.log('stdio', stdout='default')
             + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'])
+            + 1,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'update', '--clean'])
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'parents',
+                                    '--template', '{node}\\n'])
+            + ExpectShell.log('stdio', stdout='\n')
+            + ExpectShell.log('stdio',
+                stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d')
+            + 0,
+            )
+        self.expectOutcome(result=SUCCESS, status_text=["update"])
+        return self.runStep()
+
+    def test_mode_incremental_existing_repo_added_files(self):
+        self.setupStep(
+                mercurial.Mercurial(repourl='http://hg.mozilla.org',
+                                    mode='incremental', branchType='inrepo'))
+        self.expectCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', '--version'])
+            + 0,
+            Expect('stat', dict(file='wkdir/.hg',
+                                logEnviron=True))
+            + 0, # directory exists
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'pull',
+                                 'http://hg.mozilla.org', '--update'])
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'identify', '--branch'])
+            + ExpectShell.log('stdio', stdout='default')
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'])
+            + ExpectShell.log('stdio', stdout='foo\nbar/baz\n')
+            + 1,
+            Expect('rmdir', dict(dir=['wkdir/foo','wkdir/bar/baz'],
+                logEnviron=True))
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'update', '--clean'])
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'parents',
+                                    '--template', '{node}\\n'])
+            + ExpectShell.log('stdio', stdout='\n')
+            + ExpectShell.log('stdio',
+                stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d')
+            + 0,
+            )
+        self.expectOutcome(result=SUCCESS, status_text=["update"])
+        return self.runStep()
+    
+    def test_mode_incremental_existing_repo_added_files_old_rmdir(self):
+        self.setupStep(
+                mercurial.Mercurial(repourl='http://hg.mozilla.org',
+                                    mode='incremental', branchType='inrepo'))
+        self.patch_slaveVersionIsOlderThan(True)
+        self.expectCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', '--version'])
+            + 0,
+            Expect('stat', dict(file='wkdir/.hg',
+                                logEnviron=True))
+            + 0, # directory exists
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'pull',
+                                 'http://hg.mozilla.org', '--update'])
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'identify', '--branch'])
+            + ExpectShell.log('stdio', stdout='default')
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'])
+            + ExpectShell.log('stdio', stdout='foo\nbar/baz\n')
+            + 1,
+            Expect('rmdir', dict(dir='wkdir/foo',
+                logEnviron=True))
+            + 0,
+            Expect('rmdir', dict(dir='wkdir/bar/baz',
+                logEnviron=True))
+            + 0,
             ExpectShell(workdir='wkdir',
                         command=['hg', '--verbose', 'update', '--clean'])
             + 0,
@@ -410,6 +605,9 @@ def test_mode_incremental_given_revision(self):
                         command=['hg', '--verbose', 'identify', '--branch'])
             + ExpectShell.log('stdio', stdout='default')
             + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'])
+            + 1,
             ExpectShell(workdir='wkdir',
                         command=['hg', '--verbose', 'update', '--clean',
                                  '--rev', 'abcdef01'])
@@ -490,6 +688,9 @@ def test_mode_incremental_branch_change_no_clobberOnBranchChange(self):
                         command=['hg', '--verbose', 'identify', '--branch'])
             + ExpectShell.log('stdio', stdout='default')
             + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'])
+            + 1,
             ExpectShell(workdir='wkdir',
                         command=['hg', '--verbose', 'update',
                                  '--clean'])
@@ -531,6 +732,10 @@ def test_mode_full_clean_env(self):
             + ExpectShell.log('stdio',
                 stdout='default')
             + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'],
+                        env={'abc': '123'})
+            + 1,
             ExpectShell(workdir='wkdir',
                         command=['hg', '--verbose', 'update',
                                  '--clean'], env={'abc': '123'})
@@ -576,6 +781,10 @@ def test_mode_full_clean_logEnviron(self):
             + ExpectShell.log('stdio',
                 stdout='default')
             + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['hg', '--verbose', 'locate', 'set:added()'],
+                        logEnviron=False)
+            + 1,
             ExpectShell(workdir='wkdir',
                         command=['hg', '--verbose', 'update',
                                  '--clean'],