Repository: buildbot/buildbot

Issue:
buildstep: finish unfinished logs
The previous version did not call finish on
unfinished (log.finished == False) logs. Such
situations happened among others with logs not attached to
a RemoteCommand, like for example stdio on
a MasterShellCommand.

Not calling finish had at least, two side effects:
- the 'complete' of related log lines attribute was not set to 1
  in the database,
- master.data.updates.compressLog was not called on the final
  state of the log.

For these reasons, this commit calls 'finish' on unfinised
logs before ending the step. The result of the step is set
to EXCEPTION if an error occured during that operation.

Change-Id: I30ebfc51ffac12cab22f4864494b3a334a202dac



Discussion:

Resolving patch:
diff --git a/master/buildbot/steps/master.py b/master/buildbot/steps/master.py
--- a/master/buildbot/steps/master.py
+++ b/master/buildbot/steps/master.py
@@ -29,17 +29,19 @@ class MasterShellCommand(BuildStep):
     name='MasterShellCommand'
     description='Running'
     descriptionDone='Ran'
-    renderables = [ 'command', 'env' ]
+    descriptionSuffix = None
+    renderables = [ 'command', 'env', 'description', 'descriptionDone', 'descriptionSuffix' ]
     haltOnFailure = True
     flunkOnFailure = True
 
     def __init__(self, command,
-                 description=None, descriptionDone=None,
+                 description=None, descriptionDone=None, descriptionSuffix=None,
                  env=None, path=None, usePTY=0,
                  **kwargs):
         BuildStep.__init__(self, **kwargs)
         self.addFactoryArguments(description=description,
                                  descriptionDone=descriptionDone,
+                                 descriptionSuffix=descriptionSuffix,
                                  env=env, path=path, usePTY=usePTY,
                                  command=command)
 
@@ -52,6 +54,10 @@ def __init__(self, command,
             self.descriptionDone = descriptionDone
         if isinstance(self.descriptionDone, str):
             self.descriptionDone = [self.descriptionDone]
+        if descriptionSuffix:
+            self.descriptionSuffix = descriptionSuffix
+        if isinstance(self.descriptionSuffix, str):
+            self.descriptionSuffix = [self.descriptionSuffix]
         self.env=env
         self.path=path
         self.usePTY=usePTY
@@ -100,7 +106,7 @@ def start(self):
         stdio_log.addHeader("** RUNNING ON BUILDMASTER **\n")
         stdio_log.addHeader(" in dir %s\n" % os.getcwd())
         stdio_log.addHeader(" argv: %s\n" % (argv,))
-        self.step_status.setText(list(self.description))
+        self.step_status.setText(self.describe())
 
         if self.env is None:
             env = os.environ
@@ -126,8 +132,16 @@ def subst(match):
 
     def processEnded(self, status_object):
         if status_object.value.exitCode != 0:
-            self.step_status.setText(["failed (%d)" % status_object.value.exitCode])
+            self.descriptionDone = ["failed (%d)" % status_object.value.exitCode]
+            self.step_status.setText(self.describe(done=True))
             self.finished(FAILURE)
         else:
-            self.step_status.setText(list(self.descriptionDone))
+            self.step_status.setText(self.describe(done=True))
             self.finished(SUCCESS)
+
+    def describe(self, done=False):
+        desc = self.descriptionDone if done else self.description
+        if self.descriptionSuffix:
+            desc = desc[:]
+            desc.extend(self.descriptionSuffix)
+        return desc
diff --git a/master/buildbot/steps/shell.py b/master/buildbot/steps/shell.py
--- a/master/buildbot/steps/shell.py
+++ b/master/buildbot/steps/shell.py
@@ -65,9 +65,13 @@ class ShellCommand(buildstep.LoggingBuildStep):
     """
 
     name = "shell"
-    renderables = [ 'description', 'descriptionDone', 'slaveEnvironment', 'remote_kwargs', 'command', 'logfiles' ]
+    renderables = buildstep.LoggingBuildStep.renderables + [
+                   'slaveEnvironment', 'remote_kwargs', 'command']
+    
     description = None # set this to a list of short strings to override
     descriptionDone = None # alternate description when the step is complete
+    descriptionSuffix = None # extra information to append to suffix
+
     command = None # set this to a command, or set in kwargs
     # logfiles={} # you can also set 'logfiles' to a dictionary, and it
     #               will be merged with any logfiles= argument passed in
@@ -78,7 +82,7 @@ class ShellCommand(buildstep.LoggingBuildStep):
     flunkOnFailure = True
 
     def __init__(self, workdir=None,
-                 description=None, descriptionDone=None,
+                 description=None, descriptionDone=None, descriptionSuffix=None,
                  command=None,
                  usePTY="slave-config",
                  **kwargs):
@@ -95,6 +99,12 @@ def __init__(self, workdir=None,
             self.descriptionDone = descriptionDone
         if isinstance(self.descriptionDone, str):
             self.descriptionDone = [self.descriptionDone]
+
+        if descriptionSuffix:
+            self.descriptionSuffix = descriptionSuffix
+        if isinstance(self.descriptionSuffix, str):
+            self.descriptionSuffix = [self.descriptionSuffix]
+
         if command:
             self.setCommand(command)
 
@@ -108,6 +118,7 @@ def __init__(self, workdir=None,
         self.addFactoryArguments(workdir=workdir,
                                  description=description,
                                  descriptionDone=descriptionDone,
+                                 descriptionSuffix=descriptionSuffix,
                                  command=command)
 
         # everything left over goes to the RemoteShellCommand
@@ -148,6 +159,13 @@ def _flattenList(self, mainlist, commands):
              self._flattenList(mainlist, x)
 
     def describe(self, done=False):
+        desc = self._describe(done)
+        if self.descriptionSuffix:
+            desc = desc[:]
+            desc.extend(self.descriptionSuffix)
+        return desc
+
+    def _describe(self, done=False):
         """Return a list of short strings to describe this step, for the
         status display. This uses the first few words of the shell command.
         You can replace this by setting .description in your subclass, or by
diff --git a/master/buildbot/steps/source/base.py b/master/buildbot/steps/source/base.py
--- a/master/buildbot/steps/source/base.py
+++ b/master/buildbot/steps/source/base.py
@@ -26,9 +26,13 @@ class Source(LoggingBuildStep):
     starts a RemoteCommand with those arguments.
     """
 
-    renderables = [ 'workdir', 'description', 'descriptionDone' ]
+    renderables = LoggingBuildStep.renderables + [
+                     'description', 'descriptionDone', 'descriptionSuffix',
+                     'workdir' ]
+
     description = None # set this to a list of short strings to override
     descriptionDone = None # alternate description when the step is complete
+    descriptionSuffix = None # extra information to append to suffix
 
     # if the checkout fails, there's no point in doing anything else
     haltOnFailure = True
@@ -39,8 +43,8 @@ class Source(LoggingBuildStep):
 
     def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
                  timeout=20*60, retry=None, env=None, logEnviron=True,
-                 description=None, descriptionDone=None, codebase='',
-                 **kwargs):
+                 description=None, descriptionDone=None, descriptionSuffix=None,
+                 codebase='', **kwargs):
         """
         @type  workdir: string
         @param workdir: local directory (relative to the Builder's root)
@@ -128,7 +132,7 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
                            variables on the slave. In situations where the
                            environment is not relevant and is long, it may
                            be easier to set logEnviron=False.
-+
+
         @type codebase: string
         @param codebase: Specifies which changes in a build are processed by
         the step. The default codebase value is ''. The codebase must correspond
@@ -147,7 +151,8 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
                                  env=env,
                                  description=description,
                                  descriptionDone=descriptionDone,
-                                 codebase=codebase,
+                                 descriptionSuffix=descriptionSuffix,
+                                 codebase=codebase
                                  )
 
         assert mode in ("update", "copy", "clobber", "export")
@@ -164,8 +169,10 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
         self.workdir = workdir
 
         self.sourcestamp = None
-        # Codebase cannot be set yet
+
         self.codebase = codebase
+        if self.codebase:
+            self.name = ' '.join((self.name, self.codebase))
 
         self.alwaysUseLatest = alwaysUseLatest
 
@@ -183,8 +190,6 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
         else:
             self.description = [
                 descriptions_for_mode.get(mode, "updating")]
-            if self.codebase:
-                self.description.append(self.codebase)
         if isinstance(self.description, str):
             self.description = [self.description]
 
@@ -193,11 +198,16 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
         else:
             self.descriptionDone = [
                 descriptionDones_for_mode.get(mode, "update")]
-            if self.codebase:
-                self.descriptionDone.append(self.codebase)
         if isinstance(self.descriptionDone, str):
             self.descriptionDone = [self.descriptionDone]
 
+        if descriptionSuffix:
+            self.descriptionSuffix = descriptionSuffix
+        else:
+            self.descriptionSuffix = self.codebase or None # want None in lieu of ''
+        if isinstance(self.descriptionSuffix, str):
+            self.descriptionSuffix = [self.descriptionSuffix]
+
     def setProperty(self, name, value , source):
         if self.codebase != '':
             assert not isinstance(self.getProperty(name, None), str), \
@@ -219,9 +229,11 @@ def setDefaultWorkdir(self, workdir):
         self.workdir = self.workdir or workdir
 
     def describe(self, done=False):
-        if done:
-            return self.descriptionDone
-        return self.description
+        desc = self.descriptionDone if done else self.description
+        if self.descriptionSuffix:
+            desc = desc[:]
+            desc.extend(self.descriptionSuffix)
+        return desc
 
     def computeSourceRevision(self, changes):
         """Each subclass must implement this method to do something more


Test patch:
diff --git a/master/buildbot/test/unit/test_steps_master.py b/master/buildbot/test/unit/test_steps_master.py
--- a/master/buildbot/test/unit/test_steps_master.py
+++ b/master/buildbot/test/unit/test_steps_master.py
@@ -80,6 +80,9 @@ def test_constr_args(self):
                 master.MasterShellCommand(description='x', descriptionDone='y',
                                 env={'a':'b'}, path=['/usr/bin'], usePTY=True,
                                 command='true'))
+        
+        self.assertEqual(self.step.describe(), ['x'])
+        
         if runtime.platformType == 'win32':
             exp_argv = [ r'C:\WINDOWS\system32\cmd.exe', '/c', 'true' ]
         else:
@@ -127,3 +130,29 @@ def test_prop_rendering(self):
             self.expectLogfile('stdio', "BUILDBOT-TEST\nBUILDBOT-TEST\n")
         self.expectOutcome(result=SUCCESS, status_text=["Ran"])
         return self.runStep()
+
+    def test_constr_args_descriptionSuffix(self):
+        self.setupStep(
+                master.MasterShellCommand(description='x', descriptionDone='y',
+                                          descriptionSuffix='z',
+                                env={'a':'b'}, path=['/usr/bin'], usePTY=True,
+                                command='true'))
+
+        # call twice to make sure the suffix doesnt get double added
+        self.assertEqual(self.step.describe(), ['x', 'z'])        
+        self.assertEqual(self.step.describe(), ['x', 'z'])
+
+        if runtime.platformType == 'win32':
+            exp_argv = [ r'C:\WINDOWS\system32\cmd.exe', '/c', 'true' ]
+        else:
+            exp_argv = [ '/bin/sh', '-c', 'true' ]
+        self.patchSpawnProcess(
+                exp_cmd=exp_argv[0], exp_argv=exp_argv,
+                exp_path=['/usr/bin'], exp_usePTY=True, exp_env={'a':'b'},
+                outputs=[
+                    ('out', 'hello!\n'),
+                    ('err', 'world\n'),
+                    ('rc', 0),
+                ])
+        self.expectOutcome(result=SUCCESS, status_text=['y', 'z'])
+        return self.runStep()
diff --git a/master/buildbot/test/unit/test_steps_shell.py b/master/buildbot/test/unit/test_steps_shell.py
--- a/master/buildbot/test/unit/test_steps_shell.py
+++ b/master/buildbot/test/unit/test_steps_shell.py
@@ -129,6 +129,23 @@ def test_describe_custom(self):
         self.assertEqual((step.describe(), step.describe(done=True)),
                          (['echoing'], ['echoed']))
 
+    def test_describe_with_suffix(self):
+        step = shell.ShellCommand(command="echo hello", descriptionSuffix="suffix")
+        self.assertEqual((step.describe(), step.describe(done=True)),
+                        (["'echo", "hello'", 'suffix'],)*2)
+
+    def test_describe_custom_with_suffix(self):
+        step = shell.ShellCommand(command="echo hello",
+                                  description=["echoing"], descriptionDone=["echoed"],
+                                  descriptionSuffix="suffix")
+        self.assertEqual((step.describe(), step.describe(done=True)),
+                         (['echoing', 'suffix'], ['echoed', 'suffix']))
+
+    def test_describe_no_command_with_suffix(self):
+        step = shell.ShellCommand(workdir='build', descriptionSuffix="suffix")
+        self.assertEqual((step.describe(), step.describe(done=True)),
+                         (['???', 'suffix'],)*2)
+
     def test_describe_unrendered_WithProperties(self):
         step = shell.ShellCommand(command=properties.WithProperties(''))
         self.assertEqual((step.describe(), step.describe(done=True)),
diff --git a/master/buildbot/test/unit/test_steps_source_base_Source.py b/master/buildbot/test/unit/test_steps_source_base_Source.py
--- a/master/buildbot/test/unit/test_steps_source_base_Source.py
+++ b/master/buildbot/test/unit/test_steps_source_base_Source.py
@@ -73,9 +73,14 @@ def test_start_no_codebase(self):
         step.build.getSourceStamp = mock.Mock()
         step.build.getSourceStamp.return_value = None
 
+        self.assertEqual(step.describe(), ['updating'])
+        self.assertEqual(step.name, Source.name)
+
         step.startStep(mock.Mock())
         self.assertEqual(step.build.getSourceStamp.call_args[0], ('',))
         
+        self.assertEqual(step.description, ['updating'])
+
     def test_start_with_codebase(self):
         step = self.setupStep(Source(codebase='codebase'))
         step.branch = 'branch'
@@ -83,9 +88,31 @@ def test_start_with_codebase(self):
         step.build.getSourceStamp = mock.Mock()
         step.build.getSourceStamp.return_value = None
 
+        self.assertEqual(step.describe(), ['updating', 'codebase'])
+        self.assertEqual(step.name, Source.name + " codebase")
+
         step.startStep(mock.Mock())
         self.assertEqual(step.build.getSourceStamp.call_args[0], ('codebase',))        
 
+        self.assertEqual(step.describe(True), ['update', 'codebase'])
+        
+    def test_start_with_codebase_and_descriptionSuffix(self):
+        step = self.setupStep(Source(codebase='my-code',
+                                     descriptionSuffix='suffix'))
+        step.branch = 'branch'
+        step.startVC = mock.Mock()
+        step.build.getSourceStamp = mock.Mock()
+        step.build.getSourceStamp.return_value = None
+
+        self.assertEqual(step.describe(), ['updating', 'suffix'])
+        self.assertEqual(step.name, Source.name + " my-code")
+
+        step.startStep(mock.Mock())
+        self.assertEqual(step.build.getSourceStamp.call_args[0], ('my-code',))        
+        
+        self.assertEqual(step.describe(True), ['update', 'suffix'])
+
+
 class TestSourceDescription(steps.BuildStepMixin, unittest.TestCase):
 
     def setUp(self):
@@ -107,3 +134,4 @@ def test_constructor_args_lists(self):
                       descriptionDone=['svn', 'update'])
         self.assertEqual(step.description, ['svn', 'update', '(running)'])
         self.assertEqual(step.descriptionDone, ['svn', 'update'])
+