Repository: buildbot/buildbot

Issue:
Error when disconnecting libvirt's worker
When insubstantiating a libvirt worker the disconnection will raise an excepcion:

```
jun 13 18:37:15 emberci buildbot[7131]: 2019-06-13 18:37:15+0000 [-] disconnecting old worker libvirt now
jun 13 18:37:15 emberci buildbot[7131]: 2019-06-13 18:37:15+0000 [-] waiting for worker to finish disconnecting
jun 13 18:37:15 emberci buildbot[7131]: 2019-06-13 18:37:15+0000 [-] insubstantiating worker <LibVirtWorker 'libvirt'>
jun 13 18:37:15 emberci buildbot[7131]: 2019-06-13 18:37:15+0000 [-] Attempting to stop 'libvirt'
jun 13 18:37:15 emberci buildbot[7131]: 2019-06-13 18:37:15+0000 [-] Looking to start a piece of work now...
jun 13 18:37:15 emberci buildbot[7131]: 2019-06-13 18:37:15+0000 [Broker,0,192.168.122.105] Worker.detached(libvirt)
jun 13 18:37:15 emberci buildbot[7131]: 2019-06-13 18:37:15+0000 [Broker,0,192.168.122.105] releaseLocks(<LibVirtWorker 'libvirt'>): []
jun 13 18:37:15 emberci buildbot[7131]: 2019-06-13 18:37:15+0000 [-] Completed a piece of work
jun 13 18:37:15 emberci buildbot[7131]: 2019-06-13 18:37:15+0000 [-] VM destroyed (libvirt): Forcing its connection closed.
jun 13 18:37:15 emberci buildbot[7131]: 2019-06-13 18:37:15+0000 [-] We forced disconnection (libvirt), cleaning up and triggering new build
jun 13 18:37:15 emberci buildbot[7131]: 2019-06-13 18:37:15+0000 [-] while insubstantiating
jun 13 18:37:15 emberci buildbot[7131]: Traceback (most recent call last):
jun 13 18:37:15 emberci buildbot[7131]: File "/usr/local/lib64/python3.6/site-packages/twisted/internet/defer.py", line 1475, in gotResult
jun 13 18:37:15 emberci buildbot[7131]: _inlineCallbacks(r, g, status)
jun 13 18:37:15 emberci buildbot[7131]: File "/usr/local/lib64/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
jun 13 18:37:15 emberci buildbot[7131]: result = result.throwExceptionIntoGenerator(g)
jun 13 18:37:15 emberci buildbot[7131]: File "/usr/local/lib64/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
jun 13 18:37:15 emberci buildbot[7131]: return g.throw(self.type, self.value, self.tb)
jun 13 18:37:15 emberci buildbot[7131]: File "/usr/local/lib/python3.6/site-packages/buildbot/worker/latent.py", line 439, in insubstantiate
jun 13 18:37:15 emberci buildbot[7131]: log.err(e, "while insubstantiating")
jun 13 18:37:15 emberci buildbot[7131]: --- <exception caught here> ---
jun 13 18:37:15 emberci buildbot[7131]: File "/usr/local/lib/python3.6/site-packages/buildbot/worker/latent.py", line 433, in insubstantiate
jun 13 18:37:15 emberci buildbot[7131]: yield self.stop_instance(fast)
jun 13 18:37:15 emberci buildbot[7131]: File "/usr/local/lib64/python3.6/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
jun 13 18:37:15 emberci buildbot[7131]: current.result = callback(current.result, *args, **kw)
jun 13 18:37:15 emberci buildbot[7131]: File "/usr/local/lib/python3.6/site-packages/buildbot/worker/libvirt.py", line 301, in _disconnect
jun 13 18:37:15 emberci buildbot[7131]: return super().disconnect()
jun 13 18:37:15 emberci buildbot[7131]: builtins.TypeError: super(type, obj): obj must be an instance or subtype of type
[root@emberci images]# virsh list --all
```

The issue is caused by the call to `super().disconnect()` in `_disconnect`'s callback:

```
        @d.addCallback
        def _disconnect(res):
            log.msg("VM destroyed (%s): Forcing its connection closed." %
                    self.workername)
            return super().disconnect()
```


Discussion:

Resolving patch:
diff --git a/master/buildbot/worker/libvirt.py b/master/buildbot/worker/libvirt.py
--- a/master/buildbot/worker/libvirt.py
+++ b/master/buildbot/worker/libvirt.py
@@ -280,6 +280,14 @@ def stop_instance(self, fast=False):
         If the VM was using a cloned image, I remove the clone
         When everything is tidied up, I ask that bbot looks for work to do
         """
+        @defer.inlineCallbacks
+        def _destroy_domain(res, domain):
+            log.msg('Graceful shutdown failed. Force destroying domain %s' %
+                    self.workername)
+            # Don't return res to stop propagating shutdown error if destroy
+            # was successful.
+            yield domain.destroy()
+
         log.msg("Attempting to stop '%s'" % self.workername)
         if self.domain is None:
             log.msg("I don't think that domain is even running, aborting")
@@ -291,22 +299,16 @@ def stop_instance(self, fast=False):
         if self.graceful_shutdown and not fast:
             log.msg("Graceful shutdown chosen for %s" % self.workername)
             d = domain.shutdown()
+            d.addErrback(_destroy_domain, domain)
         else:
             d = domain.destroy()
 
-        @d.addCallback
-        def _disconnect(res):
-            log.msg("VM destroyed (%s): Forcing its connection closed." %
-                    self.workername)
-            return super().disconnect()
-
-        @d.addBoth
-        def _disconnected(res):
-            log.msg(
-                "We forced disconnection (%s), cleaning up and triggering new build" % self.workername)
-            if self.base_image:
+        if self.base_image:
+            @d.addBoth
+            def _remove_image(res):
+                log.msg('Removing base image %s for %s' % (self.image,
+                                                           self.workername))
                 os.remove(self.image)
-            self.botmaster.maybeStartBuildsForWorker(self.workername)
-            return res
+                return res
 
         return d


Test patch:
diff --git a/master/buildbot/test/unit/test_worker_libvirt.py b/master/buildbot/test/unit/test_worker_libvirt.py
--- a/master/buildbot/test/unit/test_worker_libvirt.py
+++ b/master/buildbot/test/unit/test_worker_libvirt.py
@@ -97,6 +97,49 @@ def test_prepare_base_image_full(self):
         utils.getProcessValue.assert_called_with(
             "cp", ["o", "p"])
 
+    @defer.inlineCallbacks
+    def _test_stop_instance(self, graceful, fast, expected_destroy,
+                            expected_shutdown, shutdown_side_effect=None):
+        bs = self.ConcreteWorker('name', 'p', self.conn,
+                                 mock.sentinel.hd_image, 'o', xml='<xml/>')
+        bs.graceful_shutdown = graceful
+        domain_mock = mock.Mock()
+        if shutdown_side_effect:
+            domain_mock.shutdown.side_effect = shutdown_side_effect
+        bs.domain = libvirtworker.Domain(mock.sentinel.connection, domain_mock)
+
+        with mock.patch('os.remove') as remove_mock:
+            yield bs.stop_instance(fast=fast)
+
+        self.assertIsNone(bs.domain)
+
+        self.assertEqual(int(expected_destroy), domain_mock.destroy.call_count)
+        self.assertEqual(int(expected_shutdown),
+                         domain_mock.shutdown.call_count)
+        remove_mock.assert_called_once_with(mock.sentinel.hd_image)
+
+    @defer.inlineCallbacks
+    def test_stop_instance_destroy(self):
+        yield self._test_stop_instance(graceful=False,
+                                       fast=False,
+                                       expected_destroy=True,
+                                       expected_shutdown=False)
+
+    @defer.inlineCallbacks
+    def test_stop_instance_shutdown(self):
+        yield self._test_stop_instance(graceful=True,
+                                       fast=False,
+                                       expected_destroy=False,
+                                       expected_shutdown=True)
+
+    @defer.inlineCallbacks
+    def test_stop_instance_shutdown_fails(self):
+        yield self._test_stop_instance(graceful=True,
+                                       fast=False,
+                                       expected_destroy=True,
+                                       expected_shutdown=True,
+                                       shutdown_side_effect=Exception)
+
     @defer.inlineCallbacks
     def test_start_instance(self):
         bs = self.ConcreteWorker('b', 'p', self.conn, 'p', 'o',