Repository: buildbot/buildbot

Issue:
Custom templates: loading fixed #5035
- master/buildbot/www/config.py:parseCustomTemplateDir
    Returns pure HTML. Conversion to JSON is done in
    www/base/src/app/layout.jade
- www/base/package.json
    Missing plugin_name fixed
- www/build_common/src/ng-template-loader.js
    Fixed missing "views/" part of template name

## Contributor Checklist:

* [x] I have updated the unit tests
* [ ] I have created a file in the `master/buildbot/newsfragments` directory (and read the `README.txt` in that directory)
* [ ] I have updated the appropriate documentation



Discussion:
# [Codecov](https://codecov.io/gh/buildbot/buildbot/pull/5041?src=pr&el=h1) Report
> Merging [#5041](https://codecov.io/gh/buildbot/buildbot/pull/5041?src=pr&el=desc) into [master](https://codecov.io/gh/buildbot/buildbot/commit/bc3be1c5213abf084e6b08dc9e88d5afb2ff7a99?src=pr&el=desc) will **increase** coverage by `0.02%`.
> The diff coverage is `100%`.

[![Impacted file tree graph](https://codecov.io/gh/buildbot/buildbot/pull/5041/graphs/tree.svg?width=650&token=3GYcEDLFWz&height=150&src=pr)](https://codecov.io/gh/buildbot/buildbot/pull/5041?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master    #5041      +/-   ##
==========================================
+ Coverage   88.99%   89.01%   +0.02%     
==========================================
  Files         332      332              
  Lines       35218    35218              
==========================================
+ Hits        31341    31349       +8     
+ Misses       3877     3869       -8
```


| [Impacted Files](https://codecov.io/gh/buildbot/buildbot/pull/5041?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [master/buildbot/www/config.py](https://codecov.io/gh/buildbot/buildbot/pull/5041/diff?src=pr&el=tree#diff-bWFzdGVyL2J1aWxkYm90L3d3dy9jb25maWcucHk=) | `95.69% <100%> (ø)` | :arrow_up: |
| [master/buildbot/db/connector.py](https://codecov.io/gh/buildbot/buildbot/pull/5041/diff?src=pr&el=tree#diff-bWFzdGVyL2J1aWxkYm90L2RiL2Nvbm5lY3Rvci5weQ==) | `100% <0%> (+1.25%)` | :arrow_up: |
| [master/buildbot/scripts/cleanupdb.py](https://codecov.io/gh/buildbot/buildbot/pull/5041/diff?src=pr&el=tree#diff-bWFzdGVyL2J1aWxkYm90L3NjcmlwdHMvY2xlYW51cGRiLnB5) | `100% <0%> (+13.46%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/buildbot/buildbot/pull/5041?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/buildbot/buildbot/pull/5041?src=pr&el=footer). Last update [bc3be1c...a4e4de0](https://codecov.io/gh/buildbot/buildbot/pull/5041?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).

Thanks a lot for looking into this!

Resolving patch:
diff --git a/master/buildbot/worker/openstack.py b/master/buildbot/worker/openstack.py
--- a/master/buildbot/worker/openstack.py
+++ b/master/buildbot/worker/openstack.py
@@ -207,7 +207,7 @@ def _determineVolumeSize(self, source_type, source_uuid):
         if source_type == 'image':
             # The size returned for an image is in bytes. Round up to the next
             # integer GiB.
-            image = nova.images.get(source_uuid)
+            image = nova.glance.get(source_uuid)
             if hasattr(image, 'OS-EXT-IMG-SIZE:size'):
                 size = getattr(image, 'OS-EXT-IMG-SIZE:size')
                 size_gb = int(math.ceil(size / 1024.0**3))
@@ -227,18 +227,29 @@ def _determineVolumeSize(self, source_type, source_uuid):
 
     @defer.inlineCallbacks
     def _getImage(self, build):
-        # If image is a callable, then pass it the list of images. The
-        # function should return the image's UUID to use.
-        image = self.image
-        if callable(image):
-            image_uuid = image(self.novaclient.images.list())
-        else:
-            image_uuid = yield build.render(image)
+        image_uuid = yield build.render(self.image)
+        # check if we got name instead of uuid
+        for image in self.novaclient.glance.list():
+            if image.name == image_uuid:
+                image_uuid = image.id
         return image_uuid
 
+    @defer.inlineCallbacks
+    def _getFlavor(self, build):
+        flavor_uuid = yield build.render(self.flavor)
+        # check if we got name instead of uuid
+        for flavor in self.novaclient.flavors.list():
+            if flavor.name == flavor_uuid:
+                flavor_uuid = flavor.id
+        return flavor_uuid
+
     @defer.inlineCallbacks
     def renderWorkerProps(self, build):
         image = yield self._getImage(build)
+        flavor = yield self._getFlavor(build)
+        nova_args = yield build.render(self.nova_args)
+        meta = yield build.render(self.meta)
+
         if self.block_devices is not None:
             block_devices = []
             for bd in self.block_devices:
@@ -246,24 +257,24 @@ def renderWorkerProps(self, build):
                 block_devices.append(rendered_block_device)
         else:
             block_devices = None
-        return (image, block_devices)
+        return (image, flavor, block_devices, nova_args, meta)
 
     @defer.inlineCallbacks
     def start_instance(self, build):
         if self.instance is not None:
             raise ValueError('instance active')
 
-        image, block_devices = yield self.renderWorkerPropsOnStart(build)
-        res = yield threads.deferToThread(self._start_instance, image,
-                                          block_devices)
+        image, flavor, block_devices, nova_args, meta = yield self.renderWorkerPropsOnStart(build)
+        res = yield threads.deferToThread(self._start_instance, image, flavor,
+                                          block_devices, nova_args, meta)
         return res
 
-    def _start_instance(self, image_uuid, block_devices):
-        boot_args = [self.workername, image_uuid, self.flavor]
+    def _start_instance(self, image_uuid, flavor_uuid, block_devices, nova_args, meta):
+        boot_args = [self.workername, image_uuid, flavor_uuid]
         boot_kwargs = dict(
-            meta=self.meta,
+            meta=meta,
             block_device_mapping_v2=block_devices,
-            **self.nova_args)
+            **nova_args)
         instance = self.novaclient.servers.create(*boot_args, **boot_kwargs)
         # There is an issue when using sessions that the status is not
         # available on the first try. Trying again will work fine. Fetch the


Test patch:
diff --git a/master/buildbot/test/fake/openstack.py b/master/buildbot/test/fake/openstack.py
--- a/master/buildbot/test/fake/openstack.py
+++ b/master/buildbot/test/fake/openstack.py
@@ -27,6 +27,7 @@
     'image': '28a65eb4-f354-4420-97dc-253b826547f7',
     'volume': '65fbb9f1-c4d5-40a8-a233-ad47c52bb837',
     'snapshot': 'ab89152d-3c26-4d30-9ae5-65b705f874b7',
+    'flavor': '853774a1-459f-4f1f-907e-c96f62472531',
 }
 
 
@@ -38,12 +39,14 @@ class FakeNovaClient():
 class Client():
 
     def __init__(self, version, session):
-        self.images = ItemManager()
-        self.images._add_items([Image(TEST_UUIDS['image'], 'CirrOS 0.3.4', 13287936)])
+        self.glance = ItemManager()
+        self.glance._add_items([Image(TEST_UUIDS['image'], 'CirrOS 0.3.4', 13287936)])
         self.volumes = ItemManager()
         self.volumes._add_items([Volume(TEST_UUIDS['volume'], 'CirrOS 0.3.4', 4)])
         self.volume_snapshots = ItemManager()
         self.volume_snapshots._add_items([Snapshot(TEST_UUIDS['snapshot'], 'CirrOS 0.3.4', 2)])
+        self.flavors = ItemManager()
+        self.flavors._add_items([Flavor(TEST_UUIDS['flavor'], 'm1.small', 0)])
         self.servers = Servers()
         self.session = session
         self.client = FakeNovaClient()
@@ -86,6 +89,10 @@ def __init__(self, *args, **kwargs):
         setattr(self, 'OS-EXT-IMG-SIZE:size', self.size)
 
 
+class Flavor(Item):
+    pass
+
+
 class Volume(Item):
     pass
 
diff --git a/master/buildbot/test/unit/worker/test_openstack.py b/master/buildbot/test/unit/worker/test_openstack.py
--- a/master/buildbot/test/unit/worker/test_openstack.py
+++ b/master/buildbot/test/unit/worker/test_openstack.py
@@ -51,7 +51,9 @@ def setUp(self):
         self.patch(openstack, "client", novaclient)
         self.patch(openstack, "loading", novaclient)
         self.patch(openstack, "session", novaclient)
-        self.build = Properties(image=novaclient.TEST_UUIDS['image'])
+        self.build = Properties(image=novaclient.TEST_UUIDS['image'],
+                                flavor=novaclient.TEST_UUIDS['flavor'],
+                                meta_value='value')
 
     @defer.inlineCallbacks
     def setupWorker(self, *args, **kwargs):
@@ -149,7 +151,7 @@ def test_constructor_block_devices_get_sizes(self):
             {'source_type': 'snapshot', 'uuid': novaclient.TEST_UUIDS['snapshot']},
         ]
 
-        def check_volume_sizes(_images, block_devices):
+        def check_volume_sizes(_images, _flavors, block_devices, nova_args, metas):
             self.assertEqual(len(block_devices), 4)
             self.assertEqual(block_devices[0]['volume_size'], 1)
             self.assertIsInstance(block_devices[0]['volume_size'], int,
@@ -214,30 +216,44 @@ def test_getImage_string(self):
         self.assertEqual('image-uuid', image_uuid)
 
     @defer.inlineCallbacks
-    def test_getImage_callable(self):
-        def image_callable(images):
-            filtered = [i for i in images if i.id == 'uuid1']
-            return filtered[0].id
-
+    def test_getImage_renderable(self):
         bs = yield self.setupWorker('bot', 'pass', flavor=1,
-                                    image=image_callable, **self.os_auth)
-        os_client = bs.novaclient
-        os_client.images._add_items([
-            novaclient.Image('uuid1', 'name1', 1),
-            novaclient.Image('uuid2', 'name2', 1),
-            novaclient.Image('uuid3', 'name3', 1),
-            ])
+                                    image=Interpolate('%(prop:image)s'),
+                                    **self.os_auth)
         image_uuid = yield bs._getImage(self.build)
-        self.assertEqual('uuid1', image_uuid)
+        self.assertEqual(novaclient.TEST_UUIDS['image'], image_uuid)
 
     @defer.inlineCallbacks
-    def test_getImage_renderable(self):
+    def test_getImage_name(self):
         bs = yield self.setupWorker('bot', 'pass', flavor=1,
-                                    image=Interpolate('%(prop:image)s'),
+                                    image='CirrOS 0.3.4',
                                     **self.os_auth)
         image_uuid = yield bs._getImage(self.build)
         self.assertEqual(novaclient.TEST_UUIDS['image'], image_uuid)
 
+    @defer.inlineCallbacks
+    def test_getFlavor_string(self):
+        bs = yield self.setupWorker(
+            'bot', 'pass', **self.bs_image_args)
+        flavor_uuid = yield bs._getFlavor(self.build)
+        self.assertEqual(1, flavor_uuid)
+
+    @defer.inlineCallbacks
+    def test_getFlavor_renderable(self):
+        bs = yield self.setupWorker('bot', 'pass', image="1",
+                                    flavor=Interpolate('%(prop:flavor)s'),
+                                    **self.os_auth)
+        flavor_uuid = yield bs._getFlavor(self.build)
+        self.assertEqual(novaclient.TEST_UUIDS['flavor'], flavor_uuid)
+
+    @defer.inlineCallbacks
+    def test_getFlavor_name(self):
+        bs = yield self.setupWorker('bot', 'pass', image="1",
+                                    flavor='m1.small',
+                                    **self.os_auth)
+        flavor_uuid = yield bs._getFlavor(self.build)
+        self.assertEqual(novaclient.TEST_UUIDS['flavor'], flavor_uuid)
+
     @defer.inlineCallbacks
     def test_start_instance_already_exists(self):
         bs = yield self.setupWorker(
@@ -291,7 +307,39 @@ def test_start_instance_check_meta(self):
         bs._poll_resolution = 0
         uuid, image_uuid, time_waiting = yield bs.start_instance(self.build)
         self.assertIn('meta', bs.instance.boot_kwargs)
-        self.assertIdentical(bs.instance.boot_kwargs['meta'], meta_arg)
+        self.assertEquals(bs.instance.boot_kwargs['meta'], meta_arg)
+
+    @defer.inlineCallbacks
+    def test_start_instance_check_meta_renderable(self):
+        meta_arg = {'some_key': Interpolate('%(prop:meta_value)s')}
+        bs = yield self.setupWorker('bot', 'pass', meta=meta_arg,
+                                    **self.bs_image_args)
+        bs._poll_resolution = 0
+        uuid, image_uuid, time_waiting = yield bs.start_instance(self.build)
+        self.assertIn('meta', bs.instance.boot_kwargs)
+        self.assertEquals(bs.instance.boot_kwargs['meta'], {'some_key': 'value'})
+
+    @defer.inlineCallbacks
+    def test_start_instance_check_nova_args(self):
+        nova_args = {'some-key': 'some-value'}
+
+        bs = yield self.setupWorker('bot', 'pass', nova_args=nova_args,
+                                    **self.bs_image_args)
+        bs._poll_resolution = 0
+        uuid, image_uuid, time_waiting = yield bs.start_instance(self.build)
+        self.assertIn('meta', bs.instance.boot_kwargs)
+        self.assertEquals(bs.instance.boot_kwargs['some-key'], 'some-value')
+
+    @defer.inlineCallbacks
+    def test_start_instance_check_nova_args_renderable(self):
+        nova_args = {'some-key': Interpolate('%(prop:meta_value)s')}
+
+        bs = yield self.setupWorker('bot', 'pass', nova_args=nova_args,
+                                    **self.bs_image_args)
+        bs._poll_resolution = 0
+        uuid, image_uuid, time_waiting = yield bs.start_instance(self.build)
+        self.assertIn('meta', bs.instance.boot_kwargs)
+        self.assertEquals(bs.instance.boot_kwargs['some-key'], 'value')
 
     @defer.inlineCallbacks
     def test_interpolate_renderables_for_new_build(self):