Repository: vmware/tern
Pull request #225 (2019-04-11 19:29:03)

Initial work on supporting custom templates
The overview of the implementation is can be found in #180.
This work:
- Creates classes to support custom templates
- Creates tests for said classes
- Modifies the Package and ImageLayer classes to use the
custom templates

At this point the build does not break but the feature is not fully implemented.

Further work:
- Modify the to_dict method of Image, Notice and NoticeOrigin to be able
to accept a template object


Patch:
diff --git a/tern/classes/image_layer.py b/tern/classes/image_layer.py
index 27e0e6e..a7c9366 100644
--- a/tern/classes/image_layer.py
+++ b/tern/classes/image_layer.py
@@ -7,6 +7,7 @@ import os
 from tern.classes.package import Package
 from tern.classes.origins import Origins
 from tern.utils import rootfs
+from tern.utils.general import prop_names
 
 
 class ImageLayer:
@@ -107,18 +108,34 @@ class ImageLayer:
             self.__packages.remove(self.__packages[rem_index])
         return success
 
-    def to_dict(self):
+    def to_dict(self, template=None):
         layer_dict = {}
         pkg_list = []
+        # for packages call each package object's to_dict method
         for pkg in self.__packages:
-            pkg_list.append(pkg.to_dict_notes())
-        layer_dict.update({self.fs_hash: {'packages': pkg_list,
-                                          'tar_file': self.tar_file,
-                                          'created_by': self.created_by,
-                                          'diff_id': self.diff_id,
-                                          'import_str': self.import_str,
-                                          'notes': self.origins.to_dict()
-                                          }})
+            pkg_list.append(pkg.to_dict(template))
+        if template:
+            # use the template mapping for key names
+            for key, prop in prop_names(self):
+                if prop in template.image_layer().keys():
+                    layer_dict.update(
+                        {template.image_layer()[prop]: self.__dict__[key]})
+        else:
+            # directly use property names
+            for key, prop in prop_names(self):
+                layer_dict.update({prop: self.__dict__[key]})
+            # update with 'packages' info
+            layer_dict.update({'packages': pkg_list})
+            # take care of the 'origins' property
+            layer_dict.update({'origins': self.origins.to_dict()})
+        # update the 'origins' if it exists in the mapping
+        if template and 'origins' in template.image_layer().keys():
+            layer_dict.update(
+                {template.image_layer()['origins']: self.origins.to_dict()})
+        # update the 'packages' if it exists in the mapping
+        if template and 'packages' in template.image_layer().keys():
+            layer_dict.update(
+                {template.image_layer()['packages']: pkg_list})
         return layer_dict
 
     def get_package_names(self):
diff --git a/tern/classes/package.py b/tern/classes/package.py
index 6c214a6..468e85f 100644
--- a/tern/classes/package.py
+++ b/tern/classes/package.py
@@ -4,6 +4,7 @@
 #
 
 from tern.classes.origins import Origins
+from tern.utils.general import prop_names
 
 
 class Package:
@@ -58,20 +59,29 @@ class Package:
     def origins(self):
         return self.__origins
 
-    def to_dict(self):
+    def to_dict(self, template=None):
+        '''Return a dictionary version of the Package object
+        If given an object which is a subclass of Template then map
+        the keys to the Package class properties'''
         pkg_dict = {}
-        pkg_dict.update({'name': self.name})
-        pkg_dict.update({'version': self.version})
-        pkg_dict.update({'license': self.license})
-        pkg_dict.update({'src_url': self.src_url})
+        if template:
+            # loop through object properties
+            for key, prop in prop_names(self):
+                # check if the property is in the mapping
+                if prop in template.package().keys():
+                    pkg_dict.update(
+                        {template.package()[prop]: self.__dict__[key]})
+        else:
+            # don't map, just use the property name as the key
+            for key, prop in prop_names(self):
+                pkg_dict.update({prop: self.__dict__[key]})
+            pkg_dict.update({'origins': self.origins.to_dict()})
+        # update the 'origins' part if it exists in the mapping
+        if template and 'origins' in template.package().keys():
+            pkg_dict.update(
+                {template.package()['origins']: self.origins.to_dict()})
         return pkg_dict
 
-    def to_dict_notes(self):
-        '''Return the package dictionary with the notices and their origins'''
-        notes_dict = self.to_dict()
-        notes_dict.update({'notes': self.origins.to_dict()})
-        return notes_dict
-
     def fill(self, package_dict):
         '''The package dict looks like this:
             name: <name>
diff --git a/tern/classes/template.py b/tern/classes/template.py
new file mode 100644
index 0000000..6053815
--- /dev/null
+++ b/tern/classes/template.py
@@ -0,0 +1,48 @@
+#
+# Copyright (c) 2017-2019 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+from abc import ABCMeta
+from abc import abstractmethod
+
+
+class Template(metaclass=ABCMeta):
+    '''This is an abstract base class for reporting templates
+    A specific type of template needs to be a subclass of this class
+    methods:
+        must implement:
+            package: mappings for the properties under 'Package'
+            layer: mappings for the properties under 'Layer'
+            image: mappings for the properties under 'Image'
+        should implement:
+            notice: mappings for the properties under 'Notice'
+            notice_origin: mappings for the properties under 'NoticeOrigin'
+            origins: mappings for the properties under 'Origins' '''
+
+    @abstractmethod
+    def package(self):
+        '''Must implement a mapping for 'Package' class properties'''
+        pass
+
+    @abstractmethod
+    def image_layer(self):
+        '''Must implement a mapping for 'ImageLayer' class properties'''
+        pass
+
+    @abstractmethod
+    def image(self):
+        '''Must implement a mapping for 'Image' class properties'''
+        pass
+
+    def notice(self):
+        '''Should implement a mapping for 'Notice' class properties'''
+        pass
+
+    def notice_origin(self):
+        '''Should implement a mapping for 'NoticeOrigin' class properties'''
+        pass
+
+    def origins(self):
+        '''Should implement a mapping for 'Origins' class properties'''
+        pass
diff --git a/tern/classes/templates/__init__.py b/tern/classes/templates/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tern/classes/templates/spdx.py b/tern/classes/templates/spdx.py
new file mode 100644
index 0000000..f42e136
--- /dev/null
+++ b/tern/classes/templates/spdx.py
@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+from tern.classes.template import Template
+
+
+class SPDXTagValue(Template):
+    '''This is the SPDX Template class
+    It provides mappings for the SPDX tag-value document format'''
+
+    def package(self):
+        return {'name': 'PackageName',
+                'version': 'PackageVersion',
+                'license': 'PackageLicenseDeclared'}
+
+    def image_layer(self):
+        return {'diff_id': 'PackageName',
+                'fs_hash': 'PackageChecksum'}
+
+    def image(self):
+        return {'name': 'PackageName',
+                'tag': 'PackageVersion',
+                'id': 'PackageChecksum'}
diff --git a/tern/utils/general.py b/tern/utils/general.py
index 23b4a38..282a094 100644
--- a/tern/utils/general.py
+++ b/tern/utils/general.py
@@ -1,18 +1,15 @@
 #
-# Copyright (c) 2017-2018 VMware, Inc. All Rights Reserved.
+# Copyright (c) 2017-2019 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: BSD-2-Clause
 #
 import os
 import random
 import re
 import subprocess  # nosec
+from contextlib import contextmanager
 
 from tern.report.formats import commit_version, packaged_version
-
 from tern import Version
-
-from contextlib import contextmanager
-
 from tern.utils import constants
 
 
@@ -90,3 +87,15 @@ def get_git_rev_or_version():
     except subprocess.CalledProcessError:
         output = packaged_version.format(version=Version)
     return output.split('\n').pop(0)
+
+
+def prop_names(obj):
+    '''Given an object, return a generator that will produce the object's
+    property key in its __dict__ representation and it's name'''
+    prop_decorators = r'^__|^_'
+    for key in obj.__dict__.keys():
+        # remove private and protected decorator characters if any
+        priv_name = '_' + obj.__class__.__name__
+        prop_name = re.sub(priv_name, '', key)
+        prop_name = re.sub(prop_decorators, '', prop_name, 1)
+        yield key, prop_name


Test patch:
diff --git a/tests/test_class_image_layer.py b/tests/test_class_image_layer.py
index 3ec5fae..f646f37 100644
--- a/tests/test_class_image_layer.py
+++ b/tests/test_class_image_layer.py
@@ -7,6 +7,8 @@ import unittest
 
 from tern.classes.image_layer import ImageLayer
 from tern.classes.package import Package
+from test_fixtures import TestTemplate1
+from test_fixtures import TestTemplate2
 
 
 class TestClassImageLayer(unittest.TestCase):
@@ -49,10 +51,25 @@ class TestClassImageLayer(unittest.TestCase):
         p1 = Package('x')
         self.layer.add_package(p1)
         a_dict = self.layer.to_dict()
-        self.assertTrue(a_dict[''])
-        self.assertEqual(len(a_dict['']['packages']), 1)
-        self.assertEqual(a_dict['']['packages'][0]['name'], 'x')
-        self.assertEqual(a_dict['']['tar_file'], 'path/to/tar')
+        self.assertEqual(a_dict['diff_id'], '123abc')
+        self.assertEqual(len(a_dict['packages']), 1)
+        self.assertEqual(a_dict['packages'][0]['name'], 'x')
+        self.assertEqual(a_dict['tar_file'], 'path/to/tar')
+
+    def testToDictTemplate(self):
+        template1 = TestTemplate1()
+        template2 = TestTemplate2()
+        p1 = Package('x')
+        self.layer.add_package(p1)
+        dict1 = self.layer.to_dict(template1)
+        dict2 = self.layer.to_dict(template2)
+        self.assertEqual(len(dict1.keys()), 3)
+        self.assertEqual(dict1['layer.diff'], '123abc')
+        self.assertEqual(dict1['layer.tarfile'], 'path/to/tar')
+        self.assertEqual(len(dict1['layer.packages']), 1)
+        self.assertEqual(len(dict2.keys()), 4)
+        self.assertFalse(dict2['notes'])
+        self.assertFalse(dict2['layer.packages'][0]['notes'])
 
     def testGetPackageNames(self):
         p1 = Package('x')
diff --git a/tests/test_class_package.py b/tests/test_class_package.py
index 3ac1254..147be59 100644
--- a/tests/test_class_package.py
+++ b/tests/test_class_package.py
@@ -6,61 +6,76 @@ SPDX-License-Identifier: BSD-2-Clause
 import unittest
 
 from tern.classes.package import Package
+from test_fixtures import TestTemplate1
+from test_fixtures import TestTemplate2
 
 
 class TestClassPackage(unittest.TestCase):
 
     def setUp(self):
-        self.package = Package('x')
+        self.p1 = Package('p1')
+        self.p1.version = '1.0'
+        self.p1.license = 'Apache 2.0'
+        self.p1.src_url = 'github.com'
+
+        self.p2 = Package('p2')
 
     def tearDown(self):
-        del self.package
+        del self.p1
+        del self.p2
 
     def testInstance(self):
-        self.assertEqual(self.package.name, 'x')
-        self.assertEqual(self.package.version, '')
-        self.assertFalse(self.package.src_url)
-        self.assertFalse(self.package.license)
+        self.assertEqual(self.p2.name, 'p2')
+        self.assertFalse(self.p2.version)
+        self.assertFalse(self.p2.src_url)
+        self.assertFalse(self.p2.license)
 
     def testSetters(self):
-        self.assertRaises(AttributeError, setattr, self.package, 'name', 'y')
-        self.package.version = '1.0'
-        self.assertEqual(self.package.version, '1.0')
-        self.package.license = 'Apache 2.0'
-        self.assertEqual(self.package.license, 'Apache 2.0')
-        self.package.src_url = 'github.com'
-        self.assertEqual(self.package.src_url, 'github.com')
+        self.assertRaises(AttributeError, setattr, self.p2, 'name', 'y')
+        self.p2.version = '2.0'
+        self.assertEqual(self.p2.version, '2.0')
+        self.p2.license = 'Apache 2.0'
+        self.assertEqual(self.p2.license, 'Apache 2.0')
+        self.p2.src_url = 'github.com'
+        self.assertEqual(self.p2.src_url, 'github.com')
 
     def testGetters(self):
-        self.package.version = '1.0'
-        self.package.license = 'Apache 2.0'
-        self.package.src_url = 'github.com'
-        self.assertEqual(self.package.name, 'x')
-        self.assertEqual(self.package.version, '1.0')
-        self.assertEqual(self.package.license, 'Apache 2.0')
-        self.assertEqual(self.package.src_url, 'github.com')
+        self.assertEqual(self.p1.name, 'p1')
+        self.assertEqual(self.p1.version, '1.0')
+        self.assertEqual(self.p1.license, 'Apache 2.0')
+        self.assertEqual(self.p1.src_url, 'github.com')
 
     def testToDict(self):
-        self.package.version = '1.0'
-        self.package.license = 'Apache 2.0'
-        self.package.src_url = 'github.com'
-        a_dict = self.package.to_dict()
-        self.assertEqual(a_dict['name'], 'x')
+        a_dict = self.p1.to_dict()
+        self.assertEqual(a_dict['name'], 'p1')
         self.assertEqual(a_dict['version'], '1.0')
         self.assertEqual(a_dict['license'], 'Apache 2.0')
         self.assertEqual(a_dict['src_url'], 'github.com')
+        self.assertFalse(a_dict['origins'])
+
+    def testToDictTemplate(self):
+        template1 = TestTemplate1()
+        template2 = TestTemplate2()
+        dict1 = self.p1.to_dict(template1)
+        dict2 = self.p1.to_dict(template2)
+        self.assertEqual(len(dict1.keys()), 3)
+        self.assertEqual(dict1['package.name'], 'p1')
+        self.assertEqual(dict1['package.version'], '1.0')
+        self.assertEqual(dict1['package.license'], 'Apache 2.0')
+        self.assertEqual(len(dict2.keys()), 5)
+        self.assertFalse(dict2['notes'])
 
     def testIsEqual(self):
-        p = Package('x')
+        p = Package('p2')
         p.license = 'TestLicense'
         p.version = '1.0'
         p.src_url = 'TestUrl'
-        self.package.license = 'TestLicense'
-        self.package.version = '2.0'
-        self.package.src_url = 'TestUrl'
-        self.assertFalse(self.package.is_equal(p))
+        self.p2.license = 'TestLicense'
+        self.p2.version = '2.0'
+        self.p2.src_url = 'TestUrl'
+        self.assertFalse(self.p2.is_equal(p))
         p.version = '2.0'
-        self.assertTrue(self.package.is_equal(p))
+        self.assertTrue(self.p2.is_equal(p))
 
 
 if __name__ == '__main__':
diff --git a/tests/test_class_template.py b/tests/test_class_template.py
new file mode 100644
index 0000000..2d23c26
--- /dev/null
+++ b/tests/test_class_template.py
@@ -0,0 +1,57 @@
+#
+# Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+import unittest
+
+from test_fixtures import TestTemplate1
+from test_fixtures import TestTemplate2
+
+
+class TestClassTemplate(unittest.TestCase):
+    '''Note that we cannot instantiate the abstract base class so we
+    will use a test fixture which is a subclass of Template'''
+
+    def setUp(self):
+        self.template1 = TestTemplate1()
+        self.template2 = TestTemplate2()
+
+    def tearDown(self):
+        del self.template1
+        del self.template2
+
+    def testPackage(self):
+        mapping = self.template1.package()
+        self.assertEqual(mapping['name'], 'package.name')
+        self.assertEqual(mapping['version'], 'package.version')
+        self.assertEqual(mapping['license'], 'package.license')
+
+    def testImageLayer(self):
+        mapping = self.template1.image_layer()
+        self.assertEqual(mapping['diff_id'], 'layer.diff')
+        self.assertEqual(mapping['tar_file'], 'layer.tarfile')
+        self.assertEqual(mapping['packages'], 'layer.packages')
+
+    def testImage(self):
+        mapping = self.template1.image()
+        self.assertEqual(mapping['id'], 'image.id')
+        self.assertEqual(mapping['layers'], 'image.layers')
+
+    def testNotice(self):
+        mapping = self.template2.notice()
+        self.assertEqual(mapping['level'], 'note.level')
+        self.assertEqual(mapping['message'], 'note.message')
+
+    def testNoticeOrigin(self):
+        mapping = self.template2.notice_origin()
+        self.assertEqual(mapping['origin_str'], 'note.source')
+        self.assertEqual(mapping['notices'], 'note.messages')
+
+    def testOrigins(self):
+        mapping = self.template2.origins()
+        self.assertEqual(mapping['origins'], 'notes')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py
new file mode 100644
index 0000000..c7b5e44
--- /dev/null
+++ b/tests/test_fixtures.py
@@ -0,0 +1,78 @@
+#
+# Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+#
+"""
+Objects to use for testing
+"""
+
+from tern.classes.package import Package
+from tern.classes.image_layer import ImageLayer
+from tern.classes.image import Image
+from tern.classes.template import Template
+
+
+class TestImage(Image):
+    def __init__(self, id):
+        super().__init__(id)
+
+    def load_image(self):
+        l1 = ImageLayer('123abc', 'path/to/tar')
+        l1.add_package(Package('p1'))
+        l1.add_package(Package('p2'))
+        self._layers.append(l1)
+
+
+class TestTemplate1(Template):
+    '''Template with no origins mapping'''
+    def package(self):
+        return {'name': 'package.name',
+                'version': 'package.version',
+                'license': 'package.license'}
+
+    def image_layer(self):
+        return {'diff_id': 'layer.diff',
+                'tar_file': 'layer.tarfile',
+                'packages': 'layer.packages'}
+
+    def image(self):
+        return {'id': 'image.id',
+                'layers': 'image.layers'}
+
+
+class TestTemplate2(Template):
+    '''Template with origins mapping'''
+    def package(self):
+        mapping = {'name': 'package.name',
+                   'version': 'package.version',
+                   'license': 'package.license',
+                   'src_url': 'package.url'}
+        # we update the mapping with another defined mapping
+        mapping.update(self.origins())
+        return mapping
+
+    def image_layer(self):
+        mapping = {'diff_id': 'layer.diff',
+                   'tar_file': 'layer.tarfile',
+                   'packages': 'layer.packages'}
+        # we update the mapping with another defined mapping
+        mapping.update(self.origins())
+        return mapping
+
+    def image(self):
+        mapping = {'id': 'image.id',
+                   'layers': 'image.layers'}
+        # we update the mapping with another defined mapping
+        mapping.update(self.origins())
+        return mapping
+
+    def notice(self):
+        return {'level': 'note.level',
+                'message': 'note.message'}
+
+    def notice_origin(self):
+        return {'origin_str': 'note.source',
+                'notices': 'note.messages'}
+
+    def origins(self):
+        return {'origins': 'notes'}