Repository: vmware/tern (python)

Issue:
Create a function to create SPDX document
**Description**
Create a function to create SPDX document.
This is probably more involved than writing one function. So we do this in a number of steps

**To Do**
1. Create a folder `report/spdxtagvalue` (this should not affect current operation)
2. Create a file called `generator.py`
3. Write out implementation of generating SPDX document here
4. Any formatting can be done in a file called `formatting.py`

**Background**
This issue is dependent on #252 
SPDX tag value example: https://spdx.org/spdx-tagvalue-example

**Super Issues**
#174 


Pull request description:
Create function to generate SPDX document
This resolves #253 

Patch:
diff --git a/tern/classes/image.py b/tern/classes/image.py
index 4be9a2e..0048878 100644
--- a/tern/classes/image.py
+++ b/tern/classes/image.py
@@ -136,3 +136,14 @@ class Image:
             image_dict.update({'layers': layer_list})
             image_dict.update({'origins': self.origins.to_dict()})
         return image_dict
+
+    def get_human_readable_id(self):
+        '''Some reports like SPDX want a unique name for the full image
+        and this is currently not supported by any image tool. So using
+        a combination of image id, name and tag instead'''
+        name = self.image_id
+        if self.name:
+            name = name + '-{}'.format(self.name)
+        if self.tag:
+            name = name + '-{}'.format(self.tag)
+        return name
diff --git a/tern/report/content.py b/tern/report/content.py
index 6fa7c56..94066d3 100644
--- a/tern/report/content.py
+++ b/tern/report/content.py
@@ -9,8 +9,19 @@ Functions to generate content for the report
 
 import json
 import yaml
+
 from tern.command_lib import command_lib
 from tern.report import formats
+from tern.utils.general import get_git_rev_or_version
+from tern import Version
+
+
+def get_tool_version():
+    '''Return a string describing the version and where it came from'''
+    ver_type, ver = get_git_rev_or_version()
+    if ver_type == 'commit':
+        return formats.commit_version.format(commit_sha=ver)
+    return formats.packaged_version.format(version=Version)
 
 
 def print_invoke_list(info_dict, info):
diff --git a/tern/report/report.py b/tern/report/report.py
index 83ac3f8..5188976 100644
--- a/tern/report/report.py
+++ b/tern/report/report.py
@@ -309,7 +309,7 @@ def generate_report(args, *images):
 def generate_verbose(is_summary, images):
     '''Generate a verbose report'''
     report = formats.disclaimer.format(
-        version_info=general.get_git_rev_or_version())
+        version_info=content.get_tool_version())
     if is_summary:
         logger.debug('Creating a summary of components in image...')
         for image in images:
diff --git a/tern/report/spdxtagvalue/__init__.py b/tern/report/spdxtagvalue/__init__.py
new file mode 100644
index 0000000..21bda99
--- /dev/null
+++ b/tern/report/spdxtagvalue/__init__.py
@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+#
diff --git a/tern/report/spdxtagvalue/formats.py b/tern/report/spdxtagvalue/formats.py
new file mode 100644
index 0000000..526fdec
--- /dev/null
+++ b/tern/report/spdxtagvalue/formats.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+#
+"""
+SPDX document formatting
+"""
+
+# basic strings
+tag_value = '{tag}: {value}'
+
+# document level strings
+spdx_version = 'SPDXVersion: SPDX-2.1'
+data_license = 'DataLicense: CC0-1.0'
+spdx_id = 'SPDXID: SPDXRef-DOCUMENT'
+document_name = 'DocumentName: Tern report for {image_name}'
+document_comment = '''DocumentComment: <text>This document was generated by '''
+'''the Tern Project at https://github.com/vmware/tern</text>'''
+document_namespace = 'DocumentNamespace: https://spdx.org/spdxdocs/tern'
+'-report-{image_id}-{version}'
+license_list_version = 'LicenseListVersion: 3.4'
+creator = 'Creator: Tool: tern-{version}'
+created = 'Created: {timestamp}'
+
+# Package level strings
+package_comment = 'PackageComment: <text>{comment}</text>'
+
+# Relationship strings
+contains = '{outer} CONTAINS {inner}'
+prereq = '{after} HAS_PREREQUISITE {before}'
diff --git a/tern/report/spdxtagvalue/generator.py b/tern/report/spdxtagvalue/generator.py
new file mode 100644
index 0000000..59b49b0
--- /dev/null
+++ b/tern/report/spdxtagvalue/generator.py
@@ -0,0 +1,225 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+#
+"""
+SPDX document generator
+"""
+
+import datetime
+
+from tern.classes.templates.spdx import SPDX
+from tern.utils.general import get_git_rev_or_version
+from tern.report import content
+from tern.report.spdxtagvalue import formats as spdx_formats
+
+
+def get_document_namespace(image_obj):
+    '''Given the image object, return a unique SPDX document uri.
+    This is a combination of the human readable id from the image
+    object and the tool name'''
+    return spdx_formats.document_namespace.format(
+        image_id=image_obj.get_human_readable_id(),
+        version=get_git_rev_or_version()[1])
+
+
+def get_package_spdxref(package_obj):
+    '''Given the package object, return an SPDX reference ID'''
+    return 'SPDXRef-{}'.format(package_obj.get_package_id())
+
+
+def get_layer_spdxref(layer_obj):
+    '''Given the layer object, return an SPDX reference ID'''
+    # here we return the shortened diff_id of the layer
+    return 'SPDXRef-{}'.format(layer_obj.diff_id[:10])
+
+
+def get_image_spdxref(image_obj):
+    '''Given the image object, return an SPDX reference ID'''
+    # here we return the image name, tag and id
+    return 'SPDXRef-{}'.format(image_obj.get_human_readable_id())
+
+
+def get_document_block(image_obj):
+    '''Return document related SPDX tag-values'''
+    block = ''
+    block = spdx_formats.spdx_version + '\n'
+    block = block + spdx_formats.data_license + '\n'
+    block = block + spdx_formats.spdx_id + '\n'
+    block = block + spdx_formats.document_name.format(
+        image_name=image_obj.get_human_readable_id()) + '\n'
+    block = block + get_document_namespace(image_obj) + '\n'
+    block = block + spdx_formats.license_list_version + '\n'
+    block = block + spdx_formats.creator.format(
+        version=get_git_rev_or_version()[1]) + '\n'
+    block = block + spdx_formats.created.format(
+        timestamp=datetime.datetime.utcnow().strftime(
+            "%Y-%m-%dT%H:%M:%SZ")) + '\n'
+    return block
+
+
+def get_package_comment(origins):
+    '''Return a PackageComment tag-value text block for a list of
+    NoticeOrigin objects'''
+    comment = ''
+    if origins:
+        for notice_origin in origins:
+            comment = comment + content.print_notices(
+                notice_origin, '', '\t')
+        return spdx_formats.package_comment.format(comment=comment)
+    return comment
+
+
+def get_main_block(level_dict, origins, **kwargs):
+    '''Given the dictionary for the level, the list of notices and a list of
+    key-value pairs, return the SPDX tag-value information for this level'''
+    block = ''
+    # insert package comment first
+    block = get_package_comment(origins) + '\n'
+    # list image key and values
+    for key, value in level_dict.items():
+        block = block + spdx_formats.tag_value.format(
+            tag=key, value=value if value else 'NOASSERTION') + '\n'
+    # list image specific tag-values
+    for key, value in kwargs.items():
+        block = block + spdx_formats.tag_value.format(
+            tag=key, value=value) + '\n'
+    return block
+
+
+def get_image_relationships(image_obj):
+    '''Given the image object, return the relationships to the layers'''
+    block = ''
+    image_reference = get_image_spdxref(image_obj)
+    for layer in image_obj.layers:
+        block = block + spdx_formats.contains.format(
+            outer=image_reference, inner=get_layer_spdxref(layer)) + '\n'
+    return block
+
+
+def get_layer_relationships(layer_obj, prev_layer_spdxref=None):
+    '''Given the layer object, return the relationships of the layer
+    objects to packages and to the previous layer'''
+    block = ''
+    layer_reference = get_layer_spdxref(layer_obj)
+    if prev_layer_spdxref:
+        block = block + spdx_formats.prereq.format(
+            after=layer_reference, before=prev_layer_spdxref)
+    for package in layer_obj.packages:
+        block = block + spdx_formats.contains.format(
+            outer=layer_reference, inner=get_package_spdxref(package)) + '\n'
+    return block
+
+
+def generate(image_obj):
+    '''Given an Image object, generate an SPDX document
+    The whole document should be stored in a string which can be written
+    to a file using the write_report function in report.py
+    First convert the image object into a dictionary. The dictionary
+    should be in this form:
+        image:{
+          origins: [...]
+          layers: [
+            {origins: [...],
+             packages: [
+               {origins: [...], package1: {...}},
+               {origins: [...], package2: {...}}...]}, ...]}
+    Then convert this into a flat format starting from top to bottom
+    So:
+        ## image
+        List all the tag-values here
+        make a PackageComment: <text> </text>
+
+        ## relationships
+        spdx-ref CONTAINS layer1
+        spdx-ref CONTAINS layer2
+        ...
+
+        ## layer1
+        List all the tag-values here
+        make a PackageComment here
+
+        # relationships
+        spdx-ref CONTAINS package1
+        spdx-ref CONTAINS package2
+        ....
+
+        # layer2
+        tag-values
+        PackageComment
+
+        # relationships
+        spdx-ref HAS_PREREQUISITE layer1
+        spdx-ref CONTAINS package3
+        spdx-ref CONTAINS package4
+
+        ....
+
+        # package1
+        tag-values
+        PackageComment
+
+        # package2
+
+        # package3
+
+        # package4
+
+
+    Everything in Origins can be in a tag-value format as
+    PackageComment: <text> </text>
+
+    For the sake of SPDX, an image is a 'Package' which 'CONTAINS'
+    each layer which is also a 'Package' which 'CONTAINS' the real Package'''
+    report = ''
+    template = SPDX()
+    # The image's PackageDownloadLocation is from a container registry
+    # This includes all the layers but the packages' download location
+    # is unknown if the download_url is blank
+    registry_repotag = image_obj.repotag if hasattr(
+        image_obj, '__repotag') else 'NOASSERTION'
+
+    # first part is the document tag-value
+    # this doesn't change at all
+    report = report + get_document_block(image_obj)
+
+    # this part is the image part and needs
+    # the image object
+    report = report + get_main_block(
+        image_obj.to_dict(template),
+        image_obj.origins.origins,
+        SPDXID=get_image_spdxref(image_obj),
+        PackageDownloadLocation=registry_repotag,
+        FilesAnalyzed='false') + '\n'
+    # Add image relationships
+    report = report + get_image_relationships(image_obj) + '\n'
+
+    # Add the layer part for each layer
+    for index, layer_obj in enumerate(image_obj.layers):
+        # this is the main block for the layer
+        report = report + get_main_block(
+            layer_obj.to_dict(template),
+            layer_obj.origins.origins,
+            SDPXID=get_layer_spdxref(layer_obj),
+            PackageDownloadLocation=registry_repotag,
+            FilesAnalyzed='false') + '\n'
+        # Add layer relationships
+        if index == 0:
+            report = report + get_layer_relationships(layer_obj)
+        else:
+            # block should contain previous layer dependency
+            report = report + get_layer_relationships(
+                layer_obj, get_layer_spdxref(image_obj.layers[index - 1]))
+
+    # Add the package part for each package
+    # There are no relationships to be listed here
+    for layer_obj in image_obj.layers:
+        for package_obj in layer_obj.packages:
+            report = report + get_main_block(
+                package_obj.to_dict(template),
+                package_obj.origins.origins,
+                SPDXID=get_package_spdxref(package_obj),
+                PackageDownloadLocation='NOASSERTION',
+                FilesAnalyzed='false') + '\n'
+    return report
diff --git a/tern/utils/general.py b/tern/utils/general.py
index eb91ced..8d74d52 100644
--- a/tern/utils/general.py
+++ b/tern/utils/general.py
@@ -9,7 +9,6 @@ import re
 import subprocess  # nosec
 from contextlib import contextmanager
 
-from tern.report.formats import commit_version, packaged_version
 from tern import Version
 from tern.utils import constants
 
@@ -77,17 +76,17 @@ def parse_command(command):
 def get_git_rev_or_version():
     '''Assuming we are operating within a git repository, get the SHA
     of the current commit'''
-    command = ['git', 'show', '--format=%H', 'HEAD']
+    command = ['git', 'rev-parse', 'HEAD']
     try:
         output = subprocess.check_output(command)  # nosec
         if isinstance(output, bytes):
             output = output.decode('utf-8')
-
-        output = commit_version.format(commit_sha=output)
+        ver_type = 'commit'
 
     except subprocess.CalledProcessError:
-        output = packaged_version.format(version=Version)
-    return output.split('\n').pop(0)
+        ver_type = 'package'
+        output = Version
+    return ver_type, output.split('\n').pop(0)
 
 
 def prop_names(obj):


Test patch:
diff --git a/tests/test_class_image.py b/tests/test_class_image.py
index d352300..cd7d378 100644
--- a/tests/test_class_image.py
+++ b/tests/test_class_image.py
@@ -36,7 +36,11 @@ class TestClassImage(unittest.TestCase):
     def testLoadImage(self):
         self.assertEqual(self.image2.image_id, '5678efgh')
         self.assertFalse(self.image2.layers)
+        self.assertFalse(self.image2.name)
+        self.assertFalse(self.image2.tag)
         self.image2.load_image()
+        self.assertEqual(self.image2.name, 'testimage')
+        self.assertEqual(self.image2.tag, 'testtag')
         self.assertEqual(len(self.image2.layers), 1)
         self.assertEqual(len(self.image2.layers[0].packages), 2)
 
@@ -70,6 +74,12 @@ class TestClassImage(unittest.TestCase):
         self.assertFalse(dict2['image.layers'][0]['notes'])
         self.assertEqual(len(dict2['image.layers'][0]['layer.packages']), 2)
 
+    def testGetHumanReadableId(self):
+        self.assertEqual(self.image1.get_human_readable_id(), '1234abcd')
+        self.image2.load_image()
+        self.assertEqual(
+            self.image2.get_human_readable_id(), '5678efgh-testimage-testtag')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py
index 256957d..0d3a946 100644
--- a/tests/test_fixtures.py
+++ b/tests/test_fixtures.py
@@ -19,6 +19,8 @@ class TestImage(Image):
 
     def load_image(self):
         l1 = ImageLayer('123abc', 'path/to/tar')
+        self.name = 'testimage'
+        self.tag = 'testtag'
         l1.add_package(Package('p1'))
         l1.add_package(Package('p2'))
         self._layers.append(l1)