Repository: tern-tools/tern

Issue:
Update SPDX format to include file level analysis
**Description**
Update the SPDX report format to include situations where there is file level data. Use the http://13.57.134.254/app/validate/ online tool to validate the generated SPDX document for various container images.

**Background**
This depends on https://github.com/vmware/tern/pull/582 to be merged.

**Super Issues**
#583 


Discussion:

Patch:
diff --git a/tern/classes/docker_image.py b/tern/classes/docker_image.py
index 126cfe5..2ed308d 100644
--- a/tern/classes/docker_image.py
+++ b/tern/classes/docker_image.py
@@ -129,6 +129,11 @@ class DockerImage(Image):
             diff_ids.append(item.split(':').pop())
         return diff_ids
 
+    def get_diff_checksum_type(self, config):
+        '''Get the checksum type that was used to calculate the diff_id
+        of the image'''
+        return config['rootfs']['diff_ids'][0].split(':')[0]
+
     def set_layer_created_by(self):
         '''Docker image history configuration consists of a list of commands
         and indication of whether the command created a filesystem or not.
@@ -156,8 +161,10 @@ class DockerImage(Image):
             self.__history = self.get_image_history(self._config)
             layer_paths = self.get_image_layers(self._manifest)
             layer_diffs = self.get_diff_ids(self._config)
+            checksum_type = self.get_diff_checksum_type(self._config)
             while layer_diffs and layer_paths:
                 layer = ImageLayer(layer_diffs.pop(0), layer_paths.pop(0))
+                layer.set_checksum(checksum_type, layer.diff_id)
                 layer.gen_fs_hash()
                 self._layers.append(layer)
             self.set_layer_created_by()
diff --git a/tern/classes/file_data.py b/tern/classes/file_data.py
index c329f39..eb40b56 100644
--- a/tern/classes/file_data.py
+++ b/tern/classes/file_data.py
@@ -26,6 +26,9 @@ class FileData:
         version: the version of the file under version control. If it came
         with a package the version would be the same as the package version.
         file_type: this is a string describing what type of file this is
+        short_file_type: this is a short string describing what type of file
+        this is. This should be one of the following:
+            SOURCE, BINARY, ARCHIVE, TEXT, OTHER
         licenses: A list of licenses that may be detected or is known
         license_expressions: This is a SPDX term used to describe how one or
         more licenses together should be understood. This list may be left
@@ -34,13 +37,16 @@ class FileData:
         authors: a list of authors if known
         packages: a list of packages where this file could come from
         urls: a list of urls from where this file could come from
-        checksums: a list of tuples of the form (checksum_type, checksum)
+        checksums: a dictionary of the form {<checksum_type>: <checksum>,...}
+        checksum types and checksums are stored in lower case
 
     methods:
         to_dict: returns a dictionary representation of the instance
         set_version: set the version of the file given the version control
         system used
         set_checksum: set the checksum of the file given the checksum type
+        get_checksum: get the checksum that matches the checksum type from
+        the property 'checksums'
         fill: fill data into the object instance from a dictionary'''
     def __init__(self,
                  name,
@@ -51,6 +57,7 @@ class FileData:
         self.__path = path
         self.date = date
         self.__file_type = file_type
+        self.__short_file_type = ''
         self.__checksum_type = ''
         self.__checksum = ''
         self.__version_control = ''
@@ -62,7 +69,7 @@ class FileData:
         self.authors = []
         self.packages = []
         self.urls = []
-        self.__checksums = []
+        self.__checksums = {}
         self.__origins = Origins()
 
     @property
@@ -123,9 +130,20 @@ class FileData:
     def checksums(self):
         return self.__checksums
 
-    @checksums.setter
-    def checksums(self, checksums):
-        self.__checksums = checksums
+    @property
+    def short_file_type(self):
+        return self.__short_file_type
+
+    @short_file_type.setter
+    def short_file_type(self, short_file_type):
+        '''short_file_type should be one of these:
+            SOURCE, BINARY, ARCHIVE, TEXT, OTHER'''
+        allowed_file_types = ('SOURCE', 'BINARY', 'ARCHIVE', 'TEXT', 'OTHER')
+        if short_file_type not in allowed_file_types:
+            raise ValueError(
+                "Incorrect short file type name, should be "
+                "SOURCE, BINARY, ARCHIVE, TEXT or OTHER")
+        self.__short_file_type = short_file_type
 
     @property
     def origins(self):
@@ -144,9 +162,14 @@ class FileData:
         self.__version = version
 
     def add_checksums(self, checksums):
-        '''Add checksum tuples to checksums property'''
-        for checksum in checksums:
-            self.__checksums.append(checksum)
+        '''Add a checksum dictionary to checksums property'''
+        for key, value in checksums.items():
+            self.__checksums[key.lower()] = value.lower()
+
+    def get_checksum(self, hash_type):
+        '''Given a hash type, return the checksum. If the hash type is not
+        available, return None'''
+        return self.__checksums.get(hash_type.lower(), None)
 
     def to_dict(self, template=None):
         '''Return a dictionary version of the FileData object
@@ -189,6 +212,7 @@ class FileData:
             path: <path to file>
             date: <date>
             file_type: <file_type>
+            short_file_type: <short_file_type>
             checksum: <checksum>
             checksum_type: <checksum_type>
             version_control: <version_control>
@@ -223,13 +247,15 @@ class FileData:
         if (self.path == other.path):
             self.date = other.date
             self.file_type = other.file_type
+            if other.short_file_type:
+                self.short_file_type = other.short_file_type
             self.licenses = other.licenses
             self.license_expressions = other.license_expressions
             self.copyrights = other.copyrights
             self.authors = other.authors
             self.packages = other.packages
             self.urls = other.urls
-            self.checksums = other.checksums
+            self.add_checksums(other.checksums)
             # collect notices
             for o in other.origins.origins:
                 for n in o.notices:
diff --git a/tern/classes/image_layer.py b/tern/classes/image_layer.py
index 1bb912a..1dba410 100644
--- a/tern/classes/image_layer.py
+++ b/tern/classes/image_layer.py
@@ -36,7 +36,7 @@ class ImageLayer:
         checksum_type: the digest algorithm used to create the image layer
         checksum
         checksum: the checksum
-        checksums: a list of tuples of the form (checksum_type, checksum)
+        checksums: a dictionary of the form {<checksum_type>: <checksum>}
     methods:
         add_package: adds a package to the layer
         remove_package: removes a package from the layer
@@ -68,7 +68,7 @@ class ImageLayer:
         self.__analyzed_output = ''
         self.__checksum_type = ''
         self.__checksum = ''
-        self.__checksums = []
+        self.__checksums = {}
 
     @property
     def diff_id(self):
@@ -110,10 +110,6 @@ class ImageLayer:
     def checksums(self):
         return self.__checksums
 
-    @checksums.setter
-    def checksums(self, checksums):
-        self.__checksums = checksums
-
     @created_by.setter
     def created_by(self, create_string):
         self.__created_by = create_string
@@ -179,9 +175,9 @@ class ImageLayer:
         self.__checksum = checksum
 
     def add_checksums(self, checksums):
-        '''Add checksum tuples to checksums property'''
-        for checksum in checksums:
-            self.__checksums.append(checksum)
+        '''Add a checksum dictionary to checksums property'''
+        for key, value in checksums.items():
+            self.__checksums[key.lower()] = value.lower()
 
     def add_package(self, package):
         if isinstance(package, Package):
diff --git a/tern/extensions/scancode/executor.py b/tern/extensions/scancode/executor.py
index e0c6e93..fecae61 100644
--- a/tern/extensions/scancode/executor.py
+++ b/tern/extensions/scancode/executor.py
@@ -30,6 +30,21 @@ from tern.utils import rootfs
 logger = logging.getLogger(constants.logger_name)
 
 
+def get_file_type(scancode_file_dict):
+    '''Scancode's file dictionary has a set of keys:
+        is_binary, is_text, is_archive, is_media, is_source, is_script
+    using this set, return a filetype recognized by SPDX'''
+    if scancode_file_dict['is_binary'] == 'true':
+        return 'BINARY'
+    if scancode_file_dict['is_source'] == 'true':
+        return 'SOURCE'
+    if scancode_file_dict['is_text'] == 'true':
+        return 'TEXT'
+    if scancode_file_dict['is_archive'] == 'true':
+        return 'ARCHIVE'
+    return 'OTHER'
+
+
 def collect_layer_data(layer_obj):
     '''Use scancode to collect data from a layer filesystem. This function will
     create a FileData object for every file found. After scanning, it will
@@ -50,13 +65,15 @@ def collect_layer_data(layer_obj):
         # make FileData objects for each result
         data = json.loads(result)
         for f in data['files']:
-            if f['type'] == 'file':
+            if f['type'] == 'file' and f['size'] != 0:
                 # scancode records paths from the target directory onwards
                 # which in tern's case is tern.utils.constants.untar_dir
                 # removing that portion of the file path
                 fspath = f['path'].replace(
                     constants.untar_dir + os.path.sep, '')
                 fd = FileData(f['name'], fspath, f['date'], f['file_type'])
+                fd.short_file_type = get_file_type(f)
+                fd.add_checksums({'sha1': f['sha1'], 'md5': f['md5']})
                 if f['licenses']:
                     fd.licenses = [l['short_name'] for l in f['licenses']]
                 fd.license_expressions = f['license_expressions']
diff --git a/tern/formats/spdx/formats.py b/tern/formats/spdx/formats.py
index 86a9024..1713a5f 100644
--- a/tern/formats/spdx/formats.py
+++ b/tern/formats/spdx/formats.py
@@ -9,7 +9,7 @@ SPDX document formatting
 
 # basic strings
 tag_value = '{tag}: {value}'
-block_text = '<text>{message}</text>'
+block_text = '<text>\n{message}\n</text>'
 
 # document level strings
 spdx_version = 'SPDXVersion: SPDX-2.1'
diff --git a/tern/formats/spdx/spdx.py b/tern/formats/spdx/spdx.py
index 6d1e34d..e72ef9a 100644
--- a/tern/formats/spdx/spdx.py
+++ b/tern/formats/spdx/spdx.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+# Copyright (c) 2019-2020 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: BSD-2-Clause
 
 from tern.classes.template import Template
@@ -12,7 +12,7 @@ class SPDX(Template):
 
     def file_data(self):
         return {'name': 'FileName',
-                'file_type': 'FileType'}
+                'short_file_type': 'FileType'}
 
     def package(self):
         return {'name': 'PackageName',
@@ -22,10 +22,7 @@ class SPDX(Template):
                 'download_url': 'PackageDownloadLocation'}
 
     def image_layer(self):
-        # TODO: hash_type should be added in the class property
-        # not hardcoded here
-        return {'diff_id': 'PackageName',
-                'fs_hash': 'PackageChecksum: SHA256'}
+        return {'tar_file': 'PackageFileName'}
 
     def image(self):
         return {'name': 'PackageName',
diff --git a/tern/formats/spdx/spdxtagvalue/file_helpers.py b/tern/formats/spdx/spdxtagvalue/file_helpers.py
new file mode 100644
index 0000000..43693bd
--- /dev/null
+++ b/tern/formats/spdx/spdxtagvalue/file_helpers.py
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+"""
+File level helpers for SPDX tag-value document generator
+"""
+import hashlib
+
+
+# basic functions
+def get_file_spdxref(filedata, layer_id):
+    '''Given a FileData object, return a unique identifier for the SPDX
+    document. According to the spec, this should be of the form: SPDXRef-<id>
+    We will use a combination of the file name, checksum and layer_id and
+    calculate a hash of this string'''
+    file_string = filedata.path + filedata.checksum[:7] + layer_id
+    fileid = hashlib.sha256(file_string.encode('utf-8')).hexdigest()[-7:]
+    return 'SPDXRef-{}'.format(fileid)
+
+
+def get_file_checksum(filedata):
+    '''Given a FileData object, return the checksum required by SPDX.
+    This should be of the form: <checksum_type>: <checksum>'''
+    return '{}: {}'.format(filedata.checksum_type.upper(), filedata.checksum)
+
+
+def get_file_comment(filedata):
+    '''Return a formatted comment string with all file level notices. Return
+    an empty string if no notices are present'''
+    comment = ''
+    for origin in filedata.origins.origins:
+        comment = comment + '{}:'.format(origin.origin_str) + '\n'
+        for notice in origin.notices:
+            comment = comment + \
+                '{}: {}'.format(notice.level, notice.message) + '\n'
+    return comment
+
+
+def get_file_notice(filedata):
+    '''Return a formatted string with all copyrights found in a file. Return
+    an empty string if there are no copyrights'''
+    notice = ''
+    for cp in filedata.copyrights:
+        notice = notice + cp + '\n'
+    return notice
+
+
+# formatting functions
+def get_license_info_block(filedata):
+    '''The SPDX spec asks to list the license expressions found in a file
+    using the format: LicenseInfoInFile: <license expression>. If the license
+    expressions list is empty, this should be "NONE"'''
+    block = ''
+    if not filedata.license_expressions:
+        block = 'LicenseInfoInFile: NONE\n'
+    else:
+        for le in filedata.license_expressions:
+            block = block + 'LicenseInfoInFile: {}'.format(le) + '\n'
+    return block
+
+
+def get_file_contributor_block(filedata):
+    '''The SPDX spec allows for an optional block listing file contributors.
+    If there are any authors found in the file, return a formatted SPDX text
+    block with the list of authors. If empty, return an empty string'''
+    block = ''
+    for author in filedata.authors:
+        block = block + 'FileContributor: {}'.format(author) + '\n'
+    return block
+
+
+# full file block
+def get_file_block(filedata, template, layer_id):
+    '''Given a FileData object, and the SPDX template mapping, return a SPDX
+    document block for the file. The mapping should have only FileName and
+    FileType keys. A layer id is used to distinguish copies of the
+    same file occuring in different places in the image'''
+    block = ''
+    mapping = filedata.to_dict(template)
+    # File Name
+    block = block + 'FileName: {}'.format(mapping['FileName']) + '\n'
+    # SPDX ID
+    block = block + 'SPDXID: {}'.format(
+        get_file_spdxref(filedata, layer_id)) + '\n'
+    # File Type
+    block = block + 'FileType: {}'.format(mapping['FileType']) + '\n'
+    # File checksum
+    block = block + 'FileChecksum: {}'.format(get_file_checksum(filedata)) + \
+        '\n'
+    # Concluded license - we won't provide this
+    block = block + 'LicenseConcluded: NOASSERTION' + '\n'
+    # License info in file
+    block = block + get_license_info_block(filedata)
+    # File copyright text - we don't know this
+    block = block + 'FileCopyrightText: NOASSERTION' + '\n'
+    # File comment - we add this only if there is a comment
+    comment = get_file_comment(filedata)
+    if comment:
+        block = block + 'FileComment: <text>\n' + comment + '</text>' + '\n'
+    # File Notice - we add this only if there is a notice
+    notice = get_file_notice(filedata)
+    if notice:
+        block = block + 'FileNotice: <text>\n' + notice + '</text>' + '\n'
+    # File Contributor - we add this only if there are contributors
+    contributors = get_file_contributor_block(filedata)
+    if contributors:
+        block = block + contributors
+    return block
diff --git a/tern/formats/spdx/spdxtagvalue/generator.py b/tern/formats/spdx/spdxtagvalue/generator.py
index 21a9832..6061eff 100755
--- a/tern/formats/spdx/spdxtagvalue/generator.py
+++ b/tern/formats/spdx/spdxtagvalue/generator.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+# Copyright (c) 2019-2020 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: BSD-2-Clause
 
 """
@@ -8,15 +8,20 @@ SPDX document generator
 """
 
 import datetime
-import hashlib
+import logging
 
 from tern.formats.spdx.spdx import SPDX
 from tern.utils.general import get_git_rev_or_version
-from tern.report import content
+from tern.utils import constants
 from tern.formats.spdx import formats as spdx_formats
+from tern.formats.spdx.spdxtagvalue import image_helpers as mhelpers
 from tern.formats import generator
 
 
+# global logger
+logger = logging.getLogger(constants.logger_name)
+
+
 def get_document_namespace(image_obj):
     '''Given the image object, return a unique SPDX document uri.
     This is a combination of the human readable id from the image
@@ -26,26 +31,6 @@ def get_document_namespace(image_obj):
         version=get_git_rev_or_version()[1])
 
 
-def get_package_spdxref(package_obj):
-    '''Given the package object, return an SPDX reference ID'''
-    return 'SPDXRef-{}'.format(
-        spdx_formats.package_id.format(
-            name=package_obj.name,
-            ver=package_obj.version).replace(':', '-', 1))
-
-
-def get_layer_spdxref(layer_obj):
-    '''Given the layer object, return an SPDX reference ID'''
-    # here we return the shortened diff_id of the layer
-    return 'SPDXRef-{}'.format(layer_obj.diff_id[:10])
-
-
-def get_image_spdxref(image_obj):
-    '''Given the image object, return an SPDX reference ID'''
-    # here we return the image name, tag and id
-    return 'SPDXRef-{}'.format(image_obj.get_human_readable_id())
-
-
 def get_document_block(image_obj):
     '''Return document related SPDX tag-values'''
     block = spdx_formats.spdx_version + '\n'
@@ -64,80 +49,6 @@ def get_document_block(image_obj):
     return block
 
 
-def get_package_comment(origins):
-    '''Return a PackageComment tag-value text block for a list of
-    NoticeOrigin objects'''
-    comment = ''
-    if origins:
-        for notice_origin in origins:
-            comment = comment + content.print_notices(
-                notice_origin, '', '\t')
-        return spdx_formats.package_comment.format(comment=comment)
-    return comment
-
-
-def get_main_block(level_dict, origins, **kwargs):
-    '''Given the dictionary for the level, the list of notices and a list of
-    key-value pairs, return the SPDX tag-value information for this level'''
-    block = ''
-    for key, value in level_dict.items():
-        block = block + spdx_formats.tag_value.format(
-            tag=key, value=value if value else 'NOASSERTION') + '\n'
-    # list specifically defined tag-values
-    for key, value in kwargs.items():
-        block = block + spdx_formats.tag_value.format(
-            tag=key, value=value) + '\n'
-    block = block + get_package_comment(origins) + '\n'
-    return block
-
-
-def get_image_relationships(image_obj):
-    '''Given the image object, return the relationships to the layers'''
-    block = ''
-    image_reference = get_image_spdxref(image_obj)
-    for layer in image_obj.layers:
-        block = block + spdx_formats.contains.format(
-            outer=image_reference, inner=get_layer_spdxref(layer)) + '\n'
-    return block
-
-
-def get_layer_relationships(layer_obj, prev_layer_spdxref=None):
-    '''Given the layer object, return the relationships of the layer
-    objects to packages and to the previous layer'''
-    block = ''
-    layer_reference = get_layer_spdxref(layer_obj)
-    if prev_layer_spdxref:
-        block = block + spdx_formats.prereq.format(
-            after=layer_reference, before=prev_layer_spdxref) + '\n'
-    for package in layer_obj.packages:
-        block = block + spdx_formats.contains.format(
-            outer=layer_reference, inner=get_package_spdxref(package)) + '\n'
-    return block
-
-
-def get_license_ref(pkg_license):
-    '''Given one license, return a LicenseRef with a unique SHA-256 ID'''
-    return 'LicenseRef-' + hashlib.sha256(pkg_license.encode(
-        'utf-8')).hexdigest()[-7:]
-
-
-def get_license_block(license_list):
-    '''Given a list of individual licenses, return a LicenseRef block of text
-    this is of the format:
-        ## License Information
-        LicenseID: LicenseRef-MIT
-        ExtractedText: <text> </text>'''
-    # make a list of individual licenses
-    block = ''
-    for l in license_list:
-        block = block + spdx_formats.license_id.format(
-            license_ref=get_license_ref(l) if l
-            else 'NOASSERTION') + '\n'
-        block = block + spdx_formats.extracted_text.format(
-            orig_license=l if l else 'NOASSERTION') + '\n\n'
-    return block
-
-
 class SpdxTagValue(generator.Generate):
     def generate(self, image_obj_list):
         '''Generate an SPDX document
@@ -154,8 +65,12 @@ class SpdxTagValue(generator.Generate):
               layers: [
                 {origins: [...],
                  packages: [
-                   {origins: [...], package1: {...}},
-                   {origins: [...], package2: {...}}...]}, ...]}
+                   {name: package1,..origins: [...]},
+                   {name: package2,..origins: [...]},..],
+                 files: [
+                   {name: file1,..origins: [...]},
+                   {name: file2,..origins: [...]},..]}
+                   ...]}
         Then convert this into a flat format starting from top to bottom
         So:
             ## image
@@ -171,7 +86,11 @@ class SpdxTagValue(generator.Generate):
             List all the tag-values here
             make a PackageComment here
 
-            # relationships
+            ## if layer1 has files analyzed
+            ### extra package info here
+            ### file level information here
+
+            ## if not then package relationships
             spdx-ref CONTAINS package1
             spdx-ref CONTAINS package2
             ....
@@ -203,73 +122,21 @@ class SpdxTagValue(generator.Generate):
 
         For the sake of SPDX, an image is a 'Package' which 'CONTAINS' each
         layer which is also a 'Package' which 'CONTAINS' the real Package'''
+        logger.debug("Generating SPDX document...")
         report = ''
-        licenses_found = []  # This is needed for unrecognized license strings
+
+        # we still don't know how SPDX documents could represent multiple
+        # images. Hence we will assume only one image is analyzed and the
+        # input is a list of length 1
         image_obj = image_obj_list[0]
         template = SPDX()
 
         # first part is the document tag-value
         # this doesn't change at all
-        report = report + get_document_block(image_obj) + '\n'
-
-        # this part is the image part and needs
-        # the image object
-        report = report + get_main_block(
-            image_obj.to_dict(template),
-            image_obj.origins.origins,
-            SPDXID=get_image_spdxref(image_obj),
-            PackageLicenseDeclared='NOASSERTION',
-            PackageLicenseConcluded='NOASSERTION',
-            PackageCopyrightText='NOASSERTION',
-            FilesAnalyzed='false') + '\n'
-        # Add image relationships
-        report = report + get_image_relationships(image_obj) + '\n'
+        report += get_document_block(image_obj) + '\n'
 
-        # Add the layer part for each layer
-        for index, layer_obj in enumerate(image_obj.layers):
-            # this is the main block for the layer
-            report = report + get_main_block(
-                layer_obj.to_dict(template),
-                layer_obj.origins.origins,
-                SPDXID=get_layer_spdxref(layer_obj),
-                PackageDownloadLocation='NOASSERTION',
-                PackageLicenseDeclared='NOASSERTION',
-                PackageLicenseConcluded='NOASSERTION',
-                PackageCopyrightText='NOASSERTION',
-                FilesAnalyzed='false') + '\n'
-            # Add layer relationships
-            if index == 0:
-                report = report + get_layer_relationships(layer_obj) + '\n'
-            else:
-                # block should contain previous layer dependency
-                report = report + get_layer_relationships(
-                    layer_obj, get_layer_spdxref(image_obj.layers[index - 1]))\
-                    + '\n'
+        # this is the image part
+        # this will bring in layer and package information
+        report += mhelpers.get_image_block(image_obj, template) + '\n'
 
-        # Add the package part for each package
-        # There are no relationships to be listed here
-        for layer_obj in image_obj.layers:
-            for package_obj in layer_obj.packages:
-                package_dict = package_obj.to_dict(template)
-                # update the PackageLicenseDeclared with a LicenseRef string
-                # only if the license data exists
-                if ('PackageLicenseDeclared' in package_dict.keys() and
-                        package_obj.pkg_license):
-                    package_dict['PackageLicenseDeclared'] = \
-                        get_license_ref(package_obj.pkg_license)
-                if ('PackageCopyrightText' in package_dict.keys() and
-                        package_obj.copyright):
-                    package_dict['PackageCopyrightText'] = \
-                        spdx_formats.block_text.format(
-                            message=package_obj.copyright)
-                # collect all the individual licenses
-                if package_obj.pkg_license and package_obj.pkg_license \
-                        not in licenses_found:
-                    licenses_found.append(package_obj.pkg_license)
-                report = report + get_main_block(
-                    package_dict,
-                    package_obj.origins.origins,
-                    SPDXID=get_package_spdxref(package_obj),
-                    PackageLicenseConcluded='NOASSERTION',
-                    FilesAnalyzed='false') + '\n'
-        return report + get_license_block(licenses_found)
+        return report
diff --git a/tern/formats/spdx/spdxtagvalue/image_helpers.py b/tern/formats/spdx/spdxtagvalue/image_helpers.py
new file mode 100644
index 0000000..e686695
--- /dev/null
+++ b/tern/formats/spdx/spdxtagvalue/image_helpers.py
@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+"""
+Helper functions for image level SPDX document blocks
+Images for SPDX act like a Package
+"""
+from tern.formats.spdx import formats as spdx_formats
+from tern.formats.spdx.spdxtagvalue import layer_helpers as lhelpers
+from tern.formats.spdx.spdxtagvalue import package_helpers as phelpers
+
+
+def get_image_spdxref(image_obj):
+    '''Given the image object, return an SPDX reference ID'''
+    # here we return the image name, tag and id
+    return 'SPDXRef-{}'.format(image_obj.get_human_readable_id())
+
+
+def get_image_layer_relationships(image_obj):
+    '''Given the image object, return the relationships to the layers'''
+    block = ''
+    image_reference = get_image_spdxref(image_obj)
+    for layer in image_obj.layers:
+        block = block + spdx_formats.contains.format(
+            outer=image_reference,
+            inner=lhelpers.get_layer_spdxref(layer)) + '\n'
+    return block
+
+
+def get_image_packages_block(image_obj, template):
+    '''Given the image object and its template, return the list of packages
+    in the image in SPDX format. The spec requires unique package references
+    to identify each package found in the image.'''
+    block = ''
+    package_refs = set()
+    for layer in image_obj.layers:
+        for package in layer.packages:
+            pkg_ref = phelpers.get_package_spdxref(package)
+            if pkg_ref not in package_refs:
+                block += phelpers.get_package_block(package, template) + '\n'
+                package_refs.add(pkg_ref)
+    return block
+
+
+def get_image_packages_license_block(image_obj):
+    '''Given the image object, get all the the licenses found for packages
+    in the image. The SPDX spec requires that each license reference be
+    unique for the document'''
+    block = ''
+    licenses = set()
+    for layer in image_obj.layers:
+        for package in layer.packages:
+            if package.pkg_license:
+                licenses.add(package.pkg_license)
+    for l in licenses:
+        block += spdx_formats.license_id.format(
+            license_ref=phelpers.get_package_license_ref(l)) + '\n'
+        block += spdx_formats.extracted_text.format(orig_license=l) + '\n'
+    return block
+
+
+def get_image_block(image_obj, template):
+    '''Given an image object and the template object for SPDX, return the
+    SPDX document block for the given image. For SPDX, the image is a package
+    and hence follows the spec for packages.
+    The mapping for images should have these keys:
+        PackageName
+        PackageVersion
+        PackageDownloadLocation'''
+    block = ''
+    mapping = image_obj.to_dict(template)
+    # Package Name
+    block += 'PackageName: {}\n'.format(mapping['PackageName'])
+    # Package SPDXID
+    block += 'SPDXID: {}\n'.format(get_image_spdxref(image_obj))
+    # Package Version
+    block += 'PackageVersion: {}\n'.format(mapping['PackageVersion'])
+    # Package Download Location
+    block += 'PackageDownloadLocation: {}\n'.format(
+        mapping['PackageDownloadLocation'])
+    # Files Analyzed (always false)
+    block += 'FilesAnalyzed: false\n'
+    # Concluded Package License (always NOASSERTION)
+    block += 'PackageLicenseConcluded: NOASSERTION\n'
+    # Declared Package License (always NOASSERTION)
+    block += 'PackageLicenseDeclared: NOASSERTION\n'
+    # Package Copyright Text (always NOASSERTION)
+    block += 'PackageCopyrightText: NOASSERTION\n'
+    # blank line
+    block += '\n'
+    # Since files are not analyzed within the image we move to relationships
+    block += get_image_layer_relationships(image_obj) + '\n'
+    # blank line
+    block += '\n'
+    # Describe each layer 'package' that the image contains
+    for index, layer in enumerate(image_obj.layers):
+        block += lhelpers.get_layer_block(
+            layer, template, mapping['PackageDownloadLocation']) + '\n'
+        # print layer relationship to previous layer if there is one
+        if index != 0:
+            block += lhelpers.get_layer_prereq(
+                image_obj.layers[index], image_obj.layers[index - 1]) + '\n'
+        # if the layer has packages, print out the relationships
+        if layer.packages:
+            block += lhelpers.get_layer_package_relationships(layer) + '\n'
+    # print out all the packages if they are known
+    pkg_block = get_image_packages_block(image_obj, template)
+    if pkg_block:
+        # add a blank line before adding the package block
+        block += pkg_block + '\n'
+        # print out the license block for packages
+        block += get_image_packages_license_block(image_obj)
+    return block
diff --git a/tern/formats/spdx/spdxtagvalue/layer_helpers.py b/tern/formats/spdx/spdxtagvalue/layer_helpers.py
new file mode 100644
index 0000000..55f05ff
--- /dev/null
+++ b/tern/formats/spdx/spdxtagvalue/layer_helpers.py
@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+"""
+Helpers for layer information
+Layers for SPDX act like a Package
+"""
+import hashlib
+import logging
+import os
+
+from tern.formats.spdx import formats as spdx_formats
+from tern.utils import constants
+from tern.formats.spdx.spdxtagvalue import file_helpers as fhelpers
+from tern.formats.spdx.spdxtagvalue import package_helpers as phelpers
+
+# global logger
+logger = logging.getLogger(constants.logger_name)
+
+
+def get_layer_spdxref(layer_obj):
+    '''Given the layer object, return an SPDX reference ID'''
+    # here we return the shortened diff_id of the layer
+    return 'SPDXRef-{}'.format(layer_obj.diff_id[:10])
+
+
+def get_layer_package_relationships(layer_obj):
+    '''Given a layer object, return the relationships of the layer
+    objects to packages. This is usually of the form:
+        layer SPDXIP CONTAINS package SPDXID'''
+    block = ''
+    layer_reference = get_layer_spdxref(layer_obj)
+    for package in layer_obj.packages:
+        block = block + spdx_formats.contains.format(
+            outer=layer_reference,
+            inner=phelpers.get_package_spdxref(package)) + '\n'
+    return block
+
+
+def get_layer_prereq(curr_layer, prev_layer):
+    '''Given the current layer and the previous layer, return the relationship
+    between the current and previous layer. This should look like this:
+        curr_layer HAS_PREREQUISITE prev_layer'''
+    return spdx_formats.prereq.format(
+        after=get_layer_spdxref(curr_layer),
+        before=get_layer_spdxref(prev_layer))
+
+
+def get_layer_verification_code(layer_obj):
+    '''Calculate the verification code from the files in an image layer. This
+    assumes that layer_obj.files_analyzed is True. The implementation follows
+    the algorithm in the SPDX spec v 2.1 which requires SHA1 to be used to
+    calculate the checksums of the file and the final verification code'''
+    sha1_list = []
+    for filedata in layer_obj.files:
+        filesha = filedata.get_checksum('sha1')
+        if not filesha:
+            # we cannot create a verification code, hence file generation
+            # is aborted
+            logger.critical(
+                'File %s does not have a sha1 checksum. Failed to generate '
+                'a SPDX tag-value report', filedata.path)
+            return None
+        sha1_list.append(filesha)
+    sha1_list.sort()
+    sha1s = ''.join(sha1_list)
+    return hashlib.sha1(sha1s.encode('utf-8')).hexdigest()  # nosec
+
+
+def get_layer_checksum(layer_obj):
+    '''Return a SPDX formatted checksum value. It should be of the form:
+        checksum_type: <checksum>'''
+    return '{}: {}'.format(layer_obj.checksum_type.upper(), layer_obj.checksum)
+
+
+def get_file_license_expressions(layer_obj):
+    '''Return a list of unique license expressions from the files analyzed
+    in the layer object. It is assumed that the files were analyzed and
+    there should be some license expressions. If there are not, an empty list
+    is returned'''
+    license_expressions = set()
+    for filedata in layer_obj.files:
+        if filedata.license_expressions:
+            for le in filedata.license_expressions:
+                license_expressions.add(le)
+    return list(license_expressions)
+
+
+def get_layer_block(layer_obj, template, image_loc=''):
+    '''Given a layer object and its SPDX template mapping, return a SPDX
+    document block for the layer. An image layer in SPDX behaves like a
+    Package with relationships to the Packages within it. If the files
+    are analyzed though, we just list the files in the block. The mapping
+    should have keys:
+        PackageFileName
+    We also pass the image location as optional for where the layers were
+    downloaded from. Registries can be implemented as distributed storage
+    or images can be stored as whole tarballs. Either way, the layers
+    would be downloaded along with the image.'''
+    block = ''
+    mapping = layer_obj.to_dict(template)
+    layer_checksum = get_layer_checksum(layer_obj)
+    # Package Name
+    block += 'PackageName: {}\n'.format(os.path.basename(layer_obj.tar_file))
+    # Package SPDXID
+    block += 'SPDXID: {}\n'.format(get_layer_spdxref(layer_obj))
+    # Package File Name
+    block += 'PackageFileName: {}\n'.format(layer_obj.tar_file)
+    # Package Download Location
+    if image_loc:
+        block += 'PackageDownloadLocation: {}\n'.format(
+            mapping['PackageFileName'])
+    else:
+        block += 'PackageDownloadLocation: NONE\n'
+    # Files Analyzed
+    if layer_obj.files_analyzed:
+        # we need a package verification code
+        block += 'FilesAnalyzed: true\n'
+        block += 'PackageVerificationCode: {}\n'.format(
+            get_layer_verification_code(layer_obj))
+    else:
+        block += 'FilesAnalyzed: false\n'
+    # Package Checksum
+    block += 'PackageChecksum: {}\n'.format(layer_checksum)
+    # Package License Concluded (always NOASSERTION)
+    block += 'PackageLicenseConcluded: NOASSERTION\n'
+    # All licenses info from files
+    if layer_obj.files_analyzed:
+        license_expressions = get_file_license_expressions(layer_obj)
+        if license_expressions:
+            for le in license_expressions:
+                block += 'PackageLicenseInfoFromFiles: {}\n'.format(le)
+        else:
+            block += 'PackageLicenseInfoFromFiles: NONE\n'
+    # Package License Declared (always NOASSERTION)
+    block += 'PackageLicenseDeclared: NOASSERTION\n'
+    # Package Copyright (always NOASSERTION)
+    block += 'PackageCopyrightText: NOASSERTION\n'
+    # put the file data here if files_analyzed is true
+    if layer_obj.files_analyzed:
+        # blank new line
+        block += '\n'
+        # some files are located in different places in the filesystem
+        # they would occur as duplicates in this block
+        # keep a list of previously printed file spdx-refs
+        file_refs = set()
+        # file data
+        for filedata in layer_obj.files:
+            # we use the layer checksum as the layer id
+            file_ref = fhelpers.get_file_spdxref(filedata, layer_checksum)
+            if file_ref not in file_refs:
+                block += fhelpers.get_file_block(
+                    filedata, template, layer_checksum) + '\n'
+                file_refs.add(file_ref)
+    return block
diff --git a/tern/formats/spdx/spdxtagvalue/package_helpers.py b/tern/formats/spdx/spdxtagvalue/package_helpers.py
new file mode 100644
index 0000000..937f633
--- /dev/null
+++ b/tern/formats/spdx/spdxtagvalue/package_helpers.py
@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+"""
+Helper functions for packages in SPDX document
+"""
+
+import hashlib
+
+from tern.formats.spdx import formats as spdx_formats
+from tern.report import content
+
+
+def get_package_spdxref(package_obj):
+    '''Given the package object, return an SPDX reference ID'''
+    return 'SPDXRef-{}'.format(
+        spdx_formats.package_id.format(
+            name=package_obj.name,
+            ver=package_obj.version).replace(':', '-', 1))
+
+
+def get_package_comment(package_obj):
+    '''Return a PackageComment tag-value text block for a list of
+    NoticeOrigin objects'''
+    comment = ''
+    if package_obj.origins.origins:
+        for notice_origin in package_obj.origins.origins:
+            comment = comment + content.print_notices(
+                notice_origin, '', '\t')
+        return spdx_formats.package_comment.format(comment=comment)
+    return comment
+
+
+def get_package_license_ref(package_license):
+    '''Return a LicenseRef with a unique SHA-256 ID for the package object
+    if it exists.'''
+    return 'LicenseRef-' + hashlib.sha256(package_license.encode(
+        'utf-8')).hexdigest()[-7:]
+
+
+def get_package_block(package_obj, template):
+    '''Given a package object and its SPDX template mapping, return a SPDX
+    document block for the package. The mapping should have keys:
+        PackageName
+        PackageVersion
+        PackageLicenseDeclared
+        PackageCopyrightText
+        PackageDownloadLocation'''
+    block = ''
+    mapping = package_obj.to_dict(template)
+    # Package Name
+    block += 'PackageName: {}\n'.format(mapping['PackageName'])
+    # SPDXID
+    block += 'SPDXID: {}\n'.format(get_package_spdxref(package_obj))
+    # Package Version
+    if mapping['PackageVersion']:
+        block += 'PackageVersion: {}\n'.format(mapping['PackageVersion'])
+    # Package Download Location
+    if mapping['PackageDownloadLocation']:
+        block += 'PackageDownloadLoaction: {}\n'.format(
+            mapping['PackageDownloadLocation'])
+    else:
+        block += 'PackageDownloadLocation: NONE\n'
+    # Files Analyzed (always false for packages)
+    block += 'FilesAnalyzed: false\n'
+    # Package License Concluded (always NOASSERTION)
+    block += 'PackageLicenseConcluded: NOASSERTION\n'
+    # Package License Declared (use the license ref for this)
+    if mapping['PackageLicenseDeclared']:
+        block += 'PackageLicenseDeclared: {}\n'.format(
+            get_package_license_ref(mapping['PackageLicenseDeclared']))
+    else:
+        block += 'PackageLicenseDeclared: NONE\n'
+    # Package Copyright Text
+    if mapping['PackageCopyrightText']:
+        block += 'PackageCopyrightText:' + spdx_formats.block_text.format(
+            message=mapping['PackageCopyrightText']) + '\n'
+    else:
+        block += 'PackageCopyrightText: NONE\n'
+    # Package Comments
+    block += get_package_comment(package_obj)
+    return block
diff --git a/tern/tools/fs_hash.sh b/tern/tools/fs_hash.sh
index 5fefaaa..6f4ef58 100755
--- a/tern/tools/fs_hash.sh
+++ b/tern/tools/fs_hash.sh
@@ -20,5 +20,5 @@ command -v getfattr || { echo "'getfattr' not found on system." >&2 ; exit 1; }
 
 cwd=`pwd`
 cd $1
-find -type f -printf "%M|%U|%G|%s|%n|" -exec sha256sum {} \; -exec getfattr -d -m - {} \;
+find -type f ! -size 0 -printf "%M|%U|%G|%s|%n|" -exec sha256sum {} \; -exec getfattr -d -m - {} \;
 cd $cwd


Test patch:
diff --git a/tests/test_class_docker_image.py b/tests/test_class_docker_image.py
index f5d1f6d..dc7b6b7 100644
--- a/tests/test_class_docker_image.py
+++ b/tests/test_class_docker_image.py
@@ -86,6 +86,8 @@ class TestClassDockerImage(unittest.TestCase):
         self.assertEqual(self.image.layers[0].diff_id, self.layer)
         self.assertEqual(len(self.image.layers), self.no_layers)
         self.assertEqual(self.image.layers[0].created_by, self.created_by)
+        self.assertEqual(self.image.layers[0].checksum_type, 'sha256')
+        self.assertEqual(self.image.layers[0].checksum, self.layer)
 
     def testGetImageOption(self):
         self.assertEqual(self.image.get_image_option(), self.image.repotag)
diff --git a/tests/test_class_file_data.py b/tests/test_class_file_data.py
index 4fecbf6..cc4c519 100644
--- a/tests/test_class_file_data.py
+++ b/tests/test_class_file_data.py
@@ -31,6 +31,7 @@ class TestClassFileData(unittest.TestCase):
         self.assertFalse(file1.version_control)
         self.assertFalse(file1.version)
         self.assertFalse(file1.file_type)
+        self.assertFalse(file1.short_file_type)
         self.assertFalse(file1.licenses)
         self.assertFalse(file1.license_expressions)
         self.assertFalse(file1.copyrights)
@@ -44,6 +45,10 @@ class TestClassFileData(unittest.TestCase):
                              '12355')
         file1.file_type = 'ELF'
         self.assertEqual(file1.file_type, 'ELF')
+        with self.assertRaises(ValueError):
+            file1.short_file_type = 'SOMETHING'
+        file1.short_file_type = 'BINARY'
+        self.assertEqual(file1.short_file_type, 'BINARY')
         file2 = FileData('file2',
                          'path/to/file2',
                          '2020-01-01',
@@ -70,11 +75,19 @@ class TestClassFileData(unittest.TestCase):
 
     def testAddChecksums(self):
         file1 = FileData('file1', 'path/to/file1')
-        file1.add_checksums([('SHA1', '12345abcde'),
-                             ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')])
+        file1.add_checksums({'SHA1': '12345abcde',
+                             'MD5': '1ff38cc592c4c5d0c8e3ca38be8f1eb1'})
         self.assertEqual(file1.checksums,
-                         [('SHA1', '12345abcde'),
-                          ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')])
+                         {'sha1': '12345abcde',
+                          'md5': '1ff38cc592c4c5d0c8e3ca38be8f1eb1'})
+
+    def testGetChecksum(self):
+        self.afile.add_checksums({'SHA1': '12345abcde',
+                                  'MD5': '1ff38cc592c4c5d0c8e3ca38be8f1eb1'})
+        self.assertEqual(self.afile.get_checksum('sha1'), '12345abcde')
+        self.assertEqual(self.afile.get_checksum('MD5'),
+                         '1ff38cc592c4c5d0c8e3ca38be8f1eb1')
+        self.assertIsNone(self.afile.get_checksum('sha256'))
 
     def testExtAttrs(self):
         file = FileData('test.txt', 'usr/abc/test.txt')
@@ -112,8 +125,8 @@ class TestClassFileData(unittest.TestCase):
                         '4ceeb204f7c3b6599834c7319541',
             'extattrs': '-rw-r--r-- 1 1000 1000 16262 Nov 13 17:57'
                         ' /usr/include/zconf.h',
-            'checksums': [('SHA1', '12345abcde'),
-                          ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')]
+            'checksums': {'sha1': '12345abcde',
+                          'md5': '1ff38cc592c4c5d0c8e3ca38be8f1eb1'}
         }
         f = FileData('zconf.h', '/usr/include/zconf.h')
         f.fill(file_dict)
@@ -125,28 +138,28 @@ class TestClassFileData(unittest.TestCase):
         self.assertEqual(f.extattrs, '-rw-r--r-- 1 1000 1000 '
                                      '16262 Nov 13 17:57 /usr/include/zconf.h')
         self.assertEqual(f.checksums,
-                         [('SHA1', '12345abcde'),
-                          ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')]
-                         )
+                         {'sha1': '12345abcde',
+                          'md5': '1ff38cc592c4c5d0c8e3ca38be8f1eb1'})
+        self.assertEqual(f.origins.origins[0].notices[0].message,
+                         'No metadata for key: date')
         self.assertEqual(f.origins.origins[0].notices[1].message,
                          'No metadata for key: file_type')
         self.assertEqual(f.origins.origins[0].notices[1].level, 'warning')
-        self.assertEqual(f.origins.origins[0].notices[0].message,
-                         'No metadata for key: date')
         self.assertEqual(f.origins.origins[0].notices[2].message,
-                         'No metadata for key: version_control')
+                         'No metadata for key: short_file_type')
 
     def testMerge(self):
         file1 = FileData('switch_root', 'sbin/switch_root')
         file1.set_checksum('sha256', '123abc456def')
         file1.extattrs = '-rwxr-xr-x|1000|1000|14408|1'
         file2 = FileData('switch_root', 'sbin/switch_root')
-        file2.checksums = [('SHA1', '12345abcde'),
-                           ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')]
+        file2.add_checksums({'SHA1': '12345abcde',
+                             'MD5': '1ff38cc592c4c5d0c8e3ca38be8f1eb1'})
         file2.set_checksum('sha256', '123abc456def')
         file2.extattrs = '-rwxr-xr-x|1000|1000|14408|1'
         file2.date = '2012-02-02'
         file2.file_type = 'binary'
+        file2.short_file_type = 'BINARY'
         file2.licenses = ['MIT', 'GPL']
         file2.license_expressions = ['MIT or GPL']
         file2.copyrights = ['copyrights']
@@ -165,10 +178,11 @@ class TestClassFileData(unittest.TestCase):
         self.assertFalse(file1.merge('astring'))
         self.assertTrue(file1.merge(file2))
         self.assertEqual(file1.checksums,
-                         [('SHA1', '12345abcde'),
-                          ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')])
+                         {'sha1': '12345abcde',
+                          'md5': '1ff38cc592c4c5d0c8e3ca38be8f1eb1'})
         self.assertEqual(file1.date, '2012-02-02')
         self.assertEqual(file1.file_type, 'binary')
+        self.assertEqual(file1.short_file_type, 'BINARY')
         self.assertEqual(file1.licenses, ['MIT', 'GPL'])
         self.assertEqual(file1.license_expressions, ['MIT or GPL'])
         self.assertEqual(file1.copyrights, ['copyrights'])
diff --git a/tests/test_class_image_layer.py b/tests/test_class_image_layer.py
index 5b5d288..30a19a0 100644
--- a/tests/test_class_image_layer.py
+++ b/tests/test_class_image_layer.py
@@ -130,11 +130,11 @@ class TestClassImageLayer(unittest.TestCase):
         self.assertEqual(self.layer.checksum, '12345abcde')
 
     def testAddChecksums(self):
-        self.layer.add_checksums([('SHA1', '12345abcde'),
-                                  ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')])
+        self.layer.add_checksums({'SHA1': '12345abcde',
+                                  'MD5': '1ff38cc592c4c5d0c8e3ca38be8f1eb1'})
         self.assertEqual(self.layer.checksums,
-                         [('SHA1', '12345abcde'),
-                          ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')])
+                         {'sha1': '12345abcde',
+                          'md5': '1ff38cc592c4c5d0c8e3ca38be8f1eb1'})
 
 
 if __name__ == '__main__':