Repository: vmware/tern
Pull request #594 (2020-03-19 19:06:44)

file_data.py: Add checksums property
This PR adds checksums property to the FileData class.
It adds the add_checksums method to add new checksum tuples.
The tests have been added for the changes made.

Closes https://github.com/vmware/tern/issues/591

Signed-off-by: PrajwalM2212 <prajwalmmath@gmail.com>

nishakm: @PrajwalM2212 will you be updating this PR?
PrajwalM2212: @nishakm Yes. I have been a little busy lately. But I will push the changes today. Sorry for the delay. 
PrajwalM2212: @nishakm I have pushed the changes.  
GH actions is broken.  CircleCI is working though. 

Patch:
diff --git a/tern/analyze/docker/analyze.py b/tern/analyze/docker/analyze.py
index 265a56c..4173725 100644
--- a/tern/analyze/docker/analyze.py
+++ b/tern/analyze/docker/analyze.py
@@ -84,17 +84,21 @@ def abort_analysis():
 
 
 def analyze_first_layer(image_obj, master_list, redo):
-    # find the binary and shell by mounting the base layer
-    target = rootfs.mount_base_layer(image_obj.layers[0].tar_file)
-    binary = common.get_base_bin()
-    shell = get_shell(image_obj, binary)
-    # set up a notice origin for the first layer
-    origin_first_layer = 'Layer: ' + image_obj.layers[0].fs_hash[:10]
-    # only extract packages if there is a known binary and the layer is not
-    # cached
-    if binary:
-        if not common.load_from_cache(image_obj.layers[0], redo):
-            # Determine pacakge/os style from binary in the image layer
+    # set shell to empty string in case one is found
+    shell = ''
+    # try to load from cache
+    loaded = common.load_from_cache(image_obj.layers[0], redo)
+    # if there is no cache, find the binary and shell by mounting the
+    # base layer
+    if not loaded:
+        target = rootfs.mount_base_layer(image_obj.layers[0].tar_file)
+        binary = common.get_base_bin()
+        shell = get_shell(image_obj, binary)
+        # set up a notice origin for the first layer
+        origin_first_layer = 'Layer: ' + image_obj.layers[0].fs_hash[:10]
+        # only extract packages if there is a known binary
+        if binary:
+            # Determine package/os style from binary in the image layer
             common.get_os_style(image_obj.layers[0], binary)
             # get the packages of the first layer
             try:
@@ -105,17 +109,18 @@ def analyze_first_layer(image_obj, master_list, redo):
                 abort_analysis()
             # unmount proc, sys and dev
             rootfs.undo_mount()
-    else:
-        logger.warning(errors.no_package_manager)
-        # /etc/os-release may still be present even if binary is not
-        common.get_os_style(image_obj.layers[0], None)
-        image_obj.layers[0].origins.add_notice_to_origins(
-            origin_first_layer, Notice(errors.no_package_manager, 'warning'))
-        # no binary means there is no shell so set to default shell
-        logger.warning('Unknown filesystem. Using default shell')
-        shell = constants.shell
-    # unmount the first layer
-    rootfs.unmount_rootfs()
+        else:
+            logger.warning(errors.no_package_manager)
+            # /etc/os-release may still be present even if binary is not
+            common.get_os_style(image_obj.layers[0], None)
+            image_obj.layers[0].origins.add_notice_to_origins(
+                origin_first_layer, Notice(
+                    errors.no_package_manager, 'warning'))
+            # no binary means there is no shell so set to default shell
+            logger.warning('Unknown filesystem. Using default shell')
+            shell = constants.shell
+        # unmount the first layer
+        rootfs.unmount_rootfs()
     # populate the master list with all packages found in the first layer
     for p in image_obj.layers[0].packages:
         master_list.append(p)
@@ -126,7 +131,8 @@ def analyze_subsequent_layers(image_obj, shell, master_list, redo, dfobj=None):
     # get packages for subsequent layers
     curr_layer = 1
     while curr_layer < len(image_obj.layers):  # pylint:disable=too-many-nested-blocks
-        if not common.load_from_cache(image_obj.layers[curr_layer], redo):
+        if (not common.load_from_cache(image_obj.layers[curr_layer], redo) and
+                shell):
             # get commands that created the layer
             # for docker images this is retrieved from the image history
             command_list = dhelper.get_commands_from_history(
diff --git a/tern/analyze/docker/helpers.py b/tern/analyze/docker/helpers.py
index 3293e86..f48c60b 100644
--- a/tern/analyze/docker/helpers.py
+++ b/tern/analyze/docker/helpers.py
@@ -64,7 +64,7 @@ def get_dockerfile_base():
         # check for scratch
         if base_image_tag[0] == 'scratch':
             # there is no base image - return no image object
-            return None
+            return None, None
         # there should be some image object here
         repotag = base_image_tag[0] + dockerfile.tag_separator + \
             base_image_tag[1]
@@ -91,7 +91,7 @@ def get_dockerfile_base():
     except ValueError as e:
         logger.warning("%s", errors.cannot_parse_base_image.format(
             dockerfile=dockerfile_global, error_msg=e))
-        return None
+        return None, None
 
 
 def get_dockerfile_image_tag():
diff --git a/tern/classes/file_data.py b/tern/classes/file_data.py
index 3bb5961..c329f39 100644
--- a/tern/classes/file_data.py
+++ b/tern/classes/file_data.py
@@ -34,6 +34,7 @@ class FileData:
         authors: a list of authors if known
         packages: a list of packages where this file could come from
         urls: a list of urls from where this file could come from
+        checksums: a list of tuples of the form (checksum_type, checksum)
 
     methods:
         to_dict: returns a dictionary representation of the instance
@@ -61,6 +62,7 @@ class FileData:
         self.authors = []
         self.packages = []
         self.urls = []
+        self.__checksums = []
         self.__origins = Origins()
 
     @property
@@ -117,6 +119,14 @@ class FileData:
     def file_type(self, file_type):
         self.__file_type = file_type
 
+    @property
+    def checksums(self):
+        return self.__checksums
+
+    @checksums.setter
+    def checksums(self, checksums):
+        self.__checksums = checksums
+
     @property
     def origins(self):
         return self.__origins
@@ -133,6 +143,11 @@ class FileData:
         self.__version_control = version_control
         self.__version = version
 
+    def add_checksums(self, checksums):
+        '''Add checksum tuples to checksums property'''
+        for checksum in checksums:
+            self.__checksums.append(checksum)
+
     def to_dict(self, template=None):
         '''Return a dictionary version of the FileData object
         If given an object which is a subclass of Template then map
@@ -185,6 +200,7 @@ class FileData:
             authors: <authors>
             packages: <packages>
             urls: <urls>
+            checksums: <checksums>
         the way to use this method is to instantiate the class with the
         name and path and then give it a file_data dictionary to fill
         in the rest return true if package name is the same as the one
@@ -213,6 +229,7 @@ class FileData:
             self.authors = other.authors
             self.packages = other.packages
             self.urls = other.urls
+            self.checksums = other.checksums
             # collect notices
             for o in other.origins.origins:
                 for n in o.notices:


Test patch:
diff --git a/tests/test_class_file_data.py b/tests/test_class_file_data.py
index 30ef358..4fecbf6 100644
--- a/tests/test_class_file_data.py
+++ b/tests/test_class_file_data.py
@@ -68,6 +68,14 @@ class TestClassFileData(unittest.TestCase):
         self.assertEqual(self.afile.checksum_type, 'sha256')
         self.assertEqual(self.afile.checksum, '12345abcde')
 
+    def testAddChecksums(self):
+        file1 = FileData('file1', 'path/to/file1')
+        file1.add_checksums([('SHA1', '12345abcde'),
+                             ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')])
+        self.assertEqual(file1.checksums,
+                         [('SHA1', '12345abcde'),
+                          ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')])
+
     def testExtAttrs(self):
         file = FileData('test.txt', 'usr/abc/test.txt')
         file.extattrs = '-rw-r--r--|1000|1000|19|1'
@@ -103,7 +111,9 @@ class TestClassFileData(unittest.TestCase):
             'checksum': '77304005ceb5f0d03ad4c37eb8386a10866e'
                         '4ceeb204f7c3b6599834c7319541',
             'extattrs': '-rw-r--r-- 1 1000 1000 16262 Nov 13 17:57'
-                        ' /usr/include/zconf.h'
+                        ' /usr/include/zconf.h',
+            'checksums': [('SHA1', '12345abcde'),
+                          ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')]
         }
         f = FileData('zconf.h', '/usr/include/zconf.h')
         f.fill(file_dict)
@@ -114,6 +124,10 @@ class TestClassFileData(unittest.TestCase):
                                      '6a10866e4ceeb204f7c3b6599834c7319541')
         self.assertEqual(f.extattrs, '-rw-r--r-- 1 1000 1000 '
                                      '16262 Nov 13 17:57 /usr/include/zconf.h')
+        self.assertEqual(f.checksums,
+                         [('SHA1', '12345abcde'),
+                          ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')]
+                         )
         self.assertEqual(f.origins.origins[0].notices[1].message,
                          'No metadata for key: file_type')
         self.assertEqual(f.origins.origins[0].notices[1].level, 'warning')
@@ -127,6 +141,8 @@ class TestClassFileData(unittest.TestCase):
         file1.set_checksum('sha256', '123abc456def')
         file1.extattrs = '-rwxr-xr-x|1000|1000|14408|1'
         file2 = FileData('switch_root', 'sbin/switch_root')
+        file2.checksums = [('SHA1', '12345abcde'),
+                           ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')]
         file2.set_checksum('sha256', '123abc456def')
         file2.extattrs = '-rwxr-xr-x|1000|1000|14408|1'
         file2.date = '2012-02-02'
@@ -148,6 +164,9 @@ class TestClassFileData(unittest.TestCase):
         self.assertEqual(file1.extattrs, '-rwxr-xr-x|1000|1000|14408|1')
         self.assertFalse(file1.merge('astring'))
         self.assertTrue(file1.merge(file2))
+        self.assertEqual(file1.checksums,
+                         [('SHA1', '12345abcde'),
+                          ('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')])
         self.assertEqual(file1.date, '2012-02-02')
         self.assertEqual(file1.file_type, 'binary')
         self.assertEqual(file1.licenses, ['MIT', 'GPL'])