Repository: vmware/tern
Pull request #561 (2020-02-28 13:20:25)

Document YAML data output that Tern produces
Listing the all the keys of the YAML output and their description.

Works towards #175 

Signed-off-by: WangJL <hazard15020@gmail.com>

nishakm: @ForgetMe17 Thanks for this PR. It was a lot of work to summarize and this looks good to me. @rnjudge can you take a look as well and see if I am not missing anything?
rnjudge: I requested just a few small changes. Otherwise, looks good! Thank you @ForgetMe17 
ForgetMe17: Thanks for your review!
nishakm: Looks good to me. @rnjudge?

Patch:
diff --git a/docs/yaml_output.md b/docs/yaml_output.md
new file mode 100644
index 0000000..05ea9df
--- /dev/null
+++ b/docs/yaml_output.md
@@ -0,0 +1,45 @@
+# YAML Format Output
+
+You can get the results in a YAML file to be consumed by a downstream tool or script.
+
+`$ tern -l report -f yaml -i golang:1.12-alpine -o output.yaml`
+
+A YAML format output file starts with line comments indicating the version of Tern. It has a `image` key(image.py) corresponding to the analyzed image. Here are the subkeys and their description:
+
+- `config`: the image config metadata
+- `image_id`: this is a unique identifier for the image - for OCI spec
+    this could be the digest. For now this is the sha256sum of the
+    config.json
+- `manifest`: the json object representing the image manifest
+- `layers`: list of layer objects in the image(image_layer.py)
+  - `diff_id`: the sha256 of the layer filesystem
+  - `fs_hash`: the hashed contents of the layer filesystem - default to empty string if there is no tarball of the layer filesystem
+  - `packages`: list of objects of type Package (package.py)
+    - `name`: package name
+    - `version`: package version
+    - `pkg_license`: package license that is declared
+    - `copyright`: copyright text
+    - `proj_url`: package source url
+    - `download_url`: package download url
+    - `origins`: a list of NoticeOrigin objects(notice_origin.py, expanded below)
+    - `checksum`: checksum as package property
+  - `origins`: list of NoticeOrigin objects (notice_origin.py, expended below)
+  - `tar_file`: the path to the layer filesystem tarball
+  - `created_by`: sometimes the metadata will contain a created_by
+      key containing the command that created the filesystem layer
+  - `import_image`: if the layer is imported from another image this is a pointer to that image. In Python terms it is just the name of the Image object or any object that uses this layer Based on how container image layers are created, this is usually the last layer of the image that was imported
+  - `import_str`: The string from a build tool (like a Dockerfile) that created this layer by importing it from another image
+  - `files_analyzed`: whether the files in this layer are analyzed or not
+  - `analyzed_output`: the result of the file analysis
+
+`origins` is the origin of a notice, which is expanded as follows.
+
+- origins
+  - origin_str: the origin string, from the input or the environment or the configuration
+  - notices
+    - message: the notice message
+    - level: notice level - error, warning or hint
+      * error: cannot continue further
+      * warning: will try to continue from here
+      * info: information only
+      * hint: message on how to make the results better
diff --git a/tern/classes/package.py b/tern/classes/package.py
index 390cd4f..96afa34 100644
--- a/tern/classes/package.py
+++ b/tern/classes/package.py
@@ -2,7 +2,7 @@
 #
 # Copyright (c) 2017-2019 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: BSD-2-Clause
-
+from tern.classes.file_data import FileData
 from tern.classes.notice import Notice
 from tern.classes.origins import Origins
 from tern.utils.general import prop_names
@@ -19,11 +19,16 @@ class Package:
         download_url: package download url
         origins: a list of NoticeOrigin objects
         checksum: checksum as package property
+        files: list of files in a package
 
     methods:
         to_dict: returns a dict representation of the instance
         fill: instantiates a package object given a dict representation.
-        is_equal: compares two package objects.'''
+        is_equal: compares two package objects.
+        get_file_paths: Returns a list of file paths
+        add_file: Add a file to list of files
+        remove_file: Removes a file from a list of files
+        '''
     def __init__(self, name):
         self.__name = name
         self.__version = ''
@@ -33,11 +38,16 @@ class Package:
         self.__download_url = ''
         self.__checksum = ''
         self.__origins = Origins()
+        self.__files = []
 
     @property
     def name(self):
         return self.__name
 
+    @property
+    def files(self):
+        return self.__files
+
     @property
     def version(self):
         return self.__version
@@ -90,11 +100,33 @@ class Package:
     def checksum(self, checksum):
         self.__checksum = checksum
 
+    def get_file_paths(self):
+        """Return a list of paths of all the files in a package"""
+        return [f.path for f in self.__files]
+
+    def add_file(self, file):
+        """Add a file to the list of files present in a package"""
+        if isinstance(file, FileData):
+            if file.path not in self.get_file_paths():
+                self.files.append(file)
+        else:
+            raise TypeError('Object type is {0}, should be FileData'.format(
+                type(file)))
+
+    def remove_file(self, file_path):
+        """Remove a file from the list of files present in a package"""
+        for file in self.__files:
+            if file.path == file_path:
+                self.__files.remove(file)
+                return True
+        return False
+
     def to_dict(self, template=None):
         '''Return a dictionary version of the Package object
         If given an object which is a subclass of Template then map
         the keys to the Package class properties'''
         pkg_dict = {}
+        file_list = [f.to_dict(template) for f in self.__files]
         if template:
             # loop through object properties
             for key, prop in prop_names(self):
@@ -106,11 +138,16 @@ class Package:
             if 'origins' in template.package().keys():
                 pkg_dict.update(
                     {template.package()['origins']: self.origins.to_dict()})
+            # update the 'files' part if it exists in the mapping
+            if 'files' in template.package().keys():
+                pkg_dict.update(
+                    {template.package()['files']: file_list})
         else:
             # don't map, just use the property name as the key
             for key, prop in prop_names(self):
                 pkg_dict.update({prop: self.__dict__[key]})
             pkg_dict.update({'origins': self.origins.to_dict()})
+            pkg_dict.update({'files': file_list})
         return pkg_dict
 
     def __fill_properties(self, package_dict):


Test patch:
diff --git a/tests/test_class_package.py b/tests/test_class_package.py
index f3aa5d1..6509ad0 100644
--- a/tests/test_class_package.py
+++ b/tests/test_class_package.py
@@ -5,6 +5,7 @@
 
 import unittest
 
+from tern.classes.file_data import FileData
 from tern.classes.package import Package
 from test_fixtures import TestTemplate1
 from test_fixtures import TestTemplate2
@@ -60,7 +61,42 @@ class TestClassPackage(unittest.TestCase):
         self.assertEqual(self.p1.download_url, 'https://github.com')
         self.assertEqual(self.p1.checksum, '123abc456')
 
+    def testAddFile(self):
+        p1 = Package('package')
+        f1 = FileData('test.java', 'abc/pqr/test.java')
+        f2 = FileData('test.c', 'abc/pqr/test.c')
+        p1.add_file(f1)
+        p1.add_file(f2)
+        self.assertEqual(len(p1.files), 2)
+        with self.assertRaises(TypeError):
+            p1.add_file('sample.cpp')
+
+    def testRemoveFile(self):
+        p1 = Package('package')
+        f1 = FileData('test.java', 'abc/pqr/test.java')
+        f2 = FileData('test.c', 'abc/pqr/test.c')
+        p1.add_file(f1)
+        p1.add_file(f2)
+        self.assertTrue(p1.remove_file('abc/pqr/test.java'))
+        self.assertEqual(len(p1.files), 1)
+        self.assertTrue(p1.remove_file('abc/pqr/test.c'))
+        self.assertEqual(len(p1.files), 0)
+        self.assertFalse(p1.remove_file('abc'))
+
+    def testGetFilePaths(self):
+        p1 = Package('package')
+        f1 = FileData('test.java', 'abc/pqr/test.java')
+        f2 = FileData('test.c', 'abc/pqr/test.c')
+        p1.add_file(f1)
+        p1.add_file(f2)
+        self.assertEqual(p1.get_file_paths(),
+                         ['abc/pqr/test.java',
+                          'abc/pqr/test.c'
+                          ])
+
     def testToDict(self):
+        f1 = FileData('test.java', 'abc/pqr/test.java')
+        self.p1.add_file(f1)
         a_dict = self.p1.to_dict()
         self.assertEqual(a_dict['name'], 'p1')
         self.assertEqual(a_dict['version'], '1.0')
@@ -70,17 +106,25 @@ class TestClassPackage(unittest.TestCase):
         self.assertEqual(a_dict['download_url'], 'https://github.com')
         self.assertFalse(a_dict['origins'])
         self.assertEqual(a_dict['checksum'], '123abc456')
+        self.assertEqual(a_dict['files'][0]['name'], 'test.java')
+        self.assertEqual(a_dict['files'][0]['path'], 'abc/pqr/test.java')
 
     def testToDictTemplate(self):
         template1 = TestTemplate1()
         template2 = TestTemplate2()
+        f1 = FileData('test.java', 'abc/pqr/test.java')
+        self.p1.add_file(f1)
         dict1 = self.p1.to_dict(template1)
         dict2 = self.p1.to_dict(template2)
-        self.assertEqual(len(dict1.keys()), 3)
+        self.assertEqual(len(dict1.keys()), 4)
         self.assertEqual(dict1['package.name'], 'p1')
+        self.assertEqual(dict1['package.files'][0]['file.name'],
+                         'test.java')
         self.assertEqual(dict1['package.version'], '1.0')
         self.assertEqual(dict1['package.license'], 'Apache 2.0')
-        self.assertEqual(len(dict2.keys()), 5)
+        self.assertEqual(dict1['package.files'][0]['file.path'],
+                         'abc/pqr/test.java')
+        self.assertEqual(len(dict2.keys()), 6)
         self.assertFalse(dict2['notes'])
 
     def testIsEqual(self):
@@ -112,7 +156,7 @@ class TestClassPackage(unittest.TestCase):
         self.assertFalse(p.proj_url)
         self.assertEqual(len(p.origins.origins), 1)
         self.assertEqual(p.origins.origins[0].origin_str, 'p1')
-        self.assertEqual(len(p.origins.origins[0].notices), 3)
+        self.assertEqual(len(p.origins.origins[0].notices), 4)
         self.assertEqual(p.origins.origins[0].notices[0].message,
                          "No metadata for key: copyright")
         self.assertEqual(p.origins.origins[0].notices[0].level, 'warning')
diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py
index e43bd78..5c48bdd 100644
--- a/tests/test_fixtures.py
+++ b/tests/test_fixtures.py
@@ -36,7 +36,8 @@ class TestTemplate1(Template):
     def package(self):
         return {'name': 'package.name',
                 'version': 'package.version',
-                'pkg_license': 'package.license'}
+                'pkg_license': 'package.license',
+                'files': 'package.files'}
 
     def image_layer(self):
         return {'diff_id': 'layer.diff',
@@ -63,7 +64,8 @@ class TestTemplate2(Template):
         mapping = {'name': 'package.name',
                    'version': 'package.version',
                    'pkg_license': 'package.license',
-                   'proj_url': 'package.url'}
+                   'proj_url': 'package.url',
+                   'files': 'package.files'}
         # we update the mapping with another defined mapping
         mapping.update(self.origins())
         return mapping