Repository: vmware/tern (python)

Issue:
Tern doesn't split shell commands separated by a ';'
**Describe the bug**
Tern doesn't split shell commands separated by a ';'

**To Reproduce**
1. Run `tern report -i golang:1.12-alpine -f output.txt`
2. See output.txt:
```
        warning:
Unrecognized Commands:set -eux;         apk add --no-cache --virtual .build-deps                bash            gcc             musl-dev                openssl                 go      ;       export          GOROOT_BOOTSTRAP="$(go env GOROOT)"             GOOS="$(go env GOOS)"           GOARCH="$(go env GOARCH)"               GOHOSTOS="$(go env GOHOSTOS)"           GOHOSTARCH="$(go env GOHOSTARCH)"       ;       apkArch="$(apk --print-arch)";  case "$apkArch" in              armhf) export GOARM='6' ;;              x86) export GO386='387' ;;      esac;           wget -O go.tgz "https://golang.org/dl/go$GOLANG_VERSION.src.tar.gz";    echo '2aa5f088cbb332e73fc3def546800616b38d3bfe6b8713b8a6404060f22503e8 *go.tgz' | sha256sum -c -;       tar -C /usr/local -xzf go.tgz;  rm go.tgz;              cd /usr/local/go/src;   ./make.bash;            rm -rf          /usr/local/go/pkg/bootstrap             /usr/local/go/pkg/obj   ;       apk del .build-deps;            export PATH="/usr/local/go/bin:$PATH";  go version
```
**Error in terminal**
No Error

**Expected behavior**
`apk` is a known package manager, so Tern should be able to report all the packages installed by this line. -> `bash gcc musl-dev openssl go`

**Environment you are running Tern on**
Enter all that apply
- VMware Fusion VM running Ubuntu 18.04 LTS


Pull request description:
Split and consolidate shell commands
This PR resolves #268 and another large bug that was hidden by this one.

Patch:
diff --git a/tern/classes/command.py b/tern/classes/command.py
index 31f2103..f4fdb0a 100644
--- a/tern/classes/command.py
+++ b/tern/classes/command.py
@@ -117,3 +117,26 @@ class Command:
     def is_ignore(self):
         '''Is this a command to ignore?'''
         return self.flags == 0b001
+
+    def merge(self, other):
+        '''Given another Command object, add and remove words based on whether
+        that command is an install command or a remove command'''
+        # check if the object is an instance of Command first
+        if isinstance(other, Command):
+            # check if names are the same
+            if self.name == other.name:
+                # check if the other command is an install command
+                if other.is_install:
+                    for word in other.words:
+                        self.__words.append(word)
+                    self.__words = list(set(self.__words))
+                    return True
+                # check if the other command is a remove command
+                if other.is_remove:
+                    for word in other.words:
+                        if word in self.__words:
+                            self.__words.remove(word)
+                    return True
+            return False
+        raise TypeError('Object type is {0}, should be Command'.format(
+            type(other)))
diff --git a/tern/helpers/common.py b/tern/helpers/common.py
index 67517c6..55adf9d 100644
--- a/tern/helpers/common.py
+++ b/tern/helpers/common.py
@@ -19,6 +19,7 @@ from tern.report import errors
 from tern.report import content
 from tern.utils import cache
 from tern.utils import constants
+from tern.utils import general
 
 # global logger
 logger = logging.getLogger(constants.logger_name)
@@ -26,10 +27,10 @@ logger = logging.getLogger(constants.logger_name)
 
 def get_shell_commands(shell_command_line):
     '''Given a shell command line, get a list of Command objects'''
-    comm_list = shell_command_line.split('&&')
+    comm_list = general.split_command(shell_command_line)
     cleaned_list = []
     for comm in comm_list:
-        cleaned_list.append(Command(comm.strip()))
+        cleaned_list.append(Command(general.clean_command(comm)))
     return cleaned_list
 
 
@@ -283,6 +284,20 @@ def remove_unrecognized_commands(command_list):
     return unrec_commands, filtered_list
 
 
+def consolidate_commands(command_list):
+    '''Given a list of Command objects, consolidate the install and remove
+    commands into one and return a list of resulting command objects'''
+    new_list = []
+    while command_list:
+        first = command_list.pop(0)
+        for _ in range(0, len(command_list)):
+            second = command_list.pop(0)
+            if not first.merge(second):
+                command_list.append(first)
+        new_list.append(first)
+    return new_list
+
+
 def filter_install_commands(shell_command_line):
     '''Given a shell command line:
         1. Create a list of Command objects
@@ -300,7 +315,8 @@ def filter_install_commands(shell_command_line):
         report = report + formats.ignored + ignore_msgs
     if unrec_msgs:
         report = report + formats.unrecognized + unrec_msgs
-    return filter2, report
+
+    return consolidate_commands(filter2), report
 
 
 def add_snippet_packages(image_layer, command, pkg_listing, shell):
diff --git a/tern/utils/dockerfile.py b/tern/utils/dockerfile.py
index 6f1b5eb..d7b8df3 100644
--- a/tern/utils/dockerfile.py
+++ b/tern/utils/dockerfile.py
@@ -9,6 +9,8 @@ Dockerfile parser and information retrieval
 
 import re
 
+from tern.utils.general import clean_command
+
 directives = ['FROM',
               'ARG',
               'ADD',
@@ -24,11 +26,9 @@ directives = ['FROM',
 # regex for matching lines in a dockerfile
 comments = re.compile('^#')
 line_indent = re.compile('.*\\\\$')
-concatenation = re.compile('&&')
 tabs = re.compile('\t')
 
 # regex strings
-cleaning = '[\t\\\\]'
 bash_var = '[\$\{\}]'  # noqa
 
 # strings
@@ -69,13 +69,6 @@ def get_command_list(dockerfile_name):
     return command_list
 
 
-def clean_command(command):
-    '''Given a command string, clean out all whitespaces, tabs and line
-    indentations
-    Leave && alone'''
-    return re.sub(cleaning, '', command).strip()
-
-
 def get_directive(line):
     '''Given a line from a Dockerfile get the Docker directive
     eg: FROM, ADD, COPY, RUN and the object in the form of a tuple'''
diff --git a/tern/utils/general.py b/tern/utils/general.py
index 08c3d2b..9f24117 100644
--- a/tern/utils/general.py
+++ b/tern/utils/general.py
@@ -13,6 +13,11 @@ from tern import Version
 from tern.utils import constants
 
 
+# regex strings
+cleaning = '[\t\\\\]'
+concat = '&&|;'
+
+
 # from https://stackoverflow.com/questions/6194499/pushd-through-os-system
 @contextmanager
 def pushd(path):
@@ -29,6 +34,18 @@ def initialize_names():
     constants.container = constants.container + "_" + str(randint)
 
 
+def clean_command(command):
+    '''Given a command string, clean out all whitespaces, tabs and line
+    indentations
+    Leave && alone'''
+    return re.sub(cleaning, '', command).strip()
+
+
+def split_command(command):
+    '''Given a string of concatenated commands, return a list of commands'''
+    return re.split(concat, command)
+
+
 def parse_command(command):
     '''Parse a unix command of the form:
         command (subcommand) [options] [arguments]


Test patch:
diff --git a/tests/test_class_command.py b/tests/test_class_command.py
index 11de90a..b4e025e 100644
--- a/tests/test_class_command.py
+++ b/tests/test_class_command.py
@@ -5,7 +5,7 @@
 
 import unittest
 
-from classes.command import Command
+from tern.classes.command import Command
 
 
 class TestClassCommand(unittest.TestCase):
@@ -15,12 +15,14 @@ class TestClassCommand(unittest.TestCase):
         self.untar = Command('tar -x -C file tarfile.tar')
         self.download = Command('wget url')
         self.remove = Command('apt-get purge git')
+        self.install2 = Command('apt-get install -y ca-certificates')
 
     def tearDown(self):
         del self.install
         del self.untar
         del self.download
         del self.remove
+        del self.install2
 
     def testInstance(self):
         self.assertEqual(self.install.shell_command, 'apt-get install -y git')
@@ -98,6 +100,28 @@ class TestClassCommand(unittest.TestCase):
         self.assertTrue(self.remove.is_set())
         self.assertTrue(self.remove.is_remove())
 
+    def testMerge(self):
+        self.install.reassign_word('install', 'subcommand')
+        self.install.set_install()
+        self.install2.reassign_word('install', 'subcommand')
+        self.install2.set_install()
+        self.remove.reassign_word('purge', 'subcommand')
+        self.remove.set_remove()
+        # merge the second install into the first install
+        self.assertTrue(self.install.merge(self.install2))
+        self.assertTrue(self.install.words, ['git', 'ca-certificates'])
+        # merge the remove command
+        self.assertTrue(self.install.merge(self.remove))
+        self.assertTrue(self.install.words, ['ca-certificates'])
+        # try to merge the download command
+        self.assertFalse(self.install.merge(self.download))
+        # try to merge an ignored command
+        self.download.set_ignore()
+        self.assertFalse(self.install.merge(self.download))
+        # try to merge some other object
+        with self.assertRaises(TypeError):
+            self.install.merge('test')
+
 
 if __name__ == '__main__':
     unittest.main()