Repository: ericsson/codechecker
Issue: failure collection error

with Codechecker version 5.9.1 collection fails

pointer.c:

```
#include <stdint.h>

void main(){
    int *p;
    int32_t i=(int32_t) p;     
}
```

if we compile this in C++ mode:

```
(venv)ednikru@md-mtas2:/ssd/ednikru/ctu/tests$ CodeChecker quickcheck --analyzers clangsa -b "gcc -Wall -std=c++11 -m32 ./pointer.c"
[10:23] - Starting build ...
cc1: warning: command line option \\u2018-std=c++11\\u2019 is valid for C++/ObjC++ but not for C [enabled by default]
./pointer.c: In function \\u2018main\\u2019:
./pointer.c:5:13: warning: \\u2018p\\u2019 is used uninitialized in this function [-Wuninitialized]
     int32_t i=(int32_t) p;
             ^
[10:23] - Build finished successfully.
[10:23] - Previous analysis results in '/tmp/codechecker-qcpW2YyO' have been removed, overwriting with current result
[10:23] - Starting static analysis ...
Traceback (most recent call last):
  File "/home/ednikru/work/codechecker/codechecker_git/codechecker/build/CodeChecker/lib/python2.7/libcodechecker/analyze/analysis_manager.py", line 244, in check
    zipfile.ZIP_DEFLATED)
  File "/usr/lib/python2.7/zipfile.py", line 1118, in write
    st = os.stat(filename)
OSError: [Errno 2] No such file or directory: 'is'
Exception in thread Thread-3:
Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/threading.py", line 763, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/lib/python2.7/multiprocessing/pool.py", line 396, in _handle_results
    cache[job]._set(i, obj)
  File "/usr/lib/python2.7/multiprocessing/pool.py", line 599, in _set
    self._callback(self._value)
  File "/home/ednikru/work/codechecker/codechecker_git/codechecker/build/CodeChecker/lib/python2.7/libcodechecker/analyze/analysis_manager.py", line 317, in <lambda>
    results, metadata, output_path)
  File "/home/ednikru/work/codechecker/codechecker_git/codechecker/build/CodeChecker/lib/python2.7/libcodechecker/analyze/analysis_manager.py", line 47, in worker_result_handler
    failed_analysis[buildaction.analyzer_type] += 1
AttributeError: 'str' object has no attribute 'analyzer_type'
```

Changed files:
libcodechecker/analyze/tidy_output_converter.py, libcodechecker/log/option_parser.py, libcodechecker/analyze/analyzers/analyzer_clangsa.py, libcodechecker/analyze/analysis_manager.py, tests/functional/analyze/__init__.py, tests/functional/analyze/test_analyze.py, libcodechecker/analyze/analyzers/analyzer_clang_tidy.py, libcodechecker/analyze/analyzers/analyzer_base.py

diff --git a/libcodechecker/analyze/analysis_manager.py b/libcodechecker/analyze/analysis_manager.py
index f63ca4c9..3eb95c7e 100644
--- a/libcodechecker/analyze/analysis_manager.py
+++ b/libcodechecker/analyze/analysis_manager.py
@@ -85,6 +85,22 @@ def create_dependencies(action):
     Transforms the given original build 'command' to a command that, when
     executed, is able to generate a dependency list.
     """
+
+    def __eliminate_argument(arg_vect, opt_strings, num_args=0):
+        """
+        This call eliminates the parameters matching the given option strings,
+        along with the number of arguments coming directly after the opt-string
+        from the command.
+        """
+        option_index = next(
+            (i for i, c in enumerate(arg_vect) if c in opt_strings), None)
+
+        if option_index:
+            arg_vect = arg_vect[0:option_index] + \\
+                       arg_vect[option_index + num_args + 1:]
+
+        return arg_vect
+
     if 'CC_LOGGER_GCC_LIKE' not in os.environ:
         os.environ['CC_LOGGER_GCC_LIKE'] = 'gcc:g++:clang:clang++:cc:c++'
 
@@ -92,17 +108,31 @@ def create_dependencies(action):
     if any(binary_substring in command[0] for binary_substring
            in os.environ['CC_LOGGER_GCC_LIKE'].split(':')):
         # gcc and clang can generate makefile-style dependency list.
-        command = [command[0], '-E', '-M', '-MQ', '__dummy'] + command[1:]
 
-        option_index = next(
-            (i for i, c in enumerate(command) if c == '-o' or c == '--output'),
-            None)
+        # If an output file is set, the dependency is not written to the
+        # standard output but rather into the given file.
+        # We need to first eliminate the output from the command.
+        command = __eliminate_argument(command, ['-o', '--output'], 1)
 
-        if option_index:
-            # If an output file is set, the dependency is not written to the
-            # standard output but rather into the given file.
-            # We need to first eliminate the output from the command.
-            command = command[0:option_index] + command[option_index+2:]
+        # Remove potential dependency-file-generator options from the string
+        # too. These arguments found in the logged build command would derail
+        # us and generate dependencies, e.g. into the build directory used.
+        command = __eliminate_argument(command, ['-MM'])
+        command = __eliminate_argument(command, ['-MF'], 1)
+        command = __eliminate_argument(command, ['-MP'])
+        command = __eliminate_argument(command, ['-MT'], 1)
+        command = __eliminate_argument(command, ['-MQ'], 1)
+        command = __eliminate_argument(command, ['-MD'])
+        command = __eliminate_argument(command, ['-MMD'])
+
+        # Clang contains some extra options.
+        command = __eliminate_argument(command, ['-MJ'], 1)
+        command = __eliminate_argument(command, ['-MV'])
+
+        # Build out custom invocation for dependency generation.
+        command = [command[0], '-E', '-M', '-MT', '__dummy'] + command[1:]
+
+        LOG.debug("Crafting build dependencies from GCC or Clang!")
 
         output, rc = util.call_command(command, env=os.environ)
         if rc == 0:
@@ -248,23 +278,60 @@ def check(check_data):
                         LOG.debug("[ZIP] Writing analyzer STDERR to /stderr")
                         archive.writestr("stderr", rh.analyzer_stderr)
 
-                    LOG.debug("Generating and writing dependent headers...")
+                    LOG.debug("Generating dependent headers via compiler...")
                     try:
                         dependencies = set(create_dependencies(rh.buildaction))
                     except Exception as ex:
                         LOG.debug("Couldn't create dependencies:")
-                        LOG.debug(ex.message)
-                        archive.writestr("no-sources", ex.message)
+                        LOG.debug(str(ex))
+                        archive.writestr("no-sources", str(ex))
                         dependencies = set()
 
+                    LOG.debug("Fetching other dependent files from analyzer "
+                              "output...")
+                    try:
+                        other_files = set()
+                        if len(rh.analyzer_stdout) > 0:
+                            other_files.update(
+                                source_analyzer.get_analyzer_mentioned_files(
+                                    rh.analyzer_stdout))
+
+                        if len(rh.analyzer_stderr) > 0:
+                            other_files.update(
+                                source_analyzer.get_analyzer_mentioned_files(
+                                    rh.analyzer_stderr))
+                    except Exception as ex:
+                        LOG.debug("Couldn't generate list of other files "
+                                  "from analyzer output:")
+                        LOG.debug(str(ex))
+                        other_files = set()
+
+                    dependencies.update(other_files)
+
+                    LOG.debug("Writing dependent files to archive.")
                     for dependent_source in dependencies:
                         LOG.debug("[ZIP] Writing '" + dependent_source + "' "
                                   "to the archive.")
-                        archive.write(
-                            dependent_source,
-                            os.path.join("sources-root",
-                                         dependent_source.lstrip('/')),
-                            zipfile.ZIP_DEFLATED)
+                        archive_path = dependent_source.lstrip('/')
+
+                        try:
+                            archive.write(
+                                dependent_source,
+                                os.path.join("sources-root",
+                                             archive_path),
+                                zipfile.ZIP_DEFLATED)
+                        except Exception as ex:
+                            # In certain cases, the output could contain
+                            # invalid tokens (such as error messages that were
+                            # printed even though the dependency generation
+                            # returned 0).
+                            LOG.debug("[ZIP] Couldn't write, because " +
+                                      str(ex))
+                            archive.writestr(
+                                os.path.join("failed-sources-root",
+                                             archive_path),
+                                "Couldn't write this file, because:\\n" +
+                                str(ex))
 
                     LOG.debug("[ZIP] Writing extra information...")
 
@@ -292,7 +359,7 @@ def check(check_data):
     except Exception as e:
         LOG.debug_analyzer(str(e))
         traceback.print_exc(file=sys.stdout)
-        return 1, skipped, action.analyzer_type, None
+        return 1, skipped, action, None
 
 
 def start_workers(actions, context, analyzer_config_map,
diff --git a/libcodechecker/analyze/analyzers/analyzer_base.py b/libcodechecker/analyze/analyzers/analyzer_base.py
index 965a2123..b1513116 100644
--- a/libcodechecker/analyze/analyzers/analyzer_base.py
+++ b/libcodechecker/analyze/analyzers/analyzer_base.py
@@ -69,6 +69,15 @@ class SourceAnalyzer(object):
         """
         pass
 
+    @abstractmethod
+    def get_analyzer_mentioned_files(self, output):
+        """
+        Return a collection of files that were mentioned by the analyzer in
+        its standard outputs, which should be analyzer_stdout or
+        analyzer_stderr from a result handler.
+        """
+        pass
+
     def analyze(self, res_handler, env=None):
         """
         Run the analyzer.
diff --git a/libcodechecker/analyze/analyzers/analyzer_clang_tidy.py b/libcodechecker/analyze/analyzers/analyzer_clang_tidy.py
index e9d72132..16b445c3 100644
--- a/libcodechecker/analyze/analyzers/analyzer_clang_tidy.py
+++ b/libcodechecker/analyze/analyzers/analyzer_clang_tidy.py
@@ -117,6 +117,31 @@ class ClangTidy(analyzer_base.SourceAnalyzer):
             LOG.error(ex)
             return []
 
+    def get_analyzer_mentioned_files(self, output):
+        """
+        Parse Clang-Tidy's output to generate a list of files that were
+        mentioned in the standard output or standard error.
+        """
+
+        # A line mentioning a file in Clang-Tidy's output looks like this:
+        # /home/.../.cpp:L:C: warning: foobar.
+        regex = re.compile(
+            # File path followed by a ':'.
+            '^(?P<path>[\\S ]+?):'
+            # Line number followed by a ':'.
+            '(?P<line>\\d+?):'
+            # Column number followed by a ':' and a space.
+            '(?P<column>\\d+?):\\ ')
+
+        paths = []
+
+        for line in output.splitlines():
+            match = re.match(regex, line)
+            if match:
+                paths.append(match.group('path'))
+
+        return set(paths)
+
     @classmethod
     def resolve_missing_binary(cls, configured_binary, env):
         """
diff --git a/libcodechecker/analyze/analyzers/analyzer_clangsa.py b/libcodechecker/analyze/analyzers/analyzer_clangsa.py
index c783de45..299956a4 100644
--- a/libcodechecker/analyze/analyzers/analyzer_clangsa.py
+++ b/libcodechecker/analyze/analyzers/analyzer_clangsa.py
@@ -172,6 +172,38 @@ class ClangSA(analyzer_base.SourceAnalyzer):
             LOG.error(ex)
             return []
 
+    def get_analyzer_mentioned_files(self, output):
+        """
+        Parse ClangSA's output to generate a list of files that were mentioned
+        in the standard output or standard error.
+        """
+
+        # A line mentioning a file in ClangSA's output looks like this:
+        # /home/.../.cpp:L:C: warning: foobar.
+        regex = re.compile(
+            # File path followed by a ':'.
+            '^(?P<path>[\\S ]+?):'
+            # Line number followed by a ':'.
+            '(?P<line>\\d+?):'
+            # Column number followed by a ':' and a space.
+            # This is optional, as lines beginning with
+            # "In file included from" don't have a column number.
+            '((?P<column>\\d+?):\\ )?')
+
+        paths = []
+
+        for line in output.splitlines():
+            # Files can also be referenced in a line that
+            # begins as "In file included from /home/...".
+            if line[0] == 'I':
+                line = line.replace("In file included from ", "", 1)
+
+            match = re.match(regex, line)
+            if match:
+                paths.append(match.group('path'))
+
+        return set(paths)
+
     @classmethod
     def resolve_missing_binary(cls, configured_binary, env):
         """
diff --git a/libcodechecker/analyze/tidy_output_converter.py b/libcodechecker/analyze/tidy_output_converter.py
index f4f13baa..f86806c8 100644
--- a/libcodechecker/analyze/tidy_output_converter.py
+++ b/libcodechecker/analyze/tidy_output_converter.py
@@ -74,11 +74,11 @@ class OutputParser(object):
     # Regex for parsing a clang-tidy message.
     message_line_re = re.compile(
         # File path followed by a ':'.
-        '^(?P<path>[\\S]+):'
+        '^(?P<path>[\\S ]+?):'
         # Line number followed by a ':'.
-        '(?P<line>\\d+):'
+        '(?P<line>\\d+?):'
         # Column number followed by a ':' and a space.
-        '(?P<column>\\d+):\\ '
+        '(?P<column>\\d+?):\\ '
         # Severity followed by a ':'.
         '(?P<severity>\\w+):'
         # Checker message.
@@ -89,11 +89,11 @@ class OutputParser(object):
     # Matches a note.
     note_line_re = re.compile(
         # File path followed by a ':'.
-        '^(?P<path>[\\S]+):'
+        '^(?P<path>[\\S ]+?):'
         # Line number followed by a ':'.
-        '(?P<line>\\d+):'
+        '(?P<line>\\d+?):'
         # Column number followed by a ':' and a space.
-        '(?P<column>\\d+):\\ '
+        '(?P<column>\\d+?):\\ '
         # Severity == note.
         'note:'
         # Checker message.
diff --git a/libcodechecker/log/option_parser.py b/libcodechecker/log/option_parser.py
index 85a6534f..24e7fcf4 100644
--- a/libcodechecker/log/option_parser.py
+++ b/libcodechecker/log/option_parser.py
@@ -107,7 +107,14 @@ REPLACE_OPTIONS_MAP = {
 
 IGNORED_OPTION_MAP = {
     '-MT': 1,
+    '-MQ': 1,
     '-MF': 1,
+    '-MJ': 1,
+    '-MM': 0,
+    '-MP': 0,
+    '-MD': 0,
+    '-MV': 0,
+    '-MMD': 0,
     '-fsyntax-only': 0,
     '-save-temps': 0,
     '-install_name': 1,
@@ -125,6 +132,10 @@ IGNORED_OPTION_MAP = {
     '--serialize-diagnostics': 1
 }
 
+IGNORED_OPTION_MAP_REGEX = {
+    '^-g(.+)?$': 0,
+}
+
 # Unknown options by clang, will be skipped.
 
 UNKNOWN_OPTIONS_MAP_REGEX = {
@@ -435,6 +446,7 @@ def arg_check(it, result):
         append_merged_to_list(LINK_OPTION_MAP_MERGED, result.link_opts),
         skip(LINKER_OPTION_MAP),
         skip(IGNORED_OPTION_MAP),
+        skip(IGNORED_OPTION_MAP_REGEX, True),
         append_to_list_from_file('-filelist', result.files),
         append_to_list({'^[^-].+': 0}, result.files, True)]
 
diff --git a/tests/functional/analyze/__init__.py b/tests/functional/analyze/__init__.py
index dd6aff10..0cc23152 100644
--- a/tests/functional/analyze/__init__.py
+++ b/tests/functional/analyze/__init__.py
@@ -27,6 +27,9 @@ def setup_package():
     global TEST_WORKSPACE
     TEST_WORKSPACE = env.get_workspace('analyze')
 
+    report_dir = os.path.join(TEST_WORKSPACE, 'reports')
+    os.makedirs(report_dir)
+
     os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE
 
 
diff --git a/tests/functional/analyze/test_analyze.py b/tests/functional/analyze/test_analyze.py
index a28bcc45..97d04d94 100644
--- a/tests/functional/analyze/test_analyze.py
+++ b/tests/functional/analyze/test_analyze.py
@@ -32,8 +32,7 @@ class TestAnalyze(unittest.TestCase):
 
         # Get the CodeChecker cmd if needed for the tests.
         self._codechecker_cmd = env.codechecker_cmd()
-        self.report_dir = os.path.join(self.test_workspace, "reports")
-        os.makedirs(self.report_dir)
+        self.report_dir = os.path.join(self.test_workspace, 'reports')
         self.test_dir = os.path.join(os.path.dirname(__file__), 'test_files')
         # Change working dir to testfile dir so CodeChecker can be run easily.
         self.__old_pwd = os.getcwd()
@@ -48,10 +47,9 @@ class TestAnalyze(unittest.TestCase):
         Test if reports/failed/<failed_file>.zip file is created
         """
         build_json = os.path.join(self.test_workspace, "build.json")
-        reports_dir = os.path.join(self.test_workspace, "reports")
-        failed_dir = os.path.join(reports_dir, "failed")
+        failed_dir = os.path.join(self.report_dir, "failed")
         analyze_cmd = [self._codechecker_cmd, "analyze", build_json,
-                       "--analyzers", "clangsa", "-o", reports_dir]
+                       "--analyzers", "clangsa", "-o", self.report_dir]
 
         source_file = os.path.join(self.test_dir, "failure.c")
         build_log = [{"directory": self.test_workspace,
@@ -77,8 +75,9 @@ class TestAnalyze(unittest.TestCase):
         self.assertEquals(len(failed_files), 1)
         self.assertIn("failure.c", failed_files[0])
 
-        with zipfile.ZipFile(os.path.join(failed_dir, failed_files[0]),
-                             'r') as archive:
+        fail_zip = os.path.join(failed_dir, failed_files[0])
+
+        with zipfile.ZipFile(fail_zip, 'r') as archive:
             files = archive.namelist()
 
             self.assertIn("build-action", files)
@@ -95,3 +94,45 @@ class TestAnalyze(unittest.TestCase):
             with archive.open(source_in_archive, 'r') as archived_code:
                 with open(source_file, 'r') as source_code:
                     self.assertEqual(archived_code.read(), source_code.read())
+
+        os.remove(os.path.join(failed_dir, failed_files[0]))
+
+    def test_robustness_for_dependencygen_failure(self):
+        """
+        Test if failure ZIP is created even if the dependency generator creates
+        an invalid output.
+        """
+        build_json = os.path.join(self.test_workspace, "build.json")
+        failed_dir = os.path.join(self.report_dir, "failed")
+        analyze_cmd = [self._codechecker_cmd, "analyze", build_json,
+                       "--analyzers", "clangsa", "-o", self.report_dir]
+
+        source_file = os.path.join(self.test_dir, "failure.c")
+        build_log = [{"directory": self.test_workspace,
+                      "command": "cc -c -std=c++11 " + source_file,
+                      "file": source_file
+                      }]
+
+        # cc -std=c++11 writes error "-std=c++11 valid for C++ but not for C"
+        # to its output when invoked as a dependency generator for this
+        # build command.
+
+        with open(build_json, 'w') as outfile:
+            json.dump(build_log, outfile)
+
+        print(analyze_cmd)
+        process = subprocess.Popen(
+                    analyze_cmd, stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE, cwd=self.test_dir)
+        out, err = process.communicate()
+        print(out+err)
+        errcode = process.returncode
+        self.assertEquals(errcode, 0)
+
+        # We expect a failure archive to be in the failed directory.
+        failed_files = os.listdir(failed_dir)
+        print(failed_files)
+        self.assertEquals(len(failed_files), 1)
+        self.assertIn("failure.c", failed_files[0])
+
+        os.remove(os.path.join(failed_dir, failed_files[0]))