Repository: ericsson/codechecker
Issue: Correctly set resource dir when not specified explicitly

Right now we detect default include directories of gcc. This might interfere with clang resource dirs. We need to include the clang resource dir with higher priority than the detected gcc includes (but potentially lower than user include dirs?). To address the priority issue we specify the clang resource dir with the `-isystem` flag. This, however, is not specified when the resource dir is not set in the package layout. From clang 5.0 it is easy to query the resource dir from clang. We should query the resource dir from clang in case it is not specified in the package layout config.

Changed files:
libcodechecker/analyze/analyzers/analyzer_clangsa.py, libcodechecker/analyze/analysis_manager.py, libcodechecker/analyze/host_check.py, libcodechecker/analyze/analyzer_env.py, libcodechecker/analyze/analyzers/ctu_triple_arch.py, libcodechecker/analyze/analyzers/analyzer_clang_tidy.py, libcodechecker/analyze/analyzers/analyzer_types.py

diff --git a/libcodechecker/analyze/analysis_manager.py b/libcodechecker/analyze/analysis_manager.py
index e5603748..175bf8d3 100644
--- a/libcodechecker/analyze/analysis_manager.py
+++ b/libcodechecker/analyze/analysis_manager.py
@@ -7,6 +7,7 @@
 """
 
 from collections import defaultdict
+import codecs
 import glob
 import multiprocessing
 import os
@@ -148,6 +149,7 @@ def create_dependencies(action):
         output, rc = util.call_command(command,
                                        env=os.environ,
                                        cwd=action.directory)
+        output = codecs.decode(output, 'utf-8', 'replace')
         if rc == 0:
             # Parse 'Makefile' syntax dependency output.
             dependencies = output.replace('__dummy: ', '') \\
diff --git a/libcodechecker/analyze/analyzer_env.py b/libcodechecker/analyze/analyzer_env.py
index d5d85b06..b792a3f1 100644
--- a/libcodechecker/analyze/analyzer_env.py
+++ b/libcodechecker/analyze/analyzer_env.py
@@ -70,7 +70,8 @@ def get_check_env(path_env_extra, ld_lib_path_extra):
     return new_env
 
 
-def extend_analyzer_cmd_with_resource_dir(analyzer_cmd, compiler_resource_dir):
+def extend_analyzer_cmd_with_resource_dir(analyzer_cmd,
+                                          compiler_resource_dir):
     """
     GCC and Clang handles the certain compiler intrinsics for specific
     instruction sets (like SSE, AVX) differently.  They have their own set of
@@ -95,6 +96,14 @@ def extend_analyzer_cmd_with_resource_dir(analyzer_cmd, compiler_resource_dir):
     and describes our intentions more cleanly if we just simply disable the
     inludes from the resource dir.
     """
-    if len(compiler_resource_dir) > 0:
-        analyzer_cmd.extend(['-nobuiltininc',
-                             '-isystem', compiler_resource_dir])
+    if len(compiler_resource_dir) == 0:
+        return
+
+    resource_inc = compiler_resource_dir
+    # Resource include path must end with "/include".
+    basename = os.path.basename(os.path.normpath(resource_inc))
+    if basename != 'include':
+        resource_inc = os.path.join(resource_inc, "include")
+
+    analyzer_cmd.extend(['-nobuiltininc',
+                         '-isystem', resource_inc])
diff --git a/libcodechecker/analyze/analyzers/analyzer_clang_tidy.py b/libcodechecker/analyze/analyzers/analyzer_clang_tidy.py
index 2eb44ef7..40680c62 100644
--- a/libcodechecker/analyze/analyzers/analyzer_clang_tidy.py
+++ b/libcodechecker/analyze/analyzers/analyzer_clang_tidy.py
@@ -94,8 +94,6 @@ class ClangTidy(analyzer_base.SourceAnalyzer):
 
             analyzer_cmd.append('-Qunused-arguments')
 
-            extend_analyzer_cmd_with_resource_dir(analyzer_cmd,
-                                                  config.compiler_resource_dir)
             # Set language.
             analyzer_cmd.extend(['-x', self.buildaction.lang])
 
@@ -106,6 +104,9 @@ class ClangTidy(analyzer_base.SourceAnalyzer):
 
             analyzer_cmd.extend(self.buildaction.compiler_includes)
 
+            extend_analyzer_cmd_with_resource_dir(analyzer_cmd,
+                                                  config.compiler_resource_dir)
+
             return analyzer_cmd
 
         except Exception as ex:
diff --git a/libcodechecker/analyze/analyzers/analyzer_clangsa.py b/libcodechecker/analyze/analyzers/analyzer_clangsa.py
index fd5e2bce..a8e9232c 100644
--- a/libcodechecker/analyze/analyzers/analyzer_clangsa.py
+++ b/libcodechecker/analyze/analyzers/analyzer_clangsa.py
@@ -90,9 +90,6 @@ class ClangSA(analyzer_base.SourceAnalyzer):
 
             analyzer_cmd = [config.analyzer_binary]
 
-            extend_analyzer_cmd_with_resource_dir(analyzer_cmd,
-                                                  config.compiler_resource_dir)
-
             # Do not warn about the unused gcc/g++ arguments.
             analyzer_cmd.append('-Qunused-arguments')
 
@@ -151,6 +148,9 @@ class ClangSA(analyzer_base.SourceAnalyzer):
 
             analyzer_cmd.extend(self.buildaction.compiler_includes)
 
+            extend_analyzer_cmd_with_resource_dir(analyzer_cmd,
+                                                  config.compiler_resource_dir)
+
             analyzer_cmd.append(self.source_file)
 
             return analyzer_cmd
diff --git a/libcodechecker/analyze/analyzers/analyzer_types.py b/libcodechecker/analyze/analyzers/analyzer_types.py
index b52a7dae..451492cb 100644
--- a/libcodechecker/analyze/analyzers/analyzer_types.py
+++ b/libcodechecker/analyze/analyzers/analyzer_types.py
@@ -266,6 +266,19 @@ def __replace_env_var(cfg_file):
     return replacer
 
 
+def __get_compiler_resource_dir(context, analyzer_binary):
+    resource_dir = ''
+    if len(context.compiler_resource_dir) > 0:
+        resource_dir = context.compiler_resource_dir
+    # If not set then ask the binary for the resource dir.
+    else:
+        # Can be None if Clang is too old.
+        resource_dir = host_check.get_resource_dir(analyzer_binary)
+        if resource_dir is None:
+            resource_dir = ""
+    return resource_dir
+
+
 def __build_clangsa_config_handler(args, context):
     """
     Build the config handler for clang static analyzer.
@@ -275,7 +288,8 @@ def __build_clangsa_config_handler(args, context):
     config_handler = config_handler_clangsa.ClangSAConfigHandler()
     config_handler.analyzer_plugins_dir = context.checker_plugin
     config_handler.analyzer_binary = context.analyzer_binaries.get(CLANG_SA)
-    config_handler.compiler_resource_dir = context.compiler_resource_dir
+    config_handler.compiler_resource_dir =\\
+        __get_compiler_resource_dir(context, config_handler.analyzer_binary)
 
     if 'ctu_phases' in args:
         config_handler.ctu_dir = os.path.join(args.output_path,
@@ -337,7 +351,24 @@ def __build_clang_tidy_config_handler(args, context):
 
     config_handler = config_handler_clang_tidy.ClangTidyConfigHandler()
     config_handler.analyzer_binary = context.analyzer_binaries.get(CLANG_TIDY)
-    config_handler.compiler_resource_dir = context.compiler_resource_dir
+
+    # FIXME We cannot get the resource dir from the clang-tidy binary,
+    # therefore now we get a clang binary which is a sibling of the clang-tidy.
+    # TODO Support "clang-tidy -print-resource-dir" .
+    check_env = analyzer_env.get_check_env(context.path_env_extra,
+                                           context.ld_lib_path_extra)
+    # Overwrite PATH to contain only the parent of the clang binary.
+    if os.path.isabs(config_handler.analyzer_binary):
+        check_env['PATH'] = os.path.dirname(config_handler.analyzer_binary)
+    clang_bin = analyzer_clangsa.ClangSA.resolve_missing_binary('clang',
+                                                                check_env)
+    if os.path.isfile(clang_bin):
+        config_handler.compiler_resource_dir =\\
+            __get_compiler_resource_dir(context, clang_bin)
+    else:
+        config_handler.compiler_resource_dir =\\
+            __get_compiler_resource_dir(context,
+                                        config_handler.analyzer_binary)
 
     try:
         with open(args.tidy_args_cfg_file, 'rb') as tidy_cfg:
diff --git a/libcodechecker/analyze/analyzers/ctu_triple_arch.py b/libcodechecker/analyze/analyzers/ctu_triple_arch.py
index 41c2571e..2023ee22 100644
--- a/libcodechecker/analyze/analyzers/ctu_triple_arch.py
+++ b/libcodechecker/analyze/analyzers/ctu_triple_arch.py
@@ -19,9 +19,9 @@ def get_compile_command(action, config, source='', output=''):
     for other operations. """
 
     cmd = [config.analyzer_binary]
+    cmd.extend(action.compiler_includes)
     extend_analyzer_cmd_with_resource_dir(cmd,
                                           config.compiler_resource_dir)
-    cmd.extend(action.compiler_includes)
     cmd.append('-c')
     cmd.extend(['-x', action.lang])
     cmd.append(config.analyzer_extra_arguments)
diff --git a/libcodechecker/analyze/host_check.py b/libcodechecker/analyze/host_check.py
index 4a10929d..7b5f021e 100644
--- a/libcodechecker/analyze/host_check.py
+++ b/libcodechecker/analyze/host_check.py
@@ -59,3 +59,29 @@ def has_analyzer_feature(clang_bin, feature):
         except OSError:
             LOG.error('Failed to run: "' + ' '.join(cmd) + '"')
             raise
+
+
+def get_resource_dir(clang_bin):
+    """
+    Returns the resource_dir of Clang or None if the switch is not supported by
+    Clang.
+    """
+    cmd = [clang_bin, "-print-resource-dir"]
+    LOG.debug('run: "' + ' '.join(cmd) + '"')
+    try:
+        proc = subprocess.Popen(cmd,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                )
+        out, err = proc.communicate()
+
+        LOG.debug("stdout:\\n" + out)
+        LOG.debug("stderr:\\n" + err)
+
+        if proc.returncode == 0:
+            return out.decode("utf-8").rstrip()
+        else:
+            return None
+    except OSError:
+        LOG.error('Failed to run: "' + ' '.join(cmd) + '"')
+        raise