Repository: sscpac/statick

Issue:
Lizard does not accept flags from config
It appears that the `lizard` plugin doesn't use any user-specified flags from the `config.yaml` file. Checking the code at [lizard_tool_plugin.py](https://github.com/sscpac/statick/blob/master/statick_tool/plugins/tool/lizard_tool_plugin.py), the subprocess call in `scan` shows that user flags aren't apparently implemented...
Investigate using lizard as Python module
A tool plugin for lizard already exists. That plugin uses `subprocess` to run the actual tool and get the output. However, there is a Python module available for lizard. We should investigate using the Python module instead of `subprocess`.

https://github.com/terryyin/lizard#using-lizard-as-python-module

Discussion:
jhdcs: So it looks like there's something hardcoded in Statick that's expecting a supbprocess to happen. At least as far as `tox` is concerned. I'm not really familiar with that area of the code, so some insight would be lovely...

Work on this will probably resume after the weekend.

Patch:
diff --git a/statick_tool/plugins/tool/lizard_tool_plugin.py b/statick_tool/plugins/tool/lizard_tool_plugin.py
index 46d79d9..f2e1d55 100644
--- a/statick_tool/plugins/tool/lizard_tool_plugin.py
+++ b/statick_tool/plugins/tool/lizard_tool_plugin.py
@@ -1,15 +1,22 @@
 """Apply lizard tool and gather results."""
+import io
 import re
-import subprocess
+from contextlib import redirect_stdout
 from typing import List, Match, Optional, Pattern
 
+import lizard
+
 from statick_tool.issue import Issue
 from statick_tool.package import Package
 from statick_tool.tool_plugin import ToolPlugin
 
 
 class LizardToolPlugin(ToolPlugin):
-    """Apply Lizard tool and gather results."""
+    """Apply Lizard tool and gather results.
+
+    Note: The `-f/--input_file`, `-o/--output_file`, and `-Edumpcomments`
+    options are unsupported.
+    """
 
     def get_name(self) -> str:
         """Get name of tool."""
@@ -20,31 +27,45 @@ class LizardToolPlugin(ToolPlugin):
         if not package.path:
             return []
 
-        try:
-            output = subprocess.check_output(
-                ["lizard", "-w", package.path], universal_newlines=True
-            )
+        # The following is a modification of lizard.py's main().
+        raw_user_flags = (
+            [lizard.__file__] + [package.path] + self.get_user_flags(level)
+        )  # Leading lizard file name is required.
+
+        # Make sure we log warnings.
+        if "-w" not in raw_user_flags:
+            raw_user_flags += ["-w"]
 
-        except subprocess.CalledProcessError as ex:
-            if ex.returncode == 1:
-                output = ex.output
-            else:
-                print("Problem {}".format(ex.returncode))
-                print("{}".format(ex.output))
-                return None
+        # Make sure unsupported arguments are not included.
+        user_flags = self.remove_invalid_flags(raw_user_flags)
 
-        except OSError as ex:
-            print("Couldn't find lizard executable! ({})".format(ex))
-            return None
+        options = lizard.parse_args(user_flags)
+        printer = options.printer or lizard.print_result
+        schema = lizard.OutputScheme(options.extensions)
+        schema.patch_for_extensions()
 
-        if self.plugin_context and self.plugin_context.args.show_tool_output:
-            print("{}".format(output))
+        result = lizard.analyze(
+            options.paths,
+            options.exclude,
+            options.working_threads,
+            options.extensions,
+            options.languages,
+        )
+        lizard_output = io.StringIO()
+        with redirect_stdout(lizard_output):
+            printer(result, options, schema, lizard.AllResult)
+        output = lizard_output.getvalue()
+        lizard.print_extension_results(options.extensions)
 
-        if self.plugin_context and self.plugin_context.args.output_directory:
-            with open(self.get_name() + ".log", "w") as fid:
-                fid.write(output)
+        if self.plugin_context:
+            if self.plugin_context.args.show_tool_output:
+                print("{}".format(output))
+            if self.plugin_context.args.output_directory:
+                with open(self.get_name() + ".log", "w") as fid:
+                    fid.write(output)
 
         issues = self.parse_output(output)
+
         return issues
 
     def parse_output(self, output: str) -> List[Issue]:
@@ -66,3 +87,15 @@ class LizardToolPlugin(ToolPlugin):
                 issues.append(issue)
 
         return issues
+
+    def remove_invalid_flags(self, flag_list: List[str]) -> List[str]:
+        """Filter out all disabled flags."""
+        return [x for x in flag_list if self.valid_flag(x)]
+
+    @classmethod
+    def valid_flag(cls, flag: str) -> bool:
+        """Indicate if passed flag is invalid."""
+        disabled_flags = ["-f", "--input_file", "-o", "--output_file", "-Edumpcomments"]
+        if flag in disabled_flags:
+            return False
+        return True
diff --git a/statick_ws b/statick_ws
index 1f4b02f..d9ac791 100755
--- a/statick_ws
+++ b/statick_ws
@@ -47,6 +47,9 @@ def main() -> None:  # pylint: disable=too-many-locals, too-many-branches, too-m
 
     packages = []
     for root, dirs, files in os.walk(parsed_args.path):
+        if "CATKIN_IGNORE" in files or "AMENT_IGNORE" in files:
+            dirs.clear()
+            continue
         for sub_dir in dirs:
             full_dir = os.path.join(root, sub_dir)
             files = os.listdir(full_dir)


Test patch:
diff --git a/tests/plugins/tool/bandit_tool_plugin/test_bandit_tool_plugin.py b/tests/plugins/tool/bandit_tool_plugin/test_bandit_tool_plugin.py
index f7123a0..d254cc6 100644
--- a/tests/plugins/tool/bandit_tool_plugin/test_bandit_tool_plugin.py
+++ b/tests/plugins/tool/bandit_tool_plugin/test_bandit_tool_plugin.py
@@ -130,9 +130,9 @@ def test_bandit_tool_plugin_scan_wrong_binary():
 @mock.patch("statick_tool.plugins.tool.bandit_tool_plugin.subprocess.check_output")
 def test_bandit_tool_plugin_scan_empty_oserror(mock_subprocess_check_output):
     """
-    Test what happens an OSError is hit (such as if bandit doesn't exist)
+    Test what happens an OSError is hit (such as if bandit doesn't exist).
 
-    Expected result: issues is an empty list
+    Expected result: issues is None
     """
     mock_subprocess_check_output.side_effect = OSError("mocked error")
     btp = setup_bandit_tool_plugin()
@@ -151,7 +151,7 @@ def test_bandit_tool_plugin_scan_empty_calledprocesserror(mock_subprocess_check_
     """
     Test what happens when a CalledProcessError is hit (such as if bandit encounters an error).
 
-    Expected result: issues is an empty list
+    Expected result: issues is None
     """
     mock_subprocess_check_output.side_effect = subprocess.CalledProcessError(
         2, "", output="mocked error"
diff --git a/tests/plugins/tool/lizard_tool_plugin/rsc/config.yaml b/tests/plugins/tool/lizard_tool_plugin/rsc/config.yaml
new file mode 100644
index 0000000..736563a
--- /dev/null
+++ b/tests/plugins/tool/lizard_tool_plugin/rsc/config.yaml
@@ -0,0 +1,9 @@
+levels:
+  default:
+    tool:
+      lizard:
+        flags: ""
+  any_silent:
+    tool:
+      lizard:
+        flags: "-Edumpcomments"
diff --git a/tests/plugins/tool/lizard_tool_plugin/rsc/profile.yaml b/tests/plugins/tool/lizard_tool_plugin/rsc/profile.yaml
new file mode 100644
index 0000000..9056aaf
--- /dev/null
+++ b/tests/plugins/tool/lizard_tool_plugin/rsc/profile.yaml
@@ -0,0 +1,1 @@
+default: default
diff --git a/tests/plugins/tool/lizard_tool_plugin/test_lizard_tool_plugin.py b/tests/plugins/tool/lizard_tool_plugin/test_lizard_tool_plugin.py
index e8278a7..6c2f89c 100644
--- a/tests/plugins/tool/lizard_tool_plugin/test_lizard_tool_plugin.py
+++ b/tests/plugins/tool/lizard_tool_plugin/test_lizard_tool_plugin.py
@@ -1,9 +1,7 @@
 """Unit tests for the lizard plugin."""
 import argparse
 import os
-import subprocess
 
-import mock
 import pytest
 from yapsy.PluginManager import PluginManager
 
@@ -16,7 +14,7 @@ from statick_tool.resources import Resources
 from statick_tool.tool_plugin import ToolPlugin
 
 
-def setup_lizard_tool_plugin():
+def setup_lizard_tool_plugin(custom_rsc_path=None):
     """Initialize and return an instance of the lizard plugin."""
     arg_parser = argparse.ArgumentParser()
     arg_parser.add_argument(
@@ -26,9 +24,12 @@ def setup_lizard_tool_plugin():
         help="Show tool output",
     )
 
-    resources = Resources(
-        [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]
-    )
+    if custom_rsc_path is not None:
+        resources = Resources([custom_rsc_path])
+    else:
+        resources = Resources(
+            [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]
+        )
     config = Config(resources.get_file("config.yaml"))
     plugin_context = PluginContext(arg_parser.parse_args([]), resources, config)
     plugin_context.args.output_directory = os.path.dirname(__file__)
@@ -128,37 +129,13 @@ def test_lizard_tool_plugin_scan_missing_fields():
     assert not issues
 
 
-@mock.patch("statick_tool.plugins.tool.lizard_tool_plugin.subprocess.check_output")
-def test_lizard_tool_plugin_scan_oserror(mock_subprocess_check_output):
-    """
-    Test what happens when an OSError is raised (usually means lizard doesn't exist).
-
-    Expected result: issues is None
-    """
-    mock_subprocess_check_output.side_effect = OSError("mocked error")
-    ltp = setup_lizard_tool_plugin()
-    package = Package(
-        "valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
-    )
-    package["src_dir"] = os.path.join(os.path.dirname(__file__), "valid_package")
-    issues = ltp.scan(package, "level")
-    assert issues is None
-
-
-@mock.patch("statick_tool.plugins.tool.lizard_tool_plugin.subprocess.check_output")
-def test_lizard_tool_plugin_scan_calledprocesserror(mock_subprocess_check_output):
+def test_lizard_tool_plugin_valid_flag_filter():
     """
-    Test what happens when a CalledProcessError is raised (usually means lizard hit an error).
+    Test that valid flag filter removes unwanted flags.
 
-    Expected result: issues is None
+    Expected result: flag list is empty
     """
-    mock_subprocess_check_output.side_effect = subprocess.CalledProcessError(
-        2, "", output="mocked error"
-    )
     ltp = setup_lizard_tool_plugin()
-    package = Package(
-        "valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
-    )
-    package["src_dir"] = os.path.join(os.path.dirname(__file__), "valid_package")
-    issues = ltp.scan(package, "level")
-    assert issues is None
+    flag_list = ["-f", "--input_file", "-o", "--output_file", "-Edumpcomments"]
+    filtered_list = ltp.remove_invalid_flags(flag_list)
+    assert not filtered_list