Repository: google/timesketch

Issue:
Rename blocklist.csv to something more neutral
Historically the blocklist.csv was used to track unvetted rules from Sigma. Now the focus has increased and the file should be called something else.

E.g. `sigma_rule_status.csv`or similar.


Discussion:

Resolving patch:
diff --git a/timesketch/lib/sigma_util.py b/timesketch/lib/sigma_util.py
--- a/timesketch/lib/sigma_util.py
+++ b/timesketch/lib/sigma_util.py
@@ -137,8 +137,8 @@ def get_sigma_rules(rule_folder, sigma_config=None):
     """
     return_array = []
 
-    blocklist_path = None
-    ignore = get_sigma_blocklist(blocklist_path)
+    rule_status_list_path = None
+    ignore = get_sigma_rule_status_list(rule_status_list_path)
     ignore_list = list(ignore.loc[ignore['status'] == 'bad']["path"].unique())
     exploratory_list = list(
         ignore.loc[ignore['status'] == 'exploratory']["path"].unique()
@@ -175,7 +175,7 @@ def get_sigma_rules(rule_folder, sigma_config=None):
                 else:
                     parsed_rule.update({'ts_use_in_analyzer': False})
 
-                # try to append any content from the reason field of the blocklist file:
+                # try to append any content from the reason field of the status file:
                 pd.set_option(
                     'display.max_colwidth', 200
                 )  # to avoid comments being truncated
@@ -377,28 +377,28 @@ def _sanitize_query(sigma_rule_query: str) -> str:
 
 
 @lru_cache(maxsize=32)
-def get_sigma_blocklist(blocklist_path=None):
+def get_sigma_rule_status_list(statuslist_path=None):
     """Get a dataframe of sigma rules to ignore.
 
     This includes filenames, paths, ids.
 
     Args:
-        blocklist_path(str): Path to a blocklist file.
+        statuslist_path(str): Path to a status file.
             The default value is None
 
     Returns:
-        Pandas dataframe with blocklist
+        Pandas dataframe with status
 
     Raises:
-        ValueError: Sigma blocklist file is not readabale.
+        ValueError: Sigma status file is not readabale.
     """
 
-    df = pd.read_csv(get_sigma_blocklist_path(blocklist_path))
+    df = pd.read_csv(get_sigma_rule_status_path(statuslist_path))
     if 'bad' in df.columns:
         df.rename(columns={"bad": "status"}, inplace=True)
         logger.warning(
             'Column name "bad" found in {0!s} - please rename to "status"'.format(
-                get_sigma_blocklist_path(blocklist_path)
+                get_sigma_rule_status_path(statuslist_path)
             )
         )
 
@@ -406,57 +406,57 @@ def get_sigma_blocklist(blocklist_path=None):
 
 
 @lru_cache(maxsize=None)
-def get_sigma_blocklist_path(blocklist_path=None):
-    """Checks and returns the Sigma blocklist path.
+def get_sigma_rule_status_path(rule_status_path=None):
+    """Checks and returns the Sigma rule_status path.
 
     This includes filenames, paths, ids.
 
     Args:
-        blocklist_path(str): Path to a blocklist file.
-            The default value is './data/sigma_blocklist.csv'
+        rule_status_path(str): Path to a rule_status file.
+            The default value is './data/sigma_rule_status.csv'
 
     Returns:
         Sigma Blocklist path
 
     Raises:
-        ValueError: Sigma blocklist file is not readabale.
+        ValueError: Sigma rule status file is not readabale.
     """
 
-    if not blocklist_path or blocklist_path == "":
-        blocklist_path = current_app.config.get(
-            "SIGMA_BLOCKLIST_CSV", "./data/sigma_blocklist.csv"
+    if not rule_status_path or rule_status_path == "":
+        rule_status_path = current_app.config.get(
+            "SIGMA_RULE_STATUS_CSV", "./data/sigma_rule_status.csv"
         )
-    if not blocklist_path:
-        raise ValueError("No blocklist_file_path set via param or config file")
+    if not rule_status_path:
+        raise ValueError("No rule_status_path set via param or config file")
 
-    if not os.path.isfile(blocklist_path):
+    if not os.path.isfile(rule_status_path):
         raise ValueError(
             "Unable to open file: [{0:s}] does not exist".format(
-                blocklist_path
+                rule_status_path
             )
         )
 
-    if not os.access(blocklist_path, os.R_OK):
+    if not os.access(rule_status_path, os.R_OK):
         raise ValueError(
             "Unable to open file: [{0:s}], cannot open it for "
-            "read, please check permissions.".format(blocklist_path)
+            "read, please check permissions.".format(rule_status_path)
         )
 
-    return blocklist_path
+    return rule_status_path
 
 
 @lru_cache(maxsize=None)
 def add_problematic_rule(filepath, rule_uuid=None, reason=None):
-    """Adds a problematic rule to the blocklist.csv.
+    """Adds a problematic rule to the sigma_rule_status.csv.
 
     Args:
         filepath: path to the sigma rule that caused problems
         rule_uuid: rule uuid
         reason: optional reason why file is moved
     """
-    blocklist_file_path = get_sigma_blocklist_path()
+    rule_status_file_path = get_sigma_rule_status_path()
 
-    # we only want to store the relative paths in the blocklist file
+    # we only want to store the relative paths in the status file
 
     try:
         sigma_rules_paths = get_sigma_rules_path()
@@ -476,7 +476,7 @@ def add_problematic_rule(filepath, rule_uuid=None, reason=None):
         rule_uuid,
     ]
 
-    with open(blocklist_file_path, "a", encoding="utf-8") as f:
+    with open(rule_status_file_path, "a", encoding="utf-8") as f:
         writer = csv.writer(f)
         writer.writerow(fields)
 


Test patch:
diff --git a/test_tools/sigma_verify_rules.py b/test_tools/sigma_verify_rules.py
--- a/test_tools/sigma_verify_rules.py
+++ b/test_tools/sigma_verify_rules.py
@@ -35,49 +35,51 @@
 logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
 
 
-def get_sigma_blocklist(blocklist_path="./data/sigma_blocklist.csv"):
-    """Get a dataframe of sigma rules to ignore.
+def get_sigma_rule_status(rule_status_path="./data/sigma_rule_status.csv"):
+    """Get a dataframe of sigma rules status.
 
     This includes filenames, paths, ids.
 
     Args:
-        blocklist_path(str): Path to a blocklist file.
-            The default value is './data/sigma_blocklist.csv'
+        rule_status_path(str): Path to a status file.
+            The default value is './data/sigma_rule_status.csv'
 
     Returns:
-        Pandas dataframe with blocklist
+        Pandas dataframe with rule status
 
     Raises:
-        ValueError: Sigma blocklist file is not readabale.
+        ValueError: Sigma rule status file is not readabale.
     """
 
-    if blocklist_path is None or blocklist_path == "":
-        blocklist_path = "./data/sigma_blocklist.csv"
+    if rule_status_path is None or rule_status_path == "":
+        rule_status_path = "./data/sigma_rule_status.csv"
 
-    if not blocklist_path:
-        raise ValueError("No blocklist_file_path set via param or config file")
+    if not rule_status_path:
+        raise ValueError("No rule_status_path set via param or config file")
 
-    if not os.path.isfile(blocklist_path):
+    if not os.path.isfile(rule_status_path):
         raise ValueError(
-            "Unable to open file: [{0:s}], it does not exist.".format(blocklist_path)
+            "Unable to open file: [{0:s}], it does not exist.".format(
+                rule_status_path
+            )
         )
 
-    if not os.access(blocklist_path, os.R_OK):
+    if not os.access(rule_status_path, os.R_OK):
         raise ValueError(
             "Unable to open file: [{0:s}], cannot open it for "
-            "read, please check permissions.".format(blocklist_path)
+            "read, please check permissions.".format(rule_status_path)
         )
 
-    return pd.read_csv(blocklist_path)
+    return pd.read_csv(rule_status_path)
 
 
-def run_verifier(rules_path, config_file_path, blocklist_path=None):
+def run_verifier(rules_path, config_file_path, rule_status_path=None):
     """Run an sigma parsing test on a dir and returns results from the run.
 
     Args:
         rules_path (str): Path to the Sigma rules.
         config_file_path (str): Path to a config file with Sigma mapping data.
-        blocklist_path (str): Optional path to a blocklist file.
+        rule_status_path (str): Optional path to a status file.
             The default value is none.
 
     Raises:
@@ -96,15 +98,19 @@ def run_verifier(rules_path, config_file_path, blocklist_path=None):
         raise IOError("Rules not found at path: {0:s}".format(rules_path))
     if not os.path.isfile(config_file_path):
         raise IOError(
-            "Config file path not found at path: {0:s}".format(config_file_path)
+            "Config file path not found at path: {0:s}".format(
+                config_file_path
+            )
         )
 
-    sigma_config = sigma_util.get_sigma_config_file(config_file=config_file_path)
+    sigma_config = sigma_util.get_sigma_config_file(
+        config_file=config_file_path
+    )
 
     return_verified_rules = []
     return_rules_with_problems = []
 
-    ignore = get_sigma_blocklist(blocklist_path)
+    ignore = get_sigma_rule_status(rule_status_path)
     ignore_list = list(ignore["path"].unique())
 
     for dirpath, dirnames, files in os.walk(rules_path):
@@ -155,7 +161,9 @@ def move_problematic_rule(filepath, move_to_path, reason=None):
         move_to_path: path to move the problematic rules to
         reason: optional reason why file is moved
     """
-    logging.info("Moving the rule: {0:s} to {1:s}".format(filepath, move_to_path))
+    logging.info(
+        "Moving the rule: {0:s} to {1:s}".format(filepath, move_to_path)
+    )
     try:
         os.makedirs(move_to_path, exist_ok=True)
         debug_path = os.path.join(move_to_path, "debug.log")
@@ -183,7 +191,9 @@ def move_problematic_rule(filepath, move_to_path, reason=None):
     )
     epilog = "Remember to feed the tool with proper rule data."
 
-    arguments = argparse.ArgumentParser(description=description, allow_abbrev=True)
+    arguments = argparse.ArgumentParser(
+        description=description, allow_abbrev=True
+    )
     arguments.add_argument(
         "--config_file",
         "--file",
@@ -195,13 +205,13 @@ def move_problematic_rule(filepath, move_to_path, reason=None):
         help=("Path to the file containing the config data to feed sigma "),
     )
     arguments.add_argument(
-        "--blocklist_file",
-        dest="blocklist_file_path",
+        "--rule_status_file",
+        dest="rule_status_path",
         action="store",
         default="",
         type=str,
-        metavar="PATH_TO_BLOCK_FILE",
-        help=("Path to the file containing the blocklist "),
+        metavar="PATH_TO_STATUS_FILE",
+        help=("Path to the file containing the rule status"),
     )
     arguments.add_argument(
         "rules_path",
@@ -211,8 +221,12 @@ def move_problematic_rule(filepath, move_to_path, reason=None):
         metavar="PATH_TO_RULES",
         help="Path to the rules to test.",
     )
-    arguments.add_argument("--debug", action="store_true", help="print debug messages ")
-    arguments.add_argument("--info", action="store_true", help="print info messages ")
+    arguments.add_argument(
+        "--debug", action="store_true", help="print debug messages "
+    )
+    arguments.add_argument(
+        "--info", action="store_true", help="print info messages "
+    )
     arguments.add_argument(
         "--move",
         dest="move_to_path",
@@ -238,18 +252,22 @@ def move_problematic_rule(filepath, move_to_path, reason=None):
         sys.exit(1)
 
     if not os.path.isdir(options.rules_path):
-        print("The path to the rules does not exist ({0:s})".format(options.rules_path))
+        print(
+            "The path to the rules does not exist ({0:s})".format(
+                options.rules_path
+            )
+        )
         sys.exit(1)
 
-    if len(options.blocklist_file_path) > 0:
-        if not os.path.isfile(options.blocklist_file_path):
-            print("Blocklist file not found.")
+    if len(options.rule_status_path) > 0:
+        if not os.path.isfile(options.rule_status_path):
+            print("rule status file not found.")
             sys.exit(1)
 
     sigma_verified_rules, sigma_rules_with_problems = run_verifier(
         rules_path=options.rules_path,
         config_file_path=options.config_file_path,
-        blocklist_path=options.blocklist_file_path,
+        rule_status_path=options.rule_status_path,
     )
 
     if len(sigma_rules_with_problems) > 0:
diff --git a/timesketch/lib/sigma_util_test.py b/timesketch/lib/sigma_util_test.py
--- a/timesketch/lib/sigma_util_test.py
+++ b/timesketch/lib/sigma_util_test.py
@@ -400,11 +400,13 @@ def test_get_sigma_config_file(self):
             sigma_util.get_sigma_config_file("/foo")
         self.assertIsNotNone(sigma_util.get_sigma_config_file())
 
-    def test_get_blocklist_file(self):
+    def test_get_rule_status_file(self):
         """Test getting sigma config file"""
-        self.assertRaises(ValueError, sigma_util.get_sigma_blocklist, "/foo")
+        self.assertRaises(
+            ValueError, sigma_util.get_sigma_rule_status_list, "/foo"
+        )
         self.assertIsNotNone(sigma_util.get_sigma_config_file())
-        blocklist = sigma_util.get_sigma_blocklist()
+        statuslist = sigma_util.get_sigma_rule_status_list()
         # self.assertIn(
         #    'exploratory',
         #    blocklist[
@@ -415,12 +417,13 @@ def test_get_blocklist_file(self):
         #    ]['status'].to_string,
         # )
         self.assertEqual(
-            'bad', blocklist[blocklist.values == 'deprecated']['status'].all()
+            'bad',
+            statuslist[statuslist.values == 'deprecated']['status'].all(),
         )
         self.assertEqual(
             'good',
-            blocklist[
-                blocklist.values
+            statuslist[
+                statuslist.values
                 == 'windows/powershell/powershell_create_local_user.yml'
             ]['status'].all(),
         )