Repository: google/timesketch

Issue:
Move Sigma rules that cause problems to a dedicated folder that is ignored by TS
Often times people just git clone the upstream Sigma project and expect the rules to work despite documented on multiple places.

One interim solution could be to just move the troubling filed to a dedicated folder and add an entry in ```https://github.com/google/timesketch/blob/master/data/sigma_blocklist.csv``` (note we can not do that by rule uuid because to get the rule uuid the rule must be parse, which in those cases would / could fail.

E.g. move the files to a deprecated sub folder.


Discussion:

Resolving patch:
diff --git a/timesketch/api/v1/resources/sigma.py b/timesketch/api/v1/resources/sigma.py
--- a/timesketch/api/v1/resources/sigma.py
+++ b/timesketch/api/v1/resources/sigma.py
@@ -49,12 +49,12 @@ def get(self):
         try:
             sigma_rules = ts_sigma_lib.get_all_sigma_rules()
 
-        except ValueError:
+        except ValueError as e:
             logger.error('OS Error, unable to get the path to the Sigma rules',
                          exc_info=True)
             abort(
                 HTTP_STATUS_CODE_NOT_FOUND,
-                'OS Error, unable to get the path to the Sigma rules')
+                f'Value Error, {e}')
         # TODO: idea for meta: add a list of folders that have been parsed
         meta = {'current_user': current_user.username,
                 'rules_count': len(sigma_rules)}
@@ -78,12 +78,12 @@ def get(self, rule_uuid):
         try:
             sigma_rules = ts_sigma_lib.get_all_sigma_rules()
 
-        except ValueError:
+        except ValueError as e:
             logger.error('OS Error, unable to get the path to the Sigma rules',
                          exc_info=True)
             abort(
                 HTTP_STATUS_CODE_NOT_FOUND,
-                'OS Error, unable to get the path to the Sigma rules')
+                f'ValueError {e}')
         for rule in sigma_rules:
             if rule is not None:
                 if rule_uuid == rule.get('id'):
diff --git a/timesketch/lib/sigma_util.py b/timesketch/lib/sigma_util.py
--- a/timesketch/lib/sigma_util.py
+++ b/timesketch/lib/sigma_util.py
@@ -15,8 +15,11 @@
 
 import os
 import codecs
+import csv
 import logging
+from datetime import datetime
 import yaml
+import pandas as pd
 
 from flask import current_app
 
@@ -27,6 +30,7 @@
 from sigma.parser import exceptions as sigma_exceptions
 from sigma.config.exceptions import SigmaConfigParseError
 
+
 logger = logging.getLogger('timesketch.lib.sigma')
 
 
@@ -122,6 +126,10 @@ def get_sigma_rules(rule_folder, sigma_config=None):
     """
     return_array = []
 
+    blocklist_path = None
+    ignore = get_sigma_blocklist(blocklist_path)
+    ignore_list = list(ignore['path'].unique())
+
     for dirpath, dirnames, files in os.walk(rule_folder):
         if 'deprecated' in [x.lower() for x in dirnames]:
             dirnames.remove('deprecated')
@@ -133,6 +141,10 @@ def get_sigma_rules(rule_folder, sigma_config=None):
                     continue
 
                 rule_file_path = os.path.join(dirpath, rule_filename)
+
+                if any(x in rule_file_path for x in ignore_list):
+                    continue
+
                 parsed_rule = get_sigma_rule(rule_file_path, sigma_config)
                 if parsed_rule:
                     return_array.append(parsed_rule)
@@ -215,18 +227,24 @@ def get_sigma_rule(filepath, sigma_config=None):
             logger.error(
                 'Error generating rule in file {0:s}: {1!s}'
                 .format(abs_path, exception))
-            raise
+            add_problematic_rule(filepath, doc.get('id'),
+                                 'Parts of the rule not implemented in TS')
+            return None
 
         except sigma_exceptions.SigmaParseError as exception:
             logger.error(
                 'Sigma parsing error generating rule in file {0:s}: {1!s}'
                 .format(abs_path, exception))
-            raise
+            add_problematic_rule(
+                filepath, doc.get('id'), 'sigma_exceptions.SigmaParseError')
+            return None
 
         except yaml.parser.ParserError as exception:
             logger.error(
                 'Yaml parsing error generating rule in file {0:s}: {1!s}'
                 .format(abs_path, exception))
+            add_problematic_rule(
+                filepath, None, 'yaml.parser.ParserError')
             return None
 
         sigma_es_query = ''
@@ -273,6 +291,91 @@ def _sanatize_sigma_rule(sigma_rule_query: str) -> str:
     return sigma_rule_query
 
 
+def get_sigma_blocklist(blocklist_path=None):
+    """Get a dataframe of sigma rules to ignore.
+
+    This includes filenames, paths, ids.
+
+    Args:
+        blocklist_path(str): Path to a blocklist file.
+            The default value is None
+
+    Returns:
+        Pandas dataframe with blocklist
+
+    Raises:
+        ValueError: Sigma blocklist file is not readabale.
+    """
+
+    return pd.read_csv(get_sigma_blocklist_path(blocklist_path))
+
+
+def get_sigma_blocklist_path(blocklist_path=None):
+    """Checks and returns the Sigma blocklist path.
+
+    This includes filenames, paths, ids.
+
+    Args:
+        blocklist_path(str): Path to a blocklist file.
+            The default value is './data/sigma_blocklist.csv'
+
+    Returns:
+        Sigma Blocklist path
+
+    Raises:
+        ValueError: Sigma blocklist file is not readabale.
+    """
+    logger.error(blocklist_path)
+
+    if not blocklist_path or blocklist_path == '':
+        blocklist_path = current_app.config.get(
+            'SIGMA_BLOCKLIST_CSV', './data/sigma_blocklist.csv')
+    if not blocklist_path:
+        raise ValueError('No blocklist_file_path set via param or config file')
+
+    if not os.path.isfile(blocklist_path):
+        raise ValueError(
+            'Unable to open file: [{0:s}], it does not exist.'.format(
+                blocklist_path))
+
+    if not os.access(blocklist_path, os.R_OK):
+        raise ValueError(
+            'Unable to open file: [{0:s}], cannot open it for '
+            'read, please check permissions.'.format(blocklist_path))
+
+    return blocklist_path
+
+
+def add_problematic_rule(filepath, rule_uuid=None, reason=None):
+    """Adds a problematic rule to the blocklist.csv.
+
+    Args:
+        filepath: path to the sigma rule that caused problems
+        rule_uuid: rule uuid
+        reason: optional reason why file is moved
+    """
+    blocklist_file_path = get_sigma_blocklist_path()
+
+    # we only want to store the relative paths in the blocklist file
+
+    try:
+        sigma_rules_paths = get_sigma_rules_path()
+    except ValueError:
+        sigma_rules_paths = None
+
+    if sigma_rules_paths:
+        for rule_path in sigma_rules_paths:
+            file_relpath = os.path.relpath(filepath, rule_path)
+
+    # path,bad,reason,last_ckecked,rule_id
+    fields = [file_relpath, 'bad', reason,
+              datetime.now().strftime('%Y-%m-%d'), rule_uuid]
+
+    with open(blocklist_file_path, 'a') as f:
+        writer = csv.writer(f)
+        writer.writerow(fields)
+
+
 def get_sigma_rule_by_text(rule_text, sigma_config=None):
     """Returns a JSON represenation for a rule
 


Test patch:
diff --git a/timesketch/lib/sigma_util_test.py b/timesketch/lib/sigma_util_test.py
--- a/timesketch/lib/sigma_util_test.py
+++ b/timesketch/lib/sigma_util_test.py
@@ -79,6 +79,11 @@ def test_get_sigma_config_file(self):
         self.assertRaises(ValueError, sigma_util.get_sigma_config_file, '/foo')
         self.assertIsNotNone(sigma_util.get_sigma_config_file())
 
+    def test_get_blocklist_file(self):
+        """Test getting sigma config file"""
+        self.assertRaises(ValueError, sigma_util.get_sigma_blocklist, '/foo')
+        self.assertIsNotNone(sigma_util.get_sigma_config_file())
+
     def test_get_sigma_rule(self):
         """Test getting sigma rule from file"""