Repository: google/timesketch

Issue:
Sigma ts_comment should not have the line number
Remove the line number in `ts_comment` in https://github.com/google/timesketch/pull/2166 as it makes it hard to write unit tests


Discussion:

Resolving patch:
diff --git a/timesketch/lib/analyzers/sigma_tagger.py b/timesketch/lib/analyzers/sigma_tagger.py
--- a/timesketch/lib/analyzers/sigma_tagger.py
+++ b/timesketch/lib/analyzers/sigma_tagger.py
@@ -47,7 +47,9 @@ def run_sigma_rule(self, query, rule_name, tag_list=None):
             tag_list = []
         return_fields = []
         tagged_events_counter = 0
-        events = self.event_stream(query_string=query, return_fields=return_fields)
+        events = self.event_stream(
+            query_string=query, return_fields=return_fields
+        )
         for event in events:
             ts_sigma_rules = event.source.get("ts_sigma_rule", [])
             ts_sigma_rules.append(rule_name)
@@ -82,7 +84,8 @@ def run(self):
 
         rule = self._rule
         if not rule:
-            logger.error("No  Sigma rules found. Check SIGMA_RULES_FOLDERS")
+            logger.error("No  Sigma rule given.")
+            return "Unable to run, no rule given to the analyzer"
         rule_name = rule.get("title", "N/A")
         problem_strings = []
         output_strings = []
@@ -91,12 +94,16 @@ def run(self):
         try:
             sigma_rule_counter += 1
             tagged_events_counter = self.run_sigma_rule(
-                rule.get("es_query"), rule.get("file_name"), tag_list=rule.get("tags")
+                rule.get("es_query"),
+                rule.get("file_name"),
+                tag_list=rule.get("tags"),
             )
             tags_applied[rule.get("file_name")] += tagged_events_counter
         except:  # pylint: disable=bare-except
             logger.error(
-                "Problem with rule in file {0:s}: ".format(rule.get("file_name")),
+                "Problem with rule in file {0:s}: ".format(
+                    rule.get("file_name")
+                ),
                 exc_info=True,
             )
             problem_strings.append("* {0:s}".format(rule.get("file_name")))
@@ -159,7 +166,11 @@ def get_kwargs():
         Returns:
             sigma_rules All Sigma rules
         """
-        sigma_rules = [{"rule": rule} for rule in ts_sigma_lib.get_all_sigma_rules()]
+        sigma_rules = []
+        for rule in ts_sigma_lib.get_all_sigma_rules():
+            if rule.get('ts_use_in_analyzer') is True:
+                sigma_rules.append({"rule": rule})
+
         return sigma_rules
 
 
diff --git a/timesketch/lib/sigma_util.py b/timesketch/lib/sigma_util.py
--- a/timesketch/lib/sigma_util.py
+++ b/timesketch/lib/sigma_util.py
@@ -19,6 +19,8 @@
 import csv
 import logging
 from datetime import datetime
+import string
+from functools import lru_cache
 import yaml
 import pandas as pd
 
@@ -72,7 +74,9 @@ def get_sigma_config_file(config_file=None):
         sigma_config_file = config_file_read.read()
 
     try:
-        sigma_config = sigma_configuration.SigmaConfiguration(sigma_config_file)
+        sigma_config = sigma_configuration.SigmaConfiguration(
+            sigma_config_file
+        )
     except SigmaConfigParseError:
         logger.error("Parsing error with {0:s}".format(sigma_config_file))
         raise
@@ -91,8 +95,11 @@ def get_sigma_rules_path():
             or the folders are not readabale.
     """
     try:
-        rules_path = current_app.config.get("SIGMA_RULES_FOLDERS", [])
+        rules_path = current_app.config.get(
+            "SIGMA_RULES_FOLDERS", ['/etc/timesketch/sigma/rules/']
+        )
     except RuntimeError as e:
+        logger.error(e)
         raise ValueError("SIGMA_RULES_FOLDERS not found in config file") from e
 
     if not rules_path:
@@ -101,7 +108,9 @@ def get_sigma_rules_path():
     for folder in rules_path:
         if not os.path.isdir(folder):
             raise ValueError(
-                "Unable to open dir: [{0:s}], it does not exist.".format(folder)
+                "Unable to open dir: [{0:s}], it does not exist.".format(
+                    folder
+                )
             )
 
         if not os.access(folder, os.R_OK):
@@ -113,6 +122,7 @@ def get_sigma_rules_path():
     return rules_path
 
 
+@lru_cache(maxsize=10)
 def get_sigma_rules(rule_folder, sigma_config=None):
     """Returns the Sigma rules for a folder including subfolders.
     Args:
@@ -129,7 +139,13 @@ def get_sigma_rules(rule_folder, sigma_config=None):
 
     blocklist_path = None
     ignore = get_sigma_blocklist(blocklist_path)
-    ignore_list = list(ignore["path"].unique())
+    ignore_list = list(ignore.loc[ignore['status'] == 'bad']["path"].unique())
+    exploratory_list = list(
+        ignore.loc[ignore['status'] == 'exploratory']["path"].unique()
+    )
+    to_be_used_in_analyzer = list(
+        ignore.loc[ignore['status'] == 'good']["path"].unique()
+    )
 
     for dirpath, dirnames, files in os.walk(rule_folder):
         if "deprecated" in [x.lower() for x in dirnames]:
@@ -147,11 +163,35 @@ def get_sigma_rules(rule_folder, sigma_config=None):
                     continue
 
                 parsed_rule = get_sigma_rule(rule_file_path, sigma_config)
+
+                if parsed_rule is None:
+                    continue
+
+                # Only assign the ts_use_in_analyzer flag to rules that are cleared
+                if any(x in rule_file_path for x in exploratory_list):
+                    parsed_rule.update({'ts_use_in_analyzer': False})
+                elif any(x in rule_file_path for x in to_be_used_in_analyzer):
+                    parsed_rule.update({'ts_use_in_analyzer': True})
+                else:
+                    parsed_rule.update({'ts_use_in_analyzer': False})
+
+                # try to append any content from the reason field of the blocklist file:
+                pd.set_option(
+                    'display.max_colwidth', 200
+                )  # to avoid comments being truncated
+                if parsed_rule.get('id') in ignore['rule_id'].unique():
+                    comment_string = ignore.loc[
+                        ignore['rule_id'] == parsed_rule.get('id')
+                    ]['reason'].to_string()
+                    comment_string = comment_string.split('    ', 1)[1]
+                    parsed_rule.update({'ts_comment': comment_string})
+
                 if parsed_rule:
                     return_array.append(parsed_rule)
     return return_array
 
 
+@lru_cache(maxsize=None)
 def get_all_sigma_rules():
     """Returns all Sigma rules
 
@@ -172,6 +212,7 @@ def get_all_sigma_rules():
     return sigma_rules
 
 
+@lru_cache(maxsize=10)
 def get_sigma_rule(filepath, sigma_config=None):
     """Returns a JSON represenation for a rule
     Args:
@@ -195,7 +236,9 @@ def get_sigma_rule(filepath, sigma_config=None):
         logger.error("Problem reading the Sigma config", exc_info=True)
         raise ValueError("Problem reading the Sigma config") from e
 
-    sigma_backend = sigma_es.ElasticsearchQuerystringBackend(sigma_conf_obj, {})
+    sigma_backend = sigma_es.ElasticsearchQuerystringBackend(
+        sigma_conf_obj, {}
+    )
 
     try:
         sigma_rules_paths = get_sigma_rules_path()
@@ -209,12 +252,22 @@ def get_sigma_rule(filepath, sigma_config=None):
     if os.path.isdir(filepath):
         raise IsADirectoryError(f"{filepath} is a directory - must be a file")
 
-    abs_path = os.path.abspath(filepath)
+    if os.stat(filepath).st_size == 0:
+        raise ValueError(f"{filepath} file is empty")
 
-    with codecs.open(abs_path, "r", encoding="utf-8", errors="replace") as file:
+    abs_path = os.path.abspath(filepath)
+    parsed_sigma_rules = None
+    with codecs.open(
+        abs_path, "r", encoding="utf-8", errors="replace"
+    ) as file:
         try:
             rule_return = {}
-            rule_yaml_data = yaml.safe_load_all(file.read())
+            rule_file_content = file.read()
+            rule_file_content = sanitize_incoming_sigma_rule_text(
+                rule_file_content
+            )
+            rule_yaml_data = yaml.safe_load_all(rule_file_content)
+
             for doc in rule_yaml_data:
                 rule_return.update(doc)
                 parser = sigma_collection.SigmaCollectionParser(
@@ -251,8 +304,9 @@ def get_sigma_rule(filepath, sigma_config=None):
 
         sigma_es_query = ""
 
-        for sigma_rule in parsed_sigma_rules:
+        assert parsed_sigma_rules is not None
 
+        for sigma_rule in parsed_sigma_rules:
             sigma_es_query = _sanitize_query(sigma_rule)
 
         rule_return.update({"es_query": sigma_es_query})
@@ -316,13 +370,13 @@ def _sanitize_query(sigma_rule_query: str) -> str:
             san.append(el)
 
     sigma_rule_query = " ".join(san)
-
     # above method might create strings that have '' in them, workaround:
     sigma_rule_query = sigma_rule_query.replace('""', '"')
 
     return sigma_rule_query
 
 
+@lru_cache(maxsize=32)
 def get_sigma_blocklist(blocklist_path=None):
     """Get a dataframe of sigma rules to ignore.
 
@@ -339,9 +393,19 @@ def get_sigma_blocklist(blocklist_path=None):
         ValueError: Sigma blocklist file is not readabale.
     """
 
-    return pd.read_csv(get_sigma_blocklist_path(blocklist_path))
+    df = pd.read_csv(get_sigma_blocklist_path(blocklist_path))
+    if 'bad' in df.columns:
+        df.rename(columns={"bad": "status"}, inplace=True)
+        logger.warning(
+            'Column name "bad" found in {0!s} - please rename to "status"'.format(
+                get_sigma_blocklist_path(blocklist_path)
+            )
+        )
 
+    return df
 
+
+@lru_cache(maxsize=None)
 def get_sigma_blocklist_path(blocklist_path=None):
     """Checks and returns the Sigma blocklist path.
 
@@ -357,7 +421,6 @@ def get_sigma_blocklist_path(blocklist_path=None):
     Raises:
         ValueError: Sigma blocklist file is not readabale.
     """
-    logger.error(blocklist_path)
 
     if not blocklist_path or blocklist_path == "":
         blocklist_path = current_app.config.get(
@@ -368,7 +431,9 @@ def get_sigma_blocklist_path(blocklist_path=None):
 
     if not os.path.isfile(blocklist_path):
         raise ValueError(
-            "Unable to open file: [{0:s}] does not exist".format(blocklist_path)
+            "Unable to open file: [{0:s}] does not exist".format(
+                blocklist_path
+            )
         )
 
     if not os.access(blocklist_path, os.R_OK):
@@ -380,6 +445,7 @@ def get_sigma_blocklist_path(blocklist_path=None):
     return blocklist_path
 
 
+@lru_cache(maxsize=None)
 def add_problematic_rule(filepath, rule_uuid=None, reason=None):
     """Adds a problematic rule to the blocklist.csv.
 
@@ -401,7 +467,7 @@ def add_problematic_rule(filepath, rule_uuid=None, reason=None):
         for rule_path in sigma_rules_paths:
             file_relpath = os.path.relpath(filepath, rule_path)
 
-    # path,bad,reason,last_ckecked,rule_id
+    # path,status,reason,last_ckecked,rule_id
     fields = [
         file_relpath,
         "bad",
@@ -415,6 +481,25 @@ def add_problematic_rule(filepath, rule_uuid=None, reason=None):
         writer.writerow(fields)
 
 
+def sanitize_incoming_sigma_rule_text(rule_text: string):
+    """Removes things that are not supportd in Timesketch
+    right now as early as possible
+
+    Args:
+        rule_text: Text of the sigma rule to be parsed
+
+    Returns:
+        Cleaned version of the rule_text
+
+    """
+
+    rule_text = rule_text.replace('|endswith', '')
+    rule_text = rule_text.replace('|startswith', '')
+
+    return rule_text
+
+
+@lru_cache(maxsize=8)
 def get_sigma_rule_by_text(rule_text, sigma_config=None):
     """Returns a JSON represenation for a rule
 
@@ -442,9 +527,14 @@ def get_sigma_rule_by_text(rule_text, sigma_config=None):
         logger.error("Problem reading the Sigma config", exc_info=True)
         raise ValueError("Problem reading the Sigma config") from e
 
-    sigma_backend = sigma_es.ElasticsearchQuerystringBackend(sigma_conf_obj, {})
+    sigma_backend = sigma_es.ElasticsearchQuerystringBackend(
+        sigma_conf_obj, {}
+    )
+
+    rule_text = sanitize_incoming_sigma_rule_text(rule_text)
 
     rule_return = {}
+    parsed_sigma_rules = None
     # TODO check if input validation is needed / useful.
     try:
         rule_yaml_data = yaml.safe_load_all(rule_text)
@@ -469,6 +559,8 @@ def get_sigma_rule_by_text(rule_text, sigma_config=None):
         logger.error("Yaml parsing error rule {0!s}".format(exception))
         raise
 
+    assert parsed_sigma_rules is not None
+
     sigma_es_query = ""
 
     for sigma_rule in parsed_sigma_rules:


Test patch:
diff --git a/timesketch/api/v1/resources_test.py b/timesketch/api/v1/resources_test.py
--- a/timesketch/api/v1/resources_test.py
+++ b/timesketch/api/v1/resources_test.py
@@ -73,7 +73,9 @@ class SketchResourceTest(BaseTest):
 
     resource_url = "/api/v1/sketches/1/"
 
-    @mock.patch("timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore)
+    @mock.patch(
+        "timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore
+    )
     def test_sketch_resource(self):
         """Authenticated request to get a sketch."""
         self.login()
@@ -230,7 +232,9 @@ class ExploreResourceTest(BaseTest):
         ],
     }
 
-    @mock.patch("timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore)
+    @mock.patch(
+        "timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore
+    )
     def test_search(self):
         """Authenticated request to query the datastore."""
         self.login()
@@ -253,7 +257,9 @@ class AggregationExploreResourceTest(BaseTest):
 
     resource_url = "/api/v1/sketches/1/aggregation/explore/"
 
-    @mock.patch("timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore)
+    @mock.patch(
+        "timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore
+    )
     def test_heatmap_aggregation(self):
         """Authenticated request to get aggregation requests."""
         self.login()
@@ -285,7 +291,9 @@ class EventResourceTest(BaseTest):
         }
     }
 
-    @mock.patch("timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore)
+    @mock.patch(
+        "timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore
+    )
     def test_get_event(self):
         """Authenticated request to get an event from the datastore."""
         self.login()
@@ -295,7 +303,9 @@ def test_get_event(self):
         self.assertDictContainsSubset(self.expected_response, response.json)
         self.assert200(response)
 
-    @mock.patch("timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore)
+    @mock.patch(
+        "timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore
+    )
     def test_invalid_index(self):
         """
         Authenticated request to get an event from the datastore, but in the
@@ -313,7 +323,9 @@ class EventAnnotationResourceTest(BaseTest):
 
     resource_url = "/api/v1/sketches/1/event/annotate/"
 
-    @mock.patch("timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore)
+    @mock.patch(
+        "timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore
+    )
     def test_post_annotate_resource(self):
         """Authenticated request to create an annotation."""
         self.login()
@@ -356,11 +368,15 @@ class SearchIndexResourceTest(BaseTest):
 
     resource_url = "/api/v1/searchindices/"
 
-    @mock.patch("timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore)
+    @mock.patch(
+        "timesketch.api.v1.resources.OpenSearchDataStore", MockDataStore
+    )
     def test_post_create_searchindex(self):
         """Authenticated request to create a searchindex."""
         self.login()
-        data = dict(searchindex_name="test3", es_index_name="test3", public=False)
+        data = dict(
+            searchindex_name="test3", es_index_name="test3", public=False
+        )
         response = self.client.post(
             self.resource_url,
             data=json.dumps(data),
@@ -429,30 +445,28 @@ class SigmaListResourceTest(BaseTest):
         "meta": {"current_user": "test1", "rules_count": 1},
         "objects": [
             {
-                "author": "Alexander Jaeger",
-                "date": "2020/06/26",
-                "description": "Detects suspicious installation of ZMap",
-                "detection": {
-                    "condition": "keywords",
-                    "keywords": ["*apt-get install zmap*"],
+                'author': 'Alexander Jaeger',
+                'date': '2020/06/26',
+                'description': 'Detects suspicious installation of ZMap',
+                'detection': {
+                    'condition': 'keywords',
+                    'keywords': ['*apt-get install zmap*'],
                 },
-                "es_query": '(data_type:("shell:zsh:history" OR '
-                '"bash:history:command" OR '
-                '"apt:history:line" OR '
-                '"selinux:line") AND '
-                '"apt-get install zmap")',
-                "falsepositives": ["Unknown"],
-                "file_name": "lnx_susp_zmap.yml",
-                "file_relpath": "lnx_susp_zmap.yml",
-                "id": "5266a592-b793-11ea-b3de-0242ac130004",
-                "level": "high",
-                "logsource": {"product": "linux", "service": "shell"},
-                "modified": "2020/06/26",
-                "tags": ["attack.discovery", "attack.t1046"],
-                "references": [
-                    "https://rmusser.net/docs/ATT&CK-Stuff/ATT&CK/Discovery.html"  # pylint: disable=line-too-long
+                'es_query': '(data_type:("shell:zsh:history" OR "bash:history:command" OR "apt:history:line" OR "selinux:line") AND "apt-get install zmap")',  # pylint: disable=line-too-long
+                'falsepositives': ['Unknown'],
+                'file_name': 'lnx_susp_zmap.yml',
+                'file_relpath': 'lnx_susp_zmap.yml',
+                'id': '5266a592-b793-11ea-b3de-0242ac130004',
+                'level': 'high',
+                'logsource': {'product': 'linux', 'service': 'shell'},
+                'modified': '2020/06/26',
+                'references': [
+                    'https://rmusser.net/docs/ATT&CK-Stuff/ATT&CK/Discovery.html'  # pylint: disable=line-too-long
                 ],
-                "title": "Suspicious Installation of ZMap",
+                'tags': ['attack.discovery', 'attack.t1046'],
+                'title': 'Suspicious Installation of ZMap',
+                'ts_comment': 'Part of Timesketch repo',
+                'ts_use_in_analyzer': True,
             }
         ],
     }
diff --git a/timesketch/lib/analyzers/sigma_tagger_test.py b/timesketch/lib/analyzers/sigma_tagger_test.py
--- a/timesketch/lib/analyzers/sigma_tagger_test.py
+++ b/timesketch/lib/analyzers/sigma_tagger_test.py
@@ -16,9 +16,23 @@ def __init__(self, *args, **kwargs):
         self.test_index = "test_index"
 
     # Mock the OpenSearch datastore.
-    @mock.patch("timesketch.lib.analyzers.interface.OpenSearchDataStore", MockDataStore)
+    @mock.patch(
+        "timesketch.lib.analyzers.interface.OpenSearchDataStore", MockDataStore
+    )
     def test_analyzer(self):
         """Test analyzer."""
-        # TODO: Add more tests
 
-        _ = sigma_tagger.RulesSigmaPlugin(sketch_id=1, index_name=self.test_index)
+        _ = sigma_tagger.RulesSigmaPlugin(
+            sketch_id=1, index_name=self.test_index
+        )
+
+    def test_get_kwargs(self):
+        analyzer_init = sigma_tagger.RulesSigmaPlugin(
+            sketch_id=1, index_name=self.test_index
+        )
+        rules = analyzer_init.get_kwargs()
+        self.assertIsNotNone(rules)
+        self.assertGreaterEqual(len(rules), 1)
+        self.assertIn("zmap", rules[0]['rule'].get("es_query"))
+        self.assertIn("b793", rules[0]['rule'].get("id"))
+        self.assertEqual(rules[0]['rule'].get("ts_use_in_analyzer"), True)
diff --git a/timesketch/lib/sigma_util_test.py b/timesketch/lib/sigma_util_test.py
--- a/timesketch/lib/sigma_util_test.py
+++ b/timesketch/lib/sigma_util_test.py
@@ -18,6 +18,7 @@
 from __future__ import unicode_literals
 
 import datetime
+import os
 
 from sigma.parser import exceptions as sigma_exceptions
 
@@ -129,6 +130,140 @@
     condition: keywords
 """
 
+MOCK_SIGMA_RULE_STARTSWITH_ENDSWITH = r"""
+title: MIXED LSASS Mock rule to test various combinations
+id: 5d2c62fe-3cbb-47c3-88e1-88ef73503a9f
+status: experimental
+author: Alexander Jaeger
+date: 2022/04/26
+references:
+    - https://github.com/SigmaHQ/sigma/blob/e4c8e62ba6a32f8966ab4216a15dd393af4ef3a3/rules/windows/process_access/proc_access_win_rare_proc_access_lsass.yml
+logsource:
+    category: process_access
+    product: windows
+detection:
+    selection:
+        TargetImage|endswith: '\foobar.exe'
+        GrantedAccess|endswith: '10'
+    # Absolute paths to programs that cause false positives
+    filter1:
+        SourceImage:
+            - 'C:\WINDOWS\system32\foo.exe'
+            - 'C:\Program Files\Malwarebytes\Anti-Malware\lorem.exe'
+            - 'C:\PROGRAMDATA\MALWAREBYTES\MBAMSERVICE\ipsum\gfdsa.exe'
+            - 'C:\WINDOWS\system32\taskhostv.exe'
+            - 'C:\Users\\*\AppData\Local\Programs\Microsoft VS Code\Microsoft.exe'
+            - 'C:\Program Files\Windows Defender\MsMpFoo.exe'
+            - 'C:\Windows\SysWOW64\gciexec.exe'
+            - 'C:\Windows\System32\gciexec.exe'
+            - 'C:\Windows\System32\lsoss.exe'
+            - 'C:\WINDOWS\System32\loremmon.exe'
+    # Windows Defender
+    filter2:
+        SourceImage|startswith: 'C:\ProgramData\Microsoft\Windows Avangers\'
+        SourceImage|endswith: '\MsMpFoo.exe'
+    # Microsoft Eating Services
+    filter3:
+        SourceImage|startswith: 'C:\Program Files\WindowsApps\'
+        SourceImage|endswith: '\EatingServices.exe'
+    # Process Drinker
+    filter4:
+        SourceImage|endswith:
+            - '\PROCDRINK64.EXE'
+            - '\PROCDRINK.EXE'
+    # BMware Tools
+    filter5:
+        SourceImage|startswith: 'C:\ProgramData\BMware\BMware Tools\'
+        SourceImage|endswith: '\bmtoolsd.exe'
+    # Provirus and MBR agents
+    filter6:
+        SourceImage|startswith:
+            - 'C:\Program Files\'
+            - 'C:\Program Files (x86)\'
+        SourceImage|contains:
+            - 'Provirus'
+    filter7:
+        SourceImage: 'C:\WINDOWS\system32\wbem\vgaprvse.exe'
+    filter8:
+        SourceImage: 'C:\Windows\sysWOW64\wbem\vgaprvse.exe'
+    filter_mcbfee:
+        SourceImage: 'C:\Program Files\Common Files\McBfee\ABBSHost\ABBSHOST.exe'
+    filter_prevtron:
+        SourceImage|startswith: 'C:\Windows\Temp\bsgbrd2-agent\'
+        SourceImage|endswith: 
+            - '\hammer64.exe'
+            - '\hammer.exe'
+    # Generic Filter for 0x1410 filter (caused by so many programs like PickBox updates etc.)
+    filter_generic:
+        SourceImage|startswith:
+            - 'C:\Program Files\'
+            - 'C:\Program Files (x86)\'
+            - 'C:\WINDOWS\system32\'
+    filter_localappdata:
+        SourceImage|contains|all:
+            - 'C:\Users\'
+            - '\AppData\Local\'
+        SourceImage|endswith:
+            - '\Maxisoft AB Cade\Cade.exe'
+            - '\software_influencer_tool.exe'
+            - '\PickUpdate.exe'
+            - '\NBAInstallerService.exe'
+    condition: selection and not 1 of filter*
+fields:
+    - User
+    - SourceImage
+    - GrantedAccess
+falsepositives:
+    - Legitimate software accessing LSASS process for legitimate reason
+level: medium
+"""
+
+SIGMA_MOCK_RULE_ENDSWITH = r"""
+title: foobar through Windows Remote Management
+id: aa35a627-33fb-4d04-abcde-asdasdasdasd
+description: Detects usage of foobar through WinRM protocol by monitoring access to lsass process by foobarhost.exe.
+references:
+    - https://github.com/SigmaHQ/sigma/blob/master/rules/windows/process_access/proc_access_win_mimikatz_trough_winrm.yml
+status: stable
+author: Alexander Jaeger
+date: 2022/04/28
+modified: 2022/04/28
+logsource:
+    category: process_access
+    product: windows
+detection:
+    selection:
+        TargetImage|endswith: '\\lsass.exe'
+        SourceImage: 'C:\\Windows\system32\foobarhost.exe'
+    filter:
+        GrantedAccess: '0x123456'
+    condition: selection and not filter
+"""
+
+SIGMA_MOCK_RULE_TEST4 = r"""
+title: Login with WMI
+id: 5af54681-df95-4c26-854f-2565e13cfab0
+status: stable
+description: Detection of logins performed with WMI
+author: Thomas Patzke
+date: 2019/12/04
+tags:
+    - attack.execution
+    - attack.t1047
+logsource:
+    product: windows
+    service: security
+detection:
+    selection:
+        EventID: 4624
+        ProcessName|endswith: '\WmiPrvSE.exe'
+    condition: selection
+falsepositives:
+    - Monitoring tools
+    - Legitimate system administration
+level: low
+"""
+
 
 class TestSigmaUtilLib(BaseTest):
     """Tests for the sigma support library."""
@@ -152,17 +287,45 @@ def test_sanitize_rule_string(self):
 
         # test that the function does not break regular queries
         self.assertEqual(
-            sigma_util._sanitize_query("*mimikatz* OR *mimikatz.exe* OR *mimilib.dll*"),
+            sigma_util._sanitize_query(
+                "*mimikatz* OR *mimikatz.exe* OR *mimilib.dll*"
+            ),
             "*mimikatz* OR *mimikatz.exe* OR *mimilib.dll*",
         )
 
         test_2 = sigma_util._sanitize_query("(*a:b* OR *c::d*)")
         self.assertEqual(test_2, r'("a:b" OR "c\:\:d")')
+
+        test_3 = sigma_util._sanitize_query(
+            '(xml_string.keyword:"\\foobar.exe" AND GrantedAccess.keyword:"10")'
+        )
+
+        self.assertEqual(
+            test_3, r'(xml_string:"\foobar.exe" AND GrantedAccess:"10")'
+        )
+
+        test_4 = sigma_util._sanitize_query(
+            '(xml_string:C:\\Program Files\\WindowsApps\\\" AND xml_string: "GamingServices.exe)'  # pylint: disable=line-too-long
+        )
+        self.assertIsNotNone(test_4)
         # pylint: enable=protected-access
 
     def test_get_rule_by_text(self):
         """Test getting sigma rule by text."""
 
+        rule = sigma_util.get_sigma_rule_by_text(SIGMA_MOCK_RULE_TEST4)
+
+        self.assertIsNotNone(SIGMA_MOCK_RULE_TEST4)
+        self.assertIsNotNone(rule)
+        self.assertEqual(
+            '(data_type:"windows:evtx:record" AND source_name:("Microsoft-Windows-Security-Auditing" OR "Microsoft-Windows-Eventlog") AND event_identifier:"4624" AND xml_string:"\\\\WmiPrvSE.exe")',  # pylint: disable=line-too-long
+            rule.get("es_query"),
+        )
+
+        rule = sigma_util.get_sigma_rule_by_text(SIGMA_MOCK_RULE_ENDSWITH)
+
+        self.assertIsNotNone(SIGMA_MOCK_RULE_ENDSWITH)
+        self.assertIsNotNone(rule)
         rule = sigma_util.get_sigma_rule_by_text(MOCK_SIGMA_RULE)
 
         self.assertIsNotNone(MOCK_SIGMA_RULE)
@@ -209,12 +372,28 @@ def test_get_rule_by_text(self):
         rule = sigma_util.get_sigma_rule_by_text(MOCK_SIGMA_RULE_DOTS)
         self.assertIsNotNone(MOCK_SIGMA_RULE_DOTS)
         self.assertIsNotNone(rule)
-        self.assertEqual("67b9a11a-03ae-490a-9156-9be9900aaaaa", rule.get("id"))
+        self.assertEqual(
+            "67b9a11a-03ae-490a-9156-9be9900aaaaa", rule.get("id")
+        )
         self.assertEqual(
             r'("aaa:bbb" OR "ccc\:\:ddd")',
             rule.get("es_query"),
         )
 
+        rule = sigma_util.get_sigma_rule_by_text(
+            MOCK_SIGMA_RULE_STARTSWITH_ENDSWITH
+        )
+
+        self.assertIsNotNone(MOCK_SIGMA_RULE_STARTSWITH_ENDSWITH)
+        self.assertIsNotNone(rule)
+        self.assertEqual(
+            "5d2c62fe-3cbb-47c3-88e1-88ef73503a9f", rule.get("id")
+        )
+        self.assertIn(
+            'event_identifier:"10" AND (xml_string:"\\\\foobar.exe" AND xml_string:"10"',  # pylint: disable=line-too-long
+            rule.get("es_query"),
+        )
+
     def test_get_sigma_config_file(self):
         """Test getting sigma config file"""
         with self.assertRaises(ValueError):
@@ -225,6 +404,27 @@ def test_get_blocklist_file(self):
         """Test getting sigma config file"""
         self.assertRaises(ValueError, sigma_util.get_sigma_blocklist, "/foo")
         self.assertIsNotNone(sigma_util.get_sigma_config_file())
+        blocklist = sigma_util.get_sigma_blocklist()
+        # self.assertIn(
+        #    'exploratory',
+        #    blocklist[
+        #        (
+        #            blocklist['rule_id']
+        #            == 'fdf135a2-9241-4f96-a114-bb404948f736'
+        #        )
+        #    ]['status'].to_string,
+        # )
+        self.assertEqual(
+            'bad', blocklist[blocklist.values == 'deprecated']['status'].all()
+        )
+        self.assertEqual(
+            'good',
+            blocklist[
+                blocklist.values
+                == 'windows/powershell/powershell_create_local_user.yml'
+            ]['status'].all(),
+        )
+        self.assertIsNotNone(False)
 
     def test_get_sigma_rule(self):
         """Test getting sigma rule from file"""
@@ -232,7 +432,28 @@ def test_get_sigma_rule(self):
         filepath = "./data/sigma/rules/lnx_susp_zmap.yml"
 
         rule = sigma_util.get_sigma_rule(filepath)
-
         self.assertIsNotNone(rule)
         self.assertIn("zmap", rule.get("es_query"))
         self.assertIn("b793", rule.get("id"))
+
+        # temp write a file with content
+
+        with open(
+            "./data/sigma/rules/temporary.yml", "w+", encoding='utf-8'
+        ) as f:
+            f.write(SIGMA_MOCK_RULE_TEST4)
+        self.assertNotEqual(0, os.stat(f.name).st_size)
+        self.assertIsNotNone(f)
+
+        rule_by_file = sigma_util.get_sigma_rule(f.name)
+
+        # Test that rule from file equals rule from text
+        rule_by_text = sigma_util.get_sigma_rule_by_text(SIGMA_MOCK_RULE_TEST4)
+
+        self.assertEqual(rule_by_file.get('id'), rule_by_text.get('id'))
+        self.assertEqual(
+            rule_by_file.get('es_query'), rule_by_text.get('es_query')
+        )
+
+        # clean up
+        os.remove(f.name)