Repository: google/timesketch

Issue:
Wrong error message in timesketch_importer
When an upload filed it gives out the timesketch timeline description instead of the status.

https://github.com/google/timesketch/blob/3907a5982bcd2b97ee03cdf816adeddceaead34f/importer_client/python/tools/timesketch_importer.py#L478


Discussion:

Resolving patch:
diff --git a/timesketch/lib/analyzers/interface.py b/timesketch/lib/analyzers/interface.py
--- a/timesketch/lib/analyzers/interface.py
+++ b/timesketch/lib/analyzers/interface.py
@@ -19,6 +19,7 @@
 import json
 import logging
 import os
+import random
 import time
 import traceback
 import yaml
@@ -48,6 +49,7 @@
 
 def _flush_datastore_decorator(func):
     """Decorator that flushes the bulk insert queue in the datastore."""
+
     def wrapper(self, *args, **kwargs):
         func_return = func(self, *args, **kwargs)
 
@@ -128,6 +130,7 @@ class Event(object):
         index_name: The name of the Elasticsearch index.
         source: Source document from Elasticsearch.
     """
+
     def __init__(self, event, datastore, sketch=None, analyzer=None):
         """Initialize Event object.
 
@@ -326,6 +329,7 @@ class Sketch(object):
         id: Sketch ID.
         sql_sketch: Instance of a SQLAlchemy Sketch object.
     """
+
     def __init__(self, sketch_id):
         """Initializes a Sketch object.
 
@@ -524,6 +528,7 @@ class AggregationGroup(object):
     Attributes:
         group (SQLAlchemy): Instance of a SQLAlchemy AggregationGroup object.
     """
+
     def __init__(self, aggregation_group):
         """Initializes the AggregationGroup object.
 
@@ -621,6 +626,7 @@ class Story(object):
     Attributes:
         story (SQLAlchemy): Instance of a SQLAlchemy Story object.
     """
+
     def __init__(self, story):
         """Initializes a Story object.
 
@@ -719,7 +725,7 @@ def add_aggregation(self, aggregation, agg_type=''):
             'updated_at': today.isoformat(),
             'parameters': json.dumps(parameter_dict),
             'user': {'username': None},
-            }
+        }
         self._commit(block)
 
     def add_aggregation_group(self, aggregation_group):
@@ -781,7 +787,6 @@ class BaseAnalyzer:
     SECONDS_PER_WAIT = 10
     MAXIMUM_WAITS = 360
 
-
     def __init__(self, index_name, sketch_id, timeline_id=None):
         """Initialize the analyzer object.
 
@@ -936,18 +941,42 @@ def event_stream(
         else:
             timeline_ids = None
 
-        event_generator = self.datastore.search_stream(
-            query_string=query_string,
-            query_filter=query_filter,
-            query_dsl=query_dsl,
-            indices=indices,
-            return_fields=return_fields,
-            enable_scroll=scroll,
-            timeline_ids=timeline_ids
-        )
-        for event in event_generator:
-            yield Event(
-                event, self.datastore, sketch=self.sketch, analyzer=self)
+        # Exponential backoff for the call to Elasticsearch. Sometimes the
+        # cluster can be a bit overloaded and timeout on requests. We want to
+        # retry a few times in order to give the cluster a chance to return
+        # results.
+        backoff_in_seconds = 3
+        retries = 5
+        for x in range(0, retries):
+            try:
+                event_generator = self.datastore.search_stream(
+                    query_string=query_string,
+                    query_filter=query_filter,
+                    query_dsl=query_dsl,
+                    indices=indices,
+                    return_fields=return_fields,
+                    enable_scroll=scroll,
+                    timeline_ids=timeline_ids
+                )
+                for event in event_generator:
+                    yield Event(
+                        event, self.datastore, sketch=self.sketch,
+                        analyzer=self)
+                break  # Query was succesful
+            except elasticsearch.TransportError as e:
+                sleep_seconds = (
+                    backoff_in_seconds * 2 ** x + random.uniform(3, 7))
+                logger.info(
+                    'Attempt: {0:d}/{1:d} sleeping {2:f} for query {3:s}'
+                    .format(x + 1, retries, sleep_seconds, query_string))
+                time.sleep(sleep_seconds)
+
+                if x == retries-1:
+                    logger.error(
+                        'Timeout executing search for {0:s}: {1!s}'
+                        .format(query_string, e), exc_info=True
+                    )
+                    raise
 
     @_flush_datastore_decorator
     def run_wrapper(self, analysis_id):
@@ -1005,6 +1034,24 @@ def run_wrapper(self, analysis_id):
 
         return result
 
+    @classmethod
+    def get_kwargs(cls):
+        """Get keyword arguments needed to instantiate the class.
+        Every analyzer gets the index_name as its first argument from Celery.
+        By default this is the only argument. If your analyzer need more
+        arguments you can override this method and return as a dictionary.
+
+        If you want more than one instance to be created for your analyzer you
+        can return a list of dictionaries with kwargs and each one will be
+        instantiated and registered in Celery. This is neat if you want to run
+        your analyzer with different arguments in parallel.
+
+        Returns:
+            List of keyword argument dicts or empty list if no extra arguments
+            are needed.
+        """
+        return []
+
     def run(self):
         """Entry point for the analyzer."""
         raise NotImplementedError
diff --git a/timesketch/lib/analyzers/sigma_tagger.py b/timesketch/lib/analyzers/sigma_tagger.py
--- a/timesketch/lib/analyzers/sigma_tagger.py
+++ b/timesketch/lib/analyzers/sigma_tagger.py
@@ -2,10 +2,6 @@
 from __future__ import unicode_literals
 
 import logging
-import time
-import elasticsearch
-
-from flask import current_app
 
 from timesketch.lib.analyzers import utils
 
@@ -24,7 +20,19 @@ class SigmaPlugin(interface.BaseAnalyzer):
     DISPLAY_NAME = 'Sigma'
     DESCRIPTION = 'Run pre-defined Sigma rules and tag matching events'
 
-    def run_sigma_rule(self, query, rule_name, tag_list = None):
+    def __init__(self, index_name, sketch_id, timeline_id=None, **kwargs):
+        """Initialize The Sigma Analyzer.
+
+        Args:
+            index_name: Elasticsearch index name
+            sketch_id: Sketch ID
+            timeline_id: The ID of the timeline.
+        """
+        self.index_name = index_name
+        self._rule = kwargs.get('rule')
+        super().__init__(index_name, sketch_id, timeline_id=timeline_id)
+
+    def run_sigma_rule(self, query, rule_name, tag_list=None):
         """Runs a sigma rule and applies the appropriate tags.
 
         Args:
@@ -45,7 +53,7 @@ def run_sigma_rule(self, query, rule_name, tag_list = None):
             event.add_attributes({'ts_sigma_rule': list(set(ts_sigma_rules))})
             ts_ttp = event.source.get('ts_ttp', [])
             for tag in tag_list:
-                # special handling for sigma tags that TS considers TTPS
+                # Special handling for sigma tags that TS considers TTPS
                 # https://car.mitre.org and https://attack.mitre.org
                 if tag.startswith(('attack.', 'car.')):
                     ts_ttp.append(tag)
@@ -66,54 +74,30 @@ def run(self):
 
         tags_applied = {}
         sigma_rule_counter = 0
-        sigma_rules = ts_sigma_lib.get_all_sigma_rules()
-        if sigma_rules is None:
+
+        rule = self._rule
+        if not rule:
             logger.error('No  Sigma rules found. Check SIGMA_RULES_FOLDERS')
+        rule_name = rule.get('title', 'N/A')
         problem_strings = []
         output_strings = []
 
-        for rule in sigma_rules:
-            tags_applied[rule.get('file_name')] = 0
-            try:
-                sigma_rule_counter += 1
-                tagged_events_counter = self.run_sigma_rule(
-                    rule.get('es_query'), rule.get('file_name'),
-                    tag_list=rule.get('tags'))
-                tags_applied[rule.get('file_name')] += tagged_events_counter
-                if sigma_rule_counter % 10 == 0:
-                    logger.debug('Rule {0:d}/{1:d}'.format(
-                        sigma_rule_counter, len(sigma_rules)))
-            except elasticsearch.TransportError as e:
-                sleep_time : int = current_app.config.get(
-                    'SIGMA_TAG_DELAY', 15)
-                logger.error(
-                    'Timeout executing search for {0:s}: '
-                    '{1!s} waiting for {2:d} seconds '
-                    '(https://github.com/google/timesketch/issues/1782)'
-                    .format(
-                        rule.get('file_name'), e, sleep_time), exc_info=True)
-                # this is caused by too many ES queries in short time range
-                # TODO: https://github.com/google/timesketch/issues/1782
-                time.sleep(sleep_time)
-                tagged_events_counter = self.run_sigma_rule(
-                    rule.get('es_query'), rule.get('file_name'),
-                    tag_list=rule.get('tags'))
-                tags_applied[rule.get('file_name')] += tagged_events_counter
-            # Wide exception handling since there are multiple exceptions that
-            # can be raised by the underlying sigma library.
-            except: # pylint: disable=bare-except
-                logger.error(
-                    'Problem with rule in file {0:s}: '.format(
-                        rule.get('file_name')), exc_info=True)
-                problem_strings.append('* {0:s}'.format(
-                    rule.get('file_name')))
-                continue
-
-        total_tagged_events = sum(tags_applied.values())
-        output_strings.append('Applied {0:d} tags'.format(total_tagged_events))
-
-        if total_tagged_events > 0:
-            self.add_sigma_match_view(sigma_rule_counter)
+        tags_applied[rule.get('file_name')] = 0
+        try:
+            sigma_rule_counter += 1
+            tagged_events_counter = self.run_sigma_rule(
+                rule.get('es_query'), rule.get('file_name'),
+                tag_list=rule.get('tags'))
+            tags_applied[rule.get('file_name')] += tagged_events_counter
+        except:  # pylint: disable=bare-except
+            logger.error(
+                'Problem with rule in file {0:s}: '.format(
+                    rule.get('file_name')), exc_info=True)
+            problem_strings.append('* {0:s}'.format(
+                rule.get('file_name')))
+
+        output_strings.append(
+            f'{tagged_events_counter} events tagged for rule [{rule_name}]')
 
         if len(problem_strings) > 0:
             output_strings.append('Problematic rules:')
@@ -159,9 +143,22 @@ def add_sigma_match_view(self, sigma_rule_counter):
         story.add_view(view)
 
 
+    @staticmethod
+    def get_kwargs():
+        """Returns an array of all rules of Timesketch.
+
+        Returns:
+            sigma_rules All Sigma rules
+        """
+        sigma_rules = [
+            {'rule': rule} for rule in ts_sigma_lib.get_all_sigma_rules()
+        ]
+        return sigma_rules
+
 class RulesSigmaPlugin(SigmaPlugin):
     """Sigma plugin to run rules."""
 
     NAME = 'sigma'
 
+
 manager.AnalysisManager.register_analyzer(RulesSigmaPlugin)
diff --git a/timesketch/lib/tasks.py b/timesketch/lib/tasks.py
--- a/timesketch/lib/tasks.py
+++ b/timesketch/lib/tasks.py
@@ -320,7 +320,8 @@ def build_sketch_analysis_pipeline(
         return None, None
 
     if not analyzer_kwargs:
-        analyzer_kwargs = current_app.config.get('ANALYZERS_DEFAULT_KWARGS', {})
+        analyzer_kwargs = current_app.config.get(
+            'ANALYZERS_DEFAULT_KWARGS', {})
 
     if user_id:
         user = User.query.get(user_id)
@@ -331,8 +332,8 @@ def build_sketch_analysis_pipeline(
     analysis_session = AnalysisSession(user, sketch)
 
     analyzers = manager.AnalysisManager.get_analyzers(analyzer_names)
-    for analyzer_name, _ in analyzers:
-        kwargs = analyzer_kwargs.get(analyzer_name, {})
+    for analyzer_name, analyzer_class in analyzers:
+        base_kwargs = analyzer_kwargs.get(analyzer_name, {})
         searchindex = SearchIndex.query.get(searchindex_id)
 
         timeline = None
@@ -343,22 +344,35 @@ def build_sketch_analysis_pipeline(
             timeline = Timeline.query.filter_by(
                 sketch=sketch, searchindex=searchindex).first()
 
-        analysis = Analysis(
-            name=analyzer_name,
-            description=analyzer_name,
-            analyzer_name=analyzer_name,
-            parameters=json.dumps(kwargs),
-            user=user,
-            sketch=sketch,
-            timeline=timeline)
-        analysis.set_status('PENDING')
-        analysis_session.analyses.append(analysis)
-        db_session.add(analysis)
-        db_session.commit()
-
-        tasks.append(run_sketch_analyzer.s(
-            sketch_id, analysis.id, analyzer_name,
-            timeline_id=timeline_id, **kwargs))
+        additional_kwargs = analyzer_class.get_kwargs()
+        if isinstance(additional_kwargs, dict):
+            additional_kwargs = [additional_kwargs]
+
+        kwargs_list = []
+        for _kwargs in additional_kwargs:
+            combined_kwargs = {**base_kwargs, **_kwargs}
+            kwargs_list.append(combined_kwargs)
+
+        if not kwargs_list:
+            kwargs_list = [base_kwargs]
+
+        for kwargs in kwargs_list:
+            analysis = Analysis(
+                name=analyzer_name,
+                description=analyzer_name,
+                analyzer_name=analyzer_name,
+                parameters=json.dumps(kwargs),
+                user=user,
+                sketch=sketch,
+                timeline=timeline)
+            analysis.set_status('PENDING')
+            analysis_session.analyses.append(analysis)
+            db_session.add(analysis)
+            db_session.commit()
+
+            tasks.append(run_sketch_analyzer.s(
+                sketch_id, analysis.id, analyzer_name,
+                timeline_id=timeline_id, **kwargs))
 
     # Commit the analysis session to the database.
     db_session.add(analysis_session)
@@ -408,7 +422,8 @@ def run_email_result_task(index_name, sketch_id=None):
     """
     # We need to get a fake request context so that url_for() will work.
     with current_app.test_request_context():
-        searchindex = SearchIndex.query.filter_by(index_name=index_name).first()
+        searchindex = SearchIndex.query.filter_by(
+            index_name=index_name).first()
         sketch = None
 
         try:
@@ -601,7 +616,6 @@ def run_plaso(
     if elastic_ssl:
         cmd.extend(['--use_ssl'])
 
-
     psort_memory = current_app.config.get('PLASO_UPPER_MEMORY_LIMIT', '')
     if psort_memory:
         cmd.extend(['--process_memory_limit', str(psort_memory)])


Test patch:
diff --git a/test_tools/sigma_verify_rules.py b/test_tools/sigma_verify_rules.py
--- a/test_tools/sigma_verify_rules.py
+++ b/test_tools/sigma_verify_rules.py
@@ -29,7 +29,7 @@
 import pandas as pd
 
 
-from timesketch.lib import sigma_util# pylint: disable=no-name-in-module
+from timesketch.lib import sigma_util  # pylint: disable=no-name-in-module
 
 logger = logging.getLogger('timesketch.test_tool.sigma-verify')
 logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO'))
@@ -69,6 +69,7 @@ def get_sigma_blocklist(blocklist_path='./data/sigma_blocklist.csv'):
 
     return pd.read_csv(blocklist_path)
 
+
 def run_verifier(rules_path, config_file_path, blocklist_path=None):
     """Run an sigma parsing test on a dir and returns results from the run.
 
@@ -106,7 +107,6 @@ def run_verifier(rules_path, config_file_path, blocklist_path=None):
     ignore = get_sigma_blocklist(blocklist_path)
     ignore_list = list(ignore['path'].unique())
 
-
     for dirpath, dirnames, files in os.walk(rules_path):
         if 'deprecated' in [x.lower() for x in dirnames]:
             dirnames.remove('deprecated')
@@ -130,10 +130,11 @@ def run_verifier(rules_path, config_file_path, blocklist_path=None):
                 try:
                     parsed_rule = sigma_util.get_sigma_rule(
                         rule_file_path, sigma_config)
+                    print(parsed_rule)
                 # This except is to keep the unknown exceptions
                 # this function is made to catch them and document
                 # them the broad exception is needed
-                except Exception:# pylint: disable=broad-except
+                except Exception:  # pylint: disable=broad-except
                     logger.debug('Rule parsing error', exc_info=True)
                     return_rules_with_problems.append(rule_file_path)