# Buggy code
def process(self):
    event = self.receive_message()
    event_dict = event.to_dict(hierarchical=False)

    for field in self.flatten_fields:
        if field in event_dict:
            val = event_dict[field]
            # if it's a string try to parse it as JSON
            if isinstance(val, str):
                try:
                    val = loads(val)
                except ValueError:
                    pass
            if isinstance(val, Mapping):
                for key, value in val.items():
                    event_dict[field + "_" + key] = value
                event_dict.pop(field)

    # For ES 2.x, replace dots with a specified replacement character
    if self.replacement_char and self.replacement_char != ".":
        event_dict = replace_keys(event_dict, replacement=self.replacement_char)

    self.es.index(
        index=self.get_index(event_dict, default_date=datetime.today().date()),
        doc_type=self.elastic_doctype,
        body=event_dict,
    )
    self.acknowledge_message()


# Traceback (elasticsearch.exceptions.RequestError)
======================================================================
ERROR: Tests whether an input event with a time.observation field (and no time.source field) is indexed according to
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/sebastian/dev/intelmq/intelmq/tests/bots/outputs/elasticsearch/test_output.py", line 200, in test_index_detected_from_time_observation
"rotate_index": "daily"})
File "/home/sebastian/dev/intelmq/intelmq/tests/bots/outputs/elasticsearch/test_output.py", line 249, in base_check_expected_index_created
self.con.indices.put_template(name=parameters['elastic_index'], body=SAMPLE_TEMPLATE)
File "/usr/lib/python3.6/site-packages/elasticsearch/client/utils.py", line 76, in _wrapped
return func(*args, params=params, **kwargs)
File "/usr/lib/python3.6/site-packages/elasticsearch/client/indices.py", line 447, in put_template
name), params=params, body=body)
File "/usr/lib/python3.6/site-packages/elasticsearch/transport.py", line 314, in perform_request
status, headers, data = connection.perform_request(method, url, params, body, headers=headers, ignore=ignore, timeout=timeout)
File "/usr/lib/python3.6/site-packages/elasticsearch/connection/http_urllib3.py", line 161, in perform_request
self._raise_error(response.status, raw_data)
File "/usr/lib/python3.6/site-packages/elasticsearch/connection/base.py", line 125, in _raise_error
raise HTTP_EXCEPTIONS.get(status_code, TransportError)(status_code, error_message, additional_info)
elasticsearch.exceptions.RequestError: TransportError(400, 'mapper_parsing_exception', 'Root mapping definition has unsupported parameters:  [events : {properties={time.observation={type=date}, time.source={type=date}, classification.type={type=keyword}, source.asn={type=integer}, source.ip={type=ip}, feed.name={type=text}}}]')
-------------------- >> begin captured logging << --------------------
urllib3.util.retry: DEBUG: Converted retries value: False -> Retry(total=False, connect=None, read=None, redirect=0, status=None)
urllib3.connectionpool: DEBUG: http://localhost:9200 "HEAD /None-2020-02-02 HTTP/1.1" 404 0
urllib3.util.retry: DEBUG: Converted retries value: False -> Retry(total=False, connect=None, read=None, redirect=0, status=None)
urllib3.connectionpool: DEBUG: http://localhost:9200 "PUT /_template/intelmq HTTP/1.1" 400 942
elasticsearch: WARNING: PUT http://localhost:9200/_template/intelmq [status:400 request:0.013s]
elasticsearch: DEBUG: > {"mappings": {"events": {"properties": {"time.observation": {"type": "date"}, "time.source": {"type": "date"}, "classification.type": {"type": "keyword"}, "source.asn": {"type": "integer"}, "feed.name": {"type": "text"}, "source.ip": {"type": "ip"}}}}, "template": "intelmq-*", "index_patterns": ["intelmq-*"]}
elasticsearch: DEBUG: < {"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters:  [events : {properties={time.observation={type=date}, time.source={type=date}, classification.type={type=keyword}, source.asn={type=integer}, source.ip={type=ip}, feed.name={type=text}}}]"}],"type":"mapper_parsing_exception","reason":"Failed to parse mapping [_doc]: Root mapping definition has unsupported parameters:  [events : {properties={time.observation={type=date}, time.source={type=date}, classification.type={type=keyword}, source.asn={type=integer}, source.ip={type=ip}, feed.name={type=text}}}]","caused_by":{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters:  [events : {properties={time.observation={type=date}, time.source={type=date}, classification.type={type=keyword}, source.asn={type=integer}, source.ip={type=ip}, feed.name={type=text}}}]"}},"status":400}
--------------------- >> end captured logging << ---------------------

======================================================================
ERROR: Tests whether an input event with a time.source field is indexed according to its time.source date.
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/sebastian/dev/intelmq/intelmq/tests/bots/outputs/elasticsearch/test_output.py", line 187, in test_index_detected_from_time_source
"rotate_index": "daily"})
File "/home/sebastian/dev/intelmq/intelmq/tests/bots/outputs/elasticsearch/test_output.py", line 249, in base_check_expected_index_created
self.con.indices.put_template(name=parameters['elastic_index'], body=SAMPLE_TEMPLATE)
File "/usr/lib/python3.6/site-packages/elasticsearch/client/utils.py", line 76, in _wrapped
return func(*args, params=params, **kwargs)
File "/usr/lib/python3.6/site-packages/elasticsearch/client/indices.py", line 447, in put_template
name), params=params, body=body)
File "/usr/lib/python3.6/site-packages/elasticsearch/transport.py", line 314, in perform_request
status, headers, data = connection.perform_request(method, url, params, body, headers=headers, ignore=ignore, timeout=timeout)
File "/usr/lib/python3.6/site-packages/elasticsearch/connection/http_urllib3.py", line 161, in perform_request
self._raise_error(response.status, raw_data)
File "/usr/lib/python3.6/site-packages/elasticsearch/connection/base.py", line 125, in _raise_error
raise HTTP_EXCEPTIONS.get(status_code, TransportError)(status_code, error_message, additional_info)
elasticsearch.exceptions.RequestError: TransportError(400, 'mapper_parsing_exception', 'Root mapping definition has unsupported parameters:  [events : {properties={time.observation={type=date}, time.source={type=date}, classification.type={type=keyword}, source.asn={type=integer}, source.ip={type=ip}, feed.name={type=text}}}]')
-------------------- >> begin captured logging << --------------------
urllib3.util.retry: DEBUG: Converted retries value: False -> Retry(total=False, connect=None, read=None, redirect=0, status=None)
urllib3.connectionpool: DEBUG: http://localhost:9200 "HEAD /None-1869-12-02 HTTP/1.1" 404 0
urllib3.util.retry: DEBUG: Converted retries value: False -> Retry(total=False, connect=None, read=None, redirect=0, status=None)
urllib3.connectionpool: DEBUG: http://localhost:9200 "PUT /_template/intelmq HTTP/1.1" 400 942
elasticsearch: WARNING: PUT http://localhost:9200/_template/intelmq [status:400 request:0.013s]
elasticsearch: DEBUG: > {"mappings": {"events": {"properties": {"time.observation": {"type": "date"}, "time.source": {"type": "date"}, "classification.type": {"type": "keyword"}, "source.asn": {"type": "integer"}, "feed.name": {"type": "text"}, "source.ip": {"type": "ip"}}}}, "template": "intelmq-*", "index_patterns": ["intelmq-*"]}
elasticsearch: DEBUG: < {"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters:  [events : {properties={time.observation={type=date}, time.source={type=date}, classification.type={type=keyword}, source.asn={type=integer}, source.ip={type=ip}, feed.name={type=text}}}]"}],"type":"mapper_parsing_exception","reason":"Failed to parse mapping [_doc]: Root mapping definition has unsupported parameters:  [events : {properties={time.observation={type=date}, time.source={type=date}, classification.type={type=keyword}, source.asn={type=integer}, source.ip={type=ip}, feed.name={type=text}}}]","caused_by":{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters:  [events : {properties={time.observation={type=date}, time.source={type=date}, classification.type={type=keyword}, source.asn={type=integer}, source.ip={type=ip}, feed.name={type=text}}}]"}},"status":400}
--------------------- >> end captured logging << ---------------------

======================================================================
ERROR: Tests whether get_index returns an expected default date
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/sebastian/dev/intelmq/intelmq/tests/bots/outputs/elasticsearch/test_output.py", line 225, in test_index_falls_back_to_default_date
parameters=parameters)
File "/home/sebastian/dev/intelmq/intelmq/tests/bots/outputs/elasticsearch/test_output.py", line 249, in base_check_expected_index_created
self.con.indices.put_template(name=parameters['elastic_index'], body=SAMPLE_TEMPLATE)
File "/usr/lib/python3.6/site-packages/elasticsearch/client/utils.py", line 76, in _wrapped
return func(*args, params=params, **kwargs)
File "/usr/lib/python3.6/site-packages/elasticsearch/client/indices.py", line 447, in put_template
name), params=params, body=body)
File "/usr/lib/python3.6/site-packages/elasticsearch/transport.py", line 314, in perform_request
status, headers, data = connection.perform_request(method, url, params, body, headers=headers, ignore=ignore, timeout=timeout)
File "/usr/lib/python3.6/site-packages/elasticsearch/connection/http_urllib3.py", line 161, in perform_request
self._raise_error(response.status, raw_data)
File "/usr/lib/python3.6/site-packages/elasticsearch/connection/base.py", line 125, in _raise_error
raise HTTP_EXCEPTIONS.get(status_code, TransportError)(status_code, error_message, additional_info)
elasticsearch.exceptions.RequestError: TransportError(400, 'mapper_parsing_exception', 'Root mapping definition has unsupported parameters:  [events : {properties={time.observation={type=date}, time.source={type=date}, classification.type={type=keyword}, source.asn={type=integer}, source.ip={type=ip}, feed.name={type=text}}}]')
-------------------- >> begin captured logging << --------------------
urllib3.util.retry: DEBUG: Converted retries value: False -> Retry(total=False, connect=None, read=None, redirect=0, status=None)
urllib3.connectionpool: DEBUG: http://localhost:9200 "HEAD /intelmq-2018-09-09 HTTP/1.1" 404 0
urllib3.util.retry: DEBUG: Converted retries value: False -> Retry(total=False, connect=None, read=None, redirect=0, status=None)
urllib3.connectionpool: DEBUG: http://localhost:9200 "PUT /_template/intelmq HTTP/1.1" 400 942
elasticsearch: WARNING: PUT http://localhost:9200/_template/intelmq [status:400 request:0.011s]
elasticsearch: DEBUG: > {"mappings": {"events": {"properties": {"time.observation": {"type": "date"}, "time.source": {"type": "date"}, "classification.type": {"type": "keyword"}, "source.asn": {"type": "integer"}, "feed.name": {"type": "text"}, "source.ip": {"type": "ip"}}}}, "template": "intelmq-*", "index_patterns": ["intelmq-*"]}
elasticsearch: DEBUG: < {"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters:  [events : {properties={time.observation={type=date}, time.source={type=date}, classification.type={type=keyword}, source.asn={type=integer}, source.ip={type=ip}, feed.name={type=text}}}]"}],"type":"mapper_parsing_exception","reason":"Failed to parse mapping [_doc]: Root mapping definition has unsupported parameters:  [events : {properties={time.observation={type=date}, time.source={type=date}, classification.type={type=keyword}, source.asn={type=integer}, source.ip={type=ip}, feed.name={type=text}}}]","caused_by":{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters:  [events : {properties={time.observation={type=date}, time.source={type=date}, classification.type={type=keyword}, source.asn={type=integer}, source.ip={type=ip}, feed.name={type=text}}}]"}},"status":400}
--------------------- >> end captured logging << ---------------------

======================================================================
ERROR: Tests whether get_index returns an expected default string
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/sebastian/dev/intelmq/intelmq/tests/bots/outputs/elasticsearch/test_output.py", line 236, in test_index_falls_back_to_default_string
"rotate_index": "daily"})
File "/home/sebastian/dev/intelmq/intelmq/lib/test.py", line 234, in prepare_bot
self.bot = self.bot_reference(self.bot_id)
File "/home/sebastian/dev/intelmq/intelmq/lib/bot.py", line 158, in __init__
self.init()
File "/home/sebastian/dev/intelmq/intelmq/bots/outputs/elasticsearch/output.py", line 99, in init
"Have you created the template?".format(self.elastic_index))
RuntimeError: No template with the name 'intelmq' exists on the Elasticsearch host, but 'rotate_index' is set. Have you created the template?
-------------------- >> begin captured logging << --------------------
test-bot: INFO: ElasticsearchOutputBot initialized with id test-bot and intelmq 2.2.0.alpha.1 and python 3.6.9 (default, Oct 29 2019, 10:39:36) [GCC] as process 4883.
test-bot: INFO: Bot is starting.
test-bot: INFO: Looking for Index template 'intelmq'.
urllib3.util.retry: DEBUG: Converted retries value: False -> Retry(total=False, connect=None, read=None, redirect=0, status=None)
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): 127.0.0.1:9200
urllib3.connectionpool: DEBUG: http://127.0.0.1:9200 "HEAD /_template/intelmq HTTP/1.1" 404 0
test-bot: ERROR: Bot initialization failed.
Traceback (most recent call last):
File "/home/sebastian/dev/intelmq/intelmq/lib/bot.py", line 158, in __init__
self.init()
File "/home/sebastian/dev/intelmq/intelmq/bots/outputs/elasticsearch/output.py", line 99, in init
"Have you created the template?".format(self.elastic_index))
RuntimeError: No template with the name 'intelmq' exists on the Elasticsearch host, but 'rotate_index' is set. Have you created the template?
test-bot: INFO: Bot stopped.
--------------------- >> end captured logging << ---------------------

----------------------------------------------------------------------
Ran 9 tests in 2.276s

FAILED (errors=4)

# Fixed code
def process(self):
    event = self.receive_message()
    event_dict = event.to_dict(hierarchical=False)

    for field in self.flatten_fields:
        if field in event_dict:
            val = event_dict[field]
            # if it's a string try to parse it as JSON
            if isinstance(val, str):
                try:
                    val = loads(val)
                except ValueError:
                    pass
            if isinstance(val, Mapping):
                for key, value in val.items():
                    event_dict[field + "_" + key] = value
                event_dict.pop(field)

    # For ES 2.x, replace dots with a specified replacement character
    if self.replacement_char and self.replacement_char != ".":
        event_dict = replace_keys(event_dict, replacement=self.replacement_char)

    self.es.index(
        index=self.get_index(event_dict, default_date=datetime.today().date()),
        body=event_dict,
    )
    self.acknowledge_message()