# Buggy code
def process(self, message):
    handler_function = self.message_types.get(type(message), None)

    if not handler_function:
        msg = 'Handler function for message type "%s" is not defined.' % type(message)
        raise ValueError(msg)

    handler_function(message)


# Traceback (ToozConnectionError)
2019-06-04 17:08:05,405 139749901439856 ERROR consumers [-] VariableMessageQueueConsumer failed to process message: ActionExecutionDB(action={u'notify': {}, u'description': u'This executes a task in the net-task library', u'runner_type': u'python-script', u'tags': [], u'enabled': True, u'name': u'net-task', u'entry_point': u'python/net-task.py', u'metadata_file': u'actions/net-task.yaml', u'output_schema': {}, u'uid': u'action:shigo:net-task', u'parameters': {u'args': {u'position': 1, u'required': False, u'type': u'array', u'description': u'Args to be passed to the task'}, u'task_name': {u'position': 0, u'required': True, u'type': u'string', u'description': u'The name of the task'}, u'kwargs': {u'position': 2, u'required': False, u'type': u'object', u'description': u'Kwargs to be passed to the task'}}, u'ref': u'shigo.net-task', u'id': u'5c1af2cedb95700082de9860', u'pack': u'shigo'}, children=[], context={u'orquesta': {u'workflow_execution_id': u'5cf6a55f5f85230022d4bf06', u'task_id': u'get_nettask_version', u'task_execution_id': u'5cf6a563d0fa0000229108c0', u'task_name': u'get_nettask_version', u'task_route': 0}, u'user': u'st2admin', u'parent': {u'user': u'st2admin', u'execution_id': u'5cf6a55ea036ab0033dc934c', u'pack': u'shigo'}, u'pack': u'shigo'}, delay=None, end_timestamp="2019-06-04 17:08:05.182673+00:00", id=5cf6a563d0fa0000229108c2, liveaction={u'callback': {}, u'runner_info': {u'hostname': u'4051aa3ae5dd', u'pid': 34}, u'parameters': {u'task_name': u'net_task.version'}, u'action': u'shigo.net-task', u'action_is_workflow': False, u'id': u'5cf6a563d0fa0000229108c1'}, log=[{u'status': u'requested', u'timestamp': datetime.datetime(2019, 6, 4, 17, 7, 47, 469000, tzinfo=<bson.tz_util.FixedOffset object at 0x7f1a0f8212d0>)}, {u'status': u'scheduled', u'timestamp': datetime.datetime(2019, 6, 4, 17, 7, 48, 199000, tzinfo=<bson.tz_util.FixedOffset object at 0x7f1a0f8212d0>)}, {u'status': u'running', u'timestamp': datetime.datetime(2019, 6, 4, 17, 7, 48, 332000, tzinfo=<bson.tz_util.FixedOffset object at 0x7f1a0f8212d0>)}, {u'status': u'succeeded', u'timestamp': datetime.datetime(2019, 6, 4, 17, 8, 5, 217000, tzinfo=<bson.tz_util.FixedOffset object at 0x7f1a0f8212d0>)}], parameters={u'task_name': u'net_task.version'}, parent="5cf6a55ea036ab0033dc934c", result={u'result': {u'task-id': u'8726afab-915d-4f97-8205-5c7f73ffa4ae', u'state': u'SUCCESS', u'result': {u'optical-tech': u'0.6.4', u'network-state-service': u'0.4.3', u'net-devices2': u'1.4.2', u'kusto-proxy': u'0.1.1', u'phynet-credentials': u'0.8.2', u'device_grouping': u'0.0.2', u'starlab': u'0.1.60', u'wan-tech': u'0.7.7', u'icm-connector': u'0.8.12', u'azconfigure-tasks': u'1.0.13', u'celery': u'4.2.1', u'quality-checks': u'0.1.22', u'atom': u'0.9.44', u'net-task-contrib': u'0.4.72', u'net-assist': u'0.1.10', u'fabric-tools': u'0.1.6', u'ask-mode-service': u'0.3.2', u'safety-check-service': u'0.2.8', u'apache-airflow': u'2.0.0dev0+3.incubating', u'net-task': u'0.13.0'}}, u'exit_code': 0, u'stderr': u"st2.actions.python.NetTaskWrapper: DEBUG    json data: {}\nst2.actions.python.NetTaskWrapper: INFO     NetTask UUID: 8726afab-915d-4f97-8205-5c7f73ffa4ae\nst2.actions.python.NetTaskWrapper: INFO     task net_task.version completed, result is {u'task-id': u'8726afab-915d-4f97-8205-5c7f73ffa4ae', u'state': u'SUCCESS', u'result': {u'net-task': u'0.13.0', u'device_grouping': u'0.0.2', u'kusto-proxy': u'0.1.1', u'phynet-credentials': u'0.8.2', u'quality-checks': u'0.1.22', u'azconfigure-tasks': u'1.0.13', u'starlab': u'0.1.60', u'ask-mode-service': u'0.3.2', u'icm-connector': u'0.8.12', u'apache-airflow': u'2.0.0dev0+3.incubating', u'celery': u'4.2.1', u'net-task-contrib': u'0.4.72', u'network-state-service': u'0.4.3', u'wan-tech': u'0.7.7', u'net-assist': u'0.1.10', u'atom': u'0.9.44', u'safety-check-service': u'0.2.8', u'fabric-tools': u'0.1.6', u'net-devices2': u'1.4.2', u'optical-tech': u'0.6.4'}}\nst2.actions.python.NetTaskWrapper: WARNING  task did not return data in (status, result) format, as expected by stackstorm\n", u'stdout': u''}, rule={}, runner={u'runner_module': u'python_runner', u'uid': u'runner_type:python-script', u'runner_package': u'python_runner', u'enabled': True, u'name': u'python-script', u'output_key': u'result', u'output_schema': {u'exit_code': {u'required': True, u'type': u'integer'}, u'result': {u'anyOf': [{u'type': u'object'}, {u'type': u'string'}, {u'type': u'integer'}, {u'type': u'number'}, {u'type': u'boolean'}, {u'type': u'array'}, {u'type': u'null'}]}, u'stderr': {u'required': True, u'type': u'string'}, u'stdout': {u'required': True, u'type': u'string'}}, u'runner_parameters': {u'debug': {u'default': False, u'required': False, u'type': u'boolean', u'description': u'Enable runner debug mode.'}, u'content_version': {u'required': False, u'type': u'string', u'description': u'Git revision of the pack content to use for this action execution (git commit sha / tag / branch). Only applies to packs which are git repositories.'}, u'log_level': {u'default': u'DEBUG', u'enum': [u'AUDIT', u'CRITICAL', u'ERROR', u'WARNING', u'INFO', u'DEBUG'], u'type': u'string', u'description': u'Default log level for Python runner actions.'}, u'timeout': {u'default': 600, u'type': u'integer', u'description': u"Action timeout in seconds. Action will get killed if it doesn't finish in timeout seconds."}, u'env': {u'type': u'object', u'description': u'Environment variables which will be available to the script.'}}, u'id': u'5c1af28f26395d0024fc2741', u'description': u'A runner for launching python actions.'}, start_timestamp="2019-06-04 17:07:47.443217+00:00", status="succeeded", task_execution="5cf6a563d0fa0000229108c0", trigger={}, trigger_instance={}, trigger_type={}, web_url="https://befdd34ce4a4/#/history/5cf6a563d0fa0000229108c2/general", workflow_execution="5cf6a55f5f85230022d4bf06")
Traceback (most recent call last):
File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2common/transport/consumers.py", line 72, in _process_message
self._handler.process(body)
File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2actions/workflows/workflows.py", line 83, in process
handler_function(message)
File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2actions/workflows/workflows.py", line 61, in handle_action_execution_with_instrumentation
return self.handle_action_execution(ac_ex_db=ac_ex_db)
File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2actions/workflows/workflows.py", line 134, in handle_action_execution
wf_svc.handle_action_execution_completion(ac_ex_db)
File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2common/services/workflows.py", line 733, in handle_action_execution_completion
with coord_svc.get_coordinator(start_heart=True).get_lock(wf_ex_id):
File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/tooz/locking.py", line 52, in __enter__
acquired = self.acquire(*args, **kwargs)
File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/tooz/drivers/redis.py", line 93, in acquire
return acquired
File "/usr/lib/python2.7/contextlib.py", line 35, in __exit__
self.gen.throw(type, value, traceback)
File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/tooz/drivers/redis.py", line 51, in _translate_failures
cause=e)
File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/tooz/utils.py", line 225, in raise_with_cause
excutils.raise_with_cause(exc_cls, message, *args, **kwargs)
File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/oslo_utils/excutils.py", line 143, in raise_with_cause
six.raise_from(exc_cls(message, *args, **kwargs), kwargs.get('cause'))
File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/six.py", line 737, in raise_from
raise value
ToozConnectionError: Error while reading from socket: (u'Connection closed by server.',)

# Fixed code
def process(self, message):
    handler_function = self.message_types.get(type(message), None)

    if not handler_function:
        msg = 'Handler function for message type "%s" is not defined.' % type(message)
        raise ValueError(msg)

    try:
        handler_function(message)
    except Exception as e:
        # If the exception is caused by DB connection error, then the following
        # error handling routine will fail as well because it will try to update
        # the database and fail the workflow execution gracefully. In this case,
        # the garbage collector will find and cancel these workflow executions.
        self.fail_workflow_execution(message, e)