Repository: bimpression/sosw

Issue:
Ecology: Design the mechanisms
- Get from properties of Labourer his dependencies.
- Get acceptable Values
- Call CloudWatch manager
- Calculate the colour status


Comments:

Patch:
diff --git a/README.md b/README.md
index b3a2627..4145fa5 100644
--- a/README.md
+++ b/README.md
@@ -34,18 +34,24 @@ $ pytest ./sosw/test/suite_3_6_unit.py
 ### Contribution Guidelines
 
 #### Release cycle
-- Master branch commits are automatically packaged and published to PyPI.
-- Branches for staging versions follow the pattern: `X_X_X`
-- Make your pull requests to the staging branch with highest number
-- Latest documentation is compiled from branch `docme`. It should be up to date with latest **staging** branch, not the master. Make PRs with documentation change directly to `docme`.
+- We follow both [Semantic Versioning](https://semver.org/) pattern
+  and [PEP440](https://www.python.org/dev/peps/pep-0440/) recommendations where comply
+- Master branch commits (merges) are automatically packaged and published to PyPI.
+- Branches for planned staging versions follow the pattern: `X_Y_Z` (Major.Minor.Micro)
+- Make your pull requests to the latest staging branch (with highest number)
+- Latest documentation is compiled from branch `docme`.
+  It should be up to date with latest **staging** branch, not the master.
+  Make PRs with documentation change directly to `docme`.
 
 #### Code formatting
-Follow [PEP8](https://www.python.org/dev/peps/pep-0008/), but both classes and functions are padded with 2 empty lines.
+Follow [PEP8](https://www.python.org/dev/peps/pep-0008/), but:
+- both classes and functions are padded with 2 empty lines
+- dictionaries are value-alligned
 
 #### Initialization
 1. Fork the repository: https://github.com/bimpression/sosw
 2. Register Account in AWS: [SignUp](https://portal.aws.amazon.com/billing/signup#/start)
-3. Run `pipenv sync –dev`to setup your virtual environment and download the required dependencies
+3. Run `pipenv sync –dev` to setup your virtual environment and download the required dependencies
 4. Create DynamoDB Tables: 
     - You can find the CloudFormation template for the databases [in the example](https://raw.githubusercontent.com/bimpression/sosw/docme/docs/yaml/sosw-shared-dynamodb.yaml).
     - If you are not familiar with CloudFormation, we highly recommend at least learning the basics from [the tutorial](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/GettingStarted.Walkthrough.html).
diff --git a/docs/orchestrator.rst b/docs/orchestrator.rst
index 7ca2af4..22f2c83 100644
--- a/docs/orchestrator.rst
+++ b/docs/orchestrator.rst
@@ -49,6 +49,31 @@ The following diagram represents the basic Task Workflow initiated by the Orches
            'some_function': {
                'arn':                          f"arn:aws:lambda:us-west-2:737060422660:function:some_function",
                'max_simultaneous_invocations': 10,
+               'health_metrics':               {
+                   'SomeDBCPU': {
+                       'details':                     {
+                           'Name':       'CPUUtilization',
+                           'Namespace':  'AWS/RDS',
+                           'Period':     60,
+                           'Statistics': ['Average'],
+                           'Dimensions': [
+                               {
+                                   'Name':  'DBInstanceIdentifier',
+                                   'Value': 'YOUR-DB'
+                               },
+                           ],
+                       },
+
+                       # These is the mapping of how the Labourer should "feel" about this metric.
+                       # See EcologyManager.ECO_STATUSES.
+                       # This is just a mapping ``ECO_STATUS: value`` using ``feeling_comparison_operator``.
+                       'feelings':                    {
+                           3: 50,
+                           4: 25,
+                       },
+                       'feeling_comparison_operator': '<='
+                   },
+               },
            },
        },
    }
diff --git a/sosw/components/helpers.py b/sosw/components/helpers.py
index 7de6b05..0f26f79 100644
--- a/sosw/components/helpers.py
+++ b/sosw/components/helpers.py
@@ -719,17 +719,28 @@ def make_hash(o):
     Makes a hash from a dictionary, list, tuple or set to any level, that contains
     only other hashable types (including any lists, tuples, sets, and
     dictionaries).
+
+    Original idea from this user:
     https://stackoverflow.com/users/660554/jomido
+
+    Plus some upgrades to work with sets and dicts having different types of keys appropriately.
+    See source unittests of this function for some more details.
     """
 
-    if isinstance(o, (set, tuple, list)):
+    if isinstance(o, (tuple, list)):
         return tuple([make_hash(e) for e in o])
 
+    # Set should be sorted (by hashes of elements) before returns
+    elif isinstance(o, set):
+        return tuple(sorted([make_hash(e) for e in o]))
+
     elif not isinstance(o, dict):
         return hash(o)
 
-    new_o = deepcopy(o)
-    for k, v in new_o.items():
-        new_o[k] = make_hash(v)
+    # We are left with a dictionary
+    new_o = dict()
+    for k, v in o.items():
+        # hash both keys and values to make sure types and order doesn't affect.
+        new_o[make_hash(k)] = make_hash(v)
 
     return hash(tuple(frozenset(sorted(new_o.items()))))
diff --git a/sosw/labourer.py b/sosw/labourer.py
index b9b08dc..09eb93f 100644
--- a/sosw/labourer.py
+++ b/sosw/labourer.py
@@ -18,8 +18,8 @@ logger.setLevel(logging.INFO)
 
 class Labourer:
     ATTRIBUTES = ('id', 'arn')
-    CUSTOM_ATTRIBUTES = ('start', 'invoked', 'expired', 'health', 'max_attempts', 'average_duration', 'max_duration',
-                         'max_simultaneous_invocations', 'arn')
+    CUSTOM_ATTRIBUTES = ('arn', 'start', 'invoked', 'expired', 'health', 'health_metrics', 'average_duration',
+                         'max_duration', 'max_attempts', 'max_simultaneous_invocations')
     id = None
     arn = None
 
diff --git a/sosw/managers/ecology.py b/sosw/managers/ecology.py
index 14ff1e8..007297f 100644
--- a/sosw/managers/ecology.py
+++ b/sosw/managers/ecology.py
@@ -5,16 +5,19 @@ __version__ = "1.0"
 import boto3
 import json
 import logging
+import operator
 import os
 import random
 import time
 
 from collections import defaultdict
-from typing import Dict, List, Optional
+from collections import OrderedDict
+from typing import Dict, List, Optional, Union
 
 from sosw.app import Processor
 from sosw.labourer import Labourer
 from sosw.components.benchmark import benchmark
+from sosw.components.helpers import make_hash
 from sosw.managers.task import TaskManager
 
 
@@ -34,7 +37,9 @@ class EcologyManager(Processor):
     DEFAULT_CONFIG = {}
 
     running_tasks = defaultdict(int)
+    health_metrics: Dict = None
     task_client: TaskManager = None  # Will be Circular import! Careful!
+    cloudwatch_client: boto3.client = None
 
 
     def __init__(self, *args, **kwargs):
@@ -42,7 +47,7 @@ class EcologyManager(Processor):
 
 
     def __call__(self, event):
-        raise NotImplemented
+        raise NotImplementedError
 
 
     def register_task_manager(self, task_manager: TaskManager):
@@ -60,16 +65,82 @@ class EcologyManager(Processor):
         logger.info("Reset cache of running_tasks counter in EcologyManager")
         self.running_tasks = defaultdict(int)
 
+        logger.info("Reset cache of health_metrics in EcologyManager")
+        self.health_metrics = dict()
+
 
     @property
     def eco_statuses(self):
         return [x[0] for x in ECO_STATUSES]
 
 
+    def fetch_metric_stats(self, **kwargs):
+
+        result = self.cloudwatch_client.get_metric_statistics(**kwargs)
+
+        return result
+
+
     def get_labourer_status(self, labourer: Labourer) -> int:
-        """ FIXME """
-        return 4
-        # return random.choice(self.eco_statuses)
+        """
+        Get the worst (lowest) health status according to preconfigured health metrics of the Labourer.
+
+        .. _ECO_STATUSES:
+
+        Current ECO_STATUSES:
+
+        - (0, 'Bad')
+        - (1, 'Poor')
+        - (2, 'Moderate')
+        - (3, 'Good')
+        - (4, 'High')
+        """
+
+        health = max(map(lambda x: x[0], ECO_STATUSES))
+
+        for health_metric in getattr(labourer, 'health_metrics', dict()).values():
+
+            metric_hash = make_hash(health_metric['details'])
+            if metric_hash not in self.health_metrics:
+                self.health_metrics[metric_hash] = self.fetch_metric_stats(**health_metric['details'])
+                logger.info(f"Updated the cache of Ecology metric {metric_hash} - {health_metric} "
+                            f"with {self.health_metrics[metric_hash]}")
+
+            value = self.health_metrics[metric_hash]
+            logger.debug(f"Ecology metric {metric_hash} has {value}")
+
+            health = min(health, self.get_health(value, metric=health_metric))
+
+        logger.info(f"Ecology health of Labourer {labourer} is {health}")
+
+        return health
+
+
+    def get_health(self, value: Union[int, float], metric: Dict) -> int:
+        """
+        Checks the value against the health_metric configuration.
+        """
+
+        op = getattr(operator, metric.get('feeling_comparison_operator'))
+
+        # Find the first configured feeling from the map that does not comply.
+        # Order and validate the feelings
+        feelings = OrderedDict([(key, metric['feelings'][key])
+                                for key in sorted(metric['feelings'].keys(), reverse=True)])
+
+        last_target = 0
+        for health, target in feelings.items():
+            if op(target, last_target):
+                raise ValueError(f"Order of values if feelings is invalid and doesn't match expected eco statuses: "
+                                 f"{feelings.items()}. Failed: {last_target} not "
+                                 f"{metric.get('feeling_comparison_operator')} {target}")
+
+            if op(value, target):
+                return health
+
+            last_target = target
+
+        return 0
 
 
     def count_running_tasks_for_labourer(self, labourer: Labourer) -> int:
@@ -85,7 +156,8 @@ class EcologyManager(Processor):
 
         if labourer.id not in self.running_tasks.keys():
             self.running_tasks[labourer.id] = self.task_client.get_count_of_running_tasks_for_labourer(labourer)
-            logger.debug(f"EcologyManager.count_running_tasks_for_labourer() recalculated cache for Labourer {labourer}")
+            logger.debug(f"EcologyManager.count_running_tasks_for_labourer() recalculated cache for Labourer "
+                         f"{labourer}")
 
         logger.debug(f"EcologyManager.count_running_tasks_for_labourer() returns: {self.running_tasks[labourer.id]}")
         return self.running_tasks[labourer.id]
diff --git a/sosw/managers/task.py b/sosw/managers/task.py
index cd19e44..303b6c1 100644
--- a/sosw/managers/task.py
+++ b/sosw/managers/task.py
@@ -17,6 +17,7 @@ from sosw.app import Processor
 from sosw.components.benchmark import benchmark
 from sosw.components.dynamo_db import DynamoDbClient
 from sosw.components.helpers import first_or_none
+# from sosw.managers.ecology import EcologyManager
 from sosw.labourer import Labourer
 
 
@@ -38,11 +39,11 @@ class TaskManager(Processor):
     """
 
     DEFAULT_CONFIG = {
-        'init_clients': ['DynamoDb', 'lambda', 'Ecology'],
-        'dynamo_db_config': {
+        'init_clients':                            ['DynamoDb', 'lambda', 'Ecology'],
+        'dynamo_db_config':                        {
             'table_name':       'sosw_tasks',
             'index_greenfield': 'sosw_tasks_greenfield',
-            'row_mapper': {
+            'row_mapper':       {
                 'task_id':             'S',
                 'labourer_id':         'S',
                 'created_at':          'N',
@@ -70,9 +71,37 @@ class TaskManager(Processor):
         'greenfield_task_step':                    1000,
         'labourers':                               {
             # 'some_function': {
-            #     'arn': 'arn:aws:lambda:us-west-2:0000000000:function:some_function',
+            #     'arn':                          'arn:aws:lambda:us-west-2:0000000000:function:some_function',
             #     'max_simultaneous_invocations': 10,
-            # }
+            #     # Health metrics for this Labourer should be stored in a dictionary.
+            #     'health_metrics':               {
+            #     # Name of the metric is just for human readability (probaly some future GUI interfaces),
+            #         'SomeDBCPU': {
+            #             # The value must have ``'details'`` as a dict with kwargs for CloudWatch client.
+            #             'details':                     {
+            #                 'Name':       'CPUUtilization',
+            #                 'Namespace':  'AWS/RDS',
+            #                 'Period':     60,
+            #                 'Statistics': ['Average'],
+            #                 'Dimensions': [
+            #                     {
+            #                         'Name':  'DBInstanceIdentifier',
+            #                         'Value': 'YOUR-DB'
+            #                     },
+            #                 ],
+            #             },
+            #
+            #             # These is the mapping of how the Labourer should "feel" about this metric.
+            #             # See EcologyManager.ECO_STATUSES.
+            #             # This is just a mapping ``ECO_STATUS: value`` using ``feeling_comparison_operator``.
+            #             'feelings':                    {
+            #                 3: 50,
+            #                 4: 25,
+            #             },
+            #             'feeling_comparison_operator': '<='
+            #         },
+            #     },
+            # },
         },
         'max_attempts':                            3,
         'max_closed_to_analyse_for_duration':      10,
@@ -82,9 +111,10 @@ class TaskManager(Processor):
     __labourers = None
 
     # these clients will be initialized by Processor constructor
+    # ecology_client: EcologyManager = None
     ecology_client = None
     dynamo_db_client: DynamoDbClient = None
-    lambda_client = None
+    lambda_client: boto3.client = None
 
 
     def get_oldest_greenfield_for_labourer(self, labourer: Labourer, reverse: bool = False) -> int:
@@ -165,6 +195,7 @@ class TaskManager(Processor):
             ('invoked', lambda x: x.get_attr('start') + self.config['greenfield_invocation_delta']),
             ('expired', lambda x: x.get_attr('invoked') - (x.duration + x.cooldown)),
             ('health', lambda x: self.ecology_client.get_labourer_status(x)),
+            ('health_metrics', lambda x: _cfg('labourers')[x.id].get('health_metrics')),
             ('max_attempts', lambda x: self.config.get(f'max_attempts_{x.id}') or self.config['max_attempts']),
             ('max_duration', lambda x: self.ecology_client.get_max_labourer_duration(x)),
             ('average_duration', lambda x: self.ecology_client.get_labourer_average_duration(x)),
@@ -538,12 +569,12 @@ class TaskManager(Processor):
         _ = self.get_db_field_name
 
         query_args = {
-            'keys':        {
+            'keys':              {
                 _('labourer_id'): labourer.id,
                 _('greenfield'):  str(time.time()),
             },
-            'comparisons': {_('greenfield'): '>='},
-            'index_name':  self.config['dynamo_db_config']['index_greenfield'],
+            'comparisons':       {_('greenfield'): '>='},
+            'index_name':        self.config['dynamo_db_config']['index_greenfield'],
             'filter_expression': f"attribute_exists {_('completed_at')}",
         }
 
@@ -611,7 +642,7 @@ class TaskManager(Processor):
 
         for task in tasks:
             assert task[_('labourer_id')] == labourer.id, f"Task labourer_id must be {labourer.id}, " \
-                f"bad value: {task[_('labourer_id')]}"
+                                                          f"bad value: {task[_('labourer_id')]}"
 
         lowest_greenfield = self.get_oldest_greenfield_for_labourer(labourer)
 
@@ -637,6 +668,7 @@ class TaskManager(Processor):
 
             self.stats['due_for_retry_tasks'] += 1
 
+
     @benchmark
     def get_average_labourer_duration(self, labourer: Labourer) -> int:
         """


Test patch:
diff --git a/sosw/components/test/unit/test_helpers.py b/sosw/components/test/unit/test_helpers.py
index cbb18b7..7d0f762 100644
--- a/sosw/components/test/unit/test_helpers.py
+++ b/sosw/components/test/unit/test_helpers.py
@@ -576,11 +576,11 @@ class helpers_UnitTestCase(unittest.TestCase):
             (("olleh", "hello"), False),
             (("hello", "hello"), True),
             (({1: 'a', 2: 'b'}, {2: 'b', 1: 'a'}), True),  # Unordered Dictionary
-            (({1: 'a', 2: {'2a': {'set', 42}}}, {1: 'a', 2: {'2a': {42, 'set'}}}), True),  # Nested Dictionary
+            (({1: 'a', 'bar': {'2a': {'set', 42}}}, {'bar': {'2a': {42, 'set'}}, 1: 'a'}), True),  # Nested Dictionary
         ]
 
         for test, expected in TESTS:
-            self.assertEqual(make_hash(test[0]) == make_hash(test[1]), expected)
+            self.assertEqual(make_hash(test[0]) == make_hash(test[1]), expected, f"Failed specific test: {test}")
 
 
 if __name__ == '__main__':
diff --git a/sosw/managers/test/unit/test_ecology.py b/sosw/managers/test/unit/test_ecology.py
index 90f552f..c4154cb 100644
--- a/sosw/managers/test/unit/test_ecology.py
+++ b/sosw/managers/test/unit/test_ecology.py
@@ -4,9 +4,10 @@ import time
 import unittest
 import os
 
-from collections import defaultdict
+from copy import deepcopy
 from unittest.mock import MagicMock, patch
 
+from sosw.components.helpers import make_hash
 
 logging.getLogger('botocore').setLevel(logging.WARNING)
 
@@ -21,6 +22,11 @@ from sosw.test.variables import TEST_ECOLOGY_CLIENT_CONFIG
 class ecology_manager_UnitTestCase(unittest.TestCase):
     TEST_CONFIG = TEST_ECOLOGY_CLIENT_CONFIG
     LABOURER = Labourer(id='some_function', arn='arn:aws:lambda:us-west-2:000000000000:function:some_function')
+    SAMPLE_HEALTH_METRICS = {
+        'test1': {'details': {'Name': 'CPUUtilization', 'Namespace': 'AWS/RDS'}},
+        'test2': {'details': {'Name': 'CPUUtilization2', 'Namespace': 'AWS/RDS'}},
+        'test3': {'details': {'Name': 'CPUUtilization3', 'Namespace': 'AWS/RDS'}},
+    }
 
 
     def setUp(self):
@@ -116,3 +122,93 @@ class ecology_manager_UnitTestCase(unittest.TestCase):
         self.manager.task_client.lambda_client.get_function_configuration.return_value = {'Timeout': 300}
 
         self.assertEqual(self.manager.get_max_labourer_duration(self.LABOURER), 300)
+
+
+    def test_get_health(self):
+        METRIC = {
+            'details':                     {},
+            'feelings':                    {
+                3: 50,
+                4: 25,
+            },
+            'feeling_comparison_operator': '__le__'
+        }
+
+        TESTS = [
+            (0, 4),
+            (1.0, 4),
+            (25, 4),
+            (25.000001, 3),
+            (30, 3),
+            (50, 3),
+            (51, 0),
+        ]
+
+        for value, expected in TESTS:
+            self.assertEqual(self.manager.get_health(value, METRIC), expected, f"Failed: {value} t")
+
+
+    def test_get_health__invalid(self):
+        METRIC = {
+            'details':                     {},
+            'feelings':                    {
+                1: 40,
+                3: 50,
+                4: 25,
+            },
+            'feeling_comparison_operator': '__le__'
+        }
+
+        self.assertRaises(ValueError, self.manager.get_health, 60, METRIC), \
+        "Did not raise while the feelings are invalid. Order of values should respect order of health statuses."
+
+
+    def test_get_labourer_status(self):
+        self.manager.get_health = MagicMock(side_effect=[3, 2, 4])
+        self.manager.register_task_manager(MagicMock())
+        self.manager.fetch_metric_stats = MagicMock()
+        self.health_metrics = dict()
+
+        labourer = deepcopy(self.LABOURER)
+        setattr(labourer, 'health_metrics', self.SAMPLE_HEALTH_METRICS)
+
+        # Calling the actual tested method.
+        result = self.manager.get_labourer_status(labourer)
+
+        # The result should be the lowest of values get_health would have returned out of three calls.
+        self.assertEqual(result, 2, f"Did not get the lowest health result. Received: {result}")
+
+        # Chech the the get_health had been called three times (for each metric).
+        self.manager.get_health.assert_called()
+        self.assertEqual(self.manager.get_health.call_count, 3)
+
+        self.manager.fetch_metric_stats.assert_called()
+        self.assertEqual(self.manager.fetch_metric_stats.call_count, 3)
+
+
+    def test_get_labourer_status__uses_cache(self):
+
+        self.manager.get_health = MagicMock(return_value=0)
+        self.manager.register_task_manager(MagicMock())
+        self.manager.fetch_metric_stats = MagicMock()
+
+        labourer = deepcopy(self.LABOURER)
+        setattr(labourer, 'health_metrics', self.SAMPLE_HEALTH_METRICS)
+
+        self.manager.health_metrics = {make_hash(labourer.health_metrics['test1']['details']): 42}
+
+        # Calling the actual tested method.
+        result = self.manager.get_labourer_status(labourer)
+
+        # Assert calculator (get_health) was called 3 times.
+        self.assertEqual(self.manager.get_health.call_count, 3)
+        self.assertEqual(self.manager.fetch_metric_stats.call_count, 2,
+                         f"Fetcher was supposed to be called only for 2 metrics. One is in cache.")
+
+
+    def test_fetch_metric_stats__calls_boto(self):
+
+        self.manager.cloudwatch_client = MagicMock()
+        self.manager.fetch_metric_stats(a=1, b={3: 42})
+
+        self.manager.cloudwatch_client.get_metric_statistics.assert_called_once_with(a=1, b={3: 42})
diff --git a/sosw/managers/test/unit/test_task.py b/sosw/managers/test/unit/test_task.py
index 7c79ea7..77455fb 100644
--- a/sosw/managers/test/unit/test_task.py
+++ b/sosw/managers/test/unit/test_task.py
@@ -521,3 +521,36 @@ class task_manager_UnitTestCase(unittest.TestCase):
             test.pop(field)
 
             self.assertFalse(self.manager.is_valid_task(test))
+
+
+    def test_health_metrics_received(self):
+        TEST_CFG = {
+            'some_function': {
+                'arn':                          'arn:aws:lambda:us-west-2:0000000000:function:some_function',
+                'max_simultaneous_invocations': 10,
+                'health_metrics':               {
+                    'SomeDBCPU': {
+                        'Name':                        'CPUUtilization',
+                        'Namespace':                   'AWS/RDS',
+                        'Period':                      60,
+                        'Statistics':                  ['Average'],
+                        'Dimensions':                  [
+                            {
+                                'Name':  'DBInstanceIdentifier',
+                                'Value': 'YOUR-DB'
+                            },
+                        ],
+
+                        # These is the mapping of how the Labourer should "feel" about this metric.
+                        # See EcologyManager.ECO_STATUSES.
+                        # This is just a mapping ``ECO_STATUS: value`` using ``feeling_comparison_operator``.
+                        'feelings':                    {
+                            3: 50,
+                            4: 25,
+                        },
+                        'feeling_comparison_operator': '<='
+                    },
+                },
+            }
+        }
+