Repository: StackStorm/st2

Issue:
Sensor fails to retrieve kv object after some time.
st2 2.0.1
mongo 3.2.10

I have a polling sensor that queries an ES database and when a datapoint crosses a threshold, it sends an alert.  Once alerted, it stores a key with a 24-hour ttl so that the sensor knows to skip alerts for the rest of the day.  I also have a mechanism for silencing alerts with the same approach.  This used to work well before I upgraded st2, but now after a few days of running st2 the sensor keeps alerting even though there’s an `alerted` key in the keystore.  When I restart st2 everything goes back to normal for a few days then the problem comes back.   

I can probably get permission to publish my entire pack, but here's the gist of the dispatch_trigger logic in my sensor:

https://gist.github.com/pixelrebel/240ca95c480d1377175b3667de0411cb

And here's the gist of my action chain that sets the `alerted` key.

https://gist.github.com/pixelrebel/6def658dfd2ee43fe71ce7735db76b29


Discussion:
I believe I've run this issue down. It would appear we're grabbing a temporary token [here](https://github.com/StackStorm/st2/blob/master/st2common/st2common/services/datastore.py#L198) and creating a `Client` with it, but the `Client` will never get unset if the token has expired. So after a day it will continue to use `Client` with the expired token. To add insult to injury we also [catch any exceptions](https://github.com/StackStorm/st2/blob/master/st2common/st2common/services/datastore.py#L93) and don't log it. Working on a fix and will test it. Will keep you posted.


Resolving patch:
diff --git a/st2common/st2common/services/datastore.py b/st2common/st2common/services/datastore.py
--- a/st2common/st2common/services/datastore.py
+++ b/st2common/st2common/services/datastore.py
@@ -13,10 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from datetime import timedelta
 from st2client.client import Client
 from st2client.models import KeyValuePair
 from st2common.services.access import create_token
 from st2common.util.api import get_full_public_api_url
+from st2common.util.date import get_datetime_utc_now
 from st2common.constants.keyvalue import DATASTORE_KEY_SEPARATOR, SYSTEM_SCOPE
 
 
@@ -34,6 +36,7 @@ def __init__(self, logger, pack_name, class_name, api_username):
         self._logger = logger
 
         self._client = None
+        self._token_expire = get_datetime_utc_now()
 
     ##################################
     # Methods for datastore management
@@ -90,7 +93,12 @@ def get_value(self, name, local=True, scope=SYSTEM_SCOPE, decrypt=False):
         try:
             params = {'decrypt': str(decrypt).lower(), 'scope': scope}
             kvp = client.keys.get_by_id(id=name, params=params)
-        except Exception:
+        except Exception as e:
+            self._logger.exception(
+                'Exception retrieving value from datastore (name=%s): %s',
+                name,
+                e
+            )
             return None
 
         if kvp:
@@ -185,7 +193,12 @@ def delete_value(self, name, local=True, scope=SYSTEM_SCOPE):
         try:
             params = {'scope': scope}
             client.keys.delete(instance=instance, params=params)
-        except Exception:
+        except Exception as e:
+            self._logger.exception(
+                'Exception deleting value from datastore (name=%s): %s',
+                name,
+                e
+            )
             return False
 
         return True
@@ -194,8 +207,12 @@ def _get_api_client(self):
         """
         Retrieve API client instance.
         """
-        if not self._client:
+        token_expire = self._token_expire <= get_datetime_utc_now()
+
+        if not self._client or token_expire:
+            self._logger.audit('Creating new Client object.')
             ttl = (24 * 60 * 60)
+            self._token_expire = get_datetime_utc_now() + timedelta(seconds=ttl)
             temporary_token = create_token(username=self._api_username, ttl=ttl)
             api_url = get_full_public_api_url()
             self._client = Client(api_url=api_url, token=temporary_token.token)


Test patch:
diff --git a/st2common/tests/unit/test_datastore.py b/st2common/tests/unit/test_datastore.py
--- a/st2common/tests/unit/test_datastore.py
+++ b/st2common/tests/unit/test_datastore.py
@@ -16,6 +16,8 @@
 
 import os
 import unittest2
+from datetime import timedelta
+from st2common.util.date import get_datetime_utc_now
 
 import mock
 
@@ -128,6 +130,47 @@ def test_datastore_unsupported_scope(self):
         self.assertRaises(ValueError, self._datastore_service.delete_value, name='test1',
             scope='NOT_SYSTEM')
 
+    def test_datastore_get_exception(self):
+        mock_api_client = mock.Mock()
+        mock_api_client.keys.get_by_id.side_effect = ValueError("Exception test")
+        self._set_mock_api_client(mock_api_client)
+        value = self._datastore_service.get_value(name='test1')
+        self.assertEquals(value, None)
+
+    def test_datastore_delete_exception(self):
+        mock_api_client = mock.Mock()
+        mock_api_client.keys.delete.side_effect = ValueError("Exception test")
+        self._set_mock_api_client(mock_api_client)
+        delete_success = self._datastore_service.delete_value(name='test1')
+        self.assertEquals(delete_success, False)
+
+    def test_datastore_token_timeout(self):
+        datastore_service = DatastoreService(logger=mock.Mock(),
+                                             pack_name='core',
+                                             class_name='TestSensor',
+                                             api_username='sensor_service')
+
+        mock_api_client = mock.Mock()
+        kvp1 = KeyValuePair()
+        kvp1.name = 'test1'
+        kvp1.value = 'bar'
+        mock_api_client.keys.get_by_id.return_value = kvp1
+
+        token_expire_time = get_datetime_utc_now() - timedelta(seconds=5)
+        datastore_service._client = mock_api_client
+        datastore_service._token_expire = token_expire_time
+
+        self._set_mock_api_client(mock_api_client)
+
+        with mock.patch(
+            'st2common.services.datastore.Client',
+            return_value=mock_api_client
+        ) as datastore_client:
+            value = datastore_service.get_value(name='test1', local=False)
+            datastore_client.assert_called()
+            self.assertEqual(value, kvp1.value)
+            self.assertGreater(datastore_service._token_expire, token_expire_time)
+
     def _set_mock_api_client(self, mock_api_client):
         mock_method = mock.Mock()
         mock_method.return_value = mock_api_client