Repository: privacyidea/privacyidea

Issue:
Rethink LDAP resolver caching for scenarios with many users
The LDAP resolver implements a process-level caching of user information, which is a good idea to improve performance. However, there are some problems in setup with many users because even expired cache entries are kept in memory until the user authenticates again (and then, they are replaced with more information).
This can get problematic if there is a large number of distinct users, because the cache will then grow and consume a lot of memory. This effect is even bigger in multi-process environments, e.g. nginx+uwsgi.

There are multiple ways to fix this:
* Implement a config option that allows to reduce the cache, e.g., to a request-level cache. But this will negatively impact performance for users that authenticate very often (i.e. every X seconds where X < ``CACHE_TIMEOUT``)
* Revise the implementation such that it actually frees expired cache entries. But this can get complicated because we need to be careful with multi-threaded environments. We could, for example, implement the cache in a way that it only keeps the 100 most recent entries.


Discussion:
I would avoid a too complicated logic, since this can quickly lead to unexpected behaviours and errors.

I would think either being able to disable it (we also have the generic user cache in SQL) or only keeping a maximum number (like a FIFO). This number might be also configurable. (In this case, setting the number to 0 would acutally disable it)
Hm, actually we already have a cache_timeout of 120 seconds default.
I do not think that in a large user setup, tens of thousands of users will be read in 120 seconds.

Well, currently we do not delete the timeouted entries, we simply check if the requested value is new enough:
https://github.com/privacyidea/privacyidea/blob/master/privacyidea/lib/resolvers/LDAPIdResolver.py#L203

THis way all resolved users one day exist in the dictionary, even if 38.000 users entries are timed out.
Of course this results in a big cache. Maybe we should simply clean up expired entries!

Resolving patch:
diff --git a/privacyidea/lib/resolvers/LDAPIdResolver.py b/privacyidea/lib/resolvers/LDAPIdResolver.py
--- a/privacyidea/lib/resolvers/LDAPIdResolver.py
+++ b/privacyidea/lib/resolvers/LDAPIdResolver.py
@@ -188,28 +188,48 @@ def cache(func):
     """
     @functools.wraps(func)
     def cache_wrapper(self, *args, **kwds):
-        # If it does not exist, create the node for this instance
-        resolver_id = self.getResolverId()
-        if not resolver_id in CACHE:
-            CACHE[resolver_id] = {"getUserId": {},
-                                  "getUserInfo": {},
-                                  "_getDN": {}}
-
-        # get the portion of the cache for this very LDAP resolver
-        r_cache = CACHE.get(resolver_id).get(func.func_name)
-        if args[0] in r_cache and \
-                        datetime.datetime.now() < r_cache[args[0]][
-                    "timestamp"] + \
-                        datetime.timedelta(seconds=self.cache_timeout):
-            log.debug("Reading {0!r} from cache for {1!r}".format(args[0],
-                                                              func.func_name))
-            return r_cache[args[0]]["value"]
+        # Only run the code, in case we have a configured cache!
+        if self.cache_timeout > 0:
+            # If it does not exist, create the node for this instance
+            resolver_id = self.getResolverId()
+            now = datetime.datetime.now()
+            tdelta = datetime.timedelta(seconds=self.cache_timeout)
+            if not resolver_id in CACHE:
+                CACHE[resolver_id] = {"getUserId": {},
+                                      "getUserInfo": {},
+                                      "_getDN": {}}
+            else:
+                # Clean up the cache in the current resolver and the current function
+                _to_be_deleted = []
+                try:
+                    for user, cached_result in CACHE[resolver_id].get(func.func_name).iteritems():
+                        if now > cached_result.get("timestamp") + tdelta:
+                            _to_be_deleted.append(user)
+                except RuntimeError:
+                    # This might happen if thread A evicts an expired
+                    # cache entry while thread B looks for expired cache entries
+                    pass
+                for user in _to_be_deleted:
+                    try:
+                        del CACHE[resolver_id][func.func_name][user]
+                    except KeyError:
+                        pass
+                del _to_be_deleted
+
+            # get the portion of the cache for this very LDAP resolver
+            r_cache = CACHE.get(resolver_id).get(func.func_name)
+            entry = r_cache.get(args[0])
+            if entry and now < entry.get("timestamp") + tdelta:
+                log.debug("Reading {0!r} from cache for {1!r}".format(args[0], func.func_name))
+                return entry.get("value")
 
         f_result = func(self, *args, **kwds)
-        # now we cache the result
-        CACHE[resolver_id][func.func_name][args[0]] = {
-            "value": f_result,
-            "timestamp": datetime.datetime.now()}
+
+        if self.cache_timeout > 0:
+            # now we cache the result
+            CACHE[resolver_id][func.func_name][args[0]] = {
+                "value": f_result,
+                "timestamp": now}
 
         return f_result
 


Test patch:
diff --git a/tests/test_lib_resolver.py b/tests/test_lib_resolver.py
--- a/tests/test_lib_resolver.py
+++ b/tests/test_lib_resolver.py
@@ -15,6 +15,7 @@
 from ldap3.core.results import RESULT_SIZE_LIMIT_EXCEEDED
 import mock
 import responses
+import datetime
 import uuid
 from privacyidea.lib.resolvers.LDAPIdResolver import IdResolver as LDAPResolver
 from privacyidea.lib.resolvers.SQLIdResolver import IdResolver as SQLResolver
@@ -1820,6 +1821,99 @@ def test_31_login_with_objectGUID_only(self):
         info = y.getUserInfo(uid)
         self.assertEqual(info['username'], objectGUIDs[0])
 
+    @ldap3mock.activate
+    def test_32_cache_expiration(self):
+        # This test checks if the short-living cache deletes expired
+        # entries. NOTE: This does not test a multi-threaded scenario!
+        ldap3mock.setLDAPDirectory(LDAPDirectory_small)
+        cache_timeout = 120
+        y = LDAPResolver()
+        y.loadConfig({'LDAPURI': 'ldap://localhost',
+                      'LDAPBASE': 'o=test',
+                      'BINDDN': 'cn=manager,ou=example,o=test',
+                      'BINDPW': 'ldaptest',
+                      'LOGINNAMEATTRIBUTE': 'cn',
+                      'LDAPSEARCHFILTER': '(&(cn=*))', # we use this weird search filter to get a unique resolver ID
+                      'USERINFO': '{ "username": "cn",'
+                                  '"phone" : "telephoneNumber", '
+                                  '"mobile" : "mobile"'
+                                  ', "email" : "mail", '
+                                  '"surname" : "sn", '
+                                  '"givenname" : "givenName" }',
+                      'UIDTYPE': 'objectGUID',
+                      'NOREFERRALS': True,
+                      'CACHE_TIMEOUT': cache_timeout
+                      })
+        from privacyidea.lib.resolvers.LDAPIdResolver import CACHE
+        # assert that the other tests haven't left anything in the cache
+        self.assertNotIn(y.getResolverId(), CACHE)
+        bob_id = y.getUserId('bob')
+        # assert the cache contains this entry
+        self.assertEqual(CACHE[y.getResolverId()]['getUserId']['bob']['value'], bob_id)
+        # assert subsequent requests for the same data hit the cache
+        with mock.patch.object(ldap3mock.Connection, 'search') as mock_search:
+            bob_id2 = y.getUserId('bob')
+            self.assertEqual(bob_id, bob_id2)
+            mock_search.assert_not_called()
+        self.assertIn('bob', CACHE[y.getResolverId()]['getUserId'])
+        # assert requests later than CACHE_TIMEOUT seconds query the directory again
+        now = datetime.datetime.now()
+        with mock.patch('privacyidea.lib.resolvers.LDAPIdResolver.datetime.datetime',
+                        wraps=datetime.datetime) as mock_datetime:
+            # we now live CACHE_TIMEOUT + 2 seconds in the future
+            mock_datetime.now.return_value = now + datetime.timedelta(seconds=cache_timeout + 2)
+            with mock.patch.object(ldap3mock.Connection, 'search', wraps=y.l.search) as mock_search:
+                bob_id3 = y.getUserId('bob')
+                self.assertEqual(bob_id, bob_id3)
+                mock_search.assert_called_once()
+        # assert the cache contains this entry, with the updated timestamp
+        self.assertEqual(CACHE[y.getResolverId()]['getUserId']['bob'],
+                         {'value': bob_id,
+                          'timestamp': now + datetime.timedelta(seconds=cache_timeout + 2)})
+        # we now go 2 * (CACHE_TIMEOUT + 2) seconds to the future and query for someone else's user ID.
+        # This will cause bob's cache entry to be evicted.
+        with mock.patch('privacyidea.lib.resolvers.LDAPIdResolver.datetime.datetime',
+                        wraps=datetime.datetime) as mock_datetime:
+            mock_datetime.now.return_value = now + datetime.timedelta(seconds=2 * (cache_timeout + 2))
+            manager_id = y.getUserId('manager')
+        self.assertEqual(CACHE[y.getResolverId()]['getUserId'].keys(), ['manager'])
+
+    @ldap3mock.activate
+    def test_33_cache_disabled(self):
+        # This test checks if the short-living cache deletes expired
+        # entries. NOTE: This does not test a multi-threaded scenario!
+        ldap3mock.setLDAPDirectory(LDAPDirectory_small)
+        y = LDAPResolver()
+        y.loadConfig({'LDAPURI': 'ldap://localhost',
+                      'LDAPBASE': 'o=test',
+                      'BINDDN': 'cn=manager,ou=example,o=test',
+                      'BINDPW': 'ldaptest',
+                      'LOGINNAMEATTRIBUTE': 'cn',
+                      'LDAPSEARCHFILTER': '(|(cn=*))', # we use this weird search filter to get a unique resolver ID
+                      'USERINFO': '{ "username": "cn",'
+                                  '"phone" : "telephoneNumber", '
+                                  '"mobile" : "mobile"'
+                                  ', "email" : "mail", '
+                                  '"surname" : "sn", '
+                                  '"givenname" : "givenName" }',
+                      'UIDTYPE': 'objectGUID',
+                      'NOREFERRALS': True,
+                      'CACHE_TIMEOUT': 0
+                      })
+        from privacyidea.lib.resolvers.LDAPIdResolver import CACHE
+        # assert that the other tests haven't left anything in the cache
+        self.assertNotIn(y.getResolverId(), CACHE)
+        bob_id = y.getUserId('bob')
+        # assert the cache does not contain this entry
+        self.assertNotIn(y.getResolverId(), CACHE)
+        # assert subsequent requests query the directory
+        with mock.patch.object(ldap3mock.Connection, 'search', wraps=y.l.search) as mock_search:
+            bob_id2 = y.getUserId('bob')
+            self.assertEqual(bob_id, bob_id2)
+            mock_search.assert_called_once()
+
+
+
 class BaseResolverTestCase(MyTestCase):
 
     def test_00_basefunctions(self):