Repository: privacyidea/privacyidea

Issue:
Recover HSM not ready
If an HSM with the AESHSM module goes to the state *hsm not ready*, it usually will not recover without any admin interaction.
This could happen if the network connection is lost for a longer time and the retry loop in the aeshsm.py module is exhausted.

We should add a way to be able to recover the hsm state and reinitialize the hsm if necessary.

Or check: is it necessary to go to the hsm_not_ready state?

If we need to recover the state, we could do this via a decorator for the encrypt and decrypt and random function. We might also think about adding some timeout for recovery?


Discussion:
The linked commit looks like a good start for me!

In addition, I was thinking about building a small mock PKCS11 module so that we could test these retry/recovery situations in the unit tests.
This would be great.We could simulate the absence of the HSM, observe the is_ready=False , reconnect the HSM and do a successful encryption.





null
I think so too! I'll look into using mocks to simulate the HSM absence tomorrow.

Resolving patch:
diff --git a/privacyidea/lib/security/aeshsm.py b/privacyidea/lib/security/aeshsm.py
--- a/privacyidea/lib/security/aeshsm.py
+++ b/privacyidea/lib/security/aeshsm.py
@@ -75,6 +75,7 @@ def __init__(self, config=None):
         config = config or {}
         self.name = "HSM"
         self.config = config
+        # Initially, we might be missing a password
         self.is_ready = False
 
         if "module" not in config:
@@ -111,8 +112,6 @@ def initialize_hsm(self):
         * get session
         :return:
         """
-        # Reset the is_ready state, so that we login again.
-        self.is_ready = False
         self.pkcs11.lib.C_Initialize()
         if self.password:
             self._login()
@@ -135,12 +134,10 @@ def setup_module(self, params):
             self.password = str(params.get("password"))
         else:
             raise HSMException("missing password")
-        return self._login()
+        self._login()
+        return self.is_ready
 
     def _login(self):
-        if self.is_ready:
-            return True
-
         slotlist = self.pkcs11.getSlotList()
         if not len(slotlist):
             raise HSMException("No HSM connected")
@@ -164,10 +161,8 @@ def _login(self):
             self.key_handles[mapping[k]] = objs[0]
 
         # self.session.logout()
-        self.is_ready = True
         log.debug("Successfully setup the security module.")
-
-        return self.is_ready
+        self.is_ready = True
 
     def random(self, length):
         """


Test patch:
diff --git a/tests/pkcs11mock.py b/tests/pkcs11mock.py
new file mode 100644
--- /dev/null
+++ b/tests/pkcs11mock.py
@@ -0,0 +1,193 @@
+# -*- coding: utf-8 -*-
+#
+#  2018-05-16 Friedrich Weber <friedrich.weber@netknights.it>
+#             Implement PKCS11 mock
+#
+#  License:  AGPLv3
+#  contact:  http://www.privacyidea.org
+#
+# This code is free software; you can redistribute it and/or
+# modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or any later version.
+#
+# This code is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNE7SS FOR A PARTICULAR PURPOSE.  See the
+# GNU AFFERO GENERAL PUBLIC LICENSE for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+"""
+Mock module for testing the handling of hardware security modules
+"""
+import sys
+import mock
+
+try:
+    import PyKCS11
+except ImportError:
+    # PyKCS11 not installed, let's use our simple mock module
+    class MockPyKCS11Error(Exception):
+        def __init__(self, code):
+            self.code = code
+
+
+    MOCK_CONSTANTS = ['CKA_CLASS', 'CKO_SECRET_KEY', 'CKA_LABEL', 'CKR_SLOT_ID_INVALID', 'CKR_DEVICE_ERROR']
+
+
+    def _setup_module_mock():
+        module = mock.MagicMock()
+        module.PyKCS11Error = MockPyKCS11Error
+        module.CK_SLOT_INFO.side_effect = lambda: mock.MagicMock()
+        for name in MOCK_CONSTANTS:
+            setattr(module, name, object())
+        return module
+
+
+    PyKCS11 = _setup_module_mock()
+    sys.modules['PyKCS11'] = PyKCS11
+
+from PyKCS11 import PyKCS11Error
+from contextlib import contextmanager
+
+SLOT_IDS = [1, 2, 3]
+
+
+def fake_encrypt(data):
+    """
+    simple fake encrypt function for testing: just add 1 to each byte.
+    :return: a list of integers
+    """
+    return [(c + 1) % 256 for c in data]
+
+
+def fake_decrypt(data):
+    """
+    simple fake decrypt function for testing: substract 1 from each byte
+    :return: a list of integers
+    """
+    return [(c - 1) % 256 for c in data]
+
+
+class PKCS11Mock(object):
+    """
+    Mock helper to simulate a HSM. Usage::
+
+        with PKCS11Mock():
+            hsm = AESHardwareSecurityModule(...)
+
+            crypted = hsm.encrypt_password(...)
+
+    Simulation of encryption and decryption is realized using
+    ``fake_encrypt`` and ``fake_decrypt``. Generated random
+    values are just "\x00\x01\x02...".
+    """
+    def __init__(self, default_mocks=True):
+        self.mock = mock.MagicMock()
+        self.lowlevel_mock = mock.MagicMock()
+        self.session_mock = mock.MagicMock()
+        if default_mocks:
+            self.setup_default_mocks()
+
+    def _mock_getSlotInfo(self, slot):
+        if slot not in SLOT_IDS:
+            raise PyKCS11Error(PyKCS11.CKR_SLOT_ID_INVALID)
+        slot_info = PyKCS11.CK_SLOT_INFO()
+        slot_info.slotDescription = "slot {!s} description".format(slot)
+        return slot_info
+
+    @contextmanager
+    def simulate_failure(self, mock_function, count, error=PyKCS11.CKR_DEVICE_ERROR):
+        """
+        Context manager to simulate external HSM failure.
+        Given a mock function, replace its side effect with a new side effect which
+        fails **count** times, raising a PyKCS11Error with the given error ID *error*
+        each time. Subsequent calls are forwarded to the original side effect.
+        When the context is left, the old side effect is restored
+
+        :param mock_function: Function for which failure should be simulated
+        :param count: number of failures
+        :param error: PyKCS11 error code
+        """
+        old_side_effect = mock_function.side_effect
+
+        # we need to modify the number of remaining failures inside our new
+        # side effect function. But modifying an outer variable is not easily possible
+        # in Python2, so we use a class to hold the value.
+        # In Python 3, we could use the ``nonlocal`` keyword
+
+        class store:
+            failures = count
+
+        def _new_side_effect(*args, **kwargs):
+            if store.failures > 0:
+                store.failures -= 1
+                raise PyKCS11Error(error)
+            return old_side_effect(*args, **kwargs)
+
+        try:
+            mock_function.side_effect = _new_side_effect
+            yield
+        finally:
+            mock_function.side_effect = old_side_effect
+
+    @contextmanager
+    def simulate_disconnect(self, count):
+        """
+        Context manager to simulate that the HSM has vanished.
+        This is just a shortcut for calling ``simulate_failure`` on ``generateRandom``, ``encrypt``,
+        ``decrypt`` and ``openSession``.
+        """
+        with self.simulate_failure(self.session_mock.generateRandom, count), \
+            self.simulate_failure(self.session_mock.encrypt, count), \
+            self.simulate_failure(self.session_mock.decrypt, count), \
+            self.simulate_failure(self.mock.openSession, count):
+            yield
+
+    def _mock_openSession(self, slot):
+        if slot not in SLOT_IDS:
+            raise PyKCS11Error(PyKCS11.CKR_SLOT_ID_INVALID)
+        return self.session_mock
+
+    def _mock_encrypt(self, key, data, mechanism):
+        return fake_encrypt(bytearray(data))
+
+    def _mock_decrypt(self, key, data, mechanism):
+        return fake_decrypt(bytearray(data))
+
+    def _mock_generateRandom(self, length):
+        return list(range(length))
+
+    def setup_default_mocks(self):
+        """
+        Configure a number of mocks.
+
+        This will simulate a HSM with three slots.
+        All ``openSession`` calls will return ``self.session_mock``.
+        On the session mock, ``encrypt``, ``decrypt`` and ``generateRandom``
+        are replaced with mock functions.
+        """
+        self.mock.getSlotList.return_value = SLOT_IDS
+        self.mock.getSlotInfo.side_effect = self._mock_getSlotInfo
+        self.mock.openSession.side_effect = self._mock_openSession
+
+        self.session_mock.encrypt.side_effect = self._mock_encrypt
+        self.session_mock.decrypt.side_effect = self._mock_decrypt
+        self.session_mock.generateRandom.side_effect = self._mock_generateRandom
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, *args):
+        self.stop()
+
+    def start(self):
+        self._patcher = mock.patch('PyKCS11.PyKCS11Lib', return_value=self.mock)
+        self._patcher.start()
+        self.mock.lib = self.lowlevel_mock
+
+    def stop(self):
+        self._patcher.stop()
\ No newline at end of file
diff --git a/tests/test_lib_crypto.py b/tests/test_lib_crypto.py
--- a/tests/test_lib_crypto.py
+++ b/tests/test_lib_crypto.py
@@ -2,17 +2,23 @@
 """
 This test file tests the lib.crypto and lib.security.default
 """
+from mock import call
 
+from privacyidea.lib.error import HSMException
 from .base import MyTestCase
+# need to import pkcs11mock before PyKCS11, because it may be replaced by a mock module
+from .pkcs11mock import PKCS11Mock
 from privacyidea.lib.crypto import (encryptPin, encryptPassword, decryptPin,
                                     decryptPassword, urandom,
                                     get_rand_digit_str, geturandom,
                                     get_alphanum_str,
-                                    hash_with_pepper, verify_with_pepper, aes_encrypt_b64, aes_decrypt_b64)
+                                    hash_with_pepper, verify_with_pepper, aes_encrypt_b64, aes_decrypt_b64, _get_hsm)
 from privacyidea.lib.security.default import (SecurityModule,
                                               DefaultSecurityModule)
+from privacyidea.lib.security.aeshsm import AESHardwareSecurityModule
 
 from flask import current_app
+from PyKCS11 import PyKCS11Error
 
 
 class SecurityModuleTestCase(MyTestCase):
@@ -219,3 +225,218 @@ def test_06_hash_pepper(self):
 
         r = verify_with_pepper(h, "super Password")
         self.assertEqual(r, False)
+
+
+class AESHardwareSecurityModuleTestCase(MyTestCase):
+    """
+    Test the AES HSM class for security modules.
+    """
+
+    def test_01_instantiate(self):
+        with PKCS11Mock() as pkcs11:
+            hsm = AESHardwareSecurityModule({
+                "module": "testmodule",
+                "password": "test123!"
+            })
+            self.assertIsNotNone(hsm)
+            self.assertTrue(hsm.is_ready)
+            self.assertIs(hsm.session, pkcs11.session_mock)
+            self.assertEqual(pkcs11.mock.openSession.call_count, 1)
+
+    def test_02_basic(self):
+        with PKCS11Mock() as pkcs11:
+            hsm = AESHardwareSecurityModule({
+                "module": "testmodule",
+            })
+            self.assertFalse(hsm.is_ready)
+            self.assertEqual(pkcs11.mock.openSession.call_count, 0)
+            hsm.setup_module({
+                "password": "test123!"
+            })
+            self.assertTrue(hsm.is_ready)
+            self.assertEqual(pkcs11.mock.openSession.call_count, 1)
+            self.assertIs(hsm.session, pkcs11.session_mock)
+
+            # mock just returns \x00\x01... for random values
+            self.assertEqual(hsm.random(4), "\x00\x01\x02\x03")
+            pkcs11.session_mock.generateRandom.assert_called_once_with(4)
+
+            password = "topSekr3t" * 16
+            crypted = hsm.encrypt_password(password)
+            # to generate the IV
+            pkcs11.session_mock.generateRandom.assert_called_with(16)
+
+            text = hsm.decrypt_password(crypted)
+            self.assertEqual(text, password)
+            self.assertEqual(pkcs11.session_mock.encrypt.call_count, 1)
+            self.assertEqual(pkcs11.session_mock.encrypt.call_count, 1)
+
+            # during the whole usage, we have only used one session
+            self.assertEqual(pkcs11.mock.openSession.call_count, 1)
+
+    def test_03_retry(self):
+        with PKCS11Mock() as pkcs11:
+            hsm = AESHardwareSecurityModule({
+                "module": "testmodule",
+            })
+            hsm.setup_module({
+                "password": "test123!"
+            })
+            self.assertTrue(hsm.is_ready)
+            self.assertIs(hsm.session, pkcs11.session_mock)
+
+            # session is opened once
+            self.assertEqual(pkcs11.mock.openSession.mock_calls, [
+                call(slot=1)
+            ])
+
+            # simulate that encryption succeeds after five tries
+            password = "topSekr3t" * 16
+            with pkcs11.simulate_failure(pkcs11.session_mock.encrypt, 5):
+                encrypted = hsm.encrypt_password(password)
+                # the session has been opened initially, and five times after that
+                self.assertEqual(pkcs11.mock.openSession.mock_calls, [call(slot=1)] * 6)
+
+            # simulate that decryption succeeds after five tries
+            with pkcs11.simulate_failure(pkcs11.session_mock.decrypt, 5):
+                self.assertEqual(hsm.decrypt_password(encrypted), password)
+                # the session has been opened initially, five times during encryption, and five times now
+                self.assertEqual(pkcs11.mock.openSession.mock_calls, [call(slot=1)] * 11)
+
+            # simulate that random generation succeeds after five tries
+            with pkcs11.simulate_failure(pkcs11.session_mock.generateRandom, 5):
+                self.assertEqual(hsm.random(4), "\x00\x01\x02\x03")
+                self.assertEqual(pkcs11.mock.openSession.mock_calls, [call(slot=1)] * 16)
+
+    def test_04_fail_encrypt(self):
+        with PKCS11Mock() as pkcs11:
+            hsm = AESHardwareSecurityModule({
+                "module": "testmodule",
+            })
+            hsm.setup_module({
+                "password": "test123!"
+            })
+            self.assertTrue(hsm.is_ready)
+            self.assertIs(hsm.session, pkcs11.session_mock)
+
+            # session is opened once
+            self.assertEqual(pkcs11.mock.openSession.mock_calls, [
+                call(slot=1)
+            ])
+
+            # simulate that encryption still fails after five tries
+            password = "topSekr3t" * 16
+            with pkcs11.simulate_failure(pkcs11.session_mock.encrypt, 6):
+                with self.assertRaises(HSMException):
+                    hsm.encrypt_password(password)
+                # the session has been opened initially, and six times after that
+                self.assertEqual(pkcs11.mock.openSession.mock_calls, [call(slot=1)] * 7)
+
+    def test_05_hsm_recovery(self):
+        with PKCS11Mock() as pkcs11:
+            hsm = AESHardwareSecurityModule({
+                "module": "testmodule",
+            })
+            hsm.setup_module({
+                "password": "test123!"
+            })
+            self.assertTrue(hsm.is_ready)
+            self.assertIs(hsm.session, pkcs11.session_mock)
+
+            self.assertEqual(pkcs11.mock.openSession.mock_calls, [
+                call(slot=1)
+            ])
+
+            # encryption+decryption succeeds once
+            password = "topSekr3t" * 16
+            crypted = hsm.encrypt_password(password)
+            text = hsm.decrypt_password(crypted)
+            self.assertEqual(text, password)
+
+            # simulate that the HSM disappears after that, so we cannot
+            # even open a session
+            with pkcs11.simulate_failure(pkcs11.session_mock.generateRandom, 1), \
+                pkcs11.simulate_failure(pkcs11.mock.openSession, 1):
+                with self.assertRaises(PyKCS11Error):
+                    hsm.encrypt_password(password)
+                self.assertEqual(pkcs11.mock.openSession.mock_calls, [call(slot=1)] * 2)
+
+            # the Security Module is in a defunct state now
+            # but we can recover from it!
+            # simulate one failure, because this will make the security module
+            # acquire a new session
+            with pkcs11.simulate_failure(pkcs11.session_mock.generateRandom, 1):
+                crypted = hsm.encrypt_password(password)
+            text = hsm.decrypt_password(crypted)
+            self.assertEqual(text, password)
+            self.assertEqual(pkcs11.mock.openSession.mock_calls, [call(slot=1)] * 3)
+
+    def test_06_wrong_password(self):
+        with PKCS11Mock() as pkcs11:
+            hsm = AESHardwareSecurityModule({
+                "module": "testmodule",
+            })
+            with pkcs11.simulate_failure(pkcs11.mock.openSession, 1):
+                with self.assertRaises(PyKCS11Error):
+                    hsm.setup_module({
+                        "password": "test123!"
+                    })
+            self.assertFalse(hsm.is_ready)
+            hsm.setup_module({
+                "password": "test123!"
+            })
+            self.assertTrue(hsm.is_ready)
+            self.assertIs(hsm.session, pkcs11.session_mock)
+
+
+class AESHardwareSecurityModuleLibLevelTestCase(MyTestCase):
+    pkcs11 = PKCS11Mock()
+
+    def setUp(self):
+        """ set up config to load the AES HSM module """
+        current_app.config["PI_HSM_MODULE"] = "privacyidea.lib.security.aeshsm.AESHardwareSecurityModule"
+        current_app.config["PI_HSM_MODULE_MODULE"] = "testmodule"
+        current_app.config["PI_HSM_MODULE_PASSWORD"] = "test123!"
+        with self.pkcs11:
+            MyTestCase.setUp(self)
+
+    def test_01_simple(self):
+        with self.pkcs11:
+            self.assertIsInstance(_get_hsm(), AESHardwareSecurityModule)
+            r = encryptPin("test")
+            pin = decryptPin(r)
+            self.assertEqual(pin, "test")
+
+            self.assertTrue(_get_hsm().is_ready)
+            self.assertEqual(self.pkcs11.session_mock.encrypt.call_count, 1)
+
+    def test_02_fault_recovery(self):
+        with self.pkcs11:
+            hsm = _get_hsm()
+            self.assertIsInstance(hsm, AESHardwareSecurityModule)
+
+            # encryption initially works
+            r = encryptPin("test")
+            pin = decryptPin(r)
+            self.assertEqual(pin, "test")
+            self.assertTrue(hsm.is_ready)
+
+            # the HSM disappears
+            generate_random_call_count = self.pkcs11.session_mock.generateRandom.call_count
+            open_session_call_count = self.pkcs11.mock.openSession.call_count
+            with self.pkcs11.simulate_disconnect(100):
+                with self.assertRaises(PyKCS11Error):
+                    encryptPin("test")
+                # we have tried to generate a random number once
+                self.assertEqual(self.pkcs11.session_mock.generateRandom.call_count,
+                                 generate_random_call_count + 1)
+                # we have tried to open a new session once
+                self.assertEqual(self.pkcs11.mock.openSession.call_count,
+                                 open_session_call_count + 1)
+
+            # HSM is now defunct
+
+            # try to recover now
+            r = encryptPin("test")
+            pin = decryptPin(r)
+            self.assertEqual(pin, "test")