Repository: privacyidea/privacyidea

Issue:
CKR TOKEN NOT PRESENT
Per Discussion:
[HSM PSKC11 Errors CKR_TOKEN_NOT_PRESENT](https://community.privacyidea.org/t/hsm-pskc11-errors-ckr-token-not-present/539)

Webserver - Nginx/Uwsgi

The connection to the HSM is lost (CKR Token Not Found).
In order to re-establish the connection, then Uwsgi service needs to be restarted.

Perhaps we can test the connection every so often and when we catch the exception, we'll preform a recovery method.

<pre>
ERROR in app: Exception on /validate/check [POST]
Traceback (most recent call last):
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/flask/app.py", line 1982, in wsgi_app
  response = self.full_dispatch_request()
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/flask/app.py", line 1614, in full_dispatch_request
  rv = self.handle_user_exception(e)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/flask/app.py", line 1517, in handle_user_exception
  reraise(exc_type, exc_value, tb)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/flask/app.py", line 1612, in full_dispatch_request
  rv = self.dispatch_request()
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/flask/app.py", line 1598, in dispatch_request return
  self.view_functions[rule.endpoint](**req.view_args)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/lib/postpolicy.py", line 101, in policy_wrapper
  response = wrapped_function(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/lib/postpolicy.py", line 101, in policy_wrapper
  response = wrapped_function(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/lib/postpolicy.py", line 101, in policy_wrapper
  response = wrapped_function(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/lib/postpolicy.py", line 101, in policy_wrapper
  response = wrapped_function(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/lib/postpolicy.py", line 101, in policy_wrapper
  response = wrapped_function(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/lib/postpolicy.py", line 101, in policy_wrapper
  response = wrapped_function(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/lib/postpolicy.py", line 101, in policy_wrapper
  response = wrapped_function(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/lib/postpolicy.py", line 101, in policy_wrapper
  response = wrapped_function(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/lib/prepolicy.py", line 117, in policy_wrapper
  return wrapped_function(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/lib/prepolicy.py", line 117, in policy_wrapper
  return wrapped_function(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/lib/prepolicy.py", line 117, in policy_wrapper
  return wrapped_function(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/decorators.py", line 91, in check_user_or_serial_in_request_wrapper
  f_result = func(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/subscriptions.py", line 278, in check_subscription_wrapper
  f_result = func(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/lib/prepolicy.py", line 117, in policy_wrapper
  return wrapped_function(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/event.py", line 60, in event_wrapper
  f_result = func(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/api/validate.py", line 299, in check result,
  details = check_user_pass(user, password, options=options)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/policydecorators.py", line 83, in policy_wrapper
  return self.decorator_function(wrapped_function, *args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/policydecorators.py", line 214, in auth_user_does_not_exist
  return wrapped_function(user_object, passw, options)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/policydecorators.py", line 83, in policy_wrapper
  return self.decorator_function(wrapped_function, *args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/policydecorators.py", line 177, in auth_user_has_no_token
  return wrapped_function(user_object, passw, options)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/policydecorators.py", line 83, in policy_wrapper
  return self.decorator_function(wrapped_function, *args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/policydecorators.py", line 292, in auth_user_timelimit res,
  reply_dict = wrapped_function(user_object, passw, options)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/policydecorators.py", line 83, in policy_wrapper
  return self.decorator_function(wrapped_function, *args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/policydecorators.py", line 378, in auth_lastauth res,
  reply_dict = wrapped_function(user_or_serial, passw, options)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/policydecorators.py", line 83, in policy_wrapper
  return self.decorator_function(wrapped_function, *args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/policydecorators.py", line 269, in auth_user_passthru
  return wrapped_function(user_object, passw, options)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/log.py", line 154, in log_wrapper
  return func(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/token.py", line 1871, in check_user_pass
  options=options)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/log.py", line 154, in log_wrapper
  return func(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/token.py", line 1930, in check_token_list
  tokenobject.authenticate(passw, user, options=options)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/decorators.py", line 44, in token_locked_wrapper
  f_result = func(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/tokenclass.py", line 443, in authenticate
  otp_counter = self.check_otp(otpval, options=options)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/decorators.py", line 44, in token_locked_wrapper
  f_result = func(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/tokens/totptoken.py", line 322, in check_otp
  symetric=True)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/log.py", line 154, in log_wrapper
  return func(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/tokens/HMAC.py", line 147, in checkOtp
  otpval = self.generate(c)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/tokens/HMAC.py", line 122, in generate
  hmac = self.hmac(counter=counter, key=key)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/tokens/HMAC.py", line 82, in hmac
  dig = str(self.secretObj.hmac_digest(data_input, self.hashfunc))
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/crypto.py", line 118, in hmac_digest
  self._setupKey_()
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/crypto.py", line 150, in _setupKey_
  akey = decrypt(self.val, self.iv)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/log.py", line 154, in log_wrapper
  return func(*args, **kwds)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/crypto.py", line 404, in decrypt
  ret = hsm.decrypt(input, iv, id)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/privacyidea/lib/security/aeshsm.py", line 177, in decrypt
  r = self.session.decrypt(k, bytes(data), m)
 File "/opt/privacyidea/local/lib/python2.7/dist-packages/PyKCS11/__init__.py", line 1123, in decrypt
  raise PyKCS11Error(rv) PyKCS11E
</pre>


Discussion:
We probably need to catch exceptions ``PyKCS11Errror`` in these lines:

https://github.com/privacyidea/privacyidea/blob/master/privacyidea/lib/security/aeshsm.py#L177
https://github.com/privacyidea/privacyidea/blob/master/privacyidea/lib/security/aeshsm.py#L168

If the error occurs we need to *initialize* and *login* again:

https://github.com/privacyidea/privacyidea/blob/master/privacyidea/lib/security/aeshsm.py#L95
``self._login()``

Ideally we also repeat the decryption or encryption.

1. We should put the C_Initialize in it's own method.
2. We could put the ``r = self.session.decrypt(k, bytes(data), m)`` (line 177) into a loop.

compare: https://bugs.launchpad.net/barbican/+bug/1582884

Resolving patch:
diff --git a/privacyidea/lib/security/aeshsm.py b/privacyidea/lib/security/aeshsm.py
--- a/privacyidea/lib/security/aeshsm.py
+++ b/privacyidea/lib/security/aeshsm.py
@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 #
+#  2017-09-25 Cornelius Kölbel <cornelius.koelbel@netknights.it>
+#             Add reinitialization of the PKCS11 module
 #  2016-09-01 Mathias Brossard <mathias@axiadids.com>
 #             Alternate PKCS11 Security Module
 #
@@ -34,6 +36,8 @@
 CONFIG_KEY = 1
 VALUE_KEY = 2
 
+MAX_RETRIES = 5
+
 mapping = {
     'token':  TOKEN_KEY,
     'config': CONFIG_KEY,
@@ -92,8 +96,10 @@ def __init__(self, config=None):
 
         self.pkcs11 = PyKCS11.PyKCS11Lib()
         self.pkcs11.load(self.module)
-        self.pkcs11.lib.C_Initialize()
+        self.initialize_hsm()
 
+    def initialize_hsm(self):
+        self.pkcs11.lib.C_Initialize()
         if self.password:
             self._login()
 
@@ -160,21 +166,43 @@ def random(self, length):
         return int_list_to_bytestring(r_integers)
 
     def encrypt(self, data, iv, key_id=TOKEN_KEY):
-        if(len(data) == 0):
+        if len(data) == 0:
             return bytes("")
         log.debug("Encrypting {} bytes with key {}".format(len(data), key_id))
         m = PyKCS11.Mechanism(PyKCS11.CKM_AES_CBC_PAD, iv)
         k = self.key_handles[key_id]
-        r = self.session.encrypt(k, bytes(data), m)
+        retries = 0
+        while True:
+            try:
+                r = self.session.encrypt(k, bytes(data), m)
+                break
+            except PyKCS11.PyKCS11Error as exx:
+                log.warning(u"Encryption failed: {0!s}".format(exx))
+                self.initialize_hsm()
+                retries += 1
+                if retries > MAX_RETRIES:
+                    raise HSMException("Failed to encrypt after multiple retries.")
+
         return int_list_to_bytestring(r)
 
     def decrypt(self, data, iv, key_id=TOKEN_KEY):
-        if(len(data) == 0):
+        if len(data) == 0:
             return bytes("")
         log.debug("Decrypting {} bytes with key {}".format(len(data), key_id))
         m = PyKCS11.Mechanism(PyKCS11.CKM_AES_CBC_PAD, iv)
         k = self.key_handles[key_id]
-        r = self.session.decrypt(k, bytes(data), m)
+        retries = 0
+        while True:
+            try:
+                r = self.session.decrypt(k, bytes(data), m)
+                break
+            except PyKCS11.PyKCS11Error as exx:
+                log.warning(u"Decryption failed: {0!s}".format(exx))
+                self.initialize_hsm()
+                retries += 1
+                if retries > MAX_RETRIES:
+                    raise HSMException("Failed to decrypt after multiple retries.")
+
         return int_list_to_bytestring(r)
 
     def decrypt_password(self, crypt_pass):


Test patch: