Repository: privacyidea/privacyidea

Issue:
PSKC import does not verify ValueMac of encrypted token secrets
In [RFC6030 (PSKC)](https://tools.ietf.org/html/rfc6030#section-6.1.1) a method is described to verify token key data with MACs. We currently [implement](https://github.com/privacyidea/privacyidea/blob/35b705a372d87ffb8e0ccbd81d1765c32e451509/privacyidea/lib/importotp.py#L627) this for export to PSKC-Files but we don't check it when importing the PSKC-File.

TODOs:
- [x] Implement verification of MAC values
- [x] Fix export of tokens / calculation of MAC values on export
- [x] Add parameter to import `validate_mac`:
    - `check_fail_hard` (default) - no import on invalid mac
    - `no_check` - ignore mac on import
    - `check_fail_soft` - only import valid mac
- [x] Add setting to UI


Discussion:
The import could contain a parameter (in the API and the UI), that tells something like "ignore MAC verification". To make verifying the default and only ignore, if there is a problematic PSKC file.

The interesting files are ``lib/importotp.py`` (https://github.com/privacyidea/privacyidea/blob/35b705a372d87ffb8e0ccbd81d1765c32e451509/privacyidea/lib/importotp.py#L455) and the tests ``tests/test_lib_importotp.py``.

The interesting Key is ValueMAC.
We have a test vector here (https://github.com/privacyidea/privacyidea/blob/35b705a372d87ffb8e0ccbd81d1765c32parsePSKCdatae451509/tests/test_lib_importotp.py#L83) and here (https://github.com/privacyidea/privacyidea/blob/35b705a372d87ffb8e0ccbd81d1765c32e451509/tests/test_lib_importotp.py#L315) which we can use for testing the code.

We need to take a look at ``test_04_import_pskc_aes.``.

Resolving patch:
diff --git a/privacyidea/api/token.py b/privacyidea/api/token.py
--- a/privacyidea/api/token.py
+++ b/privacyidea/api/token.py
@@ -3,6 +3,8 @@
 # http://www.privacyidea.org
 # (c) cornelius kölbel, privacyidea.org
 #
+# 2020-11-11 Timo Sturm <timo.sturm@netknights.it>
+#            Select how to validate PSKC imports
 # 2020-01-28 Jean-Pierre Höhmann <jean-pierre.hoehmann@netknights.it>
 #            Add WebAuthn token
 # 2018-06-07 Cornelius Kölbel <cornelius.koelbel@netknights.it>
@@ -902,6 +904,8 @@ def loadtokens_api(filename=None):
         "oathcsv" or "yubikeycsv".
     :jsonparam tokenrealms: comma separated list of realms.
     :jsonparam psk: Pre Shared Key, when importing PSKC
+    :jsonparam pskcValidateMAC: Determines how invalid MACs should be handled when importing PSKC.
+               Allowed values are 'no_check', 'check_fail_soft' and 'check_fail_hard'.
     :return: The number of the imported tokens
     :rtype: int
     """
@@ -911,6 +915,7 @@ def loadtokens_api(filename=None):
                    'Yubikey CSV', 'pskc']
     file_type = getParam(request.all_data, "type", required)
     hashlib = getParam(request.all_data, "aladdin_hashlib")
+    aes_validate_mac = getParam(request.all_data, "pskcValidateMAC", default='check_fail_hard')
     aes_psk = getParam(request.all_data, "psk")
     aes_password = getParam(request.all_data, "password")
     if aes_psk and len(aes_psk) != 32:
@@ -921,6 +926,7 @@ def loadtokens_api(filename=None):
     if trealms:
         tokenrealms = trealms.split(",")
 
+    not_imported_serials = []
     TOKENS = {}
     token_file = request.files['file']
     file_contents = ""
@@ -964,8 +970,8 @@ def loadtokens_api(filename=None):
     elif file_type in ["yubikeycsv", "Yubikey CSV"]:
         TOKENS = parseYubicoCSV(file_contents)
     elif file_type in ["pskc"]:
-        TOKENS = parsePSKCdata(file_contents, preshared_key_hex=aes_psk,
-                               password=aes_password)
+        TOKENS, not_imported_serials = parsePSKCdata(file_contents, preshared_key_hex=aes_psk,
+                                                   password=aes_password, validate_mac=aes_validate_mac)
 
     # Now import the Tokens from the dictionary
     ret = ""
@@ -987,7 +993,7 @@ def loadtokens_api(filename=None):
                         'success': True})
     # logTokenNum()
 
-    return send_result(len(TOKENS))
+    return send_result({'n_imported': len(TOKENS), 'n_not_imported': len(not_imported_serials)})
 
 
 @token_blueprint.route('/copypin', methods=['POST'])
diff --git a/privacyidea/lib/importotp.py b/privacyidea/lib/importotp.py
--- a/privacyidea/lib/importotp.py
+++ b/privacyidea/lib/importotp.py
@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 #
+#  2020-11-11 Timo Sturm <timo.sturm@netknights.it>
+#             Select how to validate PSKC imports
 #  2018-05-10 Cornelius Kölbel <cornelius.koelbel@netknights.it>
 #             Add fileversion to OATH CSV
 #  2017-11-24 Cornelius Kölbel <cornelius.koelbel@netknights.it>
@@ -47,7 +49,6 @@
 XML files, that hold the OTP secrets for eToken PASS.
 '''
 import hmac, hashlib
-
 import defusedxml.ElementTree as etree
 import re
 import binascii
@@ -55,7 +56,6 @@
 import html
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
-
 from privacyidea.lib.utils import (modhex_decode, modhex_encode,
                                    hexlify_and_unicode, to_unicode, to_utf8,
                                    b64encode_and_unicode)
@@ -438,7 +438,7 @@ def derive_key(xml, password):
         raise ImportException("The XML KeyContainer specifies a derived "
                               "encryption key, but no password given!")
 
-    keymeth= xml.keycontainer.encryptionkey.derivedkey.keyderivationmethod
+    keymeth = xml.keycontainer.encryptionkey.derivedkey.keyderivationmethod
     derivation_algo = keymeth["algorithm"].split("#")[-1]
     if derivation_algo.lower() != "pbkdf2":
         raise ImportException("We only support PBKDF2 as Key derivation "
@@ -455,6 +455,7 @@ def derive_key(xml, password):
 def parsePSKCdata(xml_data,
                   preshared_key_hex=None,
                   password=None,
+                  validate_mac='check_fail_hard',
                   do_checkserial=False):
     """
     This function parses XML data of a PSKC file, (RFC6030)
@@ -469,11 +470,18 @@ def parsePSKCdata(xml_data,
     :param password: The password that encrypted the keys
     :param do_checkserial: Check if the serial numbers conform to the OATH
         specification (not yet implemented)
+    :param validate_mac: Operation mode of hmac validation. Possible values:
+        - 'check_fail_hard' : If an invalid hmac is encountered no token gets parsed.
+        - 'check_fail_soft' : Skip tokens with invalid MAC.
+        - 'no_check' : Hmac of tokens are not checked, every token is parsed.
 
-    :return: a dictionary of token dictionaries
-        { serial : { otpkey , counter, .... }}
+    :return: tuple of a dictionary of token dictionaries and a list of serial of not imported tokens
+        { serial : { otpkey , counter, .... }}, [serial, serial, ...]
     """
 
+    abort = False
+
+    not_imported_serials = []
     tokens = {}
     xml = strip_prefix_from_soup(BeautifulSoup(xml_data, "lxml"))
 
@@ -519,18 +527,43 @@ def parsePSKCdata(xml_data,
                                           "AES128-CBC.")
                 enc_data = key.data.secret.encryptedvalue.ciphervalue.text
                 enc_data = enc_data.strip()
-                secret = aes_decrypt_b64(binascii.unhexlify(preshared_key_hex), enc_data)
+
+                preshared_key = binascii.unhexlify(preshared_key_hex)
+
+                secret = aes_decrypt_b64(preshared_key, enc_data)
+
                 if token["type"].lower() in ["hotp", "totp"]:
                     token["otpkey"] = hexlify_and_unicode(secret)
                 elif token["type"].lower() in ["pw"]:
                     token["otpkey"] = to_unicode(secret)
                 else:
                     token["otpkey"] = to_unicode(secret)
+
+                if validate_mac != 'no_check':
+                    # Validate MAC:
+                    encrypted_mac_key = xml.keycontainer.find("mackey").text
+                    mac_key = aes_decrypt_b64(preshared_key, encrypted_mac_key)
+
+                    enc_data_bin = base64.b64decode(enc_data)
+                    hm = hmac.new(key=mac_key, msg=enc_data_bin, digestmod=hashlib.sha1)
+                    mac_value_calculated = b64encode_and_unicode(hm.digest())
+
+                    mac_value_xml = key.data.find('valuemac').text.strip()
+
+                    is_invalid = not hmac.compare_digest(mac_value_xml, mac_value_calculated)
+
+                    if is_invalid and validate_mac == 'check_fail_hard':
+                        abort = True
+                    elif is_invalid and validate_mac == 'check_fail_soft':
+                        not_imported_serials.append(serial)
+                        continue
+
         except Exception as exx:
             log.error("Failed to import tokendata: {0!s}".format(exx))
             log.debug(traceback.format_exc())
             raise ImportException("Failed to import tokendata. Wrong "
                                   "encryption key? %s" % exx)
+
         if token["type"] in ["hotp", "totp"] and key.data.counter:
             token["counter"] = key.data.counter.text.strip()
         if token["type"] == "totp":
@@ -540,7 +573,12 @@ def parsePSKCdata(xml_data,
                 token["timeShift"] = key.data.timedrift.text.strip()
 
         tokens[serial] = token
-    return tokens
+
+    if abort:
+        not_imported_serials = tokens.keys()
+        tokens = {}  # reset tokens
+
+    return tokens, not_imported_serials
 
 
 class GPGImport(object):
@@ -675,7 +713,8 @@ def export_pskc(tokenobj_list, psk=None):
                 encrypted_otpkey = aes_encrypt_b64(psk, otpkey)
             else:
                 encrypted_otpkey = aes_encrypt_b64(psk, otpkey)
-            hm = hmac.new(key=mackey, msg=otpkey, digestmod=hashlib.sha1)
+
+            hm = hmac.new(key=mackey, msg=base64.b64decode(encrypted_otpkey), digestmod=hashlib.sha1)
             mac_value = b64encode_and_unicode(hm.digest())
         except TypeError:
             # Some keys might be odd string length
@@ -701,8 +740,8 @@ def export_pskc(tokenobj_list, psk=None):
                                  <xenc:CipherValue>{encrypted_otpkey}</xenc:CipherValue>
                              </xenc:CipherData>
                          </EncryptedValue>
+                         <ValueMAC>{value_mac}</ValueMAC>
                      </Secret>
-                     <ValueMAC>{value_mac}</ValueMAC>
                     <Time>
                         <PlainValue>0</PlainValue>
                     </Time>


Test patch:
diff --git a/tests/test_api_token.py b/tests/test_api_token.py
--- a/tests/test_api_token.py
+++ b/tests/test_api_token.py
@@ -892,7 +892,7 @@ def test_11_load_tokens(self):
             res = self.app.full_dispatch_request()
             self.assertTrue(res.status_code == 200, res)
             result = res.json.get("result")
-            value = result.get("value")
+            value = result.get("value")['n_imported']
             self.assertTrue(value == 3, result)
         # check for a successful audit entry
         entry = self.find_most_recent_audit_entry(action='*/token/load/*')
@@ -910,7 +910,7 @@ def test_11_load_tokens(self):
             res = self.app.full_dispatch_request()
             self.assertTrue(res.status_code == 200, res)
             result = res.json.get("result")
-            value = result.get("value")
+            value = result.get("value")['n_imported']
             self.assertTrue(value == 3, result)
 
         # Load yubico.csv
@@ -924,7 +924,7 @@ def test_11_load_tokens(self):
             res = self.app.full_dispatch_request()
             self.assertTrue(res.status_code == 200, res)
             result = res.json.get("result")
-            value = result.get("value")
+            value = result.get("value")['n_imported']
             self.assertTrue(value == 3, result)
 
         # check if the token was put into self.realm1
@@ -967,7 +967,7 @@ def test_11_load_tokens(self):
             res = self.app.full_dispatch_request()
             self.assertTrue(res.status_code == 200, res)
             result = res.json.get("result")
-            value = result.get("value")
+            value = result.get("value")['n_imported']
             self.assertTrue(value == 1, result)
 
         # Load PSKC file, encrypted Password
@@ -981,7 +981,7 @@ def test_11_load_tokens(self):
             res = self.app.full_dispatch_request()
             self.assertTrue(res.status_code == 200, res)
             result = res.json.get("result")
-            value = result.get("value")
+            value = result.get("value")['n_imported']
             self.assertTrue(value == 1, result)
 
     def test_11_load_tokens_only_to_specific_realm(self):
@@ -1002,7 +1002,7 @@ def _clean_up_tokens():
             res = self.app.full_dispatch_request()
             self.assertTrue(res.status_code == 200, res)
             result = res.json.get("result")
-            value = result.get("value")
+            value = result.get("value")['n_imported']
             self.assertTrue(value == 3, result)
         # Now check, if the tokens are in the realm
         from privacyidea.lib.token import get_realms_of_token
@@ -1023,7 +1023,7 @@ def _clean_up_tokens():
             res = self.app.full_dispatch_request()
             self.assertTrue(res.status_code == 200, res)
             result = res.json.get("result")
-            value = result.get("value")
+            value = result.get("value")['n_imported']
             self.assertTrue(value == 3, result)
         # Now check, if the tokens are in the realm
         r = get_realms_of_token("token01")
diff --git a/tests/test_api_validate.py b/tests/test_api_validate.py
--- a/tests/test_api_validate.py
+++ b/tests/test_api_validate.py
@@ -359,7 +359,7 @@ def test_00_run_complete_workflow(self):
             res = self.app.full_dispatch_request()
             self.assertTrue(res.status_code == 200, res)
             result = res.json.get("result")
-            value = result.get("value")
+            value = result.get("value")['n_imported']
             self.assertTrue(value == 1, result)
 
         from privacyidea.lib.token import set_pin
diff --git a/tests/test_lib_importotp.py b/tests/test_lib_importotp.py
--- a/tests/test_lib_importotp.py
+++ b/tests/test_lib_importotp.py
@@ -3,7 +3,7 @@
 This test file tests the lib.importotp
 
 """
-
+import pytest
 
 from .base import MyTestCase
 from privacyidea.lib.importotp import (parseOATHcsv, parseYubicoCSV,
@@ -12,8 +12,7 @@
 from privacyidea.lib.token import remove_token
 from privacyidea.lib.token import init_token
 from privacyidea.lib.importotp import export_pskc
-from privacyidea.lib.utils import hexlify_and_unicode, to_unicode
-import binascii
+from privacyidea.lib.utils import hexlify_and_unicode
 
 
 XML_PSKC_PASSWORD_PREFIX = """<?xml version="1.0" encoding="UTF-8"?>
@@ -338,8 +337,7 @@
 sccTokenData: sccKey=535CC2CB9DEA0B55B0A2D585EAB648EBCE73AC8B;sccMode=T;sccPwLen=6;sccVer=6.20;sccTick=30;sccPrTime=2013/03/12 00:00:00
 sccSignature: MC4CFQDju23MCRqmkWC7Z9sVDB0y0TeEOwIVAOIibmqMFxhPiY7mLlkt5qmRT/xn        '''
 
-
-YUBIKEYCSV= '''
+YUBIKEYCSV = '''
         Static Password: Scan Code,17.04.12 12:25,1,051212172c092728,,,,,0,0,0,0,0,0,0,0,0,0
         Static Password: Scan Code,17.04.12 12:27,1,282828051212172c092728,,,,,0,0,0,0,0,0,0,0,0,0
         LOGGING START,17.04.12 12:29
@@ -564,7 +562,7 @@ def test_02_import_yubikey(self):
     def test_03_import_pskc(self):
         self.assertRaises(ImportException, parsePSKCdata, 'not xml')
 
-        tokens = parsePSKCdata(XML_PSKC)
+        tokens, _ = parsePSKCdata(XML_PSKC)
         self.assertEqual(len(tokens), 7)
         self.assertEqual(tokens["1000133508267"].get("type"), "hotp")
         self.assertEqual(tokens["2600135004013"].get("type"), "totp")
@@ -579,23 +577,51 @@ def test_03_import_pskc(self):
         self.assertEqual(tokens["PW001"].get("otpkey"), hexlify_and_unicode("123456789012"))
 
     def test_04_import_pskc_aes(self):
+        # Test default and all tokens are valid
         encryption_key_hex = "12345678901234567890123456789012"
-        tokens = parsePSKCdata(XML_PSKC_AES,
-                               preshared_key_hex=encryption_key_hex)
+        tokens, not_imported = parsePSKCdata(XML_PSKC_AES,
+                                             preshared_key_hex=encryption_key_hex)
+        self.assertEqual(len(tokens), 1)
+        self.assertEqual(len(not_imported), 0)
+        self.assertEqual(tokens["987654321"].get("type"), "hotp")
+        self.assertEqual(tokens["987654321"].get("otplen"), "8")
+        self.assertEqual(tokens["987654321"].get("otpkey"),
+                         "3132333435363738393031323334353637383930")
+        self.assertEqual(tokens["987654321"].get("description"), "Manufacturer")
+
+        # Test 'check_fail_hard' no token parsed
+        xml_wrong_mac = XML_PSKC_AES.replace("Su+NvtQfmvfJzF6bmQiJqoLRExc=", "Su+NvtQfmvfJzF6XYZiJqoLRExc=")
+        tokens, not_imported = parsePSKCdata(xml_wrong_mac,
+                                             preshared_key_hex=encryption_key_hex)
+        self.assertEqual(len(tokens), 0)
+        self.assertEqual(len(not_imported), 1)
+
+        # Test 'no_check' all tokens parsed and no exception
+        xml_wrong_mac = XML_PSKC_AES.replace("Su+NvtQfmvfJzF6bmQiJqoLRExc=", "Su+NvtQfmvfJzF6XYZiJqoLRExc=")
+        tokens, not_imported = parsePSKCdata(xml_wrong_mac,
+                                             preshared_key_hex=encryption_key_hex, validate_mac='no_check')
         self.assertEqual(len(tokens), 1)
+        self.assertEqual(len(not_imported), 0)
         self.assertEqual(tokens["987654321"].get("type"), "hotp")
         self.assertEqual(tokens["987654321"].get("otplen"), "8")
         self.assertEqual(tokens["987654321"].get("otpkey"),
                          "3132333435363738393031323334353637383930")
         self.assertEqual(tokens["987654321"].get("description"), "Manufacturer")
 
+        # Test 'check_fail_soft' no token parsed and no exception
+        xml_wrong_mac = XML_PSKC_AES.replace("Su+NvtQfmvfJzF6bmQiJqoLRExc=", "Su+NvtQfmvfJzF6XYZiJqoLRExc=")
+        tokens, not_imported = parsePSKCdata(xml_wrong_mac,
+                                             preshared_key_hex=encryption_key_hex, validate_mac='check_fail_soft')
+        self.assertEqual(len(tokens), 0)
+        self.assertEqual(len(not_imported), 1)
+
     def test_05_import_pskc_password(self):
         password = "qwerty"
 
         self.assertRaises(ImportException, parsePSKCdata,
                           XML_PSKC_PASSWORD_PREFIX)
 
-        tokens = parsePSKCdata(XML_PSKC_PASSWORD_PREFIX, password=password)
+        tokens, _ = parsePSKCdata(XML_PSKC_PASSWORD_PREFIX, password=password)
         self.assertEqual(len(tokens), 1)
         self.assertEqual(tokens["987654321"].get("type"), "hotp")
         self.assertEqual(tokens["987654321"].get("otplen"), "8")
@@ -616,7 +642,7 @@ def test_06_export_pskc(self):
         tlist = [t1, t2, t3, t4]
         # export the tokens
         psk, token_num, soup = export_pskc(tlist)
-        # Only 2 tokens exported, the spass token does not get exported!
+        # Only 3 tokens exported, the spass token does not get exported!
         self.assertEqual(token_num, 3)
         self.assertEqual(len(psk), 32)
         export = "{0!s}".format(soup)
@@ -626,7 +652,7 @@ def test_06_export_pskc(self):
         remove_token("t3")
         remove_token("t4")
         # import the tokens again
-        tokens = parsePSKCdata(export, preshared_key_hex=psk)
+        tokens, _ = parsePSKCdata(export, preshared_key_hex=psk)
         self.assertEqual(len(tokens), 3)
         self.assertEqual(tokens.get("t1").get("type"), "hotp")
         self.assertEqual(tokens.get("t1").get("otpkey"), "123456")