Repository: privacyidea/privacyidea

Issue:
Migrate tokens: provide functions to move/copy tokens via dicts between processes
We need an easy way to move/copy tokens between processes or installations.
The requirement for this functionality keeps popping up in different migration scripts.

We should provide these functions in the genericly shipped library functions located in `lib/token.py`.

It could look like this:

To create a token with all its `tokeninfo`:

~~~~python
def create_token_from_dict(serialized_token, info_list):
    """    
    :param serialized_token: dict containing all token objects 
    :return: database ID of the token
    """
    # create database object directly, since we have the encrypted data
    r = Token(**serialized_token).save()
    for ti in info_list:
        ti["token_id"] = r
        TokenInfo(**ti).save()
    return r
~~~~

To create the dict from the database token:

~~~~python
def token_to_dict(token):
    """
    Store the database columns of the token into a dict.
    Also store the tokeninfo into a list of dicts.

    :param token: The database token object
    :return: a dict, containing the token and the tokeninfo
    """
    token_dict = {}
    columns = token.__table__.c
    for column in columns:
        # TODO: handle the secret key, needs to be decrypted
        value = getattr(token, column.key)
        if column.key not in ('id'):
            token_dict[column.key] = value
    # Now add the tokeninfo
    info_list = []
    for ti in token.info_list:
        tokeninfo = {"Description": ti.Description,
                     "Key": ti.Key,
                     "Type": ti.Type,
                     "Value": ti.Value}
        info_list.append(tokeninfo)
    token_dict["info_list"] = info_list
    # TODO: handle the user assignment. from DB table `tokenuser`
    return token_dict
~~~~

This could either be implemented as methods of the `Token` class or as functions in lib/token.py

The dict could look like this:

~~~~json
{ "serial": ....
   "active":....
   "otpkey/secret": ...
   "info_list": { "last_auth": "...." },
   "user_assignment": [ 
       {"uid": ...,
        "resolver": ....,
        "realm":...},   # maybe we rather export the realm than the realm_id
       { }
    ]
}
~~~~

Could also be used in #3005 

Maybe yaml could be a more readable format.


Discussion:
PSKC provides a simple user identification entry:
https://datatracker.ietf.org/doc/html/rfc6030#section-4.3.3
Interesting. PSKC is rather used to transport crypto data between different systems. These will most probably also handle user names or identifiers differently. This would need to be adaptable.

However, In this issue I am looking for an internal machanism to transport token data on a Python level (privacyidea.lib) to e.g. allow moving tokens between different privacyIDEA app contexts. So this should be Python objects (being either a dict or a new specific object)

Also, if we have this mentioned internal lib-function to export token, then this could also be used in an exporter script, that exports to whatever format. But this is not the current idea of this issue!

Resolving patch:
diff --git a/privacyidea/lib/token.py b/privacyidea/lib/token.py
--- a/privacyidea/lib/token.py
+++ b/privacyidea/lib/token.py
@@ -100,6 +100,7 @@
                                                          generic_challenge_response_resync)
 from privacyidea.lib.tokenclass import DATE_FORMAT
 from privacyidea.lib.tokenclass import TOKENKIND
+from privacyidea.lib.user import get_username
 from dateutil.tz import tzlocal
 
 log = logging.getLogger(__name__)
@@ -2597,3 +2598,81 @@ def list_tokengroups(tokengroup=None):
         tgs = TokenTokengroup.query.all()
 
     return tgs
+
+
+def token_dump(token, tokenowner=True):
+    """
+    Store the database columns of the token into a dict.
+    Also store the tokeninfo into a list of dicts.
+
+    :param token: A token object
+    :param tokenowner: Also dump the tokenowners
+    :type tokenowner: bool
+    :return: a dict, containing the token and the tokeninfo
+    """
+    token_dict = token._to_dict()
+    if tokenowner:
+        # handle all assigned users
+        owners = []
+        for owner in token.owners:
+            owners.append({"uid": owner.uid,
+                           "login": owner.login,
+                           "resolver": owner.resolver,
+                           "realm": owner.realm})
+        token_dict["owners"] = owners
+    return token_dict
+
+
+def token_load(token_dict, tokenowner=True, overwrite=False):
+    """
+    Load the token that has previously been dumped with the function token_dump.
+
+    :param token_dict: The token in a dict
+    :param tokenowner: The tokenowner should also be assigned. If the tokenowner can not be found or
+        identified, the token is created anyways, but not assigned to a user; an exception is raised.
+    :type tokenowner: bool
+    :param overwrite: If a token with the given serial number already exist, it should be overwritten. If the token
+        should not be overwritten but already exists, an exception is raised.
+    :type overwrite: bool
+    :return:
+    """
+    serial = token_dict.get("serial")
+    old_token_obj = get_one_token(serial=serial, silent_fail=True)
+    # Check if overwriting is not allowed and the token already exists
+    if not overwrite and old_token_obj:
+        raise TokenAdminError("Token already exists!")
+
+    tokeninfos = token_dict.get("info_list", {})
+    hashed_pin = token_dict.get("_hashed_pin")
+    # Creating a new dictionary without special keys
+    stripped_token = {k: v for k, v in token_dict.items() if k not in ["info_list", "owners", "_hashed_pin"]}
+    # Initialize or update token
+    token = init_token(stripped_token)
+    # Add token information
+    for key, value in tokeninfos.items():
+        token.add_tokeninfo(key, value)
+    # Set PIN if hashed_pin is available
+    if hashed_pin:
+        token.token.pin_hash = hashed_pin
+        token.token.save()
+
+    # Add the tokenowners
+    if tokenowner:
+        for owner in token_dict.get("owners", []):
+            # uid, login, resolver, realm
+            # We might need to create the user object
+            # Now we add the user as tokenowner
+            try:
+                loginname = get_username(owner.get("uid"), owner.get("resolver"))
+                if loginname == owner.get("login"):
+                    u = User(login=owner.get("login"),
+                             resolver=owner.get("resolver"),
+                             realm=owner.get("realm"),
+                             uid=owner.get("uid"))
+                    token.add_user(u)
+            except Exception as ex:
+                log.warning("Failed to add user {0!s} to token {1!s}: {2!s}".format(owner, token, ex))
+                log.debug(traceback.format_exc())
+                raise TokenAdminError("Token can not be assiend to the specified user!")
+
+    return token
diff --git a/privacyidea/lib/tokenclass.py b/privacyidea/lib/tokenclass.py
--- a/privacyidea/lib/tokenclass.py
+++ b/privacyidea/lib/tokenclass.py
@@ -270,6 +270,23 @@ def add_tokengroup(self, tokengroup=None, tokengroup_id=None):
                             tokengroupname=tokengroup).save()
         return r > 0
 
+    @property
+    def owners(self):
+        """
+        return all the owners of a token
+        If the token has no owner assigned, we return an empty list
+
+        :return: The owners of the token
+        :rtype: A list of user object
+        """
+        user_objects = []
+        for tokenowner in self.token.all_owners:
+            user_object = User(resolver=tokenowner.resolver,
+                               realm=tokenowner.realm.name,
+                               uid=tokenowner.user_id)
+            user_objects.append(user_object)
+        return user_objects
+
     @property
     def user(self):
         """
@@ -953,7 +970,7 @@ def get_hashlib(hLibStr):
         else:
             return hashlib.sha1
 
-    def get_tokeninfo(self, key=None, default=None):
+    def get_tokeninfo(self, key=None, default=None, decrypted=False):
         """
         return the complete token info or a single key of the tokeninfo.
         When returning the complete token info dictionary encrypted entries
@@ -965,17 +982,23 @@ def get_tokeninfo(self, key=None, default=None):
         :type key: string
         :param default: the default value, if the key does not exist
         :type default: string
+        :param decrypted: Indicates that passwords should be decrypted when fetching the whole dict
+        :type decrypted: bool
         :return: the value for the key
         :rtype: int or str or dict
         """
         tokeninfo = self.token.get_info()
+        ret = tokeninfo
+
         if key:
             ret = tokeninfo.get(key, default)
-            if tokeninfo.get(key + ".type") == "password":
-                # we need to decrypt the return value
+            key_type = tokeninfo.get(key + ".type")
+            if key_type == "password":
                 ret = decryptPassword(ret)
-        else:
-            ret = tokeninfo
+        elif decrypted:
+            ret = {x: (decryptPassword(y) if tokeninfo.get(x + ".type") == "password" else y)
+                   for x, y in tokeninfo.items()}
+
         return ret
 
     def del_tokeninfo(self, key=None):
@@ -1953,10 +1976,11 @@ def _to_dict(self, b32=False):
             "active": self.is_active(),
             "revoked": self.token.revoked,
             "locked": self.token.locked,
-            "rollout_state": self.token.rollout_state
+            "rollout_state": self.token.rollout_state,
+            "_hashed_pin": self.token.pin_hash
         }
         if b32:
             token_dict["otpkey"] = b32encode(unhexlify(token_dict.get("otpkey")))
         token_dict["otpkey"] = to_unicode(token_dict.get("otpkey"))
-        token_dict.update(self.get_tokeninfo())
+        token_dict["info_list"] = self.get_tokeninfo(decrypted=True)
         return token_dict
\ No newline at end of file
diff --git a/privacyidea/models.py b/privacyidea/models.py
--- a/privacyidea/models.py
+++ b/privacyidea/models.py
@@ -250,6 +250,10 @@ def __init__(self, serial, tokentype="",
     @property
     def first_owner(self):
         return self.owners.first()
+
+    @property
+    def all_owners(self):
+        return self.owners.all()
             
     @log_with(log)
     def delete(self):


Test patch:
diff --git a/tests/test_lib_token.py b/tests/test_lib_token.py
--- a/tests/test_lib_token.py
+++ b/tests/test_lib_token.py
@@ -1610,6 +1610,7 @@ def __init__(self, type):
         self.assertTrue(weigh_token_type(dummy_token("PUSH")) > weigh_token_type(dummy_token("hotp")))
 
 
+
 class TokenOutOfBandTestCase(MyTestCase):
 
     def test_00_create_realms(self):
@@ -2167,4 +2168,60 @@ def test_00_create_tokens(self):
             self.assertEqual(CHANGED_KEY, d.get("otpkey"))
             # Also check, that the tokeninfo for TOTP was updated
             if d.get("type") == "totp":
-                self.assertEqual("30", d.get("timeStep"), d)
+                tokeninfo = d.get("info_list")
+                self.assertEqual("30", tokeninfo.get("timeStep"), d)
+
+    def test_01_token_dump_load(self):
+        from privacyidea.lib.token import token_dump, token_load
+        self.setUp_user_realms()
+        key = "1234567890123456"
+        pin = "1234"
+        tokenobject = init_token({"serial": "s2",
+                                  "pin": pin,
+                                  "otpkey": key},
+                                 user=User("hans", self.realm1))
+        tokenobject.add_tokeninfo("secretkey", "secretvalue", value_type="password")
+        d = token_dump(tokenobject)
+        self.assertEqual(True, d.get("active"))
+        self.assertEqual("", d.get("description"))
+        self.assertEqual("software", d.get("info_list").get("tokenkind"))
+        self.assertEqual("sha1", d.get("info_list").get("hashlib"))
+        self.assertEqual(key, d.get("otpkey"))
+        self.assertNotIn("key_inc", d)
+        self.assertNotIn("key_iv", d)
+        self.assertNotIn("id", d)
+        tokeninfo = d.get("info_list")
+        # Check the encrypted values
+        self.assertEqual("secretvalue", tokeninfo.get("secretkey"))
+        self.assertEqual("password", tokeninfo.get("secretkey.type"))
+        tokenowner = d.get("owners")[0]
+        self.assertEqual("1118", tokenowner.get("uid"))
+        self.assertEqual("hans", tokenowner.get("login"))
+        self.assertEqual("resolver1", tokenowner.get("resolver"))
+        self.assertEqual("realm1", tokenowner.get("realm"))
+
+        # A token should not be overwritten, if it already exists
+        self.assertRaises(TokenAdminError, token_load, d)
+
+        # We overwrite the token
+        token_load(d, overwrite=True)
+
+        # We load a new token.
+        d["serial"] = "s3"
+        token_load(d)
+        # Read the token s3 from the database and check the PIN, that has been copied from token s2 to token s3
+        tok = get_one_token(serial="s3")
+        self.assertTrue(tok.check_pin(pin))
+
+        # Check the owner of the token
+        self.assertEqual("hans", tok.user.login)
+        self.assertEqual(self.realm1, tok.user.realm)
+
+        # Now we load the 3rd token, with a user, that does not exist
+        d["serial"] = "s4"
+        # Set an invalid uid, so that the user can not be assigned.
+        d["owners"][0]["uid"] = "987654321"
+        self.assertRaises(TokenAdminError, token_load, d)
+        tok = get_one_token(serial="s4")
+        # A user was not assigned
+        self.assertEqual(None, tok.user)
diff --git a/tests/test_lib_tokenclass.py b/tests/test_lib_tokenclass.py
--- a/tests/test_lib_tokenclass.py
+++ b/tests/test_lib_tokenclass.py
@@ -403,8 +403,15 @@ def test_11_tokeninfo_with_type(self):
         token = TokenClass(db_token)
         token.add_tokeninfo("radius.secret", "secret", value_type="password")
         info1 = token.get_tokeninfo()
-        self.assertTrue("radius.secret" in info1, info1)
-        self.assertTrue("radius.secret.type" in info1, info1)
+        self.assertIn("radius.secret", info1, info1)
+        self.assertIn("radius.secret.type", info1, info1)
+        # We get the encrypted value
+        self.assertEqual(65, len(info1.get("radius.secret")), info1)
+        # Now we get the tokenvalues, decrypted
+        info2 = token.get_tokeninfo(decrypted=True)
+        # and we can see the plain text value
+        self.assertEqual("secret", info2.get("radius.secret"), info2)
+        self.assertEqual("password", info2.get("radius.secret.type"), info2)
 
         info = token.get_tokeninfo("radius.secret")
         self.assertEqual(info, "secret", info)
@@ -426,19 +433,19 @@ def test_11_tokeninfo_encrypt(self):
         info = token.get_tokeninfo("radius.secret")
         self.assertEqual(info, "topSecret", info)
 
-        # THe same with set_tokeninfo
+        # The same with set_tokeninfo
         token.set_tokeninfo({"radius.secret": "otherSecret",
                              "radius.secret.type": "password"})
         info1 = token.get_tokeninfo()
-        self.assertTrue("radius.secret" in info1, info1)
-        self.assertTrue("radius.secret.type" in info1, info1)
+        self.assertIn("radius.secret", info1, info1)
+        self.assertIn("radius.secret.type", info1, info1)
         # get_tokeninfo without parameters does not decrypt!
-        self.assertTrue(info1.get("radius.secret") != "otherSecret",
-                        info1.get("radius.secret"))
+        self.assertNotEqual(info1.get("radius.secret"), "otherSecret",
+                            info1)
 
         # get_tokeninfo with parameter does decrypt!
         info = token.get_tokeninfo("radius.secret")
-        self.assertTrue(info == "otherSecret", info)
+        self.assertEqual(info, "otherSecret", info)
 
     def test_12_inc_otp_counter(self):
         db_token = Token.query.filter_by(serial=self.serial1).first()