internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: privacyidea/privacyidea

Issue:
Regenerate QR Code
If the user has a limited token number,
regenerating the QR Code will fail, if he has created his last token.

The limitet_token_number decorator should check during regeneration for the serial number.


Discussion:

Resolving patch:
diff --git a/privacyidea/api/lib/prepolicy.py b/privacyidea/api/lib/prepolicy.py
--- a/privacyidea/api/lib/prepolicy.py
+++ b/privacyidea/api/lib/prepolicy.py
@@ -632,6 +632,7 @@ def check_max_token_user(request=None, action=None):
     ERROR = "The number of tokens for this user is limited!"
     params = request.all_data
     user_object = get_user_from_param(params)
+    serial = getParam(params, "serial")
     if user_object.login:
         policy_object = g.policy_object
         limit_list = policy_object.get_action_values(ACTION.MAXTOKENUSER,
@@ -642,6 +643,10 @@ def check_max_token_user(request=None, action=None):
         if limit_list:
             # we need to check how many tokens the user already has assigned!
             tokenobject_list = get_tokens(user=user_object)
+            if serial and serial in [tok.token.serial for tok in tokenobject_list]:
+                # If a serial is provided and this token already exists, the
+                # token can be regenerated
+                return True
             already_assigned_tokens = len(tokenobject_list)
             if already_assigned_tokens >= max([int(x) for x in limit_list]):
                 raise PolicyError(ERROR)


Test patch:
diff --git a/tests/test_api_lib_policy.py b/tests/test_api_lib_policy.py
--- a/tests/test_api_lib_policy.py
+++ b/tests/test_api_lib_policy.py
@@ -296,6 +296,12 @@ def test_04_check_max_token_user(self):
         self.assertRaises(PolicyError,
                           check_max_token_user, req)
 
+        # Now, the token that already exists and is reenrolled must not trigger an exception
+        req.all_data = {"user": "cornelius",
+                        "realm": self.realm1,
+                        "serial": "NEW002"}
+        self.assertTrue(check_max_token_user(req))
+
         # Now we set another policy for the user max_token = 12.
         # This way, the user should be allowed to enroll tokens again, since there
         # are two policies matching for the user and the maximum is 12.
Repository: privacyidea/privacyidea

Issue:
Realmdrop down policy with wrong description
The description in the realm dropdown is wrong.
The list of realms is a whitespaced separated list, not a comma separated list.


Discussion:

Resolving patch:
diff --git a/privacyidea/lib/policy.py b/privacyidea/lib/policy.py
--- a/privacyidea/lib/policy.py
+++ b/privacyidea/lib/policy.py
@@ -1748,9 +1748,9 @@ def get_static_policy_definitions(scope=None):
             },
             ACTION.REALMDROPDOWN: {
                 'type': 'str',
-                'desc': _("A comma separated list of realm names, which are "
+                'desc': _("A list of realm names, which are "
                           "displayed in a drop down menu in the WebUI login "
-                          "screen.")
+                          "screen. Realms are separated by white spaces.")
             },
             ACTION.HIDE_WELCOME: {
                 'type': 'bool',


Test patch:
Repository: privacyidea/privacyidea

Issue:
Improve HSM retry mechanism
Apparently we have some issues in the HSM retry logic.

* It seems like some PKCS11 modules already perform network communication on module load. Thus, we need to move
https://github.com/privacyidea/privacyidea/blob/7a18294290e25aca98acf59c6e5064fa61c9d85f/privacyidea/lib/security/aeshsm.py#L105
to ``initialize_hsm``.
* The retry mechanism in ``encrypt`` and ``decrypt`` opens a new session and tries to encrypt/decrypt with the new session. However, we pass the key handles of the *old* session to the *new* session, which might cause problems. We should move
https://github.com/privacyidea/privacyidea/blob/7a18294290e25aca98acf59c6e5064fa61c9d85f/privacyidea/lib/security/aeshsm.py#L195
and
https://github.com/privacyidea/privacyidea/blob/7a18294290e25aca98acf59c6e5064fa61c9d85f/privacyidea/lib/security/aeshsm.py#L215 inside the ``try`` block.


Discussion:

Resolving patch:
diff --git a/privacyidea/lib/security/aeshsm.py b/privacyidea/lib/security/aeshsm.py
--- a/privacyidea/lib/security/aeshsm.py
+++ b/privacyidea/lib/security/aeshsm.py
@@ -101,8 +101,6 @@ def __init__(self, config=None):
         self.session = None
         self.key_handles = {}
 
-        self.pkcs11 = PyKCS11.PyKCS11Lib()
-        self.pkcs11.load(self.module)
         self.initialize_hsm()
 
     def initialize_hsm(self):
@@ -113,6 +111,8 @@ def initialize_hsm(self):
         * get session
         :return:
         """
+        self.pkcs11 = PyKCS11.PyKCS11Lib()
+        self.pkcs11.load(self.module)
         self.pkcs11.lib.C_Initialize()
         if self.password:
             self._login()
@@ -192,10 +192,10 @@ def encrypt(self, data, iv, key_id=TOKEN_KEY):
             return bytes("")
         log.debug("Encrypting {} bytes with key {}".format(len(data), key_id))
         m = PyKCS11.Mechanism(PyKCS11.CKM_AES_CBC_PAD, iv)
-        k = self.key_handles[key_id]
         retries = 0
         while True:
             try:
+                k = self.key_handles[key_id]
                 r = self.session.encrypt(k, bytes(data), m)
                 break
             except PyKCS11.PyKCS11Error as exx:
@@ -212,10 +212,10 @@ def decrypt(self, data, iv, key_id=TOKEN_KEY):
             return bytes("")
         log.debug("Decrypting {} bytes with key {}".format(len(data), key_id))
         m = PyKCS11.Mechanism(PyKCS11.CKM_AES_CBC_PAD, iv)
-        k = self.key_handles[key_id]
         retries = 0
         while True:
             try:
+                k = self.key_handles[key_id]
                 r = self.session.decrypt(k, bytes(data), m)
                 break
             except PyKCS11.PyKCS11Error as exx:


Test patch:
Repository: privacyidea/privacyidea

Issue:
setrealm does not work for nonexisting user
The setrealm policy raises a User-Does-Not-Exist error, if the original user ``user@realm1``, does not exist in ``realm1``. Problem is, that the user can not be set to ``realm2``, then.

The problem arises here:
https://github.com/privacyidea/privacyidea/blob/master/privacyidea/api/lib/prepolicy.py#L738

The realm parameter in the request is changed. But we also pass a user_object in the ``request`` object around. So we also would need to update the user object in line 739 like this:

    request.User = User(username, new_realm[0])

It is hard to say, if this is a real bug. However, it changes the behaviour so I would like to keep this for the next release and not put it into 2.23.1.


Discussion:
This change would lead to, that the PolicyError in this test https://github.com/privacyidea/privacyidea/blob/master/tests/test_api_lib_policy.py#L421 is not raised anymore

Resolving patch:
diff --git a/privacyidea/api/lib/prepolicy.py b/privacyidea/api/lib/prepolicy.py
--- a/privacyidea/api/lib/prepolicy.py
+++ b/privacyidea/api/lib/prepolicy.py
@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
 #
+#  2018-11-12 Cornelius Kölbel <cornelius.koelbel@netknights.it>
+#             In case of "setrealm" allow a user no to be in the
+#             original realm.
 #  2017-04-22 Cornelius Kölbel <cornelius.koelbel@netknights.it>
 #             Add wrapper for U2F token
 #  2017-01-18 Cornelius Kölbel <cornelius.koelbel@netknights.it>
@@ -61,7 +64,7 @@
 from flask import g, current_app
 from privacyidea.lib.policy import SCOPE, ACTION, PolicyClass
 from privacyidea.lib.user import (get_user_from_param, get_default_realm,
-                                  split_user)
+                                  split_user, User)
 from privacyidea.lib.token import (get_tokens, get_realms_of_token)
 from privacyidea.lib.utils import (generate_password, get_client_ip,
                                    parse_timedelta, is_true)
@@ -746,6 +749,8 @@ def set_realm(request=None, action=None):
     elif len(new_realm) == 1:
         # There is one specific realm, which we set in the request
         request.all_data["realm"] = new_realm[0]
+        # We also need to update the user
+        request.User = User(username, request.all_data["realm"])
 
     return True
 


Test patch:
diff --git a/tests/test_api_lib_policy.py b/tests/test_api_lib_policy.py
--- a/tests/test_api_lib_policy.py
+++ b/tests/test_api_lib_policy.py
@@ -418,6 +418,7 @@ def test_06_set_realm(self):
         g.policy_object = PolicyClass()
         # This request will trigger two policies with different realms to set
         req.all_data = {"realm": "somerealm"}
+        req.User = None
         self.assertRaises(PolicyError, set_realm, req)
 
         # finally delete policy
Repository: privacyidea/privacyidea

Issue:
Token type in trigger challenge response
It would be nice, if the response of a trigger challenge contains the token types.
For some plugins we could hide the otp input, if it is not needed (e.g. a user has only one u2f token).


Discussion:
This should be possible here:
https://github.com/privacyidea/privacyidea/blob/master/privacyidea/lib/token.py#L2029

By simply adding 

~~~~python
challenge_info["tokentype"] = token_obj.token.type
~~~~

Resolving patch:
diff --git a/privacyidea/lib/token.py b/privacyidea/lib/token.py
--- a/privacyidea/lib/token.py
+++ b/privacyidea/lib/token.py
@@ -2026,6 +2026,8 @@ def create_challenges_from_tokens(token_list, reply_dict, options=None):
                 challenge_info["transaction_id"] = transaction_id
                 challenge_info["attributes"] = attributes
                 challenge_info["serial"] = token_obj.token.serial
+                challenge_info["type"] = token_obj.get_tokentype()
+                challenge_info["message"] = message
                 # If exist, add next pin and next password change
                 next_pin = token_obj.get_tokeninfo(
                         "next_pin_change")
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@
 import sys
 
 #VERSION="2.1dev4"
-VERSION="2.23.3"
+VERSION="2.23.4dev1"
 
 # Taken from kennethreitz/requests/setup.py
 package_directory = os.path.realpath(os.path.dirname(__file__))


Test patch:
Repository: privacyidea/privacyidea

Issue:
Enhance the condition for the otp counter to also match greater than and smaller than
Currently the event handler condition for ``otp_counter`` does only an exact match:
https://github.com/privacyidea/privacyidea/blob/master/privacyidea/lib/eventhandler/base.py#L437

We should also allow to match "otp_counter > 100", since sometimes an otp_counter might be increased by 2 or more and thus the counter might actually skip the 100.



Discussion:

Resolving patch:
diff --git a/privacyidea/lib/eventhandler/base.py b/privacyidea/lib/eventhandler/base.py
--- a/privacyidea/lib/eventhandler/base.py
+++ b/privacyidea/lib/eventhandler/base.py
@@ -187,7 +187,8 @@ def conditions(cls):
             CONDITION.OTP_COUNTER: {
                 "type": "str",
                 "desc": _("Action is triggered, if the counter of the token "
-                          "equals this setting.")
+                          "equals this setting. Can also be "
+                          "'>100' or '<99' for no exact match.")
             },
             CONDITION.LAST_AUTH: {
                 "type": "str",
@@ -434,8 +435,8 @@ def check_condition(self, options):
                     return False
 
             if CONDITION.OTP_COUNTER in conditions:
-                if token_obj.token.count != \
-                      int(conditions.get(CONDITION.OTP_COUNTER)):
+                cond = conditions.get(CONDITION.OTP_COUNTER)
+                if not compare_condition(cond, token_obj.token.count):
                     return False
 
             if CONDITION.LAST_AUTH in conditions:


Test patch:
diff --git a/tests/test_lib_events.py b/tests/test_lib_events.py
--- a/tests/test_lib_events.py
+++ b/tests/test_lib_events.py
@@ -2487,6 +2487,28 @@ def test_17_check_conditions_otp_counter(self):
         # The counter of the token is 0
         self.assertEqual(r, True)
 
+        # match if counter is >100
+        tok.token.count = 101
+        tok.token.save()
+
+        r = uhandler.check_condition(
+            {"g": {},
+             "handler_def": {"conditions": {CONDITION.OTP_COUNTER: ">100"}},
+             "request": req,
+             "response": resp
+             }
+        )
+        self.assertTrue(r)
+
+        r = uhandler.check_condition(
+            {"g": {},
+             "handler_def": {"conditions": {CONDITION.OTP_COUNTER: "<100"}},
+             "request": req,
+             "response": resp
+             }
+        )
+        self.assertFalse(r)
+
         remove_token(serial)
 
     def test_18_check_conditions_last_auth(self):
Repository: privacyidea/privacyidea

Issue:
Avoid quering the userstore during the event handler
When determining the token owner there is an additional request to the user store to determine, if the user exists.
https://github.com/privacyidea/privacyidea/blob/master/privacyidea/lib/eventhandler/base.py#L270
We can get rid of this.


Discussion:

Resolving patch:
diff --git a/privacyidea/lib/eventhandler/base.py b/privacyidea/lib/eventhandler/base.py
--- a/privacyidea/lib/eventhandler/base.py
+++ b/privacyidea/lib/eventhandler/base.py
@@ -264,16 +264,10 @@ def _get_tokenowner(request):
                 log.info("Could not determine tokenowner for {0!s}. Maybe the "
                          "user does not exist anymore.".format(serial))
                 log.debug(exx)
-        # We now check, if the user exists at all!
-        try:
-            ui = user.info
-        except UserError as exx:
-            if exx.id in [904, 905]:
-                # 904: User can not be found - maybe we got here due to PassOnNoUser
-                # 905: This would be a parameter error - why is this here?
-                user = User()
-            else:
-                raise exx
+        # If the user does not exist, we set an empty user
+        if not user.exist():
+            user = User()
+
         return user
 
     @staticmethod


Test patch:
Repository: privacyidea/privacyidea

Issue:
Improved support button
Make the support button more intelligent.

The support button can direct the user to different location based on:

1. language (german, english)
2. user type:
    * not logged in
    * logged in as role user
    * logged in as role admin


Discussion:

Resolving patch:
diff --git a/doc/conf.py b/doc/conf.py
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -17,7 +17,7 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.23.3'
+version = '2.23.4'
 # The full version, including alpha/beta/rc tags.
 #release = '2.16dev5'
 release = version


Test patch:
diff --git a/tests/test_api_validate.py b/tests/test_api_validate.py
--- a/tests/test_api_validate.py
+++ b/tests/test_api_validate.py
@@ -2145,27 +2145,6 @@ def test_30_challenge_text(self):
         # Configure the SMS Gateway
         self.setup_sms_gateway()
         from privacyidea.lib.config import set_privacyidea_config
-        post_url = "http://smsgateway.com/sms_send_api.cgi"
-        success_body = "ID 12345"
-
-        identifier = "myGW"
-        provider_module = "privacyidea.lib.smsprovider.HttpSMSProvider" \
-                          ".HttpSMSProvider"
-        id = set_smsgateway(identifier, provider_module, description="test",
-                            options={"HTTP_METHOD": "POST",
-                                     "URL": post_url,
-                                     "RETURN_SUCCESS": "ID",
-                                     "text": "{otp}",
-                                     "phone": "{phone}"})
-        self.assertTrue(id > 0)
-        # set config sms.identifier = myGW
-        r = set_privacyidea_config("sms.identifier", identifier)
-        self.assertTrue(r in ["insert", "update"])
-        responses.add(responses.POST,
-                      post_url,
-                      body=success_body)
-
-
 
         self.setUp_user_realms()
         user = User("cornelius", self.realm1)
Repository: privacyidea/privacyidea

Issue:
TokenOwner migration loses assignments
At some point, the TokenOwner migration loses some assignments.

Before migration:
```
# privacyidea-token-janitor find --csv 2>/dev/null| wc -l                
10003
# privacyidea-token-janitor find --assigned true --csv 2>/dev/null| wc -l
10003
```

After migration:

```
# privacyidea-token-janitor find --csv 2>/dev/null| wc -l                
10003
# privacyidea-token-janitor find --assigned true --csv 2>/dev/null| wc -l
5988
# echo 'SELECT COUNT(*) FROM tokenowner' | mysql pi
COUNT(*)
5988
```

The migration script *does* emit some warnings (63 warnings, to be exact) of this form:
```
/opt/venv/lib/python2.7/site-packages/pymysql/cursors.py:170: Warning: (1292, u"Truncated incorrect DOUBLE value: '7cf7f772-d30d-1036-8008-cd5127ce8b3b'")
  result = self._query(query)
```
with changing user IDs. I'm not sure if that's related.


Discussion:
OK, the warnings *are* related, because the the emitted token query is

```sql
SELECT token.id AS token_id, token.serial AS token_serial, token.resolver AS token_resolver, token.resolver_type AS token_resolver_type, token.user_id AS token_user_id
FROM token
WHERE token.user_id
```

which apparently does some funky stuff and causes MariaDB to emit the warnings above. As a consequence, not all tokens are selected, so not all token owners are migrated.

Resolving patch:
diff --git a/migrations/versions/48ee74b8a7c8_.py b/migrations/versions/48ee74b8a7c8_.py
--- a/migrations/versions/48ee74b8a7c8_.py
+++ b/migrations/versions/48ee74b8a7c8_.py
@@ -83,7 +83,7 @@ def upgrade():
         bind = op.get_bind()
         session = orm.Session(bind=bind)
         # For each token, that has an owner, create a tokenowner entry
-        for token in session.query(Token).filter(Token.user_id):
+        for token in session.query(Token).filter(Token.user_id != "", Token.user_id.isnot(None)):
             token_realms = session.query(TokenRealm).filter(TokenRealm.token_id == token.id).all()
             realm_id = None
             if not token_realms:


Test patch:
Repository: privacyidea/privacyidea

Issue:
Add some debug info while splitting passwords
Splitting passwords is sometimes a root for problems.

It is hard to debug, if the password with the long length arrived, if the PIN and the OTP was split correctly...

We should add some debug information, to improve this.


Discussion:

Resolving patch:
diff --git a/privacyidea/lib/tokenclass.py b/privacyidea/lib/tokenclass.py
--- a/privacyidea/lib/tokenclass.py
+++ b/privacyidea/lib/tokenclass.py
@@ -1208,12 +1208,17 @@ def split_pin_pass(self, passw, user=None, options=None):
         """
         # The database field is always an integer
         otplen = self.token.otplen
+        log.debug("Splitting the an OTP value of length {0!s} from the password.".format(otplen))
         if get_prepend_pin():
             pin = passw[0:-otplen]
             otpval = passw[-otplen:]
+            log.debug("PIN prepended. PIN length is {0!s}, OTP length is {0!s}.".format(len(pin),
+                                                                                        len(otpval)))
         else:
             pin = passw[otplen:]
             otpval = passw[0:otplen]
+            log.debug("PIN appended. PIN length is {0!s}, OTP length is {0!s}.".format(len(pin),
+                                                                                       len(otpval)))
 
         return True, pin, otpval
 


Test patch:
Repository: privacyidea/privacyidea

Issue:
Remove sub modules
We often have problems, when building plugins. Plugins have another release pace than the server.

We should remove the submodules
* pam_python
* freeradius
* simplesamlphp
* adminclient

and give them their own versioning. 
The debian build environment has to be added/moved to those repositories.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -138,9 +138,6 @@ def get_scripts(dir):
                   'deploy/privacyidea/private.pem',
                   'deploy/privacyidea/public.pem']),
                 ('share/man/man1', get_man_pages("tools")),
-                ('lib/privacyidea/authmodules/FreeRADIUS',
-                 ["authmodules/FreeRADIUS/LICENSE",
-                  "authmodules/FreeRADIUS/privacyidea_radius.pm"]),
                ('lib/privacyidea/authmodules/OTRS',
                  ["authmodules/OTRS/privacyIDEA.pm"]),
                 ('lib/privacyidea/migrations',


Test patch:
Repository: privacyidea/privacyidea

Issue:
older passlib version on ubuntu 16.04
On ubuntu 16.04 there is an older passlib version.

The function ``hash_with_pepper`` in lib/crypto fails for older passlib versions, so that this fails:

~~~~python
pw_dig = passlib.hash.pbkdf2_sha512.hash(key + password)
~~~~

In the older version it would be called ``passlib.hash.pbkdf2_sha512.encrypt(...)``

It looks like, we can not handle this via an import, so we would have to do something like this:

~~~~python
from passlib.hash import pbkdf2_sha512
if hasattr(pbkdf2_sha512, "encrypt"):
    hash_admin_pw = pbkdf2_sha512.encrypt
elif hasattr(pbkdf2_sha512, "hash"):
    hash_admin_pw = pbkdf2_sha512.hash
else:
    raise Exception(bummer)
...

pw_dig = hash_admin_pw(key + password)
~~~~    



Discussion:

Resolving patch:
diff --git a/privacyidea/lib/crypto.py b/privacyidea/lib/crypto.py
--- a/privacyidea/lib/crypto.py
+++ b/privacyidea/lib/crypto.py
@@ -63,7 +63,6 @@
 except ImportError:
     # Bummer the version of PyCrypto has no PKCS1_15
     SIGN_WITH_RSA = True
-import passlib.hash
 import traceback
 from six import PY2, text_type
 
@@ -77,6 +76,14 @@
 from cryptography.hazmat.primitives.asymmetric import rsa
 from cryptography.hazmat.primitives import serialization
 
+import passlib.hash
+if hasattr(passlib.hash.pbkdf2_sha512, "encrypt"):
+    hash_admin_pw = passlib.hash.pbkdf2_sha512.encrypt
+elif hasattr(passlib.hash.pbkdf2_sha512, "hash"):
+    hash_admin_pw = passlib.hash.pbkdf2_sha512.hash
+else:
+    raise Exception("No password hashing method available")
+
 if not PY2:
     long = int
 
@@ -174,7 +181,7 @@ def hash_with_pepper(password):
     :rtype: str
     """
     key = get_app_config_value("PI_PEPPER", "missing")
-    pw_dig = passlib.hash.pbkdf2_sha512.hash(key + password)
+    pw_dig = hash_admin_pw(key + password)
     return pw_dig
 
 


Test patch:
Repository: privacyidea/privacyidea

Issue:
Update ldap3 dependency
The dependency for ldap3 should be updated to 2.6.
It fixes the bug with the failing objectGUID.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -51,7 +51,7 @@ def get_file_list(file_path):
                     "docutils>=0.12",
                     "funcparserlib>=0.3.6",
                     "itsdangerous>=0.24",
-                    "ldap3>=2.5",
+                    "ldap3>=2.6",
                     "netaddr>=0.7.12",
                     "passlib>=1.6.2",
                     "pyasn1>=0.4.2",


Test patch: