Repository: privacyidea/privacyidea

Issue:
no serial in polling on /ttype/push
We may have a bug in push polling. This exception occurs on polling

https://gist.github.com/laclaro/743618d11f61f8a817e273db6b804a9a

This may be related to #2534.


Discussion:
In ``test_lib_tokens_push.py`` we have a lot of tests of ``/ttype/push`` with a serial. To me it looks like as if the server shoud work correctl.y

So it could also be, that the app actually sends an empty serial, which results in the error. 
@laclaro  Which App and version was used when observing this?

See https://github.com/privacyidea/privacyidea/wiki/concept%3A-PushToken-poll#polling
Well, we might have not tests for GET /ttype/push
Actually @laclaro is right, the (API) tests use the FakeFlaskG class which sets `serial` to `None` (so it exists).
When calling the `/ttype` endpoint, the `serial` is not set in the `g`-object since it uses a different `before_request` decorator.
We should add this to the `before_request` decorator for the `/ttype` endpoint but we should also check if requiring it in the user matching 
https://github.com/privacyidea/privacyidea/blob/2bf8295da409b0f86483b11ae1fb3badd928650b/privacyidea/lib/policy.py#L2574
is necessary (All other endpoint decorators set it to `None`).
> @laclaro Which App and version was used when observing this?

Mostly, the Authenticator 3.1 Beta Channel on iOS, possibly also on Android.
We need the serial in the matching, since we may have tokeninfo conditions in every policy. I suggest to to add a line to before_request().

https://github.com/privacyidea/privacyidea/blob/0622dcbf3fb3f34fcf4fcb7331f54047a3858dcf/privacyidea/api/ttype.py#L70-L71

``` python
72      g.serial = getParam(request.all_data, "serial") or None
```




Resolving patch:
diff --git a/privacyidea/api/ttype.py b/privacyidea/api/ttype.py
--- a/privacyidea/api/ttype.py
+++ b/privacyidea/api/ttype.py
@@ -69,6 +69,7 @@ def before_request():
     # access_route contains the ip adresses of all clients, hops and proxies.
     g.client_ip = get_client_ip(request,
                                 get_from_config(SYSCONF.OVERRIDECLIENT))
+    g.serial = getParam(request.all_data, "serial") or None
     g.audit_object.log({"success": False,
                         "action_detail": "",
                         "client": g.client_ip,


Test patch:
diff --git a/tests/test_api_ttype.py b/tests/test_api_ttype.py
--- a/tests/test_api_ttype.py
+++ b/tests/test_api_ttype.py
@@ -1,17 +1,41 @@
-import json
+# -*- coding: utf-8 -*-
 from .base import MyApiTestCase
 from privacyidea.lib.user import (User)
-from privacyidea.lib.tokens.totptoken import HotpTokenClass
-from privacyidea.models import (Token)
-from privacyidea.lib.config import (set_privacyidea_config, get_token_types,
-                                    get_inc_fail_count_on_false_pin,
-                                    delete_privacyidea_config)
-from privacyidea.lib.token import (get_tokens, init_token, remove_token,
-                                   reset_token)
+from privacyidea.lib.config import (set_privacyidea_config)
+from privacyidea.lib.token import (get_tokens, init_token, remove_token)
+from privacyidea.lib.policy import (SCOPE, set_policy)
+from privacyidea.lib.smsprovider.SMSProvider import set_smsgateway
+from privacyidea.lib.smsprovider.FirebaseProvider import FIREBASE_CONFIG
+from cryptography.hazmat.primitives import serialization, hashes
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import padding, rsa
+from privacyidea.lib.utils import to_bytes, to_unicode
+from privacyidea.lib.tokens.pushtoken import (PUSH_ACTION,
+                                              strip_key,
+                                              PUBLIC_KEY_SMARTPHONE, PRIVATE_KEY_SERVER,
+                                              PUBLIC_KEY_SERVER)
+from privacyidea.lib.utils import b32encode_and_unicode
+from datetime import datetime, timedelta
+from pytz import utc
+from base64 import b32decode, b32encode
+import mock
+import responses
 
-from privacyidea.lib.error import (ParameterError, UserError)
+from .test_lib_tokens_push import myAccessTokenInfo, myCredentials
 
 PWFILE = "tests/testdata/passwords"
+HOSTSFILE = "tests/testdata/hosts"
+DICT_FILE = "tests/testdata/dictionary"
+FIREBASE_FILE = "tests/testdata/firebase-test.json"
+CLIENT_FILE = "tests/testdata/google-services.json"
+FB_CONFIG_VALS = {
+    FIREBASE_CONFIG.REGISTRATION_URL: "http://test/ttype/push",
+    FIREBASE_CONFIG.TTL: 10,
+    FIREBASE_CONFIG.API_KEY: "1",
+    FIREBASE_CONFIG.APP_ID: "2",
+    FIREBASE_CONFIG.PROJECT_NUMBER: "3",
+    FIREBASE_CONFIG.PROJECT_ID: "test-123456",
+    FIREBASE_CONFIG.JSON_CONFIG: FIREBASE_FILE}
 
 
 class TtypeAPITestCase(MyApiTestCase):
@@ -58,3 +82,242 @@ def test_02_u2f(self):
             auditdata = result.get("value").get("auditdata")
             self.assertTrue(len(auditdata) > 0)
             self.assertEqual(auditdata[0].get("token_type"), "u2f")
+
+
+class TtypePushAPITestCase(MyApiTestCase):
+    """
+    test /ttype/push
+    """
+
+    server_private_key = rsa.generate_private_key(public_exponent=65537,
+                                                  key_size=4096,
+                                                  backend=default_backend())
+    server_private_key_pem = to_unicode(server_private_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,
+        encryption_algorithm=serialization.NoEncryption()))
+    server_public_key_pem = to_unicode(server_private_key.public_key().public_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PublicFormat.SubjectPublicKeyInfo))
+
+    # We now allow white spaces in the firebase config name
+    firebase_config_name = "my firebase config"
+
+    smartphone_private_key = rsa.generate_private_key(public_exponent=65537,
+                                                      key_size=4096,
+                                                      backend=default_backend())
+    smartphone_public_key = smartphone_private_key.public_key()
+    smartphone_public_key_pem = to_unicode(smartphone_public_key.public_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PublicFormat.SubjectPublicKeyInfo))
+    # The smartphone sends the public key in URLsafe and without the ----BEGIN header
+    smartphone_public_key_pem_urlsafe = strip_key(smartphone_public_key_pem).replace("+", "-").replace("/", "_")
+    serial_push = "PIPU001"
+
+    def _create_push_token(self):
+        tparams = {'type': 'push', 'genkey': 1}
+        tparams.update(FB_CONFIG_VALS)
+        tok = init_token(param=tparams)
+        tok.add_tokeninfo(PUSH_ACTION.FIREBASE_CONFIG, self.firebase_config_name)
+        tok.add_tokeninfo(PUBLIC_KEY_SMARTPHONE, self.smartphone_public_key_pem_urlsafe)
+        tok.add_tokeninfo('firebase_token', 'firebaseT')
+        tok.add_tokeninfo(PUBLIC_KEY_SERVER, self.server_public_key_pem)
+        tok.add_tokeninfo(PRIVATE_KEY_SERVER, self.server_private_key_pem, 'password')
+        tok.del_tokeninfo("enrollment_credential")
+        tok.token.rollout_state = "enrolled"
+        tok.token.active = True
+        return tok
+
+    def test_00_create_realms(self):
+        self.setUp_user_realms()
+
+    def test_01_api_enroll_push(self):
+        self.authenticate()
+
+        # Failed enrollment due to missing policy
+        with self.app.test_request_context('/token/init',
+                                           method='POST',
+                                           data={"type": "push",
+                                                 "genkey": 1},
+                                           headers={'Authorization': self.at}):
+            res = self.app.full_dispatch_request()
+            self.assertNotEqual(res.status_code, 200)
+            error = res.json.get("result").get("error")
+            self.assertEqual(error.get("message"),
+                             "Missing enrollment policy for push token: push_firebase_configuration")
+            self.assertEqual(error.get("code"), 303)
+
+        r = set_smsgateway(self.firebase_config_name,
+                           u'privacyidea.lib.smsprovider.FirebaseProvider.FirebaseProvider',
+                           "myFB", FB_CONFIG_VALS)
+        self.assertTrue(r > 0)
+        set_policy("push1", scope=SCOPE.ENROLL,
+                   action="{0!s}={1!s}".format(PUSH_ACTION.FIREBASE_CONFIG,
+                                               self.firebase_config_name))
+
+        # 1st step
+        with self.app.test_request_context('/token/init',
+                                           method='POST',
+                                           data={"type": "push",
+                                                 "genkey": 1},
+                                           headers={'Authorization': self.at}):
+            res = self.app.full_dispatch_request()
+            self.assertEqual(res.status_code, 200)
+            detail = res.json.get("detail")
+            serial = detail.get("serial")
+            self.assertEqual(detail.get("rollout_state"), "clientwait")
+            self.assertTrue("pushurl" in detail)
+            # check that the new URL contains the serial number
+            self.assertTrue("&serial=PIPU" in detail.get("pushurl").get("value"))
+            self.assertTrue("appid=" in detail.get("pushurl").get("value"))
+            self.assertTrue("appidios=" in detail.get("pushurl").get("value"))
+            self.assertTrue("apikeyios=" in detail.get("pushurl").get("value"))
+            self.assertFalse("otpkey" in detail)
+            enrollment_credential = detail.get("enrollment_credential")
+
+        # 2nd step. Failing with wrong serial number
+        with self.app.test_request_context('/ttype/push',
+                                           method='POST',
+                                           data={"serial": "wrongserial",
+                                                 "pubkey": self.smartphone_public_key_pem_urlsafe,
+                                                 "fbtoken": "firebaseT"}):
+            res = self.app.full_dispatch_request()
+            self.assertTrue(res.status_code == 404, res)
+            status = res.json.get("result").get("status")
+            self.assertFalse(status)
+            error = res.json.get("result").get("error")
+            self.assertEqual(error.get("message"),
+                             "No token with this serial number in the rollout state 'clientwait'.")
+
+        # 2nd step. Fails with missing enrollment credential
+        with self.app.test_request_context('/ttype/push',
+                                           method='POST',
+                                           data={"serial": serial,
+                                                 "pubkey": self.smartphone_public_key_pem_urlsafe,
+                                                 "fbtoken": "firebaseT",
+                                                 "enrollment_credential": "WRonG"}):
+            res = self.app.full_dispatch_request()
+            self.assertTrue(res.status_code == 400, res)
+            status = res.json.get("result").get("status")
+            self.assertFalse(status)
+            error = res.json.get("result").get("error")
+            self.assertEqual(error.get("message"),
+                             "ERR905: Invalid enrollment credential. You are not authorized to finalize this token.")
+
+        # 2nd step: as performed by the smartphone
+        with self.app.test_request_context('/ttype/push',
+                                           method='POST',
+                                           data={"enrollment_credential": enrollment_credential,
+                                                 "serial": serial,
+                                                 "pubkey": self.smartphone_public_key_pem_urlsafe,
+                                                 "fbtoken": "firebaseT"}):
+            res = self.app.full_dispatch_request()
+            self.assertTrue(res.status_code == 200, res)
+            detail = res.json.get("detail")
+            # still the same serial number
+            self.assertEqual(serial, detail.get("serial"))
+            self.assertEqual(detail.get("rollout_state"), "enrolled")
+            # Now the smartphone gets a public key from the server
+            augmented_pubkey = "-----BEGIN RSA PUBLIC KEY-----\n{}\n-----END RSA PUBLIC KEY-----\n".format(
+                detail.get("public_key"))
+            parsed_server_pubkey = serialization.load_pem_public_key(
+                to_bytes(augmented_pubkey),
+                default_backend())
+            self.assertIsInstance(parsed_server_pubkey, rsa.RSAPublicKey)
+            pubkey = detail.get("public_key")
+
+            # Now check, what is in the token in the database
+            toks = get_tokens(serial=serial)
+            self.assertEqual(len(toks), 1)
+            token_obj = toks[0]
+            self.assertEqual(token_obj.token.rollout_state, u"enrolled")
+            self.assertTrue(token_obj.token.active)
+            tokeninfo = token_obj.get_tokeninfo()
+            self.assertEqual(tokeninfo.get("public_key_smartphone"), self.smartphone_public_key_pem_urlsafe)
+            self.assertEqual(tokeninfo.get("firebase_token"), u"firebaseT")
+            self.assertEqual(tokeninfo.get("public_key_server").strip().strip("-BEGIN END RSA PUBLIC KEY-").strip(),
+                             pubkey)
+            # The token should also contain the firebase config
+            self.assertEqual(tokeninfo.get(PUSH_ACTION.FIREBASE_CONFIG), self.firebase_config_name)
+            # remove the token
+            remove_token(serial)
+
+    def test_02_api_push_poll(self):
+        r = set_smsgateway(self.firebase_config_name,
+                           u'privacyidea.lib.smsprovider.FirebaseProvider.FirebaseProvider',
+                           "myFB", FB_CONFIG_VALS)
+        self.assertGreater(r, 0)
+
+        # create a new push token
+        tokenobj = self._create_push_token()
+        serial = tokenobj.get_serial()
+
+        # set PIN
+        tokenobj.set_pin("pushpin")
+        tokenobj.add_user(User("cornelius", self.realm1))
+
+        # We mock the ServiceAccountCredentials, since we can not directly contact the Google API
+        with mock.patch('privacyidea.lib.smsprovider.FirebaseProvider.ServiceAccountCredentials') as mySA:
+            # alternative: side_effect instead of return_value
+            mySA.from_json_keyfile_name.return_value = myCredentials(myAccessTokenInfo("my_bearer_token"))
+
+            # add responses, to simulate the communication to firebase
+            responses.add(responses.POST, 'https://fcm.googleapis.com/v1/projects/test-123456/messages:send',
+                          body="""{}""",
+                          content_type="application/json")
+
+            # Send the first authentication request to trigger the challenge.
+            # No push notification is submitted to firebase, but a challenge is created anyway
+            with self.app.test_request_context('/validate/check',
+                                               method='POST',
+                                               data={"user": "cornelius",
+                                                     "realm": self.realm1,
+                                                     "pass": "pushpin"}):
+                res = self.app.full_dispatch_request()
+                self.assertTrue(res.status_code == 400, res)
+                jsonresp = res.json
+                self.assertFalse(jsonresp.get("result").get("status"))
+                self.assertEqual(jsonresp.get("result").get("error").get("code"), 401)
+                self.assertEqual(jsonresp.get("result").get("error").get("message"),
+                                 "ERR401: Failed to submit message to firebase service.")
+
+        # first create a signature
+        ts = datetime.utcnow().isoformat()
+        sign_string = u"{serial}|{timestamp}".format(serial=serial, timestamp=ts)
+        sig = self.smartphone_private_key.sign(sign_string.encode('utf8'),
+                                               padding.PKCS1v15(),
+                                               hashes.SHA256())
+        # now check that we receive the challenge when polling
+        with self.app.test_request_context('/ttype/push',
+                                           method='GET',
+                                           data={"serial": serial,
+                                                 "timestamp": ts,
+                                                 "signature": b32encode(sig)}):
+            res = self.app.full_dispatch_request()
+
+            # check that the serial was set in flask g (via before_request in ttype.py)
+            self.assertTrue(self.app_context.g.serial, serial)
+            self.assertTrue(res.status_code == 200, res)
+            self.assertTrue(res.json['result']['status'])
+            chall = res.json['result']['value'][0]
+            self.assertTrue(chall)
+
+            challenge = chall["nonce"]
+            # This is what the smartphone answers.
+            # create the signature:
+            sign_data = "{0!s}|{1!s}".format(challenge, serial)
+            signature = b32encode_and_unicode(
+                self.smartphone_private_key.sign(sign_data.encode("utf-8"),
+                                                 padding.PKCS1v15(),
+                                                 hashes.SHA256()))
+
+            # Answer the challenge
+            with self.app.test_request_context('/ttype/push',
+                                               method='POST',
+                                               data={"serial": serial,
+                                                     "nonce": challenge,
+                                                     "signature": signature}):
+                res = self.app.full_dispatch_request()
+                self.assertTrue(res.status_code == 200, res)
+                self.assertTrue(res.json['result']['status'])
+                self.assertTrue(res.json['result']['value'])
diff --git a/tests/test_lib_tokens_push.py b/tests/test_lib_tokens_push.py
--- a/tests/test_lib_tokens_push.py
+++ b/tests/test_lib_tokens_push.py
@@ -11,10 +11,10 @@
 from privacyidea.lib.tokens.pushtoken import (PushTokenClass, PUSH_ACTION,
                                               DEFAULT_CHALLENGE_TEXT, strip_key,
                                               PUBLIC_KEY_SMARTPHONE, PRIVATE_KEY_SERVER,
+                                              PUBLIC_KEY_SERVER,
                                               PushAllowPolling, POLLING_ALLOWED)
 from privacyidea.lib.smsprovider.FirebaseProvider import FIREBASE_CONFIG
 from privacyidea.lib.token import get_tokens, remove_token, init_token
-from privacyidea.lib.tokens.pushtoken import PUBLIC_KEY_SERVER
 from privacyidea.lib.challenge import get_challenges
 from privacyidea.lib.crypto import geturandom
 from privacyidea.models import Token, Challenge
@@ -103,8 +103,8 @@ def _create_push_token(self):
         tok.add_tokeninfo(PUBLIC_KEY_SERVER, self.server_public_key_pem)
         tok.add_tokeninfo(PRIVATE_KEY_SERVER, self.server_private_key_pem, 'password')
         tok.del_tokeninfo("enrollment_credential")
-        tok.rollout_state = "enrolled"
-        tok.active = True
+        tok.token.rollout_state = "enrolled"
+        tok.token.active = True
         return tok
 
     def test_01_create_token(self):
@@ -214,21 +214,7 @@ def test_01_create_token(self):
         self.assertEqual(parsed_server_pubkey.public_numbers(), parsed_stripped_server_pubkey.public_numbers())
         remove_token(self.serial1)
 
-    def test_02_api_enroll(self):
-        self.authenticate()
-
-        # Failed enrollment due to missing policy
-        with self.app.test_request_context('/token/init',
-                                           method='POST',
-                                           data={"type": "push",
-                                                 "genkey": 1},
-                                           headers={'Authorization': self.at}):
-            res = self.app.full_dispatch_request()
-            self.assertNotEqual(res.status_code,  200)
-            error = res.json.get("result").get("error")
-            self.assertEqual(error.get("message"), "Missing enrollment policy for push token: push_firebase_configuration")
-            self.assertEqual(error.get("code"), 303)
-
+    def test_02a_lib_enroll(self):
         r = set_smsgateway(self.firebase_config_name,
                            u'privacyidea.lib.smsprovider.FirebaseProvider.FirebaseProvider',
                            "myFB", FB_CONFIG_VALS)
@@ -236,90 +222,7 @@ def test_02_api_enroll(self):
         set_policy("push1", scope=SCOPE.ENROLL,
                    action="{0!s}={1!s}".format(PUSH_ACTION.FIREBASE_CONFIG,
                                                self.firebase_config_name))
-
-        # 1st step
-        with self.app.test_request_context('/token/init',
-                                           method='POST',
-                                           data={"type": "push",
-                                                 "genkey": 1},
-                                           headers={'Authorization': self.at}):
-            res = self.app.full_dispatch_request()
-            self.assertEqual(res.status_code,  200)
-            detail = res.json.get("detail")
-            serial = detail.get("serial")
-            self.assertEqual(detail.get("rollout_state"), "clientwait")
-            self.assertTrue("pushurl" in detail)
-            # check that the new URL contains the serial number
-            self.assertTrue("&serial=PIPU" in detail.get("pushurl").get("value"))
-            self.assertTrue("appid=" in detail.get("pushurl").get("value"))
-            self.assertTrue("appidios=" in detail.get("pushurl").get("value"))
-            self.assertTrue("apikeyios=" in detail.get("pushurl").get("value"))
-            self.assertFalse("otpkey" in detail)
-            enrollment_credential = detail.get("enrollment_credential")
-
-        # 2nd step. Failing with wrong serial number
-        with self.app.test_request_context('/ttype/push',
-                                           method='POST',
-                                           data={"serial": "wrongserial",
-                                                 "pubkey": self.smartphone_public_key_pem_urlsafe,
-                                                 "fbtoken": "firebaseT"}):
-            res = self.app.full_dispatch_request()
-            self.assertTrue(res.status_code == 404, res)
-            status = res.json.get("result").get("status")
-            self.assertFalse(status)
-            error = res.json.get("result").get("error")
-            self.assertEqual(error.get("message"),
-                             "No token with this serial number in the rollout state 'clientwait'.")
-
-        # 2nd step. Fails with missing enrollment credential
-        with self.app.test_request_context('/ttype/push',
-                                           method='POST',
-                                           data={"serial": serial,
-                                                 "pubkey": self.smartphone_public_key_pem_urlsafe,
-                                                 "fbtoken": "firebaseT",
-                                                 "enrollment_credential": "WRonG"}):
-            res = self.app.full_dispatch_request()
-            self.assertTrue(res.status_code == 400, res)
-            status = res.json.get("result").get("status")
-            self.assertFalse(status)
-            error = res.json.get("result").get("error")
-            self.assertEqual(error.get("message"),
-                             "ERR905: Invalid enrollment credential. You are not authorized to finalize this token.")
-
-        # 2nd step: as performed by the smartphone
-        with self.app.test_request_context('/ttype/push',
-                                           method='POST',
-                                           data={"enrollment_credential": enrollment_credential,
-                                                 "serial": serial,
-                                                 "pubkey": self.smartphone_public_key_pem_urlsafe,
-                                                 "fbtoken": "firebaseT"}):
-            res = self.app.full_dispatch_request()
-            self.assertTrue(res.status_code == 200, res)
-            detail = res.json.get("detail")
-            # still the same serial number
-            self.assertEqual(serial, detail.get("serial"))
-            self.assertEqual(detail.get("rollout_state"), "enrolled")
-            # Now the smartphone gets a public key from the server
-            augmented_pubkey = "-----BEGIN RSA PUBLIC KEY-----\n{}\n-----END RSA PUBLIC KEY-----\n".format(
-                detail.get("public_key"))
-            parsed_server_pubkey = serialization.load_pem_public_key(
-                to_bytes(augmented_pubkey),
-                default_backend())
-            self.assertIsInstance(parsed_server_pubkey, RSAPublicKey)
-            pubkey = detail.get("public_key")
-
-            # Now check, what is in the token in the database
-            toks = get_tokens(serial=serial)
-            self.assertEqual(len(toks), 1)
-            token_obj = toks[0]
-            self.assertEqual(token_obj.token.rollout_state, u"enrolled")
-            self.assertTrue(token_obj.token.active)
-            tokeninfo = token_obj.get_tokeninfo()
-            self.assertEqual(tokeninfo.get("public_key_smartphone"), self.smartphone_public_key_pem_urlsafe)
-            self.assertEqual(tokeninfo.get("firebase_token"), u"firebaseT")
-            self.assertEqual(tokeninfo.get("public_key_server").strip().strip("-BEGIN END RSA PUBLIC KEY-").strip(), pubkey)
-            # The token should also contain the firebase config
-            self.assertEqual(tokeninfo.get(PUSH_ACTION.FIREBASE_CONFIG), self.firebase_config_name)
+        self._create_push_token()
 
     @responses.activate
     def test_03a_api_authenticate_fail(self):