Repository: privacyidea/privacyidea

Issue:
Allow more tags in email token
The email token sends an emal with an OTP value to the user's email address.
The email text can contain the tags *otp* and *serial*.

We should use the same tags and the same code as we are using in the email notification handler!
Including esepcially the **username**.


Discussion:

Resolving patch:
diff --git a/privacyidea/lib/eventhandler/usernotification.py b/privacyidea/lib/eventhandler/usernotification.py
--- a/privacyidea/lib/eventhandler/usernotification.py
+++ b/privacyidea/lib/eventhandler/usernotification.py
@@ -47,6 +47,7 @@
 from privacyidea.lib.smtpserver import get_smtpservers
 from privacyidea.lib.smsprovider.SMSProvider import get_smsgateway
 from privacyidea.lib.user import User, get_user_list
+from privacyidea.lib.utils import create_tag_dict
 from privacyidea.lib import _
 import logging
 import datetime
@@ -296,28 +297,18 @@ def do(self, action, options=None):
                 token_objects = get_tokens(user=tokenowner)
                 serial = ','.join([tok.get_serial() for tok in token_objects])
 
-            time = datetime.datetime.now().strftime("%H:%M:%S")
-            date = datetime.datetime.now().strftime("%Y-%m-%d")
-            tags = dict(admin=logged_in_user.get("username"),
-                        realm=logged_in_user.get("realm"),
-                        action=request.path,
-                        serial=serial,
-                        url=request.url_root,
-                        user=tokenowner.info.get("givenname"),
-                        surname=tokenowner.info.get("surname"),
-                        givenname=recipient.get("givenname"),
-                        username=tokenowner.login,
-                        userrealm=tokenowner.realm,
-                        tokentype=tokentype,
-                        registrationcode=registrationcode,
-                        recipient_givenname=recipient.get("givenname"),
-                        recipient_surname=recipient.get("surname"),
-                        googleurl_value=googleurl_value,
-                        time=time,
-                        date=date,
-                        client_ip=g.client_ip,
-                        ua_browser=request.user_agent.browser,
-                        ua_string=request.user_agent.string)
+            tags = create_tag_dict(logged_in_user=logged_in_user,
+                                   request=request,
+                                   client_ip=g.client_ip,
+                                   googleurl_value=googleurl_value,
+                                   recipient=recipient,
+                                   tokenowner=tokenowner,
+                                   serial=serial,
+                                   tokentype=tokentype,
+                                   registrationcode=registrationcode,
+                                   escape_html=action.lower() == "sendmail" and
+                                               handler_options.get("mimetype", "").lower() == "html")
+
             body = body.format(googleurl_img=googleurl_img,
                                **tags)
             subject = subject.format(**tags)
diff --git a/privacyidea/lib/tokens/emailtoken.py b/privacyidea/lib/tokens/emailtoken.py
--- a/privacyidea/lib/tokens/emailtoken.py
+++ b/privacyidea/lib/tokens/emailtoken.py
@@ -71,7 +71,7 @@
 from privacyidea.lib.tokens.smstoken import HotpTokenClass
 from privacyidea.lib.config import get_from_config
 from privacyidea.api.lib.utils import getParam
-from privacyidea.lib.utils import is_true
+from privacyidea.lib.utils import is_true, create_tag_dict
 from privacyidea.lib.policy import (SCOPE, ACTION, get_action_values_from_options)
 from privacyidea.lib.log import log_with
 from privacyidea.lib import _
@@ -438,12 +438,19 @@ def _compose_email(self, message="<otp>", subject="Your OTP", mimetype="plain"):
         message = message.replace("<otp>", otp)
         message = message.replace("<serial>", serial)
 
-        message = message.format(otp=otp, serial=serial)
+        tags = create_tag_dict(serial=serial,
+                               tokenowner=self.user,
+                               tokentype=self.get_tokentype(),
+                               recipient={"givenname": self.user.info.get("givenname"),
+                                          "surname": self.user.info.get("surname")},
+                               escape_html=mimetype.lower() == "html")
+
+        message = message.format(otp=otp, **tags)
 
         subject = subject.replace("<otp>", otp)
         subject = subject.replace("<serial>", serial)
 
-        subject = subject.format(otp=otp, serial=serial)
+        subject = subject.format(otp=otp, **tags)
 
         log.debug("sending Email to {0!r}".format(recipient))
 
diff --git a/privacyidea/lib/utils.py b/privacyidea/lib/utils.py
--- a/privacyidea/lib/utils.py
+++ b/privacyidea/lib/utils.py
@@ -47,6 +47,7 @@
 import threading
 import pkg_resources
 import time
+import cgi
 
 from privacyidea.lib.error import ParameterError, ResourceNotFoundError
 
@@ -1192,4 +1193,64 @@ def split_pin_pass(passw, otplen, prependpin):
         otpval = passw[0:otplen]
         log.debug("PIN appended. PIN length is {0!s}, OTP length is {0!s}.".format(len(pin),
                                                                                    len(otpval)))
-    return pin, otpval
\ No newline at end of file
+    return pin, otpval
+
+
+def create_tag_dict(logged_in_user=None,
+                    request=None,
+                    serial=None,
+                    tokenowner=None,
+                    tokentype=None,
+                    recipient=None,
+                    registrationcode=None,
+                    googleurl_value=None,
+                    client_ip=None,
+                    escape_html=False):
+    """
+    This helper function creates a dictionary with tags to be used in sending emails
+    either with email tokens or within the notification handler
+
+    :param logged_in_user: The acting logged in user (admin)
+    :param request: The HTTP request object
+    :param serial: The serial number of the token
+    :param tokenowner: The owner of the token
+    :type tokenowner: user object
+    :param tokentype: The type of the token
+    :param recipient: The recipient
+    :type recipient: dictionary with "givenname" and "surname"
+    :param registrationcode: The registration code of a token
+    :param googleurl_value: The URL for the QR code during token enrollemnt
+    :param client_ip: The IP of the client
+    :param escape_html: Whether the values for the tags should be html escaped
+    :return: The tag dictionary
+    """
+    time = datetime.now().strftime("%H:%M:%S")
+    date = datetime.now().strftime("%Y-%m-%d")
+    recipient = recipient or {}
+    tags = dict(admin=logged_in_user.get("username") if logged_in_user else "",
+                realm=logged_in_user.get("realm") if logged_in_user else "",
+                action=request.path if request else "",
+                serial=serial,
+                url=request.url_root if request else "",
+                user=tokenowner.info.get("givenname") if tokenowner else "",
+                surname=tokenowner.info.get("surname") if tokenowner else "",
+                givenname=recipient.get("givenname"),
+                username=tokenowner.login if tokenowner else "",
+                userrealm=tokenowner.realm if tokenowner else "",
+                tokentype=tokentype,
+                registrationcode=registrationcode,
+                recipient_givenname=recipient.get("givenname"),
+                recipient_surname=recipient.get("surname"),
+                googleurl_value=googleurl_value,
+                time=time,
+                date=date,
+                client_ip=client_ip,
+                ua_browser=request.user_agent.browser if request else "",
+                ua_string=request.user_agent.string if request else "")
+    if escape_html:
+        escaped_tags = {}
+        for key, value in tags.items():
+            escaped_tags[key] = cgi.escape(value) if value is not None else None
+        tags = escaped_tags
+
+    return tags
\ No newline at end of file


Test patch:
diff --git a/tests/test_lib_events.py b/tests/test_lib_events.py
--- a/tests/test_lib_events.py
+++ b/tests/test_lib_events.py
@@ -4,6 +4,7 @@
 lib/eventhandler/usernotification.py (one event handler module)
 lib/event.py (the decorator)
 """
+import email
 
 from . import smtpmock
 import responses
@@ -34,7 +35,7 @@
 from privacyidea.lib.user import create_user, User
 from privacyidea.lib.policy import ACTION
 from privacyidea.lib.error import ParameterError, ResourceNotFoundError
-from privacyidea.lib.utils import is_true
+from privacyidea.lib.utils import is_true, to_unicode
 from datetime import datetime, timedelta
 from dateutil.parser import parse as parse_date_string
 from dateutil.tz import tzlocal
@@ -2643,3 +2644,167 @@ def test_18_check_conditions_last_auth(self):
         self.assertEqual(r, False)
 
         remove_token(serial)
+
+    @smtpmock.activate
+    def test_19_sendmail_escape_html(self):
+        # setup realms
+        self.setUp_user_realms()
+
+        r = add_smtpserver(identifier="myserver", server="1.2.3.4", tls=False)
+        self.assertTrue(r > 0)
+
+        smtpmock.setdata(response={"recp@example.com": (200, "OK")},
+                         support_tls=False)
+
+        g = FakeFlaskG()
+        audit_object = FakeAudit()
+        audit_object.audit_data["serial"] = "123456"
+
+        g.logged_in_user = {"username": "admin",
+                            "role": "admin",
+                            "realm": ""}
+        g.audit_object = audit_object
+
+        # Set a user agent with HTML tags
+        builder = EnvironBuilder(method='POST',
+                                 data={'serial': "OATH123456"},
+                                 headers={"User-Agent": "<b>agent</b>"})
+
+        env = builder.get_environ()
+        env["REMOTE_ADDR"] = "10.0.0.1"
+        g.client_ip = env["REMOTE_ADDR"]
+        req = Request(env)
+        req.all_data = {"serial": "SomeSerial",
+                        "user": "cornelius"}
+        req.User = User("cornelius", self.realm1)
+        resp = Response()
+        resp.data = """{"result": {"value": true}}"""
+        # If we send a plain email, we do not escape HTML
+        options = {"g": g,
+                   "request": req,
+                   "response": resp,
+                   "handler_def": {"options":
+                                       {"emailconfig": "myserver",
+                                        "body": "{ua_string} performed an action for {user}"}
+                                   }
+                   }
+
+        un_handler = UserNotificationEventHandler()
+        res = un_handler.do("sendmail", options=options)
+        self.assertTrue(res)
+        parsed_email = email.message_from_string(smtpmock.get_sent_message())
+        payload = to_unicode(parsed_email.get_payload(decode=True))
+        self.assertEqual(parsed_email.get_content_type(), "text/plain")
+        self.assertIn("<b>agent</b>", payload)
+        # If we send a HTML email, we do escape HTML
+        options = {"g": g,
+                   "request": req,
+                   "response": resp,
+                   "handler_def": {"options":
+                                       {"emailconfig": "myserver",
+                                        "mimetype": "html",
+                                        "body": "{ua_string} performed an action for {user}"}
+                                   }
+                   }
+        un_handler = UserNotificationEventHandler()
+        res = un_handler.do("sendmail", options=options)
+        self.assertTrue(res)
+        parsed_email = email.message_from_string(smtpmock.get_sent_message())
+        payload = to_unicode(parsed_email.get_payload(decode=True))
+        self.assertEqual(parsed_email.get_content_type(), "text/html")
+        self.assertIn("&lt;b&gt;agent&lt;/b&gt;", payload)
+        self.assertNotIn("<b>", payload)
+
+    @smtpmock.activate
+    def test_20_sendmail_googleurl(self):
+        # setup realms
+        self.setUp_user_realms()
+
+        r = add_smtpserver(identifier="myserver", server="1.2.3.4", tls=False)
+        self.assertTrue(r > 0)
+
+        smtpmock.setdata(response={"recp@example.com": (200, "OK")},
+                         support_tls=False)
+
+        g = FakeFlaskG()
+        audit_object = FakeAudit()
+        audit_object.audit_data["serial"] = "123456"
+
+        g.logged_in_user = {"username": "admin",
+                            "role": "admin",
+                            "realm": ""}
+        g.audit_object = audit_object
+
+        builder = EnvironBuilder(method='POST',
+                                 data={'serial': "OATH123456"},
+                                 headers={})
+
+        env = builder.get_environ()
+        # Set the remote address so that we can filter for it
+        env["REMOTE_ADDR"] = "10.0.0.1"
+        g.client_ip = env["REMOTE_ADDR"]
+        req = Request(env)
+        req.all_data = {"serial": "SomeSerial",
+                        "user": "cornelius"}
+        req.User = User("cornelius", self.realm1)
+        resp = Response()
+        resp.data = """{
+    "detail": {
+        "googleurl": {
+            "description": "URL for google Authenticator",
+            "img": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAeoAAAHqAQAAAADjFjCXAAAD+UlEQVR4nO2dTYrkSAxGn8aGWtowB6ij2Dcb5khzA/soeYABe9lgo1mE4qeKphdpN901+WlRpDP9iEwQUnySHGXOBVv/uEKDcOHChQsXLly48HtxC+sxG08DTmMdT7N5N7M5XUK6NDOz+b7Vhb8ajru7M7m7+9a5ux/A4O7L4MlgOIhP0y2ZWL70bxf+q/G9hC/AF05j2oDVenyhc7MxPjWz/ubVhb82bvP+loopZmaRa+E0Jj9++urCXxRf3w/sr0dPinUM7jbvPbGv+8mrC38pfHD3Jb08jfX9AIYjuZ4vdO4Lp/kCxNbvztWFvxYeaiKsc6btx3/yrVITwp83/2jx7nDgy3BEmHOPV6F1i8nrhD9ltQ6SAtm0dQ4ppXYOg5cI1xWfVOVE+BXLNZB9hPXdgf00Zx+NyU9z9v5wdmD650+A7gDOjH3p3y78V+E5w6YSMKTacMmmNddmwREBTxlW+AWrXhdpNvsakWYhXA+K/x1SE8KvWNYQJdYVXytFk6ph0y3uh2Kd8CsWamLLyoEP3dcqM1I4BGCS1wm/Zp81bOtrXeOEtfsPpYYirxP+lDX7uhrrlrJzSyl1gybDlmEUeZ3w56zNpnWWaQtfayRtEhfDoX2d8JtiXXhTqf7WzR0Q83X1EqRhhd+A7z0xWmJmsFsaHnb3A6ZHX6bqIuDdu7rw18LbDFtGhnNbLJSrL4N7Uq5Lqdwpwwq/iMdG7tFjM3WWuHObAaaHmf9dJ4hvXl34q+Fl5qQEsqJhS5O/zncmwbEM6v4Lv2TV66iTJuFckU2hvgdo5kT4LXh6FGfyA5vTgxIAlIrwRp5gH0JchP/9Dl9e+JfD017N1/nNYR9x9hEYDizy6oav798Mhn/NAQekYYVfstybIPdXPza+ylhA06pN05/KsMIv4uvYOexvbvZ+NAnXF04zG6Hd5qleJ/ya5Z5D6fRnlRoNio3y8ES5VB9W+DWLPtjGh+pveUai1Ivb6ROX1wm/Be/c5sGddQwNm1NqPPbvyx5n7+jECeE3qgnIw8PU0ZKt1PCaS3XEhN+FRx/iNKZHnw7WSQU6hm8p/sURO8OBzXevLvx18NL9d8/P43gefKq6glpIgTr4pFgn/Dn7pA/qmRLRbt1ywi3d13S6k+p1wp+3XDkB2rJw5zQH69SZz3LYibxO+PMW3uS5VFeH1yP1Hm01ZSplFmVY4c9bk1dLNo2QlhJpnvRsTFVi4bfi7o+3dFYdq/WkLtlMlRmhOmz+GasLf1G8qRLTOevId47pLMNQv9mXF/418O+ewd6UT+qJE/XozhhQUYYV/qx91rBTVg5VvjaVkxgjVr1O+BUz/fc64cKFCxcuXLjw/wX+HzgPbUakdjuaAAAAAElFTkSuQmCC",
+            "value": "otpauth://hotp/OATH0001D8B6?secret=GQROHTUPBAK5N6T2HBUK4IP42R56EMV3&counter=1&digits=6&issuer=privacyIDEA"
+        },
+        "rollout_state": "",
+        "serial": "OATH0001D8B6",
+        "threadid": 140437172639168
+    },
+    "id": 1,
+    "jsonrpc": "2.0",
+    "result": {
+        "status": true,
+        "value": true
+    },
+    "signature": "foo",
+    "time": 1561549651.093083,
+    "version": "privacyIDEA 3.0.1.dev2",
+    "versionnumber": "3.0.1.dev2"
+}
+"""
+        options = {"g": g,
+                   "request": req,
+                   "response": resp,
+                   "handler_def": {
+                       "conditions": {"serial": "123.*"},
+                       "options": {"body": "<img src='{googleurl_img}' />",
+                                   "mimetype": "html",
+                                   "emailconfig": "myserver"}}}
+
+        un_handler = UserNotificationEventHandler()
+        res = un_handler.do("sendmail", options=options)
+        self.assertTrue(res)
+        parsed_email = email.message_from_string(smtpmock.get_sent_message())
+        payload = to_unicode(parsed_email.get_payload(decode=True))
+        self.assertEqual(parsed_email.get_content_type(), "text/html")
+        # Check that the base64-encoded image does not get mangled
+        self.assertEqual(payload,
+                         "<img src='data:image/png;base64,"
+                         "iVBORw0KGgoAAAANSUhEUgAAAeoAAAHqAQAAAADjFjCXAAAD+UlEQVR4nO2dTYrkSAxGn8aGWtowB6ij2Dcb5khzA"
+                         "/soeYABe9lgo1mE4qeKphdpN901+WlRpDP9iEwQUnySHGXOBVv/uEKDcOHChQsXLly48HtxC"
+                         "+sxG08DTmMdT7N5N7M5XUK6NDOz+b7Vhb8ajru7M7m7+9a5ux/A4O7L4MlgOIhP0y2ZWL70bxf+q/G9hC"
+                         "/AF05j2oDVenyhc7MxPjWz/ubVhb82bvP+loopZmaRa+E0Jj9++urCXxRf3w/sr0dPinUM7jbvPbGv"
+                         "+8mrC38pfHD3Jb08jfX9AIYjuZ4vdO4Lp/kCxNbvztWFvxYeaiKsc6btx3/yrVITwp83"
+                         "/2jx7nDgy3BEmHOPV6F1i8nrhD9ltQ6SAtm0dQ4ppXYOg5cI1xWfVOVE+BXLNZB9hPXdgf00Zx+NyU9z9v5wdmD650"
+                         "+A7gDOjH3p3y78V+E5w6YSMKTacMmmNddmwREBTxlW+AWrXhdpNvsakWYhXA+K"
+                         "/x1SE8KvWNYQJdYVXytFk6ph0y3uh2Kd8CsWamLLyoEP3dcqM1I4BGCS1wm/Zp81bOtrXeOEtfsPpYYirxP"
+                         "+lDX7uhrrlrJzSyl1gybDlmEUeZ3w56zNpnWWaQtfayRtEhfDoX2d8JtiXXhTqf7WzR0Q83X1EqRhhd"
+                         "+A7z0xWmJmsFsaHnb3A6ZHX6bqIuDdu7rw18LbDFtGhnNbLJSrL4N7Uq5Lqdwpwwq"
+                         "/iMdG7tFjM3WWuHObAaaHmf9dJ4hvXl34q+Fl5qQEsqJhS5O"
+                         "/zncmwbEM6v4Lv2TV66iTJuFckU2hvgdo5kT4LXh6FGfyA5vTgxIAlIrwRp5gH0JchP/9Dl9e+JfD017N1"
+                         "/nNYR9x9hEYDizy6oav798Mhn/NAQekYYVfstybIPdXPza+ylhA06pN05/KsMIv4uvYOexvbvZ"
+                         "+NAnXF04zG6Hd5qleJ/ya5Z5D6fRnlRoNio3y8ES5VB9W+DWLPtjGh+pveUai1Ivb6ROX1wm/Be"
+                         "/c5sGddQwNm1NqPPbvyx5n7+jECeE3qgnIw8PU0ZKt1PCaS3XEhN+FRx/iNKZHnw7WSQU6hm8p"
+                         "/sURO8OBzXevLvx18NL9d8/P43gefKq6glpIgTr4pFgn/Dn7pA/qmRLRbt1ywi3d13S6k+p1wp"
+                         "+3XDkB2rJw5zQH69SZz3LYibxO+PMW3uS5VFeH1yP1Hm01ZSplFmVY4c9bk1dLNo2QlhJpnvRsTFVi4bfi7o+3dFYdq"
+                         "/WkLtlMlRmhOmz+GasLf1G8qRLTOevId47pLMNQv9mXF/418O+ewd6UT+qJE/XozhhQUYYV"
+                         "/qx91rBTVg5VvjaVkxgjVr1O+BUz/fc64cKFCxcuXLjw/wX+HzgPbUakdjuaAAAAAElFTkSuQmCC' />")
\ No newline at end of file
diff --git a/tests/test_lib_utils.py b/tests/test_lib_utils.py
--- a/tests/test_lib_utils.py
+++ b/tests/test_lib_utils.py
@@ -21,7 +21,7 @@
                                    b64encode_and_unicode, create_png, create_img,
                                    convert_timestamp_to_utc, modhex_encode,
                                    modhex_decode, checksum, urlsafe_b64encode_and_unicode,
-                                   check_ip_in_policy, split_pin_pass)
+                                   check_ip_in_policy, split_pin_pass, create_tag_dict)
 from datetime import timedelta, datetime
 from netaddr import IPAddress, IPNetwork, AddrFormatError
 from dateutil.tz import tzlocal, tzoffset, gettz
@@ -660,4 +660,24 @@ def test_30_split_pin_pass(self):
         self.assertEqual(otp, "1234")
         pin, otp = split_pin_pass("12345678hallo", 8, False)
         self.assertEqual(pin, "hallo")
-        self.assertEqual(otp, "12345678")
\ No newline at end of file
+        self.assertEqual(otp, "12345678")
+
+    def test_31_create_tag_dict(self):
+        class UserAgentMock():
+            string = "<b>hello world</b>"
+            browser = "browser"
+
+        class RequestMock():
+            user_agent = UserAgentMock()
+            path = "/validate/check"
+            url_root = ""
+
+        recipient = {"givenname": u"<b>Sömeone</b>"}
+        dict1 = create_tag_dict(request=RequestMock(), recipient=recipient)
+        self.assertEqual(dict1["ua_string"], "<b>hello world</b>")
+        self.assertEqual(dict1["action"], "/validate/check")
+        self.assertEqual(dict1["recipient_givenname"], u"<b>Sömeone</b>")
+        dict2 = create_tag_dict(request=RequestMock(), recipient=recipient, escape_html=True)
+        self.assertEqual(dict2["ua_string"], "&lt;b&gt;hello world&lt;/b&gt;")
+        self.assertEqual(dict2["action"], "/validate/check")
+        self.assertEqual(dict2["recipient_givenname"], u"&lt;b&gt;Sömeone&lt;/b&gt;")