Repository: web2py/web2py

Issue:
IS_EMAIL does not handle Internationalized Domain Names
[Internationalized domain names](https://en.wikipedia.org/wiki/Internationalized_domain_name) allows for unicode characters in domain names.

As thus IS_EMAIL default regex should be updated to allow for such domain, else it would for example prevent people with mail addresses on IDN to subscribe.


Discussion:
[RFC 2181 §11](https://tools.ietf.org/html/rfc2181#section-11) explicit that any binary string is allowed for domain name.
[RFC 5336 §3.3](https://tools.ietf.org/html/rfc5336#section-3.3) Precise the extended mail address syntax to support for IDN.
So, a UTF8 string is permitted. That string MUST NOT contain any of the       ASCII characters (either graphics or controls) that are not permitted in "atext"; it is otherwise unrestricted.

Where:
```
atext           =       ALPHA / DIGIT / ; Any character except controls,
                        "!" / "#" /     ;  SP, and specials.
                        "$" / "%" /     ;  Used for atoms
                        "&" / "'" /
                        "*" / "+" /
                        "-" / "/" /
                        "=" / "?" /
                        "^" / "_" /
                        "`" / "{" /
                        "|" / "}" /
                        "~"
```

(See RFC 5322)
IS_EMAIL is based on a complex REGEX, see https://github.com/web2py/web2py/blob/b2e03c9dee935d46cb9512f0595dc2dacddb5166/gluon/validators.py#L1159
Related to this https://docs.python.org/2/library/codecs.html#module-encodings.idna

Resolving patch:
diff --git a/gluon/validators.py b/gluon/validators.py
--- a/gluon/validators.py
+++ b/gluon/validators.py
@@ -21,7 +21,7 @@
 import struct
 import decimal
 import unicodedata
-from gluon._compat import StringIO, long, unicodeT, to_unicode, urllib_unquote, unichr, to_bytes, PY2, to_unicode, to_native
+from gluon._compat import StringIO, long, basestring, unicodeT, to_unicode, urllib_unquote, unichr, to_bytes, PY2, to_unicode, to_native
 from gluon.utils import simple_hash, web2py_uuid, DIGEST_ALG_BY_SIZE
 from pydal.objects import Field, FieldVirtual, FieldMethod
 from functools import reduce
@@ -1156,15 +1156,16 @@ class IS_EMAIL(Validator):
 
     """
 
-    regex = re.compile('''
+    body_regex = re.compile('''
         ^(?!\.)                            # name may not begin with a dot
         (
           [-a-z0-9!\#$%&'*+/=?^_`{|}~]     # all legal characters except dot
           |
           (?<!\.)\.                        # single dots only
         )+
-        (?<!\.)                            # name may not end with a dot
-        @
+        (?<!\.)$                            # name may not end with a dot
+    ''', re.VERBOSE | re.IGNORECASE)
+    domain_regex = re.compile('''
         (
           localhost
           |
@@ -1183,6 +1184,7 @@ class IS_EMAIL(Validator):
 
     regex_proposed_but_failed = re.compile('^([\w\!\#$\%\&\'\*\+\-\/\=\?\^\`{\|\}\~]+\.)*[\w\!\#$\%\&\'\*\+\-\/\=\?\^\`{\|\}\~]+@((((([a-z0-9]{1}[a-z0-9\-]{0,62}[a-z0-9]{1})|[a-z])\.)+[a-z]{2,6})|(\d{1,3}\.){3}\d{1,3}(\:\d{1,5})?)$', re.VERBOSE | re.IGNORECASE)
 
+
     def __init__(self,
                  banned=None,
                  forced=None,
@@ -1196,14 +1198,27 @@ def __init__(self,
         self.error_message = error_message
 
     def __call__(self, value):
+        if not(isinstance(value, (basestring, unicodeT))) or not value or '@' not in value:
+          return (value, translate(self.error_message))
+
+        body, domain = value.rsplit('@', 1)
+
         try:
-            match = self.regex.match(value)
-        except TypeError:
+            match_body = self.body_regex.match(body)
+            match_domain = self.domain_regex.match(domain)
+
+            if not match_domain:
+                # check for Internationalized Domain Names
+                # see https://docs.python.org/2/library/codecs.html#module-encodings.idna
+                domain_encoded = to_unicode(domain).encode('idna').decode('ascii')
+                match_domain = self.domain_regex.match(domain_encoded)
+
+            match = (match_body != None) and (match_domain != None)
+        except (TypeError, UnicodeError):
             # Value may not be a string where we can look for matches.
             # Example: we're calling ANY_OF formatter and IS_EMAIL is asked to validate a date.
             match = None
         if match:
-            domain = value.split('@')[1]
             if (not self.banned or not self.banned.match(domain)) \
                     and (not self.forced or self.forced.match(domain)):
                 return (value, None)


Test patch:
diff --git a/gluon/tests/test_validators.py b/gluon/tests/test_validators.py
--- a/gluon/tests/test_validators.py
+++ b/gluon/tests/test_validators.py
@@ -531,6 +531,11 @@ def test_IS_EMAIL(self):
         rtn = IS_EMAIL(error_message='oops')(42)
         self.assertEqual(rtn, (42, 'oops'))
 
+        # test for Internationalized Domain Names, see https://docs.python.org/2/library/codecs.html#module-encodings.idna
+        rtn = IS_EMAIL()('web2py@Alliancefrançaise.nu')
+        self.assertEqual(rtn, ('web2py@Alliancefrançaise.nu', None))
+
+
     def test_IS_LIST_OF_EMAILS(self):
         emails = ['localguy@localhost', '_Yosemite.Sam@example.com']
         rtn = IS_LIST_OF_EMAILS()(','.join(emails))