Repository: aio-libs/multidict

Issue:
Force keys to str instances
Right now a key type is dark corner: it could be str, istr or something other.
For sake of further optimizations it should be `str` always.

Passing `istr` as key to any miltidict API is still allowed and encouraged: it saves execution time by getting rid of `s.title()` conversion.

But `istr` type is ambiguous by it nature: we can define equality for `str` and `istr` by comparing their *canonical* representations but ordering is cumbersome.
Ordering by canonical reprs leads to very tricky errors, the same for *original* reprs.
Also CPython is very optimized for `str` but not classes derived from it.

I believe the change doesn't touch our users but aiohttp might get speedup on implementing the issue. 

Comments:

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index e606232..60d2759 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -13,6 +13,8 @@
 
   If key is not present in dictionary the pair is added to the end
 
+* Force keys to `str` instances #88
+
 2.1.7 (2017-05-29)
 ------------------
 
diff --git a/benchmark.py b/benchmark.py
index 492e582..379b998 100644
--- a/benchmark.py
+++ b/benchmark.py
@@ -11,22 +11,22 @@ dct[key]
 """
 
 cython_multidict = """\
-from multidict import MultiDict
+from multidict import MultiDict, istr
 dct = MultiDict()
 """
 
 python_multidict = """\
-from multidict._multidict_py import MultiDict
+from multidict._multidict_py import MultiDict, istr
 dct = MultiDict()
 """
 
 cython_cimultidict = """\
-from multidict import CIMultiDict, upstr
+from multidict import CIMultiDict, istr
 dct = CIMultiDict()
 """
 
 python_cimultidict = """\
-from multidict._multidict_py import CIMultiDict, upstr
+from multidict._multidict_py import CIMultiDict, istr
 dct = CIMultiDict()
 """
 
@@ -37,20 +37,20 @@ for i in range(20):
 key = 'key10'
 """
 
-fill_upstr = """\
+fill_istr = """\
 for i in range(20):
-    key = upstr('key'+str(i))
+    key = istr('key'+str(i))
     dct[key] = str(i)
 
-key = upstr('key10')
+key = istr('key10')
 """
 
-upstr_from_upstr = """\
-upstr(val)
+istr_from_istr = """\
+istr(val)
 """
 
-make_upstr = """\
-val = upstr('VaLuE')
+make_istr = """\
+val = istr('VaLuE')
 """
 
 print("Cython / Python / x")
@@ -60,7 +60,14 @@ gc.collect()
 t2 = timeit.timeit(setitem, python_multidict+fill)
 gc.collect()
 
-print("MD.setitem str: {:.3f}s {:3f}s {:1f}x".format(t1, t2, t2/t1))
+print("MD.setitem str: {:.3f}s {:.3f}s {:.1f}x".format(t1, t2, t2/t1))
+
+t1 = timeit.timeit(setitem, cython_multidict+fill_istr)
+gc.collect()
+t2 = timeit.timeit(setitem, python_multidict+fill_istr)
+gc.collect()
+
+print("MD.setitem istr: {:.3f}s {:.3f}s {:.1f}x".format(t1, t2, t2/t1))
 
 
 t1 = timeit.timeit(getitem, cython_multidict+fill)
@@ -68,23 +75,23 @@ gc.collect()
 t2 = timeit.timeit(getitem, python_multidict+fill)
 gc.collect()
 
-print("MD.getitem str: {:.3f}s {:3f}s {:1f}x".format(t1, t2, t2/t1))
+print("MD.getitem str: {:.3f}s {:.3f}s {:.1f}x".format(t1, t2, t2/t1))
 
 
 t1 = timeit.timeit(getitem, cython_cimultidict+fill)
 gc.collect()
 t2 = timeit.timeit(getitem, python_cimultidict+fill)
 gc.collect()
-print("CI.getitem str: {:.3f}s {:3f}s {:1f}x".format(t1, t2, t2/t1))
+print("CI.getitem str: {:.3f}s {:.3f}s {:.1f}x".format(t1, t2, t2/t1))
 
-t1 = timeit.timeit(getitem, cython_cimultidict+fill_upstr)
+t1 = timeit.timeit(getitem, cython_cimultidict+fill_istr)
 gc.collect()
-t2 = timeit.timeit(getitem, python_cimultidict+fill_upstr)
+t2 = timeit.timeit(getitem, python_cimultidict+fill_istr)
 gc.collect()
-print("CI.getitem istr: {:.3f}s {:3f}s {:1f}x".format(t1, t2, t2/t1))
+print("CI.getitem istr: {:.3f}s {:.3f}s {:.1f}x".format(t1, t2, t2/t1))
 
-t1 = timeit.timeit(upstr_from_upstr, cython_cimultidict+make_upstr)
+t1 = timeit.timeit(istr_from_istr, cython_cimultidict+make_istr)
 gc.collect()
-t2 = timeit.timeit(upstr_from_upstr, python_cimultidict+make_upstr)
+t2 = timeit.timeit(istr_from_istr, python_cimultidict+make_istr)
 gc.collect()
-print("istr from istr: {:.3f}s {:3f}s {:1f}x".format(t1, t2, t2/t1))
+print("istr from istr: {:.3f}s {:.3f}s {:.1f}x".format(t1, t2, t2/t1))
diff --git a/multidict/_multidict.pyx b/multidict/_multidict.pyx
index 0aff712..36b6666 100644
--- a/multidict/_multidict.pyx
+++ b/multidict/_multidict.pyx
@@ -26,7 +26,9 @@ class istr(str):
             pass
         else:
             val = str(val)
-        ret = str.__new__(cls, val.title())
+        val = val.title()
+        ret = str.__new__(cls, val)
+        ret._canonical = val
         return ret
 
     def title(self):
@@ -74,19 +76,13 @@ cdef _eq(self, other):
 cdef class _Pair:
     cdef str _identity
     cdef Py_hash_t _hash
-    cdef object _key
+    cdef str _key
     cdef object _value
 
     def __cinit__(self, identity, key, value):
         self._hash = hash(identity)
-        typ = type(identity)
-        if typ is str:
-            self._identity = <str>identity
-        elif typ is _istr:
-            self._identity = <str>identity
-        else:
-            self._identity = identity
-        self._key = key
+        self._identity = <str>identity
+        self._key = <str>key
         self._value = value
 
 
@@ -99,9 +95,9 @@ cdef class _Base:
         if typ is str:
             return <str>s
         elif typ is _istr:
-            return <str>s
+            return <str>(s._canonical)
         else:
-            return s
+            return str(s)
 
     def getall(self, key, default=_marker):
         """Return a list of all values matching the key."""
@@ -261,7 +257,7 @@ cdef class CIMultiDictProxy(MultiDictProxy):
         if typ is str:
             return <str>(s.title())
         elif type(s) is _istr:
-            return <str>s
+            return <str>(s._canonical)
         return s.title()
 
     def copy(self):
@@ -272,6 +268,19 @@ cdef class CIMultiDictProxy(MultiDictProxy):
 abc.Mapping.register(CIMultiDictProxy)
 
 
+cdef str _str(key):
+    typ = type(key)
+    if typ is str:
+        return <str>key
+    if typ is _istr:
+        return <str>(key._canonical)
+    elif issubclass(typ, str):
+        return str(key)
+    else:
+        raise TypeError("MultiDict keys should be either str "
+                        "or subclasses of str")
+
+
 cdef class MultiDict(_Base):
     """An ordered dictionary that can have multiple values for each key."""
 
@@ -340,10 +349,12 @@ cdef class MultiDict(_Base):
                 self._replace(key, value)
 
     cdef _add(self, key, value):
-        self._items.append(_Pair.__new__(_Pair, self._title(key), key, value))
+        self._items.append(_Pair.__new__(
+            _Pair, self._title(key), _str(key), value))
 
     cdef _replace(self, key, value):
         cdef str identity = self._title(key)
+        cdef str k = _str(key)
         cdef Py_hash_t h = hash(identity)
         cdef Py_ssize_t i, rgt
         cdef _Pair item
@@ -355,13 +366,13 @@ cdef class MultiDict(_Base):
             if h != item._hash:
                 continue
             if item._identity == identity:
-                item._key = key
+                item._key = k
                 item._value = value
                 # i points to last found item
                 rgt = i
                 break
         else:
-            self._items.append(_Pair.__new__(_Pair, identity, key, value))
+            self._items.append(_Pair.__new__(_Pair, identity, k, value))
             return
 
         # remove all precending items
@@ -489,7 +500,7 @@ cdef class CIMultiDict(MultiDict):
         if typ is str:
             return <str>(s.title())
         elif type(s) is _istr:
-            return <str>s
+            return <str>(s._canonical)
         return s.title()
 
     def popitem(self):
diff --git a/multidict/_multidict_py.py b/multidict/_multidict_py.py
index 3428081..c8fb628 100644
--- a/multidict/_multidict_py.py
+++ b/multidict/_multidict_py.py
@@ -167,9 +167,16 @@ class MultiDict(_Base, abc.MutableMapping):
     def _title(self, key):
         return key
 
+    def _key(self, key):
+        if isinstance(key, str):
+            return str(key)
+        else:
+            raise TypeError("MultiDict keys should be either str "
+                            "or subclasses of str")
+
     def add(self, key, value):
         identity = self._title(key)
-        self._items.append((identity, key, value))
+        self._items.append((identity, self._key(key), value))
 
     def copy(self):
         """Return a copy of itself."""
@@ -272,6 +279,7 @@ class MultiDict(_Base, abc.MutableMapping):
         self._extend(args, kwargs, 'update', self._replace)
 
     def _replace(self, key, value):
+        key = self._key(key)
         identity = self._title(key)
         items = self._items
 


Test patch:
diff --git a/tests/test_multidict.py b/tests/test_multidict.py
index 3e0a345..25b13d3 100644
--- a/tests/test_multidict.py
+++ b/tests/test_multidict.py
@@ -597,6 +597,23 @@ class _BaseMutableMultiDictTests(_BaseTest):
                           ('key1', 'val'),
                           ('key2', 'val4')], list(d.items()))
 
+    def test_nonstr_key(self):
+        d = self.make_dict()
+        with self.assertRaises(TypeError):
+            d[1] = 'val'
+
+    def test_istr_key(self):
+        d = self.make_dict()
+        d[istr('1')] = 'val'
+        self.assertIs(type(list(d.keys())[0]), str)
+
+    def test_str_derived_key(self):
+        class A(str):
+            pass
+        d = self.make_dict()
+        d[A('1')] = 'val'
+        self.assertIs(type(list(d.keys())[0]), str)
+
 
 class _CIMutableMultiDictTests(_Root):