Repository: nltk/nltk

Issue:
MWE tokenizer doesnt support pickling
You can reproduce this as follows on python 2.7.11

````
from nltk.tokenize import MWETokenizer
import pickle

tokenizer = MWETokenizer([('a', 'little'), ('a', 'little', 'bit'), ('a', 'lot')])

with open('tokenizer.pkl', 'wb') as f:
    pickle.dump(tokenizer, f)

with open('tokenizer.pkl', 'rb') as f:
    t = pickle.load(f)

````

Error that i am getting is

````
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-47-205693e12b16> in <module>()
      1 with open('tokenizer.pkl', 'rb') as f:
----> 2     t = pickle.load(f)

/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.pyc in load(file)
   1382 
   1383 def load(file):
-> 1384     return Unpickler(file).load()
   1385 
   1386 def loads(str):

/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.pyc in load(self)
    862             while 1:
    863                 key = read(1)
--> 864                 dispatch[key](self)
    865         except _Stop, stopinst:
    866             return stopinst.value

/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.pyc in load_reduce(self)
   1137         args = stack.pop()
   1138         func = stack[-1]
-> 1139         value = func(*args)
   1140         stack[-1] = value
   1141     dispatch[REDUCE] = load_reduce

/Users/vinay/Sites/nltk_mwe/.env/lib/python2.7/site-packages/nltk/collections.pyc in __init__(self, strings)
    611         defaultdict.__init__(self, Trie)
    612         if strings:
--> 613             for string in strings:
    614                 self.insert(string)
    615 

TypeError: 'type' object is not iterable
````




Discussion:
The problem is with the `Trie` data structure in `nltk.util`.

```python
from collections import defaultdict
import pickle

class Trie(defaultdict):
    def __init__(self, strings=None):
        defaultdict.__init__(self, Trie)
        assert not isinstance(strings, type)

trie = Trie([1])

print("pickle")
s = pickle.dumps(trie)
print("unpickle")
trie = pickle.loads(s)
```
```
pickle
unpickle
Traceback (most recent call last):
  File "mwe.py", line 14, in <module>
    trie = pickle.loads(s)
  File "mwe.py", line 7, in __init__
    assert not isinstance(strings, type)
AssertionError
```

Tested 2.7 and 3.5, both show same behavior.

Resolving patch:
diff --git a/nltk/collections.py b/nltk/collections.py
--- a/nltk/collections.py
+++ b/nltk/collections.py
@@ -591,12 +591,12 @@ def __radd__(self, other):
 ######################################################################
 # Trie Implementation
 ######################################################################
-class Trie(defaultdict):
+class Trie(dict):
     """A Trie implementation for strings"""
     LEAF = True
 
     def __init__(self, strings=None):
-        """Builds a Trie object, which is built around a ``defaultdict``
+        """Builds a Trie object, which is built around a ``dict``
 
         If ``strings`` is provided, it will add the ``strings``, which
         consist of a ``list`` of ``strings``, to the Trie.
@@ -607,7 +607,7 @@ def __init__(self, strings=None):
         :type strings: list(str)
 
         """
-        defaultdict.__init__(self, Trie)
+        super(Trie, self).__init__()
         if strings:
             for string in strings:
                 self.insert(string)
@@ -621,9 +621,11 @@ def insert(self, string):
         :Example:
 
         >>> from nltk.collections import Trie
-        >>> trie = Trie(["ab"])
-        >>> trie
-        defaultdict(<class 'nltk.collections.Trie'>, {'a': defaultdict(<class 'nltk.collections.Trie'>, {'b': defaultdict(<class 'nltk.collections.Trie'>, {True: None})})})
+        >>> trie = Trie(["abc", "def"])
+        >>> expected = {'a': {'b': {'c': {True: None}}}, \
+                        'd': {'e': {'f': {True: None}}}}
+        >>> trie == expected
+        True
 
         """
         if len(string):
@@ -632,56 +634,7 @@ def insert(self, string):
             # mark the string is complete
             self[Trie.LEAF] = None
 
-    def __str__(self):
-        return str(self.as_dict())
-
-    def as_dict(self, d=None):
-        """Convert ``defaultdict`` to common ``dict`` representation.
-
-        :param: A defaultdict containing strings mapped to nested defaultdicts.
-            This is the structure of the trie. (Default is None)
-        :type: defaultdict(str -> defaultdict)
-        :return: Even though ``defaultdict`` is a subclass of ``dict`` and thus
-            can be converted to a simple ``dict`` using ``dict()``, in our case
-            it's a nested ``defaultdict``, so here's a quick trick to provide to
-            us the ``dict`` representation of the ``Trie`` without
-            ``defaultdict(<class 'nltk.collections.Trie'>, ...``
-        :rtype: dict(str -> dict(bool -> None))
-            Note: there can be an arbitrarily deeply nested
-            ``dict(str -> dict(str -> dict(..))``, but the last
-            level will have ``dict(str -> dict(bool -> None))``
-
-        :Example:
-
-        >>> from nltk.collections import Trie
-        >>> trie = Trie(["abc", "def"])
-        >>> expected = {'a': {'b': {'c': {True: None}}}, 'd': {'e': {'f': {True: None}}}}
-        >>> trie.as_dict() == expected
-        True
+    def __missing__(self, key):
+        self[key] = Trie()
+        return self[key]
 
-        """
-        def _default_to_regular(d):
-            """
-            Source: http://stackoverflow.com/a/26496899/4760801
-
-            :param d: Nested ``defaultdict`` to convert to regular ``dict``
-            :type d: defaultdict(str -> defaultdict(...))
-            :return: A dict representation of the defaultdict
-            :rtype: dict(str -> dict(str -> ...))
-
-            :Example:
-
-            >>> from collections import defaultdict
-            >>> d = defaultdict(defaultdict)
-            >>> d["one"]["two"] = "three"
-            >>> d
-            defaultdict(<type 'collections.defaultdict'>, {'one': defaultdict(None, {'two': 'three'})})
-            >>> _default_to_regular(d)
-            {'one': {'two': 'three'}}
-
-            """
-            if isinstance(d, defaultdict):
-                d = {k: _default_to_regular(v) for k, v in d.items()}
-            return d
-
-        return _default_to_regular(self)
diff --git a/nltk/tokenize/mwe.py b/nltk/tokenize/mwe.py
--- a/nltk/tokenize/mwe.py
+++ b/nltk/tokenize/mwe.py
@@ -71,7 +71,7 @@ def add_mwe(self, mwe):
         >>> tokenizer.add_mwe(('a', 'b', 'c'))
         >>> tokenizer.add_mwe(('a', 'x'))
         >>> expected = {'a': {'x': {True: None}, 'b': {True: None, 'c': {True: None}}}}
-        >>> tokenizer._mwes.as_dict() == expected
+        >>> tokenizer._mwes == expected
         True
 
         """


Test patch:
diff --git a/nltk/test/collections.doctest b/nltk/test/collections.doctest
new file mode 100644
--- /dev/null
+++ b/nltk/test/collections.doctest
@@ -0,0 +1,20 @@
+.. Copyright (C) 2001-2017 NLTK Project
+.. For license information, see LICENSE.TXT
+
+===========
+Collections
+===========
+
+    >>> import nltk
+    >>> from nltk.collections import *
+
+Trie
+----
+
+Trie can be pickled:
+
+    >>> import pickle
+    >>> trie = nltk.collections.Trie(['a'])
+    >>> s = pickle.dumps(trie)
+    >>> pickle.loads(s)
+    {'a': {True: None}}
\ No newline at end of file
diff --git a/nltk/test/tokenize.doctest b/nltk/test/tokenize.doctest
--- a/nltk/test/tokenize.doctest
+++ b/nltk/test/tokenize.doctest
@@ -233,3 +233,18 @@ Post-hoc alignment of tokens with a source string
     ValueError: substring "gh" not found in "ab cd ef"
     >>> list(align_tokens(['The', 'plane', ',', 'bound', 'for', 'St', 'Petersburg', ',', 'crashed', 'in', 'Egypt', "'s", 'Sinai', 'desert', 'just', '23', 'minutes', 'after', 'take-off', 'from', 'Sharm', 'el-Sheikh', 'on', 'Saturday', '.'], "The plane, bound for St Petersburg, crashed in Egypt's Sinai desert just 23 minutes after take-off from Sharm el-Sheikh on Saturday."))
     [(0, 3), (4, 9), (9, 10), (11, 16), (17, 20), (21, 23), (24, 34), (34, 35), (36, 43), (44, 46), (47, 52), (52, 54), (55, 60), (61, 67), (68, 72), (73, 75), (76, 83), (84, 89), (90, 98), (99, 103), (104, 109), (110, 119), (120, 122), (123, 131), (131, 132)]
+
+
+Regression Tests: MWETokenizer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Pickle an MWETokenizer
+
+    >>> from nltk.tokenize import MWETokenizer
+    >>> import pickle
+
+    >>> tokenizer = MWETokenizer([('hors', "d'oeuvre")], separator='+')
+    >>> p = pickle.dumps(tokenizer)
+    >>> unpickeled = pickle.loads(p)
+    >>> unpickeled.tokenize("An hors d'oeuvre tonight, sir?".split())
+    ['An', "hors+d'oeuvre", 'tonight,', 'sir?']
+