Repository: nltk/nltk

Issue:
can't pickle NgramModels because we can't pickle ConditionalFreqDists because Python can't pickle lambdas ...
Right now, we can't pickle an NgramModel, like so:

``` python
lm = nltk.model.ngram.NgramModel(3, ['a', 'b', 'c'])
pickle.dumps(lm)
```

... but I know why this is! We can't pickle any ConditionalFreqDist, and the NgramModel has one of those as a member.

``` python
from nltk.probability import ConditionalProbDist
from nltk.probability import ConditionalFreqDist
from nltk.probability import SimpleGoodTuringProbDist
cfd = ConditionalFreqDist()
cfd['hello'].inc('everybody')
cfd['hello'].inc('everybody')
model = ConditionalProbDist(cfd, SimpleGoodTuringProbDist)
pickle.dumps(model)
```

And this can't be done because ConditionalProbDist carries a lambda around.
https://github.com/nltk/nltk/blob/master/nltk/probability.py#L2091

Any thoughts on the best way to fix this, so we can pickle NgramModels again? I'll take a stab at it...



Discussion:

Resolving patch:
diff --git a/nltk/probability.py b/nltk/probability.py
--- a/nltk/probability.py
+++ b/nltk/probability.py
@@ -1990,7 +1990,7 @@ def __repr__(self):
 
 
 @compat.python_2_unicode_compatible
-class ConditionalProbDistI(defaultdict):
+class ConditionalProbDistI(dict):
     """
     A collection of probability distributions for a single experiment
     run under different conditions.  Conditional probability
@@ -2083,18 +2083,19 @@ def __init__(self, cfdist, probdist_factory,
         :type factory_kw_args: (any)
         :param factory_kw_args: Extra keyword arguments for ``probdist_factory``.
         """
-        # self._probdist_factory = probdist_factory
-        # self._cfdist = cfdist
-        # self._factory_args = factory_args
-        # self._factory_kw_args = factory_kw_args
+        self._probdist_factory = probdist_factory
+        self._factory_args = factory_args
+        self._factory_kw_args = factory_kw_args
 
-        factory = lambda: probdist_factory(FreqDist(),
-                                           *factory_args, **factory_kw_args)
-        defaultdict.__init__(self, factory)
         for condition in cfdist:
             self[condition] = probdist_factory(cfdist[condition],
                                                *factory_args, **factory_kw_args)
 
+    def __missing__(self, key):
+        self[key] = self._probdist_factory(FreqDist(),
+                                           *self._factory_args,
+                                           **self._factory_kw_args)
+        return self[key]
 
 class DictionaryConditionalProbDist(ConditionalProbDistI):
     """
@@ -2108,11 +2109,11 @@ def __init__(self, probdist_dict):
             by the conditions
         :type probdist_dict: dict any -> probdist
         """
-        defaultdict.__init__(self, DictionaryProbDist)
         self.update(probdist_dict)
 
-    def __reduce__(self):
-        return self.__class__, (self.items(),),
+    def __missing__(self, key):
+        self[key] = DictionaryProbDist()
+        return self[key]
 
 ##//////////////////////////////////////////////////////
 ## Adding in log-space.


Test patch:
diff --git a/nltk/test/probability.doctest b/nltk/test/probability.doctest
--- a/nltk/test/probability.doctest
+++ b/nltk/test/probability.doctest
@@ -238,8 +238,8 @@ they don't get smoothed
     >>> p.prob('foobar')
     0.022727272727272728...
 
-``MLEProbDist``, ``DictionaryConditionalProbDist`` and ``ConditionalFreqDist``
-can be pickled:
+``MLEProbDist``, ``ConditionalProbDist'', ``DictionaryConditionalProbDist`` and
+``ConditionalFreqDist`` can be pickled:
 
     >>> import pickle
     >>> pd = MLEProbDist(fd)
@@ -258,5 +258,9 @@ can be pickled:
     >>> cfd2 = pickle.loads(pickle.dumps(cfd))
     >>> cfd2 == cfd
     True
+    >>> cpd = ConditionalProbDist(cfd, SimpleGoodTuringProbDist)
+    >>> cpd2 = pickle.loads(pickle.dumps(cpd))
+    >>> cpd['foo'].prob('hello') == cpd2['foo'].prob('hello')
+    True