Repository: nltk/nltk

Issue:
Unigram scores incorrectly computed for Kneser-Ney smoothing causing high perplexities 
The unigram scores in KneserNey smoothing introduced in this commit https://github.com/nltk/nltk/commit/2759b03d85983fedd29b646380c713c3f4ccff66 are computed as a uniform distribution over the vocabulary, which is clearly not how any variant of KneserNey computes it. 

https://github.com/nltk/nltk/blob/5023d6b933ef1a5b1f25fba1d5ed11a8a43a47e4/nltk/lm/smoothing.py#L47


Discussion:
Hi @osmanio2, thanks for raising this!

When I implemented the interpolated smoothing I referenced the Chen & Goodman 1995 paper. If I understood the [kenlm paper](https://www.aclweb.org/anthology/P13-2121.pdf) correctly, they also represent the unigram distribution as uniform over the vocabulary.

However, I'm afraid I haven't had time to keep up with the state of the art on this front. Could you share some references that I can take a look at to get an updated understanding?
@iliakur  Thank you for replying. There are many different implementation when it comes to approximating unigram distributions:
- https://nlp.stanford.edu/~wcmac/papers/20050421-smoothing-tutorial.pdf
- http://www.foldl.me/2014/kneser-ney-smoothing/
- https://en.wikipedia.org/wiki/Kneser%E2%80%93Ney_smoothing

However, the current code **does not follow** the implementation in the [kenlm paper](https://www.aclweb.org/anthology/P13-2121.pdf) because here: https://github.com/nltk/nltk/blob/ca357e5cdcdb137f40c45346bb8bfea618dd863f/nltk/lm/models.py#L76-L78
that calls the unigram scores which is just `1.0 / len(self.vocab)`, whereas the paper interpolates between a unigram distribution and a uniform distribution.
Thanks for the links! After a quick skim I wasn't able to find anything about how they compute unigram scores when there's really no preceding context present, all of them talk about replacing raw unigram probabilities with *continuation probabilities* (the hallmark of Kneser-Ney), but none talk about the base recursive case.

However, I was able to find the place in the kenlm paper where they explicitly show that they're interpolating unigram scores with the uniform distribution over vocabulary, that convinces me that we should change the implementation.

Would you have time/interest to tackle it? No problem if not, I've accumulated a bit of a backlog of things to fix up in the code, so will likely dive into it soon.

Resolving patch:
diff --git a/nltk/lm/__init__.py b/nltk/lm/__init__.py
--- a/nltk/lm/__init__.py
+++ b/nltk/lm/__init__.py
@@ -216,6 +216,8 @@
     Laplace,
     WittenBellInterpolated,
     KneserNeyInterpolated,
+    AbsoluteDiscountingInterpolated,
+    StupidBackoff
 )
 from nltk.lm.counter import NgramCounter
 from nltk.lm.vocabulary import Vocabulary
@@ -228,4 +230,6 @@
     "Laplace",
     "WittenBellInterpolated",
     "KneserNeyInterpolated",
+    "AbsoluteDiscountingInterpolated",
+    "StupidBackoff"
 ]
diff --git a/nltk/lm/models.py b/nltk/lm/models.py
--- a/nltk/lm/models.py
+++ b/nltk/lm/models.py
@@ -2,12 +2,13 @@
 #
 # Copyright (C) 2001-2021 NLTK Project
 # Author: Ilia Kurenkov <ilia.kurenkov@gmail.com>
+#         Manu Joseph <manujosephv@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 """Language Models"""
 
 from nltk.lm.api import LanguageModel, Smoothing
-from nltk.lm.smoothing import KneserNey, WittenBell
+from nltk.lm.smoothing import KneserNey, WittenBell, AbsoluteDiscounting
 
 
 class MLE(LanguageModel):
@@ -59,6 +60,32 @@ def __init__(self, *args, **kwargs):
         super().__init__(1, *args, **kwargs)
 
 
+class StupidBackoff(LanguageModel):
+    """Provides StupidBackoff scores.
+
+    In addition to initialization arguments from BaseNgramModel also requires
+    a parameter alpha with which we scale the lower order probabilities.
+    Note that this is not a true probability distribution as scores for ngrams
+    of the same order do not sum up to unity.
+    """
+
+    def __init__(self, alpha=0.4, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.alpha = alpha
+
+    def unmasked_score(self, word, context=None):
+        if not context:
+            # Base recursion
+            return self.counts.unigrams.freq(word)
+        counts = self.context_counts(context)
+        word_count = counts[word]
+        norm_count = counts.N()
+        if word_count > 0:
+            return word_count / norm_count
+        else:
+            return self.alpha * self.unmasked_score(word, context[1:])
+
+
 class InterpolatedLanguageModel(LanguageModel):
     """Logic common to all interpolated language models.
 
@@ -67,7 +94,6 @@ class InterpolatedLanguageModel(LanguageModel):
     """
 
     def __init__(self, smoothing_cls, order, **kwargs):
-        assert issubclass(smoothing_cls, Smoothing)
         params = kwargs.pop("params", {})
         super().__init__(order, **kwargs)
         self.estimator = smoothing_cls(self.vocab, self.counts, **params)
@@ -80,8 +106,9 @@ def unmasked_score(self, word, context=None):
             # It can also happen that we have no data for this context.
             # In that case we defer to the lower-order ngram.
             # This is the same as setting alpha to 0 and gamma to 1.
-            return self.unmasked_score(word, context[1:])
-        alpha, gamma = self.estimator.alpha_gamma(word, context)
+            alpha, gamma = 0, 1
+        else:
+            alpha, gamma = self.estimator.alpha_gamma(word, context)
         return alpha + gamma * self.unmasked_score(word, context[1:])
 
 
@@ -92,8 +119,23 @@ def __init__(self, order, **kwargs):
         super().__init__(WittenBell, order, **kwargs)
 
 
+class AbsoluteDiscountingInterpolated(InterpolatedLanguageModel):
+    """Interpolated version of smoothing with absolute discount."""
+
+    def __init__(self, order, discount=0.75, **kwargs):
+        super().__init__(
+            AbsoluteDiscounting, order, params={"discount": discount}, **kwargs
+        )
+
+
 class KneserNeyInterpolated(InterpolatedLanguageModel):
     """Interpolated version of Kneser-Ney smoothing."""
 
     def __init__(self, order, discount=0.1, **kwargs):
-        super().__init__(KneserNey, order, params={"discount": discount}, **kwargs)
+        if not (0 <= discount <= 1):
+            raise ValueError(
+                "Discount must be between 0 and 1 for probabilities to sum to unity."
+            )
+        super().__init__(
+            KneserNey, order, params={"discount": discount, "order": order}, **kwargs
+        )
diff --git a/nltk/lm/smoothing.py b/nltk/lm/smoothing.py
--- a/nltk/lm/smoothing.py
+++ b/nltk/lm/smoothing.py
@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2001-2021 NLTK Project
 # Author: Ilia Kurenkov <ilia.kurenkov@gmail.com>
+#         Manu Joseph <manujosephv@gmail.com>
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 """Smoothing algorithms for language modeling.
@@ -9,12 +10,27 @@
 According to Chen & Goodman 1995 these should work with both Backoff and
 Interpolation.
 """
+from operator import methodcaller
 
 from nltk.lm.api import Smoothing
+from nltk import ConditionalFreqDist
 
 
-def _count_non_zero_vals(dictionary):
-    return sum(1.0 for c in dictionary.values() if c > 0)
+def _count_values_gt_zero(distribution):
+    """Count values that are greater than zero in a distribution.
+
+    Assumes distribution is either a mapping with counts as values or
+    an instance of `nltk.ConditionalFreqDist`.
+    """
+    as_count = (
+        methodcaller("N")
+        if isinstance(distribution, ConditionalFreqDist)
+        else lambda count: count
+    )
+    # We explicitly check that values are > 0 to guard against negative counts.
+    return sum(
+        1 for dist_or_count in distribution.values() if as_count(dist_or_count) > 0
+    )
 
 
 class WittenBell(Smoothing):
@@ -29,28 +45,83 @@ def alpha_gamma(self, word, context):
         return (1.0 - gamma) * alpha, gamma
 
     def _gamma(self, context):
-        n_plus = _count_non_zero_vals(self.counts[context])
+        n_plus = _count_values_gt_zero(self.counts[context])
         return n_plus / (n_plus + self.counts[context].N())
 
     def unigram_score(self, word):
         return self.counts.unigrams.freq(word)
 
 
+class AbsoluteDiscounting(Smoothing):
+    """Smoothing with absolute discount."""
+
+    def __init__(self, vocabulary, counter, discount=0.75, **kwargs):
+        super().__init__(vocabulary, counter, **kwargs)
+        self.discount = discount
+
+    def alpha_gamma(self, word, context):
+        alpha = (
+            max(self.counts[context][word] - self.discount, 0)
+            / self.counts[context].N()
+        )
+        gamma = self._gamma(context)
+        return alpha, gamma
+
+    def _gamma(self, context):
+        n_plus = _count_values_gt_zero(self.counts[context])
+        return (self.discount * n_plus) / self.counts[context].N()
+
+    def unigram_score(self, word):
+        return self.counts.unigrams.freq(word)
+
+
 class KneserNey(Smoothing):
-    """Kneser-Ney Smoothing."""
+    """Kneser-Ney Smoothing.
 
-    def __init__(self, vocabulary, counter, discount=0.1, **kwargs):
+    This is an extension of smoothing with a discount.
+
+    Resources:
+    - https://pages.ucsd.edu/~rlevy/lign256/winter2008/kneser_ney_mini_example.pdf
+    - https://www.youtube.com/watch?v=ody1ysUTD7o
+    - https://medium.com/@dennyc/a-simple-numerical-example-for-kneser-ney-smoothing-nlp-4600addf38b8
+    - https://www.cl.uni-heidelberg.de/courses/ss15/smt/scribe6.pdf
+    - https://www-i6.informatik.rwth-aachen.de/publications/download/951/Kneser-ICASSP-1995.pdf
+    """
+
+    def __init__(self, vocabulary, counter, order, discount=0.1, **kwargs):
         super().__init__(vocabulary, counter, **kwargs)
         self.discount = discount
+        self._order = order
 
     def unigram_score(self, word):
-        return 1.0 / len(self.vocab)
+        word_continuation_count, total_count = self._continuation_counts(word)
+        return word_continuation_count / total_count
 
     def alpha_gamma(self, word, context):
         prefix_counts = self.counts[context]
-        prefix_total_ngrams = prefix_counts.N()
-        alpha = max(prefix_counts[word] - self.discount, 0.0) / prefix_total_ngrams
-        gamma = (
-            self.discount * _count_non_zero_vals(prefix_counts) / prefix_total_ngrams
+        word_continuation_count, total_count = (
+            (prefix_counts[word], prefix_counts.N())
+            if len(context) + 1 == self._order
+            else self._continuation_counts(word, context)
         )
+        alpha = max(word_continuation_count - self.discount, 0.0) / total_count
+        gamma = self.discount * _count_values_gt_zero(prefix_counts) / total_count
         return alpha, gamma
+
+    def _continuation_counts(self, word, context=tuple()):
+        """Count continuations that end with context and word.
+
+        Continuations track unique ngram "types", regardless of how many
+        instances were observed for each "type".
+        This is different than raw ngram counts which track number of instances.
+        """
+        higher_order_ngrams_with_context = (
+            counts
+            for prefix_ngram, counts in self.counts[len(context) + 2].items()
+            if prefix_ngram[1:] == context
+        )
+        higher_order_ngrams_with_word_count, total = 0, 0
+        for counts in higher_order_ngrams_with_context:
+            higher_order_ngrams_with_word_count += int(counts[word] > 0)
+            total += _count_values_gt_zero(counts)
+        return higher_order_ngrams_with_word_count, total


Test patch:
diff --git a/nltk/test/unit/lm/test_models.py b/nltk/test/unit/lm/test_models.py
--- a/nltk/test/unit/lm/test_models.py
+++ b/nltk/test/unit/lm/test_models.py
@@ -15,11 +15,12 @@
     Laplace,
     WittenBellInterpolated,
     KneserNeyInterpolated,
+    AbsoluteDiscountingInterpolated,
+    StupidBackoff,
 )
 from nltk.lm.preprocessing import padded_everygrams
 
 
-
 def _prepare_test_data(ngram_order):
     return (
         Vocabulary(["a", "b", "c", "d", "z", "
", "
"], unk_cutoff=1),
@@ -32,6 +33,7 @@ def _prepare_test_data(ngram_order):
 
 class ParametrizedTests(type):
     """Metaclass for generating parametrized tests."""
+
     contexts = [
         ("a",),
         ("c",),
@@ -330,51 +332,137 @@ def setup_method(self):
         # count("<UNK>") = 3
         ("y", None, 3.0 / 18),
         # 2 words follow b and b occured a total of 2 times
-        # gamma(['b']) = 2/(2+2) = 0.5
+        # gamma(['b']) = 2 / (2 + 2) = 0.5
         # mle.score('c', ['b']) = 0.5
-        # mle('c') = 1/18 = 0.055
+        # mle('c') = 1 / 18 = 0.055
         # (1 - gamma) * mle + gamma * mle('c') ~= 0.27 + 0.055
         ("c", ["b"], (1 - 0.5) * 0.5 + 0.5 * 1 / 18),
         # building on that, let's try 'a b c' as the trigram
         # 1 word follows 'a b' and 'a b' occured 1 time
-        # gamma(['a', 'b']) = 1/(1+1) = 0.5
+        # gamma(['a', 'b']) = 1 / (1 + 1) = 0.5
         # mle("c", ["a", "b"]) = 1
         ("c", ["a", "b"], (1 - 0.5) + 0.5 * ((1 - 0.5) * 0.5 + 0.5 * 1 / 18)),
-        # The ngram 'z b c' was not seen, so we should simply revert to
-        # the score of the ngram 'b c'. See issue #2332.
+        # P(c|zb)
+        # The ngram 'zbc' was not seen, so we use P(c|b). See issue #2332.
         ("c", ["z", "b"], ((1 - 0.5) * 0.5 + 0.5 * 1 / 18)),
     ]
 
+###############################################################################
+#                              Notation Explained                             #
+###############################################################################
+# For all subsequent calculations we use the following notation:
+# 1. '*': Placeholder for any word/character. E.g. '*b' stands for
+#    all bigrams that end in 'b'. '*b*' stands for all trigrams that
+#    contain 'b' in the middle.
+# 1. count(ngram): Count all instances (tokens) of an ngram.
+# 1. unique(ngram): Count unique instances (types) of an ngram.
+
 
 class TestKneserNeyInterpolatedTrigram(metaclass=ParametrizedTests):
     @classmethod
     def setup_method(self):
         vocab, training_text = _prepare_test_data(3)
-        self.model = KneserNeyInterpolated(3, vocabulary=vocab)
+        self.model = KneserNeyInterpolated(3, discount=0.75, vocabulary=vocab)
+        self.model.fit(training_text)
+
+    score_tests = [
+        # P(c) = count('*c') / unique('**')
+        #      = 1 / 14
+        ("c", None, 1.0 / 14),
+        # P(z) = count('*z') / unique('**')
+        #      = 0 / 14
+        # 'z' is in the vocabulary, but it was not seen during training.
+        ("z", None, 0.0 / 14),
+        # P(y)
+        # Out of vocabulary should use "UNK" score.
+        # P(y) = P(UNK) = count('*UNK') / unique('**')
+        ("y", None, 3 / 14),
+        # We start with P(c|b)
+        # P(c|b) = alpha('bc') + gamma('b') * P(c)
+        # alpha('bc') = max(unique('*bc') - discount, 0) / unique('*b*')
+        #             = max(1 - 0.75, 0) / 2
+        #             = 0.125
+        # gamma('b')  = discount * unique('b*') / unique('*b*')
+        #             = (0.75 * 2) / 2
+        #             = 0.75
+        ("c", ["b"], (0.125 + 0.75 * (1 / 14))),
+        # Building on that, let's try P(c|ab).
+        # P(c|ab) = alpha('abc') + gamma('ab') * P(c|b)
+        # alpha('abc') = max(count('abc') - discount, 0) / count('ab*')
+        #              = max(1 - 0.75, 0) / 1
+        #              = 0.25
+        # gamma('ab')  = (discount * unique('ab*')) / count('ab*')
+        #              = 0.75 * 1 / 1
+        ("c", ["a", "b"], 0.25 + 0.75 * (0.125 + 0.75 * (1 / 14))),
+        # P(c|zb)
+        # The ngram 'zbc' was not seen, so we use P(c|b). See issue #2332.
+        ("c", ["z", "b"], (0.125 + 0.75 * (1 / 14))),
+    ]
+
+
+class TestAbsoluteDiscountingTrigram(metaclass=ParametrizedTests):
+    @classmethod
+    def setup_method(self):
+        vocab, training_text = _prepare_test_data(3)
+        self.model = AbsoluteDiscountingInterpolated(3, discount=0.75, vocabulary=vocab)
         self.model.fit(training_text)
 
     score_tests = [
         # For unigram scores revert to uniform
-        # Vocab size: 8
-        # count('c'): 1
-        ("c", None, 1.0 / 8),
-        # in vocabulary but unseen, still uses uniform
-        ("z", None, 1 / 8),
-        # out of vocabulary should use "UNK" score, i.e. again uniform
-        ("y", None, 1.0 / 8),
-        # alpha = count('bc') - discount = 1 - 0.1 = 0.9
-        # gamma(['b']) = discount * number of unique words that follow ['b'] = 0.1 * 2
-        # normalizer = total number of bigrams with this context = 2
-        # the final should be: (alpha + gamma * unigram_score("c"))
-        ("c", ["b"], (0.9 + 0.2 * (1 / 8)) / 2),
-        # building on that, let's try 'a b c' as the trigram
-        # alpha = count('abc') - discount = 1 - 0.1 = 0.9
-        # gamma(['a', 'b']) = 0.1 * 1
-        # normalizer = total number of trigrams with prefix "ab" = 1 => we can ignore it!
-        ("c", ["a", "b"], 0.9 + 0.1 * ((0.9 + 0.2 * (1 / 8)) / 2)),
-        # The ngram 'z b c' was not seen, so we should simply revert to
-        # the score of the ngram 'b c'. See issue #2332.
-        ("c", ["z", "b"], ((0.9 + 0.2 * (1 / 8)) / 2)),
+        # P(c) = count('c') / count('**')
+        ("c", None, 1.0 / 18),
+        # in vocabulary but unseen
+        # count('z') = 0
+        ("z", None, 0.0 / 18),
+        # out of vocabulary should use "UNK" score
+        # count('<UNK>') = 3
+        ("y", None, 3 / 18),
+        # P(c|b) = alpha('bc') + gamma('b') * P(c)
+        # alpha('bc') = max(count('bc') - discount, 0) / count('b*')
+        #             = max(1 - 0.75, 0) / 2
+        #             = 0.125
+        # gamma('b')  = discount * unique('b*') / count('b*')
+        #             = (0.75 * 2) / 2
+        #             = 0.75
+        ("c", ["b"], (0.125 + 0.75 * (2 / 2) * (1 / 18))),
+        # Building on that, let's try P(c|ab).
+        # P(c|ab) = alpha('abc') + gamma('ab') * P(c|b)
+        # alpha('abc') = max(count('abc') - discount, 0) / count('ab*')
+        #              = max(1 - 0.75, 0) / 1
+        #              = 0.25
+        # gamma('ab')  = (discount * unique('ab*')) / count('ab*')
+        #              = 0.75 * 1 / 1
+        ("c", ["a", "b"], 0.25 + 0.75 * (0.125 + 0.75 * (2 / 2) * (1 / 18))),
+        # P(c|zb)
+        # The ngram 'zbc' was not seen, so we use P(c|b). See issue #2332.
+        ("c", ["z", "b"], (0.125 + 0.75 * (2 / 2) * (1 / 18))),
+    ]
+
+
+class TestStupidBackoffTrigram(metaclass=ParametrizedTests):
+    @classmethod
+    def setup_method(self):
+        vocab, training_text = _prepare_test_data(3)
+        self.model = StupidBackoff(order=3, alpha=0.4, vocabulary=vocab)
+        self.model.fit(training_text)
+
+    score_tests = [
+        # For unigram scores revert to uniform
+        # total bigrams = 18
+        ("c", None, 1.0 / 18),
+        # in vocabulary but unseen
+        # bigrams ending with z = 0
+        ("z", None, 0.0 / 18),
+        # out of vocabulary should use "UNK" score
+        # count('<UNK>'): 3
+        ("y", None, 3 / 18),
+        # c follows 1 time out of 2 after b
+        ("c", ["b"], 1 / 2),
+        # c always follows ab
+        ("c", ["a", "b"], 1 / 1),
+        # The ngram 'z b c' was not seen, so we backoff to
+        # the score of the ngram 'b c' * smoothing factor
+        ("c", ["z", "b"], (0.4 * (1 / 2))),
     ]
 
 
@@ -406,22 +494,29 @@ def test_generate_cycle(self):
 
         self.model.fit(more_training_text)
         # Test that we can escape the cycle
-        assert (
-            self.model.generate(7, text_seed=("b", "d"), random_seed=5)
-            == ["b", "d", "b", "d", "b", "d", "
"]
-        )
+        assert self.model.generate(7, text_seed=("b", "d"), random_seed=5) == [
+            "b",
+            "d",
+            "b",
+            "d",
+            "b",
+            "d",
+            "
",
+        ]
 
     def test_generate_with_text_seed(self):
-        assert (
-            self.model.generate(5, text_seed=("
", "e"), random_seed=3)
-            == ["<UNK>", "a", "d", "b", "<UNK>"]
-        )
+        assert self.model.generate(5, text_seed=("
", "e"), random_seed=3) == [
+            "<UNK>",
+            "a",
+            "d",
+            "b",
+            "<UNK>",
+        ]
 
     def test_generate_oov_text_seed(self):
-        assert (
-            self.model.generate(text_seed=("aliens",), random_seed=3)
-            == self.model.generate(text_seed=("<UNK>",), random_seed=3)
-        )
+        assert self.model.generate(
+            text_seed=("aliens",), random_seed=3
+        ) == self.model.generate(text_seed=("<UNK>",), random_seed=3)
 
     def test_generate_None_text_seed(self):
         # should crash with type error when we try to look it up in vocabulary
@@ -429,7 +524,6 @@ def test_generate_None_text_seed(self):
             self.model.generate(text_seed=(None,))
 
         # This will work
-        assert (
-            self.model.generate(text_seed=None, random_seed=3)
-            == self.model.generate(random_seed=3)
-        )
+        assert self.model.generate(
+            text_seed=None, random_seed=3
+        ) == self.model.generate(random_seed=3)