Repository: nltk/nltk

Issue:
corpus ribes scorer throws division by zero error
Division by zero error is constantly thrown for ````corpus_ribes()```` function. 

To replicate: 
Take the example from the docs (https://www.nltk.org/api/nltk.translate.html) and shorten some
of the input sentences. In this case I only shortened hyp2 and hyp2a. 
````
from nltk.translate.ribes_score import corpus_ribes

hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', 'ensures', 'that', 'the', 'military', 'always', 'obeys', 'the', 'commands', 'of', 'the', 'party']
ref1a = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',  'ensures', 'that', 'the', 'military', 'will', 'forever',  'heed', 'Party', 'commands']
ref1b = ['It', 'is', 'the', 'guiding', 'principle', 'which', 'guarantees', 'the', 'military', 'forces', 'always', 'being', 'under', 'the', 'command', 'of', 'the', 'Party']
ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',  'army', 'always', 'to', 'heed', 'the', 'directions',  'of', 'the', 'party']

hyp2 = ['he', 'read', 'the']
ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', 'because', 'he']

list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]
hypotheses = [hyp1, hyp2]
corpus_ribes(list_of_references, hypotheses)
````
Output: 
````
--------------------------------------------------------------------------
ZeroDivisionError                         Traceback (most recent call last)
<ipython-input-175-b02599af69e7> in <module>
      1 list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]
      2 hypotheses = [hyp1, hyp2]
----> 3 corpus_ribes(list_of_references, hypotheses)

/anaconda3/envs/torch/lib/python3.7/site-packages/nltk/translate/ribes_score.py in corpus_ribes(list_of_references, hypotheses, alpha, beta)
    117     # Iterate through each hypothesis and their corresponding references.
    118     for references, hypothesis in zip(list_of_references, hypotheses):
--> 119         corpus_best_ribes += sentence_ribes(references, hypothesis, alpha, beta)
    120     return corpus_best_ribes / len(hypotheses)
    121 

/anaconda3/envs/torch/lib/python3.7/site-packages/nltk/translate/ribes_score.py in sentence_ribes(references, hypothesis, alpha, beta)
     53         # Collects the *worder* from the ranked correlation alignments.
     54         worder = word_rank_alignment(reference, hypothesis)
---> 55         nkt = kendall_tau(worder)
     56 
     57         # Calculates the brevity penalty

/anaconda3/envs/torch/lib/python3.7/site-packages/nltk/translate/ribes_score.py in kendall_tau(worder, normalize)
    288     num_possible_pairs = choose(worder_len, 2)
    289     # Kendall's Tau computation.
--> 290     tau = 2 * num_increasing_pairs / num_possible_pairs - 1
    291     if normalize:  # If normalized, the tau output falls between 0.0 to 1.0
    292         return (tau + 1) / 2

ZeroDivisionError: division by zero
````
corpus_ribes divide 0 error
In ribes_score.py line 290. I encounter the divide 0 error. We should prevent num_possible_pairs from being 0.


Discussion:
Hi @tibnb545, thank you for your report. 

I believe this is a duplicate of #2433.
Initially it was giving "division by zero error" for both sentence_ribes and corpus_ribes.

I made some changes in the code of nltk implementation of ribes_score it works fine when I use sentence_ribes,but outputs same error "division by zero error" when corpus_ribes is used.

Can someone please point out my mistake.
```python


from itertools import islice
import math

from nltk.util import ngrams, choose


def sentence_ribes(references, hypothesis, alpha=0.25, beta=0.10):
 
    best_ribes = -1.0
    # Calculates RIBES for each reference and returns the best score.
    
        # Collects the *worder* from the ranked correlation alignments.
    
    worder = word_rank_alignment(str(reference), str(hypothesis))       #**Change**
    nkt = kendall_tau(worder)

        # Calculates the brevity penalty
    bp = min(1.0, math.exp(1.0 - len(reference) / len(hypothesis)))

        # Calculates the unigram precision, *p1*
    p1 = len(worder) / len(hypothesis)

    _ribes = nkt * (p1 ** alpha) * (bp ** beta)

    if _ribes > best_ribes:
        best_ribes = _ribes

    return best_ribes


def corpus_ribes(list_of_references, hypotheses, alpha=0.25, beta=0.10):    #**Change**
   
    c=0
    for i in range(len(list_of_references)):
        for j in range(len(list_of_references[i])):
            print(type(list_of_references[i][j].split()))
            c+=sentence_ribes(list_of_references[i][j], hypotheses[i])
    return c/len(hypotheses)

def position_of_ngram(ngram, sentence):
    
    # Iterates through the ngrams in sentence.
    for i, sublist in enumerate(ngrams(sentence, len(ngram))):
        # Returns the index of the word when ngram matches.
        if ngram == sublist:
            return i


def word_rank_alignment(reference, hypothesis, character_based=False):
    
    worder=[]
    hypothesis=hypothesis.split()       #**Change**
    reference=reference.split()                      
    hyp_len = len(hypothesis)
    # Stores a list of possible ngrams from the reference sentence.
    # This is used for matching context window later in the algorithm.
    ref_ngrams = []
    hyp_ngrams = []
    for n in range(1, len(reference) + 1):
        for ng in ngrams(reference, n):
            ref_ngrams.append(ng)
        for ng in ngrams(hypothesis, n):
            hyp_ngrams.append(ng)
    for i, h_word in enumerate(hypothesis):
        
        # If word is not in the reference, continue.
        if h_word not in reference:
            continue
        # If we can determine one-to-one word correspondence for unigrams that
        # only appear once in both the reference and hypothesis.
        elif hypothesis.count(h_word) == reference.count(h_word) == 1:
            worder.append(reference.index(h_word))
         
        else:
            max_window_size = max(i, hyp_len - i + 1)
            for window in range(1, max_window_size):
                if i + window < hyp_len:  # If searching the right context is possible.
                    # Retrieve the right context window.
                    right_context_ngram = tuple(islice(hypothesis, i, i + window + 1))
                    num_times_in_ref = ref_ngrams.count(right_context_ngram)
                    num_times_in_hyp = hyp_ngrams.count(right_context_ngram)
                    # If ngram appears only once in both ref and hyp.
                    if num_times_in_ref == num_times_in_hyp == 1:
                        # Find the position of ngram that matched the reference.
                        pos = position_of_ngram(right_context_ngram, reference)
                        worder.append(pos)  # Add the positions of the ngram.
                        break
                if window <= i:  # If searching the left context is possible.
                    # Retrieve the left context window.
                    left_context_ngram = tuple(islice(hypothesis, i - window, i + 1))
                    num_times_in_ref = ref_ngrams.count(left_context_ngram)
                    num_times_in_hyp = hyp_ngrams.count(left_context_ngram)
                    if num_times_in_ref == num_times_in_hyp == 1:
                        # Find the position of ngram that matched the reference.
                        pos = position_of_ngram(left_context_ngram, reference)
                        # Add the positions of the ngram.
                        worder.append(pos + len(left_context_ngram) - 1)
                        break
    return worder


def find_increasing_sequences(worder):
    
    items = iter(worder)
    a, b = None, next(items, None)
    result = [b]
    while b is not None:
        a, b = b, next(items, None)
        if b is not None and a + 1 == b:
            result.append(b)
        else:
            if len(result) > 1:
                yield tuple(result)
            result = [b]


def kendall_tau(worder, normalize=True):
    
    worder_len = len(worder)
    # Extract the groups of increasing/monotonic sequences.
    increasing_sequences = find_increasing_sequences(worder)
    # Calculate no. of increasing_pairs in *worder* list.
    num_increasing_pairs = sum(choose(len(seq), 2) for seq in increasing_sequences)
    # Calculate no. of possible pairs.
    num_possible_pairs = choose(worder_len, 2)
    # Kendall's Tau computation.
    tau = 2 * num_increasing_pairs / (num_possible_pairs) - 1
    if normalize:  # If normalized, the tau output falls between 0.0 to 1.0
        return (tau + 1) / 2
    else:  # Otherwise, the tau outputs falls between -1.0 to +1.0
        return tau


def spearman_rho(worder, normalize=True):
   
    worder_len = len(worder)
    sum_d_square = sum((wi - i) ** 2 for wi, i in zip(worder, range(worder_len)))
    rho = 1 - sum_d_square / choose(worder_len + 1, 3)

    if normalize:  # If normalized, the rho output falls between 0.0 to 1.0
        return (rho + 1) / 2
    else:  # Otherwise, the rho outputs falls between -1.0 to +1.0
        return rho
```


> Division by zero error is constantly thrown for `corpus_ribes()` function.
> 
> To replicate:
> Take the example from the docs (https://www.nltk.org/api/nltk.translate.html) and shorten some
> of the input sentences. In this case I only shortened hyp2 and hyp2a.
> 
> ```
> from nltk.translate.ribes_score import corpus_ribes
> 
> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', 'ensures', 'that', 'the', 'military', 'always', 'obeys', 'the', 'commands', 'of', 'the', 'party']
> ref1a = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',  'ensures', 'that', 'the', 'military', 'will', 'forever',  'heed', 'Party', 'commands']
> ref1b = ['It', 'is', 'the', 'guiding', 'principle', 'which', 'guarantees', 'the', 'military', 'forces', 'always', 'being', 'under', 'the', 'command', 'of', 'the', 'Party']
> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',  'army', 'always', 'to', 'heed', 'the', 'directions',  'of', 'the', 'party']
> 
> hyp2 = ['he', 'read', 'the']
> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', 'because', 'he']
> 
> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]
> hypotheses = [hyp1, hyp2]
> corpus_ribes(list_of_references, hypotheses)
> ```
> 
> Output:
> 
> ```
> --------------------------------------------------------------------------
> ZeroDivisionError                         Traceback (most recent call last)
> <ipython-input-175-b02599af69e7> in <module>
>       1 list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]
>       2 hypotheses = [hyp1, hyp2]
> ----> 3 corpus_ribes(list_of_references, hypotheses)
> 
> /anaconda3/envs/torch/lib/python3.7/site-packages/nltk/translate/ribes_score.py in corpus_ribes(list_of_references, hypotheses, alpha, beta)
>     117     # Iterate through each hypothesis and their corresponding references.
>     118     for references, hypothesis in zip(list_of_references, hypotheses):
> --> 119         corpus_best_ribes += sentence_ribes(references, hypothesis, alpha, beta)
>     120     return corpus_best_ribes / len(hypotheses)
>     121 
> 
> /anaconda3/envs/torch/lib/python3.7/site-packages/nltk/translate/ribes_score.py in sentence_ribes(references, hypothesis, alpha, beta)
>      53         # Collects the *worder* from the ranked correlation alignments.
>      54         worder = word_rank_alignment(reference, hypothesis)
> ---> 55         nkt = kendall_tau(worder)
>      56 
>      57         # Calculates the brevity penalty
> 
> /anaconda3/envs/torch/lib/python3.7/site-packages/nltk/translate/ribes_score.py in kendall_tau(worder, normalize)
>     288     num_possible_pairs = choose(worder_len, 2)
>     289     # Kendall's Tau computation.
> --> 290     tau = 2 * num_increasing_pairs / num_possible_pairs - 1
>     291     if normalize:  # If normalized, the tau output falls between 0.0 to 1.0
>     292         return (tau + 1) / 2
> 
> ZeroDivisionError: division by zero
> ```

I'm having same issue
@fuzihaofzh Seems like this is only possible if `len(worder) == 2`. Should this throw an exception or return some seminal value?

Resolving patch:
diff --git a/nltk/translate/ribes_score.py b/nltk/translate/ribes_score.py
--- a/nltk/translate/ribes_score.py
+++ b/nltk/translate/ribes_score.py
@@ -36,7 +36,7 @@ def sentence_ribes(references, hypothesis, alpha=0.25, beta=0.10):
     to http://www.kecl.ntt.co.jp/icl/lirg/ribes/ for the official script.
 
     :param references: a list of reference sentences
-    :type reference: list(list(str))
+    :type references: list(list(str))
     :param hypothesis: a hypothesis sentence
     :type hypothesis: list(str)
     :param alpha: hyperparameter used as a prior for the unigram precision.
@@ -258,7 +258,7 @@ def kendall_tau(worder, normalize=True):
     Calculates the Kendall's Tau correlation coefficient given the *worder*
     list of word alignments from word_rank_alignment(), using the formula:
 
-        tau = 2 * num_increasing_pairs / num_possible pairs -1
+        tau = 2 * num_increasing_pairs / num_possible_pairs -1
 
     Note that the no. of increasing pairs can be discontinuous in the *worder*
     list and each each increasing sequence can be tabulated as choose(len(seq), 2)
@@ -273,21 +273,27 @@ def kendall_tau(worder, normalize=True):
 
     :param worder: The worder list output from word_rank_alignment
     :type worder: list(int)
-    :param normalize: Flag to indicate normalization
+    :param normalize: Flag to indicate normalization to between 0.0 and 1.0.
     :type normalize: boolean
     :return: The Kendall's Tau correlation coefficient.
     :rtype: float
     """
     worder_len = len(worder)
-    # Extract the groups of increasing/monotonic sequences.
-    increasing_sequences = find_increasing_sequences(worder)
-    # Calculate no. of increasing_pairs in *worder* list.
-    num_increasing_pairs = sum(choose(len(seq), 2) for seq in increasing_sequences)
-    # Calculate no. of possible pairs.
-    num_possible_pairs = choose(worder_len, 2)
-    # Kendall's Tau computation.
-    tau = 2 * num_increasing_pairs / num_possible_pairs - 1
-    if normalize:  # If normalized, the tau output falls between 0.0 to 1.0
+    # With worder_len < 2, `choose(worder_len, 2)` will be 0.
+    # As we divide by this, it will give a ZeroDivisionError.
+    # To avoid this, we can just return the lowest possible score.
+    if worder_len < 2:
+        tau = -1
+    else:
+        # Extract the groups of increasing/monotonic sequences.
+        increasing_sequences = find_increasing_sequences(worder)
+        # Calculate no. of increasing_pairs in *worder* list.
+        num_increasing_pairs = sum(choose(len(seq), 2) for seq in increasing_sequences)
+        # Calculate no. of possible pairs.
+        num_possible_pairs = choose(worder_len, 2)
+        # Kendall's Tau computation.
+        tau = 2 * num_increasing_pairs / num_possible_pairs - 1
+    if normalize: # If normalized, the tau output falls between 0.0 to 1.0
         return (tau + 1) / 2
     else:  # Otherwise, the tau outputs falls between -1.0 to +1.0
         return tau


Test patch:
diff --git a/nltk/test/unit/test_ribes.py b/nltk/test/unit/test_ribes.py
new file mode 100644
--- /dev/null
+++ b/nltk/test/unit/test_ribes.py
@@ -0,0 +1,246 @@
+from nltk.translate.ribes_score import corpus_ribes, word_rank_alignment
+
+
+def test_ribes_empty_worder():  # worder as in word order
+    # Verifies that these two sentences have no alignment,
+    # and hence have the lowest possible RIBES score.
+    hyp = "This is a nice sentence which I quite like".split()
+    ref = "Okay well that's neat and all but the reference's different".split()
+
+    assert word_rank_alignment(ref, hyp) == []
+
+    list_of_refs = [[ref]]
+    hypotheses = [hyp]
+    assert corpus_ribes(list_of_refs, hypotheses) == 0.0
+
+
+def test_ribes_one_worder():
+    # Verifies that these two sentences have just one match,
+    # and the RIBES score for this sentence with very little
+    # correspondence is 0.
+    hyp = "This is a nice sentence which I quite like".split()
+    ref = "Okay well that's nice and all but the reference's different".split()
+
+    assert word_rank_alignment(ref, hyp) == [3]
+
+    list_of_refs = [[ref]]
+    hypotheses = [hyp]
+    assert corpus_ribes(list_of_refs, hypotheses) == 0.0
+
+
+def test_ribes_two_worder():
+    # Verifies that these two sentences have two matches,
+    # but still get the lowest possible RIBES score due
+    # to the lack of similarity.
+    hyp = "This is a nice sentence which I quite like".split()
+    ref = "Okay well that's nice and all but the reference is different".split()
+
+    assert word_rank_alignment(ref, hyp) == [9, 3]
+
+    list_of_refs = [[ref]]
+    hypotheses = [hyp]
+    assert corpus_ribes(list_of_refs, hypotheses) == 0.0
+
+
+def test_ribes():
+    # Based on the doctest of the corpus_ribes function
+    hyp1 = [
+        "It",
+        "is",
+        "a",
+        "guide",
+        "to",
+        "action",
+        "which",
+        "ensures",
+        "that",
+        "the",
+        "military",
+        "always",
+        "obeys",
+        "the",
+        "commands",
+        "of",
+        "the",
+        "party",
+    ]
+    ref1a = [
+        "It",
+        "is",
+        "a",
+        "guide",
+        "to",
+        "action",
+        "that",
+        "ensures",
+        "that",
+        "the",
+        "military",
+        "will",
+        "forever",
+        "heed",
+        "Party",
+        "commands",
+    ]
+    ref1b = [
+        "It",
+        "is",
+        "the",
+        "guiding",
+        "principle",
+        "which",
+        "guarantees",
+        "the",
+        "military",
+        "forces",
+        "always",
+        "being",
+        "under",
+        "the",
+        "command",
+        "of",
+        "the",
+        "Party",
+    ]
+    ref1c = [
+        "It",
+        "is",
+        "the",
+        "practical",
+        "guide",
+        "for",
+        "the",
+        "army",
+        "always",
+        "to",
+        "heed",
+        "the",
+        "directions",
+        "of",
+        "the",
+        "party",
+    ]
+
+    hyp2 = [
+        "he",
+        "read",
+        "the",
+        "book",
+        "because",
+        "he",
+        "was",
+        "interested",
+        "in",
+        "world",
+        "history",
+    ]
+    ref2a = [
+        "he",
+        "was",
+        "interested",
+        "in",
+        "world",
+        "history",
+        "because",
+        "he",
+        "read",
+        "the",
+        "book",
+    ]
+
+    list_of_refs = [[ref1a, ref1b, ref1c], [ref2a]]
+    hypotheses = [hyp1, hyp2]
+
+    score = corpus_ribes(list_of_refs, hypotheses)
+
+    assert round(score, 4) == 0.3597
+
+
+def test_no_zero_div():
+    # Regression test for Issue 2529, assure that no ZeroDivisionError is thrown.
+    hyp1 = [
+        "It",
+        "is",
+        "a",
+        "guide",
+        "to",
+        "action",
+        "which",
+        "ensures",
+        "that",
+        "the",
+        "military",
+        "always",
+        "obeys",
+        "the",
+        "commands",
+        "of",
+        "the",
+        "party",
+    ]
+    ref1a = [
+        "It",
+        "is",
+        "a",
+        "guide",
+        "to",
+        "action",
+        "that",
+        "ensures",
+        "that",
+        "the",
+        "military",
+        "will",
+        "forever",
+        "heed",
+        "Party",
+        "commands",
+    ]
+    ref1b = [
+        "It",
+        "is",
+        "the",
+        "guiding",
+        "principle",
+        "which",
+        "guarantees",
+        "the",
+        "military",
+        "forces",
+        "always",
+        "being",
+        "under",
+        "the",
+        "command",
+        "of",
+        "the",
+        "Party",
+    ]
+    ref1c = [
+        "It",
+        "is",
+        "the",
+        "practical",
+        "guide",
+        "for",
+        "the",
+        "army",
+        "always",
+        "to",
+        "heed",
+        "the",
+        "directions",
+        "of",
+        "the",
+        "party",
+    ]
+
+    hyp2 = ["he", "read", "the"]
+    ref2a = ["he", "was", "interested", "in", "world", "history", "because", "he"]
+
+    list_of_refs = [[ref1a, ref1b, ref1c], [ref2a]]
+    hypotheses = [hyp1, hyp2]
+
+    score = corpus_ribes(list_of_refs, hypotheses)
+
+    assert round(score, 4) == 0.1688