Repository: nltk/nltk

Issue:
IBMModel1 aligner fails when aligning novel data (Python 3)
The method `IBMModel1.align()` fails when trying to align unseen data, evidently due to the tightened rules in Python 3 for comparing objects of different types:

``` python
from nltk import IBMModel1 
from nltk.corpus import comtrans
data = comtrans.aligned_sents("alignment-de-en.txt")
model = IBMModel1(data[:10], 20)

model.align(data[15])   # boom!
```

The training set is really tiny, so the error is inevitable. A real training set will probably lower the probability of an exception but not eliminate it completely.

Traceback:

```
    ...
    File "/Applications/anaconda/lib/python3.4/site-packages/nltk/align/ibm1.py", line 123, in
     align(self.probabilities[en_word][fr_word], i))
    TypeError: unorderable types: int() > NoneType()
```

This error does not arise under python 2 (with NLTK 3), since python 2 will never give an error when comparing objects of different types (it just compares the type names alphabetically, i.e. `NoneType < int`). Whether the resulting alignment decisions make any sense, I have not checked.



Discussion:

Resolving patch:
diff --git a/nltk/align/ibm1.py b/nltk/align/ibm1.py
--- a/nltk/align/ibm1.py
+++ b/nltk/align/ibm1.py
@@ -164,32 +164,18 @@ def align(self, sentence_pair):
 
         for j, trg_word in enumerate(sentence_pair.words):
             # Initialize trg_word to align with the NULL token
-            initial_prob = max(self.translation_table[trg_word][None],
-                               IBMModel.MIN_PROB)
-            best_alignment = (initial_prob, None)
+            best_prob = max(self.translation_table[trg_word][None],
+                            IBMModel.MIN_PROB)
+            best_alignment = None
             for i, src_word in enumerate(sentence_pair.mots):
                 align_prob = self.translation_table[trg_word][src_word]
-                best_alignment = max(best_alignment, (align_prob, i))
+                if align_prob >= best_prob: # prefer newer word in case of tie
+                    best_prob = align_prob
+                    best_alignment = i
 
             # If trg_word is not aligned to the NULL token,
             # add it to the viterbi_alignment.
-            if best_alignment[1] is not None:
-                alignment.append((j, best_alignment[1]))
+            if best_alignment is not None:
+                alignment.append((j, best_alignment))
 
         return AlignedSent(sentence_pair.words, sentence_pair.mots, alignment)
-
-        for j, en_word in enumerate(align_sent.words):
-            
-            # Initialize the maximum probability with Null token
-            max_align_prob = (self.probabilities[en_word][None], None)
-            for i, fr_word in enumerate(align_sent.mots):
-                # Find out the maximum probability
-                max_align_prob = max(max_align_prob,
-                    (self.probabilities[en_word][fr_word], i))
-
-            # If the maximum probability is not Null token,
-            # then append it to the alignment. 
-            if max_align_prob[1] is not None:
-                alignment.append((j, max_align_prob[1]))
-
-        return AlignedSent(align_sent.words, align_sent.mots, alignment)
diff --git a/nltk/align/ibm2.py b/nltk/align/ibm2.py
--- a/nltk/align/ibm2.py
+++ b/nltk/align/ibm2.py
@@ -206,19 +206,21 @@ def align(self, sentence_pair):
 
         for j, trg_word in enumerate(sentence_pair.words):
             # Initialize trg_word to align with the NULL token
-            initial_prob = (self.translation_table[trg_word][None] *
-                            self.alignment_table[0][j + 1][l][m])
-            initial_prob = max(initial_prob, IBMModel.MIN_PROB)
-            best_alignment = (initial_prob, None)
+            best_prob = (self.translation_table[trg_word][None] *
+                         self.alignment_table[0][j + 1][l][m])
+            best_prob = max(best_prob, IBMModel.MIN_PROB)
+            best_alignment = None
             for i, src_word in enumerate(sentence_pair.mots):
                 align_prob = (self.translation_table[trg_word][src_word] *
                               self.alignment_table[i + 1][j + 1][l][m])
-                best_alignment = max(best_alignment, (align_prob, i))
+                if align_prob >= best_prob:
+                    best_prob = align_prob
+                    best_alignment = i
 
             # If trg_word is not aligned to the NULL token,
             # add it to the viterbi_alignment.
-            if best_alignment[1] is not None:
-                alignment.append((j, best_alignment[1]))
+            if best_alignment is not None:
+                alignment.append((j, best_alignment))
 
         return AlignedSent(sentence_pair.words, sentence_pair.mots, alignment)
 
diff --git a/nltk/align/ibm3.py b/nltk/align/ibm3.py
--- a/nltk/align/ibm3.py
+++ b/nltk/align/ibm3.py
@@ -89,7 +89,7 @@ class IBMModel3(IBMModel):
 
     >>> align_sents = []
     >>> align_sents.append(AlignedSent(['klein', 'ist', 'das', 'Haus'], ['the', 'house', 'is', 'small']))
-    >>> align_sents.append(AlignedSent(['das', 'Haus', 'ist', 'groß'], ['the', 'house', 'is', 'big']))
+    >>> align_sents.append(AlignedSent(['das', 'Haus', 'ist', 'ja', 'groß'], ['the', 'house', 'is', 'big']))
     >>> align_sents.append(AlignedSent(['das', 'Haus'], ['the', 'house']))
     >>> align_sents.append(AlignedSent(['das', 'Buch'], ['the', 'book']))
     >>> align_sents.append(AlignedSent(['ein', 'Buch'], ['a', 'book']))
@@ -478,19 +478,21 @@ def align(self, sentence_pair):
 
         for j, trg_word in enumerate(sentence_pair.words):
             # Initialize trg_word to align with the NULL token
-            initial_prob = (self.translation_table[trg_word][None] *
-                            self.distortion_table[j + 1][0][l][m])
-            initial_prob = max(initial_prob, IBMModel.MIN_PROB)
-            best_alignment = (initial_prob, 0)
+            best_prob = (self.translation_table[trg_word][None] *
+                         self.distortion_table[j + 1][0][l][m])
+            best_prob = max(best_prob, IBMModel.MIN_PROB)
+            best_alignment = None
             for i, src_word in enumerate(sentence_pair.mots):
                 align_prob = (self.translation_table[trg_word][src_word] *
                               self.distortion_table[j + 1][i + 1][l][m])
-                best_alignment = max(best_alignment, (align_prob, i))
+                if align_prob >= best_prob:
+                    best_prob = align_prob
+                    best_alignment = i
 
             # If trg_word is not aligned to the NULL token,
             # add it to the viterbi_alignment.
-            if best_alignment[1] is not None:
-                alignment.append((j, best_alignment[1]))
+            if best_alignment is not None:
+                alignment.append((j, best_alignment))
 
         return AlignedSent(sentence_pair.words, sentence_pair.mots, alignment)
 


Test patch: