Repository: facebookresearch/mmf

Issue:
[evaluation] Fix the gap between evaluation on EvalAI and in Pythia
## 🐛 Bug

Due to contractions and other stuff there is always a gap in between the validation performance on Pythia side and EvalAI side. This gap should probably be fixed at the end evaluation for the validation and test accuracies calculation.

## Expected behavior

Accuracies from Pythia and EvalAI should be same.



Discussion:
FWIW, this only happens in LoRRA based architectures as Pythia based architectures use VQA vocabularies which are already processed using `process_answers.py`.
Running the evaluation code here : `https://github.com/GT-Vision-Lab/VQA/blob/master/PythonEvaluationTools/vqaEvaluation/vqaEval.py` gives a different accuracy compared to EvalAI for TextVQA. 

Test run on Validation phase with [submission file](https://evalai.s3.amazonaws.com/media/submission_files/submission_31880/506102f4-5916-48f2-94c8-d76bd53355e6.json) gives an accuracy of 27.4 whereas running the same file using the code given in the script results in 26.41 accuracy.

Is there any way to check if the EvalAI evaluation code is same as in the above script or are there any differences?

Resolving patch:
diff --git a/pythia/modules/metrics.py b/pythia/modules/metrics.py
--- a/pythia/modules/metrics.py
+++ b/pythia/modules/metrics.py
@@ -49,6 +49,7 @@ def calculate(self, sample_list, model_output):
 import torch
 
 from pythia.common.registry import registry
+from pythia.tasks.processors import EvalAIAnswerProcessor
 
 
 class Metrics:
@@ -157,9 +158,7 @@ def calculate(self, sample_list, model_output, *args, **kwargs):
 
         """
         # Override in your child class
-        raise NotImplementedError(
-            "'calculate' must be implemented in the child class"
-        )
+        raise NotImplementedError("'calculate' must be implemented in the child class")
 
     def __call__(self, *args, **kwargs):
         return self.calculate(*args, **kwargs)
@@ -194,10 +193,12 @@ def calculate(self, sample_list, model_output, *args, **kwargs):
         output = model_output["scores"]
         expected = sample_list["targets"]
 
-        assert output.dim() <= 2, ("Output from model shouldn't have "
-                                   " more than dim 2 for accuracy")
-        assert expected.dim() <= 2, ("Expected target shouldn't have "
-                                     "more than dim 2 for accuracy")
+        assert (
+            output.dim() <= 2
+        ), "Output from model shouldn't have more than dim 2 for accuracy"
+        assert (
+            expected.dim() <= 2
+        ), "Expected target shouldn't have more than dim 2 for accuracy"
 
         if output.dim() == 2:
             output = torch.max(output, 1)[1]
@@ -312,6 +313,77 @@ def calculate(self, sample_list, model_output, *args, **kwargs):
         return accuracy
 
 
+@registry.register_metric("vqa_evalai_accuracy")
+class VQAEvalAIAccuracy(BaseMetric):
+    """
+    Calculate Eval AI VQAAccuracy. Find more information here_
+    This is more accurate and similar comparision to Eval AI
+    but is slower compared to vqa_accuracy.
+
+    **Key**: ``vqa_evalai_accuracy``.
+
+    .. _here: https://visualqa.org/evaluation.html
+    """
+
+    def __init__(self):
+        super().__init__("vqa_evalai_accuracy")
+        self.evalai_answer_processor = EvalAIAnswerProcessor()
+
+    def _masked_unk_softmax(self, x, dim, mask_idx):
+        x1 = torch.nn.functional.softmax(x, dim=dim)
+        x1[:, mask_idx] = 0
+        x1_sum = torch.sum(x1, dim=1, keepdim=True)
+        y = x1 / x1_sum
+        return y
+
+    def calculate(self, sample_list, model_output, *args, **kwargs):
+        """Calculate vqa accuracy and return it back.
+
+        Args:
+            sample_list (SampleList): SampleList provided by DataLoader for
+                                current iteration
+            model_output (Dict): Dict returned by model.
+
+        Returns:
+            torch.FloatTensor: VQA Accuracy
+
+        """
+        output = model_output["scores"]
+        expected = sample_list["answers"]
+
+        answer_processor = registry.get(sample_list.dataset_name + "_answer_processor")
+        answer_space_size = answer_processor.get_true_vocab_size()
+
+        output = self._masked_unk_softmax(output, 1, 0)
+        output = output.argmax(dim=1).clone().tolist()
+        accuracy = []
+
+        for idx, answer_id in enumerate(output):
+            if answer_id >= answer_space_size:
+                answer_id -= answer_space_size
+                answer = sample_list["context_tokens"][idx][answer_id]
+            else:
+                answer = answer_processor.idx2word(answer_id)
+
+            answer = self.evalai_answer_processor(answer)
+
+            gt_answers = [self.evalai_answer_processor(x) for x in expected[idx]]
+            gt_answers = list(enumerate(gt_answers))
+
+            gt_acc = []
+            for gt_answer in gt_answers:
+                other_answers = [item for item in gt_answers if item != gt_answer]
+                matching_answers = [item for item in other_answers if item[1] == answer]
+                acc = min(1, float(len(matching_answers)) / 3)
+                gt_acc.append(acc)
+            avgGTAcc = float(sum(gt_acc)) / len(gt_acc)
+            accuracy.append(avgGTAcc)
+
+        accuracy = float(sum(accuracy)) / len(accuracy)
+
+        return model_output["scores"].new_tensor(accuracy, dtype=torch.float)
+
+
 class RecallAtK(BaseMetric):
     def __init__(self, name="recall@k"):
         super().__init__(name)
diff --git a/pythia/tasks/processors.py b/pythia/tasks/processors.py
--- a/pythia/tasks/processors.py
+++ b/pythia/tasks/processors.py
@@ -74,6 +74,7 @@ def __call__(self, item, *args, **kwargs):
 """
 import multiprocessing
 import os
+import re
 import warnings
 from collections import Counter
 
@@ -869,7 +870,7 @@ class CaptionProcessor(BaseProcessor):
 
     """
 
-    def __init__(self,  config, *args, **kwargs):
+    def __init__(self, config, *args, **kwargs):
         if not hasattr(config, "vocab"):
             raise AttributeError(
                 "config passed to the processor has no " "attribute vocab"
@@ -890,3 +891,216 @@ def __call__(self, item):
         ]
         caption = " ".join(tokens)
         return {"tokens": tokens, "caption": caption}
+
+
+@registry.register_processor("evalai_answer")
+class EvalAIAnswerProcessor(BaseProcessor):
+    """Processes an answer similar to Eval AI
+
+    """
+
+    CONTRACTIONS = {
+        "aint": "ain't",
+        "arent": "aren't",
+        "cant": "can't",
+        "couldve": "could've",
+        "couldnt": "couldn't",
+        "couldn'tve": "couldn't've",
+        "couldnt've": "couldn't've",
+        "didnt": "didn't",
+        "doesnt": "doesn't",
+        "dont": "don't",
+        "hadnt": "hadn't",
+        "hadnt've": "hadn't've",
+        "hadn'tve": "hadn't've",
+        "hasnt": "hasn't",
+        "havent": "haven't",
+        "hed": "he'd",
+        "hed've": "he'd've",
+        "he'dve": "he'd've",
+        "hes": "he's",
+        "howd": "how'd",
+        "howll": "how'll",
+        "hows": "how's",
+        "Id've": "I'd've",
+        "I'dve": "I'd've",
+        "Im": "I'm",
+        "Ive": "I've",
+        "isnt": "isn't",
+        "itd": "it'd",
+        "itd've": "it'd've",
+        "it'dve": "it'd've",
+        "itll": "it'll",
+        "let's": "let's",
+        "maam": "ma'am",
+        "mightnt": "mightn't",
+        "mightnt've": "mightn't've",
+        "mightn'tve": "mightn't've",
+        "mightve": "might've",
+        "mustnt": "mustn't",
+        "mustve": "must've",
+        "neednt": "needn't",
+        "notve": "not've",
+        "oclock": "o'clock",
+        "oughtnt": "oughtn't",
+        "ow's'at": "'ow's'at",
+        "'ows'at": "'ow's'at",
+        "'ow'sat": "'ow's'at",
+        "shant": "shan't",
+        "shed've": "she'd've",
+        "she'dve": "she'd've",
+        "she's": "she's",
+        "shouldve": "should've",
+        "shouldnt": "shouldn't",
+        "shouldnt've": "shouldn't've",
+        "shouldn'tve": "shouldn't've",
+        "somebody'd": "somebodyd",
+        "somebodyd've": "somebody'd've",
+        "somebody'dve": "somebody'd've",
+        "somebodyll": "somebody'll",
+        "somebodys": "somebody's",
+        "someoned": "someone'd",
+        "someoned've": "someone'd've",
+        "someone'dve": "someone'd've",
+        "someonell": "someone'll",
+        "someones": "someone's",
+        "somethingd": "something'd",
+        "somethingd've": "something'd've",
+        "something'dve": "something'd've",
+        "somethingll": "something'll",
+        "thats": "that's",
+        "thered": "there'd",
+        "thered've": "there'd've",
+        "there'dve": "there'd've",
+        "therere": "there're",
+        "theres": "there's",
+        "theyd": "they'd",
+        "theyd've": "they'd've",
+        "they'dve": "they'd've",
+        "theyll": "they'll",
+        "theyre": "they're",
+        "theyve": "they've",
+        "twas": "'twas",
+        "wasnt": "wasn't",
+        "wed've": "we'd've",
+        "we'dve": "we'd've",
+        "weve": "we've",
+        "werent": "weren't",
+        "whatll": "what'll",
+        "whatre": "what're",
+        "whats": "what's",
+        "whatve": "what've",
+        "whens": "when's",
+        "whered": "where'd",
+        "wheres": "where's",
+        "whereve": "where've",
+        "whod": "who'd",
+        "whod've": "who'd've",
+        "who'dve": "who'd've",
+        "wholl": "who'll",
+        "whos": "who's",
+        "whove": "who've",
+        "whyll": "why'll",
+        "whyre": "why're",
+        "whys": "why's",
+        "wont": "won't",
+        "wouldve": "would've",
+        "wouldnt": "wouldn't",
+        "wouldnt've": "wouldn't've",
+        "wouldn'tve": "wouldn't've",
+        "yall": "y'all",
+        "yall'll": "y'all'll",
+        "y'allll": "y'all'll",
+        "yall'd've": "y'all'd've",
+        "y'alld've": "y'all'd've",
+        "y'all'dve": "y'all'd've",
+        "youd": "you'd",
+        "youd've": "you'd've",
+        "you'dve": "you'd've",
+        "youll": "you'll",
+        "youre": "you're",
+        "youve": "you've",
+    }
+
+    NUMBER_MAP = {
+        "none": "0",
+        "zero": "0",
+        "one": "1",
+        "two": "2",
+        "three": "3",
+        "four": "4",
+        "five": "5",
+        "six": "6",
+        "seven": "7",
+        "eight": "8",
+        "nine": "9",
+        "ten": "10",
+    }
+    ARTICLES = ["a", "an", "the"]
+    PERIOD_STRIP = re.compile("(?!<=\d)(\.)(?!\d)")
+    COMMA_STRIP = re.compile("(?<=\d)(\,)+(?=\d)")
+    PUNCTUATIONS = [
+        ";",
+        r"/",
+        "[",
+        "]",
+        '"',
+        "{",
+        "}",
+        "(",
+        ")",
+        "=",
+        "+",
+        "\\",
+        "_",
+        "-",
+        ">",
+        "<",
+        "@",
+        "`",
+        ",",
+        "?",
+        "!",
+    ]
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def word_tokenize(self, word):
+        word = word.lower()
+        word = word.replace(",", "").replace("?", "").replace("'s", " 's")
+        return word.strip()
+
+    def process_punctuation(self, in_text):
+        out_text = in_text
+        for p in self.PUNCTUATIONS:
+            if (p + " " in in_text or " " + p in in_text) or (
+                re.search(self.COMMA_STRIP, in_text) is not None
+            ):
+                out_text = out_text.replace(p, "")
+            else:
+                out_text = out_text.replace(p, " ")
+        out_text = self.PERIOD_STRIP.sub("", out_text, re.UNICODE)
+        return out_text
+
+    def process_digit_article(self, in_text):
+        out_text = []
+        temp_text = in_text.lower().split()
+        for word in temp_text:
+            word = self.NUMBER_MAP.setdefault(word, word)
+            if word not in self.ARTICLES:
+                out_text.append(word)
+            else:
+                pass
+        for word_id, word in enumerate(out_text):
+            if word in self.CONTRACTIONS:
+                out_text[word_id] = self.CONTRACTIONS[word]
+        out_text = " ".join(out_text)
+        return out_text
+
+    def __call__(self, item):
+        item = self.word_tokenize(item)
+        item = item.replace("\n", " ").replace("\t", " ").strip()
+        item = self.process_punctuation(item)
+        item = self.process_digit_article(item)
+        return item
diff --git a/pythia/utils/process_answers.py b/pythia/utils/process_answers.py
--- a/pythia/utils/process_answers.py
+++ b/pythia/utils/process_answers.py
@@ -3,171 +3,8 @@
 import argparse
 import json
 import os
-import re
 
-contractions = {
-    "aint": "ain't",
-    "arent": "aren't",
-    "cant": "can't",
-    "couldve": "could've",
-    "couldnt": "couldn't",
-    "couldn'tve": "couldn't've",
-    "couldnt've": "couldn't've",
-    "didnt": "didn't",
-    "doesnt": "doesn't",
-    "dont": "don't",
-    "hadnt": "hadn't",
-    "hadnt've": "hadn't've",
-    "hadn'tve": "hadn't've",
-    "hasnt": "hasn't",
-    "havent": "haven't",
-    "hed": "he'd",
-    "hed've": "he'd've",
-    "he'dve": "he'd've",
-    "hes": "he's",
-    "howd": "how'd",
-    "howll": "how'll",
-    "hows": "how's",
-    "Id've": "I'd've",
-    "I'dve": "I'd've",
-    "Im": "I'm",
-    "Ive": "I've",
-    "isnt": "isn't",
-    "itd": "it'd",
-    "itd've": "it'd've",
-    "it'dve": "it'd've",
-    "itll": "it'll",
-    "let's": "let's",
-    "maam": "ma'am",
-    "mightnt": "mightn't",
-    "mightnt've": "mightn't've",
-    "mightn'tve": "mightn't've",
-    "mightve": "might've",
-    "mustnt": "mustn't",
-    "mustve": "must've",
-    "neednt": "needn't",
-    "notve": "not've",
-    "oclock": "o'clock",
-    "oughtnt": "oughtn't",
-    "ow's'at": "'ow's'at",
-    "'ows'at": "'ow's'at",
-    "'ow'sat": "'ow's'at",
-    "shant": "shan't",
-    "shed've": "she'd've",
-    "she'dve": "she'd've",
-    "she's": "she's",
-    "shouldve": "should've",
-    "shouldnt": "shouldn't",
-    "shouldnt've": "shouldn't've",
-    "shouldn'tve": "shouldn't've",
-    "somebody'd": "somebodyd",
-    "somebodyd've": "somebody'd've",
-    "somebody'dve": "somebody'd've",
-    "somebodyll": "somebody'll",
-    "somebodys": "somebody's",
-    "someoned": "someone'd",
-    "someoned've": "someone'd've",
-    "someone'dve": "someone'd've",
-    "someonell": "someone'll",
-    "someones": "someone's",
-    "somethingd": "something'd",
-    "somethingd've": "something'd've",
-    "something'dve": "something'd've",
-    "somethingll": "something'll",
-    "thats": "that's",
-    "thered": "there'd",
-    "thered've": "there'd've",
-    "there'dve": "there'd've",
-    "therere": "there're",
-    "theres": "there's",
-    "theyd": "they'd",
-    "theyd've": "they'd've",
-    "they'dve": "they'd've",
-    "theyll": "they'll",
-    "theyre": "they're",
-    "theyve": "they've",
-    "twas": "'twas",
-    "wasnt": "wasn't",
-    "wed've": "we'd've",
-    "we'dve": "we'd've",
-    "weve": "we've",
-    "werent": "weren't",
-    "whatll": "what'll",
-    "whatre": "what're",
-    "whats": "what's",
-    "whatve": "what've",
-    "whens": "when's",
-    "whered": "where'd",
-    "wheres": "where's",
-    "whereve": "where've",
-    "whod": "who'd",
-    "whod've": "who'd've",
-    "who'dve": "who'd've",
-    "wholl": "who'll",
-    "whos": "who's",
-    "whove": "who've",
-    "whyll": "why'll",
-    "whyre": "why're",
-    "whys": "why's",
-    "wont": "won't",
-    "wouldve": "would've",
-    "wouldnt": "wouldn't",
-    "wouldnt've": "wouldn't've",
-    "wouldn'tve": "wouldn't've",
-    "yall": "y'all",
-    "yall'll": "y'all'll",
-    "y'allll": "y'all'll",
-    "yall'd've": "y'all'd've",
-    "y'alld've": "y'all'd've",
-    "y'all'dve": "y'all'd've",
-    "youd": "you'd",
-    "youd've": "you'd've",
-    "you'dve": "you'd've",
-    "youll": "you'll",
-    "youre": "you're",
-    "youve": "you've",
-}
-
-manual_map = {
-    "none": "0",
-    "zero": "0",
-    "one": "1",
-    "two": "2",
-    "three": "3",
-    "four": "4",
-    "five": "5",
-    "six": "6",
-    "seven": "7",
-    "eight": "8",
-    "nine": "9",
-    "ten": "10",
-}
-articles = ["a", "an", "the"]
-period_strip = re.compile("(?!<=\d)(\.)(?!\d)")
-comma_strip = re.compile("(\d)(\,)(\d)")
-punct = [
-    ";",
-    r"/",
-    "[",
-    "]",
-    '"',
-    "{",
-    "}",
-    "(",
-    ")",
-    "=",
-    "+",
-    "\\",
-    "_",
-    "-",
-    ">",
-    "<",
-    "@",
-    "`",
-    ",",
-    "?",
-    "!",
-]
+from pythia.tasks.processors import EvalAIAnswerProcessor
 
 
 def get_score(occurences):
@@ -183,55 +20,21 @@ def get_score(occurences):
         return 1
 
 
-def process_punctuation(inText):
-    outText = inText
-    for p in punct:
-        if (p + " " in inText or " " + p in inText) or (
-            re.search(comma_strip, inText) is not None
-        ):
-            outText = outText.replace(p, "")
-        else:
-            outText = outText.replace(p, " ")
-    outText = period_strip.sub("", outText, re.UNICODE)
-    return outText
-
-
-def process_digit_article(inText):
-    outText = []
-    tempText = inText.lower().split()
-    for word in tempText:
-        word = manual_map.setdefault(word, word)
-        if word not in articles:
-            outText.append(word)
-        else:
-            pass
-    for wordId, word in enumerate(outText):
-        if word in contractions:
-            outText[wordId] = contractions[word]
-    outText = " ".join(outText)
-    return outText
-
-
 def multiple_replace(text, wordDict):
     for key in wordDict:
         text = text.replace(key, wordDict[key])
     return text
 
 
-def preprocess_answer(answer):
-    answer = process_digit_article(process_punctuation(answer))
-    answer = answer.replace(",", "")
-    return answer
-
-
 def filter_answers(answers_dset, min_occurence):
     """This will change the answer to preprocessed version
     """
     occurence = {}
     answer_list = []
+    evalai_answer_processor = EvalAIAnswerProcessor()
     for ans_entry in answers_dset:
         gtruth = ans_entry["multiple_choice_answer"]
-        gtruth = preprocess_answer(gtruth)
+        gtruth = evalai_answer_processor(gtruth)
         if gtruth not in occurence:
             occurence[gtruth] = set()
         occurence[gtruth].add(ans_entry["question_id"])


Test patch:
diff --git a/tests/tasks/test_processors.py b/tests/tasks/test_processors.py
--- a/tests/tasks/test_processors.py
+++ b/tests/tasks/test_processors.py
@@ -5,7 +5,11 @@
 import yaml
 import torch
 
-from pythia.tasks.processors import CaptionProcessor, MultiHotAnswerFromVocabProcessor
+from pythia.tasks.processors import (
+    CaptionProcessor,
+    MultiHotAnswerFromVocabProcessor,
+    EvalAIAnswerProcessor,
+)
 from pythia.utils.configuration import ConfigNode
 
 from ..test_utils import compare_tensors
@@ -26,7 +30,9 @@ def test_caption_processor(self):
         captioning_config = config.task_attributes.captioning.dataset_attributes.coco
         caption_processor_config = captioning_config.processors.caption_processor
 
-        vocab_path = os.path.join(os.path.abspath(__file__), "..", "..", "data", "vocab.txt")
+        vocab_path = os.path.join(
+            os.path.abspath(__file__), "..", "..", "data", "vocab.txt"
+        )
         caption_processor_config.params.vocab.vocab_file = os.path.abspath(vocab_path)
         caption_processor = CaptionProcessor(caption_processor_config.params)
 
@@ -34,26 +40,34 @@ def test_caption_processor(self):
         caption = caption_processor(tokens)
 
         # Test start, stop, pad are removed
-        self.assertNotIn('
', caption["tokens"])
-        self.assertNotIn('
', caption["tokens"])
-        self.assertNotIn('<pad>', caption["tokens"])
+        self.assertNotIn("
", caption["tokens"])
+        self.assertNotIn("
", caption["tokens"])
+        self.assertNotIn("<pad>", caption["tokens"])
 
         # Test caption is correct
         self.assertEqual(caption["caption"], "a man with a red helmet")
 
     def test_multi_hot_answer_from_vocab_processor(self):
-        config = self._get_config("../../../pythia/common/defaults/configs/tasks/vqa/clevr.yml")
+        config = self._get_config(
+            "../../../pythia/common/defaults/configs/tasks/vqa/clevr.yml"
+        )
         clevr_config = config.task_attributes.vqa.dataset_attributes.clevr
         answer_processor_config = clevr_config.processors.answer_processor
 
         # Test num_answers==1 case
-        vocab_path = os.path.join(os.path.abspath(__file__), "..", "..", "data", "vocab.txt")
+        vocab_path = os.path.join(
+            os.path.abspath(__file__), "..", "..", "data", "vocab.txt"
+        )
         answer_processor_config.params.vocab_file = os.path.abspath(vocab_path)
-        answer_processor = MultiHotAnswerFromVocabProcessor(answer_processor_config.params)
+        answer_processor = MultiHotAnswerFromVocabProcessor(
+            answer_processor_config.params
+        )
         processed = answer_processor({"answers": ["helmet"]})
         answers_indices = processed["answers_indices"]
         answers_scores = processed["answers_scores"]
-        self.assertTrue(compare_tensors(answers_indices, torch.tensor([5], dtype=torch.long)))
+        self.assertTrue(
+            compare_tensors(answers_indices, torch.tensor([5], dtype=torch.long))
+        )
         expected_answers_scores = torch.zeros(19, dtype=torch.float)
         expected_answers_scores[5] = 1.0
         self.assertTrue(compare_tensors(answers_scores, expected_answers_scores))
@@ -61,13 +75,15 @@ def test_multi_hot_answer_from_vocab_processor(self):
         # Test multihot when num answers greater than 1
         answer_processor_config.params.vocab_file = os.path.abspath(vocab_path)
         answer_processor_config.params.num_answers = 3
-        answer_processor = MultiHotAnswerFromVocabProcessor(answer_processor_config.params)
+        answer_processor = MultiHotAnswerFromVocabProcessor(
+            answer_processor_config.params
+        )
         processed = answer_processor({"answers": ["man", "with", "countryside"]})
         answers_indices = processed["answers_indices"]
         answers_scores = processed["answers_scores"]
-        self.assertTrue(compare_tensors(
-            answers_indices, torch.tensor([2, 3, 15], dtype=torch.long)
-        ))
+        self.assertTrue(
+            compare_tensors(answers_indices, torch.tensor([2, 3, 15], dtype=torch.long))
+        )
         expected_answers_scores = torch.zeros(19, dtype=torch.float)
         expected_answers_scores[2] = 1.0
         expected_answers_scores[3] = 1.0
@@ -78,10 +94,38 @@ def test_multi_hot_answer_from_vocab_processor(self):
         processed = answer_processor({"answers": ["test", "answer", "man"]})
         answers_indices = processed["answers_indices"]
         answers_scores = processed["answers_scores"]
-        self.assertTrue(compare_tensors(
-            answers_indices, torch.tensor([0, 0, 2], dtype=torch.long)
-        ))
+        self.assertTrue(
+            compare_tensors(answers_indices, torch.tensor([0, 0, 2], dtype=torch.long))
+        )
         print(answers_indices, answers_scores)
         expected_answers_scores = torch.zeros(19, dtype=torch.float)
         expected_answers_scores[2] = 1.0
         self.assertTrue(compare_tensors(answers_scores, expected_answers_scores))
+
+    def test_evalai_answer_processor(self):
+        evalai_answer_processor = EvalAIAnswerProcessor()
+
+        # Test number
+        processed = evalai_answer_processor("two")
+        expected = "2"
+        self.assertEqual(processed, expected)
+
+        # Test article
+        processed = evalai_answer_processor("a building")
+        expected = "building"
+        self.assertEqual(processed, expected)
+
+        # Test tokenize
+        processed = evalai_answer_processor("snow, mountain")
+        expected = "snow mountain"
+        self.assertEqual(processed, expected)
+
+        # Test contractions
+        processed = evalai_answer_processor("isnt")
+        expected = "isn't"
+        self.assertEqual(processed, expected)
+
+        # Test processor
+        processed = evalai_answer_processor("the two mountain's \t \n   ")
+        expected = "2 mountain 's"
+        self.assertEqual(processed, expected)