Repository: facebookresearch/ParlAI

Issue:
Empty training examples throwing runtime error
I'm getting an error with the OpenSubtitles (2009 and 2018) and the Cornell Movie datasets with length 0 examples being loaded. My guess is that there are empty messages in the datasets, which could easily be removed with some preprocessing or a flag to avoid such examples. However, I'm having trouble finding such a flag and would appreciate any help you can provide. I do not have this issue arise on other datasets that I have explored so far. 

I'm a bit puzzled as to why others haven't seemed to run into this problem and was hoping you could help point me in the right direction, especially if I'm missing an easy fix. Thanks.

**Error:**
```
Traceback (most recent call last):
 File "examples/train_model.py", line 16, in <module>
   TrainLoop(opt).train()
 File "/data/odemasi/packages/ParlAI/parlai/scripts/train_model.py", line 523, in train
   world.parley()
 File "/data/odemasi/packages/ParlAI/parlai/core/worlds.py", line 654, in parley
   batch_act = self.batch_act(agent_idx, batch_observations[agent_idx])
 File "/data/odemasi/packages/ParlAI/parlai/core/worlds.py", line 627, in batch_act
   batch_actions = a.batch_act(batch_observation)
 File "/data/odemasi/packages/ParlAI/parlai/core/torch_agent.py", line 1206, in batch_act
   output = self.train_step(batch)
 File "/data/odemasi/packages/ParlAI/parlai/core/torch_generator_agent.py", line 478, in train_step
   raise e
 File "/data/odemasi/packages/ParlAI/parlai/core/torch_generator_agent.py", line 454, in train_step
   scores, preds, _ = self.model(batch.text_vec, batch.label_vec)
 File "/data/odemasi/anaconda3/envs/parlai_venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
   result = self.forward(*input, **kwargs)
 File "/data/odemasi/packages/ParlAI/parlai/core/torch_generator_agent.py", line 214, in forward
   encoder_states = prev_enc if prev_enc is not None else self.encoder(xs)
 File "/data/odemasi/anaconda3/envs/parlai_venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
   result = self.forward(*input, **kwargs)
 File "/data/odemasi/packages/ParlAI/parlai/agents/seq2seq/modules.py", line 226, in forward
   xes = pack_padded_sequence(xes, x_lens, batch_first=True)
 File "/data/odemasi/anaconda3/envs/parlai_venv/lib/python3.7/site-packages/torch/nn/utils/rnn.py", line 148, in pack_padded_sequence
   return PackedSequence(torch._C._VariableFunctions._pack_padded_sequence(input, lengths, batch_first))
RuntimeError: Length of all samples has to be greater than 0, but found an element in 'lengths' that is <= 0
```


**Resulted from:**
python examples/train_model.py \
-t cornell_movie \
-bs 32 \
--hiddensize 1024 \
--dict-minfreq 2 \
--dict-lower True \
--dict-file cornell_dict \
-m seq2seq \
-mf cornell_seq2seq 


Discussion:
Thanks for reporting. How long does it usually take before you hit this bug?
It comes up very quickly. It's obviously within the first epoch and for the
Cornell dataset it is within the first few minutes of training.

On Fri, Mar 29, 2019, 18:00 Stephen Roller <notifications@github.com> wrote:

> Thanks for reporting. How long does it usually take before you hit this
> bug?
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/facebookresearch/ParlAI/issues/1592#issuecomment-478191314>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/ADozPzrg46AnEPpHdWnH5W3aFEuXCZ58ks5vbrc_gaJpZM4cTcKI>
> .
>


Resolving patch:
diff --git a/parlai/agents/seq2seq/seq2seq.py b/parlai/agents/seq2seq/seq2seq.py
--- a/parlai/agents/seq2seq/seq2seq.py
+++ b/parlai/agents/seq2seq/seq2seq.py
@@ -5,6 +5,7 @@
 # LICENSE file in the root directory of this source tree.
 
 from parlai.core.torch_generator_agent import TorchGeneratorAgent
+from parlai.core.utils import warn_once
 from .modules import Seq2seq, opt_to_kwargs
 
 import torch
@@ -201,3 +202,22 @@ def load(self, path):
         if 'longest_label' in states:
             self.model.longest_label = states['longest_label']
         return states
+
+    def is_valid(self, obs):
+        normally_valid = super().is_valid(obs)
+        if not normally_valid:
+            # shortcut boolean evaluation
+            return normally_valid
+        contains_empties = obs['text_vec'].shape[0] == 0
+        if self.is_training and contains_empties:
+            warn_once(
+                'seq2seq got an empty input sequence (text_vec) during training. '
+                'Skipping this example, but you should check your dataset and '
+                'preprocessing.'
+            )
+        elif not self.is_training and contains_empties:
+            warn_once(
+                'seq2seq got an empty input sequence (text_vec) in an '
+                'evaluation example! This may affect your metrics!'
+            )
+        return not contains_empties


Test patch:
diff --git a/parlai/tasks/integration_tests/agents.py b/parlai/tasks/integration_tests/agents.py
--- a/parlai/tasks/integration_tests/agents.py
+++ b/parlai/tasks/integration_tests/agents.py
@@ -13,6 +13,7 @@
 """
 
 from parlai.core.teachers import DialogTeacher
+import copy
 import random
 import itertools
 
@@ -171,5 +172,69 @@ def setup_data(self, fold):
             yield (t, a), e
 
 
+class BadExampleTeacher(CandidateTeacher):
+    """
+    Teacher which produces a variety of examples that upset verify_data.py.
+
+    Useful for checking how models respond when the following assumptions are
+    violated:
+
+        0. text is empty string
+        1. missing text
+        2. label is empty string
+        3. missing label
+        4. label candidates is empty
+        5. label candidates contains an empty string
+        6. label isn't in the candidates
+        7. missing label candidates
+
+    Note: this test may come to outlive its purpose in the future. When failing
+    this test, one should consider who is really at fault: the test, or the code.
+    """
+    NUM_CASES = 8
+
+    def __init__(self, opt, shared=None):
+        super().__init__(opt, shared)
+        # gross hack: override data.get to force things the way we want; otherwise
+        # we can't actually force some of these scenarios.
+        self.data.get = self._wrapperfn(self.data.get)
+
+    def _wrapperfn(self, oldget):
+        def newget(*args):
+            item, eod = oldget(*args)
+            item = copy.deepcopy(item)
+            newget.case = (newget.case + 1) % self.NUM_CASES
+            case = newget.case
+            if case == 0:
+                # empty string input
+                item['text'] = ''
+            elif case == 1:
+                # not text input
+                del item['text']
+            elif case == 2:
+                # empty string label
+                item['labels'] = ['']
+            elif case == 3:
+                # no label
+                del item['labels']
+            elif case == 4:
+                # no label candidates
+                item['label_candidates'] = []
+            elif case == 5:
+                # extra empty string in labels
+                item['label_candidates'] = list(item['label_candidates']) + ['']
+            elif case == 6:
+                # label candidates doesn't have the label
+                item['label_candidates'] = list(item['label_candidates'])
+                item['label_candidates'].remove(item['labels'][0])
+            elif case == 7:
+                # no label candidates field
+                del item['label_candidates']
+            return item, eod
+
+        newget.case = random.randint(0, self.NUM_CASES)
+        return newget
+
+
 class DefaultTeacher(CandidateTeacher):
     pass
diff --git a/tests/test_seq2seq.py b/tests/test_seq2seq.py
--- a/tests/test_seq2seq.py
+++ b/tests/test_seq2seq.py
@@ -106,6 +106,23 @@ def test_beamsearch(self):
             "test ppl = {}\nLOG:\n{}".format(test['ppl'], stdout)
         )
 
+    def test_badinput(self):
+        """Ensures model doesn't crash on malformed inputs."""
+        stdout, _, _ = testing_utils.train_model(dict(
+            task='integration_tests:bad_example',
+            model='seq2seq',
+            lr=LR,
+            batchsize=10,
+            datatype='train:ordered:stream',
+            num_epochs=1,
+            numthreads=1,
+            no_cuda=True,
+            embeddingsize=16,
+            hiddensize=16,
+        ))
+        self.assertIn('valid:{', stdout)
+        self.assertIn('test:{', stdout)
+
 
 class TestHogwildSeq2seq(unittest.TestCase):
     @testing_utils.skipIfGPU
diff --git a/tests/test_transformers.py b/tests/test_transformers.py
--- a/tests/test_transformers.py
+++ b/tests/test_transformers.py
@@ -245,6 +245,22 @@ def test_generator_backcomp(self):
             'test f1 = {}\nLOG:\n{}'.format(test['f1'], stdout)
         )
 
+    def test_badinput(self):
+        """Ensures model doesn't crash on malformed inputs."""
+        stdout, _, _ = testing_utils.train_model(dict(
+            task='integration_tests:bad_example',
+            model='transformer/generator',
+            batchsize=10,
+            datatype='train:ordered:stream',
+            num_epochs=1,
+            numthreads=1,
+            no_cuda=True,
+            embeddingsize=16,
+            hiddensize=16,
+        ))
+        self.assertIn('valid:{', stdout)
+        self.assertIn('test:{', stdout)
+
 
 class TestLearningRateScheduler(unittest.TestCase):
     def test_resuming(self):