Repository: pytorch/text

Issue:
py2 fails with snli example
Just FYI, seems to fail on master.
Consider adding a travis contbuild like `vision` or `tnt` packages to catch these early.
```
~/local/examples/snli] python train.py
downloading
extracting
Traceback (most recent call last):
  File "train.py", line 22, in <module>
    train, dev, test = datasets.SNLI.splits(inputs, answers)
  File "/home/soumith/local/miniconda2/lib/python2.7/site-packages/torchtext/datasets/snli.py", line 47, in splits
    filter_pred=lambda ex: ex.label != '-')
  File "/home/soumith/local/miniconda2/lib/python2.7/site-packages/torchtext/data.py", line 324, in splits
    train_data = None if train is None else cls(path + train, **kwargs)
  File "/home/soumith/local/miniconda2/lib/python2.7/site-packages/torchtext/data.py", line 398, in __init__
    examples = [make_example(line, fields) for line in f]
  File "/home/soumith/local/miniconda2/lib/python2.7/site-packages/torchtext/data.py", line 232, in fromJSON
    return cls.fromdict(json.loads(data), fields)
  File "/home/soumith/local/miniconda2/lib/python2.7/site-packages/torchtext/data.py", line 241, in fromdict
    setattr(ex, name, field.preprocess(val))
  File "/home/soumith/local/miniconda2/lib/python2.7/site-packages/torchtext/data.py", line 136, in preprocess
    x = Pipeline(str.lower)(x)
  File "/home/soumith/local/miniconda2/lib/python2.7/site-packages/torchtext/data.py", line 30, in __call__
    x = pipe.call(x)
  File "/home/soumith/local/miniconda2/lib/python2.7/site-packages/torchtext/data.py", line 36, in call
    return self.convert_token(x, *args)
TypeError: descriptor 'lower' requires a 'str' object but received a 'unicode'
```


Discussion:

Resolving patch:
diff --git a/torchtext/data.py b/torchtext/data.py
--- a/torchtext/data.py
+++ b/torchtext/data.py
@@ -1,4 +1,5 @@
 from __future__ import print_function
+import six
 import torch
 import torch.utils.data
 from torch.autograd import Variable
@@ -113,7 +114,7 @@ def __init__(
             self, sequential=True, use_vocab=True, init_token=None,
             eos_token=None, fix_length=None, tensor_type=torch.LongTensor,
             preprocessing=None, postprocessing=None, lower=False,
-            tokenize=(lambda s: s.split())):
+            tokenize=(lambda s: s.split()), include_lengths=False):
         self.sequential = sequential
         self.use_vocab = use_vocab
         self.fix_length = fix_length
@@ -122,6 +123,7 @@ def __init__(
         self.pad_token = '<pad>' if self.sequential else None
         self.tokenize = get_tokenizer(tokenize)
         self.lower = lower
+        self.include_lengths = include_lengths
         self.preprocessing = (Pipeline() if preprocessing
                               is None else preprocessing)
         self.postprocessing = (Pipeline() if postprocessing
@@ -133,7 +135,7 @@ def preprocess(self, x):
         if self.sequential and isinstance(x, str):
             x = self.tokenize(x)
         if self.lower:
-            x = Pipeline(str.lower)(x)
+            x = Pipeline(six.text_type.lower)(x)
         return self.preprocessing(x)
 
     def pad(self, minibatch):
@@ -141,7 +143,8 @@ def pad(self, minibatch):
 
         Pads to self.fix_length if provided, otherwise pads to the length of
         the longest example in the batch. Prepends self.init_token and appends
-        self.eos_token if those attributes are not None.
+        self.eos_token if those attributes are not None. Returns a tuple of the
+        padded list and a list containing lengths of each example.
         """
         minibatch = list(minibatch)
         if not self.sequential:
@@ -151,13 +154,16 @@ def pad(self, minibatch):
         else:
             max_len = self.fix_length + (
                 self.init_token, self.eos_token).count(None) - 2
-        padded = []
+        padded, lengths = [], []
         for x in minibatch:
             padded.append(
                 ([] if self.init_token is None else [self.init_token]) +
                 list(x[:max_len]) +
                 ([] if self.eos_token is None else [self.eos_token]) +
                 ['<pad>'] * max(0, max_len - len(x)))
+            lengths.append(len(padded[-1]) - max(0, max_len - len(x)))
+        if self.include_lengths:
+            return (padded, lengths)
         return padded
 
     def build_vocab(self, *args, **kwargs):
@@ -193,14 +199,20 @@ def build_vocab(self, *args, **kwargs):
     def numericalize(self, arr, device=None, train=True):
         """Turn a batch of examples that use this field into a Variable.
 
+        If the field has include_lengths=True, a tensor of lengths will be
+        included in the return value.
+
         Arguments:
-            arr: List of tokenized and padded examples.
+            arr: List of tokenized and padded examples, or tuple of a padded
+                list and a list of lengths if self.include_lengths is True.
             device: Device to create the Variable's Tensor on. Use -1 for
                 CPU and None for the currently active GPU device. Default:
                 None.
             train: Whether the batch is for a training set. If False, the
                 Variable will be created with volatile=True. Default: True.
         """
+        if isinstance(arr, tuple):
+            arr, lengths = arr
         if self.use_vocab:
             if self.sequential:
                 arr = [[self.vocab.stoi[x] for x in ex] for ex in arr]
@@ -210,6 +222,8 @@ def numericalize(self, arr, device=None, train=True):
         else:
             arr = self.postprocessing(arr, train)
         arr = self.tensor_type(arr)
+        if self.include_lengths:
+            lengths = torch.LongTensor(lengths)
         if self.sequential:
             arr.t_()
         if device == -1:
@@ -218,6 +232,10 @@ def numericalize(self, arr, device=None, train=True):
         else:
             with torch.cuda.device(device):
                 arr = arr.cuda()
+                if self.include_lengths:
+                    lengths = lengths.cuda()
+        if self.include_lengths:
+            return Variable(arr, volatile=not train), lengths
         return Variable(arr, volatile=not train)
 
 
@@ -234,11 +252,13 @@ def fromJSON(cls, data, fields):
     @classmethod
     def fromdict(cls, data, fields):
         ex = cls()
-        for key, val in data.items():
-            if key in fields:
-                name, field = fields[key]
-                if field is not None:
-                    setattr(ex, name, field.preprocess(val))
+        for key, vals in fields.items():
+            if key in data and vals is not None:
+                if not isinstance(vals, list):
+                    vals = [vals]
+                for val in vals:
+                    name, field = val
+                    setattr(ex, name, field.preprocess(data[key]))
         return ex
 
     @classmethod
@@ -371,7 +391,6 @@ def download_or_unzip(cls, root):
         return os.path.join(path, '')
 
 
-
 class TabularDataset(Dataset):
     """Defines a Dataset of columns stored in CSV, TSV, or JSON format."""
 
@@ -398,7 +417,12 @@ def __init__(self, path, format, fields, **kwargs):
             examples = [make_example(line, fields) for line in f]
 
         if make_example in (Example.fromdict, Example.fromJSON):
-            fields = fields.values()
+            fields, field_dict = [], fields
+            for field in field_dict.values():
+                if isinstance(field, list):
+                    fields.extend(field)
+                else:
+                    fields.append(field)
 
         super(TabularDataset, self).__init__(examples, fields, **kwargs)
 
@@ -536,6 +560,8 @@ def data(self):
     def init_epoch(self):
         """Set up the batch generator for a new epoch."""
         self.batches = batch(self.data(), self.batch_size)
+        if not self.repeat:
+            self.iterations = 0
 
     @property
     def epoch(self):
@@ -563,12 +589,13 @@ class BucketIterator(Iterator):
     """
 
     def init_epoch(self):
-        if self.repeat:
+        if self.sort:
+            self.batches = batch(self.data(), self.batch_size)
+        else:
             self.batches = pool(self.data(), self.batch_size,
                                 self.sort_key)
-        else:
+        if not self.repeat:
             self.iterations = 0
-            self.batches = batch(self.data(), self.batch_size)
 
 
 class BPTTIterator(Iterator):
diff --git a/torchtext/datasets/snli.py b/torchtext/datasets/snli.py
--- a/torchtext/datasets/snli.py
+++ b/torchtext/datasets/snli.py
@@ -2,6 +2,28 @@
 
 from .. import data
 
+
+class ShiftReduceField(data.Field):
+
+    def __init__(self):
+
+        super(ShiftReduceField, self).__init__(preprocessing=lambda parse: [
+            'reduce' if t == ')' else 'shift' for t in parse if t != '('])
+
+        self.build_vocab([['reduce'], ['shift']])
+
+
+class ParsedTextField(data.Field):
+
+    def __init__(self, eos_token='<pad>', lower=False):
+
+        super(ParsedTextField, self).__init__(
+            eos_token=eos_token, lower=lower, preprocessing=lambda parse: [
+                t for t in parse if t not in ('(', ')')],
+            postprocessing=lambda parse, _, __: [
+                list(reversed(p)) for p in parse])
+
+
 class SNLI(data.ZipDataset, data.TabularDataset):
 
     url = 'http://nlp.stanford.edu/projects/snli/snli_1.0.zip'
@@ -14,7 +36,7 @@ def sort_key(ex):
             len(ex.premise), len(ex.hypothesis))
 
     @classmethod
-    def splits(cls, text_field, label_field, root='.', trees=False,
+    def splits(cls, text_field, label_field, parse_field=None, root='.',
                train='train.jsonl', validation='dev.jsonl', test='test.jsonl'):
         """Create dataset objects for splits of the SNLI dataset.
 
@@ -24,8 +46,8 @@ def splits(cls, text_field, label_field, root='.', trees=False,
             text_field: The field that will be used for premise and hypothesis
                 data.
             label_field: The field that will be used for label data.
-            trees: Whether to include the parse trees provided with the SNLI
-                dataset (not implemented).
+            parse_field: The field that will be used for shift-reduce parser
+                transitions, or None to not include them.
             root: The root directory that the dataset's zip archive will be
                 expanded into; therefore the directory in whose snli_1.0
                 subdirectory the data files will be stored.
@@ -35,20 +57,28 @@ def splits(cls, text_field, label_field, root='.', trees=False,
             test: The filename of the test data, or None to not load the test
                 set. Default: 'test.jsonl'.
         """
-        if trees:
-            # TODO
-            raise NotImplementedError
         path = cls.download_or_unzip(root)
+        if parse_field is None:
+            return super(SNLI, cls).splits(
+                os.path.join(path, 'snli_1.0_'), train, validation, test,
+                format='json', fields={'sentence1': ('premise', text_field),
+                                       'sentence2': ('hypothesis', text_field),
+                                       'gold_label': ('label', label_field)},
+                filter_pred=lambda ex: ex.label != '-')
         return super(SNLI, cls).splits(
             os.path.join(path, 'snli_1.0_'), train, validation, test,
-            format='json', fields={'sentence1': ('premise', text_field),
-                                   'sentence2': ('hypothesis', text_field),
+            format='json', fields={'sentence1_binary_parse':
+                                   [('premise', text_field),
+                                    ('premise_transitions', parse_field)],
+                                   'sentence2_binary_parse':
+                                   [('hypothesis', text_field),
+                                    ('hypothesis_transitions', parse_field)],
                                    'gold_label': ('label', label_field)},
             filter_pred=lambda ex: ex.label != '-')
 
     @classmethod
     def iters(cls, batch_size=32, device=0, root='.', wv_dir='.',
-              wv_type=None, wv_dim='300d', **kwargs):
+              wv_type=None, wv_dim='300d', trees=False, **kwargs):
         """Create iterator objects for splits of the SNLI dataset.
 
         This is the simplest way to use the dataset, and assumes common
@@ -64,12 +94,20 @@ def iters(cls, batch_size=32, device=0, root='.', wv_dir='.',
             wv_dir, wv_type, wv_dim: Passed to the Vocab constructor for the
                 text field. The word vectors are accessible as
                 train.dataset.fields['text'].vocab.vectors.
+            trees: Whether to include shift-reduce parser transitions.
+                Default: False.
             Remaining keyword arguments: Passed to the splits method.
         """
-        TEXT = data.Field(tokenize='spacy')
+        if trees:
+            TEXT = ParsedTextField()
+            TRANSITIONS = ShiftReduceField()
+        else:
+            TEXT = data.Field(tokenize='spacy')
+            TRANSITIONS = None
         LABEL = data.Field(sequential=False)
 
-        train, val, test = cls.splits(TEXT, LABEL, root=root, **kwargs)
+        train, val, test = cls.splits(
+            TEXT, LABEL, TRANSITIONS, root=root, **kwargs)
 
         TEXT.build_vocab(train, wv_dir=wv_dir, wv_type=wv_type, wv_dim=wv_dim)
         LABEL.build_vocab(train)


Test patch: