Repository: pytorch/text

Issue:
TabularDataset class with header does not match fields by name
Currently, the TabularDataset class matches each feature in a csv/tsv file to its field by requiring the (name, Field) tuples passed to the fields kwarg to be in the same order as the columns csv/tsv file.

I think this is counterintuitive, especially if the csv/tsv file has a header. If there is a header, I think that the user should be allowed to specify the fields similar to the json format, where they pass in a dict mapping the column name to the (name, Field) tuple.

Here’s an example of what I’m thinking.

```
$ head -n 2 train.csv
"text","label"
"hello",0
```

```python
>>> pos = data.TabularDataset(
...    path='train.tsv', format='csv',
...    fields={'labels': ('labels', data.Field()),
...                'text': ('text', data.Field())}
...
```

I think they should also be allowed to select a subset of the rows using this method, similar to the API for the json format (columns not in the keys will be ignored).

This should provide a more consistent and intuitive API. If the maintainers are willing to accept this API, I would be happy to implement it and send a PR.


Discussion:

Resolving patch:
diff --git a/torchtext/data/dataset.py b/torchtext/data/dataset.py
--- a/torchtext/data/dataset.py
+++ b/torchtext/data/dataset.py
@@ -2,6 +2,7 @@
 import os
 import zipfile
 import tarfile
+from functools import partial
 
 import torch.utils.data
 
@@ -141,19 +142,22 @@ def __init__(self, path, format, fields, skip_header=False, **kwargs):
             path (str): Path to the data file.
             format (str): The format of the data file. One of "CSV", "TSV", or
                 "JSON" (case-insensitive).
-            fields (list(tuple(str, Field)) or dict[str: tuple(str, Field)]: For CSV and
-                TSV formats, list of tuples of (name, field). The list should be in
-                the same order as the columns in the CSV or TSV file, while tuples of
-                (name, None) represent columns that will be ignored. For JSON format,
-                dictionary whose keys are the JSON keys and whose values are tuples of
-                (name, field). This allows the user to rename columns from their JSON key
-                names and also enables selecting a subset of columns to load
-                (since JSON keys not present in the input dictionary are ignored).
+            fields (list(tuple(str, Field)) or dict[str: tuple(str, Field)]:
+                If using a list, the format must be CSV or TSV, and the values of the list
+                should be tuples of (name, field).
+                The fields should be in the same order as the columns in the CSV or TSV
+                file, while tuples of (name, None) represent columns that will be ignored.
+
+                If using a dict, the keys should be a subset of the JSON keys or CSV/TSV
+                columns, and the values should be tuples of (name, field).
+                Keys not present in the input dictionary are ignored.
+                This allows the user to rename columns from their JSON/CSV/TSV key names
+                and also enables selecting a subset of columns to load.
             skip_header (bool): Whether to skip the first line of the input file.
         """
         make_example = {
             'json': Example.fromJSON, 'dict': Example.fromdict,
-            'tsv': Example.fromTSV, 'csv': Example.fromCSV}[format.lower()]
+            'tsv': Example.fromCSV, 'csv': Example.fromCSV}[format.lower()]
 
         with io.open(os.path.expanduser(path), encoding="utf8") as f:
             if format == 'csv':
@@ -163,8 +167,18 @@ def __init__(self, path, format, fields, skip_header=False, **kwargs):
             else:
                 reader = f
 
+            if format in ['csv', 'tsv'] and isinstance(fields, dict):
+                if skip_header:
+                    raise ValueError('When using a dict to specify fields with a {} file,'
+                                     'skip_header must be False and'
+                                     'the file must have a header.'.format(format))
+                header = next(reader)
+                field_to_index = {f: header.index(f) for f in fields.keys()}
+                make_example = partial(make_example, field_to_index=field_to_index)
+
             if skip_header:
                 next(reader)
+
             examples = [make_example(line, fields) for line in reader]
 
         if make_example in (Example.fromdict, Example.fromJSON):
diff --git a/torchtext/data/example.py b/torchtext/data/example.py
--- a/torchtext/data/example.py
+++ b/torchtext/data/example.py
@@ -29,12 +29,13 @@ def fromdict(cls, data, fields):
         return ex
 
     @classmethod
-    def fromTSV(cls, data, fields):
-        return cls.fromlist(data, fields)
-
-    @classmethod
-    def fromCSV(cls, data, fields):
-        return cls.fromlist(data, fields)
+    def fromCSV(cls, data, fields, field_to_index=None):
+        if field_to_index is None:
+            return cls.fromlist(data, fields)
+        else:
+            assert(isinstance(fields, dict))
+            data_dict = {f: data[idx] for f, idx in field_to_index.items()}
+            return cls.fromdict(data_dict, fields)
 
     @classmethod
     def fromlist(cls, data, fields):


Test patch:
diff --git a/test/common/torchtext_test_case.py b/test/common/torchtext_test_case.py
--- a/test/common/torchtext_test_case.py
+++ b/test/common/torchtext_test_case.py
@@ -24,6 +24,8 @@ def setUp(self):
             self.test_dir, "test_numerical_features_dataset")
         self.test_newline_dataset_path = os.path.join(self.test_dir,
                                                       "test_newline_dataset")
+        self.test_has_header_dataset_path = os.path.join(self.test_dir,
+                                                         "test_has_header_dataset")
 
     def tearDown(self):
         try:
diff --git a/test/data/test_dataset.py b/test/data/test_dataset.py
--- a/test/data/test_dataset.py
+++ b/test/data/test_dataset.py
@@ -122,3 +122,32 @@ def test_input_with_newlines_in_text(self):
             for example in dataset:
                 self.assert_(hasattr(example, "text"))
                 self.assert_(hasattr(example, "label"))
+
+    def test_csv_file_with_header(self):
+        example_with_header = [("text", "label"),
+                               ("HELLO WORLD", "0"),
+                               ("goodbye world", "1")]
+
+        fields = {
+            "label": ("label", data.Field(sequential=False)),
+            "text": ("text", data.Field(lower=True, tokenize=lambda x: x))
+        }
+
+        for format_, delim in zip(["csv", "tsv"], [",", "\t"]):
+            with open(self.test_has_header_dataset_path, "wt") as f:
+                for line in example_with_header:
+                    f.write("{}\n".format(delim.join(line)))
+
+            # check that an error is raised here if a non-existent field is specified
+            with self.assertRaises(ValueError):
+                data.TabularDataset(
+                    path=self.test_has_header_dataset_path, format=format_,
+                    fields={"non_existent": ("label", data.Field())})
+
+            dataset = data.TabularDataset(
+                path=self.test_has_header_dataset_path, format=format_,
+                skip_header=False, fields=fields)
+
+            for i, example in enumerate(dataset):
+                self.assertEqual(example.text, example_with_header[i + 1][0].lower())
+                self.assertEqual(example.label, example_with_header[i + 1][1])