Repository: tensorflow/tensor2tensor

Issue:
MRPC dev data is being used for training
### Description

I expected that the dev dataset would be different from the training dataset. However, all dev examples of MRPC are actually included in the training dataset.

### Environment information

```
OS: macOS 10.13.4

$ pip freeze | grep tensor
mesh-tensorflow==0.0.4
-e git+git@github.com:tensorflow/tensor2tensor.git@7de63449a98375011e2a8715482dfeea946e6de7#egg=tensor2tensor
tensorboard==1.12.0
tensorflow==1.12.0
tensorflow-metadata==0.9.0
tensorflow-probability==0.5.0

$ python -V
Python 3.6.4
```

### For bugs: reproduction and error logs

```python
import tensorflow as tf
from tensor2tensor.data_generators import problem
from tensor2tensor.data_generators.mrpc import MSRParaphraseCorpus

data_dir = "/tmp/t2t_mrpc"
mrpc = MSRParaphraseCorpus()
tf.gfile.MakeDirs(data_dir)
mrpc.generate_data(data_dir, "/tmp")
encoder = mrpc.feature_encoders(data_dir).get("inputs")

tfe = tf.contrib.eager
tfe.enable_eager_execution()
train_dataset = set(
    encoder.decode(example["inputs"])
    for example in tfe.Iterator(mrpc.dataset(problem.DatasetSplit.TRAIN, data_dir)))
eval_dataset = set(
    encoder.decode(example["inputs"])
    for example in tfe.Iterator(mrpc.dataset(problem.DatasetSplit.EVAL, data_dir)))

print("TRAIN Dataset: {}".format(len(train_dataset)))
print("EVAL Dataset: {}".format(len(eval_dataset)))
print("Duplication: {}".format(len(train_dataset & eval_dataset)))
```

Output:
```
TRAIN Dataset: 8152
EVAL Dataset: 816
Duplication: 816
```



Discussion:

Resolving patch:
diff --git a/tensor2tensor/data_generators/mrpc.py b/tensor2tensor/data_generators/mrpc.py
--- a/tensor2tensor/data_generators/mrpc.py
+++ b/tensor2tensor/data_generators/mrpc.py
@@ -58,6 +58,9 @@ def dataset_splits(self):
     }, {
         "split": problem.DatasetSplit.EVAL,
         "shards": 1,
+    }, {
+        "split": problem.DatasetSplit.TEST,
+        "shards": 1,
     }]
 
   @property
@@ -89,7 +92,7 @@ def download_file(tdir, filepath, url):
 
     return mrpc_dir
 
-  def example_generator(self, filename, dev_ids):
+  def example_generator(self, filename, dev_ids, dataset_split):
     for idx, line in enumerate(tf.gfile.Open(filename, "rb")):
       if idx == 0: continue  # skip header
       if six.PY2:
@@ -97,7 +100,10 @@ def example_generator(self, filename, dev_ids):
       else:
         line = line.strip().decode("utf-8")
       l, id1, id2, s1, s2 = line.split("\t")
-      if dev_ids and [id1, id2] not in dev_ids:
+      is_dev = [id1, id2] in dev_ids
+      if dataset_split == problem.DatasetSplit.TRAIN and is_dev:
+        continue
+      if dataset_split == problem.DatasetSplit.EVAL and not is_dev:
         continue
       inputs = [[s1, s2], [s2, s1]]
       for inp in inputs:
@@ -108,14 +114,17 @@ def example_generator(self, filename, dev_ids):
 
   def generate_samples(self, data_dir, tmp_dir, dataset_split):
     mrpc_dir = self._maybe_download_corpora(tmp_dir)
-    filesplit = "msr_paraphrase_train.txt"
+    if dataset_split != problem.DatasetSplit.TEST:
+      filesplit = "msr_paraphrase_train.txt"
+    else:
+      filesplit = "msr_paraphrase_test.txt"
     dev_ids = []
-    if dataset_split != problem.DatasetSplit.TRAIN:
+    if dataset_split != problem.DatasetSplit.TEST:
       for row in tf.gfile.Open(os.path.join(mrpc_dir, "dev_ids.tsv")):
         dev_ids.append(row.strip().split("\t"))
 
     filename = os.path.join(mrpc_dir, filesplit)
-    for example in self.example_generator(filename, dev_ids):
+    for example in self.example_generator(filename, dev_ids, dataset_split):
       yield example
 
 


Test patch: