Repository: awslabs/sockeye

Issue:
skip_blank has different values during data preparation
I'm running a standard training with parallel sentences containing empty source or target lines and  `data_io.py` returns an error while building buckets:

```
IndexError: index 897268 is out of bounds for axis 0 with size 897268
```
The `parallel_iter()` function in `data_io.py` is always called with the `skip_blank` argument set as `True`, except [right here](https://github.com/awslabs/sockeye/blob/master/sockeye/data_io.py#L475). This line makes us keep the sentence pairs containing "blanks", which seems to bring the mismatch reflected in the error above. I don't get the error anymore when I set `skip_blank` to `True` (or when I remove the sentence pairs containing blanks in the data).

@mjpost This line came with [this PR](https://github.com/awslabs/sockeye/commit/02ff337e3370b351b5e2d0c19bbe80a9df8445ec#diff-6f5897a736e94787b26bb894b706251cR475). Would it be an issue to let the default `skip_blank` value here ?


Discussion:
Yeah, [this line](https://github.com/awslabs/sockeye/blob/86b8175e304a02b1c4cac4012a5776b686470478/sockeye/data_io.py#L475) seems like a bug. It should be

```python
for sentno, (sources, target) in enumerate(parallel_iter(source_iterables, target_iterable, skip_blanks=self.skip_blanks), 1):
```

Amazon folks, I can put in a PR with a test case that exercises this, but not until tonight at least.

And @franckbrl, in the long tradition of blaming the user for design problems—why aren't you cleaning your training data?!
:laughing: Fair enough!
I should also have written, thanks for tracking this down and filing a perfect bug report. It seems you covered a real hole in my use cases :)
Always a pleasure!

Resolving patch:
diff --git a/sockeye/__init__.py b/sockeye/__init__.py
--- a/sockeye/__init__.py
+++ b/sockeye/__init__.py
@@ -11,4 +11,4 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 
-__version__ = '1.18.79'
+__version__ = '1.18.80'
diff --git a/sockeye/data_io.py b/sockeye/data_io.py
--- a/sockeye/data_io.py
+++ b/sockeye/data_io.py
@@ -432,7 +432,7 @@ class RawParallelDatasetLoader:
     :param eos_id: End-of-sentence id.
     :param pad_id: Padding id.
     :param eos_id: Unknown id.
-    :param skip_blanks: Skip blank lines.
+    :param skip_blanks: Whether to skip blank lines.
     :param dtype: Data type.
     """
 
@@ -472,7 +472,7 @@ def load(self,
         num_pad_target = 0
 
         # Bucket sentences as padded np arrays
-        for sentno, (sources, target) in enumerate(parallel_iter(source_iterables, target_iterable, skip_blanks=False), 1):
+        for sentno, (sources, target) in enumerate(parallel_iter(source_iterables, target_iterable, skip_blanks=self.skip_blanks), 1):
             sources = [[] if stream is None else stream for stream in sources]
             if target is None:
                 target = []


Test patch:
diff --git a/test/common.py b/test/common.py
--- a/test/common.py
+++ b/test/common.py
@@ -135,7 +135,7 @@ def generate_fast_align_lex(lex_path: str):
 
 @contextmanager
 def tmp_digits_dataset(prefix: str,
-                       train_line_count: int, train_max_length: int,
+                       train_line_count: int, train_line_count_empty: int, train_max_length: int,
                        dev_line_count: int, dev_max_length: int,
                        test_line_count: int, test_line_count_empty: int, test_max_length: int,
                        sort_target: bool = False,
@@ -153,8 +153,8 @@ def tmp_digits_dataset(prefix: str,
         dev_target_path = os.path.join(work_dir, "dev.tgt")
         test_source_path = os.path.join(work_dir, "test.src")
         test_target_path = os.path.join(work_dir, "test.tgt")
-        generate_digits_file(train_source_path, train_target_path, train_line_count,
-                             train_max_length, sort_target=sort_target, seed=seed_train)
+        generate_digits_file(train_source_path, train_target_path, train_line_count, train_max_length,
+                             line_count_empty=train_line_count_empty, sort_target=sort_target, seed=seed_train)
         generate_digits_file(dev_source_path, dev_target_path, dev_line_count, dev_max_length, sort_target=sort_target,
                              seed=seed_dev)
         generate_digits_file(test_source_path, test_target_path, test_line_count, test_max_length,
diff --git a/test/integration/test_constraints_int.py b/test/integration/test_constraints_int.py
--- a/test/integration/test_constraints_int.py
+++ b/test/integration/test_constraints_int.py
@@ -28,6 +28,7 @@
     _TRANSLATE_PARAMS_COMMON
 
 _TRAIN_LINE_COUNT = 20
+_TRAIN_LINE_COUNT_EMPTY = 1
 _DEV_LINE_COUNT = 5
 _TEST_LINE_COUNT = 5
 _TEST_LINE_COUNT_EMPTY = 2
@@ -58,6 +59,7 @@
 def test_constraints(train_params: str, translate_params: str):
     with tmp_digits_dataset(prefix="test_constraints",
                             train_line_count=_TRAIN_LINE_COUNT,
+                            train_line_count_empty=_TRAIN_LINE_COUNT_EMPTY,
                             train_max_length=_LINE_MAX_LENGTH,
                             dev_line_count=_DEV_LINE_COUNT,
                             dev_max_length=_LINE_MAX_LENGTH,
diff --git a/test/integration/test_seq_copy_int.py b/test/integration/test_seq_copy_int.py
--- a/test/integration/test_seq_copy_int.py
+++ b/test/integration/test_seq_copy_int.py
@@ -31,6 +31,7 @@
 logger = logging.getLogger(__name__)
 
 _TRAIN_LINE_COUNT = 20
+_TRAIN_LINE_COUNT_EMPTY = 1
 _DEV_LINE_COUNT = 5
 _TEST_LINE_COUNT = 5
 _TEST_LINE_COUNT_EMPTY = 2
@@ -165,6 +166,7 @@ def test_seq_copy(train_params: str,
 
     with tmp_digits_dataset(prefix="test_seq_copy",
                             train_line_count=_TRAIN_LINE_COUNT,
+                            train_line_count_empty=_TRAIN_LINE_COUNT_EMPTY,
                             train_max_length=_LINE_MAX_LENGTH,
                             dev_line_count=_DEV_LINE_COUNT,
                             dev_max_length=_LINE_MAX_LENGTH,
@@ -173,7 +175,7 @@ def test_seq_copy(train_params: str,
                             test_max_length=_TEST_MAX_LENGTH,
                             sort_target=False,
                             with_source_factors=use_source_factors) as data:
-        
+
         # TODO: Here we temporarily switch off comparing translation and scoring scores, which
         # sometimes produces inconsistent results for --batch-size > 1 (see issue #639 on github).
         check_train_translate(train_params=train_params,
@@ -198,6 +200,7 @@ def test_other_clis(train_params: str, translate_params: str):
     """
     with tmp_digits_dataset(prefix="test_other_clis",
                             train_line_count=_TRAIN_LINE_COUNT,
+                            train_line_count_empty=_TRAIN_LINE_COUNT_EMPTY,
                             train_max_length=_LINE_MAX_LENGTH,
                             dev_line_count=_DEV_LINE_COUNT,
                             dev_max_length=_LINE_MAX_LENGTH,
diff --git a/test/system/test_seq_copy_sys.py b/test/system/test_seq_copy_sys.py
--- a/test/system/test_seq_copy_sys.py
+++ b/test/system/test_seq_copy_sys.py
@@ -25,6 +25,7 @@
 logger = logging.getLogger(__name__)
 
 _TRAIN_LINE_COUNT = 10000
+_TRAIN_LINE_COUNT_EMPTY = 100
 _DEV_LINE_COUNT = 100
 _LINE_MAX_LENGTH = 10
 _TEST_LINE_COUNT = 110
@@ -124,6 +125,7 @@ def test_seq_copy(name, train_params, translate_params, use_prepared_data, perpl
     """Task: copy short sequences of digits"""
     with tmp_digits_dataset(prefix="test_seq_copy",
                             train_line_count=_TRAIN_LINE_COUNT,
+                            train_line_count_empty=_TRAIN_LINE_COUNT_EMPTY,
                             train_max_length=_LINE_MAX_LENGTH,
                             dev_line_count=_DEV_LINE_COUNT,
                             dev_max_length=_LINE_MAX_LENGTH,
@@ -239,7 +241,7 @@ def test_seq_copy(name, train_params, translate_params, use_prepared_data, perpl
 def test_seq_sort(name, train_params, translate_params, use_prepared_data,
                   use_source_factor, perplexity_thresh, bleu_thresh):
     """Task: sort short sequences of digits"""
-    with tmp_digits_dataset("test_seq_sort.", _TRAIN_LINE_COUNT, _LINE_MAX_LENGTH, _DEV_LINE_COUNT, _LINE_MAX_LENGTH,
+    with tmp_digits_dataset("test_seq_sort.", _TRAIN_LINE_COUNT, _TRAIN_LINE_COUNT_EMPTY, _LINE_MAX_LENGTH, _DEV_LINE_COUNT, _LINE_MAX_LENGTH,
                             _TEST_LINE_COUNT, _TEST_LINE_COUNT_EMPTY, _TEST_MAX_LENGTH,
                             sort_target=True, seed_train=_SEED_TRAIN_DATA, seed_dev=_SEED_DEV_DATA,
                             with_source_factors=use_source_factor) as data:
diff --git a/test/unit/test_data_io.py b/test/unit/test_data_io.py
--- a/test/unit/test_data_io.py
+++ b/test/unit/test_data_io.py
@@ -450,6 +450,7 @@ def test_non_parallel_calculate_length_statistics(sources, target):
 
 def test_get_training_data_iters():
     train_line_count = 100
+    train_line_count_empty = 0
     train_max_length = 30
     dev_line_count = 20
     dev_max_length = 30
@@ -460,7 +461,7 @@ def test_get_training_data_iters():
     test_max_length = 30
     batch_size = 5
     with tmp_digits_dataset("tmp_corpus",
-                            train_line_count, train_max_length - C.SPACE_FOR_XOS,
+                            train_line_count, train_line_count_empty, train_max_length - C.SPACE_FOR_XOS,
                             dev_line_count, dev_max_length - C.SPACE_FOR_XOS,
                             test_line_count, test_line_count_empty,
                             test_max_length - C.SPACE_FOR_XOS) as data: