Repository: awslabs/sockeye

Issue:
No error thrown when training with source factor files but no validation source factor files given
Sockeye does not throw in the beginning of training when using plain training data + source factor files but without providing a corresponding source factor file for the validation data.
It will then crash at the first checkpoint with a cryptic error message:
```
[INFO:sockeye.training] Checkpoint [1]	Train-perplexity=928.900834
infer_shape error. Arguments:
  source: (10, 10, 1)
  target: (10, 10)
  target_label: (10, 10)
[ERROR:__main__] Uncaught exception
Traceback (most recent call last):
  File "/Users/fhieber/miniconda3/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/Users/fhieber/miniconda3/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/Users/fhieber/workspace/nmt/src/Sockeye/sockeye/train.py", line 881, in <module>
    main()
  File "/Users/fhieber/workspace/nmt/src/Sockeye/sockeye/train.py", line 756, in main
    train(args)
  File "/Users/fhieber/workspace/nmt/src/Sockeye/sockeye/train.py", line 877, in train
    existing_parameters=args.params)
  File "/Users/fhieber/workspace/nmt/src/Sockeye/sockeye/training.py", line 573, in fit
    self._evaluate(validation_iter, metric_val)
  File "/Users/fhieber/workspace/nmt/src/Sockeye/sockeye/training.py", line 711, in _evaluate
    self.model.evaluate(val_iter, val_metric)
  File "/Users/fhieber/workspace/nmt/src/Sockeye/sockeye/training.py", line 240, in evaluate
    self.module.forward(eval_batch, is_train=False)
  File "/Users/fhieber/miniconda3/lib/python3.6/site-packages/mxnet/module/bucketing_module.py", line 456, in forward
    self._curr_module.forward(data_batch, is_train=is_train)
  File "/Users/fhieber/miniconda3/lib/python3.6/site-packages/mxnet/module/module.py", line 610, in forward
    self.reshape(new_dshape, new_lshape)
  File "/Users/fhieber/miniconda3/lib/python3.6/site-packages/mxnet/module/module.py", line 471, in reshape
    self._exec_group.reshape(self._data_shapes, self._label_shapes)
  File "/Users/fhieber/miniconda3/lib/python3.6/site-packages/mxnet/module/executor_group.py", line 382, in reshape
    self.bind_exec(data_shapes, label_shapes, reshape=True)
  File "/Users/fhieber/miniconda3/lib/python3.6/site-packages/mxnet/module/executor_group.py", line 358, in bind_exec
    allow_up_sizing=True, **dict(data_shapes_i + label_shapes_i))
  File "/Users/fhieber/miniconda3/lib/python3.6/site-packages/mxnet/executor.py", line 402, in reshape
    arg_shapes, _, aux_shapes = self._symbol.infer_shape(**kwargs)
  File "/Users/fhieber/miniconda3/lib/python3.6/site-packages/mxnet/symbol/symbol.py", line 990, in infer_shape
    res = self._infer_shape_impl(False, *args, **kwargs)
  File "/Users/fhieber/miniconda3/lib/python3.6/site-packages/mxnet/symbol/symbol.py", line 1120, in _infer_shape_impl
    ctypes.byref(complete)))
  File "/Users/fhieber/miniconda3/lib/python3.6/site-packages/mxnet/base.py", line 149, in check_call
    raise MXNetError(py_str(_LIB.MXGetLastError()))
mxnet.base.MXNetError: Error in operator source_embed_factor_split: [15:36:18] src/operator/./slice_channel-inl.h:208: Check failed: dshape[real_axis] % param_.num_outputs == 0U (1 vs. 0) You are trying to split the 2-th axis of input tensor with shape [10,10,1] into num_outputs=2 evenly sized chunks, but this is not possible because 2 does not evenly divide 1
```


Discussion:
It looks like [the check is there, but is indented incorrectly](https://github.com/awslabs/sockeye/blob/3b8865e0952033c5a0a5ac0065bda8465ec9d258/sockeye/train.py#L282).

Resolving patch:
diff --git a/sockeye/__init__.py b/sockeye/__init__.py
--- a/sockeye/__init__.py
+++ b/sockeye/__init__.py
@@ -11,4 +11,4 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 
-__version__ = '1.18.49'
+__version__ = '1.18.50'
diff --git a/sockeye/train.py b/sockeye/train.py
--- a/sockeye/train.py
+++ b/sockeye/train.py
@@ -279,9 +279,9 @@ def create_data_iters_and_vocabs(args: argparse.Namespace,
             utils.check_condition(vocab.are_identical(target_vocab, model_target_vocab),
                                   "Prepared data and resumed model target vocabs do not match.")
 
-            check_condition(len(args.source_factors) == len(args.validation_source_factors),
-                            'Training and validation data must have the same number of factors: %d vs. %d.' % (
-                                len(args.source_factors), len(args.validation_source_factors)))
+        check_condition(data_config.num_source_factors == len(validation_sources),
+                        'Training and validation data must have the same number of factors, but found %d and %d.' % (
+                            data_config.num_source_factors, len(validation_sources)))
 
         return train_iter, validation_iter, data_config, source_vocabs, target_vocab
 
@@ -322,6 +322,10 @@ def create_data_iters_and_vocabs(args: argparse.Namespace,
         sources = [args.source] + args.source_factors
         sources = [str(os.path.abspath(source)) for source in sources]
 
+        check_condition(len(sources) == len(validation_sources),
+                        'Training and validation data must have the same number of factors, but found %d and %d.' % (
+                            len(source_vocabs), len(validation_sources)))
+
         train_iter, validation_iter, config_data, data_info = data_io.get_training_data_iters(
             sources=sources,
             target=os.path.abspath(args.target),


Test patch: