Repository: chainer/chainer

Issue:
NStepLSTM silently fails with invalid in_size when cuDNN is enabled
## Problem description

When cuDNN is enabled NStepLSTM (and its variants), does not raise warning or error when `in_size` is incosistent with the actual input. It will run as if nothing is wrong, and then fails with a CUDA error. A random CUDA error seems to be raised (including `CUDNN_STATUS_EXECUTION_FAILED` and `CUDA_ERROR_ILLEGAL_ADDRESS`). 

## How to reproduce

1. Download and setup `text_classification` example

https://github.com/chainer/chainer/tree/v5.0.0rc1/examples/text_classification

2. Modify `in_size` of NStepLSTM to incosistent value.

```diff
-  self.encoder = L.NStepLSTM(n_layers, n_units, n_units, dropout)
+  self.encoder = L.NStepLSTM(n_layers, 50, n_units, dropout)
```

https://github.com/chainer/chainer/blob/v5.0.0rc1/examples/text_classification/nets.py#L129

3. Run `train_text_classifier.py`

```bash
python train_text_classifier.py --gpu 0 --model rnn --batchsize 256 --epoch 100
```

I set larger `batchsize` so that failure comes earlier.
It usually reproduces within first few epochs.

4. Confirmed that it does not reproduce without cuDNN

```bash
CHAINER_CUDNN=0 python train_text_classifier.py --gpu 0 --model rnn --batchsize 256 --epoch 100
```

It should (properly) raise error that input is incosistent.


I confirmed this behavior in Chainer v5.0.0rc1 and cupy v5.0.0rc1 with cuDNN 7.1.3.16-1+cuda9.1 and CUDA 9.1 on Ubuntu 16.04.

## Suggestion

I suggest we make NStepLSTM to do input size check or make documentation that recommend user to turn off cuDNN if weird CUDA error is raised, or possibly both.

It was super hard to figure out what is wrong and seriously need solving. (cuda-memcheck seems to point at random function as well)


Discussion:
This issue has been automatically marked as stale because it has not had recent activity. It will be closed after 30 days if no further activity occurs. Thank you for your contributions.

I did reproduce this issue and will work on it.

Resolving patch:
diff --git a/chainer/functions/connection/n_step_lstm.py b/chainer/functions/connection/n_step_lstm.py
--- a/chainer/functions/connection/n_step_lstm.py
+++ b/chainer/functions/connection/n_step_lstm.py
@@ -403,6 +403,13 @@ def n_step_lstm_base(
             'Use chainer.using_config')
         argument.assert_kwargs_empty(kwargs)
 
+    # Check input size consistency with xs and ws here.
+    x_in = xs[0].shape[1]
+    w_in = ws[0][0].shape[1]
+    if x_in != w_in:
+        raise ValueError('Inconsistent input size in input values and weight '
+                         'parameters: {} != {}'.format(x_in, w_in))
+
     xp = backend.get_array_module(hx, hx.data)
 
     if xp is cuda.cupy and chainer.should_use_cudnn('>=auto', 5000):
diff --git a/chainer/functions/connection/n_step_rnn.py b/chainer/functions/connection/n_step_rnn.py
--- a/chainer/functions/connection/n_step_rnn.py
+++ b/chainer/functions/connection/n_step_rnn.py
@@ -633,6 +633,13 @@ def n_step_rnn_base(n_layers, dropout_ratio, hx, ws, bs, xs,
         raise ValueError('Invalid activation: "%s". Please select from [%s]'
                          % (activation, candidate))
 
+    # Check input size consistency with xs and ws.
+    x_in = xs[0].shape[1]
+    w_in = ws[0][0].shape[1]
+    if x_in != w_in:
+        raise ValueError('Inconsistent input size in input values and weight '
+                         'parameters: {} != {}'.format(x_in, w_in))
+
     xp = backend.get_array_module(hx)
 
     if xp is cuda.cupy and chainer.should_use_cudnn('>=auto', 5000):


Test patch:
diff --git a/tests/chainer_tests/functions_tests/connection_tests/test_n_step_lstm.py b/tests/chainer_tests/functions_tests/connection_tests/test_n_step_lstm.py
--- a/tests/chainer_tests/functions_tests/connection_tests/test_n_step_lstm.py
+++ b/tests/chainer_tests/functions_tests/connection_tests/test_n_step_lstm.py
@@ -229,6 +229,45 @@ def test_call_cudnn_backward(self):
         self.check_call_cudnn_backward('never')
         self.check_call_cudnn_backward('auto')
 
+    def check_inconsistent_input_size(
+            self, h_data, c_data, xs_data, ws_data, bs_data):
+        h = _wrap_variable(h_data)
+        c = _wrap_variable(c_data)
+        xs = _wrap_variable(xs_data)
+        ws = _wrap_variable(ws_data)
+        bs = _wrap_variable(bs_data)
+        with self.assertRaises(ValueError):
+            functions.n_step_lstm(
+                self.n_layers, self.dropout, h, c, ws, bs, xs)
+
+    def test_inconsistent_input_size_cpu(self):
+        x_in_size = 4  # inconsistent in_size with that of ws.
+        xs = [numpy.random.uniform(-1, 1, (b, x_in_size)).astype('f')
+              for b in self.batches]
+        self.check_inconsistent_input_size(
+            self.hx, self.cx, xs, self.ws, self.bs)
+
+    def check_inconsistent_input_size_gpu(self, use_cudnn):
+        x_in_size = 4  # inconsistent in_size with that of ws.
+        xs = [numpy.random.uniform(-1, 1, (b, x_in_size)).astype('f')
+              for b in self.batches]
+
+        hx = _to_gpu(self.hx)
+        cx = _to_gpu(self.cx)
+        xs = _to_gpu(xs)
+        ws = _to_gpu(self.ws)
+        bs = _to_gpu(self.bs)
+        with chainer.using_config('use_cudnn', use_cudnn):
+            self.check_inconsistent_input_size(hx, cx, xs, ws, bs)
+
+    @attr.gpu
+    def test_inconsistent_input_size_gpu_cudnn_always(self):
+        self.check_inconsistent_input_size_gpu('always')
+
+    @attr.gpu
+    def test_inconsistent_input_size_gpu_cudnn_never(self):
+        self.check_inconsistent_input_size_gpu('never')
+
 
 class TestNStepBiLSTM(unittest.TestCase):
 
@@ -459,6 +498,45 @@ def test_call_cudnn_backward(self):
         self.check_call_cudnn_backward('never')
         self.check_call_cudnn_backward('auto')
 
+    def check_inconsistent_input_size(
+            self, h_data, c_data, xs_data, ws_data, bs_data):
+        h = _wrap_variable(h_data)
+        c = _wrap_variable(c_data)
+        xs = _wrap_variable(xs_data)
+        ws = _wrap_variable(ws_data)
+        bs = _wrap_variable(bs_data)
+        with self.assertRaises(ValueError):
+            functions.n_step_bilstm(
+                self.n_layers, self.dropout, h, c, ws, bs, xs)
+
+    def test_inconsistent_input_size_cpu(self):
+        x_in_size = 4  # inconsistent in_size with that of ws.
+        xs = [numpy.random.uniform(-1, 1, (b, x_in_size)).astype('f')
+              for b in self.batches]
+        self.check_inconsistent_input_size(
+            self.hx, self.cx, xs, self.ws, self.bs)
+
+    def check_inconsistent_input_size_gpu(self, use_cudnn):
+        x_in_size = 4  # inconsistent in_size with that of ws.
+        xs = [numpy.random.uniform(-1, 1, (b, x_in_size)).astype('f')
+              for b in self.batches]
+
+        hx = _to_gpu(self.hx)
+        cx = _to_gpu(self.cx)
+        xs = _to_gpu(xs)
+        ws = _to_gpu(self.ws)
+        bs = _to_gpu(self.bs)
+        with chainer.using_config('use_cudnn', use_cudnn):
+            self.check_inconsistent_input_size(hx, cx, xs, ws, bs)
+
+    @attr.gpu
+    def test_inconsistent_input_size_gpu_cudnn_always(self):
+        self.check_inconsistent_input_size_gpu('always')
+
+    @attr.gpu
+    def test_inconsistent_input_size_gpu_cudnn_never(self):
+        self.check_inconsistent_input_size_gpu('never')
+
 
 def _stack_weight(ws):
     # TODO(unno): Input of the current LSTM implementation is shuffled
diff --git a/tests/chainer_tests/functions_tests/connection_tests/test_n_step_rnn.py b/tests/chainer_tests/functions_tests/connection_tests/test_n_step_rnn.py
--- a/tests/chainer_tests/functions_tests/connection_tests/test_n_step_rnn.py
+++ b/tests/chainer_tests/functions_tests/connection_tests/test_n_step_rnn.py
@@ -268,6 +268,42 @@ def test_call_cudnn_backward_inference(self):
         self.check_call_cudnn_backward_inference('never')
         self.check_call_cudnn_backward_inference('auto')
 
+    def check_inconsistent_input_size(self, h_data, xs_data, ws_data, bs_data):
+        h = _wrap_variable(h_data)
+        xs = _wrap_variable(xs_data)
+        ws = _wrap_variable(ws_data)
+        bs = _wrap_variable(bs_data)
+        with self.assertRaises(ValueError):
+            functions.n_step_rnn(
+                self.n_layers, self.dropout, h, ws, bs, xs,
+                activation=self.activation)
+
+    def test_inconsistent_input_size_cpu(self):
+        x_in_size = 4  # inconsistent in_size with that of ws.
+        x_shape = [(b, x_in_size) for b in self.batches]
+        xs = _shaped_random(x_shape)
+        self.check_inconsistent_input_size(self.hx, xs, self.ws, self.bs)
+
+    def check_inconsistent_input_size_gpu(self, use_cudnn):
+        x_in_size = 4  # inconsistent in_size with that of ws.
+        x_shape = [(b, x_in_size) for b in self.batches]
+        xs = _shaped_random(x_shape)
+
+        hx = _to_gpu(self.hx)
+        xs = _to_gpu(xs)
+        ws = _to_gpu(self.ws)
+        bs = _to_gpu(self.bs)
+        with chainer.using_config('use_cudnn', use_cudnn):
+            self.check_inconsistent_input_size(hx, xs, ws, bs)
+
+    @attr.gpu
+    def test_inconsistent_input_size_gpu_cudnn_always(self):
+        self.check_inconsistent_input_size_gpu('always')
+
+    @attr.gpu
+    def test_inconsistent_input_size_gpu_cudnn_never(self):
+        self.check_inconsistent_input_size_gpu('never')
+
 
 @testing.parameterize(*testing.product({
     'activation': ['tanh', 'relu']
@@ -496,5 +532,41 @@ def test_call_cudnn_backward(self):
         self.check_call_cudnn_backward('never')
         self.check_call_cudnn_backward('auto')
 
+    def check_inconsistent_input_size(self, h_data, xs_data, ws_data, bs_data):
+        h = _wrap_variable(h_data)
+        xs = _wrap_variable(xs_data)
+        ws = _wrap_variable(ws_data)
+        bs = _wrap_variable(bs_data)
+        with self.assertRaises(ValueError):
+            functions.n_step_birnn(
+                self.n_layers, self.dropout, h, ws, bs, xs,
+                activation=self.activation)
+
+    def test_inconsistent_input_size_cpu(self):
+        x_in_size = 4  # inconsistent in_size with that of ws.
+        x_shape = [(b, x_in_size) for b in self.batches]
+        xs = _shaped_random(x_shape)
+        self.check_inconsistent_input_size(self.hx, xs, self.ws, self.bs)
+
+    def check_inconsistent_input_size_gpu(self, use_cudnn):
+        x_in_size = 4  # inconsistent in_size with that of ws.
+        x_shape = [(b, x_in_size) for b in self.batches]
+        xs = _shaped_random(x_shape)
+
+        hx = _to_gpu(self.hx)
+        xs = _to_gpu(xs)
+        ws = _to_gpu(self.ws)
+        bs = _to_gpu(self.bs)
+        with chainer.using_config('use_cudnn', use_cudnn):
+            self.check_inconsistent_input_size(hx, xs, ws, bs)
+
+    @attr.gpu
+    def test_inconsistent_input_size_gpu_cudnn_always(self):
+        self.check_inconsistent_input_size_gpu('always')
+
+    @attr.gpu
+    def test_inconsistent_input_size_gpu_cudnn_never(self):
+        self.check_inconsistent_input_size_gpu('never')
+
 
 testing.run_module(__name__, __file__)