Repository: dmlc/gluon-nlp

Issue:
Flaky test_weight_drop
```
=================================== FAILURES ===================================
_______________________________ test_weight_drop _______________________________
[gw2] linux -- Python 3.6.7 /var/lib/jenkins/workspace/gluon-nlp-py3/conda/py3/bin/python

    def test_weight_drop():
        class RefBiLSTM(gluon.Block):
            def __init__(self, size, **kwargs):
                super(RefBiLSTM, self).__init__(**kwargs)
                with self.name_scope():
                    self._lstm_fwd = gluon.rnn.LSTM(size, bidirectional=False, prefix='l0')
                    self._lstm_bwd = gluon.rnn.LSTM(size, bidirectional=False, prefix='r0')
    
            def forward(self, inpt):
                fwd = self._lstm_fwd(inpt)
                bwd_inpt = mx.nd.flip(inpt, 0)
                bwd = self._lstm_bwd(bwd_inpt)
                bwd = mx.nd.flip(bwd, 0)
                return mx.nd.concat(fwd, bwd, dim=2)
        net1 = RefBiLSTM(10)
        shared_net1 = RefBiLSTM(10, params=net1.collect_params())
    
        net2 = gluon.rnn.LSTM(10)
        shared_net2 = gluon.rnn.LSTM(10, params=net2.collect_params())
    
        net3 = gluon.nn.HybridSequential()
        net3.add(gluon.rnn.LSTM(10))
        shared_net3 = gluon.nn.HybridSequential(params=net3.collect_params())
        shared_net3.add(gluon.rnn.LSTM(10, params=net3[0].collect_params()))
    
        x = mx.nd.ones((3, 4, 5))
        nets = [(net1, shared_net1),
                (net2, shared_net2),
                (net3, shared_net3)]
        for net, shared_net in nets:
            net.initialize('ones')
            mx.test_utils.assert_almost_equal(net(x).asnumpy(),
                                              shared_net(x).asnumpy())
            with mx.autograd.train_mode():
                mx.test_utils.assert_almost_equal(net(x).asnumpy(),
                                                  shared_net(x).asnumpy())
    
            grads = {}
            with mx.autograd.record():
                y = net(x)
            y.backward()
            for name, param in net.collect_params().items():
                grads[name] = param.grad().copy()
            with mx.autograd.record():
                y = shared_net(x)
            y.backward()
            for name, param in shared_net.collect_params().items():
                mx.test_utils.assert_almost_equal(grads[name].asnumpy(), param.grad().asnumpy())
    
            drop_rate = 0.5
            nlp.model.utils.apply_weight_drop(net, '.*h2h_weight', drop_rate)
            net.initialize('ones')
    
            mx.test_utils.assert_almost_equal(net(x).asnumpy(),
                                              shared_net(x).asnumpy())
            with mx.autograd.train_mode():
>               assert not mx.test_utils.almost_equal(net(x).asnumpy(),
                                                      shared_net(x).asnumpy())
E               AssertionError: assert not True
E                +  where True = <function almost_equal at 0x7ff5eb456268>(array([[[0.7536527 , 0.7536527 , 0.7536527 , 0.7536527 , 0.7536527 ,\n         0.7536527 , 0.7536527 , 0.7536527 , 0.75...3, 0.99498713, 0.9949861 ,\n         0.9949871 , 0.9949858 , 0.99498725, 0.9949858 , 0.9949871 ]]],\n      dtype=float32), array([[[0.7536527 , 0.7536527 , 0.7536527 , 0.7536527 , 0.7536527 ,\n         0.7536527 , 0.7536527 , 0.7536527 , 0.75... , 0.9949869 , 0.9949869 ,\n         0.9949869 , 0.9949869 , 0.9949869 , 0.9949869 , 0.9949869 ]]],\n      dtype=float32))
E                +    where <function almost_equal at 0x7ff5eb456268> = <module 'mxnet.test_utils' from '/var/lib/jenkins/workspace/gluon-nlp-py3/conda/py3/lib/python3.6/site-packages/mxnet/test_utils.py'>.almost_equal
E                +      where <module 'mxnet.test_utils' from '/var/lib/jenkins/workspace/gluon-nlp-py3/conda/py3/lib/python3.6/site-packages/mxnet/test_utils.py'> = mx.test_utils
E                +    and   array([[[0.7536527 , 0.7536527 , 0.7536527 , 0.7536527 , 0.7536527 ,\n         0.7536527 , 0.7536527 , 0.7536527 , 0.75...3, 0.99498713, 0.9949861 ,\n         0.9949871 , 0.9949858 , 0.99498725, 0.9949858 , 0.9949871 ]]],\n      dtype=float32) = <bound method NDArray.asnumpy of \n[[[0.7536527  0.7536527  0.7536527  0.7536527  0.7536527  0.7536527\n   0.7536527  0.... 0.99498713 0.99498713 0.9949861  0.9949871\n   0.9949858  0.99498725 0.9949858  0.9949871 ]]]\n<NDArray 3x4x10 @cpu(0)>>()
E                +      where <bound method NDArray.asnumpy of \n[[[0.7536527  0.7536527  0.7536527  0.7536527  0.7536527  0.7536527\n   0.7536527  0.... 0.99498713 0.99498713 0.9949861  0.9949871\n   0.9949858  0.99498725 0.9949858  0.9949871 ]]]\n<NDArray 3x4x10 @cpu(0)>> = \n[[[0.7536527  0.7536527  0.7536527  0.7536527  0.7536527  0.7536527\n   0.7536527  0.7536527  0.7536527  0.7536527 ]\n ...  0.99498713 0.99498713 0.9949861  0.9949871\n   0.9949858  0.99498725 0.9949858  0.9949871 ]]]\n<NDArray 3x4x10 @cpu(0)>.asnumpy
E                +        where \n[[[0.7536527  0.7536527  0.7536527  0.7536527  0.7536527  0.7536527\n   0.7536527  0.7536527  0.7536527  0.7536527 ]\n ...  0.99498713 0.99498713 0.9949861  0.9949871\n   0.9949858  0.99498725 0.9949858  0.9949871 ]]]\n<NDArray 3x4x10 @cpu(0)> = HybridSequential(\n  (0): LSTM(5 -> 10, TNC)\n)(\n[[[1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]]\n\n [[1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]\n ...1. 1. 1. 1. 1.]]\n\n [[1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]]]\n<NDArray 3x4x5 @cpu(0)>)
E                +    and   array([[[0.7536527 , 0.7536527 , 0.7536527 , 0.7536527 , 0.7536527 ,\n         0.7536527 , 0.7536527 , 0.7536527 , 0.75... , 0.9949869 , 0.9949869 ,\n         0.9949869 , 0.9949869 , 0.9949869 , 0.9949869 , 0.9949869 ]]],\n      dtype=float32) = <bound method NDArray.asnumpy of \n[[[0.7536527  0.7536527  0.7536527  0.7536527  0.7536527  0.7536527\n   0.7536527  0.... 0.9949869  0.9949869  0.9949869  0.9949869\n   0.9949869  0.9949869  0.9949869  0.9949869 ]]]\n<NDArray 3x4x10 @cpu(0)>>()
E                +      where <bound method NDArray.asnumpy of \n[[[0.7536527  0.7536527  0.7536527  0.7536527  0.7536527  0.7536527\n   0.7536527  0.... 0.9949869  0.9949869  0.9949869  0.9949869\n   0.9949869  0.9949869  0.9949869  0.9949869 ]]]\n<NDArray 3x4x10 @cpu(0)>> = \n[[[0.7536527  0.7536527  0.7536527  0.7536527  0.7536527  0.7536527\n   0.7536527  0.7536527  0.7536527  0.7536527 ]\n ...  0.9949869  0.9949869  0.9949869  0.9949869\n   0.9949869  0.9949869  0.9949869  0.9949869 ]]]\n<NDArray 3x4x10 @cpu(0)>.asnumpy
E                +        where \n[[[0.7536527  0.7536527  0.7536527  0.7536527  0.7536527  0.7536527\n   0.7536527  0.7536527  0.7536527  0.7536527 ]\n ...  0.9949869  0.9949869  0.9949869  0.9949869\n   0.9949869  0.9949869  0.9949869  0.9949869 ]]]\n<NDArray 3x4x10 @cpu(0)> = HybridSequential(\n  (0): LSTM(5 -> 10, TNC)\n)(\n[[[1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]]\n\n [[1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]\n ...1. 1. 1. 1. 1.]]\n\n [[1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]\n  [1. 1. 1. 1. 1.]]]\n<NDArray 3x4x5 @cpu(0)>)

tests/unittest/test_models.py:408: AssertionError
```

http://ci.mxnet.io/blue/rest/organizations/jenkins/pipelines/gluon-nlp/branches/PR-462/runs/3/nodes/23/steps/46/log/?start=0


Discussion:
@szha I think you added the test

Resolving patch:
diff --git a/src/gluonnlp/model/parameter.py b/src/gluonnlp/model/parameter.py
--- a/src/gluonnlp/model/parameter.py
+++ b/src/gluonnlp/model/parameter.py
@@ -40,6 +40,7 @@ class WeightDropParameter(gluon.Parameter):
     """
     def __init__(self, parameter, rate=0.0, mode='training', axes=()):
         p = parameter
+        self._deferred_init = p._deferred_init
         super(WeightDropParameter, self).__init__(
             name=p.name, grad_req=p.grad_req, shape=p._shape, dtype=p.dtype,
             lr_mult=p.lr_mult, wd_mult=p.wd_mult, init=p.init,
@@ -48,6 +49,12 @@ def __init__(self, parameter, rate=0.0, mode='training', axes=()):
         self._rate = rate
         self._mode = mode
         self._axes = axes
+        self._var = p._var
+        self._data = p._data
+        self._grad = p._grad
+        self._ctx_list = p._ctx_list
+        self._ctx_map = p._ctx_map
+        self._trainer = p._trainer
 
     def data(self, ctx=None):
         """Returns a copy of this parameter on one context. Must have been


Test patch:
diff --git a/tests/unittest/test_models.py b/tests/unittest/test_models.py
--- a/tests/unittest/test_models.py
+++ b/tests/unittest/test_models.py
@@ -98,7 +98,7 @@ def test_pretrained_bert_models():
                   'bert_24_1024_16': ['book_corpus_wiki_en_uncased','book_corpus_wiki_en_cased']}
     vocab_size = {'book_corpus_wiki_en_cased': 28996,
                   'book_corpus_wiki_en_uncased': 30522,
-                  'wiki_multilingual_cased': 119547, 
+                  'wiki_multilingual_cased': 119547,
                   'wiki_cn': 21128,
                   'wiki_multilingual': 105879}
     special_tokens = ['[UNK]', '[PAD]', '[SEP]', '[CLS]', '[MASK]']
@@ -382,12 +382,12 @@ def forward(self, inpt):
     shared_net3 = gluon.nn.HybridSequential(params=net3.collect_params())
     shared_net3.add(gluon.rnn.LSTM(10, params=net3[0].collect_params()))
 
-    x = mx.nd.ones((3, 4, 5))
+    x = mx.random.uniform(shape=(3, 4, 5))
     nets = [(net1, shared_net1),
             (net2, shared_net2),
             (net3, shared_net3)]
     for net, shared_net in nets:
-        net.initialize('ones')
+        net.initialize('uniform')
         mx.test_utils.assert_almost_equal(net(x).asnumpy(),
                                           shared_net(x).asnumpy())
         with mx.autograd.train_mode():
@@ -408,10 +408,10 @@ def forward(self, inpt):
 
         drop_rate = 0.5
         nlp.model.utils.apply_weight_drop(net, '.*h2h_weight', drop_rate)
-        net.initialize('ones')
 
-        mx.test_utils.assert_almost_equal(net(x).asnumpy(),
-                                          shared_net(x).asnumpy())
+        with mx.autograd.predict_mode():
+            mx.test_utils.assert_almost_equal(net(x).asnumpy(),
+                                              shared_net(x).asnumpy())
         with mx.autograd.train_mode():
             assert not mx.test_utils.almost_equal(net(x).asnumpy(),
                                                   shared_net(x).asnumpy())