Repository: pytorch/pytorch

Issue:
weight initializations for conv2d and linear
Allow different weight initialization schemes for conv2d and linear.
See: https://github.com/Kaixhin/nninit for some schemes.



Discussion:
I thought to add another class called `Initializers` with a function that we could pass the weights and a string with the `initialization` name. Also add a string argument to every layer called "initialization".

In a Linear layer the init should be something like this:
```python
def __init__(self, in_features, out_features, bias=True, initializer=None):
    super(Linear, self).__init__()
    self.in_features = in_features
    self.out_features = out_features
    self.weight = Parameter(torch.Tensor(out_features, in_features))
    Initializers.initialize(self.weight, initializer)
    if bias:
        self.bias = Parameter(torch.Tensor(out_features))
    else:
        self.register_parameter('bias', None)
    self.reset_parameters()
```

Do you think that this is the best approach? If not, what would be the best way to do this?
I think it's better to pass in a function as the initializer and call it with weights as argument. It's clean, verbose, and you don't have to do error checking for invalid/incompatible strings. For example:

```python
nn.Linear(20, 20, weight_init=nn.init.orthogonal)
```

Do you want to implement it, or was your comment only a suggestion?
Chainer has a nice approach, where the initializer argument is a little polymorphic (can be a callable or a scalar/matrix value). Looks like this:
```python
super().__init__(
    maxout=Maxout(maxout_size, maxout_size, 2, initialW=initializers.GlorotUniform()),
    softmax_linear=Linear(maxout_size, task.num_classes, initialW=0))
```
@apaszke, I am thinking to implement it.

I think that String-like is cleaner. But we could go the way Chainer implements it, as shown by @jekbradbury. With strings or class names being accepted.

What do you guys think?

Resolving patch:
diff --git a/torch/nn/init.py b/torch/nn/init.py
--- a/torch/nn/init.py
+++ b/torch/nn/init.py
@@ -0,0 +1,259 @@
+import math
+import random
+
+import torch
+from torch.autograd import Variable
+
+
+def uniform(tensor, a=0, b=1):
+    """Fills the input Tensor or Variable with values drawn from a uniform U(a,b)
+
+    Args:
+        tensor: a n-dimension torch.Tensor
+        a: the lower bound of the uniform distribution
+        b: the upper bound of the uniform distribution
+
+    Examples:
+        >>> w = torch.Tensor(3, 5)
+        >>> nninit.uniform(w)
+    """
+    if isinstance(tensor, Variable):
+        uniform(tensor.data, a=a, b=b)
+        return tensor
+    return tensor.uniform_(a, b)
+
+
+def normal(tensor, mean=0, std=1):
+    """Fills the input Tensor or Variable with values drawn from a normal distribution with the given mean and std
+
+    Args:
+        tensor: a n-dimension torch.Tensor
+        mean: the mean of the normal distribution
+        std: the standard deviation of the normal distribution
+
+    Examples:
+        >>> w = torch.Tensor(3, 5)
+        >>> nninit.normal(w)
+    """
+    if isinstance(tensor, Variable):
+        normal(tensor.data, mean=mean, std=std)
+        return tensor
+    return tensor.normal_(mean, std)
+
+
+def constant(tensor, val):
+    """Fills the input Tensor or Variable with the value `val`
+
+    Args:
+        tensor: a n-dimension torch.Tensor
+        val: the value to fill the tensor with
+
+    Examples:
+        >>> w = torch.Tensor(3, 5)
+        >>> nninit.constant(w)
+    """
+    if isinstance(tensor, Variable):
+        constant(tensor.data, val)
+        return tensor
+    return tensor.fill_(val)
+
+
+def _calculate_fan_in_and_fan_out(tensor):
+    if tensor.ndimension() < 2:
+        raise ValueError("fan in and fan out can not be computed for tensor of size ", tensor.size())
+
+    if tensor.ndimension() == 2:  # Linear
+        fan_in = tensor.size(1)
+        fan_out = tensor.size(0)
+    else:
+        num_input_fmaps = tensor.size(1)
+        num_output_fmaps = tensor.size(0)
+        receptive_field_size = 1
+        if tensor.dim() > 2:
+            receptive_field_size = tensor[0][0].numel()
+        fan_in = num_input_fmaps * receptive_field_size
+        fan_out = num_output_fmaps * receptive_field_size
+
+    return fan_in, fan_out
+
+
+def xavier_uniform(tensor, gain=1):
+    """Fills the input Tensor or Variable with values according to the method described in "Understanding the difficulty
+       of training deep feedforward neural networks" - Glorot, X. and Bengio, Y., using a uniform distribution.
+
+       The resulting tensor will have values sampled from U(-a, a) where a = gain * sqrt(2/(fan_in + fan_out)) * sqrt(3)
+
+    Args:
+        tensor: a n-dimension torch.Tensor
+        gain: an optional scaling factor to be applied
+
+    Examples:
+        >>> w = torch.Tensor(3, 5)
+        >>> nninit.xavier_uniform(w, gain=math.sqrt(2.0))
+    """
+    if isinstance(tensor, Variable):
+        xavier_uniform(tensor.data, gain=gain)
+        return tensor
+
+    fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
+    std = gain * math.sqrt(2.0 / (fan_in + fan_out))
+    a = math.sqrt(3.0) * std
+    return tensor.uniform_(-a, a)
+
+
+def xavier_normal(tensor, gain=1):
+    """Fills the input Tensor or Variable with values according to the method described in "Understanding the difficulty
+       of training deep feedforward neural networks" - Glorot, X. and Bengio, Y., using a normal distribution.
+
+       The resulting tensor will have values sampled from normal distribution with mean=0 and
+       std = gain * sqrt(2/(fan_in + fan_out))
+
+    Args:
+        tensor: a n-dimension torch.Tensor
+        gain: an optional scaling factor to be applied
+
+    Examples:
+        >>> w = torch.Tensor(3, 5)
+        >>> nninit.xavier_normal(w)
+    """
+    if isinstance(tensor, Variable):
+        xavier_normal(tensor.data, gain=gain)
+        return tensor
+
+    fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
+    std = gain * math.sqrt(2.0 / (fan_in + fan_out))
+    return tensor.normal_(0, std)
+
+
+def _calculate_correct_fan(tensor, mode):
+    mode = mode.lower()
+    valid_modes = ['fan_in', 'fan_out']
+    if mode not in valid_modes:
+        raise ValueError("mode {} not supported, please use one of {}".format(mode, valid_modes))
+
+    fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
+    if mode == 'fan_in':
+        return fan_in
+    else:
+        return fan_out
+
+
+def kaiming_uniform(tensor, a=0, mode='fan_in'):
+    """Fills the input Tensor or Variable with values according to the method described in "Delving deep into rectifiers:
+       Surpassing human-level performance on ImageNet classification" - He, K. et al using a uniform distribution.
+
+       The resulting tensor will have values sampled from U(-bound, bound)
+       where bound = sqrt(2/((1 + a^2) * fan_in)) * sqrt(3)
+
+    Args:
+        tensor: a n-dimension torch.Tensor
+        a: the coefficient of the slope of the rectifier used after this layer (0 for ReLU by default)
+        mode: either 'fan_in' (default) or 'fan_out'. Choosing `fan_in` preserves the magnitude of the variance of the
+              weights in the forward pass. Choosing `fan_out` preserves the magnitudes in the backwards pass.
+
+    Examples:
+        >>> w = torch.Tensor(3, 5)
+        >>> nninit.kaiming_uniform(w, mode='fan_in')
+    """
+    if isinstance(tensor, Variable):
+        kaiming_uniform(tensor.data, a=a, mode=mode)
+        return tensor
+
+    fan = _calculate_correct_fan(tensor, mode)
+    std = math.sqrt(2.0 / ((1 + a ** 2) * fan))
+    bound = math.sqrt(3.0) * std
+    return tensor.uniform_(-bound, bound)
+
+
+def kaiming_normal(tensor, a=0, mode='fan_in'):
+    """Fills the input Tensor or Variable with values according to the method described in "Delving deep into rectifiers:
+       Surpassing human-level performance on ImageNet classification" - He, K. et al using a normal distribution.
+
+       The resulting tensor will have values sampled from normal distribution with mean=0 and
+       std = sqrt(2/((1 + a^2) * fan_in))
+
+    Args:
+        tensor: a n-dimension torch.Tensor
+        a: the coefficient of the slope of the rectifier used after this layer (0 for ReLU by default)
+        mode: either 'fan_in' (default) or 'fan_out'. Choosing `fan_in` preserves the magnitude of the variance of the
+              weights in the forward pass. Choosing `fan_out` preserves the magnitudes in the backwards pass.
+
+    Examples:
+        >>> w = torch.Tensor(3, 5)
+        >>> nninit.kaiming_normal(w, mode='fan_out')
+    """
+    if isinstance(tensor, Variable):
+        kaiming_normal(tensor.data, a=a, mode=mode)
+        return tensor
+
+    fan = _calculate_correct_fan(tensor, mode)
+    std = math.sqrt(2.0 / ((1 + a ** 2) * fan))
+    return tensor.normal_(0, std)
+
+
+def orthogonal(tensor, gain=1):
+    """Fills the input Tensor or Variable with a (semi) orthogonal matrix. The input tensor must have at least 2 dimensions,
+       and for tensors with more than 2 dimensions the trailing dimensions are flattened. viewed as 2D representation
+       with rows equal to the first dimension and columns equal to the product of  as a sparse matrix, where the
+       non-zero elements will be drawn from a normal distribution with mean=0 and std=`std`.
+       Reference: "Exact solutions to the nonlinear dynamics of learning in deep linear neural networks"-Saxe, A. et al.
+
+    Args:
+        tensor: a n-dimension torch.Tensor, where n >= 2
+        gain: optional gain to be applied
+
+    Examples:
+        >>> w = torch.Tensor(3, 5)
+        >>> nninit.orthogonal(w)
+    """
+    if isinstance(tensor, Variable):
+        orthogonal(tensor.data, gain=gain)
+        return tensor
+
+    if tensor.ndimension() < 2:
+        raise ValueError("Only tensors with 2 or more dimensions are supported.")
+    rows = tensor.size(0)
+    cols = tensor[0].numel()
+    flattened = torch.Tensor(rows, cols).normal_(0, 1)
+
+    u, s, v = torch.svd(flattened, some=True)
+    if u.is_same_size(flattened):
+        tensor.view_as(u).copy_(u)
+    else:
+        tensor.view_as(v.t()).copy_(v.t())
+
+    tensor.mul_(gain)
+    return tensor
+
+
+def sparse(tensor, sparsity, std=0.01):
+    """Fills the 2D input Tensor or Variable as a sparse matrix, where the non-zero elements will be drawn from a
+       normal distribution with mean=0 and std=`std`.
+
+    Args:
+        tensor: a n-dimension torch.Tensor
+        sparsity: The fraction of elements in each column to be set to zero
+        std: the standard deviation of the normal distribution used to generate the non-zero values
+
+    Examples:
+        >>> w = torch.Tensor(3, 5)
+        >>> nninit.sparse(w, sparsity=0.1)
+    """
+    if isinstance(tensor, Variable):
+        sparse(tensor.data, sparsity, std=std)
+        return tensor
+
+    if tensor.ndimension() != 2:
+        raise ValueError("Sparse initialization only supported for 2D inputs")
+    tensor.normal_(0, std)
+    rows, cols = tensor.size(0), tensor.size(1)
+    num_zeros = int(math.ceil(cols * sparsity))
+
+    for col_idx in range(tensor.size(1)):
+        row_indices = list(range(rows))
+        random.shuffle(row_indices)
+        zero_indices = row_indices[:num_zeros]
+        for row_idx in zero_indices:
+            tensor[row_idx, col_idx] = 0
+
+    return tensor


Test patch:
diff --git a/test/common.py b/test/common.py
--- a/test/common.py
+++ b/test/common.py
@@ -3,6 +3,7 @@
 import argparse
 import unittest
 import contextlib
+from functools import wraps
 from itertools import product
 from copy import deepcopy
 
@@ -31,6 +32,24 @@ def run_tests():
 except ImportError:
     TEST_NUMPY = False
 
+TEST_SCIPY = True
+try:
+    import scipy
+except ImportError:
+    TEST_SCIPY = False
+
+
+def skipIfNoLapack(fn):
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        try:
+            fn(*args, **kwargs)
+        except Exception as e:
+            if 'Lapack library not found' in e.args[0]:
+                raise unittest.SkipTest('Compiled without Lapack')
+            raise
+    return wrapper
+
 
 def get_cpu_type(t):
     assert t.__module__ == 'torch.cuda'
diff --git a/test/test_nn.py b/test/test_nn.py
--- a/test/test_nn.py
+++ b/test/test_nn.py
@@ -1,24 +1,29 @@
 import math
-import torch
 import random
 import unittest
 import itertools
 import contextlib
 from copy import deepcopy
 from itertools import repeat, product
-from functools import wraps
+from functools import wraps, reduce
+from operator import mul
 
+import torch
 import torch.nn as nn
 import torch.nn.functional as F
 import torch.nn.parallel as dp
+import torch.nn.init as init
 import torch.nn.utils.rnn as rnn_utils
 from torch.nn.utils import clip_grad_norm
 from torch.autograd import Variable
 from torch.nn import Parameter
 from common_nn import NNTestCase, ModuleTest, CriterionTest, TestBase, \
     module_tests, criterion_tests, TEST_CUDA, TEST_MULTIGPU, TEST_CUDNN, \
-    TEST_CUDNN_VERSION, PRECISION
-from common import freeze_rng_state, run_tests
+    TEST_CUDNN_VERSION
+from common import freeze_rng_state, run_tests, TestCase, skipIfNoLapack, TEST_SCIPY
+
+if TEST_SCIPY:
+    from scipy import stats
 
 
 def default_tensor_type(type):
@@ -33,12 +38,13 @@ def wrapper(*args, **kwargs):
                 return fn(*args, **kwargs)
             finally:
                 torch.set_default_tensor_type(old_type)
+
         return wrapper
+
     return decorator
 
 
 class InputVariableMixin(object):
-
     def _get_input(self):
         input = TestBase._get_input(self)
 
@@ -49,11 +55,11 @@ def map_variables(i):
                 return Variable(i, requires_grad=True)
             else:
                 return type(i)(map_variables(elem) for elem in i)
+
         return map_variables(input)
 
 
 class NewModuleTest(InputVariableMixin, ModuleTest):
-
     def __init__(self, *args, **kwargs):
         super(NewModuleTest, self).__init__(*args, **kwargs)
         self.cudnn = kwargs.get('cudnn', False)
@@ -169,7 +175,6 @@ def _get_target(self, target):
 
 
 class TestNN(NNTestCase):
-
     def _forward(self, module, input):
         with freeze_rng_state():
             return module(input)
@@ -405,12 +410,12 @@ def num_params(module):
             return len(list(module.parameters()))
 
         class Net(nn.Module):
-
             def __init__(self):
                 super(Net, self).__init__()
                 self.l1 = l
                 self.l2 = l
                 self.param = Parameter(torch.Tensor(3, 5))
+
         l = nn.Linear(10, 20)
         n = Net()
         s = nn.Sequential(n, n, n, n)
@@ -420,12 +425,12 @@ def __init__(self):
 
     def test_modules(self):
         class Net(nn.Module):
-
             def __init__(self):
                 super(Net, self).__init__()
                 self.l1 = l
                 self.l2 = l
                 self.param = Variable(torch.Tensor(3, 5))
+
         l = nn.Linear(10, 20)
         n = Net()
         s = nn.Sequential(n, n, n, n)
@@ -454,6 +459,7 @@ def check():
                 self.assertIs(m1, m2)
             for i in range(len(modules)):
                 self.assertIs(module_list[i], modules[i])
+
         check()
         modules += [nn.Conv2d(3, 4, 3)]
         module_list += [modules[-1]]
@@ -488,6 +494,7 @@ def check():
                 self.assertIs(p1, p2)
             for i in range(len(parameters)):
                 self.assertIs(parameters[i], param_list[i])
+
         check()
         parameters += [make_param()]
         param_list += [parameters[-1]]
@@ -551,6 +558,7 @@ def test_non_leaf_parameters(self):
 
         def assign_weight():
             l2.weight = l1.weight + 2
+
         self.assertRaises(TypeError, assign_weight)
         # This should work though
         l2.weight = Parameter(torch.randn(10, 10))
@@ -841,9 +849,9 @@ def fn(input):
             return [input, (input.sin(), input.cos(), [input.add(1)]), input]
 
         class Net(nn.Module):
-
             def forward(self, input):
                 return fn(input)
+
         i = Variable(torch.randn(2, 2).float().cuda(1))
         gpus = range(torch.cuda.device_count())
         output = dp.data_parallel(Net(), i, gpus)
@@ -862,9 +870,9 @@ def fn(input):
             return input[1][0]
 
         class Net(nn.Module):
-
             def forward(self, *input):
                 return fn(input)
+
         i = Variable(torch.randn(20, 3).float().cuda(1))
         input = (i.cos(), (i.sin(), i), i.sin())
         gpus = range(torch.cuda.device_count())
@@ -956,6 +964,7 @@ def test_parameter_assignment(self):
 
         def num_params():
             return len(list(l.parameters()))
+
         self.assertEqual(num_params(), 2)
 
         new_param = Parameter(torch.randn(5, 5))
@@ -977,6 +986,7 @@ def num_params():
         # It shouldn't be possible to replace a parameter with a Variable
         def assign_var():
             l.param_attr = Variable(torch.Tensor(5, 5))
+
         self.assertRaises(TypeError, assign_var)
         # But replacing it with None should be fine
         l.param_attr = None
@@ -1072,12 +1082,10 @@ def test_MaxUnpool2d_output_size(self):
                         size = torch.LongStorage((1, 1) + size)
                     mu(output_small, indices_small, output_size=size)
                 else:
-                    self.assertRaises(ValueError, lambda:
-                                      mu(output_small, indices_small, (h, w)))
+                    self.assertRaises(ValueError, lambda: mu(output_small, indices_small, (h, w)))
 
     def test_container_copy(self):
         class Model(nn.Module):
-
             def __init__(self):
                 super(Model, self).__init__()
                 self.linear = nn.Linear(4, 5)
@@ -1567,6 +1575,277 @@ def test_batchnorm_eval(self):
             self.assertEqual(grad1, grad2)
 
 
+class TestNNInit(TestCase):
+    def setUp(self):
+        random.seed(123)
+        torch.manual_seed(123)
+
+    def _is_normal(self, tensor, mean, std):
+        if isinstance(tensor, Variable):
+            tensor = tensor.data
+        samples = list(tensor.view(-1))
+        p_value = stats.kstest(samples, 'norm', args=(mean, std)).pvalue
+        return p_value > 0.0001
+
+    def _is_uniform(self, tensor, a, b):
+        if isinstance(tensor, Variable):
+            tensor = tensor.data
+        samples = list(tensor.view(-1))
+        p_value = stats.kstest(samples, 'uniform', args=(a, (b - a))).pvalue
+        return p_value > 0.0001
+
+    def _create_random_nd_tensor(self, dims, size_min, size_max, as_variable):
+        size = [random.randint(size_min, size_max) for _ in range(dims)]
+        tensor = torch.zeros(size)
+        if as_variable:
+            tensor = Variable(tensor)
+        return tensor
+
+    def _random_float(self, a, b):
+        return (b - a) * random.random() + a
+
+    @unittest.skipIf(not TEST_SCIPY, "Scipy not found.")
+    def test_uniform(self):
+        for as_variable in [True, False]:
+            for dims in [1, 2, 4]:
+                input_tensor = self._create_random_nd_tensor(dims, size_min=30, size_max=50, as_variable=as_variable)
+                a = self._random_float(-3, 3)
+                b = a + self._random_float(1, 5)
+                init.uniform(input_tensor, a=a, b=b)
+                assert self._is_uniform(input_tensor, a, b)
+
+    @unittest.skipIf(not TEST_SCIPY, "Scipy not found.")
+    def test_normal(self):
+        for as_variable in [True, False]:
+            for dims in [1, 2, 4]:
+                input_tensor = self._create_random_nd_tensor(dims, size_min=30, size_max=50, as_variable=as_variable)
+                mean = self._random_float(-3, 3)
+                std = self._random_float(1, 5)
+                init.normal(input_tensor, mean=mean, std=std)
+
+                assert self._is_normal(input_tensor, mean, std)
+
+    def test_constant(self):
+        for as_variable in [True, False]:
+            for dims in [1, 2, 4]:
+                input_tensor = self._create_random_nd_tensor(dims, size_min=1, size_max=5, as_variable=as_variable)
+                val = self._random_float(1, 10)
+                init.constant(input_tensor, val)
+                if as_variable:
+                    input_tensor = input_tensor.data
+
+                self.assertEqual(input_tensor, input_tensor.clone().fill_(val))
+
+    def test_xavier_uniform_errors_on_inputs_smaller_than_2d(self):
+        for as_variable in [True, False]:
+            for dims in [0, 1]:
+                tensor = self._create_random_nd_tensor(dims, size_min=1, size_max=1, as_variable=as_variable)
+                with self.assertRaises(ValueError):
+                    init.xavier_uniform(tensor)
+
+    def test_xavier_normal_errors_on_inputs_smaller_than_2d(self):
+        for as_variable in [True, False]:
+            for dims in [0, 1]:
+                tensor = self._create_random_nd_tensor(dims, size_min=1, size_max=1, as_variable=as_variable)
+                with self.assertRaises(ValueError):
+                    init.xavier_normal(tensor)
+
+    @unittest.skipIf(not TEST_SCIPY, "Scipy not found.")
+    def test_xavier_uniform(self):
+        for as_variable in [True, False]:
+            for use_gain in [True, False]:
+                for dims in [2, 4]:
+                    input_tensor = self._create_random_nd_tensor(dims, size_min=20, size_max=25,
+                                                                 as_variable=as_variable)
+                    gain = 1
+
+                    if use_gain:
+                        gain = self._random_float(0.1, 2)
+                        init.xavier_uniform(input_tensor, gain=gain)
+                    else:
+                        init.xavier_uniform(input_tensor)
+
+                    if as_variable:
+                        input_tensor = input_tensor.data
+
+                    fan_in = input_tensor.size(1)
+                    fan_out = input_tensor.size(0)
+                    if input_tensor.dim() > 2:
+                        fan_in *= input_tensor[0, 0].numel()
+                        fan_out *= input_tensor[0, 0].numel()
+
+                    expected_std = gain * math.sqrt(2.0 / (fan_in + fan_out))
+                    bounds = expected_std * math.sqrt(3)
+                    assert self._is_uniform(input_tensor, -bounds, bounds)
+
+    @unittest.skipIf(not TEST_SCIPY, "Scipy not found.")
+    def test_xavier_normal(self):
+        for as_variable in [True, False]:
+            for use_gain in [True, False]:
+                for dims in [2, 4]:
+                    input_tensor = self._create_random_nd_tensor(dims, size_min=20, size_max=25,
+                                                                 as_variable=as_variable)
+                    gain = 1
+
+                    if use_gain:
+                        gain = self._random_float(0.1, 2)
+                        init.xavier_normal(input_tensor, gain=gain)
+                    else:
+                        init.xavier_normal(input_tensor)
+
+                    if as_variable:
+                        input_tensor = input_tensor.data
+
+                    fan_in = input_tensor.size(1)
+                    fan_out = input_tensor.size(0)
+                    if input_tensor.dim() > 2:
+                        fan_in *= input_tensor[0, 0].numel()
+                        fan_out *= input_tensor[0, 0].numel()
+
+                    expected_std = gain * math.sqrt(2.0 / (fan_in + fan_out))
+                    assert self._is_normal(input_tensor, 0, expected_std)
+
+    def test_kaiming_uniform_errors_on_inputs_smaller_than_2d(self):
+        for as_variable in [True, False]:
+            for dims in [0, 1]:
+                with self.assertRaises(ValueError):
+                    tensor = self._create_random_nd_tensor(dims, size_min=1, size_max=1, as_variable=as_variable)
+                    init.kaiming_uniform(tensor)
+
+    def test_kaiming_normal_errors_on_inputs_smaller_than_2d(self):
+        for as_variable in [True, False]:
+            for dims in [0, 1]:
+                with self.assertRaises(ValueError):
+                    tensor = self._create_random_nd_tensor(dims, size_min=1, size_max=1, as_variable=as_variable)
+                    init.kaiming_normal(tensor)
+
+    @unittest.skipIf(not TEST_SCIPY, "Scipy not found.")
+    def test_kaiming_uniform(self):
+        for as_variable in [True, False]:
+            for use_a in [True, False]:
+                for dims in [2, 4]:
+                    for mode in ['fan_in', 'fan_out']:
+                        input_tensor = self._create_random_nd_tensor(dims, size_min=20, size_max=25,
+                                                                     as_variable=as_variable)
+                        if use_a:
+                            a = self._random_float(0.1, 2)
+                            init.kaiming_uniform(input_tensor, a=a, mode=mode)
+                        else:
+                            a = 0
+                            init.kaiming_uniform(input_tensor, mode=mode)
+
+                        if as_variable:
+                            input_tensor = input_tensor.data
+
+                        fan_in = input_tensor.size(1)
+                        fan_out = input_tensor.size(0)
+                        if input_tensor.dim() > 2:
+                            fan_in *= input_tensor[0, 0].numel()
+                            fan_out *= input_tensor[0, 0].numel()
+
+                        if mode == 'fan_in':
+                            n = fan_in
+                        else:
+                            n = fan_out
+
+                        expected_std = math.sqrt(2.0 / ((1 + a**2) * n))
+                        bounds = expected_std * math.sqrt(3.0)
+                        assert self._is_uniform(input_tensor, -bounds, bounds)
+
+    @unittest.skipIf(not TEST_SCIPY, "Scipy not found.")
+    def test_kaiming_normal(self):
+        for as_variable in [True, False]:
+            for use_a in [True, False]:
+                for dims in [2, 4]:
+                    for mode in ['fan_in', 'fan_out']:
+                        input_tensor = self._create_random_nd_tensor(dims, size_min=20, size_max=25,
+                                                                     as_variable=as_variable)
+                        if use_a:
+                            a = self._random_float(0.1, 2)
+                            init.kaiming_normal(input_tensor, a=a, mode=mode)
+                        else:
+                            a = 0
+                            init.kaiming_normal(input_tensor, mode=mode)
+
+                        if as_variable:
+                            input_tensor = input_tensor.data
+
+                        fan_in = input_tensor.size(1)
+                        fan_out = input_tensor.size(0)
+                        if input_tensor.dim() > 2:
+                            fan_in *= input_tensor[0, 0].numel()
+                            fan_out *= input_tensor[0, 0].numel()
+
+                        if mode == 'fan_in':
+                            n = fan_in
+                        else:
+                            n = fan_out
+
+                        expected_std = math.sqrt(2.0 / ((1 + a**2) * n))
+                        assert self._is_normal(input_tensor, 0, expected_std)
+
+    def test_sparse_only_works_on_2d_inputs(self):
+        for as_variable in [True, False]:
+            for dims in [1, 3]:
+                with self.assertRaises(ValueError):
+                    sparsity = self._random_float(0.1, 0.9)
+                    tensor = self._create_random_nd_tensor(dims, size_min=1, size_max=3, as_variable=as_variable)
+                    init.sparse(tensor, sparsity)
+
+    @unittest.skipIf(not TEST_SCIPY, "Scipy not found.")
+    def test_sparse_default_std(self):
+        for as_variable in [True, False]:
+            for use_random_std in [True, False]:
+                input_tensor = self._create_random_nd_tensor(2, size_min=30, size_max=35, as_variable=as_variable)
+                rows, cols = input_tensor.size(0), input_tensor.size(1)
+                sparsity = self._random_float(0.1, 0.2)
+
+                std = 0.01  # default std
+                if use_random_std:
+                    std = self._random_float(0.01, 0.2)
+                    init.sparse(input_tensor, sparsity=sparsity, std=std)
+                else:
+                    init.sparse(input_tensor, sparsity=sparsity)
+
+                if as_variable:
+                    input_tensor = input_tensor.data
+
+                for col_idx in range(input_tensor.size(1)):
+                    column = input_tensor[:, col_idx]
+                    assert column[column == 0].nelement() >= math.ceil(sparsity * cols)
+
+                assert self._is_normal(input_tensor[input_tensor != 0], 0, std)
+
+    @skipIfNoLapack
+    def test_orthogonal(self):
+        for as_variable in [True, False]:
+            for use_gain in [True, False]:
+                for tensor_size in [[3, 4], [4, 3], [20, 2, 3, 4], [2, 3, 4, 5]]:
+                    input_tensor = torch.zeros(tensor_size)
+                    gain = 1.0
+
+                    if as_variable:
+                        input_tensor = Variable(input_tensor)
+
+                    if use_gain:
+                        gain = self._random_float(0.1, 2)
+                        init.orthogonal(input_tensor, gain=gain)
+                    else:
+                        init.orthogonal(input_tensor)
+
+                    if as_variable:
+                        input_tensor = input_tensor.data
+
+                    rows, cols = tensor_size[0], reduce(mul, tensor_size[1:])
+                    flattened_tensor = input_tensor.view(rows, cols)
+                    if rows > cols:
+                        self.assertEqual(torch.mm(flattened_tensor.t(), flattened_tensor),
+                                         torch.eye(cols) * gain ** 2, prec=1e-6)
+                    else:
+                        self.assertEqual(torch.mm(flattened_tensor, flattened_tensor.t()),
+                                         torch.eye(rows) * gain ** 2, prec=1e-6)
+
+
 def add_test(test):
     test_name = test.get_name()
     cuda_test_name = test_name + '_cuda'
@@ -1957,7 +2236,6 @@ def add_test(test):
     ),
 ]
 
-
 for test_params in module_tests + new_module_tests:
     # TODO: CUDA is not implemented yet
     if 'constructor' not in test_params:
@@ -1973,7 +2251,6 @@ def add_test(test):
 
 
 class UnpoolingNet(nn.Module):
-
     def __init__(self, pool, unpool):
         super(UnpoolingNet, self).__init__()
         self.pool = pool
@@ -2002,6 +2279,5 @@ def forward(self, input):
     input_size=(1, 1, 2, 4, 6),
     fullname='MaxUnpool3d_net'))
 
-
 if __name__ == '__main__':
     run_tests()
diff --git a/test/test_torch.py b/test/test_torch.py
--- a/test/test_torch.py
+++ b/test/test_torch.py
@@ -7,9 +7,8 @@
 import tempfile
 import unittest
 import warnings
-from itertools import product, chain
-from functools import wraps
-from common import TestCase, iter_indices, TEST_NUMPY, run_tests, download_file
+from itertools import product
+from common import TestCase, iter_indices, TEST_NUMPY, run_tests, download_file, skipIfNoLapack
 
 if TEST_NUMPY:
     import numpy as np
@@ -17,18 +16,6 @@
 SIZE = 100
 
 
-def skipIfNoLapack(fn):
-    @wraps(fn)
-    def wrapper(*args, **kwargs):
-        try:
-            fn(*args, **kwargs)
-        except Exception as e:
-            if 'Lapack library not found' in e.args[0]:
-                raise unittest.SkipTest('Compiled without Lapack')
-            raise
-    return wrapper
-
-
 class TestTorch(TestCase):
 
     def test_dot(self):