Repository: chainer/chainer

Issue:
Orthogonal initializer dimensions error
Orthogonal Initializer throws an error whenever the first dimension is larger than the second. This could be fixed by transposing the array rather than throwing an error.


e.g. 1. ` dense = L.Linear(64,128, initialW=initializers.Orthogonal())`
![image](https://user-images.githubusercontent.com/7952854/46913383-cd7d3380-cf83-11e8-86c5-4cfe5b6d99c2.png)



e.g. 2 `initializers.generate_array(initializers.Orthogonal(), (20,10), numpy, 'f')`
![image](https://user-images.githubusercontent.com/7952854/46913407-3d8bb980-cf84-11e8-9773-3e3a3c38ac76.png)



System Info:
Chainer: 5.0.0rc1
NumPy: 1.14.2
CuPy:
  CuPy Version          : 5.0.0rc1
  CUDA Root             : /usr/local/cuda
  CUDA Build Version    : 8000
  CUDA Driver Version   : 9020
  CUDA Runtime Version  : 8000
  cuDNN Build Version   : 7102
  cuDNN Version         : 7102
  NCCL Build Version    : 2213



Discussion:
I'd like to add an optional arg to `Orthogonal`
- `mode='projection'`: the current behavior
- `mode='auto'`: @MannyKayy's proposal
- `mode='embedding'`: assert first >= second
- `mode='basis'`: assert first == second
yep, sounds good.

Resolving patch:
diff --git a/chainer/initializers/orthogonal.py b/chainer/initializers/orthogonal.py
--- a/chainer/initializers/orthogonal.py
+++ b/chainer/initializers/orthogonal.py
@@ -5,6 +5,14 @@
 from chainer import utils
 
 
+_orthogonal_constraints = {  # (assert emb., assert proj.)
+    'auto': (False, False),
+    'projection': (False, True),
+    'embedding': (True, False),
+    'basis': (True, True),
+}
+
+
 # Original code forked from MIT licensed keras project
 # https://github.com/fchollet/keras/blob/master/keras/initializations.py
 
@@ -32,13 +40,24 @@ class Orthogonal(initializer.Initializer):
     Attributes:
         scale (float): A constant to be multiplied by.
         dtype: Data type specifier.
+        mode (str): Assertion on the initialized shape.
+            ``'auto'`` (default), ``'projection'`` (before v7),
+            ``'embedding'``, or ``'basis'``.
 
     Reference: Saxe et al., https://arxiv.org/abs/1312.6120
 
     """
 
-    def __init__(self, scale=1.1, dtype=None):
+    def __init__(self, scale=1.1, dtype=None, mode='auto'):
         self.scale = scale
+        self.mode = mode
+        try:
+            self._checks = _orthogonal_constraints[mode]
+        except KeyError:
+            raise ValueError(
+                'Invalid mode: {}. Choose from {}.'.format(
+                    repr(mode),
+                    ', '.join(repr(m) for m in _orthogonal_constraints)))
         super(Orthogonal, self).__init__(dtype)
 
     # TODO(Kenta Oono)
@@ -53,14 +72,22 @@ def __call__(self, array):
             raise ValueError('Array to be initialized must be non-empty.')
         else:
             # numpy.prod returns float value when the argument is empty.
-            flat_shape = (len(array), utils.size_of_shape(array.shape[1:]))
-            if flat_shape[0] > flat_shape[1]:
-                raise ValueError('Cannot make orthogonal system because'
-                                 ' # of vectors ({}) is larger than'
-                                 ' that of dimensions ({})'.format(
-                                     flat_shape[0], flat_shape[1]))
-            a = numpy.random.normal(size=flat_shape)
+            out_dim = len(array)
+            in_dim = utils.size_of_shape(array.shape[1:])
+            if (in_dim > out_dim and self._checks[0]) or (
+                    in_dim < out_dim and self._checks[1]):
+                raise ValueError(
+                    'Cannot make orthogonal {}.'
+                    'shape = {}, interpreted as '
+                    '{}-dim input and {}-dim output.'.format(
+                        self.mode, array.shape, in_dim, out_dim))
+            transpose = in_dim > out_dim
+            a = numpy.random.normal(size=(out_dim, in_dim))
+            if transpose:
+                a = a.T
             # cupy.linalg.qr requires cusolver in CUDA 8+
-            q, r = numpy.linalg.qr(a.T)
+            q, r = numpy.linalg.qr(a)
             q *= numpy.copysign(self.scale, numpy.diag(r))
-            array[...] = xp.asarray(q.T.reshape(array.shape))
+            if transpose:
+                q = q.T
+            array[...] = xp.asarray(q.reshape(array.shape))


Test patch:
diff --git a/tests/chainer_tests/initializer_tests/test_orthogonal.py b/tests/chainer_tests/initializer_tests/test_orthogonal.py
--- a/tests/chainer_tests/initializer_tests/test_orthogonal.py
+++ b/tests/chainer_tests/initializer_tests/test_orthogonal.py
@@ -14,8 +14,9 @@
     [
         {'shape': (), 'dim_in': 1, 'dim_out': 1},
         {'shape': (1,), 'dim_in': 1, 'dim_out': 1},
-        {'shape': (3, 4), 'dim_in': 4, 'dim_out': 3},
-        {'shape': (3, 4, 5), 'dim_in': 20, 'dim_out': 3}
+        {'shape': (4, 3), 'dim_in': 3, 'dim_out': 4},
+        {'shape': (6, 2, 3), 'dim_in': 6, 'dim_out': 6},
+        {'shape': (3, 4, 5), 'dim_in': 20, 'dim_out': 3},
     ],
     [
         {'scale': 2., 'dtype': numpy.float16}
@@ -70,9 +71,13 @@ def test_shaped_initializer_gpu(self):
     def check_orthogonality(self, w):
         initializer = self.target(**self.target_kwargs)
         initializer(w)
-        n = self.dim_out
-        w = w.astype(numpy.float64).reshape(n, -1)
-        dots = w.dot(w.T)
+        w = w.astype(numpy.float64).reshape(self.dim_out, self.dim_in)
+        if self.dim_in >= self.dim_out:
+            n = self.dim_out
+            dots = w.dot(w.T)
+        else:
+            n = self.dim_in
+            dots = w.T.dot(w)
         expected_scale = self.scale or 1.1
         testing.assert_allclose(
             dots, numpy.identity(n) * expected_scale**2,
@@ -99,10 +104,12 @@ def check_initializer_statistics(self, xp, n):
         sampless = cuda.to_cpu(ws.reshape(n, -1).T)
         alpha = 0.01 / len(sampless)
 
-        ab = 0.5 * (self.dim_in - 1)
+        larger_dim = max(self.dim_out, self.dim_in)
+
+        ab = 0.5 * (larger_dim - 1)
 
         for samples in sampless:
-            if self.dim_in == 1:
+            if larger_dim == 1:
                 numpy.testing.assert_allclose(abs(samples), expected_scale)
                 _, p = stats.chisquare((numpy.sign(samples) + 1) // 2)
             else:
@@ -159,14 +166,18 @@ def test_gpu(self):
         self.check_assert(cuda.to_gpu(self.w))
 
 
-@testing.parameterize(
-    {'shape': (4, 3)},
-    {'shape': (21, 4, 5)})
-class TestOverComplete(unittest.TestCase):
+@testing.parameterize(*(testing.product({
+    'shape': [(3,), (4, 3), (21, 4, 5)],
+    'mode': ['projection', 'basis'],
+}) + testing.product({
+    'shape': [(0,), (3, 4, 5)],
+    'mode': ['embedding', 'basis'],
+})))
+class TestOrthogonalMode(unittest.TestCase):
 
     def setUp(self):
         self.w = numpy.empty(self.shape, dtype=numpy.float32)
-        self.initializer = initializers.Orthogonal(scale=1.0)
+        self.initializer = initializers.Orthogonal(scale=1.0, mode=self.mode)
 
     def check_invalid(self, w):
         with self.assertRaises(ValueError):