Repository: Theano/Theano

Issue:
cumsum(x,axis=-1) does not work on random array
When working on an array generated by random number generator, it returns x unchanged. If you use a positive value for axis then it works correctly but if axis=-1 it does not apply itself to the last dimension

``` python
import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams
import pprint

rng = MRG_RandomStreams(3)
u = rng.uniform(size=(2,5))
c = T.cumsum(u,axis=-1)
d = T.cumsum(u,axis=1)
f = theano.function([], outputs=[u, c, d], allow_input_downcast=True)

pprint.pprint(f())
```

[array([[ 0.005768  ,  0.00573857,  0.79128766,  0.73698175,  0.58187205],
       [ 0.62578964,  0.73792672,  0.21239796,  0.24640237,  0.14918256]], dtype=float32),
 array([[ 0.005768  ,  0.00573857,  0.79128766,  0.73698175,  0.58187205],
       [ 0.62578964,  0.73792672,  0.21239796,  0.24640237,  0.14918256]], dtype=float32),
 array([[ 0.005768  ,  0.01150657,  0.80279422,  1.53977597,  2.12164807],
       [ 0.62578964,  1.36371636,  1.5761143 ,  1.82251668,  1.97169924]], dtype=float32)]



Discussion:
 @MarcCote
https://github.com/Theano/Theano/issues?q=is%3Apr+is%3Aopen+author%3AMarcCote
Can you look at it? I think it should raise an error or work as expected.

On Tue, Mar 31, 2015 at 7:48 PM, Ehud Ben-Reuven notifications@github.com
wrote:

> it returns x unchanged. If you use a positive value for axis then it works
> correctly but if axis=-1 it does not apply itself to the last dimension
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/Theano/Theano/issues/2715.


Resolving patch:
diff --git a/theano/sandbox/cuda/extra_ops.py b/theano/sandbox/cuda/extra_ops.py
--- a/theano/sandbox/cuda/extra_ops.py
+++ b/theano/sandbox/cuda/extra_ops.py
@@ -25,8 +25,8 @@ def __init__(self, axis):
         self.max_grid_size1 = None
         self.max_grid_size2 = None
 
-# We must reuse the same method, not reimplement and call it.
-# Otherwise DebugMode will print many warnings.
+    # We must reuse the same method, not reimplement and call it.
+    # Otherwise DebugMode will print many warnings.
     perform = Op.perform
 
     def make_node(self, x):
@@ -37,7 +37,7 @@ def make_node(self, x):
         if x.ndim > GpuCumsum.SUPPORTED_NDIMS:
             raise NotImplementedError('Only cumsum on 1D, 2D and 3D array are supported right now!')
 
-        if self.axis >= x.ndim:
+        if self.axis >= x.ndim or self.axis < -x.ndim:
             raise ValueError('axis(={1}) out of bounds'.format(self.axis))
 
         return theano.Apply(self, [x], [x.type()])
@@ -69,7 +69,7 @@ def __str__(self):
         return "%s{%s}" % (self.__class__.__name__, self.axis)
 
     def c_code_cache_version(self):
-        return (7,)
+        return (8,)
 
     def c_support_code_apply(self, node, nodename):
         return """
@@ -352,6 +352,8 @@ def c_support_code_apply(self, node, nodename):
     def c_code(self, node, nodename, inames, onames, sub):
         x, = inames
         z, = onames
+
+        # We assume array has been already flattened if needed.
         axis = self.axis if self.axis is not None else 0
         fail = sub['fail']
 
@@ -368,6 +370,12 @@ def c_code(self, node, nodename, inames, onames, sub):
             const int* shape = CudaNdarray_HOST_DIMS(%(x)s);
             bool needAllocation = !%(z)s || CudaNdarray_NDIM(%(x)s) != CudaNdarray_NDIM(%(z)s);
 
+            int axis = %(axis)s;
+            if (axis < 0) {
+                // Convert negative axis to positive axis.
+                axis += CudaNdarray_NDIM(%(x)s);
+            }
+
             // If output is already allocated, check if its shape matches the input's one.
             if (!needAllocation) {
                 for (int i= 0; i < CudaNdarray_NDIM(%(x)s); ++i) {
@@ -387,7 +395,7 @@ def c_code(self, node, nodename, inames, onames, sub):
             }
 
             { // Namespace for kernel calls //
-                if (cumSum_%(nodename)s(%(x)s, %(z)s, %(axis)s, %(max_threads_dim0)s, %(max_grid_size1)s, %(max_grid_size2)s) == -1){
+                if (cumSum_%(nodename)s(%(x)s, %(z)s, axis, %(max_threads_dim0)s, %(max_grid_size1)s, %(max_grid_size2)s) == -1){
                     %(fail)s;
                 }
 
@@ -408,11 +416,10 @@ def c_code(self, node, nodename, inames, onames, sub):
 
 def values_eq_approx_high_tol(a, b):
     """This fct is needed to don't have DebugMode raise useless
-    error due to ronding error.
+    error due to rounding error.
 
     This happen with big input size due to change in the order of
     operation.
-
     """
     rtol = None
     if a.size > 100000:
@@ -443,6 +450,7 @@ def use_gpu_cumsum(node):
         # ``gpu_cumsum`` assume array has been flattened if needed.
         if axis is None:
             axis = 0
+
         ret = host_from_gpu(GpuCumsum(axis)(x))
         ret.values_eq_approx = values_eq_approx_high_tol
         return [ret]
diff --git a/theano/tensor/extra_ops.py b/theano/tensor/extra_ops.py
--- a/theano/tensor/extra_ops.py
+++ b/theano/tensor/extra_ops.py
@@ -29,7 +29,7 @@ def make_node(self, x):
 
         if self.axis is None:
             out_type = theano.tensor.vector(dtype=x.dtype)  # Flatten
-        elif self.axis >= x.ndim:
+        elif self.axis >= x.ndim or self.axis < -x.ndim:
             raise ValueError('axis(={0}) out of bounds'.format(self.axis))
 
         return theano.Apply(self, [x], [out_type])
@@ -151,7 +151,7 @@ def make_node(self, x):
 
         if self.axis is None:
             out_type = theano.tensor.vector(dtype=x.dtype)  # Flatten
-        elif self.axis >= x.ndim:
+        elif self.axis >= x.ndim or self.axis < -x.ndim:
             raise ValueError('axis(={0}) out of bounds'.format(self.axis))
 
         return theano.Apply(self, [x], [out_type])


Test patch:
diff --git a/theano/sandbox/cuda/tests/test_extra_ops.py b/theano/sandbox/cuda/tests/test_extra_ops.py
--- a/theano/sandbox/cuda/tests/test_extra_ops.py
+++ b/theano/sandbox/cuda/tests/test_extra_ops.py
@@ -47,7 +47,7 @@ def setUp(self):
     def test_Strides1D(self):
         x = T.fvector('x')
 
-        for axis in [0, None]:
+        for axis in [0, None, -1]:
             a = np.random.random((42,)).astype("float32")
             cumsum_function = theano.function([x], cumsum(x, axis=axis),
                                               mode=self.mode)
@@ -70,7 +70,7 @@ def test_Strides1D(self):
     def test_Strides2D(self):
         x = T.fmatrix('x')
 
-        for axis in [0, 1, None]:
+        for axis in [0, 1, None, -1, -2]:
             a = np.random.random((42, 30)).astype("float32")
             cumsum_function = theano.function([x], cumsum(x, axis=axis),
                                               mode=self.mode)
@@ -93,7 +93,7 @@ def test_Strides2D(self):
     def test_Strides3D(self):
         x = T.ftensor3('x')
 
-        for axis in [0, 1, 2, None]:
+        for axis in [0, 1, 2, None, -1, -2, -3]:
             a = np.random.random((42, 30, 25)).astype("float32")
             cumsum_function = theano.function([x], cumsum(x, axis=axis),
                                               mode=self.mode)
@@ -139,7 +139,7 @@ def test_GpuCumsum2D(self):
         block_max_size = self.max_threads_dim0 * 2
 
         x = T.fmatrix('x')
-        for shape_axis, axis in zip([0, 1, 0], [0, 1, None]):
+        for shape_axis, axis in zip([0, 1, 0, 1, 0], [0, 1, None, -1, -2]):
             f = theano.function([x], cumsum(x, axis=axis), mode=self.mode)
             assert [n for n in f.maker.fgraph.toposort()
                     if isinstance(n.op, GpuCumsum)]
@@ -178,7 +178,7 @@ def test_GpuCumsum3D(self):
         block_max_size = self.max_threads_dim0 * 2
 
         x = T.ftensor3('x')
-        for shape_axis, axis in zip([0, 1, 2, 0], [0, 1, 2, None]):
+        for shape_axis, axis in zip([0, 1, 2, 0, 2, 1, 0], [0, 1, 2, None, -1, -2, -3]):
             f = theano.function([x], cumsum(x, axis=axis), mode=self.mode)
             assert [n for n in f.maker.fgraph.toposort()
                     if isinstance(n.op, GpuCumsum)]
diff --git a/theano/tensor/tests/test_extra_ops.py b/theano/tensor/tests/test_extra_ops.py
--- a/theano/tensor/tests/test_extra_ops.py
+++ b/theano/tensor/tests/test_extra_ops.py
@@ -32,12 +32,13 @@ def test_cumsumOp(self):
         a = np.random.random((3, 5, 2)).astype(config.floatX)
 
         # Test axis out of bounds
-        self.assertRaises(ValueError, cumsum, x, axis=4)
+        self.assertRaises(ValueError, cumsum, x, axis=3)
+        self.assertRaises(ValueError, cumsum, x, axis=-4)
 
         f = theano.function([x], cumsum(x))
         assert np.allclose(np.cumsum(a), f(a))  # Test axis=None
 
-        for axis in range(len(a.shape)):
+        for axis in range(-len(a.shape), len(a.shape)):
             f = theano.function([x], cumsum(x, axis=axis))
             assert np.allclose(np.cumsum(a, axis=axis), f(a))
 
@@ -51,7 +52,7 @@ def test_infer_shape(self):
                                 [a],
                                 self.op_class)
 
-        for axis in range(len(a.shape)):
+        for axis in range(-len(a.shape), len(a.shape)):
             self._compile_and_check([x],
                                     [cumsum(x, axis=axis)],
                                     [a],
@@ -62,7 +63,7 @@ def test_grad(self):
 
         utt.verify_grad(self.op, [a])  # Test axis=None
 
-        for axis in range(len(a.shape)):
+        for axis in range(-len(a.shape), len(a.shape)):
             utt.verify_grad(self.op_class(axis=axis), [a], eps=4e-4)
 
 
@@ -77,10 +78,14 @@ def test_CumprodOp(self):
         x = T.tensor3('x')
         a = np.random.random((3, 5, 2)).astype(config.floatX)
 
+        # Test axis out of bounds
+        self.assertRaises(ValueError, cumprod, x, axis=3)
+        self.assertRaises(ValueError, cumprod, x, axis=-4)
+
         f = theano.function([x], cumprod(x))
         assert np.allclose(np.cumprod(a), f(a))  # Test axis=None
 
-        for axis in range(len(a.shape)):
+        for axis in range(-len(a.shape), len(a.shape)):
             f = theano.function([x], cumprod(x, axis=axis))
             assert np.allclose(np.cumprod(a, axis=axis), f(a))
 
@@ -94,7 +99,7 @@ def test_infer_shape(self):
                                 [a],
                                 self.op_class)
 
-        for axis in range(len(a.shape)):
+        for axis in range(-len(a.shape), len(a.shape)):
             self._compile_and_check([x],
                                     [cumprod(x, axis=axis)],
                                     [a],
@@ -105,7 +110,7 @@ def test_grad(self):
 
         utt.verify_grad(self.op, [a])  # Test axis=None
 
-        for axis in range(len(a.shape)):
+        for axis in range(-len(a.shape), len(a.shape)):
             utt.verify_grad(self.op_class(axis=axis), [a])