Repository: cupy/cupy

Issue:
implement cupy.gradient
`gradient` computes n-dimensional derivatives, and is not related to autograd/autodiff - See [numpy gradient](https://docs.scipy.org/doc/numpy-1.14.0/reference/generated/numpy.gradient.html).

This function is used in a number of scientific codes and is (I believe) missing in cupy.


Discussion:
I will work on this feature.
That would be great! Thanks for working on this Tudor. 😄
Hi @TudorAndrei.  I made a CuPy-based implementation of gradient if you want to start from that for your PR. It is basically just a simple 1:1 translation of the NumPy code.

The function is in:
https://github.com/mritools/cupyimg/blob/master/cupyimg/numpy/lib/function_base.py

The main work that needs to be done for inclusion in CuPy is making some tests in the CuPy style. I have existing tests copied from NumPy in:
https://github.com/mritools/cupyimg/blob/master/cupyimg/numpy/lib/tests/test_function_base.py

However, these are not in the typical style used by CuPy. More often CuPy tests are parameterized and involve direct comparisons between NumPy and CuPy outputs. I don't have immediate plans to do this conversion for `gradient` myself and would appreciate it if you are interested!


Resolving patch:
diff --git a/cupy/__init__.py b/cupy/__init__.py
--- a/cupy/__init__.py
+++ b/cupy/__init__.py
@@ -592,6 +592,7 @@ def isscalar(element):
 from cupy.math.sumprod import nansum  # NOQA
 from cupy.math.sumprod import nanprod  # NOQA
 from cupy.math.sumprod import diff  # NOQA
+from cupy.math.sumprod import gradient  # NOQA
 from cupy.math.window import bartlett  # NOQA
 from cupy.math.window import blackman  # NOQA
 from cupy.math.window import hamming  # NOQA
diff --git a/cupy/math/sumprod.py b/cupy/math/sumprod.py
--- a/cupy/math/sumprod.py
+++ b/cupy/math/sumprod.py
@@ -3,7 +3,7 @@
 import cupy
 from cupy.core import _routines_math as _math
 from cupy.core import _fusion_thread_local
-from cupy._util import _normalize_axis_index
+from cupy import _util
 
 
 def sum(a, axis=None, dtype=None, out=None, keepdims=False):
@@ -202,7 +202,7 @@ def diff(a, n=1, axis=-1, prepend=None, append=None):
 
     a = cupy.asanyarray(a)
     nd = a.ndim
-    axis = _normalize_axis_index(axis, nd)
+    axis = _util._normalize_axis_index(axis, nd)
 
     combined = []
 
@@ -241,10 +241,225 @@ def diff(a, n=1, axis=-1, prepend=None, append=None):
     return a
 
 
-# TODO(okuta): Implement ediff1d
+def gradient(f, *varargs, axis=None, edge_order=1):
+    """Return the gradient of an N-dimensional array.
+
+    The gradient is computed using second order accurate central differences
+    in the interior points and either first or second order accurate one-sides
+    (forward or backwards) differences at the boundaries.
+    The returned gradient hence has the same shape as the input array.
+
+    Args:
+        f (cupy.ndarray): An N-dimensional array containing samples of a scalar
+            function.
+        varargs (list of scalar or array, optional): Spacing between f values.
+            Default unitary spacing for all dimensions. Spacing can be
+            specified using:
+
+            1. single scalar to specify a sample distance for all dimensions.
+            2. N scalars to specify a constant sample distance for each
+               dimension. i.e. `dx`, `dy`, `dz`, ...
+            3. N arrays to specify the coordinates of the values along each
+               dimension of F. The length of the array must match the size of
+               the corresponding dimension
+            4. Any combination of N scalars/arrays with the meaning of 2. and
+               3.
+
+            If `axis` is given, the number of varargs must equal the number of
+            axes. Default: 1.
+        edge_order ({1, 2}, optional): The gradient is calculated using N-th
+            order accurate differences at the boundaries. Default: 1.
+        axis (None or int or tuple of ints, optional): The gradient is
+            calculated only along the given axis or axes. The default
+            (axis = None) is to calculate the gradient for all the axes of the
+            input array. axis may be negative, in which case it counts from the
+            last to the first axis.
+
+    Returns:
+        gradient (cupy.ndarray or list of cupy.ndarray): A set of ndarrays
+            (or a single ndarray if there is only one dimension) corresponding
+            to the derivatives of f with respect to each dimension. Each
+            derivative has the same shape as f.
+
+    .. seealso:: :func:`numpy.gradient`
+    """
+    f = cupy.asanyarray(f)
+    ndim = f.ndim  # number of dimensions
 
+    if axis is None:
+        axes = tuple(range(ndim))
+    else:
+        axes = _util._normalize_axis_indices(axis, ndim)
 
-# TODO(okuta): Implement gradient
+    len_axes = len(axes)
+    n = len(varargs)
+    if n == 0:
+        # no spacing argument - use 1 in all axes
+        dx = [1.0] * len_axes
+    elif n == 1 and cupy.ndim(varargs[0]) == 0:
+        # single scalar for all axes
+        dx = varargs * len_axes
+    elif n == len_axes:
+        # scalar or 1d array for each axis
+        dx = list(varargs)
+        for i, distances in enumerate(dx):
+            if cupy.ndim(distances) == 0:
+                continue
+            elif cupy.ndim(distances) != 1:
+                raise ValueError("distances must be either scalars or 1d")
+            if len(distances) != f.shape[axes[i]]:
+                raise ValueError(
+                    "when 1d, distances must match "
+                    "the length of the corresponding dimension"
+                )
+            if numpy.issubdtype(distances.dtype, numpy.integer):
+                # Convert numpy integer types to float64 to avoid modular
+                # arithmetic in np.diff(distances).
+                distances = distances.astype(numpy.float64)
+            diffx = cupy.diff(distances)
+            # if distances are constant reduce to the scalar case
+            # since it brings a consistent speedup
+            if (diffx == diffx[0]).all():  # synchronize
+                diffx = diffx[0]
+            dx[i] = diffx
+    else:
+        raise TypeError("invalid number of arguments")
+
+    if edge_order > 2:
+        raise ValueError("'edge_order' greater than 2 not supported")
+
+    # use central differences on interior and one-sided differences on the
+    # endpoints. This preserves second order-accuracy over the full domain.
+
+    outvals = []
+
+    # create slice objects --- initially all are [:, :, ..., :]
+    slice1 = [slice(None)] * ndim
+    slice2 = [slice(None)] * ndim
+    slice3 = [slice(None)] * ndim
+    slice4 = [slice(None)] * ndim
+
+    otype = f.dtype
+    if numpy.issubdtype(otype, numpy.inexact):
+        pass
+    else:
+        # All other types convert to floating point.
+        # First check if f is a numpy integer type; if so, convert f to float64
+        # to avoid modular arithmetic when computing the changes in f.
+        if numpy.issubdtype(otype, numpy.integer):
+            f = f.astype(numpy.float64)
+        otype = numpy.float64
+
+    for axis, ax_dx in zip(axes, dx):
+        if f.shape[axis] < edge_order + 1:
+            raise ValueError(
+                "Shape of array too small to calculate a numerical gradient, "
+                "at least (edge_order + 1) elements are required."
+            )
+        # result allocation
+        out = cupy.empty_like(f, dtype=otype)
+
+        # spacing for the current axis
+        uniform_spacing = cupy.ndim(ax_dx) == 0
+
+        # Numerical differentiation: 2nd order interior
+        slice1[axis] = slice(1, -1)
+        slice2[axis] = slice(None, -2)
+        slice3[axis] = slice(1, -1)
+        slice4[axis] = slice(2, None)
+
+        if uniform_spacing:
+            out[tuple(slice1)] = (f[tuple(slice4)] - f[tuple(slice2)]) / (
+                2.0 * ax_dx
+            )
+        else:
+            dx1 = ax_dx[0:-1]
+            dx2 = ax_dx[1:]
+            dx_sum = dx1 + dx2
+            a = -(dx2) / (dx1 * dx_sum)
+            b = (dx2 - dx1) / (dx1 * dx2)
+            c = dx1 / (dx2 * dx_sum)
+            # fix the shape for broadcasting
+            shape = [1] * ndim
+            shape[axis] = -1
+            a.shape = b.shape = c.shape = tuple(shape)
+            # 1D equivalent -- out[1:-1] = a * f[:-2] + b * f[1:-1] + c * f[2:]
+            out[tuple(slice1)] = (a * f[tuple(slice2)] +
+                                  b * f[tuple(slice3)] +
+                                  c * f[tuple(slice4)])
+
+        # Numerical differentiation: 1st order edges
+        if edge_order == 1:
+            slice1[axis] = 0
+            slice2[axis] = 1
+            slice3[axis] = 0
+            dx_0 = ax_dx if uniform_spacing else ax_dx[0]
+            # 1D equivalent -- out[0] = (f[1] - f[0]) / (x[1] - x[0])
+            out[tuple(slice1)] = (f[tuple(slice2)] - f[tuple(slice3)]) / dx_0
+
+            slice1[axis] = -1
+            slice2[axis] = -1
+            slice3[axis] = -2
+            dx_n = ax_dx if uniform_spacing else ax_dx[-1]
+            # 1D equivalent -- out[-1] = (f[-1] - f[-2]) / (x[-1] - x[-2])
+            out[tuple(slice1)] = (f[tuple(slice2)] - f[tuple(slice3)]) / dx_n
+
+        # Numerical differentiation: 2nd order edges
+        else:
+            slice1[axis] = 0
+            slice2[axis] = 0
+            slice3[axis] = 1
+            slice4[axis] = 2
+            if uniform_spacing:
+                a = -1.5 / ax_dx
+                b = 2.0 / ax_dx
+                c = -0.5 / ax_dx
+            else:
+                dx1 = ax_dx[0]
+                dx2 = ax_dx[1]
+                dx_sum = dx1 + dx2
+                a = -(2.0 * dx1 + dx2) / (dx1 * (dx_sum))
+                b = dx_sum / (dx1 * dx2)
+                c = -dx1 / (dx2 * (dx_sum))
+            # 1D equivalent -- out[0] = a * f[0] + b * f[1] + c * f[2]
+            out[tuple(slice1)] = (a * f[tuple(slice2)] +
+                                  b * f[tuple(slice3)] +
+                                  c * f[tuple(slice4)])
+
+            slice1[axis] = -1
+            slice2[axis] = -3
+            slice3[axis] = -2
+            slice4[axis] = -1
+            if uniform_spacing:
+                a = 0.5 / ax_dx
+                b = -2.0 / ax_dx
+                c = 1.5 / ax_dx
+            else:
+                dx1 = ax_dx[-2]
+                dx2 = ax_dx[-1]
+                dx_sum = dx1 + dx2
+                a = (dx2) / (dx1 * (dx_sum))
+                b = -dx_sum / (dx1 * dx2)
+                c = (2.0 * dx2 + dx1) / (dx2 * (dx_sum))
+            # 1D equivalent -- out[-1] = a * f[-3] + b * f[-2] + c * f[-1]
+            out[tuple(slice1)] = (a * f[tuple(slice2)] +
+                                  b * f[tuple(slice3)] +
+                                  c * f[tuple(slice4)])
+        outvals.append(out)
+
+        # reset the slice object in this dimension to ":"
+        slice1[axis] = slice(None)
+        slice2[axis] = slice(None)
+        slice3[axis] = slice(None)
+        slice4[axis] = slice(None)
+
+    if len_axes == 1:
+        return outvals[0]
+    else:
+        return outvals
+
+
+# TODO(okuta): Implement ediff1d
 
 
 # TODO(okuta): Implement cross


Test patch:
diff --git a/tests/cupy_tests/math_tests/test_sumprod.py b/tests/cupy_tests/math_tests/test_sumprod.py
--- a/tests/cupy_tests/math_tests/test_sumprod.py
+++ b/tests/cupy_tests/math_tests/test_sumprod.py
@@ -4,6 +4,7 @@
 import pytest
 
 import cupy
+import cupy._util
 from cupy.core import _accelerator
 from cupy import testing
 
@@ -711,3 +712,122 @@ def test_diff_invalid_axis(self):
                 xp.diff(a, axis=3)
             with pytest.raises(numpy.AxisError):
                 xp.diff(a, axis=-4)
+
+
+# This class compares CUB results against NumPy's
+@testing.parameterize(*testing.product({
+    'shape': [(33,), (10, 20), (10, 20, 30)],
+    'spacing': ((), (1.2,), 'sequence of int', 'arrays', 'mixed'),
+    'edge_order': [1, 2],
+    'axis': [None, 0, -1, 'tuple'],
+}))
+@testing.gpu
+class TestGradient(unittest.TestCase):
+
+    def _gradient(self, xp, dtype, shape, spacing, axis, edge_order):
+        x = testing.shaped_random(shape, xp, dtype=dtype)
+        if axis == 'tuple':
+            if x.ndim == 1:
+                axis = (0,)
+            else:
+                axis = (0, -1)
+        normalized_axes = cupy._util._normalize_axis_indices(axis, x.ndim)
+        if spacing == 'sequence of int':
+            # one scalar per axis
+            spacing = tuple((ax + 1) / x.ndim for ax in normalized_axes)
+        elif spacing == 'arrays':
+            # one array per axis
+            spacing = tuple(
+                xp.arange(x.shape[ax]) * (ax + 0.5) for ax in normalized_axes
+            )
+            # make at one of the arrays have non-constant spacing
+            spacing[-1][5:] *= 2.0
+        elif spacing == 'mixed':
+            # mixture of arrays and scalars
+            spacing = [xp.arange(x.shape[normalized_axes[0]])]
+            spacing = spacing + [0.5] * (len(normalized_axes) - 1)
+        return xp.gradient(x, *spacing, axis=axis, edge_order=edge_order)
+
+    @testing.for_dtypes('fFdD')
+    @testing.numpy_cupy_allclose(atol=1e-6, rtol=1e-5)
+    def test_gradient_floating(self, xp, dtype):
+        return self._gradient(xp, dtype, self.shape, self.spacing, self.axis,
+                              self.edge_order)
+
+    # unsigned int behavior fixed in 1.18.1
+    # https://github.com/numpy/numpy/issues/15207
+    @testing.with_requires('numpy>=1.18.1')
+    @testing.for_int_dtypes(no_bool=True)
+    @testing.numpy_cupy_allclose(atol=1e-6, rtol=1e-5)
+    def test_gradient_int(self, xp, dtype):
+        return self._gradient(xp, dtype, self.shape, self.spacing, self.axis,
+                              self.edge_order)
+
+    @testing.numpy_cupy_allclose(atol=2e-2, rtol=1e-3)
+    def test_gradient_float16(self, xp):
+        return self._gradient(xp, numpy.float16, self.shape, self.spacing,
+                              self.axis, self.edge_order)
+
+
+@testing.gpu
+class TestGradientErrors(unittest.TestCase):
+
+    def test_gradient_invalid_spacings1(self):
+        # more spacings than axes
+        spacing = (1.0, 2.0, 3.0)
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random((32, 16), xp)
+            with pytest.raises(TypeError):
+                xp.gradient(x, *spacing)
+
+    def test_gradient_invalid_spacings2(self):
+        # wrong length array in spacing
+        shape = (32, 16)
+        spacing = (15, cupy.arange(shape[1] + 1))
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random(shape, xp)
+            with pytest.raises(ValueError):
+                xp.gradient(x, *spacing)
+
+    def test_gradient_invalid_spacings3(self):
+        # spacing array with ndim != 1
+        shape = (32, 16)
+        spacing = (15, cupy.arange(shape[0]).reshape(4, -1))
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random(shape, xp)
+            with pytest.raises(ValueError):
+                xp.gradient(x, *spacing)
+
+    def test_gradient_invalid_edge_order1(self):
+        # unsupported edge order
+        shape = (32, 16)
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random(shape, xp)
+            with pytest.raises(ValueError):
+                xp.gradient(x, edge_order=3)
+
+    def test_gradient_invalid_edge_order2(self):
+        # shape cannot be < edge_order
+        shape = (1, 16)
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random(shape, xp)
+            with pytest.raises(ValueError):
+                xp.gradient(x, axis=0, edge_order=2)
+
+    @testing.with_requires('numpy>=1.16')
+    def test_gradient_invalid_axis(self):
+        # axis out of range
+        shape = (4, 16)
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random(shape, xp)
+            for axis in [-3, 2]:
+                with pytest.raises(numpy.AxisError):
+                    xp.gradient(x, axis=axis)
+
+    def test_gradient_bool_input(self):
+        # axis out of range
+        shape = (4, 16)
+        for xp in [numpy, cupy]:
+            x = testing.shaped_random(shape, xp, dtype=numpy.bool)
+            with pytest.raises(TypeError):
+                xp.gradient(x)