Repository: numba/numba

Issue:
NumPy allocator routines accept heterogeneous shape
<!--

Thanks for opening an issue! To help the Numba team handle your information
efficiently, please first ensure that there is no other issue present that
already describes the issue you have
(search at https://github.com/numba/numba/issues?&q=is%3Aissue).

For more general "how do I do X?" type questions, please speak to us in real
time on https://gitter.im/numba/numba or post to the Numba mailing list
https://groups.google.com/a/continuum.io/forum/#!forum/numba-users.

-->

## Feature request

It would be useful if the np allocator functions accepted tuples with heterogeneous integer types as `shape`, e.g.:
```python
from numba import njit
import numpy as np

@njit
def foo(a, b):
    y = np.int32(a)
    x = np.int8(b)
    return np.zeros((x, y))

foo(1, 2)
```
currently fails with :
```
numba.errors.LoweringError: Failed in nopython mode pipeline (step: nopython mode backend)
Operands must be the same type, got (i64, i8)
```
LoweringError in scanpy's calculate_qc_metrics function
I have been trying to reproduce scanpy's example for the calculate_qc_metrics function:
```
adata = sc.datasets.pbmc3k()
sc.pp.calculate_qc_metrics(adata, inplace=True)
```
I get a lengthy error message which ends on:

```
Operands must be the same type, got (i64, i32)

File "..\..\..\..\..\..\..\..\ProgramData\Anaconda3\lib\site-packages\scanpy\preprocessing\_qc.py", line 392:
def _top_segment_proportions_sparse_csr(data, indptr, ns):
    <source elided>
    # Just to keep it simple, as a dense matrix
    partitioned = np.zeros((indptr.size - 1, maxidx), dtype=data.dtype)
    ^

[1] During: lowering "$0.43 = call $0.33($0.39, func=$0.33, args=[Var($0.39, C:\ProgramData\Anaconda3\lib\site-packages\scanpy\preprocessing\_qc.py (392))], kws=[('dtype', Var($0.41, C:\ProgramData\Anaconda3\lib\site-packages\scanpy\preprocessing\_qc.py (392)))], vararg=None)" at C:\ProgramData\Anaconda3\lib\site-packages\scanpy\preprocessing\_qc.py (392)
```
I have included a text file with the full error message.

Does anyone have an idea as to what might be causing that problem?

I'm running the code in a Jupyter notebook using Anaconda (python3.7), numba 0.45.1 and Windows 10.

Thanks in advance and best regards!

[full error message.txt](https://github.com/numba/numba/files/3583266/full.error.message.txt)





Discussion:

Thanks for the report. Please could you provide more information about what caused this problem:

1. The code that you were running (as a minimal working reproducer, if you are unsure how to write one see http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports).
2. Any data needed to run the code.

Thanks.

I suspect the above is an assumption made about the size of an integer, I'm not sure which package is at fault, perhaps raise an issue on `scanpy`'s issue tracker and link back to this one and gather feedback from the `scanpy` authors?
Scanpy author here. I think it was an assumption that `np.array(x, np.int)` would have the same dtype as `scipy.sparse.csr_matrix(...).indptr.dtype`. I don't have a machine where this isn't the case, but I can reproduce this error pretty exactly passing an array of 32 bit integers. Here's a cut down example:

```python
import numba
import numpy as np

@numba.njit
def foo(a, b):
    return np.zeros((a.size, b[0]))

foo(np.ones(10), np.ones(10, dtype=np.int32))
```

<details>
<summary> Traceback </summary>

```python
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/usr/local/lib/python3.7/site-packages/numba/errors.py in new_error_context(fmt_, *args, **kwargs)
    661     try:
--> 662         yield
    663     except NumbaError as e:

/usr/local/lib/python3.7/site-packages/numba/lowering.py in lower_block(self, block)
    257                                    loc=self.loc, errcls_=defaulterrcls):
--> 258                 self.lower_inst(inst)
    259 

/usr/local/lib/python3.7/site-packages/numba/lowering.py in lower_inst(self, inst)
    300             ty = self.typeof(inst.target.name)
--> 301             val = self.lower_assign(ty, inst)
    302             self.storevar(val, inst.target.name)

/usr/local/lib/python3.7/site-packages/numba/lowering.py in lower_assign(self, ty, inst)
    458         elif isinstance(value, ir.Expr):
--> 459             return self.lower_expr(ty, value)
    460 

/usr/local/lib/python3.7/site-packages/numba/lowering.py in lower_expr(self, resty, expr)
    918         elif expr.op == 'call':
--> 919             res = self.lower_call(resty, expr)
    920             return res

/usr/local/lib/python3.7/site-packages/numba/lowering.py in lower_call(self, resty, expr)
    710         else:
--> 711             res = self._lower_call_normal(fnty, expr, signature)
    712 

/usr/local/lib/python3.7/site-packages/numba/lowering.py in _lower_call_normal(self, fnty, expr, signature)
    889 
--> 890         res = impl(self.builder, argvals, self.loc)
    891         return res

/usr/local/lib/python3.7/site-packages/numba/targets/base.py in __call__(self, builder, args, loc)
   1131     def __call__(self, builder, args, loc=None):
-> 1132         res = self._imp(self._context, builder, self._sig, args, loc=loc)
   1133         self._context.add_linking_libs(getattr(self, 'libs', ()))

/usr/local/lib/python3.7/site-packages/numba/targets/base.py in wrapper(*args, **kwargs)
   1156             kwargs.pop('loc')     # drop unused loc
-> 1157             return fn(*args, **kwargs)
   1158 

/usr/local/lib/python3.7/site-packages/numba/targets/arrayobj.py in numpy_zeros_nd(context, builder, sig, args)
   3373     arrtype, shapes = _parse_empty_args(context, builder, sig, args)
-> 3374     ary = _empty_nd_impl(context, builder, arrtype, shapes)
   3375     _zero_fill_array(context, builder, ary)

/usr/local/lib/python3.7/site-packages/numba/targets/arrayobj.py in _empty_nd_impl(context, builder, arrtype, shapes)
   3260     for s in shapes:
-> 3261         arrlen = builder.mul(arrlen, s)
   3262 

/usr/local/lib/python3.7/site-packages/llvmlite/ir/builder.py in wrapped(self, lhs, rhs, name, flags)
     23                 raise ValueError("Operands must be the same type, got (%s, %s)"
---> 24                                  % (lhs.type, rhs.type))
     25             instr = cls(self.block, lhs.type, opname, (lhs, rhs), name, flags)

ValueError: Operands must be the same type, got (i64, i32)

During handling of the above exception, another exception occurred:

LoweringError                             Traceback (most recent call last)
<ipython-input-5-1bc7960d5c26> in <module>
      6     return np.zeros((a.size, b[0]))
      7 
----> 8 foo(np.ones(10), np.ones(10, dtype=np.int32))

/usr/local/lib/python3.7/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    393                     e.patch_message(''.join(e.args) + help_msg)
    394             # ignore the FULL_TRACEBACKS config, this needs reporting!
--> 395             raise e
    396 
    397     def inspect_llvm(self, signature=None):

/usr/local/lib/python3.7/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    350                 argtypes.append(self.typeof_pyval(a))
    351         try:
--> 352             return self.compile(tuple(argtypes))
    353         except errors.TypingError as e:
    354             # Intercept typing error that may be due to an argument

/usr/local/lib/python3.7/site-packages/numba/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34 

/usr/local/lib/python3.7/site-packages/numba/dispatcher.py in compile(self, sig)
    691 
    692             self._cache_misses[sig] += 1
--> 693             cres = self._compiler.compile(args, return_type)
    694             self.add_overload(cres)
    695             self._cache.save_overload(sig, cres)

/usr/local/lib/python3.7/site-packages/numba/dispatcher.py in compile(self, args, return_type)
     74 
     75     def compile(self, args, return_type):
---> 76         status, retval = self._compile_cached(args, return_type)
     77         if status:
     78             return retval

/usr/local/lib/python3.7/site-packages/numba/dispatcher.py in _compile_cached(self, args, return_type)
     88 
     89         try:
---> 90             retval = self._compile_core(args, return_type)
     91         except errors.TypingError as e:
     92             self._failed_cache[key] = e

/usr/local/lib/python3.7/site-packages/numba/dispatcher.py in _compile_core(self, args, return_type)
    106                                       args=args, return_type=return_type,
    107                                       flags=flags, locals=self.locals,
--> 108                                       pipeline_class=self.pipeline_class)
    109         # Check typing error if object mode is used
    110         if cres.typing_error is not None and not flags.enable_pyobject:

/usr/local/lib/python3.7/site-packages/numba/compiler.py in compile_extra(typingctx, targetctx, func, args, return_type, flags, locals, library, pipeline_class)
    970     pipeline = pipeline_class(typingctx, targetctx, library,
    971                               args, return_type, flags, locals)
--> 972     return pipeline.compile_extra(func)
    973 
    974 

/usr/local/lib/python3.7/site-packages/numba/compiler.py in compile_extra(self, func)
    388         self.lifted = ()
    389         self.lifted_from = None
--> 390         return self._compile_bytecode()
    391 
    392     def compile_ir(self, func_ir, lifted=(), lifted_from=None):

/usr/local/lib/python3.7/site-packages/numba/compiler.py in _compile_bytecode(self)
    901         """
    902         assert self.func_ir is None
--> 903         return self._compile_core()
    904 
    905     def _compile_ir(self):

/usr/local/lib/python3.7/site-packages/numba/compiler.py in _compile_core(self)
    888         self.define_pipelines(pm)
    889         pm.finalize()
--> 890         res = pm.run(self.status)
    891         if res is not None:
    892             # Early pipeline completion

/usr/local/lib/python3.7/site-packages/numba/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34 

/usr/local/lib/python3.7/site-packages/numba/compiler.py in run(self, status)
    264                     # No more fallback pipelines?
    265                     if is_final_pipeline:
--> 266                         raise patched_exception
    267                     # Go to next fallback pipeline
    268                     else:

/usr/local/lib/python3.7/site-packages/numba/compiler.py in run(self, status)
    255                 try:
    256                     event("-- %s" % stage_name)
--> 257                     stage()
    258                 except _EarlyPipelineCompletion as e:
    259                     return e.result

/usr/local/lib/python3.7/site-packages/numba/compiler.py in stage_nopython_backend(self)
    762         """
    763         lowerfn = self.backend_nopython_mode
--> 764         self._backend(lowerfn, objectmode=False)
    765 
    766     def stage_compile_interp_mode(self):

/usr/local/lib/python3.7/site-packages/numba/compiler.py in _backend(self, lowerfn, objectmode)
    701             self.library.enable_object_caching()
    702 
--> 703         lowered = lowerfn()
    704         signature = typing.signature(self.return_type, *self.args)
    705         self.cr = compile_result(

/usr/local/lib/python3.7/site-packages/numba/compiler.py in backend_nopython_mode(self)
    688                 self.calltypes,
    689                 self.flags,
--> 690                 self.metadata)
    691 
    692     def _backend(self, lowerfn, objectmode):

/usr/local/lib/python3.7/site-packages/numba/compiler.py in native_lowering_stage(targetctx, library, interp, typemap, restype, calltypes, flags, metadata)
   1141         lower = lowering.Lower(targetctx, library, fndesc, interp,
   1142                                metadata=metadata)
-> 1143         lower.lower()
   1144         if not flags.no_cpython_wrapper:
   1145             lower.create_cpython_wrapper(flags.release_gil)

/usr/local/lib/python3.7/site-packages/numba/lowering.py in lower(self)
    175         if self.generator_info is None:
    176             self.genlower = None
--> 177             self.lower_normal_function(self.fndesc)
    178         else:
    179             self.genlower = self.GeneratorLower(self)

/usr/local/lib/python3.7/site-packages/numba/lowering.py in lower_normal_function(self, fndesc)
    216         # Init argument values
    217         self.extract_function_arguments()
--> 218         entry_block_tail = self.lower_function_body()
    219 
    220         # Close tail of entry block

/usr/local/lib/python3.7/site-packages/numba/lowering.py in lower_function_body(self)
    241             bb = self.blkmap[offset]
    242             self.builder.position_at_end(bb)
--> 243             self.lower_block(block)
    244 
    245         self.post_lower()

/usr/local/lib/python3.7/site-packages/numba/lowering.py in lower_block(self, block)
    256             with new_error_context('lowering "{inst}" at {loc}', inst=inst,
    257                                    loc=self.loc, errcls_=defaulterrcls):
--> 258                 self.lower_inst(inst)
    259 
    260     def create_cpython_wrapper(self, release_gil=False):

/usr/local/Cellar/python/3.7.3/Frameworks/Python.framework/Versions/3.7/lib/python3.7/contextlib.py in __exit__(self, type, value, traceback)
    128                 value = type()
    129             try:
--> 130                 self.gen.throw(type, value, traceback)
    131             except StopIteration as exc:
    132                 # Suppress StopIteration *unless* it's the same exception that

/usr/local/lib/python3.7/site-packages/numba/errors.py in new_error_context(fmt_, *args, **kwargs)
    668         from numba import config
    669         tb = sys.exc_info()[2] if config.FULL_TRACEBACKS else None
--> 670         six.reraise(type(newerr), newerr, tb)
    671 
    672 

/usr/local/lib/python3.7/site-packages/numba/six.py in reraise(tp, value, tb)
    657         if value.__traceback__ is not tb:
    658             raise value.with_traceback(tb)
--> 659         raise value
    660 
    661 else:

LoweringError: Failed in nopython mode pipeline (step: nopython mode backend)
Operands must be the same type, got (i64, i32)

File "<ipython-input-5-1bc7960d5c26>", line 6:
def foo(a, b):
    return np.zeros((a.size, b[0]))
    ^

[1] During: lowering "$0.9 = call $0.2($0.8, func=$0.2, args=[Var($0.8, <ipython-input-5-1bc7960d5c26> (6))], kws=(), vararg=None)" at <ipython-input-5-1bc7960d5c26> (6)
-------------------------------------------------------------------------------
This should not have happened, a problem has occurred in Numba's internals.
You are currently using Numba version 0.45.1.

Please report the error message and traceback, along with a minimal reproducer
at: https://github.com/numba/numba/issues/new

If more help is needed please feel free to speak to the Numba core developers
directly at: https://gitter.im/numba/numba

Thanks in advance for your help in improving Numba!

```

</details>

On our end, I think I need to figure out what assumptions I can make about the dtypes of sparse matrices constituent arrays.
Actually realizing the issue is a little more subtle than that. I think the issue was I had assumed `np.array(..., dtype=np.int)[-1]` has the same dtype as `sparse.csr_matrix(...).indptr.size - 1`. Is this actually an incorrect assumption to make?

[Here's a link to the relevant function in scanpy](https://github.com/theislab/scanpy/blob/26fd7707d4b2881cb07ce507733cc458977238c1/scanpy/preprocessing/_qc.py#L383-L405) and [here's how it get's called](https://github.com/theislab/scanpy/blob/26fd7707d4b2881cb07ce507733cc458977238c1/scanpy/preprocessing/_qc.py#L351-L353).
I imagine that this is a windows vs *nix problem where `np.int` is the same as cpython `int()`. https://github.com/numba/numba/issues/2729 is related as is https://github.com/numpy/numpy/issues/9464.
Seems like a can of worms. I think I'd be fine to just force dtypes on my end (pending being able to confirm that fixes the problem). Alternatively, any chance non-homogenous `shape` tuples could be supported by numba?
https://github.com/numba/numba/issues/4575
Actually, having just opened that ticket, it reminds me that I've half fixed this already here https://github.com/numba/numba/pull/3505, it just gets really involved.
Thank you very much for the information! The data used in the code example is sample data from scanpy. I just tried it and converting the data matrix to a dense array fixed the error.
Does that mean that the default data type for a sparse matrix and an array are different on windows?

Resolving patch:
diff --git a/numba/targets/arrayobj.py b/numba/targets/arrayobj.py
--- a/numba/targets/arrayobj.py
+++ b/numba/targets/arrayobj.py
@@ -3337,8 +3337,13 @@ def _empty_nd_impl(context, builder, arrtype, shapes):
 
     # compute array length
     arrlen = context.get_constant(types.intp, 1)
+    overflow = ir.Constant(ir.IntType(1), 0)
     for s in shapes:
-        arrlen = builder.mul(arrlen, s)
+        arrlen_mult = builder.smul_with_overflow(arrlen, s)
+        arrlen = builder.extract_value(arrlen_mult, 0)
+        overflow = builder.or_(
+            overflow, builder.extract_value(arrlen_mult, 1)
+        )
 
     if arrtype.ndim == 0:
         strides = ()
@@ -3357,7 +3362,20 @@ def _empty_nd_impl(context, builder, arrtype, shapes):
             "Don't know how to allocate array with layout '{0}'.".format(
                 arrtype.layout))
 
-    allocsize = builder.mul(itemsize, arrlen)
+    # Check overflow, numpy also does this after checking order
+    allocsize_mult = builder.smul_with_overflow(arrlen, itemsize)
+    allocsize = builder.extract_value(allocsize_mult, 0)
+    overflow = builder.or_(overflow, builder.extract_value(allocsize_mult, 1))
+
+    with builder.if_then(overflow, likely=False):
+        # Raise same error as numpy, see:
+        # https://github.com/numpy/numpy/blob/2a488fe76a0f732dc418d03b452caace161673da/numpy/core/src/multiarray/ctors.c#L1095-L1101    # noqa: E501
+        context.call_conv.return_user_exc(
+            builder, ValueError,
+            ("array is too big; `arr.size * arr.dtype.itemsize` is larger than"
+             " the maximum possible size.",)
+        )
+
     align = context.get_preferred_array_alignment(arrtype.dtype)
     meminfo = context.nrt.meminfo_alloc_aligned(builder, size=allocsize,
                                                 align=align)
@@ -3389,13 +3407,38 @@ def _parse_shape(context, builder, ty, val):
     """
     Parse the shape argument to an array constructor.
     """
+    def safecast_intp(context, builder, src_t, src):
+        """Cast src to intp only if value can be maintained"""
+        intp_t = context.get_value_type(types.intp)
+        intp_width = intp_t.width
+        intp_ir = ir.IntType(intp_width)
+        maxval = ir.Constant(intp_ir, ((1 << intp_width - 1) - 1))
+        if src_t.width < intp_width:
+            res = builder.sext(src, intp_ir)
+        elif src_t.width >= intp_width:
+            is_larger = builder.icmp_signed(">", src, maxval)
+            with builder.if_then(is_larger, likely=False):
+                context.call_conv.return_user_exc(
+                    builder, ValueError,
+                    ("Cannot safely convert value to intp",)
+                )
+            if src_t.width > intp_width:
+                res = builder.trunc(src, intp_ir)
+            else:
+                res = src
+        return res
+
     if isinstance(ty, types.Integer):
         ndim = 1
-        shapes = [context.cast(builder, val, ty, types.intp)]
+        passed_shapes = [context.cast(builder, val, ty, types.intp)]
     else:
         assert isinstance(ty, types.BaseTuple)
         ndim = ty.count
-        shapes = cgutils.unpack_tuple(builder, val, count=ndim)
+        passed_shapes = cgutils.unpack_tuple(builder, val, count=ndim)
+
+    shapes = []
+    for s in passed_shapes:
+        shapes.append(safecast_intp(context, builder, s.type, s))
 
     zero = context.get_constant_generic(builder, types.intp, 0)
     for dim in range(ndim):


Test patch:
diff --git a/numba/tests/test_dyn_array.py b/numba/tests/test_dyn_array.py
--- a/numba/tests/test_dyn_array.py
+++ b/numba/tests/test_dyn_array.py
@@ -9,7 +9,9 @@
 
 from numba import unittest_support as unittest
 from numba.errors import TypingError
+from numba import config
 from numba import njit
+from numba import types
 from numba import utils
 from numba.numpy_support import version as numpy_version
 from .support import MemoryLeakMixin, TestCase, tag
@@ -575,6 +577,16 @@ def check_2d(self, pyfunc):
             cfunc(2, -1)
         self.assertEqual(str(cm.exception), "negative dimensions not allowed")
 
+    def check_alloc_size(self, pyfunc):
+        """Checks that pyfunc will error, not segfaulting due to array size."""
+        cfunc = nrtjit(pyfunc)
+        with self.assertRaises(ValueError) as e:
+            cfunc()
+        self.assertIn(
+            "array is too big",
+            str(e.exception)
+        )
+
 
 class TestNdZeros(ConstructorBaseTest, TestCase):
 
@@ -617,6 +629,22 @@ def func(m, n):
             return pyfunc((m, n))
         self.check_2d(func)
 
+    def test_2d_shape_dtypes(self):
+        # Test for issue #4575
+        pyfunc = self.pyfunc
+        def func1(m, n):
+            return pyfunc((np.int16(m), np.int32(n)))
+        self.check_2d(func1)
+        # Using a 64-bit value checks that 32 bit systems will downcast to intp
+        def func2(m, n):
+            return pyfunc((np.int64(m), np.int8(n)))
+        self.check_2d(func2)
+        # Make sure an error is thrown if we can't downcast safely
+        if config.IS_32BITS:
+            cfunc = nrtjit(lambda m, n: pyfunc((m, n)))
+            with self.assertRaises(ValueError):
+                cfunc(np.int64(1 << (32 - 1)), 1)
+
     @tag('important')
     def test_2d_dtype_kwarg(self):
         pyfunc = self.pyfunc
@@ -624,6 +652,15 @@ def func(m, n):
             return pyfunc((m, n), dtype=np.complex64)
         self.check_2d(func)
 
+    def test_alloc_size(self):
+        pyfunc = self.pyfunc
+        width = types.intp.bitwidth
+        def gen_func(shape, dtype):
+            return lambda : pyfunc(shape, dtype)
+        # Under these values numba will segfault, but thats another issue
+        self.check_alloc_size(gen_func(1 << width - 2, np.intp))
+        self.check_alloc_size(gen_func((1 << width - 8, 64), np.intp))
+
 
 class TestNdOnes(TestNdZeros):
 
@@ -686,6 +723,29 @@ def func(m, n):
             return np.full((m, n), 1, dtype=np.int8)
         self.check_2d(func)
 
+    def test_2d_shape_dtypes(self):
+        # Test for issue #4575
+        def func1(m, n):
+            return np.full((np.int16(m), np.int32(n)), 4.5)
+        self.check_2d(func1)
+        # Using a 64-bit value checks that 32 bit systems will downcast to intp
+        def func2(m, n):
+            return np.full((np.int64(m), np.int8(n)), 4.5)
+        self.check_2d(func2)
+        # Make sure an error is thrown if we can't downcast safely
+        if config.IS_32BITS:
+            cfunc = nrtjit(lambda m, n: np.full((m, n), 4.5))
+            with self.assertRaises(ValueError):
+                cfunc(np.int64(1 << (32 - 1)), 1)
+
+    def test_alloc_size(self):
+        width = types.intp.bitwidth
+        def gen_func(shape, value):
+            return lambda : np.full(shape, value)
+        # Under these values numba will segfault, but thats another issue
+        self.check_alloc_size(gen_func(1 << width - 2, 1))
+        self.check_alloc_size(gen_func((1 << width - 8, 64), 1))
+
 
 class ConstructorLikeBaseTest(object):