Repository: scverse/scanpy

Issue:
calculate_qc_metrics error in scanpy 1.4.5 and above
Hi, I was trying to use the most recent version but saw this error in 1.4.5 and above

```
scanpy==1.4.5 anndata==0.7rc2 umap==0.3.10 numpy==1.17.3 scipy==1.4.1 pandas==0.25.3 scikit-learn==0.22.1 statsmodels==0.11.0rc1 python-igraph==0.7.1
```

```
adata = sc.datasets.pbmc3k()
sc.pp.calculate_qc_metrics(adata, inplace=True)
```
output:
```
---------------------------------------------------------------------------
TypingError                               Traceback (most recent call last)
<ipython-input-5-0d8cf2779f18> in <module>
      1 adata = sc.datasets.pbmc3k()
----> 2 sc.pp.calculate_qc_metrics(adata, inplace=True)

~/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py in calculate_qc_metrics(adata, expr_type, var_type, qc_vars, percent_top, layer, use_raw, inplace, parallel)
    281         percent_top=percent_top,
    282         inplace=inplace,
--> 283         X=X,
    284     )
    285     var_metrics = describe_var(

~/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py in describe_obs(adata, expr_type, var_type, qc_vars, percent_top, layer, use_raw, inplace, X, parallel)
    107     if percent_top:
    108         percent_top = sorted(percent_top)
--> 109         proportions = top_segment_proportions(X, percent_top)
    110         for i, n in enumerate(percent_top):
    111             obs_metrics[f"pct_{expr_type}_in_top_{n}_{var_type}"] = (

~/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py in top_segment_proportions(mtx, ns)
    364             mtx = csr_matrix(mtx)
    365         return top_segment_proportions_sparse_csr(
--> 366             mtx.data, mtx.indptr, np.array(ns, dtype=np.int)
    367         )
    368     else:

~/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    399                 e.patch_message(msg)
    400 
--> 401             error_rewrite(e, 'typing')
    402         except errors.UnsupportedError as e:
    403             # Something unsupported is present in the user code, add help info

~/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/numba/dispatcher.py in error_rewrite(e, issue_type)
    342                 raise e
    343             else:
--> 344                 reraise(type(e), e, None)
    345 
    346         argtypes = []

~/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/numba/six.py in reraise(tp, value, tb)
    666             value = tp()
    667         if value.__traceback__ is not tb:
--> 668             raise value.with_traceback(tb)
    669         raise value
    670 

TypingError: Failed in nopython mode pipeline (step: nopython mode backend)
Failed in nopython mode pipeline (step: nopython frontend)
Invalid use of Function(<intrinsic wrap_index>) with argument(s) of type(s): (int32, int64)
 * parameterized
In definition 0:
    ValueError: Argument types for wrap_index must match
    raised from /home/gzhang/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/numba/array_analysis.py:72
In definition 1:
    ValueError: Argument types for wrap_index must match
    raised from /home/gzhang/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/numba/array_analysis.py:72
This error is usually caused by passing an argument of a type that is unsupported by the named function.
[1] During: resolving callee type: Function(<intrinsic wrap_index>)
[2] During: typing of call at /home/gzhang/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py (399)


File "../../../../../packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py", line 399:
def top_segment_proportions_sparse_csr(data, indptr, ns):
    <source elided>
        start, end = indptr[i], indptr[i + 1]
        sums[i] = np.sum(data[start:end])
        ^

[1] During: lowering "id=13[LoopNest(index_variable = parfor_index.271, range = (0, $100.6, 1))]{386: <ir.Block at /home/gzhang/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py (403)>, 388: <ir.Block at /home/gzhang/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py (404)>, 264: <ir.Block at /home/gzhang/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py (401)>, 306: <ir.Block at /home/gzhang/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py (402)>, 118: <ir.Block at /home/gzhang/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py (397)>}Var(parfor_index.271, _qc.py:397)" at /home/gzhang/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py (397)

This is not usually a problem with Numba itself but instead often caused by
the use of unsupported features or an issue in resolving types.

To see Python/NumPy features supported by the latest release of Numba visit:
http://numba.pydata.org/numba-doc/latest/reference/pysupported.html
and
http://numba.pydata.org/numba-doc/latest/reference/numpysupported.html

For more information about typing errors and how to debug them visit:
http://numba.pydata.org/numba-doc/latest/user/troubleshoot.html#my-code-doesn-t-compile

If you think your code should work with Numba, please report the error message
and traceback, along with a minimal reproducer at:
https://github.com/numba/numba/issues/new

```
numba is 0.47.0 but 0.43.1 gave the same error.
It seems that ```top_segment_proportions_sparse_csr``` is new for scanpy 1.4.5

Please help. Thank you very much.


Discussion:
Are you sure about numba 0.43? This very much looks like a bug in numba.

> It seems that `top_segment_proportions_sparse_csr` is new for scanpy 1.4.5

What makes you think that? It’s been there since @ivirshup added `calculate_qc_metrics` in #316.

A second way for this to fail is:

```pytb
NotImplementedError: No definition for lowering UniTuple(int64 x 2).shape
...
numba.errors.LoweringError: Failed in nopython mode pipeline (step: nopython mode backend)
No definition for lowering UniTuple(int64 x 2).shape

File "_qc_.py", line 390:
def top_segment_proportions_sparse_csr(data, indptr, ns):
    <source elided>
    prev = 0
    for j, n in enumerate(ns):
    ^

[1] During: lowering "$phi382.1_shape.158 = getattr(value=$380for_iter.2, attr=shape)" at _qc.py (408)
```
> Are you sure about numba 0.43? This very much looks like a bug in numba.
> 
> > It seems that top_segment_proportions_sparse_csr is new for scanpy 1.4.5
> 
> What makes you think that? It’s been there since @ivirshup added `calculate_qc_metrics` in #316.

Yes. see

```py
print(sc.__version__)

import numba
print(numba.__version__)
```

```
1.4.5
0.43.1
```

```py
adata = sc.datasets.pbmc3k()
sc.pp.calculate_qc_metrics(adata, inplace=True)
```

```pytb
---------------------------------------------------------------------------
TypingError                               Traceback (most recent call last)
<ipython-input-4-c3b3d6ea75c2> in <module>
----> 1 sc.pp.calculate_qc_metrics(adata, inplace=True)

~/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py in calculate_qc_metrics(adata, expr_type, var_type, qc_vars, percent_top, layer, use_raw, inplace, parallel)
    281         percent_top=percent_top,
    282         inplace=inplace,
--> 283         X=X,
    284     )
    285     var_metrics = describe_var(

~/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py in describe_obs(adata, expr_type, var_type, qc_vars, percent_top, layer, use_raw, inplace, X, parallel)
    107     if percent_top:
    108         percent_top = sorted(percent_top)
--> 109         proportions = top_segment_proportions(X, percent_top)
    110         for i, n in enumerate(percent_top):
    111             obs_metrics[f"pct_{expr_type}_in_top_{n}_{var_type}"] = (

~/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/scanpy/preprocessing/_qc.py in top_segment_proportions(mtx, ns)
    364             mtx = csr_matrix(mtx)
    365         return top_segment_proportions_sparse_csr(
--> 366             mtx.data, mtx.indptr, np.array(ns, dtype=np.int)
    367         )
    368     else:

~/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    348                 e.patch_message(msg)
    349 
--> 350             error_rewrite(e, 'typing')
    351         except errors.UnsupportedError as e:
    352             # Something unsupported is present in the user code, add help info

~/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/numba/dispatcher.py in error_rewrite(e, issue_type)
    315                 raise e
    316             else:
--> 317                 reraise(type(e), e, None)
    318 
    319         argtypes = []

~/packages/anaconda3/envs/testscanpy145/lib/python3.6/site-packages/numba/six.py in reraise(tp, value, tb)
    656             value = tp()
    657         if value.__traceback__ is not tb:
--> 658             raise value.with_traceback(tb)
    659         raise value
    660 
```


I was also surprised since this should be the first few functions people run.
```calculate_qc_metrics``` was there for a long time. But ```top_segment_proportions_sparse_csr``` seems to be a new version since 1.4.5- I checked the _qc.py in the tar.gz files. Sorry my previous description was not accurate.
It’s right there in 683b965a38e8ac51342f75e182ac4b9882b3ff39, the commit that introduced `calculate_qc_metrics`, so I still don’t see what’s different in 1.4.5:

https://github.com/theislab/scanpy/blob/683b965a38e8ac51342f75e182ac4b9882b3ff39/scanpy/preprocessing/qc.py#L192-L216
Looks like numba/numba#2625 is involved. Why did they close it?
same error in 1.4.4
```
sc.pp.calculate_qc_metrics(adata, inplace=True, parallel=True)
```
Maybe because sc.pp.calculate_qc_metrics was running in non parallel by default in 1.4.4 and before. the parallel= option has been removed since 1.4.5 and calculate_qc_metrics is running in parallel by default. That's why the error wasn't reported.

I don't know enough about numba.
Aaah, that makes a lot of sense.

I filed numba/numba#5056 and am working on a workaround. Already have one for numba/numba#2625.

Resolving patch:
diff --git a/scanpy/preprocessing/_qc.py b/scanpy/preprocessing/_qc.py
--- a/scanpy/preprocessing/_qc.py
+++ b/scanpy/preprocessing/_qc.py
@@ -388,6 +388,8 @@ def top_segment_proportions_dense(
 
 @numba.njit(cache=True, parallel=True)
 def top_segment_proportions_sparse_csr(data, indptr, ns):
+    # work around https://github.com/numba/numba/issues/5056
+    indptr = indptr.astype(np.int64)
     ns = np.sort(ns)
     maxidx = ns[-1]
     sums = np.zeros((indptr.size - 1), dtype=data.dtype)
@@ -405,8 +407,9 @@ def top_segment_proportions_sparse_csr(data, indptr, ns):
     partitioned = partitioned[:, ::-1][:, :ns[-1]]
     acc = np.zeros((indptr.size - 1), dtype=data.dtype)
     prev = 0
-    for j, n in enumerate(ns):
-        acc += partitioned[:, prev:n].sum(axis=1)
+    # can’t use enumerate due to https://github.com/numba/numba/issues/2625
+    for j in range(ns.size):
+        acc += partitioned[:, prev:ns[j]].sum(axis=1)
         values[:, j] = acc
-        prev = n
+        prev = ns[j]
     return values / sums.reshape((indptr.size - 1, 1))


Test patch:
diff --git a/scanpy/tests/test_qc_metrics.py b/scanpy/tests/test_qc_metrics.py
--- a/scanpy/tests/test_qc_metrics.py
+++ b/scanpy/tests/test_qc_metrics.py
@@ -108,27 +108,30 @@ def test_qc_metrics():
         assert np.allclose(adata.var[col], old_var[col])
 
 
-def test_qc_metrics_format():
+def adata_mito():
     a = np.random.binomial(100, 0.005, (1000, 1000))
     init_var = pd.DataFrame(
         dict(mito=np.concatenate((np.ones(100, dtype=bool), np.zeros(900, dtype=bool))))
     )
     adata_dense = AnnData(X=a, var=init_var.copy())
+    return adata_dense, init_var
+
+
+@pytest.mark.parametrize(
+    "cls", [np.asarray, sparse.csr_matrix, sparse.csc_matrix, sparse.coo_matrix]
+)
+def test_qc_metrics_format(cls):
+    adata_dense, init_var = adata_mito()
     sc.pp.calculate_qc_metrics(adata_dense, qc_vars=["mito"], inplace=True)
-    for fmt in [sparse.csr_matrix, sparse.csc_matrix, sparse.coo_matrix]:
-        adata = AnnData(X=fmt(a), var=init_var.copy())
-        sc.pp.calculate_qc_metrics(adata, qc_vars=["mito"], inplace=True)
-        assert np.allclose(adata.obs, adata_dense.obs)
-        for col in adata.var:  # np.allclose doesn't like mix of types
-            assert np.allclose(adata.var[col], adata_dense.var[col])
+    adata = AnnData(X=cls(adata_dense.X), var=init_var.copy())
+    sc.pp.calculate_qc_metrics(adata, qc_vars=["mito"], inplace=True)
+    assert np.allclose(adata.obs, adata_dense.obs)
+    for col in adata.var:  # np.allclose doesn't like mix of types
+        assert np.allclose(adata.var[col], adata_dense.var[col])
 
 
 def test_qc_metrics_percentage():  # In response to #421
-    a = np.random.binomial(100, 0.005, (1000, 1000))
-    init_var = pd.DataFrame(
-        dict(mito=np.concatenate((np.ones(100, dtype=bool), np.zeros(900, dtype=bool))))
-    )
-    adata_dense = AnnData(X=a, var=init_var.copy())
+    adata_dense, init_var = adata_mito()
     sc.pp.calculate_qc_metrics(adata_dense, percent_top=[])
     sc.pp.calculate_qc_metrics(adata_dense, percent_top=())
     sc.pp.calculate_qc_metrics(adata_dense, percent_top=None)