Repository: scverse/scanpy

Issue:
sc.pl.matrixplot index error
<!-- Please give a clear and concise description of what the bug is: -->
Since I update to scanpy==1.4.5.1 I am getting multiple plotting errors. `sc.pl.rank_genes_groups() and sc.pl.violin()` are still working fine but I am getting errors in the rank_genes functions like `sc.pl.rank_genes_groups_violin(`) and `sc.pl.tracksplot()`

<!-- Put a minimal reproducible example that reproduces the bug in the code block below: -->
```python
##still working fine
sc.tl.rank_genes_groups(adata, 'louvain', method='wilcoxon')
sc.pl.rank_genes_groups(adata, n_genes=25, sharey=False)
result = adata.uns['rank_genes_groups']
groups = result['names'].dtype.names

pd.DataFrame(
    {group + '_' + key[:1]: result[key][group]
    for group in groups for key in ['names', 'pvals']}).head(5)
##gives error
sc.pl.rank_genes_groups_violin(adata, groups='2', n_genes=8)

```

<!-- Put your Error output in this code block (if applicable, else delete the block): -->
```pytb
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
~/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py in _ensure_valid_index(self, value)
   3169             try:
-> 3170                 value = Series(value)
   3171             except:

~/anaconda3/lib/python3.7/site-packages/pandas/core/series.py in __init__(self, data, index, dtype, name, copy, fastpath)
    273                 data = _sanitize_array(data, index, dtype, copy,
--> 274                                        raise_cast_failure=True)
    275 

~/anaconda3/lib/python3.7/site-packages/pandas/core/series.py in _sanitize_array(data, index, dtype, copy, raise_cast_failure)
   4160         if isinstance(data, np.ndarray):
-> 4161             raise Exception('Data must be 1-dimensional')
   4162         else:

Exception: Data must be 1-dimensional

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-23-ccdbf8b7836c> in <module>
----> 1 sc.pl.rank_genes_groups_violin(adata, groups='2', n_genes=8) ## 200316 error fix later, also when I run the entire script from the start

~/anaconda3/lib/python3.7/site-packages/scanpy/plotting/_tools/__init__.py in rank_genes_groups_violin(adata, groups, n_genes, gene_names, gene_symbols, use_raw, key, split, scale, strip, jitter, size, ax, show, save)
    727             if issparse(X_col): X_col = X_col.toarray().flatten()
    728             new_gene_names.append(g)
--> 729             df[g] = X_col
    730         df['hue'] = adata.obs[groups_key].astype(str).values
    731         if reference == 'rest':

~/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py in __setitem__(self, key, value)
   3114         else:
   3115             # set column
-> 3116             self._set_item(key, value)
   3117 
   3118     def _setitem_slice(self, key, value):

~/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py in _set_item(self, key, value)
   3188         """
   3189 
-> 3190         self._ensure_valid_index(value)
   3191         value = self._sanitize_column(key, value)
   3192         NDFrame._set_item(self, key, value)

~/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py in _ensure_valid_index(self, value)
   3170                 value = Series(value)
   3171             except:
-> 3172                 raise ValueError('Cannot set a frame with no defined index '
   3173                                  'and a value that cannot be converted to a '
   3174                                  'Series')

ValueError: Cannot set a frame with no defined index and a value that cannot be converted to a Series
```

#### Versions:
<!-- Output of scanpy.logging.print_versions() -->
> scanpy==1.4.5.1 anndata==0.7.1 umap==0.3.10 numpy==1.18.1 scipy==1.4.1 pandas==0.23.0 scikit-learn==0.21.3 statsmodels==0.11.0 python-igraph==0.7.1 louvain==0.6.1
scvelo==0.1.25  scanpy==1.4.5.1  anndata==0.7.1  loompy==3.0.6  numpy==1.18.1  scipy==1.4.1  matplotlib==3.1.3  sklearn==0.21.3  pandas==0.23.0  



Discussion:
Hi,

I just tried with a minimum reproducible example and it seemed to work:
```python
sc.__version__
>>> '1.4.5.1'
```
```python
adata = sc.datasets.pbmc68k_reduced()
sc.pp.neighbors(adata)
sc.tl.louvain(adata)
sc.tl.rank_genes_groups(adata, 'louvain', method='wilcoxon')
sc.pl.rank_genes_groups_violin(adata, groups='2', n_genes=8)
```
![image](https://user-images.githubusercontent.com/25887487/76887535-b7635900-6882-11ea-9a1c-65bd7d2e6721.png)

Could you provide more inputs on the anndata object?

Also, does upgrading pandas help?

<details>
<summary> Versions</summary>

```python
anndata==0.7.1
appnope==0.1.0
attrs==19.3.0
backcall==0.1.0
bleach==3.1.0
certifi==2019.11.28
cycler==0.10.0
decorator==4.4.2
defusedxml==0.6.0
entrypoints==0.3
get-version==2.1
h5py==2.10.0
importlib-metadata==1.5.0
ipykernel==5.1.4
ipython==7.13.0
ipython-genutils==0.2.0
jedi==0.16.0
Jinja2==2.11.1
joblib==0.14.1
json5==0.9.1
jsonschema==3.2.0
jupyter-client==5.3.4
jupyter-core==4.6.1
jupyterlab==1.2.6
jupyterlab-server==1.0.6
kiwisolver==1.1.0
legacy-api-wrap==1.2
leidenalg==0.7.0
llvmlite==0.31.0
louvain==0.6.1
MarkupSafe==1.1.1
matplotlib==3.2.0
mistune==0.8.4
natsort==7.0.1
nbconvert==5.6.1
nbformat==5.0.4
networkx==2.4
notebook==6.0.3
numba==0.48.0
numexpr==2.7.1
numpy==1.18.2
packaging==20.3
pandas==1.0.2
pandocfilters==1.4.2
parso==0.6.1
patsy==0.5.1
pexpect==4.8.0
pickleshare==0.7.5
prometheus-client==0.7.1
prompt-toolkit==3.0.3
ptyprocess==0.6.0
pycairo==1.19.0
Pygments==2.5.2
pyparsing==2.4.6
pyrsistent==0.15.7
python-dateutil==2.8.1
python-igraph==0.7.1.post7
pytz==2019.3
pyzmq==18.1.1
scanpy==1.4.5.1
scikit-learn==0.22.2.post1
scipy==1.4.1
seaborn==0.10.0
Send2Trash==1.5.0
setuptools-scm==3.5.0
six==1.14.0
statsmodels==0.11.1
tables==3.6.1
terminado==0.8.3
testpath==0.4.4
tornado==6.0.4
tqdm==4.43.0
traitlets==4.3.3
umap-learn==0.3.10
wcwidth==0.1.8
webencodings==0.5.1
zipp==2.2.0
```

</details>

Thanks @giovp for your quick reply! I upgraded pandas and ran your code with the pbmc dataset. This ran fine. On my own dataset it is still giving the same error. So maybe something is wrong with the way I created adata. Because my code ran fine before upgrading scanpy and I found this issue: https://github.com/theislab/single-cell-tutorial/issues/28#issue-576248363 I thought it might be a real bug. 
After running your example I will just look into how I created adata to see if I can find the error. 
This is what it looks like now: 
```
AnnData object with n_obs × n_vars = 2773 × 3783 
    obs: 'n_genes', 'plate', 'platebatch', 'stage', 'well_no', 'ERCC_genes', 'n_total_counts', 'percent_mito', 'n_counts', 'percent_ribo', 'percent_protein_coding', 'percent_lincRNA', 'sum_lincRNA', 'percent_antisense', 'sum_antisense', 'percent_miRNA', 'sum_miRNA', 'percent_bidirectional_promoter_lncRNA', 'sum_bidirectional_promoter_lncRNA', 'percent_snoRNA', 'n_counts_norm', 'Chat_norm_expr', 'cellnr', 'louvain', 'velocity_self_transition', 'lineages', 'root_cells', 'end_points', 'velocity_pseudotime'
    var: 'ENS_names', 'geneid', 'feature', 'chr', 'fullname', 'n_cells', 'highly_variable', 'means', 'dispersions', 'dispersions_norm', 'velocity_gamma', 'velocity_r2', 'velocity_genes'
    uns: 'louvain', 'louvain_colors', 'neighbors', 'pca', 'plate_colors', 'stage_colors', 'umap', 'velocity_graph', 'velocity_graph_neg', 'velocity_settings', 'rank_genes_groups'
    obsm: 'X_pca', 'X_tsne', 'X_umap', 'velocity_tsne', 'velocity_umap'
    varm: 'PCs'
    layers: 'Ms', 'Mu', 'spliced', 'unspliced', 'variance_velocity', 'velocity'
```

The adata.X of the pbmc data is `scipy.sparse.csr.csr_matrix`
My adata.X is `numpy.ndarray`

This probably results in the problem of the difference in dimensions.
@fidelram, any thoughts on this?

@liekevandehaar is this still occurring? Are you using the newest version of `anndata`?
Yes I am still getting this error, my version of anndata==0.7.1
I'm experiencing a similar issue with my data. Like in the case of @liekevandehaar , sc.pl.rank_genes_groups() and sc.pl.violin() work fine, but I am getting the same exception saying 'Data must be 1-dimensional' when I attempt to run sc.pl.rank_genes_groups_violin. However, unlike in her case, sc.pl.tracksplot() works fine for me.
```

sc.pl.rank_genes_groups_violin(tmp,
                               groups='0',
                               n_genes=8)

Traceback (most recent call last):

  File "<ipython-input-126-3db850431424>", line 3, in <module>
    n_genes=8)

  File "/opt/anaconda3/lib/python3.7/site-packages/scanpy/plotting/_tools/__init__.py", line 729, in rank_genes_groups_violin
    df[g] = X_col

  File "/opt/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py", line 3487, in __setitem__
    self._set_item(key, value)

  File "/opt/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py", line 3563, in _set_item
    self._ensure_valid_index(value)

  File "/opt/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py", line 3540, in _ensure_valid_index
    value = Series(value)

  File "/opt/anaconda3/lib/python3.7/site-packages/pandas/core/series.py", line 314, in __init__
    data = sanitize_array(data, index, dtype, copy, raise_cast_failure=True)

  File "/opt/anaconda3/lib/python3.7/site-packages/pandas/core/internals/construction.py", line 729, in sanitize_array
    raise Exception("Data must be 1-dimensional")

Exception: Data must be 1-dimensional

```
Btw, I can confirm what @liekevandehaar has observed. I am experiencing the issue with a data set in which adata.X was initially imported as a dense matrix, but NOT with another data set, in which adata.X was imported as a sparse matrix.
Can you provide an example to reproduce. From this issue #28 it seems to be related to dense matrices. Can try transforming `adata=sc.datasets.pbmc68k_reduced()` to see if you trigger the error.
same issue with dense data:
AnnData object with n_obs × n_vars = 1008 × 30810
    obs: 'batch', 'n_genes_by_counts', 'total_counts', 'total_counts_mt', 'pct_counts_mt', 'leiden'
    var: 'mt', 'n_cells_by_counts', 'mean_counts', 'pct_dropout_by_counts', 'total_counts', 'highly_variable', 'means', 'dispersions', 'dispersions_norm', 'mean', 'std'
    uns: 'log1p', 'hvg', 'pca', 'neighbors', 'leiden', 'paga', 'leiden_sizes', 'leiden_colors', 'umap', 'rank_genes_groups'
    obsm: 'X_pca', 'X_umap'
    varm: 'PCs'
    obsp: 'distances', 'connectivities'
We (mainly my student :smile: ) also gets this problem now, minimal reproducable example with data from https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE134432:

```
import pandas as pd
import scanpy as sc
import anndata

A375 = sc.read_csv("GSM3946506_10x_A375_BL.tsv", delimiter="\t").transpose()
MM001 = sc.read_csv("GSM3946506_10x_MM001_BL.tsv", delimiter="\t").transpose()
MM011 = sc.read_csv("GSM3946506_10x_MM011_BL.tsv", delimiter="\t").transpose()
MM029 = sc.read_csv("GSM3946506_10x_MM029_BL.tsv", delimiter="\t").transpose()
MM031 = sc.read_csv("GSM3946506_10x_MM031_BL.tsv", delimiter="\t").transpose()
MM047 = sc.read_csv("GSM3946506_10x_MM047_BL.tsv", delimiter="\t").transpose()
MM057 = sc.read_csv("GSM3946506_10x_MM057_BL.tsv", delimiter="\t").transpose()
MM074 = sc.read_csv("GSM3946506_10x_MM074_BL.tsv", delimiter="\t").transpose()
MM087 = sc.read_csv("GSM3946506_10x_MM087_BL.tsv", delimiter="\t").transpose()
MM099 = sc.read_csv("GSM3946506_10x_MM099_BL.tsv", delimiter="\t").transpose()

AnnDatas = [A375, MM001, MM011, MM029, MM031, MM047, MM057, MM074, MM087, MM099]
AnnData_names = ["A375", "MM001", "MM011", "MM029", "MM031", "MM047", "MM057", "MM074", "MM087", "MM099"]

All = anndata.concat(AnnDatas, label="batch", keys=AnnData_names)

sc.tl.rank_genes_groups(All, 'batch', groups=['A375'], reference='MM001', method='wilcoxon')
sc.pl.rank_genes_groups(All, groups=['A375'], n_genes=20)

sc.pl.rank_genes_groups_violin(All, groups='A375', n_genes=8)
```

Which results in:

```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/scratch/estroucken/miniconda3/envs/scepia/lib/python3.8/site-packages/pandas/core/frame.py in _ensure_valid_index(self, value)
   3291             try:
-> 3292                 value = Series(value)
   3293             except (ValueError, NotImplementedError, TypeError) as err:

/scratch/estroucken/miniconda3/envs/scepia/lib/python3.8/site-packages/pandas/core/series.py in __init__(self, data, index, dtype, name, copy, fastpath)
    363             else:
--> 364                 data = sanitize_array(data, index, dtype, copy, raise_cast_failure=True)
    365 

/scratch/estroucken/miniconda3/envs/scepia/lib/python3.8/site-packages/pandas/core/construction.py in sanitize_array(data, index, dtype, copy, raise_cast_failure)
    528         if isinstance(data, np.ndarray):
--> 529             raise ValueError("Data must be 1-dimensional")
    530         else:

ValueError: Data must be 1-dimensional

The above exception was the direct cause of the following exception:

ValueError                                Traceback (most recent call last)
<ipython-input-6-edeb6431ddac> in <module>
----> 1 sc.pl.rank_genes_groups_violin(All, groups='A375', n_genes=8)

/scratch/estroucken/miniconda3/envs/scepia/lib/python3.8/site-packages/scanpy/plotting/_tools/__init__.py in rank_genes_groups_violin(adata, groups, n_genes, gene_names, gene_symbols, use_raw, key, split, scale, strip, jitter, size, ax, show, save)
    911                 X_col = X_col.toarray().flatten()
    912             new_gene_names.append(g)
--> 913             df[g] = X_col
    914         df['hue'] = adata.obs[groups_key].astype(str).values
    915         if reference == 'rest':

/scratch/estroucken/miniconda3/envs/scepia/lib/python3.8/site-packages/pandas/core/frame.py in __setitem__(self, key, value)
   3161         else:
   3162             # set column
-> 3163             self._set_item(key, value)
   3164 
   3165     def _setitem_slice(self, key: slice, value):

/scratch/estroucken/miniconda3/envs/scepia/lib/python3.8/site-packages/pandas/core/frame.py in _set_item(self, key, value)
   3239         ensure homogeneity.
   3240         """
-> 3241         self._ensure_valid_index(value)
   3242         value = self._sanitize_column(key, value)
   3243         NDFrame._set_item(self, key, value)

/scratch/estroucken/miniconda3/envs/scepia/lib/python3.8/site-packages/pandas/core/frame.py in _ensure_valid_index(self, value)
   3292                 value = Series(value)
   3293             except (ValueError, NotImplementedError, TypeError) as err:
-> 3294                 raise ValueError(
   3295                     "Cannot set a frame with no defined index "
   3296                     "and a value that cannot be converted to a Series"

ValueError: Cannot set a frame with no defined index and a value that cannot be converted to a Series
```
@Maarten-vd-Sande, it looks like the issue here is a bug with handling dense data. With your particular example here, you should be fine to work around this by using a sparse matrix.

```python
from scipy import sparse
All.X = sparse.csr_matrix(All.X)
```

Resolving patch:
diff --git a/scanpy/tools/_rank_genes_groups.py b/scanpy/tools/_rank_genes_groups.py
--- a/scanpy/tools/_rank_genes_groups.py
+++ b/scanpy/tools/_rank_genes_groups.py
@@ -429,7 +429,7 @@ def compute_statistics(
 def rank_genes_groups(
     adata: AnnData,
     groupby: str,
-    use_raw: bool = True,
+    use_raw: Optional[bool] = None,
     groups: Union[Literal['all'], Iterable[str]] = 'all',
     reference: str = 'rest',
     n_genes: Optional[int] = None,
@@ -533,6 +533,11 @@ def rank_genes_groups(
     >>> # to visualize the results
     >>> sc.pl.rank_genes_groups(adata)
     """
+    if use_raw is None:
+        use_raw = adata.raw is not None
+    elif use_raw is True and adata.raw is not None:
+        raise ValueError("Received `use_raw=True`, but `adata.raw` is empty.")
+
     if method is None:
         logg.warning(
             "Default of the method has been changed to 't-test' from 't-test_overestim_var'"


Test patch:
diff --git a/scanpy/tests/test_rank_genes_groups.py b/scanpy/tests/test_rank_genes_groups.py
--- a/scanpy/tests/test_rank_genes_groups.py
+++ b/scanpy/tests/test_rank_genes_groups.py
@@ -73,6 +73,7 @@ def test_results_dense():
     seed(1234)
 
     adata = get_example_data()
+    assert adata.raw is None  # Assumption for later checks
 
     (
         true_names_t_test,
@@ -92,6 +93,7 @@ def test_results_dense():
             true_scores_t_test[name], adata.uns['rank_genes_groups']['scores'][name]
         )
     assert np.array_equal(true_names_t_test, adata.uns['rank_genes_groups']['names'])
+    assert adata.uns["rank_genes_groups"]["params"]["use_raw"] is False
 
     rank_genes_groups(adata, 'true_groups', n_genes=20, method='wilcoxon')
 
@@ -107,6 +109,7 @@ def test_results_dense():
     assert np.array_equal(
         true_names_wilcoxon[:7], adata.uns['rank_genes_groups']['names'][:7]
     )
+    assert adata.uns["rank_genes_groups"]["params"]["use_raw"] is False
 
 
 def test_results_sparse():
@@ -132,6 +135,7 @@ def test_results_sparse():
             true_scores_t_test[name], adata.uns['rank_genes_groups']['scores'][name]
         )
     assert np.array_equal(true_names_t_test, adata.uns['rank_genes_groups']['names'])
+    assert adata.uns["rank_genes_groups"]["params"]["use_raw"] is False
 
     rank_genes_groups(adata, 'true_groups', n_genes=20, method='wilcoxon')
 
@@ -147,6 +151,7 @@ def test_results_sparse():
     assert np.array_equal(
         true_names_wilcoxon[:7], adata.uns['rank_genes_groups']['names'][:7]
     )
+    assert adata.uns["rank_genes_groups"]["params"]["use_raw"] is False
 
 
 def test_results_layers():
@@ -169,9 +174,9 @@ def test_results_layers():
         'true_groups',
         method='wilcoxon',
         layer="to_test",
-        use_raw=False,
         n_genes=20,
     )
+    assert adata.uns["rank_genes_groups"]["params"]["use_raw"] is False
     for name in true_scores_t_test.dtype.names:
         assert np.allclose(
             true_scores_wilcoxon[name][:7],
@@ -236,6 +241,7 @@ def test_wilcoxon_symmetry():
         method='wilcoxon',
         rankby_abs=True,
     )
+    assert pbmc.uns["rank_genes_groups"]["params"]["use_raw"] is True
 
     stats_mono = (
         rank_genes_groups_df(pbmc, group="CD14+ Monocyte")