Repository: dask/dask

Issue:
Upstream failures with `idxmaxmin`
I am taking a look now. But these failures just started showing up on upstream CI
```python-traceback
=================================== FAILURES ===================================
__________________________ test_idxmaxmin[idx2-True] ___________________________

idx = DatetimeIndex(['2015-01-01', '2015-01-02', '2015-01-03', '2015-01-04',
               '2015-01-05', '2015-01-06', '201...               '2015-04-07', '2015-04-08', '2015-04-09', '2015-04-10'],
              dtype='datetime64[ns]', freq='D')
skipna = True

    @pytest.mark.parametrize("skipna", [True, False])
    @pytest.mark.parametrize(
        "idx",
        [
            np.arange(100),
            sorted(np.random.random(size=100)),
            pd.date_range("20150101", periods=100),
        ],
    )
    def test_idxmaxmin(idx, skipna):
        df = pd.DataFrame(np.random.randn(100, 5), columns=list("abcde"), index=idx)
        df.b.iloc[31] = np.nan
        df.d.iloc[78] = np.nan
        ddf = dd.from_pandas(df, npartitions=3)
    
        with warnings.catch_warnings(record=True):
            assert_eq(df.idxmax(axis=1, skipna=skipna), ddf.idxmax(axis=1, skipna=skipna))
            assert_eq(df.idxmin(axis=1, skipna=skipna), ddf.idxmin(axis=1, skipna=skipna))
    
            assert_eq(df.idxmax(skipna=skipna), ddf.idxmax(skipna=skipna))
            assert_eq(df.idxmax(skipna=skipna), ddf.idxmax(skipna=skipna, split_every=2))
            assert (
                ddf.idxmax(skipna=skipna)._name
                != ddf.idxmax(skipna=skipna, split_every=2)._name
            )
    
            assert_eq(df.idxmin(skipna=skipna), ddf.idxmin(skipna=skipna))
            assert_eq(df.idxmin(skipna=skipna), ddf.idxmin(skipna=skipna, split_every=2))
            assert (
                ddf.idxmin(skipna=skipna)._name
                != ddf.idxmin(skipna=skipna, split_every=2)._name
            )
    
            assert_eq(df.a.idxmax(skipna=skipna), ddf.a.idxmax(skipna=skipna))
>           assert_eq(
                df.a.idxmax(skipna=skipna), ddf.a.idxmax(skipna=skipna, split_every=2)
            )

dask/dataframe/tests/test_dataframe.py:3497: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
dask/dataframe/utils.py:817: in assert_eq
    b = _check_dask(b, check_names=check_names, check_dtypes=check_dtypes)
dask/dataframe/utils.py:729: in _check_dask
    result = dsk.compute(scheduler="sync")
dask/base.py:284: in compute
    (result,) = compute(self, traverse=False, **kwargs)
dask/base.py:566: in compute
    results = schedule(dsk, keys, **kwargs)
dask/local.py:560: in get_sync
    return get_async(
dask/local.py:503: in get_async
    for key, res_info, failed in queue_get(queue).result():
/usr/share/miniconda3/envs/test-environment/lib/python3.8/concurrent/futures/_base.py:432: in result
    return self.__get_result()
/usr/share/miniconda3/envs/test-environment/lib/python3.8/concurrent/futures/_base.py:388: in __get_result
    raise self._exception
dask/local.py:545: in submit
    fut.set_result(fn(*args, **kwargs))
dask/local.py:237: in batch_execute_tasks
    return [execute_task(*a) for a in it]
dask/local.py:237: in <listcomp>
    return [execute_task(*a) for a in it]
dask/local.py:228: in execute_task
    result = pack_exception(e, dumps)
dask/local.py:223: in execute_task
    result = _execute_task(task, data)
dask/core.py:121: in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
dask/utils.py:34: in apply
    return func(*args, **kwargs)
dask/dataframe/core.py:6522: in idxmaxmin_combine
    x.groupby(level=0)
/usr/share/miniconda3/envs/test-environment/lib/python3.8/site-packages/pandas/core/frame.py:5565: in reset_index
    level = [self.index._get_level_number(lev) for lev in level]
/usr/share/miniconda3/envs/test-environment/lib/python3.8/site-packages/pandas/core/frame.py:5565: in <listcomp>
    level = [self.index._get_level_number(lev) for lev in level]
/usr/share/miniconda3/envs/test-environment/lib/python3.8/site-packages/pandas/core/indexes/base.py:1677: in _get_level_number
    self._validate_index_level(level)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = Int64Index([0], dtype='int64'), level = 1

    @final
    def _validate_index_level(self, level) -> None:
        """
        Validate index level.
    
        For single-level Index getting level number is a no-op, but some
        verification must be done like in MultiIndex.
    
        """
        if isinstance(level, int):
            if level < 0 and level != -1:
                raise IndexError(
                    "Too many levels: Index has only 1 level, "
                    f"{level} is not a valid level number"
                )
            elif level > 0:
>               raise IndexError(
                    f"Too many levels: Index has only 1 level, not {level + 1}"
E                   IndexError: Too many levels: Index has only 1 level, not 2

/usr/share/miniconda3/envs/test-environment/lib/python3.8/site-packages/pandas/core/indexes/base.py:1668: IndexError
__________________________ test_idxmaxmin[idx2-False] __________________________

idx = DatetimeIndex(['2015-01-01', '2015-01-02', '2015-01-03', '2015-01-04',
               '2015-01-05', '2015-01-06', '201...               '2015-04-07', '2015-04-08', '2015-04-09', '2015-04-10'],
              dtype='datetime64[ns]', freq='D')
skipna = False

    @pytest.mark.parametrize("skipna", [True, False])
    @pytest.mark.parametrize(
        "idx",
        [
            np.arange(100),
            sorted(np.random.random(size=100)),
            pd.date_range("20150101", periods=100),
        ],
    )
    def test_idxmaxmin(idx, skipna):
        df = pd.DataFrame(np.random.randn(100, 5), columns=list("abcde"), index=idx)
        df.b.iloc[31] = np.nan
        df.d.iloc[78] = np.nan
        ddf = dd.from_pandas(df, npartitions=3)
    
        with warnings.catch_warnings(record=True):
            assert_eq(df.idxmax(axis=1, skipna=skipna), ddf.idxmax(axis=1, skipna=skipna))
            assert_eq(df.idxmin(axis=1, skipna=skipna), ddf.idxmin(axis=1, skipna=skipna))
    
            assert_eq(df.idxmax(skipna=skipna), ddf.idxmax(skipna=skipna))
            assert_eq(df.idxmax(skipna=skipna), ddf.idxmax(skipna=skipna, split_every=2))
            assert (
                ddf.idxmax(skipna=skipna)._name
                != ddf.idxmax(skipna=skipna, split_every=2)._name
            )
    
            assert_eq(df.idxmin(skipna=skipna), ddf.idxmin(skipna=skipna))
            assert_eq(df.idxmin(skipna=skipna), ddf.idxmin(skipna=skipna, split_every=2))
            assert (
                ddf.idxmin(skipna=skipna)._name
                != ddf.idxmin(skipna=skipna, split_every=2)._name
            )
    
            assert_eq(df.a.idxmax(skipna=skipna), ddf.a.idxmax(skipna=skipna))
>           assert_eq(
                df.a.idxmax(skipna=skipna), ddf.a.idxmax(skipna=skipna, split_every=2)
            )

dask/dataframe/tests/test_dataframe.py:3497: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
dask/dataframe/utils.py:817: in assert_eq
    b = _check_dask(b, check_names=check_names, check_dtypes=check_dtypes)
dask/dataframe/utils.py:729: in _check_dask
    result = dsk.compute(scheduler="sync")
dask/base.py:284: in compute
    (result,) = compute(self, traverse=False, **kwargs)
dask/base.py:566: in compute
    results = schedule(dsk, keys, **kwargs)
dask/local.py:560: in get_sync
    return get_async(
dask/local.py:503: in get_async
    for key, res_info, failed in queue_get(queue).result():
/usr/share/miniconda3/envs/test-environment/lib/python3.8/concurrent/futures/_base.py:432: in result
    return self.__get_result()
/usr/share/miniconda3/envs/test-environment/lib/python3.8/concurrent/futures/_base.py:388: in __get_result
    raise self._exception
dask/local.py:545: in submit
    fut.set_result(fn(*args, **kwargs))
dask/local.py:237: in batch_execute_tasks
    return [execute_task(*a) for a in it]
dask/local.py:237: in <listcomp>
    return [execute_task(*a) for a in it]
dask/local.py:228: in execute_task
    result = pack_exception(e, dumps)
dask/local.py:223: in execute_task
    result = _execute_task(task, data)
dask/core.py:121: in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
dask/utils.py:34: in apply
    return func(*args, **kwargs)
dask/dataframe/core.py:6522: in idxmaxmin_combine
    x.groupby(level=0)
/usr/share/miniconda3/envs/test-environment/lib/python3.8/site-packages/pandas/core/frame.py:5565: in reset_index
    level = [self.index._get_level_number(lev) for lev in level]
/usr/share/miniconda3/envs/test-environment/lib/python3.8/site-packages/pandas/core/frame.py:5565: in <listcomp>
    level = [self.index._get_level_number(lev) for lev in level]
/usr/share/miniconda3/envs/test-environment/lib/python3.8/site-packages/pandas/core/indexes/base.py:1677: in _get_level_number
    self._validate_index_level(level)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = Int64Index([0], dtype='int64'), level = 1

    @final
    def _validate_index_level(self, level) -> None:
        """
        Validate index level.
    
        For single-level Index getting level number is a no-op, but some
        verification must be done like in MultiIndex.
    
        """
        if isinstance(level, int):
            if level < 0 and level != -1:
                raise IndexError(
                    "Too many levels: Index has only 1 level, "
                    f"{level} is not a valid level number"
                )
            elif level > 0:
>               raise IndexError(
                    f"Too many levels: Index has only 1 level, not {level + 1}"
E                   IndexError: Too many levels: Index has only 1 level, not 2

/usr/share/miniconda3/envs/test-environment/lib/python3.8/site-packages/pandas/core/indexes/base.py:1668: IndexError
```


Discussion:
So far I think something is wrong with `split_every`. This test passes if you set the npartitions to 4 which is evenly divisible by 3. 

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -6516,7 +6516,7 @@ def idxmaxmin_row(x, fn=None, skipna=True):
 
 
 def idxmaxmin_combine(x, fn=None, skipna=True):
-    if len(x) == 0:
+    if len(x) <= 1:
         return x
     return (
         x.groupby(level=0)


Test patch: