Repository: pandas-dev/pandas
Pull request: ENH: support Ellipsis in loc/iloc
- [ ] closes #xxxx
- [x] tests added / passed
- [x] passes `black pandas`
- [ ] passes `git diff upstream/master -u -- "*.py" | flake8 --diff`
- [x] whatsnew entry

closes #10956

Problem statement:
support Ellipsis (...) in pandas indexing
Indexing like `x[..., 0]` is a convenient way to indexing the last dimension of an array with NumPy. `...` gets turned into an `Ellipsis` object, which NumPy expands to the appropriate number of full slice objects.

It would be nice to support this in pandas, too. Currently, attempting to do this throws KeyError:

```
>>> df[...]
KeyError: Ellipsis
```

Hints from issue discussion:
Can you give some usecases?

Because for direct getitem indexing on a dataframe, with a single item it already does index in the last dimension, so `df[..., 'col']` doesn't seem very useful.

I could imagine it being useful in multi-indexing, eg with multiple levels of row indices, being able to easily select in the last level: `df.loc[pd.IndexSlice[..., 'label'], :]`

The most compelling use case for me is with `.iloc` and `.loc`, e.g., `df.iloc[..., 0]`, for users who are writing indexing code that they want to work on data structures of any dimensionality.

I agree this is not so common/useful with pandas itself but it would be nice for cross-compatibility with NumPy and xray.

A workaround is to reset the index and reshuffle the order such that the relevant indices are first in the order



Will #33616 enable ellipsis indexing also for `.loc`?
My current workaround is:
```
df.loc[:, (*(df.columns.nlevels - 1) * [slice(None)], 'key')]
```
I know that this is a really cumbersome workaround, but perhaps it will help someone until the current v1.1 ETA. :)

Where I need it: Inplace operations such as `*=` or overwriting specific values with nan, when applying these operations only to a subset of columns in a MultiIndex DataFrame. Of course I can select these subsets with f.i. `df.xs('key', axis=1, level=mysubsetlevel)`, but assigning values like this is not possible. Is there also a subtle way to do this for assigning values?
@JoElfner this is an open issue; if you want it fixed then a PR would do it

Commit message: Merge branch 'master' of https://github.com/pandas-dev/pandas into enh-indexing-ellipses

Patch to review:
diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py
index c5e331a104726..489e7c11819a9 100644
--- a/pandas/core/indexing.py
+++ b/pandas/core/indexing.py
@@ -1,5 +1,5 @@
 from contextlib import suppress
-from typing import TYPE_CHECKING, Any, Hashable, List, Sequence, Tuple, Union
+from typing import TYPE_CHECKING, Hashable, List, Tuple, Union
 import warnings
 
 import numpy as np
@@ -705,11 +705,31 @@ def _validate_key(self, key, axis: int):
         """
         raise AbstractMethodError(self)
 
+    def _expand_ellipsis(self, tup: Tuple) -> Tuple:
+        """
+        If a tuple key includes an Ellipsis, replace it with an appropriate
+        number of null slices.
+        """
+        if any(x is Ellipsis for x in tup):
+
+            if len(tup) == self.ndim:
+                # It is unambiguous what axis this Ellipsis is indexing,
+                #  treat as a single null slice.
+                i = tup.index(Ellipsis)
+                # FIXME: this assumes only one Ellipsis
+                new_key = tup[:i] + (_NS,) + tup[i + 1 :]
+                return new_key
+
+            # TODO: other cases?  only one test gets here, and that is covered
+            #  by _validate_key_length
+        return tup
+
     def _has_valid_tuple(self, key: Tuple):
         """
         Check the key for valid keys across my indexer.
         """
-        self._validate_key_length(key)
+        key = self._validate_key_length(key)
+        key = self._expand_ellipsis(key)
         for i, k in enumerate(key):
             try:
                 self._validate_key(k, i)
@@ -718,6 +738,7 @@ def _has_valid_tuple(self, key: Tuple):
                     "Location based indexing can only have "
                     f"[{self._valid_types}] types"
                 ) from err
+        return key
 
     def _is_nested_tuple_indexer(self, tup: Tuple) -> bool:
         """
@@ -748,9 +769,14 @@ def _convert_tuple(self, key, is_setter: bool = False):
 
         return tuple(keyidx)
 
-    def _validate_key_length(self, key: Sequence[Any]) -> None:
+    def _validate_key_length(self, key: Tuple) -> Tuple:
         if len(key) > self.ndim:
+            if key[0] is Ellipsis:
+                # e.g. Series.iloc[..., 3] reduces to just Series.iloc[3]
+                key = key[1:]
+                return self._validate_key_length(key)
             raise IndexingError("Too many indexers")
+        return key
 
     def _getitem_tuple_same_dim(self, tup: Tuple):
         """
@@ -790,7 +816,7 @@ def _getitem_lowerdim(self, tup: Tuple):
             with suppress(IndexingError):
                 return self._handle_lowerdim_multi_index_axis0(tup)
 
-        self._validate_key_length(tup)
+        tup = self._validate_key_length(tup)
 
         for i, key in enumerate(tup):
             if is_label_like(key):
@@ -1049,10 +1075,11 @@ def _getitem_iterable(self, key, axis: int):
 
     def _getitem_tuple(self, tup: Tuple):
         with suppress(IndexingError):
+            tup = self._expand_ellipsis(tup)
             return self._getitem_lowerdim(tup)
 
         # no multi-index, so validate all of the indexers
-        self._has_valid_tuple(tup)
+        tup = self._has_valid_tuple(tup)
 
         # ugly hack for GH #836
         if self._multi_take_opportunity(tup):
@@ -1447,7 +1474,7 @@ def _validate_integer(self, key: int, axis: int) -> None:
 
     def _getitem_tuple(self, tup: Tuple):
 
-        self._has_valid_tuple(tup)
+        tup = self._has_valid_tuple(tup)
         with suppress(IndexingError):
             return self._getitem_lowerdim(tup)
 
@@ -2329,7 +2356,11 @@ def is_label_like(key) -> bool:
     bool
     """
     # select a label or row
-    return not isinstance(key, slice) and not is_list_like_indexer(key)
+    return (
+        not isinstance(key, slice)
+        and not is_list_like_indexer(key)
+        and key is not Ellipsis
+    )
 
 
 def need_slice(obj) -> bool:
diff --git a/pandas/tests/indexing/test_loc.py b/pandas/tests/indexing/test_loc.py
index 0d40b5f38e48a..71cd270845814 100644
--- a/pandas/tests/indexing/test_loc.py
+++ b/pandas/tests/indexing/test_loc.py
@@ -1126,6 +1126,27 @@ def test_loc_setitem_listlike_with_timedelta64index(self, indexer, expected):
 
         tm.assert_frame_equal(expected, df)
 
+    def test_loc_getitem_ellipses_series(self):
+        ser = Series(range(5))
+
+        result = ser.loc[..., [3]]
+        expected = ser.loc[[3]]
+        tm.assert_series_equal(result, expected)
+
+        # also check iloc while we're here
+        result = ser.iloc[..., [3]]
+        expected = ser.iloc[[3]]
+        tm.assert_series_equal(result, expected)
+
+        df = ser.to_frame()
+        result = df.loc[..., [0]]
+        expected = df.loc[:, [0]]
+        tm.assert_frame_equal(result, expected)
+
+        result = df.iloc[..., [0]]
+        expected = df.iloc[:, [0]]
+        tm.assert_frame_equal(result, expected)
+
 
 class TestLocWithMultiIndex:
     @pytest.mark.parametrize(


Code review comments:
diff_hunk: @@ -705,11 +705,31 @@ def _validate_key(self, key, axis: int):
         """
         raise AbstractMethodError(self)
 
+    def _expand_ellipsis(self, tup: Tuple) -> Tuple:
+        """
+        If a tuple key includes an Ellipsis, replace it with an appropriate
+        number of null slices.
+        """
+        if any(x is Ellipsis for x in tup):
+
+            if len(tup) == self.ndim:
+                # It is unambiguous what axis this Ellipsis is indexing,
+                #  treat as a single null slice.
+                i = tup.index(Ellipsis)
+                # FIXME: this assumes only one Ellipsis
+                new_key = tup[:i] + (_NS,) + tup[i + 1 :]
+                return new_key
+
+            # TODO: other cases?  only one test gets here, and that is covered
+            #  by _validate_key_length
+        return tup
+
     def _has_valid_tuple(self, key: Tuple):
path: pandas/core/indexing.py
text: @user1:
Can you rename this method if it now actually returns a modified key?