Repository: pandas-dev/pandas

Issue:
Unexpected output for `nlargest` function with multiple columns
#### Code Sample

```python
import pandas as pd

aas = [2,2,2,1,1,1]
bbs = [1,2,3,3,2,1]
n = 4

df = pd.DataFrame({'a': aas, 'b':  bbs})

print('-- First --')
nlargest = df.nlargest(n, columns=['a', 'b']).sort_values(['a', 'b'], ascending=False)
print(nlargest)

print('-- Second --')
pseudo_nlargest = df.sort_values(['a', 'b'], ascending=False).head(n)
print(pseudo_nlargest)
```

#### Actual Output

```
-- First --
   a  b
2  2  3
1  2  2
3  1  3
4  1  2
-- Second --
   a  b
2  2  3 <same>
1  2  2 <same>
0  2  1 <different!>
3  1  3 <different!>
```
_Text within square brackets added to call attention to rows with unexpected output._

#### Problem description

According to [the documentation for nlargest](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.nlargest.html#pandas-dataframe-nlargest), the `nlargest` function should function identically to `df.sort_values(columns, ascending=False).head(n)` but be more performant. Presumably this is more performant due to not needing to sort the entire dataframe.

I am observing different behavior. In the example above, I expect the first and second dataframes to be the same in both indices and values. (Note that I've sorted the output of the `nlargest` function to remove sort order as a difference). 

#### Similar issues, but different enough that I opened a new one

https://github.com/pandas-dev/pandas/issues/21426 - Deals with unsigned ints, this issue uses signed `int64`s.
https://github.com/pandas-dev/pandas/issues/19563 - Different by sort order only, this issue is different in that the rows themselves are a different, non-unique subset of the original rows.


#### Expected Output

```
-- First --
   a  b
2  2  3
1  2  2
0  2  1
3  1  3
-- Second --
   a  b
2  2  3 <same>
1  2  2 <same>
0  2  1 <same>
3  1  3 <same>
```

#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: None
python: 3.5.2.final.0
python-bits: 64
OS: Linux
OS-release: 4.15.0-34-generic
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: None
LOCALE: None.None

pandas: 0.23.4
pytest: None
pip: 9.0.3
setuptools: 40.4.1
Cython: None
numpy: 1.15.1
scipy: 1.1.0
pyarrow: None
xarray: None
IPython: 6.5.0
sphinx: None
patsy: 0.5.0
dateutil: 2.7.3
pytz: 2018.5
blosc: None
bottleneck: None
tables: None
numexpr: None
feather: None
matplotlib: 2.2.3
openpyxl: 2.5.7
xlrd: 1.1.0
xlwt: None
xlsxwriter: None
lxml: None
bs4: None
html5lib: 1.0.1
sqlalchemy: 1.2.11
pymysql: None
psycopg2: 2.7.5 (dt dec pq3 ext lo64)
jinja2: 2.10
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: None

</details>


Discussion:

Resolving patch:
diff --git a/asv_bench/benchmarks/frame_methods.py b/asv_bench/benchmarks/frame_methods.py
--- a/asv_bench/benchmarks/frame_methods.py
+++ b/asv_bench/benchmarks/frame_methods.py
@@ -505,14 +505,21 @@ class NSort(object):
     param_names = ['keep']
 
     def setup(self, keep):
-        self.df = DataFrame(np.random.randn(1000, 3), columns=list('ABC'))
+        self.df = DataFrame(np.random.randn(100000, 3),
+                            columns=list('ABC'))
 
-    def time_nlargest(self, keep):
+    def time_nlargest_one_column(self, keep):
         self.df.nlargest(100, 'A', keep=keep)
 
-    def time_nsmallest(self, keep):
+    def time_nlargest_two_columns(self, keep):
+        self.df.nlargest(100, ['A', 'B'], keep=keep)
+
+    def time_nsmallest_one_column(self, keep):
         self.df.nsmallest(100, 'A', keep=keep)
 
+    def time_nsmallest_two_columns(self, keep):
+        self.df.nsmallest(100, ['A', 'B'], keep=keep)
+
 
 class Describe(object):
 
diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py
--- a/pandas/core/algorithms.py
+++ b/pandas/core/algorithms.py
@@ -1214,41 +1214,56 @@ def get_indexer(current_indexer, other_indexer):
         indexer = Int64Index([])
 
         for i, column in enumerate(columns):
-
             # For each column we apply method to cur_frame[column].
-            # If it is the last column in columns, or if the values
-            # returned are unique in frame[column] we save this index
-            # and break
-            # Otherwise we must save the index of the non duplicated values
-            # and set the next cur_frame to cur_frame filtered on all
-            # duplcicated values (#GH15297)
+            # If it's the last column or if we have the number of
+            # results desired we are done.
+            # Otherwise there are duplicates of the largest/smallest
+            # value and we need to look at the rest of the columns
+            # to determine which of the rows with the largest/smallest
+            # value in the column to keep.
             series = cur_frame[column]
-            values = getattr(series, method)(cur_n, keep=self.keep)
             is_last_column = len(columns) - 1 == i
-            if is_last_column or values.nunique() == series.isin(values).sum():
+            values = getattr(series, method)(
+                cur_n,
+                keep=self.keep if is_last_column else 'all')
 
-                # Last column in columns or values are unique in
-                # series => values
-                # is all that matters
+            if is_last_column or len(values) <= cur_n:
                 indexer = get_indexer(indexer, values.index)
                 break
 
-            duplicated_filter = series.duplicated(keep=False)
-            duplicated = values[duplicated_filter]
-            non_duplicated = values[~duplicated_filter]
-            indexer = get_indexer(indexer, non_duplicated.index)
+            # Now find all values which are equal to
+            # the (nsmallest: largest)/(nlarrgest: smallest)
+            # from our series.
+            border_value = values == values[values.index[-1]]
+
+            # Some of these values are among the top-n
+            # some aren't.
+            unsafe_values = values[border_value]
+
+            # These values are definitely among the top-n
+            safe_values = values[~border_value]
+            indexer = get_indexer(indexer, safe_values.index)
 
-            # Must set cur frame to include all duplicated values
-            # to consider for the next column, we also can reduce
-            # cur_n by the current length of the indexer
-            cur_frame = cur_frame[series.isin(duplicated)]
+            # Go on and separate the unsafe_values on the remaining
+            # columns.
+            cur_frame = cur_frame.loc[unsafe_values.index]
             cur_n = n - len(indexer)
 
         frame = frame.take(indexer)
 
         # Restore the index on frame
         frame.index = original_index.take(indexer)
-        return frame
+
+        # If there is only one column, the frame is already sorted.
+        if len(columns) == 1:
+            return frame
+
+        ascending = method == 'nsmallest'
+
+        return frame.sort_values(
+            columns,
+            ascending=ascending,
+            kind='mergesort')
 
 
 # ------- ## ---- #


Test patch:
diff --git a/pandas/tests/frame/test_analytics.py b/pandas/tests/frame/test_analytics.py
--- a/pandas/tests/frame/test_analytics.py
+++ b/pandas/tests/frame/test_analytics.py
@@ -2095,6 +2095,24 @@ def test_n_all_dtypes(self, df_main_dtypes):
         df.nsmallest(2, list(set(df) - {'category_string', 'string'}))
         df.nlargest(2, list(set(df) - {'category_string', 'string'}))
 
+    @pytest.mark.parametrize('method,expected', [
+        ('nlargest',
+         pd.DataFrame({'a': [2, 2, 2, 1], 'b': [3, 2, 1, 3]},
+                      index=[2, 1, 0, 3])),
+        ('nsmallest',
+         pd.DataFrame({'a': [1, 1, 1, 2], 'b': [1, 2, 3, 1]},
+                      index=[5, 4, 3, 0]))])
+    def test_duplicates_on_starter_columns(self, method, expected):
+        # regression test for #22752
+
+        df = pd.DataFrame({
+            'a': [2, 2, 2, 1, 1, 1],
+            'b': [1, 2, 3, 3, 2, 1]
+        })
+
+        result = getattr(df, method)(4, columns=['a', 'b'])
+        tm.assert_frame_equal(result, expected)
+
     def test_n_identical_values(self):
         # GH15297
         df = pd.DataFrame({'a': [1] * 5, 'b': [1, 2, 3, 4, 5]})