Repository: mars-project/mars

Issue:
[BUG] `df.sort_values` produces incorrect result
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
`df.sort_values`'s result is incorrect.

**To Reproduce**
``` Python
In [10]: df = pd.DataFrame(
    ...:         np.random.rand(100, 10), columns=["a" + str(i) for i in range(10)]
    ...:     )

In [11]: mdf = md.DataFrame(df, chunk_size=10)

In [12]: r = mdf.sort_values(["a3", "a4"], ascending=[False, True]).execute()
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100.0/100 [00:00<00:00, 631.75it/s]

In [13]: r
Out[13]:
          a0        a1        a2        a3        a4        a5        a6        a7        a8        a9
16  0.107025  0.476072  0.335091  0.982758  0.100404  0.535478  0.843878  0.993314  0.519650  0.039456
22  0.918816  0.782424  0.685651  0.981538  0.054176  0.204336  0.164184  0.545094  0.626901  0.001013
27  0.102054  0.078257  0.412166  0.977549  0.943098  0.095842  0.908522  0.078090  0.321839  0.579264
82  0.948665  0.387145  0.140989  0.962591  0.253510  0.053363  0.695930  0.322598  0.434367  0.831326
37  0.921237  0.795837  0.419291  0.957214  0.029907  0.224950  0.239270  0.694234  0.494518  0.698810
..       ...       ...       ...       ...       ...       ...       ...       ...       ...       ...
38  0.479132  0.447279  0.262018  0.047719  0.232861  0.967857  0.608678  0.285415  0.385973  0.443056
1   0.318765  0.664569  0.631351  0.045131  0.163595  0.965267  0.037361  0.044477  0.963650  0.140346
11  0.391554  0.169611  0.384232  0.040313  0.397935  0.822954  0.042206  0.522298  0.944956  0.611841
67  0.022653  0.053233  0.813252  0.020961  0.366821  0.261931  0.592673  0.948731  0.476598  0.604238
20  0.955453  0.521866  0.419302  0.007513  0.303412  0.231128  0.984855  0.439811  0.755543  0.441908

[200 rows x 10 columns]

In [14]: mdf.shape
Out[14]: (100, 10)
```



Discussion:
It only happens when ascending is a list, probably caused by `_calc_poses` in psrs.

Resolving patch:
diff --git a/mars/dataframe/sort/psrs.py b/mars/dataframe/sort/psrs.py
--- a/mars/dataframe/sort/psrs.py
+++ b/mars/dataframe/sort/psrs.py
@@ -20,7 +20,13 @@
 from ... import opcodes as OperandDef
 from ...core.operand import OperandStage, MapReduceOperand
 from ...utils import lazy_import, calc_nsplits
-from ...serialization.serializables import Int32Field, ListField, StringField, BoolField
+from ...serialization.serializables import (
+    AnyField,
+    Int32Field,
+    ListField,
+    StringField,
+    BoolField,
+)
 from ...tensor.base.psrs import PSRSOperandMixin
 from ..core import IndexValue, OutputType
 from ..utils import standardize_range_index, parse_index, is_cudf
@@ -48,6 +54,23 @@ def __gt__(self, other):
 _largest = _Largest()
 
 
+class _ReversedValue:
+    def __init__(self, value):
+        self._value = value
+
+    def __lt__(self, other):
+        if type(other) is _ReversedValue:
+            # may happen when call searchsorted
+            return self._value >= other._value
+        return self._value >= other
+
+    def __gt__(self, other):
+        return self._value <= other
+
+    def __repr__(self):
+        return repr(self._value)
+
+
 class DataFramePSRSOperandMixin(DataFrameOperandMixin, PSRSOperandMixin):
     @classmethod
     def _collect_op_properties(cls, op):
@@ -380,90 +403,23 @@ def execute_sort_index(data, op, inplace=None):
 
 class DataFramePSRSChunkOperand(DataFrameOperand):
     # sort type could be 'sort_values' or 'sort_index'
-    _sort_type = StringField("sort_type")
+    sort_type = StringField("sort_type")
 
-    _axis = Int32Field("axis")
-    _by = ListField("by")
-    _ascending = BoolField("ascending")
-    _inplace = BoolField("inplace")
-    _kind = StringField("kind")
-    _na_position = StringField("na_position")
+    axis = Int32Field("axis")
+    by = ListField("by", default=None)
+    ascending = AnyField("ascending")
+    inplace = BoolField("inplace")
+    kind = StringField("kind")
+    na_position = StringField("na_position")
 
     # for sort_index
-    _level = ListField("level")
-    _sort_remaining = BoolField("sort_remaining")
-
-    _n_partition = Int32Field("n_partition")
-
-    def __init__(
-        self,
-        sort_type=None,
-        by=None,
-        axis=None,
-        ascending=None,
-        inplace=None,
-        kind=None,
-        na_position=None,
-        level=None,
-        sort_remaining=None,
-        n_partition=None,
-        output_types=None,
-        **kw
-    ):
-        super().__init__(
-            _sort_type=sort_type,
-            _by=by,
-            _axis=axis,
-            _ascending=ascending,
-            _inplace=inplace,
-            _kind=kind,
-            _na_position=na_position,
-            _level=level,
-            _sort_remaining=sort_remaining,
-            _n_partition=n_partition,
-            _output_types=output_types,
-            **kw
-        )
+    level = ListField("level")
+    sort_remaining = BoolField("sort_remaining")
 
-    @property
-    def sort_type(self):
-        return self._sort_type
+    n_partition = Int32Field("n_partition")
 
-    @property
-    def axis(self):
-        return self._axis
-
-    @property
-    def by(self):
-        return self._by
-
-    @property
-    def ascending(self):
-        return self._ascending
-
-    @property
-    def inplace(self):
-        return self._inplace
-
-    @property
-    def kind(self):
-        return self._kind
-
-    @property
-    def na_position(self):
-        return self._na_position
-
-    @property
-    def level(self):
-        return self._level
-
-    @property
-    def sort_remaining(self):
-        return self._sort_remaining
-
-    @property
-    def n_partition(self):
-        return self._n_partition
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
 
 
 class DataFramePSRSSortRegularSample(DataFramePSRSChunkOperand, DataFrameOperandMixin):
@@ -567,92 +523,25 @@ def execute(cls, ctx, op):
 class DataFramePSRSShuffle(MapReduceOperand, DataFrameOperandMixin):
     _op_type_ = OperandDef.PSRS_SHUFFLE
 
-    _sort_type = StringField("sort_type")
+    sort_type = StringField("sort_type")
 
     # for shuffle map
-    _axis = Int32Field("axis")
-    _by = ListField("by")
-    _ascending = BoolField("ascending")
-    _inplace = BoolField("inplace")
-    _na_position = StringField("na_position")
-    _n_partition = Int32Field("n_partition")
+    axis = Int32Field("axis")
+    by = ListField("by")
+    ascending = AnyField("ascending")
+    inplace = BoolField("inplace")
+    na_position = StringField("na_position")
+    n_partition = Int32Field("n_partition")
 
     # for sort_index
-    _level = ListField("level")
-    _sort_remaining = BoolField("sort_remaining")
+    level = ListField("level")
+    sort_remaining = BoolField("sort_remaining")
 
     # for shuffle reduce
-    _kind = StringField("kind")
-
-    def __init__(
-        self,
-        sort_type=None,
-        by=None,
-        axis=None,
-        ascending=None,
-        n_partition=None,
-        na_position=None,
-        inplace=None,
-        kind=None,
-        level=None,
-        sort_remaining=None,
-        output_types=None,
-        **kw
-    ):
-        super().__init__(
-            _sort_type=sort_type,
-            _by=by,
-            _axis=axis,
-            _ascending=ascending,
-            _n_partition=n_partition,
-            _na_position=na_position,
-            _inplace=inplace,
-            _kind=kind,
-            _level=level,
-            _sort_remaining=sort_remaining,
-            _output_types=output_types,
-            **kw
-        )
-
-    @property
-    def sort_type(self):
-        return self._sort_type
-
-    @property
-    def by(self):
-        return self._by
-
-    @property
-    def axis(self):
-        return self._axis
-
-    @property
-    def ascending(self):
-        return self._ascending
+    kind = StringField("kind")
 
-    @property
-    def inplace(self):
-        return self._inplace
-
-    @property
-    def na_position(self):
-        return self._na_position
-
-    @property
-    def level(self):
-        return self._level
-
-    @property
-    def sort_remaining(self):
-        return self._sort_remaining
-
-    @property
-    def n_partition(self):
-        return self._n_partition
-
-    @property
-    def kind(self):
-        return self._kind
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
 
     @property
     def output_limit(self):
@@ -660,6 +549,23 @@ def output_limit(self):
 
     @staticmethod
     def _calc_poses(src_cols, pivots, ascending=True):
+        if isinstance(ascending, list):
+            for asc, col in zip(ascending, pivots.columns):
+                # Make pivots available to use ascending order when mixed order specified
+                if not asc:
+                    if pd.api.types.is_numeric_dtype(pivots.dtypes[col]):
+                        # for numeric dtypes, convert to negative is more efficient
+                        pivots[col] = -pivots[col]
+                        src_cols[col] = -src_cols[col]
+                    else:
+                        # for other types, convert to ReversedValue
+                        pivots[col] = pivots[col].map(
+                            lambda x: x
+                            if type(x) is _ReversedValue
+                            else _ReversedValue(x)
+                        )
+            ascending = True
+
         records = src_cols.to_records(index=False)
         p_records = pivots.to_records(index=False)
         if ascending:
diff --git a/mars/dataframe/sort/sort_values.py b/mars/dataframe/sort/sort_values.py
--- a/mars/dataframe/sort/sort_values.py
+++ b/mars/dataframe/sort/sort_values.py
@@ -252,6 +252,13 @@ def dataframe_sort_values(
         raise NotImplementedError("Only support sort on axis 0")
     psrs_kinds = _validate_sort_psrs_kinds(psrs_kinds)
     by = by if isinstance(by, (list, tuple)) else [by]
+    if isinstance(ascending, list):  # pragma: no cover
+        if all(ascending):
+            # all are True, convert to True
+            ascending = True
+        elif not any(ascending):
+            # all are False, convert to False
+            ascending = False
     op = DataFrameSortValues(
         by=by,
         axis=axis,


Test patch:
diff --git a/mars/dataframe/sort/tests/test_sort_execution.py b/mars/dataframe/sort/tests/test_sort_execution.py
--- a/mars/dataframe/sort/tests/test_sort_execution.py
+++ b/mars/dataframe/sort/tests/test_sort_execution.py
@@ -27,10 +27,9 @@
     "distinct_opt", ["0"] if sys.platform.lower().startswith("win") else ["0", "1"]
 )
 def test_sort_values_execution(setup, distinct_opt):
+    ns = np.random.RandomState(0)
     os.environ["PSRS_DISTINCT_COL"] = distinct_opt
-    df = pd.DataFrame(
-        np.random.rand(100, 10), columns=["a" + str(i) for i in range(10)]
-    )
+    df = pd.DataFrame(ns.rand(100, 10), columns=["a" + str(i) for i in range(10)])
 
     # test one chunk
     mdf = DataFrame(df)
@@ -67,6 +66,38 @@ def test_sort_values_execution(setup, distinct_opt):
 
     pd.testing.assert_frame_equal(result, expected)
 
+    # test ascending is a list
+    result = (
+        mdf.sort_values(["a3", "a4", "a5", "a6"], ascending=[False, True, True, False])
+        .execute()
+        .fetch()
+    )
+    expected = df.sort_values(
+        ["a3", "a4", "a5", "a6"], ascending=[False, True, True, False]
+    )
+    pd.testing.assert_frame_equal(result, expected)
+
+    in_df = pd.DataFrame(
+        {
+            "col1": ns.choice([f"a{i}" for i in range(5)], size=(100,)),
+            "col2": ns.choice([f"b{i}" for i in range(5)], size=(100,)),
+            "col3": ns.choice([f"c{i}" for i in range(5)], size=(100,)),
+            "col4": ns.randint(10, 20, size=(100,)),
+        }
+    )
+    mdf = DataFrame(in_df, chunk_size=10)
+    result = (
+        mdf.sort_values(
+            ["col1", "col4", "col3", "col2"], ascending=[False, False, True, False]
+        )
+        .execute()
+        .fetch()
+    )
+    expected = in_df.sort_values(
+        ["col1", "col4", "col3", "col2"], ascending=[False, False, True, False]
+    )
+    pd.testing.assert_frame_equal(result, expected)
+
     # test multiindex
     df2 = df.copy(deep=True)
     df2.columns = pd.MultiIndex.from_product([list("AB"), list("CDEFG")])