Repository: mars-project/mars

Issue:
Implements `DataFrame.transpose`
Implements `DataFrame.transpose`.

Need #1095 to be implemented first.


Discussion:

Resolving patch:
diff --git a/mars/dataframe/base/__init__.py b/mars/dataframe/base/__init__.py
--- a/mars/dataframe/base/__init__.py
+++ b/mars/dataframe/base/__init__.py
@@ -43,6 +43,7 @@
 from .check_monotonic import check_monotonic, is_monotonic, \
     is_monotonic_increasing, is_monotonic_decreasing
 from .pct_change import pct_change
+from .transpose import transpose
 
 
 def _install():
@@ -80,6 +81,7 @@ def _install():
         setattr(t, 'eval', df_eval)
         setattr(t, 'query', df_query)
         setattr(t, 'pct_change', pct_change)
+        setattr(t, 'transpose', transpose)
 
     for t in SERIES_TYPE:
         setattr(t, 'to_gpu', to_gpu)
diff --git a/mars/dataframe/base/transpose.py b/mars/dataframe/base/transpose.py
new file mode 100644
--- /dev/null
+++ b/mars/dataframe/base/transpose.py
@@ -0,0 +1,161 @@
+# Copyright 1999-2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ... import opcodes
+from ...core import OutputType
+from ..operands import DataFrameOperand, DataFrameOperandMixin
+from ..utils import parse_index
+
+
+class DataFrameTranspose(DataFrameOperand, DataFrameOperandMixin):
+    _op_code_ = opcodes.TRANSPOSE
+
+    def __init__(self, **kw):
+        super().__init__(**kw)
+        self.output_types = [OutputType.dataframe]
+
+    def __call__(self, args):
+        arg = args[0]
+        new_shape = arg.shape[::-1]
+        columns_value = arg.index_value
+        index_value = parse_index(arg.dtypes.index)
+        return self.new_dataframe([arg], shape=new_shape, dtypes=None,
+                                  columns_value=columns_value, index_value=index_value)
+
+    @classmethod
+    def tile(cls, op):
+        out_chunks = []
+        for c in op.inputs[0].chunks:
+            chunk_op = op.copy().reset_key()
+            chunk_shape = tuple(s if np.isnan(s) else int(s)
+                                for s in c.shape[::-1])
+            chunk_idx = c.index[::-1]
+            index_value = parse_index(c.dtypes.index)
+            columns_value = c.index_value
+            out_chunk = chunk_op.new_chunk([c], shape=chunk_shape,
+                                           index=chunk_idx,
+                                           index_value=index_value,
+                                           columns_value=columns_value,
+                                           dtypes=None)
+            out_chunks.append(out_chunk)
+
+        new_op = op.copy()
+        nsplits = op.inputs[0].nsplits[::-1]
+        params = op.outputs[0].params
+        return new_op.new_dataframe(op.inputs, chunks=out_chunks,
+                                    nsplits=nsplits, **params)
+
+    @classmethod
+    def execute(cls, ctx, op):
+        inp = ctx[op.inputs[0].key]
+        ctx[op.outputs[0].key] = inp.transpose()
+
+
+def transpose(*args):
+    """
+    Transpose index and columns.
+
+    Reflect the DataFrame over its main diagonal by writing rows as columns
+    and vice-versa. The property :attr:`.T` is an accessor to the method
+    :meth:`transpose`.
+
+    Parameters
+    ----------
+    *args : tuple, optional
+            Accepted for compatibility with NumPy.
+
+    Returns
+    -------
+    DataFrame
+        The transposed DataFrame.
+
+    See Also
+    --------
+    numpy.transpose : Permute the dimensions of a given array.
+
+    Notes
+    -----
+    Transposing a DataFrame with mixed dtypes will result in a homogeneous
+    DataFrame with the `object` dtype.
+
+    Examples
+    --------
+    **Square DataFrame with homogeneous dtype**
+
+    >>> import mars.dataframe as md
+    >>> d1 = {'col1': [1, 2], 'col2': [3, 4]}
+    >>> df1 = md.DataFrame(data=d1).execute()
+    >>> df1
+        col1  col2
+    0     1     3
+    1     2     4
+
+    >>> df1_transposed = df1.T.execute() # or df1.transpose().execute()
+    >>> df1_transposed
+          0  1
+    col1  1  2
+    col2  3  4
+
+    When the dtype is homogeneous in the original DataFrame, we get a
+    transposed DataFrame with the same dtype:
+
+    >>> df1.dtypes
+    col1    int64
+    col2    int64
+    dtype: object
+
+    >>> df1_transposed.dtypes
+    0    int64
+    1    int64
+    dtype: object
+
+    **Non-square DataFrame with mixed dtypes**
+
+    >>> d2 = {'name': ['Alice', 'Bob'],
+    ...       'score': [9.5, 8],
+    ...       'employed': [False, True],
+    ...       'kids': [0, 0]}
+    >>> df2 = md.DataFrame(data=d2).execute()
+    >>> df2
+        name  score  employed  kids
+    0  Alice    9.5     False     0
+    1    Bob    8.0      True     0
+
+    >>> df2_transposed = df2.T.execute() # or df2.transpose().execute()
+    >>> df2_transposed
+                  0     1
+    name      Alice   Bob
+    score       9.5   8.0
+    employed  False  True
+    kids          0     0
+
+    When the DataFrame has mixed dtypes, we get a transposed DataFrame with
+    the `object` dtype:
+
+    >>> df2.dtypes
+    name         object
+    score       float64
+    employed       bool
+    kids          int64
+    dtype: object
+
+    >>> df2_transposed.dtypes
+    0    object
+    1    object
+    dtype: object
+    """
+    op = DataFrameTranspose()
+    return op(args)
diff --git a/mars/dataframe/core.py b/mars/dataframe/core.py
--- a/mars/dataframe/core.py
+++ b/mars/dataframe/core.py
@@ -668,6 +668,11 @@ def _set_df_or_series(self, df_or_series, axis):
         self._parent_key = df_or_series.key
         self._axis = axis
 
+    @property
+    def T(self):
+        """Return the transpose, which is by definition self."""
+        return self
+
     @property
     def name(self):
         return self._data.name
@@ -1071,6 +1076,11 @@ def to_tensor(self, dtype=None):
     def from_tensor(self, in_tensor, index=None, name=None):
         return self._data.from_tensor(in_tensor, index=index, name=name)
 
+    @property
+    def T(self):
+        """Return the transpose, which is by definition self."""
+        return self
+
     @property
     def ndim(self):
         """
@@ -1724,6 +1734,10 @@ def __dir__(self):
         result = list(super().__dir__())
         return sorted(result + [k for k in self.dtypes.index if isinstance(k, str) and k.isidentifier()])
 
+    @property
+    def T(self):
+        return self.transpose()
+
     @property
     def ndim(self):
         """


Test patch:
diff --git a/mars/dataframe/base/tests/test_base.py b/mars/dataframe/base/tests/test_base.py
--- a/mars/dataframe/base/tests/test_base.py
+++ b/mars/dataframe/base/tests/test_base.py
@@ -701,6 +701,19 @@ def test_cut():
     assert r.dtype == e.dtype
 
 
+def test_transpose():
+    s = pd.DataFrame({"a": [1, 2, 3], "b": ['5', '-6', '7'], "c": [1, 2, 3]})
+    df = from_pandas_df(s, chunk_size=2)
+
+    r = tile(df.transpose())
+    assert len(r.chunks) == 4
+    assert isinstance(r, DATAFRAME_TYPE)
+
+    r = tile(df.T)
+    assert len(r.chunks) == 4
+    assert isinstance(r, DATAFRAME_TYPE)
+
+
 def test_to_numeric():
     raw = pd.DataFrame({"a": [1.0, 2, 3, -3]})
     df = from_pandas_df(raw, chunk_size=2)
diff --git a/mars/dataframe/base/tests/test_base_execution.py b/mars/dataframe/base/tests/test_base_execution.py
--- a/mars/dataframe/base/tests/test_base_execution.py
+++ b/mars/dataframe/base/tests/test_base_execution.py
@@ -24,7 +24,9 @@
     pa = None
 
 from ....config import options, option_context
-from ....tensor import tensor
+from ....dataframe import DataFrame
+from ....tensor import arange, tensor
+from ....tensor.random import rand
 from ....tests.core import require_cudf
 from ....utils import lazy_import
 from ... import eval as mars_eval, cut, qcut
@@ -859,6 +861,48 @@ def test_cut_execution(setup):
         cut(s3, 3).execute()
 
 
+def test_transpose_execution(setup):
+    raw = pd.DataFrame({"a": ['1', '2', '3'], "b": ['5', '-6', '7'], "c": ['1', '2', '3']})
+
+    # test 1 chunk
+    df = from_pandas_df(raw)
+    result = df.transpose().execute().fetch()
+    pd.testing.assert_frame_equal(result, raw.transpose())
+
+    # test multi chunks
+    df = from_pandas_df(raw, chunk_size=2)
+    result = df.transpose().execute().fetch()
+    pd.testing.assert_frame_equal(result, raw.transpose())
+
+    df = from_pandas_df(raw, chunk_size=2)
+    result = df.T.execute().fetch()
+    pd.testing.assert_frame_equal(result, raw.transpose())
+
+    # dtypes are varied
+    raw = pd.DataFrame({"a": [1.1, 2.2, 3.3], "b": [5, -6, 7], "c": [1, 2, 3]})
+
+    df = from_pandas_df(raw, chunk_size=2)
+    result = df.transpose().execute().fetch()
+    pd.testing.assert_frame_equal(result, raw.transpose())
+
+    raw = pd.DataFrame({"a": [1.1, 2.2, 3.3], "b": ['5', '-6', '7']})
+
+    df = from_pandas_df(raw, chunk_size=2)
+    result = df.transpose().execute().fetch()
+    pd.testing.assert_frame_equal(result, raw.transpose())
+
+    # Transposing from results of other operands
+    raw = pd.DataFrame(np.arange(0, 100).reshape(10, 10))
+    df = DataFrame(arange(0, 100, chunk_size=5).reshape(10, 10))
+    result = df.transpose().execute().fetch()
+    pd.testing.assert_frame_equal(result, raw.transpose())
+
+    df = DataFrame(rand(100, 100, chunk_size=10))
+    raw = df.to_pandas()
+    result = df.transpose().execute().fetch()
+    pd.testing.assert_frame_equal(result, raw.transpose())
+
+
 def test_to_numeric_execition(setup):
     rs = np.random.RandomState(0)
     s = pd.Series(rs.randint(5, size=100))
diff --git a/mars/dataframe/tests/test_core.py b/mars/dataframe/tests/test_core.py
--- a/mars/dataframe/tests/test_core.py
+++ b/mars/dataframe/tests/test_core.py
@@ -53,6 +53,8 @@ def test_series_params():
     series = tile(series)
     c = series.chunks[0]
 
+    assert series.T is series
+
     assert any(np.isnan(s) for s in c.params['shape'])
     assert np.isnan(c.params['index_value'].min_val)
     c.params = c.get_params_from_data(raw[raw < 2])
@@ -76,6 +78,8 @@ def test_index_params():
     index = tile(index)
     c = index.chunks[0]
 
+    assert index.T is index
+
     assert any(np.isnan(s) for s in c.params['shape'])
     assert np.isnan(c.params['index_value'].min_val)
     c.params = c.get_params_from_data(raw[raw < 2].index)
diff --git a/mars/tests/core.py b/mars/tests/core.py
--- a/mars/tests/core.py
+++ b/mars/tests/core.py
@@ -66,8 +66,10 @@ def patch_method(method, *args, **kwargs):
     if hasattr(method, '__qualname__'):
         return mock.patch(method.__module__ + '.' + method.__qualname__, *args, **kwargs)
     elif hasattr(method, 'im_class'):
-        return mock.patch('.'.join([method.im_class.__module__, method.im_class.__name__, method.__name__]),
-                          *args, **kwargs)
+        return mock.patch('.'.join(
+            [method.im_class.__module__, method.im_class.__name__, method.__name__]),
+            *args, **kwargs
+        )
     else:
         return mock.patch(method.__module__ + '.' + method.__name__, *args, **kwargs)
 
@@ -288,7 +290,9 @@ def assert_dataframe_consistent(self, expected, real):
         if not isinstance(real, dataframe_types):
             raise AssertionError(f'Type of real value ({type(real)}) not DataFrame')
         self.assert_shape_consistent(expected.shape, real.shape)
-        if not np.isnan(expected.shape[1]):  # ignore when columns length is nan
+        if not np.isnan(expected.shape[1]) \
+                and expected.dtypes is not None:
+            # ignore check when columns length is nan or dtypes undefined
             pd.testing.assert_index_equal(expected.dtypes.index,
                                           self.adapt_index_value(real.dtypes.index))