Repository: mars-project/mars

Issue:
Support DataFrame operations on data which generates unknown dtypes
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Is your feature request related to a problem? Please describe.**

Some DataFrame operands would generate output with unknown dtypes, like `tranpose`, `get_dummies`, we should add some mechanism that upcoming operations on these objects would fail.



Discussion:

Resolving patch:
diff --git a/mars/core/operand/core.py b/mars/core/operand/core.py
--- a/mars/core/operand/core.py
+++ b/mars/core/operand/core.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import sys
-from typing import Any, Callable, Dict, List, Type, Union
+from typing import Any, Callable, Dict, List, Tuple, Type, Union
 
 import numpy as np
 try:
@@ -25,7 +25,7 @@
 from ...utils import calc_data_size
 from ..context import Context
 from ..mode import is_eager_mode
-from ..entity import OutputType, ExecutableTuple, \
+from ..entity import OutputType, TILEABLE_TYPE, ExecutableTuple, \
     get_chunk_types, get_tileable_types, \
     get_output_types, get_fetch_class
 
@@ -36,11 +36,24 @@
 class TileableOperandMixin:
     __slots__ = ()
 
-    def check_inputs(self, inputs):
-        pass
+    def check_inputs(self, inputs: List[TileableType]):
+        if not inputs:
+            return
+        for inp in inputs:
+            if isinstance(inp, TILEABLE_TYPE):
+                i = inp.extra_params['_i']
+                if not inp.op.output_types:
+                    continue
+                if inp.op.output_types[i] != OutputType.dataframe:
+                    continue
+                dtypes = getattr(inp, 'dtypes', None)
+                if dtypes is None:
+                    raise ValueError(
+                        f'{inp} has unknown dtypes, '
+                        f'it must be executed first before {str(type(self))}')
 
     @classmethod
-    def _check_if_gpu(cls, inputs):
+    def _check_if_gpu(cls, inputs: List[TileableType]):
         if inputs is not None and \
                 len([inp for inp in inputs
                      if inp is not None and getattr(inp, 'op', None) is not None]) > 0:
@@ -49,7 +62,10 @@ def _check_if_gpu(cls, inputs):
             elif all(inp.op.gpu is False for inp in inputs):
                 return False
 
-    def _create_chunk(self, output_idx, index, **kw) -> ChunkType:
+    def _create_chunk(self,
+                      output_idx: int,
+                      index: Tuple[int],
+                      **kw) -> ChunkType:
         output_type = kw.pop('output_type', self._get_output_type(output_idx))
         if not output_type:
             raise ValueError('output_type should be specified')
@@ -66,7 +82,10 @@ def _create_chunk(self, output_idx, index, **kw) -> ChunkType:
         data = chunk_data_type(**kw)
         return chunk_type(data)
 
-    def _new_chunks(self, inputs, kws=None, **kw) -> List[ChunkType]:
+    def _new_chunks(self,
+                    inputs: List[ChunkType],
+                    kws: dict = None,
+                    **kw) -> List[ChunkType]:
         output_limit = kw.pop('output_limit', None)
         if output_limit is None:
             output_limit = getattr(self, 'output_limit')
@@ -97,16 +116,31 @@ def _new_chunks(self, inputs, kws=None, **kw) -> List[ChunkType]:
                 t.data._siblings = [c.data for c in chunks[:j] + chunks[j + 1:]]
         return chunks
 
-    def new_chunks(self, inputs, kws=None, **kwargs) -> List[ChunkType]:
+    def new_chunks(self,
+                   inputs: List[ChunkType],
+                   kws: dict = None,
+                   **kwargs) -> List[ChunkType]:
         """
         Create chunks.
+
         A chunk is a node in a fine grained graph, all the chunk objects are created by
         calling this function, it happens mostly in tiles.
         The generated chunks will be set as this operand's outputs and each chunk will
         hold this operand as it's op.
-        :param inputs: input chunks
-        :param kws: kwargs for each output
-        :param kwargs: common kwargs for all outputs
+
+        Parameters
+        ----------
+        inputs : list
+            Input chunks.
+        kws : dict
+            Kwargs for each output.
+        kwargs : dict
+            common kwargs for all outputs
+
+        Returns
+        -------
+        chunks : list
+            Output chunks.
 
         .. note::
             It's a final method, do not override.
@@ -114,14 +148,17 @@ def new_chunks(self, inputs, kws=None, **kwargs) -> List[ChunkType]:
         """
         return self._new_chunks(inputs, kws=kws, **kwargs)
 
-    def new_chunk(self, inputs, kws=None, **kw) -> ChunkType:
+    def new_chunk(self,
+                  inputs: List[ChunkType],
+                  kws: dict = None,
+                  **kw) -> ChunkType:
         if getattr(self, 'output_limit') != 1:
             raise TypeError('cannot new chunk with more than 1 outputs')
 
         return self.new_chunks(inputs, kws=kws, **kw)[0]
 
     @staticmethod
-    def _fill_nan_shape(kw):
+    def _fill_nan_shape(kw: dict):
         nsplits = kw.get('nsplits')
         shape = kw.get('shape')
         if nsplits is not None and shape is not None:
@@ -137,7 +174,9 @@ def _fill_nan_shape(kw):
             kw['nsplits'] = nsplits
         return kw
 
-    def _create_tileable(self, output_idx, **kw) -> TileableType:
+    def _create_tileable(self,
+                         output_idx: int,
+                         **kw) -> TileableType:
         output_type = kw.pop('output_type', self._get_output_type(output_idx))
         if output_type is None:
             raise ValueError('output_type should be specified')
@@ -155,7 +194,10 @@ def _create_tileable(self, output_idx, **kw) -> TileableType:
         data = tileable_data_type(**kw)
         return tileable_type(data)
 
-    def _new_tileables(self, inputs, kws=None, **kw) -> List[TileableType]:
+    def _new_tileables(self,
+                       inputs: List[TileableType],
+                       kws: dict = None,
+                       **kw) -> List[TileableType]:
         output_limit = kw.pop('output_limit', None)
         if output_limit is None:
             output_limit = getattr(self, 'output_limit')
@@ -183,27 +225,44 @@ def _new_tileables(self, inputs, kws=None, **kw) -> List[TileableType]:
                 t.data._siblings = [tileable.data for tileable in tileables[:j] + tileables[j + 1:]]
         return tileables
 
-    def new_tileables(self, inputs, kws=None, **kw) -> List[TileableType]:
+    def new_tileables(self,
+                      inputs: List[TileableType],
+                      kws=None,
+                      **kw) -> List[TileableType]:
         """
         Create tileable objects(Tensors or DataFrames).
+
         This is a base function for create tileable objects like tensors or dataframes,
         it will be called inside the `new_tensors` and `new_dataframes`.
         If eager mode is on, it will trigger the execution after tileable objects are created.
-        :param inputs: input tileables
-        :param kws: kwargs for each output
-        :param kw: common kwargs for all outputs
+
+        Parameters
+        ----------
+        inputs : list
+            Input tileables
+        kws : dict
+            Kwargs for each output.
+        kw : dict
+            Common kwargs for all outputs.
+
+        Returns
+        -------
+        tileables : list
+            Output tileables.
 
         .. note::
             It's a final method, do not override.
             Override the method `_new_tileables` if needed.
         """
-
         tileables = self._new_tileables(inputs, kws=kws, **kw)
         if is_eager_mode():
             ExecutableTuple(tileables).execute()
         return tileables
 
-    def new_tileable(self, inputs, kws=None, **kw) -> TileableType:
+    def new_tileable(self,
+                     inputs: List[TileableType],
+                     kws: dict = None,
+                     **kw) -> TileableType:
         if getattr(self, 'output_limit') != 1:
             raise TypeError('cannot new chunk with more than 1 outputs')
 
@@ -225,7 +284,9 @@ def tile(cls, op: OperandType):
         raise NotImplementedError
 
     @classmethod
-    def post_tile(cls, op: OperandType, results: List[TileableType]):
+    def post_tile(cls,
+                  op: OperandType,
+                  results: List[TileableType]):
         """
         Operation after tile.
 
@@ -238,7 +299,9 @@ def post_tile(cls, op: OperandType, results: List[TileableType]):
         """
 
     @classmethod
-    def pre_execute(cls, ctx: Union[dict, Context], op: OperandType):
+    def pre_execute(cls,
+                    ctx: Union[dict, Context],
+                    op: OperandType):
         """
         Operation before execute.
 
@@ -251,11 +314,15 @@ def pre_execute(cls, ctx: Union[dict, Context], op: OperandType):
         """
 
     @classmethod
-    def execute(cls, ctx: Union[dict, Context], op: OperandType):
+    def execute(cls,
+                ctx: Union[dict, Context],
+                op: OperandType):
         raise NotImplementedError
 
     @classmethod
-    def post_execute(cls, ctx: Union[dict, Context], op: OperandType):
+    def post_execute(cls,
+                     ctx: Union[dict, Context],
+                     op: OperandType):
         """
         Operand before execute.
 
@@ -268,7 +335,9 @@ def post_execute(cls, ctx: Union[dict, Context], op: OperandType):
         """
 
     @classmethod
-    def estimate_size(cls, ctx: dict, op: OperandType):
+    def estimate_size(cls,
+                      ctx: dict,
+                      op: OperandType):
         from .fetch import FetchShuffle
 
         exec_size = 0
@@ -337,14 +406,19 @@ def estimate_size(cls, ctx: dict, op: OperandType):
             ctx[out.key] = (result_size, exec_size * memory_scale // len(outputs))
 
     @classmethod
-    def concat_tileable_chunks(cls, tileable):
+    def concat_tileable_chunks(cls,
+                               tileable: TileableType):
         raise NotImplementedError
 
     @classmethod
-    def create_tileable_from_chunks(cls, chunks, inputs=None, **kw):
+    def create_tileable_from_chunks(cls,
+                                    chunks: List[ChunkType],
+                                    inputs: List[TileableType] = None,
+                                    **kw) -> TileableType:
         raise NotImplementedError
 
-    def get_fetch_op_cls(self, obj):
+    def get_fetch_op_cls(self,
+                         obj: ChunkType):
         from .shuffle import ShuffleProxy
 
         output_types = get_output_types(obj, unknown_as=OutputType.object)
@@ -359,7 +433,8 @@ def _inner(**kw):
 
         return _inner
 
-    def get_fuse_op_cls(self, obj):
+    def get_fuse_op_cls(self,
+                        obj: ChunkType):
         raise NotImplementedError
 
     @classmethod


Test patch:
diff --git a/mars/core/operand/tests/test_core.py b/mars/core/operand/tests/test_core.py
--- a/mars/core/operand/tests/test_core.py
+++ b/mars/core/operand/tests/test_core.py
@@ -14,6 +14,7 @@
 
 import pytest
 
+from mars.core import OutputType
 from mars.core.operand import Operand, TileableOperandMixin, \
     execute, estimate_size
 
@@ -60,3 +61,12 @@ def test_estimate_size():
     MyOperand2.unregister_size_estimator()
     with pytest.raises(KeyError):
         estimate_size(dict(), MyOperand2(_key='1'))
+
+
+def test_unknown_dtypes():
+    op = MyOperand(_output_types=[OutputType.dataframe])
+    df = op.new_tileable(None, dtypes=None)
+    op2 = MyOperand(_output_types=[OutputType.scalar])
+    with pytest.raises(ValueError) as exc_info:
+        op2.new_tileable([df])
+    assert 'executed first' in exc_info.value.args[0]
diff --git a/mars/tensor/stats/tests/test_stats_execute.py b/mars/tensor/stats/tests/test_stats_execute.py
--- a/mars/tensor/stats/tests/test_stats_execute.py
+++ b/mars/tensor/stats/tests/test_stats_execute.py
@@ -199,23 +199,16 @@ def test_t_test_execution(setup):
 
 
 @pytest.mark.parametrize('chunk_size', [5, 15])
-@pytest.mark.parametrize('mode', ['auto', 'asymp', 'approx'])
-def test_ks_1samp(setup, chunk_size, mode):
+@pytest.mark.parametrize('mode, alternative', [
+    ('auto', 'greater'),
+    ('auto', 'less'),
+    ('auto', 'two-sided'),
+    ('asymp', 'two-sided'),
+    ('approx', 'two-sided'),
+])
+def test_ks_1samp(setup, chunk_size, mode, alternative):
     x = tensor(np.linspace(-15, 15, 9), chunk_size=5)
 
-    if mode == 'auto':
-        result = ks_1samp(x, sp_norm.cdf,
-                          alternative='greater').execute().fetch()
-        expected = sp_ks_1samp(x, sp_norm.cdf,
-                               alternative='greater')
-        assert result == expected
-
-        result = ks_1samp(x, sp_norm.cdf,
-                          alternative='less').execute().fetch()
-        expected = sp_ks_1samp(x, sp_norm.cdf,
-                               alternative='less')
-        assert result == expected
-
     result = ks_1samp(x, sp_norm.cdf, mode=mode).execute().fetch()
     expected = sp_ks_1samp(x, sp_norm.cdf, mode=mode)
     assert result == expected