Repository: mars-project/mars

Issue:
make mars type inference optional
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Is your feature request related to a problem? Please describe.**
Mars type inferrence will generate mock data, then feed the data into user provided function, if the user function is time-consuming  which take minutes or hours, there will be much time wasting in type inference, and the real computing didn't even happen which is unacceptable in some cases.

**Describe the solution you'd like**
It would be great if we can make dtypes lazy and type inferrence optional. If so, the expression call cost would be minimal.




Discussion:

Resolving patch:
diff --git a/mars/dataframe/base/apply.py b/mars/dataframe/base/apply.py
--- a/mars/dataframe/base/apply.py
+++ b/mars/dataframe/base/apply.py
@@ -269,19 +269,21 @@ def tile(cls, op):
 
     def _infer_df_func_returns(self, df, dtypes, dtype=None, name=None, index=None):
         if isinstance(self._func, np.ufunc):
-            output_type, new_dtypes, index_value, new_elementwise = (
-                OutputType.dataframe,
-                None,
-                "inherit",
-                True,
-            )
+            output_type = OutputType.dataframe
+            new_dtypes = None
+            index_value = "inherit"
+            new_elementwise = True
         else:
-            output_type, new_dtypes, index_value, new_elementwise = (
-                None,
-                None,
-                None,
-                False,
-            )
+            if self.output_types is not None and (
+                dtypes is not None or dtype is not None
+            ):
+                ret_dtypes = dtypes if dtypes is not None else (dtype, name)
+                ret_index_value = parse_index(index) if index is not None else None
+                self._elementwise = False
+                return ret_dtypes, ret_index_value
+
+            output_type = new_dtypes = index_value = None
+            new_elementwise = False
 
         try:
             empty_df = build_df(df, size=2)
@@ -374,14 +376,19 @@ def _call_series(self, series, dtypes=None, dtype=None, name=None, index=None):
         # for backward compatibility
         dtype = dtype if dtype is not None else dtypes
         if self._convert_dtype:
-            test_series = build_series(series, size=2, name=series.name)
-            try:
-                with np.errstate(all="ignore"), quiet_stdio():
-                    infer_series = test_series.apply(
-                        self._func, args=self.args, **self.kwds
-                    )
-            except:  # noqa: E722  # nosec  # pylint: disable=bare-except
-                infer_series = None
+            if self.output_types is not None and (
+                dtypes is not None or dtype is not None
+            ):
+                infer_series = test_series = None
+            else:
+                test_series = build_series(series, size=2, name=series.name)
+                try:
+                    with np.errstate(all="ignore"), quiet_stdio():
+                        infer_series = test_series.apply(
+                            self._func, args=self.args, **self.kwds
+                        )
+                except:  # noqa: E722  # nosec  # pylint: disable=bare-except
+                    infer_series = None
 
             output_type = self._output_types[0]
 
diff --git a/mars/dataframe/groupby/apply.py b/mars/dataframe/groupby/apply.py
--- a/mars/dataframe/groupby/apply.py
+++ b/mars/dataframe/groupby/apply.py
@@ -139,10 +139,15 @@ def tile(cls, op):
             return [auto_merge_chunks(get_context(), ret)]
 
     def _infer_df_func_returns(
-        self, in_groupby, in_df, dtypes, dtype=None, name=None, index=None
+        self, in_groupby, in_df, dtypes=None, dtype=None, name=None, index=None
     ):
         index_value, output_type, new_dtypes = None, None, None
 
+        if self.output_types is not None and (dtypes is not None or dtype is not None):
+            ret_dtypes = dtypes if dtypes is not None else (dtype, name)
+            ret_index_value = parse_index(index) if index is not None else None
+            return ret_dtypes, ret_index_value
+
         try:
             infer_df = in_groupby.op.build_mock_groupby().apply(
                 self.func, *self.args, **self.kwds
diff --git a/mars/utils.py b/mars/utils.py
--- a/mars/utils.py
+++ b/mars/utils.py
@@ -1566,7 +1566,12 @@ def __str__(self):
         return message
 
     return type(
-        name,
+        bases[-1].__name__,
         bases,
-        {"__init__": __init__, "__getattr__": __getattr__, "__str__": __str__},
+        {
+            "__init__": __init__,
+            "__getattr__": __getattr__,
+            "__str__": __str__,
+            "__basename__": name,
+        },
     )().with_traceback(traceback)


Test patch:
diff --git a/mars/dataframe/base/tests/test_base.py b/mars/dataframe/base/tests/test_base.py
--- a/mars/dataframe/base/tests/test_base.py
+++ b/mars/dataframe/base/tests/test_base.py
@@ -376,6 +376,12 @@ def test_series_apply():
     pd.testing.assert_series_equal(r.dtypes, dtypes)
     assert r.shape == (2, 3)
 
+    def apply_with_error(_):
+        raise ValueError
+
+    r = series.apply(apply_with_error, output_type="dataframe", dtypes=dtypes)
+    assert r.ndim == 2
+
     r = series.apply(
         pd.Series, output_type="dataframe", dtypes=dtypes, index=pd.RangeIndex(2)
     )
diff --git a/mars/dataframe/groupby/tests/test_groupby.py b/mars/dataframe/groupby/tests/test_groupby.py
--- a/mars/dataframe/groupby/tests/test_groupby.py
+++ b/mars/dataframe/groupby/tests/test_groupby.py
@@ -153,6 +153,9 @@ def test_groupby_apply():
         }
     )
 
+    def apply_call_with_err(_):
+        raise ValueError
+
     def apply_df(df):
         return df.sort_index()
 
@@ -165,6 +168,14 @@ def apply_series(s):
 
     mdf = md.DataFrame(df1, chunk_size=3)
 
+    # when dtype and output_type specified, apply function
+    # shall not be called
+    applied = mdf.groupby("b").apply(
+        apply_call_with_err, output_type="series", dtype=int
+    )
+    assert applied.dtype == int
+    assert applied.op.output_types[0] == OutputType.series
+
     with pytest.raises(TypeError):
         mdf.groupby("b").apply(apply_df_with_error)
 
diff --git a/mars/deploy/oscar/tests/test_fault_injection.py b/mars/deploy/oscar/tests/test_fault_injection.py
--- a/mars/deploy/oscar/tests/test_fault_injection.py
+++ b/mars/deploy/oscar/tests/test_fault_injection.py
@@ -283,8 +283,10 @@ async def test_rerun_subtask_fail(fault_cluster, fault_config):
 
     with expect_raises as e:
         b.execute(extra_config=extra_config)
-    assert e.typename == exception_typename, "".join(traceback.format_tb(e.tb))
-    assert e.traceback[-1].name == stack_string, "".join(traceback.format_tb(e.tb))
+
+    tb_str = "".join(traceback.format_tb(e.tb))
+    assert e.value.__basename__ == exception_typename, tb_str
+    assert e.traceback[-1].name == stack_string, tb_str
 
 
 @pytest.mark.parametrize(
@@ -325,7 +327,7 @@ def f(x):
     r = spawn(f, args=(1,), retry_when_fail=False)
     with expect_raises as e:
         r.execute(extra_config=extra_config)
-    assert e.typename == exception_typename, "".join(traceback.format_tb(e.tb))
-    assert stack_string == "*" or e.traceback[-1].name == stack_string, "".join(
-        traceback.format_tb(e.tb)
-    )
+
+    tb_str = "".join(traceback.format_tb(e.tb))
+    assert e.value.__basename__ == exception_typename, tb_str
+    assert stack_string == "*" or e.traceback[-1].name == stack_string, tb_str